diff --git a/krfb/invitationsrfbclient.cpp b/krfb/invitationsrfbclient.cpp
index 5a2a2961..7212ebe2 100644
--- a/krfb/invitationsrfbclient.cpp
+++ b/krfb/invitationsrfbclient.cpp
@@ -52,10 +52,6 @@ PendingInvitationsRfbClient::PendingInvitationsRfbClient(rfbClientPtr client, QO
     d(new Private(client))
 {
     d->client->clientGoneHook = clientGoneHookNoop;
-    d->notifier = new QSocketNotifier(client->sock, QSocketNotifier::Read, this);
-    d->notifier->setEnabled(true);
-    connect(d->notifier, &QSocketNotifier::activated,
-            this, &PendingInvitationsRfbClient::onSocketActivated);
 }
 
 PendingInvitationsRfbClient::~PendingInvitationsRfbClient()
@@ -90,39 +86,6 @@ void PendingInvitationsRfbClient::processNewClient()
     }
 }
 
-void PendingInvitationsRfbClient::onSocketActivated()
-{
-    //Process not only one, but all pending messages.
-    //poll() idea/code copied from vino:
-    // Copyright (C) 2003 Sun Microsystems, Inc.
-    // License: GPL v2 or later
-    struct pollfd pollfd = { d->client->sock, POLLIN|POLLPRI, 0 };
-
-    while(poll(&pollfd, 1, 0) == 1) {
-
-        if(d->client->state == rfbClientRec::RFB_INITIALISATION) {
-            d->notifier->setEnabled(false);
-            //Client is Authenticated
-            processNewClient();
-            break;
-        }
-        rfbProcessClientMessage(d->client);
-
-        //This is how we handle disconnection.
-        //if rfbProcessClientMessage() finds out that it can't read the socket,
-        //it closes it and sets it to -1. So, we just have to check this here
-        //and call rfbClientConnectionGone() if necessary. This will call
-        //the clientGoneHook which in turn will remove this RfbClient instance
-        //from the server manager and will call deleteLater() to delete it
-        if (d->client->sock == -1) {
-            qCDebug(KRFB) << "disconnected from socket signal";
-            d->notifier->setEnabled(false);
-            rfbClientConnectionGone(d->client);
-            break;
-        }
-    }
-}
-
 bool PendingInvitationsRfbClient::checkPassword(const QByteArray & encryptedPassword)
 {
     qCDebug(KRFB) << "about to start authentication";
diff --git a/krfb/invitationsrfbclient.h b/krfb/invitationsrfbclient.h
index 3e4fe51a..40bc271e 100644
--- a/krfb/invitationsrfbclient.h
+++ b/krfb/invitationsrfbclient.h
@@ -37,7 +37,6 @@ public:
 
 protected Q_SLOTS:
     void processNewClient() override;
-    virtual void onSocketActivated();
     bool checkPassword(const QByteArray & encryptedPassword) override;
 
 private Q_SLOTS:
diff --git a/krfb/invitationsrfbserver.cpp b/krfb/invitationsrfbserver.cpp
index 074db829..1fb21dc6 100644
--- a/krfb/invitationsrfbserver.cpp
+++ b/krfb/invitationsrfbserver.cpp
@@ -21,7 +21,6 @@
 #include "invitationsrfbserver.h"
 #include "invitationsrfbclient.h"
 #include "krfbconfig.h"
-#include "rfbservermanager.h"
 #include "krfbdebug.h"
 #include <QTimer>
 #include <QApplication>
diff --git a/krfb/rfbclient.cpp b/krfb/rfbclient.cpp
index 95d99981..6a40b3b3 100644
--- a/krfb/rfbclient.cpp
+++ b/krfb/rfbclient.cpp
@@ -170,8 +170,13 @@ void RfbClient::update()
 
 PendingRfbClient::PendingRfbClient(rfbClientPtr client, QObject *parent)
     : QObject(parent), m_rfbClient(client)
+    , m_notifier(new QSocketNotifier(client->sock, QSocketNotifier::Read, this))
 {
     m_rfbClient->clientData = this;
+
+    m_notifier->setEnabled(true);
+    connect(m_notifier, &QSocketNotifier::activated,
+            this, &PendingRfbClient::onSocketActivated);
 }
 
 PendingRfbClient::~PendingRfbClient()
@@ -230,3 +235,35 @@ bool PendingRfbClient::vncAuthCheckPassword(const QByteArray& password, const QB
     rfbEncryptBytes(challenge, passwd);
     return memcmp(challenge, encryptedPassword.constData(), encryptedPassword.size()) == 0;
 }
+
+void PendingRfbClient::onSocketActivated()
+{
+    //Process not only one, but all pending messages.
+    //poll() idea/code copied from vino:
+    // Copyright (C) 2003 Sun Microsystems, Inc.
+    // License: GPL v2 or later
+    struct pollfd pollfd = { m_rfbClient->sock, POLLIN|POLLPRI, 0 };
+
+    while(poll(&pollfd, 1, 0) == 1) {
+        if(m_rfbClient->state == rfbClientRec::RFB_INITIALISATION) {
+            m_notifier->setEnabled(false);
+            //Client is Authenticated
+            processNewClient();
+            break;
+        }
+        rfbProcessClientMessage(m_rfbClient);
+
+        //This is how we handle disconnection.
+        //if rfbProcessClientMessage() finds out that it can't read the socket,
+        //it closes it and sets it to -1. So, we just have to check this here
+        //and call rfbClientConnectionGone() if necessary. This will call
+        //the clientGoneHook which in turn will remove this RfbClient instance
+        //from the server manager and will call deleteLater() to delete it
+        if (m_rfbClient->sock == -1) {
+            qCDebug(KRFB) << "disconnected from socket signal";
+            m_notifier->setEnabled(false);
+            rfbClientConnectionGone(m_rfbClient);
+            break;
+        }
+    }
+}
diff --git a/krfb/rfbclient.h b/krfb/rfbclient.h
index 4b3b02df..030ac742 100644
--- a/krfb/rfbclient.h
+++ b/krfb/rfbclient.h
@@ -23,6 +23,8 @@
 #include "rfb.h"
 #include <QObject>
 
+class QSocketNotifier;
+
 class RfbClient : public QObject
 {
     Q_OBJECT
@@ -104,6 +106,11 @@ protected:
     bool vncAuthCheckPassword(const QByteArray & password, const QByteArray & encryptedPassword) const;
 
     rfbClientPtr m_rfbClient;
+
+private:
+    void onSocketActivated();
+
+    QSocketNotifier *const m_notifier;
 };
 
 #endif // RFBCLIENT_H