diff --git a/src/IRCServer.py b/src/IRCServer.py
index b23d03b0..1454fedb 100644
--- a/src/IRCServer.py
+++ b/src/IRCServer.py
@@ -40,7 +40,6 @@ class Server(IRCObject.Object):
         self.batches = {} # type: typing.Dict[str, IRCLine.IRCBatch]
 
         self.users = {} # type: typing.Dict[str, IRCUser.User]
-        self.new_users = set([]) #type: typing.Set[IRCUser.User]
         self.channels = IRCChannels.Channels(self, self.bot, self.events)
         self.own_modes = {} # type: typing.Dict[str, typing.Optional[str]]
 
@@ -176,7 +175,6 @@ class Server(IRCObject.Object):
             user_id = self.get_user_id(nickname)
             new_user = IRCUser.User(nickname, user_id, self, self.bot)
             self.users[new_user.nickname_lower] = new_user
-            self.new_users.add(new_user)
 
         user = self.users.get(self.irc_lower(nickname), None)
         if user:
@@ -204,6 +202,12 @@ class Server(IRCObject.Object):
         for channel in user.channels:
             channel.remove_user(user)
 
+    def quit_user(self, user: IRCUser.User):
+        self.remove_user(user)
+    def part_user(self, channel: IRCChannel.Channel, user: IRCUser.User):
+        user.part_channel(channel)
+        channel.remove_user(user)
+
     def is_channel(self, name: str) -> bool:
         return name[0] in self.channel_types
 
@@ -234,10 +238,12 @@ class Server(IRCObject.Object):
                 line=IRCLine.parse_line(line))
             self.check_users()
     def check_users(self):
-        for user in self.new_users:
+        prune: typing.List[IRCUser.User] = []
+        for user in self.users.values():
             if not len(user.channels):
-                self.remove_user(user)
-        self.new_users.clear()
+                prune.append(user)
+        for user in prune:
+            self.remove_user(user)
 
     def until_next_ping(self) -> typing.Optional[float]:
         if self.ping_sent:
diff --git a/src/core_modules/line_handler/channel.py b/src/core_modules/line_handler/channel.py
index ce3ca90d..1d1d5d61 100644
--- a/src/core_modules/line_handler/channel.py
+++ b/src/core_modules/line_handler/channel.py
@@ -99,10 +99,7 @@ def part(events, event):
     user = event["server"].get_user(event["line"].source.nickname)
     reason = event["line"].args.get(1)
 
-    channel.remove_user(user)
-    user.part_channel(channel)
-    if not len(user.channels):
-        event["server"].remove_user(user)
+    event["server"].part_user(channel, user)
 
     if not event["server"].is_own_nickname(event["line"].source.nickname):
         events.on("received.part").call(channel=channel, reason=reason,
@@ -148,8 +145,6 @@ def kick(events, event):
 
     channel.remove_user(target_user)
     target_user.part_channel(channel)
-    if not len(target_user.channels):
-        event["server"].remove_user(target_user)
 
 def rename(events, event):
     old_name = event["line"].args[0]
diff --git a/src/core_modules/line_handler/user.py b/src/core_modules/line_handler/user.py
index 765d1da8..a5d407fb 100644
--- a/src/core_modules/line_handler/user.py
+++ b/src/core_modules/line_handler/user.py
@@ -27,9 +27,9 @@ def quit(events, event):
         if (not event["server"].is_own_nickname(nickname) and
                 not event["line"].source.hostmask == "*"):
             user = event["server"].get_user(nickname)
+            event["server"].quit_user(user)
             events.on("received.quit").call(reason=reason, user=user,
                 server=event["server"])
-            event["server"].remove_user(user)
         else:
             event["server"].disconnect()
     else: