diff --git a/authd/provider.c b/authd/provider.c
index 3b64c55a..b058928e 100644
--- a/authd/provider.c
+++ b/authd/provider.c
@@ -87,9 +87,13 @@ destroy_providers(void)
 	/* Cancel outstanding connections */
 	RB_DICTIONARY_FOREACH(auth, &iter, auth_clients)
 	{
+		auth_client_ref(auth);
+
 		/* TBD - is this the right thing? */
 		reject_client(auth, UINT32_MAX, "destroy",
 			"Authentication system is down... try reconnecting in a few seconds");
+
+		auth_client_unref(auth);
 	}
 
 	RB_DLINK_FOREACH_SAFE(ptr, nptr, auth_providers.head)
@@ -187,7 +191,7 @@ cancel_providers(struct auth_client *auth)
 
 	auth->providers_cancelled = true;
 
-	if(auth->refcount > 0)
+	if(auth->providers_active > 0)
 	{
 		rb_dlink_node *ptr;
 
@@ -215,7 +219,7 @@ provider_done(struct auth_client *auth, uint32_t id)
 
 	set_provider_done(auth, id);
 
-	if(auth->refcount == 0 && !auth->providers_starting)
+	if(auth->providers_active == 0 && !auth->providers_starting)
 	{
 		/* All done */
 		accept_client(auth, UINT32_MAX);
@@ -237,7 +241,6 @@ provider_done(struct auth_client *auth, uint32_t id)
 void
 reject_client(struct auth_client *auth, uint32_t id, const char *data, const char *fmt, ...)
 {
-	unsigned int refcount = auth->refcount;
 	char buf[BUFSIZE];
 	va_list args;
 
@@ -265,8 +268,6 @@ reject_client(struct auth_client *auth, uint32_t id, const char *data, const cha
 void
 accept_client(struct auth_client *auth, uint32_t id)
 {
-	unsigned int refcount = auth->refcount;
-
 	rb_helper_write(authd_helper, "A %x %s %s", auth->cid, auth->username, auth->hostname);
 
 	if(id != UINT32_MAX)
@@ -287,6 +288,7 @@ start_auth(const char *cid, const char *l_ip, const char *l_port, const char *c_
 		return;
 
 	auth = rb_malloc(sizeof(struct auth_client));
+	auth_client_ref(auth);
 	auth->cid = (uint32_t)lcid;
 
 	if(rb_dictionary_find(auth_clients, RB_UINT_TO_POINTER(auth->cid)) == NULL)
@@ -328,7 +330,7 @@ start_auth(const char *cid, const char *l_ip, const char *l_port, const char *c_
 		/* Execute providers */
 		if(!provider->start(auth))
 			/* Rejected immediately */
-			return;
+			goto done;
 
 		if(auth->providers_cancelled)
 			break;
@@ -336,8 +338,11 @@ start_auth(const char *cid, const char *l_ip, const char *l_port, const char *c_
 	auth->providers_starting = false;
 
 	/* If no providers are running, accept the client */
-	if(auth->refcount == 0)
+	if(auth->providers_active == 0)
 		accept_client(auth, UINT32_MAX);
+
+done:
+	auth_client_unref(auth);
 }
 
 /* Callback for the initiation */
@@ -379,7 +384,9 @@ handle_cancel_connection(int parc, char *parv[])
 		return;
 	}
 
+	auth_client_ref(auth);
 	cancel_providers(auth);
+	auth_client_unref(auth);
 }
 
 static void
@@ -393,6 +400,8 @@ provider_timeout_event(void *notused __unused)
 	{
 		rb_dlink_node *ptr;
 
+		auth_client_ref(auth);
+
 		RB_DLINK_FOREACH(ptr, auth_providers.head)
 		{
 			struct auth_provider *provider = ptr->data;
@@ -404,5 +413,7 @@ provider_timeout_event(void *notused __unused)
 				provider->timeout(auth);
 			}
 		}
+
+		auth_client_unref(auth);
 	}
 }
diff --git a/authd/provider.h b/authd/provider.h
index fc7d5fea..fa606d50 100644
--- a/authd/provider.h
+++ b/authd/provider.h
@@ -59,6 +59,7 @@ struct auth_client
 
 	bool providers_starting;		/* Providers are still warming up */
 	bool providers_cancelled;		/* Providers are being cancelled */
+	unsigned int providers_active;		/* Number of active providers */
 	unsigned int refcount;			/* Held references */
 
 	struct auth_client_data *data;		/* Provider-specific data */
@@ -188,7 +189,7 @@ set_provider_status(struct auth_client *auth, uint32_t provider, provider_status
 static inline void
 set_provider_running(struct auth_client *auth, uint32_t provider)
 {
-	auth_client_ref(auth);
+	auth->providers_active++;
 	set_provider_status(auth, provider, PROVIDER_STATUS_RUNNING);
 }
 
@@ -198,7 +199,7 @@ static inline void
 set_provider_done(struct auth_client *auth, uint32_t provider)
 {
 	set_provider_status(auth, provider, PROVIDER_STATUS_DONE);
-	auth_client_unref(auth);
+	auth->providers_active--;
 }
 
 /* Check if provider is operating on this auth client */
diff --git a/authd/providers/blacklist.c b/authd/providers/blacklist.c
index 769a34a4..b0522c70 100644
--- a/authd/providers/blacklist.c
+++ b/authd/providers/blacklist.c
@@ -266,6 +266,8 @@ blacklist_dns_callback(const char *result, bool status, query_type type, void *d
 		set_provider_data(auth, SELF_PID, NULL);
 		set_provider_timeout_absolute(auth, SELF_PID, 0);
 		provider_done(auth, SELF_PID);
+
+		auth_client_unref(auth);
 	}
 }
 
@@ -352,6 +354,8 @@ blacklists_start(struct auth_client *auth)
 		/* Nothing to do... */
 		return true;
 
+	auth_client_ref(auth);
+
 	set_provider_data(auth, SELF_PID, rb_malloc(sizeof(struct blacklist_user)));
 
 	if((!get_provider_id("rdns", &rdns_pid) || is_provider_done(auth, rdns_pid)) &&
@@ -416,6 +420,8 @@ blacklists_cancel(struct auth_client *auth)
 	set_provider_data(auth, SELF_PID, NULL);
 	set_provider_timeout_absolute(auth, SELF_PID, 0);
 	provider_done(auth, SELF_PID);
+
+	auth_client_unref(auth);
 }
 
 static void
@@ -427,6 +433,7 @@ blacklists_destroy(void)
 	RB_DICTIONARY_FOREACH(auth, &iter, auth_clients)
 	{
 		blacklists_cancel(auth);
+		/* auth is now invalid as we have no reference */
 	}
 
 	delete_all_blacklists();
diff --git a/authd/providers/ident.c b/authd/providers/ident.c
index 959bdb57..e6f08b19 100644
--- a/authd/providers/ident.c
+++ b/authd/providers/ident.c
@@ -188,6 +188,8 @@ client_fail(struct auth_client *auth, ident_message report)
 
 	notice_client(auth->cid, messages[report]);
 	provider_done(auth, SELF_PID);
+
+	auth_client_unref(auth);
 }
 
 static void
@@ -206,6 +208,8 @@ client_success(struct auth_client *auth)
 
 	notice_client(auth->cid, messages[REPORT_FOUND]);
 	provider_done(auth, SELF_PID);
+
+	auth_client_unref(auth);
 }
 
 /* get_valid_ident
@@ -283,6 +287,7 @@ ident_destroy(void)
 	{
 		if(get_provider_data(auth, SELF_PID) != NULL)
 			client_fail(auth, REPORT_FAIL);
+		/* auth is now invalid as we have no reference */
 	}
 }
 
@@ -303,6 +308,8 @@ ident_start(struct auth_client *auth)
 		return true;
 	}
 
+	auth_client_ref(auth);
+
 	notice_client(auth->cid, messages[REPORT_LOOKUP]);
 
 	set_provider_data(auth, SELF_PID, query);
diff --git a/authd/providers/opm.c b/authd/providers/opm.c
index 560eecee..8087bc97 100644
--- a/authd/providers/opm.c
+++ b/authd/providers/opm.c
@@ -563,6 +563,7 @@ create_listener(const char *ip, uint16_t port)
 	RB_DICTIONARY_FOREACH(auth, &iter, auth_clients)
 	{
 		opm_cancel(auth);
+		/* auth is now invalid as we have no reference */
 	}
 
 	/* Copy data */
@@ -631,6 +632,8 @@ opm_start(struct auth_client *auth)
 		/* Nothing to do... */
 		return true;
 
+	auth_client_ref(auth);
+
 	set_provider_data(auth, SELF_PID, rb_malloc(sizeof(struct opm_lookup)));
 
 	if((!get_provider_id("rdns", &rdns_pid) || is_provider_done(auth, rdns_pid)) &&
@@ -668,6 +671,8 @@ opm_cancel(struct auth_client *auth)
 		set_provider_data(auth, SELF_PID, NULL);
 		set_provider_timeout_absolute(auth, SELF_PID, 0);
 		provider_done(auth, SELF_PID);
+
+		auth_client_unref(auth);
 	}
 }
 
@@ -681,6 +686,7 @@ opm_destroy(void)
 	RB_DICTIONARY_FOREACH(auth, &iter, auth_clients)
 	{
 		opm_cancel(auth);
+		/* auth is now invalid as we have no reference */
 	}
 }
 
@@ -723,6 +729,7 @@ set_opm_enabled(const char *key __unused, int parc __unused, const char **parv)
 			RB_DICTIONARY_FOREACH(auth, &iter, auth_clients)
 			{
 				opm_cancel(auth);
+				/* auth is now invalid as we have no reference */
 			}
 		}
 	}
@@ -847,6 +854,8 @@ delete_opm_scanner(const char *key __unused, int parc __unused, const char **par
 		if(lookup == NULL)
 			continue;
 
+		auth_client_ref(auth);
+
 		RB_DLINK_FOREACH(ptr, lookup->scans.head)
 		{
 			struct opm_scan *scan = ptr->data;
@@ -863,6 +872,8 @@ delete_opm_scanner(const char *key __unused, int parc __unused, const char **par
 				break;
 			}
 		}
+
+		auth_client_unref(auth);
 	}
 
 	rb_dlinkDelete(&proxy->node, &proxy_scanners);
@@ -888,6 +899,7 @@ delete_opm_scanner_all(const char *key __unused, int parc __unused, const char *
 	RB_DICTIONARY_FOREACH(auth, &iter, auth_clients)
 	{
 		opm_cancel(auth);
+		/* auth is now invalid as we have no reference */
 	}
 
 	opm_enable = false;
diff --git a/authd/providers/rdns.c b/authd/providers/rdns.c
index d263415a..ff0bb078 100644
--- a/authd/providers/rdns.c
+++ b/authd/providers/rdns.c
@@ -90,6 +90,8 @@ client_fail(struct auth_client *auth, dns_message report)
 	set_provider_data(auth, SELF_PID, NULL);
 	set_provider_timeout_absolute(auth, SELF_PID, 0);
 	provider_done(auth, SELF_PID);
+
+	auth_client_unref(auth);
 }
 
 static void
@@ -107,6 +109,8 @@ client_success(struct auth_client *auth)
 	set_provider_data(auth, SELF_PID, NULL);
 	set_provider_timeout_absolute(auth, SELF_PID, 0);
 	provider_done(auth, SELF_PID);
+
+	auth_client_unref(auth);
 }
 
 static void
@@ -119,6 +123,7 @@ rdns_destroy(void)
 	{
 		if(get_provider_data(auth, SELF_PID) != NULL)
 			client_fail(auth, REPORT_FAIL);
+		/* auth is now invalid as we have no reference */
 	}
 }
 
@@ -127,6 +132,8 @@ rdns_start(struct auth_client *auth)
 {
 	struct user_query *query = rb_malloc(sizeof(struct user_query));
 
+	auth_client_ref(auth);
+
 	set_provider_data(auth, SELF_PID, query);
 	set_provider_timeout_relative(auth, SELF_PID, rdns_timeout);