diff --git a/server/render_client.c b/server/render_client.c
index ee75095..3b12d79 100644
--- a/server/render_client.c
+++ b/server/render_client.c
@@ -51,9 +51,8 @@ render_client_detach_all_records(struct render_client *client)
    list_splicetail(&client->context_records, &client->reap_records);
    list_for_each_entry_safe (struct render_context_record, rec, &client->reap_records,
                              head) {
-      render_worker_destroy(rec->worker);
+      render_worker_destroy(srv->worker_jail, rec->worker);
       free(rec);
-      srv->current_worker_count--;
    }
 
    list_inithead(&client->context_records);
@@ -90,11 +89,9 @@ render_client_reap_all_records(struct render_client *client, bool wait)
       if (!render_worker_reap(rec->worker, wait))
          continue;
 
-      render_worker_destroy(rec->worker);
+      render_worker_destroy(srv->worker_jail, rec->worker);
       list_del(&rec->head);
       free(rec);
-
-      srv->current_worker_count--;
    }
 }
 
@@ -133,11 +130,6 @@ render_client_create_context(struct render_client *client,
 {
    struct render_server *srv = client->server;
 
-   if (srv->current_worker_count >= srv->max_worker_count) {
-      render_log("too many context workers");
-      return false;
-   }
-
    struct render_context_record *rec = calloc(1, sizeof(*rec));
    if (!rec)
       return false;
@@ -171,7 +163,6 @@ render_client_create_context(struct render_client *client,
 
    rec->ctx_id = req->ctx_id;
    list_addtail(&rec->head, &client->context_records);
-   srv->current_worker_count++;
 
    if (!render_worker_is_record(rec->worker)) {
       /* this is the child process */
diff --git a/server/render_server.c b/server/render_server.c
index b38df26..f4e1d7c 100644
--- a/server/render_server.c
+++ b/server/render_server.c
@@ -31,7 +31,6 @@ render_server_fini(struct render_server *srv)
 {
    if (srv->client)
       render_client_destroy(srv->client);
-   assert(srv->current_worker_count == 0);
 
    if (srv->worker_jail)
       render_worker_jail_destroy(srv->worker_jail);
@@ -109,7 +108,6 @@ render_server_init(struct render_server *srv,
    srv->state = RENDER_SERVER_STATE_RUN;
    srv->context_args = ctx_args;
    srv->client_fd = -1;
-   srv->max_worker_count = RENDER_SERVER_MAX_WORKER_COUNT;
 
    if (!render_server_parse_options(srv, argc, argv))
       return false;
@@ -128,7 +126,8 @@ render_server_init(struct render_server *srv,
       seccomp_path = srv->worker_seccomp_minijail_policy;
    }
 
-   srv->worker_jail = render_worker_jail_create(seccomp_filter, seccomp_path);
+   srv->worker_jail = render_worker_jail_create(RENDER_SERVER_MAX_WORKER_COUNT,
+                                                seccomp_filter, seccomp_path);
    if (!srv->worker_jail) {
       render_log("failed to create worker jail");
       goto fail;
diff --git a/server/render_server.h b/server/render_server.h
index ec74c0a..ba71ce2 100644
--- a/server/render_server.h
+++ b/server/render_server.h
@@ -26,8 +26,6 @@ struct render_server {
    bool worker_seccomp_minijail_log;
 
    struct render_worker_jail *worker_jail;
-   int max_worker_count;
-   int current_worker_count;
 
    /* only one client in the current design */
    struct render_client *client;
diff --git a/server/render_worker.c b/server/render_worker.c
index 922df0a..c59d8da 100644
--- a/server/render_worker.c
+++ b/server/render_worker.c
@@ -31,7 +31,11 @@
 struct minijail;
 
 struct render_worker_jail {
+   int max_worker_count;
+
    struct minijail *minijail;
+
+   int worker_count;
 };
 
 struct render_worker {
@@ -148,14 +152,33 @@ fork_minijail(const struct minijail *template)
 
 #endif /* ENABLE_RENDER_SERVER_WORKER_MINIJAIL */
 
+static void
+render_worker_jail_add_worker(struct render_worker_jail *jail,
+                              UNUSED struct render_worker *worker)
+{
+   jail->worker_count++;
+}
+
+static void
+render_worker_jail_remove_worker(struct render_worker_jail *jail,
+                                 struct render_worker *worker)
+{
+   jail->worker_count--;
+
+   free(worker);
+}
+
 struct render_worker_jail *
-render_worker_jail_create(enum render_worker_jail_seccomp_filter seccomp_filter,
+render_worker_jail_create(int max_worker_count,
+                          enum render_worker_jail_seccomp_filter seccomp_filter,
                           const char *seccomp_path)
 {
    struct render_worker_jail *jail = calloc(1, sizeof(*jail));
    if (!jail)
       return NULL;
 
+   jail->max_worker_count = max_worker_count;
+
 #if defined(ENABLE_RENDER_SERVER_WORKER_MINIJAIL)
    jail->minijail = create_minijail(seccomp_filter, seccomp_path);
    if (!jail->minijail)
@@ -177,6 +200,8 @@ fail:
 void
 render_worker_jail_destroy(struct render_worker_jail *jail)
 {
+   assert(!jail->worker_count);
+
 #if defined(ENABLE_RENDER_SERVER_WORKER_MINIJAIL)
    minijail_destroy(jail->minijail);
 #endif
@@ -190,6 +215,11 @@ render_worker_create(struct render_worker_jail *jail,
                      void *thread_data,
                      size_t thread_data_size)
 {
+   if (jail->worker_count >= jail->max_worker_count) {
+      render_log("too many workers");
+      return NULL;
+   }
+
    struct render_worker *worker = calloc(1, sizeof(*worker) + thread_data_size);
    if (!worker)
       return NULL;
@@ -200,11 +230,9 @@ render_worker_create(struct render_worker_jail *jail,
 #if defined(ENABLE_RENDER_SERVER_WORKER_PROCESS)
    worker->pid = fork();
    ok = worker->pid >= 0;
-   (void)jail;
    (void)thread_func;
 #elif defined(ENABLE_RENDER_SERVER_WORKER_THREAD)
    ok = thrd_create(&worker->thread, thread_func, worker->thread_data) == thrd_success;
-   (void)jail;
 #elif defined(ENABLE_RENDER_SERVER_WORKER_MINIJAIL)
    worker->pid = fork_minijail(jail->minijail);
    ok = worker->pid >= 0;
@@ -215,6 +243,8 @@ render_worker_create(struct render_worker_jail *jail,
       return NULL;
    }
 
+   render_worker_jail_add_worker(jail, worker);
+
    return worker;
 }
 
@@ -265,7 +295,7 @@ render_worker_reap(struct render_worker *worker, bool wait)
 }
 
 void
-render_worker_destroy(struct render_worker *worker)
+render_worker_destroy(struct render_worker_jail *jail, struct render_worker *worker)
 {
-   free(worker);
+   render_worker_jail_remove_worker(jail, worker);
 }
diff --git a/server/render_worker.h b/server/render_worker.h
index 158dc7d..4d2d0fb 100644
--- a/server/render_worker.h
+++ b/server/render_worker.h
@@ -19,7 +19,8 @@ enum render_worker_jail_seccomp_filter {
 };
 
 struct render_worker_jail *
-render_worker_jail_create(enum render_worker_jail_seccomp_filter seccomp_filter,
+render_worker_jail_create(int max_worker_count,
+                          enum render_worker_jail_seccomp_filter seccomp_filter,
                           const char *seccomp_path);
 
 void
@@ -41,6 +42,6 @@ bool
 render_worker_reap(struct render_worker *worker, bool wait);
 
 void
-render_worker_destroy(struct render_worker *worker);
+render_worker_destroy(struct render_worker_jail *jail, struct render_worker *worker);
 
 #endif /* RENDER_WORKER_H */