virgl: add proxy context

A proxy context is a virgl_context that forwards context method calls to
a remote process.

On virgl_renderer_init, it starts the render server and connects to it.
On each virgl_renderer_context_create, it requests the render server to
fork a context process such that it can forward context method calls to
the context process.

Signed-off-by: Chia-I Wu <olvaffe@gmail.com>
Reviewed-by: Yiwei Zhang <zzyiwei@chromium.org>
Reviewed-by: Ryan Neph <ryanneph@google.com>

src/meson.build

@@ -133,6 +133,15 @@ venus_codegen = custom_target(
    command : [prog_python, '@INPUT0@', '-o', '@OUTDIR@', '@INPUT1@'],
 )
 
+proxy_sources = [
+   'proxy/proxy_client.c',
+   'proxy/proxy_common.c',
+   'proxy/proxy_context.c',
+   'proxy/proxy_renderer.c',
+   'proxy/proxy_server.c',
+   'proxy/proxy_socket.c',
+]
+
 virgl_depends = [
    gallium_dep,
    epoxy_dep,
@@ -167,6 +176,10 @@ if with_venus
    virgl_depends += [venus_dep]
 endif
 
+if with_render_server
+   virgl_sources += proxy_sources
+endif
+
 libvirgl = static_library(
    'virgl',
    virgl_sources,

src/proxy/proxy_client.c

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "proxy_client.h"
+
+#include <unistd.h>
+
+#include "server/render_protocol.h"
+
+#include "proxy_server.h"
+
+bool
+proxy_client_destroy_context(struct proxy_client *client, uint32_t ctx_id)
+{
+   const struct render_client_op_destroy_context_request req = {
+      .header.op = RENDER_CLIENT_OP_DESTROY_CONTEXT,
+      .ctx_id = ctx_id,
+   };
+
+   return proxy_socket_send_request(&client->socket, &req, sizeof(req));
+}
+
+bool
+proxy_client_create_context(struct proxy_client *client,
+                            uint32_t ctx_id,
+                            size_t ctx_name_len,
+                            const char *ctx_name,
+                            int *out_ctx_fd)
+{
+   struct render_client_op_create_context_request req = {
+      .header.op = RENDER_CLIENT_OP_CREATE_CONTEXT,
+      .ctx_id = ctx_id,
+   };
+
+   const size_t len = MIN2(ctx_name_len, sizeof(req.ctx_name) - 1);
+   memcpy(req.ctx_name, ctx_name, len);
+
+   if (!proxy_socket_send_request(&client->socket, &req, sizeof(req)))
+      return false;
+
+   struct render_client_op_create_context_reply reply;
+   int fd_count;
+   int ctx_fd;
+   if (!proxy_socket_receive_reply_with_fds(&client->socket, &reply, sizeof(reply),
+                                            &ctx_fd, 1, &fd_count))
+      return false;
+
+   if (reply.ok != fd_count) {
+      if (fd_count)
+         close(ctx_fd);
+      return false;
+   } else if (!reply.ok) {
+      return false;
+   }
+
+   if (!proxy_socket_is_seqpacket(ctx_fd)) {
+      close(ctx_fd);
+      return false;
+   }
+
+   *out_ctx_fd = ctx_fd;
+   return true;
+}
+
+bool
+proxy_client_reset(struct proxy_client *client)
+{
+   const struct render_client_op_reset_request req = {
+      .header.op = RENDER_CLIENT_OP_RESET,
+   };
+   return proxy_socket_send_request(&client->socket, &req, sizeof(req));
+}
+
+void
+proxy_client_destroy(struct proxy_client *client)
+{
+   proxy_socket_fini(&client->socket);
+   free(client);
+}
+
+static bool
+proxy_client_init(struct proxy_client *client, uint32_t flags)
+{
+   const struct render_client_op_init_request req = {
+      .header.op = RENDER_CLIENT_OP_INIT,
+      .flags = flags,
+   };
+   return proxy_socket_send_request(&client->socket, &req, sizeof(req));
+}
+
+struct proxy_client *
+proxy_client_create(struct proxy_server *srv, uint32_t flags)
+{
+   struct proxy_client *client = calloc(1, sizeof(*client));
+   if (!client)
+      return NULL;
+
+   const int client_fd = proxy_server_connect(srv);
+   if (client_fd < 0) {
+      free(client);
+      return NULL;
+   }
+
+   proxy_socket_init(&client->socket, client_fd);
+
+   if (!proxy_client_init(client, flags)) {
+      proxy_socket_fini(&client->socket);
+      free(client);
+      return NULL;
+   }
+
+   return client;
+}

src/proxy/proxy_client.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef PROXY_CLIENT_H
+#define PROXY_CLIENT_H
+
+#include "proxy_common.h"
+
+struct proxy_client {
+   struct proxy_socket socket;
+};
+
+struct proxy_client *
+proxy_client_create(struct proxy_server *srv, uint32_t flags);
+
+void
+proxy_client_destroy(struct proxy_client *client);
+
+bool
+proxy_client_reset(struct proxy_client *client);
+
+bool
+proxy_client_create_context(struct proxy_client *client,
+                            uint32_t ctx_id,
+                            size_t ctx_name_len,
+                            const char *ctx_name,
+                            int *out_ctx_fd);
+
+bool
+proxy_client_destroy_context(struct proxy_client *client, uint32_t ctx_id);
+
+#endif /* PROXY_CLIENT_H */

src/proxy/proxy_common.c

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "proxy_common.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+
+struct proxy_renderer proxy_renderer;
+
+void
+proxy_log(const char *fmt, ...)
+{
+   va_list va;
+
+   va_start(va, fmt);
+
+   fprintf(stderr, "proxy: ");
+   vfprintf(stderr, fmt, va);
+   fprintf(stderr, "\n");
+
+   va_end(va);
+}

src/proxy/proxy_common.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef PROXY_COMMON_H
+#define PROXY_COMMON_H
+
+#include <assert.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <stdatomic.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "util/list.h"
+#include "util/macros.h"
+#include "virgl_util.h"
+#include "virglrenderer.h"
+
+#include "proxy_renderer.h"
+#include "proxy_socket.h"
+
+struct proxy_client;
+struct proxy_context;
+struct proxy_server;
+struct proxy_socket;
+
+struct proxy_renderer {
+   const struct proxy_renderer_cbs *cbs;
+   uint32_t flags;
+
+   struct proxy_server *server;
+   struct proxy_client *client;
+};
+
+extern struct proxy_renderer proxy_renderer;
+
+void
+proxy_log(const char *fmt, ...);
+
+#endif /* PROXY_COMMON_H */

src/proxy/proxy_context.c

@@ -0,0 +1,614 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "proxy_context.h"
+
+#include <fcntl.h>
+#include <poll.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "server/render_protocol.h"
+#include "util/anon_file.h"
+#include "util/bitscan.h"
+
+#include "proxy_client.h"
+
+struct proxy_fence {
+   uint32_t flags;
+   uint32_t seqno;
+   void *cookie;
+   struct list_head head;
+};
+
+static bool
+proxy_fence_is_signaled(const struct proxy_fence *fence, uint32_t cur_seqno)
+{
+   /* takes wrapping into account */
+   const uint32_t d = cur_seqno - fence->seqno;
+   return d < INT32_MAX;
+}
+
+static struct proxy_fence *
+proxy_context_alloc_fence(struct proxy_context *ctx)
+{
+   struct proxy_fence *fence = NULL;
+
+   if (proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB)
+      mtx_lock(&ctx->free_fences_mutex);
+
+   if (!list_is_empty(&ctx->free_fences)) {
+      fence = list_first_entry(&ctx->free_fences, struct proxy_fence, head);
+      list_del(&fence->head);
+   }
+
+   if (proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB)
+      mtx_unlock(&ctx->free_fences_mutex);
+
+   return fence ? fence : malloc(sizeof(*fence));
+}
+
+static void
+proxy_context_free_fence(struct proxy_context *ctx, struct proxy_fence *fence)
+{
+   if (proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB)
+      mtx_lock(&ctx->free_fences_mutex);
+
+   list_add(&fence->head, &ctx->free_fences);
+
+   if (proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB)
+      mtx_unlock(&ctx->free_fences_mutex);
+}
+
+static uint32_t
+proxy_context_load_timeline_seqno(struct proxy_context *ctx, uint32_t ring_idx)
+{
+   return atomic_load(&ctx->timeline_seqnos[ring_idx]);
+}
+
+static bool
+proxy_context_retire_timeline_fences_locked(struct proxy_context *ctx,
+                                            uint32_t ring_idx,
+                                            uint32_t cur_seqno)
+{
+   struct proxy_timeline *timeline = &ctx->timelines[ring_idx];
+   bool force_retire_all = false;
+
+   /* check if the socket has been disconnected (i.e., the other end has
+    * crashed) if no progress is made after a while
+    */
+   if (timeline->cur_seqno == cur_seqno && !list_is_empty(&timeline->fences)) {
+      timeline->cur_seqno_stall_count++;
+      if (timeline->cur_seqno_stall_count < 100 ||
+          proxy_socket_is_connected(&ctx->socket))
+         return false;
+
+      /* socket has been disconnected */
+      force_retire_all = true;
+   }
+
+   timeline->cur_seqno = cur_seqno;
+   timeline->cur_seqno_stall_count = 0;
+
+   list_for_each_entry_safe(struct proxy_fence, fence, &timeline->fences, head)
+   {
+      if (!proxy_fence_is_signaled(fence, timeline->cur_seqno) && !force_retire_all)
+         return false;
+
+      ctx->base.fence_retire(&ctx->base, ring_idx, fence->cookie);
+
+      list_del(&fence->head);
+      proxy_context_free_fence(ctx, fence);
+   }
+
+   return true;
+}
+
+static void
+proxy_context_retire_fences_internal(struct proxy_context *ctx)
+{
+   if (ctx->sync_thread.fence_eventfd >= 0)
+      flush_eventfd(ctx->sync_thread.fence_eventfd);
+
+   if (proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB)
+      mtx_lock(&ctx->timeline_mutex);
+
+   uint64_t new_busy_mask = 0;
+   uint64_t old_busy_mask = ctx->timeline_busy_mask;
+   while (old_busy_mask) {
+      const uint32_t ring_idx = u_bit_scan64(&old_busy_mask);
+      const uint32_t cur_seqno = proxy_context_load_timeline_seqno(ctx, ring_idx);
+      if (!proxy_context_retire_timeline_fences_locked(ctx, ring_idx, cur_seqno))
+         new_busy_mask |= 1 << ring_idx;
+   }
+
+   ctx->timeline_busy_mask = new_busy_mask;
+
+   if (proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB)
+      mtx_unlock(&ctx->timeline_mutex);
+}
+
+static int
+proxy_context_sync_thread(void *arg)
+{
+   struct proxy_context *ctx = arg;
+   struct pollfd poll_fds[2] = {
+      [0] = {
+         .fd = ctx->sync_thread.fence_eventfd,
+         .events = POLLIN,
+      },
+      [1] = {
+         .fd = ctx->socket.fd,
+      },
+   };
+
+   assert(proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB);
+
+   while (!ctx->sync_thread.stop) {
+      const int ret = poll(poll_fds, ARRAY_SIZE(poll_fds), -1);
+      if (ret <= 0) {
+         if (ret < 0 && (errno == EINTR || errno == EAGAIN))
+            continue;
+
+         proxy_log("failed to poll fence eventfd");
+         break;
+      }
+
+      proxy_context_retire_fences_internal(ctx);
+   }
+
+   return 0;
+}
+
+static int
+proxy_context_submit_fence(struct virgl_context *base,
+                           uint32_t flags,
+                           uint64_t queue_id,
+                           void *fence_cookie)
+{
+   struct proxy_context *ctx = (struct proxy_context *)base;
+
+   /* TODO fix virglrenderer to match virtio-gpu spec which uses ring_idx */
+   const uint32_t ring_idx = queue_id;
+   if (ring_idx >= PROXY_CONTEXT_TIMELINE_COUNT)
+      return -EINVAL;
+
+   struct proxy_fence *fence = proxy_context_alloc_fence(ctx);
+   if (!fence)
+      return -ENOMEM;
+
+   struct proxy_timeline *timeline = &ctx->timelines[ring_idx];
+   const uint32_t seqno = timeline->next_seqno++;
+   const struct render_context_op_submit_fence_request req = {
+      .header.op = RENDER_CONTEXT_OP_SUBMIT_FENCE,
+      .flags = flags,
+      .ring_index = ring_idx,
+      .seqno = seqno,
+   };
+   if (!proxy_socket_send_request(&ctx->socket, &req, sizeof(req))) {
+      proxy_log("failed to submit fence");
+      proxy_context_free_fence(ctx, fence);
+      return -1;
+   }
+
+   fence->flags = flags;
+   fence->seqno = seqno;
+   fence->cookie = fence_cookie;
+
+   if (proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB)
+      mtx_lock(&ctx->timeline_mutex);
+
+   list_addtail(&fence->head, &timeline->fences);
+   ctx->timeline_busy_mask |= 1 << ring_idx;
+
+   if (proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB)
+      mtx_unlock(&ctx->timeline_mutex);
+
+   return 0;
+}
+
+static void
+proxy_context_retire_fences(struct virgl_context *base)
+{
+   struct proxy_context *ctx = (struct proxy_context *)base;
+
+   assert(!(proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB));
+   proxy_context_retire_fences_internal(ctx);
+}
+
+static int
+proxy_context_get_fencing_fd(struct virgl_context *base)
+{
+   struct proxy_context *ctx = (struct proxy_context *)base;
+
+   assert(!(proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB));
+   return ctx->sync_thread.fence_eventfd;
+}
+
+static int
+proxy_context_submit_cmd(struct virgl_context *base, const void *buffer, size_t size)
+{
+   struct proxy_context *ctx = (struct proxy_context *)base;
+
+   if (!size)
+      return 0;
+
+   struct render_context_op_submit_cmd_request req = {
+      .header.op = RENDER_CONTEXT_OP_SUBMIT_CMD,
+      .size = size,
+   };
+
+   const size_t inlined = MIN2(size, sizeof(req.cmd));
+   memcpy(req.cmd, buffer, inlined);
+
+   if (!proxy_socket_send_request(&ctx->socket, &req, sizeof(req))) {
+      proxy_log("failed to submit cmd");
+      return -1;
+   }
+
+   if (size > inlined) {
+      if (!proxy_socket_send_request(&ctx->socket, (const char *)buffer + inlined,
+                                     size - inlined)) {
+         proxy_log("failed to submit large cmd buffer");
+         return -1;
+      }
+   }
+
+   /* XXX this is forced a roundtrip to avoid surprises; vtest requires this
+    * at least
+    */
+   struct render_context_op_submit_cmd_reply reply;
+   if (!proxy_socket_receive_reply(&ctx->socket, &reply, sizeof(reply))) {
+      proxy_log("failed to get submit result");
+      return -1;
+   }
+
+   return reply.ok ? 0 : -1;
+}
+
+static int
+alloc_memfd(const char *name, size_t size, void **out_ptr)
+{
+   int fd = os_create_anonymous_file(size, name);
+   if (fd < 0)
+      return -1;
+
+   int ret = fcntl(fd, F_ADD_SEALS, F_SEAL_SEAL | F_SEAL_SHRINK | F_SEAL_GROW);
+   if (ret)
+      goto fail;
+
+   if (!out_ptr)
+      return fd;
+
+   void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+   if (ptr == MAP_FAILED)
+      goto fail;
+
+   *out_ptr = ptr;
+   return fd;
+
+fail:
+   close(fd);
+   return -1;
+}
+
+static int
+proxy_context_get_blob(struct virgl_context *base,
+                       uint64_t blob_id,
+                       uint64_t blob_size,
+                       uint32_t blob_flags,
+                       struct virgl_context_blob *blob)
+{
+   struct proxy_context *ctx = (struct proxy_context *)base;
+
+   /* hijack blob_id == 0 && blob_flags == MMAPABLE to save roundtrips */
+   if (!blob_id && blob_flags == VIRGL_RENDERER_BLOB_FLAG_USE_MAPPABLE) {
+      int fd = alloc_memfd("proxy-blob", blob_size, NULL);
+      if (fd < 0)
+         return -ENOMEM;
+
+      blob->type = VIRGL_RESOURCE_FD_SHM;
+      blob->u.fd = fd;
+      blob->map_info = VIRGL_RENDERER_MAP_CACHE_CACHED;
+      return 0;
+   }
+
+   const struct render_context_op_get_blob_request req = {
+      .header.op = RENDER_CONTEXT_OP_GET_BLOB,
+      .blob_id = blob_id,
+      .blob_size = blob_size,
+      .blob_flags = blob_flags,
+   };
+   if (!proxy_socket_send_request(&ctx->socket, &req, sizeof(req))) {
+      proxy_log("failed to get blob %" PRIu64, blob_id);
+      return -1;
+   }
+
+   struct render_context_op_get_blob_reply reply;
+   int reply_fd;
+   int reply_fd_count;
+   if (!proxy_socket_receive_reply_with_fds(&ctx->socket, &reply, sizeof(reply),
+                                            &reply_fd, 1, &reply_fd_count)) {
+      proxy_log("failed to get reply of blob %" PRIu64, blob_id);
+      return -1;
+   }
+
+   if (!reply_fd_count) {
+      proxy_log("invalid reply for blob %" PRIu64, blob_id);
+      return -1;
+   }
+
+   bool reply_fd_valid = false;
+   switch (reply.fd_type) {
+   case VIRGL_RESOURCE_FD_DMABUF:
+      /* TODO validate the fd is dmabuf >= blob_size */
+      reply_fd_valid = true;
+      break;
+   case VIRGL_RESOURCE_FD_OPAQUE:
+      /* this will be validated when imported by the client */
+      reply_fd_valid = true;
+      break;
+   case VIRGL_RESOURCE_FD_SHM:
+      /* we don't expect shm, otherwise we should validate seals and size */
+      reply_fd_valid = false;
+      break;
+   default:
+      break;
+   }
+   if (!reply_fd_valid) {
+      proxy_log("invalid fd type %d for blob %" PRIu64, reply.fd_type, blob_id);
+      close(reply_fd);
+      return -1;
+   }
+
+   blob->type = reply.fd_type;
+   blob->u.fd = reply_fd;
+   blob->map_info = reply.map_info;
+
+   return 0;
+}
+
+static int
+proxy_context_transfer_3d(struct virgl_context *base,
+                          struct virgl_resource *res,
+                          UNUSED const struct vrend_transfer_info *info,
+                          UNUSED int transfer_mode)
+{
+   struct proxy_context *ctx = (struct proxy_context *)base;
+
+   proxy_log("no transfer support for ctx %d and res %d", ctx->base.ctx_id, res->res_id);
+   return -1;
+}
+
+static void
+proxy_context_detach_resource(struct virgl_context *base, struct virgl_resource *res)
+{
+   struct proxy_context *ctx = (struct proxy_context *)base;
+
+   const struct render_context_op_detach_resource_request req = {
+      .header.op = RENDER_CONTEXT_OP_DETACH_RESOURCE,
+      .res_id = res->res_id,
+   };
+   if (!proxy_socket_send_request(&ctx->socket, &req, sizeof(req)))
+      proxy_log("failed to detach res %d", res->res_id);
+}
+
+static void
+proxy_context_attach_resource(struct virgl_context *base, struct virgl_resource *res)
+{
+   struct proxy_context *ctx = (struct proxy_context *)base;
+
+   if (res->iov_count) {
+      proxy_log("failed to attach resource with iov");
+      return;
+   }
+
+   enum virgl_resource_fd_type res_fd_type = res->fd_type;
+   int res_fd = res->fd;
+   bool close_res_fd = false;
+   if (res_fd_type == VIRGL_RESOURCE_FD_INVALID) {
+      res_fd_type = virgl_resource_export_fd(res, &res_fd);
+      if (res_fd_type == VIRGL_RESOURCE_FD_INVALID) {
+         proxy_log("failed to export res %d", res->res_id);
+         return;
+      }
+
+      close_res_fd = true;
+   }
+
+   const struct render_context_op_attach_resource_request req = {
+      .header.op = RENDER_CONTEXT_OP_ATTACH_RESOURCE,
+      .res_id = res->res_id,
+      .fd_type = res_fd_type,
+      .size = res->map_size,
+   };
+   if (!proxy_socket_send_request_with_fds(&ctx->socket, &req, sizeof(req), &res_fd, 1))
+      proxy_log("failed to attach res %d", res->res_id);
+
+   if (res_fd >= 0 && close_res_fd)
+      close(res_fd);
+}
+
+static void
+proxy_context_destroy(struct virgl_context *base)
+{
+   struct proxy_context *ctx = (struct proxy_context *)base;
+
+   /* ask the server process to terminate the context process */
+   if (!proxy_client_destroy_context(ctx->client, ctx->base.ctx_id))
+      proxy_log("failed to destroy ctx %d", ctx->base.ctx_id);
+
+   if (ctx->shmem.ptr)
+      munmap(ctx->shmem.ptr, ctx->shmem.size);
+   if (ctx->shmem.fd >= 0)
+      close(ctx->shmem.fd);
+
+   if (ctx->timeline_seqnos) {
+      for (uint32_t i = 0; i < PROXY_CONTEXT_TIMELINE_COUNT; i++) {
+         struct proxy_timeline *timeline = &ctx->timelines[i];
+         list_for_each_entry_safe(struct proxy_fence, fence, &timeline->fences, head)
+            free(fence);
+      }
+   }
+   mtx_destroy(&ctx->timeline_mutex);
+
+   list_for_each_entry_safe(struct proxy_fence, fence, &ctx->free_fences, head)
+      free(fence);
+   mtx_destroy(&ctx->free_fences_mutex);
+
+   if (ctx->sync_thread.fence_eventfd >= 0) {
+      if (ctx->sync_thread.created) {
+         ctx->sync_thread.stop = true;
+         write_eventfd(ctx->sync_thread.fence_eventfd, 1);
+         thrd_join(ctx->sync_thread.thread, NULL);
+      }
+
+      close(ctx->sync_thread.fence_eventfd);
+   }
+
+   proxy_socket_fini(&ctx->socket);
+
+   free(ctx);
+}
+
+static void
+proxy_context_init_base(struct proxy_context *ctx)
+{
+   ctx->base.destroy = proxy_context_destroy;
+   ctx->base.attach_resource = proxy_context_attach_resource;
+   ctx->base.detach_resource = proxy_context_detach_resource;
+   ctx->base.transfer_3d = proxy_context_transfer_3d;
+   ctx->base.get_blob = proxy_context_get_blob;
+   ctx->base.get_blob_done = NULL;
+   ctx->base.submit_cmd = proxy_context_submit_cmd;
+
+   ctx->base.get_fencing_fd = proxy_context_get_fencing_fd;
+   ctx->base.retire_fences = proxy_context_retire_fences;
+   ctx->base.submit_fence = proxy_context_submit_fence;
+}
+
+static bool
+proxy_context_init_fencing(struct proxy_context *ctx)
+{
+   /* The render server updates the shmem for the current seqnos and
+    * optionally notifies using the eventfd.  That means, when only
+    * VIRGL_RENDERER_THREAD_SYNC is set, we just need to set up the eventfd.
+    * When VIRGL_RENDERER_ASYNC_FENCE_CB is also set, we need to create a sync
+    * thread as well.
+    *
+    * Fence polling can always check the shmem directly.
+    */
+   if (!(proxy_renderer.flags & VIRGL_RENDERER_THREAD_SYNC))
+      return true;
+
+   ctx->sync_thread.fence_eventfd = create_eventfd(0);
+   if (ctx->sync_thread.fence_eventfd < 0) {
+      proxy_log("failed to create fence eventfd");
+      return false;
+   }
+
+   if (proxy_renderer.flags & VIRGL_RENDERER_ASYNC_FENCE_CB) {
+      int ret = thrd_create(&ctx->sync_thread.thread, proxy_context_sync_thread, ctx);
+      if (ret != thrd_success) {
+         proxy_log("failed to create sync thread");
+         return false;
+      }
+      ctx->sync_thread.created = true;
+   }
+
+   return true;
+}
+
+static bool
+proxy_context_init_timelines(struct proxy_context *ctx)
+{
+   atomic_uint *timeline_seqnos = ctx->shmem.ptr;
+   for (uint32_t i = 0; i < ARRAY_SIZE(ctx->timelines); i++) {
+      atomic_init(&timeline_seqnos[i], 0);
+
+      struct proxy_timeline *timeline = &ctx->timelines[i];
+      timeline->cur_seqno = 0;
+      timeline->next_seqno = 1;
+      list_inithead(&timeline->fences);
+   }
+
+   ctx->timeline_seqnos = timeline_seqnos;
+
+   return true;
+}
+
+static bool
+proxy_context_init_shmem(struct proxy_context *ctx)
+{
+   const size_t shmem_size = sizeof(*ctx->timeline_seqnos) * PROXY_CONTEXT_TIMELINE_COUNT;
+   ctx->shmem.fd = alloc_memfd("proxy-ctx", shmem_size, &ctx->shmem.ptr);
+   if (ctx->shmem.fd < 0)
+      return false;
+
+   ctx->shmem.size = shmem_size;
+
+   return true;
+}
+
+static bool
+proxy_context_init(struct proxy_context *ctx, uint32_t ctx_flags)
+{
+   if (!proxy_context_init_shmem(ctx) || !proxy_context_init_timelines(ctx) ||
+       !proxy_context_init_fencing(ctx))
+      return false;
+
+   const struct render_context_op_init_request req = {
+      .header.op = RENDER_CONTEXT_OP_INIT,
+      .flags = ctx_flags,
+      .shmem_size = ctx->shmem.size,
+   };
+   const int req_fds[2] = { ctx->shmem.fd, ctx->sync_thread.fence_eventfd };
+   const int req_fd_count = req_fds[1] >= 0 ? 2 : 1;
+   if (!proxy_socket_send_request_with_fds(&ctx->socket, &req, sizeof(req), req_fds,
+                                           req_fd_count)) {
+      proxy_log("failed to initialize context");
+      return false;
+   }
+
+   return true;
+}
+
+struct virgl_context *
+proxy_context_create(uint32_t ctx_id,
+                     uint32_t ctx_flags,
+                     size_t debug_len,
+                     const char *debug_name)
+{
+   struct proxy_client *client = proxy_renderer.client;
+   struct proxy_context *ctx;
+
+   int ctx_fd;
+   if (!proxy_client_create_context(client, ctx_id, debug_len, debug_name, &ctx_fd)) {
+      proxy_log("failed to create a context");
+      return NULL;
+   }
+
+   ctx = calloc(1, sizeof(*ctx));
+   if (!ctx) {
+      close(ctx_fd);
+      return NULL;
+   }
+
+   proxy_context_init_base(ctx);
+   ctx->client = client;
+   proxy_socket_init(&ctx->socket, ctx_fd);
+   ctx->shmem.fd = -1;
+   mtx_init(&ctx->timeline_mutex, mtx_plain);
+   mtx_init(&ctx->free_fences_mutex, mtx_plain);
+   list_inithead(&ctx->free_fences);
+   ctx->sync_thread.fence_eventfd = -1;
+
+   if (!proxy_context_init(ctx, ctx_flags)) {
+      proxy_context_destroy(&ctx->base);
+      return NULL;
+   }
+
+   return &ctx->base;
+}

src/proxy/proxy_context.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef PROXY_CONTEXT_H
+#define PROXY_CONTEXT_H
+
+#include "proxy_common.h"
+
+#include "c11/threads.h"
+#include "virgl_context.h"
+
+/* matches virtio-gpu */
+#define PROXY_CONTEXT_TIMELINE_COUNT 64
+
+static_assert(ATOMIC_INT_LOCK_FREE == 2, "proxy renderer requires lock-free atomic_uint");
+
+struct proxy_timeline {
+   uint32_t cur_seqno;
+   uint32_t next_seqno;
+   struct list_head fences;
+
+   int cur_seqno_stall_count;
+};
+
+struct proxy_context {
+   struct virgl_context base;
+
+   struct proxy_client *client;
+   struct proxy_socket socket;
+
+   /* this is shared with the render worker */
+   struct {
+      int fd;
+      size_t size;
+      void *ptr;
+   } shmem;
+
+   mtx_t timeline_mutex;
+   struct proxy_timeline timelines[PROXY_CONTEXT_TIMELINE_COUNT];
+   /* which timelines have fences */
+   uint64_t timeline_busy_mask;
+   /* this points a region of shmem updated by the render worker */
+   const volatile atomic_uint *timeline_seqnos;
+
+   mtx_t free_fences_mutex;
+   struct list_head free_fences;
+
+   struct {
+      /* when VIRGL_RENDERER_THREAD_SYNC is set */
+      int fence_eventfd;
+
+      /* when VIRGL_RENDERER_ASYNC_FENCE_CB is also set */
+      thrd_t thread;
+      bool created;
+      bool stop;
+   } sync_thread;
+};
+
+#endif /* PROXY_CONTEXT_H */

src/proxy/proxy_renderer.c

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "proxy_common.h"
+
+#include "proxy_client.h"
+#include "proxy_renderer.h"
+#include "proxy_server.h"
+
+int
+proxy_renderer_init(const struct proxy_renderer_cbs *cbs, uint32_t flags)
+{
+   proxy_renderer.cbs = cbs;
+   proxy_renderer.flags = flags;
+
+   proxy_renderer.server = proxy_server_create();
+   if (!proxy_renderer.server)
+      goto fail;
+
+   proxy_renderer.client =
+      proxy_client_create(proxy_renderer.server, proxy_renderer.flags);
+   if (!proxy_renderer.client)
+      goto fail;
+
+   return 0;
+
+fail:
+   proxy_renderer_fini();
+   return -1;
+}
+
+void
+proxy_renderer_fini(void)
+{
+   if (proxy_renderer.server)
+      proxy_server_destroy(proxy_renderer.server);
+
+   if (proxy_renderer.client)
+      proxy_client_destroy(proxy_renderer.client);
+
+   memset(&proxy_renderer, 0, sizeof(struct proxy_renderer));
+}
+
+void
+proxy_renderer_reset(void)
+{
+   proxy_client_reset(proxy_renderer.client);
+}

src/proxy/proxy_renderer.h

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef PROXY_RENDERER_H
+#define PROXY_RENDERER_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+struct iovec;
+struct virgl_context;
+
+struct proxy_renderer_cbs {
+   int (*get_server_fd)(uint32_t version);
+};
+
+#ifdef ENABLE_RENDER_SERVER
+
+int
+proxy_renderer_init(const struct proxy_renderer_cbs *cbs, uint32_t flags);
+
+void
+proxy_renderer_fini(void);
+
+void
+proxy_renderer_reset(void);
+
+struct virgl_context *
+proxy_context_create(uint32_t ctx_id,
+                     uint32_t ctx_flags,
+                     size_t debug_len,
+                     const char *debug_name);
+
+#else /* ENABLE_RENDER_SERVER */
+
+static inline int
+proxy_renderer_init(UNUSED const struct proxy_renderer_cbs *cbs, UNUSED uint32_t flags)
+{
+   virgl_log("Render server support was not enabled in virglrenderer\n");
+   return -1;
+}
+
+static inline void
+proxy_renderer_fini(void)
+{
+}
+
+static inline void
+proxy_renderer_reset(void)
+{
+}
+
+static inline struct virgl_context *
+proxy_context_create(UNUSED uint32_t ctx_id,
+                     UNUSED uint32_t ctx_flags,
+                     UNUSED size_t debug_len,
+                     UNUSED const char *debug_name)
+{
+   return NULL;
+}
+
+#endif /* ENABLE_RENDER_SERVER */
+
+#endif /* PROXY_RENDERER_H */

src/proxy/proxy_server.c

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "proxy_server.h"
+
+#include <signal.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "server/render_protocol.h"
+
+int
+proxy_server_connect(struct proxy_server *srv)
+{
+   int client_fd = srv->client_fd;
+   /* transfer ownership */
+   srv->client_fd = -1;
+   return client_fd;
+}
+
+void
+proxy_server_destroy(struct proxy_server *srv)
+{
+   if (srv->pid >= 0) {
+      kill(srv->pid, SIGKILL);
+
+      siginfo_t siginfo = { 0 };
+      waitid(P_PID, srv->pid, &siginfo, WEXITED);
+   }
+
+   if (srv->client_fd >= 0)
+      close(srv->client_fd);
+
+   free(srv);
+}
+
+static bool
+proxy_server_fork(struct proxy_server *srv)
+{
+   int socket_fds[2];
+   if (!proxy_socket_pair(socket_fds))
+      return false;
+   const int client_fd = socket_fds[0];
+   const int remote_fd = socket_fds[1];
+
+   pid_t pid = fork();
+   if (pid < 0) {
+      proxy_log("failed to fork proxy server");
+      close(client_fd);
+      close(remote_fd);
+      return false;
+   }
+
+   if (pid > 0) {
+      srv->pid = pid;
+      srv->client_fd = client_fd;
+      close(remote_fd);
+   } else {
+      close(client_fd);
+
+      /* do not receive signals from terminal */
+      setpgid(0, 0);
+
+      char fd_str[16];
+      snprintf(fd_str, sizeof(fd_str), "%d", remote_fd);
+
+      char *const argv[] = {
+         RENDER_SERVER_EXEC_PATH,
+         "--socket-fd",
+         fd_str,
+         NULL,
+      };
+      execv(argv[0], argv);
+
+      proxy_log("failed to exec %s: %s", argv[0], strerror(errno));
+      close(remote_fd);
+      exit(-1);
+   }
+
+   return true;
+}
+
+static bool
+proxy_server_init_fd(struct proxy_server *srv)
+{
+   /* the fd represents a connection to the server */
+   srv->client_fd = proxy_renderer.cbs->get_server_fd(RENDER_SERVER_VERSION);
+   if (srv->client_fd < 0)
+      return false;
+
+   return true;
+}
+
+struct proxy_server *
+proxy_server_create(void)
+{
+   struct proxy_server *srv = calloc(1, sizeof(*srv));
+   if (!srv)
+      return NULL;
+
+   srv->pid = -1;
+
+   if (!proxy_server_init_fd(srv)) {
+      /* start the render server on demand when the client does not provide a
+       * server fd
+       */
+      if (!proxy_server_fork(srv)) {
+         free(srv);
+         return NULL;
+      }
+   }
+
+   if (!proxy_socket_is_seqpacket(srv->client_fd)) {
+      proxy_log("invalid client fd type");
+      close(srv->client_fd);
+      free(srv);
+      return NULL;
+   }
+
+   proxy_log("proxy server with pid %d", srv->pid);
+
+   return srv;
+}

src/proxy/proxy_server.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef PROXY_SERVER_H
+#define PROXY_SERVER_H
+
+#include "proxy_common.h"
+
+#include <sys/types.h>
+
+struct proxy_server {
+   pid_t pid;
+   int client_fd;
+};
+
+struct proxy_server *
+proxy_server_create(void);
+
+void
+proxy_server_destroy(struct proxy_server *srv);
+
+int
+proxy_server_connect(struct proxy_server *srv);
+
+#endif /* PROXY_SERVER_H */

src/proxy/proxy_socket.c

@@ -0,0 +1,258 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "proxy_socket.h"
+
+#include <poll.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <unistd.h>
+
+#define PROXY_SOCKET_MAX_FD_COUNT 8
+
+/* this is only used when the render server is started on demand */
+bool
+proxy_socket_pair(int out_fds[static 2])
+{
+   int ret = socketpair(AF_UNIX, SOCK_SEQPACKET, 0, out_fds);
+   if (ret) {
+      proxy_log("failed to create socket pair");
+      return false;
+   }
+
+   return true;
+}
+
+bool
+proxy_socket_is_seqpacket(int fd)
+{
+   int type;
+   socklen_t len = sizeof(type);
+   if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &type, &len)) {
+      proxy_log("fd %d err %s", fd, strerror(errno));
+      return false;
+   }
+   return type == SOCK_SEQPACKET;
+}
+
+void
+proxy_socket_init(struct proxy_socket *socket, int fd)
+{
+   /* TODO make fd non-blocking and perform io with timeout */
+   assert(fd >= 0);
+   *socket = (struct proxy_socket){
+      .fd = fd,
+   };
+}
+
+void
+proxy_socket_fini(struct proxy_socket *socket)
+{
+   close(socket->fd);
+}
+
+bool
+proxy_socket_is_connected(const struct proxy_socket *socket)
+{
+   struct pollfd poll_fd = {
+      .fd = socket->fd,
+   };
+
+   while (true) {
+      const int ret = poll(&poll_fd, 1, 0);
+      if (ret == 0) {
+         return true;
+      } else if (ret < 0) {
+         if (errno == EINTR || errno == EAGAIN)
+            continue;
+
+         proxy_log("failed to poll socket");
+         return false;
+      }
+
+      if (poll_fd.revents & (POLLERR | POLLHUP | POLLNVAL)) {
+         proxy_log("socket disconnected");
+         return false;
+      }
+
+      return true;
+   }
+}
+
+static const int *
+get_received_fds(const struct msghdr *msg, int *out_count)
+{
+   const struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg);
+   if (unlikely(!cmsg || cmsg->cmsg_level != SOL_SOCKET ||
+                cmsg->cmsg_type != SCM_RIGHTS || cmsg->cmsg_len < CMSG_LEN(0))) {
+      *out_count = 0;
+      return NULL;
+   }
+
+   *out_count = (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int);
+   return (const int *)CMSG_DATA(cmsg);
+}
+
+static bool
+proxy_socket_recvmsg(struct proxy_socket *socket, struct msghdr *msg)
+{
+   do {
+      const ssize_t s = recvmsg(socket->fd, msg, MSG_CMSG_CLOEXEC);
+      if (unlikely(s < 0)) {
+         if (errno == EAGAIN || errno == EINTR)
+            continue;
+
+         proxy_log("failed to receive message: %s", strerror(errno));
+         return false;
+      }
+
+      assert(msg->msg_iovlen == 1);
+      if (unlikely((msg->msg_flags & (MSG_TRUNC | MSG_CTRUNC)) ||
+                   msg->msg_iov[0].iov_len != (size_t)s)) {
+         proxy_log("failed to receive message: truncated or incomplete");
+
+         int fd_count;
+         const int *fds = get_received_fds(msg, &fd_count);
+         for (int i = 0; i < fd_count; i++)
+            close(fds[i]);
+
+         return false;
+      }
+
+      return true;
+   } while (true);
+}
+
+static bool
+proxy_socket_receive_reply_internal(struct proxy_socket *socket,
+                                    void *data,
+                                    size_t size,
+                                    int *fds,
+                                    int max_fd_count,
+                                    int *out_fd_count)
+{
+   assert(data && size);
+   struct msghdr msg = {
+      .msg_iov =
+         &(struct iovec){
+            .iov_base = data,
+            .iov_len = size,
+         },
+      .msg_iovlen = 1,
+   };
+
+   char cmsg_buf[CMSG_SPACE(sizeof(*fds) * PROXY_SOCKET_MAX_FD_COUNT)];
+   if (max_fd_count) {
+      assert(fds && max_fd_count <= PROXY_SOCKET_MAX_FD_COUNT);
+      msg.msg_control = cmsg_buf;
+      msg.msg_controllen = CMSG_SPACE(sizeof(*fds) * max_fd_count);
+
+      struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+      memset(cmsg, 0, sizeof(*cmsg));
+   }
+
+   if (!proxy_socket_recvmsg(socket, &msg))
+      return false;
+
+   if (max_fd_count) {
+      int received_fd_count;
+      const int *received_fds = get_received_fds(&msg, &received_fd_count);
+      assert(received_fd_count <= max_fd_count);
+
+      memcpy(fds, received_fds, sizeof(*fds) * received_fd_count);
+      *out_fd_count = received_fd_count;
+   } else if (out_fd_count) {
+      *out_fd_count = 0;
+   }
+
+   return true;
+}
+
+bool
+proxy_socket_receive_reply(struct proxy_socket *socket, void *data, size_t size)
+{
+   return proxy_socket_receive_reply_internal(socket, data, size, NULL, 0, NULL);
+}
+
+bool
+proxy_socket_receive_reply_with_fds(struct proxy_socket *socket,
+                                    void *data,
+                                    size_t size,
+                                    int *fds,
+                                    int max_fd_count,
+                                    int *out_fd_count)
+{
+   return proxy_socket_receive_reply_internal(socket, data, size, fds, max_fd_count,
+                                              out_fd_count);
+}
+
+static bool
+proxy_socket_sendmsg(struct proxy_socket *socket, const struct msghdr *msg)
+{
+   do {
+      const ssize_t s = sendmsg(socket->fd, msg, MSG_NOSIGNAL);
+      if (unlikely(s < 0)) {
+         if (errno == EAGAIN || errno == EINTR)
+            continue;
+
+         proxy_log("failed to send message: %s", strerror(errno));
+         return false;
+      }
+
+      /* no partial send since the socket type is SOCK_SEQPACKET */
+      assert(msg->msg_iovlen == 1 && msg->msg_iov[0].iov_len == (size_t)s);
+      return true;
+   } while (true);
+}
+
+static bool
+proxy_socket_send_request_internal(struct proxy_socket *socket,
+                                   const void *data,
+                                   size_t size,
+                                   const int *fds,
+                                   int fd_count)
+{
+   assert(data && size);
+   struct msghdr msg = {
+      .msg_iov =
+         &(struct iovec){
+            .iov_base = (void *)data,
+            .iov_len = size,
+         },
+      .msg_iovlen = 1,
+   };
+
+   char cmsg_buf[CMSG_SPACE(sizeof(*fds) * PROXY_SOCKET_MAX_FD_COUNT)];
+   if (fd_count) {
+      assert(fds && fd_count <= PROXY_SOCKET_MAX_FD_COUNT);
+      msg.msg_control = cmsg_buf;
+      msg.msg_controllen = CMSG_SPACE(sizeof(*fds) * fd_count);
+
+      struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+      cmsg->cmsg_level = SOL_SOCKET;
+      cmsg->cmsg_type = SCM_RIGHTS;
+      cmsg->cmsg_len = CMSG_LEN(sizeof(*fds) * fd_count);
+      memcpy(CMSG_DATA(cmsg), fds, sizeof(*fds) * fd_count);
+   }
+
+   return proxy_socket_sendmsg(socket, &msg);
+}
+
+bool
+proxy_socket_send_request(struct proxy_socket *socket, const void *data, size_t size)
+{
+   return proxy_socket_send_request_internal(socket, data, size, NULL, 0);
+}
+
+bool
+proxy_socket_send_request_with_fds(struct proxy_socket *socket,
+                                   const void *data,
+                                   size_t size,
+                                   const int *fds,
+                                   int fd_count)
+{
+   return proxy_socket_send_request_internal(socket, data, size, fds, fd_count);
+}

src/proxy/proxy_socket.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: MIT
+ */
+
+#ifndef PROXY_SOCKET_H
+#define PROXY_SOCKET_H
+
+#include "proxy_common.h"
+
+struct proxy_socket {
+   int fd;
+};
+
+bool
+proxy_socket_pair(int out_fds[static 2]);
+
+bool
+proxy_socket_is_seqpacket(int fd);
+
+void
+proxy_socket_init(struct proxy_socket *socket, int fd);
+
+void
+proxy_socket_fini(struct proxy_socket *socket);
+
+bool
+proxy_socket_is_connected(const struct proxy_socket *socket);
+
+bool
+proxy_socket_receive_reply(struct proxy_socket *socket, void *data, size_t size);
+
+bool
+proxy_socket_receive_reply_with_fds(struct proxy_socket *socket,
+                                    void *data,
+                                    size_t size,
+                                    int *fds,
+                                    int max_fd_count,
+                                    int *out_fd_count);
+
+bool
+proxy_socket_send_request(struct proxy_socket *socket, const void *data, size_t size);
+
+bool
+proxy_socket_send_request_with_fds(struct proxy_socket *socket,
+                                   const void *data,
+                                   size_t size,
+                                   const int *fds,
+                                   int fd_count);
+
+#endif /* PROXY_SOCKET_H */