compositor-drm: Add hardware accelerated capture of screen using libva

This patch adds a feature to the DRM backend that uses libva for
encoding the screen contents in H.264. Screen recording can be
activated by pressing mod-shift-space q. A file named capture.h264
will be created in the current directory, which can be muxed into
an MP4 file with gstreamer using

gst-launch filesrc location=capture.h264 ! h264parse ! mp4mux ! \
           filesink location=file.mp4

This is limitted to the DRM compositor in order to avoid a copy when
submitting the front buffer to libva. The code in vaapi-recorder.c
takes a dma_buf fd referencing it, does a colorspace conversion using
the video post processing pipeline and then uses that as input to the
encoder.

I'm sending this now so I get comments, but this is not ready for
prime time yet. I have a somewhat consistent GPU hang when using
i915 with SandyBridge. Sometimes a page flip never completes. If you
want to try this anyway and your system get stuck, you might need to
run the following:

  # echo 1 > /sys/kernel/debug/dri/0/i915_wedged

After that, alt-sysrq [rv] should work.

Once that's fixed it would also be good to make the parameters used by
the encoder more flexible. For now the QP parameter is hardcoded to 0
and we have only I and P frames (no B frames), which causes the
resulting files to be very large.

configure.ac

@@ -240,6 +240,11 @@ PKG_CHECK_MODULES(WEBP, [libwebp], [have_webp=yes], [have_webp=no])
 AS_IF([test "x$have_webp" = "xyes"],
       [AC_DEFINE([HAVE_WEBP], [1], [Have webp])])
 
+PKG_CHECK_MODULES(LIBVA, [libva >= 0.34.0 libva-drm >= 0.34.0], [have_libva=yes], [have_libva=no])
+AS_IF([test "x$have_libva" = "xyes"],
+      [AC_DEFINE([HAVE_LIBVA], [1], [Have libva])])
+AM_CONDITIONAL(ENABLE_LIBVA, test "x$have_libva" = "xyes")
+
 AC_CHECK_LIB([jpeg], [jpeg_CreateDecompress], have_jpeglib=yes)
 if test x$have_jpeglib = xyes; then
   JPEG_LIBS="-ljpeg"
@@ -480,4 +485,5 @@ AC_MSG_RESULT([
 	LCMS2 Support			${have_lcms}
 	libwebp Support			${have_webp}
 	libunwind Support		${have_libunwind}
+	VA H.264 encoding Support	${have_libva}
 ])

src/Makefile.am

@@ -152,6 +152,12 @@ drm_backend_la_SOURCES =			\
 	launcher-util.h				\
 	libbacklight.c				\
 	libbacklight.h
+
+if ENABLE_LIBVA
+drm_backend_la_SOURCES += vaapi-recorder.c
+drm_backend_la_LIBADD += $(LIBVA_LIBS)
+drm_backend_la_CFLAGS += $(LIBVA_CFLAGS)
+endif
 endif
 
 if ENABLE_WAYLAND_COMPOSITOR

src/compositor-drm.c

@@ -47,6 +47,7 @@
 #include "pixman-renderer.h"
 #include "udev-seat.h"
 #include "launcher-util.h"
+#include "vaapi-recorder.h"
 
 #ifndef DRM_CAP_TIMESTAMP_MONOTONIC
 #define DRM_CAP_TIMESTAMP_MONOTONIC 0x6
@@ -75,6 +76,7 @@ struct drm_compositor {
 	struct {
 		int id;
 		int fd;
+		char *filename;
 	} drm;
 	struct gbm_device *gbm;
 	uint32_t *crtcs;
@@ -159,6 +161,9 @@ struct drm_output {
 	pixman_image_t *image[2];
 	int current_image;
 	pixman_region32_t previous_damage;
+
+	struct vaapi_recorder *recorder;
+	struct wl_listener recorder_frame_listener;
 };
 
 /*
@@ -717,6 +722,11 @@ page_flip_handler(int fd, unsigned int frame,
 	if (!output->vblank_pending) {
 		msecs = sec * 1000 + usec / 1000;
 		weston_output_finish_frame(&output->base, msecs);
+
+		/* We can't call this from frame_notify, because the output's
+		 * repaint needed flag is cleared just after that */
+		if (output->recorder)
+			weston_output_schedule_repaint(&output->base);
 	}
 }
 
@@ -1215,6 +1225,7 @@ init_drm(struct drm_compositor *ec, struct udev_device *device)
 	weston_log("using %s\n", filename);
 
 	ec->drm.fd = fd;
+	ec->drm.filename = strdup(filename);
 
 	ret = drmGetCap(fd, DRM_CAP_TIMESTAMP_MONOTONIC, &cap);
 	if (ret == 0 && cap == 1)
@@ -2435,6 +2446,102 @@ planes_binding(struct weston_seat *seat, uint32_t time, uint32_t key, void *data
 	}
 }
 
+#ifdef HAVE_LIBVA
+static void
+recorder_frame_notify(struct wl_listener *listener, void *data)
+{
+	struct drm_output *output;
+	struct drm_compositor *c;
+	int fd, ret;
+
+	output = container_of(listener, struct drm_output,
+			      recorder_frame_listener);
+	c = (struct drm_compositor *) output->base.compositor;
+
+	if (!output->recorder)
+		return;
+
+	ret = drmPrimeHandleToFD(c->drm.fd, output->current->handle,
+				 DRM_CLOEXEC, &fd);
+	if (ret) {
+		weston_log("[libva recorder] "
+			   "failed to create prime fd for front buffer\n");
+		return;
+	}
+
+	vaapi_recorder_frame(output->recorder, fd, output->current->stride / 4);
+
+	close(fd);
+}
+
+static void *
+create_recorder(struct drm_compositor *c, int width, int height,
+		const char *filename)
+{
+	int fd;
+	drm_magic_t magic;
+
+	fd = open(c->drm.filename, O_RDWR | O_CLOEXEC);
+	if (fd < 0)
+		return NULL;
+
+	drmGetMagic(fd, &magic);
+	drmAuthMagic(c->drm.fd, magic);
+
+	return vaapi_recorder_create(fd, width, height, filename);
+}
+
+static void
+recorder_binding(struct weston_seat *seat, uint32_t time, uint32_t key,
+		 void *data)
+{
+	struct drm_compositor *c = data;
+	struct drm_output *output;
+	int width, height;
+
+	output = container_of(c->base.output_list.next,
+			      struct drm_output, base.link);
+
+	if (!output->recorder) {
+		width = output->base.current->width;
+		height = output->base.current->height;
+
+		output->recorder =
+			create_recorder(c, width, height, "capture.h264");
+		if (!output->recorder) {
+			weston_log("failed to create vaapi recorder\n");
+			return;
+		}
+
+		output->base.disable_planes++;
+
+		output->recorder_frame_listener.notify = recorder_frame_notify;
+		wl_signal_add(&output->base.frame_signal,
+			      &output->recorder_frame_listener);
+
+		weston_output_schedule_repaint(&output->base);
+
+		weston_log("[libva recorder] initialized\n");
+	} else {
+		vaapi_recorder_destroy(output->recorder);
+		/* FIXME: close drm fd passed to recorder */
+		output->recorder = NULL;
+
+		output->base.disable_planes--;
+
+		wl_list_remove(&output->recorder_frame_listener.link);
+		weston_log("[libva recorder] done\n");
+	}
+}
+#else
+static void
+recorder_binding(struct weston_seat *seat, uint32_t time, uint32_t key,
+		 void *data)
+{
+	weston_log("Compiled without libva support\n");
+}
+#endif
+
 static struct weston_compositor *
 drm_compositor_create(struct wl_display *display,
 		      int connector, const char *seat_id, int tty, int pixman,
@@ -2568,6 +2675,8 @@ drm_compositor_create(struct wl_display *display,
 					    planes_binding, ec);
 	weston_compositor_add_debug_binding(&ec->base, KEY_V,
 					    planes_binding, ec);
+	weston_compositor_add_debug_binding(&ec->base, KEY_Q,
+					    recorder_binding, ec);
 
 	return &ec->base;
 

src/vaapi-recorder.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright © 2013 Intel Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and
+ * its documentation for any purpose is hereby granted without fee, provided
+ * that the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of the copyright holders not be used in
+ * advertising or publicity pertaining to distribution of the software
+ * without specific, written prior permission.  The copyright holders make
+ * no representations about the suitability of this software for any
+ * purpose.  It is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+ * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+ * CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _VAAPI_RECORDER_H_
+#define _VAAPI_RECORDER_H_
+
+struct vaapi_recorder;
+
+struct vaapi_recorder *
+vaapi_recorder_create(int drm_fd, int width, int height, const char *filename);
+void
+vaapi_recorder_destroy(struct vaapi_recorder *r);
+void
+vaapi_recorder_frame(struct vaapi_recorder *r, int fd, int stride);
+
+#endif /* _VAAPI_RECORDER_H_ */