backend-drm: Cache fb<->plane format compatibility list

Walking the format/modifier list to try to find out if our FB is
compatible with the plane is surprisingly expensive. Since the plane's
capabilities are static over the lifetime of the KMS device, cache the
set of planes for which the FB is theoretically
format/modifier-compatible when it's created, and use that to do an
early cull of the set of acceptable planes.

Signed-off-by: Daniel Stone <daniels@collabora.com>

libweston/backend-drm/drm-internal.h

@@ -276,6 +276,7 @@ struct drm_backend {
 	int min_height, max_height;
 
 	struct wl_list plane_list;
+	uint32_t next_plane_idx;
 
 	void *repaint_data;
 
@@ -341,6 +342,8 @@ struct drm_fb {
 	int width, height;
 	int fd;
 
+	uint32_t plane_mask;
+
 	/* Used by gbm fbs */
 	struct gbm_bo *bo;
 	struct gbm_surface *gbm_surface;
@@ -463,6 +466,7 @@ struct drm_plane {
 
 	uint32_t possible_crtcs;
 	uint32_t plane_id;
+	uint32_t plane_idx;
 
 	struct drm_property_info props[WDRM_PLANE__COUNT];
 

libweston/backend-drm/drm.c

@@ -764,6 +764,7 @@ drm_plane_create(struct drm_backend *b, const drmModePlane *kplane)
 	}
 
 	plane->backend = b;
+	plane->plane_idx = b->next_plane_idx++;
 	plane->state_cur = drm_plane_state_alloc(NULL, plane);
 	plane->state_cur->complete = true;
 	plane->possible_crtcs = kplane->possible_crtcs;

libweston/backend-drm/fb.c

@@ -461,6 +461,43 @@ drm_can_scanout_dmabuf(struct weston_compositor *ec,
 	return ret;
 }
 
+static bool
+drm_fb_compatible_with_plane(struct drm_fb *fb, struct drm_plane *plane)
+{
+	struct drm_backend *b = plane->backend;
+	struct weston_drm_format *fmt;
+
+	/* Check whether the format is supported */
+	fmt = weston_drm_format_array_find_format(&plane->formats,
+						  fb->format->format);
+	if (fmt) {
+		/* We never try to promote a dmabuf with DRM_FORMAT_MOD_INVALID
+		 * to a KMS plane (see drm_fb_get_from_dmabuf() for more details).
+		 * So if fb->modifier == DRM_FORMAT_MOD_INVALID, we are sure
+		 * that this is for the legacy GBM import path, in which a
+		 * wl_drm is being used for scanout. Mesa is the only user we
+		 * care in this case (even though recent versions are also using
+		 * dmabufs), and it should know better what works or not. */
+		if (fb->modifier == DRM_FORMAT_MOD_INVALID)
+			return true;
+
+		if (weston_drm_format_has_modifier(fmt, fb->modifier))
+			return true;
+	}
+
+	drm_debug(b, "\t\t\t\t[%s] not placing view on %s: "
+		  "no free %s planes matching format %s (0x%lx) "
+		  "modifier 0x%llx\n",
+		  drm_output_get_plane_type_name(plane),
+		  drm_output_get_plane_type_name(plane),
+		  drm_output_get_plane_type_name(plane),
+		  fb->format->drm_format_name,
+		  (unsigned long) fb->format->format,
+		  (unsigned long long) fb->modifier);
+
+	return false;
+}
+
 static void
 drm_fb_handle_buffer_destroy(struct wl_listener *listener, void *data)
 {
@@ -487,6 +524,7 @@ drm_fb_get_from_view(struct drm_output_state *state, struct weston_view *ev,
 	bool is_opaque = weston_view_is_opaque(ev, &ev->transform.boundingbox);
 	struct linux_dmabuf_buffer *dmabuf;
 	struct drm_fb *fb;
+	struct drm_plane *plane;
 
 	if (ev->alpha != 1.0f)
 		return NULL;
@@ -540,6 +578,18 @@ drm_fb_get_from_view(struct drm_output_state *state, struct weston_view *ev,
 		}
 	}
 
+	/* Check if this buffer can ever go on any planes. If it can't, we have
+	 * no reason to ever have a drm_fb, so we fail it here. */
+	wl_list_for_each(plane, &b->plane_list, link) {
+		if (drm_fb_compatible_with_plane(fb, plane))
+			fb->plane_mask |= (1 << plane->plane_idx);
+	}
+	if (fb->plane_mask == 0) {
+		drm_fb_unref(fb);
+		buf_fb->failure_reasons |= FAILURE_REASONS_FB_FORMAT_INCOMPATIBLE;
+		goto unsuitable;
+	}
+
 	/* The caller holds its own ref to the drm_fb, so when creating a new
 	 * drm_fb we take an additional ref for the weston_buffer's cache. */
 	buf_fb->fb = drm_fb_ref(fb);

libweston/backend-drm/state-propose.c

@@ -124,64 +124,6 @@ drm_output_check_plane_has_view_assigned(struct drm_plane *plane,
 	return false;
 }
 
-static bool
-drm_output_plane_has_valid_format(struct drm_plane *plane,
-				  struct drm_output_state *state,
-				  struct drm_fb *fb)
-{
-	struct drm_backend *b = plane->backend;
-	struct weston_drm_format *fmt;
-
-	if (!fb)
-		return false;
-
-	/* Check whether the format is supported */
-	fmt = weston_drm_format_array_find_format(&plane->formats,
-						  fb->format->format);
-	if (fmt) {
-		/* We never try to promote a dmabuf with DRM_FORMAT_MOD_INVALID
-		 * to a KMS plane (see drm_fb_get_from_dmabuf() for more details).
-		 * So if fb->modifier == DRM_FORMAT_MOD_INVALID, we are sure
-		 * that this is for the legacy GBM import path, in which a
-		 * wl_drm is being used for scanout. Mesa is the only user we
-		 * care in this case (even though recent versions are also using
-		 * dmabufs), and it should know better what works or not. */
-		if (fb->modifier == DRM_FORMAT_MOD_INVALID)
-			return true;
-
-		if (weston_drm_format_has_modifier(fmt, fb->modifier))
-			return true;
-	}
-
-	drm_debug(b, "\t\t\t\t[%s] not placing view on %s: "
-		  "no free %s planes matching format %s (0x%lx) "
-		  "modifier 0x%llx\n",
-		  drm_output_get_plane_type_name(plane),
-		  drm_output_get_plane_type_name(plane),
-		  drm_output_get_plane_type_name(plane),
-		  fb->format->drm_format_name,
-		  (unsigned long) fb->format->format,
-		  (unsigned long long) fb->modifier);
-
-	return false;
-}
-
-static bool
-drm_output_plane_cursor_has_valid_format(struct weston_view *ev)
-{
-	struct wl_shm_buffer *shmbuf =
-		wl_shm_buffer_get(ev->surface->buffer_ref.buffer->resource);
-
-	/* When we have cursor planes we've already checked for wl_shm buffer in
-	 * the view before calling this function. */
-	assert(shmbuf);
-
-	if (wl_shm_buffer_get_format(shmbuf) == WL_SHM_FORMAT_ARGB8888)
-		return true;
-
-	return false;
-}
-
 static struct drm_plane_state *
 drm_output_prepare_overlay_view(struct drm_plane *plane,
 				struct drm_output_state *output_state,
@@ -497,28 +439,6 @@ err:
 	return NULL;
 }
 
-static bool
-drm_output_plane_view_has_valid_format(struct drm_plane *plane,
-				       struct drm_output_state *state,
-				       struct weston_view *ev,
-				       struct drm_fb *fb)
-{
-	/* depending on the type of the plane we have different requirements */
-	switch (plane->type) {
-	case WDRM_PLANE_TYPE_CURSOR:
-		return drm_output_plane_cursor_has_valid_format(ev);
-	case WDRM_PLANE_TYPE_OVERLAY:
-		return drm_output_plane_has_valid_format(plane, state, fb);
-	case WDRM_PLANE_TYPE_PRIMARY:
-		return drm_output_plane_has_valid_format(plane, state, fb);
-	default:
-		assert(0);
-		return false;
-	}
-
-	return false;
-}
-
 static struct drm_plane_state *
 drm_output_try_view_on_plane(struct drm_plane *plane,
 			     struct drm_output_state *state,
@@ -834,15 +754,17 @@ drm_output_prepare_plane_view(struct drm_output_state *state,
 			continue;
 		}
 
-		if (plane->type == WDRM_PLANE_TYPE_CURSOR && !shmbuf) {
+		if (plane->type == WDRM_PLANE_TYPE_CURSOR &&
+		    (!shmbuf || wl_shm_buffer_get_format(shmbuf) != WL_SHM_FORMAT_ARGB8888)) {
 			drm_debug(b, "\t\t\t\t[plane] not adding plane %d, type cursor to "
-				     "candidate list: cursor planes only support wl_shm "
+				     "candidate list: cursor planes only support ARGB8888"
-				     "buffers and the view buffer is of another type\n",
+				     "wl_shm buffers and the view buffer is of another type\n",
 				     plane->plane_id);
 			continue;
 		}
 
-		if (!drm_output_plane_view_has_valid_format(plane, state, ev, fb)) {
+		if (plane->type != WDRM_PLANE_TYPE_CURSOR &&
+		    (!fb || !(fb->plane_mask & (1 << plane->plane_idx)))) {
 			*try_view_on_plane_failure_reasons |=
 				FAILURE_REASONS_FB_FORMAT_INCOMPATIBLE;
 			drm_debug(b, "\t\t\t\t[plane] not adding plane %d to "