compositor-drm: Support overlays with transformed buffers

Make overlays work when the client uses a buffer with the same
transformation as the output.

In order to calculate the destination rectangle, the same logic in
weston_surface_to_buffer_float() is needed, but with the output
dimensions instead. For that reason, this patch generalizes this
function into weston_transformed_{coord,rect} and moves it to util.c.
The surface functions are then implemented using those.

src/compositor-drm.c

@@ -572,11 +572,11 @@ drm_output_prepare_overlay_surface(struct weston_output *output_base,
 	int found = 0;
 	struct gbm_bo *bo;
 	pixman_region32_t dest_rect, src_rect;
-	pixman_box32_t *box;
+	pixman_box32_t *box, tbox;
 	uint32_t format;
 	wl_fixed_t sx1, sy1, sx2, sy2;
 
-	if (output_base->transform != WL_OUTPUT_TRANSFORM_NORMAL)
+	if (es->buffer_transform != output_base->transform)
 		return NULL;
 
 	if (c->sprites_are_broken)
@@ -645,10 +645,13 @@ drm_output_prepare_overlay_surface(struct weston_output *output_base,
 				  &output_base->region);
 	pixman_region32_translate(&dest_rect, -output_base->x, -output_base->y);
 	box = pixman_region32_extents(&dest_rect);
-	s->dest_x = box->x1;
+	tbox = weston_transformed_rect(output_base->width,
-	s->dest_y = box->y1;
+				       output_base->height,
-	s->dest_w = box->x2 - box->x1;
+				       output_base->transform, *box);
-	s->dest_h = box->y2 - box->y1;
+	s->dest_x = tbox.x1;
+	s->dest_y = tbox.y1;
+	s->dest_w = tbox.x2 - tbox.x1;
+	s->dest_h = tbox.y2 - tbox.y1;
 	pixman_region32_fini(&dest_rect);
 
 	pixman_region32_init(&src_rect);
@@ -674,10 +677,19 @@ drm_output_prepare_overlay_surface(struct weston_output *output_base,
 	if (sy2 > wl_fixed_from_int(es->geometry.height))
 		sy2 = wl_fixed_from_int(es->geometry.height);
 
-	s->src_x = sx1 << 8;
+	tbox.x1 = sx1;
-	s->src_y = sy1 << 8;
+	tbox.y1 = sy1;
-	s->src_w = (sx2 - sx1) << 8;
+	tbox.x2 = sx2;
-	s->src_h = (sy2 - sy1) << 8;
+	tbox.y2 = sy2;
+
+	tbox = weston_transformed_rect(wl_fixed_from_int(es->geometry.width),
+				       wl_fixed_from_int(es->geometry.height),
+				       es->buffer_transform, tbox);
+
+	s->src_x = tbox.x1 << 8;
+	s->src_y = tbox.y1 << 8;
+	s->src_w = (tbox.x2 - tbox.x1) << 8;
+	s->src_h = (tbox.y2 - tbox.y1) << 8;
 	pixman_region32_fini(&src_rect);
 
 	return &s->plane;

src/compositor.c

@@ -294,71 +294,19 @@ WL_EXPORT void
 weston_surface_to_buffer_float(struct weston_surface *surface,
 			       float sx, float sy, float *bx, float *by)
 {
-	switch (surface->buffer_transform) {
+	weston_transformed_coord(surface->geometry.width,
-	case WL_OUTPUT_TRANSFORM_NORMAL:
+				 surface->geometry.height,
-	default:
+				 surface->buffer_transform,
-		*bx = sx;
+				 sx, sy, bx, by);
-		*by = sy;
-		break;
-	case WL_OUTPUT_TRANSFORM_FLIPPED:
-		*bx = surface->geometry.width - sx;
-		*by = sy;
-		break;
-	case WL_OUTPUT_TRANSFORM_90:
-		*bx = surface->geometry.height - sy;
-		*by = sx;
-		break;
-	case WL_OUTPUT_TRANSFORM_FLIPPED_90:
-		*bx = surface->geometry.height - sy;
-		*by = surface->geometry.width - sx;
-		break;
-	case WL_OUTPUT_TRANSFORM_180:
-		*bx = surface->geometry.width - sx;
-		*by = surface->geometry.height - sy;
-		break;
-	case WL_OUTPUT_TRANSFORM_FLIPPED_180:
-		*bx = sx;
-		*by = surface->geometry.height - sy;
-		break;
-	case WL_OUTPUT_TRANSFORM_270:
-		*bx = sy;
-		*by = surface->geometry.width - sx;
-		break;
-	case WL_OUTPUT_TRANSFORM_FLIPPED_270:
-		*bx = sy;
-		*by = sx;
-		break;
-	}
 }
 
 WL_EXPORT pixman_box32_t
 weston_surface_to_buffer_rect(struct weston_surface *surface,
 			      pixman_box32_t rect)
 {
-	float x1, x2, y1, y2;
+	return weston_transformed_rect(surface->geometry.width,
-
+				       surface->geometry.height,
-	pixman_box32_t ret;
+				       surface->buffer_transform, rect);
-
-	weston_surface_to_buffer_float(surface, rect.x1, rect.y1, &x1, &y1);
-	weston_surface_to_buffer_float(surface, rect.x2, rect.y2, &x2, &y2);
-
-	if (x1 <= x2) {
-		ret.x1 = x1;
-		ret.x2 = x2;
-	} else {
-		ret.x1 = x2;
-		ret.x2 = x1;
-	}
-
-	if (y1 <= y2) {
-		ret.y1 = y1;
-		ret.y2 = y2;
-	} else {
-		ret.y1 = y2;
-		ret.y2 = y1;
-	}
-
-	return ret;
 }
 
 WL_EXPORT void

src/compositor.h

@@ -831,4 +831,13 @@ backend_init(struct wl_display *display, int argc, char *argv[],
 int
 module_init(struct weston_compositor *compositor);
 
+void
+weston_transformed_coord(int width, int height,
+			 enum wl_output_transform transform,
+			 float sx, float sy, float *bx, float *by);
+pixman_box32_t
+weston_transformed_rect(int width, int height,
+			enum wl_output_transform transform,
+			pixman_box32_t rect);
+
 #endif

src/util.c

@@ -557,3 +557,78 @@ weston_environment_get_fd(const char *env)
 
 	return fd;
 }
+
+WL_EXPORT void
+weston_transformed_coord(int width, int height,
+			 enum wl_output_transform transform,
+			 float sx, float sy, float *bx, float *by)
+{
+	switch (transform) {
+	case WL_OUTPUT_TRANSFORM_NORMAL:
+	default:
+		*bx = sx;
+		*by = sy;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED:
+		*bx = width - sx;
+		*by = sy;
+		break;
+	case WL_OUTPUT_TRANSFORM_90:
+		*bx = height - sy;
+		*by = sx;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED_90:
+		*bx = height - sy;
+		*by = width - sx;
+		break;
+	case WL_OUTPUT_TRANSFORM_180:
+		*bx = width - sx;
+		*by = height - sy;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED_180:
+		*bx = sx;
+		*by = height - sy;
+		break;
+	case WL_OUTPUT_TRANSFORM_270:
+		*bx = sy;
+		*by = width - sx;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED_270:
+		*bx = sy;
+		*by = sx;
+		break;
+	}
+}
+
+WL_EXPORT pixman_box32_t
+weston_transformed_rect(int width, int height,
+			enum wl_output_transform transform,
+			pixman_box32_t rect)
+{
+	float x1, x2, y1, y2;
+
+	pixman_box32_t ret;
+
+	weston_transformed_coord(width, height, transform,
+				 rect.x1, rect.y1, &x1, &y1);
+	weston_transformed_coord(width, height, transform,
+				 rect.x2, rect.y2, &x2, &y2);
+
+	if (x1 <= x2) {
+		ret.x1 = x1;
+		ret.x2 = x2;
+	} else {
+		ret.x1 = x2;
+		ret.x2 = x1;
+	}
+
+	if (y1 <= y2) {
+		ret.y1 = y1;
+		ret.y2 = y2;
+	} else {
+		ret.y1 = y2;
+		ret.y2 = y1;
+	}
+
+	return ret;
+}