compositor: Fix rendering with shm transformed buffers

The implementation of buffer transformation didn't handle transformed
shm buffers properly. The partial texture upload was broken since the
damage is in surface coordinates that don't necessarily match the
buffer's coordinates. It also wouldn't handle the buffer stride
properly, resulting in incorrect rendering if it didn't match the
buffer's width.

The logic used for converting texture coordinates was generalized and
moved out of the renderer, since this conversion may be useful in other
places, such as the backends.

src/compositor.c

@@ -310,6 +310,77 @@ weston_surface_to_global_float(struct weston_surface *surface,
 	}
 }
 
+WL_EXPORT void
+weston_surface_to_buffer_float(struct weston_surface *surface,
+			       float sx, float sy, float *bx, float *by)
+{
+	switch (surface->buffer_transform) {
+	case WL_OUTPUT_TRANSFORM_NORMAL:
+	default:
+		*bx = sx;
+		*by = sy;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED:
+		*bx = surface->geometry.width - sx;
+		*by = sy;
+		break;
+	case WL_OUTPUT_TRANSFORM_90:
+		*bx = surface->geometry.height - sy;
+		*by = sx;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED_90:
+		*bx = surface->geometry.height - sy;
+		*by = surface->geometry.width - sx;
+		break;
+	case WL_OUTPUT_TRANSFORM_180:
+		*bx = surface->geometry.width - sx;
+		*by = surface->geometry.height - sy;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED_180:
+		*bx = sx;
+		*by = surface->geometry.height - sy;
+		break;
+	case WL_OUTPUT_TRANSFORM_270:
+		*bx = sy;
+		*by = surface->geometry.width - sx;
+		break;
+	case WL_OUTPUT_TRANSFORM_FLIPPED_270:
+		*bx = sy;
+		*by = sx;
+		break;
+	}
+}
+
+WL_EXPORT pixman_box32_t
+weston_surface_to_buffer_rect(struct weston_surface *surface,
+			      pixman_box32_t rect)
+{
+	float x1, x2, y1, y2;
+
+	pixman_box32_t ret;
+
+	weston_surface_to_buffer_float(surface, rect.x1, rect.y1, &x1, &y1);
+	weston_surface_to_buffer_float(surface, rect.x2, rect.y2, &x2, &y2);
+
+	if (x1 <= x2) {
+		ret.x1 = x1;
+		ret.x2 = x2;
+	} else {
+		ret.x1 = x2;
+		ret.x2 = x1;
+	}
+
+	if (y1 <= y2) {
+		ret.y1 = y1;
+		ret.y2 = y2;
+	} else {
+		ret.y1 = y2;
+		ret.y2 = y1;
+	}
+
+	return ret;
+}
+
 WL_EXPORT void
 weston_surface_move_to_plane(struct weston_surface *surface,
 			     struct weston_plane *plane)

src/compositor.h

@@ -506,6 +506,13 @@ weston_surface_buffer_width(struct weston_surface *surface);
 int32_t
 weston_surface_buffer_height(struct weston_surface *surface);
 
+WL_EXPORT void
+weston_surface_to_buffer_float(struct weston_surface *surface,
+			       float x, float y, float *bx, float *by);
+pixman_box32_t
+weston_surface_to_buffer_rect(struct weston_surface *surface,
+			      pixman_box32_t rect);
+
 void
 weston_spring_init(struct weston_spring *spring,
 		   double k, double current, double target);

src/gl-renderer.c

@@ -517,47 +517,6 @@ calculate_edges(struct weston_surface *es, pixman_box32_t *rect,
 	return n;
 }
 
-static void
-transform_texcoord(struct weston_surface *es, GLfloat sx, GLfloat sy,
-		   GLfloat *tx, GLfloat *ty)
-{
-	switch(es->buffer_transform) {
-	case WL_OUTPUT_TRANSFORM_NORMAL:
-	default:
-		*tx = sx;
-		*ty = sy;
-		break;
-	case WL_OUTPUT_TRANSFORM_FLIPPED:
-		*tx = 1.0 - sx;
-		*ty = sy;
-		break;
-	case WL_OUTPUT_TRANSFORM_90:
-		*tx = 1.0 - sy;
-		*ty = sx;
-		break;
-	case WL_OUTPUT_TRANSFORM_FLIPPED_90:
-		*tx = 1.0 - sy;
-		*ty = 1.0 - sx;
-		break;
-	case WL_OUTPUT_TRANSFORM_180:
-		*tx = 1.0 - sx;
-		*ty = 1.0 - sy;
-		break;
-	case WL_OUTPUT_TRANSFORM_FLIPPED_180:
-		*tx = sx;
-		*ty = 1.0 - sy;
-		break;
-	case WL_OUTPUT_TRANSFORM_270:
-		*tx = sy;
-		*ty = 1.0 - sx;
-		break;
-	case WL_OUTPUT_TRANSFORM_FLIPPED_270:
-		*tx = sy;
-		*ty = sx;
-		break;
-	}
-}
-
 static int
 texture_region(struct weston_surface *es, pixman_region32_t *region,
 		pixman_region32_t *surf_region)
@@ -578,13 +537,23 @@ texture_region(struct weston_surface *es, pixman_region32_t *region,
 	vtxcnt = wl_array_add(&ec->vtxcnt, nrects * nsurf * sizeof *vtxcnt);
 
 	inv_width = 1.0 / es->pitch;
+
+	switch (es->buffer_transform) {
+	case WL_OUTPUT_TRANSFORM_90:
+	case WL_OUTPUT_TRANSFORM_270:
+	case WL_OUTPUT_TRANSFORM_FLIPPED_90:
+	case WL_OUTPUT_TRANSFORM_FLIPPED_270:
+		inv_height = 1.0 / es->geometry.width;
+		break;
+	default:
 		inv_height = 1.0 / es->geometry.height;
+	}
 
 	for (i = 0; i < nrects; i++) {
 		pixman_box32_t *rect = &rects[i];
 		for (j = 0; j < nsurf; j++) {
 			pixman_box32_t *surf_rect = &surf_rects[j];
-			GLfloat sx, sy, tx, ty;
+			GLfloat sx, sy, bx, by;
 			GLfloat ex[8], ey[8];          /* edge points in screen space */
 			int n;
 
@@ -613,12 +582,10 @@ texture_region(struct weston_surface *es, pixman_region32_t *region,
 				*(v++) = ex[k];
 				*(v++) = ey[k];
 				/* texcoord: */
-				transform_texcoord(es,
+				weston_surface_to_buffer_float(es, sx, sy,
-						   sx * inv_width,
+							       &bx, &by);
-						   sy * inv_height,
+				*(v++) = bx * inv_width;
-						   &tx, &ty);
+				*(v++) = by * inv_height;
-				*(v++) = tx;
-				*(v++) = ty;
 			}
 
 			vtxcnt[nvtx++] = n;
@@ -1107,12 +1074,14 @@ gl_renderer_flush_damage(struct weston_surface *surface)
 	data = wl_shm_buffer_get_data(surface->buffer);
 	rectangles = pixman_region32_rectangles(&surface->texture_damage, &n);
 	for (i = 0; i < n; i++) {
-		glPixelStorei(GL_UNPACK_SKIP_PIXELS, rectangles[i].x1);
+		pixman_box32_t r;
-		glPixelStorei(GL_UNPACK_SKIP_ROWS, rectangles[i].y1);
+
-		glTexSubImage2D(GL_TEXTURE_2D, 0,
+		r = weston_surface_to_buffer_rect(surface, rectangles[i]);
-				rectangles[i].x1, rectangles[i].y1,
+
-				rectangles[i].x2 - rectangles[i].x1,
+		glPixelStorei(GL_UNPACK_SKIP_PIXELS, r.x1);
-				rectangles[i].y2 - rectangles[i].y1,
+		glPixelStorei(GL_UNPACK_SKIP_ROWS, r.y1);
+		glTexSubImage2D(GL_TEXTURE_2D, 0, r.x1, r.y1,
+				r.x2 - r.x1, r.y2 - r.y1,
 				GL_BGRA_EXT, GL_UNSIGNED_BYTE, data);
 	}
 #endif
@@ -1228,16 +1197,7 @@ gl_renderer_attach(struct weston_surface *es, struct wl_buffer *buffer)
 						    gs->images[i]);
 		}
 
-		switch(es->buffer_transform) {
-		case WL_OUTPUT_TRANSFORM_90:
-		case WL_OUTPUT_TRANSFORM_270:
-		case WL_OUTPUT_TRANSFORM_FLIPPED_90:
-		case WL_OUTPUT_TRANSFORM_FLIPPED_270:
-			es->pitch = buffer->height;
-			break;
-		default:
 		es->pitch = buffer->width;
-		}
 	} else {
 		weston_log("unhandled buffer type!\n");
 	}