compositor: add a masking mechanism to weston_layer

this adds a mechanism to mask the views belonging to a layer
to an arbitrary rect, in the global space. The parts that don't fit
in that rect will be clipped away.
Supported by the gl and pixman renderer only for now.

src/compositor.c

@@ -349,8 +349,11 @@ weston_view_create(struct weston_surface *surface)
 
 	view->plane = NULL;
 	view->layer_link.layer = NULL;
+	view->parent_view = NULL;
 
 	pixman_region32_init(&view->clip);
+	pixman_region32_init(&view->transform.masked_boundingbox);
+	pixman_region32_init(&view->transform.masked_opaque);
 
 	view->alpha = 1.0;
 	pixman_region32_init(&view->transform.opaque);
@@ -783,7 +786,7 @@ weston_view_damage_below(struct weston_view *view)
 	pixman_region32_t damage;
 
 	pixman_region32_init(&damage);
-	pixman_region32_subtract(&damage, &view->transform.boundingbox,
+	pixman_region32_subtract(&damage, &view->transform.masked_boundingbox,
 				 &view->clip);
 	if (view->plane)
 		pixman_region32_union(&view->plane->damage,
@@ -1009,10 +1012,20 @@ weston_view_update_transform_enable(struct weston_view *view)
 	return 0;
 }
 
+static struct weston_layer *
+get_view_layer(struct weston_view *view)
+{
+	if (view->parent_view)
+		return get_view_layer(view->parent_view);
+	return view->layer_link.layer;
+}
+
 WL_EXPORT void
 weston_view_update_transform(struct weston_view *view)
 {
 	struct weston_view *parent = view->geometry.parent;
+	struct weston_layer *layer;
+	pixman_region32_t mask;
 
 	if (!view->transform.dirty)
 		return;
@@ -1040,6 +1053,16 @@ weston_view_update_transform(struct weston_view *view)
 			weston_view_update_transform_disable(view);
 	}
 
+	layer = get_view_layer(view);
+	if (layer) {
+		pixman_region32_init_with_extents(&mask, &layer->mask);
+		pixman_region32_intersect(&view->transform.masked_boundingbox,
+					&view->transform.boundingbox, &mask);
+		pixman_region32_intersect(&view->transform.masked_opaque,
+					&view->transform.opaque, &mask);
+		pixman_region32_fini(&mask);
+	}
+
 	weston_view_damage_below(view);
 
 	weston_view_assign_output(view);
@@ -1331,10 +1354,15 @@ weston_compositor_pick_view(struct weston_compositor *compositor,
 			    wl_fixed_t *vx, wl_fixed_t *vy)
 {
 	struct weston_view *view;
+        int ix = wl_fixed_to_int(x);
+        int iy = wl_fixed_to_int(y);
 
 	wl_list_for_each(view, &compositor->view_list, link) {
 		weston_view_from_global_fixed(view, x, y, vx, vy);
-		if (pixman_region32_contains_point(&view->surface->input,
+		if (pixman_region32_contains_point(
+			&view->transform.masked_boundingbox,
+						   ix, iy, NULL) &&
+		    pixman_region32_contains_point(&view->surface->input,
 						   wl_fixed_to_int(*vx),
 						   wl_fixed_to_int(*vy),
 						   NULL))
@@ -1426,6 +1454,8 @@ weston_view_destroy(struct weston_view *view)
 
 	pixman_region32_fini(&view->clip);
 	pixman_region32_fini(&view->transform.boundingbox);
+	pixman_region32_fini(&view->transform.masked_boundingbox);
+	pixman_region32_fini(&view->transform.masked_opaque);
 
 	weston_view_set_transform_parent(view, NULL);
 
@@ -1625,7 +1655,7 @@ view_accumulate_damage(struct weston_view *view,
 			      &view->plane->damage, &damage);
 	pixman_region32_fini(&damage);
 	pixman_region32_copy(&view->clip, opaque);
-	pixman_region32_union(opaque, opaque, &view->transform.opaque);
+	pixman_region32_union(opaque, opaque, &view->transform.masked_opaque);
 }
 
 static void
@@ -1740,6 +1770,7 @@ view_list_add_subsurface_view(struct weston_compositor *compositor,
 		weston_view_set_transform_parent(view, parent);
 	}
 
+	view->parent_view = parent;
 	weston_view_update_transform(view);
 
 	if (wl_list_empty(&sub->surface->subsurface_list)) {
@@ -1933,10 +1964,34 @@ weston_layer_init(struct weston_layer *layer, struct wl_list *below)
 {
 	wl_list_init(&layer->view_list.link);
 	layer->view_list.layer = layer;
+	weston_layer_set_mask_infinite(layer);
 	if (below != NULL)
 		wl_list_insert(below, &layer->link);
 }
 
+WL_EXPORT void
+weston_layer_set_mask(struct weston_layer *layer,
+		      int x, int y, int width, int height)
+{
+	struct weston_view *view;
+
+	layer->mask.x1 = x;
+	layer->mask.x2 = x + width;
+	layer->mask.y1 = y;
+	layer->mask.y2 = y + height;
+
+	wl_list_for_each(view, &layer->view_list.link, layer_link.link) {
+		weston_view_geometry_dirty(view);
+	}
+}
+
+WL_EXPORT void
+weston_layer_set_mask_infinite(struct weston_layer *layer)
+{
+	weston_layer_set_mask(layer, INT32_MIN, INT32_MIN,
+				     UINT32_MAX, UINT32_MAX);
+}
+
 WL_EXPORT void
 weston_output_schedule_repaint(struct weston_output *output)
 {

src/compositor.h

@@ -534,6 +534,7 @@ struct weston_layer_entry {
 struct weston_layer {
 	struct weston_layer_entry view_list;
 	struct wl_list link;
+	pixman_box32_t mask;
 };
 
 struct weston_plane {
@@ -738,6 +739,7 @@ struct weston_view {
 	struct wl_list link;
 	struct weston_layer_entry layer_link;
 	struct weston_plane *plane;
+	struct weston_view *parent_view;
 
 	pixman_region32_t clip;
 	float alpha;                     /* part of geometry, see below */
@@ -769,6 +771,8 @@ struct weston_view {
 
 		pixman_region32_t boundingbox;
 		pixman_region32_t opaque;
+		pixman_region32_t masked_boundingbox;
+		pixman_region32_t masked_opaque;
 
 		/* matrix and inverse are used only if enabled = 1.
 		 * If enabled = 0, use x, y, width, height directly.
@@ -1009,6 +1013,12 @@ weston_layer_entry_remove(struct weston_layer_entry *entry);
 void
 weston_layer_init(struct weston_layer *layer, struct wl_list *below);
 
+void
+weston_layer_set_mask(struct weston_layer *layer, int x, int y, int width, int height);
+
+void
+weston_layer_set_mask_infinite(struct weston_layer *layer);
+
 void
 weston_plane_init(struct weston_plane *plane,
 			struct weston_compositor *ec,

src/gl-renderer.c

@@ -542,7 +542,7 @@ draw_view(struct weston_view *ev, struct weston_output *output,
 
 	pixman_region32_init(&repaint);
 	pixman_region32_intersect(&repaint,
-				  &ev->transform.boundingbox, damage);
+				  &ev->transform.masked_boundingbox, damage);
 	pixman_region32_subtract(&repaint, &repaint, &ev->clip);
 
 	if (!pixman_region32_not_empty(&repaint))

src/pixman-renderer.c

@@ -397,7 +397,7 @@ draw_view(struct weston_view *ev, struct weston_output *output,
 
 	pixman_region32_init(&repaint);
 	pixman_region32_intersect(&repaint,
-				  &ev->transform.boundingbox, damage);
+				  &ev->transform.masked_boundingbox, damage);
 	pixman_region32_subtract(&repaint, &repaint, &ev->clip);
 
 	if (!pixman_region32_not_empty(&repaint))