diff --git a/src/compositor-drm.c b/src/compositor-drm.c
index da36cd3f..11d99818 100644
--- a/src/compositor-drm.c
+++ b/src/compositor-drm.c
@@ -805,6 +805,16 @@ drm_assign_planes(struct weston_output *output)
 	pixman_region32_init(&overlap);
 	primary = &c->base.primary_plane;
 	wl_list_for_each_safe(es, next, &c->base.surface_list, link) {
+		/* test whether this buffer can ever go into a plane:
+		 * non-shm, or small enough to be a cursor
+		 */
+		if ((es->buffer_ref.buffer &&
+		     !wl_buffer_is_shm(es->buffer_ref.buffer)) ||
+		    (es->geometry.width <= 64 && es->geometry.height <= 64))
+			es->keep_buffer = 1;
+		else
+			es->keep_buffer = 0;
+
 		pixman_region32_init(&surface_overlap);
 		pixman_region32_intersect(&surface_overlap, &overlap,
 					  &es->transform.boundingbox);
diff --git a/src/compositor-rpi.c b/src/compositor-rpi.c
index e77c5c6f..df9ce79d 100644
--- a/src/compositor-rpi.c
+++ b/src/compositor-rpi.c
@@ -798,6 +798,9 @@ rpi_output_assign_planes(struct weston_output *base)
 
 	pixman_region32_init(&overlap);
 	wl_list_for_each(surface, &compositor->base.surface_list, link) {
+		/* always, since all buffers are shm on rpi */
+		surface->keep_buffer = 1;
+
 		pixman_region32_init(&surface_overlap);
 		pixman_region32_intersect(&surface_overlap, &overlap,
 					  &surface->transform.boundingbox);
diff --git a/src/compositor.c b/src/compositor.c
index 869fffa2..38bc6bc5 100644
--- a/src/compositor.c
+++ b/src/compositor.c
@@ -1084,9 +1084,21 @@ weston_output_repaint(struct weston_output *output, uint32_t msecs)
 	pixman_region32_fini(&ec->primary_plane.opaque);
 	pixman_region32_init(&ec->primary_plane.opaque);
 
-	wl_list_for_each(es, &ec->surface_list, link)
+	wl_list_for_each(es, &ec->surface_list, link) {
 		surface_accumulate_damage(es, &opaque);
 
+		/* Both the renderer and the backend have seen the buffer
+		 * by now. If renderer needs the buffer, it has its own
+		 * reference set. If the backend wants to keep the buffer
+		 * around for migrating the surface into a non-primary plane
+		 * later, keep_buffer is true. Otherwise, drop the core
+		 * reference now, and allow early buffer release. This enables
+		 * clients to use single-buffering.
+		 */
+		if (!es->keep_buffer)
+			weston_buffer_reference(&es->buffer_ref, NULL);
+	}
+
 	pixman_region32_fini(&opaque);
 
 	pixman_region32_init(&output_damage);
diff --git a/src/compositor.h b/src/compositor.h
index 53f9cfb7..bbf9bde9 100644
--- a/src/compositor.h
+++ b/src/compositor.h
@@ -444,6 +444,7 @@ struct weston_surface {
 
 	struct weston_buffer_reference buffer_ref;
 	uint32_t buffer_transform;
+	int keep_buffer; /* bool for backends to prevent early release */
 
 	/* All the pending state, that wl_surface.commit will apply. */
 	struct {