diff --git a/libweston/backend-drm/drm-internal.h b/libweston/backend-drm/drm-internal.h
index 1d3170c3..9cd80d66 100644
--- a/libweston/backend-drm/drm-internal.h
+++ b/libweston/backend-drm/drm-internal.h
@@ -340,8 +340,6 @@ struct drm_fb {
 	uint64_t modifier;
 	int width, height;
 	int fd;
-	struct weston_buffer_reference buffer_ref;
-	struct weston_buffer_release_reference buffer_release_ref;
 
 	/* Used by gbm fbs */
 	struct gbm_bo *bo;
@@ -411,6 +409,10 @@ struct drm_plane_state {
 	struct drm_output_state *output_state;
 
 	struct drm_fb *fb;
+	struct {
+		struct weston_buffer_reference buffer;
+		struct weston_buffer_release_reference release;
+	} fb_ref;
 
 	struct weston_view *ev; /**< maintained for drm_assign_planes only */
 
diff --git a/libweston/backend-drm/fb.c b/libweston/backend-drm/fb.c
index a0581dac..dd1cfb06 100644
--- a/libweston/backend-drm/fb.c
+++ b/libweston/backend-drm/fb.c
@@ -48,8 +48,6 @@ drm_fb_destroy(struct drm_fb *fb)
 {
 	if (fb->fb_id != 0)
 		drmModeRmFB(fb->fd, fb->fb_id);
-	weston_buffer_reference(&fb->buffer_ref, NULL);
-	weston_buffer_release_reference(&fb->buffer_release_ref, NULL);
 	free(fb);
 }
 
diff --git a/libweston/backend-drm/state-helpers.c b/libweston/backend-drm/state-helpers.c
index 8fc2e03a..75a2a18c 100644
--- a/libweston/backend-drm/state-helpers.c
+++ b/libweston/backend-drm/state-helpers.c
@@ -93,6 +93,8 @@ drm_plane_state_free(struct drm_plane_state *state, bool force)
 
 	if (force || state != state->plane->state_cur) {
 		drm_fb_unref(state->fb);
+		weston_buffer_reference(&state->fb_ref.buffer, NULL);
+		weston_buffer_release_reference(&state->fb_ref.release, NULL);
 		free(state);
 	}
 }
@@ -128,8 +130,21 @@ drm_plane_state_duplicate(struct drm_output_state *state_output,
 	}
 
 	wl_list_insert(&state_output->plane_list, &dst->link);
-	if (src->fb)
+
+	/* Take a reference on the src framebuffer; if it wraps a client
+	 * buffer, then we must also transfer the reference on the client
+	 * buffer. */
+	if (src->fb) {
 		dst->fb = drm_fb_ref(src->fb);
+		memset(&dst->fb_ref, 0, sizeof(dst->fb_ref));
+		weston_buffer_reference(&dst->fb_ref.buffer,
+					src->fb_ref.buffer.buffer);
+		weston_buffer_release_reference(&dst->fb_ref.release,
+						src->fb_ref.release.buffer_release);
+	} else {
+		assert(!src->fb_ref.buffer.buffer);
+		assert(!src->fb_ref.release.buffer_release);
+	}
 	dst->output_state = state_output;
 	dst->complete = false;
 
diff --git a/libweston/backend-drm/state-propose.c b/libweston/backend-drm/state-propose.c
index 7242af9e..6926855c 100644
--- a/libweston/backend-drm/state-propose.c
+++ b/libweston/backend-drm/state-propose.c
@@ -607,14 +607,15 @@ out:
 		/* Take a reference on the buffer so that we don't release it
 		 * back to the client until we're done with it; cursor buffers
 		 * don't require a reference since we copy them. */
-		assert(fb->buffer_ref.buffer == NULL);
-		assert(fb->buffer_release_ref.buffer_release == NULL);
+		assert(ps->fb_ref.buffer.buffer == NULL);
+		assert(ps->fb_ref.release.buffer_release == NULL);
 		if (ps->plane->type == WDRM_PLANE_TYPE_CURSOR) {
 			assert(ps->fb->type == BUFFER_CURSOR);
 		} else if (fb->type == BUFFER_CLIENT || fb->type == BUFFER_DMABUF) {
 			assert(ps->fb == fb);
-			weston_buffer_reference(&fb->buffer_ref, surface->buffer_ref.buffer);
-			weston_buffer_release_reference(&fb->buffer_release_ref,
+			weston_buffer_reference(&ps->fb_ref.buffer,
+						surface->buffer_ref.buffer);
+			weston_buffer_release_reference(&ps->fb_ref.release,
 							surface->buffer_release_ref.buffer_release);
 		}
 		break;