diff --git a/include/libweston/libweston.h b/include/libweston/libweston.h
index 1fe3e4a5..2f60dc8e 100644
--- a/include/libweston/libweston.h
+++ b/include/libweston/libweston.h
@@ -1201,8 +1201,15 @@ struct weston_buffer {
 	uint64_t format_modifier;
 };
 
+enum weston_buffer_reference_type {
+	BUFFER_REF_NONE,
+	BUFFER_MAY_BE_ACCESSED,
+	BUFFER_WILL_NOT_BE_ACCESSED,
+};
+
 struct weston_buffer_reference {
 	struct weston_buffer *buffer;
+	enum weston_buffer_reference_type type;
 };
 
 struct weston_buffer_viewport {
diff --git a/libweston/backend-drm/state-helpers.c b/libweston/backend-drm/state-helpers.c
index 75a2a18c..245ee087 100644
--- a/libweston/backend-drm/state-helpers.c
+++ b/libweston/backend-drm/state-helpers.c
@@ -93,7 +93,8 @@ drm_plane_state_free(struct drm_plane_state *state, bool force)
 
 	if (force || state != state->plane->state_cur) {
 		drm_fb_unref(state->fb);
-		weston_buffer_reference(&state->fb_ref.buffer, NULL);
+		weston_buffer_reference(&state->fb_ref.buffer, NULL,
+					BUFFER_WILL_NOT_BE_ACCESSED);
 		weston_buffer_release_reference(&state->fb_ref.release, NULL);
 		free(state);
 	}
@@ -135,10 +136,20 @@ drm_plane_state_duplicate(struct drm_output_state *state_output,
 	 * buffer, then we must also transfer the reference on the client
 	 * buffer. */
 	if (src->fb) {
+		struct weston_buffer *buffer;
+
 		dst->fb = drm_fb_ref(src->fb);
 		memset(&dst->fb_ref, 0, sizeof(dst->fb_ref));
-		weston_buffer_reference(&dst->fb_ref.buffer,
-					src->fb_ref.buffer.buffer);
+
+		if (src->fb->type == BUFFER_CLIENT ||
+		    src->fb->type == BUFFER_DMABUF) {
+			buffer = src->fb_ref.buffer.buffer;
+		} else {
+			buffer = NULL;
+		}
+		weston_buffer_reference(&dst->fb_ref.buffer, buffer,
+					buffer ? BUFFER_MAY_BE_ACCESSED :
+					         BUFFER_WILL_NOT_BE_ACCESSED);
 		weston_buffer_release_reference(&dst->fb_ref.release,
 						src->fb_ref.release.buffer_release);
 	} else {
diff --git a/libweston/backend-drm/state-propose.c b/libweston/backend-drm/state-propose.c
index 5acaa809..799d4825 100644
--- a/libweston/backend-drm/state-propose.c
+++ b/libweston/backend-drm/state-propose.c
@@ -139,7 +139,8 @@ drm_output_try_view_on_plane(struct drm_plane *plane,
 	assert(state->fb_ref.buffer.buffer == NULL);
 	assert(state->fb_ref.release.buffer_release == NULL);
 	weston_buffer_reference(&state->fb_ref.buffer,
-				surface->buffer_ref.buffer);
+				surface->buffer_ref.buffer,
+				BUFFER_MAY_BE_ACCESSED);
 	weston_buffer_release_reference(&state->fb_ref.release,
 					surface->buffer_release_ref.buffer_release);
 
diff --git a/libweston/compositor.c b/libweston/compositor.c
index 8561eed0..62eff5f1 100644
--- a/libweston/compositor.c
+++ b/libweston/compositor.c
@@ -2334,7 +2334,8 @@ weston_surface_destroy(struct weston_surface *surface)
 
 	weston_surface_state_fini(&surface->pending);
 
-	weston_buffer_reference(&surface->buffer_ref, NULL);
+	weston_buffer_reference(&surface->buffer_ref, NULL,
+				BUFFER_WILL_NOT_BE_ACCESSED);
 	weston_buffer_release_reference(&surface->buffer_release_ref, NULL);
 
 	pixman_region32_fini(&surface->damage);
@@ -2472,8 +2473,11 @@ fail:
 
 WL_EXPORT void
 weston_buffer_reference(struct weston_buffer_reference *ref,
-			struct weston_buffer *buffer)
+			struct weston_buffer *buffer,
+			enum weston_buffer_reference_type type)
 {
+	assert(buffer != NULL || type == BUFFER_WILL_NOT_BE_ACCESSED);
+
 	if (buffer == ref->buffer)
 		return;
 
@@ -2562,7 +2566,9 @@ static void
 weston_surface_attach(struct weston_surface *surface,
 		      struct weston_buffer *buffer)
 {
-	weston_buffer_reference(&surface->buffer_ref, buffer);
+	weston_buffer_reference(&surface->buffer_ref, buffer,
+				buffer ? BUFFER_MAY_BE_ACCESSED :
+					 BUFFER_WILL_NOT_BE_ACCESSED);
 
 	if (!buffer) {
 		if (weston_surface_is_mapped(surface))
@@ -2699,7 +2705,9 @@ output_accumulate_damage(struct weston_output *output)
 		 * clients to use single-buffering.
 		 */
 		if (!pnode->surface->keep_buffer) {
-			weston_buffer_reference(&pnode->surface->buffer_ref, NULL);
+			weston_buffer_reference(&pnode->surface->buffer_ref,
+						NULL,
+						BUFFER_WILL_NOT_BE_ACCESSED);
 			weston_buffer_release_reference(
 				&pnode->surface->buffer_release_ref, NULL);
 		}
@@ -4226,7 +4234,8 @@ weston_subsurface_commit_from_cache(struct weston_subsurface *sub)
 	struct weston_surface *surface = sub->surface;
 
 	weston_surface_commit_state(surface, &sub->cached);
-	weston_buffer_reference(&sub->cached_buffer_ref, NULL);
+	weston_buffer_reference(&sub->cached_buffer_ref, NULL,
+				BUFFER_WILL_NOT_BE_ACCESSED);
 
 	weston_surface_commit_subsurface_order(surface);
 
@@ -4263,7 +4272,10 @@ weston_subsurface_commit_to_cache(struct weston_subsurface *sub)
 		weston_surface_state_set_buffer(&sub->cached,
 						surface->pending.buffer);
 		weston_buffer_reference(&sub->cached_buffer_ref,
-					surface->pending.buffer);
+					surface->pending.buffer,
+					surface->pending.buffer ?
+						BUFFER_MAY_BE_ACCESSED :
+						BUFFER_WILL_NOT_BE_ACCESSED);
 		weston_presentation_feedback_discard_list(
 					&sub->cached.feedback_list);
 		/* zwp_surface_synchronization_v1.set_acquire_fence */
@@ -4845,7 +4857,8 @@ weston_subsurface_destroy(struct weston_subsurface *sub)
 			weston_subsurface_unlink_parent(sub);
 
 		weston_surface_state_fini(&sub->cached);
-		weston_buffer_reference(&sub->cached_buffer_ref, NULL);
+		weston_buffer_reference(&sub->cached_buffer_ref, NULL,
+					BUFFER_WILL_NOT_BE_ACCESSED);
 
 		sub->surface->committed = NULL;
 		sub->surface->committed_private = NULL;
diff --git a/libweston/libweston-internal.h b/libweston/libweston-internal.h
index 02c4ad3b..c40ba12c 100644
--- a/libweston/libweston-internal.h
+++ b/libweston/libweston-internal.h
@@ -50,7 +50,8 @@ weston_buffer_send_server_error(struct weston_buffer *buffer,
 				const char *msg);
 void
 weston_buffer_reference(struct weston_buffer_reference *ref,
-			struct weston_buffer *buffer);
+			struct weston_buffer *buffer,
+			enum weston_buffer_reference_type type);
 
 void
 weston_buffer_release_move(struct weston_buffer_release_reference *dest,
diff --git a/libweston/pixman-renderer.c b/libweston/pixman-renderer.c
index be2df1d5..32d28349 100644
--- a/libweston/pixman-renderer.c
+++ b/libweston/pixman-renderer.c
@@ -627,7 +627,9 @@ pixman_renderer_attach(struct weston_surface *es, struct weston_buffer *buffer)
 	struct wl_shm_buffer *shm_buffer;
 	const struct pixel_format_info *pixel_info;
 
-	weston_buffer_reference(&ps->buffer_ref, buffer);
+	weston_buffer_reference(&ps->buffer_ref, buffer,
+				buffer ? BUFFER_MAY_BE_ACCESSED :
+					 BUFFER_WILL_NOT_BE_ACCESSED);
 	weston_buffer_release_reference(&ps->buffer_release_ref,
 					es->buffer_release_ref.buffer_release);
 
@@ -646,7 +648,8 @@ pixman_renderer_attach(struct weston_surface *es, struct weston_buffer *buffer)
 
 	if (buffer->type != WESTON_BUFFER_SHM) {
 		weston_log("Pixman renderer supports only SHM buffers\n");
-		weston_buffer_reference(&ps->buffer_ref, NULL);
+		weston_buffer_reference(&ps->buffer_ref, NULL,
+					BUFFER_WILL_NOT_BE_ACCESSED);
 		weston_buffer_release_reference(&ps->buffer_release_ref, NULL);
 		return;
 	}
@@ -657,7 +660,8 @@ pixman_renderer_attach(struct weston_surface *es, struct weston_buffer *buffer)
 	if (!pixel_info || !pixman_format_supported_source(pixel_info->pixman_format)) {
 		weston_log("Unsupported SHM buffer format 0x%x\n",
 			wl_shm_buffer_get_format(shm_buffer));
-		weston_buffer_reference(&ps->buffer_ref, NULL);
+		weston_buffer_reference(&ps->buffer_ref, NULL,
+					BUFFER_WILL_NOT_BE_ACCESSED);
 		weston_buffer_release_reference(&ps->buffer_release_ref, NULL);
 		weston_buffer_send_server_error(buffer,
 			"disconnecting due to unhandled buffer type");
@@ -693,7 +697,8 @@ pixman_renderer_surface_state_destroy(struct pixman_surface_state *ps)
 		pixman_image_unref(ps->image);
 		ps->image = NULL;
 	}
-	weston_buffer_reference(&ps->buffer_ref, NULL);
+	weston_buffer_reference(&ps->buffer_ref, NULL,
+				BUFFER_WILL_NOT_BE_ACCESSED);
 	weston_buffer_release_reference(&ps->buffer_release_ref, NULL);
 	free(ps);
 }
diff --git a/libweston/renderer-gl/gl-renderer.c b/libweston/renderer-gl/gl-renderer.c
index aef37499..b442caaf 100644
--- a/libweston/renderer-gl/gl-renderer.c
+++ b/libweston/renderer-gl/gl-renderer.c
@@ -1931,7 +1931,8 @@ done:
 	pixman_region32_init(&gs->texture_damage);
 	gs->needs_full_upload = false;
 
-	weston_buffer_reference(&gs->buffer_ref, NULL);
+	weston_buffer_reference(&gs->buffer_ref, buffer,
+				BUFFER_WILL_NOT_BE_ACCESSED);
 	weston_buffer_release_reference(&gs->buffer_release_ref, NULL);
 }
 
@@ -2971,7 +2972,9 @@ gl_renderer_attach(struct weston_surface *es, struct weston_buffer *buffer)
 	EGLint format;
 	int i;
 
-	weston_buffer_reference(&gs->buffer_ref, buffer);
+	weston_buffer_reference(&gs->buffer_ref, buffer,
+				buffer ? BUFFER_MAY_BE_ACCESSED :
+					 BUFFER_WILL_NOT_BE_ACCESSED);
 	weston_buffer_release_reference(&gs->buffer_release_ref,
 					es->buffer_release_ref.buffer_release);
 
@@ -3015,7 +3018,8 @@ gl_renderer_attach(struct weston_surface *es, struct weston_buffer *buffer)
 		weston_log("eglQueryWaylandBufferWL failed\n");
 		gl_renderer_print_egl_error_state();
 	}
-	weston_buffer_reference(&gs->buffer_ref, NULL);
+	weston_buffer_reference(&gs->buffer_ref, NULL,
+				BUFFER_WILL_NOT_BE_ACCESSED);
 	weston_buffer_release_reference(&gs->buffer_release_ref, NULL);
 	gs->buffer_type = BUFFER_TYPE_NULL;
 	gs->y_inverted = true;
@@ -3199,7 +3203,8 @@ surface_state_destroy(struct gl_surface_state *gs, struct gl_renderer *gr)
 	for (i = 0; i < gs->num_images; i++)
 		egl_image_unref(gs->images[i]);
 
-	weston_buffer_reference(&gs->buffer_ref, NULL);
+	weston_buffer_reference(&gs->buffer_ref, NULL,
+				BUFFER_WILL_NOT_BE_ACCESSED);
 	weston_buffer_release_reference(&gs->buffer_release_ref, NULL);
 	pixman_region32_fini(&gs->texture_damage);
 	free(gs);