diff --git a/libweston/backend-drm/drm-internal.h b/libweston/backend-drm/drm-internal.h
index c838c407..29b3a91a 100644
--- a/libweston/backend-drm/drm-internal.h
+++ b/libweston/backend-drm/drm-internal.h
@@ -445,6 +445,7 @@ struct drm_output {
 	int disable_pending;
 	int dpms_off_pending;
 
+	uint32_t gbm_cursor_handle[2];
 	struct drm_fb *gbm_cursor_fb[2];
 	struct drm_plane *cursor_plane;
 	struct weston_view *cursor_view;
diff --git a/libweston/backend-drm/drm.c b/libweston/backend-drm/drm.c
index 13c30155..07ae2643 100644
--- a/libweston/backend-drm/drm.c
+++ b/libweston/backend-drm/drm.c
@@ -1307,6 +1307,7 @@ drm_output_init_cursor_egl(struct drm_output *output, struct drm_backend *b)
 			gbm_bo_destroy(bo);
 			goto err;
 		}
+		output->gbm_cursor_handle[i] = gbm_bo_get_handle(bo).s32;
 	}
 
 	return 0;
diff --git a/libweston/backend-drm/kms.c b/libweston/backend-drm/kms.c
index e5ff31ee..b5b1ade6 100644
--- a/libweston/backend-drm/kms.c
+++ b/libweston/backend-drm/kms.c
@@ -514,8 +514,7 @@ drm_output_set_cursor(struct drm_output_state *output_state)
 	struct drm_backend *b = to_drm_backend(output->base.compositor);
 	struct drm_plane *plane = output->cursor_plane;
 	struct drm_plane_state *state;
-	int handle;
-	struct gbm_bo *bo;
+	uint32_t handle;
 
 	if (!plane)
 		return;
@@ -534,9 +533,8 @@ drm_output_set_cursor(struct drm_output_state *output_state)
 	assert(state->fb == output->gbm_cursor_fb[output->current_cursor]);
 	assert(!plane->state_cur->output || plane->state_cur->output == output);
 
+	handle = output->gbm_cursor_handle[output->current_cursor];
 	if (plane->state_cur->fb != state->fb) {
-		bo = state->fb->bo;
-		handle = gbm_bo_get_handle(bo).s32;
 		if (drmModeSetCursor(b->drm.fd, output->crtc_id, handle,
 				     b->cursor_width, b->cursor_height)) {
 			weston_log("failed to set cursor: %s\n",