diff --git a/src/launcher-util.c b/src/launcher-util.c
index a599686b..5ccf3025 100644
--- a/src/launcher-util.c
+++ b/src/launcher-util.c
@@ -202,6 +202,11 @@ weston_launcher_restore(struct weston_launcher *launcher)
 	if (ioctl(launcher->tty, KDSETMODE, KD_TEXT))
 		weston_log("failed to set KD_TEXT mode on tty: %m\n");
 
+	/* We have to drop master before we switch the VT back in
+	 * VT_AUTO, so we don't risk switching to a VT with another
+	 * display server, that will then fail to set drm master. */
+	drm_drop_master(launcher->drm_fd);
+
 	mode.mode = VT_AUTO;
 	if (ioctl(launcher->tty, VT_SETMODE, &mode) < 0)
 		weston_log("could not reset vt handling\n");
diff --git a/src/weston-launch.c b/src/weston-launch.c
index 03a6a430..d8364c85 100644
--- a/src/weston-launch.c
+++ b/src/weston-launch.c
@@ -407,6 +407,11 @@ quit(struct weston_launch *wl, int status)
 	if (ioctl(wl->tty, KDSETMODE, KD_TEXT))
 		fprintf(stderr, "failed to set KD_TEXT mode on tty: %m\n");
 
+	/* We have to drop master before we switch the VT back in
+	 * VT_AUTO, so we don't risk switching to a VT with another
+	 * display server, that will then fail to set drm master. */
+	drmDropMaster(wl->drm_fd);
+
 	mode.mode = VT_AUTO;
 	if (ioctl(wl->tty, VT_SETMODE, &mode) < 0)
 		fprintf(stderr, "could not reset vt handling\n");