diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index 4c13d01a..cd3208d3c 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -259,7 +259,8 @@ static void wlr_drm_connector_enable(struct wlr_output *output, bool enable) {
 	}
 }
 
-static void realloc_planes(struct wlr_drm_backend *drm, const uint32_t *crtc_in) {
+static void realloc_planes(struct wlr_drm_backend *drm, const uint32_t *crtc_in,
+		bool *changed_outputs) {
 	// overlay, primary, cursor
 	for (int type = 0; type < 3; ++type) {
 		if (drm->num_type_planes[type] == 0) {
@@ -298,6 +299,7 @@ static void realloc_planes(struct wlr_drm_backend *drm, const uint32_t *crtc_in)
 			struct wlr_drm_plane *new = &drm->type_planes[type][crtc_res[i]];
 
 			if (*old != new) {
+				changed_outputs[crtc_res[i]] = true;
 				if (*old) {
 					wlr_drm_surface_finish(&(*old)->surf);
 				}
@@ -308,7 +310,8 @@ static void realloc_planes(struct wlr_drm_backend *drm, const uint32_t *crtc_in)
 	}
 }
 
-static void realloc_crtcs(struct wlr_drm_backend *drm, struct wlr_drm_connector *conn) {
+static void realloc_crtcs(struct wlr_drm_backend *drm,
+		struct wlr_drm_connector *conn, bool *changed_outputs) {
 	uint32_t crtc[drm->num_crtcs];
 	uint32_t crtc_res[drm->num_crtcs];
 	uint32_t possible_crtc[wl_list_length(&drm->outputs)];
@@ -356,12 +359,15 @@ static void realloc_crtcs(struct wlr_drm_backend *drm, struct wlr_drm_connector
 		return;
 	}
 
+	changed_outputs[index] = true;
+
 	for (size_t i = 0; i < drm->num_crtcs; ++i) {
 		if (crtc_res[i] == UNMATCHED) {
 			continue;
 		}
 
 		if (crtc_res[i] != crtc[i]) {
+			changed_outputs[crtc_res[i]] = true;
 			struct wlr_drm_connector *c;
 			size_t pos = 0;
 			wl_list_for_each(c, &drm->outputs, link) {
@@ -374,7 +380,7 @@ static void realloc_crtcs(struct wlr_drm_backend *drm, struct wlr_drm_connector
 		}
 	}
 
-	realloc_planes(drm, crtc_res);
+	realloc_planes(drm, crtc_res, changed_outputs);
 }
 
 static uint32_t get_possible_crtcs(int fd, uint32_t conn_id) {
@@ -413,6 +419,7 @@ static bool wlr_drm_connector_set_mode(struct wlr_output *output,
 		struct wlr_output_mode *mode) {
 	struct wlr_drm_connector *conn = (struct wlr_drm_connector *)output;
 	struct wlr_drm_backend *drm = (struct wlr_drm_backend *)output->backend;
+	bool changed_outputs[wl_list_length(&drm->outputs)];
 
 	wlr_log(L_INFO, "Modesetting '%s' with '%ux%u@%u mHz'", conn->output.name,
 			mode->width, mode->height, mode->refresh);
@@ -422,7 +429,8 @@ static bool wlr_drm_connector_set_mode(struct wlr_output *output,
 		goto error_conn;
 	}
 
-	realloc_crtcs(drm, conn);
+	memset(changed_outputs, false, sizeof(changed_outputs));
+	realloc_crtcs(drm, conn, changed_outputs);
 
 	if (!conn->crtc) {
 		wlr_log(L_ERROR, "Unable to match %s with a CRTC", conn->output.name);
@@ -445,12 +453,15 @@ static bool wlr_drm_connector_set_mode(struct wlr_output *output,
 	}
 
 	// Since realloc_crtcs can deallocate planes on OTHER outputs,
-	// we actually need to reinitalise all of them
+	// we actually need to reinitalise any than has changed
+	ssize_t output_index = -1;
 	wl_list_for_each(conn, &drm->outputs, link) {
+		output_index += 1;
 		struct wlr_output_mode *mode = conn->output.current_mode;
 		struct wlr_drm_crtc *crtc = conn->crtc;
 
-		if (conn->state != WLR_DRM_CONN_CONNECTED) {
+		if (conn->state != WLR_DRM_CONN_CONNECTED ||
+				!changed_outputs[output_index]) {
 			continue;
 		}