backend/drm: use atomic test-only commits for modifier fallback

Instead of trying to perform a real modeset in init_renderer,
perform an atomic test-only commit to find out whether disabling
modifiers is necessary because of bandwidth limitations.

This decouples init_renderer from the actual commit, making it
possible to modeset an output with a user-supplied buffer instead
of a black frame.

We loose the ability to make sure the buffers coming from the
swapchain will work fine when using the legacy interface. This
can break i915 when atomic is disabled and modifiers enabled.
But i915 always has atomic (so the user must explicitly disable it
to run into potential bandwidth limitations) and is the only known
problematic driver.

backend/drm/drm.c

@@ -631,27 +631,32 @@ struct wlr_drm_fb *plane_get_next_fb(struct wlr_drm_plane *plane) {
 	return plane->current_fb;
 }
 
-static bool drm_connector_pageflip_renderer(struct wlr_drm_connector *conn,
+static bool drm_connector_test_renderer(struct wlr_drm_connector *conn,
 		const struct wlr_output_state *state) {
 	struct wlr_drm_backend *drm = conn->backend;
-	struct wlr_drm_crtc *crtc = conn->crtc;
+
-	if (!crtc) {
+	if (drm->iface == &legacy_iface) {
-		wlr_drm_conn_log(conn, WLR_ERROR, "Page-flip failed: no CRTC");
+		return true;
-		return false;
 	}
 
-	// drm_crtc_page_flip expects a FB to be available
+	struct wlr_drm_plane *plane = conn->crtc->primary;
-	struct wlr_drm_plane *plane = crtc->primary;
+
-	if (!plane_get_next_fb(plane)) {
+	struct wlr_drm_fb *prev_fb = NULL;
+	drm_fb_move(&prev_fb, &plane->pending_fb);
+
+	bool ok = false;
 	if (!drm_surface_render_black_frame(&plane->surf)) {
-			return false;
+		goto out;
 	}
 	if (!drm_plane_lock_surface(plane, drm)) {
-			return false;
+		goto out;
-		}
 	}
 
-	return drm_crtc_page_flip(conn, state);
+	ok = drm_crtc_commit(conn, state, DRM_MODE_ATOMIC_TEST_ONLY);
+
+out:
+	drm_fb_move(&plane->pending_fb, &prev_fb);
+	return ok;
 }
 
 static bool drm_connector_init_renderer(struct wlr_drm_connector *conn,
@@ -663,29 +668,22 @@ static bool drm_connector_init_renderer(struct wlr_drm_connector *conn,
 		return false;
 	}
 
-	wlr_drm_conn_log(conn, WLR_DEBUG, "Initializing renderer");
+	assert(conn->crtc != NULL);
 
-	struct wlr_drm_crtc *crtc = conn->crtc;
+	wlr_drm_conn_log(conn, WLR_DEBUG, "Initializing renderer");
-	if (!crtc) {
-		wlr_drm_conn_log(conn, WLR_ERROR,
-			"Failed to initialize renderer: no CRTC");
-		return false;
-	}
-	struct wlr_drm_plane *plane = crtc->primary;
 
 	drmModeModeInfo mode = {0};
 	drm_connector_state_mode(conn, state, &mode);
 
+	struct wlr_drm_plane *plane = conn->crtc->primary;
 	int width = mode.hdisplay;
 	int height = mode.vdisplay;
 
-	bool modifiers = drm->addfb2_modifiers;
+	if (drm->addfb2_modifiers) {
-	if (!drm_plane_init_surface(plane, drm, width, height, modifiers) ||
+		// Modifiers are supported, try to use them
-			!drm_connector_pageflip_renderer(conn, state)) {
+		if (drm_plane_init_surface(plane, drm, width, height, true) &&
-		if (!modifiers) {
+				drm_connector_test_renderer(conn, state)) {
-			wlr_drm_conn_log(conn, WLR_ERROR, "Failed to initialize renderer:"
+			return true;
-				"initial page-flip failed");
-			return false;
 		}
 
 		// If page-flipping with modifiers enabled doesn't work, retry without
@@ -693,20 +691,17 @@ static bool drm_connector_init_renderer(struct wlr_drm_connector *conn,
 		wlr_drm_conn_log(conn, WLR_INFO,
 			"Page-flip failed with primary FB modifiers enabled, "
 			"retrying without modifiers");
-		modifiers = false;
-
-		if (!drm_plane_init_surface(plane, drm, width, height, modifiers)) {
-			return false;
 	}
-		if (!drm_connector_pageflip_renderer(conn, state)) {
+
+	if (drm_plane_init_surface(plane, drm, width, height, false) &&
+			drm_connector_test_renderer(conn, state)) {
+		return true;
+	}
+
 	wlr_drm_conn_log(conn, WLR_ERROR, "Failed to initialize renderer: "
 		"initial page-flip failed");
 	return false;
 }
-	}
-
-	return true;
-}
 
 static void realloc_crtcs(struct wlr_drm_backend *drm);
 
@@ -787,6 +782,21 @@ static bool drm_connector_set_mode(struct wlr_drm_connector *conn,
 		return false;
 	}
 
+	// drm_crtc_page_flip expects a FB to be available
+	struct wlr_drm_plane *plane = conn->crtc->primary;
+	if (!plane_get_next_fb(plane)) {
+		if (!drm_surface_render_black_frame(&plane->surf)) {
+			return false;
+		}
+		if (!drm_plane_lock_surface(plane, drm)) {
+			return false;
+		}
+	}
+
+	if (!drm_crtc_page_flip(conn, state)) {
+		return false;
+	}
+
 	conn->state = WLR_DRM_CONN_CONNECTED;
 	conn->desired_mode = NULL;
 	wlr_output_update_mode(&conn->output, wlr_mode);