output: introduce wlr_output_configure_primary_swapchain()

The goal is to simplify wlr_output by moving all of its rendering
API into separate helpers. Here is a first step to sunset
wlr_output_attach_render(). Instead, compositors call
wlr_output_configure_primary_swapchain(), wlr_swapchain_acquire(),
wlr_renderer_begin_with_buffer() and wlr_output_attach_buffer().

Note that compositors can supply a base struct wlr_output_state.
This is useful to allocate a buffer suitable for submission with
a modeset, for instance.

Closes: https://gitlab.freedesktop.org/wlroots/wlroots/-/issues/3079
References: https://gitlab.freedesktop.org/wlroots/wlroots/-/merge_requests/3197
References: https://gitlab.freedesktop.org/wlroots/wlroots/-/merge_requests/3984

include/wlr/types/wlr_output.h

@@ -563,6 +563,20 @@ void wlr_output_state_set_buffer(struct wlr_output_state *state,
 	struct wlr_buffer *buffer);
 
 
+/**
+ * Re-configure the swapchain as required for the output's primary buffer.
+ *
+ * If a NULL swapchain is passed in, a new swapchain is allocated. If the
+ * swapchain is already suitable for the output's primary buffer, this function
+ * is a no-op.
+ *
+ * The state describes the output changes the swapchain's buffers will be
+ * committed with. A NULL state indicates no change.
+ */
+bool wlr_output_configure_primary_swapchain(struct wlr_output *output,
+	const struct wlr_output_state *state, struct wlr_swapchain **swapchain);
+
+
 /**
  * Returns the transform that, when composed with `tr`, gives
  * `WL_OUTPUT_TRANSFORM_NORMAL`.

types/meson.build

@@ -7,6 +7,7 @@ wlr_files += files(
 	'output/output.c',
 	'output/render.c',
 	'output/state.c',
+	'output/swapchain.c',
 	'output/transform.c',
 	'scene/drag_icon.c',
 	'scene/subsurface_tree.c',

types/output/swapchain.c

@@ -0,0 +1,109 @@
+#include <assert.h>
+#include <drm_fourcc.h>
+#include <stdlib.h>
+#include <wlr/render/allocator.h>
+#include <wlr/render/swapchain.h>
+#include <wlr/util/log.h>
+#include <xf86drm.h>
+
+#include "render/drm_format_set.h"
+#include "types/wlr_output.h"
+
+static struct wlr_swapchain *create_swapchain(struct wlr_output *output,
+		int width, int height, bool allow_modifiers) {
+	struct wlr_allocator *allocator = output->allocator;
+	assert(output->allocator != NULL);
+
+	const struct wlr_drm_format_set *display_formats =
+		wlr_output_get_primary_formats(output, allocator->buffer_caps);
+	struct wlr_drm_format *format = output_pick_format(output, display_formats,
+		output->render_format);
+	if (format == NULL) {
+		wlr_log(WLR_ERROR, "Failed to pick primary buffer format for output '%s'",
+			output->name);
+		return NULL;
+	}
+
+	char *format_name = drmGetFormatName(format->format);
+	wlr_log(WLR_DEBUG, "Choosing primary buffer format %s (0x%08"PRIX32") for output '%s'",
+		format_name ? format_name : "<unknown>", format->format, output->name);
+	free(format_name);
+
+	if (!allow_modifiers && (format->len != 1 || format->modifiers[0] != DRM_FORMAT_MOD_LINEAR)) {
+		if (!wlr_drm_format_has(format, DRM_FORMAT_MOD_INVALID)) {
+			wlr_log(WLR_DEBUG, "Implicit modifiers not supported");
+			free(format);
+			return NULL;
+		}
+
+		format->len = 0;
+		wlr_drm_format_add(&format, DRM_FORMAT_MOD_INVALID);
+	}
+
+	struct wlr_swapchain *swapchain = wlr_swapchain_create(allocator, width, height, format);
+	free(format);
+	return swapchain;
+}
+
+static bool test_swapchain(struct wlr_output *output,
+		struct wlr_swapchain *swapchain, const struct wlr_output_state *state) {
+	struct wlr_buffer *buffer = wlr_swapchain_acquire(swapchain, NULL);
+	if (buffer == NULL) {
+		return false;
+	}
+
+	struct wlr_output_state copy = *state;
+	copy.committed |= WLR_OUTPUT_STATE_BUFFER;
+	copy.buffer = buffer;
+	bool ok = wlr_output_test_state(output, &copy);
+	wlr_buffer_unlock(buffer);
+	return ok;
+}
+
+bool wlr_output_configure_primary_swapchain(struct wlr_output *output,
+		const struct wlr_output_state *state, struct wlr_swapchain **swapchain_ptr) {
+	const struct wlr_output_state empty_state = {0};
+	if (state == NULL) {
+		state = &empty_state;
+	}
+
+	int width, height;
+	output_pending_resolution(output, state, &width, &height);
+
+	// Re-use the existing swapchain if possible
+	struct wlr_swapchain *old_swapchain = *swapchain_ptr;
+	if (old_swapchain != NULL &&
+			old_swapchain->width == width && old_swapchain->height == height &&
+			old_swapchain->format->format == output->render_format) {
+		return true;
+	}
+
+	struct wlr_swapchain *swapchain = create_swapchain(output, width, height, true);
+	if (swapchain == NULL) {
+		wlr_log(WLR_ERROR, "Failed to create swapchain for output '%s'", output->name);
+		return false;
+	}
+
+	wlr_log(WLR_DEBUG, "Testing swapchain for output '%s'", output->name);
+	if (!test_swapchain(output, swapchain, state)) {
+		wlr_log(WLR_DEBUG, "Output test failed on '%s', retrying without modifiers",
+			output->name);
+		wlr_swapchain_destroy(swapchain);
+		swapchain = create_swapchain(output, width, height, false);
+		if (swapchain == NULL) {
+			wlr_log(WLR_ERROR, "Failed to create modifier-less swapchain for output '%s'",
+				output->name);
+			return NULL;
+		}
+		wlr_log(WLR_DEBUG, "Testing modifier-less swapchain for output '%s'", output->name);
+		if (!test_swapchain(output, swapchain, state)) {
+			wlr_log(WLR_ERROR, "Swapchain for output '%s' failed test", output->name);
+			wlr_swapchain_destroy(swapchain);
+			return NULL;
+		}
+	}
+
+	wlr_swapchain_destroy(*swapchain_ptr);
+	*swapchain_ptr = swapchain;
+	return true;
+}