output-swapchain-manager: new helper

This new helper assists compositors in allocating buffers for
modesets. It degrades to different allocation parameters as
needed, and should help with screens not turning on when multiple
outputs are connected on some hardware (e.g. Intel).

For simplicity, the old logic to try allocating with explicit
modifiers first and then fallback to implicit modifiers later is
left as-is. We'll probably want to have more complicated logic
instead in the future: try the fallback on one output at a time,
and try dropping modifiers one by one instead of using implicit
modifiers (at the cost of some combinatorial explosion).

Closes: https://gitlab.freedesktop.org/wlroots/wlroots/-/issues/1873
Co-authored-by: Kenny Levinsen <kl@kl.wtf>

include/wlr/types/wlr_output_swapchain_manager.h

@@ -0,0 +1,90 @@
+/*
+ * This an unstable interface of wlroots. No guarantees are made regarding the
+ * future consistency of this API.
+ */
+#ifndef WLR_USE_UNSTABLE
+#error "Add -DWLR_USE_UNSTABLE to enable unstable wlroots features"
+#endif
+
+#ifndef WLR_TYPES_WLR_OUTPUT_SWAPCHAIN_MANAGER_H
+#define WLR_TYPES_WLR_OUTPUT_SWAPCHAIN_MANAGER_H
+
+#include <wayland-server-core.h>
+
+struct wlr_backend;
+struct wlr_backend_output_state;
+
+/**
+ * Helper to allocate swapchains for mode-setting.
+ *
+ * Compositors are expected to call wlr_output_swapchain_manager_init(), then
+ * pass the new desired output states to wlr_output_swapchain_manager_prepare().
+ * Compositors may retry that step with different desired output states until
+ * they find a working configuration. Then, compositors should use
+ * wlr_output_swapchain_manager_get_swapchain() to get pending swapchains for
+ * outputs, render onto a new buffer acquired from the swapchain, and call
+ * wlr_backend_commit(). If that succeeds, wlr_output_swapchain_manager_apply()
+ * should be called. After compositors are done with the manager, be it after a
+ * success or failure, they should call wlr_output_swapchain_manager_finish().
+ */
+struct wlr_output_swapchain_manager {
+	struct wlr_backend *backend;
+
+	// private state
+
+	struct wl_array outputs; // struct wlr_output_swapchain_manager_output
+};
+
+/**
+ * Initialize the manager.
+ *
+ * Compositors should call wlr_output_swapchain_manager_finish() to cleanup the
+ * manager.
+ */
+void wlr_output_swapchain_manager_init(struct wlr_output_swapchain_manager *manager,
+	struct wlr_backend *backend);
+
+/**
+ * Prepare a commit for a mode-setting backend commit.
+ *
+ * This function allocates (and potentially re-allocates) swapchains suitable
+ * for the new output configuration. On success, compositors should call
+ * wlr_output_swapchain_manager_get_swapchain() to get the pending swapchain,
+ * repaint with a buffer acquired from the swapchain, call wlr_backend_commit()
+ * and then wlr_output_swapchain_manager_apply().
+ *
+ * Compositors should include all enabled outputs to maximize the chance to
+ * find a working configuration, even if an output state is unchanged by the
+ * compositor. This function might re-create swapchains for already-enabled
+ * outputs.
+ */
+bool wlr_output_swapchain_manager_prepare(struct wlr_output_swapchain_manager *manager,
+	const struct wlr_backend_output_state *states, size_t states_len);
+
+/**
+ * Get the pending swapchain for an output.
+ *
+ * This can only be called after a successful
+ * wlr_output_swapchain_manager_prepare(), if the output was passed in.
+ *
+ * If the output is disabled, NULL is returned.
+ */
+struct wlr_swapchain *wlr_output_swapchain_manager_get_swapchain(
+	struct wlr_output_swapchain_manager *manager, struct wlr_output *output);
+
+/**
+ * Apply swapchains allocated for the last successful call to
+ * wlr_output_swapchain_manager_prepare().
+ *
+ * This function swaps output swapchains with new swapchains suitable for the
+ * new output configuration. It should be called after a successful
+ * wlr_backend_commit().
+ */
+void wlr_output_swapchain_manager_apply(struct wlr_output_swapchain_manager *manager);
+
+/**
+ * Cleanup resources allocated by the manager.
+ */
+void wlr_output_swapchain_manager_finish(struct wlr_output_swapchain_manager *manager);
+
+#endif

types/meson.build

@@ -59,6 +59,7 @@ wlr_files += files(
 	'wlr_output_layout.c',
 	'wlr_output_management_v1.c',
 	'wlr_output_power_management_v1.c',
+	'wlr_output_swapchain_manager.c',
 	'wlr_pointer_constraints_v1.c',
 	'wlr_pointer_gestures_v1.c',
 	'wlr_pointer.c',

types/wlr_output_swapchain_manager.c

@@ -0,0 +1,251 @@
+#include <assert.h>
+#include <drm_fourcc.h>
+#include <stdlib.h>
+#include <string.h>
+#include <wlr/backend.h>
+#include <wlr/render/allocator.h>
+#include <wlr/render/swapchain.h>
+#include <wlr/types/wlr_output_swapchain_manager.h>
+#include <wlr/util/log.h>
+#include "render/drm_format_set.h"
+#include "types/wlr_output.h"
+
+struct wlr_output_swapchain_manager_output {
+	struct wlr_output *output;
+	// Newly allocated swapchain. Can be NULL if the old swapchain is re-used
+	// or if the output is disabled.
+	struct wlr_swapchain *new_swapchain;
+	// True if the output was included in the last successful call to
+	// wlr_output_swapchain_manager_prepare().
+	bool test_success;
+	// Pending swapchain which will replace the old one when
+	// wlr_output_swapchain_manager_apply() is called. Can be either a pointer
+	// to the newly allocated swapchain, or the old swapchain, or NULL.
+	struct wlr_swapchain *pending_swapchain;
+};
+
+void wlr_output_swapchain_manager_init(struct wlr_output_swapchain_manager *manager,
+		struct wlr_backend *backend) {
+	*manager = (struct wlr_output_swapchain_manager){
+		.backend = backend,
+	};
+}
+
+static struct wlr_output_swapchain_manager_output *manager_get_output(
+		struct wlr_output_swapchain_manager *manager, struct wlr_output *output) {
+	struct wlr_output_swapchain_manager_output *manager_output;
+	wl_array_for_each(manager_output, &manager->outputs) {
+		if (manager_output->output == output) {
+			return manager_output;
+		}
+	}
+	return NULL;
+}
+
+static struct wlr_output_swapchain_manager_output *manager_get_or_add_output(
+		struct wlr_output_swapchain_manager *manager, struct wlr_output *output) {
+	struct wlr_output_swapchain_manager_output *manager_output = manager_get_output(manager, output);
+	if (manager_output != NULL) {
+		return manager_output;
+	}
+
+	manager_output = wl_array_add(&manager->outputs, sizeof(*manager_output));
+	if (manager_output == NULL) {
+		return NULL;
+	}
+	*manager_output = (struct wlr_output_swapchain_manager_output){
+		.output = output,
+	};
+	return manager_output;
+}
+
+static bool swapchain_is_compatible(struct wlr_swapchain *swapchain,
+		int width, int height, const struct wlr_drm_format *format) {
+	if (swapchain == NULL) {
+		return false;
+	}
+	if (swapchain->width != width || swapchain->height != height) {
+		return false;
+	}
+	if (swapchain->format.format != format->format || swapchain->format.len != format->len) {
+		return false;
+	}
+	assert(format->len > 0);
+	return memcmp(swapchain->format.modifiers, format->modifiers, format->len * sizeof(format->modifiers[0])) == 0;
+}
+
+static struct wlr_swapchain *manager_output_get_swapchain(
+		struct wlr_output_swapchain_manager_output *manager_output,
+		int width, int height, const struct wlr_drm_format *format) {
+	struct wlr_output *output = manager_output->output;
+
+	if (swapchain_is_compatible(output->swapchain, width, height, format)) {
+		return output->swapchain;
+	}
+	if (swapchain_is_compatible(manager_output->new_swapchain, width, height, format)) {
+		return manager_output->new_swapchain;
+	}
+
+	struct wlr_swapchain *swapchain = wlr_swapchain_create(output->allocator, width, height, format);
+	if (swapchain == NULL) {
+		return NULL;
+	}
+	wlr_swapchain_destroy(manager_output->new_swapchain);
+	manager_output->new_swapchain = swapchain;
+	return swapchain;
+}
+
+static bool manager_output_prepare(struct wlr_output_swapchain_manager_output *manager_output,
+		struct wlr_output_state *state, bool explicit_modifiers) {
+	struct wlr_output *output = manager_output->output;
+	struct wlr_allocator *allocator = output->allocator;
+	assert(allocator != NULL);
+
+	if (!output_pending_enabled(output, state)) {
+		manager_output->pending_swapchain = NULL;
+		return true;
+	}
+
+	int width, height;
+	output_pending_resolution(output, state, &width, &height);
+
+	uint32_t fmt = output->render_format;
+	if (state->committed & WLR_OUTPUT_STATE_RENDER_FORMAT) {
+		fmt = state->render_format;
+	}
+
+	const struct wlr_drm_format_set *display_formats =
+		wlr_output_get_primary_formats(output, allocator->buffer_caps);
+	struct wlr_drm_format format = {0};
+	if (!output_pick_format(output, display_formats, &format, fmt)) {
+		return false;
+	}
+
+	if (!explicit_modifiers && (format.len != 1 || format.modifiers[0] != DRM_FORMAT_MOD_LINEAR)) {
+		if (!wlr_drm_format_has(&format, DRM_FORMAT_MOD_INVALID)) {
+			wlr_log(WLR_DEBUG, "Implicit modifiers not supported");
+			wlr_drm_format_finish(&format);
+			return false;
+		}
+
+		format.len = 0;
+		if (!wlr_drm_format_add(&format, DRM_FORMAT_MOD_INVALID)) {
+			wlr_log(WLR_DEBUG, "Failed to add implicit modifier to format");
+			wlr_drm_format_finish(&format);
+			return false;
+		}
+	}
+
+	struct wlr_swapchain *swapchain =
+		manager_output_get_swapchain(manager_output, width, height, &format);
+	wlr_drm_format_finish(&format);
+	if (swapchain == NULL) {
+		return false;
+	}
+
+	struct wlr_buffer *buffer = wlr_swapchain_acquire(swapchain, NULL);
+	if (buffer == NULL) {
+		return false;
+	}
+
+	wlr_output_state_set_buffer(state, buffer);
+	wlr_buffer_unlock(buffer);
+	manager_output->pending_swapchain = swapchain;
+	return true;
+}
+
+static bool manager_test(struct wlr_output_swapchain_manager *manager,
+		struct wlr_backend_output_state *states, size_t states_len,
+		bool explicit_modifiers) {
+	wlr_log(WLR_DEBUG, "Preparing test commit for %zu outputs with %s modifiers",
+		states_len, explicit_modifiers ? "explicit": "implicit");
+
+	struct wlr_output_swapchain_manager_output *manager_output;
+	wl_array_for_each(manager_output, &manager->outputs) {
+		manager_output->test_success = false;
+	}
+
+	for (size_t i = 0; i < states_len; i++) {
+		struct wlr_backend_output_state *state = &states[i];
+		struct wlr_output_swapchain_manager_output *manager_output =
+			manager_get_or_add_output(manager, state->output);
+		if (manager_output == NULL) {
+			return false;
+		}
+		if (!manager_output_prepare(manager_output, &state->base, explicit_modifiers)) {
+			return false;
+		}
+	}
+
+	bool ok = wlr_backend_test(manager->backend, states, states_len);
+	wlr_log(WLR_DEBUG, "Test commit for %zu outputs %s",
+		states_len, ok ? "succeeded" : "failed");
+	if (!ok) {
+		return false;
+	}
+
+	for (size_t i = 0; i < states_len; i++) {
+		struct wlr_output_swapchain_manager_output *manager_output =
+			manager_get_output(manager, states[i].output);
+		assert(manager_output != NULL);
+		manager_output->test_success = true;
+	}
+
+	return true;
+}
+
+bool wlr_output_swapchain_manager_prepare(struct wlr_output_swapchain_manager *manager,
+		const struct wlr_backend_output_state *states, size_t states_len) {
+	bool ok = false;
+	struct wlr_backend_output_state *pending = malloc(states_len * sizeof(states[0]));
+	if (pending == NULL) {
+		return false;
+	}
+	for (size_t i = 0; i < states_len; i++) {
+		pending[i] = states[i];
+		pending[i].base.buffer = NULL;
+	}
+
+	ok = manager_test(manager, pending, states_len, true);
+	if (!ok) {
+		ok = manager_test(manager, pending, states_len, false);
+	}
+
+	for (size_t i = 0; i < states_len; i++) {
+		wlr_buffer_unlock(pending[i].base.buffer);
+	}
+	free(pending);
+
+	return ok;
+}
+
+struct wlr_swapchain *wlr_output_swapchain_manager_get_swapchain(
+		struct wlr_output_swapchain_manager *manager, struct wlr_output *output) {
+	struct wlr_output_swapchain_manager_output *manager_output =
+		manager_get_output(manager, output);
+	assert(manager_output != NULL && manager_output->test_success);
+	return manager_output->pending_swapchain;
+}
+
+void wlr_output_swapchain_manager_apply(struct wlr_output_swapchain_manager *manager) {
+	struct wlr_output_swapchain_manager_output *manager_output;
+	wl_array_for_each(manager_output, &manager->outputs) {
+		struct wlr_output *output = manager_output->output;
+		if (!manager_output->test_success || manager_output->pending_swapchain == output->swapchain) {
+			continue;
+		}
+
+		wlr_swapchain_destroy(output->swapchain);
+		output->swapchain = manager_output->new_swapchain;
+		manager_output->new_swapchain = NULL;
+		manager_output->test_success = false;
+	}
+}
+
+void wlr_output_swapchain_manager_finish(struct wlr_output_swapchain_manager *manager) {
+	struct wlr_output_swapchain_manager_output *manager_output;
+	wl_array_for_each(manager_output, &manager->outputs) {
+		wlr_swapchain_destroy(manager_output->new_swapchain);
+	}
+	wl_array_release(&manager->outputs);
+}