output: fix back buffer checks

The back buffer is no longer set at commit time since 0556aa0c59
("output: rejigger attach/clear for back buffer").

Instead, check whether the buffer belongs to the output swapchain.
This is more robust.

Closes: https://gitlab.freedesktop.org/wlroots/wlroots/-/issues/3496

include/types/wlr_output.h

@@ -8,6 +8,8 @@ void output_pending_resolution(struct wlr_output *output,
 	const struct wlr_output_state *state, int *width, int *height);
 void output_state_attach_buffer(struct wlr_output_state *state,
 	struct wlr_buffer *buffer);
+bool output_is_direct_scanout(struct wlr_output *output,
+	struct wlr_buffer *buffer);
 
 struct wlr_drm_format *output_pick_format(struct wlr_output *output,
 	const struct wlr_drm_format_set *display_formats, uint32_t format);

types/output/output.c

@@ -575,7 +575,7 @@ static bool output_basic_test(struct wlr_output *output,
 			return false;
 		}
 
-		if (output->back_buffer == NULL) {
+		if (output_is_direct_scanout(output, state->buffer)) {
 			if (output->attach_render_locks > 0) {
 				wlr_log(WLR_DEBUG, "Direct scan-out disabled by lock");
 				return false;

types/output/render.c

@@ -329,3 +329,18 @@ uint32_t wlr_output_preferred_read_format(struct wlr_output *output) {
 
 	return fmt;
 }
+
+bool output_is_direct_scanout(struct wlr_output *output,
+		struct wlr_buffer *buffer) {
+	if (output->swapchain == NULL) {
+		return true;
+	}
+
+	for (size_t i = 0; i < WLR_SWAPCHAIN_CAP; i++) {
+		if (output->swapchain->slots[i].buffer == buffer) {
+			return false;
+		}
+	}
+
+	return true;
+}

types/wlr_output_damage.c

@@ -5,6 +5,7 @@
 #include <wlr/types/wlr_output_damage.h>
 #include <wlr/types/wlr_output.h>
 #include <wlr/util/box.h>
+#include "types/wlr_output.h"
 
 static void output_handle_destroy(struct wl_listener *listener, void *data) {
 	struct wlr_output_damage *output_damage =
@@ -53,7 +54,8 @@ static void output_handle_precommit(struct wl_listener *listener, void *data) {
 	if (state->committed & WLR_OUTPUT_STATE_BUFFER) {
 		// TODO: find a better way to access this info without a precommit
 		// handler
-		output_damage->pending_attach_render = output->back_buffer != NULL;
+		output_damage->pending_attach_render =
+			output_is_direct_scanout(output, state->buffer);
 	}
 }