compositor: introduce wlr_surface_reject_pending()

include/wlr/types/wlr_compositor.h

@@ -231,6 +231,9 @@ struct wlr_surface {
 
 	bool opaque;
 
+	bool handling_commit;
+	bool pending_rejected;
+
 	int32_t preferred_buffer_scale;
 	bool preferred_buffer_transform_sent;
 	enum wl_output_transform preferred_buffer_transform;
@@ -290,6 +293,15 @@ void wlr_surface_map(struct wlr_surface *surface);
  */
 void wlr_surface_unmap(struct wlr_surface *surface);
 
+/**
+ * Mark the pending state of a surface as rejected due to a protocol violation,
+ * preventing it from being cached or committed.
+ *
+ * This function must only be used while processing a commit request.
+ */
+void wlr_surface_reject_pending(struct wlr_surface *surface, struct wl_resource *resource,
+	uint32_t code, const char *msg, ...);
+
 /**
  * Whether or not this surface currently has an attached buffer. A surface has
  * an attached buffer when it commits with a non-null buffer in its pending

types/wlr_compositor.c

@@ -1,4 +1,5 @@
 #include <assert.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <wayland-server-core.h>
 #include <wlr/render/interface.h>
@@ -200,7 +201,7 @@ static void surface_finalize_pending(struct wlr_surface *surface) {
 				"is not divisible by scale (%d)", pending->buffer_width,
 				pending->buffer_height, pending->scale);
 		} else {
-			wl_resource_post_error(surface->resource,
+			wlr_surface_reject_pending(surface, surface->resource,
 				WL_SURFACE_ERROR_INVALID_SIZE,
 				"Buffer size (%dx%d) is not divisible by scale (%d)",
 				pending->buffer_width, pending->buffer_height, pending->scale);
@@ -567,6 +568,8 @@ static void surface_commit_state(struct wlr_surface *surface,
 static void surface_handle_commit(struct wl_client *client,
 		struct wl_resource *resource) {
 	struct wlr_surface *surface = wlr_surface_from_resource(resource);
+	surface->handling_commit = true;
+
 	surface_finalize_pending(surface);
 
 	if (surface->role != NULL && surface->role->client_commit != NULL &&
@@ -576,6 +579,11 @@ static void surface_handle_commit(struct wl_client *client,
 
 	wl_signal_emit_mutable(&surface->events.client_commit, NULL);
 
+	surface->handling_commit = false;
+	if (surface->pending_rejected) {
+		return;
+	}
+
 	if (surface->pending.cached_state_locks > 0 || !wl_list_empty(&surface->cached)) {
 		surface_cache_pending(surface);
 	} else {
@@ -853,6 +861,26 @@ void wlr_surface_unmap(struct wlr_surface *surface) {
 	}
 }
 
+void wlr_surface_reject_pending(struct wlr_surface *surface, struct wl_resource *resource,
+		uint32_t code, const char *msg, ...) {
+	assert(surface->handling_commit);
+	if (surface->pending_rejected) {
+		return;
+	}
+
+	va_list args;
+	va_start(args, msg);
+
+	// XXX: libwayland could expose wl_resource_post_error_vargs() instead
+	char buffer[128]; // Matches the size of the buffer used in libwayland
+	vsnprintf(buffer, sizeof(buffer), msg, args);
+
+	wl_resource_post_error(resource, code, "%s", buffer);
+	surface->pending_rejected = true;
+
+	va_end(args);
+}
+
 bool wlr_surface_set_role(struct wlr_surface *surface, const struct wlr_surface_role *role,
 		struct wl_resource *error_resource, uint32_t error_code) {
 	assert(role != NULL);

types/wlr_layer_shell_v1.c

@@ -331,7 +331,7 @@ static void layer_surface_role_client_commit(struct wlr_surface *wlr_surface) {
 	}
 
 	if (wlr_surface_state_has_buffer(&wlr_surface->pending) && !surface->configured) {
-		wl_resource_post_error(surface->resource,
+		wlr_surface_reject_pending(wlr_surface, surface->resource,
 			ZWLR_LAYER_SHELL_V1_ERROR_ALREADY_CONSTRUCTED,
 			"layer_surface has never been configured");
 		return;
@@ -341,7 +341,7 @@ static void layer_surface_role_client_commit(struct wlr_surface *wlr_surface) {
 		ZWLR_LAYER_SURFACE_V1_ANCHOR_RIGHT;
 	if (surface->pending.desired_width == 0 &&
 			(surface->pending.anchor & horiz) != horiz) {
-		wl_resource_post_error(surface->resource,
+		wlr_surface_reject_pending(wlr_surface, surface->resource,
 			ZWLR_LAYER_SURFACE_V1_ERROR_INVALID_SIZE,
 			"width 0 requested without setting left and right anchors");
 		return;
@@ -351,7 +351,7 @@ static void layer_surface_role_client_commit(struct wlr_surface *wlr_surface) {
 		ZWLR_LAYER_SURFACE_V1_ANCHOR_BOTTOM;
 	if (surface->pending.desired_height == 0 &&
 			(surface->pending.anchor & vert) != vert) {
-		wl_resource_post_error(surface->resource,
+		wlr_surface_reject_pending(wlr_surface, surface->resource,
 			ZWLR_LAYER_SURFACE_V1_ERROR_INVALID_SIZE,
 			"height 0 requested without setting top and bottom anchors");
 		return;

types/wlr_session_lock_v1.c

@@ -157,14 +157,14 @@ static void lock_surface_role_client_commit(struct wlr_surface *surface) {
 	}
 
 	if (!wlr_surface_state_has_buffer(&surface->pending)) {
-		wl_resource_post_error(lock_surface->resource,
+		wlr_surface_reject_pending(surface, lock_surface->resource,
 			EXT_SESSION_LOCK_SURFACE_V1_ERROR_NULL_BUFFER,
 			"session lock surface is committed with a null buffer");
 		return;
 	}
 
 	if (!lock_surface->configured) {
-		wl_resource_post_error(lock_surface->resource,
+		wlr_surface_reject_pending(surface, lock_surface->resource,
 			EXT_SESSION_LOCK_SURFACE_V1_ERROR_COMMIT_BEFORE_FIRST_ACK,
 			"session lock surface has never been configured");
 		return;
@@ -172,7 +172,7 @@ static void lock_surface_role_client_commit(struct wlr_surface *surface) {
 
 	if ((uint32_t)surface->pending.width != lock_surface->pending.width ||
 			(uint32_t)surface->pending.height != lock_surface->pending.height) {
-		wl_resource_post_error(lock_surface->resource,
+		wlr_surface_reject_pending(surface, lock_surface->resource,
 			EXT_SESSION_LOCK_SURFACE_V1_ERROR_DIMENSIONS_MISMATCH,
 			"committed surface dimensions do not match last acked configure");
 		return;

types/wlr_viewporter.c

@@ -151,7 +151,8 @@ static void viewport_handle_surface_client_commit(struct wl_listener *listener,
 	if (!state->viewport.has_dst &&
 			(floor(state->viewport.src.width) != state->viewport.src.width ||
 			floor(state->viewport.src.height) != state->viewport.src.height)) {
-		wl_resource_post_error(viewport->resource, WP_VIEWPORT_ERROR_BAD_SIZE,
+		wlr_surface_reject_pending(viewport->surface,
+			viewport->resource, WP_VIEWPORT_ERROR_BAD_SIZE,
 			"wl_viewport.set_source width and height must be integers "
 			"when the destination rectangle is unset");
 		return;
@@ -159,7 +160,8 @@ static void viewport_handle_surface_client_commit(struct wl_listener *listener,
 
 	if (state->viewport.has_src && state->buffer != NULL &&
 			!check_src_buffer_bounds(state)) {
-		wl_resource_post_error(viewport->resource, WP_VIEWPORT_ERROR_OUT_OF_BUFFER,
+		wlr_surface_reject_pending(viewport->surface,
+			viewport->resource, WP_VIEWPORT_ERROR_OUT_OF_BUFFER,
 			"source rectangle out of buffer bounds");
 		return;
 	}

types/xdg_shell/wlr_xdg_popup.c

@@ -240,9 +240,8 @@ static struct wlr_xdg_popup_grab *get_xdg_shell_popup_grab_from_seat(
 
 void handle_xdg_popup_client_commit(struct wlr_xdg_popup *popup) {
 	if (!popup->parent) {
-		wl_resource_post_error(popup->base->resource,
-			XDG_SURFACE_ERROR_NOT_CONSTRUCTED,
-			"xdg_popup has no parent");
+		wlr_surface_reject_pending(popup->base->surface, popup->base->resource,
+			XDG_SURFACE_ERROR_NOT_CONSTRUCTED, "xdg_popup has no parent");
 		return;
 	}
 }

types/xdg_shell/wlr_xdg_surface.c

@@ -261,16 +261,14 @@ static void xdg_surface_role_client_commit(struct wlr_surface *wlr_surface) {
 	assert(surface != NULL);
 
 	if (wlr_surface_state_has_buffer(&wlr_surface->pending) && !surface->configured) {
-		wl_resource_post_error(surface->resource,
-			XDG_SURFACE_ERROR_UNCONFIGURED_BUFFER,
-			"xdg_surface has never been configured");
+		wlr_surface_reject_pending(wlr_surface, surface->resource,
+			XDG_SURFACE_ERROR_UNCONFIGURED_BUFFER, "xdg_surface has never been configured");
 		return;
 	}
 
 	if (surface->role_resource == NULL) {
-		wl_resource_post_error(surface->resource,
-			XDG_SURFACE_ERROR_NOT_CONSTRUCTED,
-			"xdg_surface must have a role object");
+		wlr_surface_reject_pending(wlr_surface, surface->resource,
+			XDG_SURFACE_ERROR_NOT_CONSTRUCTED, "xdg_surface must have a role object");
 		return;
 	}
 

types/xdg_shell/wlr_xdg_toplevel.c

@@ -125,9 +125,8 @@ void handle_xdg_toplevel_client_commit(struct wlr_xdg_toplevel *toplevel) {
 			pending->max_width < 0 || pending->max_height < 0 ||
 			(pending->max_width != 0 && pending->max_width < pending->min_width) ||
 			(pending->max_height != 0 && pending->max_height < pending->min_height)) {
-		wl_resource_post_error(toplevel->resource,
-			XDG_TOPLEVEL_ERROR_INVALID_SIZE,
-			"client provided an invalid min or max size");
+		wlr_surface_reject_pending(toplevel->base->surface, toplevel->resource,
+			XDG_TOPLEVEL_ERROR_INVALID_SIZE, "client provided an invalid min or max size");
 		return;
 	}
 }