render/drm-format-set: add wlr_drm_format_set_intersect

This intersects two DRM format sets. This is useful for implementing
DMA-BUF feedback in compositors, see e.g. the Sway PR [1].

[1]: https://github.com/swaywm/sway/pull/6313

include/wlr/render/drm_format_set.h

@@ -63,4 +63,13 @@ bool wlr_drm_format_set_has(const struct wlr_drm_format_set *set,
 bool wlr_drm_format_set_add(struct wlr_drm_format_set *set, uint32_t format,
 	uint64_t modifier);
 
+/**
+ * Intersect two DRM format sets `a` and `b`, storing in the destination set
+ * `dst` the format + modifier pairs which are in both source sets.
+ *
+ * Returns false on failure or when the intersection is empty.
+ */
+bool wlr_drm_format_set_intersect(struct wlr_drm_format_set *dst,
+	const struct wlr_drm_format_set *a, const struct wlr_drm_format_set *b);
+
 #endif

render/drm_format_set.c

@@ -175,3 +175,41 @@ struct wlr_drm_format *wlr_drm_format_intersect(
 
 	return format;
 }
+
+bool wlr_drm_format_set_intersect(struct wlr_drm_format_set *dst,
+		const struct wlr_drm_format_set *a, const struct wlr_drm_format_set *b) {
+	assert(dst != a && dst != b);
+
+	struct wlr_drm_format_set out = {0};
+	out.capacity = a->len < b->len ? a->len : b->len;
+	out.formats = calloc(out.capacity, sizeof(struct wlr_drm_format *));
+	if (out.formats == NULL) {
+		wlr_log_errno(WLR_ERROR, "Allocation failed");
+		return false;
+	}
+
+	for (size_t i = 0; i < a->len; i++) {
+		for (size_t j = 0; j < b->len; j++) {
+			if (a->formats[i]->format == b->formats[j]->format) {
+				// When the two formats have no common modifier, keep
+				// intersecting the rest of the formats: they may be compatible
+				// with each other
+				struct wlr_drm_format *format =
+					wlr_drm_format_intersect(a->formats[i], b->formats[j]);
+				if (format != NULL) {
+					out.formats[out.len] = format;
+					out.len++;
+				}
+				break;
+			}
+		}
+	}
+
+	if (out.len == 0) {
+		wlr_drm_format_set_finish(&out);
+		return false;
+	}
+
+	*dst = out;
+	return true;
+}