screencast: fixate modifier from PipeWire format negotiation result

This implements the modifier fixation procedure. The producer of a
stream has to ensure that it can create a buffer with the negotiated
properties. To do that we will take the result of the intersection of
supported modifiers by PipeWire and select the "best" modifier from that
list.

To do this we do the following allocations and fixate on the
modifier of the created buffer should the allocation suceed.
* Try to allocate a buffer with explicit modifiers using the list
  provided by PipeWire
* Walk the list of modifiers and do single allocations for implicit and
  linear modifiers using the old api.

If none of the allocations above succeed we fall back to shm buffers.
This is implemented in the next commit.

If an allocation was successfull we fixate the modifier with
fixate_format().
This function creates an EnumFormat like build_format, but will only
announce support for a single modifier.
It is used to finish the negotiation process by announcing it together
with EnumFormats of our full supported formats.

src/screencast/pipewire_screencast.c

@@ -35,6 +35,44 @@ static struct spa_pod *build_buffer(struct spa_pod_builder *b, uint32_t blocks,
 	return spa_pod_builder_pop(b, &f[0]);
 }
 
+static struct spa_pod *fixate_format(struct spa_pod_builder *b, enum spa_video_format format,
+		uint32_t width, uint32_t height, uint32_t framerate, uint64_t *modifier)
+{
+	struct spa_pod_frame f[1];
+
+	enum spa_video_format format_without_alpha = xdpw_format_pw_strip_alpha(format);
+
+	spa_pod_builder_push_object(b, &f[0], SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat);
+	spa_pod_builder_add(b, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), 0);
+	spa_pod_builder_add(b, SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), 0);
+	/* format */
+	if (modifier || format_without_alpha == SPA_VIDEO_FORMAT_UNKNOWN) {
+		spa_pod_builder_add(b, SPA_FORMAT_VIDEO_format, SPA_POD_Id(format), 0);
+	} else {
+		spa_pod_builder_add(b, SPA_FORMAT_VIDEO_format,
+				SPA_POD_CHOICE_ENUM_Id(3, format, format, format_without_alpha), 0);
+	}
+	/* modifiers */
+	if (modifier) {
+		// implicit modifier
+		spa_pod_builder_prop(b, SPA_FORMAT_VIDEO_modifier, SPA_POD_PROP_FLAG_MANDATORY);
+		spa_pod_builder_long(b, *modifier);
+	}
+	spa_pod_builder_add(b, SPA_FORMAT_VIDEO_size,
+		SPA_POD_Rectangle(&SPA_RECTANGLE(width, height)),
+		0);
+	// variable framerate
+	spa_pod_builder_add(b, SPA_FORMAT_VIDEO_framerate,
+		SPA_POD_Fraction(&SPA_FRACTION(0, 1)), 0);
+	spa_pod_builder_add(b, SPA_FORMAT_VIDEO_maxFramerate,
+		SPA_POD_CHOICE_RANGE_Fraction(
+			&SPA_FRACTION(framerate, 1),
+			&SPA_FRACTION(1, 1),
+			&SPA_FRACTION(framerate, 1)),
+		0);
+	return spa_pod_builder_pop(b, &f[0]);
+}
+
 static struct spa_pod *build_format(struct spa_pod_builder *b, enum spa_video_format format,
 		uint32_t width, uint32_t height, uint32_t framerate,
 		uint64_t *modifiers, int modifier_count) {
@@ -146,7 +184,7 @@ static void pwr_handle_stream_param_changed(void *data, uint32_t id,
 	uint8_t params_buffer[1024];
 	struct spa_pod_builder b =
 		SPA_POD_BUILDER_INIT(params_buffer, sizeof(params_buffer));
-	const struct spa_pod *params[2];
+	const struct spa_pod *params[3];
 	uint32_t blocks;
 	uint32_t data_type;
 
@@ -163,8 +201,56 @@ static void pwr_handle_stream_param_changed(void *data, uint32_t id,
 		data_type = 1<<SPA_DATA_DmaBuf;
 		assert(cast->pwr_format.format == xdpw_format_pw_from_drm_fourcc(cast->screencopy_frame_info[DMABUF].format));
 		if ((prop_modifier->flags & SPA_POD_PROP_FLAG_DONT_FIXATE) > 0) {
-			// TODO: fixate
+			const struct spa_pod *pod_modifier = &prop_modifier->value;
+
+			uint32_t n_modifiers = SPA_POD_CHOICE_N_VALUES(pod_modifier) - 1;
+			uint64_t *modifiers = SPA_POD_CHOICE_VALUES(pod_modifier);
+			modifiers++;
+			uint32_t flags = GBM_BO_USE_RENDERING;
+			uint64_t modifier;
+
+			struct gbm_bo *bo = gbm_bo_create_with_modifiers2(cast->ctx->gbm,
+				cast->screencopy_frame_info[cast->buffer_type].width, cast->screencopy_frame_info[cast->buffer_type].height,
+				cast->screencopy_frame_info[cast->buffer_type].format, modifiers, n_modifiers, flags);
+			if (bo) {
+				modifier = gbm_bo_get_modifier(bo);
+				gbm_bo_destroy(bo);
+				goto fixate_format;
+			}
+
+			logprint(INFO, "pipewire: unable to allocate a dmabuf with modifiers. Falling back to the old api");
+			for (uint32_t i = 0; i < n_modifiers; i++) {
+				switch (modifiers[i]) {
+				case DRM_FORMAT_MOD_INVALID:
+					modifiers = NULL;
+					n_modifiers = 0;
+					flags = GBM_BO_USE_RENDERING;
+					break;
+				default:
+					continue;
+				}
+				bo = gbm_bo_create_with_modifiers2(cast->ctx->gbm,
+					cast->screencopy_frame_info[cast->buffer_type].width, cast->screencopy_frame_info[cast->buffer_type].height,
+					cast->screencopy_frame_info[cast->buffer_type].format, modifiers, n_modifiers, flags);
+				if (bo) {
+					modifier = gbm_bo_get_modifier(bo);
+					gbm_bo_destroy(bo);
+					goto fixate_format;
+				}
+			}
+
+			logprint(ERROR, "pipewire: unable to allocate a dmabuf");
 			abort();
+
+fixate_format:
+			params[0] = fixate_format(&b, xdpw_format_pw_from_drm_fourcc(cast->screencopy_frame_info[cast->buffer_type].format),
+					cast->screencopy_frame_info[cast->buffer_type].width, cast->screencopy_frame_info[cast->buffer_type].height, cast->framerate, &modifier);
+
+			uint32_t n_params = build_formats(&b, cast, &params[1]);
+			n_params++;
+
+			pw_stream_update_params(stream, params, n_params);
+			return;
 		}
 
 		if (cast->pwr_format.modifier == DRM_FORMAT_MOD_INVALID) {