screencast: use drm format instead of wl_shm

drm format defined by drm_fourcc.h is the standard to describe the
format of a buffer. This will be used when dealing with dmabufs and to
simplify things we should drm_formats for all internal structs.

include/screencast_common.h

@@ -63,7 +63,7 @@ struct xdpw_screencopy_frame_info {
 	uint32_t height;
 	uint32_t size;
 	uint32_t stride;
-	enum wl_shm_format format;
+	uint32_t format;
 };
 
 struct xdpw_buffer {
@@ -71,7 +71,7 @@ struct xdpw_buffer {
 
 	uint32_t width;
 	uint32_t height;
-	enum wl_shm_format format;
+	uint32_t format;
 
 	int fd;
 	uint32_t size;
@@ -154,7 +154,9 @@ void randname(char *buf);
 struct xdpw_buffer *xdpw_buffer_create(struct xdpw_screencast_instance *cast,
 	struct xdpw_screencopy_frame_info *frame_info);
 void xdpw_buffer_destroy(struct xdpw_buffer *buffer);
-enum spa_video_format xdpw_format_pw_from_wl_shm(enum wl_shm_format format);
+enum wl_shm_format xdpw_format_wl_shm_from_drm_fourcc(uint32_t format);
+uint32_t xdpw_format_drm_fourcc_from_wl_shm(enum wl_shm_format format);
+enum spa_video_format xdpw_format_pw_from_drm_fourcc(uint32_t format);
 enum spa_video_format xdpw_format_pw_strip_alpha(enum spa_video_format format);
 
 enum xdpw_chooser_types get_chooser_type(const char *chooser_type);

src/screencast/pipewire_screencast.c

@@ -268,7 +268,7 @@ void pwr_update_stream_param(struct xdpw_screencast_instance *cast) {
 		SPA_POD_BUILDER_INIT(params_buffer, sizeof(params_buffer));
 	const struct spa_pod *params[1];
 
-	enum spa_video_format format = xdpw_format_pw_from_wl_shm(cast->screencopy_frame_info.format);
+	enum spa_video_format format = xdpw_format_pw_from_drm_fourcc(cast->screencopy_frame_info.format);
 
 	params[0] = build_format(&b, format,
 			cast->screencopy_frame_info.width, cast->screencopy_frame_info.height, cast->framerate);
@@ -298,7 +298,7 @@ void xdpw_pwr_stream_create(struct xdpw_screencast_instance *cast) {
 	}
 	cast->pwr_stream_state = false;
 
-	enum spa_video_format format = xdpw_format_pw_from_wl_shm(cast->screencopy_frame_info.format);
+	enum spa_video_format format = xdpw_format_pw_from_drm_fourcc(cast->screencopy_frame_info.format);
 
 	const struct spa_pod *param = build_format(&b, format,
 			cast->screencopy_frame_info.width, cast->screencopy_frame_info.height, cast->framerate);

src/screencast/screencast_common.c

@@ -5,6 +5,7 @@
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <libdrm/drm_fourcc.h>
 
 #include "logger.h"
 
@@ -78,7 +79,7 @@ struct xdpw_buffer *xdpw_buffer_create(struct xdpw_screencast_instance *cast,
 		return NULL;
 	}
 
-	buffer->buffer = import_wl_shm_buffer(cast, buffer->fd, frame_info->format,
+	buffer->buffer = import_wl_shm_buffer(cast, buffer->fd, xdpw_format_wl_shm_from_drm_fourcc(frame_info->format),
 			frame_info->width, frame_info->height, frame_info->stride);
 	if (buffer->buffer == NULL) {
 		logprint(ERROR, "xdpw: unable to create wl_buffer");
@@ -97,44 +98,102 @@ void xdpw_buffer_destroy(struct xdpw_buffer *buffer) {
 	free(buffer);
 }
 
-enum spa_video_format xdpw_format_pw_from_wl_shm(enum wl_shm_format format) {
+enum wl_shm_format xdpw_format_wl_shm_from_drm_fourcc(uint32_t format) {
+	switch (format) {
+	case DRM_FORMAT_ARGB8888:
+		return WL_SHM_FORMAT_ARGB8888;
+	case DRM_FORMAT_XRGB8888:
+		return WL_SHM_FORMAT_XRGB8888;
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_BGRA8888:
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_RGBX1010102:
+	case DRM_FORMAT_BGRX1010102:
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_ABGR2101010:
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_BGRA1010102:
+		return (enum wl_shm_format)format;
+	default:
+		logprint(ERROR, "xdg-desktop-portal-wlr: unsupported drm "
+			"format 0x%08x", format);
+		abort();
+	}
+}
+
+uint32_t xdpw_format_drm_fourcc_from_wl_shm(enum wl_shm_format format) {
 	switch (format) {
 	case WL_SHM_FORMAT_ARGB8888:
-		return SPA_VIDEO_FORMAT_BGRA;
+		return DRM_FORMAT_ARGB8888;
 	case WL_SHM_FORMAT_XRGB8888:
-		return SPA_VIDEO_FORMAT_BGRx;
+		return DRM_FORMAT_XRGB8888;
 	case WL_SHM_FORMAT_RGBA8888:
-		return SPA_VIDEO_FORMAT_ABGR;
 	case WL_SHM_FORMAT_RGBX8888:
-		return SPA_VIDEO_FORMAT_xBGR;
 	case WL_SHM_FORMAT_ABGR8888:
-		return SPA_VIDEO_FORMAT_RGBA;
 	case WL_SHM_FORMAT_XBGR8888:
-		return SPA_VIDEO_FORMAT_RGBx;
 	case WL_SHM_FORMAT_BGRA8888:
-		return SPA_VIDEO_FORMAT_ARGB;
 	case WL_SHM_FORMAT_BGRX8888:
-		return SPA_VIDEO_FORMAT_xRGB;
 	case WL_SHM_FORMAT_NV12:
-		return SPA_VIDEO_FORMAT_NV12;
 	case WL_SHM_FORMAT_XRGB2101010:
-		return SPA_VIDEO_FORMAT_xRGB_210LE;
 	case WL_SHM_FORMAT_XBGR2101010:
-		return SPA_VIDEO_FORMAT_xBGR_210LE;
 	case WL_SHM_FORMAT_RGBX1010102:
-		return SPA_VIDEO_FORMAT_RGBx_102LE;
 	case WL_SHM_FORMAT_BGRX1010102:
-		return SPA_VIDEO_FORMAT_BGRx_102LE;
 	case WL_SHM_FORMAT_ARGB2101010:
-		return SPA_VIDEO_FORMAT_ARGB_210LE;
 	case WL_SHM_FORMAT_ABGR2101010:
-		return SPA_VIDEO_FORMAT_ABGR_210LE;
 	case WL_SHM_FORMAT_RGBA1010102:
-		return SPA_VIDEO_FORMAT_RGBA_102LE;
 	case WL_SHM_FORMAT_BGRA1010102:
+		return (uint32_t)format;
+	default:
+		logprint(ERROR, "xdg-desktop-portal-wlr: unsupported wl_shm "
+			"format 0x%08x", format);
+		abort();
+	}
+}
+
+enum spa_video_format xdpw_format_pw_from_drm_fourcc(uint32_t format) {
+	switch (format) {
+	case DRM_FORMAT_ARGB8888:
+		return SPA_VIDEO_FORMAT_BGRA;
+	case DRM_FORMAT_XRGB8888:
+		return SPA_VIDEO_FORMAT_BGRx;
+	case DRM_FORMAT_RGBA8888:
+		return SPA_VIDEO_FORMAT_ABGR;
+	case DRM_FORMAT_RGBX8888:
+		return SPA_VIDEO_FORMAT_xBGR;
+	case DRM_FORMAT_ABGR8888:
+		return SPA_VIDEO_FORMAT_RGBA;
+	case DRM_FORMAT_XBGR8888:
+		return SPA_VIDEO_FORMAT_RGBx;
+	case DRM_FORMAT_BGRA8888:
+		return SPA_VIDEO_FORMAT_ARGB;
+	case DRM_FORMAT_BGRX8888:
+		return SPA_VIDEO_FORMAT_xRGB;
+	case DRM_FORMAT_NV12:
+		return SPA_VIDEO_FORMAT_NV12;
+	case DRM_FORMAT_XRGB2101010:
+		return SPA_VIDEO_FORMAT_xRGB_210LE;
+	case DRM_FORMAT_XBGR2101010:
+		return SPA_VIDEO_FORMAT_xBGR_210LE;
+	case DRM_FORMAT_RGBX1010102:
+		return SPA_VIDEO_FORMAT_RGBx_102LE;
+	case DRM_FORMAT_BGRX1010102:
+		return SPA_VIDEO_FORMAT_BGRx_102LE;
+	case DRM_FORMAT_ARGB2101010:
+		return SPA_VIDEO_FORMAT_ARGB_210LE;
+	case DRM_FORMAT_ABGR2101010:
+		return SPA_VIDEO_FORMAT_ABGR_210LE;
+	case DRM_FORMAT_RGBA1010102:
+		return SPA_VIDEO_FORMAT_RGBA_102LE;
+	case DRM_FORMAT_BGRA1010102:
 		return SPA_VIDEO_FORMAT_BGRA_102LE;
 	default:
-		logprint(ERROR, "xdg-desktop-portal-wlr: failed to convert wl_shm "
+		logprint(ERROR, "xdg-desktop-portal-wlr: failed to convert drm "
 			"format 0x%08x to spa_video_format", format);
 		abort();
 	}

src/screencast/wlr_screencast.c

@@ -91,7 +91,7 @@ static void wlr_frame_buffer(void *data, struct zwlr_screencopy_frame_v1 *frame,
 	cast->screencopy_frame_info.height = height;
 	cast->screencopy_frame_info.stride = stride;
 	cast->screencopy_frame_info.size = stride * height;
-	cast->screencopy_frame_info.format = format;
+	cast->screencopy_frame_info.format = xdpw_format_drm_fourcc_from_wl_shm(format);
 
 	if (zwlr_screencopy_manager_v1_get_version(cast->ctx->screencopy_manager) < 3) {
 		wlr_frame_buffer_done(cast, frame);
@@ -116,8 +116,8 @@ static void wlr_frame_buffer_done(void *data,
 	}
 
 	// Check if announced screencopy information is compatible with pipewire meta
-	if ((cast->pwr_format.format != xdpw_format_pw_from_wl_shm(cast->screencopy_frame_info.format) &&
+	if ((cast->pwr_format.format != xdpw_format_pw_from_drm_fourcc(cast->screencopy_frame_info.format) &&
-			cast->pwr_format.format != xdpw_format_pw_strip_alpha(xdpw_format_pw_from_wl_shm(cast->screencopy_frame_info.format))) ||
+			cast->pwr_format.format != xdpw_format_pw_strip_alpha(xdpw_format_pw_from_drm_fourcc(cast->screencopy_frame_info.format))) ||
 			cast->pwr_format.size.width != cast->screencopy_frame_info.width ||
 			cast->pwr_format.size.height != cast->screencopy_frame_info.height) {
 		logprint(DEBUG, "wlroots: pipewire and wlroots metadata are incompatible. Renegotiate stream");