render/vulkan: make shm/dmabuf split clearer in wlr_vk_format_props

struct wlr_vk_format_props contains a mix of properties for shm
and dmabuf, and it's not immediately clear which fields are for
which kind of buffer. Use a nested struct to group the fields.

include/render/vulkan.h

@@ -96,14 +96,19 @@ struct wlr_vk_format_modifier_props {
 
 struct wlr_vk_format_props {
 	struct wlr_vk_format format;
-	VkExtent2D max_extent; // relevant if not created as dma_buf
-	VkFormatFeatureFlags features; // relevant if not created as dma_buf
 
-	uint32_t render_mod_count;
+	struct {
-	struct wlr_vk_format_modifier_props *render_mods;
+		VkExtent2D max_extent;
+		VkFormatFeatureFlags features;
+	} shm;
 
-	uint32_t texture_mod_count;
+	struct {
-	struct wlr_vk_format_modifier_props *texture_mods;
+		uint32_t render_mod_count;
+		struct wlr_vk_format_modifier_props *render_mods;
+
+		uint32_t texture_mod_count;
+		struct wlr_vk_format_modifier_props *texture_mods;
+	} dmabuf;
 };
 
 void vulkan_format_props_query(struct wlr_vk_device *dev,

render/vulkan/pixel_format.c

@@ -264,15 +264,15 @@ static bool query_modifier_support(struct wlr_vk_device *dev,
 
 	vkGetPhysicalDeviceFormatProperties2(dev->phdev, props->format.vk, &fmtp);
 
-	props->render_mods =
+	props->dmabuf.render_mods =
-		calloc(modp.drmFormatModifierCount, sizeof(*props->render_mods));
+		calloc(modp.drmFormatModifierCount, sizeof(*props->dmabuf.render_mods));
-	props->texture_mods =
+	props->dmabuf.texture_mods =
-		calloc(modp.drmFormatModifierCount, sizeof(*props->texture_mods));
+		calloc(modp.drmFormatModifierCount, sizeof(*props->dmabuf.texture_mods));
-	if (!props->render_mods || !props->texture_mods) {
+	if (!props->dmabuf.render_mods || !props->dmabuf.texture_mods) {
 		wlr_log_errno(WLR_ERROR, "Allocation failed");
 		free(modp.pDrmFormatModifierProperties);
-		free(props->render_mods);
+		free(props->dmabuf.render_mods);
-		free(props->texture_mods);
+		free(props->dmabuf.texture_mods);
 		return false;
 	}
 
@@ -288,7 +288,7 @@ static bool query_modifier_support(struct wlr_vk_device *dev,
 				props->format.is_srgb) {
 			struct wlr_vk_format_modifier_props p = {0};
 			if (query_modifier_usage_support(dev, props->format.vk, render_usage, &m, &p, &errmsg)) {
-				props->render_mods[props->render_mod_count++] = p;
+				props->dmabuf.render_mods[props->dmabuf.render_mod_count++] = p;
 				wlr_drm_format_set_add(&dev->dmabuf_render_formats,
 					props->format.drm, m.drmFormatModifier);
 				found = true;
@@ -307,7 +307,7 @@ static bool query_modifier_support(struct wlr_vk_device *dev,
 		if ((m.drmFormatModifierTilingFeatures & dma_tex_features) == dma_tex_features) {
 			struct wlr_vk_format_modifier_props p = {0};
 			if (query_modifier_usage_support(dev, props->format.vk, dma_tex_usage, &m, &p, &errmsg)) {
-				props->texture_mods[props->texture_mod_count++] = p;
+				props->dmabuf.texture_mods[props->dmabuf.texture_mod_count++] = p;
 				wlr_drm_format_set_add(&dev->dmabuf_texture_formats,
 					props->format.drm, m.drmFormatModifier);
 				found = true;
@@ -380,9 +380,9 @@ void vulkan_format_props_query(struct wlr_vk_device *dev,
 			}
 		} else {
 			VkExtent3D me = ifmtp.imageFormatProperties.maxExtent;
-			props.max_extent.width = me.width;
+			props.shm.max_extent.width = me.width;
-			props.max_extent.height = me.height;
+			props.shm.max_extent.height = me.height;
-			props.features = fmtp.formatProperties.optimalTilingFeatures;
+			props.shm.features = fmtp.formatProperties.optimalTilingFeatures;
 
 			shm_texture_status = "✓ texture";
 
@@ -410,25 +410,26 @@ void vulkan_format_props_query(struct wlr_vk_device *dev,
 }
 
 void vulkan_format_props_finish(struct wlr_vk_format_props *props) {
-	free(props->texture_mods);
+	free(props->dmabuf.texture_mods);
-	free(props->render_mods);
+	free(props->dmabuf.render_mods);
 }
 
 const struct wlr_vk_format_modifier_props *vulkan_format_props_find_modifier(
 		struct wlr_vk_format_props *props, uint64_t mod, bool render) {
+	uint32_t len;
+	const struct wlr_vk_format_modifier_props *mods;
 	if (render) {
-		for (unsigned i = 0u; i < props->render_mod_count; ++i) {
+		len = props->dmabuf.render_mod_count;
-			if (props->render_mods[i].props.drmFormatModifier == mod) {
+		mods = props->dmabuf.render_mods;
-				return &props->render_mods[i];
-			}
-		}
 	} else {
-		for (unsigned i = 0u; i < props->texture_mod_count; ++i) {
+		len = props->dmabuf.texture_mod_count;
-			if (props->texture_mods[i].props.drmFormatModifier == mod) {
+		mods = props->dmabuf.texture_mods;
-				return &props->texture_mods[i];
-			}
-		}
 	}
 
+	for (uint32_t i = 0; i < len; ++i) {
+		if (mods[i].props.drmFormatModifier == mod) {
+			return &mods[i];
+		}
+	}
 	return NULL;
 }