From c4635c68d25b92564a4568f435526108448fa0ca Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Fri, 4 Dec 2020 17:09:00 +0100 Subject: [PATCH] render/drm_format_set: add special case for LINEAR-only formats Our wlr_format_set structs don't hold GBM usage flags. Instead, users who want to get a LINEAR buffer can use the DRM_FORMAT_MOD_LINEAR modifier even if the kernel driver doesn't support modifiers. Add a special case to wlr_drm_format_intersect to properly handle this situation. --- render/drm_format_set.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/render/drm_format_set.c b/render/drm_format_set.c index 634d9070..bd8f0cb7 100644 --- a/render/drm_format_set.c +++ b/render/drm_format_set.c @@ -152,6 +152,16 @@ struct wlr_drm_format *wlr_drm_format_intersect( const struct wlr_drm_format *a, const struct wlr_drm_format *b) { assert(a->format == b->format); + // Special case: if a format only supports LINEAR and the other doesn't + // support any modifier, force LINEAR. This will force the allocator to + // create a buffer with a LINEAR layout instead of an implicit modifier. + if (a->len == 0 && b->len == 1 && b->modifiers[0] == DRM_FORMAT_MOD_LINEAR) { + return wlr_drm_format_dup(b); + } + if (b->len == 0 && a->len == 1 && a->modifiers[0] == DRM_FORMAT_MOD_LINEAR) { + return wlr_drm_format_dup(a); + } + size_t format_cap = a->len < b->len ? a->len : b->len; size_t format_size = sizeof(struct wlr_drm_format) + format_cap * sizeof(a->modifiers[0]);