From a7d77c60ee5aaa0fef7c4b5c00ff9e8c885c6985 Mon Sep 17 00:00:00 2001
From: moetayuko <loli@yuko.moe>
Date: Fri, 26 Jul 2024 04:41:46 +0800
Subject: [PATCH] gbm: manually set modifier for implicit bo allocation (#30)

Ported from wlroots

Fixes https://github.com/hyprwm/Hyprland/issues/7001
---
 src/allocator/GBM.cpp | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/allocator/GBM.cpp b/src/allocator/GBM.cpp
index 42666a6..773fc00 100644
--- a/src/allocator/GBM.cpp
+++ b/src/allocator/GBM.cpp
@@ -53,8 +53,7 @@ static SDRMFormat guessFormatFrom(std::vector<SDRMFormat> formats, bool cursor)
 }
 
 Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hyprutils::Memory::CWeakPointer<CGBMAllocator> allocator_,
-                                   Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain) :
-    allocator(allocator_) {
+                                   Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain) : allocator(allocator_) {
     if (!allocator)
         return;
 
@@ -128,8 +127,6 @@ Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hypruti
     uint32_t flags = GBM_BO_USE_RENDERING;
     if (params.scanout)
         flags |= GBM_BO_USE_SCANOUT;
-    if (CURSOR)
-        flags |= GBM_BO_USE_CURSOR; // make implicit fail for nvidia - avoids freezing with incorrect formats for cursor plane
 
     if (explicitModifiers.empty()) {
         allocator->backend->log(AQ_LOG_WARNING, "GBM: Using modifier-less allocation");
@@ -148,7 +145,11 @@ Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hypruti
         }
 
         if (!bo) {
-            allocator->backend->log(AQ_LOG_ERROR, "GBM: Allocating with modifiers failed, falling back to implicit");
+            if (explicitModifiers.size() == 1 && explicitModifiers[0] == DRM_FORMAT_MOD_LINEAR) {
+                flags |= GBM_BO_USE_LINEAR;
+                allocator->backend->log(AQ_LOG_ERROR, "GBM: Allocating with modifiers failed, falling back to modifier-less allocation");
+            } else
+                allocator->backend->log(AQ_LOG_ERROR, "GBM: Allocating with modifiers failed, falling back to implicit");
             bo = gbm_bo_create(allocator->gbmDevice, attrs.size.x, attrs.size.y, attrs.format, flags);
         }
     }
@@ -159,7 +160,7 @@ Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hypruti
     }
 
     attrs.planes   = gbm_bo_get_plane_count(bo);
-    attrs.modifier = gbm_bo_get_modifier(bo);
+    attrs.modifier = (flags & GBM_BO_USE_LINEAR) ? DRM_FORMAT_MOD_LINEAR : gbm_bo_get_modifier(bo);
 
     for (size_t i = 0; i < (size_t)attrs.planes; ++i) {
         attrs.strides.at(i) = gbm_bo_get_stride_for_plane(bo, i);