From 838ed87d6ffae0dbdc8a3ecaac2c8be006f6d053 Mon Sep 17 00:00:00 2001
From: Ikalco <73481042+ikalco@users.noreply.github.com>
Date: Sat, 31 Aug 2024 08:07:52 -0500
Subject: [PATCH] renderer: minor direct scanout fixes (#7594)

---
 src/helpers/Monitor.cpp | 15 +++++++++++++++
 src/helpers/Monitor.hpp |  3 ++-
 src/render/Renderer.cpp |  9 ++++++---
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/src/helpers/Monitor.cpp b/src/helpers/Monitor.cpp
index 685009fd..5b01d651 100644
--- a/src/helpers/Monitor.cpp
+++ b/src/helpers/Monitor.cpp
@@ -832,6 +832,21 @@ bool CMonitor::attemptDirectScanout() {
 
     // FIXME: make sure the buffer actually follows the available scanout dmabuf formats
     // and comes from the appropriate device. This may implode on multi-gpu!!
+
+    const auto params = PSURFACE->current.buffer->buffer->dmabuf();
+    // scanout buffer isn't dmabuf, so no scanout
+    if (!params.success)
+        return false;
+
+    // entering into scanout, so save monitor format
+    if (lastScanout.expired())
+        prevDrmFormat = drmFormat;
+
+    if (drmFormat != params.format) {
+        output->state->setFormat(params.format);
+        drmFormat = params.format;
+    }
+
     output->state->setBuffer(PSURFACE->current.buffer->buffer.lock());
     output->state->setPresentationMode(tearingState.activelyTearing ? Aquamarine::eOutputPresentationMode::AQ_OUTPUT_PRESENTATION_IMMEDIATE :
                                                                       Aquamarine::eOutputPresentationMode::AQ_OUTPUT_PRESENTATION_VSYNC);
diff --git a/src/helpers/Monitor.hpp b/src/helpers/Monitor.hpp
index 01a5d28d..203cac90 100644
--- a/src/helpers/Monitor.hpp
+++ b/src/helpers/Monitor.hpp
@@ -100,7 +100,8 @@ class CMonitor {
     std::optional<Vector2D>     forceSize;
     SP<Aquamarine::SOutputMode> currentMode;
     SP<Aquamarine::CSwapchain>  cursorSwapchain;
-    uint32_t                    drmFormat = DRM_FORMAT_INVALID;
+    uint32_t                    drmFormat     = DRM_FORMAT_INVALID;
+    uint32_t                    prevDrmFormat = DRM_FORMAT_INVALID;
 
     bool                        dpmsStatus       = true;
     bool                        vrrActive        = false; // this can be TRUE even if VRR is not active in the case that this display does not support it.
diff --git a/src/render/Renderer.cpp b/src/render/Renderer.cpp
index c5866e4a..27fde129 100644
--- a/src/render/Renderer.cpp
+++ b/src/render/Renderer.cpp
@@ -1277,7 +1277,8 @@ void CHyprRenderer::renderMonitor(CMonitor* pMonitor) {
             pMonitor->lastScanout.reset();
 
             // reset DRM format, make sure it's the one we want.
-            pMonitor->output->state->setFormat(pMonitor->drmFormat);
+            pMonitor->output->state->setFormat(pMonitor->prevDrmFormat);
+            pMonitor->drmFormat = pMonitor->prevDrmFormat;
         }
     }
 
@@ -1978,7 +1979,8 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
     pMonitor->currentMode   = nullptr;
 
     pMonitor->output->state->setFormat(DRM_FORMAT_XRGB8888);
-    pMonitor->drmFormat = DRM_FORMAT_XRGB8888;
+    pMonitor->prevDrmFormat = pMonitor->drmFormat;
+    pMonitor->drmFormat     = DRM_FORMAT_XRGB8888;
     pMonitor->output->state->resetExplicitFences();
 
     bool autoScale = false;
@@ -2219,7 +2221,8 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
 
     for (auto const& fmt : formats[(int)!RULE->enable10bit]) {
         pMonitor->output->state->setFormat(fmt.second);
-        pMonitor->drmFormat = fmt.second;
+        pMonitor->prevDrmFormat = pMonitor->drmFormat;
+        pMonitor->drmFormat     = fmt.second;
 
         if (!pMonitor->state.test()) {
             Debug::log(ERR, "output {} failed basic test on format {}", pMonitor->szName, fmt.first);