some fixes to dmabuf

src/Compositor.cpp

@@ -2327,8 +2327,12 @@ void CCompositor::setWindowFullscreen(PHLWINDOW pWindow, bool on, eFullscreenMod
 
     g_pInputManager->recheckIdleInhibitorStatus();
 
-    // DMAbuf stuff for direct scanout
+    // further updates require a monitor
-    g_pHyprRenderer->setWindowScanoutMode(pWindow);
+    if (!PMONITOR)
+        return;
+
+    // send a scanout tranche if we are entering fullscreen, and send a regular one if we aren't.
+    g_pHyprRenderer->setSurfaceScanoutMode(pWindow->m_pWLSurface->resource(), on ? PMONITOR->self.lock() : nullptr);
 
     g_pConfigManager->ensureVRR(PMONITOR);
 }

src/helpers/Format.cpp

@@ -3,6 +3,7 @@
 #include "../includes.hpp"
 #include "debug/Log.hpp"
 #include "../macros.hpp"
+#include <xf86drm.h>
 
 /*
     DRM formats are LE, while OGL is BE. The two primary formats
@@ -309,3 +310,17 @@ uint32_t FormatUtils::glFormatToType(uint32_t gl) {
 #endif
         GL_UNSIGNED_BYTE;
 }
+
+std::string FormatUtils::drmFormatName(DRMFormat drm) {
+    auto        n    = drmGetFormatName(drm);
+    std::string name = n;
+    free(n);
+    return name;
+}
+
+std::string FormatUtils::drmModifierName(uint64_t mod) {
+    auto        n    = drmGetFormatModifierName(mod);
+    std::string name = n;
+    free(n);
+    return name;
+}

src/helpers/Format.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cstdint>
+#include <string>
 #include "math/Math.hpp"
 #include <aquamarine/backend/Misc.hpp>
 
@@ -32,4 +33,6 @@ namespace FormatUtils {
     int                 minStride(const SPixelFormat* const fmt, int32_t width);
     uint32_t            drmFormatToGL(DRMFormat drm);
     uint32_t            glFormatToType(uint32_t gl);
+    std::string         drmFormatName(DRMFormat drm);
+    std::string         drmModifierName(uint64_t mod);
 };

src/protocols/LinuxDMABUF.cpp

@@ -63,7 +63,7 @@ CCompiledDMABUFFeedback::CCompiledDMABUFFeedback(dev_t device, std::vector<SDMAB
 
     mainDevice    = device;
     tableFD       = fds[1];
-    tranches   = formatsVec;
+    this->formats = formatsVec;
 
     // TODO: maybe calculate indices? currently we send all as available which could be wrong? I ain't no kernel dev tho.
 }
@@ -189,7 +189,7 @@ void CLinuxDMABBUFParamsResource::create(uint32_t id) {
         return;
     }
 
-    LOGM(LOG, "Creating a dmabuf, with id {}: size {}, fmt {}, planes {}", id, attrs->size, attrs->format, attrs->planes);
+    LOGM(LOG, "Creating a dmabuf, with id {}: size {}, fmt {}, planes {}", id, attrs->size, FormatUtils::drmFormatName(attrs->format), attrs->planes);
     for (int i = 0; i < attrs->planes; ++i) {
         LOGM(LOG, " | plane {}: mod {} fd {} stride {} offset {}", i, attrs->modifier, attrs->fds[i], attrs->strides[i], attrs->offsets[i]);
     }
@@ -276,14 +276,25 @@ CLinuxDMABUFFeedbackResource::CLinuxDMABUFFeedbackResource(SP<CZwpLinuxDmabufFee
     resource->setOnDestroy([this](CZwpLinuxDmabufFeedbackV1* r) { PROTO::linuxDma->destroyResource(this); });
     resource->setDestroy([this](CZwpLinuxDmabufFeedbackV1* r) { PROTO::linuxDma->destroyResource(this); });
 
-    if (surface)
-        LOGM(ERR, "FIXME: surface feedback stub");
-
     auto* feedback = PROTO::linuxDma->defaultFeedback.get();
 
     resource->sendFormatTable(feedback->tableFD, feedback->tableLen);
 
     // send default feedback
+    sendDefault();
+}
+
+CLinuxDMABUFFeedbackResource::~CLinuxDMABUFFeedbackResource() {
+    ;
+}
+
+bool CLinuxDMABUFFeedbackResource::good() {
+    return resource->resource();
+}
+
+void CLinuxDMABUFFeedbackResource::sendDefault() {
+    auto*           feedback = PROTO::linuxDma->defaultFeedback.get();
+
     struct wl_array deviceArr = {
         .size = sizeof(feedback->mainDevice),
         .data = (void*)&feedback->mainDevice,
@@ -295,40 +306,12 @@ CLinuxDMABUFFeedbackResource::CLinuxDMABUFFeedbackResource(SP<CZwpLinuxDmabufFee
 
     wl_array indices;
     wl_array_init(&indices);
-    for (size_t i = 0; i < feedback->tranches.size(); ++i) {
+    for (size_t i = 0; i < feedback->formats.size(); ++i) {
         *((uint16_t*)wl_array_add(&indices, sizeof(uint16_t))) = i;
     }
     resource->sendTrancheFormats(&indices);
     wl_array_release(&indices);
     resource->sendTrancheDone();
-
-    // Scanout tranche
-    // FIXME: jesus fucking christ this SUCKSSSSSS ASSSSSS
-    resource->sendTrancheTargetDevice(&deviceArr);
-    resource->sendTrancheFlags(ZWP_LINUX_DMABUF_FEEDBACK_V1_TRANCHE_FLAGS_SCANOUT);
-
-    wl_array indices2;
-    wl_array_init(&indices2);
-    for (size_t i = 0; i < feedback->tranches.size(); ++i) {
-        // FIXME: if the monitor gets the wrong format we'll be FUCKED by drm and no scanout will happen
-        if (feedback->tranches.at(i).first != DRM_FORMAT_XRGB8888 && feedback->tranches.at(i).first != DRM_FORMAT_XRGB2101010)
-            continue;
-
-        *((uint16_t*)wl_array_add(&indices2, sizeof(uint16_t))) = i;
-    }
-    resource->sendTrancheFormats(&indices2);
-    wl_array_release(&indices2);
-    resource->sendTrancheDone();
-
-    resource->sendDone();
-}
-
-CLinuxDMABUFFeedbackResource::~CLinuxDMABUFFeedbackResource() {
-    ;
-}
-
-bool CLinuxDMABUFFeedbackResource::good() {
-    return resource->resource();
 }
 
 CLinuxDMABUFResource::CLinuxDMABUFResource(SP<CZwpLinuxDmabufV1> resource_) : resource(resource_) {
@@ -379,7 +362,7 @@ bool CLinuxDMABUFResource::good() {
 }
 
 void CLinuxDMABUFResource::sendMods() {
-    for (auto& [fmt, mod] : PROTO::linuxDma->defaultFeedback->tranches) {
+    for (auto& [fmt, mod] : PROTO::linuxDma->defaultFeedback->formats) {
         if (resource->version() < 3) {
             if (mod == DRM_FORMAT_MOD_INVALID || mod == DRM_FORMAT_MOD_LINEAR)
                 resource->sendFormat(fmt);
@@ -475,3 +458,62 @@ void CLinuxDMABufV1Protocol::destroyResource(CLinuxDMABBUFParamsResource* resour
 void CLinuxDMABufV1Protocol::destroyResource(CLinuxDMABuffer* resource) {
     std::erase_if(m_vBuffers, [&](const auto& other) { return other.get() == resource; });
 }
+
+void CLinuxDMABufV1Protocol::updateScanoutTranche(SP<CWLSurfaceResource> surface, SP<CMonitor> pMonitor) {
+    // TODO: iterating isn't particularly efficient, maybe add a system for addons for surfaces
+
+    SP<CLinuxDMABUFFeedbackResource> feedbackResource;
+    for (auto& f : m_vFeedbacks) {
+        if (f->surface != surface)
+            continue;
+
+        feedbackResource = f;
+        break;
+    }
+
+    if (!feedbackResource) {
+        LOGM(LOG, "updateScanoutTranche: surface has no dmabuf_feedback");
+        return;
+    }
+
+    if (!pMonitor) {
+        LOGM(LOG, "updateScanoutTranche: resetting feedback");
+        feedbackResource->sendDefault();
+        return;
+    }
+
+    auto* feedback = PROTO::linuxDma->defaultFeedback.get();
+
+    LOGM(LOG, "updateScanoutTranche: sending a scanout tranche");
+
+    // send a dedicated scanout tranche that contains formats that:
+    //  - match the format of the output
+    //  - are not linear or implicit
+
+    struct wl_array deviceArr = {
+        .size = sizeof(feedback->mainDevice),
+        .data = (void*)&feedback->mainDevice,
+    };
+    feedbackResource->resource->sendTrancheTargetDevice(&deviceArr);
+    feedbackResource->resource->sendTrancheFlags(ZWP_LINUX_DMABUF_FEEDBACK_V1_TRANCHE_FLAGS_SCANOUT);
+
+    wl_array indices2;
+    wl_array_init(&indices2);
+    for (size_t i = 0; i < feedback->formats.size(); ++i) {
+        if (feedback->formats.at(i).first != pMonitor->output->state->state().drmFormat)
+            continue;
+
+        if (feedback->formats.at(i).second == DRM_FORMAT_MOD_LINEAR || feedback->formats.at(i).second == DRM_FORMAT_MOD_INVALID)
+            continue;
+
+        LOGM(TRACE, "updateScanoutTranche: Format {} with modifier {} aka {} passed", FormatUtils::drmFormatName(feedback->formats.at(i).first), feedback->formats.at(i).second,
+             FormatUtils::drmModifierName(feedback->formats.at(i).second));
+        *((uint16_t*)wl_array_add(&indices2, sizeof(uint16_t))) = i;
+    }
+
+    feedbackResource->resource->sendTrancheFormats(&indices2);
+    wl_array_release(&indices2);
+    feedbackResource->resource->sendTrancheDone();
+
+    feedbackResource->resource->sendDone();
+}

src/protocols/LinuxDMABUF.hpp

@@ -57,7 +57,7 @@ class CCompiledDMABUFFeedback {
     dev_t                                      mainDevice = 0;
     int                                        tableFD    = -1;
     size_t                                     tableLen   = 0;
-    std::vector<std::pair<uint32_t, uint64_t>> tranches;
+    std::vector<std::pair<uint32_t, uint64_t>> formats;
 };
 
 class CLinuxDMABBUFParamsResource {
@@ -85,11 +85,14 @@ class CLinuxDMABUFFeedbackResource {
     ~CLinuxDMABUFFeedbackResource();
 
     bool                   good();
+    void                   sendDefault();
 
     SP<CWLSurfaceResource> surface; // optional, for surface feedbacks
 
   private:
     SP<CZwpLinuxDmabufFeedbackV1> resource;
+
+    friend class CLinuxDMABufV1Protocol;
 };
 
 class CLinuxDMABUFResource {
@@ -109,6 +112,7 @@ class CLinuxDMABufV1Protocol : public IWaylandProtocol {
     ~CLinuxDMABufV1Protocol();
 
     virtual void bindManager(wl_client* client, void* data, uint32_t ver, uint32_t id);
+    void         updateScanoutTranche(SP<CWLSurfaceResource> surface, SP<CMonitor> pMonitor);
 
   private:
     void destroyResource(CLinuxDMABUFResource* resource);

src/render/Renderer.cpp

@@ -15,6 +15,7 @@
 #include "../protocols/core/DataDevice.hpp"
 #include "../protocols/core/Compositor.hpp"
 #include "../protocols/DRMSyncobj.hpp"
+#include "../protocols/LinuxDMABUF.hpp"
 #include "../helpers/sync/SyncTimeline.hpp"
 
 extern "C" {
@@ -1515,8 +1516,11 @@ void CHyprRenderer::sendFrameEventsToWorkspace(CMonitor* pMonitor, PHLWORKSPACE
     }
 }
 
-void CHyprRenderer::setWindowScanoutMode(PHLWINDOW pWindow) {
+void CHyprRenderer::setSurfaceScanoutMode(SP<CWLSurfaceResource> surface, SP<CMonitor> monitor) {
-    // FIXME: fix when moved to new impl
+    if (!PROTO::linuxDma)
+        return;
+
+    PROTO::linuxDma->updateScanoutTranche(surface, monitor);
 }
 
 // taken from Sway.

src/render/Renderer.hpp

@@ -89,7 +89,7 @@ class CHyprRenderer {
     damageTrackingModeFromStr(const std::string&);
 
     bool             attemptDirectScanout(CMonitor*);
-    void             setWindowScanoutMode(PHLWINDOW);
+    void             setSurfaceScanoutMode(SP<CWLSurfaceResource> surface, SP<CMonitor> monitor); // nullptr monitor resets
     void             initiateManualCrash();
 
     bool             m_bCrashingInProgress = false;