gbm: Nvidia fixes for scanout gbm allocation (#27)

* nv fixes

* fix

include/aquamarine/backend/Backend.hpp

@@ -77,6 +77,7 @@ namespace Aquamarine {
         virtual std::vector<SDRMFormat>                                 getCursorFormats()                         = 0;
         virtual bool                                                    createOutput(const std::string& name = "") = 0; // "" means auto
         virtual Hyprutils::Memory::CSharedPointer<IAllocator>           preferredAllocator()                       = 0;
+        virtual std::vector<SDRMFormat>                                 getRenderableFormats(); // empty = use getRenderFormats
     };
 
     class CBackend {

include/aquamarine/backend/DRM.hpp

@@ -342,6 +342,7 @@ namespace Aquamarine {
         virtual std::vector<SDRMFormat>                                 getCursorFormats();
         virtual bool                                                    createOutput(const std::string& name = "");
         virtual Hyprutils::Memory::CSharedPointer<IAllocator>           preferredAllocator();
+        virtual std::vector<SDRMFormat>                                 getRenderableFormats();
 
         Hyprutils::Memory::CWeakPointer<CDRMBackend>                    self;
 
@@ -366,6 +367,7 @@ namespace Aquamarine {
         void scanLeases();
         void restoreAfterVT();
         void recheckCRTCs();
+        void buildGlFormats(const std::vector<SGLFormat>& fmts);
 
         Hyprutils::Memory::CSharedPointer<CSessionDevice>     gpu;
         Hyprutils::Memory::CSharedPointer<IDRMImplementation> impl;
@@ -383,6 +385,7 @@ namespace Aquamarine {
         std::vector<Hyprutils::Memory::CSharedPointer<SDRMPlane>>     planes;
         std::vector<Hyprutils::Memory::CSharedPointer<SDRMConnector>> connectors;
         std::vector<SDRMFormat>                                       formats;
+        std::vector<SDRMFormat>                                       glFormats;
 
         bool                                                          atomic = false;
 

include/aquamarine/backend/Misc.hpp

@@ -4,6 +4,12 @@
 #include <vector>
 
 namespace Aquamarine {
+    struct SGLFormat {
+        uint32_t drmFormat = 0;
+        uint64_t modifier  = 0;
+        bool     external  = false;
+    };
+
     struct SDRMFormat {
         uint32_t              drmFormat = 0; /* DRM_FORMAT_INVALID */
         std::vector<uint64_t> modifiers;

src/allocator/GBM.cpp

@@ -67,7 +67,9 @@ Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hypruti
     TRACE(allocator->backend->log(AQ_LOG_TRACE,
                                   std::format("GBM: Allocating a buffer: size {}, format {}, cursor: {}, multigpu: {}", attrs.size, fourccToName(attrs.format), CURSOR, MULTIGPU)));
 
-    const auto            FORMATS = CURSOR ? swapchain->backendImpl->getCursorFormats() : swapchain->backendImpl->getRenderFormats();
+    const auto            FORMATS = CURSOR ?
+                   swapchain->backendImpl->getCursorFormats() :
+                   (swapchain->backendImpl->getRenderableFormats().size() == 0 ? swapchain->backendImpl->getRenderFormats() : swapchain->backendImpl->getRenderableFormats());
 
     std::vector<uint64_t> explicitModifiers;
 

src/backend/Backend.cpp

@@ -328,3 +328,7 @@ int Aquamarine::CBackend::reopenDRMNode(int drmFD, bool allowRenderNode) {
 
     return newFD;
 }
+
+std::vector<SDRMFormat> Aquamarine::IBackendImplementation::getRenderableFormats() {
+    return {};
+}

src/backend/drm/DRM.cpp

@@ -495,9 +495,32 @@ bool Aquamarine::CDRMBackend::initMgpu() {
 
     mgpu.renderer->self = mgpu.renderer;
 
+    buildGlFormats(mgpu.renderer->formats);
+
     return true;
 }
 
+void Aquamarine::CDRMBackend::buildGlFormats(const std::vector<SGLFormat>& fmts) {
+    std::vector<SDRMFormat> result;
+
+    for (auto& fmt : fmts) {
+        if (fmt.external)
+            continue;
+
+        if (auto it = std::find_if(result.begin(), result.end(), [fmt] (const auto& e) { return fmt.drmFormat == e.drmFormat; }); it != result.end()) {
+            it->modifiers.emplace_back(fmt.modifier);
+            continue;
+        }
+
+        result.emplace_back(SDRMFormat{
+            fmt.drmFormat,
+            {fmt.modifier},
+        });
+    }
+
+    glFormats = result;
+}
+
 void Aquamarine::CDRMBackend::recheckCRTCs() {
     if (connectors.empty() || crtcs.empty())
         return;
@@ -802,6 +825,25 @@ bool Aquamarine::CDRMBackend::setCursor(SP<IBuffer> buffer, const Hyprutils::Mat
 void Aquamarine::CDRMBackend::onReady() {
     backend->log(AQ_LOG_DEBUG, std::format("drm: Connectors size2 {}", connectors.size()));
 
+    // init a drm renderer to gather gl formats.
+    // if we are secondary, initMgpu will have done that
+    if (!primary) {
+        auto a = CGBMAllocator::create(backend->reopenDRMNode(gpu->fd), backend);
+        if (!a)
+            backend->log(AQ_LOG_ERROR, "drm: onReady: no renderer for gl formats");
+        else {
+            auto r = CDRMRenderer::attempt(a, backend.lock());
+            if (!r)
+                backend->log(AQ_LOG_ERROR, "drm: onReady: no renderer for gl formats");
+            else {
+                TRACE(backend->log(AQ_LOG_TRACE, std::format("drm: onReady: gathered {} gl formats", r->formats.size())));
+                buildGlFormats(r->formats);
+                r.reset();
+                a.reset();
+            }
+        }
+    }
+
     for (auto& c : connectors) {
         backend->log(AQ_LOG_DEBUG, std::format("drm: onReady: connector {}", c->id));
         if (!c->output)
@@ -834,6 +876,10 @@ std::vector<SDRMFormat> Aquamarine::CDRMBackend::getRenderFormats() {
     return {};
 }
 
+std::vector<SDRMFormat> Aquamarine::CDRMBackend::getRenderableFormats() {
+    return glFormats;
+}
+
 std::vector<SDRMFormat> Aquamarine::CDRMBackend::getCursorFormats() {
     for (auto& p : planes) {
         if (p->type != DRM_PLANE_TYPE_CURSOR)

src/backend/drm/Renderer.cpp

@@ -153,7 +153,7 @@ bool CDRMRenderer::initDRMFormats() {
 
     TRACE(backend->log(AQ_LOG_TRACE, "EGL: Supported formats:"));
 
-    std::vector<GLFormat> dmaFormats;
+    std::vector<SGLFormat> dmaFormats;
 
     for (auto& fmt : formats) {
         std::vector<std::pair<uint64_t, bool>> mods;
@@ -170,7 +170,7 @@ bool CDRMRenderer::initDRMFormats() {
         mods.push_back({DRM_FORMAT_MOD_INVALID, true});
 
         for (auto& [mod, external] : mods) {
-            dmaFormats.push_back(GLFormat{
+            dmaFormats.push_back(SGLFormat{
                 .drmFormat = (uint32_t)fmt,
                 .modifier  = mod,
                 .external  = external,

src/backend/drm/Renderer.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <aquamarine/backend/DRM.hpp>
+#include "FormatUtils.hpp"
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
 #include <GLES2/gl2.h>
@@ -19,6 +20,7 @@ namespace Aquamarine {
         GLuint   texid  = 0;
         GLuint   target = GL_TEXTURE_2D;
     };
+
     class CDRMRendererBufferAttachment : public IAttachment {
       public:
         CDRMRendererBufferAttachment(Hyprutils::Memory::CWeakPointer<CDRMRenderer> renderer_, Hyprutils::Memory::CSharedPointer<IBuffer> buffer, EGLImageKHR image, GLuint fbo_,
@@ -78,16 +80,10 @@ namespace Aquamarine {
             EGLSurface draw = nullptr, read = nullptr;
         } savedEGLState;
 
-        struct GLFormat {
-            uint32_t drmFormat = 0;
-            uint64_t modifier  = 0;
-            bool     external  = false;
-        };
-
         SGLTex                                        glTex(Hyprutils::Memory::CSharedPointer<IBuffer> buf);
 
         Hyprutils::Memory::CWeakPointer<CDRMRenderer> self;
-        std::vector<GLFormat>                         formats;
+        std::vector<SGLFormat>                        formats;
 
       private:
         CDRMRenderer() = default;