DRM: Add a dumb allocator (#104)

Adds a drm_dumb allocator to the DRM allocator for nvidia hw cursors.

include/aquamarine/allocator/Allocator.hpp

@@ -16,6 +16,7 @@ namespace Aquamarine {
 
     enum eAllocatorType {
         AQ_ALLOCATOR_TYPE_GBM = 0,
+        AQ_ALLOCATOR_TYPE_DRM_DUMB,
     };
 
     class IAllocator {

include/aquamarine/allocator/DRMDumb.hpp

@@ -0,0 +1,68 @@
+#pragma once
+
+#include "Allocator.hpp"
+
+namespace Aquamarine {
+    class CDRMDumbAllocator;
+    class CBackend;
+    class CSwapchain;
+
+    class CDRMDumbBuffer : public IBuffer {
+      public:
+        virtual ~CDRMDumbBuffer();
+
+        virtual eBufferCapability                      caps();
+        virtual eBufferType                            type();
+        virtual void                                   update(const Hyprutils::Math::CRegion& damage);
+        virtual bool                                   isSynchronous();
+        virtual bool                                   good();
+        virtual SDMABUFAttrs                           dmabuf();
+        virtual std::tuple<uint8_t*, uint32_t, size_t> beginDataPtr(uint32_t flags);
+        virtual void                                   endDataPtr();
+
+      private:
+        CDRMDumbBuffer(const SAllocatorBufferParams& params, Hyprutils::Memory::CWeakPointer<CDRMDumbAllocator> allocator_,
+                       Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain);
+
+        Hyprutils::Memory::CWeakPointer<CDRMDumbAllocator> allocator;
+
+        //
+        Hyprutils::Math::Vector2D pixelSize;
+        uint32_t                  stride = 0, handle = 0;
+        uint64_t                  bufferLen = 0;
+        uint8_t*                  data      = nullptr;
+        int                       primeFD   = -1;
+
+        //
+        SDMABUFAttrs attrs{.success = false};
+
+        friend class CDRMDumbAllocator;
+    };
+
+    class CDRMDumbAllocator : public IAllocator {
+      public:
+        ~CDRMDumbAllocator();
+        static Hyprutils::Memory::CSharedPointer<CDRMDumbAllocator> create(int drmfd_, Hyprutils::Memory::CWeakPointer<CBackend> backend_);
+
+        virtual Hyprutils::Memory::CSharedPointer<IBuffer>          acquire(const SAllocatorBufferParams& params, Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain_);
+        virtual Hyprutils::Memory::CSharedPointer<CBackend>         getBackend();
+        virtual int                                                 drmFD();
+        virtual eAllocatorType                                      type();
+
+        //
+        Hyprutils::Memory::CWeakPointer<CDRMDumbAllocator> self;
+
+      private:
+        CDRMDumbAllocator(int fd_, Hyprutils::Memory::CWeakPointer<CBackend> backend_);
+
+        // a vector purely for tracking (debugging) the buffers and nothing more
+        std::vector<Hyprutils::Memory::CWeakPointer<CDRMDumbBuffer>> buffers;
+
+        Hyprutils::Memory::CWeakPointer<CBackend>                    backend;
+
+        int                                                          drmfd = -1;
+
+        friend class CDRMDumbBuffer;
+        friend class CDRMRenderer;
+    };
+};

include/aquamarine/backend/Backend.hpp

@@ -78,6 +78,7 @@ namespace Aquamarine {
         virtual bool                                                       createOutput(const std::string& name = "") = 0; // "" means auto
         virtual Hyprutils::Memory::CSharedPointer<IAllocator>              preferredAllocator()                       = 0;
         virtual std::vector<SDRMFormat>                                    getRenderableFormats(); // empty = use getRenderFormats
+        virtual std::vector<Hyprutils::Memory::CSharedPointer<IAllocator>> getAllocators() = 0;
     };
 
     class CBackend {

include/aquamarine/backend/DRM.hpp

@@ -15,6 +15,7 @@ namespace Aquamarine {
     class CDRMOutput;
     struct SDRMConnector;
     class CDRMRenderer;
+    class CDRMDumbAllocator;
 
     typedef std::function<void(void)> FIdleCallback;
 
@@ -353,6 +354,7 @@ namespace Aquamarine {
         virtual bool                                                       createOutput(const std::string& name = "");
         virtual Hyprutils::Memory::CSharedPointer<IAllocator>              preferredAllocator();
         virtual std::vector<SDRMFormat>                                    getRenderableFormats();
+        virtual std::vector<Hyprutils::Memory::CSharedPointer<IAllocator>> getAllocators();
 
         Hyprutils::Memory::CWeakPointer<CDRMBackend>                       self;
 
@@ -396,6 +398,8 @@ namespace Aquamarine {
         std::vector<SDRMFormat>                                       formats;
         std::vector<SDRMFormat>                                       glFormats;
 
+        Hyprutils::Memory::CSharedPointer<CDRMDumbAllocator>          dumbAllocator;
+
         bool                                                          atomic = false;
 
         struct {

include/aquamarine/backend/Headless.hpp

@@ -8,6 +8,7 @@
 namespace Aquamarine {
     class CBackend;
     class CHeadlessBackend;
+    class IAllocator;
 
     class CHeadlessOutput : public IOutput {
       public:
@@ -47,6 +48,7 @@ namespace Aquamarine {
         virtual std::vector<SDRMFormat>                                    getCursorFormats();
         virtual bool                                                       createOutput(const std::string& name = "");
         virtual Hyprutils::Memory::CSharedPointer<IAllocator>              preferredAllocator();
+        virtual std::vector<Hyprutils::Memory::CSharedPointer<IAllocator>> getAllocators();
 
         Hyprutils::Memory::CWeakPointer<CHeadlessBackend>                  self;
 

include/aquamarine/backend/Wayland.hpp

@@ -135,6 +135,7 @@ namespace Aquamarine {
         virtual std::vector<SDRMFormat>                                    getCursorFormats();
         virtual bool                                                       createOutput(const std::string& name = "");
         virtual Hyprutils::Memory::CSharedPointer<IAllocator>              preferredAllocator();
+        virtual std::vector<Hyprutils::Memory::CSharedPointer<IAllocator>> getAllocators();
 
         Hyprutils::Memory::CWeakPointer<CWaylandBackend>                   self;
 

src/allocator/DRMDumb.cpp

@@ -0,0 +1,159 @@
+#include <aquamarine/allocator/DRMDumb.hpp>
+#include <aquamarine/backend/Backend.hpp>
+#include <aquamarine/backend/DRM.hpp>
+#include <aquamarine/allocator/Swapchain.hpp>
+#include "FormatUtils.hpp"
+#include "Shared.hpp"
+#include <fcntl.h>
+#include <xf86drm.h>
+#include <xf86drmMode.h>
+#include <unistd.h>
+#include <cstring>
+#include <sys/mman.h>
+#include "../backend/drm/Renderer.hpp"
+
+using namespace Aquamarine;
+using namespace Hyprutils::Memory;
+#define SP CSharedPointer
+#define WP CWeakPointer
+
+Aquamarine::CDRMDumbBuffer::CDRMDumbBuffer(const SAllocatorBufferParams& params, Hyprutils::Memory::CWeakPointer<CDRMDumbAllocator> allocator_,
+                                           Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain) : allocator(allocator_) {
+    attrs.format = params.format;
+
+    if (int ret = drmModeCreateDumbBuffer(allocator->drmFD(), params.size.x, params.size.y, 32, 0, &handle, &stride, &bufferLen); ret < 0) {
+        allocator->backend->log(AQ_LOG_ERROR, std::format("failed to create a drm_dumb buffer: {}", strerror(-ret)));
+        return;
+    }
+
+    pixelSize = {(double)params.size.x, (double)params.size.y};
+
+    attrs.size          = pixelSize;
+    attrs.strides.at(0) = stride;
+    attrs.planes        = 1;
+    size                = pixelSize;
+
+    uint64_t offset = 0;
+    if (int ret = drmModeMapDumbBuffer(allocator->drmFD(), handle, &offset); ret < 0) {
+        allocator->backend->log(AQ_LOG_ERROR, std::format("failed to map a drm_dumb buffer: {}", strerror(-ret)));
+        return;
+    }
+
+    data = (uint8_t*)mmap(nullptr, bufferLen, PROT_READ | PROT_WRITE, MAP_SHARED, allocator->drmFD(), offset);
+    if (data == MAP_FAILED) {
+        allocator->backend->log(AQ_LOG_ERROR, "failed to mmap a drm_dumb buffer");
+        return;
+    }
+
+    // set the entire buffer so we dont get garbage
+    memset(data, 0xFF, bufferLen);
+
+    if (int ret = drmPrimeHandleToFD(allocator->drmFD(), handle, DRM_CLOEXEC, &primeFD); ret < 0) {
+        allocator->backend->log(AQ_LOG_ERROR, std::format("failed to map a drm_dumb buffer: {}", strerror(-ret)));
+        return;
+    }
+
+    attrs.fds.at(0) = primeFD;
+
+    attrs.success = true;
+
+    allocator->backend->log(AQ_LOG_DEBUG, std::format("DRM Dumb: Allocated a new buffer with primeFD {}, size {} and format {}", primeFD, attrs.size, fourccToName(attrs.format)));
+}
+
+Aquamarine::CDRMDumbBuffer::~CDRMDumbBuffer() {
+    events.destroy.emit();
+
+    TRACE(allocator->backend->log(AQ_LOG_TRACE, std::format("DRM Dumb: dropping buffer {}", primeFD)));
+
+    if (handle == 0)
+        return;
+
+    if (data)
+        munmap(data, bufferLen);
+
+    drmModeDestroyDumbBuffer(allocator->drmFD(), handle);
+}
+
+eBufferCapability Aquamarine::CDRMDumbBuffer::caps() {
+    return eBufferCapability::BUFFER_CAPABILITY_DATAPTR;
+}
+
+eBufferType Aquamarine::CDRMDumbBuffer::type() {
+    return eBufferType::BUFFER_TYPE_DMABUF;
+}
+
+void Aquamarine::CDRMDumbBuffer::update(const Hyprutils::Math::CRegion& damage) {
+    ; // nothing to do
+}
+
+bool Aquamarine::CDRMDumbBuffer::isSynchronous() {
+    return true;
+}
+
+bool Aquamarine::CDRMDumbBuffer::good() {
+    return attrs.success && data;
+}
+
+SDMABUFAttrs Aquamarine::CDRMDumbBuffer::dmabuf() {
+    return attrs;
+}
+
+std::tuple<uint8_t*, uint32_t, size_t> Aquamarine::CDRMDumbBuffer::beginDataPtr(uint32_t flags) {
+    return {data, attrs.format, bufferLen};
+}
+
+void Aquamarine::CDRMDumbBuffer::endDataPtr() {
+    ; // nothing to do
+}
+
+Aquamarine::CDRMDumbAllocator::~CDRMDumbAllocator() {
+    ; // nothing to do
+}
+
+SP<CDRMDumbAllocator> Aquamarine::CDRMDumbAllocator::create(int drmfd_, Hyprutils::Memory::CWeakPointer<CBackend> backend_) {
+    if (drmGetNodeTypeFromFd(drmfd_) != DRM_NODE_PRIMARY) {
+        backend_->log(AQ_LOG_ERROR, "DRM Dumb: Cannot create allocator when drmfd is not the primary node");
+        return nullptr;
+    }
+
+    uint64_t hasDumb = 0;
+    if (drmGetCap(drmfd_, DRM_CAP_DUMB_BUFFER, &hasDumb) < 0) {
+        backend_->log(AQ_LOG_ERROR, "DRM Dumb: Failed to query hasDumb");
+        return nullptr;
+    }
+
+    if (!hasDumb) {
+        backend_->log(AQ_LOG_ERROR, "DRM Dumb: hasDumb is false, gpu driver doesn't support dumb buffers!");
+        return nullptr;
+    }
+
+    auto a  = SP<CDRMDumbAllocator>(new CDRMDumbAllocator(drmfd_, backend_));
+    a->self = a;
+
+    backend_->log(AQ_LOG_DEBUG, "DRM Dumb: created a dumb allocator");
+
+    return a;
+}
+
+SP<IBuffer> Aquamarine::CDRMDumbAllocator::acquire(const SAllocatorBufferParams& params, SP<CSwapchain> swapchain_) {
+    auto buf = SP<IBuffer>(new CDRMDumbBuffer(params, self, swapchain_));
+    if (!buf->good())
+        return nullptr;
+    return buf;
+}
+
+SP<CBackend> Aquamarine::CDRMDumbAllocator::getBackend() {
+    return backend.lock();
+}
+
+int Aquamarine::CDRMDumbAllocator::drmFD() {
+    return drmfd;
+}
+
+eAllocatorType Aquamarine::CDRMDumbAllocator::type() {
+    return eAllocatorType::AQ_ALLOCATOR_TYPE_DRM_DUMB;
+}
+
+Aquamarine::CDRMDumbAllocator::CDRMDumbAllocator(int fd_, Hyprutils::Memory::CWeakPointer<CBackend> backend_) : drmfd(fd_), backend(backend_) {
+    ; // nothing to do
+}

src/backend/Headless.cpp

@@ -187,6 +187,10 @@ SP<IAllocator> Aquamarine::CHeadlessBackend::preferredAllocator() {
     return backend->primaryAllocator;
 }
 
+std::vector<SP<IAllocator>> Aquamarine::CHeadlessBackend::getAllocators() {
+    return {backend->primaryAllocator};
+}
+
 bool Aquamarine::CHeadlessBackend::CTimer::expired() {
     return std::chrono::steady_clock::now() > when;
 }

src/backend/Wayland.cpp

@@ -442,6 +442,10 @@ SP<IAllocator> Aquamarine::CWaylandBackend::preferredAllocator() {
     return backend->primaryAllocator;
 }
 
+std::vector<SP<IAllocator>> Aquamarine::CWaylandBackend::getAllocators() {
+    return {backend->primaryAllocator};
+}
+
 Aquamarine::CWaylandOutput::CWaylandOutput(const std::string& name_, Hyprutils::Memory::CWeakPointer<CWaylandBackend> backend_) : backend(backend_) {
     name = name_;
 

src/backend/drm/DRM.cpp

@@ -3,6 +3,7 @@
 #include <aquamarine/backend/drm/Legacy.hpp>
 #include <aquamarine/backend/drm/Atomic.hpp>
 #include <aquamarine/allocator/GBM.hpp>
+#include <aquamarine/allocator/DRMDumb.hpp>
 #include <hyprutils/string/VarList.hpp>
 #include <chrono>
 #include <thread>
@@ -255,6 +256,8 @@ std::vector<SP<CDRMBackend>> Aquamarine::CDRMBackend::attempt(SP<CBackend> backe
             newPrimary = drmBackend;
         }
 
+        drmBackend->dumbAllocator = CDRMDumbAllocator::create(gpu->fd, backend);
+
         backends.emplace_back(drmBackend);
 
         // so that session can handle udev change/remove events for this gpu
@@ -961,6 +964,10 @@ SP<IAllocator> Aquamarine::CDRMBackend::preferredAllocator() {
     return backend->primaryAllocator;
 }
 
+std::vector<SP<IAllocator>> Aquamarine::CDRMBackend::getAllocators() {
+    return {backend->primaryAllocator, dumbAllocator};
+}
+
 bool Aquamarine::SDRMPlane::init(drmModePlane* plane) {
     id = plane->plane_id;
 
@@ -1647,33 +1654,45 @@ SP<IBackendImplementation> Aquamarine::CDRMOutput::getBackend() {
 }
 
 bool Aquamarine::CDRMOutput::setCursor(SP<IBuffer> buffer, const Vector2D& hotspot) {
-    if (buffer && !buffer->dmabuf().success) {
-        backend->backend->log(AQ_LOG_ERROR, "drm: Cursor buffer has to be a dmabuf");
-        return false;
-    }
-
     if (!connector->crtc)
         return false;
 
     if (!buffer)
         setCursorVisible(false);
     else {
+        auto bufferType = buffer->type();
+
+        if (!buffer->good()) {
+            backend->backend->log(AQ_LOG_ERROR, "drm: bad buffer passed to setCursor");
+            return false;
+        }
+
+        if ((bufferType == eBufferType::BUFFER_TYPE_SHM && !buffer->shm().success) || (bufferType == eBufferType::BUFFER_TYPE_DMABUF && !buffer->dmabuf().success)) {
+            backend->backend->log(AQ_LOG_ERROR, "drm: Invalid buffer passed to setCursor");
+            return false;
+        }
+
         SP<CDRMFB> fb;
 
         if (backend->primary) {
             TRACE(backend->backend->log(AQ_LOG_TRACE, "drm: Backend requires cursor blit, blitting"));
 
+            // TODO: will this not implode on drm_dumb?!
+
             if (!mgpu.cursorSwapchain) {
                 TRACE(backend->backend->log(AQ_LOG_TRACE, "drm: No cursorSwapchain for blit, creating"));
                 mgpu.cursorSwapchain = CSwapchain::create(backend->rendererState.allocator, backend.lock());
             }
 
+            const auto FORMAT = bufferType == eBufferType::BUFFER_TYPE_SHM ? buffer->shm().format : buffer->dmabuf().format;
+            const auto SIZE   = bufferType == eBufferType::BUFFER_TYPE_SHM ? buffer->shm().size : buffer->dmabuf().size;
+
             auto       OPTIONS = mgpu.cursorSwapchain->currentOptions();
             OPTIONS.multigpu   = false;
             OPTIONS.scanout    = true;
             OPTIONS.cursor     = true;
-            OPTIONS.format   = buffer->dmabuf().format;
+            OPTIONS.format     = FORMAT;
-            OPTIONS.size     = buffer->dmabuf().size;
+            OPTIONS.size       = SIZE;
             OPTIONS.length     = 2;
 
             if (!mgpu.cursorSwapchain->reconfigure(OPTIONS)) {
@@ -1808,7 +1827,7 @@ Aquamarine::CDRMFB::CDRMFB(SP<IBuffer> buffer_, Hyprutils::Memory::CWeakPointer<
 void Aquamarine::CDRMFB::import() {
     auto attrs = buffer->dmabuf();
     if (!attrs.success) {
-        backend->backend->log(AQ_LOG_ERROR, "drm: Buffer submitted has no dmabuf");
+        backend->backend->log(AQ_LOG_ERROR, "drm: Buffer submitted has no dmabuf or a drm handle");
         return;
     }
 
@@ -1818,7 +1837,6 @@ void Aquamarine::CDRMFB::import() {
     }
 
     // TODO: check format
-
     for (int i = 0; i < attrs.planes; ++i) {
         int ret = drmPrimeFDToHandle(backend->gpu->fd, attrs.fds.at(i), &boHandles[i]);
         if (ret) {
@@ -1831,6 +1849,7 @@ void Aquamarine::CDRMFB::import() {
     }
 
     id = submitBuffer();
+
     if (!id) {
         backend->backend->log(AQ_LOG_ERROR, "drm: Failed to submit a buffer to KMS");
         buffer->attachments.add(makeShared<CDRMBufferUnimportable>());
@@ -1840,7 +1859,6 @@ void Aquamarine::CDRMFB::import() {
 
     TRACE(backend->backend->log(AQ_LOG_TRACE, std::format("drm: new buffer {}", id)));
 
-    // FIXME: why does this implode when it doesnt on wlroots or kwin?
     closeHandles();
 
     listeners.destroyBuffer = buffer->events.destroy.registerListener([this](std::any d) {
@@ -1915,8 +1933,12 @@ void Aquamarine::CDRMFB::drop() {
 }
 
 uint32_t Aquamarine::CDRMFB::submitBuffer() {
-    auto                    attrs = buffer->dmabuf();
     uint32_t newID = 0;
+
+    if (!buffer->dmabuf().success)
+        return 0;
+
+    auto                    attrs = buffer->dmabuf();
     std::array<uint64_t, 4> mods  = {0, 0, 0, 0};
     for (size_t i = 0; i < attrs.planes; ++i) {
         mods[i] = attrs.modifier;