drm: split allocators per drm device

include/aquamarine/allocator/Allocator.hpp

@@ -11,13 +11,14 @@ namespace Aquamarine {
     struct SAllocatorBufferParams {
         Hyprutils::Math::Vector2D size;
         uint32_t                  format  = DRM_FORMAT_INVALID;
-        bool                      scanout = false, cursor = false;
+        bool                      scanout = false, cursor = false, multigpu = false;
     };
 
     class IAllocator {
       public:
-        virtual ~IAllocator() = default;
+        virtual ~IAllocator()                                                                                                                                      = default;
         virtual Hyprutils::Memory::CSharedPointer<IBuffer>  acquire(const SAllocatorBufferParams& params, Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain) = 0;
         virtual Hyprutils::Memory::CSharedPointer<CBackend> getBackend()                                                                                           = 0;
+        virtual int                                         drmFD()                                                                                                = 0;
     };
 };

include/aquamarine/allocator/GBM.hpp

@@ -40,6 +40,7 @@ namespace Aquamarine {
 
         virtual Hyprutils::Memory::CSharedPointer<IBuffer>      acquire(const SAllocatorBufferParams& params, Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain_);
         virtual Hyprutils::Memory::CSharedPointer<CBackend>     getBackend();
+        virtual int                                             drmFD();
 
         //
         Hyprutils::Memory::CWeakPointer<CGBMAllocator> self;

include/aquamarine/allocator/Swapchain.hpp

@@ -10,7 +10,7 @@ namespace Aquamarine {
         size_t                    length = 0;
         Hyprutils::Math::Vector2D size;
         uint32_t                  format  = DRM_FORMAT_INVALID; // if you leave this on invalid, the swapchain will choose an appropriate format (and modifier) for you.
-        bool                      scanout = false, cursor = false /* requires scanout = true */;
+        bool                      scanout = false, cursor = false /* requires scanout = true */, multigpu = false /* if true, will force linear */;
     };
 
     class CSwapchain {
@@ -23,6 +23,7 @@ namespace Aquamarine {
         bool                                                 contains(Hyprutils::Memory::CSharedPointer<IBuffer> buffer);
         Hyprutils::Memory::CSharedPointer<IBuffer>           next(int* age);
         const SSwapchainOptions&                             currentOptions();
+        Hyprutils::Memory::CSharedPointer<IAllocator>        getAllocator();
 
         // rolls the buffers back, marking the last consumed as the next valid.
         // useful if e.g. a commit fails and we don't wanna write to the previous buffer that is

include/aquamarine/backend/Backend.hpp

@@ -76,6 +76,7 @@ namespace Aquamarine {
         virtual std::vector<SDRMFormat>                                 getRenderFormats()                         = 0;
         virtual std::vector<SDRMFormat>                                 getCursorFormats()                         = 0;
         virtual bool                                                    createOutput(const std::string& name = "") = 0; // "" means auto
+        virtual Hyprutils::Memory::CSharedPointer<IAllocator>           preferredAllocator()                       = 0;
     };
 
     class CBackend {
@@ -122,9 +123,10 @@ namespace Aquamarine {
             Hyprutils::Signal::CSignal newTabletPad;
         } events;
 
-        Hyprutils::Memory::CSharedPointer<IAllocator> allocator;
+        Hyprutils::Memory::CSharedPointer<IAllocator>              primaryAllocator;
-        bool                                          ready = false;
+        std::vector<Hyprutils::Memory::CSharedPointer<IAllocator>> allocators;
-        Hyprutils::Memory::CSharedPointer<CSession>   session;
+        bool                                                       ready = false;
+        Hyprutils::Memory::CSharedPointer<CSession>                session;
 
       private:
         CBackend();

include/aquamarine/backend/DRM.hpp

@@ -331,6 +331,7 @@ namespace Aquamarine {
         virtual std::vector<SDRMFormat>                                 getRenderFormats();
         virtual std::vector<SDRMFormat>                                 getCursorFormats();
         virtual bool                                                    createOutput(const std::string& name = "");
+        virtual Hyprutils::Memory::CSharedPointer<IAllocator>           preferredAllocator();
 
         Hyprutils::Memory::CWeakPointer<CDRMBackend>                    self;
 
@@ -340,6 +341,7 @@ namespace Aquamarine {
 
         std::vector<FIdleCallback>                                      idleCallbacks;
         std::string                                                     gpuName;
+        Hyprutils::Memory::CWeakPointer<IAllocator>                     allocator;
 
       private:
         CDRMBackend(Hyprutils::Memory::CSharedPointer<CBackend> backend);

include/aquamarine/backend/Headless.hpp

@@ -45,6 +45,7 @@ namespace Aquamarine {
         virtual std::vector<SDRMFormat>                                 getRenderFormats();
         virtual std::vector<SDRMFormat>                                 getCursorFormats();
         virtual bool                                                    createOutput(const std::string& name = "");
+        virtual Hyprutils::Memory::CSharedPointer<IAllocator>           preferredAllocator();
 
         Hyprutils::Memory::CWeakPointer<CHeadlessBackend>               self;
 

include/aquamarine/backend/Wayland.hpp

@@ -133,6 +133,7 @@ namespace Aquamarine {
         virtual std::vector<SDRMFormat>                                 getRenderFormats();
         virtual std::vector<SDRMFormat>                                 getCursorFormats();
         virtual bool                                                    createOutput(const std::string& name = "");
+        virtual Hyprutils::Memory::CSharedPointer<IAllocator>           preferredAllocator();
 
         Hyprutils::Memory::CWeakPointer<CWaylandBackend>                self;
 

include/aquamarine/output/Output.hpp

@@ -137,7 +137,11 @@ namespace Aquamarine {
         //
         std::vector<Hyprutils::Memory::CSharedPointer<SOutputMode>> modes;
         Hyprutils::Memory::CSharedPointer<COutputState>             state = Hyprutils::Memory::makeShared<COutputState>();
-        Hyprutils::Memory::CSharedPointer<CSwapchain>               swapchain;
+
+        // please note that for multigpu setups, this swapchain resides on the target gpu,
+        // so it's recommended that if this swapchain's allocator FD doesn't match the primary
+        // drmFD used, you should render to a buffer on the main gpu and only perform the final copy to this swapchain.
+        Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain;
 
         //
 

src/allocator/GBM.cpp

@@ -53,8 +53,7 @@ static SDRMFormat guessFormatFrom(std::vector<SDRMFormat> formats, bool cursor)
 }
 
 Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hyprutils::Memory::CWeakPointer<CGBMAllocator> allocator_,
-                                   Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain) :
+                                   Hyprutils::Memory::CSharedPointer<CSwapchain> swapchain) : allocator(allocator_) {
-    allocator(allocator_) {
     if (!allocator)
         return;
 
@@ -62,7 +61,8 @@ Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hypruti
     attrs.format = params.format;
     size         = attrs.size;
 
-    const bool            CURSOR = params.cursor && params.scanout;
+    const bool            CURSOR   = params.cursor && params.scanout;
+    const bool            MULTIGPU = params.multigpu && params.scanout;
 
     const auto            FORMATS = CURSOR ? swapchain->backendImpl->getCursorFormats() : swapchain->backendImpl->getRenderFormats();
 
@@ -93,9 +93,12 @@ Aquamarine::CGBMBuffer::CGBMBuffer(const SAllocatorBufferParams& params, Hypruti
         }
     }
 
+    // FIXME: Nvidia cannot render to linear buffers. What do?
+    if (MULTIGPU)
+        explicitModifiers = {DRM_FORMAT_MOD_LINEAR};
+
     if (explicitModifiers.empty()) {
         // fall back to using a linear buffer.
-        // FIXME: Nvidia cannot render to linear buffers.
         explicitModifiers.push_back(DRM_FORMAT_MOD_LINEAR);
     }
 
@@ -245,3 +248,7 @@ SP<IBuffer> Aquamarine::CGBMAllocator::acquire(const SAllocatorBufferParams& par
 Hyprutils::Memory::CSharedPointer<CBackend> Aquamarine::CGBMAllocator::getBackend() {
     return backend.lock();
 }
+
+int Aquamarine::CGBMAllocator::drmFD() {
+    return fd;
+}

src/allocator/Swapchain.cpp

@@ -72,8 +72,9 @@ SP<IBuffer> Aquamarine::CSwapchain::next(int* age) {
 bool Aquamarine::CSwapchain::fullReconfigure(const SSwapchainOptions& options_) {
     buffers.clear();
     for (size_t i = 0; i < options_.length; ++i) {
-        auto buf =
+        auto buf = allocator->acquire(
-            allocator->acquire(SAllocatorBufferParams{.size = options_.size, .format = options_.format, .scanout = options_.scanout, .cursor = options_.cursor}, self.lock());
+            SAllocatorBufferParams{.size = options_.size, .format = options_.format, .scanout = options_.scanout, .cursor = options_.cursor, .multigpu = options_.multigpu},
+            self.lock());
         if (!buf) {
             allocator->getBackend()->log(AQ_LOG_ERROR, "Swapchain: Failed acquiring a buffer");
             return false;
@@ -107,7 +108,7 @@ bool Aquamarine::CSwapchain::resize(size_t newSize) {
     return true;
 }
 
-bool Aquamarine::CSwapchain::contains(Hyprutils::Memory::CSharedPointer<IBuffer> buffer) {
+bool Aquamarine::CSwapchain::contains(SP<IBuffer> buffer) {
     return std::find(buffers.begin(), buffers.end(), buffer) != buffers.end();
 }
 
@@ -120,3 +121,7 @@ void Aquamarine::CSwapchain::rollback() {
     if (lastAcquired < 0)
         lastAcquired = options.length - 1;
 }
+
+SP<IAllocator> Aquamarine::CSwapchain::getAllocator() {
+    return allocator;
+}

src/backend/Backend.cpp

@@ -144,12 +144,13 @@ bool Aquamarine::CBackend::start() {
     // TODO: obviously change this when (if) we add different allocators.
     for (auto& b : implementations) {
         if (b->drmFD() >= 0) {
-            allocator = CGBMAllocator::create(b->drmFD(), self);
+            allocators.emplace_back(CGBMAllocator::create(b->drmFD(), self));
-            break;
+            if (!primaryAllocator)
+                primaryAllocator = allocators.front();
         }
     }
 
-    if (!allocator)
+    if (allocators.empty() || !primaryAllocator)
         return false;
 
     ready = true;

src/backend/Headless.cpp

@@ -125,7 +125,7 @@ std::vector<SDRMFormat> Aquamarine::CHeadlessBackend::getCursorFormats() {
 bool Aquamarine::CHeadlessBackend::createOutput(const std::string& name) {
     auto output = SP<CHeadlessOutput>(new CHeadlessOutput(name.empty() ? std::format("HEADLESS-{}", ++outputIDCounter) : name, self.lock()));
     outputs.emplace_back(output);
-    output->swapchain = CSwapchain::create(backend->allocator, self.lock());
+    output->swapchain = CSwapchain::create(backend->primaryAllocator, self.lock());
     output->self      = output;
     backend->events.newOutput.emit(SP<IOutput>(output));
 
@@ -176,6 +176,10 @@ void Aquamarine::CHeadlessBackend::updateTimerFD() {
         backend->log(AQ_LOG_ERROR, std::format("headless: failed to arm timerfd: {}", strerror(errno)));
 }
 
+SP<IAllocator> Aquamarine::CHeadlessBackend::preferredAllocator() {
+    return backend->primaryAllocator;
+}
+
 bool Aquamarine::CHeadlessBackend::CTimer::expired() {
     return std::chrono::steady_clock::now() > when;
 }

src/backend/Wayland.cpp

@@ -139,7 +139,7 @@ bool Aquamarine::CWaylandBackend::createOutput(const std::string& szName) {
     auto o  = outputs.emplace_back(SP<CWaylandOutput>(new CWaylandOutput(szName.empty() ? std::format("WAYLAND-{}", ++lastOutputID) : szName, self)));
     o->self = o;
     if (backend->ready)
-        o->swapchain = CSwapchain::create(backend->allocator, self.lock());
+        o->swapchain = CSwapchain::create(backend->primaryAllocator, self.lock());
     idleCallbacks.emplace_back([this, o]() { backend->events.newOutput.emit(SP<IOutput>(o)); });
     return true;
 }
@@ -188,7 +188,7 @@ bool Aquamarine::CWaylandBackend::setCursor(Hyprutils::Memory::CSharedPointer<IB
 
 void Aquamarine::CWaylandBackend::onReady() {
     for (auto& o : outputs) {
-        o->swapchain = CSwapchain::create(backend->allocator, self.lock());
+        o->swapchain = CSwapchain::create(backend->primaryAllocator, self.lock());
         if (!o->swapchain) {
             backend->log(AQ_LOG_ERROR, std::format("Output {} failed: swapchain creation failed", o->name));
             continue;
@@ -435,6 +435,10 @@ std::vector<SDRMFormat> Aquamarine::CWaylandBackend::getCursorFormats() {
     return dmabufFormats;
 }
 
+SP<IAllocator> Aquamarine::CWaylandBackend::preferredAllocator() {
+    return backend->primaryAllocator;
+}
+
 Aquamarine::CWaylandOutput::CWaylandOutput(const std::string& name_, Hyprutils::Memory::CWeakPointer<CWaylandBackend> backend_) : backend(backend_) {
     name = name_;
 

src/backend/drm/DRM.cpp

@@ -293,9 +293,6 @@ void Aquamarine::CDRMBackend::restoreAfterVT() {
             if (!drmFB)
                 backend->log(AQ_LOG_ERROR, "drm: Buffer failed to import to KMS");
 
-            if (!isNew && primary && drmFB)
-                drmFB->reimport();
-
             data.mainFB = drmFB;
         }
 
@@ -747,9 +744,23 @@ void Aquamarine::CDRMBackend::onReady() {
 
         backend->log(AQ_LOG_DEBUG, std::format("drm: onReady: connector {} has output name {}", c->id, c->output->name));
 
+        // find our allocator, for multigpu setups there will be 2
+        for (auto& alloc : backend->allocators) {
+            if (alloc->drmFD() != gpu->fd)
+                continue;
+
+            allocator = alloc;
+            break;
+        }
+
+        if (!allocator) {
+            backend->log(AQ_LOG_ERROR, std::format("drm: backend for gpu {} doesn't have an allocator?!", gpu->path));
+            return;
+        }
+
         // swapchain has to be created here because allocator is absent in connect if not ready
-        c->output->swapchain = CSwapchain::create(backend->allocator, self.lock());
+        c->output->swapchain = CSwapchain::create(allocator.lock(), self.lock());
-        c->output->swapchain->reconfigure(SSwapchainOptions{.length = 0, .scanout = true}); // mark the swapchain for scanout
+        c->output->swapchain->reconfigure(SSwapchainOptions{.length = 0, .scanout = true, .multigpu = !!primary}); // mark the swapchain for scanout
         c->output->needsFrame = true;
 
         backend->events.newOutput.emit(SP<IOutput>(c->output));
@@ -824,6 +835,10 @@ int Aquamarine::CDRMBackend::getNonMasterFD() {
     return fd;
 }
 
+SP<IAllocator> Aquamarine::CDRMBackend::preferredAllocator() {
+    return allocator.lock();
+}
+
 bool Aquamarine::SDRMPlane::init(drmModePlane* plane) {
     id = plane->plane_id;
 
@@ -1141,7 +1156,7 @@ void Aquamarine::SDRMConnector::connect(drmModeConnector* connector) {
     if (!backend->backend->ready)
         return;
 
-    output->swapchain = CSwapchain::create(backend->backend->allocator, backend->self.lock());
+    output->swapchain = CSwapchain::create(backend->allocator.lock(), backend->self.lock());
     backend->backend->events.newOutput.emit(output);
     output->scheduleFrame(IOutput::AQ_SCHEDULE_NEW_CONNECTOR);
 }
@@ -1311,13 +1326,6 @@ bool Aquamarine::CDRMOutput::commitState(bool onlyTest) {
             return false;
         }
 
-        if (!isNew && backend->primary) {
-            // this is not a new buffer, and we are not on a primary GPU, which means
-            // this buffer lives on the primary. We need to re-import it to update
-            // the contents that have possibly (probably) changed
-            drmFB->reimport();
-        }
-
         data.mainFB = drmFB;
     }
 
@@ -1381,13 +1389,6 @@ bool Aquamarine::CDRMOutput::setCursor(SP<IBuffer> buffer, const Vector2D& hotsp
         connector->crtc->pendingCursor = fb;
 
         cursorVisible = true;
-
-        if (!isNew && backend->primary) {
-            // this is not a new buffer, and we are not on a primary GPU, which means
-            // this buffer lives on the primary. We need to re-import it to update
-            // the contents that have possibly (probably) changed
-            fb->reimport();
-        }
     }
 
     needsFrame = true;