wayland/core: add support for hw cursors

include/aquamarine/allocator/Swapchain.hpp

@@ -18,6 +18,7 @@ namespace Aquamarine {
 
         bool                                       contains(Hyprutils::Memory::CSharedPointer<IBuffer> buffer);
         Hyprutils::Memory::CSharedPointer<IBuffer> next(int* age);
+        const SSwapchainOptions&                   currentOptions();
 
       private:
         bool fullReconfigure(const SSwapchainOptions& options_);

include/aquamarine/backend/Backend.hpp

@@ -7,6 +7,7 @@
 #include <mutex>
 #include <condition_variable>
 #include "../allocator/Allocator.hpp"
+#include "Misc.hpp"
 
 namespace Aquamarine {
     enum eBackendType {
@@ -66,6 +67,8 @@ namespace Aquamarine {
         virtual bool                    dispatchEvents()   = 0;
         virtual uint32_t                capabilities()     = 0;
         virtual void                    onReady()          = 0;
+        virtual std::vector<SDRMFormat> getRenderFormats() = 0;
+        virtual std::vector<SDRMFormat> getCursorFormats() = 0;
     };
 
     class CBackend {

include/aquamarine/backend/Wayland.hpp

@@ -15,6 +15,7 @@ namespace Aquamarine {
     class CBackend;
     class CWaylandBackend;
     class CWaylandOutput;
+    class CWaylandPointer;
 
     typedef std::function<void(void)> FIdleCallback;
 
@@ -46,6 +47,7 @@ namespace Aquamarine {
         virtual bool                                                      setCursor(Hyprutils::Memory::CSharedPointer<IBuffer> buffer, const Hyprutils::Math::Vector2D& hotspot);
         virtual void                                                      moveCursor(const Hyprutils::Math::Vector2D& coord);
         virtual void                                                      scheduleFrame();
+        virtual Hyprutils::Math::Vector2D                                 maxCursorSize();
 
         Hyprutils::Memory::CWeakPointer<CWaylandOutput>                   self;
 
@@ -58,6 +60,7 @@ namespace Aquamarine {
 
         void                                              sendFrameAndSetCallback();
         void                                              onFrameDone();
+        void                                              onEnter(Hyprutils::Memory::CSharedPointer<CCWlPointer> pointer, uint32_t serial);
 
         // frame loop
         bool frameScheduledWhileWaiting = false;
@@ -68,6 +71,14 @@ namespace Aquamarine {
             std::vector<std::pair<Hyprutils::Memory::CWeakPointer<IBuffer>, Hyprutils::Memory::CSharedPointer<CWaylandBuffer>>> buffers;
         } backendState;
 
+        struct {
+            Hyprutils::Memory::CSharedPointer<IBuffer>     cursorBuffer;
+            Hyprutils::Memory::CSharedPointer<CCWlSurface> cursorSurface;
+            Hyprutils::Memory::CSharedPointer<CCWlBuffer>  cursorWlBuffer;
+            uint32_t                                       serial = 0;
+            Hyprutils::Math::Vector2D                      hotspot;
+        } cursorState;
+
         struct {
             Hyprutils::Memory::CSharedPointer<CCWlSurface>   surface;
             Hyprutils::Memory::CSharedPointer<CCXdgSurface>  xdgSurface;
@@ -118,6 +129,8 @@ namespace Aquamarine {
         virtual uint32_t                                 capabilities();
         virtual bool                                     setCursor(Hyprutils::Memory::CSharedPointer<IBuffer> buffer, const Hyprutils::Math::Vector2D& hotspot);
         virtual void                                     onReady();
+        virtual std::vector<SDRMFormat>                  getRenderFormats();
+        virtual std::vector<SDRMFormat>                  getCursorFormats();
 
         Hyprutils::Memory::CWeakPointer<CWaylandBackend> self;
 
@@ -140,12 +153,16 @@ namespace Aquamarine {
         Hyprutils::Memory::CWeakPointer<CWaylandOutput> focusedOutput;
         uint32_t                                        lastEnterSerial = 0;
 
+        // dmabuf formats
+        std::vector<SDRMFormat> dmabufFormats;
+
         struct {
             wl_display* display = nullptr;
 
             // hw-s types
             Hyprutils::Memory::CSharedPointer<CCWlRegistry>               registry;
             Hyprutils::Memory::CSharedPointer<CCWlSeat>                   seat;
+            Hyprutils::Memory::CSharedPointer<CCWlShm>                    shm;
             Hyprutils::Memory::CSharedPointer<CCXdgWmBase>                xdg;
             Hyprutils::Memory::CSharedPointer<CCWlCompositor>             compositor;
             Hyprutils::Memory::CSharedPointer<CCZwpLinuxDmabufV1>         dmabuf;

include/aquamarine/output/Output.hpp

@@ -62,6 +62,7 @@ namespace Aquamarine {
         virtual Hyprutils::Memory::CSharedPointer<SOutputMode>            preferredMode();
         virtual bool                                                      setCursor(Hyprutils::Memory::CSharedPointer<IBuffer> buffer, const Hyprutils::Math::Vector2D& hotspot);
         virtual void                                                      moveCursor(const Hyprutils::Math::Vector2D& coord); // includes the hotspot
+        virtual Hyprutils::Math::Vector2D                                 maxCursorSize();                                    // -1, -1 means no limit, 0, 0 means error
         virtual void                                                      scheduleFrame();
 
         std::string                                                       name, description, make, model, serial;

src/allocator/Swapchain.cpp

@@ -91,3 +91,7 @@ bool Aquamarine::CSwapchain::resize(size_t newSize) {
 bool Aquamarine::CSwapchain::contains(Hyprutils::Memory::CSharedPointer<IBuffer> buffer) {
     return std::find(buffers.begin(), buffers.end(), buffer) != buffers.end();
 }
+
+const SSwapchainOptions& Aquamarine::CSwapchain::currentOptions() {
+    return options;
+}

src/backend/Wayland.cpp

@@ -2,16 +2,62 @@
 #include <wayland.hpp>
 #include <xdg-shell.hpp>
 #include "Shared.hpp"
+#include "FormatUtils.hpp"
 #include <string.h>
 #include <xf86drm.h>
 #include <gbm.h>
 #include <fcntl.h>
+#include <sys/mman.h>
 
 using namespace Aquamarine;
 using namespace Hyprutils::Memory;
 using namespace Hyprutils::Math;
 #define SP CSharedPointer
 
+static std::pair<int, std::string> openExclusiveShm() {
+    // Only absolute paths can be shared across different shm_open() calls
+    srand(time(nullptr));
+    std::string name = std::format("/aq{:x}", rand() % RAND_MAX);
+
+    for (size_t i = 0; i < 69; ++i) {
+        int fd = shm_open(name.c_str(), O_RDWR | O_CREAT | O_EXCL, 0600);
+        if (fd >= 0)
+            return {fd, name};
+    }
+
+    return {-1, ""};
+}
+
+static int allocateSHMFile(size_t len) {
+    auto [fd, name] = openExclusiveShm();
+    if (fd < 0)
+        return -1;
+
+    shm_unlink(name.c_str());
+
+    int ret;
+    do {
+        ret = ftruncate(fd, len);
+    } while (ret < 0 && errno == EINTR);
+
+    if (ret < 0) {
+        close(fd);
+        return -1;
+    }
+
+    return fd;
+}
+
+wl_shm_format shmFormatFromDRM(uint32_t drmFormat) {
+    switch (drmFormat) {
+        case DRM_FORMAT_XRGB8888: return WL_SHM_FORMAT_XRGB8888;
+        case DRM_FORMAT_ARGB8888: return WL_SHM_FORMAT_ARGB8888;
+        default: return (wl_shm_format)drmFormat;
+    }
+
+    return (wl_shm_format)drmFormat;
+}
+
 Aquamarine::CWaylandBackend::~CWaylandBackend() {
     if (drmState.fd >= 0)
         close(drmState.fd);
@@ -55,6 +101,9 @@ bool Aquamarine::CWaylandBackend::start() {
         } else if (NAME == "wl_compositor") {
             backend->log(AQ_LOG_TRACE, std::format("  > binding to global: {} (version {}) with id {}", name, 6, id));
             waylandState.compositor = makeShared<CCWlCompositor>((wl_proxy*)wl_registry_bind((wl_registry*)waylandState.registry->resource(), id, &wl_compositor_interface, 6));
+        } else if (NAME == "wl_shm") {
+            backend->log(AQ_LOG_TRACE, std::format("  > binding to global: {} (version {}) with id {}", name, 1, id));
+            waylandState.shm = makeShared<CCWlShm>((wl_proxy*)wl_registry_bind((wl_registry*)waylandState.registry->resource(), id, &wl_shm_interface, 1));
         } else if (NAME == "zwp_linux_dmabuf_v1") {
             backend->log(AQ_LOG_TRACE, std::format("  > binding to global: {} (version {}) with id {}", name, 5, id));
             waylandState.dmabuf =
@@ -69,7 +118,7 @@ bool Aquamarine::CWaylandBackend::start() {
 
     wl_display_roundtrip(waylandState.display);
 
-    if (!waylandState.xdg || !waylandState.compositor || !waylandState.seat || !waylandState.dmabuf || waylandState.dmabufFailed) {
+    if (!waylandState.xdg || !waylandState.compositor || !waylandState.seat || !waylandState.dmabuf || waylandState.dmabufFailed || !waylandState.shm) {
         backend->log(AQ_LOG_ERROR, "Wayland backend cannot start: Missing protocols");
         return false;
     }
@@ -208,10 +257,20 @@ Aquamarine::CWaylandPointer::CWaylandPointer(SP<CCWlPointer> pointer_, Hyprutils
 
             backend->focusedOutput = o;
             backend->backend->log(AQ_LOG_DEBUG, std::format("[wayland] focus changed: {}", o->name));
+            o->onEnter(pointer, serial);
             break;
         }
     });
 
+    pointer->setLeave([this](CCWlPointer* r, uint32_t serial, wl_proxy* surface) {
+        for (auto& o : backend->outputs) {
+            if (o->waylandState.surface->resource() != surface)
+                continue;
+
+            o->cursorState.serial = 0;
+        }
+    });
+
     pointer->setButton([this](CCWlPointer* r, uint32_t serial, uint32_t timeMs, uint32_t button, wl_pointer_button_state state) {
         events.button.emit(SButtonEvent{
             .timeMs  = timeMs,
@@ -310,7 +369,42 @@ bool Aquamarine::CWaylandBackend::initDmabuf() {
         backend->log(AQ_LOG_DEBUG, std::format("zwp_linux_dmabuf_v1: Got node {}", drmState.nodeName));
     });
 
-    // TODO: format table and tranche
+    waylandState.dmabufFeedback->setFormatTable([this](CCZwpLinuxDmabufFeedbackV1* r, int32_t fd, uint32_t size) {
+#pragma pack(push, 1)
+        struct wlDrmFormatMarshalled {
+            uint32_t drmFormat;
+            char     pad[4];
+            uint64_t modifier;
+        };
+#pragma pack(pop)
+        static_assert(sizeof(wlDrmFormatMarshalled) == 16);
+
+        auto formatTable = mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, 0);
+        if (formatTable == MAP_FAILED) {
+            backend->log(AQ_LOG_ERROR, std::format("zwp_linux_dmabuf_v1: Failed to mmap the format table"));
+            return;
+        }
+
+        const auto FORMATS = (wlDrmFormatMarshalled*)formatTable;
+
+        for (size_t i = 0; i < size / 16; ++i) {
+            auto& fmt = FORMATS[i];
+
+            auto  modName = drmGetFormatModifierName(fmt.modifier);
+            backend->log(AQ_LOG_DEBUG, std::format("zwp_linux_dmabuf_v1: Got format {} with modifier {}", fourccToName(fmt.drmFormat), modName ? modName : "UNKNOWN"));
+            free(modName);
+
+            auto it = std::find_if(dmabufFormats.begin(), dmabufFormats.end(), [&fmt](const auto& e) { return e.drmFormat == fmt.drmFormat; });
+            if (it == dmabufFormats.end()) {
+                dmabufFormats.emplace_back(SDRMFormat{.drmFormat = fmt.drmFormat, .modifiers = {fmt.modifier}});
+                continue;
+            }
+
+            it->modifiers.emplace_back(fmt.modifier);
+        }
+
+        munmap(formatTable, size);
+    });
 
     wl_display_roundtrip(waylandState.display);
 
@@ -327,6 +421,14 @@ bool Aquamarine::CWaylandBackend::initDmabuf() {
     return true;
 }
 
+std::vector<SDRMFormat> Aquamarine::CWaylandBackend::getRenderFormats() {
+    return dmabufFormats;
+}
+
+std::vector<SDRMFormat> Aquamarine::CWaylandBackend::getCursorFormats() {
+    return dmabufFormats;
+}
+
 Aquamarine::CWaylandOutput::CWaylandOutput(const std::string& name_, Hyprutils::Memory::CWeakPointer<CWaylandBackend> backend_) : backend(backend_) {
     name = name_;
 
@@ -496,6 +598,84 @@ void Aquamarine::CWaylandOutput::onFrameDone() {
 }
 
 bool Aquamarine::CWaylandOutput::setCursor(Hyprutils::Memory::CSharedPointer<IBuffer> buffer, const Hyprutils::Math::Vector2D& hotspot) {
+    if (!cursorState.cursorSurface)
+        cursorState.cursorSurface = makeShared<CCWlSurface>(backend->waylandState.compositor->sendCreateSurface());
+
+    if (!cursorState.cursorSurface) {
+        backend->backend->log(AQ_LOG_ERROR, std::format("Output {}: Failed to create a wl_surface for the cursor", name));
+        return false;
+    }
+
+    if (!buffer) {
+        cursorState.cursorBuffer.reset();
+        cursorState.cursorWlBuffer.reset();
+        backend->pointers.at(0)->pointer->sendSetCursor(cursorState.serial, nullptr, cursorState.hotspot.x, cursorState.hotspot.y);
+        return true;
+    }
+
+    cursorState.cursorBuffer = buffer;
+    cursorState.hotspot      = hotspot;
+
+    if (buffer->shm().success) {
+        auto attrs                    = buffer->shm();
+        auto [pixelData, fmt, bufLen] = buffer->beginDataPtr(0);
+
+        int fd = allocateSHMFile(bufLen);
+        if (fd < 0) {
+            backend->backend->log(AQ_LOG_ERROR, std::format("Output {}: Failed to allocate a shm file", name));
+            return false;
+        }
+
+        void* data = mmap(nullptr, bufLen, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+        if (data == MAP_FAILED) {
+            backend->backend->log(AQ_LOG_ERROR, std::format("Output {}: Failed to mmap the cursor pixel data", name));
+            close(fd);
+            return false;
+        }
+
+        memcpy(data, pixelData, bufLen);
+        munmap(data, bufLen);
+
+        auto pool = makeShared<CCWlShmPool>(backend->waylandState.shm->sendCreatePool(fd, bufLen));
+        if (!pool) {
+            backend->backend->log(AQ_LOG_ERROR, std::format("Output {}: Failed to submit a wl_shm pool", name));
+            close(fd);
+            return false;
+        }
+
+        cursorState.cursorWlBuffer = makeShared<CCWlBuffer>(pool->sendCreateBuffer(0, attrs.size.x, attrs.size.y, attrs.stride, shmFormatFromDRM(attrs.format)));
+
+        pool.reset();
+
+        close(fd);
+    } else if (auto attrs = buffer->dmabuf(); attrs.success) {
+        auto params = makeShared<CCZwpLinuxBufferParamsV1>(backend->waylandState.dmabuf->sendCreateParams());
+
+        for (int i = 0; i < attrs.planes; ++i) {
+            params->sendAdd(attrs.fds.at(i), i, attrs.offsets.at(i), attrs.strides.at(i), attrs.modifier >> 32, attrs.modifier & 0xFFFFFFFF);
+        }
+
+        cursorState.cursorWlBuffer = makeShared<CCWlBuffer>(params->sendCreateImmed(attrs.size.x, attrs.size.y, attrs.format, (zwpLinuxBufferParamsV1Flags)0));
+    } else {
+        backend->backend->log(AQ_LOG_ERROR, std::format("Output {}: Failed to create a buffer for cursor: No known attrs (tried dmabuf / shm)", name));
+        return false;
+    }
+
+    if (!cursorState.cursorWlBuffer) {
+        backend->backend->log(AQ_LOG_ERROR, std::format("Output {}: Failed to create a buffer for cursor", name));
+        return false;
+    }
+
+    cursorState.cursorSurface->sendSetBufferScale(1);
+    cursorState.cursorSurface->sendSetBufferTransform(WL_OUTPUT_TRANSFORM_NORMAL);
+    cursorState.cursorSurface->sendAttach(cursorState.cursorWlBuffer.get(), 0, 0);
+    cursorState.cursorSurface->sendDamage(0, 0, INT32_MAX, INT32_MAX);
+    cursorState.cursorSurface->sendCommit();
+
+    // this may fail if we are not in focus
+    if (!backend->pointers.empty() && cursorState.serial)
+        backend->pointers.at(0)->pointer->sendSetCursor(cursorState.serial, cursorState.cursorSurface.get(), cursorState.hotspot.x, cursorState.hotspot.y);
+
     return true;
 }
 
@@ -503,6 +683,19 @@ void Aquamarine::CWaylandOutput::moveCursor(const Hyprutils::Math::Vector2D& coo
     return;
 }
 
+void Aquamarine::CWaylandOutput::onEnter(SP<CCWlPointer> pointer, uint32_t serial) {
+    cursorState.serial = serial;
+
+    if (!cursorState.cursorSurface)
+        return;
+
+    pointer->sendSetCursor(serial, cursorState.cursorSurface.get(), cursorState.hotspot.x, cursorState.hotspot.y);
+}
+
+Hyprutils::Math::Vector2D Aquamarine::CWaylandOutput::maxCursorSize() {
+    return {-1, -1}; // no limit
+}
+
 void Aquamarine::CWaylandOutput::scheduleFrame() {
     if (frameScheduled)
         return;

src/output/Output.cpp

@@ -22,3 +22,7 @@ bool Aquamarine::IOutput::setCursor(Hyprutils::Memory::CSharedPointer<IBuffer> b
 void Aquamarine::IOutput::scheduleFrame() {
     ;
 }
+
+Hyprutils::Math::Vector2D Aquamarine::IOutput::maxCursorSize() {
+    return {}; // error
+}