Added direct scanout

Makefile

@@ -91,6 +91,16 @@ wlr-output-power-management-unstable-v1-protocol.c:
 
 wlr-output-power-management-unstable-v1-protocol.o: wlr-output-power-management-unstable-v1-protocol.h
 
+linux-dmabuf-unstable-v1-protocol.h:
+	$(WAYLAND_SCANNER) server-header \
+		$(WAYLAND_PROTOCOLS)/unstable/linux-dmabuf/linux-dmabuf-unstable-v1.xml $@
+
+linux-dmabuf-unstable-v1-protocol.c:
+	$(WAYLAND_SCANNER) private-code \
+		$(WAYLAND_PROTOCOLS)/unstable/linux-dmabuf/linux-dmabuf-unstable-v1.xml $@
+
+linux-dmabuf-unstable-v1-protocol.o: linux-dmabuf-unstable-v1-protocol.h
+
 legacyrenderer:
 	mkdir -p build && cmake --no-warn-unused-cli -DCMAKE_BUILD_TYPE:STRING=Release -DLEGACY_RENDERER:STRING=true -H./ -B./build -G Ninja
 	cmake --build ./build --config Release --target all -j$(shell nproc)
@@ -166,7 +176,7 @@ uninstall:
 	rm -f ${PREFIX}/share/man/man1/Hyprland.1
 	rm -f ${PREFIX}/share/man/man1/hyprctl.1
 
-protocols: xdg-shell-protocol.o wlr-layer-shell-unstable-v1-protocol.o wlr-screencopy-unstable-v1-protocol.o idle-protocol.o ext-workspace-unstable-v1-protocol.o pointer-constraints-unstable-v1-protocol.o tablet-unstable-v2-protocol.o wlr-output-power-management-unstable-v1-protocol.o
+protocols: xdg-shell-protocol.o wlr-layer-shell-unstable-v1-protocol.o wlr-screencopy-unstable-v1-protocol.o idle-protocol.o ext-workspace-unstable-v1-protocol.o pointer-constraints-unstable-v1-protocol.o tablet-unstable-v2-protocol.o wlr-output-power-management-unstable-v1-protocol.o linux-dmabuf-unstable-v1-protocol.o
 
 fixwlr:
 	sed -i -E 's/(soversion = 11)([^032]|$$)/soversion = 11032/g' subprojects/wlroots/meson.build

src/Compositor.cpp

@@ -73,7 +73,15 @@ CCompositor::CCompositor() {
         throw std::runtime_error("wlr_gles2_renderer_create_with_drm_fd() failed!");
     }
 
-    wlr_renderer_init_wl_display(m_sWLRRenderer, m_sWLDisplay);
+    wlr_renderer_init_wl_shm(m_sWLRRenderer, m_sWLDisplay);
+
+    if (wlr_renderer_get_dmabuf_texture_formats(m_sWLRRenderer)) {
+        if (wlr_renderer_get_drm_fd(m_sWLRRenderer) >= 0) {
+            wlr_drm_create(m_sWLDisplay, m_sWLRRenderer);
+        }
+
+        m_sWLRLinuxDMABuf = wlr_linux_dmabuf_v1_create(m_sWLDisplay, m_sWLRRenderer);
+    }
 
     m_sWLRAllocator = wlr_allocator_autocreate(m_sWLRBackend, m_sWLRRenderer);
 
@@ -1699,6 +1707,9 @@ void CCompositor::setWindowFullscreen(CWindow* pWindow, bool on, eFullscreenMode
     forceReportSizesToWindowsOnWorkspace(pWindow->m_iWorkspaceID);
 
     g_pInputManager->recheckIdleInhibitorStatus();
+
+    // DMAbuf stuff for direct scanout
+    g_pHyprRenderer->setWindowScanoutMode(pWindow);
 }
 
 void CCompositor::moveUnmanagedX11ToWindows(CWindow* pWindow) {

src/Compositor.hpp

@@ -70,6 +70,7 @@ public:
     wlr_input_method_manager_v2*     m_sWLRIMEMgr;
     wlr_text_input_manager_v3*       m_sWLRTextInputMgr;
     wlr_xdg_activation_v1*           m_sWLRActivation;
+    wlr_linux_dmabuf_v1*             m_sWLRLinuxDMABuf;
     // ------------------------------------------------- //
 
 

src/config/ConfigManager.cpp

@@ -44,6 +44,8 @@ void CConfigManager::setDefaultVars() {
     configValues["general:cursor_inactive_timeout"].intValue = 0;
     configValues["general:no_cursor_warps"].intValue = 0;
 
+    configValues["general:no_direct_scanout"].intValue = 0;
+
     configValues["general:layout"].strValue = "dwindle";
 
     configValues["misc:disable_hyprland_logo"].intValue = 0;

src/events/Monitors.cpp

@@ -109,6 +109,7 @@ void Events::listener_monitorFrame(void* owner, void* data) {
     static auto *const PDAMAGETRACKINGMODE = &g_pConfigManager->getConfigValuePtr("debug:damage_tracking")->intValue;
     static auto *const PDAMAGEBLINK = &g_pConfigManager->getConfigValuePtr("debug:damage_blink")->intValue;
     static auto *const PNOVFR = &g_pConfigManager->getConfigValuePtr("misc:no_vfr")->intValue;
+    static auto *const PNODIRECTSCANOUT = &g_pConfigManager->getConfigValuePtr("general:no_direct_scanout")->intValue;
 
     static int damageBlinkCleanup = 0; // because double-buffered
 
@@ -154,6 +155,16 @@ void Events::listener_monitorFrame(void* owner, void* data) {
         g_pLayoutManager->getCurrentLayout()->recalculateMonitor(PMONITOR->ID);
     }
 
+    // Direct scanout first
+    if (!*PNODIRECTSCANOUT) {
+        if (g_pHyprRenderer->attemptDirectScanout(PMONITOR)) {
+            return;
+        } else if (g_pHyprRenderer->m_pLastScanout) {
+            Debug::log(LOG, "Left a direct scanout.");
+            g_pHyprRenderer->m_pLastScanout = nullptr;
+        }
+    }
+
     timespec now;
     clock_gettime(CLOCK_MONOTONIC, &now);
 

src/render/Renderer.cpp

@@ -1,5 +1,6 @@
 #include "Renderer.hpp"
 #include "../Compositor.hpp"
+#include "linux-dmabuf-unstable-v1-protocol.h"
 
 void renderSurface(struct wlr_surface* surface, int x, int y, void* data) {
     const auto TEXTURE = wlr_surface_get_texture(surface);
@@ -533,6 +534,150 @@ void CHyprRenderer::calculateUVForWindowSurface(CWindow* pWindow, wlr_surface* p
     }
 }
 
+void countSubsurfacesIter(wlr_surface* pSurface, int x, int y, void* data) {
+    *(int*)data += 1;
+}
+
+bool CHyprRenderer::attemptDirectScanout(CMonitor* pMonitor) {
+    const auto PWORKSPACE = g_pCompositor->getWorkspaceByID(pMonitor->activeWorkspace);
+
+    if (!PWORKSPACE->m_bHasFullscreenWindow || g_pInputManager->m_sDrag.drag || g_pCompositor->m_sSeat.exclusiveClient)
+        return false;
+
+    const auto PCANDIDATE = g_pCompositor->getFullscreenWindowOnWorkspace(PWORKSPACE->m_iID);
+
+    if (!PCANDIDATE)
+        return false; // ????
+
+    if (PCANDIDATE->m_fAlpha.fl() != 255.f || PCANDIDATE->m_fActiveInactiveAlpha.fl() != 1.f || PWORKSPACE->m_fAlpha.fl() != 255.f)
+        return false;
+
+    if (PCANDIDATE->m_vRealSize.vec() != pMonitor->vecSize || PCANDIDATE->m_vRealPosition.vec() != pMonitor->vecPosition || PCANDIDATE->m_vRealPosition.isBeingAnimated() || PCANDIDATE->m_vRealSize.isBeingAnimated())
+        return false;
+
+    if (!pMonitor->m_aLayerSurfaceLists[ZWLR_LAYER_SHELL_V1_LAYER_OVERLAY].empty())
+        return false;
+
+    for (auto& topls : pMonitor->m_aLayerSurfaceLists[ZWLR_LAYER_SHELL_V1_LAYER_TOP]) {
+        if (topls->alpha.fl() != 0.f)
+            return false;
+    }
+
+    // check if it did not open any subsurfaces or shit
+    int surfaceCount = 0;
+    if (PCANDIDATE->m_bIsX11) {
+        surfaceCount = 1;
+
+        // check opaque
+        if (PCANDIDATE->m_uSurface.xwayland->has_alpha)
+            return false;
+    } else {
+        wlr_xdg_surface_for_each_surface(PCANDIDATE->m_uSurface.xdg, countSubsurfacesIter, &surfaceCount);
+        wlr_xdg_surface_for_each_popup_surface(PCANDIDATE->m_uSurface.xdg, countSubsurfacesIter, &surfaceCount);
+
+        if (!PCANDIDATE->m_uSurface.xdg->surface->opaque)
+            return false;
+    }
+
+    if (surfaceCount != 1)
+        return false;
+
+    const auto PSURFACE = g_pXWaylandManager->getWindowSurface(PCANDIDATE);
+
+    if (!PSURFACE || PSURFACE->current.scale != pMonitor->output->scale || PSURFACE->current.transform != pMonitor->output->transform)
+        return false;
+
+    // finally, we should be GTG.
+    wlr_output_attach_buffer(pMonitor->output, &PSURFACE->buffer->base);
+
+    if (!wlr_output_test(pMonitor->output)) {
+        Debug::log(ERR, "Direct scanout test failed for %x", PCANDIDATE);
+        return false;
+    }
+
+    wlr_presentation_surface_sampled_on_output(g_pCompositor->m_sWLRPresentation, PSURFACE, pMonitor->output);
+
+    if (wlr_output_commit(pMonitor->output)) {
+        if (!m_pLastScanout) {
+            m_pLastScanout = PCANDIDATE;
+            Debug::log(LOG, "Entered a direct scanout to %x: \"%s\"", PCANDIDATE, PCANDIDATE->m_szTitle.c_str());
+        }
+    } else {
+        Debug::log(ERR, "Direct scanout failed for %x: \"%s\"", PCANDIDATE, PCANDIDATE->m_szTitle.c_str());
+        m_pLastScanout = nullptr;
+        return false;
+    }
+
+    return true;
+}
+
+void CHyprRenderer::setWindowScanoutMode(CWindow* pWindow) {
+    if (!g_pCompositor->m_sWLRLinuxDMABuf)
+        return;
+
+    if (!pWindow->m_bIsFullscreen) {
+        wlr_linux_dmabuf_v1_set_surface_feedback(g_pCompositor->m_sWLRLinuxDMABuf, g_pXWaylandManager->getWindowSurface(pWindow), nullptr);
+        Debug::log(LOG, "Scanout mode OFF set for %x", pWindow);
+        return;
+    }
+
+    const auto RENDERERDRMFD = wlr_renderer_get_drm_fd(g_pCompositor->m_sWLRRenderer);
+    const auto BACKENDDRMFD = wlr_backend_get_drm_fd(g_pCompositor->m_sWLRBackend);
+
+    if (RENDERERDRMFD < 0 || BACKENDDRMFD < 0)
+        return;
+
+    auto deviceIDFromFD = [](int fd, unsigned long* deviceID) -> bool {
+        struct stat stat;
+        if (fstat(fd, &stat) != 0) {
+            return false;
+        }
+        *deviceID = stat.st_rdev;
+        return true;
+    };
+
+    unsigned long rendererDevice, scanoutDevice;
+    if (!deviceIDFromFD(RENDERERDRMFD, &rendererDevice) || !deviceIDFromFD(BACKENDDRMFD, &scanoutDevice))
+        return;
+
+    const auto PMONITOR = g_pCompositor->getMonitorFromID(pWindow->m_iMonitorID);
+
+    const auto POUTPUTFORMATS = wlr_output_get_primary_formats(PMONITOR->output, WLR_BUFFER_CAP_DMABUF);
+    if (!POUTPUTFORMATS)
+        return;
+
+    const auto PRENDERERFORMATS = wlr_renderer_get_dmabuf_texture_formats(g_pCompositor->m_sWLRRenderer);
+    wlr_drm_format_set scanoutFormats = { 0 };
+
+    if (!wlr_drm_format_set_intersect(&scanoutFormats, POUTPUTFORMATS, PRENDERERFORMATS))
+        return;
+
+    const wlr_linux_dmabuf_feedback_v1_tranche TRANCHES[] = {
+        {
+            .target_device = scanoutDevice,
+            .flags = ZWP_LINUX_DMABUF_FEEDBACK_V1_TRANCHE_FLAGS_SCANOUT,
+            .formats = &scanoutFormats
+        }, {
+            .target_device = rendererDevice,
+            .formats = PRENDERERFORMATS
+        }
+    };
+
+    const wlr_linux_dmabuf_feedback_v1 FEEDBACK = {
+        .main_device = rendererDevice,
+        .tranches_len = sizeof(TRANCHES) / sizeof(TRANCHES[0]),
+        .tranches = TRANCHES
+    };
+
+    if (!wlr_linux_dmabuf_v1_set_surface_feedback(g_pCompositor->m_sWLRLinuxDMABuf, g_pXWaylandManager->getWindowSurface(pWindow), &FEEDBACK)) {
+        Debug::log(ERR, "Error in scanout mode setting: wlr_linux_dmabuf_v1_set_surface_feedback returned false.");
+    }
+
+    wlr_drm_format_set_finish(&scanoutFormats);
+
+    Debug::log(LOG, "Scanout mode ON set for %x", pWindow);
+}
+
 void CHyprRenderer::outputMgrApplyTest(wlr_output_configuration_v1* config, bool test) {
     wlr_output_configuration_head_v1* head;
     bool noError = true;

src/render/Renderer.hpp

@@ -44,9 +44,13 @@ public:
     void                calculateUVForWindowSurface(CWindow*, wlr_surface*, bool main = false);
 
     bool                m_bWindowRequestedCursorHide = false;
+    CWindow*            m_pLastScanout = nullptr;
 
     DAMAGETRACKINGMODES damageTrackingModeFromStr(const std::string&);
 
+    bool                attemptDirectScanout(CMonitor*);
+    void                setWindowScanoutMode(CWindow*);
+
 private:
     void                arrangeLayerArray(CMonitor*, const std::vector<std::unique_ptr<SLayerSurface>>&, bool, wlr_box*);
     void                renderWorkspaceWithFullscreenWindow(CMonitor*, CWorkspace*, timespec*);