renderer: some fixes for renderModif

src/protocols/Screencopy.cpp

@@ -485,7 +485,9 @@ bool CScreencopyProtocolManager::copyFrameShm(SScreencopyFrame* frame, timespec*
 
     CBox monbox = CBox{0, 0, frame->pMonitor->vecTransformedSize.x, frame->pMonitor->vecTransformedSize.y}.translate({-frame->box.x, -frame->box.y});
     g_pHyprOpenGL->setMonitorTransformEnabled(true);
+    g_pHyprOpenGL->setRenderModifEnabled(false);
     g_pHyprOpenGL->renderTexture(sourceTex, &monbox, 1);
+    g_pHyprOpenGL->setRenderModifEnabled(true);
     g_pHyprOpenGL->setMonitorTransformEnabled(false);
 
 #ifndef GLES2
@@ -544,7 +546,9 @@ bool CScreencopyProtocolManager::copyFrameDmabuf(SScreencopyFrame* frame) {
                       .translate({-frame->box.x, -frame->box.y}) // vvvv kinda ass-backwards but that's how I designed the renderer... sigh.
                       .transform(wlr_output_transform_invert(frame->pMonitor->output->transform), frame->pMonitor->vecPixelSize.x, frame->pMonitor->vecPixelSize.y);
     g_pHyprOpenGL->setMonitorTransformEnabled(true);
+    g_pHyprOpenGL->setRenderModifEnabled(false);
     g_pHyprOpenGL->renderTexture(sourceTex, &monbox, 1);
+    g_pHyprOpenGL->setRenderModifEnabled(true);
     g_pHyprOpenGL->setMonitorTransformEnabled(false);
 
     g_pHyprRenderer->endRender();

src/render/OpenGL.cpp

@@ -795,6 +795,8 @@ void CHyprOpenGLImpl::renderRectWithDamage(CBox* box, const CColor& col, CRegion
     }
 
     glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shQUAD.posAttrib);
+
+    scissor((CBox*)nullptr);
 }
 
 void CHyprOpenGLImpl::renderTexture(wlr_texture* tex, CBox* pBox, float alpha, int round, bool allowCustomUV) {
@@ -1480,6 +1482,8 @@ void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, CBox* pBox, flo
     if (texDamage.empty())
         return;
 
+    m_RenderData.renderModif.applyToRegion(texDamage);
+
     if (*PBLURENABLED == 0 || (*PNOBLUROVERSIZED && m_RenderData.primarySurfaceUVTopLeft != Vector2D(-1, -1)) ||
         (m_pCurrentWindow && (m_pCurrentWindow->m_sAdditionalConfigData.forceNoBlur || m_pCurrentWindow->m_sAdditionalConfigData.forceRGBX))) {
         renderTexture(tex, pBox, a, round, false, true);
@@ -1509,7 +1513,9 @@ void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, CBox* pBox, flo
 
     CFramebuffer* POUTFB = nullptr;
     if (!USENEWOPTIMIZE) {
-        inverseOpaque.translate({pBox->x, pBox->y}).intersect(texDamage);
+        inverseOpaque.translate({pBox->x, pBox->y});
+        m_RenderData.renderModif.applyToRegion(inverseOpaque);
+        inverseOpaque.intersect(texDamage);
 
         POUTFB = blurMainFramebufferWithDamage(a, &inverseOpaque);
     } else {
@@ -1542,12 +1548,11 @@ void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, CBox* pBox, flo
     CBox MONITORBOX = {0, 0, m_RenderData.pMonitor->vecTransformedSize.x, m_RenderData.pMonitor->vecTransformedSize.y};
     // render our great blurred FB
     static auto PBLURIGNOREOPACITY = CConfigValue<Hyprlang::INT>("decoration:blur:ignore_opacity");
-    m_bEndFrame                    = true; // fix transformed
+    setMonitorTransformEnabled(true);
-    const auto SAVEDRENDERMODIF    = m_RenderData.renderModif;
+    setRenderModifEnabled(false);
-    m_RenderData.renderModif       = {}; // fix shit
     renderTextureInternalWithDamage(POUTFB->m_cTex, &MONITORBOX, *PBLURIGNOREOPACITY ? blurA : a * blurA, &texDamage, 0, false, false, false);
-    m_bEndFrame              = false;
+    setMonitorTransformEnabled(false);
-    m_RenderData.renderModif = SAVEDRENDERMODIF;
+    setRenderModifEnabled(true);
 
     // render the window, but clear stencil
     glClearStencil(0);
@@ -1587,6 +1592,7 @@ void CHyprOpenGLImpl::renderBorder(CBox* box, const CGradientValueData& grad, in
         return;
 
     int scaledBorderSize = std::round(borderSize * m_RenderData.pMonitor->scale);
+    scaledBorderSize     = std::round(scaledBorderSize * m_RenderData.renderModif.combinedScale());
 
     // adjust box
     box->x -= scaledBorderSize;
@@ -2223,6 +2229,10 @@ void CHyprOpenGLImpl::setMonitorTransformEnabled(bool enabled) {
     m_bEndFrame = enabled;
 }
 
+void CHyprOpenGLImpl::setRenderModifEnabled(bool enabled) {
+    m_RenderData.renderModif.enabled = enabled;
+}
+
 inline const SGLPixelFormat GLES2_FORMATS[] = {
     {
         .drmFormat = DRM_FORMAT_ARGB8888,
@@ -2358,6 +2368,9 @@ const SGLPixelFormat* CHyprOpenGLImpl::getPixelFormatFromDRM(uint32_t drmFormat)
 }
 
 void SRenderModifData::applyToBox(CBox& box) {
+    if (!enabled)
+        return;
+
     for (auto& [type, val] : modifs) {
         try {
             switch (type) {
@@ -2378,3 +2391,39 @@ void SRenderModifData::applyToBox(CBox& box) {
         } catch (std::bad_any_cast& e) { Debug::log(ERR, "BUG THIS OR PLUGIN ERROR: caught a bad_any_cast in SRenderModifData::applyToBox!"); }
     }
 }
+
+void SRenderModifData::applyToRegion(CRegion& rg) {
+    if (!enabled)
+        return;
+
+    for (auto& [type, val] : modifs) {
+        try {
+            switch (type) {
+                case RMOD_TYPE_SCALE: rg.scale(std::any_cast<float>(val)); break;
+                case RMOD_TYPE_SCALECENTER: rg.scale(std::any_cast<float>(val)); break;
+                case RMOD_TYPE_TRANSLATE: rg.translate(std::any_cast<Vector2D>(val)); break;
+                case RMOD_TYPE_ROTATE: /* TODO */
+                case RMOD_TYPE_ROTATECENTER: break;
+            }
+        } catch (std::bad_any_cast& e) { Debug::log(ERR, "BUG THIS OR PLUGIN ERROR: caught a bad_any_cast in SRenderModifData::applyToRegion!"); }
+    }
+}
+
+float SRenderModifData::combinedScale() {
+    if (!enabled)
+        return 1;
+
+    float scale = 1.f;
+    for (auto& [type, val] : modifs) {
+        try {
+            switch (type) {
+                case RMOD_TYPE_SCALE: scale *= std::any_cast<float>(val); break;
+                case RMOD_TYPE_SCALECENTER:
+                case RMOD_TYPE_TRANSLATE:
+                case RMOD_TYPE_ROTATE:
+                case RMOD_TYPE_ROTATECENTER: break;
+            }
+        } catch (std::bad_any_cast& e) { Debug::log(ERR, "BUG THIS OR PLUGIN ERROR: caught a bad_any_cast in SRenderModifData::combinedScale!"); }
+    }
+    return scale;
+}

src/render/OpenGL.hpp

@@ -47,6 +47,10 @@ struct SRenderModifData {
     std::vector<std::pair<eRenderModifType, std::any>> modifs;
 
     void                                               applyToBox(CBox& box);
+    void                                               applyToRegion(CRegion& rg);
+    float                                              combinedScale();
+
+    bool                                               enabled = true;
 };
 
 struct SGLPixelFormat {
@@ -138,6 +142,7 @@ class CHyprOpenGLImpl {
     void                  renderTextureMatte(const CTexture& tex, CBox* pBox, CFramebuffer& matte);
 
     void                  setMonitorTransformEnabled(bool enabled);
+    void                  setRenderModifEnabled(bool enabled);
 
     void                  saveMatrix();
     void                  setMatrixScaleTranslate(const Vector2D& translate, const float& scale);

src/render/Renderer.cpp

@@ -760,13 +760,35 @@ void CHyprRenderer::renderAllClientsForWorkspace(CMonitor* pMonitor, PHLWORKSPAC
     // g_pHyprOpenGL->setMatrixScaleTranslate(translate, scale);
     g_pHyprOpenGL->m_RenderData.renderModif = RENDERMODIFDATA;
 
+    if (!pWorkspace) {
+        // allow rendering without a workspace. In this case, just render layers.
+        g_pHyprOpenGL->blend(true);
+
+        for (auto& ls : pMonitor->m_aLayerSurfaceLayers[ZWLR_LAYER_SHELL_V1_LAYER_BACKGROUND]) {
+            renderLayer(ls.get(), pMonitor, time);
+        }
+        for (auto& ls : pMonitor->m_aLayerSurfaceLayers[ZWLR_LAYER_SHELL_V1_LAYER_BOTTOM]) {
+            renderLayer(ls.get(), pMonitor, time);
+        }
+        for (auto& ls : pMonitor->m_aLayerSurfaceLayers[ZWLR_LAYER_SHELL_V1_LAYER_TOP]) {
+            renderLayer(ls.get(), pMonitor, time);
+        }
+        for (auto& ls : pMonitor->m_aLayerSurfaceLayers[ZWLR_LAYER_SHELL_V1_LAYER_OVERLAY]) {
+            renderLayer(ls.get(), pMonitor, time);
+        }
+
+        g_pHyprOpenGL->m_RenderData.renderModif = {};
+
+        return;
+    }
+
     // for storing damage when we optimize for occlusion
     CRegion preOccludedDamage{g_pHyprOpenGL->m_RenderData.damage};
 
     // Render layer surfaces below windows for monitor
     // if we have a fullscreen, opaque window that convers the screen, we can skip this.
     // TODO: check better with solitary after MR for tearing.
-    const auto PFULLWINDOW = g_pCompositor->getFullscreenWindowOnWorkspace(pWorkspace->m_iID);
+    const auto PFULLWINDOW = pWorkspace ? g_pCompositor->getFullscreenWindowOnWorkspace(pWorkspace->m_iID) : nullptr;
     if (!pWorkspace->m_bHasFullscreenWindow || pWorkspace->m_efFullscreenMode != FULLSCREEN_FULL || !PFULLWINDOW || PFULLWINDOW->m_vRealSize.isBeingAnimated() ||
         !PFULLWINDOW->opaque() || pWorkspace->m_vRenderOffset.value() != Vector2D{}) {
 
@@ -804,8 +826,6 @@ void CHyprRenderer::renderAllClientsForWorkspace(CMonitor* pMonitor, PHLWORKSPAC
 
     g_pHyprOpenGL->m_RenderData.damage = preOccludedDamage;
 
-    g_pHyprOpenGL->m_RenderData.renderModif = {};
-
     // and then special
     for (auto& ws : g_pCompositor->m_vWorkspaces) {
         if (ws->m_iMonitorID == pMonitor->ID && ws->m_fAlpha.value() > 0.f && ws->m_bIsSpecialWorkspace) {
@@ -2450,6 +2470,7 @@ void CHyprRenderer::setOccludedForBackLayers(CRegion& region, PHLWORKSPACE pWork
         CBox           box = {POS.x, POS.y, SIZE.x, SIZE.y};
 
         box.scale(PMONITOR->scale);
+        g_pHyprOpenGL->m_RenderData.renderModif.applyToBox(box);
 
         rg.add(box);
     }

src/render/decorations/CHyprDropShadowDecoration.cpp

@@ -171,6 +171,7 @@ void CHyprDropShadowDecoration::draw(CMonitor* pMonitor, float a) {
 
         g_pHyprOpenGL->m_RenderData.damage = fullBox;
         g_pHyprOpenGL->m_RenderData.damage.subtract(windowBox.copy().expand(-ROUNDING * pMonitor->scale)).intersect(saveDamage);
+        g_pHyprOpenGL->m_RenderData.renderModif.applyToRegion(g_pHyprOpenGL->m_RenderData.damage);
 
         alphaFB.bind();
 
@@ -194,7 +195,9 @@ void CHyprDropShadowDecoration::draw(CMonitor* pMonitor, float a) {
 
         CBox monbox = {0, 0, pMonitor->vecTransformedSize.x, pMonitor->vecTransformedSize.y};
         g_pHyprOpenGL->setMonitorTransformEnabled(true);
+        g_pHyprOpenGL->setRenderModifEnabled(false);
         g_pHyprOpenGL->renderTextureMatte(alphaSwapFB.m_cTex, &monbox, alphaFB);
+        g_pHyprOpenGL->setRenderModifEnabled(true);
         g_pHyprOpenGL->setMonitorTransformEnabled(false);
 
         g_pHyprOpenGL->m_RenderData.damage = saveDamage;