fixed blur rendering, stencil and gles3

src/includes.hpp

@@ -85,6 +85,6 @@ extern "C" {
 #endif
 
 #include <GLES3/gl32.h>
-#include <GLES2/gl2ext.h>
+#include <GLES3/gl3ext.h>
 
 #include "helpers/Vector2D.hpp"

src/render/Framebuffer.cpp

@@ -29,6 +29,17 @@ bool CFramebuffer::alloc(int w, int h) {
         glBindTexture(GL_TEXTURE_2D, m_cTex.m_iTexID);
         glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, m_cTex.m_iTexID, 0);
 
+
+        if (m_pStencilTex) {
+            glBindTexture(GL_TEXTURE_2D, m_pStencilTex->m_iTexID);
+            glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH24_STENCIL8, w, h, 0, GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8, 0);
+
+            glBindFramebuffer(GL_FRAMEBUFFER, m_iFb);
+            glBindTexture(GL_TEXTURE_2D, m_pStencilTex->m_iTexID);
+
+            glFramebufferTexture2D(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_TEXTURE_2D, m_pStencilTex->m_iTexID, 0);
+        }
+
         auto status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
         RASSERT((status == GL_FRAMEBUFFER_COMPLETE), "Framebuffer incomplete, couldn't create! (FB status: %i)", status);
 

src/render/Framebuffer.hpp

@@ -18,5 +18,7 @@ public:
     CTexture        m_cTex;
     GLuint          m_iFb = -1;
 
+    CTexture*       m_pStencilTex = nullptr;
+
     wl_output_transform m_tTransform; // for saving state
 };

src/render/OpenGL.cpp

@@ -127,13 +127,20 @@ void CHyprOpenGLImpl::begin(SMonitor* pMonitor) {
     m_iWLROutputFb = m_iCurrentOutputFb;
 
     // ensure a framebuffer for the monitor exists
-    if (m_mMonitorFramebuffers.find(pMonitor) == m_mMonitorFramebuffers.end() || m_mMonitorFramebuffers[pMonitor].m_Size != pMonitor->vecSize) {
+    if (m_mMonitorRenderResources.find(pMonitor) == m_mMonitorRenderResources.end() || m_mMonitorRenderResources[pMonitor].primaryFB.m_Size != pMonitor->vecSize) {
-        m_mMonitorFramebuffers[pMonitor].alloc(pMonitor->vecSize.x, pMonitor->vecSize.y);
+        m_mMonitorRenderResources[pMonitor].stencilTex.allocate();
+
+        m_mMonitorRenderResources[pMonitor].primaryFB.m_pStencilTex = &m_mMonitorRenderResources[pMonitor].stencilTex;
+        m_mMonitorRenderResources[pMonitor].mirrorFB.m_pStencilTex = &m_mMonitorRenderResources[pMonitor].stencilTex;
+
+        m_mMonitorRenderResources[pMonitor].primaryFB.alloc(pMonitor->vecSize.x, pMonitor->vecSize.y);
+        m_mMonitorRenderResources[pMonitor].mirrorFB.alloc(pMonitor->vecSize.x, pMonitor->vecSize.y);
+
         createBGTextureForMonitor(pMonitor);
     }
 
-    // bind the Hypr Framebuffer
+    // bind the primary Hypr Framebuffer
-    m_mMonitorFramebuffers[pMonitor].bind();
+    m_mMonitorRenderResources[pMonitor].primaryFB.bind();
     clear(CColor(11, 11, 11, 255));
 }
 
@@ -147,7 +154,7 @@ void CHyprOpenGLImpl::end() {
 
     clear(CColor(11, 11, 11, 255));
 
-    renderTexture(m_mMonitorFramebuffers[m_RenderData.pMonitor].m_cTex, matrix, 255.f, 0);
+    renderTexture(m_mMonitorRenderResources[m_RenderData.pMonitor].primaryFB.m_cTex, matrix, 255.f, 0);
 
     // reset our data
     m_RenderData.pMonitor = nullptr;
@@ -292,22 +299,29 @@ void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, float matrix[9]
         return;
     }
 
-    // create a stencil for our thang
+    // bind the mirror FB and clear it.
-    glEnable(GL_STENCIL_TEST);
+    m_mMonitorRenderResources[m_RenderData.pMonitor].mirrorFB.bind();
-    glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+    clear(CColor(0, 0, 0, 0));
 
-    // we clear the stencil and then draw our texture with GL_ALWAYS to make a stencil mask.
+    // init stencil for blurring only behind da window
+    glClearStencil(0);
     glClear(GL_STENCIL_BUFFER_BIT);
 
-    glStencilMask(0xFF);
+    glEnable(GL_STENCIL_TEST);
 
-    glStencilFunc(GL_ALWAYS, 1, 0xFF);
+    glStencilFunc(GL_ALWAYS, 1, -1);
+    glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
 
-    renderTexture(tex, matrix, 0, round); // 0 alpha because we dont want to draw to the FB
+    // render our window to the mirror FB while also writing to the stencil
+    renderTexture(tex, matrix, a, round);
 
     // then we disable writing to the mask and ONLY accept writing within the stencil
-    glStencilMask(0x00);
+    glStencilFunc(GL_EQUAL, 1, -1);
-    glStencilFunc(GL_EQUAL, 1, 0xFF);
+    glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+
+    // now we bind back the primary FB
+    // the mirror FB now has only our window.
+    m_mMonitorRenderResources[m_RenderData.pMonitor].primaryFB.bind();
 
     glEnable(GL_BLEND);
 
@@ -327,7 +341,7 @@ void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, float matrix[9]
 
     const auto RADIUS = g_pConfigManager->getInt("decoration:blur_size") + 2;
     const auto BLURPASSES = g_pConfigManager->getInt("decoration:blur_passes");
-    const auto PFRAMEBUFFER = &m_mMonitorFramebuffers[m_RenderData.pMonitor];
+    const auto PFRAMEBUFFER = &m_mMonitorRenderResources[m_RenderData.pMonitor].primaryFB;
 
     auto drawWithShader = [&](CShader* pShader) {
         glActiveTexture(GL_TEXTURE0);
@@ -363,10 +377,11 @@ void CHyprOpenGLImpl::renderTextureWithBlur(const CTexture& tex, float matrix[9]
     glBindTexture(tex.m_iTarget, 0);
 
     // when the blur is done, let's render the window itself
-    renderTexture(tex, matrix, a, round);
+    // we get it from the mirrored FB                                 full because it's the FB   255 alpha cuz we rendered with a before, same for rounding
+    renderTexture(m_mMonitorRenderResources[m_RenderData.pMonitor].mirrorFB.m_cTex, matrixFull, 255.f, 0);
 
     // and disable the stencil
-    glStencilMask(0xFF);
+    glStencilMask(-1);
     glStencilFunc(GL_ALWAYS, 1, 0xFF);
     glDisable(GL_STENCIL_TEST);
 }

src/render/OpenGL.hpp

@@ -29,6 +29,13 @@ struct SCurrentRenderData {
     float       projection[9];
 };
 
+struct SMonitorRenderData {
+    CFramebuffer primaryFB;
+    CFramebuffer mirrorFB;
+
+    CTexture     stencilTex;
+};
+
 class CHyprOpenGLImpl {
 public:
 
@@ -56,7 +63,7 @@ public:
     GLint  m_iWLROutputFb = 0;
 
     std::unordered_map<CWindow*, CFramebuffer> m_mWindowFramebuffers;
-    std::unordered_map<SMonitor*, CFramebuffer> m_mMonitorFramebuffers;
+    std::unordered_map<SMonitor*, SMonitorRenderData> m_mMonitorRenderResources;
     std::unordered_map<SMonitor*, CTexture> m_mMonitorBGTextures;
 
 private:

src/render/Shaders.hpp

@@ -56,13 +56,13 @@ void main() {
 		if (pixCoord[1] < topLeft[1]) {
 			// top
 			if (distance(topLeft, pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		} else if (pixCoord[1] > bottomRight[1]) {
 			// bottom
 			if (distance(vec2(topLeft[0], bottomRight[1]), pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		}
@@ -72,13 +72,13 @@ void main() {
 		if (pixCoord[1] < topLeft[1]) {
 			// top
 			if (distance(vec2(bottomRight[0], topLeft[1]), pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		} else if (pixCoord[1] > bottomRight[1]) {
 			// bottom
 			if (distance(bottomRight, pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		}
@@ -107,13 +107,13 @@ void main() {
 		if (pixCoord[1] < topLeft[1]) {
 			// top
 			if (distance(topLeft, pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		} else if (pixCoord[1] > bottomRight[1]) {
 			// bottom
 			if (distance(vec2(topLeft[0], bottomRight[1]), pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		}
@@ -123,13 +123,13 @@ void main() {
 		if (pixCoord[1] < topLeft[1]) {
 			// top
 			if (distance(vec2(bottomRight[0], topLeft[1]), pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		} else if (pixCoord[1] > bottomRight[1]) {
 			// bottom
 			if (distance(bottomRight, pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		}
@@ -261,13 +261,13 @@ void main() {
 		if (pixCoord[1] < topLeft[1]) {
 			// top
 			if (distance(topLeft, pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		} else if (pixCoord[1] > bottomRight[1]) {
 			// bottom
 			if (distance(vec2(topLeft[0], bottomRight[1]), pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		}
@@ -277,13 +277,13 @@ void main() {
 		if (pixCoord[1] < topLeft[1]) {
 			// top
 			if (distance(vec2(bottomRight[0], topLeft[1]), pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		} else if (pixCoord[1] > bottomRight[1]) {
 			// bottom
 			if (distance(bottomRight, pixCoord) > radius) {
-				gl_FragColor = vec4(0,0,0,0);
+				discard;
 				return;
 			}
 		}