From 6c10c384813890ed137f73ee79452cbe8c6f1dca Mon Sep 17 00:00:00 2001
From: vaxerski <vaxry@vaxry.net>
Date: Sun, 6 Aug 2023 20:33:36 +0200
Subject: [PATCH] renderer: fully switch to premultiplied alpha

---
 src/render/OpenGL.cpp         | 31 ++++++++++---------------------
 src/render/shaders/Border.hpp |  4 +---
 src/render/shaders/Shadow.hpp |  5 +++++
 3 files changed, 16 insertions(+), 24 deletions(-)

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 1040aa86..b56c876b 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -118,8 +118,6 @@ void CHyprOpenGLImpl::begin(CMonitor* pMonitor, CRegion* pDamage, bool fake) {
 
     m_RenderData.pCurrentMonData = &m_mMonitorRenderResources[pMonitor];
 
-    glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
-
     glGetIntegerv(GL_FRAMEBUFFER_BINDING, &m_iCurrentOutputFb);
     m_iWLROutputFb = m_iCurrentOutputFb;
 
@@ -421,9 +419,10 @@ void CHyprOpenGLImpl::clear(const CColor& color) {
 }
 
 void CHyprOpenGLImpl::blend(bool enabled) {
-    if (enabled)
+    if (enabled) {
         glEnable(GL_BLEND);
-    else
+        glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA); // everything is premultiplied
+    } else
         glDisable(GL_BLEND);
 
     m_bBlend = enabled;
@@ -494,8 +493,6 @@ void CHyprOpenGLImpl::renderRectWithDamage(wlr_box* box, const CColor& col, CReg
     float glMatrix[9];
     wlr_matrix_multiply(glMatrix, m_RenderData.projection, matrix);
 
-    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
-
     glUseProgram(m_RenderData.pCurrentMonData->m_shQUAD.program);
 
 #ifndef GLES2
@@ -504,7 +501,9 @@ void CHyprOpenGLImpl::renderRectWithDamage(wlr_box* box, const CColor& col, CReg
     wlr_matrix_transpose(glMatrix, glMatrix);
     glUniformMatrix3fv(m_RenderData.pCurrentMonData->m_shQUAD.proj, 1, GL_FALSE, glMatrix);
 #endif
-    glUniform4f(m_RenderData.pCurrentMonData->m_shQUAD.color, col.r, col.g, col.b, col.a);
+
+    // premultiply the color as well as we don't work with straight alpha
+    glUniform4f(m_RenderData.pCurrentMonData->m_shQUAD.color, col.r * col.a, col.g * col.a, col.b * col.a, col.a);
 
     wlr_box transformedBox;
     wlr_box_transform(&transformedBox, box, wlr_output_transform_invert(m_RenderData.pMonitor->transform), m_RenderData.pMonitor->vecTransformedSize.x,
@@ -543,8 +542,6 @@ void CHyprOpenGLImpl::renderRectWithDamage(wlr_box* box, const CColor& col, CReg
     }
 
     glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shQUAD.posAttrib);
-
-    glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
 }
 
 void CHyprOpenGLImpl::renderTexture(wlr_texture* tex, wlr_box* pBox, float alpha, int round, bool allowCustomUV) {
@@ -588,9 +585,7 @@ void CHyprOpenGLImpl::renderTextureInternalWithDamage(const CTexture& tex, wlr_b
     float glMatrix[9];
     wlr_matrix_multiply(glMatrix, m_RenderData.projection, matrix);
 
-    CShader* shader = nullptr;
-
-    glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
+    CShader*   shader = nullptr;
 
     bool       usingFinalShader = false;
 
@@ -955,9 +950,6 @@ CFramebuffer* CHyprOpenGLImpl::blurMainFramebufferWithDamage(float a, wlr_box* p
     }
 
     // finish
-    glEnable(GL_BLEND);
-    glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
-
     glBindTexture(PMIRRORFB->m_cTex.m_iTarget, 0);
 
     blend(BLENDBEFORE);
@@ -1227,8 +1219,8 @@ void CHyprOpenGLImpl::renderBorder(wlr_box* box, const CGradientValueData& grad,
     float glMatrix[9];
     wlr_matrix_multiply(glMatrix, m_RenderData.projection, matrix);
 
-    glEnable(GL_BLEND);
-    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+    const auto BLEND = m_bBlend;
+    blend(true);
 
     glUseProgram(m_RenderData.pCurrentMonData->m_shBORDER1.program);
 
@@ -1286,7 +1278,7 @@ void CHyprOpenGLImpl::renderBorder(wlr_box* box, const CGradientValueData& grad,
     glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shBORDER1.posAttrib);
     glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shBORDER1.texAttrib);
 
-    glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
+    blend(BLEND);
 }
 
 void CHyprOpenGLImpl::makeRawWindowSnapshot(CWindow* pWindow, CFramebuffer* pFramebuffer) {
@@ -1561,7 +1553,6 @@ void CHyprOpenGLImpl::renderRoundedShadow(wlr_box* box, int round, int range, fl
     wlr_matrix_multiply(glMatrix, m_RenderData.projection, matrix);
 
     glEnable(GL_BLEND);
-    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
 
     glUseProgram(m_RenderData.pCurrentMonData->m_shSHADOW.program);
 
@@ -1610,8 +1601,6 @@ void CHyprOpenGLImpl::renderRoundedShadow(wlr_box* box, int round, int range, fl
 
     glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.posAttrib);
     glDisableVertexAttribArray(m_RenderData.pCurrentMonData->m_shSHADOW.texAttrib);
-
-    glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
 }
 
 void CHyprOpenGLImpl::saveBufferForMirror() {
diff --git a/src/render/shaders/Border.hpp b/src/render/shaders/Border.hpp
index e89251a8..4e5bfd97 100644
--- a/src/render/shaders/Border.hpp
+++ b/src/render/shaders/Border.hpp
@@ -111,10 +111,8 @@ void main() {
     if (pixColor[3] == 0.0)
         discard;
 
-    float pixColor3 = pixColor[3];
     pixColor = getColorForCoord(v_texcoord);
-    pixColor[3] *= alpha * pixColor3;
 
-    gl_FragColor = pixColor;
+    gl_FragColor = pixColor * alpha;
 }
 )#";
diff --git a/src/render/shaders/Shadow.hpp b/src/render/shaders/Shadow.hpp
index 8fd25382..49f548d7 100644
--- a/src/render/shaders/Shadow.hpp
+++ b/src/render/shaders/Shadow.hpp
@@ -78,5 +78,10 @@ void main() {
         discard; return;
     }
 
+    // premultiply
+    pixColor[0] *= pixColor[3];
+    pixColor[1] *= pixColor[3];
+    pixColor[2] *= pixColor[3];
+
 	gl_FragColor = pixColor;
 })#";