From 1797319a0785cf9ae4f406d40bc27b032c6f5fb2 Mon Sep 17 00:00:00 2001
From: Vaxry <vaxry@vaxry.net>
Date: Mon, 22 Jul 2024 13:01:55 +0200
Subject: [PATCH] renderer: untransform textures matching display transform

fixes #6754

This will break if the client uses a transform that is not equal to the display, reverting to old behavior. Combining transforms is left as a todo for the future.
---
 src/protocols/core/Compositor.cpp |  3 +++
 src/render/OpenGL.cpp             | 10 +++++++---
 src/render/Texture.hpp            | 13 ++++++++-----
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/src/protocols/core/Compositor.cpp b/src/protocols/core/Compositor.cpp
index 126478111..f1c835311 100644
--- a/src/protocols/core/Compositor.cpp
+++ b/src/protocols/core/Compositor.cpp
@@ -406,6 +406,9 @@ void CWLSurfaceResource::commitPendingState() {
     pending.damage.clear();
     pending.bufferDamage.clear();
 
+    if (current.buffer && current.buffer->texture)
+        current.buffer->texture->m_eTransform = wlTransformToHyprutils(current.transform);
+
     if (current.buffer && !bufferReleased) {
         current.buffer->update(accumulateCurrentBufferDamage());
 
diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index b2b046ce5..fef69f7c3 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -1277,9 +1277,13 @@ void CHyprOpenGLImpl::renderTextureInternalWithDamage(SP<CTexture> tex, CBox* pB
     static auto PDIMINACTIVE = CConfigValue<Hyprlang::INT>("decoration:dim_inactive");
     static auto PDT          = CConfigValue<Hyprlang::INT>("debug:damage_tracking");
 
-    // get transform
-    const auto TRANSFORM = wlTransformToHyprutils(invertTransform(!m_bEndFrame ? WL_OUTPUT_TRANSFORM_NORMAL : m_RenderData.pMonitor->transform));
-    float      matrix[9];
+    // get the needed transform for this texture
+    const bool TRANSFORMS_MATCH = wlTransformToHyprutils(m_RenderData.pMonitor->transform) == tex->m_eTransform; // FIXME: combine them properly!!!
+    eTransform TRANSFORM        = HYPRUTILS_TRANSFORM_NORMAL;
+    if (m_bEndFrame || TRANSFORMS_MATCH)
+        TRANSFORM = wlTransformToHyprutils(invertTransform(m_RenderData.pMonitor->transform));
+
+    float matrix[9];
     projectBox(matrix, newBox, TRANSFORM, newBox.rot, m_RenderData.monitorProjection.data());
 
     float glMatrix[9];
diff --git a/src/render/Texture.hpp b/src/render/Texture.hpp
index 0da95300c..a54be8c5d 100644
--- a/src/render/Texture.hpp
+++ b/src/render/Texture.hpp
@@ -2,9 +2,11 @@
 
 #include "../defines.hpp"
 #include <aquamarine/buffer/Buffer.hpp>
+#include <hyprutils/math/Misc.hpp>
 
 class IHLBuffer;
 HYPRUTILS_FORWARD(Math, CRegion);
+using namespace Hyprutils::Math;
 
 enum TEXTURETYPE {
     TEXTURE_INVALID,  // Invalid
@@ -33,11 +35,12 @@ class CTexture {
     void        allocate();
     void        update(uint32_t drmFormat, uint8_t* pixels, uint32_t stride, const CRegion& damage);
 
-    TEXTURETYPE m_iType   = TEXTURE_RGBA;
-    GLenum      m_iTarget = GL_TEXTURE_2D;
-    GLuint      m_iTexID  = 0;
-    Vector2D    m_vSize;
-    void*       m_pEglImage = nullptr;
+    TEXTURETYPE m_iType      = TEXTURE_RGBA;
+    GLenum      m_iTarget    = GL_TEXTURE_2D;
+    GLuint      m_iTexID     = 0;
+    Vector2D    m_vSize      = {};
+    void*       m_pEglImage  = nullptr;
+    eTransform  m_eTransform = HYPRUTILS_TRANSFORM_NORMAL;
 
   private:
     void createFromShm(uint32_t drmFormat, uint8_t* pixels, uint32_t stride, const Vector2D& size);