xwayland: use a completely separate coordinate system

Use a completely separate coordinate system for XWayland. This fixes all issues with force_zero_scaling overlaps.

Fixes #2916

src/Compositor.cpp

@@ -2487,6 +2487,8 @@ void CCompositor::setIdleActivityInhibit(bool enabled) {
 }
 
 void CCompositor::arrangeMonitors() {
+    static auto* const     PXWLFORCESCALEZERO = &g_pConfigManager->getConfigValuePtr("xwayland:force_zero_scaling")->intValue;
+
     std::vector<CMonitor*> toArrange;
     std::vector<CMonitor*> arranged;
 
@@ -2522,4 +2524,12 @@ void CCompositor::arrangeMonitors() {
         m->moveTo({maxOffset, 0});
         maxOffset += m->vecPosition.x + m->vecSize.x;
     }
+
+    // reset maxOffset (reuse)
+    // and set xwayland positions aka auto for all
+    maxOffset = 0;
+    for (auto& m : m_vMonitors) {
+        m->vecXWaylandPosition = {maxOffset, 0};
+        maxOffset += (*PXWLFORCESCALEZERO ? m->vecTransformedSize.x : m->vecSize.x);
+    }
 }

src/Window.cpp

@@ -356,6 +356,9 @@ void CWindow::moveToWorkspace(int workspaceID) {
         m_pSwallowed->moveToWorkspace(workspaceID);
         m_pSwallowed->m_iMonitorID = m_iMonitorID;
     }
+
+    // update xwayland coords
+    g_pXWaylandManager->setWindowSize(this, m_vRealSize.vec());
 }
 
 CWindow* CWindow::X11TransientFor() {

src/helpers/Monitor.hpp

@@ -31,6 +31,7 @@ class CMonitor {
     ~CMonitor();
 
     Vector2D        vecPosition         = Vector2D(-1, -1); // means unset
+    Vector2D        vecXWaylandPosition = Vector2D(-1, -1); // means unset
     Vector2D        vecSize             = Vector2D(0, 0);
     Vector2D        vecPixelSize        = Vector2D(0, 0);
     Vector2D        vecTransformedSize  = Vector2D(0, 0);

src/managers/XWaylandManager.cpp

@@ -54,6 +54,7 @@ void CHyprXWaylandManager::activateSurface(wlr_surface* pSurface, bool activate)
 
 void CHyprXWaylandManager::activateWindow(CWindow* pWindow, bool activate) {
     if (pWindow->m_bIsX11) {
+        setWindowSize(pWindow, pWindow->m_vRealSize.vec()); // update xwayland output pos
 
         if (activate) {
             wlr_xwayland_surface_set_minimized(pWindow->m_uSurface.xwayland, false);
@@ -142,14 +143,28 @@ void CHyprXWaylandManager::sendCloseWindow(CWindow* pWindow) {
 
 void CHyprXWaylandManager::setWindowSize(CWindow* pWindow, Vector2D size, bool force) {
 
-    if (!force &&
+    static auto* const PXWLFORCESCALEZERO = &g_pConfigManager->getConfigValuePtr("xwayland:force_zero_scaling")->intValue;
-        ((pWindow->m_vReportedSize == size && pWindow->m_vRealPosition.vec() == pWindow->m_vReportedPosition) || (pWindow->m_vReportedSize == size && !pWindow->m_bIsX11)))
+
+    const auto         PMONITOR = g_pCompositor->getMonitorFromID(pWindow->m_iMonitorID);
+    if (!PMONITOR)
         return;
 
-    pWindow->m_vReportedPosition = pWindow->m_vRealPosition.vec();
+    // calculate pos
-    pWindow->m_vReportedSize     = size;
+    // TODO: this should be decoupled from setWindowSize IMO
+    Vector2D windowPos = pWindow->m_vRealPosition.vec();
 
-    static auto* const PXWLFORCESCALEZERO = &g_pConfigManager->getConfigValuePtr("xwayland:force_zero_scaling")->intValue;
+    if (pWindow->m_bIsX11) {
+        windowPos = windowPos - PMONITOR->vecPosition; // normalize to monitor
+        if (*PXWLFORCESCALEZERO)
+            windowPos = windowPos * PMONITOR->scale;           // scale if applicable
+        windowPos = windowPos + PMONITOR->vecXWaylandPosition; // move to correct position for xwayland
+    }
+
+    if (!force && ((pWindow->m_vReportedSize == size && windowPos == pWindow->m_vReportedPosition) || (pWindow->m_vReportedSize == size && !pWindow->m_bIsX11)))
+        return;
+
+    pWindow->m_vReportedPosition = windowPos;
+    pWindow->m_vReportedSize     = size;
 
     pWindow->m_fX11SurfaceScaledBy = 1.f;
 
@@ -160,10 +175,8 @@ void CHyprXWaylandManager::setWindowSize(CWindow* pWindow, Vector2D size, bool f
         }
     }
 
-    const Vector2D POS = *PXWLFORCESCALEZERO && pWindow->m_bIsX11 ? pWindow->m_vRealPosition.vec() * pWindow->m_fX11SurfaceScaledBy : pWindow->m_vRealPosition.vec();
-
     if (pWindow->m_bIsX11)
-        wlr_xwayland_surface_configure(pWindow->m_uSurface.xwayland, POS.x, POS.y, size.x, size.y);
+        wlr_xwayland_surface_configure(pWindow->m_uSurface.xwayland, windowPos.x, windowPos.y, size.x, size.y);
     else
         wlr_xdg_toplevel_set_size(pWindow->m_uSurface.xdg->toplevel, size.x, size.y);
 }

src/protocols/XDGOutput.cpp

@@ -107,7 +107,7 @@ void CXDGOutputProtocol::onManagerGetXDGOutput(wl_client* client, wl_resource* r
 void CXDGOutputProtocol::updateOutputDetails(SXDGOutput* pOutput) {
     static auto* const PXWLFORCESCALEZERO = &g_pConfigManager->getConfigValuePtr("xwayland:force_zero_scaling")->intValue;
 
-    const auto         POS = pOutput->overridePosition.value_or(pOutput->monitor->vecPosition);
+    const auto         POS = pOutput->isXWayland ? pOutput->monitor->vecXWaylandPosition : pOutput->monitor->vecPosition;
     zxdg_output_v1_send_logical_position(pOutput->resource->resource(), POS.x, POS.y);
 
     if (*PXWLFORCESCALEZERO && pOutput->isXWayland)
@@ -120,47 +120,9 @@ void CXDGOutputProtocol::updateOutputDetails(SXDGOutput* pOutput) {
 }
 
 void CXDGOutputProtocol::updateAllOutputs() {
-    static auto* const       PXWLFORCESCALEZERO = &g_pConfigManager->getConfigValuePtr("xwayland:force_zero_scaling")->intValue;
-    std::vector<SXDGOutput*> xwlOutputs;
-
     for (auto& o : m_vXDGOutputs) {
-        if (o->isXWayland && *PXWLFORCESCALEZERO) {
-            xwlOutputs.push_back(o.get());
-            o->overridePosition.reset();
-            continue;
-        }
-
         updateOutputDetails(o.get());
 
         wlr_output_schedule_done(o->monitor->output);
     }
-
-    // all of this will be noop if force_zero_scaling is off
-    std::sort(xwlOutputs.begin(), xwlOutputs.end(), [&](const auto& a, const auto& b) { return a->monitor->vecPosition < b->monitor->vecPosition; });
-
-    for (auto& o : xwlOutputs) {
-        Vector2D suggestedPos = o->monitor->vecPosition;
-
-        // check for overlaps
-        for (auto& o1 : xwlOutputs) {
-            if (!o1->overridePosition.has_value())
-                continue;
-
-            const auto OPOS = o1->overridePosition.value();
-
-            if (OPOS.x + o1->monitor->vecTransformedSize.x > suggestedPos.x && /* Y overlap */
-                !(suggestedPos.y + o->monitor->vecTransformedSize.y <= OPOS.y || OPOS.y + o1->monitor->vecTransformedSize.y <= suggestedPos.y))
-                suggestedPos.x = OPOS.x + o1->monitor->vecTransformedSize.x;
-
-            if (OPOS.y + o1->monitor->vecTransformedSize.y > suggestedPos.y && /* X overlap */
-                !(suggestedPos.x + o->monitor->vecTransformedSize.x <= OPOS.x || OPOS.x + o1->monitor->vecTransformedSize.x <= suggestedPos.x))
-                suggestedPos.y = OPOS.y + o1->monitor->vecTransformedSize.y;
-        }
-
-        o->overridePosition = suggestedPos;
-
-        updateOutputDetails(o);
-
-        wlr_output_schedule_done(o->monitor->output);
-    }
 }