xdgoutput: separate logic for zero scaling positions

if a scaled monitor was directly to the left of another monitor, we'd change the size to have zero scaling without pushing other monitors out of the way, creating overlaps in xwayland.

src/protocols/XDGOutput.cpp

@@ -97,7 +97,7 @@ void CXDGOutputProtocol::onManagerGetXDGOutput(wl_client* client, wl_resource* r
     if (XDGVER >= ZXDG_OUTPUT_V1_DESCRIPTION_SINCE_VERSION && PMONITOR->output->description)
         zxdg_output_v1_send_description(pXDGOutput->resource->resource(), PMONITOR->output->description);
 
-    updateOutputDetails(pXDGOutput);
+    updateAllOutputs();
 
     const auto OUTPUTVER = wl_resource_get_version(outputResource);
     if (OUTPUTVER >= WL_OUTPUT_DONE_SINCE_VERSION && XDGVER >= OUTPUT_DONE_DEPRECATED_SINCE_VERSION)
@@ -107,10 +107,11 @@ void CXDGOutputProtocol::onManagerGetXDGOutput(wl_client* client, wl_resource* r
 void CXDGOutputProtocol::updateOutputDetails(SXDGOutput* pOutput) {
     static auto* const PXWLFORCESCALEZERO = &g_pConfigManager->getConfigValuePtr("xwayland:force_zero_scaling")->intValue;
 
-    zxdg_output_v1_send_logical_position(pOutput->resource->resource(), pOutput->monitor->vecPosition.x, pOutput->monitor->vecPosition.y);
+    const auto         POS = pOutput->overridePosition.value_or(pOutput->monitor->vecPosition);
+    zxdg_output_v1_send_logical_position(pOutput->resource->resource(), POS.x, POS.y);
 
     if (*PXWLFORCESCALEZERO && pOutput->isXWayland)
-        zxdg_output_v1_send_logical_size(pOutput->resource->resource(), pOutput->monitor->vecPixelSize.x, pOutput->monitor->vecPixelSize.y);
+        zxdg_output_v1_send_logical_size(pOutput->resource->resource(), pOutput->monitor->vecTransformedSize.x, pOutput->monitor->vecTransformedSize.y);
     else
         zxdg_output_v1_send_logical_size(pOutput->resource->resource(), pOutput->monitor->vecSize.x, pOutput->monitor->vecSize.y);
 
@@ -119,9 +120,47 @@ void CXDGOutputProtocol::updateOutputDetails(SXDGOutput* pOutput) {
 }
 
 void CXDGOutputProtocol::updateAllOutputs() {
+    static auto* const       PXWLFORCESCALEZERO = &g_pConfigManager->getConfigValuePtr("xwayland:force_zero_scaling")->intValue;
+    std::vector<SXDGOutput*> xwlOutputs;
+
     for (auto& o : m_vXDGOutputs) {
+        if (o->isXWayland && *PXWLFORCESCALEZERO) {
+            xwlOutputs.push_back(o.get());
+            o->overridePosition.reset();
+            continue;
+        }
+
         updateOutputDetails(o.get());
 
         wlr_output_schedule_done(o->monitor->output);
     }
+
+    // all of this will be noop if force_zero_scaling is off
+    std::sort(xwlOutputs.begin(), xwlOutputs.end(), [&](const auto& a, const auto& b) { return a->monitor->vecPosition < b->monitor->vecPosition; });
+
+    for (auto& o : xwlOutputs) {
+        Vector2D suggestedPos = o->monitor->vecPosition;
+
+        // check for overlaps
+        for (auto& o1 : xwlOutputs) {
+            if (!o1->overridePosition.has_value())
+                continue;
+
+            const auto OPOS = o1->overridePosition.value();
+
+            if (OPOS.x + o1->monitor->vecTransformedSize.x > suggestedPos.x && /* Y overlap */
+                !(suggestedPos.y + o->monitor->vecTransformedSize.y <= OPOS.y || OPOS.y + o1->monitor->vecTransformedSize.y <= suggestedPos.y))
+                suggestedPos.x = OPOS.x + o1->monitor->vecTransformedSize.x;
+
+            if (OPOS.y + o1->monitor->vecTransformedSize.y > suggestedPos.y && /* X overlap */
+                !(suggestedPos.x + o->monitor->vecTransformedSize.x <= OPOS.x || OPOS.x + o1->monitor->vecTransformedSize.x <= suggestedPos.x))
+                suggestedPos.y = OPOS.y + o1->monitor->vecTransformedSize.y;
+        }
+
+        o->overridePosition = suggestedPos;
+
+        updateOutputDetails(o);
+
+        wlr_output_schedule_done(o->monitor->output);
+    }
 }

src/protocols/XDGOutput.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "WaylandProtocol.hpp"
+#include <optional>
 
 class CMonitor;
 
@@ -8,6 +9,8 @@ struct SXDGOutput {
     CMonitor*                         monitor = nullptr;
     std::unique_ptr<CWaylandResource> resource;
 
+    std::optional<Vector2D>           overridePosition;
+
     wl_client*                        client     = nullptr;
     bool                              isXWayland = false;
 };