compositor: find windows in direction on floating

src/Compositor.cpp

@@ -1501,6 +1501,9 @@ void CCompositor::addToFadingOutSafe(CWindow* pWindow) {
 
 CWindow* CCompositor::getWindowInDirection(CWindow* pWindow, char dir) {
 
+    if (!isDirection(dir))
+        return nullptr;
+
     // 0 -> history, 1 -> shared length
     static auto* const PMETHOD = &g_pConfigManager->getConfigValuePtr("binds:focus_preferred_method")->intValue;
 
@@ -1515,9 +1518,13 @@ CWindow* CCompositor::getWindowInDirection(CWindow* pWindow, char dir) {
     const auto POSA  = Vector2D(WINDOWIDEALBB.x, WINDOWIDEALBB.y);
     const auto SIZEA = Vector2D(WINDOWIDEALBB.width, WINDOWIDEALBB.height);
 
+    const auto PWORKSPACE   = g_pCompositor->getWorkspaceByID(pWindow->m_iWorkspaceID);
     auto       leaderValue  = -1;
     CWindow*   leaderWindow = nullptr;
 
+    if (!pWindow->m_bIsFloating) {
+
+        // for tiled windows, we calc edges
         for (auto& w : m_vWindows) {
             if (w.get() == pWindow || !w->m_bIsMapped || w->isHidden() || w->m_bIsFloating || !isWorkspaceVisible(w->m_iWorkspaceID))
                 continue;
@@ -1525,7 +1532,6 @@ CWindow* CCompositor::getWindowInDirection(CWindow* pWindow, char dir) {
             if (pWindow->m_iMonitorID == w->m_iMonitorID && pWindow->m_iWorkspaceID != w->m_iWorkspaceID)
                 continue;
 
-        const auto PWORKSPACE = g_pCompositor->getWorkspaceByID(w->m_iWorkspaceID);
             if (PWORKSPACE->m_bHasFullscreenWindow && !w->m_bIsFullscreen && !w->m_bCreatedOverFullscreen)
                 continue;
 
@@ -1587,6 +1593,53 @@ CWindow* CCompositor::getWindowInDirection(CWindow* pWindow, char dir) {
                 }
             }
         }
+    } else {
+        // for floating windows, we calculate best distance and angle.
+        // if there is a window with angle better than THRESHOLD, only distance counts
+
+        if (dir == 'u')
+            dir = 't';
+        if (dir == 'd')
+            dir = 'b';
+
+        static const std::unordered_map<char, Vector2D> VECTORS = {{'r', {1, 0}}, {'t', {0, -1}}, {'b', {0, 1}}, {'l', {-1, 0}}};
+
+        //
+        auto vectorAngles = [](Vector2D a, Vector2D b) -> double {
+            double dot = a.x * b.x + a.y * b.y;
+            double ang = std::acos(dot / (a.size() * b.size()));
+            return ang;
+        };
+
+        float           bestAngleAbs = 2.0 * M_PI;
+        constexpr float THRESHOLD    = 0.3 * M_PI;
+
+        for (auto& w : m_vWindows) {
+            if (w.get() == pWindow || !w->m_bIsMapped || w->isHidden() || !w->m_bIsFloating || !isWorkspaceVisible(w->m_iWorkspaceID))
+                continue;
+
+            if (pWindow->m_iMonitorID == w->m_iMonitorID && pWindow->m_iWorkspaceID != w->m_iWorkspaceID)
+                continue;
+
+            if (PWORKSPACE->m_bHasFullscreenWindow && !w->m_bIsFullscreen && !w->m_bCreatedOverFullscreen)
+                continue;
+
+            const auto DIST  = w->middle().distance(pWindow->middle());
+            const auto ANGLE = vectorAngles(Vector2D{w->middle() - pWindow->middle()}, VECTORS.at(dir));
+
+            if (ANGLE > M_PI_2)
+                continue; // if the angle is over 90 degrees, ignore. Wrong direction entirely.
+
+            if ((bestAngleAbs < THRESHOLD && DIST < leaderValue && ANGLE < THRESHOLD) || (ANGLE < bestAngleAbs && bestAngleAbs > THRESHOLD) || leaderValue == -1) {
+                leaderValue  = DIST;
+                bestAngleAbs = ANGLE;
+                leaderWindow = w.get();
+            }
+        }
+
+        if (!leaderWindow && PWORKSPACE->m_bHasFullscreenWindow)
+            leaderWindow = g_pCompositor->getFullscreenWindowOnWorkspace(PWORKSPACE->m_iID);
+    }
 
     if (leaderValue != -1)
         return leaderWindow;

src/helpers/MiscFunctions.cpp

@@ -242,6 +242,10 @@ bool isDirection(const std::string& arg) {
     return arg == "l" || arg == "r" || arg == "u" || arg == "d" || arg == "t" || arg == "b";
 }
 
+bool isDirection(const char& arg) {
+    return arg == 'l' || arg == 'r' || arg == 'u' || arg == 'd' || arg == 't' || arg == 'b';
+}
+
 int getWorkspaceIDFromString(const std::string& in, std::string& outName) {
     int result = INT_MAX;
     if (in.starts_with("special")) {

src/helpers/MiscFunctions.hpp

@@ -19,6 +19,7 @@ std::string                      escapeJSONStrings(const std::string& str);
 std::string                      removeBeginEndSpacesTabs(std::string);
 bool                             isNumber(const std::string&, bool allowfloat = false);
 bool                             isDirection(const std::string&);
+bool                             isDirection(const char&);
 int                              getWorkspaceIDFromString(const std::string&, std::string&);
 std::optional<std::string>       cleanCmdForWorkspace(const std::string&, std::string);
 float                            vecToRectDistanceSquared(const Vector2D& vec, const Vector2D& p1, const Vector2D& p2);

src/helpers/Vector2D.cpp

@@ -45,3 +45,7 @@ bool Vector2D::inTriangle(const Vector2D& p1, const Vector2D& p2, const Vector2D
 
     return 0 <= a && a <= 1 && 0 <= b && b <= 1 && 0 <= c && c <= 1;
 }
+
+double Vector2D::size() const {
+    return std::sqrt(x * x + y * y);
+}

src/helpers/Vector2D.hpp

@@ -57,7 +57,7 @@ class Vector2D {
     }
 
     double   distance(const Vector2D& other) const;
-
+    double   size() const;
     Vector2D clamp(const Vector2D& min, const Vector2D& max = Vector2D()) const;
 
     Vector2D floor() const;