diff --git a/src/events/Layers.cpp b/src/events/Layers.cpp
index dae25391..da57bb68 100644
--- a/src/events/Layers.cpp
+++ b/src/events/Layers.cpp
@@ -199,8 +199,29 @@ void Events::listener_unmapLayerSurface(void* owner, void* data) {
 
     // refocus if needed
     if (layersurface->layerSurface->surface == g_pCompositor->m_pLastFocus) {
+        
+        Vector2D surfaceCoords;
+        SLayerSurface* pFoundLayerSurface = nullptr;
+        wlr_surface* foundSurface = nullptr;
+
         g_pCompositor->m_pLastFocus = nullptr;
-        g_pInputManager->refocus();
+
+        // find LS-es to focus
+        foundSurface = g_pCompositor->vectorToLayerSurface(g_pInputManager->getMouseCoordsInternal(), &PMONITOR->m_aLayerSurfaceLists[ZWLR_LAYER_SHELL_V1_LAYER_OVERLAY], &surfaceCoords, &pFoundLayerSurface);
+
+        if (!foundSurface)
+            foundSurface = g_pCompositor->vectorToLayerSurface(g_pInputManager->getMouseCoordsInternal(), &PMONITOR->m_aLayerSurfaceLists[ZWLR_LAYER_SHELL_V1_LAYER_TOP], &surfaceCoords, &pFoundLayerSurface);
+
+
+        if (!foundSurface) {
+            // if there isn't any, focus the last window
+            const auto PLASTWINDOW = g_pCompositor->m_pLastWindow;
+            g_pCompositor->focusWindow(nullptr);
+            g_pCompositor->focusWindow(PLASTWINDOW);
+        } else {
+            // otherwise, full refocus
+            g_pInputManager->refocus();
+        }
     }
 
     wlr_box geomFixed = {layersurface->geometry.x + PMONITOR->vecPosition.x, layersurface->geometry.y + PMONITOR->vecPosition.y, layersurface->geometry.width, layersurface->geometry.height};
diff --git a/src/managers/XWaylandManager.cpp b/src/managers/XWaylandManager.cpp
index e9980075..4a0b397d 100644
--- a/src/managers/XWaylandManager.cpp
+++ b/src/managers/XWaylandManager.cpp
@@ -62,8 +62,10 @@ void CHyprXWaylandManager::activateWindow(CWindow* pWindow, bool activate) {
     else
         wlr_xdg_toplevel_set_activated(pWindow->m_uSurface.xdg->toplevel, activate);
 
-    g_pCompositor->m_pLastFocus = getWindowSurface(pWindow);
-    g_pCompositor->m_pLastWindow = pWindow;
+    if (activate) {
+        g_pCompositor->m_pLastFocus = getWindowSurface(pWindow);
+        g_pCompositor->m_pLastWindow = pWindow;
+    }
 
     if (!pWindow->m_bPinned)
         g_pCompositor->getWorkspaceByID(pWindow->m_iWorkspaceID)->m_pLastFocusedWindow = pWindow;
diff --git a/src/managers/input/InputManager.cpp b/src/managers/input/InputManager.cpp
index 64963041..d76810e3 100644
--- a/src/managers/input/InputManager.cpp
+++ b/src/managers/input/InputManager.cpp
@@ -311,7 +311,6 @@ void CInputManager::mouseMoveUnified(uint32_t time, bool refocus) {
     } else {
         if (pFoundLayerSurface && pFoundLayerSurface->layerSurface->current.keyboard_interactive && *PFOLLOWMOUSE != 3 && allowKeyboardRefocus) {
             g_pCompositor->focusSurface(foundSurface);
-            g_pCompositor->m_pLastWindow = nullptr; // reset last window as we have a full focus on a LS
         }
 
         if (pFoundLayerSurface)