diff --git a/src/managers/InputManager.cpp b/src/managers/InputManager.cpp
index 10b1ad95..1ab991d3 100644
--- a/src/managers/InputManager.cpp
+++ b/src/managers/InputManager.cpp
@@ -109,6 +109,8 @@ void CInputManager::onMouseButton(wlr_pointer_button_event* e) {
 
     switch (e->state) {
         case WLR_BUTTON_PRESSED:
+            refocus();
+
             if ((e->button == BTN_LEFT || e->button == BTN_RIGHT) && wlr_keyboard_get_modifiers(PKEYBOARD) == (uint32_t)g_pConfigManager->getInt("general:main_mod_internal")) {
                 currentlyDraggedWindow = g_pCompositor->windowFloatingFromCursor();
                 dragButton = e->button;
@@ -124,8 +126,6 @@ void CInputManager::onMouseButton(wlr_pointer_button_event* e) {
             break;
     }
 
-    refocus();
-
     // notify app if we didnt handle it
     if (g_pCompositor->doesSeatAcceptInput(g_pCompositor->m_pLastFocus)) {
         wlr_seat_pointer_notify_button(g_pCompositor->m_sSeat.seat, e->time_msec, e->button, e->state);