diff --git a/kitty/mouse.c b/kitty/mouse.c
index 4f00d84c9..23e0f81e6 100644
--- a/kitty/mouse.c
+++ b/kitty/mouse.c
@@ -372,7 +372,7 @@ open_url(Window *w) {
 HANDLER(handle_button_event) {
     Tab *t = global_state.callback_os_window->tabs + global_state.callback_os_window->active_tab;
     bool is_release = !global_state.callback_os_window->mouse_button_pressed[button];
-    if (window_idx != t->active_window) {
+    if (window_idx != t->active_window && !is_release) {
         call_boss(switch_focus_to, "I", window_idx);
     }
     Screen *screen = w->render_data.screen;