diff --git a/kitty/constants.py b/kitty/constants.py
index fe141248e..aec603b20 100644
--- a/kitty/constants.py
+++ b/kitty/constants.py
@@ -66,6 +66,7 @@ def queue_action(func, *args):
 
 
 is_key_pressed = defaultdict(lambda: False)
+mouse_button_pressed = defaultdict(lambda: False)
 viewport_size = ViewportSize()
 cell_size = ViewportSize()
 terminfo_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'terminfo')
diff --git a/kitty/tabs.py b/kitty/tabs.py
index 542954036..251119d16 100644
--- a/kitty/tabs.py
+++ b/kitty/tabs.py
@@ -15,7 +15,10 @@ from time import monotonic
 from queue import Queue, Empty
 
 from .child import Child
-from .constants import viewport_size, shell_path, appname, set_tab_manager, tab_manager, wakeup, cell_size, MODIFIER_KEYS, main_thread
+from .constants import (
+    viewport_size, shell_path, appname, set_tab_manager, tab_manager, wakeup,
+    cell_size, MODIFIER_KEYS, main_thread, mouse_button_pressed
+)
 from .fast_data_types import (
     glViewport, glBlendFunc, GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GLFW_PRESS,
     GLFW_REPEAT, GLFW_MOUSE_BUTTON_1, glfw_post_empty_event,
@@ -345,6 +348,7 @@ class TabManager(Thread):
 
     @callback
     def on_mouse_button(self, window, button, action, mods):
+        mouse_button_pressed[button] = action == GLFW_PRESS
         self.show_mouse_cursor()
         w = self.window_for_pos(*window.get_cursor_pos())
         if w is None:
diff --git a/kitty/window.py b/kitty/window.py
index 5a01efaf7..d156e5143 100644
--- a/kitty/window.py
+++ b/kitty/window.py
@@ -4,12 +4,12 @@
 
 import os
 import weakref
-from collections import deque, defaultdict
+from collections import deque
 from functools import partial
 from time import monotonic
 
 from .char_grid import CharGrid
-from .constants import wakeup, tab_manager, appname, WindowGeometry, is_key_pressed
+from .constants import wakeup, tab_manager, appname, WindowGeometry, is_key_pressed, mouse_button_pressed
 from .fast_data_types import (
     BRACKETED_PASTE_START, BRACKETED_PASTE_END, Screen, read_bytes_dump,
     read_bytes, GLFW_MOD_SHIFT, GLFW_MOUSE_BUTTON_1, GLFW_PRESS,
@@ -27,7 +27,6 @@ class Window:
 
     def __init__(self, tab, child, opts, args):
         self.tabref = weakref.ref(tab)
-        self.mouse_button_pressed = defaultdict(lambda: False)
         self.last_mouse_cursor_pos = 0, 0
         self.destroyed = False
         self.click_queue = deque(maxlen=3)
@@ -154,7 +153,6 @@ class Window:
             glfw_post_empty_event()
 
     def on_mouse_button(self, button, action, mods):
-        self.mouse_button_pressed[button] = action == GLFW_PRESS
         mode = self.screen.mouse_tracking_mode()
         send_event = mods != GLFW_MOD_SHIFT and mode > 0
         x, y = self.last_mouse_cursor_pos
@@ -182,7 +180,7 @@ class Window:
     def on_mouse_move(self, x, y):
         button = None
         for b in range(0, GLFW_MOUSE_BUTTON_5 + 1):
-            if self.mouse_button_pressed[b]:
+            if mouse_button_pressed[b]:
                 button = b
                 break
         action = MOVE if button is None else DRAG