diff --git a/kitty/boss.py b/kitty/boss.py
index cb9bd238a..ba4ac13c0 100644
--- a/kitty/boss.py
+++ b/kitty/boss.py
@@ -7,11 +7,11 @@ from weakref import WeakValueDictionary
 
 from .config import MINIMUM_FONT_SIZE
 from .constants import (
-    MODIFIER_KEYS, cell_size, is_key_pressed, mouse_button_pressed,
+    MODIFIER_KEYS, cell_size, is_key_pressed,
     mouse_cursor_pos, set_boss, viewport_size, wakeup
 )
 from .fast_data_types import (
-    GLFW_MOUSE_BUTTON_1, GLFW_PRESS, GLFW_REPEAT, ChildMonitor,
+    GLFW_PRESS, GLFW_REPEAT, ChildMonitor,
     destroy_global_data, destroy_sprite_map, glfw_post_empty_event,
     layout_sprite_map
 )
@@ -230,55 +230,19 @@ class Boss:
                     SpecialWindow(
                         self.opts.scrollback_pager, data, _('History')))
 
-    def window_for_pos(self, x, y):
+    def switch_focus_to(self, window_idx):
         tab = self.active_tab
-        if tab is not None:
-            for w in tab:
-                if w.is_visible_in_layout and w.contains(x, y):
-                    return w
-
-    def in_tab_bar(self, y):
-        th = self.current_tab_bar_height
-        return th > 0 and y >= viewport_size.height - th
-
-    def on_mouse_button(self, window, button, action, mods):
-        mouse_button_pressed[button] = action == GLFW_PRESS
-        x, y = mouse_cursor_pos
-        w = self.window_for_pos(x, y)
-        if w is None:
-            if self.in_tab_bar(y):
-                if button == GLFW_MOUSE_BUTTON_1 and action == GLFW_PRESS:
-                    self.tab_manager.activate_tab_at(x)
-            return
-        focus_moved = False
-        old_focus = self.active_window
-        tab = self.active_tab
-        if button == GLFW_MOUSE_BUTTON_1 and w is not old_focus:
-            tab.set_active_window(w)
-            focus_moved = True
-        if focus_moved:
-            if old_focus is not None and not old_focus.destroyed:
-                old_focus.focus_changed(False)
-            w.focus_changed(True)
-        w.on_mouse_button(button, action, mods)
-
-    def on_mouse_move(self, window, xpos, ypos):
-        mouse_cursor_pos[:2] = xpos, ypos = int(
-            xpos * viewport_size.x_ratio), int(ypos * viewport_size.y_ratio)
-        w = self.window_for_pos(xpos, ypos)
-        if w is not None:
-            w.on_mouse_move(xpos, ypos)
-        else:
-            self.change_mouse_cursor(self.in_tab_bar(ypos))
+        tab.set_active_window_idx(window_idx)
+        old_focus = tab.active_window
+        if not old_focus.destroyed:
+            old_focus.focus_changed(False)
+        tab.active_window.focus_changed(True)
 
     def on_mouse_scroll(self, window, x, y):
         w = self.window_for_pos(*mouse_cursor_pos)
         if w is not None:
             w.on_mouse_scroll(x, y)
 
-    def change_mouse_cursor(self, click=False):
-        self.glfw_window.set_click_cursor(click)
-
     def request_attention(self):
         try:
             self.glfw_window.request_window_attention()
diff --git a/kitty/mouse.c b/kitty/mouse.c
index deec0db35..3ae0b3809 100644
--- a/kitty/mouse.c
+++ b/kitty/mouse.c
@@ -39,7 +39,7 @@ cell_for_pos(Window *w, unsigned int *x, unsigned int *y) {
 }
 
 void 
-handle_move_event(Window *w, int UNUSED button, int UNUSED modifiers) {
+handle_move_event(Window *w, int UNUSED button, int UNUSED modifiers, unsigned int UNUSED window_idx) {
     unsigned int x, y;
     if (cell_for_pos(w, &x, &y)) {
         Line *line = screen_visual_line(w->render_data.screen, y);
@@ -51,16 +51,26 @@ handle_move_event(Window *w, int UNUSED button, int UNUSED modifiers) {
 }
 
 void 
-handle_event(Window *w, int button, int modifiers) {
+handle_button_event(Window UNUSED *w, int button, int UNUSED modifiers, unsigned int window_idx) {
+    Tab *t = global_state.tabs + global_state.active_tab;
+    if (window_idx != t->active_window && button == GLFW_MOUSE_BUTTON_LEFT) {
+        call_boss(switch_focus_to, "I", window_idx);
+        return;
+    }
+}
+
+void 
+handle_event(Window *w, int button, int modifiers, unsigned int window_idx) {
     switch(button) {
         case -1:
-            handle_move_event(w, button, modifiers);
+            handle_move_event(w, button, modifiers, window_idx);
             break;
         case GLFW_MOUSE_BUTTON_LEFT:  
         case GLFW_MOUSE_BUTTON_RIGHT:
         case GLFW_MOUSE_BUTTON_MIDDLE:
         case GLFW_MOUSE_BUTTON_4:
         case GLFW_MOUSE_BUTTON_5:
+            handle_button_event(w, button, modifiers, window_idx);
             break;
         default:
             break;
@@ -82,9 +92,9 @@ mouse_event(int button, int modifiers) {
         handle_tab_bar_mouse(button, modifiers); 
     } else {
         Tab *t = global_state.tabs + global_state.active_tab;
-        for (size_t i = 0; i < t->num_windows; i++) {
+        for (unsigned int i = 0; i < t->num_windows; i++) {
             if (contains_mouse(t->windows + i) && t->windows[i].render_data.screen) {
-                handle_event(t->windows + i, button, modifiers);
+                handle_event(t->windows + i, button, modifiers, i);
                 break;
             }
         }