diff --git a/kitty/screen.c b/kitty/screen.c
index df2f7071e..9d1c81e30 100644
--- a/kitty/screen.c
+++ b/kitty/screen.c
@@ -3131,7 +3131,8 @@ screen_update_selection(Screen *self, index_type x, index_type y, bool in_left_h
         set_abs(abs_initial_start, s->initial_extent.start, s->initial_extent.scrolled_by);
         set_abs(abs_initial_end, s->initial_extent.end, s->initial_extent.scrolled_by);
         if (self->selections.extend_mode == EXTEND_WORD) {
-            s->adjusting_start = selection_boundary_less_than(&abs_current_input, &abs_initial_end);
+            if (s->adjusting_start) s->adjusting_start = selection_boundary_less_than(&abs_current_input, &abs_initial_end);
+            else s->adjusting_start = selection_boundary_less_than(&abs_current_input, &abs_initial_start);
         } else {
             const unsigned int initial_line = abs_initial_start.y;
             if (initial_line == abs_current_input.y) {
diff --git a/kitty_tests/mouse.py b/kitty_tests/mouse.py
index 12363d481..eb5048bac 100644
--- a/kitty_tests/mouse.py
+++ b/kitty_tests/mouse.py
@@ -155,6 +155,20 @@ class TestMouse(BaseTest):
         move(0.2)
         release()
         self.ae(sel(), 'ab cd')
+        multi_click(x=4.4)
+        self.ae(sel(), 'cd')
+        move(x=4.4, y=1)
+        self.ae(sel(), 'cd f gh')
+        move(x=4.4, y=0)
+        self.ae(sel(), 'cd')
+        release()
+        multi_click(x=4.4, y=1)
+        self.ae(sel(), 'gh')
+        move(x=4.4, y=0)
+        self.ae(sel(), 'cd f gh')
+        move(x=4.4, y=1)
+        self.ae(sel(), 'gh')
+        release()
 
         # Line select with drag
         s.reset()