diff --git a/kitty/mouse.c b/kitty/mouse.c
index 18ca25a71..0e12e0fa5 100644
--- a/kitty/mouse.c
+++ b/kitty/mouse.c
@@ -23,7 +23,6 @@ typedef enum MouseActions { PRESS, RELEASE, DRAG, MOVE } MouseAction;
 #define MOTION_INDICATOR  (1 << 5)
 #define EXTRA_BUTTON_INDICATOR (1 << 6)
 
-static int last_multi_clicks = 0;
 
 static inline unsigned int
 button_map(int button) {
@@ -121,10 +120,9 @@ update_drag(bool from_button, Window *w, bool is_release, int modifiers) {
     Screen *screen = w->render_data.screen;
     if (from_button) {
         if (is_release) screen_update_selection(screen, w->mouse_cell_x, w->mouse_cell_y, true);
-        else screen_start_selection(screen, w->mouse_cell_x, w->mouse_cell_y, modifiers == (int)OPT(rectangle_select_modifiers));
+        else screen_start_selection(screen, w->mouse_cell_x, w->mouse_cell_y, modifiers == (int)OPT(rectangle_select_modifiers), EXTEND_CELL);
     } else if (screen->selection.in_progress) {
         screen_update_selection(screen, w->mouse_cell_x, w->mouse_cell_y, false);
-        call_boss(set_primary_selection, NULL);
     }
 }
 
@@ -155,12 +153,8 @@ drag_scroll(Window *w, OSWindow *frame) {
 static inline void
 extend_selection(Window *w) {
     Screen *screen = w->render_data.screen;
-    index_type start, end;
     if (screen_has_selection(screen)) {
-        bool found_selectable_word = screen_selection_range_for_word(screen, w->mouse_cell_x, w->mouse_cell_y, &start, &end);
-        if (last_multi_clicks >= 2 && found_selectable_word) screen_update_selection(screen, end, w->mouse_cell_y, true);
-        else screen_update_selection(screen, w->mouse_cell_x, w->mouse_cell_y, false);
-        call_boss(set_primary_selection, NULL);
+        screen_update_selection(screen, w->mouse_cell_x, w->mouse_cell_y, false);
     }
 }
 
@@ -237,20 +231,22 @@ multi_click(Window *w, unsigned int count) {
     Screen *screen = w->render_data.screen;
     index_type start, end;
     bool found_selection = false;
+    SelectionExtendMode mode = EXTEND_CELL;
     switch(count) {
         case 2:
             found_selection = screen_selection_range_for_word(screen, w->mouse_cell_x, w->mouse_cell_y, &start, &end);
+            mode = EXTEND_WORD;
             break;
         case 3:
             found_selection = screen_selection_range_for_line(screen, w->mouse_cell_y, &start, &end);
+            mode = EXTEND_LINE;
             break;
         default:
             break;
     }
     if (found_selection) {
-        screen_start_selection(screen, start, w->mouse_cell_y, false);
-        screen_update_selection(screen, end, w->mouse_cell_y, true);
-        call_boss(set_primary_selection, NULL);
+        screen_start_selection(screen, start, w->mouse_cell_y, false, mode);
+        screen_update_selection(screen, end, w->mouse_cell_y, false);
     }
 }
 
@@ -263,11 +259,9 @@ HANDLER(add_click) {
     q->length++;
     // Now dispatch the multi-click if any
     if (q->length > 2 && N(1).at - N(3).at <= 2 * OPT(click_interval)) {
-        last_multi_clicks = 3;
         multi_click(w, 3);
         q->length = 0;
     } else if (q->length > 1 && N(1).at - N(2).at <= OPT(click_interval)) {
-        last_multi_clicks = 2;
         multi_click(w, 2);
     }
 #undef N
@@ -297,14 +291,9 @@ HANDLER(handle_button_event) {
         switch(button) {
             case GLFW_MOUSE_BUTTON_LEFT:
                 update_drag(true, w, is_release, modifiers);
-                last_multi_clicks = 0;
                 if (is_release) {
-                    if (modifiers == (int)OPT(open_url_modifiers)) {
-                        open_url(w);
-                    } else {
-                        if (is_release) add_click(w, button, modifiers, window_idx);
-                    }
-                }
+                    if (modifiers == (int)OPT(open_url_modifiers)) open_url(w);
+                } else add_click(w, button, modifiers, window_idx);
                 break;
             case GLFW_MOUSE_BUTTON_MIDDLE:
                 if (is_release && !modifiers) { call_boss(paste_from_selection, NULL); return; }
diff --git a/kitty/screen.c b/kitty/screen.c
index f0f9e5b22..9c55fb263 100644
--- a/kitty/screen.c
+++ b/kitty/screen.c
@@ -1626,9 +1626,17 @@ WRAP0(carriage_return)
 WRAP2(resize, 1, 1)
 WRAP2(set_margins, 1, 1)
 WRAP2(rescale_images, 1, 1)
-WRAP2B(start_selection)
 WRAP2B(update_selection)
 
+static PyObject*
+start_selection(Screen *self, PyObject *args) {
+    unsigned int x, y;
+    int rectangle_select = 0, extend_mode = EXTEND_CELL;
+    if (!PyArg_ParseTuple(args, "II|pp", &x, &y, &rectangle_select, &extend_mode)) return NULL;
+    screen_start_selection(self, x, y, rectangle_select, extend_mode);
+    Py_RETURN_NONE;
+}
+
 static PyObject*
 change_scrollback_size(Screen *self, PyObject *args) {
     unsigned int count = 1;
@@ -1733,10 +1741,10 @@ screen_is_selection_dirty(Screen *self) {
 }
 
 void
-screen_start_selection(Screen *self, index_type x, index_type y, bool rectangle_select) {
+screen_start_selection(Screen *self, index_type x, index_type y, bool rectangle_select, SelectionExtendMode extend_mode) {
 #define A(attr, val) self->selection.attr = val;
     A(start_x, x); A(end_x, x); A(start_y, y); A(end_y, y); A(start_scrolled_by, self->scrolled_by); A(end_scrolled_by, self->scrolled_by);
-    A(in_progress, true); A(rectangle_select, rectangle_select);
+    A(in_progress, true); A(rectangle_select, rectangle_select); A(extend_mode, extend_mode);
 #undef A
 }
 
@@ -1751,6 +1759,20 @@ void
 screen_update_selection(Screen *self, index_type x, index_type y, bool ended) {
     self->selection.end_x = x; self->selection.end_y = y; self->selection.end_scrolled_by = self->scrolled_by;
     if (ended) self->selection.in_progress = false;
+    index_type start, end;
+    bool found = false;
+    switch(self->selection.extend_mode) {
+        case EXTEND_WORD:
+            found = screen_selection_range_for_word(self, x, y, &start, &end);
+            break;
+        case EXTEND_LINE:
+            found = screen_selection_range_for_line(self, y, &start, &end);
+            break;
+        case EXTEND_CELL:
+            break;
+    }
+    if (found) self->selection.end_x = end;
+    call_boss(set_primary_selection, NULL);
 }
 
 static PyObject*
diff --git a/kitty/screen.h b/kitty/screen.h
index d99e1cbe5..68dad0b3e 100644
--- a/kitty/screen.h
+++ b/kitty/screen.h
@@ -22,9 +22,12 @@ typedef struct {
     unsigned int x, y;
 } SelectionBoundary;
 
+typedef enum SelectionExtendModes { EXTEND_CELL, EXTEND_WORD, EXTEND_LINE } SelectionExtendMode;
+
 typedef struct {
     unsigned int start_x, start_y, start_scrolled_by, end_x, end_y, end_scrolled_by;
     bool in_progress, rectangle_select;
+    SelectionExtendMode extend_mode;
 } Selection;
 
 #define SAVEPOINTS_SZ 256
@@ -156,7 +159,7 @@ void screen_update_cell_data(Screen *self, void *address, size_t sz);
 bool screen_is_cursor_visible(Screen *self);
 bool screen_selection_range_for_line(Screen *self, index_type y, index_type *start, index_type *end);
 bool screen_selection_range_for_word(Screen *self, index_type x, index_type y, index_type *start, index_type *end);
-void screen_start_selection(Screen *self, index_type x, index_type y, bool);
+void screen_start_selection(Screen *self, index_type x, index_type y, bool, SelectionExtendMode);
 void screen_update_selection(Screen *self, index_type x, index_type y, bool ended);
 bool screen_history_scroll(Screen *self, int amt, bool upwards);
 Line* screen_visual_line(Screen *self, index_type y);