diff --git a/kitty/boss.py b/kitty/boss.py
index 3bcccc0b0..8af397550 100644
--- a/kitty/boss.py
+++ b/kitty/boss.py
@@ -910,10 +910,10 @@ class Boss:
         else:
             self._new_window(cmd)
 
-    def switch_focus_to(self, window_idx: int) -> None:
+    def switch_focus_to(self, window_id: int) -> None:
         tab = self.active_tab
         if tab:
-            tab.set_active_window_idx(window_idx)
+            tab.set_active_window(window_id)
 
     def open_url(self, url: str, program: Optional[Union[str, List[str]]] = None, cwd: Optional[str] = None) -> None:
         if url:
diff --git a/kitty/fast_data_types.pyi b/kitty/fast_data_types.pyi
index 5baa94da3..1b129d7a2 100644
--- a/kitty/fast_data_types.pyi
+++ b/kitty/fast_data_types.pyi
@@ -544,7 +544,7 @@ def update_window_title(
 
 
 def update_window_visibility(
-    os_window_id: int, tab_id: int, window_id: int, window_idx: int,
+    os_window_id: int, tab_id: int, window_id: int,
     visible: bool
 ) -> None:
     pass
@@ -774,15 +774,11 @@ def swap_tabs(os_window_id: int, a: int, b: int) -> None:
     pass
 
 
-def swap_windows(os_window_id: int, tab_id: int, a: int, b: int) -> None:
-    pass
-
-
 def set_active_tab(os_window_id: int, a: int) -> None:
     pass
 
 
-def set_active_window(os_window_id: int, tab_id: int, window_idx: int) -> None:
+def set_active_window(os_window_id: int, tab_id: int, window_id: int) -> None:
     pass
 
 
@@ -1063,7 +1059,7 @@ def set_tab_bar_render_data(
 
 
 def set_window_render_data(
-    os_window_id: int, tab_id: int, window_id: int, window_idx: int,
+    os_window_id: int, tab_id: int, window_id: int,
     xstart: float, ystart: float, dx: float, dy: float, screen: Screen,
     left: int, top: int, right: int, bottom: int
 ) -> None:
diff --git a/kitty/layout/base.py b/kitty/layout/base.py
index 35bb3f294..19c87d9e3 100644
--- a/kitty/layout/base.py
+++ b/kitty/layout/base.py
@@ -11,7 +11,7 @@ from typing import (
 
 from kitty.constants import Edges, WindowGeometry
 from kitty.fast_data_types import (
-    Region, set_active_window, swap_windows, viewport_for_window
+    Region, set_active_window, viewport_for_window
 )
 from kitty.options_stub import Options
 from kitty.typing import TypedDict, WindowType
@@ -220,7 +220,6 @@ class Layout:
         self.os_window_id = os_window_id
         self.tab_id = tab_id
         self.set_active_window_in_os_window = partial(set_active_window, os_window_id, tab_id)
-        self.swap_windows_in_os_window = partial(swap_windows, os_window_id, tab_id)
         # A set of rectangles corresponding to the blank spaces at the edges of
         # this layout, i.e. spaces that are not covered by any window
         self.blank_rects: List[Rect] = []
@@ -275,7 +274,7 @@ class Layout:
         aidx = (idx + num_slots + delta) % num_slots
         return self.set_active_window(all_windows, aidx)
 
-    def neighbors(self, all_windows: WindowList, active_window_idx: int) -> NeighborsMap:
+    def neighbors(self, all_windows: WindowList) -> NeighborsMap:
         w = all_windows.active_window_for_idx(active_window_idx)
         assert w is not None
         n = self.neighbors_for_window(w, all_windows)
@@ -328,20 +327,18 @@ class Layout:
         nidx = qidx
         idx = active_window_idx
         self.swap_windows_in_layout(all_windows, nidx, idx)
-        self.swap_windows_in_os_window(nidx, idx)
         return self.set_active_window(all_windows, nidx)
 
     def swap_windows_in_layout(self, all_windows: WindowList, a: int, b: int) -> None:
         all_windows[a], all_windows[b] = all_windows[b], all_windows[a]
 
-    def add_window(self, all_windows: WindowList, window: WindowType, current_active_window_idx: int, location: Optional[str] = None) -> int:
+    def add_window(self, all_windows: WindowList, window: WindowType, location: Optional[str] = None, overlay_for: Optional[int] = None) -> int:
         active_window_idx = None
         if window.overlay_for is not None:
             i = idx_for_id(window.overlay_for, all_windows)
             if i is not None:
                 # put the overlay window in the position occupied by the
                 # overlaid window and move the overlaid window to the end
-                self.swap_windows_in_os_window(len(all_windows), i)
                 all_windows.append(all_windows[i])
                 all_windows[i] = window
                 active_window_idx = i
@@ -364,8 +361,6 @@ class Layout:
             elif location == 'first':
                 active_window_idx = 0
             if active_window_idx is not None:
-                for i in range(len(all_windows), active_window_idx, -1):
-                    self.swap_windows_in_os_window(i, i - 1)
                 all_windows.insert(active_window_idx, window)
 
         if active_window_idx is None:
@@ -383,7 +378,6 @@ class Layout:
             if nidx is not None:
                 idx = all_windows.index(window)
                 all_windows[nidx], all_windows[idx] = all_windows[idx], all_windows[nidx]
-                self.swap_windows_in_os_window(nidx, idx)
                 return self.remove_window(all_windows, window, current_active_window_idx, swapped=True)
 
         position = all_windows.index(window)
@@ -429,7 +423,7 @@ class Layout:
     def _set_dimensions(self) -> None:
         lgd.central, tab_bar, vw, vh, lgd.cell_width, lgd.cell_height = viewport_for_window(self.os_window_id)
 
-    def __call__(self, all_windows: WindowList, active_window_idx: int) -> int:
+    def __call__(self, all_windows: WindowList) -> int:
         self._set_dimensions()
         active_window = all_windows[active_window_idx]
         overlaid_windows, windows = process_overlaid_windows(all_windows)
@@ -548,5 +542,5 @@ class Layout:
             else:
                 yield no_borders
 
-    def layout_action(self, action_name: str, args: Sequence[str], all_windows: WindowList, active_window_idx: int) -> Optional[Union[bool, int]]:
+    def layout_action(self, action_name: str, args: Sequence[str], all_windows: WindowList) -> Optional[bool]:
         pass
diff --git a/kitty/layout/splits.py b/kitty/layout/splits.py
index 83fd45a9f..5593fc2c2 100644
--- a/kitty/layout/splits.py
+++ b/kitty/layout/splits.py
@@ -443,7 +443,7 @@ class Splits(Layout):
             else:
                 yield no_borders
 
-    def layout_action(self, action_name: str, args: Sequence[str], all_windows: WindowList, active_window_idx: int) -> Optional[Union[bool, int]]:
+    def layout_action(self, action_name: str, args: Sequence[str], all_windows: WindowList, active_window_idx: int) -> Optional[bool]:
         if action_name == 'rotate':
             args = args or ('90',)
             try:
diff --git a/kitty/mouse.c b/kitty/mouse.c
index dae9a853d..308f934b0 100644
--- a/kitty/mouse.c
+++ b/kitty/mouse.c
@@ -321,7 +321,7 @@ HANDLER(handle_move_event) {
     if (OPT(focus_follows_mouse)) {
         Tab *t = global_state.callback_os_window->tabs + global_state.callback_os_window->active_tab;
         if (window_idx != t->active_window) {
-            call_boss(switch_focus_to, "I", window_idx);
+            call_boss(switch_focus_to, "K", t->windows[window_idx].id);
         }
     }
     bool in_left_half_of_cell = false;
diff --git a/kitty/state.c b/kitty/state.c
index 98a58c4a9..4d4574cca 100644
--- a/kitty/state.c
+++ b/kitty/state.c
@@ -382,11 +382,12 @@ set_active_tab(id_type os_window_id, unsigned int idx) {
 }
 
 static inline void
-set_active_window(id_type os_window_id, id_type tab_id, unsigned int idx) {
-    WITH_TAB(os_window_id, tab_id)
-        tab->active_window = idx;
+set_active_window(id_type os_window_id, id_type tab_id, id_type window_id) {
+    WITH_WINDOW(os_window_id, tab_id, window_id)
+        (void)window;
+        tab->active_window = w;
         osw->needs_render = true;
-    END_WITH_TAB;
+    END_WITH_WINDOW;
 }
 
 static inline void
@@ -398,15 +399,6 @@ swap_tabs(id_type os_window_id, unsigned int a, unsigned int b) {
     END_WITH_OS_WINDOW
 }
 
-static inline void
-swap_windows(id_type os_window_id, id_type tab_id, unsigned int a, unsigned int b) {
-    WITH_TAB(os_window_id, tab_id);
-    Window w = tab->windows[b];
-    tab->windows[b] = tab->windows[a];
-    tab->windows[a] = w;
-    END_WITH_TAB;
-}
-
 static void
 add_borders_rect(id_type os_window_id, id_type tab_id, uint32_t left, uint32_t top, uint32_t right, uint32_t bottom, uint32_t color) {
     WITH_TAB(os_window_id, tab_id)
@@ -466,7 +458,9 @@ mark_os_window_for_close(OSWindow* w, CloseRequest cr) {
 #define KI(name) PYWRAP1(name) { id_type a; unsigned int b; PA("KI", &a, &b); name(a, b); Py_RETURN_NONE; }
 #define KII(name) PYWRAP1(name) { id_type a; unsigned int b, c; PA("KII", &a, &b, &c); name(a, b, c); Py_RETURN_NONE; }
 #define KKI(name) PYWRAP1(name) { id_type a, b; unsigned int c; PA("KKI", &a, &b, &c); name(a, b, c); Py_RETURN_NONE; }
+#define KKK(name) PYWRAP1(name) { id_type a, b, c; PA("KKK", &a, &b, &c); name(a, b, c); Py_RETURN_NONE; }
 #define KKII(name) PYWRAP1(name) { id_type a, b; unsigned int c, d; PA("KKII", &a, &b, &c, &d); name(a, b, c, d); Py_RETURN_NONE; }
+#define KKKK(name) PYWRAP1(name) { id_type a, b, c, d; PA("KKKK", &a, &b, &c, &d); name(a, b, c, d); Py_RETURN_NONE; }
 #define KK5I(name) PYWRAP1(name) { id_type a, b; unsigned int c, d, e, f, g; PA("KKIIIII", &a, &b, &c, &d, &e, &f, &g); name(a, b, c, d, e, f, g); Py_RETURN_NONE; }
 #define BOOL_SET(name) PYWRAP1(set_##name) { global_state.name = PyObject_IsTrue(args); Py_RETURN_NONE; }
 
@@ -872,14 +866,6 @@ PYWRAP1(background_opacity_of) {
     Py_RETURN_NONE;
 }
 
-static inline bool
-fix_window_idx(Tab *tab, id_type window_id, unsigned int *window_idx) {
-    for (id_type fix = 0; fix < tab->num_windows; fix++) {
-        if (tab->windows[fix].id == window_id) { *window_idx = fix; return true; }
-    }
-    return false;
-}
-
 PYWRAP1(set_window_padding) {
     id_type os_window_id, tab_id, window_id;
     unsigned int left, top, right, bottom;
@@ -894,22 +880,18 @@ PYWRAP1(set_window_render_data) {
 #define A(name) &(d.name)
 #define B(name) &(g.name)
     id_type os_window_id, tab_id, window_id;
-    unsigned int window_idx;
     ScreenRenderData d = {0};
     WindowGeometry g = {0};
-    PA("KKKIffffOIIII", &os_window_id, &tab_id, &window_id, &window_idx, A(xstart), A(ystart), A(dx), A(dy), A(screen), B(left), B(top), B(right), B(bottom));
+    PA("KKKffffOIIII", &os_window_id, &tab_id, &window_id, A(xstart), A(ystart), A(dx), A(dy), A(screen), B(left), B(top), B(right), B(bottom));
 
-    WITH_TAB(os_window_id, tab_id);
-        if (tab->windows[window_idx].id != window_id) {
-            if (!fix_window_idx(tab, window_id, &window_idx)) Py_RETURN_NONE;
-        }
-        Py_CLEAR(tab->windows[window_idx].render_data.screen);
-        d.vao_idx = tab->windows[window_idx].render_data.vao_idx;
-        d.gvao_idx = tab->windows[window_idx].render_data.gvao_idx;
-        tab->windows[window_idx].render_data = d;
-        tab->windows[window_idx].geometry = g;
-        Py_INCREF(tab->windows[window_idx].render_data.screen);
-    END_WITH_TAB;
+    WITH_WINDOW(os_window_id, tab_id, window_id);
+        Py_CLEAR(window->render_data.screen);
+        d.vao_idx = window->render_data.vao_idx;
+        d.gvao_idx = window->render_data.gvao_idx;
+        window->render_data = d;
+        window->geometry = g;
+        Py_INCREF(window->render_data.screen);
+    END_WITH_WINDOW;
     Py_RETURN_NONE;
 #undef A
 #undef B
@@ -917,15 +899,11 @@ PYWRAP1(set_window_render_data) {
 
 PYWRAP1(update_window_visibility) {
     id_type os_window_id, tab_id, window_id;
-    unsigned int window_idx;
     int visible;
-    PA("KKKIp", &os_window_id, &tab_id, &window_id, &window_idx, &visible);
-    WITH_TAB(os_window_id, tab_id);
-        if (tab->windows[window_idx].id != window_id) {
-            if (!fix_window_idx(tab, window_id, &window_idx)) Py_RETURN_NONE;
-        }
-        tab->windows[window_idx].visible = visible & 1;
-    END_WITH_TAB;
+    PA("KKKp", &os_window_id, &tab_id, &window_id, &visible);
+    WITH_WINDOW(os_window_id, tab_id, window_id);
+        window->visible = visible & 1;
+    END_WITH_WINDOW;
     Py_RETURN_NONE;
 }
 
@@ -1112,9 +1090,8 @@ PYWRAP1(add_window) { PyObject *title; id_type a, b; PA("KKO", &a, &b, &title);
 PYWRAP0(current_os_window) { OSWindow *w = current_os_window(); if (!w) Py_RETURN_NONE; return PyLong_FromUnsignedLongLong(w->id); }
 TWO_ID(remove_tab)
 KI(set_active_tab)
-KKI(set_active_window)
+KKK(set_active_window)
 KII(swap_tabs)
-KKII(swap_windows)
 KK5I(add_borders_rect)
 
 #define M(name, arg_type) {#name, (PyCFunction)name, arg_type, NULL}
@@ -1138,7 +1115,6 @@ static PyMethodDef module_methods[] = {
     MW(set_active_tab, METH_VARARGS),
     MW(set_active_window, METH_VARARGS),
     MW(swap_tabs, METH_VARARGS),
-    MW(swap_windows, METH_VARARGS),
     MW(add_borders_rect, METH_VARARGS),
     MW(set_tab_bar_render_data, METH_VARARGS),
     MW(set_window_render_data, METH_VARARGS),
diff --git a/kitty/tabs.py b/kitty/tabs.py
index 51e55e7c0..1ab1faa81 100644
--- a/kitty/tabs.py
+++ b/kitty/tabs.py
@@ -9,7 +9,7 @@ from functools import partial
 from operator import attrgetter
 from typing import (
     Any, Deque, Dict, Generator, Iterator, List, NamedTuple, Optional, Pattern,
-    Sequence, Tuple, cast
+    Sequence, Tuple, Union, cast
 )
 
 from .borders import Borders
@@ -19,12 +19,10 @@ from .constants import appname, is_macos, is_wayland
 from .fast_data_types import (
     add_tab, attach_window, detach_window, get_boss, mark_tab_bar_dirty,
     next_window_id, remove_tab, remove_window, ring_bell, set_active_tab,
-    swap_tabs, sync_os_window_title, x11_window_id
+    set_active_window, swap_tabs, sync_os_window_title, x11_window_id
 )
 from .layout.base import Layout, Rect
-from .layout.interface import (
-    create_layout_object_for, evict_cached_layouts
-)
+from .layout.interface import create_layout_object_for, evict_cached_layouts
 from .options_stub import Options
 from .tab_bar import TabBar, TabBarData
 from .typing import SessionTab, SessionType, TypedDict
@@ -82,18 +80,16 @@ class Tab:  # {{{
         cwd_from: Optional[int] = None,
         no_initial_window: bool = False
     ):
-        self._active_window_idx = 0
         self.tab_manager_ref = weakref.ref(tab_manager)
         self.os_window_id: int = tab_manager.os_window_id
         self.id: int = add_tab(self.os_window_id)
-        self.active_window_history: Deque[int] = deque()
         if not self.id:
             raise Exception('No OS window with id {} found, or tab counter has wrapped'.format(self.os_window_id))
         self.opts, self.args = tab_manager.opts, tab_manager.args
         self.name = getattr(session_tab, 'name', '')
         self.enabled_layouts = [x.lower() for x in getattr(session_tab, 'enabled_layouts', None) or self.opts.enabled_layouts]
         self.borders = Borders(self.os_window_id, self.id, self.opts)
-        self.windows = WindowList()
+        self.windows = WindowList(self)
         for i, which in enumerate('first second third fourth fifth sixth seventh eighth ninth tenth'.split()):
             setattr(self, which + '_window', partial(self.nth_window, num=i))
         self._last_used_layout: Optional[str] = None
@@ -121,15 +117,11 @@ class Tab:  # {{{
         if other_tab._current_layout_name:
             self._set_current_layout(other_tab._current_layout_name)
         self._last_used_layout = other_tab._last_used_layout
-        orig_history = deque(other_tab.active_window_history)
-        orig_active = other_tab._active_window_idx
         for window in other_tab.windows:
             detach_window(other_tab.os_window_id, other_tab.id, window.id)
-        other_tab._active_window_idx = 0
-        self.active_window_history = orig_history
         self.windows = other_tab.windows
-        other_tab.windows = WindowList()
-        self._active_window_idx = orig_active
+        self.windows.change_tab(self)
+        other_tab.windows = WindowList(other_tab)
         for window in self.windows:
             window.change_tab(self)
             attach_window(self.os_window_id, self.id, window.id)
@@ -147,38 +139,23 @@ class Tab:  # {{{
                 self.new_special_window(cmd)
             else:
                 self.new_window(cmd=cmd)
-        self.set_active_window_idx(session_tab.active_window_idx)
+        self.windows.active_window = self.windows.all_windows[session_tab.active_window_idx]
 
     def serialize_state(self) -> Dict[str, Any]:
         return {
             'version': 1,
             'id': self.id,
-            'active_window_idx': self.active_window_idx,
-            'windows': [w.serialize_state() for w in self],
+            'window_list': self.windows.serialize_state(),
             'current_layout': self._current_layout_name,
             'last_used_layout': self._last_used_layout,
-            'active_window_history': list(self.active_window_history),
             'name': self.name,
         }
 
-    @property
-    def active_window_idx(self) -> int:
-        return self._active_window_idx
-
-    @active_window_idx.setter
-    def active_window_idx(self, val: int) -> None:
-        old_active_window = self.windows.active_window_for_idx(self._active_window_idx)
-        if old_active_window is not None:
-            add_active_id_to_history(self.active_window_history, self.windows.overlaid_window_for(old_active_window))
-        self._active_window_idx = max(0, min(val, self.windows.max_active_idx))
-        new_active_window = self.windows.active_window_for_idx(self._active_window_idx)
-        if old_active_window is not new_active_window:
-            if old_active_window is not None:
-                old_active_window.focus_changed(False)
-            if new_active_window is not None:
-                new_active_window.focus_changed(True)
-            self.relayout_borders()
-            self.mark_tab_bar_dirty()
+    def active_window_changed(self) -> None:
+        w = self.active_window
+        set_active_window(self.os_window_id, self.id, 0 if w is None else w.id)
+        self.mark_tab_bar_dirty()
+        self.relayout_borders()
 
     def mark_tab_bar_dirty(self) -> None:
         tm = self.tab_manager_ref()
@@ -187,7 +164,7 @@ class Tab:  # {{{
 
     @property
     def active_window(self) -> Optional[Window]:
-        return self.windows.active_window_for_idx(self.active_window_idx)
+        return self.windows.active_window
 
     @property
     def title(self) -> str:
@@ -195,9 +172,7 @@ class Tab:  # {{{
 
     def set_title(self, title: str) -> None:
         self.name = title or ''
-        tm = self.tab_manager_ref()
-        if tm is not None:
-            tm.mark_tab_bar_dirty()
+        self.mark_tab_bar_dirty()
 
     def title_changed(self, window: Window) -> None:
         if window is self.active_window:
@@ -206,10 +181,7 @@ class Tab:  # {{{
                 tm.title_changed(self)
 
     def on_bell(self, window: Window) -> None:
-        tm = self.tab_manager_ref()
-        if tm is not None:
-            self.relayout_borders()
-            tm.mark_tab_bar_dirty()
+        self.mark_tab_bar_dirty()
 
     def visible_windows(self) -> Generator[Window, None, None]:
         for w in self.windows:
@@ -218,7 +190,7 @@ class Tab:  # {{{
 
     def relayout(self) -> None:
         if self.windows:
-            self.active_window_idx = self.current_layout(self.windows, self.active_window_idx)
+            self.current_layout(self.windows)
         self.relayout_borders()
 
     def relayout_borders(self) -> None:
@@ -287,12 +259,10 @@ class Tab:  # {{{
             self.relayout()
 
     def layout_action(self, action_name: str, args: Sequence[str]) -> None:
-        ret = self.current_layout.layout_action(action_name, args, self.windows, self.active_window_idx)
+        ret = self.current_layout.layout_action(action_name, args, self.windows)
         if ret is None:
             ring_bell()
             return
-        if not isinstance(ret, bool) and isinstance(ret, int):
-            self.active_window_idx = ret
         self.relayout()
 
     def launch_child(
@@ -324,8 +294,8 @@ class Tab:  # {{{
         ans.fork()
         return ans
 
-    def _add_window(self, window: Window, location: Optional[str] = None) -> None:
-        self.active_window_idx = self.current_layout.add_window(self.windows, window, self.active_window_idx, location)
+    def _add_window(self, window: Window, location: Optional[str] = None, overlay_for: Optional[int] = None) -> None:
+        self.current_layout.add_window(self.windows, window, location, overlay_for)
         self.relayout_borders()
         self.mark_tab_bar_dirty()
 
@@ -351,11 +321,9 @@ class Tab:  # {{{
             self, child, self.opts, self.args, override_title=override_title,
             copy_colors_from=copy_colors_from, watchers=watchers
         )
-        if overlay_for is not None:
-            window.overlay_for = overlay_for
         # Must add child before laying out so that resize_pty succeeds
         get_boss().add_child(window)
-        self._add_window(window, location=location)
+        self._add_window(window, location=location, overlay_for=overlay_for)
         if marker:
             try:
                 window.set_marker(marker)
@@ -391,39 +359,12 @@ class Tab:  # {{{
                 if window is not active_window:
                     self.remove_window(window)
 
-    def previous_active_window_idx(self, num: int) -> Optional[int]:
-        try:
-            old_window_id = self.active_window_history[-num]
-        except IndexError:
-            return None
-        return self.windows.idx_for_window(old_window_id)
-
     def remove_window(self, window: Window, destroy: bool = True) -> None:
-        next_window_id = self.windows.next_id_in_stack_on_remove(window)
-        if next_window_id is None:
-            try:
-                next_window_id = self.active_window_history[-1]
-            except IndexError:
-                pass
-
-        active_window_idx = self.current_layout.remove_window(self.windows, window, self.active_window_idx)
+        self.windows.remove_window(window)
         if destroy:
             remove_window(self.os_window_id, self.id, window.id)
         else:
             detach_window(self.os_window_id, self.id, window.id)
-        if next_window_id is None:
-            w = self.windows.active_window_for_idx(active_window_idx)
-            if w is not None:
-                next_window_id = w.id
-        if next_window_id is not None:
-            for idx, window in enumerate(self.windows):
-                if window.id == next_window_id:
-                    self.active_window_idx = self.current_layout.set_active_window(self.windows, idx)
-                    break
-            else:
-                self.active_window_idx = active_window_idx
-        else:
-            self.active_window_idx = active_window_idx
         self.relayout_borders()
         self.mark_tab_bar_dirty()
         active_window = self.active_window
@@ -431,7 +372,7 @@ class Tab:  # {{{
             self.title_changed(active_window)
 
     def detach_window(self, window: Window) -> Tuple[Window, ...]:
-        windows = list(self.windows.iter_stack_for_window(window))
+        windows = list(self.windows.windows_in_group_of(window))
         windows.sort(key=attrgetter('id'))  # since ids increase in order of creation
         for w in reversed(windows):
             self.remove_window(w, destroy=False)
@@ -442,15 +383,9 @@ class Tab:  # {{{
         attach_window(self.os_window_id, self.id, window.id)
         self._add_window(window)
 
-    def set_active_window_idx(self, idx: int) -> None:
-        if idx != self.active_window_idx:
-            self.active_window_idx = self.current_layout.set_active_window(self.windows, idx)
-            self.relayout_borders()
-
-    def set_active_window(self, window: Window) -> None:
-        idx = self.windows.idx_for_window(window)
-        if idx is not None:
-            self.set_active_window_idx(idx)
+    def set_active_window(self, x: Union[Window, int]) -> None:
+        q = self.windows.id_map[x] if isinstance(x, int) else x
+        self.windows.active_window = q
 
     def get_nth_window(self, n: int) -> Optional[Window]:
         if self.windows:
@@ -459,17 +394,14 @@ class Tab:  # {{{
     def nth_window(self, num: int = 0) -> None:
         if self.windows:
             if num < 0:
-                idx = self.previous_active_window_idx(-num)
-                if idx is None:
-                    return
-                self.active_window_idx = self.current_layout.set_active_window(self.windows, idx)
+                self.windows.make_previous_group_active(-num)
             else:
-                self.active_window_idx = self.current_layout.activate_nth_window(self.windows, num)
+                self.current_layout.activate_nth_window(self.windows, num)
             self.relayout_borders()
 
     def _next_window(self, delta: int = 1) -> None:
         if len(self.windows) > 1:
-            self.active_window_idx = self.current_layout.next_window(self.windows, self.active_window_idx, delta)
+            self.current_layout.next_window(self.windows, delta)
             self.relayout_borders()
 
     def next_window(self) -> None:
@@ -481,18 +413,20 @@ class Tab:  # {{{
     prev_window = previous_window
 
     def neighboring_window(self, which: str) -> None:
-        neighbors = self.current_layout.neighbors(self.windows, self.active_window_idx)
+        neighbors = self.current_layout.neighbors(self.windows)
         candidates = cast(Optional[Tuple[int, ...]], neighbors.get(which))
         if candidates:
-            self.active_window_idx = self.current_layout.set_active_window(self.windows, candidates[0])
+            self.current_layout.set_active_window(self.windows, candidates[0])
             self.relayout_borders()
 
     def move_window(self, delta: int = 1) -> None:
-        self.active_window_idx = self.current_layout.move_window(self.windows, self.active_window_idx, delta)
+        self.current_layout.move_window(self.windows, delta)
         self.relayout()
 
     def move_window_to_top(self) -> None:
-        self.move_window(-self.active_window_idx)
+        n = self.windows.num_groups
+        if n > 1:
+            self.move_window(1 - n)
 
     def move_window_forward(self) -> None:
         self.move_window()
@@ -524,7 +458,7 @@ class Tab:  # {{{
         evict_cached_layouts(self.id)
         for w in self.windows:
             w.destroy()
-        self.windows = WindowList()
+        self.windows = WindowList(self)
 
     def __repr__(self) -> str:
         return 'Tab(title={}, id={})'.format(self.name or self.title, hex(id(self)))
@@ -679,7 +613,7 @@ class TabManager:  # {{{
                 'title': tab.name or tab.title,
                 'layout': str(tab.current_layout.name),
                 'windows': list(tab.list_windows(active_window)),
-                'active_window_history': list(tab.active_window_history),
+                'active_window_history': list(tab.windows.active_window_history),
             }
 
     def serialize_state(self) -> Dict[str, Any]:
diff --git a/kitty/window.py b/kitty/window.py
index 570820c30..4b35e8f7a 100644
--- a/kitty/window.py
+++ b/kitty/window.py
@@ -224,7 +224,6 @@ class Window:
         self.pty_resized_once = False
         self.needs_attention = False
         self.override_title = override_title
-        self.overlay_for: Optional[int] = None
         self.default_title = os.path.basename(child.argv[0] or appname)
         self.child_title = self.default_title
         self.title_stack: Deque[str] = deque(maxlen=10)
@@ -296,8 +295,8 @@ class Window:
         return self.override_title or self.child_title
 
     def __repr__(self) -> str:
-        return 'Window(title={}, id={}, overlay_for={})'.format(
-                self.title, self.id, self.overlay_for)
+        return 'Window(title={}, id={})'.format(
+                self.title, self.id)
 
     def as_dict(self, is_focused: bool = False) -> WindowDict:
         return dict(
@@ -320,7 +319,6 @@ class Window:
             'default_title': self.default_title,
             'title_stack': list(self.title_stack),
             'allow_remote_control': self.allow_remote_control,
-            'overlay_for': self.overlay_for,
             'cwd': self.child.current_cwd or self.child.cwd,
             'env': self.child.environ,
             'cmdline': self.child.cmdline,
@@ -360,11 +358,11 @@ class Window:
             return False
         return False
 
-    def set_visible_in_layout(self, window_idx: int, val: bool) -> None:
+    def set_visible_in_layout(self, val: bool) -> None:
         val = bool(val)
         if val is not self.is_visible_in_layout:
             self.is_visible_in_layout = val
-            update_window_visibility(self.os_window_id, self.tab_id, self.id, window_idx, val)
+            update_window_visibility(self.os_window_id, self.tab_id, self.id, val)
             if val:
                 self.refresh()
 
@@ -377,7 +375,7 @@ class Window:
         self.screen_geometry = sg = calculate_gl_geometry(window_geometry, vw, vh, cw, ch)
         return sg
 
-    def set_geometry(self, window_idx: int, new_geometry: WindowGeometry) -> None:
+    def set_geometry(self, new_geometry: WindowGeometry) -> None:
         if self.destroyed:
             return
         if self.needs_layout or new_geometry.xnum != self.screen.columns or new_geometry.ynum != self.screen.lines:
@@ -396,7 +394,7 @@ class Window:
         else:
             sg = self.update_position(new_geometry)
         self.geometry = g = new_geometry
-        set_window_render_data(self.os_window_id, self.tab_id, self.id, window_idx, sg.xstart, sg.ystart, sg.dx, sg.dy, self.screen, *g[:4])
+        set_window_render_data(self.os_window_id, self.tab_id, self.id, sg.xstart, sg.ystart, sg.dx, sg.dy, self.screen, *g[:4])
         self.update_effective_padding()
 
     def contains(self, x: int, y: int) -> bool:
diff --git a/kitty/window_list.py b/kitty/window_list.py
index aa648af83..557cd45ec 100644
--- a/kitty/window_list.py
+++ b/kitty/window_list.py
@@ -2,20 +2,70 @@
 # vim:fileencoding=utf-8
 # License: GPLv3 Copyright: 2020, Kovid Goyal <kovid at kovidgoyal.net>
 
-from typing import Dict, Generator, Iterator, List, Optional, Union
+import weakref
+from collections import deque
+from contextlib import suppress
+from itertools import count
+from typing import Any, Deque, Dict, Iterator, List, Optional, Union
 
-from .typing import WindowType
+from .typing import TabType, WindowType
+
+WindowOrId = Union[WindowType, int]
+group_id_counter = count()
+
+
+class WindowGroup:
+
+    def __init__(self) -> None:
+        self.windows: List[WindowType] = []
+        self.id = next(group_id_counter)
+
+    def __len__(self) -> int:
+        return len(self.windows)
+
+    def __bool__(self) -> bool:
+        return bool(self.windows)
+
+    def __iter__(self) -> Iterator[WindowType]:
+        return iter(self.windows)
+
+    def __contains__(self, window: WindowType) -> bool:
+        for w in self.windows:
+            if w is window:
+                return True
+        return False
+
+    @property
+    def base_window_id(self) -> int:
+        return self.windows[0].id if self.windows else 0
+
+    @property
+    def active_window_id(self) -> int:
+        return self.windows[-1].id if self.windows else 0
+
+    def add_window(self, window: WindowType) -> None:
+        self.windows.append(window)
+
+    def remove_window(self, window: WindowType) -> None:
+        with suppress(ValueError):
+            self.windows.remove(window)
+
+    def serialize_state(self) -> Dict[str, Any]:
+        return {
+            'id': self.id,
+            'windows': [w.serialize_state() for w in self.windows]
+        }
 
 
 class WindowList:
 
-    def __init__(self) -> None:
+    def __init__(self, tab: TabType) -> None:
         self.all_windows: List[WindowType] = []
         self.id_map: Dict[int, WindowType] = {}
-        self.overlay_stacks: Dict[int, List[int]] = {}
-        self.id_to_idx_map: Dict[int, int] = {}
-        self.idx_to_base_id_map: Dict[int, int] = {}
-        self.max_active_idx = 0
+        self.groups: List[WindowGroup] = []
+        self.active_group_idx: int = -1
+        self.active_group_history: Deque[int] = deque((), 64)
+        self.tabref = weakref.ref(tab)
 
     def __len__(self) -> int:
         return len(self.all_windows)
@@ -29,62 +79,144 @@ class WindowList:
     def __contains__(self, window: WindowType) -> bool:
         return window.id in self.id_map
 
-    def stack_for_window_id(self, q: int) -> List[int]:
-        ' The stack of overlaid windows this window belongs to '
-        w = self.id_map[q]
-        if w.overlay_for is not None and w.overlay_for in self.id_map:
-            q = self.id_map[w.overlay_for].id
-        return self.overlay_stacks[q]
+    def serialize_state(self) -> Dict[str, Any]:
+        return {
+            'active_group_idx': self.active_group_idx,
+            'active_group_history': list(self.active_group_history),
+            'window_groups': [g.serialize_state() for g in self.groups]
+        }
 
-    def iter_top_level_windows(self) -> Generator[WindowType, None, None]:
-        ' Iterator over all top level windows '
-        for stack in self.overlay_stacks.values():
-            yield self.id_map[stack[-1]]
+    @property
+    def active_window_history(self) -> List[int]:
+        ans = []
+        seen = set()
+        gid_map = {g.id: g for g in self.groups}
+        for gid in self.active_group_history:
+            g = gid_map[gid]
+            w = g.active_window_id
+            if w > 0 and w not in seen:
+                seen.add(w)
+                ans.append(w)
+        return ans
 
-    def iter_stack_for_window(self, x: Union[WindowType, int], reverse: bool = False) -> Generator[WindowType, None, None]:
-        ' Iterator over all windows in the stack for this window '
-        q = x if isinstance(x, int) else x.id
-        stack = self.stack_for_window_id(q)
-        y = reversed(stack) if reverse else iter(stack)
-        for wid in y:
-            yield self.id_map[wid]
+    def set_active_group_idx(self, i: int) -> None:
+        if i != self.active_group_idx and 0 <= i < len(self.groups):
+            old_active_window = self.active_window
+            g = self.active_group
+            if g is not None:
+                with suppress(ValueError):
+                    self.active_group_history.remove(g.id)
+                self.active_group_history.append(g.id)
+            self.active_group_idx = i
+            new_active_window = self.active_window
+            if old_active_window is not new_active_window:
+                if old_active_window is not None:
+                    old_active_window.focus_changed(False)
+                if new_active_window is not None:
+                    new_active_window.focus_changed(True)
+                tab = self.tabref()
+                if tab is not None:
+                    tab.active_window_changed()
 
-    def overlay_for(self, x: Union[WindowType, int]) -> int:
-        ' id of the top-most window overlaying this window, same as this window id if not overlaid '
-        q = x if isinstance(x, int) else x.id
-        return self.stack_for_window_id(q)[-1]
+    def change_tab(self, tab: TabType) -> None:
+        self.tabref = weakref.ref(tab)
 
-    def overlaid_window_for(self, x: Union[WindowType, int]) -> int:
-        ' id of the bottom-most window in this windows overlay stack '
-        q = x if isinstance(x, int) else x.id
-        return self.stack_for_window_id(q)[0]
+    def make_previous_group_active(self, which: int = 1) -> None:
+        which = max(1, which)
+        self.active_group_idx = len(self.groups) - 1
+        gid_map = {g.id: i for i, g in enumerate(self.groups)}
+        num = len(self.active_group_history)
+        for i in range(num):
+            idx = num - i - 1
+            gid = self.active_group_history[idx]
+            x = gid_map.get(gid)
+            if x is not None:
+                which -= 1
+                if which < 1:
+                    self.set_active_group_idx(x)
+                    return
 
-    def is_overlaid(self, x: Union[WindowType, int]) -> bool:
-        ' Return False if there is a window overlaying this one '
-        q = x if isinstance(x, int) else x.id
-        return self.overlay_for(q) != q
+    @property
+    def num_groups(self) -> int:
+        return len(self.groups)
 
-    def idx_for_window(self, x: Union[WindowType, int]) -> Optional[int]:
-        ' Return the index of the window in the list of top-level windows '
-        q = x if isinstance(x, int) else x.id
-        return self.id_to_idx_map[q]
-
-    def active_window_for_idx(self, idx: int, clamp: bool = False) -> Optional[WindowType]:
-        ' Return the active window at the specified index '
-        if clamp:
-            idx = max(0, min(idx, self.max_active_idx))
-        q = self.idx_to_base_id_map.get(idx)
-        if q is not None:
-            return self.id_map[self.overlay_stacks[q][-1]]
+    def group_for_window(self, x: WindowOrId) -> Optional[WindowGroup]:
+        q = self.id_map[x] if isinstance(x, int) else x
+        for g in self.groups:
+            if q in g:
+                return g
         return None
 
-    def next_id_in_stack_on_remove(self, x: Union[WindowType, int]) -> Optional[int]:
-        ' The id of the window that should become active when this window is removed, or None if there is no other window in the stack '
-        q = x if isinstance(x, int) else x.id
-        stack = self.stack_for_window_id(q)
-        idx = stack.index(q)
-        if idx < len(stack) - 1:
-            return stack[idx + 1]
-        if idx > 0:
-            return stack[idx - 1]
+    def windows_in_group_of(self, x: WindowOrId) -> Iterator[WindowType]:
+        g = self.group_for_window(x)
+        if g is not None:
+            return iter(g)
+
+    @property
+    def active_group(self) -> Optional[WindowGroup]:
+        if self.active_group_idx >= 0:
+            return self.groups[self.active_group_idx]
         return None
+
+    @property
+    def active_window(self) -> Optional[WindowType]:
+        if self.active_group_idx >= 0:
+            return self.id_map[self.groups[self.active_group_idx].active_window_id]
+        return None
+
+    @active_window.setter
+    def active_window(self, x: WindowOrId) -> None:
+        q = self.id_map[x] if isinstance(x, int) else x
+        for i, group in enumerate(self.groups):
+            if q in group:
+                self.set_active_group_idx(i)
+                break
+
+    def add_window(
+        self,
+        window: WindowType,
+        group_of: Optional[WindowOrId] = None,
+        next_to: Optional[WindowOrId] = None,
+        before: bool = False,
+        make_active: bool = True
+    ) -> None:
+        self.all_windows.append(window)
+        self.id_map[window.id] = window
+        target_group: Optional[WindowGroup] = None
+
+        if group_of is not None:
+            target_group = self.group_for_window(group_of)
+        if target_group is None and next_to is not None:
+            q = self.id_map[next_to] if isinstance(next_to, int) else next_to
+            pos = -1
+            for i, g in enumerate(self.groups):
+                if q in g:
+                    pos = i
+                    break
+            if pos > -1:
+                target_group = WindowGroup()
+                self.groups.insert(pos + (0 if before else 1), target_group)
+        if target_group is None:
+            target_group = WindowGroup()
+            self.groups.append(target_group)
+
+        target_group.add_window(window)
+        if make_active:
+            for i, g in enumerate(self.groups):
+                if g is target_group:
+                    self.set_active_group_idx(i)
+                    break
+
+    def remove_window(self, x: WindowOrId) -> None:
+        q = self.id_map[x] if isinstance(x, int) else x
+        try:
+            self.all_windows.remove(q)
+        except ValueError:
+            pass
+        self.id_map.pop(q.id, None)
+        for i, g in enumerate(tuple(self.groups)):
+            g.remove_window(q)
+            if not g:
+                del self.groups[i]
+                if self.active_group_idx == i:
+                    self.make_previous_group_active()