diff --git a/docs/changelog.rst b/docs/changelog.rst
index 206d76749..84df53e8e 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -22,6 +22,9 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
 
 - macOS: Fix :kbd:`cmd+plus` not changing font size (:iss:`2839`)
 
+- Make neighboring window selection in grid and splits layouts more intelligent
+  (:pull:`2840`)
+
 - Allow passing the current selection to kittens (:iss:`2796`)
 
 - Allow setting the :opt:`background_opacity` of new OS windows created via
diff --git a/kitty/tabs.py b/kitty/tabs.py
index e792f4048..452adfbab 100644
--- a/kitty/tabs.py
+++ b/kitty/tabs.py
@@ -25,7 +25,7 @@ from .layout.base import Layout, Rect
 from .layout.interface import create_layout_object_for, evict_cached_layouts
 from .options_stub import Options
 from .tab_bar import TabBar, TabBarData
-from .typing import SessionTab, SessionType, TypedDict
+from .typing import EdgeLiteral, SessionTab, SessionType, TypedDict
 from .utils import log_error, resolved_shell
 from .window import Watchers, Window, WindowDict
 from .window_list import WindowList
@@ -410,28 +410,29 @@ class Tab:  # {{{
 
     prev_window = previous_window
 
-    def most_recent_group(self, groups: List[int]) -> int:
-        groups_set = set(groups)
+    def most_recent_group(self, groups: Sequence[int]) -> Optional[int]:
+        groups_set = frozenset(groups)
 
         for window_id in reversed(self.windows.active_window_history):
             group = self.windows.group_for_window(window_id)
             if group and group.id in groups_set:
                 return group.id
 
-        return groups[0]
+        if groups:
+            return groups[0]
 
-    def neighboring_group_id(self, which: str) -> Optional[int]:
+    def neighboring_group_id(self, which: EdgeLiteral) -> Optional[int]:
         neighbors = self.current_layout.neighbors(self.windows)
-        candidates = cast(Optional[List[int]], neighbors.get(which))
+        candidates = neighbors.get(which)
         if candidates:
             return self.most_recent_group(candidates)
 
-    def neighboring_window(self, which: str) -> None:
+    def neighboring_window(self, which: EdgeLiteral) -> None:
         neighbor = self.neighboring_group_id(which)
         if neighbor:
             self.windows.set_active_group(neighbor)
 
-    def move_window(self, delta: Union[str, int] = 1) -> None:
+    def move_window(self, delta: Union[EdgeLiteral, int] = 1) -> None:
         if isinstance(delta, int):
             if self.current_layout.move_window(self.windows, delta):
                 self.relayout()