diff --git a/kitty/layout.py b/kitty/layout.py
index e03d229ed..14d8ddcd5 100644
--- a/kitty/layout.py
+++ b/kitty/layout.py
@@ -235,10 +235,13 @@ class Layout:  # {{{
 
     def neighbors(self, all_windows, active_window_idx):
         w = all_windows[active_window_idx]
-        windows = process_overlaid_windows(all_windows)[1]
+        if self.needs_all_windows:
+            windows = all_windows
+        else:
+            windows = process_overlaid_windows(all_windows)[1]
         ans = self.neighbors_for_window(w, windows)
         for values in ans.values():
-            values[:] = [idx_for_id(w.id, all_windows) for w in values]
+            values[:] = [idx_for_id(getattr(w, 'id', w), all_windows) for w in values]
         return ans
 
     def move_window(self, all_windows, active_window_idx, delta=1):
@@ -1328,13 +1331,6 @@ class Splits(Layout):
             if pair.between_border is not None:
                 yield pair.between_border
 
-    def neighbors(self, all_windows, active_window_idx):
-        w = all_windows[active_window_idx]
-        ans = self.neighbors_for_window(w, all_windows)
-        for values in ans.values():
-            values[:] = [idx_for_id(wid, all_windows) for wid in values]
-        return ans
-
     def neighbors_for_window(self, window, windows):
         window_id = window.overlay_for or window.id
         pair = self.pairs_root.pair_for_window(window_id)