diff --git a/kitty/borders.py b/kitty/borders.py
index 6fabe093e..cdcc034c8 100644
--- a/kitty/borders.py
+++ b/kitty/borders.py
@@ -7,7 +7,6 @@ from threading import Lock
 
 from .constants import viewport_size, GLfloat, GLint, GLuint
 from .fast_data_types import glUniform3fv, GL_TRIANGLE_FAN, glMultiDrawArrays
-from .layout import available_height
 from .utils import get_dpi
 from .shaders import ShaderProgram
 
@@ -72,26 +71,10 @@ class Borders:
             *as_color(opts.inactive_border_color)
         )
 
-    def __call__(self, windows, active_window, draw_window_borders=True):
-        vw, vh = viewport_size.width, available_height()
-        if windows:
-            left_edge = min(w.geometry.left for w in windows)
-            right_edge = max(w.geometry.right for w in windows)
-            top_edge = min(w.geometry.top for w in windows)
-            bottom_edge = max(w.geometry.bottom for w in windows)
-        else:
-            left_edge = top_edge = 0
-            right_edge = vw
-            bottom_edge = vh
+    def __call__(self, windows, active_window, current_layout, draw_window_borders=True):
         rects = []
-        if left_edge > 0:
-            rects.extend(as_rect(0, 0, left_edge, vh))
-        if top_edge > 0:
-            rects.extend(as_rect(0, 0, vw, top_edge))
-        if right_edge < vw:
-            rects.extend(as_rect(right_edge, 0, vw, vh))
-        if bottom_edge < vh:
-            rects.extend(as_rect(0, bottom_edge, vw, vh))
+        for br in current_layout.blank_rects:
+            rects.extend(as_rect(*br))
         if draw_window_borders and self.border_width > 0:
             bw = self.border_width
             for w in windows:
diff --git a/kitty/layout.py b/kitty/layout.py
index 50402f479..21083dc9d 100644
--- a/kitty/layout.py
+++ b/kitty/layout.py
@@ -2,6 +2,7 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
+from collections import namedtuple
 from itertools import islice
 
 from .constants import WindowGeometry, viewport_size, cell_size, get_boss
@@ -31,6 +32,9 @@ def layout_dimension(length, cell_length, number_of_windows=1, border_length=0,
         pos += window_length
 
 
+Rect = namedtuple('Rect', 'left top right bottom')
+
+
 class Layout:
 
     name = None
@@ -39,6 +43,9 @@ class Layout:
     def __init__(self, opts, border_width, windows):
         self.opts = opts
         self.border_width = border_width
+        # A set of rectangles corresponding to the blank spaces at the edges of
+        # this layout, i.e. spaces that are not covered by any window
+        self.blank_rects = ()
 
     def next_window(self, windows, active_window_idx, delta=1):
         active_window_idx = (active_window_idx + len(windows) + delta) % len(windows)
@@ -75,6 +82,33 @@ def layout_single_window():
     return window_geometry(xstart, xnum, ystart, ynum)
 
 
+def left_blank_rect(w, rects, vh):
+    if w.geometry.left > 0:
+        rects.append(Rect(0, 0, w.geometry.left, vh))
+
+
+def right_blank_rect(w, rects, vh):
+    if w.geometry.right < viewport_size.width:
+        rects.append(Rect(w.geometry.right, 0, viewport_size.width, vh))
+
+
+def top_blank_rect(w, rects, vh):
+    if w.geometry.top > 0:
+        rects.append(Rect(0, 0, viewport_size.width, w.geometry.top))
+
+
+def bottom_blank_rect(w, rects, vh):
+    if w.geometry.bottom < available_height():
+        rects.append(Rect(0, w.geometry.bottom, viewport_size.width, vh))
+
+
+def blank_rects_for_window(w):
+    ans = []
+    vh = available_height()
+    left_blank_rect(w, ans, vh), top_blank_rect(w, ans, vh), right_blank_rect(w, ans, vh), bottom_blank_rect(w, ans, vh)
+    return ans
+
+
 class Stack(Layout):
 
     name = 'stack'
@@ -85,10 +119,13 @@ class Stack(Layout):
             w.is_visible_in_layout = i == active_window_idx
 
     def __call__(self, windows, active_window_idx):
+        self.blank_rects = []
         wg = layout_single_window()
         for i, w in enumerate(windows):
             w.is_visible_in_layout = i == active_window_idx
             w.set_geometry(wg)
+            if w.is_visible_in_layout:
+                self.blank_rects = blank_rects_for_window(w)
 
 
 class Tall(Layout):
@@ -96,18 +133,25 @@ class Tall(Layout):
     name = 'tall'
 
     def __call__(self, windows, active_window_idx):
+        self.blank_rects = br = []
         if len(windows) == 1:
             wg = layout_single_window()
             windows[0].set_geometry(wg)
+            self.blank_rects = blank_rects_for_window(windows[0])
             return
         xlayout = layout_dimension(viewport_size.width, cell_size.width, 2, self.border_width)
         xstart, xnum = next(xlayout)
         ystart, ynum = next(layout_dimension(available_height(), cell_size.height, 1, self.border_width, left_align=True))
         windows[0].set_geometry(window_geometry(xstart, xnum, ystart, ynum))
+        vh = available_height()
         xstart, xnum = next(xlayout)
         ylayout = layout_dimension(available_height(), cell_size.height, len(windows) - 1, self.border_width, left_align=True)
         for w, (ystart, ynum) in zip(islice(windows, 1, None), ylayout):
             w.set_geometry(window_geometry(xstart, xnum, ystart, ynum))
+        left_blank_rect(windows[0], br, vh), top_blank_rect(windows[0], br, vh), right_blank_rect(windows[-1], br, vh)
+        br.append(Rect(windows[0].geometry.right, 0, windows[1].geometry.left, vh))
+        br.append(Rect(0, windows[0].geometry.bottom, windows[0].geometry.right, vh))
+        br.append(Rect(windows[-1].geometry.left, windows[-1].geometry.bottom, viewport_size.width, vh))
 
 
 all_layouts = {o.name: o for o in globals().values() if isinstance(o, type) and issubclass(o, Layout) and o is not Layout}
diff --git a/kitty/tabs.py b/kitty/tabs.py
index 2cb445651..f6a740287 100644
--- a/kitty/tabs.py
+++ b/kitty/tabs.py
@@ -79,7 +79,7 @@ class Tab:
         self.relayout_borders()
 
     def relayout_borders(self):
-        self.borders(self.windows, self.active_window, self.current_layout.needs_window_borders and len(self.windows) > 1)
+        self.borders(self.windows, self.active_window, self.current_layout, self.current_layout.needs_window_borders and len(self.windows) > 1)
 
     def next_layout(self):
         if len(self.opts.enabled_layouts) > 1:
@@ -110,7 +110,7 @@ class Tab:
             window.title = window.override_title = override_title
         get_boss().add_child_fd(child.child_fd, window.read_ready, window.write_ready)
         self.active_window_idx = self.current_layout.add_window(self.windows, window, self.active_window_idx)
-        self.borders(self.windows, self.active_window, self.current_layout.needs_window_borders and len(self.windows) > 1)
+        self.relayout_borders()
         glfw_post_empty_event()
         return window
 
@@ -123,14 +123,14 @@ class Tab:
 
     def remove_window(self, window):
         self.active_window_idx = self.current_layout.remove_window(self.windows, window, self.active_window_idx)
-        self.borders(self.windows, self.active_window, self.current_layout.needs_window_borders and len(self.windows) > 1)
+        self.relayout_borders()
         glfw_post_empty_event()
 
     def set_active_window_idx(self, idx):
         if idx != self.active_window_idx:
             self.current_layout.set_active_window(self.windows, idx)
             self.active_window_idx = idx
-            self.borders(self.windows, self.active_window, self.current_layout.needs_window_borders and len(self.windows) > 1)
+            self.relayout_borders()
             glfw_post_empty_event()
 
     def set_active_window(self, window):
@@ -147,7 +147,7 @@ class Tab:
     def _next_window(self, delta=1):
         if len(self.windows) > 1:
             self.active_window_idx = self.current_layout.next_window(self.windows, self.active_window_idx, delta)
-            self.borders(self.windows, self.active_window, self.current_layout.needs_window_borders and len(self.windows) > 1)
+            self.relayout_borders()
             glfw_post_empty_event()
 
     def next_window(self):