diff --git a/docs/changelog.rst b/docs/changelog.rst
index 68c551e06..9621cb879 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -16,6 +16,10 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
 - Fix ``kitty @ close-tab`` not working with pipe based remote control
   (:iss:`3510`)
 
+- Fix removal of inactive tab that is before the currently active tab causing
+  the highlighted tab to be incorrect (:iss:`3516`)
+
+
 0.20.1 [2021-04-19]
 ----------------------
 
diff --git a/kitty/tabs.py b/kitty/tabs.py
index 94b0865bc..137bfb90f 100644
--- a/kitty/tabs.py
+++ b/kitty/tabs.py
@@ -687,7 +687,10 @@ class TabManager:  # {{{
 
     @property
     def active_tab(self) -> Optional[Tab]:
-        return self.tabs[self.active_tab_idx] if self.tabs else None
+        try:
+            return self.tabs[self.active_tab_idx] if self.tabs else None
+        except Exception:
+            return None
 
     @property
     def active_window(self) -> Optional[Window]:
@@ -750,6 +753,7 @@ class TabManager:  # {{{
         return self.tabs[idx]
 
     def remove(self, tab: Tab) -> None:
+        active_tab_before_removal = self.active_tab
         self._remove_tab(tab)
         try:
             active_tab_needs_to_change = self.active_tab is None or self.active_tab is tab
@@ -779,6 +783,13 @@ class TabManager:  # {{{
                 next_active_tab = max(0, min(self.active_tab_idx, len(self.tabs) - 1))
 
             self._set_active_tab(next_active_tab)
+        elif active_tab_before_removal is not None:
+            try:
+                idx = self.tabs.index(active_tab_before_removal)
+            except Exception:
+                pass
+            else:
+                self._active_tab_idx = idx
         self.mark_tab_bar_dirty()
         tab.destroy()