diff --git a/kitty/boss.py b/kitty/boss.py
index f494ab97e..9099f1b2c 100644
--- a/kitty/boss.py
+++ b/kitty/boss.py
@@ -262,8 +262,10 @@ class Boss:
             self.allow_remote_control = 'y'
         elif self.allow_remote_control in ('n', 'no', 'false'):
             self.allow_remote_control = 'n'
+        self.listening_on = ''
         if args.listen_on and self.allow_remote_control in ('y', 'socket', 'socket-only', 'password'):
             listen_fd = listen_on(args.listen_on)
+            self.listening_on = args.listen_on
         self.prewarm = prewarm
         self.child_monitor = ChildMonitor(
             self.on_child_death,
diff --git a/kitty/child.py b/kitty/child.py
index aedc33873..647662527 100644
--- a/kitty/child.py
+++ b/kitty/child.py
@@ -241,6 +241,7 @@ class Child:
     def final_env(self) -> Dict[str, str]:
         from kitty.options.utils import DELETE_ENV_VAR
         env = default_env().copy()
+        boss = fast_data_types.get_boss()
         if is_macos and env.get('LC_CTYPE') == 'UTF-8' and not getattr(sys, 'kitty_run_data').get(
                 'lc_ctype_before_python') and not getattr(default_env, 'lc_ctype_set_by_user', False):
             del env['LC_CTYPE']
@@ -248,9 +249,13 @@ class Child:
         env['TERM'] = fast_data_types.get_options().term
         env['COLORTERM'] = 'truecolor'
         env['KITTY_PID'] = getpid()
-        env['KITTY_PUBLIC_KEY'] = fast_data_types.get_boss().encryption_public_key
+        env['KITTY_PUBLIC_KEY'] = boss.encryption_public_key
+        if boss.listening_on:
+            env['KITTY_LISTEN_ON'] = boss.listening_on
+        else:
+            env.pop('KITTY_LISTEN_ON', None)
         if not self.is_prewarmed:
-            env['KITTY_PREWARM_SOCKET'] = fast_data_types.get_boss().prewarm.socket_env_var()
+            env['KITTY_PREWARM_SOCKET'] = boss.prewarm.socket_env_var()
             env['KITTY_PREWARM_SOCKET_REAL_TTY'] = ' ' * 32
         if self.cwd:
             # needed in case cwd is a symlink, in which case shells
diff --git a/kitty/main.py b/kitty/main.py
index bdf46dd64..0aed81c5d 100644
--- a/kitty/main.py
+++ b/kitty/main.py
@@ -325,7 +325,6 @@ def setup_environment(opts: Options, cli_opts: CLIOptions) -> None:
         from_config_file = True
     if cli_opts.listen_on:
         cli_opts.listen_on = expand_listen_on(cli_opts.listen_on, from_config_file)
-        os.environ['KITTY_LISTEN_ON'] = cli_opts.listen_on
     env = opts.env.copy()
     ensure_kitty_in_path()
     kitty_path = shutil.which('kitty')