version 0.25.0

Fixes #4913

.github/workflows/ci.py

@@ -30,13 +30,17 @@ def install_deps():
     print('Installing kitty dependencies...')
     sys.stdout.flush()
     if is_macos:
-        items = (x.strip() for x in open('Brewfile').readlines() if not x.startswith('#'))
-        run('brew', 'install', *items)
+        items = (x.split()[1].strip('"') for x in open('Brewfile').readlines() if x.strip().startswith('brew '))
+        run('brew', 'install', 'fish', *items)
     else:
         run('sudo apt-get update')
         run('sudo apt-get install -y libgl1-mesa-dev libxi-dev libxrandr-dev libxinerama-dev ca-certificates'
-            ' libxcursor-dev libxcb-xkb-dev libdbus-1-dev libxkbcommon-dev libharfbuzz-dev libx11-xcb-dev'
-            ' libpng-dev liblcms2-dev libfontconfig-dev libxkbcommon-x11-dev libcanberra-dev librsync-dev uuid-dev')
+            ' libxcursor-dev libxcb-xkb-dev libdbus-1-dev libxkbcommon-dev libharfbuzz-dev libx11-xcb-dev zsh'
+            ' libpng-dev liblcms2-dev libfontconfig-dev libxkbcommon-x11-dev libcanberra-dev librsync-dev uuid-dev'
+            ' zsh bash dash')
+        # for some reason these directories are world writable which causes zsh
+        # compinit to break
+        run('sudo chmod -R og-w /usr/share/zsh')
     if is_bundle:
         install_bundle()
     else:
@@ -48,7 +52,7 @@ def install_deps():
 
 def build_kitty():
     python = shutil.which('python3') if is_bundle else sys.executable
-    cmd = '{} setup.py build --verbose'.format(python)
+    cmd = f'{python} setup.py build --verbose'
     if os.environ.get('KITTY_SANITIZE') == '1':
         cmd += ' --debug --sanitize'
     run(cmd)
@@ -59,8 +63,8 @@ def test_kitty():
 
 
 def package_kitty():
-    py = 'python3' if is_macos else 'python'
-    run(py + ' setup.py linux-package --update-check-interval=0 --verbose')
+    python = 'python3' if is_macos else 'python'
+    run(f'{python} setup.py linux-package --update-check-interval=0 --verbose')
     if is_macos:
         run('python3 setup.py kitty.app --update-check-interval=0 --verbose')
         run('kitty.app/Contents/MacOS/kitty +runpy "from kitty.constants import *; print(kitty_exe())"')
@@ -76,11 +80,11 @@ def replace_in_file(path, src, dest):
 def setup_bundle_env():
     global SW
     os.environ['SW'] = SW = '/Users/Shared/kitty-build/sw/sw' if is_macos else os.path.join(os.environ['GITHUB_WORKSPACE'], 'sw')
-    os.environ['PKG_CONFIG_PATH'] = SW + '/lib/pkgconfig'
+    os.environ['PKG_CONFIG_PATH'] = os.path.join(SW, 'lib', 'pkgconfig')
     if is_macos:
         os.environ['PATH'] = '{}:{}'.format('/usr/local/opt/sphinx-doc/bin', os.environ['PATH'])
     else:
-        os.environ['LD_LIBRARY_PATH'] = SW + '/lib'
+        os.environ['LD_LIBRARY_PATH'] = os.path.join(SW, 'lib')
         os.environ['PYTHONHOME'] = SW
     os.environ['PATH'] = '{}:{}'.format(os.path.join(SW, 'bin'), os.environ['PATH'])
 
@@ -111,7 +115,7 @@ def main():
         setup_bundle_env()
     else:
         if not is_macos and 'pythonLocation' in os.environ:
-            os.environ['LD_LIBRARY_PATH'] = '{}/lib'.format(os.environ['pythonLocation'])
+            os.environ['LD_LIBRARY_PATH'] = os.path.join(os.environ['pythonLocation'], 'lib')
     action = sys.argv[-1]
     if action in ('build', 'package'):
         install_deps()
@@ -122,7 +126,7 @@ def main():
     elif action == 'test':
         test_kitty()
     else:
-        raise SystemExit('Unknown action: ' + action)
+        raise SystemExit(f'Unknown action: {action}')
 
 
 if __name__ == '__main__':

.github/workflows/ci.yml

@@ -42,11 +42,11 @@ jobs:
 
         steps:
             - name: Checkout source code
-              uses: actions/checkout@master
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 10
             - name: Set up Python ${{ matrix.pyver }}
-              uses: actions/setup-python@master
+              uses: actions/setup-python@v3
               with:
                 python-version: ${{ matrix.pyver }}
 
@@ -63,7 +63,7 @@ jobs:
             CFLAGS: -funsigned-char
         steps:
             - name: Checkout source code
-              uses: actions/checkout@master
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 10
 
@@ -71,7 +71,7 @@ jobs:
               run: if grep -Inr '\s$' kitty kitty_tests kittens docs *.py *.asciidoc *.rst .gitattributes .gitignore; then echo Trailing whitespace found, aborting.; exit 1; fi
 
             - name: Set up Python
-              uses: actions/setup-python@master
+              uses: actions/setup-python@v3
               with:
                 python-version: 3.8
 
@@ -88,10 +88,10 @@ jobs:
               run: python test.py mypy
 
             - name: Build man page
-              run: make FAIL_WARN=-W man
+              run: make FAIL_WARN=1 man
 
             - name: Build HTML docs
-              run: make FAIL_WARN=-W html
+              run: make FAIL_WARN=1 html
 
     bundle:
         name: Bundle test (${{ matrix.os }})

.github/workflows/codeql-analysis.yml

@@ -16,7 +16,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         # We must fetch at least the immediate parents so that if this is
         # a pull request then we can checkout the head.
@@ -27,6 +27,7 @@ jobs:
       uses: github/codeql-action/init@v1
       with:
         languages: python, c
+        setup-python-dependencies: false
 
     - name: Build kitty
       run: python3 .github/workflows/ci.py build

Brewfile

@@ -1,7 +1,7 @@
-pkg-config
-zlib
-librsync
-python
-imagemagick
-harfbuzz
-sphinx-doc
+brew "pkg-config"
+brew "zlib"
+brew "librsync"
+brew "python"
+brew "imagemagick"
+brew "harfbuzz"
+brew "sphinx-doc"

Makefile

@@ -5,6 +5,10 @@ ifdef VERBOSE
 	VVAL=--verbose
 endif
 
+ifdef FAIL_WARN
+export FAIL_WARN
+endif
+
 all:
 	python3 setup.py $(VVAL)
 
@@ -32,13 +36,16 @@ app:
 	python3 setup.py kitty.app $(VVAL)
 
 man:
-	$(MAKE) FAIL_WARN=$(FAIL_WARN) -C docs man
+	$(MAKE) -C docs man
 
 html:
-	$(MAKE) FAIL_WARN=$(FAIL_WARN) -C docs html
+	$(MAKE) -C docs html
+
+dirhtml:
+	$(MAKE) -C docs dirhtml
 
 linkcheck:
-	$(MAKE) FAIL_WARN=$(FAIL_WARN) -C docs linkcheck
+	$(MAKE) -C docs linkcheck
 
 website:
 	./publish.py --only website

README.asciidoc

@@ -11,4 +11,4 @@ https://www.reddit.com/r/KittyTerminal[Reddit community]
 
 Packaging status in various repositories:
 
-image:https://repology.org/badge/vertical-allrepos/kitty.svg[https://repology.org/project/kitty/versions]
+image:https://repology.org/badge/vertical-allrepos/kitty.svg["Packaging status", link="https://repology.org/project/kitty/versions"]

__main__.py

@@ -31,14 +31,17 @@ def runpy(args: List[str]) -> None:
 
 def hold(args: List[str]) -> None:
     import subprocess
-    import tty
-    from contextlib import suppress
-    ret = subprocess.Popen(args[1:]).wait()
-    with suppress(BaseException):
-        print('\n\x1b[1;32mPress any key to exit', end='', flush=True)
-    with suppress(BaseException):
-        tty.setraw(sys.stdin.fileno())
-        sys.stdin.buffer.read(1)
+    ret = 1
+    try:
+        ret = subprocess.Popen(args[1:]).wait()
+    except KeyboardInterrupt:
+        pass
+    except FileNotFoundError:
+        print(f'Could not find {args[1]!r} to execute', file=sys.stderr)
+    except Exception as e:
+        print(e, file=sys.stderr)
+    from kitty.utils import hold_till_enter
+    hold_till_enter()
     raise SystemExit(ret)
 
 
@@ -47,6 +50,13 @@ def complete(args: List[str]) -> None:
     complete_main(args[1:], entry_points, namespaced_entry_points)
 
 
+def open_urls(args: List[str]) -> None:
+    setattr(sys, 'cmdline_args_for_open', True)
+    sys.argv = ['kitty'] + args[1:]
+    from kitty.main import main as kitty_main
+    kitty_main()
+
+
 def launch(args: List[str]) -> None:
     import runpy
     sys.argv = args[1:]
@@ -69,6 +79,27 @@ def launch(args: List[str]) -> None:
     runpy.run_path(exe, run_name='__main__')
 
 
+def shebang(args: List[str]) -> None:
+    script_path = args[1]
+    cmd = args[2:]
+    if cmd == ['__ext__']:
+        cmd = [os.path.splitext(script_path)[1][1:].lower()]
+    try:
+        f = open(script_path)
+    except FileNotFoundError:
+        raise SystemExit(f'The file {script_path} does not exist')
+    with f:
+        if f.read(2) == '#!':
+            line = f.readline().strip()
+            _plat = sys.platform.lower()
+            is_macos: bool = 'darwin' in _plat
+            if is_macos:
+                cmd = line.split(' ')
+            else:
+                cmd = line.split(' ', maxsplit=1)
+    os.execvp(cmd[0], cmd + [script_path])
+
+
 def run_kitten(args: List[str]) -> None:
     try:
         kitten = args[1]
@@ -104,10 +135,6 @@ entry_points = {
     # These two are here for backwards compat
     'icat': icat,
     'list-fonts': list_fonts,
-    'runpy': runpy,
-    'launch': launch,
-    'kitten': run_kitten,
-    'edit-config': edit_config_file,
 
     '@': remote_control,
     '+': namespaced,
@@ -115,6 +142,12 @@ entry_points = {
 namespaced_entry_points = {k: v for k, v in entry_points.items() if k[0] not in '+@'}
 namespaced_entry_points['hold'] = hold
 namespaced_entry_points['complete'] = complete
+namespaced_entry_points['runpy'] = runpy
+namespaced_entry_points['launch'] = launch
+namespaced_entry_points['open'] = open_urls
+namespaced_entry_points['kitten'] = run_kitten
+namespaced_entry_points['edit-config'] = edit_config_file
+namespaced_entry_points['shebang'] = shebang
 
 
 def setup_openssl_environment() -> None:

build-terminfo

@@ -13,7 +13,7 @@ import tempfile
 
 def compile_terminfo(base):
     with tempfile.TemporaryDirectory() as tdir:
-        proc = subprocess.run(['tic', '-x', '-o' + tdir, 'terminfo/kitty.terminfo'], check=True, stderr=subprocess.PIPE)
+        proc = subprocess.run(['tic', '-x', f'-o{tdir}', 'terminfo/kitty.terminfo'], check=True, stderr=subprocess.PIPE)
         regex = '^"terminfo/kitty.terminfo", line [0-9]+, col [0-9]+, terminal \'xterm-kitty\': older tic versions may treat the description field as an alias$'
         for error in proc.stderr.decode('utf-8').splitlines():
             if not re.match(regex, error):

bypy/init_env.py

@@ -26,7 +26,7 @@ def initialize_constants():
     kitty_constants = {}
     src = read_src_file('constants.py')
     nv = re.search(r'Version\((\d+), (\d+), (\d+)\)', src)
-    kitty_constants['version'] = '%s.%s.%s' % (nv.group(1), nv.group(2), nv.group(3))
+    kitty_constants['version'] = f'{nv.group(1)}.{nv.group(2)}.{nv.group(3)}'
     kitty_constants['appname'] = re.search(
             r'appname: str\s+=\s+(u{0,1})[\'"]([^\'"]+)[\'"]', src
     ).group(2)

bypy/linux/__main__.py

@@ -35,10 +35,11 @@ def binary_includes():
             'expat', 'sqlite3', 'ffi', 'z', 'lzma', 'png16', 'lcms2', 'crypt',
             'iconv', 'pcre', 'graphite2', 'glib-2.0', 'freetype', 'rsync',
             'harfbuzz', 'xkbcommon', 'xkbcommon-x11',
-            'ncursesw', 'readline', 'brotlicommon', 'brotlienc', 'brotlidec'
+            'ncursesw', 'readline', 'brotlicommon', 'brotlienc', 'brotlidec',
+            'wayland-client', 'wayland-cursor',
         ))) + (
                 get_dll_path('bz2', 2), get_dll_path('ssl', 2), get_dll_path('crypto', 2),
-                get_dll_path('python' + py_ver, 2),
+                get_dll_path(f'python{py_ver}', 2),
         )
 
 
@@ -47,7 +48,7 @@ class Env:
     def __init__(self, package_dir):
         self.base = package_dir
         self.lib_dir = j(self.base, 'lib')
-        self.py_dir = j(self.lib_dir, 'python' + py_ver)
+        self.py_dir = j(self.lib_dir, f'python{py_ver}')
         os.makedirs(self.py_dir)
         self.bin_dir = j(self.base, 'bin')
         self.obj_dir = mkdtemp('launchers-')
@@ -107,7 +108,7 @@ def add_ca_certs(env):
 
 def copy_python(env):
     print('Copying python...')
-    srcdir = j(PREFIX, 'lib/python' + py_ver)
+    srcdir = j(PREFIX, f'lib/python{py_ver}')
 
     for x in os.listdir(srcdir):
         y = j(srcdir, x)
@@ -187,11 +188,11 @@ def strip_files(files, argv_max=(256 * 1024)):
 
 
 def strip_binaries(files):
-    print('Stripping %d files...' % len(files))
+    print(f'Stripping {len(files)} files...')
     before = sum(os.path.getsize(x) for x in files)
     strip_files(files)
     after = sum(os.path.getsize(x) for x in files)
-    print('Stripped %.1f MB' % ((before - after) / (1024 * 1024.)))
+    print('Stripped {:.1f} MB'.format((before - after) / (1024 * 1024.)))
 
 
 def create_tarfile(env, compression_level='9'):
@@ -203,7 +204,7 @@ def create_tarfile(env, compression_level='9'):
         if err.errno != errno.ENOENT:
             raise
     os.mkdir(base)
-    dist = os.path.join(base, '%s-%s-%s.tar' % (kitty_constants['appname'], kitty_constants['version'], arch))
+    dist = os.path.join(base, f'{kitty_constants["appname"]}-{kitty_constants["version"]}-{arch}.tar')
     with tarfile.open(dist, mode='w', format=tarfile.PAX_FORMAT) as tf:
         cwd = os.getcwd()
         os.chdir(env.base)
@@ -213,13 +214,13 @@ def create_tarfile(env, compression_level='9'):
         finally:
             os.chdir(cwd)
     print('Compressing archive...')
-    ans = dist.rpartition('.')[0] + '.txz'
+    ans = f'{dist.rpartition(".")[0]}.txz'
     start_time = time.time()
-    subprocess.check_call(['xz', '--threads=0', '-f', '-' + compression_level, dist])
+    subprocess.check_call(['xz', '--threads=0', '-f', f'-{compression_level}', dist])
     secs = time.time() - start_time
-    print('Compressed in %d minutes %d seconds' % (secs // 60, secs % 60))
-    os.rename(dist + '.xz', ans)
-    print('Archive %s created: %.2f MB' % (
+    print('Compressed in {} minutes {} seconds'.format(secs // 60, secs % 60))
+    os.rename(f'{dist}.xz', ans)
+    print('Archive {} created: {:.2f} MB'.format(
         os.path.basename(ans), os.stat(ans).st_size / (1024.**2)))
 
 

bypy/macos/__main__.py

@@ -90,7 +90,7 @@ def strip_files(files, argv_max=(256 * 1024)):
 def files_in(folder):
     for record in os.walk(folder):
         for f in record[-1]:
-            yield os.path.join(record[0], f)
+            yield join(record[0], f)
 
 
 def expand_dirs(items, exclude=lambda x: x.endswith('.so')):
@@ -103,7 +103,7 @@ def expand_dirs(items, exclude=lambda x: x.endswith('.so')):
 
 
 def do_sign(app_dir):
-    with current_dir(os.path.join(app_dir, 'Contents')):
+    with current_dir(join(app_dir, 'Contents')):
         # Sign all .so files
         so_files = {x for x in files_in('.') if x.endswith('.so')}
         codesign(so_files)
@@ -153,7 +153,7 @@ class Freeze(object):
         self.to_strip = []
         self.warnings = []
         self.py_ver = py_ver
-        self.python_stdlib = join(self.resources_dir, 'Python', 'lib', 'python' + self.py_ver)
+        self.python_stdlib = join(self.resources_dir, 'Python', 'lib', f'python{self.py_ver}')
         self.site_packages = self.python_stdlib  # hack to avoid needing to add site-packages to path
         self.obj_dir = mkdtemp('launchers-')
 
@@ -177,7 +177,7 @@ class Freeze(object):
             self.run_tests()
         # self.run_shell()
 
-        ret = self.makedmg(self.build_dir, APPNAME + '-' + VERSION)
+        ret = self.makedmg(self.build_dir, f'{APPNAME}-{VERSION}')
 
         return ret
 
@@ -185,8 +185,17 @@ class Freeze(object):
     def add_ca_certs(self):
         print('\nDownloading CA certs...')
         from urllib.request import urlopen
-        cdata = urlopen(kitty_constants['cacerts_url']).read()
-        dest = os.path.join(self.contents_dir, 'Resources', 'cacert.pem')
+        cdata = None
+        for i in range(5):
+            try:
+                cdata = urlopen(kitty_constants['cacerts_url']).read()
+                break
+            except Exception as e:
+                print(f'Downloading CA certs failed with error: {e}, retrying...')
+
+        if cdata is None:
+            raise SystemExit('Downloading C certs failed, giving up')
+        dest = join(self.contents_dir, 'Resources', 'cacert.pem')
         with open(dest, 'wb') as f:
             f.write(cdata)
 
@@ -197,7 +206,7 @@ class Freeze(object):
 
     @flush
     def run_tests(self):
-        iv['run_tests'](os.path.join(self.contents_dir, 'MacOS', 'kitty'))
+        iv['run_tests'](join(self.contents_dir, 'MacOS', 'kitty'))
 
     @flush
     def set_id(self, path_to_lib, new_id):
@@ -222,10 +231,10 @@ class Freeze(object):
     @flush
     def get_local_dependencies(self, path_to_lib):
         for x, is_id in self.get_dependencies(path_to_lib):
-            for y in (PREFIX + '/lib/', PREFIX + '/python/Python.framework/', '@rpath/'):
+            for y in (f'{PREFIX}/lib/', f'{PREFIX}/python/Python.framework/', '@rpath/'):
                 if x.startswith(y):
-                    if y == PREFIX + '/python/Python.framework/':
-                        y = PREFIX + '/python/'
+                    if y == f'{PREFIX}/python/Python.framework/':
+                        y = f'{PREFIX}/python/'
                     yield x, x[len(y):], is_id
                     break
 
@@ -239,7 +248,7 @@ class Freeze(object):
         self.to_strip.append(path_to_lib)
         old_mode = flipwritable(path_to_lib)
         for dep, bname, is_id in self.get_local_dependencies(path_to_lib):
-            ndep = self.FID + '/' + bname
+            ndep = f'{self.FID}/{bname}'
             self.change_dep(dep, ndep, is_id, path_to_lib)
         ldeps = list(self.get_local_dependencies(path_to_lib))
         if ldeps:
@@ -252,7 +261,7 @@ class Freeze(object):
     @flush
     def add_python_framework(self):
         print('\nAdding Python framework')
-        src = join(PREFIX + '/python', 'Python.framework')
+        src = join(f'{PREFIX}/python', 'Python.framework')
         x = join(self.frameworks_dir, 'Python.framework')
         curr = os.path.realpath(join(src, 'Versions', 'Current'))
         currd = join(x, 'Versions', basename(curr))
@@ -262,12 +271,12 @@ class Freeze(object):
         shutil.copy2(join(curr, 'Python'), currd)
         self.set_id(
             join(currd, 'Python'),
-            self.FID + '/Python.framework/Versions/%s/Python' % basename(curr))
+            f'{self.FID}/Python.framework/Versions/{basename(curr)}/Python')
         # The following is needed for codesign
         with current_dir(x):
             os.symlink(basename(curr), 'Versions/Current')
             for y in ('Python', 'Resources'):
-                os.symlink('Versions/Current/%s' % y, y)
+                os.symlink(f'Versions/Current/{y}', y)
 
     @flush
     def install_dylib(self, path, set_id=True):
@@ -275,7 +284,7 @@ class Freeze(object):
         if set_id:
             self.set_id(
                 join(self.frameworks_dir, basename(path)),
-                self.FID + '/' + basename(path))
+                f'{self.FID}/{basename(path)}')
         self.fix_dependencies_in_lib(join(self.frameworks_dir, basename(path)))
 
     @flush
@@ -291,11 +300,11 @@ class Freeze(object):
                 'rsync.2',
         ):
             print('\nAdding', x)
-            x = 'lib%s.dylib' % x
+            x = f'lib{x}.dylib'
             src = join(PREFIX, 'lib', x)
             shutil.copy2(src, self.frameworks_dir)
             dest = join(self.frameworks_dir, x)
-            self.set_id(dest, self.FID + '/' + x)
+            self.set_id(dest, f'{self.FID}/{x}')
             self.fix_dependencies_in_lib(dest)
 
     @flush
@@ -321,7 +330,7 @@ class Freeze(object):
     @flush
     def add_stdlib(self):
         print('\nAdding python stdlib')
-        src = PREFIX + '/python/Python.framework/Versions/Current/lib/python' + self.py_ver
+        src = f'{PREFIX}/python/Python.framework/Versions/Current/lib/python{self.py_ver}'
         dest = self.python_stdlib
         if not os.path.exists(dest):
             os.makedirs(dest)
@@ -345,19 +354,19 @@ class Freeze(object):
         kitty_dir = join(self.resources_dir, 'kitty')
         bases = ('kitty', 'kittens', 'kitty_tests')
         for x in bases:
-            dest = os.path.join(self.python_stdlib, x)
-            os.rename(os.path.join(kitty_dir, x), dest)
+            dest = join(self.python_stdlib, x)
+            os.rename(join(kitty_dir, x), dest)
             if x == 'kitty':
-                shutil.rmtree(os.path.join(dest, 'launcher'))
-        os.rename(os.path.join(kitty_dir, '__main__.py'), os.path.join(self.python_stdlib, 'kitty_main.py'))
-        shutil.rmtree(os.path.join(kitty_dir, '__pycache__'))
-        pdir = os.path.join(dirname(self.python_stdlib), 'kitty-extensions')
+                shutil.rmtree(join(dest, 'launcher'))
+        os.rename(join(kitty_dir, '__main__.py'), join(self.python_stdlib, 'kitty_main.py'))
+        shutil.rmtree(join(kitty_dir, '__pycache__'))
+        pdir = join(dirname(self.python_stdlib), 'kitty-extensions')
         os.mkdir(pdir)
         print('Extracting extension modules from', self.python_stdlib, 'to', pdir)
         ext_map = extract_extension_modules(self.python_stdlib, pdir)
-        shutil.copy(os.path.join(os.path.dirname(self_dir), 'site.py'), os.path.join(self.python_stdlib, 'site.py'))
+        shutil.copy(join(os.path.dirname(self_dir), 'site.py'), join(self.python_stdlib, 'site.py'))
         for x in bases:
-            iv['sanitize_source_folder'](os.path.join(self.python_stdlib, x))
+            iv['sanitize_source_folder'](join(self.python_stdlib, x))
         self.compile_py_modules()
         freeze_python(self.python_stdlib, pdir, self.obj_dir, ext_map, develop_mode_env_var='KITTY_DEVELOP_FROM', remove_pyc_files=True)
         iv['build_frozen_launcher']([path_to_freeze_dir(), self.obj_dir])
@@ -434,23 +443,23 @@ class Freeze(object):
         ''' Copy a directory d into a dmg named volname '''
         print('\nMaking dmg...')
         sys.stdout.flush()
-        destdir = os.path.join(SW, 'dist')
+        destdir = join(SW, 'dist')
         try:
             shutil.rmtree(destdir)
         except FileNotFoundError:
             pass
         os.mkdir(destdir)
-        dmg = os.path.join(destdir, volname + '.dmg')
+        dmg = join(destdir, f'{volname}.dmg')
         if os.path.exists(dmg):
             os.unlink(dmg)
         tdir = tempfile.mkdtemp()
-        appdir = os.path.join(tdir, os.path.basename(d))
+        appdir = join(tdir, os.path.basename(d))
         shutil.copytree(d, appdir, symlinks=True)
         if self.sign_installers:
             with timeit() as times:
                 sign_app(appdir, self.notarize)
-            print('Signing completed in %d minutes %d seconds' % tuple(times))
-        os.symlink('/Applications', os.path.join(tdir, 'Applications'))
+            print('Signing completed in {} minutes {} seconds'.format(*times))
+        os.symlink('/Applications', join(tdir, 'Applications'))
         size_in_mb = int(
             subprocess.check_output(['du', '-s', '-k', tdir]).decode('utf-8')
             .split()[0]) / 1024.
@@ -466,10 +475,10 @@ class Freeze(object):
         print('\nCreating dmg...')
         with timeit() as times:
             subprocess.check_call(cmd + [dmg])
-        print('dmg created in %d minutes and %d seconds' % tuple(times))
+        print('dmg created in {} minutes and {} seconds'.format(*times))
         shutil.rmtree(tdir)
         size = os.stat(dmg).st_size / (1024 * 1024.)
-        print('\nInstaller size: %.2fMB\n' % size)
+        print(f'\nInstaller size: {size:.2f}MB\n')
         return dmg
 
 
@@ -477,7 +486,7 @@ def main():
     args = globals()['args']
     ext_dir = globals()['ext_dir']
     Freeze(
-        os.path.join(ext_dir, kitty_constants['appname'] + '.app'),
+        join(ext_dir, f'{kitty_constants["appname"]}.app'),
         dont_strip=args.dont_strip,
         sign_installers=args.sign_installers,
         notarize=args.notarize,

bypy/sources.json

@@ -166,8 +166,8 @@
     {
         "name": "pygments",
         "unix": {
-            "filename": "Pygments-2.7.4.tar.gz",
-            "hash": "sha256:df49d09b498e83c1a73128295860250b0b7edd4c723a32e9bc0d295c7c2ec337",
+            "filename": "Pygments-2.11.2.tar.gz",
+            "hash": "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a",
             "urls": ["pypi"]
         }
     },
@@ -273,8 +273,8 @@
         "name": "wayland",
         "os": "linux",
         "unix": {
-            "filename": "wayland-1.19.0.tar.xz",
-            "hash": "sha256:baccd902300d354581cd5ad3cc49daa4921d55fb416a5883e218750fef166d15",
+            "filename": "wayland-1.20.0.tar.xz",
+            "hash": "sha256:b8a034154c7059772e0fdbd27dbfcda6c732df29cae56a82274f6ec5d7cd8725",
             "urls": ["https://wayland.freedesktop.org/releases/{filename}"]
         }
     },
@@ -283,8 +283,8 @@
         "name": "wayland-protocols",
         "os": "linux",
         "unix": {
-            "filename": "wayland-protocols-1.20.tar.xz",
-            "hash": "sha256:9782b7a1a863d82d7c92478497d13c758f52e7da4f197aa16443f73de77e4de7",
+            "filename": "wayland-protocols-1.25.tar.xz",
+            "hash": "sha256:f1ff0f7199d0a0da337217dd8c99979967808dc37731a1e759e822b75b571460",
             "urls": ["https://wayland.freedesktop.org/releases/{filename}"]
         }
     }

docs/Makefile

@@ -1,6 +1,10 @@
 # Minimal makefile for Sphinx documentation
 #
 
+ifdef FAIL_WARN
+override FAIL_WARN=-W
+endif
+
 # You can set these variables from the command line.
 SPHINXOPTS    = -n -q -j auto -T $(FAIL_WARN) $(OPTS)
 SPHINXBUILD   = sphinx-build

docs/_static/custom.css

@@ -28,3 +28,9 @@ details > summary {
     text-decoration-color: var(--color-link-underline);
     text-decoration-line: underline;
 }
+
+/* pygments adds an underline to some white-space, this is particularly visible in
+ * dark mode. Remove it. */
+.highlight .w {
+    text-decoration: none
+}

docs/basic.rst

@@ -77,8 +77,11 @@ You can also define a shortcut to switch to the previously active window::
 
    map ctrl+p nth_window -1
 
-``nth_window`` will focus the nth window for positive numbers and the
-previously active windows for negative numbers.
+``nth_window`` will focus the nth window for positive numbers (starting from
+zero) and the previously active windows for negative numbers.
+
+To switch to the nth OS window, you can define ``nth_os_window``. Only positive
+numbers are accepted, starting from one.
 
 .. _detach_window:
 
@@ -127,12 +130,12 @@ Restore font size                   :sc:`reset_font_size` (also :kbd:`⌘+0` on
 Toggle fullscreen                   :sc:`toggle_fullscreen` (also :kbd:`⌃+⌘+f` on macOS)
 Toggle maximized                    :sc:`toggle_maximized`
 Input unicode character             :sc:`input_unicode_character` (also :kbd:`⌃+⌘+space` on macOS)
-Click URL using the keyboard        :sc:`open_url`
-Reset the terminal                  :sc:`reset_terminal`
+Open URL in web browser             :sc:`open_url`
+Reset the terminal                  :sc:`reset_terminal` (also :kbd:`⌥+⌘+r` on macOS)
 Reload :file:`kitty.conf`           :sc:`reload_config_file` (also :kbd:`⌃+⌘+,` on macOS)
-Debug :file:`kitty.conf`            :sc:`debug_config` (also :kbd:`⌘+⌥+f6` on macOS)
+Debug :file:`kitty.conf`            :sc:`debug_config` (also :kbd:`⌥+⌘+,` on macOS)
 Pass current selection to program   :sc:`pass_selection_to_program`
-Edit |kitty| config file            :sc:`edit_config_file`
+Edit |kitty| config file            :sc:`edit_config_file` (also :kbd:`⌘+,` on macOS)
 Open a |kitty| shell                :sc:`kitty_shell`
 Increase background opacity         :sc:`increase_background_opacity`
 Decrease background opacity         :sc:`decrease_background_opacity`

docs/binary.rst

@@ -49,12 +49,15 @@ particular desktop, but it should work for most major desktop environments.
 .. code-block:: sh
 
     # Create a symbolic link to add kitty to PATH (assuming ~/.local/bin is in
-    # your PATH)
+    # your system-wide PATH)
     ln -s ~/.local/kitty.app/bin/kitty ~/.local/bin/
     # Place the kitty.desktop file somewhere it can be found by the OS
     cp ~/.local/kitty.app/share/applications/kitty.desktop ~/.local/share/applications/
-    # Update the path to the kitty icon in the kitty.desktop file
-    sed -i "s|Icon=kitty|Icon=/home/$USER/.local/kitty.app/share/icons/hicolor/256x256/apps/kitty.png|g" ~/.local/share/applications/kitty.desktop
+    # If you want to open text files and images in kitty via your file manager also add the kitty-open.desktop file
+    cp ~/.local/kitty.app/share/applications/kitty-open.desktop ~/.local/share/applications/
+    # Update the paths to the kitty and its icon in the kitty.desktop file(s)
+    sed -i "s|Icon=kitty|Icon=/home/$USER/.local/kitty.app/share/icons/hicolor/256x256/apps/kitty.png|g" ~/.local/share/applications/kitty*.desktop
+    sed -i "s|Exec=kitty|Exec=/home/$USER/.local/kitty.app/bin/kitty|g" ~/.local/share/applications/kitty*.desktop
 
 .. note::
     If you use the venerable `stow <https://www.gnu.org/software/stow/>`_

docs/build.rst

@@ -125,6 +125,19 @@ make them available in the newly spawned shell.
 Then proceed with ``make`` or ``make app`` according to the platform specific instructions above.
 
 
+Debug builds
+--------------
+
+A basic debug build can be done with::
+
+    make debug
+
+This includes debug info in the binary for better traces. To build with address sanitizer, use::
+
+    make asan
+
+Which will result in a debug binary that uses the address sanitizer as well.
+
 .. _packagers:
 
 Notes for Linux/macOS packagers
@@ -173,32 +186,4 @@ servers into which they ssh, without needing to install all of |kitty|.
    ``python -m pip -r docs/requirements.txt``.
 
 This applies to creating packages for |kitty| for macOS package managers such as
-brew or MacPorts as well.
-
-
-Changing defaults for packages
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-|kitty| has its defaults chosen to be suitable for a standalone distributable.
-If you are packaging it a few of these might need to be changed.
-
-update-checking
-   |kitty| has its own update check mechanism, if you would like to turn
-   it off for your package, use::
-
-       ./setup.py linux-package --update-check-interval=0
-
-shell-integration
-   |kitty| by default injects its :ref:`shell_integration` code into the user's
-   shell using environment variables or (for bash only) modifying
-   the user's :file:`~/.bashrc` file.
-   For a package, it might make more sense to distribute the shell
-   integration scripts into the system-wide shell vendor locations. The
-   shell integration files are found in the :file:`shell-integration`
-   directory. Copy them to the system wide shell vendor locations for each
-   shell, and use::
-
-       ./setup.py linux-package --shell-integration=enabled\ no-rc
-
-   This will prevent kitty from modifying the user's shell environment to load
-   the integration scripts.
+Homebrew or MacPorts as well.

docs/changelog.rst

@@ -4,26 +4,19 @@ Changelog
 |kitty| is a feature-rich, cross-platform, *fast*, GPU based terminal.
 To update |kitty|, :doc:`follow the instructions <binary>`.
 
+.. recent major features {{{
+
 Recent major new features
 ---------------------------
 
-.. only:: dirhtml
+Truly convenient SSH
+~~~~~~~~~~~~~~~~~~~~~~~~
 
-   A demo video
-   ~~~~~~~~~~~~~~~~
+The :doc:`ssh kitten <kittens/ssh>` is redesigned with powerful new features:
 
-   A new video showcasing some of kitty's many productivity enhancing features.
-
-   .. raw:: html
-
-        <video controls width="640" height="360">
-            <source src="https://download.calibre-ebook.com/videos/kitty.mp4" type="video/mp4">
-            <source src="https://download.calibre-ebook.com/videos/kitty.webm" type="video/webm">
-        </video>
-
-   .. rst-class:: caption caption-text
-
-       Watch kitty in action!
+* Automatic :ref:`shell_integration` on remote machines
+* Easily :ref:`clone local shell/editor config <real_world_ssh_kitten_config>` on remote machines
+* Automatic :opt:`re-use of existing connections <kitten-ssh.share_connections>` to avoid connection setup latency
 
 
 Shell integration
@@ -36,43 +29,248 @@ scrollback by pressing :sc:`scroll_to_previous_prompt` and clicking with the
 mouse anywhere in the current command to move the cursor there. See
 :doc:`shell-integration` for details.
 
-Logos for windows
-~~~~~~~~~~~~~~~~~~~~
-
-kitty has the ability to display arbitrary ``PNG`` images at the corner
-of any kitty window to serve as a *logo* for quick window identification.
-Can be controlled via :opt:`window_logo_path` and also and changed via
-the :ref:`at_set-window-logo` remote control command.
-Useful to quickly identify special windows or just for some *bling*.
-
-.. figure:: screenshots/window-logo.png
-    :alt: A screenshot of the kitty shell window showing the kitty logo
-    :align: center
-    :width: 100%
-
-    A screenshot of the kitty shell window showing the kitty logo
-
-
-Visual keyboard based window select
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Select a kitty window visually using the keyboard, like the
-:doc:`kittens/hints`, but for kitty windows. Can also be used from shell
-scripts/third party integrations via the :ref:`at_select-window` remote control
-command.
-
-.. figure:: screenshots/select-window.png
-    :alt: A screenshot of the kitty select window function
-    :align: center
-    :width: 100%
-
-    Hints for selecting kitty windows visually
-
-
+.. }}}
 
 Detailed list of changes
 -------------------------------------
 
+0.25.0 [2022-04-11]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- :doc:`kittens/ssh`: automatic shell integration when using SSH. Easily
+  clone local shell and editor configuration on remote machines, and automatic
+  re-use of existing connections to avoid connection setup latency.
+
+- When pasting URLs at shell prompts automatically quote them. Also allow filtering pasted text and confirm pastes. See :opt:`paste_actions` for details. (:iss:`4873`)
+
+- macOS: When using Apple's less as the pager for viewing scrollback strip out OSC codes as it cant parse them (:iss:`4788`)
+
+- diff kitten: Fix incorrect rendering in rare circumstances when scrolling after changing the context size (:iss:`4831`)
+
+- icat kitten: Fix a regression that broke :option:`kitty +kitten icat --print-window-size` (:pull:`4818`)
+
+- Wayland: Fix :opt:`hide_window_decorations` causing docked windows to be resized on blur (:iss:`4797`)
+
+- Bash integration: Prevent shell integration code from running twice if user enables both automatic and manual integration
+
+- Bash integration: Handle existing PROMPT_COMMAND ending with a literal newline
+
+- Fix continued lines not having their continued status reset on line feed (:iss:`4837`)
+
+- macOS: Allow the New kitty Tab/Window Here services to open multiple selected folders.  (:pull:`4848`)
+
+- Wayland: Fix a regression that broke IME when changing windows/tabs (:iss:`4853`)
+
+- macOS: Fix Unicode paths not decoded correctly when dropping files (:pull:`4879`)
+
+- Avoid flicker when starting kittens such as the hints kitten (:iss:`4674`)
+
+- A new action :ac:`scroll_prompt_to_top` to move the current prompt to the top (:pull:`4891`)
+
+- :ac:`select_tab`: Use stable numbers when selecting the tab (:iss:`4792`)
+
+- Only check for updates in the official binary builds. Distro packages or source builds will no longer check for updates, regardless of the
+  value of :opt:`update_check_interval`.
+
+- Fix :opt:`inactive_text_alpha` still being applied to the cursor hidden window after focus (:iss:`4928`)
+
+- Fix resizing window that is extra tall/wide because of left-over cells not
+  working reliably (:iss:`4913`)
+
+- A new action :ac:`close_other_tabs_in_os_window` to close other tabs in the active OS window (:pull:`4944`)
+
+
+0.24.4 [2022-03-03]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Shell integration: Fix the default Bash :code:`$HISTFILE` changing to :file:`~/.sh_history` instead of :file:`~/.bash_history` (:iss:`4765`)
+
+- Linux binaries: Fix binaries not working on systems with older Wayland client libraries (:iss:`4760`)
+
+- Fix a regression in the previous release that broke kittens launched with :code:`STDIN` not connected to a terminal (:iss:`4763`)
+
+- Wayland: Fix surface configure events not being acknowledged before commit
+  the resized buffer (:pull:`4768`)
+
+
+0.24.3 [2022-02-28]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Bash integration: No longer modify :file:`~/.bashrc` to load :ref:`shell integration <shell_integration>`.
+  It is recommended to remove the lines used to load the shell integration from :file:`~/.bashrc` as they are no-ops.
+
+- macOS: Allow kitty to handle various URL types. Can be configured via
+  :ref:`launch_actions` (:pull:`4618`)
+
+- macOS: Add a new service ``Open with kitty`` to open file types that are not
+  recognized by the system (:pull:`4641`)
+
+- Splits layout: A new value for :option:`launch --location` to auto-select the split axis when splitting existing windows.
+  Wide windows are split side-by-side and tall windows are split one-above-the-other
+
+- hints kitten: Fix a regression that broke recognition of path:linenumber:colnumber (:iss:`4675`)
+
+- Fix a regression in the previous release that broke :opt:`active_tab_foreground` (:iss:`4620`)
+
+- Fix :ac:`show_last_command_output` not working when the output is stored
+  partially in the scrollback pager history buffer (:iss:`4435`)
+
+- When dropping URLs/files onto kitty at a shell prompt insert them appropriately quoted and space
+  separated (:iss:`4734`)
+
+- Improve CWD detection when there are multiple foreground processes in the TTY process group
+
+- A new option :opt:`narrow_symbols` to turn off opportunistic wide rendering of private use codepoints
+
+- ssh kitten: Fix location of generated terminfo files on NetBSD (:iss:`4622`)
+
+- A new action to clear the screen up to the line containing the cursor, see
+  :ac:`clear_terminal`
+
+- A new action :ac:`copy_ansi_to_clipboard` to copy the current selection with ANSI formatting codes
+  (:iss:`4665`)
+
+- Linux: Do not rescale fallback fonts to match the main font cell height, instead just
+  set the font size and let FreeType take care of it. This matches
+  rendering on macOS (:iss:`4707`)
+
+- macOS: Fix a regression in the previous release that broke switching input
+  sources by keyboard (:iss:`4621`)
+
+- macOS: Add the default shortcut :kbd:`cmd+k` to clear the terminal screen and
+  scrollback up to the cursor (:iss:`4625`)
+
+- Fix a regression in the previous release that broke strikethrough (:disc:`4632`)
+
+- A new action :ac:`scroll_prompt_to_bottom` to move the current prompt
+  to the bottom, filling in the window from the scrollback (:pull:`4634`)
+
+- Add two special arguments ``@first-line-on-screen`` and ``@last-line-on-screen``
+  for the :doc:`launch <launch>` command to be used for pager positioning.
+  (:iss:`4462`)
+
+- Linux: Fix rendering of emoji when using scalable fonts such as Segoe UI Emoji
+
+- Shell integration: bash: Dont fail if an existing PROMPT_COMMAND ends with a semi-colon (:iss:`4645`)
+
+- Shell integration: bash: Fix rendering of multiline prompts with more than two lines (:iss:`4681`)
+
+- Shell integration: fish: Check fish version 3.3.0+ and exit on outdated versions (:pull:`4745`)
+
+- Shell integration: fish: Fix pipestatus being overwritten (:pull:`4756`)
+
+- Linux: Fix fontconfig alias not being used if the aliased font is dual spaced instead of monospaced (:iss:`4649`)
+
+- macOS: Add an option :opt:`macos_menubar_title_max_length` to control the max length of the window title displayed in the global menubar (:iss:`2132`)
+
+- Fix :opt:`touch_scroll_multiplier` also taking effect in terminal programs such as vim that handle mouse events themselves (:iss:`4680`)
+
+- Fix symbol/PUA glyphs loaded via :opt:`symbol_map` instead of as fallbacks not using following spaces to render larger versions (:iss:`4670`)
+
+- macOS: Fix regression in previous release that caused Apple's global shortcuts to not work if they had never been configured on a particular machine (:iss:`4657`)
+
+- Fix a fast *click, move mouse, click* sequence causing the first click event to be discarded (:iss:`4603`)
+
+- Wayland: Fix wheel mice with line based scrolling being incorrectly handled as high precision devices (:iss:`4694`)
+
+- Wayland: Fix touchpads and high resolution wheels not scrolling at the same speed on monitors with different scales (:iss:`4703`)
+
+- Add an option :opt:`wheel_scroll_min_lines` to set the minimum number of lines for mouse wheel scrolling when using a mouse with a wheel that generates very small offsets when slow scrolling (:pull:`4710`)
+
+- macOS: Make the shortcut to toggle full screen configurable (:pull:`4714`)
+
+- macOS: Fix the mouse cursor being set to arrow after switching desktops or toggling full screen (:pull:`4716`)
+
+- Fix copying of selection after selection has been scrolled off history buffer raising an error (:iss:`4713`)
+
+
+0.24.2 [2022-02-03]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- macOS: Allow opening text files, images and directories with kitty when
+  launched using "Open with" in Finder (:iss:`4460`)
+
+- Allow including config files matching glob patterns in :file:`kitty.conf`
+  (:iss:`4533`)
+
+- Shell integration: Fix bash integration not working when ``PROMPT_COMMAND``
+  is used to change the prompt variables (:iss:`4476`)
+
+- Shell integration: Fix cursor shape not being restored to default when
+  running commands in the shell
+
+- Improve the UI of the ask kitten (:iss:`4545`)
+
+- Allow customizing the placement and formatting of the
+  :opt:`tab_activity_symbol` and :opt:`bell_on_tab` symbols
+  by adding them to the :opt:`tab_title_template` (:iss:`4581`, :pull:`4507`)
+
+- macOS: Persist "Secure Keyboard Entry" across restarts to match the behavior
+  of Terminal.app (:iss:`4471`)
+
+- hints kitten: Fix common single letter extension files not being detected
+  (:iss:`4491`)
+
+- Support dotted and dashed underline styles (:pull:`4529`)
+
+- For the vertical and horizontal layouts have the windows arranged on a ring
+  rather than a plane. This means the first and last window are considered
+  neighbors (:iss:`4494`)
+
+- A new action to clear the current selection (:iss:`4600`)
+
+- Shell integration: fish: Fix cursor shape not working with fish's vi mode
+  (:iss:`4508`)
+
+- Shell integration: fish: Dont override fish's native title setting functionality.
+  See `discussion <https://github.com/fish-shell/fish-shell/issues/8641>`__.
+
+- macOS: Fix hiding via :kbd:`cmd+h` not working on macOS 10.15.7 (:iss:`4472`)
+
+- Draw the dots for braille characters more evenly spaced at all font sizes (:iss:`4499`)
+
+- icat kitten: Add options to mirror images and remove their transparency
+  before displaying them (:iss:`4513`)
+
+- macOS: Respect the users system-wide global keyboard shortcut preferences
+  (:iss:`4501`)
+
+- macOS: Fix a few key-presses causing beeps from Cocoa's text input system
+  (:iss:`4489`)
+
+- macOS: Fix using shortcuts from the global menu bar as subsequent key presses
+  in a multi key mapping not working (:iss:`4519`)
+
+- Fix getting last command output not working correctly when the screen is
+  scrolled (:pull:`4522`)
+
+- Show number of windows per tab in the :ac:`select_tab` action (:pull:`4523`)
+
+- macOS: Fix the shift key not clearing pre-edit text in IME (:iss:`4541`)
+
+- Fix clicking in a window to focus it and typing immediately sometimes having
+  unexpected effects if at a shell prompt (:iss:`4128`)
+
+- themes kitten: Allow writing to a different file than :file:`kitty.conf`.
+
+
+0.24.1 [2022-01-06]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Shell integration: Work around conflicts with some zsh plugins (:iss:`4428`)
+
+- Have the zero width space and various other characters from the *Other,
+  formatting* Unicode category be treated as combining characters (:iss:`4439`)
+
+- Fix using ``--shell-integration`` with :file:`setup.py` broken (:iss:`4434`)
+
+- Fix showing debug information not working if kitty's :file:`STDIN` is not a tty
+  (:iss:`4424`)
+
+- Linux: Fix a regression that broke rendering of emoji with variation selectors
+  (:iss:`4444`)
+
+
 0.24.0 [2022-01-04]
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -331,7 +529,7 @@ Detailed list of changes
 0.22.0 [2021-07-26]
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Add a new :ref:`action-toggle_layout` action to easily zoom/unzoom a window
+- Add a new :ac:`toggle_layout` action to easily zoom/unzoom a window
 
 - When right clicking to extend a selection, move the nearest selection
   boundary rather than the end of the selection. To restore previous behavior

docs/conf.py

@@ -13,7 +13,7 @@ import subprocess
 import sys
 import time
 from functools import partial
-from typing import Any, Callable, Dict, Iterable, List, Match, Optional, Tuple
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
 
 from docutils import nodes
 from docutils.parsers.rst.roles import set_classes
@@ -22,16 +22,13 @@ from pygments.token import (  # type: ignore
     Comment, Keyword, Literal, Name, Number, String, Whitespace
 )
 from sphinx import addnodes, version_info
-from sphinx.builders.html.transforms import KeyboardTransform
 from sphinx.util.logging import getLogger
 
-KeyboardTransform.builders = ('html', 'dirhtml')  # type: ignore
-
 kitty_src = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if kitty_src not in sys.path:
     sys.path.insert(0, kitty_src)
 
-from kitty.conf.types import Definition  # noqa
+from kitty.conf.types import Definition, expand_opt_references  # noqa
 from kitty.constants import str_version, website_url  # noqa
 
 # config {{{
@@ -62,6 +59,7 @@ extensions = [
     'sphinx.ext.ifconfig',
     'sphinx.ext.viewcode',
     'sphinx.ext.githubpages',
+    'sphinx.ext.extlinks',
     'sphinx_copybutton',
     'sphinx_inline_tabs',
     "sphinxext.opengraph",
@@ -170,24 +168,11 @@ texinfo_documents = [
 
 # GitHub linking inline roles {{{
 
-def num_role(
-    which: str, name: str, rawtext: str, text: str, lineno: int, inliner: Any, options: Any = {}, content: Any = []
-) -> Tuple[List[nodes.reference], List[nodes.problematic]]:
-    ' Link to a github issue '
-    try:
-        issue_num = int(text)
-        if issue_num <= 0:
-            raise ValueError
-    except ValueError:
-        msg = inliner.reporter.error(
-            'GitHub issue number must be a number greater than or equal to 1; '
-            '"%s" is invalid.' % text, line=lineno)
-        prb = inliner.problematic(rawtext, rawtext, msg)
-        return [prb], [msg]
-    url = f'https://github.com/kovidgoyal/kitty/{which}/{issue_num}'
-    set_classes(options)
-    node = nodes.reference(rawtext, f'#{issue_num}', refuri=url, **options)
-    return [node], []
+extlinks = {
+    'iss': ('https://github.com/kovidgoyal/kitty/issues/%s', '#%s'),
+    'pull': ('https://github.com/kovidgoyal/kitty/pull/%s', '#%s'),
+    'disc': ('https://github.com/kovidgoyal/kitty/discussions/%s', '#%s'),
+}
 
 
 def commit_role(
@@ -215,6 +200,13 @@ def commit_role(
 def write_cli_docs(all_kitten_names: Iterable[str]) -> None:
     from kitty.cli import option_spec_as_rst
     from kitty.launch import options_spec as launch_options_spec
+    from kittens.ssh.copy import option_text
+    from kittens.ssh.options.definition import copy_message
+    with open('generated/ssh-copy.rst', 'w') as f:
+        f.write(option_spec_as_rst(
+            appname='copy', ospec=option_text, heading_char='^',
+            usage='file-or-dir-to-copy ...', message=copy_message
+        ))
     with open('generated/launch.rst', 'w') as f:
         f.write(option_spec_as_rst(
             appname='launch', ospec=launch_options_spec, heading_char='_',
@@ -232,7 +224,8 @@ if you specify a program-to-run you can use the special placeholder
     from kitty.remote_control import cli_msg, global_options_spec
     with open('generated/cli-kitty-at.rst', 'w') as f:
         p = partial(print, file=f)
-        p('kitty @\n' + '-' * 80)
+        p('kitty @')
+        p('-' * 80)
         p('.. program::', 'kitty @')
         p('\n\n' + as_rst(
             global_options_spec, message=cli_msg, usage='command ...', appname='kitty @'))
@@ -240,7 +233,8 @@ if you specify a program-to-run you can use the special placeholder
         for cmd_name in sorted(all_command_names()):
             func = command_for_name(cmd_name)
             p(f'.. _at_{func.name}:\n')
-            p('kitty @', func.name + '\n' + '-' * 120)
+            p('kitty @', func.name)
+            p('-' * 120)
             p('.. program::', 'kitty @', func.name)
             p('\n\n' + as_rst(*cli_params_for(func)))
     from kittens.runner import get_kitten_cli_docs
@@ -249,12 +243,12 @@ if you specify a program-to-run you can use the special placeholder
         if data:
             with open(f'generated/cli-kitten-{kitten}.rst', 'w') as f:
                 p = partial(print, file=f)
-                p('.. program::', f'kitty +kitten {kitten}')
-                p(f'\nSource code for {kitten}')
+                p('.. program::', 'kitty +kitten', kitten)
+                p('\nSource code for', kitten)
                 p('-' * 72)
                 p(f'\nThe source code for this kitten is `available on GitHub <https://github.com/kovidgoyal/kitty/tree/master/kittens/{kitten}>`_.')
                 p('\nCommand Line Interface')
-                p('-' * 72, file=f)
+                p('-' * 72)
                 p('\n\n' + option_spec_as_rst(
                     data['options'], message=data['help_text'], usage=data['usage'], appname=f'kitty +kitten {kitten}',
                     heading_char='^'))
@@ -379,19 +373,6 @@ def link_role(
     return [node], []
 
 
-def expand_opt_references(conf_name: str, text: str) -> str:
-    conf_name += '.'
-
-    def expand(m: Match[str]) -> str:
-        ref = m.group(1)
-        if '<' not in ref and '.' not in ref:
-            full_ref = conf_name + ref
-            return ':opt:`{} <{}>`'.format(ref, full_ref)
-        return str(m.group())
-
-    return re.sub(r':opt:`(.+?)`', expand, text)
-
-
 opt_aliases: Dict[str, str] = {}
 shortcut_slugs: Dict[str, Tuple[str, str]] = {}
 
@@ -414,7 +395,7 @@ def parse_opt_node(env: Any, sig: str, signode: Any) -> str:
             opt_aliases[optname] = firstname
         count += 1
     if not firstname:
-        raise ValueError('{} is not a valid opt'.format(sig))
+        raise ValueError(f'{sig} is not a valid opt')
     return firstname
 
 
@@ -425,23 +406,33 @@ def parse_shortcut_node(env: Any, sig: str, signode: Any) -> str:
     return sig
 
 
+def parse_action_node(env: Any, sig: str, signode: Any) -> str:
+    """Transform an action description into RST nodes."""
+    signode += addnodes.desc_name(sig, sig)
+    return sig
+
+
 def process_opt_link(env: Any, refnode: Any, has_explicit_title: bool, title: str, target: str) -> Tuple[str, str]:
     conf_name, opt = target.partition('.')[::2]
     if not opt:
         conf_name, opt = 'kitty', conf_name
-    full_name = conf_name + '.' + opt
+    full_name = f'{conf_name}.{opt}'
     return title, opt_aliases.get(full_name, full_name)
 
 
+def process_action_link(env: Any, refnode: Any, has_explicit_title: bool, title: str, target: str) -> Tuple[str, str]:
+    return title, target
+
+
 def process_shortcut_link(env: Any, refnode: Any, has_explicit_title: bool, title: str, target: str) -> Tuple[str, str]:
     conf_name, slug = target.partition('.')[::2]
     if not slug:
         conf_name, slug = 'kitty', conf_name
-    full_name = conf_name + '.' + slug
+    full_name = f'{conf_name}.{slug}'
     try:
         target, stitle = shortcut_slugs[full_name]
     except KeyError:
-        logger.warning('Unknown shortcut: {}'.format(target), location=refnode)
+        logger.warning(f'Unknown shortcut: {target}', location=refnode)
     else:
         if not has_explicit_title:
             title = stitle
@@ -470,6 +461,15 @@ def write_conf_docs(app: Any, all_kitten_names: Iterable[str]) -> None:
     sc_role.process_link = process_shortcut_link
     shortcut_slugs.clear()
 
+    app.add_object_type(
+        'action', 'ac',
+        indextemplate="pair: %s; Action",
+        parse_node=parse_action_node,
+    )
+    ac_role = app.registry.domain_roles['std']['ac']
+    ac_role.warn_dangling = True
+    ac_role.process_link = process_action_link
+
     def generate_default_config(definition: Definition, name: str) -> None:
         with open(f'generated/conf-{name}.rst', 'w', encoding='utf-8') as f:
             print('.. highlight:: conf\n', file=f)
@@ -520,9 +520,6 @@ def setup(app: Any) -> None:
     app.connect('html-page-context', add_html_context)
     app.add_lexer('session', SessionLexer() if version_info[0] < 3 else SessionLexer)
     app.add_role('link', link_role)
-    app.add_role('iss', partial(num_role, 'issues'))
-    app.add_role('pull', partial(num_role, 'pull'))
-    app.add_role('disc', partial(num_role, 'discussions'))
     app.add_role('commit', commit_role)
     # monkey patch sphinx_inline_tabs to avoid a warning about parallel reads
     # see https://github.com/pradyunsg/sphinx-inline-tabs/issues/26

docs/conf.rst

@@ -7,11 +7,11 @@ kitty.conf
 rendering frames-per-second. See below for an overview of all customization
 possibilities.
 
-You can open the config file within kitty by pressing :sc:`edit_config_file`.
+You can open the config file within kitty by pressing :sc:`edit_config_file` (:kbd:`⌘+,` on macOS).
 You can reload the config file within kitty by pressing
 :sc:`reload_config_file` (:kbd:`⌃+⌘+,` on macOS) or sending kitty the ``SIGUSR1`` signal.
-You can also display the current configuration by pressing the :sc:`debug_config`
-(:kbd:`⌘+⌥+f6` on macOS) key.
+You can also display the current configuration by pressing :sc:`debug_config`
+(:kbd:`⌥+⌘+,` on macOS).
 
 .. _confloc:
 
@@ -24,13 +24,21 @@ Comments can be added to the config file as lines starting with the ``#``
 character. This works only if the ``#`` character is the first character
 in the line.
 
+.. _include:
+
 You can include secondary config files via the :code:`include` directive.  If
-you use a relative path for include, it is resolved with respect to the
+you use a relative path for :code:`include`, it is resolved with respect to the
 location of the current config file. Note that environment variables are
 expanded, so :code:`${USER}.conf` becomes :file:`name.conf` if
-:code:`USER=name`.  For example::
+:code:`USER=name`. Also, you can use :code:`globinclude` to include files
+matching a shell glob pattern and :code:`envinclude` to include configuration
+from environment variables. For example::
 
      include other.conf
+     # Include *.conf files from all subdirs of kitty.d inside the kitty config dir
+     globinclude kitty.d/**/*.conf
+     # Include the *contents* of all env vars starting with KITTY_CONF_
+     envinclude KITTY_CONF_*
 
 
 .. note:: Syntax highlighting for :file:`kitty.conf` in vim is available via

docs/deccara.rst

@@ -11,7 +11,7 @@ region of the screen.
 The motivation for this extension is the various problems with the existing
 solution for erasing to background color, namely the *background color erase
 (bce)* capability. See
-`this discussion <https://github.com/kovidgoyal/kitty/issues/160#issuecomment-346470545>`_
+:iss:`this discussion <160#issuecomment-346470545>`
 and `this FAQ <https://invisible-island.net/ncurses/ncurses.faq.html#bce_mismatches>`_
 for a summary of problems with *bce*.
 

docs/faq.rst

@@ -6,21 +6,22 @@ Frequently Asked Questions
 Some special symbols are rendered small/truncated in kitty?
 -----------------------------------------------------------
 
-The number of cells a unicode character takes up are controlled by the unicode
-standard.  All characters are rendered in a single cell unless the unicode
+The number of cells a Unicode character takes up are controlled by the Unicode
+standard. All characters are rendered in a single cell unless the Unicode
 standard says they should be rendered in two cells. When a symbol does not fit,
 it will either be rescaled to be smaller or truncated (depending on how much
 extra space it needs). This is often different from other terminals which just
 let the character overflow into neighboring cells, which is fine if the
 neighboring cell is empty, but looks terrible if it is not.
 
-Some programs, like powerline, vim with fancy gutter symbols/status-bar, etc.
-misuse unicode characters from the private use area to represent symbols. Often
-these symbols are square and should be rendered in two cells.  However, since
-private use area symbols all have their width set to one in the unicode
+Some programs, like Powerline, vim with fancy gutter symbols/status-bar, etc.
+use Unicode characters from the private use area to represent symbols. Often
+these symbols are wide and should be rendered in two cells. However, since
+private use area symbols all have their width set to one in the Unicode
 standard, |kitty| renders them either smaller or truncated. The exception is if
 these characters are followed by a space or empty cell in which case kitty
-makes use of the extra cell to render them in two cells.
+makes use of the extra cell to render them in two cells. This behavior can be
+turned off for specific symbols using :opt:`narrow_symbols`.
 
 
 Using a color theme with a background color does not work well in vim?
@@ -45,30 +46,21 @@ terminfo files to the server::
 
     kitty +kitten ssh myserver
 
-This ssh kitten takes all the same command line arguments
-as ssh, you can alias it to ssh in your shell's rc files to avoid having to
-type it each time::
+This :doc:`ssh kitten <kittens/ssh>` takes all the same command line arguments
+as ssh, you can alias it to something small in your shell's rc files to avoid
+having to type it each time::
 
-    alias ssh="kitty +kitten ssh"
+    alias s="kitty +kitten ssh"
 
-Remember to also setup :ref:`shell_integration` for completion and other
-niceties.
-
-If for some reason that does not work (typically because the server is using a
-non POSIX compliant shell as ``/bin/sh``), you can try using it with ``python``
-instead::
-
-    kitty +kitten ssh use-python myserver
-
-If that also fails, perhaps because python is not installed on the remote
-server, use the following one-liner instead (it
+If the ssh kitten fails, use the following one-liner instead (it
 is slower as it needs to ssh into the server twice, but will work with most
 servers)::
 
     infocmp -a xterm-kitty | ssh myserver tic -x -o \~/.terminfo /dev/stdin
 
-If you are behind a proxy (like Balabit) that prevents this, you must redirect the
-1st command to a file, copy that to the server and run ``tic`` manually.  If you
+If you are behind a proxy (like Balabit) that prevents this, or ``tic`` comes
+with macOS that does not support reading from STDIN, you must redirect the 1st
+command to a file, copy that to the server and run ``tic`` manually.  If you
 connect to a server, embedded or Android system that doesn't have ``tic``, copy over
 your local file terminfo to the other system as :file:`~/.terminfo/x/xterm-kitty`.
 
@@ -96,6 +88,12 @@ Make sure the TERM environment variable, is ``xterm-kitty``.  And either the
 TERMINFO environment variable points to a directory containing :file:`x/xterm-kitty`
 or that file is under :file:`~/.terminfo/x/`.
 
+For macOS, you may also need to put that file under :file:`~/.terminfo/78/`::
+
+    mkdir -p ~/.terminfo/{78,x}
+    ln -snf ../x/xterm-kitty ~/.terminfo/78/xterm-kitty
+    tic -x -o ~/.terminfo "$KITTY_INSTALLATION_DIR/terminfo/kitty.terminfo"
+
 Note that ``sudo`` might remove TERMINFO.  Then setting it at the shell prompt can
 be too late, because command line editing may not be reinitialized.  In that case
 you can either ask ``sudo`` to set it or if that is not supported, insert an ``env``
@@ -202,13 +200,14 @@ bitmapped fonts.
 .. note::
    If you are trying to use a font patched with NERD font symbols, dont do that
    as patching destroys fonts. There is no need, simply install the standalone
-   NERD font, kitty should pick up symbols from it automatically, and you can
-   tell it to do so explicitly in case it doesnt with the :opt:`symbol_map`
-   directive::
+   NERD font (the file :file:`NerdFontsSymbolsOnly.zip` from the `NERD font
+   releases page <https://github.com/ryanoasis/nerd-fonts/releases>`__). kitty
+   should pick up symbols from it automatically, and you can tell it to do so
+   explicitly in case it doesnt with the :opt:`symbol_map` directive::
 
-        symbol_map U+23FB-U+23FE,U+2B58,U+E200-U+E2A9,U+E0A0-U+E0A3,U+E0B0-U+E0BF,U+E0C0-U+E0C8,U+E0CC-U+E0CF,U+E0D0-U+E0D2,U+E0D4,U+E700-U+E7C5,U+F000-U+F2E0,U+2665,U+26A1,U+F400-U+F4A8,U+F67C,U+E000-U+E00A,U+F300-U+F313,U+E5FA-U+E62B Symbols Nerd Font
+        symbol_map U+23FB-U+23FE,U+2665,U+26A1,U+2B58,U+E000-U+E00A,U+E0A0-U+E0A3,U+E0B0-U+E0C8,U+E0CA,U+E0CC-U+E0D2,U+E0D4,U+E200-U+E2A9,U+E300-U+E3E3,U+E5FA-U+E62F,U+E700-U+E7C5,U+F000-U+F2E0,U+F300-U+F31C,U+F400-U+F4A9,U+F500-U+F8FF Symbols Nerd Font
 
-If your font is not listed in ``kitty list-fonts`` it means that it is not
+If your font is not listed in ``kitty +list-fonts`` it means that it is not
 monospace or is a bitmapped font. On Linux you can list all monospace fonts with::
 
     fc-list : family spacing outline scalable | grep -e spacing=100 -e spacing=90 | grep -e outline=True | grep -e scalable=True
@@ -236,7 +235,7 @@ command to rebuild your fontconfig cache::
 
     fc-cache -r
 
-Then, the font will be available in ``kitty list-fonts``.
+Then, the font will be available in ``kitty +list-fonts``.
 
 
 How can I assign a single global shortcut to bring up the kitty terminal?
@@ -245,7 +244,7 @@ How can I assign a single global shortcut to bring up the kitty terminal?
 Bringing up applications on a single key press is the job of the window
 manager/desktop environment. For ways to do it with kitty (or indeed any
 terminal) in different environments,
-see `here <https://github.com/kovidgoyal/kitty/issues/45>`_.
+see :iss:`here <45>`.
 
 
 I do not like the kitty icon!
@@ -270,6 +269,10 @@ homepage:
    :target: https://github.com/hristost/kitty-alternative-icon
    :width: 256
 
+.. image:: https://github.com/igrmk/whiskers/raw/main/whiskers.svg
+   :target: https://github.com/igrmk/whiskers
+   :width: 256
+
 On macOS you can change the icon by following the steps:
 
 #. Find :file:`kitty.app` in the Applications folder, select it and press :kbd:`⌘+i`
@@ -337,8 +340,7 @@ sorts of places where they may or may not work.
 I am using tmux and have a problem
 --------------------------------------
 
-First, terminal multiplexers are `a bad idea
-<https://github.com/kovidgoyal/kitty/issues/391#issuecomment-638320745>`_, do
+First, terminal multiplexers are :iss:`a bad idea <391#issuecomment-638320745>`, do
 not use them, if at all possible. kitty contains features that do all of what
 tmux does, but better, with the exception of remote persistence (:iss:`391`).
 If you still want to use tmux, read on.

docs/glossary.rst

@@ -57,6 +57,12 @@ Variables that influence kitty behavior
    Controls where kitty stores cache files. Defaults to :file:`~/.cache/kitty`
    or :file:`~/Library/Caches/kitty` on macOS.
 
+.. envvar:: KITTY_RUNTIME_DIRECTORY
+
+   Controls where kitty stores runtime files like sockets. Defaults to
+   the :code:`XDG_RUNTIME_DIR` environment variable if that is defined
+   otherwise the run directory inside the kitty cache directory is used.
+
 .. envvar:: VISUAL
 
    The terminal editor (such as ``vi`` or ``nano``) kitty uses, when, for

docs/graphics-protocol.rst

@@ -12,8 +12,7 @@ emulator. The major design goals are:
   below as well as above the text, with alpha blending. The graphics should also scroll with the text, automatically.
 * Should use optimizations when the client is running on the same computer as the terminal emulator.
 
-For some discussion regarding the design choices, see `#33
-<https://github.com/kovidgoyal/kitty/issues/33>`_.
+For some discussion regarding the design choices, see :iss:`33`.
 
 To see a quick demo, inside a |kitty| terminal run::
 
@@ -47,6 +46,8 @@ Some programs and libraries that use the kitty graphics protocol:
 Other terminals that have implemented the graphics protocol:
 
 * `WezTerm <https://github.com/wez/wezterm/issues/986>`_
+* `Konsole <https://invent.kde.org/utilities/konsole/-/merge_requests/594>`_
+* `wayst <https://github.com/91861/wayst>`_
 
 
 Getting the window size
@@ -109,7 +110,7 @@ features of the graphics protocol:
 
     def serialize_gr_command(**cmd):
         payload = cmd.pop('payload', None)
-        cmd = ','.join('{}={}'.format(k, v) for k, v in cmd.items())
+        cmd = ','.join(f'{k}={v}' for k, v in cmd.items())
         ans = []
         w = ans.append
         w(b'\033_G'), w(cmd.encode('ascii'))
@@ -326,14 +327,14 @@ use the *query action*, set ``a=q``. Then the terminal emulator will try to load
 the image and respond with either OK or an error, as above, but it will not
 replace an existing image with the same id, nor will it store the image.
 
-As of September 2021, kitty and WezTerm are the only terminal emulators to support this
-graphics protocol. We intend that any terminal emulator that wishes to support
-it can do so. To check if a terminal emulator supports the graphics protocol the best way
-is to send the above *query action* followed by a request for the
-`primary device attributes <https://vt100.net/docs/vt510-rm/DA1.html>`_. If you
-get back an answer for the device attributes without getting back an answer for
-the *query action* the terminal emulator does not support the graphics
-protocol.
+As of April 2022, kitty and WezTerm are the only terminal emulators to
+support this graphics protocol completely, with Konsole and wayst having partial support.
+We intend that any terminal emulator that wishes to support it can do so. To
+check if a terminal emulator supports the graphics protocol the best way is to
+send the above *query action* followed by a request for the `primary device
+attributes <https://vt100.net/docs/vt510-rm/DA1.html>`_. If you get back an
+answer for the device attributes without getting back an answer for the *query
+action* the terminal emulator does not support the graphics protocol.
 
 This means that terminal emulators that support the graphics protocol, **must**
 reply to *query actions* immediately without processing other input. Most

docs/index.rst

@@ -14,13 +14,13 @@ kitty
     changelog
     integrations
     protocol-extensions
+    press-mentions
 
 
 .. tab:: Fast
 
    * Offloads rendering to the GPU for :doc:`lower system load <performance>`
-   * Uses threaded rendering for
-     `absolutely minimal latency <https://github.com/kovidgoyal/kitty/issues/2701#issuecomment-636497270>`_
+   * Uses threaded rendering for :iss:`absolutely minimal latency <2701#issuecomment-636497270>`
    * Performance tradeoffs can be :ref:`tuned <conf-kitty-performance>`
 
 .. tab:: Capable
@@ -61,7 +61,7 @@ kitty
 
     .. raw:: html
 
-        <video controls width="640" height="360">
+        <video controls width="640" height="360" poster="_static/poster.png">
             <source src="https://download.calibre-ebook.com/videos/kitty.mp4" type="video/mp4">
             <source src="https://download.calibre-ebook.com/videos/kitty.webm" type="video/webm">
         </video>

docs/installer.py

@@ -1,261 +0,0 @@
-#!/usr/bin/env python3
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-
-import atexit
-import json
-import os
-import platform
-import re
-import shlex
-import shutil
-import subprocess
-import sys
-import tempfile
-
-py3 = sys.version_info[0] > 2
-is64bit = platform.architecture()[0] == '64bit'
-is_macos = 'darwin' in sys.platform.lower()
-is_linux_arm = is_linux_arm64 = False
-if is_macos:
-    mac_ver = tuple(map(int, platform.mac_ver()[0].split('.')))
-    if mac_ver[:2] < (10, 12):
-        raise SystemExit('Your version of macOS is too old, at least 10.12 is required')
-else:
-    machine = (os.uname()[4] or '').lower()
-    if machine.startswith('arm') or machine.startswith('aarch64'):
-        is_linux_arm = True
-        is_linux_arm64 = machine.startswith('arm64') or machine.startswith('aarch64')
-
-try:
-    __file__
-    from_file = True
-except NameError:
-    from_file = False
-
-if py3:
-    unicode = str
-    raw_input = input
-    import urllib.request as urllib
-
-    def encode_for_subprocess(x):
-        return x
-else:
-    from future_builtins import map
-    import urllib2 as urllib
-
-    def encode_for_subprocess(x):
-        if isinstance(x, unicode):
-            x = x.encode('utf-8')
-        return x
-
-
-def run(*args):
-    if len(args) == 1:
-        args = shlex.split(args[0])
-    args = list(map(encode_for_subprocess, args))
-    ret = subprocess.Popen(args).wait()
-    if ret != 0:
-        raise SystemExit(ret)
-
-
-class Reporter:  # {{{
-
-    def __init__(self, fname):
-        self.fname = fname
-        self.last_percent = 0
-
-    def __call__(self, blocks, block_size, total_size):
-        percent = (blocks*block_size)/float(total_size)
-        report = '\rDownloaded {:.1%}         '.format(percent)
-        if percent - self.last_percent > 0.05:
-            self.last_percent = percent
-            print(report, end='')
-            sys.stdout.flush()
-# }}}
-
-
-def get_release_data(relname='latest'):
-    print('Checking for latest release on GitHub...')
-    req = urllib.Request(
-        'https://api.github.com/repos/kovidgoyal/kitty/releases/' + relname,
-        headers={'Accept': 'application/vnd.github.v3+json'})
-    try:
-        res = urllib.urlopen(req).read().decode('utf-8')
-    except Exception as err:
-        raise SystemExit('Failed to contact {} with error: {}'.format(req.get_full_url(), err))
-    data = json.loads(res)
-    html_url = data['html_url'].replace('/tag/', '/download/').rstrip('/')
-    for asset in data.get('assets', ()):
-        name = asset['name']
-        if is_macos:
-            if name.endswith('.dmg'):
-                return html_url + '/' + name, asset['size']
-        else:
-            if name.endswith('.txz'):
-                if is64bit:
-                    q = '-arm64.txz' if is_linux_arm64 else '-x86_64.txz'
-                    if name.endswith(q):
-                        return html_url + '/' + name, asset['size']
-                else:
-                    if name.endswith('-i686.txz'):
-                        return html_url + '/' + name, asset['size']
-    raise SystemExit('Failed to find the installer package on github')
-
-
-def do_download(url, size, dest):
-    print('Will download and install', os.path.basename(dest))
-    reporter = Reporter(os.path.basename(dest))
-
-    # Get content length and check if range is supported
-    rq = urllib.urlopen(url)
-    headers = rq.info()
-    sent_size = int(headers['content-length'])
-    if sent_size != size:
-        raise SystemExit('Failed to download from {} Content-Length ({}) != {}'.format(url, sent_size, size))
-    with open(dest, 'wb') as f:
-        while f.tell() < size:
-            raw = rq.read(8192)
-            if not raw:
-                break
-            f.write(raw)
-            reporter(f.tell(), 1, size)
-    rq.close()
-    if os.path.getsize(dest) < size:
-        raise SystemExit('Download failed, try again later')
-    print('\rDownloaded {} bytes'.format(os.path.getsize(dest)))
-
-
-def clean_cache(cache, fname):
-    for x in os.listdir(cache):
-        if fname not in x:
-            os.remove(os.path.join(cache, x))
-
-
-def download_installer(url, size):
-    fname = url.rpartition('/')[-1]
-    tdir = tempfile.gettempdir()
-    cache = os.path.join(tdir, 'kitty-installer-cache')
-    if not os.path.exists(cache):
-        os.makedirs(cache)
-    clean_cache(cache, fname)
-    dest = os.path.join(cache, fname)
-    if os.path.exists(dest) and os.path.getsize(dest) == size:
-        print('Using previously downloaded', fname)
-        return dest
-    if os.path.exists(dest):
-        os.remove(dest)
-    do_download(url, size, dest)
-    return dest
-
-
-def macos_install(dmg, dest='/Applications', launch=True):
-    mp = tempfile.mkdtemp()
-    atexit.register(shutil.rmtree, mp)
-    run('hdiutil', 'attach', dmg, '-mountpoint', mp)
-    try:
-        os.chdir(mp)
-        app = 'kitty.app'
-        d = os.path.join(dest, app)
-        if os.path.exists(d):
-            shutil.rmtree(d)
-        dest = os.path.join(dest, app)
-        run('ditto', '-v', app, dest)
-        print('Successfully installed kitty into', dest)
-        if launch:
-            run('open', dest)
-    finally:
-        os.chdir('/')
-        run('hdiutil', 'detach', mp)
-
-
-def linux_install(installer, dest=os.path.expanduser('~/.local'), launch=True):
-    dest = os.path.join(dest, 'kitty.app')
-    if os.path.exists(dest):
-        shutil.rmtree(dest)
-    os.makedirs(dest)
-    print('Extracting tarball...')
-    run('tar', '-C', dest, '-xJof', installer)
-    print('kitty successfully installed to', dest)
-    kitty = os.path.join(dest, 'bin', 'kitty')
-    print('Use', kitty, 'to run kitty')
-    if launch:
-        run(kitty, '--detach')
-
-
-def main(dest=None, launch=True, installer=None):
-    if not dest:
-        if is_macos:
-            dest = '/Applications'
-        else:
-            dest = os.path.expanduser('~/.local')
-    if is_linux_arm and not is_linux_arm64:
-        raise SystemExit(
-            'You are running on a 32-bit ARM system. The kitty binaries are only'
-            ' available for 64 bit ARM systems. You will have to build from'
-            ' source.')
-    if not installer:
-        url, size = get_release_data()
-        installer = download_installer(url, size)
-    else:
-        if installer == 'nightly':
-            url, size = get_release_data('tags/nightly')
-            installer = download_installer(url, size)
-        else:
-            installer = os.path.abspath(installer)
-            if not os.access(installer, os.R_OK):
-                raise SystemExit('Could not read from: {}'.format(installer))
-    if is_macos:
-        macos_install(installer, dest=dest, launch=launch)
-    else:
-        linux_install(installer, dest=dest, launch=launch)
-
-
-def script_launch():
-    # To test: python3 -c "import runpy; runpy.run_path('installer.py', run_name='script_launch')"
-    def path(x):
-        return os.path.expandvars(os.path.expanduser(x))
-
-    def to_bool(x):
-        return x.lower() in {'y', 'yes', '1', 'true'}
-
-    type_map = {x: path for x in 'dest installer'.split()}
-    type_map['launch'] = to_bool
-    kwargs = {}
-
-    for arg in sys.argv[1:]:
-        if arg:
-            m = re.match('([a-z_]+)=(.+)', arg)
-            if m is None:
-                raise SystemExit('Unrecognized command line argument: ' + arg)
-            k = m.group(1)
-            if k not in type_map:
-                raise SystemExit('Unrecognized command line argument: ' + arg)
-            kwargs[k] = type_map[k](m.group(2))
-    main(**kwargs)
-
-
-def update_installer_wrapper():
-    # To run: python3 -c "import runpy; runpy.run_path('installer.py', run_name='update_wrapper')" installer.sh
-    with open(__file__, 'rb') as f:
-        src = f.read().decode('utf-8')
-    wrapper = sys.argv[-1]
-    with open(wrapper, 'r+b') as f:
-        raw = f.read().decode('utf-8')
-        nraw = re.sub(r'^# HEREDOC_START.+^# HEREDOC_END', lambda m: '# HEREDOC_START\n{}\n# HEREDOC_END'.format(src), raw, flags=re.MULTILINE | re.DOTALL)
-        if 'update_intaller_wrapper()' not in nraw:
-            raise SystemExit('regex substitute of HEREDOC failed')
-        f.seek(0), f.truncate()
-        f.write(nraw.encode('utf-8'))
-
-
-if __name__ == '__main__' and from_file:
-    main()
-elif __name__ == 'update_wrapper':
-    update_installer_wrapper()
-elif __name__ == 'script_launch':
-    script_launch()

docs/installer.sh

@@ -1,292 +1,174 @@
 #!/bin/sh
-#
-# installer.sh
 # Copyright (C) 2018 Kovid Goyal <kovid at kovidgoyal.net>
 #
 # Distributed under terms of the GPLv3 license.
-#
 
-python=$(command -v python3)
-if [ -z "$python" ]; then
-    python=$(command -v python2)
-fi
-if [ -z "$python" ]; then
-    python=$(command -v python2.7)
-fi
-if [ -z "$python" ]; then
-    python=$(command -v python)
-fi
-if [ -z "$python" ]; then
-    python=python
-fi
+{ \unalias command; \unset -f command; } >/dev/null 2>&1
+tdir=''
+cleanup() {
+    [ -n "$tdir" ] && {
+        command rm -rf "$tdir"
+        tdir=''
+    }
+}
 
-echo Using python executable: $python
+die() {
+    cleanup
+    printf "\033[31m%s\033[m\n\r" "$*" > /dev/stderr;
+    exit 1;
+}
 
-$python -c "import sys; script_launch=lambda:sys.exit('Download of installer failed!'); exec(sys.stdin.read()); script_launch()" "$@" <<'INSTALLER_HEREDOC'
-# {{{
-# HEREDOC_START
-#!/usr/bin/env python3
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-from __future__ import (
-    absolute_import, division, print_function, unicode_literals
-)
-
-import atexit
-import json
-import os
-import platform
-import re
-import shlex
-import shutil
-import subprocess
-import sys
-import tempfile
-
-py3 = sys.version_info[0] > 2
-is64bit = platform.architecture()[0] == '64bit'
-is_macos = 'darwin' in sys.platform.lower()
-is_linux_arm = is_linux_arm64 = False
-if is_macos:
-    mac_ver = tuple(map(int, platform.mac_ver()[0].split('.')))
-    if mac_ver[:2] < (10, 12):
-        raise SystemExit('Your version of macOS is too old, at least 10.12 is required')
-else:
-    machine = (os.uname()[4] or '').lower()
-    if machine.startswith('arm') or machine.startswith('aarch64'):
-        is_linux_arm = True
-        is_linux_arm64 = machine.startswith('arm64') or machine.startswith('aarch64')
-
-try:
-    __file__
-    from_file = True
-except NameError:
-    from_file = False
-
-if py3:
-    unicode = str
-    raw_input = input
-    import urllib.request as urllib
-
-    def encode_for_subprocess(x):
-        return x
-else:
-    from future_builtins import map
-    import urllib2 as urllib
-
-    def encode_for_subprocess(x):
-        if isinstance(x, unicode):
-            x = x.encode('utf-8')
-        return x
+detect_network_tool() {
+    if command -v curl 2> /dev/null > /dev/null; then
+        fetch() {
+            command curl -fL "$1"
+        }
+        fetch_quiet() {
+            command curl -fsSL "$1"
+        }
+    elif command -v wget 2> /dev/null > /dev/null; then
+        fetch() {
+            command wget -O- "$1"
+        }
+        fetch_quiet() {
+            command wget --quiet -O- "$1"
+        }
+    else
+        die "Neither curl nor wget available, cannot download kitty"
+    fi
+}
 
 
-def run(*args):
-    if len(args) == 1:
-        args = shlex.split(args[0])
-    args = list(map(encode_for_subprocess, args))
-    ret = subprocess.Popen(args).wait()
-    if ret != 0:
-        raise SystemExit(ret)
+detect_os() {
+    arch=""
+    case "$(command uname)" in
+        'Darwin') OS="macos";;
+        'Linux')
+            OS="linux"
+            case "$(command uname -m)" in
+                x86_64) arch="x86_64";;
+                aarch64*) arch="arm64";;
+                armv8*) arch="arm64";;
+                i386) arch="i686";;
+                i686) arch="i686";;
+                *) die "Unknown CPU architecture $(command uname -m)";;
+            esac
+            ;;
+        *) die "kitty binaries are not available for $(command uname)"
+    esac
+}
+
+expand_tilde() {
+    tilde_less="${1#\~/}"
+    [ "$1" != "$tilde_less" ] && tilde_less="$HOME/$tilde_less"
+    printf '%s' "$tilde_less"
+}
+
+parse_args() {
+    dest='~/.local'
+    [ "$OS" = "macos" ] && dest="/Applications"
+    launch='y'
+    installer=''
+    while :; do
+        case "$1" in
+            dest=*) dest="${1#*=}";;
+            launch=*) launch="${1#*=}";;
+            installer=*) installer="${1#*=}";;
+            "") break;;
+            *) die "Unrecognized command line option: $1";;
+        esac
+        shift
+    done
+    dest=$(expand_tilde "${dest}")
+    [ "$launch" != "y" -a "$launch" != "n" ] && die "Unrecognized command line option: launch=$launch"
+    dest="$dest/kitty.app"
+}
 
 
-class Reporter:  # {{{
+get_file_url() {
+    url="https://github.com/kovidgoyal/kitty/releases/download/$1/kitty-$2"
+    if [ "$OS" = "macos" ]; then
+        url="$url.dmg"
+    else
+        url="$url-$arch.txz"
+    fi
+}
 
-    def __init__(self, fname):
-        self.fname = fname
-        self.last_percent = 0
-
-    def __call__(self, blocks, block_size, total_size):
-        percent = (blocks*block_size)/float(total_size)
-        report = '\rDownloaded {:.1%}         '.format(percent)
-        if percent - self.last_percent > 0.05:
-            self.last_percent = percent
-            print(report, end='')
-            sys.stdout.flush()
-# }}}
+get_release_url() {
+    release_version=$(fetch_quiet "https://sw.kovidgoyal.net/kitty/current-version.txt")
+    [ $? -ne 0 -o -z "$release_version" ] && die "Could not get kitty latest release version"
+    get_file_url "v$release_version" "$release_version"
+}
 
 
-def get_release_data(relname='latest'):
-    print('Checking for latest release on GitHub...')
-    req = urllib.Request(
-        'https://api.github.com/repos/kovidgoyal/kitty/releases/' + relname,
-        headers={'Accept': 'application/vnd.github.v3+json'})
-    try:
-        res = urllib.urlopen(req).read().decode('utf-8')
-    except Exception as err:
-        raise SystemExit('Failed to contact {} with error: {}'.format(req.get_full_url(), err))
-    data = json.loads(res)
-    html_url = data['html_url'].replace('/tag/', '/download/').rstrip('/')
-    for asset in data.get('assets', ()):
-        name = asset['name']
-        if is_macos:
-            if name.endswith('.dmg'):
-                return html_url + '/' + name, asset['size']
-        else:
-            if name.endswith('.txz'):
-                if is64bit:
-                    q = '-arm64.txz' if is_linux_arm64 else '-x86_64.txz'
-                    if name.endswith(q):
-                        return html_url + '/' + name, asset['size']
-                else:
-                    if name.endswith('-i686.txz'):
-                        return html_url + '/' + name, asset['size']
-    raise SystemExit('Failed to find the installer package on github')
+get_nightly_url() {
+    get_file_url "nightly" "nightly"
+}
 
+get_download_url() {
+    installer_is_file="n"
+    case "$installer" in
+        "nightly") get_nightly_url ;;
+        "") get_release_url ;;
+        *) installer_is_file="y" ;;
+    esac
+}
 
-def do_download(url, size, dest):
-    print('Will download and install', os.path.basename(dest))
-    reporter = Reporter(os.path.basename(dest))
+linux_install() {
+    if [ "$installer_is_file" = "y" ]; then
+        command tar -C "$dest" "-xJof" "$installer"
+    else
+        printf '%s\n\n' "Downloading from: $url"
+        fetch "$url" | command tar -C "$dest" "-xJof" "-"
+    fi
+}
 
-    # Get content length and check if range is supported
-    rq = urllib.urlopen(url)
-    headers = rq.info()
-    sent_size = int(headers['content-length'])
-    if sent_size != size:
-        raise SystemExit('Failed to download from {} Content-Length ({}) != {}'.format(url, sent_size, size))
-    with open(dest, 'wb') as f:
-        while f.tell() < size:
-            raw = rq.read(8192)
-            if not raw:
-                break
-            f.write(raw)
-            reporter(f.tell(), 1, size)
-    rq.close()
-    if os.path.getsize(dest) < size:
-        raise SystemExit('Download failed, try again later')
-    print('\rDownloaded {} bytes'.format(os.path.getsize(dest)))
+macos_install() {
+    tdir=$(command mktemp -d "/tmp/kitty-install-XXXXXXXXXXXX")
+    [ "$installer_is_file" != "y" ] && {
+        installer="$tdir/kitty.dmg"
+        printf '%s\n\n' "Downloading from: $url"
+        fetch "$url" > "$installer" || die "Failed to download: $url"
+    }
+    command mkdir "$tdir/mp"
+    command hdiutil attach "$installer" "-mountpoint" "$tdir/mp" || die "Failed to mount kitty.dmg"
+    command ditto -v "$tdir/mp/kitty.app" "$dest"
+    rc="$?"
+    command hdiutil detach "$tdir/mp"
+    command rm -rf "$tdir"
+    tdir=''
+    [ "$rc" != "0" ] && die "Failed to copy kitty.app from mounted dmg"
+}
 
+prepare_install_dest() {
+    printf "%s\n" "Installing to $dest"
+    command rm -rf "$dest"
+    command mkdir -p "$dest" || die "Failed to create the directory: $dest"
+}
 
-def clean_cache(cache, fname):
-    for x in os.listdir(cache):
-        if fname not in x:
-            os.remove(os.path.join(cache, x))
+exec_kitty() {
+    if [ "$OS" = "macos" ]; then
+        exec "open" "$dest"
+    else
+        exec "$dest/bin/kitty" "--detach"
+    fi
+    die "Failed to launch kitty"
+}
 
+main() {
+    [ -n "$KITTY_WINDOW_ID" ] && die "You should not try to update kitty from within kitty itself. For best results, quit all kitty instances and run this script from another terminal"
+    detect_os
+    parse_args "$@"
+    detect_network_tool
+    get_download_url
+    prepare_install_dest
+    if [ "$OS" = "macos" ]; then
+        macos_install
+    else
+        linux_install
+    fi
+    [ "$launch" = "y" ] && exec_kitty
+    exit 0
+}
 
-def download_installer(url, size):
-    fname = url.rpartition('/')[-1]
-    tdir = tempfile.gettempdir()
-    cache = os.path.join(tdir, 'kitty-installer-cache')
-    if not os.path.exists(cache):
-        os.makedirs(cache)
-    clean_cache(cache, fname)
-    dest = os.path.join(cache, fname)
-    if os.path.exists(dest) and os.path.getsize(dest) == size:
-        print('Using previously downloaded', fname)
-        return dest
-    if os.path.exists(dest):
-        os.remove(dest)
-    do_download(url, size, dest)
-    return dest
-
-
-def macos_install(dmg, dest='/Applications', launch=True):
-    mp = tempfile.mkdtemp()
-    atexit.register(shutil.rmtree, mp)
-    run('hdiutil', 'attach', dmg, '-mountpoint', mp)
-    try:
-        os.chdir(mp)
-        app = 'kitty.app'
-        d = os.path.join(dest, app)
-        if os.path.exists(d):
-            shutil.rmtree(d)
-        dest = os.path.join(dest, app)
-        run('ditto', '-v', app, dest)
-        print('Successfully installed kitty into', dest)
-        if launch:
-            run('open', dest)
-    finally:
-        os.chdir('/')
-        run('hdiutil', 'detach', mp)
-
-
-def linux_install(installer, dest=os.path.expanduser('~/.local'), launch=True):
-    dest = os.path.join(dest, 'kitty.app')
-    if os.path.exists(dest):
-        shutil.rmtree(dest)
-    os.makedirs(dest)
-    print('Extracting tarball...')
-    run('tar', '-C', dest, '-xJof', installer)
-    print('kitty successfully installed to', dest)
-    kitty = os.path.join(dest, 'bin', 'kitty')
-    print('Use', kitty, 'to run kitty')
-    if launch:
-        run(kitty, '--detach')
-
-
-def main(dest=None, launch=True, installer=None):
-    if not dest:
-        if is_macos:
-            dest = '/Applications'
-        else:
-            dest = os.path.expanduser('~/.local')
-    if is_linux_arm and not is_linux_arm64:
-        raise SystemExit(
-            'You are running on a 32-bit ARM system. The kitty binaries are only'
-            ' available for 64 bit ARM systems. You will have to build from'
-            ' source.')
-    if not installer:
-        url, size = get_release_data()
-        installer = download_installer(url, size)
-    else:
-        if installer == 'nightly':
-            url, size = get_release_data('tags/nightly')
-            installer = download_installer(url, size)
-        else:
-            installer = os.path.abspath(installer)
-            if not os.access(installer, os.R_OK):
-                raise SystemExit('Could not read from: {}'.format(installer))
-    if is_macos:
-        macos_install(installer, dest=dest, launch=launch)
-    else:
-        linux_install(installer, dest=dest, launch=launch)
-
-
-def script_launch():
-    # To test: python3 -c "import runpy; runpy.run_path('installer.py', run_name='script_launch')"
-    def path(x):
-        return os.path.expandvars(os.path.expanduser(x))
-
-    def to_bool(x):
-        return x.lower() in {'y', 'yes', '1', 'true'}
-
-    type_map = {x: path for x in 'dest installer'.split()}
-    type_map['launch'] = to_bool
-    kwargs = {}
-
-    for arg in sys.argv[1:]:
-        if arg:
-            m = re.match('([a-z_]+)=(.+)', arg)
-            if m is None:
-                raise SystemExit('Unrecognized command line argument: ' + arg)
-            k = m.group(1)
-            if k not in type_map:
-                raise SystemExit('Unrecognized command line argument: ' + arg)
-            kwargs[k] = type_map[k](m.group(2))
-    main(**kwargs)
-
-
-def update_installer_wrapper():
-    # To run: python3 -c "import runpy; runpy.run_path('installer.py', run_name='update_wrapper')" installer.sh
-    with open(__file__, 'rb') as f:
-        src = f.read().decode('utf-8')
-    wrapper = sys.argv[-1]
-    with open(wrapper, 'r+b') as f:
-        raw = f.read().decode('utf-8')
-        nraw = re.sub(r'^# HEREDOC_START.+^# HEREDOC_END', lambda m: '# HEREDOC_START\n{}\n# HEREDOC_END'.format(src), raw, flags=re.MULTILINE | re.DOTALL)
-        if 'update_intaller_wrapper()' not in nraw:
-            raise SystemExit('regex substitute of HEREDOC failed')
-        f.seek(0), f.truncate()
-        f.write(nraw.encode('utf-8'))
-
-
-if __name__ == '__main__' and from_file:
-    main()
-elif __name__ == 'update_wrapper':
-    update_installer_wrapper()
-elif __name__ == 'script_launch':
-    script_launch()
-
-# HEREDOC_END
-# }}}
-INSTALLER_HEREDOC
+main "$@"

docs/keyboard-protocol.rst

@@ -38,6 +38,7 @@ In addition to kitty, this protocol is also implemented in:
 * The `notcurses library
   <https://github.com/dankamongmen/notcurses/issues/2131>`__
 * The `kakoune text editor <https://github.com/mawww/kakoune/issues/4103>`__
+* The `dte text editor <https://gitlab.com/craigbarnes/dte/-/issues/138>`__
 
 .. versionadded:: 0.20.0
 
@@ -441,6 +442,7 @@ mode* (the ``smkx/rmkx`` terminfo capabilities). This form is used only in
     "F10",       "kf10",       "CSI 21 ~"
     "F11",       "kf11",       "CSI 23 ~"
     "F12",       "kf12",       "CSI 24 ~"
+    "MENU",      "kf16",       "CSI 29 ~"
 
 There are a few more functional keys that have special cased legacy encodings.
 These are present because they are commonly used and for the sake of legacy

docs/kittens/custom.rst

@@ -35,7 +35,7 @@ your machine).
         # get the kitty window into which to paste answer
         w = boss.window_id_map.get(target_window_id)
         if w is not None:
-            w.paste(answer)
+            w.paste_text(answer)
 
 
 Now in :file:`kitty.conf` add the lines::
@@ -134,7 +134,7 @@ terminal program, you can tell the kittens system to run the
 
 For example, here is a kitten that "zooms/unzooms" the current terminal window
 by switching to the stack layout or back to the previous layout. This is
-equivalent to the builtin :ref:`action-toggle_layout` action.
+equivalent to the builtin :ac:`toggle_layout` action.
 
 Create a file in the kitty config folder, :file:`~/.config/kitty/zoom_toggle.py`
 
@@ -325,7 +325,7 @@ Kittens created by kitty users
 `smart-scroll <https://github.com/yurikhan/kitty-smart-scroll>`_
     Makes the kitty scroll bindings work in full screen applications
 
-`insert password <https://github.com/kovidgoyal/kitty/issues/1222>`_
+:iss:`insert password <1222>`
     Insert a password from a CLI password manager, taking care to only do it at
     a password prompt.
 

docs/kittens/hints.rst

@@ -33,6 +33,12 @@ that has been marked as such by the program running in the terminal,
 for example, by ``ls --hyperlink=auto``. You can also :doc:`customize what actions are
 taken for different types of URLs <../open_actions>`.
 
+.. note:: If there are more hints than letters, hints will use multiple
+   letters. In this case, when you press the first letter, only hints
+   starting with that letter are displayed. Pressing the second letter will
+   select that hint or press :kbd:`Enter` or :kbd:`Space` to select the empty
+   hint.
+
 The hints kitten is very powerful to see more detailed help on its various
 options and modes of operation, see below. You can use these options to
 create mappings in :file:`kitty.conf` to select various different text

docs/kittens/hyperlinked_grep.rst

@@ -45,17 +45,31 @@ You can now run searches with::
     hg some-search-term
 
 If you want to enable completion, for the kitten, you can delegate completion
-to rg. For that, instead of using an alias create a simple wrapper script named
-:file:`hg` somewhere in your ``PATH``:
+to rg. How to do that varies based on the shell:
 
-.. code-block:: sh
 
-    #!/bin/sh
-    exec kitty +kitten hyperlinked_grep "$@"
+.. tab:: zsh
 
-Then, for example, for ZSH, add the following to :file:`.zshrc`::
+    Instead of using an alias create a simple wrapper script named
+    :file:`hg` somewhere in your ``PATH``:
 
-    compdef _rg hg
+    .. code-block:: sh
+
+        #!/bin/sh
+        exec kitty +kitten hyperlinked_grep "$@"
+
+    Then, add the following to :file:`.zshrc`::
+
+        compdef _rg hg
+
+.. tab:: fish
+
+    You can combine both the aliasing/wrapping and pointing fish
+    to rg's autocompletion with a fish "wrapper" function in your :file:`config.fish`:
+
+    .. code-block:: sh
+
+        function hg --wraps rg; kitty +kitten hyperlinked_grep $argv; end
 
 To learn more about kitty's powerful framework for customizing URL click
 actions, :doc:`see here </open_actions>`.

docs/kittens/icat.rst

@@ -35,6 +35,18 @@ Then you can simply use ``icat image.png`` to view images.
 
 The ``icat`` kitten has various command line arguments to allow it to be used
 from inside other programs to display images. In particular, :option:`--place`,
-:option:`--detect-support` and :option:`--print-window-size`.
+:option:`--detect-support`, :option:`--silent` and :option:`--print-window-size`.
+
+If you are trying to integrate icat into a complex program like a file
+manager or editor, there are a few things to keep in mind. icat works by
+communicating over the TTY device, it both writes to and reads from the TTY.
+So it is imperative that while it is running the host program does not do
+any TTY I/O. Any key presses or other input from the user on the TTY device
+will be discarded. At a minimum, you should use the :option:`--silent` and
+:option:`--transfer-mode` command line arguments. To be
+really robust you should consider writing proper support for the
+:doc:`../graphics-protocol` in the program instead.  Nowadays there are many
+libraries that have support for it.
+
 
 .. include:: /generated/cli-kitten-icat.rst

docs/kittens/ssh.rst

@@ -0,0 +1,154 @@
+Truly convenient SSH
+=========================================
+
+* Automatic :ref:`shell_integration` on remote hosts
+
+* Easily :ref:`clone local shell/editor config <real_world_ssh_kitten_config>` on remote hosts
+
+* Automatic :opt:`re-use of existing connections <kitten-ssh.share_connections>` to avoid connection setup latency
+
+* Make kitty itself available in the remote host :opt:`on demand <kitten-ssh.remote_kitty>`
+
+* Easily :opt:`change terminal colors <kitten-ssh.color_scheme>` when connecting to remote hosts
+
+.. versionadded:: 0.25.0
+   Automatic shell integration, file transfer and reuse of connections
+
+The ssh kitten allows you to login easily to remote hosts, and automatically
+setup the environment there to be as comfortable as your local shell. You
+can specify environment variables to set on the remote host and
+files to copy there, making your remote experience just like your
+local shell. Additionally, it automatically sets up :ref:`shell_integration` on
+the remote host and copies the kitty terminfo database there.
+
+The ssh kitten is a thin wrapper around the traditional `ssh <https://man.openbsd.org/ssh>`__
+command line program and supports all the same options and arguments and configuration.
+In interactive usage scenarios it is a drop in replacement for ``ssh``. To try it
+out, simply run:
+
+.. code-block:: sh
+
+    kitty +kitten ssh some-hostname-to-connect-to
+
+You should end up at a shell prompt on the remote host, with shell
+integration enabled. If you like it you can add an alias to it in your shell's
+rc files:
+
+.. code-block:: sh
+
+    alias s=kitty +kitten ssh
+
+So now you can just type ``s hostname`` to connect.
+
+If you define a mapping in :file:`kitty.conf` such as::
+
+    map f1 new_window_with_cwd
+
+Then, pressing :kbd:`F1` will open a new window automatically logged
+into the same host using the ssh kitten, at the same directory.
+
+The ssh kitten can be configured using the :file:`~/.config/kitty/ssh.conf`
+file where you can specify environment variables to set on the remote host
+and files to copy from the local to the remote host. Let's see a
+quick example:
+
+.. code-block:: conf
+
+   # Copy the files and directories needed to setup some common tools
+   copy .zshrc .vimrc .vim
+   # Setup some environment variables
+   env SOME_VAR=x
+   # COPIED_VAR will have the same value on the remote host as it does locally
+   env COPIED_VAR=_kitty_copy_env_var_
+
+   # Create some per hostname settings
+   hostname someserver-*
+   copy env-files
+   env SOMETHING=else
+
+   hostname someuser@somehost
+   copy --dest=foo/bar some-file
+   copy --glob some/files.*
+
+
+See below for full details on the syntax and options of :file:`ssh.conf`.
+Additionally, you can pass config options on the command line:
+
+.. code-block:: sh
+
+   kitty +kitten ssh --kitten interpreter=python servername
+
+The :code:`--kitten` argument can be specified multiple times, with directives
+from :file:`ssh.conf`. These are merged with :file:`ssh.conf` as if they were
+appended to the end of that file. They apply only to the host being SSHed to
+by this invocation, so any :opt:`hostname <kitten-ssh.hostname>` directives are ignored.
+
+.. warning::
+
+   Due to limitations in the design of SSH, any typing you do before the
+   shell prompt appears may be lost. So ideally don't start typing till you see
+   the shell prompt. 😇
+
+
+.. _real_world_ssh_kitten_config:
+
+A real world example
+----------------------
+
+Suppose you often SSH into a production server, and you would like to setup
+your shell and editor there using your custom settings. However, other people
+could SSH in as well and you don't want to clobber their settings. Here is how
+this could be achieved using the ssh kitten with zsh and vim as the shell and
+editor, respectively:
+
+.. code-block:: conf
+
+   # Have these settings apply to servers in my organization
+   hostname myserver-*
+
+   # Setup zsh to read its files from my-conf/zsh
+   env ZDOTDIR $HOME/my-conf/zsh
+   copy --dest my-conf/zsh/.zshrc .zshrc
+   copy --dest my-conf/zsh/.zshenv .zshenv
+   # If you use other zsh init files add them in a similar manner
+
+   # Setup vim to read its config from my-conf/vim
+   env VIMINIT $HOME/my-conf/vim/vimrc
+   env VIMRUNTIME $HOME/my-conf/vim
+   copy --dest my-conf/vim .vim
+   copy --dest my-conf/vim/vimrc .vimrc
+
+
+How it works
+----------------
+
+The ssh kitten works by having SSH transmit and execute a POSIX sh (or
+:opt:`optionally <kitten-ssh.interpreter>` Python) bootstrap script on the
+remote host using an :opt:`interpreter <kitten-ssh.interpreter>`. This script
+reads setup data over the tty device, which kitty sends as a base64 encoded
+compressed tarball. The script extracts it and places the :opt:`files <kitten-ssh.copy>`
+and sets the :opt:`environment variables <kitten-ssh.env>` before finally
+launching the :opt:`login shell <kitten-ssh.login_shell>` with :opt:`shell
+integration <kitten-ssh.shell_integration>` enabled. The data is requested by
+the kitten over the TTY with a random one time password. kitty reads the request
+and if the password matches a password pre-stored in shared memory on the
+localhost by the kitten, the transmission is allowed. If your OpenSSH version is
+>= 8.4 then the data is transmitted instantly without any roundtrip delay.
+
+.. note::
+
+   When connecting to BSD hosts, it is possible the bootstrap script will
+   fail or run slowly, because the default shells are crippled in various ways.
+   Your best bet is to install Python on the remote, make sure the login shell
+   is something POSIX sh compliant, and use :code:`python` as the :opt:`interpreter
+   <kitten-ssh.interpreter>` in :file:`ssh.conf`.
+
+.. include:: /generated/conf-kitten-ssh.rst
+
+
+.. _ssh_copy_command:
+
+The copy command
+--------------------
+
+.. include:: /generated/ssh-copy.rst

docs/kittens_intro.rst

@@ -15,6 +15,7 @@ Extend with kittens
    kittens/remote_file
    kittens/hyperlinked_grep
    kittens/transfer
+   kittens/ssh
    kittens/custom
    kittens/*
 
@@ -62,6 +63,11 @@ Some prominent kittens:
     other :term:`kitty windows <window>`.
 
 
+:doc:`SSH <kittens/ssh>`
+    SSH with automatic shell integration, connection re-use for low latency and easy cloning
+    of local shell and editor configuration to the remote machine.
+
+
 :doc:`Panel <kittens/panel>`
     Draw a GPU accelerated dock panel on your desktop showing the output
     from an arbitrary terminal program.

docs/launch.rst

@@ -95,6 +95,12 @@ the command line:
 ``@cursor-y``
     replaced by the current cursor y position with 1 being the topmost cell
 
+``@first-line-on-screen``
+    replaced by the first line on screen. Can be used for pager positioning.
+
+``@last-line-on-screen``
+    replaced by the last line on screen. Can be used for pager positioning.
+
 
 For example::
 

docs/layouts.rst

@@ -128,33 +128,47 @@ The Splits Layout
 
 This is the most flexible layout. You can create any arrangement of windows
 by splitting exiting windows repeatedly. To best use this layout you should
-define a few extra keybindings in :file:`kitty.conf`::
+define a few extra key bindings in :file:`kitty.conf`::
 
+    # Create a new window splitting the space used by the existing one so that
+    # the two windows are placed one above the other
     map F5 launch --location=hsplit
+
+    # Create a new window splitting the space used by the existing one so that
+    # the two windows are placed side by side
     map F6 launch --location=vsplit
+
+    # Create a new window splitting the space used by the existing one so that
+    # the two windows are placed side by side if the existing window is wide or
+    # one above the other if the existing window is tall.
+    map F4 launch --location=split
+
+    # Rotate the current split, chaging its split axis from vertical to
+    # horizontal or vice versa
     map F7 layout_action rotate
 
+    # Move the active window in the indicated direction
     map shift+up move_window up
     map shift+left move_window left
     map shift+right move_window right
     map shift+down move_window down
 
+    # Switch focus to the neighboring window in the indicated direction
     map ctrl+left neighboring_window left
     map ctrl+right neighboring_window right
     map ctrl+up neighboring_window up
     map ctrl+down neighboring_window down
 
-Now you can create horizontal and vertical splits by using :kbd:`F5` and
-:kbd:`F6`. You can move them around using :kbd:`shift+arrow` keys
-and you can move focus to neighboring windows using :kbd:`ctrl+arrow` keys.
-You can switch an existing split from horizontal to vertical and vice versa
-using :kbd:`F7`. Finally, windows can be resized using :ref:`window_resizing`.
-
-Note that you can swap the windows in a split using the ``rotate`` action with
-an argument of ``180`` and rotate and swap with an argument of ``270``.
+Windows can be resized using :ref:`window_resizing`.  You can swap the windows
+in a split using the ``rotate`` action with an argument of ``180`` and rotate
+and swap with an argument of ``270``.
 
 This layout takes one option, ``split_axis`` that controls whether new windows
-are placed into vertical or horizontal splits, by default::
+are placed into vertical or horizontal splits when a ``--location`` is not
+specified. A value of ``horizontal`` (same as ``--location=vsplit``)
+means when a new split is created the two windows will be placed side by side
+and a value of ``vertical`` (same as ``--location=hsplit``) means the two
+windows will be placed one on top of the other. By default::
 
     enabled_layouts splits:split_axis=horizontal
 
@@ -239,6 +253,8 @@ action, for example::
    map ctrl+right resize_window wider
    map ctrl+up resize_window taller
    map ctrl+down resize_window shorter 3
+   # reset all windows in the tab to default sizes
+   map ctrl+home resize_window reset
 
 The ``resize_window`` action has a second, optional argument to control
 the resizing increment (a positive integer that defaults to 1).

docs/open_actions.rst

@@ -100,3 +100,69 @@ lines. The various available criteria are:
 ``file``
     A shell glob pattern that must match the filename, for example:
     ``image-??.png``
+
+
+.. _launch_actions:
+
+Scripting the opening of files with kitty on macOS
+-------------------------------------------------------
+
+On macOS you can use :guilabel:`Open With` in Finder or drag and drop files and
+URLs onto the kitty dock icon to open them with kitty. The default actions are:
+
+* Open text files in your editor and images using the icat kitten.
+* Run shell scripts in a shell
+* Open SSH urls using the ssh command
+
+These actions can also be executed from the command line by running::
+
+    open -a kitty.app file_or_url ... (on macOS only)
+    or
+    kitty +open file_or_url ...
+
+Since macOS lacks an official interface to set default URL scheme handlers,
+kitty has a command you can use for it. The first argument for is the URL
+scheme, and the second optional argument is the bundle id of the app, which
+defaults to kitty, if not specified. For example:
+
+.. code-block:: sh
+
+    # Set kitty as the handler for ssh:// URLs
+    kitty +runpy 'from kitty.fast_data_types import cocoa_set_url_handler; import sys; cocoa_set_url_handler(*sys.argv[1:]); print("OK")' ssh
+    # Set someapp as the handler for xyz:// URLs
+    kitty +runpy 'from kitty.fast_data_types import cocoa_set_url_handler; import sys; cocoa_set_url_handler(*sys.argv[1:]); print("OK")' xyz someapp.bundle.identifier
+
+You can customize these actions by creating a :file:`launch-actions.conf` file
+in the kitty config directory, just like
+the :file:`open-actions.conf` file above. For example:
+
+.. code:: conf
+
+    # Open script files based on their shebangs
+    protocol file
+    ext sh,command,tool
+    action launch --hold --type=os-window kitty +shebang $FILE_PATH {SHELL}
+
+    # Open shell specific script files
+    protocol file
+    ext fish,bash,zsh
+    action launch --hold --type=os-window kitty +shebang $FILE_PATH __ext__
+
+    # Open directories
+    protocol file
+    mime inode/directory
+    action launch --type=os-window --cwd $FILE_PATH
+
+    # Open text files without fragments in the editor
+    protocol file
+    mime text/*
+    action launch --type=os-window $EDITOR $FILE_PATH
+
+    # Open image files with icat
+    protocol file
+    mime image/*
+    action launch --type=os-window kitty +kitten icat --hold $FILE_PATH
+
+    # Open ssh URLs with ssh command
+    protocol ssh
+    action launch --type=os-window ssh $URL

docs/overview.rst

@@ -247,8 +247,7 @@ Would open the scrollback buffer in a new :term:`window` when you press the :kbd
 key. See :sc:`show_scrollback <show_scrollback>` for details.
 
 If you want to use it with an editor such as vim to get more powerful features,
-you can see tips for doing so, in
-`this thread <https://github.com/kovidgoyal/kitty/issues/719>`_.
+you can see tips for doing so, in :iss:`this thread <719>`.
 
 If you wish to store very large amounts of scrollback to view using the piping or
 :sc:`show_scrollback <show_scrollback>` features, you can use the

docs/press-mentions.rst

@@ -0,0 +1,18 @@
+Press mentions of kitty
+========================
+
+`Python Bytes 272 <https://youtu.be/8HKliSbA-gQ?t=815>`__ (Feb 2022)
+    A podcast demoing some of kitty's coolness
+
+`Console #88 <https://console.substack.com/p/console-88>`__ (Jan 2022)
+    An interview with Kovid about kitty
+
+
+Video reviews
+--------------
+
+`Review (Jan 2021) <https://www.youtube.com/watch?v=TTzP2zYJn2k>`__
+    A kitty review by distrotube
+
+`Review (Dec 2020) <https://www.youtube.com/watch?v=KUMkLhFeBrI>`__
+    A kitty review/intro by TechHut

docs/shell-integration.rst

@@ -26,7 +26,7 @@ Features
   to view it in a pager
 
 * The current working directory or the command being executed are automatically
-  displayed in the kitty window titlebar/tab title.
+  displayed in the kitty window titlebar/tab title
 
 * The text cursor is changed to a bar when editing commands at the shell prompt
 
@@ -48,17 +48,32 @@ off or it can be disabled entirely as well. The :opt:`shell_integration` option
 takes a space separated list of keywords:
 
 disabled
-    Turn off all shell integration
+    Turn off all shell integration. The shell's launch environment is not
+    modified and :envvar:`KITTY_SHELL_INTEGRATION` is not set. Useful for
+    :ref:`manual integration <manual_shell_integration>`.
 
 no-rc
     Do not modify the shell's launch environment to enable integration. Useful if you prefer
-    to :ref:`manually enable integration <manual_shell_integration>`.
+    to load the kitty shell integration code yourself, either as part of
+    :ref:`manually integration <manual_shell_integration>` or because you have
+    some other software that sets up shell integration.
+    This will still set the :envvar:`KITTY_SHELL_INTEGRATION` environment variable when kitty runs the shell.
 
 no-cursor
     Turn off changing of the text cursor to a bar when editing text
 
 no-title
-    Turn off setting the kitty window/tab title based on shell state
+    Turn off setting the kitty window/tab title based on shell state.
+    Note that for the ``fish`` shell kitty relies on fish's native title
+    setting functionality instead.
+
+no-cwd
+    Turn off reporting the current working directory. This is used to allow
+    :ac:`new_window_with_cwd` and similar to open windows logged
+    into remote machines using the :doc:`ssh kitten <kittens/ssh>`
+    automatically with the same working directory as the current window.
+    Note that for the ``fish`` shell this will not disable its built-in current
+    working directory reporting.
 
 no-prompt-mark
     Turn off marking of prompts. This disables jumping to prompt, browsing
@@ -66,6 +81,8 @@ no-prompt-mark
 
 no-complete
     Turn off completion for the kitty command.
+    Note that for the ``fish`` shell this does not take effect, since fish
+    already comes with a kitty completion script.
 
 
 More ways to browse command output
@@ -84,7 +101,7 @@ to be copied.
 
 The feature to jump to previous prompts (
 :sc:`scroll_to_previous_prompt` and :sc:`scroll_to_next_prompt`) and mouse
-actions (:ref:`action-mouse_select_command_output` and :ref:`action-mouse_show_command_output`) can
+actions (:ac:`mouse_select_command_output` and :ac:`mouse_show_command_output`) can
 be integrated with browsing command output as well. For example, define the
 following mapping in :file:`kitty.conf`:
 
@@ -129,25 +146,29 @@ different shells.
    and sources the original :file:`.zshenv`. It then loads the shell integration code.
    The remainder of zsh's startup process proceeds as normal.
 
-.. tab:: bash
-
-    For bash, kitty adds a couple of lines to the bottom of :file:`~/.bashrc`
-    (in an atomic manner) to load the shell integration code.
-
 .. tab:: fish
 
     For fish, to make it automatically load the integration code provided by
     kitty, the integration script directory path is prepended to the
-    :code:`XDG_DATA_DIRS` environment variable. This is only applied to the fish
+    ``XDG_DATA_DIRS`` environment variable. This is only applied to the fish
     process and will be cleaned up by the integration script after startup. No files
     are added or modified.
 
+.. tab:: bash
+
+    For bash, kitty starts bash in POSIX mode, using the environment variable
+    ``ENV`` to load the shell integration script. This prevents bash from
+    loading any startup files itself. The loading of the startup files is done
+    by the integration script, after disabling POSIX mode. From the
+    perspective of those scripts there should be no difference to running
+    vanilla bash.
+
+
 Then, when launching the shell, kitty sets the environment variable
 :envvar:`KITTY_SHELL_INTEGRATION` to the value of the :opt:`shell_integration`
 option. The shell integration code reads the environment variable, turns on the
 specified integration functionality and then unsets the variable so as to not
-pollute the system. This has the nice effect that the changes to the shell's rc
-files become no-ops when running the shell in anything other than kitty itself.
+pollute the system.
 
 The actual shell integration code uses hooks provided by each shell to send
 special escape codes to kitty, to perform the various tasks. You can see the
@@ -168,6 +189,7 @@ code used for each shell below:
 
     .. literalinclude:: ../shell-integration/fish/vendor_conf.d/kitty-shell-integration.fish
         :language: fish
+        :force:
 
 .. tab:: bash
 
@@ -179,13 +201,27 @@ code used for each shell below:
    </details>
 
 
+Shell integration over SSH
+----------------------------
+
+The easiest way to have shell integration work when SSHing into remote systems
+is to use the :doc:`ssh kitten <kittens/ssh>`. Simply run::
+
+    kitty +kitten ssh hostname
+
+And, by magic, you will be logged into the remote system with fully functional
+shell integration. Alternately, you can :ref:`setup shell integration manually
+<manual_shell_integration>`, by copying the kitty shell integration scripts to
+the remote server and editing the shell rc files there, as described below.
+
+
 .. _manual_shell_integration:
 
 Manual shell integration
 ----------------------------
 
 The automatic shell integration is designed to be minimally intrusive, as such
-it wont work for sub-shells, terminal multiplexers, containers, remote systems, etc.
+it wont work for sub-shells, terminal multiplexers, containers, etc.
 For such systems, you should setup manual shell integration by adding some code
 to your shells startup files to load the shell integration script.
 
@@ -197,15 +233,6 @@ First, in :file:`kitty.conf` set:
 
 Then in your shell's rc file, add the lines:
 
-.. tab:: bash
-
-    .. code-block:: sh
-
-        if test -n "$KITTY_INSTALLATION_DIR"; then
-            export KITTY_SHELL_INTEGRATION="enabled"
-            source "$KITTY_INSTALLATION_DIR/shell-integration/bash/kitty.bash"
-        fi
-
 .. tab:: zsh
 
     .. code-block:: sh
@@ -228,15 +255,31 @@ Then in your shell's rc file, add the lines:
         end
 
 
+.. tab:: bash
+
+    .. code-block:: sh
+
+        if test -n "$KITTY_INSTALLATION_DIR"; then
+            export KITTY_SHELL_INTEGRATION="enabled"
+            source "$KITTY_INSTALLATION_DIR/shell-integration/bash/kitty.bash"
+        fi
+
 The value of :envvar:`KITTY_SHELL_INTEGRATION` is the same as that for
 :opt:`shell_integration`, except if you want to disable shell integration
 completely, in which case simply do not set the
 :envvar:`KITTY_SHELL_INTEGRATION` variable at all.
 
-If you want this to work while SSHing into a remote system, then you will
-need to add some code to the snippets above to check if :code:`KITTY_INSTALLATION_DIR`
-is empty and if so to set it to some hard coded location with the shell
-integration scripts that need to be copied onto the remote system.
+In a container, you will need to install the kitty shell integration scripts
+and make sure the :envvar:`KITTY_INSTALLATION_DIR` environment variable is set
+to point to the location of the scripts.
+
+Integration with other shells
+-------------------------------
+
+There exist third-party integrations to use these features for various other shells:
+
+* Jupyter console via a patch (:iss:`4475`)
+* `xonsh <https://github.com/xonsh/xonsh/issues/4623>`__
 
 
 Notes for shell developers

docs/underlines.rst

@@ -13,8 +13,8 @@ To set the underline style::
     <ESC>[4:1m  # this is a straight underline
     <ESC>[4:2m  # this is a double underline
     <ESC>[4:3m  # this is a curly underline
-    <ESC>[4:4m  # this is a dotted underline (not implemented in kitty)
-    <ESC>[4:5m  # this is a dashed underline (not implemented in kitty)
+    <ESC>[4:4m  # this is a dotted underline
+    <ESC>[4:5m  # this is a dashed underline
     <ESC>[4m    # this is a straight underline (for backwards compat)
     <ESC>[24m   # this is no underline (for backwards compat)
 

gen-apc-parsers.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # License: GPLv3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
+import os
 import subprocess
 from collections import defaultdict
 from typing import Any, DefaultDict, Dict, FrozenSet, List, Tuple, Union
@@ -42,7 +43,7 @@ def parse_flag(keymap: KeymapType, type_map: Dict[str, Any], command_class: str)
     lines = []
     for ch in type_map['flag']:
         attr, allowed_values = keymap[ch]
-        q = ' && '.join(f"g.{attr} != '{x}'" for x in allowed_values)
+        q = ' && '.join(f"g.{attr} != '{x}'" for x in sorted(allowed_values))
         lines.append(f'''
             case {attr}: {{
                 g.{attr} = screen->parser_buf[pos++] & 0xff;
@@ -67,7 +68,7 @@ def cmd_for_report(report_name: str, keymap: KeymapType, type_map: Dict[str, Any
         flag_fmt, flag_attrs = [], []
         cv = {'flag': 'c', 'int': 'i', 'uint': 'I'}[atype]
         for ch in type_map[atype]:
-            flag_fmt.append('s' + cv)
+            flag_fmt.append(f's{cv}')
             attr = keymap[ch][0]
             flag_attrs.append(f'"{attr}", {conv}g.{attr}')
         return ' '.join(flag_fmt), ', '.join(flag_attrs)
@@ -240,7 +241,7 @@ static inline void
 
 def write_header(text: str, path: str) -> None:
     with open(path, 'w') as f:
-        print(f'// This file is generated by {__file__} do not edit!', file=f, end='\n\n')
+        print(f'// This file is generated by {os.path.basename(__file__)} do not edit!', file=f, end='\n\n')
         print('#pragma once', file=f)
         print(text, file=f)
     subprocess.check_call(['clang-format', '-i', path])

gen-config.py

@@ -38,6 +38,8 @@ def main() -> None:
 
     from kittens.diff.options.definition import definition as kd
     write_output('kittens.diff', kd)
+    from kittens.ssh.options.definition import definition as sd
+    write_output('kittens.ssh', sd)
 
 
 if __name__ == '__main__':

gen-key-constants.py

@@ -238,7 +238,7 @@ def patch_file(path: str, what: str, text: str, start_marker: str = '/* ', end_m
             end = raw.index(end_q)
         except ValueError:
             raise SystemExit(f'Failed to find "{end_q}" in {path}')
-        raw = raw[:start] + start_q + '\n' + text + '\n' + raw[end:]
+        raw = f'{raw[:start]}{start_q}\n{text}\n{raw[end:]}'
         f.seek(0)
         f.truncate(0)
         f.write(raw)
@@ -368,7 +368,7 @@ def generate_ctrl_mapping() -> None:
         val = str(ctrl_mapping[k])
         items.append(val)
         if k in "\\'":
-            k = '\\' + k
+            k = f'\\{k}'
         mi.append(f"        case '{k}': return {val};")
 
     for line_items in chunks(items, 6):

gen-wcwidth.py

@@ -49,11 +49,10 @@ class_maps: Dict[str, Set[int]] = {}
 all_symbols: Set[int] = set()
 name_map: Dict[int, str] = {}
 word_search_map: DefaultDict[str, Set[int]] = defaultdict(set)
-zwj = 0x200d
 soft_hyphen = 0xad
 flag_codepoints = frozenset(range(0x1F1E6, 0x1F1E6 + 26))
 # See https://github.com/harfbuzz/harfbuzz/issues/169
-marks = set(emoji_skin_tone_modifiers) | {zwj} | flag_codepoints
+marks = set(emoji_skin_tone_modifiers) | flag_codepoints
 not_assigned = set(range(0, sys.maxunicode))
 property_maps: Dict[str, Set[int]] = defaultdict(set)
 
@@ -69,7 +68,6 @@ def parse_prop_list() -> None:
         property_maps[name] |= chars
     # see https://www.unicode.org/faq/unsup_char.html#3
     marks |= property_maps['Other_Default_Ignorable_Code_Point']
-    marks.add(soft_hyphen)
 
 
 def parse_ucd() -> None:
@@ -113,6 +111,12 @@ def parse_ucd() -> None:
                 marks.add(codepoint)
             elif category.startswith('S'):
                 all_symbols.add(codepoint)
+            elif category == 'Cf':
+                # we add Cf to marks as it contains things like tags and zero
+                # width chars. Not sure if *all* of Cf should be treated as
+                # combining chars, might need to add individual exceptions in
+                # the future.
+                marks.add(codepoint)
 
     with open('nerd-fonts-glyphs.txt') as f:
         for line in f:
@@ -382,9 +386,9 @@ def gen_ucd() -> None:
             ascii_range='false'
         )
         category_test(
-            'is_non_rendered_char', p, 'Cc Cs'.split(),
+            'is_non_rendered_char', p, 'Cc Cs Cf'.split(),
             'Other_Default_Ignorable_Code_Point and soft hyphen',
-            extra_chars=property_maps['Other_Default_Ignorable_Code_Point'] | {soft_hyphen},
+            extra_chars=property_maps['Other_Default_Ignorable_Code_Point'] | set(range(0xfe00, 0xfe0f + 1)),
             ascii_range='false'
         )
         category_test('is_word_char', p, {c for c in class_maps if c[0] in 'LN'}, 'L and N categories')

glfw/cocoa_init.m

@@ -264,6 +264,8 @@ display_reconfigured(CGDirectDisplayID display UNUSED, CGDisplayChangeSummaryFla
     }
 }
 
+static NSDictionary<NSString*,NSNumber*> *global_shortcuts = nil;
+
 @interface GLFWHelper : NSObject
 @end
 
@@ -280,6 +282,16 @@ display_reconfigured(CGDirectDisplayID display UNUSED, CGDisplayChangeSummaryFla
     (void)object;
 }
 
+// watch for settings change and rebuild global_shortcuts using key/value observing on NSUserDefaults
+- (void)observeValueForKeyPath:(NSString *)keyPath ofObject:(id)object change:(NSDictionary *)change context:(void *)context
+{
+    (void)keyPath; (void)object; (void)change; (void)context;
+    if (global_shortcuts != nil) {
+        [global_shortcuts release];
+        global_shortcuts = nil;
+    }
+}
+
 @end // GLFWHelper
 
 // Delegate for application related notifications {{{
@@ -346,32 +358,49 @@ static GLFWapplicationwillfinishlaunchingfun finish_launching_callback = NULL;
         finish_launching_callback();
 }
 
-- (BOOL)application:(NSApplication *)theApplication openFile:(NSString *)filename {
-    (void)theApplication;
-    if (!filename || !_glfw.ns.file_open_callback) return NO;
-    const char *path = NULL;
+- (BOOL)application:(NSApplication *)sender openFile:(NSString *)filename {
+    (void)sender;
+    if (!filename || !_glfw.ns.url_open_callback) return NO;
+    const char *url = NULL;
     @try {
-        path = [[NSFileManager defaultManager] fileSystemRepresentationWithPath: filename];
+        url = [[[NSURL fileURLWithPath:filename] absoluteString] UTF8String];
     } @catch(NSException *exc) {
         NSLog(@"Converting openFile filename: %@ failed with error: %@", filename, exc.reason);
         return NO;
     }
-    if (!path) return NO;
-    return _glfw.ns.file_open_callback(path);
+    if (!url) return NO;
+    return _glfw.ns.url_open_callback(url);
 }
 
 - (void)application:(NSApplication *)sender openFiles:(NSArray *)filenames {
     (void)sender;
-    if (!_glfw.ns.file_open_callback || !filenames) return;
+    if (!_glfw.ns.url_open_callback || !filenames) return;
     for (id x in filenames) {
         NSString *filename = x;
-        const char *path = NULL;
+        const char *url = NULL;
         @try {
-            path = [[NSFileManager defaultManager] fileSystemRepresentationWithPath: filename];
+            url = [[[NSURL fileURLWithPath:filename] absoluteString] UTF8String];
         } @catch(NSException *exc) {
             NSLog(@"Converting openFiles filename: %@ failed with error: %@", filename, exc.reason);
         }
-        if (path) _glfw.ns.file_open_callback(path);
+        if (url) _glfw.ns.url_open_callback(url);
+    }
+}
+
+// Remove openFile and openFiles when the minimum supported macOS version is 10.13
+- (void)application:(NSApplication *)sender openURLs:(NSArray<NSURL *> *)urls
+{
+    (void)sender;
+    if (!_glfw.ns.url_open_callback || !urls) return;
+    for (id x in urls) {
+        NSURL *ns_url = x;
+        const char *url = NULL;
+        @try {
+            url = [[ns_url absoluteString] UTF8String];
+        } @catch(NSException *exc) {
+            NSLog(@"Converting openURLs url: %@ failed with error: %@", ns_url, exc.reason);
+        }
+        if (url) _glfw.ns.url_open_callback(url);
     }
 }
 
@@ -379,7 +408,6 @@ static GLFWapplicationwillfinishlaunchingfun finish_launching_callback = NULL;
 {
     (void)notification;
     [NSApp stop:nil];
-    if (_glfw.ns.file_open_callback) _glfw.ns.file_open_callback(":cocoa::application launched::");
 
     CGDisplayRegisterReconfigurationCallback(display_reconfigured, NULL);
     _glfwCocoaPostEmptyEvent();
@@ -453,72 +481,293 @@ void* _glfwLoadLocalVulkanLoaderNS(void)
 //////                       GLFW platform API                      //////
 //////////////////////////////////////////////////////////////////////////
 
-static bool
-is_modified_tab(NSEvent *event, NSEventModifierFlags modifierFlags) {
-    switch ((NSUInteger)modifierFlags) {
-        // No need to handle shift+tab, [shift]+option+tab
-        case NSEventModifierFlagShift:
-        case NSEventModifierFlagOption:
-        case (NSEventModifierFlagShift | NSEventModifierFlagOption):
-        // Do not intercept cmd+tab, shift+cmd+tab
-        case NSEventModifierFlagCommand:
-        case (NSEventModifierFlagShift | NSEventModifierFlagCommand):
-            return false;
-        default:
-            break;
-    }
-    // ctrl+whatever+tab, option+cmd+tab
-    if (
-            (
-                (modifierFlags & NSEventModifierFlagControl) ||
-                    modifierFlags == (NSEventModifierFlagOption | NSEventModifierFlagCommand)
-            ) && [event.charactersIgnoringModifiers isEqualToString:@"\t"]
-       ) return true;
-    // shift+whatever+tab
-    if (
-            (modifierFlags & NSEventModifierFlagShift) &&
-                [event.charactersIgnoringModifiers isEqualToString:@"\x19"]
-       ) return true;
-    return false;
-}
+/**
+ * Apple Symbolic HotKeys Ids
+ * To find this symbolic hot keys indices do:
+ * 1. open Terminal
+ * 2. restore defaults in System Preferences > Keyboard > Shortcuts
+ * 3. defaults read com.apple.symbolichotkeys > current.txt
+ * 4. enable/disable given symbolic hot key in System Preferences > Keyboard > Shortcuts
+ * 5. defaults read com.apple.symbolichotkeys | diff -C 5 current.txt -
+ * 6. restore defaults in System Preferences > Keyboard > Shortcuts
+ */
+typedef enum AppleShortcutNames {
+    // launchpad & dock
+    kSHKTurnDockHidingOnOrOff                   = 52,   // Opt, Cmd, D
+    kSHKShowLaunchpad                           = 160,  //
+
+    // display
+    kSHKDecreaseDisplayBrightness1              = 53,   // F14 (Fn)
+    kSHKDecreaseDisplayBrightness2              = 55,   // F14 (Fn, Ctrl)
+    kSHKIncreaseDisplayBrightness1              = 54,   // F15 (Fn)
+    kSHKIncreaseDisplayBrightness2              = 56,   // F15 (Fn, Ctrl)
+
+    // mission control
+    kSHKMissionControl                          = 32,   // Ctrl, Arrow Up
+    kSHKShowNotificationCenter                  = 163,  //
+    kSHKTurnDoNotDisturbOnOrOff                 = 175,  //
+    kSHKApplicationWindows                      = 33,   // Ctrl, Arrow Down
+    kSHKShowDesktop                             = 36,   // F11
+    kSHKMoveLeftASpace                          = 79,   // Ctrl, Arrow Left
+    kSHKMoveRightASpace                         = 81,   // Ctrl, Arrow Right
+    kSHKSwitchToDesktop1                        = 118,  // Ctrl, 1
+    kSHKSwitchToDesktop2                        = 119,  // Ctrl, 2
+    kSHKSwitchToDesktop3                        = 120,  // Ctrl, 3
+    kSHKSwitchToDesktop4                        = 121,  // Ctrl, 4
+    kSHKQuickNote                               = 190,  // Fn, Q
+
+    // keyboard
+    kSHKChangeTheWayTabMovesFocus               = 13,   // Ctrl, F7
+    kSHKTurnKeyboardAccessOnOrOff               = 12,   // Ctrl, F1
+    kSHKMoveFocusToTheMenuBar                   = 7,    // Ctrl, F2
+    kSHKMoveFocusToTheDock                      = 8,    // Ctrl, F3
+    kSHKMoveFocusToActiveOrNextWindow           = 9,    // Ctrl, F4
+    kSHKMoveFocusToTheWindowToolbar             = 10,   // Ctrl, F5
+    kSHKMoveFocusToTheFloatingWindow            = 11,   // Ctrl, F6
+    kSHKMoveFocusToNextWindow                   = 27,   // Cmd, `
+    kSHKMoveFocusToStatusMenus                  = 57,   // Ctrl, F8
+
+    // input sources
+    kSHKSelectThePreviousInputSource            = 60,   // Ctrl, Space bar
+    kSHKSelectNextSourceInInputMenu             = 61,   // Ctrl, Opt, Space bar
+
+    // screenshots
+    kSHKSavePictureOfScreenAsAFile              = 28,   // Shift, Cmd, 3
+    kSHKCopyPictureOfScreenToTheClipboard       = 29,   // Ctrl, Shift, Cmd, 3
+    kSHKSavePictureOfSelectedAreaAsAFile        = 30,   // Shift, Cmd, 4
+    kSHKCopyPictureOfSelectedAreaToTheClipboard = 31,   // Ctrl, Shift, Cmd, 4
+    kSHKScreenshotAndRecordingOptions           = 184,  // Shift, Cmd, 5
+
+    // spotlight
+    kSHKShowSpotlightSearch                     = 64,   // Cmd, Space bar
+    kSHKShowFinderSearchWindow                  = 65,   // Opt, Cmd, Space bar
+
+    // accessibility
+    kSHKTurnZoomOnOrOff                         = 15,   // Opt, Cmd, 8
+    kSHKTurnImageSmoothingOnOrOff               = 23,   // Opt, Cmd, Backslash "\"
+    kSHKZoomOut                                 = 19,   // Opt, Cmd, -
+    kSHKZoomIn                                  = 17,   // Opt, Cmd, =
+    kSHKTurnFocusFollowingOnOrOff               = 179,  //
+
+    kSHKIncreaseContrast                        = 25,   // Ctrl, Opt, Cmd, .
+    kSHKDecreaseContrast                        = 26,   // Ctrl, Opt, Cmd, ,
+
+    kSHKInvertColors                            = 21,   // Ctrl, Opt, Cmd, 8
+    kSHKTurnVoiceOverOnOrOff                    = 59,   // Cmd, F5
+    kSHKShowAccessibilityControls               = 162,  // Opt, Cmd, F5
+
+    // app shortcuts
+    kSHKShowHelpMenu                            = 98,   // Shift, Cmd, /
+
+    // deprecated (Not shown on macOS Monterey)
+    kSHKMoveFocusToTheWindowDrawer              = 51,   // Opt, Cmd, `
+    kSHKShowDashboard                           = 62,   // F12
+    kSHKLookUpInDictionary                      = 70,   // Shift, Cmd, E
+    kSHKHideAndShowFrontRow                     = 73,   // Cmd, Esc
+    kSHKActivateSpaces                          = 75,   // F8
+
+    // unknown
+    kSHKUnknown                                 = 0,    //
+} AppleShortcutNames;
 
 static bool
-is_cmd_period(NSEvent *event, NSEventModifierFlags modifierFlags) {
-    if (modifierFlags != NSEventModifierFlagCommand) return false;
-    if ([event.charactersIgnoringModifiers isEqualToString:@"."]) return true;
-    return false;
+is_shiftable_shortcut(int scv) {
+    return scv == kSHKMoveFocusToActiveOrNextWindow || scv == kSHKMoveFocusToNextWindow;
 }
 
-static bool
-is_modified_special_key(NSEvent *event, NSEventModifierFlags modifierFlags) {
-    // really one should use [[NSUserDefaults standardUserDefaults] valueForDefaultsDomain:@"com.apple.symbolichotkeys" key:@"AppleSymbolicHotKeys"]
-    // to get the list of global shortcuts and pass through the important ones,
-    // see https://stackoverflow.com/questions/21878482/what-do-the-parameter-values-in-applesymbolichotkeys-plist-dict-represent
-    // however given that in order to know which integers are which actions in that dict one needs reverse engineering
-    // see https://stackoverflow.com/questions/866056/how-do-i-programmatically-get-the-shortcut-keys-reserved-by-mac-os-x
-    // it's too much effort.
-    if ([event.charactersIgnoringModifiers length] != 1) return false;
-    const unichar ch = [event.charactersIgnoringModifiers characterAtIndex:0];
-    if (modifierFlags == (NSEventModifierFlagControl | NSEventModifierFlagShift)) {
-        switch (ch) {
-            case 0x1b:  // Esc
-            case NSF1FunctionKey: case NSF2FunctionKey: case NSF3FunctionKey: case NSF4FunctionKey:
-            case NSF5FunctionKey: case NSF6FunctionKey: case NSF7FunctionKey: case NSF8FunctionKey:
-            case NSF9FunctionKey: case NSF10FunctionKey: case NSF11FunctionKey: case NSF12FunctionKey:
-            case NSF13FunctionKey: case NSF14FunctionKey: case NSF15FunctionKey: case NSF16FunctionKey:
-            case NSF17FunctionKey: case NSF18FunctionKey: case NSF19FunctionKey:
-                return true;
+#define USEFUL_MODS(x) (x & (NSEventModifierFlagShift | NSEventModifierFlagOption | NSEventModifierFlagCommand | NSEventModifierFlagControl))
+
+static void
+build_global_shortcuts_lookup(void) {
+    // dump these in a terminal with: defaults read com.apple.symbolichotkeys
+    NSMutableDictionary<NSString*, NSNumber*> *temp = [NSMutableDictionary dictionaryWithCapacity:128];  // will be autoreleased
+    NSMutableSet<NSNumber*> *temp_configured = [NSMutableSet setWithCapacity:128];  // will be autoreleased
+    NSMutableSet<NSNumber*> *temp_missing_value = [NSMutableSet setWithCapacity:128];  // will be autoreleased
+    NSDictionary *apple_settings = [[NSUserDefaults standardUserDefaults] persistentDomainForName:@"com.apple.symbolichotkeys"];
+    if (apple_settings) {
+        NSDictionary<NSString*, id> *symbolic_hotkeys = [apple_settings objectForKey:@"AppleSymbolicHotKeys"];
+        if (symbolic_hotkeys) {
+            for (NSString *key in symbolic_hotkeys) {
+                id obj = symbolic_hotkeys[key];
+                if (![key isKindOfClass:[NSString class]] || ![obj isKindOfClass:[NSDictionary class]]) continue;
+                NSInteger sc = [key integerValue];
+                NSDictionary *sc_value = obj;
+                id enabled = [sc_value objectForKey:@"enabled"];
+                if (!enabled || ![enabled isKindOfClass:[NSNumber class]]) continue;
+                [temp_configured addObject:@(sc)];
+                if (![enabled boolValue]) continue;
+                id v = [sc_value objectForKey:@"value"];
+                if (!v || ![v isKindOfClass:[NSDictionary class]]) {
+                    if ([enabled boolValue]) [temp_missing_value addObject:@(sc)];
+                    continue;
+                }
+                NSDictionary *value = v;
+                id t = [value objectForKey:@"type"];
+                if (!t || ![t isKindOfClass:[NSString class]] || ![t isEqualToString:@"standard"]) continue;
+                id p = [value objectForKey:@"parameters"];
+                if (!p || ![p isKindOfClass:[NSArray class]] || [(NSArray*)p count] < 2) continue;
+                NSArray<NSNumber*> *parameters = p;
+                NSInteger ch = [parameters[0] isKindOfClass:[NSNumber class]] ? [parameters[0] integerValue] : 0xffff;
+                NSInteger vk = [parameters[1] isKindOfClass:[NSNumber class]] ? [parameters[1] integerValue] : 0xffff;
+                NSEventModifierFlags mods = ([parameters count] > 2 && [parameters[2] isKindOfClass:[NSNumber class]]) ? [parameters[2] unsignedIntegerValue] : 0;
+                mods = USEFUL_MODS(mods);
+                static char buf[64];
+#define S(x, k) snprintf(buf, sizeof(buf) - 1, #x":%lx:%ld", (unsigned long)mods, (long)k)
+                if (ch == 0xffff) { if (vk == 0xffff) continue; S(v, vk); } else S(c, ch);
+                temp[@(buf)] = @(sc);
+                // the move to next window shortcuts also respond to the same shortcut + shift
+                if (is_shiftable_shortcut([key intValue]) && !(mods & NSEventModifierFlagShift)) {
+                    mods |= NSEventModifierFlagShift;
+                    if (ch == 0xffff) S(v, vk); else S(c, ch);
+                    temp[@(buf)] = @(sc);
+                }
+#undef S
+            }
         }
     }
-    switch (ch) {
-        case 0x1b:  // Esc
-            if (modifierFlags & (NSEventModifierFlagCommand | NSEventModifierFlagControl)) return true;
-            break;
-        case NSHelpFunctionKey:  // For some reason keyboards with an insert key have it mapped to help
-            if (!modifierFlags || modifierFlags == NSEventModifierFlagShift) return true;
-            break;
+
+    // Add global shortcut definitions when the default enabled shortcut is not defined,
+    // or when the default enabled shortcut is not disabled and is missing a value.
+    // Here are the shortcuts that are enabled by default in the standard ANSI (US) layout.
+    // macOS provides separate configurations for some languages or keyboards.
+    // In general, the rules here will not take effect.
+    static char buf[64];
+#define S(i, t, m, k) if ([temp_configured member:@(i)] == nil || [temp_missing_value member:@(i)] != nil) { \
+        snprintf(buf, sizeof(buf) - 1, #t":%lx:%ld", (unsigned long)m, (long)k); \
+        temp[@(buf)] = @(i); \
+    }
+
+    // launchpad & dock
+    S(kSHKTurnDockHidingOnOrOff, c, (NSEventModifierFlagOption | NSEventModifierFlagCommand), 'd'); // Opt, Cmd, D
+    // mission control
+    S(kSHKMissionControl, v, NSEventModifierFlagControl, 126); // Ctrl, Arrow Up
+    S(kSHKApplicationWindows, v, NSEventModifierFlagControl, 125); // Ctrl, Arrow Down
+    // keyboard
+    S(kSHKMoveFocusToTheMenuBar, v, NSEventModifierFlagControl, 120); // Ctrl, F2
+    S(kSHKMoveFocusToTheDock, v, NSEventModifierFlagControl, 99); // Ctrl, F3
+    S(kSHKMoveFocusToActiveOrNextWindow, v, NSEventModifierFlagControl, 118); // Ctrl, F4
+    S(kSHKMoveFocusToActiveOrNextWindow, v, (NSEventModifierFlagShift | NSEventModifierFlagControl), 118); // Shift, Ctrl, F4
+    S(kSHKMoveFocusToNextWindow, c, NSEventModifierFlagCommand, 96); // Cmd, `
+    S(kSHKMoveFocusToNextWindow, c, (NSEventModifierFlagShift | NSEventModifierFlagCommand), 96); // Shift, Cmd, `
+    S(kSHKMoveFocusToStatusMenus, v, NSEventModifierFlagControl, 100); // Ctrl, F8
+    // input sources
+    S(kSHKSelectThePreviousInputSource, c, NSEventModifierFlagControl, 32); // Ctrl, Space bar
+    S(kSHKSelectNextSourceInInputMenu, c, (NSEventModifierFlagControl | NSEventModifierFlagOption), 32); // Ctrl, Opt, Space bar
+    // spotlight
+    S(kSHKShowSpotlightSearch, c, NSEventModifierFlagCommand, 32); // Cmd, Space bar
+    S(kSHKShowFinderSearchWindow, c, (NSEventModifierFlagOption | NSEventModifierFlagCommand), 32); // Opt, Cmd, Space bar
+
+#undef S
+    global_shortcuts = [[NSDictionary dictionaryWithDictionary:temp] retain];
+    /* NSLog(@"global_shortcuts: %@", global_shortcuts); */
+}
+
+static int
+is_active_apple_global_shortcut(NSEvent *event) {
+    if (global_shortcuts == nil) build_global_shortcuts_lookup();
+    NSEventModifierFlags modifierFlags = USEFUL_MODS([event modifierFlags]);
+    static char lookup_key[64];
+#define LOOKUP(t, k) \
+    snprintf(lookup_key, sizeof(lookup_key) - 1, #t":%lx:%ld", (unsigned long)modifierFlags, (long)k); \
+    NSNumber *sc = global_shortcuts[@(lookup_key)]; \
+    if (sc != nil) return [sc intValue]; \
+
+    if ([event.charactersIgnoringModifiers length] == 1) {
+        if (modifierFlags & NSEventModifierFlagShift) {
+            const uint32_t ch_without_shift = vk_to_unicode_key_with_current_layout([event keyCode]);
+            if (ch_without_shift < GLFW_FKEY_FIRST || ch_without_shift > GLFW_FKEY_LAST) {
+                LOOKUP(c, ch_without_shift);
+            }
+        }
+        const unichar ch = [event.charactersIgnoringModifiers characterAtIndex:0];
+        LOOKUP(c, ch);
+    }
+    unsigned short vk = [event keyCode];
+    if (vk != 0xffff) {
+        LOOKUP(v, vk);
+    }
+#undef LOOKUP
+    return kSHKUnknown;
+}
+
+static bool
+is_useful_apple_global_shortcut(int sc) {
+    switch(sc) {
+        // launchpad & dock
+        case kSHKTurnDockHidingOnOrOff:                   // Opt, Cmd, D
+        case kSHKShowLaunchpad:                           //
+
+        // display
+        case kSHKDecreaseDisplayBrightness1:              // F14 (Fn)
+        case kSHKDecreaseDisplayBrightness2:              // F14 (Fn, Ctrl)
+        case kSHKIncreaseDisplayBrightness1:              // F15 (Fn)
+        case kSHKIncreaseDisplayBrightness2:              // F14 (Fn, Ctrl)
+
+        // mission control
+        case kSHKMissionControl:                          // Ctrl, Arrow Up
+        case kSHKShowNotificationCenter:                  //
+        case kSHKTurnDoNotDisturbOnOrOff:                 //
+        case kSHKApplicationWindows:                      // Ctrl, Arrow Down
+        case kSHKShowDesktop:                             // F11
+        case kSHKMoveLeftASpace:                          // Ctrl, Arrow Left
+        case kSHKMoveRightASpace:                         // Ctrl, Arrow Right
+        case kSHKSwitchToDesktop1:                        // Ctrl, 1
+        case kSHKSwitchToDesktop2:                        // Ctrl, 2
+        case kSHKSwitchToDesktop3:                        // Ctrl, 3
+        case kSHKSwitchToDesktop4:                        // Ctrl, 4
+        case kSHKQuickNote:                               // Fn, Q
+
+        // keyboard
+        /* case kSHKChangeTheWayTabMovesFocus:               // Ctrl, F7 */
+        /* case kSHKTurnKeyboardAccessOnOrOff:               // Ctrl, F1 */
+        case kSHKMoveFocusToTheMenuBar:                   // Ctrl, F2
+        case kSHKMoveFocusToTheDock:                      // Ctrl, F3
+        case kSHKMoveFocusToActiveOrNextWindow:           // Ctrl, F4
+        /* case kSHKMoveFocusToTheWindowToolbar:             // Ctrl, F5 */
+        /* case kSHKMoveFocusToTheFloatingWindow:            // Ctrl, F6 */
+        case kSHKMoveFocusToNextWindow:                   // Cmd, `
+        case kSHKMoveFocusToStatusMenus:                  // Ctrl, F8
+
+        // input sources
+        case kSHKSelectThePreviousInputSource:            // Ctrl, Space bar
+        case kSHKSelectNextSourceInInputMenu:             // Ctrl, Opt, Space bar
+
+        // screenshots
+        /* case kSHKSavePictureOfScreenAsAFile:              // Shift, Cmd, 3 */
+        /* case kSHKCopyPictureOfScreenToTheClipboard:       // Ctrl, Shift, Cmd, 3 */
+        /* case kSHKSavePictureOfSelectedAreaAsAFile:        // Shift, Cmd, 4 */
+        /* case kSHKCopyPictureOfSelectedAreaToTheClipboard: // Ctrl, Shift, Cmd, 4 */
+        /* case kSHKScreenshotAndRecordingOptions:           // Shift, Cmd, 5 */
+
+        // spotlight
+        case kSHKShowSpotlightSearch:                     // Cmd, Space bar
+        case kSHKShowFinderSearchWindow:                  // Opt, Cmd, Space bar
+
+        // accessibility
+        /* case kSHKTurnZoomOnOrOff:                         // Opt, Cmd, 8 */
+        /* case kSHKTurnImageSmoothingOnOrOff:               // Opt, Cmd, Backslash "\" */
+        /* case kSHKZoomOut:                                 // Opt, Cmd, - */
+        /* case kSHKZoomIn:                                  // Opt, Cmd, = */
+        /* case kSHKTurnFocusFollowingOnOrOff:               // */
+        /* case kSHKIncreaseContrast:                        // Ctrl, Opt, Cmd, . */
+        /* case kSHKDecreaseContrast:                        // Ctrl, Opt, Cmd, , */
+        /* case kSHKInvertColors:                            // Ctrl, Opt, Cmd, 8 */
+        /* case kSHKTurnVoiceOverOnOrOff:                    // Cmd, F5 */
+        /* case kSHKShowAccessibilityControls:               // Opt, Cmd, F5 */
+
+        // app shortcuts
+        /* case kSHKShowHelpMenu:                            // Shift, Cmd, / */
+
+        // deprecated (Not shown on macOS Monterey)
+        /* case kSHKMoveFocusToTheWindowDrawer:              // Opt, Cmd, ` */
+        /* case kSHKShowDashboard:                           // F12 */
+        /* case kSHKLookUpInDictionary:                      // Shift, Cmd, E */
+        /* case kSHKHideAndShowFrontRow:                     // Cmd, Esc */
+        /* case kSHKActivateSpaces:                          // F8 */
+            return true;
+        default:
+            return false;
     }
-    return false;
 }
 
 GLFWAPI GLFWapplicationshouldhandlereopenfun glfwSetApplicationShouldHandleReopen(GLFWapplicationshouldhandlereopenfun callback) {
@@ -557,35 +806,76 @@ int _glfwPlatformInit(void)
     }
 
     [NSApp setDelegate:_glfw.ns.delegate];
+    static struct {
+        unsigned short virtual_key_code;
+        NSEventModifierFlags input_source_switch_modifiers;
+        NSTimeInterval timestamp;
+    } last_keydown_shortcut_event;
+    last_keydown_shortcut_event.virtual_key_code = 0xffff;
+    last_keydown_shortcut_event.input_source_switch_modifiers = 0;
 
     NSEvent* (^keydown_block)(NSEvent*) = ^ NSEvent* (NSEvent* event)
     {
-        NSEventModifierFlags modifierFlags = [event modifierFlags] & (NSEventModifierFlagShift | NSEventModifierFlagOption | NSEventModifierFlagCommand | NSEventModifierFlagControl);
-        if (is_modified_special_key(event, modifierFlags) || is_modified_tab(event, modifierFlags) || is_cmd_period(event, modifierFlags)) {
-            // Cocoa swallows various key presses, so route them explicitly
-            [[NSApp keyWindow].contentView keyDown:event];
-            return nil;
+        debug_key("---------------- key down -------------------\n");
+        debug_key("%s\n", [[event description] UTF8String]);
+        if (!_glfw.ignoreOSKeyboardProcessing) {
+            // first check if there is a global menu bar shortcut
+            if ([[NSApp mainMenu] performKeyEquivalent:event]) {
+                debug_key("keyDown triggerred global menu bar action ignoring\n");
+                last_keydown_shortcut_event.virtual_key_code = [event keyCode];
+                last_keydown_shortcut_event.input_source_switch_modifiers = 0;
+                last_keydown_shortcut_event.timestamp = [event timestamp];
+                return nil;
+            }
+            // now check if there is a useful apple shortcut
+            int global_shortcut = is_active_apple_global_shortcut(event);
+            if (is_useful_apple_global_shortcut(global_shortcut)) {
+                debug_key("keyDown triggerred global macOS shortcut ignoring\n");
+                last_keydown_shortcut_event.virtual_key_code = [event keyCode];
+                // record the modifier keys if switching to the next input source
+                last_keydown_shortcut_event.input_source_switch_modifiers = (global_shortcut == kSHKSelectNextSourceInInputMenu) ? USEFUL_MODS([event modifierFlags]) : 0;
+                last_keydown_shortcut_event.timestamp = [event timestamp];
+                return event;
+            }
         }
-
-        return event;
+        last_keydown_shortcut_event.virtual_key_code = 0xffff;
+        NSWindow *kw = [NSApp keyWindow];
+        if (kw && kw.contentView) [kw.contentView keyDown:event];
+        else debug_key("keyDown ignored as no keyWindow present\n");
+        return nil;
     };
 
     NSEvent* (^keyup_block)(NSEvent*) = ^ NSEvent* (NSEvent* event)
     {
-        NSEventModifierFlags modifierFlags = [event modifierFlags] & (NSEventModifierFlagShift | NSEventModifierFlagOption | NSEventModifierFlagCommand | NSEventModifierFlagControl);
-        if (modifierFlags & NSEventModifierFlagCommand) {
-            // From https://cocoadev.github.io/GameKeyboardHandlingAlmost/
-            // This works around an AppKit bug, where key up events while holding
-            // down the command key don't get sent to the key window.
-            [[NSApp keyWindow] sendEvent:event];
-        }
-        if (is_modified_special_key(event, modifierFlags) || is_modified_tab(event, modifierFlags) || is_cmd_period(event, modifierFlags)) {
-            // Cocoa swallows various key presses, so route them explicitly
-            [[NSApp keyWindow].contentView keyUp:event];
+        debug_key("----------------- key up --------------------\n");
+        debug_key("%s\n", [[event description] UTF8String]);
+        if (last_keydown_shortcut_event.virtual_key_code != 0xffff && last_keydown_shortcut_event.virtual_key_code == [event keyCode]) {
+            // ignore as the corresponding key down event triggered a menu bar or macOS shortcut
+            last_keydown_shortcut_event.virtual_key_code = 0xffff;
+            debug_key("keyUp ignored as corresponds to previous keyDown that trigerred a shortcut\n");
             return nil;
         }
+        NSWindow *kw = [NSApp keyWindow];
+        if (kw && kw.contentView) [kw.contentView keyUp:event];
+        else debug_key("keyUp ignored as no keyWindow present\n");
+        return nil;
+    };
 
-        return event;
+    NSEvent* (^flags_changed_block)(NSEvent*) = ^ NSEvent* (NSEvent* event)
+    {
+        debug_key("-------------- flags changed -----------------\n");
+        debug_key("%s\n", [[event description] UTF8String]);
+        last_keydown_shortcut_event.virtual_key_code = 0xffff;
+        // switching to the next input source is only confirmed when all modifier keys are released
+        if (last_keydown_shortcut_event.input_source_switch_modifiers) {
+            if (!([event modifierFlags] & last_keydown_shortcut_event.input_source_switch_modifiers))
+                last_keydown_shortcut_event.input_source_switch_modifiers = 0;
+            return event;
+        }
+        NSWindow *kw = [NSApp keyWindow];
+        if (kw && kw.contentView) [kw.contentView flagsChanged:event];
+        else debug_key("flagsChanged ignored as no keyWindow present\n");
+        return nil;
     };
 
     _glfw.ns.keyUpMonitor =
@@ -594,6 +884,9 @@ int _glfwPlatformInit(void)
     _glfw.ns.keyDownMonitor =
         [NSEvent addLocalMonitorForEventsMatchingMask:NSEventMaskKeyDown
                                               handler:keydown_block];
+    _glfw.ns.flagsChangedMonitor =
+        [NSEvent addLocalMonitorForEventsMatchingMask:NSEventMaskFlagsChanged
+                                              handler:flags_changed_block];
 
     if (_glfw.hints.init.ns.chdir)
         changeToResourcesDirectory();
@@ -606,6 +899,13 @@ int _glfwPlatformInit(void)
     };
     [[NSUserDefaults standardUserDefaults] registerDefaults:defaults];
 
+    NSUserDefaults *apple_settings = [[NSUserDefaults alloc] initWithSuiteName:@"com.apple.symbolichotkeys"];
+    [apple_settings addObserver:_glfw.ns.helper
+                     forKeyPath:@"AppleSymbolicHotKeys"
+                        options:NSKeyValueObservingOptionNew
+                        context:NULL];
+    _glfw.ns.appleSettings = apple_settings;
+
     [[NSNotificationCenter defaultCenter]
         addObserver:_glfw.ns.helper
            selector:@selector(selectedKeyboardInputSourceChanged:)
@@ -661,6 +961,8 @@ void _glfwPlatformTerminate(void)
                     object:nil];
         [[NSNotificationCenter defaultCenter]
             removeObserver:_glfw.ns.helper];
+        if (_glfw.ns.appleSettings)
+            [_glfw.ns.appleSettings removeObserver:_glfw.ns.helper forKeyPath:@"AppleSymbolicHotKeys"];
         [_glfw.ns.helper release];
         _glfw.ns.helper = nil;
     }
@@ -669,10 +971,18 @@ void _glfwPlatformTerminate(void)
         [NSEvent removeMonitor:_glfw.ns.keyUpMonitor];
     if (_glfw.ns.keyDownMonitor)
         [NSEvent removeMonitor:_glfw.ns.keyDownMonitor];
+    if (_glfw.ns.flagsChangedMonitor)
+        [NSEvent removeMonitor:_glfw.ns.flagsChangedMonitor];
+
+    if (_glfw.ns.appleSettings != nil) {
+        [_glfw.ns.appleSettings release];
+        _glfw.ns.appleSettings = nil;
+    }
 
     free(_glfw.ns.clipboardString);
 
     _glfwTerminateNSGL();
+    if (global_shortcuts != nil) { [global_shortcuts release]; global_shortcuts = nil; }
 
     } // autoreleasepool
 }

glfw/cocoa_platform.h

@@ -64,10 +64,11 @@ typedef void* CVDisplayLinkRef;
  #define NSOpenGLContextParameterSurfaceOpacity NSOpenGLCPSurfaceOpacity
 #endif
 
+#define debug_key(...) if (_glfw.hints.init.debugKeyboard) { fprintf(stderr, __VA_ARGS__); fflush(stderr); }
 
 typedef int (* GLFWcocoatextinputfilterfun)(int,int,unsigned int, unsigned long);
 typedef bool (* GLFWapplicationshouldhandlereopenfun)(int);
-typedef bool (* GLFWhandlefileopen)(const char*);
+typedef bool (* GLFWhandleurlopen)(const char*);
 typedef void (* GLFWapplicationwillfinishlaunchingfun)(void);
 typedef bool (* GLFWcocoatogglefullscreenfun)(GLFWwindow*);
 typedef void (* GLFWcocoarenderframefun)(GLFWwindow*);
@@ -130,6 +131,7 @@ typedef struct _GLFWwindowNS
     bool            maximized;
     bool            retina;
     bool            in_traditional_fullscreen;
+    bool            in_fullscreen_transition;
     bool            titlebar_hidden;
     unsigned long   pre_full_screen_style_mask;
 
@@ -152,6 +154,8 @@ typedef struct _GLFWwindowNS
     // Whether a render frame has been requested for this window
     bool renderFrameRequested;
     GLFWcocoarenderframefun renderFrameCallback;
+    // update cursor after switching desktops with Mission Control
+    bool delayed_cursor_update_requested;
 } _GLFWwindowNS;
 
 typedef struct _GLFWDisplayLinkNS
@@ -173,8 +177,8 @@ typedef struct _GLFWlibraryNS
     IOHIDManagerRef     hidManager;
     id                  unicodeData;
     id                  helper;
-    id                  keyUpMonitor;
-    id                  keyDownMonitor;
+    id                  keyUpMonitor, keyDownMonitor, flagsChangedMonitor;
+    id                  appleSettings;
     id                  nibObjects;
 
     char                keyName[64];
@@ -198,8 +202,8 @@ typedef struct _GLFWlibraryNS
         _GLFWDisplayLinkNS entries[256];
         size_t count;
     } displayLinks;
-    // the callback to handle file open events
-    GLFWhandlefileopen file_open_callback;
+    // the callback to handle url open events
+    GLFWhandleurlopen url_open_callback;
 
 } _GLFWlibraryNS;
 
@@ -231,7 +235,6 @@ typedef struct _GLFWtimerNS
 
 } _GLFWtimerNS;
 
-
 void _glfwPollMonitorsNS(void);
 void _glfwSetVideoModeNS(_GLFWmonitor* monitor, const GLFWvidmode* desired);
 void _glfwRestoreVideoModeNS(_GLFWmonitor* monitor);
@@ -248,3 +251,4 @@ void _glfwShutdownCVDisplayLink(unsigned long long, void*);
 void _glfwCocoaPostEmptyEvent(void);
 void _glfw_create_cv_display_link(_GLFWDisplayLinkNS *entry);
 _GLFWDisplayLinkNS* _glfw_create_display_link(CGDirectDisplayID);
+uint32_t vk_to_unicode_key_with_current_layout(uint16_t keycode);

glfw/cocoa_window.m

@@ -32,11 +32,6 @@
 #include <float.h>
 #include <string.h>
 
-#if (MAC_OS_X_VERSION_MAX_ALLOWED < 101300)
-#define NSControlStateValueOn NSOnState
-#define NSControlStateValueOff NSOffState
-#define NSControlStateValueMixed NSMixedState
-#endif
 
 static const char*
 polymorphic_string_as_utf8(id string) {
@@ -259,7 +254,7 @@ is_pua_char(uint32_t ch) {
     return (0xE000 <= ch && ch <= 0xF8FF) || (0xF0000 <= ch && ch <= 0xFFFFF) || (0x100000 <= ch && ch <= 0x10FFFF);
 }
 
-static uint32_t
+uint32_t
 vk_to_unicode_key_with_current_layout(uint16_t keycode)
 {
     UInt32 dead_key_state = 0;
@@ -513,8 +508,6 @@ translateFlags(NSUInteger flags)
     return mods;
 }
 
-#define debug_key(...) if (_glfw.hints.init.debugKeyboard) { fprintf(stderr, __VA_ARGS__); fflush(stderr); }
-
 static const char*
 format_mods(int mods) {
     static char buf[128];
@@ -576,166 +569,11 @@ get_window_size_without_border_in_logical_pixels(_GLFWwindow *window) {
     return [window->ns.object contentRectForFrameRect:[window->ns.object frame]];
 }
 
-// Translate a GLFW keycode to a Cocoa modifier flag
-//
-static NSUInteger
-translateKeyToModifierFlag(uint32_t key)
-{
-    switch (key)
-    {
-        case GLFW_FKEY_LEFT_SHIFT:
-        case GLFW_FKEY_RIGHT_SHIFT:
-            return NSEventModifierFlagShift;
-        case GLFW_FKEY_LEFT_CONTROL:
-        case GLFW_FKEY_RIGHT_CONTROL:
-            return NSEventModifierFlagControl;
-        case GLFW_FKEY_LEFT_ALT:
-        case GLFW_FKEY_RIGHT_ALT:
-            return NSEventModifierFlagOption;
-        case GLFW_FKEY_LEFT_SUPER:
-        case GLFW_FKEY_RIGHT_SUPER:
-            return NSEventModifierFlagCommand;
-        case GLFW_FKEY_CAPS_LOCK:
-            return NSEventModifierFlagCapsLock;
-    }
-
-    return 0;
-}
-
 // Defines a constant for empty ranges in NSTextInputClient
 //
 static const NSRange kEmptyRange = { NSNotFound, 0 };
 
 
-// SecureKeyboardEntryController {{{
-@interface SecureKeyboardEntryController : NSObject
-
-@property (nonatomic, readonly) BOOL isDesired;
-@property (nonatomic, readonly, getter=isEnabled) BOOL enabled;
-
-+ (instancetype)sharedInstance;
-
-- (void)toggle;
-- (void)update;
-
-@end
-
-@implementation SecureKeyboardEntryController {
-    int _count;
-    BOOL _desired;
-}
-
-+ (instancetype)sharedInstance {
-    static id instance;
-    static dispatch_once_t onceToken;
-    dispatch_once(&onceToken, ^{
-        instance = [[self alloc] init];
-    });
-    return instance;
-}
-
-- (instancetype)init {
-    self = [super init];
-    if (self) {
-        _desired = false;
-
-        [[NSNotificationCenter defaultCenter] addObserver:self
-                                                 selector:@selector(applicationDidResignActive:)
-                                                     name:NSApplicationDidResignActiveNotification
-                                                   object:nil];
-        [[NSNotificationCenter defaultCenter] addObserver:self
-                                                 selector:@selector(applicationDidBecomeActive:)
-                                                     name:NSApplicationDidBecomeActiveNotification
-                                                   object:nil];
-        if ([NSApp isActive]) {
-            [self update];
-        }
-    }
-    return self;
-}
-
-#pragma mark - API
-
-- (void)toggle {
-    // Set _desired to the opposite of the current state.
-    _desired = !_desired;
-    debug_key("toggle called. Setting desired to %d", _desired);
-
-    // Try to set the system's state of secure input to the desired state.
-    [self update];
-}
-
-- (BOOL)isEnabled {
-    return !!IsSecureEventInputEnabled();
-}
-
-- (BOOL)isDesired {
-    return _desired;
-}
-
-#pragma mark - Notifications
-
-- (void)applicationDidResignActive:(NSNotification *)notification {
-    (void)notification;
-    if (_count > 0) {
-        debug_key("Application resigning active.");
-        [self update];
-    }
-}
-
-- (void)applicationDidBecomeActive:(NSNotification *)notification {
-    (void)notification;
-    if (self.isDesired) {
-        debug_key("Application became active.");
-        [self update];
-    }
-}
-
-#pragma mark - Private
-
-- (BOOL)allowed {
-    return [NSApp isActive];
-}
-
-- (void)update {
-    debug_key("Update secure keyboard entry. desired=%d active=%d\n",
-         (int)self.isDesired, (int)[NSApp isActive]);
-    const BOOL secure = self.isDesired && [self allowed];
-
-    if (secure && _count > 0) {
-        debug_key("Want to turn on secure input but it's already on\n");
-        return;
-    }
-
-    if (!secure && _count == 0) {
-        debug_key("Want to turn off secure input but it's already off\n");
-        return;
-    }
-
-    debug_key("Before: IsSecureEventInputEnabled returns %d ", (int)self.isEnabled);
-    if (secure) {
-        OSErr err = EnableSecureEventInput();
-        debug_key("EnableSecureEventInput err=%d ", (int)err);
-        if (err) {
-            debug_key("EnableSecureEventInput failed with error %d ", (int)err);
-        } else {
-            _count += 1;
-        }
-    } else {
-        OSErr err = DisableSecureEventInput();
-        debug_key("DisableSecureEventInput err=%d ", (int)err);
-        if (err) {
-            debug_key("DisableSecureEventInput failed with error %d ", (int)err);
-        } else {
-            _count -= 1;
-        }
-    }
-    debug_key("After: IsSecureEventInputEnabled returns %d\n", (int)self.isEnabled);
-}
-
-@end
-// }}}
-
 // Delegate for window related notifications {{{
 
 @interface GLFWWindowDelegate : NSObject
@@ -744,6 +582,7 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
 }
 
 - (instancetype)initWithGlfwWindow:(_GLFWwindow *)initWindow;
+- (void)request_delayed_cursor_update:(id)sender;
 
 @end
 
@@ -854,6 +693,9 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
         _glfwPlatformGetCursorPos(window, &x, &y);
         _glfwInputCursorPos(window, x, y);
     }
+    // macOS will send a delayed event to update the cursor to arrow after switching desktops.
+    // So we need to delay and update the cursor once after that.
+    [self performSelector:@selector(request_delayed_cursor_update:) withObject:nil afterDelay:0.3];
 }
 
 - (void)windowDidResignKey:(NSNotification *)notification
@@ -866,9 +708,11 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
     _glfwInputWindowFocus(window, false);
     // IME is cancelled when losing the focus
     if ([window->ns.view hasMarkedText]) {
+        [[window->ns.view inputContext] discardMarkedText];
         [window->ns.view unmarkText];
         GLFWkeyevent dummy = {.action = GLFW_RELEASE, .ime_state = GLFW_IME_PREEDIT_CHANGED};
         _glfwInputKeyboard(window, &dummy);
+        _glfw.ns.text[0] = 0;
     }
 }
 
@@ -882,8 +726,54 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
     }
 }
 
+- (void)request_delayed_cursor_update:(id)sender
+{
+    (void)sender;
+    if (window) window->ns.delayed_cursor_update_requested = true;
+}
+
+- (void)windowWillEnterFullScreen:(NSNotification *)notification
+{
+    (void)notification;
+    if (window) window->ns.in_fullscreen_transition = true;
+}
+
+- (void)windowDidEnterFullScreen:(NSNotification *)notification
+{
+    (void)notification;
+    if (window) window->ns.in_fullscreen_transition = false;
+    [self performSelector:@selector(request_delayed_cursor_update:) withObject:nil afterDelay:0.3];
+}
+
+- (void)windowWillExitFullScreen:(NSNotification *)notification
+{
+    (void)notification;
+    if (window) window->ns.in_fullscreen_transition = true;
+}
+
+- (void)windowDidExitFullScreen:(NSNotification *)notification
+{
+    (void)notification;
+    if (window) window->ns.in_fullscreen_transition = false;
+    [self performSelector:@selector(request_delayed_cursor_update:) withObject:nil afterDelay:0.3];
+}
+
 @end // }}}
 
+// Text input context class for the GLFW content view {{{
+
+@interface GLFWTextInputContext : NSTextInputContext
+@end
+
+@implementation GLFWTextInputContext
+- (void)doCommandBySelector:(SEL)selector
+{
+    // interpretKeyEvents: May call insertText: or doCommandBySelector:.
+    // With the default macOS keybindings, pressing certain key combinations
+    // (e.g. Ctrl+/, Ctrl+Cmd+Down/Left/Right) will produce a beep sound.
+    debug_key("\n\tTextInputCtx: doCommandBySelector: (%s)\n", [NSStringFromSelector(selector) UTF8String]);
+}
+@end // }}}
 
 // Content view class for the GLFW window {{{
 
@@ -891,9 +781,11 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
 {
     _GLFWwindow* window;
     NSTrackingArea* trackingArea;
+    GLFWTextInputContext* input_context;
     NSMutableAttributedString* markedText;
     NSRect markedRect;
     bool marked_text_cleared_by_insert;
+    int in_key_handler;
     NSString *input_source_at_last_key_event;
 }
 
@@ -911,9 +803,11 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
     {
         window = initWindow;
         trackingArea = nil;
+        input_context = [[GLFWTextInputContext alloc] initWithClient:self];
         markedText = [[NSMutableAttributedString alloc] init];
         markedRect = NSMakeRect(0.0, 0.0, 0.0, 0.0);
         input_source_at_last_key_event = nil;
+        in_key_handler = 0;
 
         [self updateTrackingAreas];
         [self registerForDraggedTypes:@[NSPasteboardTypeFileURL, NSPasteboardTypeString]];
@@ -927,6 +821,7 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
     [trackingArea release];
     [markedText release];
     if (input_source_at_last_key_event) [input_source_at_last_key_event release];
+    [input_context release];
     [super dealloc];
 }
 
@@ -1045,6 +940,11 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
 
     window->ns.cursorWarpDeltaX = 0;
     window->ns.cursorWarpDeltaY = 0;
+
+    if (window->ns.delayed_cursor_update_requested) {
+        window->ns.delayed_cursor_update_requested = false;
+        if (cursorInContentArea(window)) updateCursorImage(window);
+    }
 }
 
 - (void)rightMouseDown:(NSEvent *)event
@@ -1144,6 +1044,8 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
 
 - (void)updateTrackingAreas
 {
+    if (window && [window->ns.object areCursorRectsEnabled])
+        [window->ns.object disableCursorRects];
     if (trackingArea != nil)
     {
         [self removeTrackingArea:trackingArea];
@@ -1166,6 +1068,11 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
     [super updateTrackingAreas];
 }
 
+- (NSTextInputContext *)inputContext
+{
+    return input_context;
+}
+
 static UInt32
 convert_cocoa_to_carbon_modifiers(NSUInteger flags) {
     UInt32 mods = 0;
@@ -1231,15 +1138,16 @@ is_ascii_control_char(char x) {
 #define UPDATE_PRE_EDIT_TEXT glfw_keyevent.text = [[markedText string] UTF8String]; glfw_keyevent.ime_state = GLFW_IME_PREEDIT_CHANGED; _glfwInputKeyboard(window, &glfw_keyevent);
 
     const bool previous_has_marked_text = [self hasMarkedText];
-    NSTextInputContext *inpctx = [NSTextInputContext currentInputContext];
-    if (inpctx && (!input_source_at_last_key_event || ![input_source_at_last_key_event isEqualToString:inpctx.selectedKeyboardInputSource])) {
+    if (input_context && (!input_source_at_last_key_event || ![input_source_at_last_key_event isEqualToString:input_context.selectedKeyboardInputSource])) {
         if (input_source_at_last_key_event) {
             debug_key("Input source changed, clearing pre-edit text and resetting deadkey state\n");
             GLFWkeyevent dummy = {.action = GLFW_RELEASE, .ime_state = GLFW_IME_PREEDIT_CHANGED};
             window->ns.deadKeyState = 0;
             _glfwInputKeyboard(window, &dummy); // clear pre-edit text
+            [input_source_at_last_key_event release];
+            input_source_at_last_key_event = nil;
         }
-        input_source_at_last_key_event = [inpctx.selectedKeyboardInputSource retain];
+        input_source_at_last_key_event = [input_context.selectedKeyboardInputSource retain];
         [self unmarkText];
     }
 
@@ -1247,7 +1155,7 @@ is_ascii_control_char(char x) {
     const NSUInteger flags = [event modifierFlags];
     const int mods = translateFlags(flags);
     const uint32_t key = translateKey(keycode, true);
-    const bool process_text = !window->ns.textInputFilterCallback || window->ns.textInputFilterCallback(key, mods, keycode, flags) != 1;
+    const bool process_text = !_glfw.ignoreOSKeyboardProcessing && (!window->ns.textInputFilterCallback || window->ns.textInputFilterCallback(key, mods, keycode, flags) != 1);
     _glfw.ns.text[0] = 0;
     if (keycode == 0x33 /* backspace */ || keycode == 0x35 /* escape */) [self unmarkText];
     GLFWkeyevent glfw_keyevent = {.key = key, .native_key = keycode, .native_key_id = keycode, .action = GLFW_PRESS, .mods = mods};
@@ -1257,7 +1165,7 @@ is_ascii_control_char(char x) {
         // keyboard unicode data is not available.
         if (process_text) {
             // this will call insertText with the text for this event, if any
-            [self interpretKeyEvents:[NSArray arrayWithObject:event]];
+            [self interpretKeyEvents:@[event]];
         }
     } else {
         static UniChar text[256];
@@ -1284,8 +1192,10 @@ is_ascii_control_char(char x) {
                 keycode, safe_name_for_keycode(keycode), key, format_mods(mods), char_count, window->ns.deadKeyState, event.ARepeat);
         marked_text_cleared_by_insert = false;
         if (process_text) {
+            in_key_handler = 1;
             // this will call insertText which will fill up _glfw.ns.text
-            [self interpretKeyEvents:[NSArray arrayWithObject:event]];
+            [self interpretKeyEvents:@[event]];
+            in_key_handler = 0;
         } else {
             window->ns.deadKeyState = 0;
         }
@@ -1314,6 +1224,7 @@ is_ascii_control_char(char x) {
         }
         if (([self hasMarkedText] || previous_has_marked_text) && !_glfw.ns.text[0]) {
             // do not pass keys like BACKSPACE while there's pre-edit text, let IME handle it
+            debug_key("Ignoring key press as IME is active and it generated no text\n");
             return;
         }
     }
@@ -1329,35 +1240,60 @@ is_ascii_control_char(char x) {
         // insertText followed by setMarkedText
         UPDATE_PRE_EDIT_TEXT;
     }
-#undef CLEAR_PRE_EDIT_TEXT
-#undef UPDATE_PRE_EDIT_TEXT
 }
 
 - (void)flagsChanged:(NSEvent *)event
 {
     int action = GLFW_RELEASE;
-    const unsigned int modifierFlags =
-        [event modifierFlags] & NSEventModifierFlagDeviceIndependentFlagsMask;
-    const uint32_t key = translateKey([event keyCode], false);
+    const char old_first_char = _glfw.ns.text[0];
+    _glfw.ns.text[0] = 0;
+    const NSUInteger modifierFlags = [event modifierFlags] & NSEventModifierFlagDeviceIndependentFlagsMask;
+    const uint32_t key = vk_code_to_functional_key_code([event keyCode]);
+    const unsigned int keycode = [event keyCode];
     const int mods = translateFlags(modifierFlags);
-    const NSUInteger keyFlag = translateKeyToModifierFlag(key);
+    const bool process_text = !_glfw.ignoreOSKeyboardProcessing && (!window->ns.textInputFilterCallback || window->ns.textInputFilterCallback(key, mods, keycode, modifierFlags) != 1);
+    const char *mod_name = "unknown";
 
-    if (keyFlag & modifierFlags)
-    {
-        int current_action = GLFW_RELEASE;
-        for (unsigned i = 0; i < arraysz(window->activated_keys); i++) {
-            if (window->activated_keys[i].key == key) {
-                current_action = window->activated_keys[i].action;
-                break;
-            }
-        }
-        if (current_action == GLFW_PRESS)
-            action = GLFW_RELEASE;
-        else
-            action = GLFW_PRESS;
+    switch(key) {
+        case GLFW_FKEY_CAPS_LOCK:
+            mod_name = "capslock";
+            action = modifierFlags & NSEventModifierFlagCapsLock ? GLFW_PRESS : GLFW_RELEASE; break;
+        case GLFW_FKEY_LEFT_SUPER:
+        case GLFW_FKEY_RIGHT_SUPER:
+            mod_name = "super";
+            action = modifierFlags & NSEventModifierFlagCommand ? GLFW_PRESS : GLFW_RELEASE; break;
+        case GLFW_FKEY_LEFT_CONTROL:
+        case GLFW_FKEY_RIGHT_CONTROL:
+            mod_name = "ctrl";
+            action = modifierFlags & NSEventModifierFlagControl ? GLFW_PRESS : GLFW_RELEASE; break;
+        case GLFW_FKEY_LEFT_ALT:
+        case GLFW_FKEY_RIGHT_ALT:
+            mod_name = "alt";
+            action = modifierFlags & NSEventModifierFlagOption ? GLFW_PRESS : GLFW_RELEASE; break;
+        case GLFW_FKEY_LEFT_SHIFT:
+        case GLFW_FKEY_RIGHT_SHIFT:
+            mod_name = "shift";
+            action = modifierFlags & NSEventModifierFlagShift ? GLFW_PRESS : GLFW_RELEASE; break;
+        default:
+            return;
     }
-
-    GLFWkeyevent glfw_keyevent = {.key = key, .native_key = [event keyCode], .native_key_id = [event keyCode], .action = action, .mods = mods};
+    GLFWkeyevent glfw_keyevent = {.key = key, .native_key = keycode, .native_key_id = keycode, .action = action, .mods = mods};
+    debug_key("\x1b[33mflagsChanged:\x1b[m modifier: %s native_key: 0x%x (%s) glfw_key: 0x%x %s\n",
+            mod_name, keycode, safe_name_for_keycode(keycode), key, format_mods(mods));
+    marked_text_cleared_by_insert = false;
+    if (process_text && input_context) {
+        // this will call insertText which will fill up _glfw.ns.text
+        in_key_handler = 2;
+        [input_context handleEvent:event];
+        in_key_handler = 0;
+        if (marked_text_cleared_by_insert) {
+            debug_key("Clearing pre-edit text because insertText called from flagsChanged\n");
+            CLEAR_PRE_EDIT_TEXT;
+            if (_glfw.ns.text[0]) glfw_keyevent.text = _glfw.ns.text;
+            else _glfw.ns.text[0] = old_first_char;
+        }
+    }
+    glfw_keyevent.ime_state = GLFW_IME_NONE;
     _glfwInputKeyboard(window, &glfw_keyevent);
 }
 
@@ -1374,6 +1310,9 @@ is_ascii_control_char(char x) {
     _glfwInputKeyboard(window, &glfw_keyevent);
 }
 
+#undef CLEAR_PRE_EDIT_TEXT
+#undef UPDATE_PRE_EDIT_TEXT
+
 - (void)scrollWheel:(NSEvent *)event
 {
     double deltaX = [event scrollingDeltaX];
@@ -1429,14 +1368,17 @@ is_ascii_control_char(char x) {
                                               options:options];
     if (!objs) return NO;
     const NSUInteger count = [objs count];
+    NSMutableString *uri_list = [NSMutableString stringWithCapacity:4096];  // auto-released
     if (count)
     {
         for (NSUInteger i = 0;  i < count;  i++)
         {
             id obj = objs[i];
             if ([obj isKindOfClass:[NSURL class]]) {
-                const char *path = [obj fileSystemRepresentation];
-                _glfwInputDrop(window, "text/plain;charset=utf-8", path, strlen(path));
+                NSURL *url = (NSURL*)obj;
+                if ([uri_list length] > 0) [uri_list appendString:@("\n")];
+                if (url.fileURL) [uri_list appendString:url.filePathURL.absoluteString];
+                else [uri_list appendString:url.absoluteString];
             } else if ([obj isKindOfClass:[NSString class]]) {
                 const char *text = [obj UTF8String];
                 _glfwInputDrop(window, "text/plain;charset=utf-8", text, strlen(text));
@@ -1446,6 +1388,7 @@ is_ascii_control_char(char x) {
             }
         }
     }
+    if ([uri_list length] > 0) _glfwInputDrop(window, "text/uri-list", uri_list.UTF8String, strlen(uri_list.UTF8String));
 
     return YES;
 }
@@ -1472,8 +1415,19 @@ is_ascii_control_char(char x) {
         selectedRange:(NSRange)selectedRange
      replacementRange:(NSRange)replacementRange
 {
-    debug_key("\n\tsetMarkedText: %s selectedRange: (%lu, %lu) replacementRange: (%lu, %lu)\n", polymorphic_string_as_utf8(string), selectedRange.location, selectedRange.length, replacementRange.location, replacementRange.length);
-    if (string == nil) { [self unmarkText]; return; }
+    const char *s = polymorphic_string_as_utf8(string);
+    debug_key("\n\tsetMarkedText: %s selectedRange: (%lu, %lu) replacementRange: (%lu, %lu)\n", s, selectedRange.location, selectedRange.length, replacementRange.location, replacementRange.length);
+    if (string == nil || !s[0]) {
+        bool had_marked_text = [self hasMarkedText];
+        [self unmarkText];
+        if (had_marked_text && (!in_key_handler || in_key_handler == 2)) {
+            debug_key("Clearing pre-edit because setMarkedText called from %s\n", in_key_handler ? "flagsChanged" : "event loop");
+            GLFWkeyevent glfw_keyevent = {.ime_state = GLFW_IME_PREEDIT_CHANGED};
+            _glfwInputKeyboard(window, &glfw_keyevent);
+            _glfw.ns.text[0] = 0;
+        }
+        return;
+    }
     if ([string isKindOfClass:[NSAttributedString class]]) {
         if (((NSMutableAttributedString*)string).length == 0) { [self unmarkText]; return; }
         [markedText release];
@@ -1483,6 +1437,12 @@ is_ascii_control_char(char x) {
         [markedText release];
         markedText = [[NSMutableAttributedString alloc] initWithString:string];
     }
+    if (!in_key_handler || in_key_handler == 2) {
+        debug_key("Updating IME text in kitty from setMarkedText called from %s: %s\n", in_key_handler ? "flagsChanged" : "event loop", _glfw.ns.text);
+        GLFWkeyevent glfw_keyevent = {.text=[[markedText string] UTF8String], .ime_state = GLFW_IME_PREEDIT_CHANGED};
+        _glfwInputKeyboard(window, &glfw_keyevent);
+        _glfw.ns.text[0] = 0;
+    }
 }
 
 - (void)unmarkText
@@ -1491,15 +1451,23 @@ is_ascii_control_char(char x) {
 }
 
 void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
-    [w->ns.view updateIMEStateFor: ev->type left:(CGFloat)ev->cursor.left top:(CGFloat)ev->cursor.top cellWidth:(CGFloat)ev->cursor.width cellHeight:(CGFloat)ev->cursor.height];
+    [w->ns.view updateIMEStateFor: ev->type focused:(bool)ev->focused left:(CGFloat)ev->cursor.left top:(CGFloat)ev->cursor.top cellWidth:(CGFloat)ev->cursor.width cellHeight:(CGFloat)ev->cursor.height];
 }
 
 - (void)updateIMEStateFor:(GLFWIMEUpdateType)which
+                  focused:(bool)focused
                      left:(CGFloat)left
                       top:(CGFloat)top
                 cellWidth:(CGFloat)cellWidth
                cellHeight:(CGFloat)cellHeight
 {
+    if (which == GLFW_IME_UPDATE_FOCUS && !focused && [self hasMarkedText] && window) {
+        [input_context discardMarkedText];
+        [self unmarkText];
+        GLFWkeyevent glfw_keyevent = {.ime_state = GLFW_IME_PREEDIT_CHANGED};
+        _glfwInputKeyboard(window, &glfw_keyevent);
+        _glfw.ns.text[0] = 0;
+    }
     if (which != GLFW_IME_UPDATE_CURSOR_POSITION) return;
     left /= window->ns.xscale;
     top /= window->ns.yscale;
@@ -1543,14 +1511,28 @@ void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
 {
     const char *utf8 = polymorphic_string_as_utf8(string);
     debug_key("\n\tinsertText: %s replacementRange: (%lu, %lu)\n", utf8, replacementRange.location, replacementRange.length);
-    if ([self hasMarkedText]) {
+    if ([self hasMarkedText] && !is_ascii_control_char(utf8[0])) {
         [self unmarkText];
         marked_text_cleared_by_insert = true;
+        if (!in_key_handler) {
+            debug_key("Clearing pre-edit because insertText called from event loop\n");
+            GLFWkeyevent glfw_keyevent = {.ime_state = GLFW_IME_PREEDIT_CHANGED};
+            _glfwInputKeyboard(window, &glfw_keyevent);
+            _glfw.ns.text[0] = 0;
+        }
     }
     // insertText can be called multiple times for a single key event
     char *s = _glfw.ns.text + strnlen(_glfw.ns.text, sizeof(_glfw.ns.text));
     snprintf(s, sizeof(_glfw.ns.text) - (s - _glfw.ns.text), "%s", utf8);
     _glfw.ns.text[sizeof(_glfw.ns.text) - 1] = 0;
+    if ((!in_key_handler || in_key_handler == 2) && _glfw.ns.text[0]) {
+        if (!is_ascii_control_char(_glfw.ns.text[0])) {
+            debug_key("Sending text to kitty from insertText called from %s: %s\n", in_key_handler ? "flagsChanged" : "event loop", _glfw.ns.text);
+            GLFWkeyevent glfw_keyevent = {.text=_glfw.ns.text, .ime_state=GLFW_IME_COMMIT_TEXT};
+            _glfwInputKeyboard(window, &glfw_keyevent);
+        }
+        _glfw.ns.text[0] = 0;
+    }
 }
 
 - (void)doCommandBySelector:(SEL)selector
@@ -1597,19 +1579,6 @@ void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
 
 - (BOOL)validateMenuItem:(NSMenuItem *)item {
     if (item.action == @selector(performMiniaturize:)) return YES;
-    if (item.action == @selector(toggleSecureInput:)) {
-      SecureKeyboardEntryController *controller = [SecureKeyboardEntryController sharedInstance];
-      if (controller.isEnabled) {
-          if (controller.isDesired) {
-              item.state = NSControlStateValueOn;
-          } else {
-              item.state = NSControlStateValueMixed;
-          }
-      } else {
-          item.state = controller.isDesired ? NSControlStateValueOn : NSControlStateValueOff;
-      }
-      return YES;
-    }
     return [super validateMenuItem:item];
 }
 
@@ -1619,11 +1588,6 @@ void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
     else [super performMiniaturize:sender];
 }
 
-- (void)toggleSecureInput:(id)sender {
-    (void)sender;
-    [[SecureKeyboardEntryController sharedInstance] toggle];
-}
-
 - (BOOL)canBecomeKeyWindow
 {
     // Required for NSWindowStyleMaskBorderless windows
@@ -1637,8 +1601,11 @@ void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
 
 - (void)toggleFullScreen:(nullable id)sender
 {
-    if (glfw_window && glfw_window->ns.toggleFullscreenCallback && glfw_window->ns.toggleFullscreenCallback((GLFWwindow*)glfw_window) == 1)
-            return;
+    if (glfw_window) {
+        if (glfw_window->ns.in_fullscreen_transition) return;
+        if (glfw_window->ns.toggleFullscreenCallback && glfw_window->ns.toggleFullscreenCallback((GLFWwindow*)glfw_window) == 1) return;
+        glfw_window->ns.in_fullscreen_transition = true;
+    }
     // When resizeIncrements is set, Cocoa cannot restore the original window size after returning from fullscreen.
     const NSSize original = [self resizeIncrements];
     [self setResizeIncrements:NSMakeSize(1.0, 1.0)];
@@ -2449,7 +2416,7 @@ void _glfwPlatformSetCursor(_GLFWwindow* window, _GLFWcursor* cursor UNUSED)
 bool _glfwPlatformIsFullscreen(_GLFWwindow* w, unsigned int flags) {
     NSWindow *window = w->ns.object;
     bool traditional = !(flags & 1);
-    if (traditional) { if(@available(macOS 11.0, *)) return w->ns.in_traditional_fullscreen; }
+    if (traditional) { if(@available(macOS 10.15.7, *)) return w->ns.in_traditional_fullscreen; }
     NSWindowStyleMask sm = [window styleMask];
     return sm & NSWindowStyleMaskFullScreen;
 }
@@ -2460,8 +2427,9 @@ bool _glfwPlatformToggleFullscreen(_GLFWwindow* w, unsigned int flags) {
     bool traditional = !(flags & 1);
     NSWindowStyleMask sm = [window styleMask];
     if (traditional) {
-        if (@available(macOS 11.0, *)) {
+        if (@available(macOS 10.15.7, *)) {
             // As of Big Turd NSWindowStyleMaskFullScreen is no longer useable
+            // Also no longer compatible after a minor release of macOS 10.15.7
             if (!w->ns.in_traditional_fullscreen) {
                 w->ns.pre_full_screen_style_mask = sm;
                 [window setStyleMask: NSWindowStyleMaskBorderless];
@@ -2724,10 +2692,10 @@ GLFWAPI GLFWcocoatextinputfilterfun glfwSetCocoaTextInputFilter(GLFWwindow *hand
     return previous;
 }
 
-GLFWAPI GLFWhandlefileopen glfwSetCocoaFileOpenCallback(GLFWhandlefileopen callback) {
+GLFWAPI GLFWhandleurlopen glfwSetCocoaURLOpenCallback(GLFWhandleurlopen callback) {
     _GLFW_REQUIRE_INIT_OR_RETURN(nil);
-    GLFWhandlefileopen prev = _glfw.ns.file_open_callback;
-    _glfw.ns.file_open_callback = callback;
+    GLFWhandleurlopen prev = _glfw.ns.url_open_callback;
+    _glfw.ns.url_open_callback = callback;
     return prev;
 }
 

glfw/dbus_glfw.c

@@ -263,7 +263,7 @@ call_method_with_msg(DBusConnection *conn, DBusMessage *msg, int timeout, dbus_p
 
 static bool
 call_method(DBusConnection *conn, const char *node, const char *path, const char *interface, const char *method, int timeout, dbus_pending_callback callback, void *user_data, va_list ap) {
-    if (!conn) return false;
+    if (!conn || !path) return false;
     DBusMessage *msg = dbus_message_new_method_call(node, path, interface, method);
     if (!msg) return false;
     bool retval = false;

glfw/glfw.py

@@ -67,7 +67,7 @@ class Env:
 
 def wayland_protocol_file_name(base: str, ext: str = 'c') -> str:
     base = os.path.basename(base).rpartition('.')[0]
-    return 'wayland-{}-client-protocol.{}'.format(base, ext)
+    return f'wayland-{base}-client-protocol.{ext}'
 
 
 def init_env(
@@ -80,7 +80,7 @@ def init_env(
 ) -> Env:
     ans = env.copy()
     ans.cflags.append('-fPIC')
-    ans.cppflags.append('-D_GLFW_' + module.upper())
+    ans.cppflags.append(f'-D_GLFW_{module.upper()}')
     ans.cppflags.append('-D_GLFW_BUILD_DLL')
 
     with open(os.path.join(base, 'source-info.json')) as f:
@@ -111,7 +111,7 @@ def init_env(
 
     elif module == 'cocoa':
         ans.cppflags.append('-DGL_SILENCE_DEPRECATION')
-        for f_ in 'Cocoa Carbon IOKit CoreFoundation CoreVideo'.split():
+        for f_ in 'Cocoa IOKit CoreFoundation CoreVideo'.split():
             ans.ldpaths.extend(('-framework', f_))
 
     elif module == 'wayland':
@@ -150,14 +150,14 @@ def build_wayland_protocols(
     for protocol in env.wayland_protocols:
         src = os.path.join(env.wayland_packagedir, protocol)
         if not os.path.exists(src):
-            raise SystemExit('The wayland-protocols package on your system is missing the {} protocol definition file'.format(protocol))
+            raise SystemExit(f'The wayland-protocols package on your system is missing the {protocol} protocol definition file')
         for ext in 'hc':
             dest = wayland_protocol_file_name(src, ext)
             dest = os.path.join(dest_dir, dest)
             if newer(dest, src):
                 q = 'client-header' if ext == 'h' else env.wayland_scanner_code
                 items.append(Command(
-                    'Generating {} ...'.format(emphasis(os.path.basename(dest))),
+                    f'Generating {emphasis(os.path.basename(dest))} ...',
                     [env.wayland_scanner, q, src, dest], lambda: True))
     if items:
         parallel_run(items)
@@ -174,7 +174,7 @@ class Arg:
             self.type = self.type + '*'
 
     def __repr__(self) -> str:
-        return 'Arg({}, {})'.format(self.type, self.name)
+        return f'Arg({self.type}, {self.name})'
 
 
 class Function:
@@ -233,7 +233,7 @@ def generate_wrappers(glfw_header: str) -> None:
     void* glfwGetNSGLContext(GLFWwindow *window)
     uint32_t glfwGetCocoaMonitor(GLFWmonitor* monitor)
     GLFWcocoatextinputfilterfun glfwSetCocoaTextInputFilter(GLFWwindow* window, GLFWcocoatextinputfilterfun callback)
-    GLFWhandlefileopen glfwSetCocoaFileOpenCallback(GLFWhandlefileopen callback)
+    GLFWhandleurlopen glfwSetCocoaURLOpenCallback(GLFWhandleurlopen callback)
     GLFWcocoatogglefullscreenfun glfwSetCocoaToggleFullscreenIntercept(GLFWwindow *window, GLFWcocoatogglefullscreenfun callback)
     GLFWapplicationshouldhandlereopenfun glfwSetApplicationShouldHandleReopen(GLFWapplicationshouldhandlereopenfun callback)
     GLFWapplicationwillfinishlaunchingfun glfwSetApplicationWillFinishLaunching(GLFWapplicationwillfinishlaunchingfun callback)
@@ -277,7 +277,7 @@ const char *action_text, int32_t timeout, GLFWDBusnotificationcreatedfun callbac
 
 typedef int (* GLFWcocoatextinputfilterfun)(int,int,unsigned int,unsigned long);
 typedef bool (* GLFWapplicationshouldhandlereopenfun)(int);
-typedef bool (* GLFWhandlefileopen)(const char*);
+typedef bool (* GLFWhandleurlopen)(const char*);
 typedef void (* GLFWapplicationwillfinishlaunchingfun)(void);
 typedef bool (* GLFWcocoatogglefullscreenfun)(GLFWwindow*);
 typedef void (* GLFWcocoarenderframefun)(GLFWwindow*);
@@ -292,6 +292,8 @@ const char* load_glfw(const char* path);
         f.write(header)
 
     code = '''
+// generated by glfw.py DO NOT edit
+
 #define GFW_EXTERN
 #include "data-types.h"
 #include "glfw-wrapper.h"

glfw/glfw3.h

@@ -4119,6 +4119,9 @@ GLFWAPI GLFWwindowcontentscalefun glfwSetWindowContentScaleCallback(GLFWwindow*
  */
 GLFWAPI void glfwPostEmptyEvent(void);
 
+GLFWAPI bool glfwGetIgnoreOSKeyboardProcessing(void);
+GLFWAPI void glfwSetIgnoreOSKeyboardProcessing(bool enabled);
+
 /*! @brief Returns the value of an input option for the specified window.
  *
  *  This function returns the value of an input option for the specified window.

glfw/ibus_glfw.c

@@ -24,6 +24,17 @@
 //
 //========================================================================
 
+/* To test under X11 start IBUS as:
+ * ibus-daemon -drxR
+ * Setup the input sources you want with:
+ * ibus-setup
+ * Switch to the input source you want to test with:
+ * ibus engine name
+ * You can list available engines with:
+ * ibus list-engine
+ * Then run kitty as:
+ * GLFW_IM_MODULE=ibus kitty
+ */
 #define _GNU_SOURCE
 #include <stdio.h>
 #include <errno.h>
@@ -226,6 +237,7 @@ message_handler(DBusConnection *conn UNUSED, DBusMessage *msg, void *user_data)
             break;
         case 2:
             debug("IBUS: HidePreeditText\n");
+            send_text("", GLFW_IME_PREEDIT_CHANGED);
             break;
         case 3:
             debug("IBUS: ShowPreeditText\n");

glfw/init.c

@@ -227,6 +227,7 @@ GLFWAPI int glfwInit(monotonic_t start_time)
 
     memset(&_glfw, 0, sizeof(_glfw));
     _glfw.hints.init = _glfwInitHints;
+    _glfw.ignoreOSKeyboardProcessing = false;
 
     if (!_glfwPlatformInit())
     {

glfw/input.c

@@ -667,6 +667,14 @@ void _glfwCenterCursorInContentArea(_GLFWwindow* window)
 //////                        GLFW public API                       //////
 //////////////////////////////////////////////////////////////////////////
 
+GLFWAPI bool glfwGetIgnoreOSKeyboardProcessing(void) {
+    return _glfw.ignoreOSKeyboardProcessing;
+}
+
+GLFWAPI void glfwSetIgnoreOSKeyboardProcessing(bool enabled) {
+    _glfw.ignoreOSKeyboardProcessing = enabled;
+}
+
 GLFWAPI int glfwGetInputMode(GLFWwindow* handle, int mode)
 {
     _GLFWwindow* window = (_GLFWwindow*) handle;

glfw/internal.h

@@ -584,6 +584,8 @@ struct _GLFWlibrary
     _GLFWtls            contextSlot;
     _GLFWmutex          errorLock;
 
+    bool                ignoreOSKeyboardProcessing;
+
     struct {
         bool            available;
         void*           handle;

glfw/linux_desktop_settings.c

@@ -10,77 +10,118 @@
 #include <strings.h>
 #include <string.h>
 
-static const char *DESKTOP_SERVICE = "org.freedesktop.portal.Desktop";
-static const char *DESKTOP_PATH = "/org/freedesktop/portal/desktop";
-static const char *DESKTOP_INTERFACE = "org.freedesktop.portal.Settings";
-static const char *GNOME_DESKTOP_NAMESPACE = "org.gnome.desktop.interface";
+#define DESKTOP_SERVICE "org.freedesktop.portal.Desktop"
+#define DESKTOP_PATH "/org/freedesktop/portal/desktop"
+#define DESKTOP_INTERFACE "org.freedesktop.portal.Settings"
+#define GNOME_DESKTOP_NAMESPACE "org.gnome.desktop.interface"
+#define FDO_DESKTOP_NAMESPACE "org.freedesktop.appearance"
+#define FDO_APPEARANCE_KEY "color-scheme"
 
 
-static char theme_name[64] = {0};
+static char theme_name[128] = {0};
 static int theme_size = -1;
-static bool gnome_cursor_theme_read = false, gnome_cursor_size_read = false;
+static uint32_t appearance = 0;
+static bool is_gnome = false;
+static bool cursor_theme_changed = false;
 
-static bool
-parse_dbus_message_for_type(DBusMessage *const reply, const char *errmsg, const int type, void *value) {
-	DBusMessageIter iter[3];
-	dbus_message_iter_init(reply, &iter[0]);
-#define FAIL { _glfwInputError(GLFW_PLATFORM_ERROR, "%s", errmsg); return false; }
-	if (dbus_message_iter_get_arg_type(&iter[0]) != DBUS_TYPE_VARIANT) FAIL;
-	dbus_message_iter_recurse(&iter[0], &iter[1]);
-	if (dbus_message_iter_get_arg_type(&iter[1]) != DBUS_TYPE_VARIANT) FAIL;
-	dbus_message_iter_recurse(&iter[1], &iter[2]);
-	if (dbus_message_iter_get_arg_type(&iter[2]) != type) FAIL;
-	dbus_message_iter_get_basic(&iter[2], value);
-	return true;
-#undef FAIL
-}
-
-#define HANDLER(name) void name(DBusMessage *msg, const char* errmsg, void *data) { \
+#define HANDLER(name) static void name(DBusMessage *msg, const char* errmsg, void *data) { \
     (void)data; \
     if (errmsg) { \
         _glfwInputError(GLFW_PLATFORM_ERROR, "%s: failed with error: %s", #name, errmsg); \
         return; \
     }
 
-HANDLER(on_gnome_cursor_theme_read)
-    const char *name;
-    if (!parse_dbus_message_for_type(msg, "Failed to get cursor theme name from reply", DBUS_TYPE_STRING, &name)) return;
-    if (name && name[0]) {
-        gnome_cursor_theme_read = true;
-        strncpy(theme_name, name, sizeof(theme_name) - 1);
-        if (gnome_cursor_size_read) _glfwPlatformChangeCursorTheme();
+static void
+process_fdo_setting(const char *key, DBusMessageIter *value) {
+    if (strcmp(key, FDO_APPEARANCE_KEY) == 0) {
+        if (dbus_message_iter_get_arg_type(value) == DBUS_TYPE_UINT32) {
+            dbus_message_iter_get_basic(value, &appearance);
+            if (appearance > 2) appearance = 0;
+        }
     }
 }
 
-HANDLER(on_gnome_cursor_size_read)
-    int32_t sz;
-    if (!parse_dbus_message_for_type(msg, "Failed to get cursor theme size from reply", DBUS_TYPE_INT32, &sz)) return;
-    gnome_cursor_size_read = true;
-    theme_size = sz;
-    if (gnome_cursor_theme_read) _glfwPlatformChangeCursorTheme();
-}
-#undef HANDLER
-
-
-static bool
-call_read(DBusConnection *session_bus, dbus_pending_callback callback, const char *namespace, const char *key) {
-    return glfw_dbus_call_method_with_reply(
-            session_bus, DESKTOP_SERVICE, DESKTOP_PATH, DESKTOP_INTERFACE, "Read", DBUS_TIMEOUT_USE_DEFAULT,
-            callback, NULL, DBUS_TYPE_STRING, &namespace, DBUS_TYPE_STRING, &key, DBUS_TYPE_INVALID);
-}
-
 static void
-get_from_gnome(void) {
-    theme_size = 32;
-    DBusConnection *session_bus = glfw_dbus_session_bus();
-    if (session_bus) {
-        const char *theme_key = "cursor-theme";
-        call_read(session_bus, on_gnome_cursor_theme_read, GNOME_DESKTOP_NAMESPACE, theme_key);
-        const char *size_key = "cursor-size";
-        call_read(session_bus, on_gnome_cursor_size_read, GNOME_DESKTOP_NAMESPACE, size_key);
+process_gnome_setting(const char *key, DBusMessageIter *value) {
+    if (strcmp(key, "cursor-size") == 0) {
+        if (dbus_message_iter_get_arg_type(value) == DBUS_TYPE_INT32) {
+            int32_t sz;
+            dbus_message_iter_get_basic(value, &sz);
+            if (sz > 0 && sz != theme_size) {
+                theme_size = sz;
+                cursor_theme_changed = true;
+            }
+        }
+    } else if (strcmp(key, "cursor-theme") == 0) {
+        if (dbus_message_iter_get_arg_type(value) == DBUS_TYPE_STRING) {
+            const char *name;
+            dbus_message_iter_get_basic(value, &name);
+            if (name) {
+                strncpy(theme_name, name, sizeof(theme_name) - 1);
+                cursor_theme_changed = true;
+            }
+        }
     }
 }
 
+static void
+process_settings_dict(DBusMessageIter *array_iter, void(process_setting)(const char *, DBusMessageIter*)) {
+    DBusMessageIter item_iter, value_iter;
+    while (dbus_message_iter_get_arg_type(array_iter) == DBUS_TYPE_DICT_ENTRY) {
+        dbus_message_iter_recurse(array_iter, &item_iter);
+        if (dbus_message_iter_get_arg_type(&item_iter) == DBUS_TYPE_STRING) {
+            const char *key;
+            dbus_message_iter_get_basic(&item_iter, &key);
+            if (dbus_message_iter_next(&item_iter) && dbus_message_iter_get_arg_type(&item_iter) == DBUS_TYPE_VARIANT) {
+                dbus_message_iter_recurse(&item_iter, &value_iter);
+                process_setting(key, &value_iter);
+            }
+        }
+        if (!dbus_message_iter_next(array_iter)) break;
+    }
+}
+
+HANDLER(process_desktop_settings)
+    cursor_theme_changed = false;
+	DBusMessageIter root, array, item, settings;
+	dbus_message_iter_init(msg, &root);
+#define die(...) { _glfwInputError(GLFW_PLATFORM_ERROR, __VA_ARGS__); return; }
+    if (dbus_message_iter_get_arg_type(&root) != DBUS_TYPE_ARRAY) die("Reply to request for desktop settings is not an array");
+    dbus_message_iter_recurse(&root, &array);
+    while (dbus_message_iter_get_arg_type(&array) == DBUS_TYPE_DICT_ENTRY) {
+        dbus_message_iter_recurse(&array, &item);
+        if (dbus_message_iter_get_arg_type(&item) == DBUS_TYPE_STRING) {
+            const char *namespace;
+            dbus_message_iter_get_basic(&item, &namespace);
+            if (dbus_message_iter_next(&item) && dbus_message_iter_get_arg_type(&item) == DBUS_TYPE_ARRAY) {
+                dbus_message_iter_recurse(&item, &settings);
+                if (strcmp(namespace, FDO_DESKTOP_NAMESPACE) == 0) {
+                    process_settings_dict(&settings, process_fdo_setting);
+                } else if (is_gnome && strcmp(namespace, GNOME_DESKTOP_NAMESPACE) == 0) {
+                    process_settings_dict(&settings, process_gnome_setting);
+                }
+            }
+        }
+        if (!dbus_message_iter_next(&array)) break;
+    }
+#undef die
+    if (cursor_theme_changed) _glfwPlatformChangeCursorTheme();
+}
+
+#undef HANDLER
+
+static bool
+read_desktop_settings(DBusConnection *session_bus) {
+    DBusMessage *msg = dbus_message_new_method_call(DESKTOP_SERVICE, DESKTOP_PATH, DESKTOP_INTERFACE, "ReadAll");
+    if (!msg) return false;
+    DBusMessageIter iter, array_iter;
+    dbus_message_iter_init_append(msg, &iter);
+    if (!dbus_message_iter_open_container(&iter, DBUS_TYPE_ARRAY, "s", &array_iter)) { dbus_message_unref(msg); return false; }
+    if (!dbus_message_iter_close_container(&iter, &array_iter)) { dbus_message_unref(msg); return false; }
+    bool ok = call_method_with_msg(session_bus, msg, DBUS_TIMEOUT_USE_DEFAULT, process_desktop_settings, NULL);
+    dbus_message_unref(msg);
+    return ok;
+}
 
 void
 glfw_current_cursor_theme(const char **theme, int *size) {
@@ -100,10 +141,55 @@ get_cursor_theme_from_env(void) {
     }
 }
 
+static void
+on_color_scheme_change(DBusMessage *message) {
+    DBusMessageIter iter[2];
+    dbus_message_iter_init (message, &iter[0]);
+    int current_type;
+    while ((current_type = dbus_message_iter_get_arg_type (&iter[0])) != DBUS_TYPE_INVALID) {
+        if (current_type == DBUS_TYPE_VARIANT) {
+            dbus_message_iter_recurse(&iter[0], &iter[1]);
+            if (dbus_message_iter_get_arg_type(&iter[1]) == DBUS_TYPE_UINT32) {
+                uint32_t val = 0;
+                dbus_message_iter_get_basic(&iter[1], &val);
+                if (val > 2) val = 0;
+                if (val != appearance) {
+                    appearance = val;
+                }
+            }
+            break;
+        }
+        dbus_message_iter_next(&iter[0]);
+    }
+}
+
+static DBusHandlerResult
+setting_changed(DBusConnection *conn UNUSED, DBusMessage *msg, void *user_data UNUSED) {
+    /* printf("session_bus settings_changed invoked interface: %s member: %s\n", dbus_message_get_interface(msg), dbus_message_get_member(msg)); */
+    if (dbus_message_is_signal(msg, DESKTOP_INTERFACE, "SettingChanged")) {
+        const char *namespace = NULL, *key = NULL;
+        if (glfw_dbus_get_args(msg, "Failed to get namespace and key from SettingChanged notification signal", DBUS_TYPE_STRING, &namespace, DBUS_TYPE_STRING, &key, DBUS_TYPE_INVALID)) {
+            if (strcmp(namespace, FDO_DESKTOP_NAMESPACE) == 0) {
+                if (strcmp(key, FDO_APPEARANCE_KEY) == 0) {
+                    on_color_scheme_change(msg);
+                }
+            }
+        }
+
+    }
+    return DBUS_HANDLER_RESULT_NOT_YET_HANDLED;
+}
+
+
 void
 glfw_initialize_desktop_settings(void) {
     get_cursor_theme_from_env();
     const char *desktop = getenv("XDG_CURRENT_DESKTOP");
-    bool is_gnome = desktop && strncasecmp(desktop, "GNOME", sizeof("GNOME") - 1) == 0;
-    if (is_gnome) get_from_gnome();
+    is_gnome = desktop && strstr(desktop, "GNOME");
+    DBusConnection *session_bus = glfw_dbus_session_bus();
+    if (session_bus) {
+        if (!read_desktop_settings(session_bus)) _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to read desktop settings, make sure you have the desktop portal running.");
+        dbus_bus_add_match(session_bus, "type='signal',interface='" DESKTOP_INTERFACE "',member='SettingChanged'", NULL);
+        dbus_connection_add_filter(session_bus, setting_changed, NULL, NULL);
+    }
 }

glfw/wl_client_side_decorations.c

@@ -353,6 +353,7 @@ ensure_csd_resources(_GLFWwindow *window) {
         !decs.mapping.data
     );
     const bool needs_update = focus_changed || size_changed || !decs.left.surface;
+    debug("CSD: old.size: %dx%d new.size: %dx%d needs_update: %d size_changed: %d\n", decs.for_window_state.width, decs.for_window_state.height, window->wl.width, window->wl.height, needs_update, size_changed);
     if (!needs_update) return false;
     if (size_changed) {
         free_csd_buffers(window);
@@ -409,7 +410,7 @@ change_csd_title(_GLFWwindow *window) {
 
 void
 set_csd_window_geometry(_GLFWwindow *window, int32_t *width, int32_t *height) {
-    bool has_csd = window->decorated && !window->wl.decorations.serverSide && window->wl.decorations.left.surface && !(window->wl.toplevel_states & TOPLEVEL_STATE_FULLSCREEN);
+    bool has_csd = window->decorated && !window->wl.decorations.serverSide && window->wl.decorations.left.surface && !(window->wl.current.toplevel_states & TOPLEVEL_STATE_FULLSCREEN);
     bool size_specified_by_compositor = *width > 0 && *height > 0;
     if (!size_specified_by_compositor) {
         *width = window->wl.user_requested_content_size.width;

glfw/wl_init.c

@@ -54,6 +54,8 @@
 #endif
 
 
+#define debug(...) if (_glfw.hints.init.debugRendering) fprintf(stderr, __VA_ARGS__);
+
 static int min(int n1, int n2)
 {
     return n1 < n2 ? n1 : n2;
@@ -96,7 +98,6 @@ static void pointerHandleEnter(void* data UNUSED,
         if (!window)
             return;
     }
-
     window->wl.decorations.focus = focus;
     _glfw.wl.serial = serial;
     _glfw.wl.pointerFocus = window;
@@ -243,7 +244,7 @@ static void pointerHandleButton(void* data UNUSED,
                     window->wl.decorations.last_click_on_top_decoration_at = monotonic();
                     if (window->wl.decorations.last_click_on_top_decoration_at - last_click_at <= _glfwPlatformGetDoubleClickInterval(window)) {
                         window->wl.decorations.last_click_on_top_decoration_at = 0;
-                        if (window->wl.toplevel_states & TOPLEVEL_STATE_MAXIMIZED)
+                        if (window->wl.current.toplevel_states & TOPLEVEL_STATE_MAXIMIZED)
                             xdg_toplevel_unset_maximized(window->wl.xdg.toplevel);
                         else
                             xdg_toplevel_set_maximized(window->wl.xdg.toplevel);
@@ -331,20 +332,83 @@ static void pointerHandleAxis(void* data UNUSED,
     assert(axis == WL_POINTER_AXIS_HORIZONTAL_SCROLL ||
            axis == WL_POINTER_AXIS_VERTICAL_SCROLL);
 
-    if (axis == WL_POINTER_AXIS_HORIZONTAL_SCROLL)
-        x = -wl_fixed_to_double(value);
-    else if (axis == WL_POINTER_AXIS_VERTICAL_SCROLL)
-        y = -wl_fixed_to_double(value);
+    if (axis == WL_POINTER_AXIS_HORIZONTAL_SCROLL) {
+        if (window->wl.axis_discrete_count.x) {
+            window->wl.axis_discrete_count.x--;
+            return;
+        }
+        x = -wl_fixed_to_double(value) * (window->wl.scale);
+    }
+    else if (axis == WL_POINTER_AXIS_VERTICAL_SCROLL) {
+        if (window->wl.axis_discrete_count.y) {
+            window->wl.axis_discrete_count.y--;
+            return;
+        }
+        y = -wl_fixed_to_double(value) * (window->wl.scale);
+    }
 
     _glfwInputScroll(window, x, y, 1, _glfw.wl.xkb.states.modifiers);
 }
 
+static void pointerHandleFrame(void* data UNUSED,
+                               struct wl_pointer* pointer UNUSED)
+{
+    _GLFWwindow* window = _glfw.wl.pointerFocus;
+    if (window) {
+        window->wl.axis_discrete_count.x = 0;
+        window->wl.axis_discrete_count.y = 0;
+    }
+}
+
+static void pointerHandleAxisSource(void* data UNUSED,
+                                    struct wl_pointer* pointer UNUSED,
+                                    uint32_t source UNUSED)
+{
+}
+
+static void pointerHandleAxisStop(void *data UNUSED,
+                  struct wl_pointer *wl_pointer UNUSED,
+                  uint32_t time UNUSED,
+                  uint32_t axis UNUSED)
+{
+}
+
+
+static void pointerHandleAxisDiscrete(void *data UNUSED,
+                  struct wl_pointer *wl_pointer UNUSED,
+                  uint32_t axis,
+                  int32_t discrete)
+{
+    _GLFWwindow* window = _glfw.wl.pointerFocus;
+    double x = 0.0, y = 0.0;
+    if (!window)
+        return;
+
+    assert(axis == WL_POINTER_AXIS_HORIZONTAL_SCROLL ||
+           axis == WL_POINTER_AXIS_VERTICAL_SCROLL);
+
+    if (axis == WL_POINTER_AXIS_HORIZONTAL_SCROLL) {
+        x = -discrete;
+        window->wl.axis_discrete_count.x++;
+    }
+    else if (axis == WL_POINTER_AXIS_VERTICAL_SCROLL) {
+        y = -discrete;
+        window->wl.axis_discrete_count.y++;
+    }
+
+    _glfwInputScroll(window, x, y, 0, _glfw.wl.xkb.states.modifiers);
+}
+
 static const struct wl_pointer_listener pointerListener = {
     pointerHandleEnter,
     pointerHandleLeave,
     pointerHandleMotion,
     pointerHandleButton,
     pointerHandleAxis,
+    pointerHandleFrame,
+    pointerHandleAxisSource,
+    pointerHandleAxisStop,
+    pointerHandleAxisDiscrete,
 };
 
 static void keyboardHandleKeymap(void* data UNUSED,
@@ -571,7 +635,7 @@ static void registryHandleGlobal(void* data UNUSED,
     {
         if (!_glfw.wl.seat)
         {
-            _glfw.wl.seatVersion = min(4, version);
+            _glfw.wl.seatVersion = min(5, version);
             _glfw.wl.seat =
                 wl_registry_bind(registry, name, &wl_seat_interface,
                                  _glfw.wl.seatVersion);

glfw/wl_platform.h

@@ -127,6 +127,10 @@ typedef enum WaylandWindowState {
 
 static const WaylandWindowState TOPLEVEL_STATE_DOCKED = TOPLEVEL_STATE_MAXIMIZED | TOPLEVEL_STATE_FULLSCREEN | TOPLEVEL_STATE_TILED_TOP | TOPLEVEL_STATE_TILED_LEFT | TOPLEVEL_STATE_TILED_RIGHT | TOPLEVEL_STATE_TILED_BOTTOM;
 
+enum WaylandWindowPendingState {
+    PENDING_STATE_TOPLEVEL = 1,
+    PENDING_STATE_DECORATION = 2
+};
 
 // Wayland-specific per-window data
 //
@@ -210,9 +214,20 @@ typedef struct _GLFWwindowWayland
         int32_t width, height;
     } user_requested_content_size;
 
-    uint32_t toplevel_states;
     bool maximize_on_first_show;
+    // counters for ignoring axis events following axis_discrete events in the
+    // same frame along the same axis
+    struct {
+        unsigned int x, y;
+    } axis_discrete_count;
+    bool surface_configured_once;
 
+    uint32_t pending_state;
+    struct {
+        int width, height;
+        uint32_t toplevel_states;
+        uint32_t decoration_mode;
+    } current, pending;
 } _GLFWwindowWayland;
 
 typedef enum _GLFWWaylandOfferType

glfw/wl_text_input.c

@@ -47,7 +47,7 @@ static void
 send_text(const char *text, GLFWIMEState ime_state) {
     _GLFWwindow *w = _glfwFocusedWindow();
     if (w && w->callbacks.keyboard) {
-        GLFWkeyevent fake_ev = {.action = GLFW_PRESS};
+        GLFWkeyevent fake_ev = {.action = text ? GLFW_PRESS : GLFW_RELEASE};
         fake_ev.text = text;
         fake_ev.ime_state = ime_state;
         w->callbacks.keyboard((GLFWwindow*) w, &fake_ev);
@@ -93,6 +93,9 @@ text_input_done(void *data UNUSED, struct zwp_text_input_v3 *txt_input UNUSED, u
     if (pending_pre_edit) {
         send_text(pending_pre_edit, GLFW_IME_PREEDIT_CHANGED);
         free(pending_pre_edit); pending_pre_edit = NULL;
+    } else {
+        // Clear pre-edit text
+        send_text(NULL, GLFW_IME_PREEDIT_CHANGED);
     }
     if (pending_commit) {
         send_text(pending_commit, GLFW_IME_COMMIT_TEXT);
@@ -139,7 +142,20 @@ _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
     switch(ev->type) {
         case GLFW_IME_UPDATE_FOCUS:
             debug("\ntext-input: updating IME focus state, focused: %d\n", ev->focused);
-            if (ev->focused) zwp_text_input_v3_enable(text_input); else zwp_text_input_v3_disable(text_input);
+            if (ev->focused) {
+                zwp_text_input_v3_enable(text_input);
+                zwp_text_input_v3_set_content_type(text_input, ZWP_TEXT_INPUT_V3_CONTENT_HINT_NONE, ZWP_TEXT_INPUT_V3_CONTENT_PURPOSE_TERMINAL);
+            } else {
+                if (pending_pre_edit) {
+                    // Clear pre-edit text
+                    send_text(NULL, GLFW_IME_PREEDIT_CHANGED);
+                    free(pending_pre_edit); pending_pre_edit = NULL;
+                }
+                if (pending_commit) {
+                    free(pending_commit); pending_commit = NULL;
+                }
+                zwp_text_input_v3_disable(text_input);
+            }
             commit();
             break;
         case GLFW_IME_UPDATE_CURSOR_POSITION: {

glfw/wl_window.c

@@ -234,7 +234,7 @@ clipboard_mime(void) {
     return buf;
 }
 
-static void
+static bool
 dispatchChangesAfterConfigure(_GLFWwindow *window, int32_t width, int32_t height) {
     bool size_changed = width != window->wl.width || height != window->wl.height;
     bool scale_changed = checkScaleChange(window);
@@ -246,12 +246,14 @@ dispatchChangesAfterConfigure(_GLFWwindow *window, int32_t width, int32_t height
     }
 
     if (scale_changed) {
-        if (!size_changed)
-            resizeFramebuffer(window);
+        debug("Scale changed to %d in dispatchChangesAfterConfigure\n", window->wl.scale);
+        if (!size_changed) resizeFramebuffer(window);
         _glfwInputWindowContentScale(window, window->wl.scale, window->wl.scale);
     }
 
     _glfwInputWindowDamage(window);
+
+    return size_changed || scale_changed;
 }
 
 static void
@@ -269,24 +271,9 @@ xdgDecorationHandleConfigure(void* data,
                                          uint32_t mode)
 {
     _GLFWwindow* window = data;
-
-    bool has_server_side_decorations = (mode == ZXDG_TOPLEVEL_DECORATION_V1_MODE_SERVER_SIDE);
-    debug("XDG decoration configure event received: has_server_side_decorations: %d\n", has_server_side_decorations);
-    if (has_server_side_decorations == window->wl.decorations.serverSide) return;
-    window->wl.decorations.serverSide = has_server_side_decorations;
-    int width = window->wl.width, height = window->wl.height;
-    if (window->wl.decorations.serverSide) {
-        free_csd_surfaces(window);
-        height += window->wl.decorations.metrics.visible_titlebar_height;
-    } else {
-        ensure_csd_resources(window);
-    }
-    set_csd_window_geometry(window, &width, &height);
-    dispatchChangesAfterConfigure(window, width, height);
-    ensure_csd_resources(window);
-    wl_surface_commit(window->wl.surface);
-    debug("final window content size: %dx%d\n", window->wl.width, window->wl.height);
-    inform_compositor_of_window_geometry(window, "configure-decorations");
+    window->wl.pending.decoration_mode = mode;
+    window->wl.pending_state |= PENDING_STATE_DECORATION;
+    debug("XDG decoration configure event received: has_server_side_decorations: %d\n", (mode == ZXDG_TOPLEVEL_DECORATION_V1_MODE_SERVER_SIDE));
 }
 
 static const struct zxdg_toplevel_decoration_v1_listener xdgDecorationListener = {
@@ -311,6 +298,7 @@ static void surfaceHandleEnter(void *data,
     window->wl.monitors[window->wl.monitorsCount++] = monitor;
 
     if (checkScaleChange(window)) {
+        debug("Scale changed to %d in surface enter event\n", window->wl.scale);
         resizeFramebuffer(window);
         _glfwInputWindowContentScale(window, window->wl.scale, window->wl.scale);
         ensure_csd_resources(window);
@@ -336,6 +324,7 @@ static void surfaceHandleLeave(void *data,
     window->wl.monitors[--window->wl.monitorsCount] = NULL;
 
     if (checkScaleChange(window)) {
+        debug("Scale changed to %d in surface leave event\n", window->wl.scale);
         resizeFramebuffer(window);
         _glfwInputWindowContentScale(window, window->wl.scale, window->wl.scale);
         ensure_csd_resources(window);
@@ -416,7 +405,7 @@ static void setFullscreen(_GLFWwindow* window, _GLFWmonitor* monitor, bool on)
 
 bool
 _glfwPlatformIsFullscreen(_GLFWwindow *window, unsigned int flags UNUSED) {
-    return window->wl.toplevel_states & TOPLEVEL_STATE_FULLSCREEN;
+    return window->wl.current.toplevel_states & TOPLEVEL_STATE_FULLSCREEN;
 }
 
 bool
@@ -458,7 +447,7 @@ xdgToplevelHandleConfigure(void* data,
     if (new_states & TOPLEVEL_STATE_RESIZING) {
         if (width) window->wl.user_requested_content_size.width = width;
         if (height) window->wl.user_requested_content_size.height = height;
-        if (!(window->wl.toplevel_states & TOPLEVEL_STATE_RESIZING)) _glfwInputLiveResize(window, true);
+        if (!(window->wl.current.toplevel_states & TOPLEVEL_STATE_RESIZING)) _glfwInputLiveResize(window, true);
     }
     if (width != 0 && height != 0)
     {
@@ -475,16 +464,11 @@ xdgToplevelHandleConfigure(void* data,
             }
         }
     }
-    bool live_resize_done = !(new_states & TOPLEVEL_STATE_RESIZING) && (window->wl.toplevel_states & TOPLEVEL_STATE_RESIZING);
-    window->wl.toplevel_states = new_states;
-    set_csd_window_geometry(window, &width, &height);
-    dispatchChangesAfterConfigure(window, width, height);
-    debug("final window content size: %dx%d\n", window->wl.width, window->wl.height);
-    _glfwInputWindowFocus(window, window->wl.toplevel_states & TOPLEVEL_STATE_ACTIVATED);
-    ensure_csd_resources(window);
-    wl_surface_commit(window->wl.surface);
-    inform_compositor_of_window_geometry(window, "configure");
-    if (live_resize_done) _glfwInputLiveResize(window, false);
+
+    window->wl.pending.toplevel_states = new_states;
+    window->wl.pending.width = width;
+    window->wl.pending.height = height;
+    window->wl.pending_state |= PENDING_STATE_TOPLEVEL;
 }
 
 static void xdgToplevelHandleClose(void* data,
@@ -499,11 +483,64 @@ static const struct xdg_toplevel_listener xdgToplevelListener = {
     xdgToplevelHandleClose
 };
 
-static void xdgSurfaceHandleConfigure(void* data UNUSED,
+static void xdgSurfaceHandleConfigure(void* data,
                                       struct xdg_surface* surface,
                                       uint32_t serial)
 {
+    _GLFWwindow* window = data;
     xdg_surface_ack_configure(surface, serial);
+    if (window->wl.pending_state & PENDING_STATE_TOPLEVEL) {
+        uint32_t new_states = window->wl.pending.toplevel_states;
+        int width = window->wl.pending.width;
+        int height = window->wl.pending.height;
+        if (!window->wl.surface_configured_once) {
+            window->wl.surface_configured_once = true;
+            if (!width && !height && !new_states && !window->wl.decorations.serverSide && getenv("XAUTHORITY") && strstr(getenv("XAUTHORITY"), "mutter")) {
+                // https://github.com/kovidgoyal/kitty/issues/4802
+                debug("Ignoring first empty surface configure event on mutter.\n");
+                return;
+            }
+        }
+
+        if (new_states != window->wl.current.toplevel_states ||
+                width != window->wl.current.width ||
+                height != window->wl.current.height) {
+
+            bool live_resize_done = !(new_states & TOPLEVEL_STATE_RESIZING) && (window->wl.current.toplevel_states & TOPLEVEL_STATE_RESIZING);
+            window->wl.current.toplevel_states = new_states;
+            window->wl.current.width = width;
+            window->wl.current.height = height;
+            _glfwInputWindowFocus(window, window->wl.current.toplevel_states & TOPLEVEL_STATE_ACTIVATED);
+            if (live_resize_done) _glfwInputLiveResize(window, false);
+        }
+    }
+
+    if (window->wl.pending_state & PENDING_STATE_DECORATION) {
+        uint32_t mode = window->wl.pending.decoration_mode;
+        bool has_server_side_decorations = (mode == ZXDG_TOPLEVEL_DECORATION_V1_MODE_SERVER_SIDE);
+        window->wl.decorations.serverSide = has_server_side_decorations;
+        window->wl.current.decoration_mode = mode;
+    }
+
+    bool resized = false;
+    if (window->wl.pending_state) {
+        int width = window->wl.pending.width, height = window->wl.pending.height;
+        set_csd_window_geometry(window, &width, &height);
+        resized = dispatchChangesAfterConfigure(window, width, height);
+        if (window->wl.decorations.serverSide) {
+            free_csd_surfaces(window);
+        } else {
+            ensure_csd_resources(window);
+        }
+        debug("final window content size: %dx%d resized: %d\n", width, height, resized);
+    }
+
+    inform_compositor_of_window_geometry(window, "configure");
+
+    // if a resize happened there will be a commit at the next render frame so
+    // dont commit here, GNOME doesnt like it and its not really needed anyway
+    if (!resized) wl_surface_commit(window->wl.surface);
+    window->wl.pending_state = 0;
 }
 
 static const struct xdg_surface_listener xdgSurfaceListener = {
@@ -991,7 +1028,7 @@ void _glfwPlatformRestoreWindow(_GLFWwindow* window)
     {
         if (window->monitor)
             xdg_toplevel_unset_fullscreen(window->wl.xdg.toplevel);
-        if (window->wl.toplevel_states & TOPLEVEL_STATE_MAXIMIZED)
+        if (window->wl.current.toplevel_states & TOPLEVEL_STATE_MAXIMIZED)
             xdg_toplevel_unset_maximized(window->wl.xdg.toplevel);
         // There is no way to unset minimized, or even to know if we are
         // minimized, so there is nothing to do in this case.
@@ -1092,7 +1129,7 @@ int _glfwPlatformWindowVisible(_GLFWwindow* window)
 
 int _glfwPlatformWindowMaximized(_GLFWwindow* window)
 {
-    return window->wl.toplevel_states & TOPLEVEL_STATE_MAXIMIZED;
+    return window->wl.current.toplevel_states & TOPLEVEL_STATE_MAXIMIZED;
 }
 
 int _glfwPlatformWindowHovered(_GLFWwindow* window)

glfw/xkb_glfw.c

@@ -33,7 +33,7 @@
 #include <X11/XKBlib.h>
 #endif
 
-#define debug(...) if (_glfw.hints.init.debugKeyboard) printf(__VA_ARGS__);
+#define debug(...) if (_glfw.hints.init.debugKeyboard) fprintf(stderr, __VA_ARGS__);
 
 #ifdef XKB_HAS_NO_UTF32
 #include "xkb-compat-shim.h"
@@ -827,7 +827,7 @@ glfw_xkb_key_from_ime(_GLFWIBUSKeyEvent *ev, bool handled_by_ime, bool failed) {
     xkb_keycode_t prev_handled_press = last_handled_press_keycode;
     last_handled_press_keycode = 0;
     bool is_release = ev->glfw_ev.action == GLFW_RELEASE;
-    debug("From IBUS: native_key: 0x%x name: %s is_release: %d\n", ev->glfw_ev.native_key, glfw_xkb_keysym_name(ev->glfw_ev.key), is_release);
+    debug("From IBUS: native_key: 0x%x name: %s is_release: %d handled_by_ime: %d\n", ev->glfw_ev.native_key, glfw_xkb_keysym_name(ev->glfw_ev.key), is_release, handled_by_ime);
     if (window && !handled_by_ime && !(is_release && ev->glfw_ev.native_key == (int) prev_handled_press)) {
         debug("↳ to application: glfw_keycode: 0x%x (%s) keysym: 0x%x (%s) action: %s %s text: %s\n",
             ev->glfw_ev.native_key, _glfwGetKeyName(ev->glfw_ev.native_key), ev->glfw_ev.key, glfw_xkb_keysym_name(ev->glfw_ev.key),

kittens/ask/main.py

@@ -4,13 +4,15 @@
 import os
 import sys
 from contextlib import suppress
-from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple
+from typing import (
+    TYPE_CHECKING, Callable, Dict, List, NamedTuple, Optional, Tuple
+)
 
 from kitty.cli import parse_args
 from kitty.cli_stub import AskCLIOptions
 from kitty.constants import cache_dir
-from kitty.fast_data_types import wcswidth
-from kitty.typing import BossType, TypedDict
+from kitty.fast_data_types import truncate_point_for_length, wcswidth
+from kitty.typing import BossType, KeyEventType, TypedDict
 from kitty.utils import ScreenSize
 
 from ..tui.handler import Handler, result_handler
@@ -72,7 +74,7 @@ class HistoryCompleter:
 def option_text() -> str:
     return '''\
 --type -t
-choices=line,yesno,choices
+choices=line,yesno,choices,password
 default=line
 Type of input. Defaults to asking for a line of text.
 
@@ -95,6 +97,17 @@ letter is the accelerator key and text is the corresponding text.  There can be
 an optional color specification after the letter to indicate what color it should
 be.
 For example: y:Yes and n;red:No
+
+
+--default -d
+A default choice or text. If unspecified, it is "y" for :code:`yesno`, the first choice
+for :code:`choices` and empty for others. The default choice is selected when the user
+presses the Enter key.
+
+
+--prompt -p
+default="> "
+The prompt to use when inputting a line of text or a password.
 '''
 
 
@@ -107,6 +120,7 @@ class Choice(NamedTuple):
     text: str
     idx: int
     color: str
+    letter: str
 
 
 class Range(NamedTuple):
@@ -118,27 +132,93 @@ class Range(NamedTuple):
         return y == self.y and self.start <= x <= self.end
 
 
+def truncate_at_space(text: str, width: int) -> Tuple[str, str]:
+    p = truncate_point_for_length(text, width)
+    if p < len(text):
+        i = text.rfind(' ', 0, p + 1)
+        if i > 0 and p - i < 12:
+            p = i + 1
+    return text[:p], text[p:]
+
+
+def extra_for(width: int, screen_width: int) -> int:
+    return max(0, screen_width - width) // 2 + 1
+
+
+class Password(Handler):
+
+    def __init__(self, cli_opts: AskCLIOptions, prompt: str) -> None:
+        self.cli_opts = cli_opts
+        self.prompt = prompt
+        from kittens.tui.line_edit import LineEdit
+        self.line_edit = LineEdit(is_password=True)
+
+    def initialize(self) -> None:
+        self.cmd.set_cursor_shape('beam')
+        self.draw_screen()
+
+    @Handler.atomic_update
+    def draw_screen(self) -> None:
+        self.cmd.clear_screen()
+        if self.cli_opts.message:
+            for line in self.cli_opts.message.splitlines():
+                self.print(line)
+            self.print()
+        self.line_edit.write(self.write, self.prompt)
+
+    def on_text(self, text: str, in_bracketed_paste: bool = False) -> None:
+        self.line_edit.on_text(text, in_bracketed_paste)
+        self.draw_screen()
+
+    def on_key(self, key_event: KeyEventType) -> None:
+        if self.line_edit.on_key(key_event):
+            self.draw_screen()
+            return
+        if key_event.matches('enter'):
+            self.quit_loop(0)
+        if key_event.matches('esc'):
+            self.quit_loop(1)
+
+    def on_resize(self, screen_size: ScreenSize) -> None:
+        self.screen_size = screen_size
+        self.draw_screen()
+
+    def on_interrupt(self) -> None:
+        self.quit_loop(1)
+    on_eot = on_interrupt
+
+    @property
+    def response(self) -> str:
+        if self._tui_loop.return_code == 0:
+            return self.line_edit.current_input
+        return ''
+
+
 class Choose(Handler):
     mouse_tracking = MouseTracking.buttons_only
 
     def __init__(self, cli_opts: AskCLIOptions) -> None:
         self.cli_opts = cli_opts
-        self.response = 'n' if cli_opts.type == 'yesno' else ''
-        self.allowed = frozenset('yn')
         self.choices: Dict[str, Choice] = {}
-        self.clickable_ranges: Dict[str, Range] = {}
-        if cli_opts.type != 'yesno':
+        self.clickable_ranges: Dict[str, List[Range]] = {}
+        if cli_opts.type == 'yesno':
+            self.allowed = frozenset('yn')
+        else:
             allowed = []
             for choice in cli_opts.choices:
-                color = 'green'
                 letter, text = choice.split(':', maxsplit=1)
+                color = ''
                 if ';' in letter:
                     letter, color = letter.split(';', maxsplit=1)
                 letter = letter.lower()
                 idx = text.lower().index(letter)
                 allowed.append(letter)
-                self.choices[letter] = Choice(text, idx, color)
+                self.choices[letter] = Choice(text, idx, color, letter)
             self.allowed = frozenset(allowed)
+        self.response = ''
+        self.response_on_accept = cli_opts.default or ''
+        if cli_opts.type in ('yesno', 'choices') and self.response_on_accept not in self.allowed:
+            self.response_on_accept = 'y' if cli_opts.type == 'yesno' else tuple(self.choices.keys())[0]
 
     def initialize(self) -> None:
         self.cmd.set_cursor_visible(False)
@@ -147,52 +227,164 @@ class Choose(Handler):
     def finalize(self) -> None:
         self.cmd.set_cursor_visible(True)
 
+    def draw_long_text(self, text: str) -> int:
+        y = 0
+        width = self.screen_size.cols - 2
+        while text:
+            t, text = truncate_at_space(text, width)
+            t = t.strip()
+            self.print(' ' * extra_for(wcswidth(t), width), styled(t, bold=True), sep='')
+            y += 1
+        return y
+
     @Handler.atomic_update
     def draw_screen(self) -> None:
         self.cmd.clear_screen()
-        y = 1
+        y = max(0, self.screen_size.rows // 2 - 2)
+        self.print(end='\r\n'*y)
         if self.cli_opts.message:
-            self.cmd.styled(self.cli_opts.message, bold=True)
-            y += wcswidth(self.cli_opts.message) // self.screen_size.cols
-        self.print()
+            for line in self.cli_opts.message.splitlines():
+                y += self.draw_long_text(line)
+        if self.screen_size.rows > 2:
+            self.print()
+            y += 1
         if self.cli_opts.type == 'yesno':
             self.draw_yesno(y)
         else:
             self.draw_choice(y)
 
-    def draw_choice(self, y: int) -> None:
-        x = 0
+    def draw_choice_boxes(self, y: int, *choices: Choice) -> None:
         self.clickable_ranges.clear()
+        width = self.screen_size.cols - 2
+        current_line_length = 0
+        current_line: List[Tuple[str, str]] = []
+        lines: List[List[Tuple[str, str]]] = []
+        sep, sep_sz = '  ', 2
+
+        for choice in choices:
+            self.clickable_ranges[choice.letter] = []
+            text = ' ' + choice.text[:choice.idx]
+            text += styled(choice.text[choice.idx], fg=choice.color or 'green')
+            text += choice.text[choice.idx + 1:] + ' '
+            sz = wcswidth(text)
+            if sz + sep_sz + current_line_length > width:
+                lines.append(current_line)
+                current_line = []
+                current_line_length = 0
+            current_line.append((choice.letter, text))
+            current_line_length += sz
+        if current_line:
+            lines.append(current_line)
+
+        def top(text: str) -> str:
+            return '╭' + '─' * wcswidth(text) + '╮'
+
+        def middle(text: str) -> str:
+            return f'│{text}│'
+
+        def bottom(text: str) -> str:
+            return '╰' + '─' * wcswidth(text) + '╯'
+
+        def highlight(text: str, only_edges: bool = False) -> str:
+            if only_edges:
+                return styled(text[0], fg='yellow') + text[1:-1] + styled(text[-1], fg='yellow')
+            return styled(text, fg='yellow')
+
+        def print_line(add_borders: Callable[[str], str], *items: Tuple[str, str], is_last: bool = False) -> None:
+            nonlocal y
+            texts = []
+            positions = []
+            x = 0
+            for (letter, text) in items:
+                positions.append((letter, x, wcswidth(text) + 2))
+                text = add_borders(text)
+                if letter == self.response_on_accept:
+                    text = highlight(text, only_edges=add_borders is middle)
+                text += sep
+                x += wcswidth(text)
+                texts.append(text)
+            line = ''.join(texts).rstrip()
+            offset = extra_for(wcswidth(line), width)
+            for (letter, x, sz) in positions:
+                x += offset
+                self.clickable_ranges[letter].append(Range(x, x + sz - 1, y))
+            self.print(' ' * offset, line, sep='', end='' if is_last else '\r\n')
+            y += 1
+
+        for boxed_line in lines:
+            print_line(top, *boxed_line)
+            print_line(middle, *boxed_line)
+            print_line(bottom, *boxed_line, is_last=boxed_line is lines[-1])
+
+    def draw_choice(self, y: int) -> None:
+        if y + 3 <= self.screen_size.rows:
+            self.draw_choice_boxes(y, *self.choices.values())
+            return
+        self.clickable_ranges.clear()
+        current_line = ''
+        current_ranges: Dict[str, int] = {}
+        width = self.screen_size.cols - 2
+
+        def commit_line(end: str = '\r\n') -> None:
+            nonlocal current_line, y
+            x = extra_for(wcswidth(current_line), width)
+            self.print(' ' * x + current_line, end=end)
+            for letter, sz in current_ranges.items():
+                self.clickable_ranges[letter] = [Range(x, x + sz - 3, y)]
+                x += sz
+            current_ranges.clear()
+            y += 1
+            current_line = ''
+
         for letter, choice in self.choices.items():
             text = choice.text[:choice.idx]
-            text += styled(choice.text[choice.idx], fg=choice.color)
+            text += styled(choice.text[choice.idx], fg=choice.color or 'green', underline='straight' if letter == self.response_on_accept else None)
             text += choice.text[choice.idx + 1:]
             text += '  '
             sz = wcswidth(text)
-            if sz + x >= self.screen_size.cols:
-                y += 1
-                x = 0
-                self.print()
-            self.clickable_ranges[letter] = Range(x, x + sz - 1, y)
-            x += sz
-            self.print(text, end='')
+            if sz + wcswidth(current_line) >= width:
+                commit_line()
+            current_line += text
+            current_ranges[letter] = sz
+        if current_line:
+            commit_line(end='')
 
     def draw_yesno(self, y: int) -> None:
-        self.clickable_ranges = {'y': Range(2, 4, y), 'n': Range(8, 9, y)}
-        self.print(' ', styled('Y', fg='green') + 'es', ' ', styled('N', fg='red') + 'o')
+        yes = styled('Y', fg='green') + 'es'
+        no = styled('N', fg='red') + 'o'
+        if y + 3 <= self.screen_size.rows:
+            self.draw_choice_boxes(y, Choice('Yes', 0, 'green', 'y'), Choice('No', 0, 'red', 'n'))
+            return
+        sep = ' ' * 3
+        text = yes + sep + no
+        w = wcswidth(text)
+        x = extra_for(w, self.screen_size.cols - 2)
+        nx = x + wcswidth(yes) + len(sep)
+        self.clickable_ranges = {'y': [Range(x, x + wcswidth(yes) - 1, y)], 'n': [Range(nx, nx + wcswidth(no) - 1, y)]}
+        self.print(' ' * x + text, end='')
 
     def on_text(self, text: str, in_bracketed_paste: bool = False) -> None:
         text = text.lower()
         if text in self.allowed:
             self.response = text
             self.quit_loop(0)
+        elif self.cli_opts.type == 'yesno':
+            self.on_interrupt()
+
+    def on_key(self, key_event: KeyEventType) -> None:
+        if key_event.matches('esc'):
+            self.on_interrupt()
+        elif key_event.matches('enter'):
+            self.response = self.response_on_accept
+            self.quit_loop(0)
 
     def on_click(self, ev: MouseEvent) -> None:
-        for letter, r in self.clickable_ranges.items():
-            if r.has_point(ev.cell_x, ev.cell_y):
-                self.response = letter
-                self.quit_loop(0)
-                break
+        for letter, ranges in self.clickable_ranges.items():
+            for r in ranges:
+                if r.has_point(ev.cell_x, ev.cell_y):
+                    self.response = letter
+                    self.quit_loop(0)
+                    return
 
     def on_resize(self, screen_size: ScreenSize) -> None:
         self.screen_size = screen_size
@@ -223,6 +415,15 @@ def main(args: List[str]) -> Response:
         loop.loop(handler)
         return {'items': items, 'response': handler.response}
 
+    prompt = cli_opts.prompt
+    if prompt[0] == prompt[-1] and prompt[0] in '\'"':
+        prompt = prompt[1:-1]
+    if cli_opts.type == 'password':
+        loop = Loop()
+        phandler = Password(cli_opts, prompt)
+        loop.loop(phandler)
+        return {'items': items, 'response': phandler.response}
+
     import readline as rl
     readline = rl
     from kitty.shell import init_readline
@@ -233,9 +434,16 @@ def main(args: List[str]) -> Response:
         if cli_opts.message:
             print(styled(cli_opts.message, bold=True))
 
-        prompt = '> '
         with suppress(KeyboardInterrupt, EOFError):
-            response = input(prompt)
+            if cli_opts.default:
+                def prefill_text() -> None:
+                    readline.insert_text(cli_opts.default or '')
+                    readline.redisplay()
+                readline.set_pre_input_hook(prefill_text)
+                response = input(prompt)
+                readline.set_pre_input_hook()
+            else:
+                response = input(prompt)
     return {'items': items, 'response': response}
 
 

kittens/diff/collect.py

@@ -32,8 +32,7 @@ class Segment:
         self.end_code: Optional[str] = None
 
     def __repr__(self) -> str:
-        return 'Segment(start={!r}, start_code={!r}, end={!r}, end_code={!r})'.format(
-            self.start, self.start_code, self.end, self.end_code)
+        return f'Segment(start={self.start!r}, start_code={self.start_code!r}, end={self.end!r}, end_code={self.end_code!r})'
 
 
 class Collection:
@@ -102,7 +101,7 @@ def remote_hostname(path: str) -> Tuple[Optional[str], Optional[str]]:
 def resolve_remote_name(path: str, default: str) -> str:
     remote_dir, rh = remote_hostname(path)
     if remote_dir and rh:
-        return rh + ':' + os.path.relpath(path, remote_dir)
+        return f'{rh}:{os.path.relpath(path, remote_dir)}'
     return default
 
 

kittens/diff/highlight.py

@@ -4,7 +4,10 @@
 import concurrent
 import os
 import re
-from typing import IO, Dict, Iterable, List, Optional, Tuple, Union, cast
+from concurrent.futures import ProcessPoolExecutor
+from typing import (
+    IO, Dict, Iterable, Iterator, List, Optional, Tuple, Union, cast
+)
 
 from pygments import highlight  # type: ignore
 from pygments.formatter import Formatter  # type: ignore
@@ -94,7 +97,7 @@ def highlight_data(code: str, filename: str, aliases: Optional[Dict[str, str]] =
         base, ext = os.path.splitext(filename)
         alias = aliases.get(ext[1:])
         if alias is not None:
-            filename = base + '.' + alias
+            filename = f'{base}.{alias}'
     try:
         lexer = get_lexer_for_filename(filename, stripnl=False)
     except ClassNotFound:
@@ -136,10 +139,22 @@ def highlight_for_diff(path: str, aliases: Dict[str, str]) -> DiffHighlight:
     return ans
 
 
+process_pool_executor: Optional[ProcessPoolExecutor] = None
+
+
+def get_highlight_processes() -> Iterator[int]:
+    if process_pool_executor is None:
+        return
+    for pid in process_pool_executor._processes:
+        yield pid
+
+
 def highlight_collection(collection: Collection, aliases: Optional[Dict[str, str]] = None) -> Union[str, Dict[str, DiffHighlight]]:
+    global process_pool_executor
     jobs = {}
     ans: Dict[str, DiffHighlight] = {}
     with get_process_pool_executor(prefer_fork=True) as executor:
+        process_pool_executor = executor
         for path, item_type, other_path in collection:
             if item_type != 'rename':
                 for p in (path, other_path):
@@ -152,15 +167,18 @@ def highlight_collection(collection: Collection, aliases: Optional[Dict[str, str
             try:
                 highlights = future.result()
             except Exception as e:
-                return f'Running syntax highlighting for {path} generated an exception: {e}'
+                import traceback
+                tb = traceback.format_exc()
+                return f'Running syntax highlighting for {path} generated an exception: {e} with traceback:\n{tb}'
             ans[path] = highlights
     return ans
 
 
 def main() -> None:
     # kitty +runpy "from kittens.diff.highlight import main; main()" file
-    from .options.types import defaults
     import sys
+
+    from .options.types import defaults
     initialize_highlighter()
     with open(sys.argv[-1]) as f:
         highlighted = highlight_data(f.read(), f.name, defaults.syntax_aliases)

kittens/diff/main.py

@@ -10,10 +10,11 @@ import tempfile
 import warnings
 from collections import defaultdict
 from contextlib import suppress
+from enum import Enum, auto
 from functools import partial
 from gettext import gettext as _
 from typing import (
-    Any, DefaultDict, Dict, Iterable, List, Optional, Tuple, Union
+    Any, DefaultDict, Dict, Iterable, Iterator, List, Optional, Tuple, Union
 )
 
 from kitty.cli import CONFIG_HELP, parse_args
@@ -44,7 +45,8 @@ from .search import BadRegex, Search
 
 try:
     from .highlight import (
-        DiffHighlight, highlight_collection, initialize_highlighter
+        DiffHighlight, get_highlight_processes, highlight_collection,
+        initialize_highlighter
     )
     has_highlighter = True
     DiffHighlight
@@ -54,8 +56,24 @@ except ImportError:
     def highlight_collection(collection: 'Collection', aliases: Optional[Dict[str, str]] = None) -> Union[str, Dict[str, 'DiffHighlight']]:
         return ''
 
+    def get_highlight_processes() -> Iterator[int]:
+        if has_highlighter:
+            yield -1
 
-INITIALIZING, COLLECTED, DIFFED, COMMAND, MESSAGE = range(5)
+
+class State(Enum):
+    initializing = auto()
+    collected = auto()
+    diffed = auto()
+    command = auto()
+    message = auto()
+
+
+class BackgroundWork(Enum):
+    none = auto()
+    collecting = auto()
+    diffing = auto()
+    highlighting = auto()
 
 
 def generate_diff(collection: Collection, context: int) -> Union[str, Dict[str, Patch]]:
@@ -76,7 +94,7 @@ class DiffHandler(Handler):
     image_manager_class = ImageManager
 
     def __init__(self, args: DiffCLIOptions, opts: DiffOptions, left: str, right: str) -> None:
-        self.state = INITIALIZING
+        self.state = State.initializing
         self.message = ''
         self.current_search_is_regex = True
         self.current_search: Optional[Search] = None
@@ -90,16 +108,20 @@ class DiffHandler(Handler):
         if self.current_context_count < 0:
             self.current_context_count = self.original_context_count = self.opts.num_context_lines
         self.highlighting_done = False
+        self.doing_background_work = BackgroundWork.none
         self.restore_position: Optional[Reference] = None
         for key_def, action in self.opts.key_definitions.items():
             self.add_shortcut(action, key_def)
 
+    def terminate(self, return_code: int = 0) -> None:
+        self.quit_loop(return_code)
+
     def perform_action(self, action: KeyAction) -> None:
         func, args = action
         if func == 'quit':
-            self.quit_loop(0)
+            self.terminate()
             return
-        if self.state <= DIFFED:
+        if self.state.value <= State.diffed.value:
             if func == 'scroll_by':
                 return self.scroll_lines(int(args[0] or 0))
             if func == 'scroll_to':
@@ -130,8 +152,9 @@ class DiffHandler(Handler):
     def create_collection(self) -> None:
 
         def collect_done(collection: Collection) -> None:
+            self.doing_background_work = BackgroundWork.none
             self.collection = collection
-            self.state = COLLECTED
+            self.state = State.collected
             self.generate_diff()
 
         def collect(left: str, right: str) -> None:
@@ -139,15 +162,17 @@ class DiffHandler(Handler):
             self.asyncio_loop.call_soon_threadsafe(collect_done, collection)
 
         self.asyncio_loop.run_in_executor(None, collect, self.left, self.right)
+        self.doing_background_work = BackgroundWork.collecting
 
     def generate_diff(self) -> None:
 
         def diff_done(diff_map: Union[str, Dict[str, Patch]]) -> None:
+            self.doing_background_work = BackgroundWork.none
             if isinstance(diff_map, str):
                 self.report_traceback_on_exit = diff_map
-                self.quit_loop(1)
+                self.terminate(1)
                 return
-            self.state = DIFFED
+            self.state = State.diffed
             self.diff_map = diff_map
             self.calculate_statistics()
             self.render_diff()
@@ -163,7 +188,7 @@ class DiffHandler(Handler):
                     initialize_highlighter(self.opts.pygments_style)
                 except StyleNotFound as e:
                     self.report_traceback_on_exit = str(e)
-                    self.quit_loop(1)
+                    self.terminate(1)
                     return
                 self.syntax_highlight()
 
@@ -172,13 +197,15 @@ class DiffHandler(Handler):
             self.asyncio_loop.call_soon_threadsafe(diff_done, diff_map)
 
         self.asyncio_loop.run_in_executor(None, diff, self.collection, self.current_context_count)
+        self.doing_background_work = BackgroundWork.diffing
 
     def syntax_highlight(self) -> None:
 
         def highlighting_done(hdata: Union[str, Dict[str, 'DiffHighlight']]) -> None:
+            self.doing_background_work = BackgroundWork.none
             if isinstance(hdata, str):
                 self.report_traceback_on_exit = hdata
-                self.quit_loop(1)
+                self.terminate(1)
                 return
             set_highlight_data(hdata)
             self.render_diff()
@@ -189,6 +216,7 @@ class DiffHandler(Handler):
             self.asyncio_loop.call_soon_threadsafe(highlighting_done, result)
 
         self.asyncio_loop.run_in_executor(None, highlight, self.collection, self.opts.syntax_aliases)
+        self.doing_background_work = BackgroundWork.highlighting
 
     def calculate_statistics(self) -> None:
         self.added_count = self.collection.added_count
@@ -265,10 +293,11 @@ class DiffHandler(Handler):
 
     def scroll_lines(self, amt: int = 1) -> None:
         new_pos = max(0, min(self.scroll_pos + amt, self.max_scroll_pos))
+        amt = new_pos - self.scroll_pos
         if new_pos == self.scroll_pos:
             self.cmd.bell()
             return
-        if abs(new_pos - self.scroll_pos) >= self.num_lines - 1:
+        if abs(amt) >= self.num_lines - 1:
             self.scroll_pos = new_pos
             self.draw_screen()
             return
@@ -304,7 +333,7 @@ class DiffHandler(Handler):
         self.create_collection()
 
     def enforce_cursor_state(self) -> None:
-        self.cmd.set_cursor_visible(self.state == COMMAND)
+        self.cmd.set_cursor_visible(self.state is State.command)
 
     def draw_lines(self, num: int, offset: int = 0) -> None:
         offset += self.scroll_pos
@@ -319,7 +348,7 @@ class DiffHandler(Handler):
                 text = line.text
                 if line.image_data is not None:
                     image_involved = True
-            self.write('\r\x1b[K' + text + '\x1b[0m')
+            self.write(f'\r\x1b[K{text}\x1b[0m')
             if self.current_search is not None:
                 self.current_search.highlight_line(self.write, lpos)
             if i < num - 1:
@@ -407,7 +436,7 @@ class DiffHandler(Handler):
 
     def draw_screen(self) -> None:
         self.enforce_cursor_state()
-        if self.state < DIFFED:
+        if self.state.value < State.diffed.value:
             self.cmd.clear_screen()
             self.write(_('Calculating diff, please wait...'))
             return
@@ -417,14 +446,14 @@ class DiffHandler(Handler):
         self.draw_status_line()
 
     def draw_status_line(self) -> None:
-        if self.state < DIFFED:
+        if self.state.value < State.diffed.value:
             return
         self.enforce_cursor_state()
         self.cmd.set_cursor_position(0, self.num_lines)
         self.cmd.clear_to_eol()
-        if self.state is COMMAND:
+        if self.state is State.command:
             self.line_edit.write(self.write)
-        elif self.state is MESSAGE:
+        elif self.state is State.message:
             self.cmd.styled(self.message, reverse=True)
         else:
             sp = f'{self.scroll_pos/self.max_scroll_pos:.0%}' if self.scroll_pos and self.max_scroll_pos else '0%'
@@ -437,7 +466,7 @@ class DiffHandler(Handler):
                 )
             else:
                 counts = styled(f'{len(self.current_search)} matches', fg=self.opts.margin_fg)
-            suffix = counts + '  ' + scroll_frac
+            suffix = f'{counts}  {scroll_frac}'
             prefix = styled(':', fg=self.opts.margin_fg)
             filler = self.screen_size.cols - wcswidth(prefix) - wcswidth(suffix)
             text = '{}{}{}'.format(prefix, ' ' * filler, suffix)
@@ -447,16 +476,16 @@ class DiffHandler(Handler):
         new_ctx = max(0, new_ctx)
         if new_ctx != self.current_context_count:
             self.current_context_count = new_ctx
-            self.state = COLLECTED
+            self.state = State.collected
             self.generate_diff()
             self.restore_position = self.current_position
             self.draw_screen()
 
     def start_search(self, is_regex: bool, is_backward: bool) -> None:
-        if self.state != DIFFED:
+        if self.state is not State.diffed:
             self.cmd.bell()
             return
-        self.state = COMMAND
+        self.state = State.command
         self.line_edit.clear()
         self.line_edit.add_text('?' if is_backward else '/')
         self.current_search_is_regex = is_regex
@@ -470,50 +499,50 @@ class DiffHandler(Handler):
         try:
             self.current_search = Search(self.opts, query[1:], self.current_search_is_regex, query[0] == '?')
         except BadRegex:
-            self.state = MESSAGE
+            self.state = State.message
             self.message = sanitize(_('Bad regex: {}').format(query[1:]))
             self.cmd.bell()
         else:
             if self.current_search(self.diff_lines, self.margin_size, self.screen_size.cols):
                 self.scroll_to_next_match(include_current=True)
             else:
-                self.state = MESSAGE
+                self.state = State.message
                 self.message = sanitize(_('No matches found'))
                 self.cmd.bell()
 
     def on_key_event(self, key_event: KeyEvent, in_bracketed_paste: bool = False) -> None:
         if key_event.text:
-            if self.state is COMMAND:
+            if self.state is State.command:
                 self.line_edit.on_text(key_event.text, in_bracketed_paste)
                 self.draw_status_line()
                 return
-            if self.state is MESSAGE:
-                self.state = DIFFED
+            if self.state is State.message:
+                self.state = State.diffed
                 self.draw_status_line()
                 return
         else:
-            if self.state is MESSAGE:
+            if self.state is State.message:
                 if key_event.type is not EventType.RELEASE:
-                    self.state = DIFFED
+                    self.state = State.diffed
                     self.draw_status_line()
                 return
-            if self.state is COMMAND:
+            if self.state is State.command:
                 if self.line_edit.on_key(key_event):
                     if not self.line_edit.current_input:
-                        self.state = DIFFED
+                        self.state = State.diffed
                     self.draw_status_line()
                     return
                 if key_event.matches('enter'):
-                    self.state = DIFFED
+                    self.state = State.diffed
                     self.do_search()
                     self.line_edit.clear()
                     self.draw_screen()
                     return
                 if key_event.matches('esc'):
-                    self.state = DIFFED
+                    self.state = State.diffed
                     self.draw_status_line()
                     return
-            if self.state >= DIFFED and self.current_search is not None and key_event.matches('esc'):
+            if self.state.value >= State.diffed.value and self.current_search is not None and key_event.matches('esc'):
                 self.current_search = None
                 self.draw_screen()
                 return
@@ -526,16 +555,16 @@ class DiffHandler(Handler):
     def on_resize(self, screen_size: ScreenSize) -> None:
         self.screen_size = screen_size
         self.set_scrolling_region()
-        if self.state > COLLECTED:
+        if self.state.value > State.collected.value:
             self.image_manager.delete_all_sent_images()
             self.render_diff()
         self.draw_screen()
 
     def on_interrupt(self) -> None:
-        self.quit_loop(1)
+        self.terminate(1)
 
     def on_eot(self) -> None:
-        self.quit_loop(1)
+        self.terminate(1)
 
 
 OPTIONS = partial('''\
@@ -639,9 +668,10 @@ def main(args: List[str]) -> None:
     for message in showwarning.warnings:
         from kitty.utils import safe_print
         safe_print(message, file=sys.stderr)
-    highlight_processes = getattr(highlight_collection, 'processes', ())
-    terminate_processes(tuple(highlight_processes))
-    terminate_processes(tuple(worker_processes))
+    if handler.doing_background_work is BackgroundWork.highlighting:
+        terminate_processes(tuple(get_highlight_processes()))
+    elif handler.doing_background_work == BackgroundWork.diffing:
+        terminate_processes(tuple(worker_processes))
     if loop.return_code != 0:
         if handler.report_traceback_on_exit:
             print(handler.report_traceback_on_exit, file=sys.stderr)

kittens/diff/options/parse.py

@@ -93,7 +93,7 @@ def create_result_dict() -> typing.Dict[str, typing.Any]:
     }
 
 
-actions = frozenset(('map',))
+actions: typing.FrozenSet[str] = frozenset(('map',))
 
 
 def merge_result_dicts(defaults: typing.Dict[str, typing.Any], vals: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:

kittens/diff/patch.py

@@ -244,14 +244,14 @@ class Differ:
             except Exception as e:
                 return f'Running git diff for {left_path} vs. {right_path} generated an exception: {e}'
             if not ok:
-                return output + f'\nRunning git diff for {left_path} vs. {right_path} failed'
+                return f'{output}\nRunning git diff for {left_path} vs. {right_path} failed'
             left_lines = lines_for_path(left_path)
             right_lines = lines_for_path(right_path)
             try:
                 patch = parse_patch(output)
             except Exception:
                 import traceback
-                return traceback.format_exc() + f'\nParsing diff for {left_path} vs. {right_path} failed'
+                return f'{traceback.format_exc()}\nParsing diff for {left_path} vs. {right_path} failed'
             else:
                 ans[key] = patch
         return ans

kittens/diff/render.py

@@ -101,7 +101,7 @@ def human_readable(size: int, sep: str = ' ') -> str:
         s = s[:s.find(".")+2]
     if s.endswith('.0'):
         s = s[:-2]
-    return s + sep + suffix
+    return f'{s}{sep}{suffix}'
 
 
 def fit_in(text: str, count: int) -> str:
@@ -110,7 +110,7 @@ def fit_in(text: str, count: int) -> str:
         return text
     if count > 1:
         p = truncate_point_for_length(text, count - 1)
-    return text[:p] + '…'
+    return f'{text[:p]}…'
 
 
 def fill_in(text: str, sz: int) -> str:
@@ -127,8 +127,8 @@ def place_in(text: str, sz: int) -> str:
 def format_func(which: str) -> Callable[[str], str]:
     def formatted(text: str) -> str:
         fmt = formats[which]
-        return '\x1b[' + fmt + 'm' + text + '\x1b[0m'
-    formatted.__name__ = which + '_format'
+        return f'\x1b[{fmt}m{text}\x1b[0m'
+    formatted.__name__ = f'{which}_format'
     return formatted
 
 
@@ -148,8 +148,8 @@ highlight_map = {'remove': ('removed_highlight', 'removed'), 'add': ('added_high
 
 def highlight_boundaries(ltype: str) -> Tuple[str, str]:
     s, e = highlight_map[ltype]
-    start = '\x1b[' + formats[s] + 'm'
-    stop = '\x1b[' + formats[e] + 'm'
+    start = f'\x1b[{formats[s]}m'
+    stop = f'\x1b[{formats[e]}m'
     return start, stop
 
 

kittens/hints/main.py

@@ -20,12 +20,14 @@ from kitty.fast_data_types import get_options, set_clipboard_string
 from kitty.key_encoding import KeyEvent
 from kitty.typing import BossType, KittyCommonOpts
 from kitty.utils import (
-    ScreenSize, resolve_custom_file, screen_size_function, set_primary_selection
+    ScreenSize, kitty_ansi_sanitizer_pat, resolve_custom_file,
+    screen_size_function, set_primary_selection
 )
 
 from ..tui.handler import Handler, result_handler
 from ..tui.loop import Loop
 from ..tui.operations import faint, styled
+from ..tui.utils import report_error, report_unhandled_error
 
 
 @lru_cache()
@@ -40,6 +42,8 @@ def kitty_common_opts() -> KittyCommonOpts:
 
 DEFAULT_HINT_ALPHABET = string.digits + string.ascii_lowercase
 DEFAULT_REGEX = r'(?m)^\s*(.+)\s*$'
+FILE_EXTENSION = r'\.(?:[a-zA-Z0-9]{2,7}|[ahcmo])(?!\.)'
+PATH_REGEX = fr'(?:\S*?/[\r\S]+)|(?:\S[\r\S]*{FILE_EXTENSION})\b'
 
 
 class Mark:
@@ -114,6 +118,9 @@ def render(text: str, current_input: str, all_marks: Sequence[Mark], ignore_mark
 
 class Hints(Handler):
 
+    use_alternate_screen = False  # disabled to avoid screen being blanked at exit causing flicker
+    overlay_ready_report_needed = True
+
     def __init__(self, text: str, all_marks: Sequence[Mark], index_map: Dict[int, Mark], args: HintsCLIOptions):
         self.text, self.index_map = text, index_map
         self.alphabet = args.alphabet or DEFAULT_HINT_ALPHABET
@@ -184,7 +191,7 @@ class Hints(Handler):
             self.current_input = self.current_input[:-1]
             self.current_text = None
             self.draw_screen()
-        elif key_event.matches('enter') and self.current_input:
+        elif (key_event.matches('enter') or key_event.matches('space')) and self.current_input:
             try:
                 idx = decode_hint(self.current_input, self.alphabet)
                 self.chosen.append(self.index_map[idx])
@@ -228,7 +235,7 @@ def regex_finditer(pat: 'Pattern[str]', minimum_match_length: int, text: str) ->
             yield s, e, m.groupdict()
 
 
-closing_bracket_map = {'(': ')', '[': ']', '{': '}', '<': '>', '*': '*', '"': '"', "'": "'"}
+closing_bracket_map = {'(': ')', '[': ']', '{': '}', '<': '>', '*': '*', '"': '"', "'": "'", "“": "”", "‘": "’"}
 opening_brackets = ''.join(closing_bracket_map)
 PostprocessorFunc = Callable[[str, int, int], Tuple[int, int]]
 postprocessor_map: Dict[str, PostprocessorFunc] = {}
@@ -271,9 +278,13 @@ def brackets(text: str, s: int, e: int) -> Tuple[int, int]:
     # Remove matching brackets
     if s < e <= len(text):
         before = text[s]
-        if before in '({[<' and text[e-1] == closing_bracket_map[before]:
-            s += 1
-            e -= 1
+        if before in '({[<':
+            q = closing_bracket_map[before]
+            if text[e-1] == q:
+                s += 1
+                e -= 1
+            elif text[e:e+1] == q:
+                s += 1
     return s, e
 
 
@@ -282,9 +293,13 @@ def quotes(text: str, s: int, e: int) -> Tuple[int, int]:
     # Remove matching quotes
     if s < e <= len(text):
         before = text[s]
-        if before in '\'"' and text[e-1] == before:
-            s += 1
-            e -= 1
+        if before in '\'"“‘':
+            q = closing_bracket_map[before]
+            if text[e-1] == q:
+                s += 1
+                e -= 1
+            elif text[e:e+1] == q:
+                s += 1
     return s, e
 
 
@@ -348,7 +363,7 @@ def functions_for(args: HintsCLIOptions) -> Tuple[str, List[PostprocessorFunc]]:
         )
         post_processors.append(url)
     elif args.type == 'path':
-        pattern = r'(?:\S*?/[\r\S]+)|(?:\S[\r\S]*\.[a-zA-Z0-9\r]{2,7})'
+        pattern = PATH_REGEX
         post_processors.extend((brackets, quotes))
     elif args.type == 'line':
         pattern = '(?m)^\\s*(.+)[\\s\0]*$'
@@ -405,7 +420,7 @@ def parse_input(text: str) -> str:
 def linenum_marks(text: str, args: HintsCLIOptions, Mark: Type[Mark], extra_cli_args: Sequence[str], *a: Any) -> Generator[Mark, None, None]:
     regex = args.regex
     if regex == DEFAULT_REGEX:
-        regex = r'(?P<path>(?:\S*/\S+?)|(?:\S+[.][a-zA-Z0-9]{2,7})):(?P<line>\d+)'
+        regex = fr'(?P<path>{PATH_REGEX}):(?P<line>\d+)'
     yield from mark(regex, [brackets, quotes], text, args)
 
 
@@ -421,11 +436,7 @@ def load_custom_processor(customize_processing: str) -> Any:
     return runpy.run_path(custom_path, run_name='__main__')
 
 
-def remove_sgr(text: str) -> str:
-    return re.sub(r'\x1b\[.*?m', '', text)
-
-
-def process_hyperlinks(text: str) -> Tuple[str, Tuple[Mark, ...]]:
+def process_escape_codes(text: str) -> Tuple[str, Tuple[Mark, ...]]:
     hyperlinks: List[Mark] = []
     removed_size = idx = 0
     active_hyperlink_url: Optional[str] = None
@@ -448,6 +459,9 @@ def process_hyperlinks(text: str) -> Tuple[str, Tuple[Mark, ...]]:
     def process_hyperlink(m: 're.Match[str]') -> str:
         nonlocal removed_size, active_hyperlink_url, active_hyperlink_id, active_hyperlink_start_offset
         raw = m.group()
+        if not raw.startswith('\x1b]8'):
+            removed_size += len(raw)
+            return ''
         start = m.start() - removed_size
         removed_size += len(raw)
         if active_hyperlink_url is not None:
@@ -465,7 +479,7 @@ def process_hyperlinks(text: str) -> Tuple[str, Tuple[Mark, ...]]:
 
         return ''
 
-    text = re.sub(r'\x1b\]8.+?\x1b\\', process_hyperlink, text)
+    text = kitty_ansi_sanitizer_pat().sub(process_hyperlink, text)
     if active_hyperlink_url is not None:
         add_hyperlink(len(text))
     return text, tuple(hyperlinks)
@@ -473,8 +487,8 @@ def process_hyperlinks(text: str) -> Tuple[str, Tuple[Mark, ...]]:
 
 def run(args: HintsCLIOptions, text: str, extra_cli_args: Sequence[str] = ()) -> Optional[Dict[str, Any]]:
     try:
-        text = parse_input(remove_sgr(text))
-        text, hyperlinks = process_hyperlinks(text)
+        text = parse_input(text)
+        text, hyperlinks = process_escape_codes(text)
         pattern, post_processors = functions_for(args)
         if args.type == 'linenum':
             args.customize_processing = '::linenum::'
@@ -490,7 +504,7 @@ def run(args: HintsCLIOptions, text: str, extra_cli_args: Sequence[str] = ()) ->
             all_marks = tuple(mark(pattern, post_processors, text, args))
         if not all_marks:
             none_of = {'url': 'URLs', 'hyperlink': 'hyperlinks'}.get(args.type, 'matches')
-            input(_('No {} found, press Enter to quit.').format(none_of))
+            report_error(_('No {} found.').format(none_of))
             return None
 
         largest_index = all_marks[-1].index
@@ -502,11 +516,7 @@ def run(args: HintsCLIOptions, text: str, extra_cli_args: Sequence[str] = ()) ->
                 m.index = largest_index - m.index + offset
         index_map = {m.index: m for m in all_marks}
     except Exception:
-        import traceback
-        traceback.print_exc()
-        input('Press Enter to quit.')
-        raise SystemExit(1)
-
+        report_unhandled_error()
     return run_loop(args, text, all_marks, index_map, extra_cli_args)
 
 
@@ -667,9 +677,7 @@ def main(args: List[str]) -> Optional[Dict[str, Any]]:
     text = ''
     if sys.stdin.isatty():
         if '--help' not in args and '-h' not in args:
-            print('You must pass the text to be hinted on STDIN', file=sys.stderr)
-            input(_('Press Enter to quit'))
-            return None
+            report_unhandled_error('You must pass the text to be hinted on STDIN')
     else:
         text = sys.stdin.buffer.read().decode('utf-8')
         sys.stdin = open(os.ctermid())
@@ -677,29 +685,37 @@ def main(args: List[str]) -> Optional[Dict[str, Any]]:
         opts, items = parse_hints_args(args[1:])
     except SystemExit as e:
         if e.code != 0:
-            print(e.args[0], file=sys.stderr)
-            input(_('Press Enter to quit'))
+            report_unhandled_error(e.args[0])
         return None
     if items and not (opts.customize_processing or opts.type == 'linenum'):
-        print('Extra command line arguments present: {}'.format(' '.join(items)), file=sys.stderr)
-        input(_('Press Enter to quit'))
+        report_unhandled_error('Extra command line arguments present: {}'.format(' '.join(items)))
     try:
         return run(opts, text, items)
     except Exception:
-        import traceback
-        traceback.print_exc()
-        input(_('Press Enter to quit'))
+        report_unhandled_error()
     return None
 
 
-def linenum_handle_result(args: List[str], data: Dict[str, Any], target_window_id: int, boss: BossType, extra_cli_args: Sequence[str], *a: Any) -> None:
+def linenum_process_result(data: Dict[str, Any]) -> Tuple[str, int]:
+    pat = re.compile(r':(\d+)$')
     for m, g in zip(data['match'], data['groupdicts']):
         if m:
             path, line = g['path'], g['line']
-            path = os.path.expanduser(path.split(':')[-1])
-            line = int(line)
-            break
-    else:
+            # look for trailers on path of the for :number
+            while True:
+                m = pat.search(path)
+                if m is None:
+                    break
+                line = m.group(1)
+                path = path[:-len(m.group())]
+
+            return os.path.expanduser(path), int(line)
+    return '', -1
+
+
+def linenum_handle_result(args: List[str], data: Dict[str, Any], target_window_id: int, boss: BossType, extra_cli_args: Sequence[str], *a: Any) -> None:
+    path, line = linenum_process_result(data)
+    if not path:
         return
 
     cmd = [x.format(path=path, line=line) for x in extra_cli_args or ('vim', '+{line}', '{path}')]
@@ -720,7 +736,7 @@ def linenum_handle_result(args: List[str], data: Dict[str, Any], target_window_i
             else:
                 import shlex
                 text = ' '.join(shlex.quote(arg) for arg in cmd)
-                w.paste_bytes(text + '\r')
+                w.paste_bytes(f'{text}\r')
     elif action == 'background':
         import subprocess
         subprocess.Popen(cmd, cwd=data['cwd'])
@@ -730,7 +746,7 @@ def linenum_handle_result(args: List[str], data: Dict[str, Any], target_window_i
             }[action])(*cmd)
 
 
-@result_handler(type_of_input='screen-ansi')
+@result_handler(type_of_input='screen-ansi', has_ready_notification=Hints.overlay_ready_report_needed)
 def handle_result(args: List[str], data: Dict[str, Any], target_window_id: int, boss: BossType) -> None:
     if data['customize_processing']:
         m = load_custom_processor(data['customize_processing'])
@@ -772,7 +788,7 @@ def handle_result(args: List[str], data: Dict[str, Any], target_window_id: int,
         if program == '-':
             w = boss.window_id_map.get(target_window_id)
             if w is not None:
-                w.paste(joined_text())
+                w.paste_text(joined_text())
         elif program == '@':
             set_clipboard_string(joined_text())
         elif program == '*':

kittens/hyperlinked_grep/main.py

@@ -23,7 +23,10 @@ def main() -> None:
     if not sys.stdout.isatty() and '--pretty' not in sys.argv:
         os.execlp('rg', 'rg', *sys.argv[1:])
     cmdline = ['rg', '--pretty', '--with-filename'] + sys.argv[1:]
-    p = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
+    try:
+        p = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
+    except FileNotFoundError:
+        raise SystemExit('Could not find the rg executable in your PATH. Is ripgrep installed?')
     assert p.stdout is not None
     write: Callable[[bytes], None] = cast(Callable[[bytes], None], sys.stdout.buffer.write)
     sgr_pat = re.compile(br'\x1b\[.*?m')

kittens/icat/main.py

@@ -19,6 +19,7 @@ from kitty.cli import parse_args
 from kitty.cli_stub import IcatCLIOptions
 from kitty.constants import appname
 from kitty.guess_mime_type import guess_type
+from kitty.rgb import to_color
 from kitty.types import run_once
 from kitty.typing import GRT_f, GRT_t
 from kitty.utils import (
@@ -55,6 +56,19 @@ are smaller than the specified area to be scaled up to use as much
 of the specified area as possible.
 
 
+--background
+default=none
+Specify a background color, this will cause transparent images to be composited on
+top of the specified color.
+
+
+--mirror
+default=none
+type=choices
+choices=none,horizontal,vertical,both
+Mirror the image about a horizontal or vertical axis or both.
+
+
 --clear
 type=bool-set
 Remove all images currently displayed on the screen.
@@ -88,7 +102,7 @@ detecting image display support.
 --print-window-size
 type=bool-set
 Print out the window size as :italic:`widthxheight` (in pixels) and quit. This is a
-convenience method to query the window size if using kitty icat from a
+convenience method to query the window size if using :code:`kitty +kitten icat` from a
 scripting language that cannot make termios calls.
 
 
@@ -307,6 +321,9 @@ class ParsedOpts:
 
     place: Optional['Place'] = None
     z_index: int = 0
+    remove_alpha: str = ''
+    flip: bool = False
+    flop: bool = False
 
 
 def process(path: str, args: IcatCLIOptions, parsed_opts: ParsedOpts, is_tempfile: bool) -> bool:
@@ -316,9 +333,10 @@ def process(path: str, args: IcatCLIOptions, parsed_opts: ParsedOpts, is_tempfil
     available_height = parsed_opts.place.height * (ss.height // ss.rows) if parsed_opts.place else 10 * m.height
     needs_scaling = m.width > available_width or m.height > available_height
     needs_scaling = needs_scaling or args.scale_up
+    needs_conversion = needs_scaling or bool(parsed_opts.remove_alpha) or parsed_opts.flip or parsed_opts.flop
     file_removed = False
     use_number = 0
-    if m.fmt == 'png' and not needs_scaling:
+    if m.fmt == 'png' and not needs_conversion:
         outfile = path
         transmit_mode: 'GRT_t' = 't' if is_tempfile else 'f'
         fmt: 'GRT_f' = 100
@@ -328,13 +346,17 @@ def process(path: str, args: IcatCLIOptions, parsed_opts: ParsedOpts, is_tempfil
         fmt = 24 if m.mode == 'rgb' else 32
         transmit_mode = 't'
         if len(m) == 1 or args.loop == 0:
-            outfile, width, height = render_as_single_image(path, m, available_width, available_height, args.scale_up)
+            outfile, width, height = render_as_single_image(
+                path, m, available_width, available_height, args.scale_up,
+                remove_alpha=parsed_opts.remove_alpha, flip=parsed_opts.flip, flop=parsed_opts.flop)
         else:
             import struct
             use_number = max(1, struct.unpack('@I', os.urandom(4))[0])
             with NamedTemporaryFile() as f:
                 prefix = f.name
-            frame_data = render_image(path, prefix, m, available_width, available_height, args.scale_up)
+            frame_data = render_image(
+                path, prefix, m, available_width, available_height, args.scale_up,
+                remove_alpha=parsed_opts.remove_alpha, flip=parsed_opts.flip, flop=parsed_opts.flop)
             outfile, width, height = frame_data.frames[0].path, frame_data.width, frame_data.height
     show(
         outfile, width, height, parsed_opts.z_index, fmt, transmit_mode,
@@ -449,9 +471,8 @@ def process_single_item(
     file_removed = False
     try:
         if isinstance(item, bytes):
-            tf = NamedTemporaryFile(prefix='stdin-image-data-', delete=False)
-            tf.write(item)
-            tf.close()
+            with NamedTemporaryFile(prefix='stdin-image-data-', delete=False) as tf:
+                tf.write(item)
             item = tf.name
             is_tempfile = True
         if url_pat is not None and url_pat.match(item) is not None:
@@ -494,7 +515,12 @@ def main(args: List[str] = sys.argv) -> None:
     if cli_opts.print_window_size:
         screen_size_function.cache_clear()
         with open(os.ctermid()) as tty:
-            ss = screen_size_function(tty)()
+            try:
+                fd = tty.fileno()
+            except AttributeError:
+                # use default value for fd if ctermid is not available
+                fd = None
+            ss = screen_size_function(fd)()
         print(f'{ss.width}x{ss.height}', end='')
         raise SystemExit(0)
 
@@ -527,6 +553,13 @@ def main(args: List[str] = sys.argv) -> None:
         parsed_opts.z_index = parse_z_index(cli_opts.z_index)
     except Exception:
         raise SystemExit(f'Not a valid z-index specification: {cli_opts.z_index}')
+    if cli_opts.background != 'none':
+        ra = to_color(cli_opts.background)
+        if ra is None:
+            raise SystemExit(f'Not a valid color specification: {cli_opts.background}')
+        parsed_opts.remove_alpha = ra.as_sharp
+    parsed_opts.flip = cli_opts.mirror in ('both', 'vertical')
+    parsed_opts.flop = cli_opts.mirror in ('both', 'horizontal')
 
     if cli_opts.detect_support:
         if not detect_support(wait_for=cli_opts.detection_timeout, silent=True):
@@ -550,16 +583,26 @@ def main(args: List[str] = sys.argv) -> None:
             raise SystemExit(f'The --place option can only be used with a single image, not {items}')
         sys.stdout.buffer.write(b'\0337')  # save cursor
     url_pat = re.compile(r'(?:https?|ftp)://', flags=re.I)
+
+    def hold_if_needed(exit_code_or_msg: Union[int, str]) -> None:
+        if cli_opts.hold:
+            if isinstance(exit_code_or_msg, str):
+                print(exit_code_or_msg, file=sys.stderr, flush=True)
+                exit_code_or_msg = 1
+            with open(os.ctermid()) as tty, raw_mode(tty.fileno()):
+                tty.buffer.read(1)
+        raise SystemExit(exit_code_or_msg)
+
     for item in items:
         try:
             process_single_item(item, cli_opts, parsed_opts, url_pat)
         except NoImageMagick as e:
-            raise SystemExit(str(e))
+            hold_if_needed(str(e))
         except OutdatedImageMagick as e:
             print(e.detailed_error, file=sys.stderr)
-            raise SystemExit(str(e))
+            hold_if_needed(str(e))
         except ConvertFailed as e:
-            raise SystemExit(str(e))
+            hold_if_needed(str(e))
         except OpenFailed as e:
             errors.append(e)
     if parsed_opts.place:
@@ -567,11 +610,8 @@ def main(args: List[str] = sys.argv) -> None:
     if errors:
         for err in errors:
             print(err, file=sys.stderr)
-    if cli_opts.hold:
-        with open(os.ctermid()) as tty:
-            with raw_mode(tty.fileno()):
-                tty.buffer.read(1)
-    raise SystemExit(1 if errors else 0)
+    hold_if_needed(1 if errors else 0)
+    raise SystemExit()
 
 
 if __name__ == '__main__':

kittens/panel/main.py

@@ -85,9 +85,9 @@ def create_strut(
             '-id',
             str(int(win_id)), '-format', '_NET_WM_STRUT_PARTIAL', '32cccccccccccc',
             '-set', '_NET_WM_STRUT_PARTIAL',
-            '{left},{right},{top},{bottom},'
-            '{left_start_y},{left_end_y},{right_start_y},{right_end_y},'
-            '{top_start_x},{top_end_x},{bottom_start_x},{bottom_end_x}'.format(**locals())
+            f'{left},{right},{top},{bottom},'
+            f'{left_start_y},{left_end_y},{right_start_y},{right_end_y},'
+            f'{top_start_x},{top_end_x},{bottom_start_x},{bottom_end_x}'
     )
 
 

kittens/query_terminal/main.py

@@ -245,7 +245,7 @@ def main(args: List[str] = sys.argv) -> None:
             raise SystemExit(f'Unknown queries: {", ".join(extra)}')
 
     for key, val in do_queries(queries, cli_opts).items():
-        print(key + ':', val)
+        print(f'{key}:', val)
 
 
 if __name__ == '__main__':

kittens/remote_file/main.py

@@ -122,10 +122,17 @@ class ControlMaster:
             cmd.extend(['-i', conn_data.identity_file])
         self.batch_cmd_prefix = cmd + ['-o', 'BatchMode=yes']
 
+    def check_call(self, cmd: List[str]) -> None:
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL)
+        stdout = p.communicate()[0]
+        if p.wait() != 0:
+            out = stdout.decode('utf-8', 'replace')
+            raise Exception(f'The ssh command: {shlex.join(cmd)} failed with exit code {p.returncode} and output: {out}')
+
     def __enter__(self) -> 'ControlMaster':
-        subprocess.check_call(
+        self.check_call(
             self.cmd_prefix + ['-o', 'ControlMaster=auto', '-fN', self.conn_data.hostname])
-        subprocess.check_call(
+        self.check_call(
             self.batch_cmd_prefix + ['-O', 'check', self.conn_data.hostname])
         if not self.dest:
             self.tdir = tempfile.mkdtemp()

kittens/runner.py

@@ -9,10 +9,10 @@ from contextlib import contextmanager
 from functools import partial
 from typing import TYPE_CHECKING, Any, Dict, FrozenSet, Generator, List, cast
 
+from kitty.constants import list_kitty_resources
 from kitty.types import run_once
 from kitty.utils import resolve_abs_or_config_path
 
-
 aliases = {'url_hints': 'hints'}
 if TYPE_CHECKING:
     from kitty.conf.types import Definition
@@ -69,6 +69,7 @@ def create_kitten_handler(kitten: str, orig_args: List[str]) -> Any:
     ans = partial(m['end'], [kitten] + orig_args)
     setattr(ans, 'type_of_input', getattr(m['end'], 'type_of_input', None))
     setattr(ans, 'no_ui', getattr(m['end'], 'no_ui', False))
+    setattr(ans, 'has_ready_notification', getattr(m['end'], 'has_ready_notification', False))
     return ans
 
 
@@ -85,42 +86,31 @@ def launch(args: List[str]) -> None:
     del args[:2]
     args = [kitten] + args
     os.environ['KITTY_CONFIG_DIRECTORY'] = config_dir
-    from kittens.tui.operations import clear_screen, reset_mode, Mode
     set_debug(kitten)
     m = import_kitten_main_module(config_dir, kitten)
     try:
         result = m['start'](args)
     finally:
         sys.stdin = sys.__stdin__
-    print(reset_mode(Mode.ALTERNATE_SCREEN) + clear_screen(), end='')
     if result is not None:
         import json
-        data = json.dumps(result)
-        print('OK:', len(data), data)
+        import base64
+        data = base64.b85encode(json.dumps(result).encode('utf-8'))
+        sys.stdout.buffer.write(b'\x1bP@kitty-kitten-result|')
+        sys.stdout.buffer.write(data)
+        sys.stdout.buffer.write(b'\x1b\\')
     sys.stderr.flush()
     sys.stdout.flush()
 
 
-def deserialize(output: str) -> Any:
-    import json
-    if output.startswith('OK: '):
-        try:
-            prefix, sz, rest = output.split(' ', 2)
-            return json.loads(rest[:int(sz)])
-        except Exception:
-            raise ValueError(f'Failed to parse kitten output: {output!r}')
-
-
 def run_kitten(kitten: str, run_name: str = '__main__') -> None:
     import runpy
     original_kitten_name = kitten
     kitten = resolved_kitten(kitten)
     set_debug(kitten)
-    try:
+    if kitten in all_kitten_names():
         runpy.run_module(f'kittens.{kitten}.main', run_name=run_name)
         return
-    except ImportError:
-        pass
     # Look for a custom kitten
     if not kitten.endswith('.py'):
         kitten += '.py'
@@ -137,7 +127,6 @@ def run_kitten(kitten: str, run_name: str = '__main__') -> None:
 
 @run_once
 def all_kitten_names() -> FrozenSet[str]:
-    from kitty.constants import list_kitty_resources
     ans = []
     for name in list_kitty_resources('kittens'):
         if '__' not in name and '.' not in name and name != 'tui':

kittens/show_error/main.py

@@ -23,21 +23,22 @@ def real_main(args: List[str]) -> None:
     cli_opts, items = parse_args(args[1:], OPTIONS, '', msg, 'hints', result_class=ErrorCLIOptions)
     error_message = sys.stdin.buffer.read().decode('utf-8')
     sys.stdin = open(os.ctermid())
-    print(styled(cli_opts.title, fg_intense=True, fg='red', bold=True))
-    print()
+    if cli_opts.title:
+        print(styled(cli_opts.title, fg_intense=True, fg='red', bold=True))
+        print()
     print(error_message)
     print()
-    input('Press Enter to close.')
+    input('\x1b[1;32mPress Enter to close\x1b[m')
 
 
 def main(args: List[str]) -> None:
     try:
-        with suppress(KeyboardInterrupt):
+        with suppress(KeyboardInterrupt, EOFError):
             real_main(args)
     except Exception:
         import traceback
         traceback.print_exc()
-        input('Press Enter to close.')
+        input('Press Enter to close')
 
 
 if __name__ == '__main__':

kittens/show_key/kitty_mode.py

@@ -52,7 +52,7 @@ class KeysHandler(Handler):
         self.cmd.colored(etype + ' ', 'yellow')
         self.cmd.styled(key_event.text, italic=True)
         self.print()
-        rep = 'CSI ' + encode_key_event(key_event)[2:]
+        rep = f'CSI {encode_key_event(key_event)[2:]}'
         rep = rep.replace(';', ' ; ').replace(':', ' : ')[:-1] + ' ' + rep[-1]
         self.cmd.styled(rep, fg='magenta')
         if (key_event.shifted_key or key_event.alternate_key):

kittens/show_key/main.py

@@ -17,7 +17,7 @@ def print_key(raw: bytearray) -> None:
     unix = ''
     for ch in raw:
         if ch < len(ctrl_keys):
-            unix += '^' + ctrl_keys[ch]
+            unix += f'^{ctrl_keys[ch]}'
         elif ch == 127:
             unix += '^?'
         else:

kittens/ssh/completion.py

@@ -281,7 +281,7 @@ def complete_choices(ans: Completions, prefix: str, title: str, choices: Iterabl
         if q.startswith(effective_prefix):
             if comma_separated:
                 tq = q
-                q = hidden_prefix + q + ','
+                q = f'{hidden_prefix}{q},'
                 word_transforms[q] = tq
             matches[q] = ''
     ans.add_match_group(title, matches, trailing_space=not comma_separated, word_transforms=word_transforms)

kittens/ssh/config.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# License: GPLv3 Copyright: 2022, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+import fnmatch
+import os
+from typing import Any, Dict, Iterable, Optional
+
+from kitty.conf.utils import (
+    load_config as _load_config, parse_config_base, resolve_config
+)
+from kitty.constants import config_dir
+
+from .options.types import Options as SSHOptions, defaults
+
+SYSTEM_CONF = '/etc/xdg/kitty/ssh.conf'
+defconf = os.path.join(config_dir, 'ssh.conf')
+
+
+def host_matches(mpat: str, hostname: str, username: str) -> bool:
+    for pat in mpat.split():
+        upat = '*'
+        if '@' in pat:
+            upat, pat = pat.split('@', 1)
+        if fnmatch.fnmatchcase(hostname, pat) and fnmatch.fnmatchcase(username, upat):
+            return True
+    return False
+
+
+def load_config(*paths: str, overrides: Optional[Iterable[str]] = None, hostname: str = '!', username: str = '') -> SSHOptions:
+    from .options.parse import (
+        create_result_dict, merge_result_dicts, parse_conf_item
+    )
+    from .options.utils import (
+        first_seen_positions, get_per_hosts_dict, init_results_dict
+    )
+
+    def merge_dicts(base: Dict[str, Any], vals: Dict[str, Any]) -> Dict[str, Any]:
+        base_phd = get_per_hosts_dict(base)
+        vals_phd = get_per_hosts_dict(vals)
+        for hostname in base_phd:
+            vals_phd[hostname] = merge_result_dicts(base_phd[hostname], vals_phd.get(hostname, {}))
+        ans: Dict[str, Any] = vals_phd.pop(vals['hostname'])
+        ans['per_host_dicts'] = vals_phd
+        return ans
+
+    def parse_config(lines: Iterable[str]) -> Dict[str, Any]:
+        ans: Dict[str, Any] = init_results_dict(create_result_dict())
+        parse_config_base(lines, parse_conf_item, ans)
+        return ans
+
+    overrides = tuple(overrides) if overrides is not None else ()
+    first_seen_positions.clear()
+    first_seen_positions['*'] = 0
+    opts_dict, paths = _load_config(
+        defaults, parse_config, merge_dicts, *paths, overrides=overrides, initialize_defaults=init_results_dict)
+    phd = get_per_hosts_dict(opts_dict)
+    final_dict: Dict[str, Any] = {}
+    for hostname_pat in sorted(phd, key=first_seen_positions.__getitem__):
+        if host_matches(hostname_pat, hostname, username):
+            od = phd[hostname_pat]
+            for k, v in od.items():
+                if isinstance(v, dict):
+                    bv = final_dict.setdefault(k, {})
+                    bv.update(v)
+                else:
+                    final_dict[k] = v
+    first_seen_positions.clear()
+    return SSHOptions(final_dict)
+
+
+def init_config(hostname: str, username: str, overrides: Optional[Iterable[str]] = None) -> SSHOptions:
+    config = tuple(resolve_config(SYSTEM_CONF, defconf))
+    return load_config(*config, overrides=overrides, hostname=hostname, username=username)

kittens/ssh/copy.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# License: GPLv3 Copyright: 2022, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+import glob
+import os
+import shlex
+import uuid
+from typing import (
+    Dict, Iterable, Iterator, List, NamedTuple, Optional, Sequence, Tuple
+)
+
+from kitty.cli import parse_args
+from kitty.cli_stub import CopyCLIOptions
+from kitty.types import run_once
+
+from ..transfer.utils import expand_home, home_path
+
+
+@run_once
+def option_text() -> str:
+    return '''
+--glob
+type=bool-set
+Interpret file arguments as glob patterns.
+
+
+--dest
+The destination on the remote host to copy to. Relative paths are resolved
+relative to HOME on the remote host. When this option is not specified, the
+local file path is used as the remote destination (with the HOME directory
+getting automatically replaced by the remote HOME). Note that environment
+variables and ~ are not expanded.
+
+
+--exclude
+type=list
+A glob pattern. Files with names matching this pattern are excluded from being
+transferred. Useful when adding directories. Can
+be specified multiple times, if any of the patterns match the file will be
+excluded.
+'''
+
+
+def parse_copy_args(args: Optional[Sequence[str]] = None) -> Tuple[CopyCLIOptions, List[str]]:
+    args = list(args or ())
+    try:
+        opts, args = parse_args(result_class=CopyCLIOptions, args=args, ospec=option_text)
+    except SystemExit as e:
+        raise CopyCLIError from e
+    return opts, args
+
+
+def resolve_file_spec(spec: str, is_glob: bool) -> Iterator[str]:
+    ans = os.path.expandvars(expand_home(spec))
+    if not os.path.isabs(ans):
+        ans = expand_home(f'~/{ans}')
+    if is_glob:
+        files = glob.glob(ans)
+        if not files:
+            raise CopyCLIError(f'{spec} does not exist')
+    else:
+        if not os.path.exists(ans):
+            raise CopyCLIError(f'{spec} does not exist')
+        files = [ans]
+    for x in files:
+        yield os.path.normpath(x).replace(os.sep, '/')
+
+
+class CopyCLIError(ValueError):
+    pass
+
+
+def get_arcname(loc: str, dest: Optional[str], home: str) -> str:
+    if dest:
+        arcname = dest
+    else:
+        arcname = os.path.normpath(loc)
+        if arcname.startswith(home):
+            arcname = os.path.relpath(arcname, home)
+    arcname = os.path.normpath(arcname).replace(os.sep, '/')
+    prefix = 'root' if arcname.startswith('/') else 'home/'
+    return prefix + arcname
+
+
+class CopyInstruction(NamedTuple):
+    local_path: str
+    arcname: str
+    exclude_patterns: Tuple[str, ...]
+
+
+def parse_copy_instructions(val: str, current_val: Dict[str, str]) -> Iterable[Tuple[str, CopyInstruction]]:
+    opts, args = parse_copy_args(shlex.split(val))
+    locations: List[str] = []
+    for a in args:
+        locations.extend(resolve_file_spec(a, opts.glob))
+    if not locations:
+        raise CopyCLIError('No files to copy specified')
+    if len(locations) > 1 and opts.dest:
+        raise CopyCLIError('Specifying a remote location with more than one file is not supported')
+    home = home_path()
+    for loc in locations:
+        arcname = get_arcname(loc, opts.dest, home)
+        yield str(uuid.uuid4()), CopyInstruction(loc, arcname, tuple(opts.exclude))

kittens/ssh/main.py

@@ -1,137 +1,307 @@
 #!/usr/bin/env python3
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
+import atexit
+import fnmatch
+import glob
+import io
+import json
 import os
+import re
+import secrets
 import shlex
+import shutil
+import stat
 import subprocess
 import sys
-from contextlib import suppress
-from typing import List, NoReturn, Optional, Set, Tuple
-from .completion import ssh_options, complete
+import tarfile
+import tempfile
+import termios
+import time
+import traceback
+from base64 import standard_b64decode, standard_b64encode
+from contextlib import contextmanager, suppress
+from getpass import getuser
+from select import select
+from typing import (
+    Callable, Dict, Iterator, List, NoReturn, Optional, Sequence, Set, Tuple,
+    Union
+)
 
-from kitty.utils import SSHConnectionData
+from kittens.tui.operations import restore_colors, save_colors
+from kitty.constants import (
+    cache_dir, runtime_dir, shell_integration_dir, ssh_control_master_template,
+    str_version, terminfo_dir
+)
+from kitty.options.types import Options
+from kitty.shm import SharedMemory
+from kitty.types import run_once
+from kitty.utils import SSHConnectionData, set_echo as turn_off_echo, expandvars, resolve_abs_or_config_path
 
-SHELL_SCRIPT = '''\
-#!/bin/sh
-# macOS ships with an ancient version of tic that cannot read from stdin, so we
-# create a temp file for it
-tmp=$(mktemp)
-cat >$tmp << 'TERMEOF'
-TERMINFO
-TERMEOF
-
-tic_out=$(tic -x -o $HOME/.terminfo $tmp 2>&1)
-rc=$?
-rm $tmp
-if [ "$rc" != "0" ]; then echo "$tic_out"; exit 1; fi
-if [ -z "$USER" ]; then export USER=$(whoami); fi
-export TERMINFO="$HOME/.terminfo"
-login_shell=""
-python=""
-
-login_shell_is_ok() {
-    if [ -z "$login_shell" ] || [ ! -x "$login_shell" ]; then return 1; fi
-    case "$login_shell" in
-        *sh) return 0;
-    esac
-    return 1;
-}
-
-detect_python() {
-    python=$(command -v python3)
-    if [ -z "$python" ]; then python=$(command -v python2); fi
-    if [ -z "$python" ]; then python=python; fi
-}
-
-using_getent() {
-    cmd=$(command -v getent)
-    if [ -z "$cmd" ]; then return; fi
-    output=$($cmd passwd $USER 2>/dev/null)
-    if [ $? = 0 ]; then login_shell=$(echo $output | cut -d: -f7); fi
-}
-
-using_id() {
-    cmd=$(command -v id)
-    if [ -z "$cmd" ]; then return; fi
-    output=$($cmd -P $USER 2>/dev/null)
-    if [ $? = 0 ]; then login_shell=$(echo $output | cut -d: -f7); fi
-}
-
-using_passwd() {
-    cmd=$(command -v grep)
-    if [ -z "$cmd" ]; then return; fi
-    output=$($cmd "^$USER:" /etc/passwd 2>/dev/null)
-    if [ $? = 0 ]; then login_shell=$(echo $output | cut -d: -f7); fi
-}
-
-using_python() {
-    detect_python
-    if [ ! -x "$python" ]; then return; fi
-    output=$($python -c "import pwd, os; print(pwd.getpwuid(os.geteuid()).pw_shell)")
-    if [ $? = 0 ]; then login_shell=$output; fi
-}
-
-execute_with_python() {
-    detect_python
-    exec $python -c "import os; os.execl('$login_shell', '-' '$shell_name')"
-}
-
-die() { echo "$*" 1>&2 ; exit 1; }
-
-using_getent
-if ! login_shell_is_ok; then using_id; fi
-if ! login_shell_is_ok; then using_python; fi
-if ! login_shell_is_ok; then using_passwd; fi
-if ! login_shell_is_ok; then die "Could not detect login shell"; fi
+from .completion import complete, ssh_options
+from .config import init_config
+from .copy import CopyInstruction
+from .options.types import Options as SSHOptions
+from .options.utils import DELETE_ENV_VAR
 
 
-# If a command was passed to SSH execute it here
-EXEC_CMD
-
-# We need to pass the first argument to the executed program with a leading -
-# to make sure the shell executes as a login shell. Note that not all shells
-# support exec -a so we use the below to try to detect such shells
-shell_name=$(basename $login_shell)
-if [ -z "$PIPESTATUS" ]; then
-    # the dash shell does not support exec -a and also does not define PIPESTATUS
-    execute_with_python
-fi
-exec -a "-$shell_name" $login_shell
-'''
+@run_once
+def ssh_exe() -> str:
+    return shutil.which('ssh') or 'ssh'
 
 
-PYTHON_SCRIPT = '''\
-#!/usr/bin/env python
-from __future__ import print_function
-from tempfile import NamedTemporaryFile
-import subprocess, os, sys, pwd, binascii, json
+def is_kitten_cmdline(q: List[str]) -> bool:
+    if len(q) < 4:
+        return False
+    if os.path.basename(q[0]).lower() != 'kitty':
+        return False
+    return q[1:3] == ['+kitten', 'ssh'] or q[1:4] == ['+', 'kitten', 'ssh']
 
-# macOS ships with an ancient version of tic that cannot read from stdin, so we
-# create a temp file for it
-with NamedTemporaryFile() as tmp:
-    tmp.write(binascii.unhexlify('{terminfo}'))
-    p = subprocess.Popen(['tic', '-x', '-o', os.path.expanduser('~/.terminfo'), tmp.name], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = p.communicate()
-    if p.wait() != 0:
-        getattr(sys.stderr, 'buffer', sys.stderr).write(stdout + stderr)
-        raise SystemExit('Failed to compile terminfo using tic')
-command_to_execute = json.loads(binascii.unhexlify('{command_to_execute}'))
-try:
-    shell_path = pwd.getpwuid(os.geteuid()).pw_shell or '/bin/sh'
-except KeyError:
-    shell_path = '/bin/sh'
-shell_name = '-' + os.path.basename(shell_path)
-if command_to_execute:
-    os.execlp(shell_path, shell_path, '-c', command_to_execute)
-os.execlp(shell_path, shell_name)
-'''
+
+def set_cwd_in_cmdline(cwd: str, argv: List[str]) -> None:
+    for i, arg in enumerate(tuple(argv)):
+        if arg.startswith('--kitten=cwd'):
+            argv[i] = f'--kitten=cwd={cwd}'
+            return
+        elif i > 0 and argv[i-1] == '--kitten' and (arg.startswith('cwd=') or arg.startswith('cwd ')):
+            argv[i] = cwd
+            return
+    idx = argv.index('ssh')
+    argv.insert(idx + 1, f'--kitten=cwd={cwd}')
+
+
+# See https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html
+quote_pat = re.compile('([\\`"\n])')
+
+
+def quote_env_val(x: str) -> str:
+    x = quote_pat.sub(r'\\\1', x)
+    x = x.replace('$(', r'\$(')  # prevent execution with $()
+    return f'"{x}"'
+
+
+def serialize_env(env: Dict[str, str], base_env: Dict[str, str]) -> bytes:
+    lines = []
+
+    def a(k: str, val: str) -> None:
+        lines.append(f'export {shlex.quote(k)}={quote_env_val(val)}')
+
+    for k in sorted(env):
+        v = env[k]
+        if v == DELETE_ENV_VAR:
+            lines.append(f'unset {shlex.quote(k)}')
+        elif v == '_kitty_copy_env_var_':
+            q = base_env.get(k)
+            if q is not None:
+                a(k, q)
+        else:
+            a(k, v)
+    return '\n'.join(lines).encode('utf-8')
+
+
+@run_once
+def kitty_opts() -> Options:
+    from kitty.cli import create_default_opts
+    return create_default_opts()
+
+
+def make_tarfile(ssh_opts: SSHOptions, base_env: Dict[str, str], compression: str = 'gz') -> bytes:
+
+    def normalize_tarinfo(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo:
+        tarinfo.uname = tarinfo.gname = ''
+        tarinfo.uid = tarinfo.gid = 0
+        return tarinfo
+
+    def add_data_as_file(tf: tarfile.TarFile, arcname: str, data: Union[str, bytes]) -> tarfile.TarInfo:
+        ans = tarfile.TarInfo(arcname)
+        ans.mtime = 0
+        ans.type = tarfile.REGTYPE
+        if isinstance(data, str):
+            data = data.encode('utf-8')
+        ans.size = len(data)
+        normalize_tarinfo(ans)
+        tf.addfile(ans, io.BytesIO(data))
+        return ans
+
+    def filter_from_globs(*pats: str) -> Callable[[tarfile.TarInfo], Optional[tarfile.TarInfo]]:
+        def filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]:
+            for junk_dir in ('.DS_Store', '__pycache__'):
+                for pat in (f'*/{junk_dir}', f'*/{junk_dir}/*'):
+                    if fnmatch.fnmatch(tarinfo.name, pat):
+                        return None
+            for pat in pats:
+                if fnmatch.fnmatch(tarinfo.name, pat):
+                    return None
+            return normalize_tarinfo(tarinfo)
+        return filter
+
+    from kitty.shell_integration import get_effective_ksi_env_var
+    if ssh_opts.shell_integration == 'inherited':
+        ksi = get_effective_ksi_env_var(kitty_opts())
+    else:
+        from kitty.options.utils import shell_integration
+        ksi = get_effective_ksi_env_var(Options({'shell_integration': shell_integration(ssh_opts.shell_integration)}))
+
+    env = {
+        'TERM': os.environ.get('TERM') or kitty_opts().term,
+        'COLORTERM': 'truecolor',
+    }
+    for q in ('KITTY_WINDOW_ID', 'WINDOWID'):
+        val = os.environ.get(q)
+        if val is not None:
+            env[q] = val
+    env.update(ssh_opts.env)
+    env['KITTY_SHELL_INTEGRATION'] = ksi or DELETE_ENV_VAR
+    env['KITTY_SSH_KITTEN_DATA_DIR'] = ssh_opts.remote_dir
+    if ssh_opts.login_shell:
+        env['KITTY_LOGIN_SHELL'] = ssh_opts.login_shell
+    if ssh_opts.cwd:
+        env['KITTY_LOGIN_CWD'] = ssh_opts.cwd
+    if ssh_opts.remote_kitty != 'no':
+        env['KITTY_REMOTE'] = ssh_opts.remote_kitty
+    env_script = serialize_env(env, base_env)
+    buf = io.BytesIO()
+    with tarfile.open(mode=f'w:{compression}', fileobj=buf, encoding='utf-8') as tf:
+        rd = ssh_opts.remote_dir.rstrip('/')
+        for ci in ssh_opts.copy.values():
+            tf.add(ci.local_path, arcname=ci.arcname, filter=filter_from_globs(*ci.exclude_patterns))
+        add_data_as_file(tf, 'data.sh', env_script)
+        if compression == 'gz':
+            tf.add(f'{shell_integration_dir}/ssh/bootstrap-utils.sh', arcname='bootstrap-utils.sh', filter=normalize_tarinfo)
+        if ksi:
+            arcname = 'home/' + rd + '/shell-integration'
+            tf.add(shell_integration_dir, arcname=arcname, filter=filter_from_globs(
+                f'{arcname}/ssh/*',          # bootstrap files are sent as command line args
+                f'{arcname}/zsh/kitty.zsh',  # present for legacy compat not needed by ssh kitten
+            ))
+        if ssh_opts.remote_kitty != 'no':
+            arcname = 'home/' + rd + '/kitty'
+            add_data_as_file(tf, arcname + '/version', str_version.encode('ascii'))
+            tf.add(shell_integration_dir + '/ssh/kitty', arcname=arcname + '/bin/kitty', filter=normalize_tarinfo)
+        tf.add(f'{terminfo_dir}/kitty.terminfo', arcname='home/.terminfo/kitty.terminfo', filter=normalize_tarinfo)
+        tf.add(glob.glob(f'{terminfo_dir}/*/xterm-kitty')[0], arcname='home/.terminfo/x/xterm-kitty', filter=normalize_tarinfo)
+    return buf.getvalue()
+
+
+def get_ssh_data(msg: str, request_id: str) -> Iterator[bytes]:
+    yield b'\nKITTY_DATA_START\n'  # to discard leading data
+    try:
+        msg = standard_b64decode(msg).decode('utf-8')
+        md = dict(x.split('=', 1) for x in msg.split(':'))
+        pw = md['pw']
+        pwfilename = md['pwfile']
+        rq_id = md['id']
+    except Exception:
+        traceback.print_exc()
+        yield b'invalid ssh data request message\n'
+    else:
+        try:
+            with SharedMemory(pwfilename, readonly=True) as shm:
+                shm.unlink()
+                if shm.stats.st_uid != os.geteuid() or shm.stats.st_gid != os.getegid():
+                    raise ValueError('Incorrect owner on pwfile')
+                mode = stat.S_IMODE(shm.stats.st_mode)
+                if mode != stat.S_IREAD:
+                    raise ValueError('Incorrect permissions on pwfile')
+                env_data = json.loads(shm.read_data_with_size())
+                if pw != env_data['pw']:
+                    raise ValueError('Incorrect password')
+                if rq_id != request_id:
+                    raise ValueError('Incorrect request id')
+        except Exception as e:
+            traceback.print_exc()
+            yield f'{e}\n'.encode('utf-8')
+        else:
+            yield b'OK\n'
+            ssh_opts = SSHOptions(env_data['opts'])
+            ssh_opts.copy = {k: CopyInstruction(*v) for k, v in ssh_opts.copy.items()}
+            encoded_data = memoryview(env_data['tarfile'].encode('ascii'))
+            # macOS has a 255 byte limit on its input queue as per man stty.
+            # Not clear if that applies to canonical mode input as well, but
+            # better to be safe.
+            line_sz = 254
+            while encoded_data:
+                yield encoded_data[:line_sz]
+                yield b'\n'
+                encoded_data = encoded_data[line_sz:]
+            yield b'KITTY_DATA_END\n'
+
+
+def safe_remove(x: str) -> None:
+    with suppress(OSError):
+        os.remove(x)
+
+
+def prepare_script(ans: str, replacements: Dict[str, str], script_type: str) -> str:
+    for k in ('EXEC_CMD', 'EXPORT_HOME_CMD'):
+        replacements[k] = replacements.get(k, '')
+
+    def sub(m: 're.Match[str]') -> str:
+        return replacements[m.group()]
+
+    return re.sub('|'.join(fr'\b{k}\b' for k in replacements), sub, ans)
+
+
+def prepare_exec_cmd(remote_args: Sequence[str], is_python: bool) -> str:
+    # ssh simply concatenates multiple commands using a space see
+    # line 1129 of ssh.c and on the remote side sshd.c runs the
+    # concatenated command as shell -c cmd
+    if is_python:
+        return standard_b64encode(' '.join(remote_args).encode('utf-8')).decode('ascii')
+    args = ' '.join(c.replace("'", """'"'"'""") for c in remote_args)
+    return f"""unset KITTY_SHELL_INTEGRATION; exec "$login_shell" -c '{args}'"""
+
+
+def prepare_export_home_cmd(ssh_opts: SSHOptions, is_python: bool) -> str:
+    home = ssh_opts.env.get('HOME')
+    if home == '_kitty_copy_env_var_':
+        home = os.environ.get('HOME')
+    if home:
+        if is_python:
+            return standard_b64encode(home.encode('utf-8')).decode('ascii')
+        else:
+            return f'export HOME={quote_env_val(home)}; cd "$HOME"'
+    return ''
+
+
+def bootstrap_script(
+    ssh_opts: SSHOptions, script_type: str = 'sh', remote_args: Sequence[str] = (),
+    test_script: str = '', request_id: Optional[str] = None, cli_hostname: str = '', cli_uname: str = '',
+    request_data: bool = False, echo_on: bool = True
+) -> Tuple[str, Dict[str, str], SharedMemory]:
+    if request_id is None:
+        request_id = os.environ['KITTY_PID'] + '-' + os.environ['KITTY_WINDOW_ID']
+    is_python = script_type == 'py'
+    export_home_cmd = prepare_export_home_cmd(ssh_opts, is_python) if 'HOME' in ssh_opts.env else ''
+    exec_cmd = prepare_exec_cmd(remote_args, is_python) if remote_args else ''
+    with open(os.path.join(shell_integration_dir, 'ssh', f'bootstrap.{script_type}')) as f:
+        ans = f.read()
+    pw = secrets.token_hex()
+    tfd = standard_b64encode(make_tarfile(ssh_opts, dict(os.environ), 'gz' if script_type == 'sh' else 'bz2')).decode('ascii')
+    data = {'pw': pw, 'opts': ssh_opts._asdict(), 'hostname': cli_hostname, 'uname': cli_uname, 'tarfile': tfd}
+    db = json.dumps(data)
+    with SharedMemory(size=len(db) + SharedMemory.num_bytes_for_size, mode=stat.S_IREAD, prefix=f'kssh-{os.getpid()}-') as shm:
+        shm.write_data_with_size(db)
+        shm.flush()
+        atexit.register(shm.unlink)
+    sensitive_data = {'REQUEST_ID': request_id, 'DATA_PASSWORD': pw, 'PASSWORD_FILENAME': shm.name}
+    replacements = {
+        'EXPORT_HOME_CMD': export_home_cmd,
+        'EXEC_CMD': exec_cmd, 'TEST_SCRIPT': test_script,
+        'REQUEST_DATA': '1' if request_data else '0', 'ECHO_ON': '1' if echo_on else '0',
+    }
+    sd = replacements.copy()
+    if request_data:
+        sd.update(sensitive_data)
+    replacements.update(sensitive_data)
+    return prepare_script(ans, sd, script_type), replacements, shm
 
 
 def get_ssh_cli() -> Tuple[Set[str], Set[str]]:
     other_ssh_args: Set[str] = set()
     boolean_ssh_args: Set[str] = set()
     for k, v in ssh_options().items():
-        k = '-' + k
+        k = f'-{k}'
         if v:
             other_ssh_args.add(k)
         else:
@@ -139,13 +309,22 @@ def get_ssh_cli() -> Tuple[Set[str], Set[str]]:
     return boolean_ssh_args, other_ssh_args
 
 
-def get_connection_data(args: List[str], cwd: str = '') -> Optional[SSHConnectionData]:
+def is_extra_arg(arg: str, extra_args: Tuple[str, ...]) -> str:
+    for x in extra_args:
+        if arg == x or arg.startswith(f'{x}='):
+            return x
+    return ''
+
+
+def get_connection_data(args: List[str], cwd: str = '', extra_args: Tuple[str, ...] = ()) -> Optional[SSHConnectionData]:
     boolean_ssh_args, other_ssh_args = get_ssh_cli()
     port: Optional[int] = None
     expecting_port = expecting_identity = False
     expecting_option_val = False
     expecting_hostname = False
+    expecting_extra_val = ''
     host_name = identity_file = found_ssh = ''
+    found_extra_args: List[Tuple[str, str]] = []
 
     for i, arg in enumerate(args):
         if not found_ssh:
@@ -173,6 +352,15 @@ def get_connection_data(args: List[str], cwd: str = '') -> Optional[SSHConnectio
                 else:
                     identity_file = arg[2:]
                     continue
+            if arg.startswith('--') and extra_args:
+                matching_ex = is_extra_arg(arg, extra_args)
+                if matching_ex:
+                    if '=' in arg:
+                        exval = arg.partition('=')[-1]
+                        found_extra_args.append((matching_ex, exval))
+                        continue
+                    expecting_extra_val = matching_ex
+
             expecting_option_val = True
             continue
 
@@ -183,6 +371,9 @@ def get_connection_data(args: List[str], cwd: str = '') -> Optional[SSHConnectio
                 expecting_port = False
             elif expecting_identity:
                 identity_file = arg
+            elif expecting_extra_val:
+                found_extra_args.append((expecting_extra_val, arg))
+                expecting_extra_val = ''
             expecting_option_val = False
             continue
 
@@ -190,13 +381,22 @@ def get_connection_data(args: List[str], cwd: str = '') -> Optional[SSHConnectio
             host_name = arg
     if not host_name:
         return None
+    if host_name.startswith('ssh://'):
+        from urllib.parse import urlparse
+        purl = urlparse(host_name)
+        if purl.hostname:
+            host_name = purl.hostname
+        if purl.username:
+            host_name = f'{purl.username}@{host_name}'
+        if port is None and purl.port:
+            port = purl.port
     if identity_file:
         if not os.path.isabs(identity_file):
             identity_file = os.path.expanduser(identity_file)
         if not os.path.isabs(identity_file):
             identity_file = os.path.normpath(os.path.join(cwd or os.getcwd(), identity_file))
 
-    return SSHConnectionData(found_ssh, host_name, port, identity_file)
+    return SSHConnectionData(found_ssh, host_name, port, identity_file, tuple(found_extra_args))
 
 
 class InvalidSSHArgs(ValueError):
@@ -208,17 +408,19 @@ class InvalidSSHArgs(ValueError):
     def system_exit(self) -> None:
         if self.err_msg:
             print(self.err_msg, file=sys.stderr)
-        os.execlp('ssh', 'ssh')
+        os.execlp(ssh_exe(), 'ssh')
 
 
-def parse_ssh_args(args: List[str]) -> Tuple[List[str], List[str], bool]:
+def parse_ssh_args(args: List[str], extra_args: Tuple[str, ...] = ()) -> Tuple[List[str], List[str], bool, Tuple[str, ...]]:
     boolean_ssh_args, other_ssh_args = get_ssh_cli()
-    passthrough_args = {'-' + x for x in 'Nnf'}
+    passthrough_args = {f'-{x}' for x in 'NnfG'}
     ssh_args = []
     server_args: List[str] = []
     expecting_option_val = False
     passthrough = False
     stop_option_processing = False
+    found_extra_args: List[str] = []
+    expecting_extra_val = ''
     for argument in args:
         if len(server_args) > 1 or stop_option_processing:
             server_args.append(argument)
@@ -227,10 +429,20 @@ def parse_ssh_args(args: List[str]) -> Tuple[List[str], List[str], bool]:
             if argument == '--':
                 stop_option_processing = True
                 continue
+            if extra_args:
+                matching_ex = is_extra_arg(argument, extra_args)
+                if matching_ex:
+                    if '=' in argument:
+                        exval = argument.partition('=')[-1]
+                        found_extra_args.extend((matching_ex, exval))
+                    else:
+                        expecting_extra_val = matching_ex
+                        expecting_option_val = True
+                    continue
             # could be a multi-character option
             all_args = argument[1:]
             for i, arg in enumerate(all_args):
-                arg = '-' + arg
+                arg = f'-{arg}'
                 if arg in passthrough_args:
                     passthrough = True
                 if arg in boolean_ssh_args:
@@ -247,75 +459,264 @@ def parse_ssh_args(args: List[str]) -> Tuple[List[str], List[str], bool]:
                 raise InvalidSSHArgs(f'unknown option -- {arg[1:]}')
             continue
         if expecting_option_val:
-            ssh_args.append(argument)
+            if expecting_extra_val:
+                found_extra_args.extend((expecting_extra_val, argument))
+                expecting_extra_val = ''
+            else:
+                ssh_args.append(argument)
             expecting_option_val = False
             continue
         server_args.append(argument)
     if not server_args:
         raise InvalidSSHArgs()
-    return ssh_args, server_args, passthrough
+    return ssh_args, server_args, passthrough, tuple(found_extra_args)
 
 
-def quote(x: str) -> str:
-    # we have to escape unbalanced quotes and other unparsable
-    # args as they will break the shell script
-    # But we do not want to quote things like * or 'echo hello'
-    # See https://github.com/kovidgoyal/kitty/issues/1787
-    try:
-        shlex.split(x)
-    except ValueError:
-        x = shlex.quote(x)
-    return x
+def wrap_bootstrap_script(sh_script: str, interpreter: str) -> List[str]:
+    # sshd will execute the command we pass it by join all command line
+    # arguments with a space and passing it as a single argument to the users
+    # login shell with -c. If the user has a non POSIX login shell it might
+    # have different escaping semantics and syntax, so the command it should
+    # execute has to be as simple as possible, basically of the form
+    # interpreter -c unwrap_script escaped_bootstrap_script
+    # The unwrap_script is responsible for unescaping the bootstrap script and
+    # executing it.
+    q = os.path.basename(interpreter).lower()
+    is_python = 'python' in q
+    if is_python:
+        es = standard_b64encode(sh_script.encode('utf-8')).decode('ascii')
+        unwrap_script = '''"import base64, sys; eval(compile(base64.standard_b64decode(sys.argv[-1]), 'bootstrap.py', 'exec'))"'''
+    else:
+        # We cant rely on base64 being available on the remote system, so instead
+        # we quote the bootstrap script by replacing ' and \ with \v and \f
+        # also replacing \n and ! with \r and \b for tcsh
+        # finally surrounding with '
+        es = "'" + sh_script.replace("'", '\v').replace('\\', '\f').replace('\n', '\r').replace('!', '\b') + "'"
+        unwrap_script = r"""'eval "$(echo "$0" | tr \\\v\\\f\\\r\\\b \\\047\\\134\\\n\\\041)"' """
+    # exec is supported by all sh like shells, and fish and csh
+    return ['exec', interpreter, '-c', unwrap_script, es]
 
 
-def get_posix_cmd(terminfo: str, remote_args: List[str]) -> List[str]:
-    sh_script = SHELL_SCRIPT.replace('TERMINFO', terminfo, 1)
-    command_to_execute = ''
-    if remote_args:
-        # ssh simply concatenates multiple commands using a space see
-        # line 1129 of ssh.c and on the remote side sshd.c runs the
-        # concatenated command as shell -c cmd
-        args = [c.replace("'", """'"'"'""") for c in remote_args]
-        command_to_execute = "exec $login_shell -c '{}'".format(' '.join(args))
-    sh_script = sh_script.replace('EXEC_CMD', command_to_execute)
-    return [f'sh -c {shlex.quote(sh_script)}']
+def get_remote_command(
+    remote_args: List[str], ssh_opts: SSHOptions, cli_hostname: str = '', cli_uname: str = '',
+    echo_on: bool = True, request_data: bool = False
+) -> Tuple[List[str], Dict[str, str], str]:
+    interpreter = ssh_opts.interpreter
+    q = os.path.basename(interpreter).lower()
+    is_python = 'python' in q
+    sh_script, replacements, shm = bootstrap_script(
+        ssh_opts, script_type='py' if is_python else 'sh', remote_args=remote_args,
+        cli_hostname=cli_hostname, cli_uname=cli_uname, echo_on=echo_on, request_data=request_data)
+    return wrap_bootstrap_script(sh_script, interpreter), replacements, shm.name
 
 
-def get_python_cmd(terminfo: str, command_to_execute: List[str]) -> List[str]:
-    import json
-    script = PYTHON_SCRIPT.format(
-        terminfo=terminfo.encode('utf-8').hex(),
-        command_to_execute=json.dumps(' '.join(command_to_execute)).encode('utf-8').hex()
-    )
-    return [f'python -c "{script}"']
+def connection_sharing_args(opts: SSHOptions, kitty_pid: int) -> List[str]:
+    rd = runtime_dir()
+    # Bloody OpenSSH generates a 40 char hash and in creating the socket
+    # appends a 27 char temp suffix to it. Socket max path length is approx
+    # ~104 chars. macOS has no system runtime dir so we use a cache dir in
+    # /Users/WHY_DOES_ANYONE_USE_MACOS/Library/Caches/APPLE_ARE_IDIOTIC
+    if len(rd) > 35 and os.path.isdir('/tmp'):
+        idiotic_design = f'/tmp/kssh-rdir-{os.getuid()}'
+        try:
+            os.symlink(rd, idiotic_design)
+        except FileExistsError:
+            try:
+                dest = os.readlink(idiotic_design)
+            except OSError as e:
+                raise ValueError(f'The {idiotic_design} symlink could not be created as something with that name exists already') from e
+            else:
+                if dest != rd:
+                    with tempfile.TemporaryDirectory(dir='/tmp') as tdir:
+                        tlink = os.path.join(tdir, 'sigh')
+                        os.symlink(rd, tlink)
+                        os.rename(tlink, idiotic_design)
+        rd = idiotic_design
+
+    cp = os.path.join(rd, ssh_control_master_template.format(kitty_pid=kitty_pid, ssh_placeholder='%C'))
+    ans: List[str] = [
+        '-o', 'ControlMaster=auto',
+        '-o', f'ControlPath={cp}',
+        '-o', 'ControlPersist=yes',
+        '-o', 'ServerAliveInterval=60',
+        '-o', 'ServerAliveCountMax=5',
+        '-o', 'TCPKeepAlive=no',
+    ]
+    return ans
+
+
+@contextmanager
+def restore_terminal_state() -> Iterator[bool]:
+    with open(os.ctermid()) as f:
+        val = termios.tcgetattr(f.fileno())
+        try:
+            yield bool(val[3] & termios.ECHO)
+        finally:
+            termios.tcsetattr(f.fileno(), termios.TCSAFLUSH, val)
+
+
+def dcs_to_kitty(payload: Union[bytes, str], type: str = 'ssh') -> bytes:
+    if isinstance(payload, str):
+        payload = payload.encode('utf-8')
+    payload = standard_b64encode(payload)
+    return b'\033P@kitty-' + type.encode('ascii') + b'|' + payload + b'\033\\'
+
+
+@run_once
+def ssh_version() -> Tuple[int, int]:
+    o = subprocess.check_output([ssh_exe(), '-V'], stderr=subprocess.STDOUT).decode()
+    m = re.match(r'OpenSSH_(\d+).(\d+)', o)
+    if m is None:
+        raise ValueError(f'Invalid version string for OpenSSH: {o}')
+    return int(m.group(1)), int(m.group(2))
+
+
+@contextmanager
+def drain_potential_tty_garbage(p: 'subprocess.Popen[bytes]', data_request: str) -> Iterator[None]:
+    ssh_started_at = time.monotonic()
+    with open(os.open(os.ctermid(), os.O_CLOEXEC | os.O_RDWR | os.O_NOCTTY), 'wb') as tty:
+        if data_request:
+            turn_off_echo(tty.fileno())
+            tty.write(dcs_to_kitty(data_request))
+            tty.flush()
+        try:
+            yield
+        finally:
+            if p.returncode and time.monotonic() - ssh_started_at < 30:
+                # discard queued input data on tty in case data transmission was
+                # interrupted due to SSH failure, avoids spewing garbage to
+                # screen
+                data = b''
+                give_up_at = time.monotonic() + 1
+                tty_fd = tty.fileno()
+                while time.monotonic() < give_up_at and b'KITTY_DATA_END' not in data:
+                    rd, wr, err = select([tty_fd], [], [tty_fd], max(0, give_up_at - time.monotonic()))
+                    if err or not rd:
+                        break
+                    q = os.read(tty_fd, io.DEFAULT_BUFFER_SIZE)
+                    if not q:
+                        break
+                    data += q
+
+
+def change_colors(color_scheme: str) -> bool:
+    if not color_scheme:
+        return False
+    from kittens.themes.collection import load_themes, NoCacheFound, text_as_opts
+    from kittens.themes.main import colors_as_escape_codes
+    if color_scheme.endswith('.conf'):
+        conf_file = resolve_abs_or_config_path(color_scheme)
+        try:
+            with open(conf_file) as f:
+                opts = text_as_opts(f.read())
+        except FileNotFoundError:
+            raise SystemExit(f'Failed to find the color conf file: {expandvars(conf_file)}')
+    else:
+        try:
+            themes = load_themes(-1)
+        except NoCacheFound:
+            themes = load_themes()
+        cs = expandvars(color_scheme)
+        try:
+            theme = themes[cs]
+        except KeyError:
+            raise SystemExit(f'Failed to find the color theme: {cs}')
+        opts = theme.kitty_opts
+    raw = colors_as_escape_codes(opts)
+    print(save_colors(), sep='', end=raw, flush=True)
+    return True
+
+
+def run_ssh(ssh_args: List[str], server_args: List[str], found_extra_args: Tuple[str, ...]) -> NoReturn:
+    cmd = [ssh_exe()] + ssh_args
+    hostname, remote_args = server_args[0], server_args[1:]
+    if not remote_args:
+        cmd.append('-t')
+    insertion_point = len(cmd)
+    cmd.append('--')
+    cmd.append(hostname)
+    uname = getuser()
+    if hostname.startswith('ssh://'):
+        from urllib.parse import urlparse
+        purl = urlparse(hostname)
+        hostname_for_match = purl.hostname or hostname[6:].split('/', 1)[0]
+        uname = purl.username or uname
+    elif '@' in hostname and hostname[0] != '@':
+        uname, hostname_for_match = hostname.split('@', 1)
+    else:
+        hostname_for_match = hostname
+    hostname_for_match = hostname_for_match.split('@', 1)[-1].split(':', 1)[0]
+    overrides = []
+    pat = re.compile(r'^([a-zA-Z0-9_]+)[ \t]*=')
+    for i, a in enumerate(found_extra_args):
+        if i % 2 == 1:
+            aq = pat.sub(r'\1 ', a.lstrip())
+            if aq.split(maxsplit=1)[0] != 'hostname':
+                overrides.append(aq)
+    if overrides:
+        overrides.insert(0, f'hostname {uname}@{hostname_for_match}')
+    host_opts = init_config(hostname_for_match, uname, overrides)
+    if host_opts.share_connections:
+        cmd[insertion_point:insertion_point] = connection_sharing_args(host_opts, int(os.environ['KITTY_PID']))
+    use_kitty_askpass = host_opts.askpass == 'native' or (host_opts.askpass == 'unless-set' and 'SSH_ASKPASS' not in os.environ)
+    need_to_request_data = True
+    if use_kitty_askpass:
+        sentinel = os.path.join(cache_dir(), 'openssh-is-new-enough-for-askpass')
+        sentinel_exists = os.path.exists(sentinel)
+        if sentinel_exists or ssh_version() >= (8, 4):
+            if not sentinel_exists:
+                open(sentinel, 'w').close()
+            # SSH_ASKPASS_REQUIRE was introduced in 8.4 release on 2020-09-27
+            need_to_request_data = False
+            os.environ['SSH_ASKPASS_REQUIRE'] = 'force'
+        os.environ['SSH_ASKPASS'] = os.path.join(shell_integration_dir, 'ssh', 'askpass.py')
+    if need_to_request_data and host_opts.share_connections:
+        cp = subprocess.run(cmd[:1] + ['-O', 'check'] + cmd[1:], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        if cp.returncode == 0:
+            # we will use the master connection so SSH does not need to use the tty
+            need_to_request_data = False
+    with restore_terminal_state() as echo_on:
+        rcmd, replacements, shm_name = get_remote_command(
+            remote_args, host_opts, hostname_for_match, uname, echo_on, request_data=need_to_request_data)
+        cmd += rcmd
+        colors_changed = change_colors(host_opts.color_scheme)
+        try:
+            p = subprocess.Popen(cmd)
+        except FileNotFoundError:
+            raise SystemExit('Could not find the ssh executable, is it in your PATH?')
+        else:
+            rq = '' if need_to_request_data else 'id={REQUEST_ID}:pwfile={PASSWORD_FILENAME}:pw={DATA_PASSWORD}'.format(**replacements)
+            with drain_potential_tty_garbage(p, rq):
+                try:
+                    raise SystemExit(p.wait())
+                except KeyboardInterrupt:
+                    raise SystemExit(1)
+        finally:
+            if colors_changed:
+                print(end=restore_colors(), flush=True)
 
 
 def main(args: List[str]) -> NoReturn:
     args = args[1:]
-    use_posix = True
     if args and args[0] == 'use-python':
-        args = args[1:]
-        use_posix = False
+        args = args[1:]  # backwards compat from when we had a python implementation
     try:
-        ssh_args, server_args, passthrough = parse_ssh_args(args)
+        ssh_args, server_args, passthrough, found_extra_args = parse_ssh_args(args, extra_args=('--kitten',))
     except InvalidSSHArgs as e:
         e.system_exit()
-    cmd = ['ssh'] + ssh_args
+    if not os.environ.get('KITTY_WINDOW_ID') or not os.environ.get('KITTY_PID'):
+        raise SystemExit('The SSH kitten is meant to run inside a kitty window')
     if passthrough:
-        cmd += server_args
-    else:
-        hostname, remote_args = server_args[0], server_args[1:]
-        if not remote_args:
-            cmd.append('-t')
-        cmd.append('--')
-        cmd.append(hostname)
-        terminfo = subprocess.check_output(['infocmp', '-a']).decode('utf-8')
-        f = get_posix_cmd if use_posix else get_python_cmd
-        cmd += f(terminfo, remote_args)
-    os.execvp('ssh', cmd)
+        raise SystemExit('The SSH kitten is meant for interactive use via SSH only')
+    if not sys.stdin.isatty():
+        raise SystemExit('The SSH kitten is meant for interactive use only, STDIN must be a terminal')
+    run_ssh(ssh_args, server_args, found_extra_args)
 
 
 if __name__ == '__main__':
     main(sys.argv)
 elif __name__ == '__completer__':
     setattr(sys, 'kitten_completer', complete)
+elif __name__ == '__conf__':
+    from .options.definition import definition
+    sys.options_definition = definition  # type: ignore

kittens/ssh/options/definition.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+# After editing this file run ./gen-config.py to apply the changes
+
+from kitty.conf.types import Definition
+
+
+copy_message = '''\
+Copy files and directories from local to remote hosts. The specified files are
+assumed to be relative to the HOME directory and copied to the HOME on the
+remote. Directories are copied recursively. If absolute paths are used, they are
+copied as is.'''
+
+definition = Definition(
+    'kittens.ssh',
+)
+
+agr = definition.add_group
+egr = definition.end_group
+opt = definition.add_option
+
+agr('bootstrap', 'Host bootstrap configuration')  # {{{
+
+opt('hostname', '*', option_type='hostname', long_text='''
+The hostname that the following options apply to. A glob pattern to match multiple
+hosts can be used. Multiple hostnames can also be specified separated by spaces.
+The hostname can include an optional username in the form :code:`user@host`.
+When not specified options apply to all hosts, until the
+first hostname specification is found. Note that matching of hostname is done against
+the name you specify on the command line to connect to the remote host.
+If you wish to include the same basic configuration for many
+different hosts, you can do so with the :ref:`include <include>` directive.
+''')
+
+opt('interpreter', 'sh', long_text='''
+The interpreter to use on the remote host. Must be either a POSIX complaint shell
+or a python executable. If the default sh is not available or broken, using
+an alternate interpreter can be useful.
+''')
+
+opt('remote_dir', '.local/share/kitty-ssh-kitten', option_type='relative_dir', long_text='''
+The location on the remote host where the files needed for this kitten
+are installed. The location is relative to the HOME directory. Absolute paths or paths
+that resolve to a location outside the HOME are not allowed.
+''')
+
+opt('+copy', '', option_type='copy', add_to_default=False, long_text=f'''
+{copy_message} For example::
+
+    copy .vimrc .zshrc .config/some-dir
+
+If a file should be copied to some other destination on the remote host,
+use :code:`--dest`::
+
+    copy --dest some-other-name some-file
+
+Glob patterns can be specified to copy multiple files, with :code:`--glob`::
+
+    copy --glob images/*.png
+
+Files can be excluded when copying with :code:`--exclude`::
+
+    copy --glob --exclude *.jpg --exclude *.bmp images/*
+
+Files whose remote name matches the exclude pattern will not be copied.
+For more details, see :ref:`ssh_copy_command`.
+''')
+egr()  # }}}
+
+agr('shell', 'Login shell environment')  # {{{
+
+opt('shell_integration', 'inherited', long_text='''
+Control the shell integration on the remote host. See :ref:`shell_integration`
+for details on how this setting works. The special value :code:`inherited` means
+use the setting from :file:`kitty.conf`. This setting is useful for overriding
+integration on a per-host basis.
+''')
+
+opt('login_shell', '', long_text='''
+The login shell to execute on the remote host. By default, the remote user account's
+login shell is used.
+''')
+
+opt('+env', '', option_type='env', add_to_default=False, long_text='''
+Specify environment variables to set on the remote host. Note that
+environment variables can refer to each other, so if you use::
+
+    env MYVAR1=a
+    env MYVAR2=$MYVAR1/$HOME/b
+
+The value of MYVAR2 will be :code:`a/<path to home directory>/b`. Using
+:code:`VAR=` will set it to the empty string and using just :code:`VAR`
+will delete the variable from the child process' environment. The definitions
+are processed alphabetically. The special value :code:`_kitty_copy_env_var_`
+will cause the value of the variable to be copied from the local environment.
+''')
+
+opt('cwd', '', long_text='''
+The working directory on the remote host to change to. Env vars in this
+value are expanded. The default is empty so no changing is done, which
+usually means the home directory is used.
+''')
+
+opt('color_scheme', '', long_text='''
+Specify a color scheme to use when connecting to the remote host.  If the
+color_scheme ends with :code:`.conf` it is assumed to be the name of a config
+file to load from the kitty config directory, otherwise it is assumed to be the
+name of a color theme to load via the themes kitten. Note that only colors
+applying to the text/background are changed, other config settings in the .conf
+files/themes are ignored.
+''')
+
+opt('remote_kitty', 'if-needed', choices=('if-needed', 'no', 'yes'), long_text='''
+Make kitty available on the remote server. Useful to run kittens such as the
+icat kitten to display images or the transfer file kitten to transfer files.
+Only works if the remote server has an architecture for which pre-compiled
+kitty binaries are available. Note that kitty is not actually copied to the
+remote server, instead a small bootstrap script is copied which will download
+and run kitty when kitty is first executed on the remote server. A value of
+:code:`if-needed` means kitty is installed only if not already present in the
+system-wide PATH. A value of :code:`yes` means that kitty is installed even if
+already present, and the installed kitty takes precedence. Finally, :code:`no`
+means no kitty is installed on the remote machine. The installed kitty
+can be updated by running: :code:`kitty +update-kitty` on the remote machine.
+''')
+egr()  # }}}
+
+agr('ssh', 'SSH configuration')  # {{{
+
+opt('share_connections', 'yes', option_type='to_bool', long_text='''
+Within a single kitty instance, all connections to a particular server can be
+shared. This reduces startup latency for subsequent connections and means that you have
+to enter the password only once. Under the hood, it uses SSH ControlMasters and
+these are automatically cleaned up by kitty when it quits.
+''')
+
+opt('askpass', 'unless-set', choices=('unless-set', 'ssh', 'native'), long_text='''
+Control the program SSH uses to ask for passwords or confirmation of host keys
+etc. The default is to use kitty's native askpass, unless the SSH_ASKPASS
+environment variable is set. Set it to :code:`ssh` to not interfere with the
+normal ssh askpass mechanism at all, which typically means that ssh will prompt
+at the terminal. Set it to :code:`native` to always use kitty's native,
+built-in askpass implementation. Note that not using the kitty askpass implementation
+means that SSH might need to use the terminal before the connection is established
+so the kitten cannot use the terminal to send data without an extra roundtrip,
+adding to initial connection latency.
+''')
+egr()  # }}}

kittens/ssh/options/parse.py

@@ -0,0 +1,89 @@
+# generated by gen-config.py DO NOT edit
+
+import typing
+from kittens.ssh.options.utils import copy, env, hostname, relative_dir
+from kitty.conf.utils import merge_dicts, to_bool
+
+
+class Parser:
+
+    def askpass(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        val = val.lower()
+        if val not in self.choices_for_askpass:
+            raise ValueError(f"The value {val} is not a valid choice for askpass")
+        ans["askpass"] = val
+
+    choices_for_askpass = frozenset(('unless-set', 'ssh', 'native'))
+
+    def color_scheme(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['color_scheme'] = str(val)
+
+    def copy(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        for k, v in copy(val, ans["copy"]):
+            ans["copy"][k] = v
+
+    def cwd(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['cwd'] = str(val)
+
+    def env(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        for k, v in env(val, ans["env"]):
+            ans["env"][k] = v
+
+    def hostname(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        hostname(val, ans)
+
+    def interpreter(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['interpreter'] = str(val)
+
+    def login_shell(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['login_shell'] = str(val)
+
+    def remote_dir(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['remote_dir'] = relative_dir(val)
+
+    def remote_kitty(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        val = val.lower()
+        if val not in self.choices_for_remote_kitty:
+            raise ValueError(f"The value {val} is not a valid choice for remote_kitty")
+        ans["remote_kitty"] = val
+
+    choices_for_remote_kitty = frozenset(('if-needed', 'no', 'yes'))
+
+    def share_connections(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['share_connections'] = to_bool(val)
+
+    def shell_integration(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['shell_integration'] = str(val)
+
+
+def create_result_dict() -> typing.Dict[str, typing.Any]:
+    return {
+        'copy': {},
+        'env': {},
+    }
+
+
+actions: typing.FrozenSet[str] = frozenset(())
+
+
+def merge_result_dicts(defaults: typing.Dict[str, typing.Any], vals: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:
+    ans = {}
+    for k, v in defaults.items():
+        if isinstance(v, dict):
+            ans[k] = merge_dicts(v, vals.get(k, {}))
+        elif k in actions:
+            ans[k] = v + vals.get(k, [])
+        else:
+            ans[k] = vals.get(k, v)
+    return ans
+
+
+parser = Parser()
+
+
+def parse_conf_item(key: str, val: str, ans: typing.Dict[str, typing.Any]) -> bool:
+    func = getattr(parser, key, None)
+    if func is not None:
+        func(val, ans)
+        return True
+    return False

kittens/ssh/options/types.py

@@ -0,0 +1,92 @@
+# generated by gen-config.py DO NOT edit
+
+import typing
+import kittens.ssh.copy
+
+if typing.TYPE_CHECKING:
+    choices_for_askpass = typing.Literal['unless-set', 'ssh', 'native']
+    choices_for_remote_kitty = typing.Literal['if-needed', 'no', 'yes']
+else:
+    choices_for_askpass = str
+    choices_for_remote_kitty = str
+
+option_names = (  # {{{
+ 'askpass',
+ 'color_scheme',
+ 'copy',
+ 'cwd',
+ 'env',
+ 'hostname',
+ 'interpreter',
+ 'login_shell',
+ 'remote_dir',
+ 'remote_kitty',
+ 'share_connections',
+ 'shell_integration')  # }}}
+
+
+class Options:
+    askpass: choices_for_askpass = 'unless-set'
+    color_scheme: str = ''
+    cwd: str = ''
+    hostname: str = '*'
+    interpreter: str = 'sh'
+    login_shell: str = ''
+    remote_dir: str = '.local/share/kitty-ssh-kitten'
+    remote_kitty: choices_for_remote_kitty = 'if-needed'
+    share_connections: bool = True
+    shell_integration: str = 'inherited'
+    copy: typing.Dict[str, kittens.ssh.copy.CopyInstruction] = {}
+    env: typing.Dict[str, str] = {}
+    config_paths: typing.Tuple[str, ...] = ()
+    config_overrides: typing.Tuple[str, ...] = ()
+
+    def __init__(self, options_dict: typing.Optional[typing.Dict[str, typing.Any]] = None) -> None:
+        if options_dict is not None:
+            null = object()
+            for key in option_names:
+                val = options_dict.get(key, null)
+                if val is not null:
+                    setattr(self, key, val)
+
+    @property
+    def _fields(self) -> typing.Tuple[str, ...]:
+        return option_names
+
+    def __iter__(self) -> typing.Iterator[str]:
+        return iter(self._fields)
+
+    def __len__(self) -> int:
+        return len(self._fields)
+
+    def _copy_of_val(self, name: str) -> typing.Any:
+        ans = getattr(self, name)
+        if isinstance(ans, dict):
+            ans = ans.copy()
+        elif isinstance(ans, list):
+            ans = ans[:]
+        return ans
+
+    def _asdict(self) -> typing.Dict[str, typing.Any]:
+        return {k: self._copy_of_val(k) for k in self}
+
+    def _replace(self, **kw: typing.Any) -> "Options":
+        ans = Options()
+        for name in self:
+            setattr(ans, name, self._copy_of_val(name))
+        for name, val in kw.items():
+            setattr(ans, name, val)
+        return ans
+
+    def __getitem__(self, key: typing.Union[int, str]) -> typing.Any:
+        k = option_names[key] if isinstance(key, int) else key
+        try:
+            return getattr(self, k)
+        except AttributeError:
+            pass
+        raise KeyError(f"No option named: {k}")
+
+
+defaults = Options()
+defaults.copy = {}
+defaults.env = {}

kittens/ssh/options/utils.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# License: GPLv3 Copyright: 2022, Kovid Goyal <kovid at kovidgoyal.net>
+
+import posixpath
+from typing import Any, Dict, Iterable, Optional, Tuple
+
+from ..copy import CopyInstruction, parse_copy_instructions
+
+DELETE_ENV_VAR = '_delete_this_env_var_'
+
+
+def relative_dir(val: str) -> str:
+    if posixpath.isabs(val):
+        raise ValueError(f'Absolute paths not allowed. {val} is invalid.')
+    base = '/ffjdg'
+    q = posixpath.normpath(posixpath.join(base, val))
+    if q == base or not q.startswith(base):
+        raise ValueError(f'Paths that escape their parent dir are not allowed. {val} is not valid')
+    return posixpath.normpath(val)
+
+
+def env(val: str, current_val: Dict[str, str]) -> Iterable[Tuple[str, str]]:
+    val = val.strip()
+    if val:
+        if '=' in val:
+            key, v = val.split('=', 1)
+            key, v = key.strip(), v.strip()
+            if key:
+                yield key, v
+        else:
+            yield val, DELETE_ENV_VAR
+
+
+def copy(val: str, current_val: Dict[str, str]) -> Iterable[Tuple[str, CopyInstruction]]:
+    yield from parse_copy_instructions(val, current_val)
+
+
+def init_results_dict(ans: Dict[str, Any]) -> Dict[str, Any]:
+    ans['hostname'] = '*'
+    ans['per_host_dicts'] = {}
+    return ans
+
+
+def get_per_hosts_dict(results_dict: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
+    ans: Dict[str, Dict[str, Any]] = results_dict.get('per_host_dicts', {}).copy()
+    h = results_dict['hostname']
+    hd = {k: v for k, v in results_dict.items() if k != 'per_host_dicts'}
+    ans[h] = hd
+    return ans
+
+
+first_seen_positions: Dict[str, int] = {}
+
+
+def hostname(val: str, dict_with_parse_results: Optional[Dict[str, Any]] = None) -> str:
+    if dict_with_parse_results is not None:
+        ch = dict_with_parse_results['hostname']
+        if val != ch:
+            from .parse import create_result_dict
+            phd = get_per_hosts_dict(dict_with_parse_results)
+            dict_with_parse_results.clear()
+            dict_with_parse_results.update(phd.pop(val, create_result_dict()))
+            dict_with_parse_results['per_host_dicts'] = phd
+            dict_with_parse_results['hostname'] = val
+            if val not in first_seen_positions:
+                first_seen_positions[val] = len(first_seen_positions)
+    return val

kittens/themes/collection.py

@@ -36,7 +36,7 @@ def patch_conf(raw: str, theme_name: str) -> str:
         nraw = raw + addition
     # comment out all existing color definitions
     color_conf_items = (  # {{{
-        # generated by gen_config.py do not EDIT
+        # generated by gen-config.py DO NOT edit
         # ALL_COLORS_START
         'active_border_color',
         'active_tab_background',
@@ -376,7 +376,7 @@ def fetch_themes(
 
     needs_delete = False
     try:
-        with tempfile.NamedTemporaryFile(suffix='-' + os.path.basename(dest_path), dir=os.path.dirname(dest_path), delete=False) as f:
+        with tempfile.NamedTemporaryFile(suffix=f'-{os.path.basename(dest_path)}', dir=os.path.dirname(dest_path), delete=False) as f:
             needs_delete = True
             shutil.copyfileobj(res, f)
             f.flush()
@@ -405,7 +405,7 @@ def theme_name_from_file_name(fname: str) -> str:
     ans = ans.replace('_', ' ')
 
     def camel_case(m: 'Match[str]') -> str:
-        return str(m.group(1) + ' ' + m.group(2))
+        return f'{m.group(1)} {m.group(2)}'
 
     ans = re.sub(r'([a-z])([A-Z])', camel_case, ans)
     return ' '.join(x.capitalize() for x in filter(None, ans.split()))
@@ -483,6 +483,10 @@ def update_theme_file(path: str) -> bool:
     return True
 
 
+def text_as_opts(text: str) -> KittyOptions:
+    return KittyOptions(options_dict=parse_config(text.splitlines()))
+
+
 class Theme:
     name: str = ''
     author: str = ''
@@ -516,16 +520,16 @@ class Theme:
     @property
     def kitty_opts(self) -> KittyOptions:
         if self._opts is None:
-            self._opts = KittyOptions(options_dict=parse_config(self.raw.splitlines()))
+            self._opts = text_as_opts(self.raw)
         return self._opts
 
     def save_in_dir(self, dirpath: str) -> None:
         atomic_save(self.raw.encode('utf-8'), os.path.join(dirpath, f'{self.name}.conf'))
 
-    def save_in_conf(self, confdir: str, reload_in: str) -> None:
+    def save_in_conf(self, confdir: str, reload_in: str, config_file_name: str = 'kitty.conf') -> None:
         os.makedirs(confdir, exist_ok=True)
         atomic_save(self.raw.encode('utf-8'), os.path.join(confdir, 'current-theme.conf'))
-        confpath = os.path.realpath(os.path.join(confdir, 'kitty.conf'))
+        confpath = os.path.realpath(os.path.join(confdir, config_file_name))
         try:
             with open(confpath) as f:
                 raw = f.read()
@@ -533,7 +537,7 @@ class Theme:
             raw = ''
         nraw = patch_conf(raw, self.name)
         if raw:
-            with open(confpath + '.bak', 'w') as f:
+            with open(f'{confpath}.bak', 'w') as f:
                 f.write(raw)
         atomic_save(nraw.encode('utf-8'), confpath)
         if reload_in == 'parent':

kittens/themes/main.py

@@ -14,6 +14,7 @@ from typing import (
 from kitty.cli import create_default_opts, parse_args
 from kitty.cli_stub import ThemesCLIOptions
 from kitty.config import cached_values_for
+from kitty.options.types import Options as KittyOptions
 from kitty.constants import config_dir
 from kitty.fast_data_types import truncate_point_for_length, wcswidth
 from kitty.rgb import color_as_sharp, color_from_int
@@ -23,7 +24,7 @@ from kitty.utils import ScreenSize
 from ..tui.handler import Handler
 from ..tui.line_edit import LineEdit
 from ..tui.loop import Loop
-from ..tui.operations import color_code, styled
+from ..tui.operations import color_code, set_default_colors, styled
 from .collection import MARK_AFTER, NoCacheFound, Theme, Themes, load_themes
 
 separator = '║'
@@ -37,7 +38,7 @@ def limit_length(text: str, limit: int = 32) -> str:
     x = truncate_point_for_length(text, limit - 1)
     if x >= len(text):
         return text
-    return text[:x] + '…'
+    return f'{text[:x]}…'
 
 
 class State(Enum):
@@ -132,6 +133,17 @@ class ThemesList:
         return self.themes[self.current_idx]
 
 
+def colors_as_escape_codes(o: KittyOptions) -> str:
+    ans = set_default_colors(
+        fg=o.foreground, bg=o.background, cursor=o.cursor, select_bg=o.selection_background, select_fg=o.selection_foreground
+    )
+    cmds = []
+    for i in range(256):
+        col = color_as_sharp(color_from_int(o.color_table[i]))
+        cmds.append(f'{i};{col}')
+    return ans + '\033]4;' + ';'.join(cmds) + '\033\\'
+
+
 class ThemesHandler(Handler):
 
     def __init__(self, cached_values: Dict[str, Any], cli_opts: ThemesCLIOptions) -> None:
@@ -195,15 +207,9 @@ class ThemesHandler(Handler):
             o = self.themes_list.current_theme.kitty_opts
         else:
             o = create_default_opts()
-        self.cmd.set_default_colors(
-            fg=o.foreground, bg=o.background, cursor=o.cursor, select_bg=o.selection_background, select_fg=o.selection_foreground
-        )
         self.current_opts = o
-        cmds = []
-        for i in range(256):
-            col = color_as_sharp(color_from_int(o.color_table[i]))
-            cmds.append(f'{i};{col}')
-        self.print(end='\033]4;' + ';'.join(cmds) + '\033\\')
+        cmd = colors_as_escape_codes(o)
+        self.write(cmd)
         return True
 
     def redraw_after_category_change(self) -> None:
@@ -332,7 +338,7 @@ class ThemesHandler(Handler):
         for line, width, is_current in self.themes_list.lines(num_rows):
             num_rows -= 1
             if is_current:
-                line = line.replace(MARK_AFTER, '\033[' + color_code('green') + 'm')
+                line = line.replace(MARK_AFTER, f'\033[{color_code("green")}m')
             self.cmd.styled('>' if is_current else ' ', fg='green')
             self.cmd.styled(line, bold=is_current, fg='green' if is_current else None)
             self.cmd.move_cursor_by(mw - width, 'right')
@@ -433,7 +439,7 @@ class ThemesHandler(Handler):
     def draw_accepting_screen(self) -> None:
         name = self.themes_list.current_theme.name
         name = styled(name, bold=True, fg="green")
-        kc = styled('kitty.conf', italic=True)
+        kc = styled(self.cli_opts.config_file_name, italic=True)
 
         def ac(x: str) -> str:
             return styled(x, fg='red')
@@ -465,7 +471,7 @@ class ThemesHandler(Handler):
             self.quit_on_next_key_release = 0
             return
         if key_event.matches_text('m'):
-            self.themes_list.current_theme.save_in_conf(config_dir, self.cli_opts.reload_in)
+            self.themes_list.current_theme.save_in_conf(config_dir, self.cli_opts.reload_in, self.cli_opts.config_file_name)
             self.update_recent()
             self.quit_on_next_key_release = 0
             return
@@ -536,6 +542,13 @@ type=bool-set
 default=false
 When running non-interactively, dump the specified theme to STDOUT
 instead of changing kitty.conf.
+
+
+--config-file-name
+default=kitty.conf
+The name or path to the config file to edit. Relative paths are interpreted
+with respect to the kitty config directory. By default the kitty config file,
+kitty.conf is edited.
 '''.format
 
 
@@ -559,7 +572,7 @@ def non_interactive(cli_opts: ThemesCLIOptions, theme_name: str) -> None:
     if cli_opts.dump_theme:
         print(theme.raw)
         return
-    theme.save_in_conf(config_dir, cli_opts.reload_in)
+    theme.save_in_conf(config_dir, cli_opts.reload_in, cli_opts.config_file_name)
 
 
 def main(args: List[str]) -> None:

kittens/transfer/librsync.py

@@ -126,7 +126,7 @@ class PatchFile(StreamingJob):
 
     def read_from_src(self, b: memoryview, pos: int) -> int:
         self.src_file.seek(pos)
-        return self.src_file.readinto(b)  # type: ignore
+        return self.src_file.readinto(b)
 
     def close(self) -> None:
         if not self.src_file.closed:
@@ -155,12 +155,12 @@ def develop() -> None:
     import sys
     src = sys.argv[-1]
     sig_loader = LoadSignature()
-    with open(src + '.sig', 'wb') as f:
+    with open(f'{src}.sig', 'wb') as f:
         for chunk in signature_of_file(src):
             sig_loader.add_chunk(chunk)
             f.write(chunk)
     sig_loader.commit()
-    with open(src + '.delta', 'wb') as f, PatchFile(src, src + '.output') as patcher:
+    with open(f'{src}.delta', 'wb') as f, PatchFile(src, f'{src}.output') as patcher:
         for chunk in delta_for_file(src, sig_loader.signature):
             f.write(chunk)
             patcher.write(chunk)

kittens/transfer/utils.py

@@ -45,7 +45,7 @@ def render_path_in_width(path: str, width: int) -> str:
     if wcswidth(path) <= width:
         return path
     x = truncate_point_for_length(path, width - 1)
-    return path[:x] + '…'
+    return f'{path[:x]}…'
 
 
 def render_seconds(val: float) -> str:

kittens/tui/dircolors.py

@@ -331,7 +331,7 @@ class Dircolors:
                 # change .xyz to *.xyz
                 yield '*' + pair[0], pair[1]
 
-        return ':'.join('%s=%s' % pair for pair in gen_pairs())
+        return ':'.join('{}={}'.format(*pair) for pair in gen_pairs())
 
     def _format_code(self, text: str, code: str) -> str:
         val = self.codes.get(code)

kittens/tui/handler.py

@@ -44,6 +44,7 @@ class Handler:
     use_alternate_screen = True
     mouse_tracking = MouseTracking.none
     terminal_io_ended = False
+    overlay_ready_report_needed = False
 
     def _initialize(
         self,
@@ -206,18 +207,23 @@ class HandleResult:
     type_of_input: Optional[str] = None
     no_ui: bool = False
 
-    def __init__(self, impl: Callable[..., Any], type_of_input: Optional[str], no_ui: bool):
+    def __init__(self, impl: Callable[..., Any], type_of_input: Optional[str], no_ui: bool, has_ready_notification: bool):
         self.impl = impl
         self.no_ui = no_ui
         self.type_of_input = type_of_input
+        self.has_ready_notification = has_ready_notification
 
     def __call__(self, args: Sequence[str], data: Any, target_window_id: int, boss: BossType) -> Any:
         return self.impl(args, data, target_window_id, boss)
 
 
-def result_handler(type_of_input: Optional[str] = None, no_ui: bool = False) -> Callable[[Callable[..., Any]], HandleResult]:
+def result_handler(
+    type_of_input: Optional[str] = None,
+    no_ui: bool = False,
+    has_ready_notification: bool = Handler.overlay_ready_report_needed
+) -> Callable[[Callable[..., Any]], HandleResult]:
 
     def wrapper(impl: Callable[..., Any]) -> HandleResult:
-        return HandleResult(impl, type_of_input, no_ui)
+        return HandleResult(impl, type_of_input, no_ui, has_ready_notification)
 
     return wrapper

kittens/tui/images.py

@@ -189,7 +189,9 @@ def render_image(
     m: ImageData,
     available_width: int, available_height: int,
     scale_up: bool,
-    only_first_frame: bool = False
+    only_first_frame: bool = False,
+    remove_alpha: str = '',
+    flip: bool = False, flop: bool = False,
 ) -> RenderedImage:
     import tempfile
     has_multiple_frames = len(m) > 1
@@ -202,7 +204,15 @@ def render_image(
         if exe is None:
             raise OSError('Failed to find the ImageMagick convert executable, make sure it is present in PATH')
         cmd = [exe]
-    cmd += ['-background', 'none', '--', path]
+    if remove_alpha:
+        cmd += ['-background', remove_alpha, '-alpha', 'remove']
+    else:
+        cmd += ['-background', 'none']
+    if flip:
+        cmd.append('-flip')
+    if flop:
+        cmd.append('-flop')
+    cmd += ['--', path]
     if only_first_frame and has_multiple_frames:
         cmd[-1] += '[0]'
     cmd.append('-auto-orient')
@@ -287,12 +297,15 @@ def render_as_single_image(
     path: str, m: ImageData,
     available_width: int, available_height: int,
     scale_up: bool,
-    tdir: Optional[str] = None
+    tdir: Optional[str] = None,
+    remove_alpha: str = '', flip: bool = False, flop: bool = False,
 ) -> Tuple[str, int, int]:
     import tempfile
     fd, output = tempfile.mkstemp(prefix='icat-', suffix=f'.{m.mode}', dir=tdir)
     os.close(fd)
-    result = render_image(path, output, m, available_width, available_height, scale_up, only_first_frame=True)
+    result = render_image(
+        path, output, m, available_width, available_height, scale_up,
+        only_first_frame=True, remove_alpha=remove_alpha, flip=flip, flop=flop)
     os.rename(result.frames[0].path, output)
     return output, result.width, result.height
 

kittens/tui/line_edit.py

@@ -13,8 +13,9 @@ from .operations import (
 
 class LineEdit:
 
-    def __init__(self) -> None:
+    def __init__(self, is_password: bool = False) -> None:
         self.clear()
+        self.is_password = is_password
 
     def clear(self) -> None:
         self.current_input = ''
@@ -31,7 +32,10 @@ class LineEdit:
         if self.pending_bell:
             write('\a')
             self.pending_bell = False
-        text = prompt + self.current_input
+        ci = self.current_input
+        if self.is_password:
+            ci = '*' * wcswidth(ci)
+        text = prompt + ci
         cursor_pos = self.cursor_pos + wcswidth(prompt)
         if screen_cols:
             write(SAVE_CURSOR + text + RESTORE_CURSOR)

kittens/tui/loop.py

@@ -110,14 +110,16 @@ class TermManager:
 
 class MouseButton(IntFlag):
     NONE, LEFT, MIDDLE, RIGHT, FOURTH, FIFTH = 0, 1, 2, 4, 8, 16
+    WHEEL_UP, WHEEL_DOWN = -1, -2
 
 
 bmap = {0: MouseButton.LEFT, 1: MouseButton.MIDDLE, 2: MouseButton.RIGHT}
-MOTION_INDICATOR = 1 << 5
-EXTRA_BUTTON_INDICATOR = 1 << 6
 SHIFT_INDICATOR = 1 << 2
 ALT_INDICATOR = 1 << 3
 CTRL_INDICATOR = 1 << 4
+MOTION_INDICATOR = 1 << 5
+SCROLL_BUTTON_INDICATOR = 1 << 6
+EXTRA_BUTTON_INDICATOR = 1 << 7
 
 
 class EventType(Enum):
@@ -149,7 +151,9 @@ def decode_sgr_mouse(text: str, screen_size: ScreenSize) -> MouseEvent:
     buttons: MouseButton = MouseButton.NONE
     cb3 = cb & 3
     if cb3 != 3:
-        if cb & EXTRA_BUTTON_INDICATOR:
+        if cb & SCROLL_BUTTON_INDICATOR:
+            buttons = MouseButton.WHEEL_DOWN if cb3 & 1 else MouseButton.WHEEL_UP
+        elif cb & EXTRA_BUTTON_INDICATOR:
             buttons |= MouseButton.FIFTH if cb3 & 1 else MouseButton.FOURTH
         else:
             buttons |= bmap[cb3]
@@ -178,7 +182,7 @@ class UnhandledException(Handler):
         self.cmd.set_default_colors()
         self.write(self.tb.replace('\n', '\r\n'))
         self.write('\r\n')
-        self.write('Press the Enter key to quit')
+        self.write('Press Enter to quit')
 
     def on_key(self, key_event: KeyEventType) -> None:
         if key_event.key == 'ENTER':
@@ -232,6 +236,7 @@ class Loop:
         else:
             self.asyncio_loop = asyncio.get_event_loop()
         self.return_code = 0
+        self.overlay_ready_reported = False
         self.optional_actions = optional_actions
         self.read_buf = ''
         self.decoder = codecs.getincrementaldecoder('utf-8')('ignore')
@@ -431,6 +436,8 @@ class Loop:
         self.asyncio_loop.set_exception_handler(handle_exception)
         handler._initialize(self._get_screen_size(), term_manager, schedule_write, self, debug, image_manager)
         with handler:
+            if handler.overlay_ready_report_needed:
+                handler.cmd.overlay_ready()
             self.asyncio_loop.add_reader(
                     tty_fd, self._read_ready, handler, tty_fd)
             self.asyncio_loop.add_writer(
@@ -463,9 +470,12 @@ class Loop:
 
             term_manager.extra_finalize = b''.join(self.write_buf).decode('utf-8')
             if tb is not None:
+                report_overlay_ready = handler.overlay_ready_report_needed and not self.overlay_ready_reported
                 self.return_code = 1
                 if not handler.terminal_io_ended:
-                    self._report_error_loop(tb, term_manager)
+                    self._report_error_loop(tb, term_manager, report_overlay_ready)
 
-    def _report_error_loop(self, tb: str, term_manager: TermManager) -> None:
-        self.loop_impl(UnhandledException(tb), term_manager)
+    def _report_error_loop(self, tb: str, term_manager: TermManager, overlay_ready_report_needed: bool) -> None:
+        handler = UnhandledException(tb)
+        handler.overlay_ready_report_needed = overlay_ready_report_needed
+        self.loop_impl(handler, term_manager)

kittens/tui/operations.py

@@ -5,13 +5,13 @@ import sys
 from contextlib import contextmanager
 from enum import Enum, auto
 from functools import wraps
-from typing import (
-    IO, Any, Callable, Dict, Generator, Optional, TypeVar, Union
-)
+from typing import IO, Any, Callable, Dict, Generator, Optional, TypeVar, Union
 
 from kitty.fast_data_types import Color
 from kitty.rgb import color_as_sharp, to_color
-from kitty.typing import GraphicsCommandType, HandlerType, ScreenSize
+from kitty.typing import (
+    GraphicsCommandType, HandlerType, ScreenSize, UnderlineLiteral
+)
 
 from .operations_stub import CMD
 
@@ -163,14 +163,11 @@ def set_scrolling_region(screen_size: Optional['ScreenSize'] = None, top: Option
 
 @cmd
 def scroll_screen(amt: int = 1) -> str:
-    return '\033[' + str(abs(amt)) + ('T' if amt < 0 else 'S')
+    return f'\033[{abs(amt)}{"T" if amt < 0 else "S"}'
 
 
-STANDARD_COLORS = {name: i for i, name in enumerate(
-    'black red green yellow blue magenta cyan gray'.split())}
-STANDARD_COLORS['white'] = STANDARD_COLORS['gray']
-UNDERLINE_STYLES = {name: i + 1 for i, name in enumerate(
-    'straight double curly'.split())}
+STANDARD_COLORS = {'black': 0, 'red': 1, 'green': 2, 'yellow': 3, 'blue': 4, 'magenta': 5, 'cyan': 6, 'gray': 7, 'white': 7}
+UNDERLINE_STYLES = {'straight': 1, 'double': 2, 'curly': 3, 'dotted': 4, 'dashed': 5}
 
 
 ColorSpec = Union[int, str, Color]
@@ -217,7 +214,7 @@ def styled(
     bg_intense: bool = False,
     italic: Optional[bool] = None,
     bold: Optional[bool] = None,
-    underline: Optional[str] = None,
+    underline: Optional[UnderlineLiteral] = None,
     underline_color: Optional[ColorSpec] = None,
     reverse: Optional[bool] = None,
     dim: Optional[bool] = None,
@@ -291,7 +288,7 @@ class MouseTracking(Enum):
     full = auto()
 
 
-def init_state(alternate_screen: bool = True, mouse_tracking: MouseTracking = MouseTracking.none) -> str:
+def init_state(alternate_screen: bool = True, mouse_tracking: MouseTracking = MouseTracking.none, kitty_keyboard_mode: bool = True) -> str:
     sc = SAVE_CURSOR if alternate_screen else ''
     ans = (
         S7C1T + sc + SAVE_PRIVATE_MODE_VALUES + reset_mode(Mode.LNM) +
@@ -314,7 +311,10 @@ def init_state(alternate_screen: bool = True, mouse_tracking: MouseTracking = Mo
             ans += set_mode(Mode.MOUSE_MOTION_TRACKING)
         elif mouse_tracking is MouseTracking.full:
             ans += set_mode(Mode.MOUSE_MOVE_TRACKING)
-    ans += '\033[>31u'  # extended keyboard mode
+    if kitty_keyboard_mode:
+        ans += '\033[>31u'  # extended keyboard mode
+    else:
+        ans += '\033[>u'  # legacy keyboard mode
     return ans
 
 
@@ -413,6 +413,11 @@ def restore_colors() -> str:
     return '\x1b[#Q'
 
 
+@cmd
+def overlay_ready() -> str:
+    return '\x1bP@kitty-overlay-ready|\x1b\\'
+
+
 @cmd
 def write_to_clipboard(data: Union[str, bytes], use_primary: bool = False) -> str:
     from base64 import standard_b64encode
@@ -465,7 +470,7 @@ def as_type_stub() -> str:
     for name, func in all_cmds.items():
         args = ', '.join(func_sig(func))
         if args:
-            args = ', ' + args
+            args = f', {args}'
         methods.append(f'    def {name}(self{args}) -> str: pass')
     ans += ['', '', 'class CMD:'] + methods
 

kittens/tui/utils.py

@@ -48,3 +48,24 @@ def human_size(
     from math import log
     exponent = min(int(log(size, 1024)), len(unit_list) - 1)
     return format_number(size / 1024**exponent, max_num_of_decimals) + sep + unit_list[exponent]
+
+
+def report_error(msg: str = '', return_code: int = 1, print_exc: bool = False) -> None:
+    ' Report an error also sending the overlay ready message to ensure kitten is visible '
+    from .operations import overlay_ready
+    print(end=overlay_ready())
+    if msg:
+        print(msg, file=sys.stderr)
+    if print_exc:
+        cls, e, tb = sys.exc_info()
+        if e and not isinstance(e, (SystemExit, KeyboardInterrupt)):
+            import traceback
+            traceback.print_exc()
+    with suppress(KeyboardInterrupt, EOFError):
+        input('Press Enter to quit')
+    raise SystemExit(return_code)
+
+
+def report_unhandled_error(msg: str = '') -> None:
+    ' Report an unhandled exception with the overlay ready message '
+    return report_error(msg, print_exc=True)