make clean should not clean bypy build artifacts

Fixes #3773

.gitattributes

@@ -7,6 +7,11 @@ kitty/rgb.py linguist-generated=true
 kitty/gl-wrapper.* linguist-generated=true
 kitty/glfw-wrapper.* linguist-generated=true
 kitty/parse-graphics-command.h linguist-generated=true
+kitty/options/types.py linguist-generated=true
+kitty/options/parse.py linguist-generated=true
+kitty/options/to-c-generated.h linguist-generated=true
+kittens/diff/options/types.py linguist-generated=true
+kittens/diff/options/parse.py linguist-generated=true
 glfw/*.c linguist-vendored=true
 glfw/*.h linguist-vendored=true
 kittens/unicode_input/names.h linguist-generated=true

.github/ISSUE_TEMPLATE/bug_report.md

@@ -17,17 +17,15 @@ Steps to reproduce the behavior:
 3. ZZZ
 4. See error
 
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
 **Environment details**
-OS: Name and version of operating system(s)
-
 ```
-Output of kitty --debug-config
+Press Ctrl+Shift+F6 (cmd+option+, on macOS) in kitty, to copy debug output about kitty and its
+configuration to the clipboard and paste it here. 
+
+On older versions of kitty, run kitty --debug-config instead
 ```
 
 **Additional context**

.github/workflows/ci.yml

@@ -76,7 +76,7 @@ jobs:
                 python-version: 3.8
 
             - name: Install build-only deps
-              run: pip install flake8 mypy sphinx
+              run: pip install flake8 mypy types-requests types-docutils sphinx
 
             - name: Run pyflakes
               run: python -m flake8 --count .

__main__.py

@@ -2,8 +2,8 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
 
-import sys
 import os
+import sys
 from typing import List
 
 
@@ -30,8 +30,8 @@ def runpy(args: List[str]) -> None:
 
 def hold(args: List[str]) -> None:
     import subprocess
-    from contextlib import suppress
     import tty
+    from contextlib import suppress
     ret = subprocess.Popen(args[1:]).wait()
     with suppress(BaseException):
         print('\n\x1b[1;32mPress any key to exit', end='', flush=True)
@@ -72,13 +72,22 @@ def run_kitten(args: List[str]) -> None:
 
 
 def edit_config_file(args: List[str]) -> None:
+    from kitty.cli import create_default_opts
+    from kitty.fast_data_types import set_options
     from kitty.utils import edit_config_file as f
+    set_options(create_default_opts())
     f()
 
 
 def namespaced(args: List[str]) -> None:
-    func = namespaced_entry_points[args[1]]
-    func(args[1:])
+    try:
+        func = namespaced_entry_points[args[1]]
+    except KeyError:
+        pass
+    else:
+        func(args[1:])
+        return
+    raise SystemExit(f'{args[1]} is not a known entry point. Choices are: ' + ', '.join(namespaced_entry_points))
 
 
 entry_points = {

bypy/init_env.py

@@ -46,9 +46,12 @@ def run(*args, **extra_env):
     return subprocess.call(list(args), env=env, cwd=cwd)
 
 
+SETUP_CMD = [PYTHON, 'setup.py', '--build-universal-binary']
+
+
 def build_frozen_launcher(extra_include_dirs):
     inc_dirs = [f'--extra-include-dirs={x}' for x in extra_include_dirs]
-    cmd = [PYTHON, 'setup.py', '--prefix', build_frozen_launcher.prefix] + inc_dirs + ['build-frozen-launcher']
+    cmd = SETUP_CMD + ['--prefix', build_frozen_launcher.prefix] + inc_dirs + ['build-frozen-launcher']
     if run(*cmd, cwd=build_frozen_launcher.writeable_src_dir) != 0:
         print('Building of frozen kitty launcher failed', file=sys.stderr)
         os.chdir(KITTY_DIR)
@@ -89,7 +92,7 @@ def build_c_extensions(ext_dir, args):
     with suppress(FileNotFoundError):
         os.unlink(os.path.join(writeable_src_dir, 'kitty', 'launcher', 'kitty'))
 
-    cmd = [PYTHON, 'setup.py', 'macos-freeze' if ismacos else 'linux-freeze']
+    cmd = SETUP_CMD + ['macos-freeze' if ismacos else 'linux-freeze']
     if args.dont_strip:
         cmd.append('--debug')
     dest = kitty_constants['appname'] + ('.app' if ismacos else '')

bypy/linux/__main__.py

@@ -124,7 +124,7 @@ def copy_python(env):
     for x in bases:
         iv['sanitize_source_folder'](os.path.join(env.py_dir, x))
     py_compile(env.py_dir)
-    freeze_python(env.py_dir, pdir, env.obj_dir, ext_map, develop_mode_env_var='KITTY_DEVELOP_FROM')
+    freeze_python(env.py_dir, pdir, env.obj_dir, ext_map, develop_mode_env_var='KITTY_DEVELOP_FROM', remove_pyc_files=True)
 
 
 def build_launcher(env):

bypy/macos.conf

@@ -1,6 +1,8 @@
 # Requires installation of XCode 10.3 and Python 3 and
 # python3 -m pip install certifi
 
-vm_name 'macos-kitty-build'
+vm_name 'macos-kitty'
 root '/Users/Shared/kitty-build'
 python '/usr/local/bin/python3'
+universal 'true'
+deploy_target '10.14'

bypy/macos/__main__.py

@@ -278,7 +278,6 @@ class Freeze(object):
                 'lcms2.2',
                 'crypto.1.1',
                 'ssl.1.1',
-                'ffi.7',
         ):
             print('\nAdding', x)
             x = 'lib%s.dylib' % x
@@ -349,7 +348,7 @@ class Freeze(object):
         for x in bases:
             iv['sanitize_source_folder'](os.path.join(self.python_stdlib, x))
         self.compile_py_modules()
-        freeze_python(self.python_stdlib, pdir, self.obj_dir, ext_map, develop_mode_env_var='KITTY_DEVELOP_FROM')
+        freeze_python(self.python_stdlib, pdir, self.obj_dir, ext_map, develop_mode_env_var='KITTY_DEVELOP_FROM', remove_pyc_files=True)
         iv['build_frozen_launcher']([path_to_freeze_dir(), self.obj_dir])
         os.rename(join(dirname(self.contents_dir), 'bin', 'kitty'), join(self.contents_dir, 'MacOS', 'kitty'))
         shutil.rmtree(join(dirname(self.contents_dir), 'bin'))
@@ -420,7 +419,7 @@ class Freeze(object):
         py_compile(join(self.resources_dir, 'Python'))
 
     @flush
-    def makedmg(self, d, volname, internet_enable=True, format='ULFO'):
+    def makedmg(self, d, volname, format='ULFO'):
         ''' Copy a directory d into a dmg named volname '''
         print('\nMaking dmg...')
         sys.stdout.flush()
@@ -456,9 +455,6 @@ class Freeze(object):
         print('\nCreating dmg...')
         with timeit() as times:
             subprocess.check_call(cmd + [dmg])
-            if internet_enable:
-                subprocess.check_call(
-                    ['/usr/bin/hdiutil', 'internet-enable', '-yes', dmg])
         print('dmg created in %d minutes and %d seconds' % tuple(times))
         shutil.rmtree(tdir)
         size = os.stat(dmg).st_size / (1024 * 1024.)

bypy/sources.json

@@ -28,6 +28,16 @@
         }
     },
 
+    {
+        "name": "cmake",
+        "os": "macos",
+        "unix": {
+            "filename": "cmake-3.19.4.tar.gz",
+            "hash": "sha256:7d0232b9f1c57e8de81f38071ef8203e6820fe7eec8ae46a1df125d88dbcc2e1",
+            "urls": ["https://cmake.org/files/v3.19/{filename}"]
+        }
+    },
+
 
     {
         "name": "expat",
@@ -72,6 +82,7 @@
 
     {
         "name": "libffi",
+        "os": "linux",
         "unix": {
             "filename": "libffi-3.3.0.tar.gz",
             "hash": "sha256:72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056",
@@ -130,9 +141,9 @@
     {
         "name": "python",
         "unix": {
-            "filename": "Python-3.9.1.tar.xz",
-            "hash": "sha256:991c3f8ac97992f3d308fefeb03a64db462574eadbff34ce8bc5bb583d9903ff",
-            "urls": ["https://www.python.org/ftp/python/3.9.1/{filename}"]
+            "filename": "Python-3.9.4.tar.xz",
+            "hash": "sha256:4b0e6644a76f8df864ae24ac500a51bbf68bd098f6a173e27d3b61cdca9aa134",
+            "urls": ["https://www.python.org/ftp/python/3.9.4/{filename}"]
         }
     },
 
@@ -154,16 +165,6 @@
         }
     },
 
-    {
-        "name": "cmake",
-        "os": "macos",
-        "unix": {
-            "filename": "cmake-3.19.4.tar.gz",
-            "hash": "sha256:7d0232b9f1c57e8de81f38071ef8203e6820fe7eec8ae46a1df125d88dbcc2e1",
-            "urls": ["https://cmake.org/files/v3.19/{filename}"]
-        }
-    },
-
     {
         "name": "libpng",
         "unix": {

count-lines-of-code

@@ -16,6 +16,11 @@ kitty/glfw-wrapper.c
 kitty/emoji.h
 kittens/unicode_input/names.h
 kitty/parse-graphics-command.h
+kitty/options/types.py
+kitty/options/parse.py
+kitty/options/to-c-generated.h
+kittens/diff/options/types.py
+kittens/diff/options/parse.py
 '''
 
 p = subprocess.Popen([

docs/basic.rst

@@ -116,6 +116,8 @@ Toggle maximized                    :sc:`toggle_maximized`
 Input unicode character             :sc:`input_unicode_character` (also :kbd:`^+⌘+space` on macOS)
 Click URL using the keyboard        :sc:`open_url`
 Reset the terminal                  :sc:`reset_terminal`
+Reload :file:`kitty.conf`           :sc:`reload_config_file` (also :kbd:`^+⌘+f5` on macOS)
+Debug :file:`kitty.conf`            :sc:`debug_config` (also :kbd:`⌘+option+f6` on macOS)
 Pass current selection to program   :sc:`pass_selection_to_program`
 Edit |kitty| config file            :sc:`edit_config_file`
 Open a |kitty| shell                :sc:`kitty_shell`

docs/binary.rst

@@ -48,6 +48,13 @@ particular desktop, but it should work for most major desktop environments.
     # Update the path to the kitty icon in the kitty.desktop file
     sed -i "s|Icon=kitty|Icon=/home/$USER/.local/kitty.app/share/icons/hicolor/256x256/apps/kitty.png|g" ~/.local/share/applications/kitty.desktop
 
+.. note::
+    If you use the venerable `stow <https://www.gnu.org/software/stow/>`_
+    command to manage your manual installations, the following takes care of the
+    above for you (use with :file:`dest=~/.local/stow`)::
+
+        cd ~/.local/stow
+        stow -v kitty.app
 
 
 Customizing the installation

docs/build.rst

@@ -6,23 +6,34 @@ Building kitty from source
   :target: https://github.com/kovidgoyal/kitty/actions?query=workflow%3ACI
 
 
-|kitty| is designed to run from source, for easy hackability. Make sure
+|kitty| is designed to run from source, for easy hack-ability. Make sure
 the following dependencies are installed first.
 
+
+.. note::
+   If you are making small changes only to the python parts of kitty, there is no need to
+   build kitty at all, instead, assuming you have installed the official kitty
+   binaries, you can simply set the KITTY_DEVELOP_FROM enviroment variable to
+   point to the directory into which you have checked out the kitty source
+   code. kitty will then load its python code from there. You should use a
+   version of the source that matches the binary version as closely as
+   possible, since the two are tightly coupled.
+
+
 Dependencies
 ----------------
 
 Run-time dependencies:
 
-    * python >= 3.5
-    * harfbuzz >= 1.5.0
+    * python >= 3.6
+    * harfbuzz >= 2.2.0
     * zlib
     * libpng
     * liblcms2
     * freetype (not needed on macOS)
     * fontconfig (not needed on macOS)
     * libcanberra (not needed on macOS)
-    * ImageMagick (optional, needed to use the ``kitty icat`` tool to display images in the terminal)
+    * ImageMagick (optional, needed to use the ``kitty +kitten icat`` tool to display images in the terminal)
     * pygments (optional, need for syntax highlighting in ``kitty +kitten diff``)
 
 Build-time dependencies:
@@ -67,8 +78,7 @@ you might have to rebuild the app.
 .. note::
    The released :file:`kitty.dmg` includes all dependencies, unlike the
    :file:`kitty.app` built above and is built automatically by using the
-   :file:`kitty` branch of `build-calibre
-   <https://github.com/kovidgoyal/build-calibre>`_ however, that is designed to
+   <https://github.com/kovidgoyal/bypy>`_ however, that is designed to
    run on Linux and is not for the faint of heart.
 
 

docs/changelog.rst

@@ -4,6 +4,176 @@ Changelog
 |kitty| is a feature-rich, cross-platform, *fast*, GPU based terminal.
 To update |kitty|, :doc:`follow the instructions <binary>`.
 
+0.21.2 [2021-06-28]
+----------------------
+
+- A new :opt:`adjust_baseline` option to adjust the vertical alignment of text
+  inside a line (:pull:`3734`)
+
+- A new :opt:`url_excluded_characters` option to exclude additional characters
+  when detecting URLs under the mouse (:pull:`3738`)
+
+- Fix a regression in 0.21.0 that broke rendering of private use Unicode symbols followed
+  by spaces, when they also exist not followed by spaces (:iss:`3729`)
+
+- ssh kitten: Support systems where the login shell is a non-POSIX shell
+  (:iss:`3405`)
+
+- ssh kitten: Add completion (:iss:`3760`)
+
+- ssh kitten: Fix "Connection closed" message being printed by ssh when running
+  remote commands
+
+- Add support for the XTVERSION escape code
+
+- macOS: Fix a regression in 0.21.0 that broke middle-click to paste from clipboard (:iss:`3730`)
+
+- macOS: Fix shortcuts in the global menu bar responding slowly when cursor blink
+  is disabled/timed out (:iss:`3693`)
+
+- When displaying scrollback ensure that the window does not quit if the amount
+  of scrollback is less than a screen and the user has the ``--quit-if-one-screen``
+  option enabled for less (:iss:`3740`)
+
+- Linux: Fix Emoji/bitmapped fonts not use able in symbol_map
+
+- query terminal kitten: Allow querying font face and size information
+  (:iss:`3756`)
+
+- hyperlinked grep kitten: Fix context options not generating contextual output (:iss:`3759`)
+
+- Allow using superscripts in tab titles (:iss:`3763`)
+
+- Unicode input kitten: Fix searching when a word has more than 1024 matches (:iss:`3773`)
+
+
+0.21.1 [2021-06-14]
+----------------------
+
+- macOS: Fix a regression in the previous release that broke rendering of
+  strikeout (:iss:`3717`)
+
+- macOS: Fix a crash when rendering ligatures larger than 128 characters
+  (:iss:`3724`)
+
+- Fix a regression in the previous release that could cause a crash when
+  changing layouts and mousing (:iss:`3713`)
+
+
+0.21.0 [2021-06-12]
+----------------------
+
+- Allow reloading the :file:`kitty.conf` config file by pressing
+  :sc:`reload_config_file`. (:iss:`1292`)
+
+- Allow clicking URLs to open them without needing to also hold
+  :kbd:`ctrl+shift`
+
+- Allow remapping all mouse button press/release events to perform arbitrary
+  actions. :ref:`See details <conf-kitty-mouse.mousemap>` (:iss:`1033`)
+
+- Support infinite length ligatures (:iss:`3504`)
+
+- **Backward incompatibility**: The options to control which modifiers keys to
+  press for various mouse actions have been removed, if you used these options,
+  you will need to replace them with configuration using the new
+  :ref:`mouse actions framework <conf-kitty-mouse.mousemap>` as they will be
+  ignored. The options were: ``terminal_select_modifiers``,
+  ``rectangle_select_modifiers`` and ``open_url_modifiers``.
+
+- Add a configurable mouse action (:kbd:`ctrl+alt+triplepress` to select from the
+  clicked point to the end of the line. (:iss:`3585`)
+
+- Add the ability to un-scroll the screen to the ``kitty @ scroll-window``
+  remote control command (:iss:`3604`)
+
+- A new option, :opt:`tab_bar_margin_height` to add margins around the
+  top and bottom edges of the tab bar (:iss:`3247`)
+
+- Unicode input kitten: Fix a regression in 0.20.0 that broke keyboard handling
+  when the NumLock or CapsLock modifiers were engaged. (:iss:`3587`)
+
+- Fix a regression in 0.20.0 that sent incorrect bytes for the :kbd:`F1-F4` keys
+  in rmkx mode (:iss:`3586`)
+
+- macOS: When the Apple Color Emoji font lacks an emoji glyph search for it in other
+  installed fonts (:iss:`3591`)
+
+- macOS: Fix rendering getting stuck on some machines after sleep/screensaver
+  (:iss:`2016`)
+
+- macOS: Add a new ``Shell`` menu to the global menubar with some commonly used
+  actions (:pull:`3653`)
+
+- macOS: Fix the baseline for text not matching other CoreText based
+  applications for some fonts (:iss:`2022`)
+
+- Add a few more special commandline arguments for the launch command. Now all
+  ``KITTY_PIPE_DATA`` is also available via command line argument substitution
+  (:iss:`3593`)
+
+- Fix dynamically changing the background color in a window causing rendering
+  artifacts in the tab bar (:iss:`3595`)
+
+- Fix passing STDIN to launched background processes causing them to not inherit
+  environment variables (:pull:`3603`)
+
+- Fix deleting windows that are not the last window via remote control leaving
+  no window focused (:iss:`3619`)
+
+- Add an option :option:`kitty @ get-text --add-cursor` to also get the current
+  cursor position and state as ANSI escape codes (:iss:`3625`)
+
+- Add an option :option:`kitty @ get-text --add-wrap-markers` to add line wrap
+  markers to the output (:pull:`3633`)
+
+- Improve rendering of curly underlines on HiDPI screens (:pull:`3637`)
+
+- ssh kitten: Mimic behavior of ssh command line client more closely by
+  executing any command specified on the command line via the users' shell
+  just as ssh does (:iss:`3638`)
+
+- Fix trailing parentheses in URLs not being detected (:iss:`3688`)
+
+- Tab bar: Use a lower contrast color for tab separators (:pull:`3666`)
+
+- Fix a regression that caused using the ``title`` command in session files
+  to stop working (:iss:`3676`)
+
+- macOS: Fix a rare crash on exit (:iss:`3686`)
+
+- Fix ligatures not working with the `Iosevka
+  <https://github.com/be5invis/Iosevka>`_ font (requires Iosevka >= 7.0.4)
+  (:iss:`297`)
+
+- Remote control: Allow matching tabs by index number in currently active OS
+  Window (:iss:`3708`)
+
+- ssh kitten: Fix non-standard properties in terminfo such as the ones used for
+  true color not being copied (:iss:`312`)
+
+
+0.20.3 [2021-05-06]
+----------------------
+
+- macOS: Distribute universal binaries with both ARM and Intel architectures
+
+- A new ``show_key`` kitten to easily see the bytes generated by the terminal
+  for key presses in the various keyboard modes (:pull:`3556`)
+
+- Linux: Fix keyboard layout change keys defined via compose rules not being
+  ignored
+
+- macOS: Fix Spotlight search of global menu not working in non-English locales
+  (:pull:`3567`)
+
+- Fix tab activity symbol not appearing if no other changes happen in tab bar even when
+  there is activity in a tab (:iss:`3571`)
+
+- Fix focus changes not being sent to windows when focused window changes
+  because of the previously focused window being closed (:iss:`3571`)
+
+
 0.20.2 [2021-04-28]
 ----------------------
 
@@ -659,7 +829,7 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
   beam and underline cursors (:iss:`2337` and :pull:`2342`)
 
 - When the application running in the terminal grabs the mouse, pass middle
-  clicks to the application unless :opt:`terminal_select_modifiers` are
+  clicks to the application unless `terminal_select_modifiers` are
   pressed (:iss:`2368`)
 
 - A new ``copy_and_clear_or_interrupt`` function (:iss:`2403`)
@@ -767,7 +937,7 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
 - macOS: Add a :code:`titlebar-only` setting to
   :opt:`hide_window_decorations` to only hide the title bar (:pull:`2286`)
 
-- Fix a segfault when using :option:`kitty --debug-config` with maps (:iss:`2270`)
+- Fix a segfault when using ``--debug-config`` with maps (:iss:`2270`)
 
 - ``goto_tab`` now maps numbers larger than the last tab to the last tab
   (:iss:`2291`)
@@ -939,7 +1109,7 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
   the mouse pointer shape when the terminal programs grabs the pointer
   (:iss:`1808`)
 
-- Add an option :opt:`terminal_select_modifiers` to control which modifiers
+- Add an option `terminal_select_modifiers` to control which modifiers
   are used to override mouse selection even when a terminal application has
   grabbed the mouse (:iss:`1774`)
 
@@ -1643,8 +1813,8 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
 
 - Support "-T" as an alias for "--title" (:pull:`659`)
 
-- Fix a regression in the previous release that broke using :option:`kitty
-  --debug-config` with custom key mappings (:iss:`695`)
+- Fix a regression in the previous release that broke using
+  ``--debug-config`` with custom key mappings (:iss:`695`)
 
 
 0.11.1 [2018-06-17]

docs/conf.py

@@ -13,10 +13,7 @@ import subprocess
 import sys
 import time
 from functools import partial
-from typing import (
-    Any, Callable, Dict, Iterable, List, Match, Optional, Sequence, Tuple,
-    Union
-)
+from typing import Any, Callable, Dict, Iterable, List, Match, Optional, Tuple
 
 from docutils import nodes
 from docutils.parsers.rst.roles import set_classes
@@ -32,10 +29,9 @@ kitty_src = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if kitty_src not in sys.path:
     sys.path.insert(0, kitty_src)
 
-from kitty.conf.definition import Option, Shortcut  # noqa
+from kitty.conf.types import Definition  # noqa
 from kitty.constants import str_version  # noqa
 
-
 # config {{{
 # -- Project information -----------------------------------------------------
 
@@ -248,8 +244,8 @@ def add_html_context(app: Any, pagename: str, templatename: str, context: Any, *
 
 # CLI docs {{{
 def write_cli_docs(all_kitten_names: Iterable[str]) -> None:
-    from kitty.launch import options_spec as launch_options_spec
     from kitty.cli import option_spec_as_rst
+    from kitty.launch import options_spec as launch_options_spec
     with open('generated/launch.rst', 'w') as f:
         f.write(option_spec_as_rst(
             appname='launch', ospec=launch_options_spec, heading_char='_',
@@ -264,7 +260,7 @@ if you specify a program-to-run you can use the special placeholder
             'kitty --to', 'kitty @ --to'))
     as_rst = partial(option_spec_as_rst, heading_char='_')
     from kitty.rc.base import all_command_names, command_for_name
-    from kitty.remote_control import global_options_spec, cli_msg
+    from kitty.remote_control import cli_msg, global_options_spec
     with open('generated/cli-kitty-at.rst', 'w') as f:
         p = partial(print, file=f)
         p('kitty @\n' + '-' * 80)
@@ -293,7 +289,9 @@ if you specify a program-to-run you can use the special placeholder
 
 
 def write_remote_control_protocol_docs() -> None:  # {{{
-    from kitty.rc.base import all_command_names, command_for_name, RemoteCommand
+    from kitty.rc.base import (
+        RemoteCommand, all_command_names, command_for_name
+    )
     field_pat = re.compile(r'\s*([a-zA-Z0-9_+]+)\s*:\s*(.+)')
 
     def format_cmd(p: Callable, name: str, cmd: RemoteCommand) -> None:
@@ -390,12 +388,14 @@ class SessionLexer(RegexLexer):
 
 
 def link_role(name: str, rawtext: str, text: str, lineno: int, inliner: Any, options: Any = {}, content: Any = []) -> Tuple[List, List]:
+    text = text.replace('\n', ' ')
     m = re.match(r'(.+)\s+<(.+?)>', text)
     if m is None:
         msg = inliner.reporter.error(f'link "{text}" not recognized', line=lineno)
         prb = inliner.problematic(rawtext, rawtext, msg)
         return [prb], [msg]
     text, url = m.group(1, 2)
+    url = url.replace(' ', '')
     set_classes(options)
     node = nodes.reference(rawtext, text, refuri=url, **options)
     return [node], []
@@ -447,87 +447,6 @@ def parse_shortcut_node(env: Any, sig: str, signode: Any) -> str:
     return sig
 
 
-def render_conf(conf_name: str, all_options: Iterable[Union['Option', Sequence['Shortcut']]]) -> str:
-    from kitty.conf.definition import merged_opts, Option, Group
-    ans = ['.. default-domain:: conf', '']
-    a = ans.append
-    current_group: Optional[Group] = None
-    all_options_ = list(all_options)
-    kitty_mod = 'kitty_mod'
-
-    def render_group(group: Group) -> None:
-        a('')
-        a(f'.. _conf-{conf_name}-{group.name}:')
-        a('')
-        a(group.short_text)
-        heading_level = '+' if '.' in group.name else '^'
-        a(heading_level * (len(group.short_text) + 20))
-        a('')
-        if group.start_text:
-            a(group.start_text)
-            a('')
-
-    def handle_group_end(group: Group) -> None:
-        if group.end_text:
-            assert current_group is not None
-            a(''), a(current_group.end_text)
-
-    def handle_group(new_group: Group, new_group_is_shortcut: bool = False) -> None:
-        nonlocal current_group
-        if new_group is not current_group:
-            if current_group:
-                handle_group_end(current_group)
-            current_group = new_group
-            render_group(current_group)
-
-    def handle_option(i: int, opt: Option) -> None:
-        nonlocal kitty_mod
-        if not opt.long_text or not opt.add_to_docs:
-            return
-        handle_group(opt.group)
-        if opt.name == 'kitty_mod':
-            kitty_mod = opt.defval_as_string
-        mopts = list(merged_opts(all_options_, opt, i))
-        a('.. opt:: ' + ', '.join(conf_name + '.' + mo.name for mo in mopts))
-        a('.. code-block:: conf')
-        a('')
-        sz = max(len(x.name) for x in mopts)
-        for mo in mopts:
-            a(('    {:%ds} {}' % sz).format(mo.name, mo.defval_as_string))
-        a('')
-        if opt.long_text:
-            a(expand_opt_references(conf_name, opt.long_text))
-            a('')
-
-    def handle_shortcuts(shortcuts: Sequence[Shortcut]) -> None:
-        sc = shortcuts[0]
-        handle_group(sc.group, True)
-        sc_text = f'{conf_name}.{sc.short_text}'
-        a('.. shortcut:: ' + sc_text)
-        shortcuts = [s for s in shortcuts if s.add_to_default]
-        shortcut_slugs[f'{conf_name}.{sc.name}'] = (sc_text, sc.key.replace('kitty_mod', kitty_mod))
-        if shortcuts:
-            a('.. code-block:: conf')
-            a('')
-            for x in shortcuts:
-                if x.add_to_default:
-                    a('    map {} {}'.format(x.key.replace('kitty_mod', kitty_mod), x.action_def))
-        a('')
-        if sc.long_text:
-            a(expand_opt_references(conf_name, sc.long_text))
-            a('')
-
-    for i, opt in enumerate(all_options_):
-        if isinstance(opt, Option):
-            handle_option(i, opt)
-        else:
-            handle_shortcuts(opt)
-
-    if current_group:
-        handle_group_end(current_group)
-    return '\n'.join(ans)
-
-
 def process_opt_link(env: Any, refnode: Any, has_explicit_title: bool, title: str, target: str) -> Tuple[str, str]:
     conf_name, opt = target.partition('.')[::2]
     if not opt:
@@ -571,26 +490,26 @@ def write_conf_docs(app: Any, all_kitten_names: Iterable[str]) -> None:
     sc_role = app.registry.domain_roles['std']['sc']
     sc_role.warn_dangling = True
     sc_role.process_link = process_shortcut_link
+    shortcut_slugs.clear()
 
-    def generate_default_config(all_options: Dict[str, Union[Option, Sequence[Shortcut]]], name: str) -> None:
-        from kitty.conf.definition import as_conf_file
+    def generate_default_config(definition: Definition, name: str) -> None:
         with open(f'generated/conf-{name}.rst', 'w', encoding='utf-8') as f:
             print('.. highlight:: conf\n', file=f)
-            f.write(render_conf(name, all_options.values()))
+            f.write('\n'.join(definition.as_rst(name, shortcut_slugs)))
 
         conf_name = re.sub(r'^kitten-', '', name) + '.conf'
         with open(f'generated/conf/{conf_name}', 'w', encoding='utf-8') as f:
-            text = '\n'.join(as_conf_file(all_options.values()))
+            text = '\n'.join(definition.as_conf())
             print(text, file=f)
 
-    from kitty.config_data import all_options
-    generate_default_config(all_options, 'kitty')
+    from kitty.options.definition import definition
+    generate_default_config(definition, 'kitty')
 
     from kittens.runner import get_kitten_conf_docs
     for kitten in all_kitten_names:
-        all_options = get_kitten_conf_docs(kitten)
-        if all_options:
-            generate_default_config(all_options, f'kitten-{kitten}')
+        definition = get_kitten_conf_docs(kitten)
+        if definition:
+            generate_default_config(definition, f'kitten-{kitten}')
 # }}}
 
 

docs/conf.rst

@@ -5,13 +5,14 @@ Configuring kitty
 
 .. highlight:: conf
 
-|kitty| is highly customizable, everything from keyboard shortcuts, to painting
-frames-per-second. See below for an overview of all customization
+|kitty| is highly customizable, everything from keyboard shortcuts, to
+rendering frames-per-second. See below for an overview of all customization
 possibilities.
 
 You can open the config file within kitty by pressing :sc:`edit_config_file`.
-You can also display the current configuration by running ``kitty
---debug-config``.
+You can reload the config file within kitty by pressing
+:sc:`reload_config_file` or sending kitty the ``SIGUSR1`` signal.  You can also
+display the current configuration by pressing the :sc:`debug_config` key.
 
 .. _confloc:
 

docs/faq.rst

@@ -4,6 +4,7 @@ Frequently Asked Questions
 .. highlight:: sh
 
 .. contents::
+   :local:
 
 Some special symbols are rendered small/truncated in kitty?
 -----------------------------------------------------------
@@ -53,8 +54,11 @@ type it each time::
 
     alias ssh="kitty +kitten ssh"
 
+Remember to also setup :ref:`completion`.
+
 If for some reason that does not work (typically because the server is using a
-non POSIX compliant shell), you can try using it with python instead::
+non POSIX compliant shell as ``/bin/sh``), you can try using it with ``python``
+instead::
 
     kitty +kitten ssh use-python myserver
 
@@ -63,7 +67,7 @@ server, use the following one-liner instead (it
 is slower as it needs to ssh into the server twice, but will work with most
 servers)::
 
-    infocmp xterm-kitty | ssh myserver tic -x -o \~/.terminfo /dev/stdin
+    infocmp -a xterm-kitty | ssh myserver tic -x -o \~/.terminfo /dev/stdin
 
 If you are behind a proxy (like Balabit) that prevents this, you must redirect the
 1st command to a file, copy that to the server and run ``tic`` manually.  If you
@@ -169,6 +173,19 @@ You can, of course, also run |kitty| from a terminal with command line options,
 And within |kitty| itself, you can always run |kitty| using just `kitty` as it
 cleverly adds itself to the ``PATH``.
 
+I catted a binary file and now kitty is hung?
+-----------------------------------------------
+
+**Never** output unknown binary data directly into a terminal.
+
+Terminals have a single channel for both data and control. Certain bytes
+are control codes. Some of these control codes are of arbitrary length, so
+if the binary data you output into the terminal happens to contain the starting
+sequence for one of these control codes, the terminal will hang waiting for
+the closing sequence. Press :kbd:`ctrl+shift+delete` to reset the terminal.
+
+If you do want to cat unknown data, use ``cat -v``.
+
 
 kitty is not able to use my favorite font?
 ---------------------------------------------
@@ -226,26 +243,14 @@ How do I map key presses in kitty to different keys in the terminal program?
 This is accomplished by using ``map`` with :sc:`send_text <send_text>` in :file:`kitty.conf`.
 For example::
 
-    map alt+s send_text all \x13
+    map alt+s send_text normal,application \x13
 
 This maps :kbd:`alt+s` to :kbd:`ctrl+s`. To figure out what bytes to use for
-the :sc:`send_text <send_text>` you can use the ``showkey`` utility. Run::
+the :sc:`send_text <send_text>` you can use the ``show_key`` kitten. Run::
 
-    showkey -a
+    kitty +kitten show_key
 
-Then press the key you want to emulate. On macOS, this utility is currently not
-available. The manual way to figure it out is:
-
-    1. Look up your key's decimal value in the table at the bottom of `this
-       page <http://ascii-table.com/ansi-escape-sequences.php>`_ or any
-       ANSI escape sequence table. There are different modifiers for :kbd:`ctrl`,
-       :kbd:`alt`, etc. For e.g., for :kbd:`ctrl+s`, find the ``S`` row and look at
-       the third column value, ``19``.
-
-    2. Convert the decimal value to hex with ``kitty +runpy "print(hex(19))"``.
-       This shows the hex value, ``13`` in this case.
-
-    3. Use ``\x(hexval)`` in your ``send_text`` command in kitty. So in this example, ``\x13``
+Then press the key you want to emulate.
 
 How do I open a new window or tab with the same working directory as the current window?
 --------------------------------------------------------------------------------------------

docs/graphics-protocol.rst

@@ -46,6 +46,7 @@ Some programs and libraries that use the kitty graphics protocol:
 
 
 .. contents::
+   :local:
 
 
 Getting the window size
@@ -100,7 +101,8 @@ features of the graphics protocol:
    import sys
    from base64 import standard_b64encode
 
-   def serialize_gr_command(cmd, payload=None):
+   def serialize_gr_command(**cmd):
+      payload = cmd.pop('payload', None)
       cmd = ','.join('{}={}'.format(k, v) for k, v in cmd.items())
       ans = []
       w = ans.append
@@ -111,18 +113,17 @@ features of the graphics protocol:
       w(b'\033\\')
       return b''.join(ans)
 
-   def write_chunked(cmd, data):
-      data = standard_b64encode(data)
+   def write_chunked(**cmd):
+      data = standard_b64encode(cmd.pop('data'))
       while data:
          chunk, data = data[:4096], data[4096:]
          m = 1 if data else 0
-         cmd['m'] = m
-         sys.stdout.buffer.write(serialize_gr_command(cmd, chunk))
+         sys.stdout.buffer.write(serialize_gr_command(payload=chunk, m=m, **cmd))
          sys.stdout.flush()
          cmd.clear()
 
    with open(sys.argv[-1], 'rb') as f:
-      write_chunked({'a': 'T', 'f': 100}, f.read())
+      write_chunked(a='T', f=100, data=f.read())
 
 
 Save this script as :file:`png.py`, then you can use it to display any PNG
@@ -225,8 +226,12 @@ Value of `t`          Meaning
                       is in a known temporary directory, such as :file:`/tmp`,
                       :file:`/dev/shm`, :file:`TMPDIR env var if present` and any platform
                       specific temporary directories.
-``s``                 A `POSIX shared memory object <http://man7.org/linux/man-pages/man7/shm_overview.7.html>`_.
-                      The terminal emulator will delete it after reading the pixel data
+``s``                 A *shared memory object*, which on POSIX systems is a `POSIX shared memory object
+                      <http://man7.org/linux/man-pages/man7/shm_overview.7.html>`_ and on Windows is a
+                      `Named shared memory object <https://docs.microsoft.com/en-us/windows/win32/memory/creating-named-shared-memory>`_.
+                      The terminal emulator must read the data from the memory
+                      object and then unlink and close it on POSIX and just
+                      close it on Windows.
 ==================    ============
 
 Local client
@@ -311,7 +316,7 @@ While as of May 2020, kitty is the only terminal emulator to support this
 graphics protocol, we intend that any terminal emulator that wishes to support
 it can. To check if a terminal emulator supports the graphics protocol the best way
 is to send the above *query action* followed by a request for the
-`primary device attributes <https://vt100.net/docs/vt510-rm/DA1.html>`. If you
+`primary device attributes <https://vt100.net/docs/vt510-rm/DA1.html>`_. If you
 get back an answer for the device attributes without getting back an answer for
 the *query action* the terminal emulator does not support the graphics
 protocol.
@@ -471,7 +476,7 @@ Requesting image ids from the terminal
 If you are writing a program that is going to share the screen with other
 programs and you still want to use image ids, it is not possible to know
 what image ids are free to use. In this case, instead of using the ``i``
-key to specify and image id use the ``I`` key to specify and image number
+key to specify an image id use the ``I`` key to specify an image number
 instead. These numbers are not unique.
 When creating a new image, even if an existing image has the same number a new
 one is created. And the terminal will reply with the id of the newly created

docs/index.rst

@@ -49,35 +49,34 @@ kitty - the fast, featureful, GPU based terminal emulator
     Screenshot, showing vim, tig and git running in |kitty| with the 'Tall' layout
 
 
+
+.. contents::
+   :local:
+   :depth: 1
+
+
 .. _quickstart:
 
 Quickstart
 --------------
 
 Pre-built binaries of |kitty| are available for both macOS and Linux.
-See the :doc:`binary install instructions </binary>`. You can
+See the :doc:`binary install instructions </binary>`. You can also
 :doc:`build from source </build>`.
 
-You can also use your favorite package manager to install the |kitty| package.
-|kitty| packages are available for:
-`macOS with Homebrew (Cask) <https://formulae.brew.sh/cask/kitty>`_,
-`macOS and Linux with Nix <https://search.nixos.org/packages?channel=unstable&show=kitty&sort=relevance&query=kitty>`_,
-`Ubuntu <https://launchpad.net/ubuntu/+source/kitty>`_,
-`Debian <https://packages.debian.org/buster/kitty>`_,
-`openSUSE <https://build.opensuse.org/package/show/X11:terminals/kitty>`_,
-`Arch Linux <https://www.archlinux.org/packages/community/x86_64/kitty/>`_,
-`Gentoo <https://packages.gentoo.org/packages/x11-terms/kitty>`_,
-`Fedora <https://src.fedoraproject.org/rpms/kitty>`_,
-`Void Linux <https://github.com/void-linux/void-packages/blob/master/srcpkgs/kitty/template>`_,
-and `Solus <https://dev.getsol.us/source/kitty/>`_.
+Additionally, you can use your favorite package manager to install the |kitty|
+package, but note that some Linux distribution packages are woefully outdated.
+|kitty| is available in a vast number of package repositories for macOS
+and Linux.
+
+.. image:: https://repology.org/badge/tiny-repos/kitty.svg
+   :target: https://repology.org/project/kitty/versions
+   :alt: Number of repositories kitty is available in
 
 See :doc:`Configuring kitty <conf>` for help on configuring |kitty| and
 :doc:`Invocation <invocation>` for the command line arguments |kitty| supports.
 
 
-.. contents::
-
-
 Design philosophy
 -------------------
 
@@ -109,7 +108,7 @@ Layouts
 A layout is an arrangement of multiple kitty *windows* inside a top-level OS window. You can create a new window
 using the :sc:`new_window` key combination.
 
-Currently, there are six layouts available:
+Currently, there are seven layouts available:
 
 * **Fat** -- One (or optionally more) windows are shown full width on the top, the rest of the windows are shown side-by-side on the bottom
 * **Grid** -- All windows are shown in a grid
@@ -222,8 +221,7 @@ For example:
     # vim in it
     launch --env FOO=BAR vim
     # Set the title for the next window
-    title Chat with x
-    launch irssi --profile x
+    launch --title "Chat with x" irssi --profile x
 
     # Create a new tab (the part after new_tab is the optional tab
     # name which will be displayed in the tab bar, if omitted, the
@@ -255,17 +253,23 @@ For example:
 Mouse features
 -------------------
 
-* You can hold down :kbd:`ctrl+shift` and click on a URL to open it in a browser.
+* You can click on a URL to open it in a browser.
 * You can double click to select a word and then drag to select more words.
 * You can triple click to select a line and then drag to select more lines.
+* You can triple click while holding :kbd:`ctrl+alt` to select from clicked
+  point to end of line.
 * You can right click to extend a previous selection.
 * You can hold down :kbd:`ctrl+alt` and drag with the mouse to select in
-  columns (see also :opt:`rectangle_select_modifiers`).
+  columns.
 * Selecting text automatically copies it to the primary clipboard (on
   platforms with a primary clipboard).
+* You can middle click to paste from the primary clipboard (on platforms
+  with a primary clipboard).
 * You can select text with kitty even when a terminal program has grabbed
-  the mouse by holding down the :kbd:`shift` key (see also
-  :opt:`terminal_select_modifiers`).
+  the mouse by holding down the :kbd:`shift` key.
+
+All these actions can be customized in :file:`kitty.conf` as described
+:ref:`here <conf-kitty-mouse.mousemap>`.
 
 
 Font control
@@ -341,8 +345,6 @@ Frequently Asked Questions
 
 The list of Frequently Asked Questions (*FAQ*) is :doc:`available here <faq>`.
 
-.. _completion:
-
 
 Cool integrations for kitty with other CLI tools
 --------------------------------------------------
@@ -352,6 +354,8 @@ kitty provides extremely powerful interfaces such as :doc:`remote-control` and
 that allow it to be integrated with other tools seamlessly. For a list of such
 user created integrations, see: :doc:`integrations`.
 
+.. _completion:
+
 Completion for kitty
 ---------------------------------
 

docs/integrations.rst

@@ -59,6 +59,10 @@ System and data visualisation tools
 `KittyTerminalImages.jl <https://github.com/simonschoelly/KittyTerminalImages.jl>`_
     show images from Julia directly in kitty
 
+`euporie <https://github.com/joouha/euporie>`_
+    a text-based user interface for running and editing Jupyter notebooks,
+    powered by kitty's graphics protocol for displaying plots
+
 `gnuplot <http://www.gnuplot.info/>`_
     a graphing and data visualization tool that can be made to display its
     output in kitty with the following bash snippet::
@@ -82,6 +86,10 @@ System and data visualisation tools
 `onefetch <https://github.com/o2sh/onefetch>`_
     a tool to fetch information about your git repositories
 
+`patat <https://github.com/jaspervdj/patat>`_
+    terminal based presentations using pandoc and kitty's image protocol for
+    images
+
 `wttr.in <https://github.com/chubin/wttr.in>`_
     a tool to display weather information in your terminal with curl
 
@@ -111,6 +119,9 @@ such a split windows, previews, REPLs etc.
 `vim-test <https://github.com/vim-test/vim-test>`_
     allows easily running tests in a terminal window
 
+`hologram.nvim <https://github.com/edluffy/hologram.nvim>`_
+    terminal image viewer for nvim
+
 
 Scrollback manipulation
 -------------------------

docs/keyboard-protocol.rst

@@ -28,7 +28,7 @@ issues in that proposal, listed at the :ref:`bottom of this document
 
 You can see this protocol with all enhancements in action by running::
 
-    kitty +kitten key_demo
+    kitty +kitten show_key -m kitty
 
 inside the kitty terminal to report key events.
 
@@ -41,12 +41,14 @@ If you are an application or library developer just interested in using this
 protocol to make keyboard handling simpler and more robust in your application,
 without too many changes, do the following:
 
-#. Emit the escape code ``CSI > 1 u`` at application startup or when entering
-   alternate screen mode
+#. Emit the escape code ``CSI > 1 u`` at application startup if using the main
+   screen or when entering alternate screen mode, if using the alternate
+   screen.
 #. All key events will now be sent in only a few forms to your application,
    that are easy to parse unambiguously.
-#. Emit the escape sequence ``CSI < u`` at application exit or just before
-   leaving alternate screen mode to restore the previously used keyboard mode.
+#. Emit the escape sequence ``CSI < u`` at application exit if using the main
+   screen or just before leaving alternate screen mode if using the alternate screen,
+   to restore the previously used keyboard mode.
 
 Key events will all be delivered to your application either as plain UTF-8
 text, or using the following escape codes, for those keys that do not produce
@@ -110,7 +112,7 @@ decimal number. For example, the :kbd:`A` key is represented as ``97`` which is
 the unicode code for lowercase ``a``. Note that the codepoint used is *always*
 the lower-case (or more technically, un-shifted) version of the key. If the
 user presses, for example, :kbd:`ctrl+shift+a` the escape code would be ``CSI
-97;modifiers u``. It *must not* by ``CSI 65; modifiers u``.
+97;modifiers u``. It *must not* be ``CSI 65; modifiers u``.
 
 If *alternate key reporting* is requested by the program running in the
 terminal, the terminal can send two additional Unicode codepoints, the
@@ -266,6 +268,15 @@ and alternate screens. If a pop request is received that empties the stack,
 all flags are reset. If a push request is received and the stack is full, the
 oldest entry from the stack must be evicted.
 
+.. note:: The main and alternate screens in the terminal emulator must maintain
+   their own, independent, keyboard mode stacks. This is so that a program that
+   uses the alternate screen such as an editor, can change the keyboard mode
+   in the alternate screen only, without affecting the mode in the main screen
+   or even knowing what that mode is. Without this, and if no stack is
+   implemented for keyboard modes (such as in some legacy terminal emulators)
+   the editor would have to somehow know what the keyboard mode of the main
+   screen is and restore to that mode on exit.
+
 .. note:: In the interests of interoperation, the XTerm specific sequences
    `CSI > 4; 1 m` and `CSI > 4; 0 m` are treated as `CSI > 1 u` and `CSI < 1 u`.
    These codes cause XTerm to use the CSI u encoding for more keys and are therefore
@@ -296,7 +307,7 @@ represented in one of the following two forms::
 
 This makes it very easy to parse key events in an application. In particular,
 :kbd:`ctrl+c` will no longer generate the ``SIGINT`` signal, but instead be
-delivers as a ``CSI u`` escape code. This has the nice side effect of making it
+delivered as a ``CSI u`` escape code. This has the nice side effect of making it
 much easier to integrate into the application event loop. The only exceptions
 are the :kbd:`Enter, Tab and Backspace` keys which still generate the same
 bytes as in legacy mode this is to allow the user to type and execute commands

docs/kittens/custom.rst

@@ -185,6 +185,104 @@ The output of print statements will go to the ``STDOUT`` of the kitty process.
 So if you run kitty from another kitty instance, the output will be visible
 in the first kitty instance.
 
+Adding options to kittens
+----------------------------
+
+If you would like to use kitty's config framework to make your kittens
+configurable, you will need some boilerplate. In the directory
+of your kitten make the following files.
+
+:file:`kitten_options_definition.py`
+
+.. code-block:: python
+
+    from kitty.conf.types import Action, Definition
+
+    definition = Definition(
+        '!kitten_options_utils',
+        Action(
+            'map', 'parse_map',
+            {'key_definitions': 'kitty.conf.utils.KittensKeyMap'},
+            ['kitty.types.ParsedShortcut', 'kitty.conf.utils.KeyAction']
+        ),
+    )
+
+    agr = definition.add_group
+    egr = definition.end_group
+    opt = definition.add_option
+    map = definition.add_map
+
+    # main options {{{
+    agr('main', 'Main')
+
+    opt('some_option', '33',
+        option_type='some_option_parser',
+        long_text='''
+    Help text for this option
+    '''
+        )
+    egr()  # }}}
+
+    # shortcuts {{{
+    agr('shortcuts', 'Keyboard shortcuts')
+
+    map('Quit', 'quit q quit')
+    egr()  # }}}
+
+
+:file:`kitten_options_utils.py`
+
+.. code-block:: python
+
+    from kitty.conf.utils import KittensKeyDefinition, key_func, parse_kittens_key
+
+    func_with_args, args_funcs = key_func()
+    FuncArgsType = Tuple[str, Sequence[Any]]
+
+    def some_option_parser(val: str) -> int:
+        return int(val) + 3000
+
+    def parse_map(val: str) -> Iterable[KittensKeyDefinition]:
+        x = parse_kittens_key(val, args_funcs)
+        if x is not None:
+            yield x
+
+Then run::
+
+    kitty +runpy 'from kitty.conf.generate import main; main()' /path/to/kitten_options_definition.py
+
+You can parse and read the options in your kitten using the following code:
+
+.. code-block:: python
+
+    from .kitten_options_types import Options, defaults
+    from kitty.conf.utils import load_config as _load_config, parse_config_base
+    from typing import Optional, Iterable, Dict, Any
+
+    def load_config(*paths: str, overrides: Optional[Iterable[str]] = None) -> Options:
+        from .kitten_options_parse import  (
+            create_result_dict, merge_result_dicts, parse_conf_item
+        )
+
+        def parse_config(lines: Iterable[str]) -> Dict[str, Any]:
+            ans: Dict[str, Any] = create_result_dict()
+            parse_config_base(
+                lines,
+                parse_conf_item,
+                ans,
+            )
+            return ans
+
+        overrides = tuple(overrides) if overrides is not None else ()
+        opts_dict, paths = _load_config(defaults, parse_config, merge_result_dicts, *paths, overrides=overrides)
+        opts = Options(opts_dict)
+        opts.config_paths = paths
+        opts.config_overrides = overrides
+        return opts
+
+See the code for the builtin diff kitten for examples of creating more options
+and keyboard shortcuts.
+
 .. _external_kittens:
 
 Kittens created by kitty users

docs/kittens/diff.rst

@@ -26,6 +26,7 @@ Major Features
    Screenshot, showing a sample diff
 
 .. contents::
+   :local:
 
 
 Installation

docs/kittens/unicode-input.rst

@@ -24,7 +24,8 @@ also type a space followed by a period and the index for the match if you don't
 like to use arrow keys.
 
 You can switch between modes using either the function keys or by pressing
-:kbd:`Ctrl+[` and :kbd:`Ctrl+]`.
+:kbd:`Ctrl+[` and :kbd:`Ctrl+]` or by pressing :kbd:`Ctrl+Tab` and
+:kbd:`Ctrl+Shift+Tab`.
 
 
 Command Line Interface

docs/launch.rst

@@ -19,7 +19,6 @@ launch::
 
     map f1 launch vim path/to/some/file
 
-
 To open a new window with the same working directory as the currently
 active window::
 
@@ -29,6 +28,10 @@ To open the new window in a new tab::
 
     map f1 launch --type=tab
 
+To run multiple commands in a shell, use::
+
+    map f1 launch sh -c "ls && zsh"
+
 To pass the contents of the current screen and scrollback to the started process::
 
     map f1 launch --stdin-source=@screen_scrollback less
@@ -54,9 +57,34 @@ Special arguments
 -------------------
 
 There are a few special placeholder arguments that can be specified as part of
-the command line. Namely ``@selection`` which is replaced by the current
-selection and ``@active-kitty-window-id`` which is replaced by the id of the
-currently active kitty window. For example::
+the command line:
+
+
+``@selection``
+    replaced by the currently selected text
+
+``@active-kitty-window-id``
+    replaced by the id of the currently active kitty window
+
+``@line-count``
+    replaced by the number of lines in STDIN. Only present when passing some
+    data to STDIN
+
+``@input-line-number``
+    replaced the number of lines a pager should scroll to match the current
+    scroll position in kitty. See :opt:`scrollback_pager` for details
+
+``@scrolled-by``
+    replaced by the number of lines kitty is currently scrolled by
+
+``@cursor-x``
+    replaced by the current cursor x position with 1 being the leftmost cell
+
+``@cursor-y``
+    replaced by the current cursor y position with 1 being the topmost cell
+
+
+For example::
 
     map f1 launch my-program @active-kitty-window-id
 

docs/layouts.rst

@@ -28,6 +28,7 @@ are available use :opt:`enabled_layouts`, the first listed layout becomes
 the default. Individual layouts and how to use them are described below.
 
 .. contents::
+   :local:
 
 
 The Stack Layout

docs/open_actions.rst

@@ -63,7 +63,7 @@ Matching criteria
 ------------------
 
 An entry in :file:`open-actions.conf` must have one or more matching criteria.
-URLs that match all criteria for an entry will trigger that entries' actions.
+URLs that match all criteria for an entry will trigger that entry's actions.
 Processing stops at the first matching entry, so put more specific matching
 criteria at the start of the list. Entries in the file are separated by blank
 lines. The various available criteria are:

docs/protocol-extensions.rst

@@ -16,9 +16,11 @@ is to make it as easy to implement these protocol extensions as possible,
 thereby hopefully encouraging their widespread adoption.
 
 If you wish to discuss these extensions, propose additions/changes to them
-please do so by opening issues in the github bug tracker.
+please do so by opening issues in the `GitHub
+<https://github.com/kovidgoyal/kitty/issues>`_ bug tracker.
 
 .. contents::
+   :local:
 
 Colored and styled underlines
 -------------------------------
@@ -177,9 +179,7 @@ screen. This escape code allows that text to be restored.
 If the scrollback buffer is empty or there is no scrollback buffer, such as for
 the alternate screen, then the newly inserted lines must be empty, just as with
 the original ``SD`` escape code. The maximum number of lines that can be
-scrolled down is the height of the screen, i.e. the number of lines on the screen.
-If the escape code requests more than that number of lines, the terminal *must*
-clip the request to the current screen size.
+scrolled down is implementation defined, but must be at least one screen worth.
 
 The syntax of the escape code is identical to that of ``SD`` except that it has
 a trailing ``+`` modifier. This is legal under the `ECMA 48 standard

gen-config.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+from kitty.conf.generate import write_output
+
+
+def main() -> None:
+    from kitty.options.definition import definition
+    write_output('kitty', definition)
+    from kittens.diff.options.definition import definition as kd
+    write_output('kittens.diff', kd)
+
+
+if __name__ == '__main__':
+    main()

glfw/cocoa_monitor.m

@@ -47,7 +47,7 @@ static char* getDisplayName(CGDirectDisplayID displayID)
     io_service_t service;
     CFDictionaryRef info;
 
-    if (IOServiceGetMatchingServices(kIOMasterPortDefault,
+    if (IOServiceGetMatchingServices(kIOMainPortDefault,
                                      IOServiceMatching("IODisplayConnect"),
                                      &it) != 0)
     {
@@ -245,7 +245,7 @@ static double getFallbackRefreshRate(CGDirectDisplayID displayID)
     io_iterator_t it;
     io_service_t service;
 
-    if (IOServiceGetMatchingServices(kIOMasterPortDefault,
+    if (IOServiceGetMatchingServices(kIOMainPortDefault,
                                      IOServiceMatching("IOFramebuffer"),
                                      &it) != 0)
     {
@@ -316,6 +316,7 @@ void _glfwClearDisplayLinks() {
             CVDisplayLinkRelease(_glfw.ns.displayLinks.entries[i].displayLink);
             _glfw.ns.displayLinks.entries[i].displayLink = nil;
             _glfw.ns.displayLinks.entries[i].lastRenderFrameRequestedAt = 0;
+            _glfw.ns.displayLinks.entries[i].first_unserviced_render_frame_request_at = 0;
         }
     }
     _glfw.ns.displayLinks.count = 0;
@@ -333,7 +334,14 @@ static CVReturn displayLinkCallback(
     return kCVReturnSuccess;
 }
 
-static inline void createDisplayLink(CGDirectDisplayID displayID) {
+void
+_glfw_create_cv_display_link(_GLFWDisplayLinkNS *entry) {
+    CVDisplayLinkCreateWithCGDisplay(entry->displayID, &entry->displayLink);
+    CVDisplayLinkSetOutputCallback(entry->displayLink, &displayLinkCallback, (void*)(uintptr_t)entry->displayID);
+}
+
+static void
+createDisplayLink(CGDirectDisplayID displayID) {
     if (_glfw.ns.displayLinks.count >= sizeof(_glfw.ns.displayLinks.entries)/sizeof(_glfw.ns.displayLinks.entries[0]) - 1) return;
     for (size_t i = 0; i < _glfw.ns.displayLinks.count; i++) {
         if (_glfw.ns.displayLinks.entries[i].displayID == displayID) return;
@@ -341,8 +349,7 @@ static inline void createDisplayLink(CGDirectDisplayID displayID) {
     _GLFWDisplayLinkNS *entry = &_glfw.ns.displayLinks.entries[_glfw.ns.displayLinks.count++];
     memset(entry, 0, sizeof(_GLFWDisplayLinkNS));
     entry->displayID = displayID;
-    CVDisplayLinkCreateWithCGDisplay(displayID, &entry->displayLink);
-    CVDisplayLinkSetOutputCallback(entry->displayLink, &displayLinkCallback, (void*)(uintptr_t)displayID);
+    _glfw_create_cv_display_link(entry);
 }
 
 // Poll for changes in the set of connected monitors

glfw/cocoa_platform.h

@@ -39,6 +39,9 @@ typedef void* CVDisplayLinkRef;
 // NOTE: Many Cocoa enum values have been renamed and we need to build across
 //       SDK versions where one is unavailable or the other deprecated
 //       We use the newer names in code and these macros to handle compatibility
+#if (MAC_OS_X_VERSION_MAX_ALLOWED < 120000) // Before macOS 12 Monterey
+  #define kIOMainPortDefault kIOMasterPortDefault
+#endif
 #if MAC_OS_X_VERSION_MAX_ALLOWED < 101200
  #define NSBitmapFormatAlphaNonpremultiplied NSAlphaNonpremultipliedBitmapFormat
  #define NSEventMaskAny NSAnyEventMask
@@ -158,7 +161,7 @@ typedef struct _GLFWDisplayLinkNS
 {
     CVDisplayLinkRef displayLink;
     CGDirectDisplayID displayID;
-    monotonic_t lastRenderFrameRequestedAt;
+    monotonic_t lastRenderFrameRequestedAt, first_unserviced_render_frame_request_at;
 } _GLFWDisplayLinkNS;
 
 // Cocoa-specific global data
@@ -246,3 +249,4 @@ void _glfwDispatchTickCallback(void);
 void _glfwDispatchRenderFrame(CGDirectDisplayID);
 void _glfwShutdownCVDisplayLink(unsigned long long, void*);
 void _glfwCocoaPostEmptyEvent(void);
+void _glfw_create_cv_display_link(_GLFWDisplayLinkNS *entry);

glfw/cocoa_window.m

@@ -317,6 +317,7 @@ _glfwShutdownCVDisplayLink(unsigned long long timer_id UNUSED, void *user_data U
         _GLFWDisplayLinkNS *dl = &_glfw.ns.displayLinks.entries[i];
         if (dl->displayLink) CVDisplayLinkStop(dl->displayLink);
         dl->lastRenderFrameRequestedAt = 0;
+        dl->first_unserviced_render_frame_request_at = 0;
     }
 }
 
@@ -340,10 +341,22 @@ requestRenderFrame(_GLFWwindow *w, GLFWcocoarenderframefun callback) {
         _GLFWDisplayLinkNS *dl = &_glfw.ns.displayLinks.entries[i];
         if (dl->displayID == displayID) {
             dl->lastRenderFrameRequestedAt = now;
+            if (!dl->first_unserviced_render_frame_request_at) dl->first_unserviced_render_frame_request_at = now;
             if (!CVDisplayLinkIsRunning(dl->displayLink)) CVDisplayLinkStart(dl->displayLink);
+            else if (now - dl->first_unserviced_render_frame_request_at > s_to_monotonic_t(1ll)) {
+                // display link is stuck need to recreate it because Apple cant even
+                // get a simple timer right
+                CVDisplayLinkRelease(dl->displayLink); dl->displayLink = nil;
+                dl->first_unserviced_render_frame_request_at = now;
+                _glfw_create_cv_display_link(dl);
+                _glfwInputError(GLFW_PLATFORM_ERROR,
+                    "CVDisplayLink stuck possibly because of sleep/screensaver + Apple's incompetence, recreating.");
+                if (!CVDisplayLinkIsRunning(dl->displayLink)) CVDisplayLinkStart(dl->displayLink);
+            }
         } else if (dl->displayLink && dl->lastRenderFrameRequestedAt && now - dl->lastRenderFrameRequestedAt >= DISPLAY_LINK_SHUTDOWN_CHECK_INTERVAL) {
             CVDisplayLinkStop(dl->displayLink);
             dl->lastRenderFrameRequestedAt = 0;
+            dl->first_unserviced_render_frame_request_at = 0;
         }
     }
 }
@@ -2141,6 +2154,12 @@ _glfwDispatchRenderFrame(CGDirectDisplayID displayID) {
         }
         w = w->next;
     }
+    for (size_t i = 0; i < _glfw.ns.displayLinks.count; i++) {
+        _GLFWDisplayLinkNS *dl = &_glfw.ns.displayLinks.entries[i];
+        if (dl->displayID == displayID) {
+            dl->first_unserviced_render_frame_request_at = 0;
+        }
+    }
 }
 
 void _glfwPlatformGetCursorPos(_GLFWwindow* window, double* xpos, double* ypos)

glfw/glfw3.h

@@ -523,18 +523,20 @@ typedef enum {
  *
  *  @ingroup input
  *  @{ */
-#define GLFW_MOUSE_BUTTON_1         0
-#define GLFW_MOUSE_BUTTON_2         1
-#define GLFW_MOUSE_BUTTON_3         2
-#define GLFW_MOUSE_BUTTON_4         3
-#define GLFW_MOUSE_BUTTON_5         4
-#define GLFW_MOUSE_BUTTON_6         5
-#define GLFW_MOUSE_BUTTON_7         6
-#define GLFW_MOUSE_BUTTON_8         7
-#define GLFW_MOUSE_BUTTON_LAST      GLFW_MOUSE_BUTTON_8
-#define GLFW_MOUSE_BUTTON_LEFT      GLFW_MOUSE_BUTTON_1
-#define GLFW_MOUSE_BUTTON_RIGHT     GLFW_MOUSE_BUTTON_2
-#define GLFW_MOUSE_BUTTON_MIDDLE    GLFW_MOUSE_BUTTON_3
+typedef enum GLFWMouseButton {
+    GLFW_MOUSE_BUTTON_1 = 0,
+    GLFW_MOUSE_BUTTON_LEFT = 0,
+    GLFW_MOUSE_BUTTON_2 = 1,
+    GLFW_MOUSE_BUTTON_RIGHT = 1,
+    GLFW_MOUSE_BUTTON_3 = 2,
+    GLFW_MOUSE_BUTTON_MIDDLE = 2,
+    GLFW_MOUSE_BUTTON_4 = 3,
+    GLFW_MOUSE_BUTTON_5 = 4,
+    GLFW_MOUSE_BUTTON_6 = 5,
+    GLFW_MOUSE_BUTTON_7 = 6,
+    GLFW_MOUSE_BUTTON_8 = 7,
+    GLFW_MOUSE_BUTTON_LAST = 7
+} GLFWMouseButton;
 /*! @} */
 
 /*! @defgroup joysticks Joysticks

glfw/window.c

@@ -701,8 +701,8 @@ GLFWAPI void glfwSetWindowSizeIncrements(GLFWwindow* handle, int widthincr, int
 {
     _GLFWwindow* window = (_GLFWwindow*) handle;
     assert(window != NULL);
-    assert(widthincr >= 0);
-    assert(heightincr >= 0);
+    assert(widthincr >= 0 || widthincr == GLFW_DONT_CARE);
+    assert(heightincr >= 0 || heightincr == GLFW_DONT_CARE);
 
     _GLFW_REQUIRE_INIT();
 

glfw/xkb_glfw.c

@@ -702,6 +702,11 @@ glfw_xkb_key_from_ime(_GLFWIBUSKeyEvent *ev, bool handled_by_ime, bool failed) {
       last_handled_press_keycode = ev->glfw_ev.native_key;
 }
 
+static bool
+is_switch_layout_key(xkb_keysym_t xkb_sym) {
+    return xkb_sym == XKB_KEY_ISO_First_Group || xkb_sym == XKB_KEY_ISO_Last_Group || xkb_sym == XKB_KEY_ISO_Next_Group || xkb_sym == XKB_KEY_ISO_Prev_Group || xkb_sym == XKB_KEY_Mode_switch;
+}
+
 void
 glfw_xkb_handle_key_event(_GLFWwindow *window, _GLFWXKBData *xkb, xkb_keycode_t xkb_keycode, int action) {
     static char key_text[64] = {0};
@@ -745,7 +750,7 @@ glfw_xkb_handle_key_event(_GLFWwindow *window, _GLFWXKBData *xkb, xkb_keycode_t
             xkb_mod_mask_t consumed_unknown_mods = xkb_state_key_get_consumed_mods(sg->state, code_for_sym) & sg->activeUnknownModifiers;
             if (sg->activeUnknownModifiers) debug("%s", format_xkb_mods(xkb, "active_unknown_mods", sg->activeUnknownModifiers));
             if (consumed_unknown_mods) { debug("%s", format_xkb_mods(xkb, "consumed_unknown_mods", consumed_unknown_mods)); }
-            else xkb_sym = clean_syms[0];
+            else if (!is_switch_layout_key(xkb_sym)) xkb_sym = clean_syms[0];
             // xkb returns text even if alt and/or super are pressed
             if ( ((GLFW_MOD_CONTROL | GLFW_MOD_ALT | GLFW_MOD_SUPER | GLFW_MOD_HYPER | GLFW_MOD_META) & sg->modifiers) == 0) {
               xkb_state_key_get_utf8(sg->state, code_for_sym, key_text, sizeof(key_text));
@@ -757,9 +762,7 @@ glfw_xkb_handle_key_event(_GLFWwindow *window, _GLFWXKBData *xkb, xkb_keycode_t
         }
         if (key_text[0]) { debug("%s: %s ", text_type, key_text); }
     }
-    if (xkb_sym == XKB_KEY_ISO_First_Group || xkb_sym == XKB_KEY_ISO_Last_Group || xkb_sym == XKB_KEY_ISO_Next_Group || xkb_sym == XKB_KEY_ISO_Prev_Group || xkb_sym == XKB_KEY_Mode_switch) {
-      return;
-    }
+    if (is_switch_layout_key(xkb_sym)) { debug(" is a keyboard layout shift key, ignoring.\n"); return; }
     if (sg->modifiers & GLFW_MOD_NUM_LOCK && XKB_KEY_KP_Space <= xkb_sym && xkb_sym <= XKB_KEY_KP_9) {
         xkb_sym = xkb_state_key_get_one_sym(sg->state, code_for_sym);
     }

kittens/diff/config.py

@@ -3,21 +3,16 @@
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
 import os
-from typing import Any, Dict, Iterable, Optional, Tuple, Type, Union
+from typing import Any, Dict, Iterable, Optional
 
-from kitty.conf.definition import config_lines
+from kitty.cli_stub import DiffCLIOptions
 from kitty.conf.utils import (
-    init_config as _init_config, key_func, load_config as _load_config,
-    merge_dicts, parse_config_base, parse_kittens_key, resolve_config
+    load_config as _load_config, parse_config_base, resolve_config
 )
 from kitty.constants import config_dir
-from kitty.options_stub import DiffOptions
-from kitty.cli_stub import DiffCLIOptions
 from kitty.rgb import color_as_sgr
 
-from .config_data import all_options, type_convert
-
-defaults: Optional[DiffOptions] = None
+from .options.types import Options as DiffOptions, defaults
 
 formats: Dict[str, str] = {
     'title': '',
@@ -42,94 +37,30 @@ def set_formats(opts: DiffOptions) -> None:
     formats['added_highlight'] = '48' + color_as_sgr(opts.highlight_added_bg)
 
 
-func_with_args, args_funcs = key_func()
-
-
-@func_with_args('scroll_by')
-def parse_scroll_by(func: str, rest: str) -> Tuple[str, int]:
-    try:
-        return func, int(rest)
-    except Exception:
-        return func, 1
-
-
-@func_with_args('scroll_to')
-def parse_scroll_to(func: str, rest: str) -> Tuple[str, str]:
-    rest = rest.lower()
-    if rest not in {'start', 'end', 'next-change', 'prev-change', 'next-page', 'prev-page', 'next-match', 'prev-match'}:
-        rest = 'start'
-    return func, rest
-
-
-@func_with_args('change_context')
-def parse_change_context(func: str, rest: str) -> Tuple[str, Union[int, str]]:
-    rest = rest.lower()
-    if rest in {'all', 'default'}:
-        return func, rest
-    try:
-        amount = int(rest)
-    except Exception:
-        amount = 5
-    return func, amount
-
-
-@func_with_args('start_search')
-def parse_start_search(func: str, rest: str) -> Tuple[str, Tuple[bool, bool]]:
-    rest_ = rest.lower().split()
-    is_regex = bool(rest_ and rest_[0] == 'regex')
-    is_backward = bool(len(rest_) > 1 and rest_[1] == 'backward')
-    return func, (is_regex, is_backward)
-
-
-def special_handling(key: str, val: str, ans: Dict) -> bool:
-    if key == 'map':
-        x = parse_kittens_key(val, args_funcs)
-        if x is not None:
-            action, key_def = x
-            ans['key_definitions'][key_def] = action
-            return True
-    return False
-
-
-def parse_config(lines: Iterable[str], check_keys: bool = True) -> Dict[str, Any]:
-    ans: Dict[str, Any] = {'key_definitions': {}}
-    parse_config_base(
-        lines,
-        defaults,
-        type_convert,
-        special_handling,
-        ans,
-        check_keys=check_keys
-    )
-    return ans
-
-
-def merge_configs(defaults: Dict, vals: Dict) -> Dict:
-    ans = {}
-    for k, v in defaults.items():
-        if isinstance(v, dict):
-            newvals = vals.get(k, {})
-            ans[k] = merge_dicts(v, newvals)
-        else:
-            ans[k] = vals.get(k, v)
-    return ans
-
-
-def parse_defaults(lines: Iterable[str], check_keys: bool = False) -> Dict[str, Any]:
-    return parse_config(lines, check_keys)
-
-
-x = _init_config(config_lines(all_options), parse_defaults)
-Options: Type[DiffOptions] = x[0]
-defaults = x[1]
+SYSTEM_CONF = '/etc/xdg/kitty/diff.conf'
+defconf = os.path.join(config_dir, 'diff.conf')
 
 
 def load_config(*paths: str, overrides: Optional[Iterable[str]] = None) -> DiffOptions:
-    return _load_config(Options, defaults, parse_config, merge_configs, *paths, overrides=overrides)
+    from .options.parse import (
+        create_result_dict, merge_result_dicts, parse_conf_item
+    )
 
+    def parse_config(lines: Iterable[str]) -> Dict[str, Any]:
+        ans: Dict[str, Any] = create_result_dict()
+        parse_config_base(
+            lines,
+            parse_conf_item,
+            ans,
+        )
+        return ans
 
-SYSTEM_CONF = '/etc/xdg/kitty/diff.conf'
-defconf = os.path.join(config_dir, 'diff.conf')
+    overrides = tuple(overrides) if overrides is not None else ()
+    opts_dict, paths = _load_config(defaults, parse_config, merge_result_dicts, *paths, overrides=overrides)
+    opts = DiffOptions(opts_dict)
+    opts.config_paths = paths
+    opts.config_overrides = overrides
+    return opts
 
 
 def init_config(args: DiffCLIOptions) -> DiffOptions:
@@ -137,4 +68,6 @@ def init_config(args: DiffCLIOptions) -> DiffOptions:
     overrides = (a.replace('=', ' ', 1) for a in args.override or ())
     opts = load_config(*config, overrides=overrides)
     set_formats(opts)
+    for (sc, action) in opts.map:
+        opts.key_definitions[sc] = action
     return opts

kittens/diff/config_data.py

@@ -1,131 +0,0 @@
-#!/usr/bin/env python3
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-
-# Utils  {{{
-from functools import partial
-from gettext import gettext as _
-from typing import Any, Dict, Sequence, Union
-
-from kitty.conf.definition import Option, Shortcut, option_func
-from kitty.conf.utils import python_string, to_color, to_color_or_none
-from kitty.utils import positive_int
-
-# }}}
-
-all_options: Dict[str, Union[Option, Sequence[Shortcut]]] = {}
-o, k, g, all_groups = option_func(all_options, {
-    'colors': [_('Colors')],
-    'diff': [_('Diffing'), ],
-    'shortcuts': [_('Keyboard shortcuts')],
-})
-
-
-g('diff')
-
-
-def syntax_aliases(raw: str) -> Dict[str, str]:
-    ans = {}
-    for x in raw.split():
-        a, b = x.partition(':')[::2]
-        if a and b:
-            ans[a.lower()] = b
-    return ans
-
-
-o('syntax_aliases', 'pyj:py pyi:py recipe:py', option_type=syntax_aliases, long_text=_('''
-File extension aliases for syntax highlight
-For example, to syntax highlight :file:`file.xyz` as
-:file:`file.abc` use a setting of :code:`xyz:abc`
-'''))
-
-o('num_context_lines', 3, option_type=positive_int, long_text=_('''
-The number of lines of context to show around each change.'''))
-
-o('diff_cmd', 'auto', long_text=_('''
-The diff command to use. Must contain the placeholder :code:`_CONTEXT_`
-which will be replaced by the number of lines of context. The default
-is to search the system for either git or diff and use that, if found.
-'''))
-
-o('replace_tab_by', r'\x20\x20\x20\x20', option_type=python_string, long_text=_('''
-The string to replace tabs with. Default is to use four spaces.'''))
-
-
-g('colors')
-
-o('pygments_style', 'default', long_text=_('''
-The pygments color scheme to use for syntax highlighting.
-See :link:`pygments colors schemes <https://help.farbox.com/pygments.html>` for a list of schemes.'''))
-
-
-c = partial(o, option_type=to_color)
-c('foreground', 'black', long_text=_('Basic colors'))
-c('background', 'white')
-
-c('title_fg', 'black', long_text=_('Title colors'))
-c('title_bg', 'white')
-
-c('margin_bg', '#fafbfc', long_text=_('Margin colors'))
-c('margin_fg', '#aaaaaa')
-
-c('removed_bg', '#ffeef0', long_text=_('Removed text backgrounds'))
-c('highlight_removed_bg', '#fdb8c0')
-c('removed_margin_bg', '#ffdce0')
-
-c('added_bg', '#e6ffed', long_text=_('Added text backgrounds'))
-c('highlight_added_bg', '#acf2bd')
-c('added_margin_bg', '#cdffd8')
-
-c('filler_bg', '#fafbfc', long_text=_('Filler (empty) line background'))
-c('margin_filler_bg', 'none', option_type=to_color_or_none, long_text=_(
-    'Filler (empty) line background in margins, defaults to the filler background'))
-
-c('hunk_margin_bg', '#dbedff', long_text=_('Hunk header colors'))
-c('hunk_bg', '#f1f8ff')
-
-c('search_bg', '#444', long_text=_('Highlighting'))
-c('search_fg', 'white')
-c('select_bg', '#b4d5fe')
-o('select_fg', 'black', option_type=to_color_or_none)
-
-g('shortcuts')
-k('quit', 'q', 'quit', _('Quit'))
-k('quit', 'esc', 'quit', _('Quit'))
-
-k('scroll_down', 'j', 'scroll_by 1', _('Scroll down'))
-k('scroll_down', 'down', 'scroll_by 1', _('Scroll down'))
-k('scroll_up', 'k', 'scroll_by -1', _('Scroll up'))
-k('scroll_up', 'up', 'scroll_by -1', _('Scroll up'))
-
-k('scroll_top', 'home', 'scroll_to start', _('Scroll to top'))
-k('scroll_bottom', 'end', 'scroll_to end', _('Scroll to bottom'))
-
-k('scroll_page_down', 'page_down', 'scroll_to next-page', _('Scroll to next page'))
-k('scroll_page_down', 'space', 'scroll_to next-page', _('Scroll to next page'))
-k('scroll_page_up', 'page_up', 'scroll_to prev-page', _('Scroll to previous page'))
-
-k('next_change', 'n', 'scroll_to next-change', _('Scroll to next change'))
-k('prev_change', 'p', 'scroll_to prev-change', _('Scroll to previous change'))
-
-k('all_context', 'a', 'change_context all', _('Show all context'))
-k('default_context', '=', 'change_context default', _('Show default context'))
-k('increase_context', '+', 'change_context 5', _('Increase context'))
-k('decrease_context', '-', 'change_context -5', _('Decrease context'))
-
-k('search_forward', '/', 'start_search regex forward', _('Search forward'))
-k('search_backward', '?', 'start_search regex backward', _('Search backward'))
-k('next_match', '.', 'scroll_to next-match', _('Scroll to next search match'))
-k('prev_match', ',', 'scroll_to prev-match', _('Scroll to previous search match'))
-k('next_match', '>', 'scroll_to next-match', _('Scroll to next search match'))
-k('prev_match', '<', 'scroll_to prev-match', _('Scroll to previous search match'))
-k('search_forward_simple', 'f', 'start_search substring forward', _('Search forward (no regex)'))
-k('search_backward_simple', 'b', 'start_search substring backward', _('Search backward (no regex)'))
-
-
-def type_convert(name: str, val: Any) -> Any:
-    o = all_options.get(name)
-    if isinstance(o, Option):
-        val = o.option_type(val)
-    return val

kittens/diff/highlight.py

@@ -159,13 +159,12 @@ def highlight_collection(collection: Collection, aliases: Optional[Dict[str, str
 
 
 def main() -> None:
-    from .config import defaults
     # kitty +runpy "from kittens.diff.highlight import main; main()" file
+    from .options.types import defaults
     import sys
     initialize_highlighter()
-    if defaults is not None:
-        with open(sys.argv[-1]) as f:
-            highlighted = highlight_data(f.read(), f.name, defaults.syntax_aliases)
-        if highlighted is None:
-            raise SystemExit('Unknown filetype: {}'.format(sys.argv[-1]))
-        print(highlighted)
+    with open(sys.argv[-1]) as f:
+        highlighted = highlight_data(f.read(), f.name, defaults.syntax_aliases)
+    if highlighted is None:
+        raise SystemExit('Unknown filetype: {}'.format(sys.argv[-1]))
+    print(highlighted)

kittens/diff/main.py

@@ -19,11 +19,10 @@ from typing import (
 
 from kitty.cli import CONFIG_HELP, parse_args
 from kitty.cli_stub import DiffCLIOptions
-from kitty.conf.utils import KittensKeyAction
+from kitty.conf.utils import KeyAction
 from kitty.constants import appname
 from kitty.fast_data_types import wcswidth
 from kitty.key_encoding import EventType, KeyEvent
-from kitty.options_stub import DiffOptions
 from kitty.utils import ScreenSize
 
 from ..tui.handler import Handler
@@ -33,10 +32,11 @@ from ..tui.loop import Loop
 from ..tui.operations import styled
 from . import global_data
 from .collect import (
-    Collection, create_collection, data_for_path, lines_for_path, sanitize,
-    set_highlight_data, add_remote_dir
+    Collection, add_remote_dir, create_collection, data_for_path,
+    lines_for_path, sanitize, set_highlight_data
 )
 from .config import init_config
+from .options.types import Options as DiffOptions
 from .patch import Differ, Patch, set_diff_command, worker_processes
 from .render import (
     ImagePlacement, ImageSupportWarning, Line, LineRef, Reference, render_diff
@@ -95,14 +95,14 @@ class DiffHandler(Handler):
         for key_def, action in self.opts.key_definitions.items():
             self.add_shortcut(action, key_def)
 
-    def perform_action(self, action: KittensKeyAction) -> None:
+    def perform_action(self, action: KeyAction) -> None:
         func, args = action
         if func == 'quit':
             self.quit_loop(0)
             return
         if self.state <= DIFFED:
             if func == 'scroll_by':
-                return self.scroll_lines(int(args[0]))
+                return self.scroll_lines(int(args[0] or 0))
             if func == 'scroll_to':
                 where = str(args[0])
                 if 'change' in where:
@@ -122,7 +122,7 @@ class DiffHandler(Handler):
                 elif to == 'default':
                     new_ctx = self.original_context_count
                 else:
-                    new_ctx += int(to)
+                    new_ctx += int(to or 0)
                 return self.change_context_count(new_ctx)
             if func == 'start_search':
                 self.start_search(bool(args[0]), bool(args[1]))
@@ -658,5 +658,5 @@ elif __name__ == '__doc__':
     cd['options'] = OPTIONS
     cd['help_text'] = help_text
 elif __name__ == '__conf__':
-    from .config_data import all_options
-    sys.all_options = all_options  # type: ignore
+    from .options.definition import definition
+    sys.options_definition = definition  # type: ignore

kittens/diff/options/definition.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+# After editing this file run ./gen-config.py to apply the changes
+
+from kitty.conf.types import Action, Definition
+
+
+definition = Definition(
+    'kittens.diff',
+    Action('map', 'parse_map', {'key_definitions': 'kitty.conf.utils.KittensKeyMap'}, ['kitty.types.ParsedShortcut', 'kitty.conf.utils.KeyAction']),
+)
+
+agr = definition.add_group
+egr = definition.end_group
+opt = definition.add_option
+map = definition.add_map
+mma = definition.add_mouse_map
+
+# diff {{{
+agr('diff', 'Diffing')
+
+opt('syntax_aliases', 'pyj:py pyi:py recipe:py',
+    option_type='syntax_aliases',
+    long_text='''
+File extension aliases for syntax highlight For example, to syntax highlight
+:file:`file.xyz` as :file:`file.abc` use a setting of :code:`xyz:abc`
+'''
+    )
+
+opt('num_context_lines', '3',
+    option_type='positive_int',
+    long_text='The number of lines of context to show around each change.'
+    )
+
+opt('diff_cmd', 'auto',
+    long_text='''
+The diff command to use. Must contain the placeholder :code:`_CONTEXT_` which
+will be replaced by the number of lines of context. The default is to search the
+system for either git or diff and use that, if found.
+'''
+    )
+
+opt('replace_tab_by', '\\x20\\x20\\x20\\x20',
+    option_type='python_string',
+    long_text='The string to replace tabs with. Default is to use four spaces.'
+    )
+egr()  # }}}
+
+# colors {{{
+agr('colors', 'Colors')
+
+opt('pygments_style', 'default',
+    long_text='''
+The pygments color scheme to use for syntax highlighting. See :link:`pygments
+colors schemes <https://help.farbox.com/pygments.html>` for a list of schemes.
+'''
+    )
+
+opt('foreground', 'black',
+    option_type='to_color',
+    long_text='Basic colors'
+    )
+
+opt('background', 'white',
+    option_type='to_color',
+    )
+
+opt('title_fg', 'black',
+    option_type='to_color',
+    long_text='Title colors'
+    )
+
+opt('title_bg', 'white',
+    option_type='to_color',
+    )
+
+opt('margin_bg', '#fafbfc',
+    option_type='to_color',
+    long_text='Margin colors'
+    )
+
+opt('margin_fg', '#aaaaaa',
+    option_type='to_color',
+    )
+
+opt('removed_bg', '#ffeef0',
+    option_type='to_color',
+    long_text='Removed text backgrounds'
+    )
+
+opt('highlight_removed_bg', '#fdb8c0',
+    option_type='to_color',
+    )
+
+opt('removed_margin_bg', '#ffdce0',
+    option_type='to_color',
+    )
+
+opt('added_bg', '#e6ffed',
+    option_type='to_color',
+    long_text='Added text backgrounds'
+    )
+
+opt('highlight_added_bg', '#acf2bd',
+    option_type='to_color',
+    )
+
+opt('added_margin_bg', '#cdffd8',
+    option_type='to_color',
+    )
+
+opt('filler_bg', '#fafbfc',
+    option_type='to_color',
+    long_text='Filler (empty) line background'
+    )
+
+opt('margin_filler_bg', 'none',
+    option_type='to_color_or_none',
+    long_text='Filler (empty) line background in margins, defaults to the filler background'
+    )
+
+opt('hunk_margin_bg', '#dbedff',
+    option_type='to_color',
+    long_text='Hunk header colors'
+    )
+
+opt('hunk_bg', '#f1f8ff',
+    option_type='to_color',
+    )
+
+opt('search_bg', '#444',
+    option_type='to_color',
+    long_text='Highlighting'
+    )
+
+opt('search_fg', 'white',
+    option_type='to_color',
+    )
+
+opt('select_bg', '#b4d5fe',
+    option_type='to_color',
+    )
+
+opt('select_fg', 'black',
+    option_type='to_color_or_none',
+    )
+egr()  # }}}
+
+# shortcuts {{{
+agr('shortcuts', 'Keyboard shortcuts')
+
+map('Quit',
+    'quit q quit',
+    )
+map('Quit',
+    'quit esc quit',
+    )
+
+map('Scroll down',
+    'scroll_down j scroll_by 1',
+    )
+map('Scroll down',
+    'scroll_down down scroll_by 1',
+    )
+
+map('Scroll up',
+    'scroll_up k scroll_by -1',
+    )
+map('Scroll up',
+    'scroll_up up scroll_by -1',
+    )
+
+map('Scroll to top',
+    'scroll_top home scroll_to start',
+    )
+
+map('Scroll to bottom',
+    'scroll_bottom end scroll_to end',
+    )
+
+map('Scroll to next page',
+    'scroll_page_down page_down scroll_to next-page',
+    )
+map('Scroll to next page',
+    'scroll_page_down space scroll_to next-page',
+    )
+
+map('Scroll to previous page',
+    'scroll_page_up page_up scroll_to prev-page',
+    )
+
+map('Scroll to next change',
+    'next_change n scroll_to next-change',
+    )
+
+map('Scroll to previous change',
+    'prev_change p scroll_to prev-change',
+    )
+
+map('Show all context',
+    'all_context a change_context all',
+    )
+
+map('Show default context',
+    'default_context = change_context default',
+    )
+
+map('Increase context',
+    'increase_context + change_context 5',
+    )
+
+map('Decrease context',
+    'decrease_context - change_context -5',
+    )
+
+map('Search forward',
+    'search_forward / start_search regex forward',
+    )
+
+map('Search backward',
+    'search_backward ? start_search regex backward',
+    )
+
+map('Scroll to next search match',
+    'next_match . scroll_to next-match',
+    )
+map('Scroll to next search match',
+    'next_match > scroll_to next-match',
+    )
+
+map('Scroll to previous search match',
+    'prev_match , scroll_to prev-match',
+    )
+map('Scroll to previous search match',
+    'prev_match < scroll_to prev-match',
+    )
+
+map('Search forward (no regex)',
+    'search_forward_simple f start_search substring forward',
+    )
+
+map('Search backward (no regex)',
+    'search_backward_simple b start_search substring backward',
+    )
+egr()  # }}}

kittens/diff/options/parse.py

@@ -0,0 +1,120 @@
+# generated by gen-config.py DO NOT edit
+# vim:fileencoding=utf-8
+
+import typing
+from kittens.diff.options.utils import parse_map, syntax_aliases
+from kitty.conf.utils import merge_dicts, positive_int, python_string, to_color, to_color_or_none
+
+
+class Parser:
+
+    def added_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['added_bg'] = to_color(val)
+
+    def added_margin_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['added_margin_bg'] = to_color(val)
+
+    def background(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['background'] = to_color(val)
+
+    def diff_cmd(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['diff_cmd'] = str(val)
+
+    def filler_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['filler_bg'] = to_color(val)
+
+    def foreground(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['foreground'] = to_color(val)
+
+    def highlight_added_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['highlight_added_bg'] = to_color(val)
+
+    def highlight_removed_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['highlight_removed_bg'] = to_color(val)
+
+    def hunk_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['hunk_bg'] = to_color(val)
+
+    def hunk_margin_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['hunk_margin_bg'] = to_color(val)
+
+    def margin_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['margin_bg'] = to_color(val)
+
+    def margin_fg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['margin_fg'] = to_color(val)
+
+    def margin_filler_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['margin_filler_bg'] = to_color_or_none(val)
+
+    def num_context_lines(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['num_context_lines'] = positive_int(val)
+
+    def pygments_style(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['pygments_style'] = str(val)
+
+    def removed_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['removed_bg'] = to_color(val)
+
+    def removed_margin_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['removed_margin_bg'] = to_color(val)
+
+    def replace_tab_by(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['replace_tab_by'] = python_string(val)
+
+    def search_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['search_bg'] = to_color(val)
+
+    def search_fg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['search_fg'] = to_color(val)
+
+    def select_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['select_bg'] = to_color(val)
+
+    def select_fg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['select_fg'] = to_color_or_none(val)
+
+    def syntax_aliases(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['syntax_aliases'] = syntax_aliases(val)
+
+    def title_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['title_bg'] = to_color(val)
+
+    def title_fg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        ans['title_fg'] = to_color(val)
+
+    def map(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
+        for k in parse_map(val):
+            ans['map'].append(k)
+
+
+def create_result_dict() -> typing.Dict[str, typing.Any]:
+    return {
+        'map': [],
+    }
+
+
+actions = frozenset(('map',))
+
+
+def merge_result_dicts(defaults: typing.Dict[str, typing.Any], vals: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:
+    ans = {}
+    for k, v in defaults.items():
+        if isinstance(v, dict):
+            ans[k] = merge_dicts(v, vals.get(k, {}))
+        elif k in actions:
+            ans[k] = v + vals.get(k, [])
+        else:
+            ans[k] = vals.get(k, v)
+    return ans
+
+
+parser = Parser()
+
+
+def parse_conf_item(key: str, val: str, ans: typing.Dict[str, typing.Any]) -> bool:
+    func = getattr(parser, key, None)
+    if func is not None:
+        func(val, ans)
+        return True
+    return False

kittens/diff/options/types.py

@@ -0,0 +1,168 @@
+# generated by gen-config.py DO NOT edit
+# vim:fileencoding=utf-8
+
+import typing
+from kitty.conf.utils import KeyAction, KittensKeyMap
+import kitty.conf.utils
+from kitty.rgb import Color
+import kitty.rgb
+from kitty.types import ParsedShortcut
+import kitty.types
+
+
+option_names = (  # {{{
+ 'added_bg',
+ 'added_margin_bg',
+ 'background',
+ 'diff_cmd',
+ 'filler_bg',
+ 'foreground',
+ 'highlight_added_bg',
+ 'highlight_removed_bg',
+ 'hunk_bg',
+ 'hunk_margin_bg',
+ 'map',
+ 'margin_bg',
+ 'margin_fg',
+ 'margin_filler_bg',
+ 'num_context_lines',
+ 'pygments_style',
+ 'removed_bg',
+ 'removed_margin_bg',
+ 'replace_tab_by',
+ 'search_bg',
+ 'search_fg',
+ 'select_bg',
+ 'select_fg',
+ 'syntax_aliases',
+ 'title_bg',
+ 'title_fg')  # }}}
+
+
+class Options:
+    added_bg: Color = Color(red=230, green=255, blue=237)
+    added_margin_bg: Color = Color(red=205, green=255, blue=216)
+    background: Color = Color(red=255, green=255, blue=255)
+    diff_cmd: str = 'auto'
+    filler_bg: Color = Color(red=250, green=251, blue=252)
+    foreground: Color = Color(red=0, green=0, blue=0)
+    highlight_added_bg: Color = Color(red=172, green=242, blue=189)
+    highlight_removed_bg: Color = Color(red=253, green=184, blue=192)
+    hunk_bg: Color = Color(red=241, green=248, blue=255)
+    hunk_margin_bg: Color = Color(red=219, green=237, blue=255)
+    margin_bg: Color = Color(red=250, green=251, blue=252)
+    margin_fg: Color = Color(red=170, green=170, blue=170)
+    margin_filler_bg: typing.Optional[kitty.rgb.Color] = None
+    num_context_lines: int = 3
+    pygments_style: str = 'default'
+    removed_bg: Color = Color(red=255, green=238, blue=240)
+    removed_margin_bg: Color = Color(red=255, green=220, blue=224)
+    replace_tab_by: str = '    '
+    search_bg: Color = Color(red=68, green=68, blue=68)
+    search_fg: Color = Color(red=255, green=255, blue=255)
+    select_bg: Color = Color(red=180, green=213, blue=254)
+    select_fg: typing.Optional[kitty.rgb.Color] = Color(red=0, green=0, blue=0)
+    syntax_aliases: typing.Dict[str, str] = {'pyj': 'py', 'pyi': 'py', 'recipe': 'py'}
+    title_bg: Color = Color(red=255, green=255, blue=255)
+    title_fg: Color = Color(red=0, green=0, blue=0)
+    map: typing.List[typing.Tuple[kitty.types.ParsedShortcut, kitty.conf.utils.KeyAction]] = []
+    key_definitions: KittensKeyMap = {}
+    config_paths: typing.Tuple[str, ...] = ()
+    config_overrides: typing.Tuple[str, ...] = ()
+
+    def __init__(self, options_dict: typing.Optional[typing.Dict[str, typing.Any]] = None) -> None:
+        if options_dict is not None:
+            for key in option_names:
+                setattr(self, key, options_dict[key])
+
+    @property
+    def _fields(self) -> typing.Tuple[str, ...]:
+        return option_names
+
+    def __iter__(self) -> typing.Iterator[str]:
+        return iter(self._fields)
+
+    def __len__(self) -> int:
+        return len(self._fields)
+
+    def _copy_of_val(self, name: str) -> typing.Any:
+        ans = getattr(self, name)
+        if isinstance(ans, dict):
+            ans = ans.copy()
+        elif isinstance(ans, list):
+            ans = ans[:]
+        return ans
+
+    def _asdict(self) -> typing.Dict[str, typing.Any]:
+        return {k: self._copy_of_val(k) for k in self}
+
+    def _replace(self, **kw: typing.Any) -> "Options":
+        ans = Options()
+        for name in self:
+            setattr(ans, name, self._copy_of_val(name))
+        for name, val in kw.items():
+            setattr(ans, name, val)
+        return ans
+
+    def __getitem__(self, key: typing.Union[int, str]) -> typing.Any:
+        k = option_names[key] if isinstance(key, int) else key
+        try:
+            return getattr(self, k)
+        except AttributeError:
+            pass
+        raise KeyError(f"No option named: {k}")
+
+
+defaults = Options()
+defaults.map = [
+    # quit
+    (ParsedShortcut(mods=0, key_name='q'), KeyAction('quit')),
+    # quit
+    (ParsedShortcut(mods=0, key_name='ESCAPE'), KeyAction('quit')),
+    # scroll_down
+    (ParsedShortcut(mods=0, key_name='j'), KeyAction('scroll_by', (1,))),
+    # scroll_down
+    (ParsedShortcut(mods=0, key_name='DOWN'), KeyAction('scroll_by', (1,))),
+    # scroll_up
+    (ParsedShortcut(mods=0, key_name='k'), KeyAction('scroll_by', (-1,))),
+    # scroll_up
+    (ParsedShortcut(mods=0, key_name='UP'), KeyAction('scroll_by', (-1,))),
+    # scroll_top
+    (ParsedShortcut(mods=0, key_name='HOME'), KeyAction('scroll_to', ('start',))),
+    # scroll_bottom
+    (ParsedShortcut(mods=0, key_name='END'), KeyAction('scroll_to', ('end',))),
+    # scroll_page_down
+    (ParsedShortcut(mods=0, key_name='PAGE_DOWN'), KeyAction('scroll_to', ('next-page',))),
+    # scroll_page_down
+    (ParsedShortcut(mods=0, key_name=' '), KeyAction('scroll_to', ('next-page',))),
+    # scroll_page_up
+    (ParsedShortcut(mods=0, key_name='PAGE_UP'), KeyAction('scroll_to', ('prev-page',))),
+    # next_change
+    (ParsedShortcut(mods=0, key_name='n'), KeyAction('scroll_to', ('next-change',))),
+    # prev_change
+    (ParsedShortcut(mods=0, key_name='p'), KeyAction('scroll_to', ('prev-change',))),
+    # all_context
+    (ParsedShortcut(mods=0, key_name='a'), KeyAction('change_context', ('all',))),
+    # default_context
+    (ParsedShortcut(mods=0, key_name='='), KeyAction('change_context', ('default',))),
+    # increase_context
+    (ParsedShortcut(mods=0, key_name='+'), KeyAction('change_context', (5,))),
+    # decrease_context
+    (ParsedShortcut(mods=0, key_name='-'), KeyAction('change_context', (-5,))),
+    # search_forward
+    (ParsedShortcut(mods=0, key_name='/'), KeyAction('start_search', (True, False))),
+    # search_backward
+    (ParsedShortcut(mods=0, key_name='?'), KeyAction('start_search', (True, True))),
+    # next_match
+    (ParsedShortcut(mods=0, key_name='.'), KeyAction('scroll_to', ('next-match',))),
+    # next_match
+    (ParsedShortcut(mods=0, key_name='>'), KeyAction('scroll_to', ('next-match',))),
+    # prev_match
+    (ParsedShortcut(mods=0, key_name=','), KeyAction('scroll_to', ('prev-match',))),
+    # prev_match
+    (ParsedShortcut(mods=0, key_name='<'), KeyAction('scroll_to', ('prev-match',))),
+    # search_forward_simple
+    (ParsedShortcut(mods=0, key_name='f'), KeyAction('start_search', (False, False))),
+    # search_backward_simple
+    (ParsedShortcut(mods=0, key_name='b'), KeyAction('start_search', (False, True))),
+]

kittens/diff/options/utils.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+from typing import Any, Dict, Iterable, Sequence, Tuple, Union
+
+from kitty.conf.utils import KittensKeyDefinition, key_func, parse_kittens_key
+
+func_with_args, args_funcs = key_func()
+FuncArgsType = Tuple[str, Sequence[Any]]
+
+
+@func_with_args('scroll_by')
+def parse_scroll_by(func: str, rest: str) -> Tuple[str, int]:
+    try:
+        return func, int(rest)
+    except Exception:
+        return func, 1
+
+
+@func_with_args('scroll_to')
+def parse_scroll_to(func: str, rest: str) -> Tuple[str, str]:
+    rest = rest.lower()
+    if rest not in {'start', 'end', 'next-change', 'prev-change', 'next-page', 'prev-page', 'next-match', 'prev-match'}:
+        rest = 'start'
+    return func, rest
+
+
+@func_with_args('change_context')
+def parse_change_context(func: str, rest: str) -> Tuple[str, Union[int, str]]:
+    rest = rest.lower()
+    if rest in {'all', 'default'}:
+        return func, rest
+    try:
+        amount = int(rest)
+    except Exception:
+        amount = 5
+    return func, amount
+
+
+@func_with_args('start_search')
+def parse_start_search(func: str, rest: str) -> Tuple[str, Tuple[bool, bool]]:
+    rest_ = rest.lower().split()
+    is_regex = bool(rest_ and rest_[0] == 'regex')
+    is_backward = bool(len(rest_) > 1 and rest_[1] == 'backward')
+    return func, (is_regex, is_backward)
+
+
+def syntax_aliases(raw: str) -> Dict[str, str]:
+    ans = {}
+    for x in raw.split():
+        a, b = x.partition(':')[::2]
+        if a and b:
+            ans[a.lower()] = b
+    return ans
+
+
+def parse_map(val: str) -> Iterable[KittensKeyDefinition]:
+    x = parse_kittens_key(val, args_funcs)
+    if x is not None:
+        yield x

kittens/diff/search.py

@@ -6,7 +6,7 @@ import re
 from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Tuple
 
 from kitty.fast_data_types import wcswidth
-from kitty.options_stub import DiffOptions
+from .options.types import Options as DiffOptions
 
 from ..tui.operations import styled
 

kittens/hyperlinked_grep/main.py

@@ -29,7 +29,7 @@ def main() -> None:
     write: Callable[[bytes], None] = cast(Callable[[bytes], None], sys.stdout.buffer.write)
     sgr_pat = re.compile(br'\x1b\[.*?m')
     osc_pat = re.compile(b'\x1b\\].*?\x1b\\\\')
-    num_pat = re.compile(b'^(\\d+):')
+    num_pat = re.compile(br'^(\d+)[:-]')
 
     in_result: bytes = b''
     hostname = socket.gethostname().encode('utf-8')
@@ -46,6 +46,8 @@ def main() -> None:
                 m = num_pat.match(clean_line)
                 if m is not None:
                     write_hyperlink(write, in_result, line, frag=m.group(1))
+                else:
+                    write(line)
             else:
                 if line.strip():
                     path = quote_from_bytes(os.path.abspath(clean_line)).encode('utf-8')
@@ -55,8 +57,9 @@ def main() -> None:
                     write(line)
     except KeyboardInterrupt:
         p.send_signal(signal.SIGINT)
-        p.stdout.close()
     except EOFError:
+        pass
+    finally:
         p.stdout.close()
     raise SystemExit(p.wait())
 

kittens/query_terminal/main.py

@@ -4,22 +4,27 @@
 
 import re
 import sys
-from binascii import unhexlify, hexlify
+from binascii import hexlify, unhexlify
 from contextlib import suppress
-from typing import Dict, Iterable, List, Type
+from typing import Dict, Iterable, List, Type, Optional
 
 from kitty.cli import parse_args
 from kitty.cli_stub import QueryTerminalCLIOptions
-from kitty.constants import appname
-from kitty.utils import TTYIO
+from kitty.constants import appname, str_version
+from kitty.options.types import Options
 from kitty.terminfo import names
+from kitty.utils import TTYIO
 
 
 class Query:
     name: str = ''
     ans: str = ''
-    query_name: str = ''
     help_text: str = ''
+    override_query_name: str = ''
+
+    @property
+    def query_name(self) -> str:
+        return self.override_query_name or f'kitty-query-{self.name}'
 
     def __init__(self) -> None:
         self.encoded_query_name = hexlify(self.query_name.encode('utf-8')).decode('ascii')
@@ -45,6 +50,10 @@ class Query:
     def output_line(self) -> str:
         return self.ans
 
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        raise NotImplementedError()
+
 
 all_queries: Dict[str, Type[Query]] = {}
 
@@ -57,23 +66,99 @@ def query(cls: Type[Query]) -> Type[Query]:
 @query
 class TerminalName(Query):
     name: str = 'name'
-    query_name: str = 'TN'
+    override_query_name: str = 'name'
     help_text: str = f'Terminal name ({names[0]})'
 
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        return appname
+
 
 @query
 class TerminalVersion(Query):
     name: str = 'version'
-    query_name: str = 'kitty-query-version'
     help_text: str = 'Terminal version, for e.g.: 0.19.2'
 
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        return str_version
+
 
 @query
 class AllowHyperlinks(Query):
     name: str = 'allow_hyperlinks'
-    query_name: str = 'kitty-query-allow_hyperlinks'
     help_text: str = 'yes, no or ask'
 
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        return 'ask' if opts.allow_hyperlinks == 0b11 else ('yes' if opts.allow_hyperlinks else 'no')
+
+
+@query
+class FontFamily(Query):
+    name: str = 'font_family'
+    help_text: str = 'The current font\'s PostScript name'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        from kitty.fast_data_types import current_fonts
+        cf = current_fonts()
+        return str(cf['medium'].display_name())
+
+
+@query
+class BoldFont(Query):
+    name: str = 'bold_font'
+    help_text: str = 'The current bold font\'s PostScript name'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        from kitty.fast_data_types import current_fonts
+        cf = current_fonts()
+        return str(cf['bold'].display_name())
+
+
+@query
+class ItalicFont(Query):
+    name: str = 'italic_font'
+    help_text: str = 'The current italic font\'s PostScript name'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        from kitty.fast_data_types import current_fonts
+        cf = current_fonts()
+        return str(cf['italic'].display_name())
+
+
+@query
+class BiFont(Query):
+    name: str = 'bold_italic_font'
+    help_text: str = 'The current bold-italic font\'s PostScript name'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        from kitty.fast_data_types import current_fonts
+        cf = current_fonts()
+        return str(cf['bi'].display_name())
+
+
+@query
+class FontSize(Query):
+    name: str = 'font_size'
+    help_text: str = 'The current overall font size (individual windows can have different per window font sizes)'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        return f'{opts.font_size:g}'
+
+
+def get_result(name: str) -> Optional[str]:
+    from kitty.fast_data_types import get_options
+    q = all_queries.get(name)
+    if q is None:
+        return None
+    return q.get_result(get_options())
+
 
 def do_queries(queries: Iterable, cli_opts: QueryTerminalCLIOptions) -> Dict[str, str]:
     actions = tuple(all_queries[x]() for x in queries)

kittens/remote_file/main.py

@@ -151,7 +151,7 @@ class ControlMaster:
         cp = subprocess.run(self.batch_cmd_prefix + [self.conn_data.hostname, 'hostname', '-f'], stdout=subprocess.PIPE,
                             stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL)
         if cp.returncode == 0:
-            q = cp.stdout.decode('utf-8').strip()
+            q = tuple(filter(None, cp.stdout.decode('utf-8').strip().splitlines()))[-1]
             if not hostname_matches(self.cli_opts.hostname or '', q):
                 print(reset_terminal(), end='')
                 print(f'The remote hostname {styled(q, fg="green")} does not match the')
@@ -317,7 +317,8 @@ def handle_action(action: str, cli_opts: RemoteFileCLIOptions) -> Result:
 @result_handler()
 def handle_result(args: List[str], data: Result, target_window_id: int, boss: BossType) -> None:
     if data:
-        cmd = command_for_open(boss.opts.open_url_with)
+        from kitty.fast_data_types import get_options
+        cmd = command_for_open(get_options().open_url_with)
         open_cmd(cmd, data)
 
 

kittens/resize_window/main.py

@@ -74,7 +74,7 @@ class Resize(Handler):
         if key_event.matches('esc'):
             self.quit_loop(0)
             return
-        if key_event.key in ('w', 'n', 't', 's') and key_event.mods == CTRL:
+        if key_event.key in ('w', 'n', 't', 's') and key_event.mods_without_locks == CTRL:
             self.do_window_resize(is_decrease=key_event.key in 'ns', is_horizontal=key_event.key in 'wn', multiplier=2)
 
     def on_resize(self, new_size: ScreenSize) -> None:

kittens/runner.py

@@ -6,12 +6,17 @@
 import importlib
 import os
 import sys
+from contextlib import contextmanager
 from functools import partial
-from typing import Any, Dict, FrozenSet, List
+from typing import TYPE_CHECKING, Any, Dict, FrozenSet, Generator, List, cast
 
 from kitty.types import run_once
 
 aliases = {'url_hints': 'hints'}
+if TYPE_CHECKING:
+    from kitty.conf.types import Definition
+else:
+    Definition = object
 
 
 def resolved_kitten(k: str) -> str:
@@ -26,21 +31,29 @@ def path_to_custom_kitten(config_dir: str, kitten: str) -> str:
     return path
 
 
+@contextmanager
+def preserve_sys_path() -> Generator[None, None, None]:
+    orig = sys.path[:]
+    try:
+        yield
+    finally:
+        if sys.path != orig:
+            del sys.path[:]
+            sys.path.extend(orig)
+
+
 def import_kitten_main_module(config_dir: str, kitten: str) -> Dict[str, Any]:
     if kitten.endswith('.py'):
-        path_modified = False
-        path = path_to_custom_kitten(config_dir, kitten)
-        if os.path.dirname(path):
-            sys.path.insert(0, os.path.dirname(path))
-            path_modified = True
-        with open(path) as f:
-            src = f.read()
-        code = compile(src, path, 'exec')
-        g = {'__name__': 'kitten'}
-        exec(code, g)
-        hr = g.get('handle_result', lambda *a, **kw: None)
-        if path_modified:
-            del sys.path[0]
+        with preserve_sys_path():
+            path = path_to_custom_kitten(config_dir, kitten)
+            if os.path.dirname(path):
+                sys.path.insert(0, os.path.dirname(path))
+            with open(path) as f:
+                src = f.read()
+            code = compile(src, path, 'exec')
+            g = {'__name__': 'kitten'}
+            exec(code, g)
+            hr = g.get('handle_result', lambda *a, **kw: None)
         return {'start': g['main'], 'end': hr}
 
     kitten = resolved_kitten(kitten)
@@ -117,7 +130,7 @@ def run_kitten(kitten: str, run_name: str = '__main__') -> None:
         for kitten in all_kitten_names():
             print(kitten, file=sys.stderr)
         raise SystemExit('No kitten named {}'.format(original_kitten_name))
-    m = runpy.run_path(path, init_globals={'sys': sys, 'os': os}, run_name='__run_kitten__')  # type: ignore
+    m = runpy.run_path(path, init_globals={'sys': sys, 'os': os}, run_name='__run_kitten__')
     m['main'](sys.argv)
 
 
@@ -151,14 +164,22 @@ def get_kitten_cli_docs(kitten: str) -> Any:
         return ans
 
 
-def get_kitten_conf_docs(kitten: str) -> Any:
-    setattr(sys, 'all_options', None)
-    run_kitten(kitten, run_name='__conf__')
-    ans = getattr(sys, 'all_options')
-    delattr(sys, 'all_options')
+def get_kitten_completer(kitten: str) -> Any:
+    run_kitten(kitten, run_name='__completer__')
+    ans = getattr(sys, 'kitten_completer', None)
+    if ans is not None:
+        delattr(sys, 'kitten_completer')
     return ans
 
 
+def get_kitten_conf_docs(kitten: str) -> Definition:
+    setattr(sys, 'options_definition', None)
+    run_kitten(kitten, run_name='__conf__')
+    ans = getattr(sys, 'options_definition')
+    delattr(sys, 'options_definition')
+    return cast(Definition, ans)
+
+
 def main() -> None:
     try:
         args = sys.argv[1:]

kittens/show_key/kitty_mode.py

@@ -2,16 +2,13 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
-import sys
-from typing import List
-
 from kitty.key_encoding import (
     ALT, CAPS_LOCK, CTRL, HYPER, META, NUM_LOCK, PRESS, RELEASE, REPEAT, SHIFT,
     SUPER, KeyEvent, encode_key_event
 )
 
-from ..tui.handler import Handler
-from ..tui.loop import Loop
+from kittens.tui.handler import Handler
+from kittens.tui.loop import Loop
 
 
 class KeysHandler(Handler):
@@ -69,12 +66,8 @@ class KeysHandler(Handler):
         self.quit_loop(0)
 
 
-def main(args: List[str]) -> None:
+def main() -> None:
     loop = Loop()
     handler = KeysHandler()
     loop.loop(handler)
     raise SystemExit(loop.return_code)
-
-
-if __name__ == '__main__':
-    main(sys.argv)

kittens/show_key/main.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+import os
+import sys
+from typing import List
+
+from kitty.cli import parse_args
+from kitty.cli_stub import ShowKeyCLIOptions
+from kittens.tui.operations import raw_mode, styled
+
+ctrl_keys = '@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_'
+
+
+def print_key(raw: bytearray) -> None:
+    unix = ''
+    for ch in raw:
+        if ch < len(ctrl_keys):
+            unix += '^' + ctrl_keys[ch]
+        elif ch == 127:
+            unix += '^?'
+        else:
+            unix += chr(ch)
+    print(unix + '\t\t', end='')
+    for ch in raw:
+        x = chr(ch).encode('ascii')
+        print(styled(repr(x)[2:-1], fg='yellow'), end='')
+    print(end='\r\n', flush=True)
+
+
+def read_keys() -> None:
+    fd = sys.stdin.fileno()
+    while True:
+        try:
+            raw = bytearray(os.read(fd, 64))
+        except OSError as err:
+            print(err, file=sys.stderr, flush=True)
+            break
+        if not raw:
+            break
+        print_key(raw)
+        if len(raw) == 1 and raw[0] == 4:
+            break
+
+
+def legacy_main() -> None:
+    print('Press any keys - Ctrl-D will terminate this program', end='\r\n', flush=True)
+    print(styled('UNIX', italic=True, fg='green'), styled('send_text', italic=True, fg='green'), sep='\t\t', end='\r\n')
+
+    with raw_mode():
+        read_keys()
+
+
+OPTIONS = r'''
+--key-mode -m
+default=normal
+type=choices
+choices=normal,application,kitty,unchanged
+The keyboard mode to use when showing keys. "normal" mode is with DECCKM reset and "application" mode is with
+DECCKM set. "kitty" is the full kitty extended keyboard protocol.
+'''.format
+
+
+def main(args: List[str]) -> None:
+    cli_opts, items = parse_args(args[1:], OPTIONS, '', '', 'kitty +kitten clipboard', result_class=ShowKeyCLIOptions)
+    if cli_opts.key_mode == 'kitty':
+        from .kitty_mode import main as kitty_main
+        return kitty_main()
+    if cli_opts.key_mode != 'unchanged':
+        print(end='\x1b[?1' + ('l' if cli_opts.key_mode == 'normal' else 'h'), flush=True)
+    try:
+        return legacy_main()
+    finally:
+        if cli_opts.key_mode != 'unchanged':
+            print(end='\x1b[?1l', flush=True)
+
+
+if __name__ == '__main__':
+    main(sys.argv)

kittens/ssh/completion.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+import re
+import subprocess
+from typing import Callable, Dict, Iterable, Iterator, Sequence, Tuple
+
+from kitty.complete import Completions, complete_files_and_dirs, debug
+from kitty.types import run_once
+
+debug
+
+
+def lines_from_file(path: str) -> Iterator[str]:
+    try:
+        f = open(os.path.expanduser(path))
+    except OSError:
+        pass
+    else:
+        yield from f
+
+
+def lines_from_command(*cmd: str) -> Iterator[str]:
+    try:
+        output = subprocess.check_output(cmd).decode('utf-8')
+    except Exception:
+        return
+    yield from output.splitlines()
+
+
+def parts_yielder(lines: Iterable[str], pfilter: Callable[[str], Iterator[str]]) -> Iterator[str]:
+    for line in lines:
+        yield from pfilter(line)
+
+
+def hosts_from_config_lines(line: str) -> Iterator[str]:
+    parts = line.strip().split()
+    if len(parts) > 1 and parts[0] == 'Host':
+        yield parts[1]
+
+
+def hosts_from_known_hosts(line: str) -> Iterator[str]:
+    parts = line.strip().split()
+    if parts:
+        yield re.sub(r':\d+$', '', parts[0])
+
+
+def hosts_from_hosts(line: str) -> Iterator[str]:
+    line = line.strip()
+    if not line.startswith('#'):
+        parts = line.split()
+        if parts:
+            yield parts[0]
+        if len(parts) > 1:
+            yield parts[1]
+        if len(parts) > 2:
+            yield parts[2]
+
+
+def iter_known_hosts() -> Iterator[str]:
+    yield from parts_yielder(lines_from_file('~/.ssh/config'), hosts_from_config_lines)
+    yield from parts_yielder(lines_from_file('~/.ssh/known_hosts'), hosts_from_known_hosts)
+    yield from parts_yielder(lines_from_file('/etc/ssh/ssh_known_hosts'), hosts_from_known_hosts)
+    yield from parts_yielder(lines_from_file('/etc/hosts'), hosts_from_hosts)
+    yield from parts_yielder(lines_from_command('getent', 'hosts'), hosts_from_hosts)
+
+
+@run_once
+def known_hosts() -> Tuple[str, ...]:
+    return tuple(sorted(filter(lambda x: '*' not in x and '[' not in x, set(iter_known_hosts()))))
+
+
+@run_once
+def ssh_options() -> Dict[str, str]:
+    stderr = subprocess.Popen(['ssh'], stderr=subprocess.PIPE).stderr
+    assert stderr is not None
+    raw = stderr.read().decode('utf-8')
+    ans: Dict[str, str] = {}
+    pos = 0
+    while True:
+        pos = raw.find('[', pos)
+        if pos < 0:
+            break
+        num = 1
+        epos = pos
+        while num > 0:
+            epos += 1
+            if raw[epos] not in '[]':
+                continue
+            num += 1 if raw[epos] == '[' else -1
+        q = raw[pos+1:epos]
+        pos = epos
+        if len(q) < 2 or q[0] != '-':
+            continue
+        if ' ' in q:
+            opt, desc = q.split(' ', 1)
+            ans[opt[1:]] = desc
+        else:
+            ans.update(dict.fromkeys(q[1:], ''))
+    return ans
+
+
+# option help {{{
+@run_once
+def option_help_map() -> Dict[str, str]:
+    ans: Dict[str, str] = {}
+    lines = '''
+-4  -- force ssh to use IPv4 addresses only
+-6  -- force ssh to use IPv6 addresses only
+-a  -- disable forwarding of authentication agent connection
+-A  -- enable forwarding of the authentication agent connection
+-B  -- bind to specified interface before attempting to connect
+-b  -- specify interface to transmit on
+-C  -- compress data
+-c  -- select encryption cipher
+-D  -- specify a dynamic port forwarding
+-E  -- append log output to file instead of stderr
+-e  -- set escape character
+-f  -- go to background
+-F  -- specify alternate config file
+-g  -- allow remote hosts to connect to local forwarded ports
+-G  -- output configuration and exit
+-i  -- select identity file
+-I  -- specify smartcard device
+-J  -- connect via a jump host
+-k  -- disable forwarding of GSSAPI credentials
+-K  -- enable GSSAPI-based authentication and forwarding
+-L  -- specify local port forwarding
+-l  -- specify login name
+-M  -- master mode for connection sharing
+-m  -- specify mac algorithms
+-N  -- don't execute a remote command
+-n  -- redirect stdin from /dev/null
+-O  -- control an active connection multiplexing master process
+-o  -- specify extra options
+-p  -- specify port on remote host
+-P  -- use non privileged port
+-Q  -- query parameters
+-q  -- quiet operation
+-R  -- specify remote port forwarding
+-s  -- invoke subsystem
+-S  -- specify location of control socket for connection sharing
+-T  -- disable pseudo-tty allocation
+-t  -- force pseudo-tty allocation
+-V  -- show version number
+-v  -- verbose mode (multiple increase verbosity, up to 3)
+-W  -- forward standard input and output to host
+-w  -- request tunnel device forwarding
+-x  -- disable X11 forwarding
+-X  -- enable (untrusted) X11 forwarding
+-Y  -- enable trusted X11 forwarding
+-y  -- send log info via syslog instead of stderr
+'''.splitlines()
+    for line in lines:
+        line = line.strip()
+        if line:
+            parts = line.split(maxsplit=2)
+            ans[parts[0]] = parts[2]
+    return ans
+# }}}
+
+
+# option names {{{
+@run_once
+def option_names() -> Tuple[str, ...]:
+    return tuple(filter(None, (
+        line.strip() for line in '''
+AddKeysToAgent
+AddressFamily
+BatchMode
+BindAddress
+CanonicalDomains
+CanonicalizeFallbackLocal
+CanonicalizeHostname
+CanonicalizeMaxDots
+CanonicalizePermittedCNAMEs
+CASignatureAlgorithms
+CertificateFile
+ChallengeResponseAuthentication
+CheckHostIP
+Ciphers
+ClearAllForwardings
+Compression
+ConnectionAttempts
+ConnectTimeout
+ControlMaster
+ControlPath
+ControlPersist
+DynamicForward
+EscapeChar
+ExitOnForwardFailure
+FingerprintHash
+ForwardAgent
+ForwardX11
+ForwardX11Timeout
+ForwardX11Trusted
+GatewayPorts
+GlobalKnownHostsFile
+GSSAPIAuthentication
+GSSAPIDelegateCredentials
+HashKnownHosts
+Host
+HostbasedAcceptedAlgorithms
+HostbasedAuthentication
+HostKeyAlgorithms
+HostKeyAlias
+Hostname
+IdentitiesOnly
+IdentityAgent
+IdentityFile
+IPQoS
+KbdInteractiveAuthentication
+KbdInteractiveDevices
+KexAlgorithms
+KnownHostsCommand
+LocalCommand
+LocalForward
+LogLevel
+MACs
+Match
+NoHostAuthenticationForLocalhost
+NumberOfPasswordPrompts
+PasswordAuthentication
+PermitLocalCommand
+PermitRemoteOpen
+PKCS11Provider
+Port
+PreferredAuthentications
+ProxyCommand
+ProxyJump
+ProxyUseFdpass
+PubkeyAcceptedAlgorithms
+PubkeyAuthentication
+RekeyLimit
+RemoteCommand
+RemoteForward
+RequestTTY
+SendEnv
+ServerAliveInterval
+ServerAliveCountMax
+SetEnv
+StreamLocalBindMask
+StreamLocalBindUnlink
+StrictHostKeyChecking
+TCPKeepAlive
+Tunnel
+TunnelDevice
+UpdateHostKeys
+User
+UserKnownHostsFile
+VerifyHostKeyDNS
+VisualHostKey
+XAuthLocation
+'''.splitlines())))
+# }}}
+
+
+def complete_choices(ans: Completions, prefix: str, title: str, choices: Iterable[str], comma_separated: bool = False) -> None:
+    matches: Dict[str, str] = {}
+    word_transforms = {}
+    effective_prefix = prefix
+    hidden_prefix = ''
+    if comma_separated:
+        effective_prefix = prefix.split(',')[-1]
+        hidden_prefix = ','.join(prefix.split(',')[:-1])
+        if hidden_prefix:
+            hidden_prefix += ','
+    for q in choices:
+        if q.startswith(effective_prefix):
+            if comma_separated:
+                tq = q
+                q = hidden_prefix + q + ','
+                word_transforms[q] = tq
+            matches[q] = ''
+    ans.add_match_group(title, matches, trailing_space=not comma_separated, word_transforms=word_transforms)
+
+
+def complete_q_choices(ans: Completions, prefix: str, title: str, key: str, comma_separated: bool) -> None:
+    choices = (line.strip() for line in lines_from_command('ssh', '-Q', key))
+    complete_choices(ans, prefix, title, choices, comma_separated)
+
+
+def complete_arg(ans: Completions, option_flag: str, prefix: str = '') -> None:
+    options = ssh_options()
+    option_name = options.get(option_flag[1:])
+    if option_name.endswith('file') or option_name.endswith('path'):
+        return complete_files_and_dirs(ans, prefix, option_name)
+    choices = {
+        'mac_spec': ('MAC algorithm', 'mac', True),
+        'cipher_spec': ('encryption cipher', 'cipher', True),
+        'query_option': ('query option', 'help', False),
+    }
+    if option_name in choices:
+        return complete_q_choices(ans, prefix, *choices[option_name])
+    if option_name == 'destination':
+        return complete_destination(ans, prefix)
+    if option_name == 'ctl_cmd':
+        return complete_choices(ans, prefix, 'control command', ('check', 'forward', 'cancel', 'exit'))
+    if option_name == 'option':
+        matches = (x+'=' for x in option_names() if x.startswith(prefix))
+        word_transforms = {x+'=': x for x in option_names()}
+        ans.add_match_group('configure file option', matches, trailing_space=False, word_transforms=word_transforms)
+
+
+def complete_destination(ans: Completions, prefix: str = '') -> None:
+    result = (k for k in known_hosts() if k.startswith(prefix))
+    ans.add_match_group('remote host name', result)
+
+
+def complete_option(ans: Completions, prefix: str = '-') -> None:
+    hm = option_help_map()
+    if len(prefix) <= 1:
+        result = {k: v for k, v in hm.items() if k.startswith(prefix)}
+        ans.add_match_group('option', result)
+    else:
+        ans.add_match_group('option', {prefix: ''})
+
+
+def complete(ans: Completions, words: Sequence[str], new_word: bool) -> None:
+    options = ssh_options()
+    expecting_arg = False
+    types = ['' for i in range(len(words))]
+    for i, word in enumerate(words):
+        if expecting_arg:
+            types[i] = 'arg'
+            expecting_arg = False
+            continue
+        if word.startswith('-'):
+            types[i] = 'option'
+            if len(word) == 2 and options.get(word[1]):
+                expecting_arg = True
+            continue
+        types[i] = 'destination'
+        break
+    if new_word:
+        if words:
+            if expecting_arg:
+                return complete_arg(ans, words[-1])
+        return complete_destination(ans)
+    if words:
+        if types[-1] == 'arg' and len(words) > 1:
+            return complete_arg(ans, words[-2], words[-1])
+        if types[-1] == 'destination':
+            return complete_destination(ans, words[-1])
+        if types[-1] == 'option':
+            return complete_option(ans, words[-1])

kittens/ssh/main.py

@@ -3,12 +3,12 @@
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
 import os
-import re
 import shlex
 import subprocess
 import sys
 from contextlib import suppress
 from typing import List, NoReturn, Optional, Set, Tuple
+from .completion import ssh_options, complete
 
 from kitty.utils import SSHConnectionData
 
@@ -21,28 +21,83 @@ cat >$tmp << 'TERMEOF'
 TERMINFO
 TERMEOF
 
-tic_out=$(tic -x -o ~/.terminfo $tmp 2>&1)
+tic_out=$(tic -x -o $HOME/.terminfo $tmp 2>&1)
 rc=$?
 rm $tmp
 if [ "$rc" != "0" ]; then echo "$tic_out"; exit 1; fi
 if [ -z "$USER" ]; then export USER=$(whoami); fi
+export TERMINFO="$HOME/.terminfo"
+login_shell=""
+python=""
+
+login_shell_is_ok() {
+    if [ -z "$login_shell" ] || [ ! -x "$login_shell" ]; then return 1; fi
+    case "$login_shell" in
+        *sh) return 0;
+    esac
+    return 1;
+}
+
+detect_python() {
+    python=$(command -v python3)
+    if [ -z "$python" ]; then python=$(command -v python2); fi
+    if [ -z "$python" ]; then python=python; fi
+}
+
+using_getent() {
+    cmd=$(command -v getent)
+    if [ -z "$cmd" ]; then return; fi
+    output=$($cmd passwd $USER 2>/dev/null)
+    if [ $? = 0 ]; then login_shell=$(echo $output | cut -d: -f7); fi
+}
+
+using_id() {
+    cmd=$(command -v id)
+    if [ -z "$cmd" ]; then return; fi
+    output=$($cmd -P $USER 2>/dev/null)
+    if [ $? = 0 ]; then login_shell=$(echo $output | cut -d: -f7); fi
+}
+
+using_passwd() {
+    cmd=$(command -v grep)
+    if [ -z "$cmd" ]; then return; fi
+    output=$($cmd "^$USER:" /etc/passwd 2>/dev/null)
+    if [ $? = 0 ]; then login_shell=$(echo $output | cut -d: -f7); fi
+}
+
+using_python() {
+    detect_python
+    if [ ! -x "$python" ]; then return; fi
+    output=$($python -c "import pwd, os; print(pwd.getpwuid(os.geteuid()).pw_shell)")
+    if [ $? = 0 ]; then login_shell=$output; fi
+}
+
+execute_with_python() {
+    detect_python
+    exec $python -c "import os; os.execl('$login_shell', '-' '$shell_name')"
+}
+
+die() { echo "$*" 1>&2 ; exit 1; }
+
+using_getent
+if ! login_shell_is_ok; then using_id; fi
+if ! login_shell_is_ok; then using_python; fi
+if ! login_shell_is_ok; then using_passwd; fi
+if ! login_shell_is_ok; then die "Could not detect login shell"; fi
+
+
+# If a command was passed to SSH execute it here
 EXEC_CMD
-shell_name=$(basename $0)
 
 # We need to pass the first argument to the executed program with a leading -
 # to make sure the shell executes as a login shell. Note that not all shells
 # support exec -a so we use the below to try to detect such shells
-
-case "dash" in
-    *$shell_name*)
-        python=$(command -v python3)
-        if [ -z "$python" ]; then python=$(command -v python2); fi
-        if [ -z "$python" ]; then python=python; fi
-        exec $python -c "import os; os.execlp('$0', '-' '$shell_name')"
-    ;;
-esac
-
-exec -a "-$shell_name" "$0"
+shell_name=$(basename $login_shell)
+if [ -z "$PIPESTATUS" ]; then
+    # the dash shell does not support exec -a and also does not define PIPESTATUS
+    execute_with_python
+fi
+exec -a "-$shell_name" $login_shell
 '''
 
 
@@ -62,32 +117,27 @@ with NamedTemporaryFile() as tmp:
         getattr(sys.stderr, 'buffer', sys.stderr).write(stdout + stderr)
         raise SystemExit('Failed to compile terminfo using tic')
 command_to_execute = json.loads(binascii.unhexlify('{command_to_execute}'))
-if command_to_execute:
-    os.execlp(command_to_execute[0], *command_to_execute)
 try:
     shell_path = pwd.getpwuid(os.geteuid()).pw_shell or '/bin/sh'
 except KeyError:
     shell_path = '/bin/sh'
 shell_name = '-' + os.path.basename(shell_path)
+if command_to_execute:
+    os.execlp(shell_path, shell_path, '-c', command_to_execute)
 os.execlp(shell_path, shell_name)
 '''
 
 
 def get_ssh_cli() -> Tuple[Set[str], Set[str]]:
-    other_ssh_args: List[str] = []
-    boolean_ssh_args: List[str] = []
-    stderr = subprocess.Popen(['ssh'], stderr=subprocess.PIPE).stderr
-    assert stderr is not None
-    raw = stderr.read().decode('utf-8')
-    for m in re.finditer(r'\[(.+?)\]', raw):
-        q = m.group(1)
-        if len(q) < 2 or q[0] != '-':
-            continue
-        if ' ' in q:
-            other_ssh_args.append(q[1])
+    other_ssh_args: Set[str] = set()
+    boolean_ssh_args: Set[str] = set()
+    for k, v in ssh_options().items():
+        k = '-' + k
+        if v:
+            other_ssh_args.add(k)
         else:
-            boolean_ssh_args.extend(q[1:])
-    return set('-' + x for x in boolean_ssh_args), set('-' + x for x in other_ssh_args)
+            boolean_ssh_args.add(k)
+    return boolean_ssh_args, other_ssh_args
 
 
 def get_connection_data(args: List[str]) -> Optional[SSHConnectionData]:
@@ -179,20 +229,22 @@ def quote(x: str) -> str:
 
 def get_posix_cmd(terminfo: str, remote_args: List[str]) -> List[str]:
     sh_script = SHELL_SCRIPT.replace('TERMINFO', terminfo, 1)
+    command_to_execute = ''
     if remote_args:
-        command_to_executeg = (quote(c) for c in remote_args)
-        command_to_execute = 'exec ' + ' '.join(command_to_executeg)
-    else:
-        command_to_execute = ''
+        # ssh simply concatenates multiple commands using a space see
+        # line 1129 of ssh.c and on the remote side sshd.c runs the
+        # concatenated command as shell -c cmd
+        args = [c.replace("'", """'"'"'""") for c in remote_args]
+        command_to_execute = "exec $login_shell -c '{}'".format(' '.join(args))
     sh_script = sh_script.replace('EXEC_CMD', command_to_execute)
-    return [sh_script] + remote_args
+    return [f'sh -c {shlex.quote(sh_script)}']
 
 
 def get_python_cmd(terminfo: str, command_to_execute: List[str]) -> List[str]:
     import json
     script = PYTHON_SCRIPT.format(
         terminfo=terminfo.encode('utf-8').hex(),
-        command_to_execute=json.dumps(command_to_execute).encode('utf-8').hex()
+        command_to_execute=json.dumps(' '.join(command_to_execute)).encode('utf-8').hex()
     )
     return [f'python -c "{script}"']
 
@@ -209,8 +261,10 @@ def main(args: List[str]) -> NoReturn:
         cmd += server_args
     else:
         hostname, remote_args = server_args[0], server_args[1:]
-        cmd += ['-t', hostname]
-        terminfo = subprocess.check_output(['infocmp']).decode('utf-8')
+        if not remote_args:
+            cmd.append('-t')
+        cmd.append(hostname)
+        terminfo = subprocess.check_output(['infocmp', '-a']).decode('utf-8')
         f = get_posix_cmd if use_posix else get_python_cmd
         cmd += f(terminfo, remote_args)
     os.execvp('ssh', cmd)
@@ -218,3 +272,5 @@ def main(args: List[str]) -> NoReturn:
 
 if __name__ == '__main__':
     main(sys.argv)
+elif __name__ == '__completer__':
+    setattr(sys, 'kitten_completer', complete)

kittens/tui/handler.py

@@ -11,7 +11,7 @@ from typing import (
 from kitty.types import ParsedShortcut
 from kitty.typing import (
     AbstractEventLoop, BossType, Debug, ImageManagerType, KeyEventType,
-    KittensKeyActionType, LoopType, MouseEvent, ScreenSize, TermManagerType
+    KeyActionType, LoopType, MouseEvent, ScreenSize, TermManagerType
 )
 
 
@@ -46,15 +46,15 @@ class Handler:
     def asyncio_loop(self) -> AbstractEventLoop:
         return self._tui_loop.asycio_loop
 
-    def add_shortcut(self, action: KittensKeyActionType, spec: Union[str, ParsedShortcut]) -> None:
+    def add_shortcut(self, action: KeyActionType, spec: Union[str, ParsedShortcut]) -> None:
         if not hasattr(self, '_key_shortcuts'):
-            self._key_shortcuts: Dict[ParsedShortcut, KittensKeyActionType] = {}
+            self._key_shortcuts: Dict[ParsedShortcut, KeyActionType] = {}
         if isinstance(spec, str):
             from kitty.key_encoding import parse_shortcut
             spec = parse_shortcut(spec)
         self._key_shortcuts[spec] = action
 
-    def shortcut_action(self, key_event: KeyEventType) -> Optional[KittensKeyActionType]:
+    def shortcut_action(self, key_event: KeyEventType) -> Optional[KeyActionType]:
         for sc, action in self._key_shortcuts.items():
             if key_event.matches(sc):
                 return action

kittens/tui/images.py

@@ -15,13 +15,12 @@ from typing import (
     Sequence, Tuple, Union
 )
 
+from kitty.conf.utils import positive_float, positive_int
 from kitty.fast_data_types import create_canvas
 from kitty.typing import (
     CompletedProcess, GRT_a, GRT_d, GRT_f, GRT_m, GRT_o, GRT_t, HandlerType
 )
-from kitty.utils import (
-    ScreenSize, find_exe, fit_image, positive_float, positive_int
-)
+from kitty.utils import ScreenSize, find_exe, fit_image
 
 from .operations import cursor
 

kittens/tui/operations_stub.py

@@ -9,6 +9,6 @@ class CMD:
 
 def generate_stub() -> None:
     from kittens.tui.operations import as_type_stub
-    from kitty.conf.definition import save_type_stub
+    from kitty.conf.utils import save_type_stub
     text = as_type_stub()
     save_type_stub(text, __file__)

kittens/unicode_input/main.py

@@ -67,7 +67,7 @@ def name(cp: Union[int, str]) -> str:
 
 
 @lru_cache(maxsize=256)
-def codepoints_matching_search(parts: Sequence[str]) -> List[int]:
+def codepoints_matching_search(parts: Tuple[str, ...]) -> List[int]:
     ans = []
     if parts and parts[0] and len(parts[0]) > 1:
         codepoints = points_for_word(parts[0])
@@ -416,7 +416,7 @@ class UnicodeInput(Handler):
         self.refresh()
 
     def on_key(self, key_event: KeyEvent) -> None:
-        if self.mode is HEX and key_event.type is not EventType.RELEASE and not key_event.mods:
+        if self.mode is HEX and key_event.type is not EventType.RELEASE and not key_event.has_mods:
             try:
                 val = int(self.line_edit.current_input, 16)
             except Exception:
@@ -434,7 +434,7 @@ class UnicodeInput(Handler):
                     self.line_edit.current_input = hex(val - 1)[2:]
                     self.refresh()
                     return
-        if self.mode is NAME and key_event.type is not EventType.RELEASE and not key_event.mods:
+        if self.mode is NAME and key_event.type is not EventType.RELEASE and not key_event.has_mods:
             if key_event.matches('shift+tab'):
                 self.table.move_current(cols=-1)
                 self.refresh()

kittens/unicode_input/unicode_names.c

@@ -19,27 +19,31 @@ all_words(PYNOARG) {
     return ans;
 }
 
-static inline void
-add_matches(const word_trie *wt, char_type *codepoints, size_t *pos, const size_t sz) {
+static void
+add_matches(const word_trie *wt, PyObject *ans) {
     size_t num = mark_groups[wt->match_offset];
-    for (size_t i = wt->match_offset + 1; i < wt->match_offset + 1 + num && *pos < sz; i++, (*pos)++) {
-        codepoints[*pos] = mark_to_cp[mark_groups[i]];
+    for (size_t i = wt->match_offset + 1; i < wt->match_offset + 1 + num; i++) {
+        PyObject *t = PyLong_FromUnsignedLong(mark_to_cp[mark_groups[i]]);
+        if (!t) return;
+        int ret = PySet_Add(ans, t);
+        Py_DECREF(t);
+        if (ret != 0) return;
     }
 }
 
 static void
-process_trie_node(const word_trie *wt, char_type *codepoints, size_t *pos, const size_t sz) {
-    if (wt->match_offset) add_matches(wt, codepoints, pos, sz);
+process_trie_node(const word_trie *wt, PyObject *ans) {
+    if (wt->match_offset) { add_matches(wt, ans); if (PyErr_Occurred()) return; }
     size_t num_children = children_array[wt->children_offset];
     if (!num_children) return;
     for (size_t c = wt->children_offset + 1; c < wt->children_offset + 1 + num_children; c++) {
-        if (*pos > sz) return;
         uint32_t x = children_array[c];
-        process_trie_node(&all_trie_nodes[x >> 8], codepoints, pos, sz);
+        process_trie_node(&all_trie_nodes[x >> 8], ans);
+        if (PyErr_Occurred()) return;
     }
 }
 
-static inline PyObject*
+static PyObject*
 codepoints_for_word(const char *word, size_t len) {
     const word_trie *wt = all_trie_nodes;
     for (size_t i = 0; i < len; i++) {
@@ -57,14 +61,10 @@ codepoints_for_word(const char *word, size_t len) {
         }
         if (!found) return PyFrozenSet_New(NULL);
     }
-    static char_type codepoints[1024];
-    size_t cpos = 0;
-    process_trie_node(wt, codepoints, &cpos, arraysz(codepoints));
-    PyObject *ans = PyFrozenSet_New(NULL); if (ans == NULL) return NULL;
-    for (size_t i = 0; i < cpos; i++) {
-        PyObject *t = PyLong_FromUnsignedLong(codepoints[i]); if (t == NULL) { Py_DECREF(ans); return NULL; }
-        int ret = PySet_Add(ans, t); Py_DECREF(t); if (ret != 0) { Py_DECREF(ans); return NULL; }
-    }
+    PyObject *ans = PyFrozenSet_New(NULL);
+    if (!ans) return NULL;
+    process_trie_node(wt, ans);
+    if (PyErr_Occurred()) return NULL;
     return ans;
 }
 

kitty/borders.py

@@ -7,10 +7,9 @@ from itertools import chain
 from typing import Sequence, Tuple
 
 from .fast_data_types import (
-    BORDERS_PROGRAM, add_borders_rect, compile_program, init_borders_program,
-    os_window_has_background_image
+    BORDERS_PROGRAM, add_borders_rect, compile_program, get_options,
+    init_borders_program, os_window_has_background_image
 )
-from .options_stub import Options
 from .typing import LayoutType
 from .utils import load_shaders
 from .window_list import WindowGroup, WindowList
@@ -65,11 +64,9 @@ def load_borders_program() -> None:
 
 class Borders:
 
-    def __init__(self, os_window_id: int, tab_id: int, opts: Options):
+    def __init__(self, os_window_id: int, tab_id: int):
         self.os_window_id = os_window_id
         self.tab_id = tab_id
-        self.draw_active_borders = opts.active_border_color is not None
-        self.draw_minimal_borders = opts.draw_minimal_borders and max(opts.window_margin_width) < 1
 
     def __call__(
         self,
@@ -78,6 +75,9 @@ class Borders:
         extra_blank_rects: Sequence[Tuple[int, int, int, int]],
         draw_window_borders: bool = True,
     ) -> None:
+        opts = get_options()
+        draw_active_borders = opts.active_border_color is not None
+        draw_minimal_borders = opts.draw_minimal_borders and max(opts.window_margin_width) < 1
         add_borders_rect(self.os_window_id, self.tab_id, 0, 0, 0, 0, BorderColor.default_bg)
         has_background_image = os_window_has_background_image(self.os_window_id)
         if not has_background_image:
@@ -94,9 +94,9 @@ class Borders:
         for i, wg in enumerate(groups):
             window_bg = wg.default_bg
             window_bg = (window_bg << 8) | BorderColor.window_bg
-            if draw_borders and not self.draw_minimal_borders:
+            if draw_borders and not draw_minimal_borders:
                 # Draw the border rectangles
-                if wg is active_group and self.draw_active_borders:
+                if wg is active_group and draw_active_borders:
                     color = BorderColor.active
                 else:
                     color = BorderColor.bell if wg.needs_attention else BorderColor.inactive
@@ -106,7 +106,7 @@ class Borders:
                 colors = window_bg, window_bg, window_bg, window_bg
                 draw_edges(self.os_window_id, self.tab_id, colors, wg)
 
-        if self.draw_minimal_borders:
-            for border_line in current_layout.minimal_borders(all_windows):
+        if draw_minimal_borders:
+            for border_line in current_layout.get_minimal_borders(all_windows):
                 left, top, right, bottom = border_line.edges
                 add_borders_rect(self.os_window_id, self.tab_id, left, top, right, bottom, border_line.color)

kitty/boss.py

@@ -15,33 +15,34 @@ from typing import (
 )
 from weakref import WeakValueDictionary
 
-from .child import cached_process_data, cwd_of_process
+from .child import (
+    cached_process_data, cwd_of_process, default_env, set_default_env
+)
 from .cli import create_opts, parse_args
 from .cli_stub import CLIOptions
-from .conf.utils import BadLine, to_cmdline
-from .config import (
-    KeyAction, SubSequenceMap, common_opts_as_dict,
-    prepare_config_file_for_editing
-)
-from .config_data import MINIMUM_FONT_SIZE
+from .conf.utils import BadLine, KeyAction, to_cmdline
+from .config import common_opts_as_dict, prepare_config_file_for_editing
 from .constants import (
-    appname, config_dir, is_macos, kitty_exe, supports_primary_selection
+    appname, config_dir, is_macos, is_wayland, kitty_exe,
+    supports_primary_selection
 )
 from .fast_data_types import (
     CLOSE_BEING_CONFIRMED, IMPERATIVE_CLOSE_REQUESTED, NO_CLOSE_REQUESTED,
-    ChildMonitor, KeyEvent, add_timer, background_opacity_of,
-    change_background_opacity, change_os_window_state, cocoa_set_menubar_title,
-    create_os_window, current_application_quit_request, current_os_window,
-    destroy_global_data, focus_os_window, get_clipboard_string,
-    global_font_size, mark_os_window_for_close, os_window_font_size,
-    patch_global_colors, safe_pipe, set_application_quit_request,
-    set_background_image, set_boss, set_clipboard_string, set_in_sequence_mode,
-    thread_write, toggle_fullscreen, toggle_maximized
+    ChildMonitor, KeyEvent, add_timer, apply_options_update,
+    background_opacity_of, change_background_opacity, change_os_window_state,
+    cocoa_set_menubar_title, create_os_window,
+    current_application_quit_request, current_os_window, destroy_global_data,
+    focus_os_window, get_clipboard_string, get_options, global_font_size,
+    mark_os_window_for_close, os_window_font_size, patch_global_colors,
+    safe_pipe, set_application_quit_request, set_background_image, set_boss,
+    set_clipboard_string, set_in_sequence_mode, set_options, thread_write,
+    toggle_fullscreen, toggle_maximized
 )
 from .keys import get_shortcut, shortcut_matches
 from .layout.base import set_layout_options
 from .notify import notification_activated
-from .options_stub import Options
+from .options.types import Options
+from .options.utils import MINIMUM_FONT_SIZE, SubSequenceMap
 from .os_window_size import initial_window_size_func
 from .rgb import Color, color_from_int
 from .session import Session, create_sessions, get_os_window_sizing_data
@@ -53,8 +54,8 @@ from .typing import PopenType, TypedDict
 from .utils import (
     func_name, get_editor, get_primary_selection, is_path_in_temp_dir,
     log_error, open_url, parse_address_spec, parse_uri_list,
-    platform_window_id, remove_socket_file, safe_print, set_primary_selection,
-    single_instance, startup_notification_handler
+    platform_window_id, read_shell_environment, remove_socket_file, safe_print,
+    set_primary_selection, single_instance, startup_notification_handler
 )
 from .window import MatchPatternType, Window
 
@@ -167,19 +168,23 @@ class Boss:
             talk_fd, listen_fd
         )
         set_boss(self)
-        self.opts, self.args = opts, args
-        self.keymap = self.opts.keymap.copy()
+        self.args = args
         self.global_shortcuts_map = {v: KeyAction(k) for k, v in global_shortcuts.items()}
-        for sc in global_shortcuts.values():
-            self.keymap.pop(sc, None)
+        self.global_shortcuts = global_shortcuts
+        self.update_keymap()
         if is_macos:
             from .fast_data_types import (
                 cocoa_set_notification_activated_callback
             )
             cocoa_set_notification_activated_callback(notification_activated)
 
+    def update_keymap(self) -> None:
+        self.keymap = get_options().keymap.copy()
+        for sc in self.global_shortcuts.values():
+            self.keymap.pop(sc, None)
+
     def startup_first_child(self, os_window_id: Optional[int]) -> None:
-        startup_sessions = create_sessions(self.opts, self.args, default_session=self.opts.startup_session)
+        startup_sessions = create_sessions(get_options(), self.args, default_session=get_options().startup_session)
         for startup_session in startup_sessions:
             self.add_os_window(startup_session, os_window_id=os_window_id)
             os_window_id = None
@@ -199,7 +204,7 @@ class Boss:
         startup_id: Optional[str] = None
     ) -> int:
         if os_window_id is None:
-            size_data = get_os_window_sizing_data(opts_for_size or self.opts, startup_session)
+            size_data = get_os_window_sizing_data(opts_for_size or get_options(), startup_session)
             wclass = wclass or getattr(startup_session, 'os_window_class', None) or self.args.cls or appname
             wname = wname or self.args.name or wclass
             with startup_notification_handler(do_notify=startup_id is not None, startup_id=startup_id) as pre_show_callback:
@@ -210,7 +215,7 @@ class Boss:
         else:
             wname = self.args.name or self.args.cls or appname
             wclass = self.args.cls or appname
-        tm = TabManager(os_window_id, self.opts, self.args, wclass, wname, startup_session)
+        tm = TabManager(os_window_id, self.args, wclass, wname, startup_session)
         self.os_window_map[os_window_id] = tm
         return os_window_id
 
@@ -294,6 +299,12 @@ class Boss:
                 if q:
                     found = True
                     yield q
+        elif field == 'index':
+            tm = self.active_tab_manager
+            if tm is not None and len(tm.tabs) > 0:
+                idx = (int(pat.pattern) + len(tm.tabs)) % len(tm.tabs)
+                found = True
+                yield tm.tabs[idx]
         if not found:
             tabs = {self.tab_for_window(w) for w in self.match_windows(match)}
             for q in tabs:
@@ -317,7 +328,7 @@ class Boss:
             sw: Optional[SpecialWindowInstance] = args
         else:
             sw = self.args_to_special_window(args, cwd_from) if args else None
-        startup_session = next(create_sessions(self.opts, special_window=sw, cwd_from=cwd_from))
+        startup_session = next(create_sessions(get_options(), special_window=sw, cwd_from=cwd_from))
         return self.add_os_window(startup_session)
 
     def new_os_window(self, *args: str) -> None:
@@ -347,7 +358,7 @@ class Boss:
         from .remote_control import handle_cmd
         response = None
         window = window or None
-        if self.opts.allow_remote_control == 'y' or from_peer or getattr(window, 'allow_remote_control', False):
+        if get_options().allow_remote_control == 'y' or from_peer or getattr(window, 'allow_remote_control', False):
             try:
                 response = handle_cmd(self, window, cmd)
             except Exception as err:
@@ -411,7 +422,7 @@ class Boss:
                 args.directory = os.path.join(data['cwd'], args.directory)
             for session in create_sessions(opts, args, respect_cwd=True):
                 os_window_id = self.add_os_window(session, wclass=args.cls, wname=args.name, opts_for_size=opts, startup_id=startup_id)
-                if opts.background_opacity != self.opts.background_opacity:
+                if opts.background_opacity != get_options().background_opacity:
                     self._set_os_window_background_opacity(os_window_id, opts.background_opacity)
                 if data.get('notify_on_os_window_death'):
                     self.os_window_death_actions[os_window_id] = partial(self.notify_on_os_window_death, data['notify_on_os_window_death'])
@@ -436,6 +447,7 @@ class Boss:
                         mark_os_window_for_close(src_tab.os_window_id)
 
     def on_child_death(self, window_id: int) -> None:
+        prev_active_window = self.active_window
         window = self.window_id_map.pop(window_id, None)
         if window is None:
             return
@@ -464,6 +476,12 @@ class Boss:
                 import traceback
                 traceback.print_exc()
         window.action_on_close = window.action_on_removal = None
+        window = self.active_window
+        if window is not prev_active_window:
+            if prev_active_window is not None:
+                prev_active_window.focus_changed(False)
+            if window is not None:
+                window.focus_changed(True)
 
     def close_window(self, window: Optional[Window] = None) -> None:
         window = window or self.active_window
@@ -477,7 +495,7 @@ class Boss:
 
     def confirm_tab_close(self, tab: Tab) -> None:
         windows = tuple(tab)
-        needs_confirmation = self.opts.confirm_os_window_close > 0 and len(windows) >= self.opts.confirm_os_window_close
+        needs_confirmation = get_options().confirm_os_window_close > 0 and len(windows) >= get_options().confirm_os_window_close
         if not needs_confirmation:
             self.close_tab_no_confirm(tab)
             return
@@ -514,9 +532,9 @@ class Boss:
             self.io_thread_started = True
             self.startup_first_child(first_os_window_id)
 
-        if self.opts.update_check_interval > 0 and not hasattr(self, 'update_check_started'):
+        if get_options().update_check_interval > 0 and not hasattr(self, 'update_check_started'):
             from .update_check import run_update_check
-            run_update_check(self.opts.update_check_interval * 60 * 60)
+            run_update_check(get_options().update_check_interval * 60 * 60)
             self.update_check_started = True
 
     def activate_tab_at(self, os_window_id: int, x: int, is_double: bool = False) -> int:
@@ -554,14 +572,14 @@ class Boss:
 
     def increase_font_size(self) -> None:  # legacy
         cfs = global_font_size()
-        self.set_font_size(min(self.opts.font_size * 5, cfs + 2.0))
+        self.set_font_size(min(get_options().font_size * 5, cfs + 2.0))
 
     def decrease_font_size(self) -> None:  # legacy
         cfs = global_font_size()
         self.set_font_size(max(MINIMUM_FONT_SIZE, cfs - 2.0))
 
     def restore_font_size(self) -> None:  # legacy
-        self.set_font_size(self.opts.font_size)
+        self.set_font_size(get_options().font_size)
 
     def set_font_size(self, new_size: float) -> None:  # legacy
         self.change_font_size(True, None, new_size)
@@ -570,13 +588,13 @@ class Boss:
         def calc_new_size(old_size: float) -> float:
             new_size = old_size
             if amt == 0:
-                new_size = self.opts.font_size
+                new_size = get_options().font_size
             else:
                 if increment_operation:
                     new_size += (1 if increment_operation == '+' else -1) * amt
                 else:
                     new_size = amt
-                new_size = max(MINIMUM_FONT_SIZE, min(new_size, self.opts.font_size * 5))
+                new_size = max(MINIMUM_FONT_SIZE, min(new_size, get_options().font_size * 5))
             return new_size
 
         if all_windows:
@@ -626,7 +644,7 @@ class Boss:
         window = self.active_window
         if window is None or not opacity:
             return
-        if not self.opts.dynamic_background_opacity:
+        if not get_options().dynamic_background_opacity:
             self.show_error(
                     _('Cannot change background opacity'),
                     _('You must set the dynamic_background_opacity option in kitty.conf to be able to change background opacity'))
@@ -638,7 +656,7 @@ class Boss:
                 return
             fin_opacity = old_opacity + float(opacity)
         elif opacity == 'default':
-            fin_opacity = self.opts.background_opacity
+            fin_opacity = get_options().background_opacity
         else:
             fin_opacity = float(opacity)
         self._set_os_window_background_opacity(os_window_id, fin_opacity)
@@ -665,7 +683,7 @@ class Boss:
         # Handles shortcuts, return True if the key was consumed
         key_action = get_shortcut(self.keymap, ev)
         if key_action is None:
-            sequences = get_shortcut(self.opts.sequence_map, ev)
+            sequences = get_shortcut(get_options().sequence_map, ev)
             if sequences and not isinstance(sequences, KeyAction):
                 self.pending_sequences = sequences
                 set_in_sequence_mode(True)
@@ -703,8 +721,8 @@ class Boss:
         if w is None:
             return
         overlay_window = self._run_kitten('resize_window', args=[
-            '--horizontal-increment={}'.format(self.opts.window_resize_step_cells),
-            '--vertical-increment={}'.format(self.opts.window_resize_step_lines)
+            '--horizontal-increment={}'.format(get_options().window_resize_step_cells),
+            '--vertical-increment={}'.format(get_options().window_resize_step_lines)
         ])
         if overlay_window is not None:
             overlay_window.allow_remote_control = True
@@ -729,27 +747,37 @@ class Boss:
                 if t is not None:
                     t.relayout_borders()
 
-    def dispatch_action(self, key_action: KeyAction) -> bool:
+    def dispatch_action(
+        self,
+        key_action: KeyAction,
+        window_for_dispatch: Optional[Window] = None,
+        dispatch_type: str = 'KeyPress'
+    ) -> bool:
+
+        def report_match(f: Callable) -> None:
+            if self.args.debug_keyboard:
+                print(f'\x1b[35m{dispatch_type}\x1b[m matched action:', func_name(f), flush=True)
+
         if key_action is not None:
             f = getattr(self, key_action.func, None)
             if f is not None:
-                if self.args.debug_keyboard:
-                    print('Keypress matched action:', func_name(f))
+                report_match(f)
                 passthrough = f(*key_action.args)
                 if passthrough is not True:
                     return True
-        tab = self.active_tab
-        if tab is None:
-            return False
-        window = self.active_window
-        if window is None:
+        if window_for_dispatch is None:
+            tab = self.active_tab
+            window = self.active_window
+        else:
+            window = window_for_dispatch
+            tab = window.tabref()
+        if tab is None or window is None:
             return False
         if key_action is not None:
             f = getattr(tab, key_action.func, getattr(window, key_action.func, None))
             if f is not None:
                 passthrough = f(*key_action.args)
-                if self.args.debug_keyboard:
-                    print('Keypress matched action:', func_name(f))
+                report_match(f)
                 if passthrough is not True:
                     return True
         return False
@@ -768,6 +796,12 @@ class Boss:
                     cocoa_set_menubar_title(w.title or '')
             tm.mark_tab_bar_dirty()
 
+    def on_activity_since_last_focus(self, window: Window) -> None:
+        os_window_id = window.os_window_id
+        tm = self.os_window_map.get(os_window_id)
+        if tm is not None:
+            tm.mark_tab_bar_dirty()
+
     def update_tab_bar_data(self, os_window_id: int) -> None:
         tm = self.os_window_map.get(os_window_id)
         if tm is not None:
@@ -790,7 +824,7 @@ class Boss:
 
     def confirm_os_window_close(self, os_window_id: int) -> None:
         tm = self.os_window_map.get(os_window_id)
-        needs_confirmation = tm is not None and self.opts.confirm_os_window_close > 0 and tm.number_of_windows >= self.opts.confirm_os_window_close
+        needs_confirmation = tm is not None and get_options().confirm_os_window_close > 0 and tm.number_of_windows >= get_options().confirm_os_window_close
         if not needs_confirmation:
             mark_os_window_for_close(os_window_id)
             return
@@ -827,7 +861,7 @@ class Boss:
         num = 0
         for q in self.os_window_map.values():
             num += q.number_of_windows
-        needs_confirmation = tm is not None and self.opts.confirm_os_window_close > 0 and num >= self.opts.confirm_os_window_close
+        needs_confirmation = tm is not None and get_options().confirm_os_window_close > 0 and num >= get_options().confirm_os_window_close
         if not needs_confirmation:
             set_application_quit_request(IMPERATIVE_CLOSE_REQUESTED)
             return
@@ -854,11 +888,31 @@ class Boss:
                 s.shutdown(socket.SHUT_RDWR)
             s.close()
 
-    def display_scrollback(self, window: Window, data: Optional[bytes], cmd: List[str]) -> None:
+    def display_scrollback(self, window: Window, data: Union[bytes, str], input_line_number: int = 0, title: str = '') -> None:
+        def prepare_arg(x: str) -> str:
+            x = x.replace('INPUT_LINE_NUMBER', str(input_line_number))
+            x = x.replace('CURSOR_LINE', str(window.screen.cursor.y + 1))
+            x = x.replace('CURSOR_COLUMN', str(window.screen.cursor.x + 1))
+            return x
+
+        cmd = list(map(prepare_arg, get_options().scrollback_pager))
+        if not os.path.isabs(cmd[0]):
+            import shutil
+            exe = shutil.which(cmd[0])
+            if not exe:
+                env = read_shell_environment(get_options())
+                if env and 'PATH' in env:
+                    exe = shutil.which(cmd[0], path=env['PATH'])
+                    if exe:
+                        cmd[0] = exe
+
+        if os.path.basename(cmd[0]) == 'less':
+            cmd.append('-+F')  # reset --quit-if-one-screen
         tab = self.active_tab
         if tab is not None:
+            bdata = data.encode('utf-8') if isinstance(data, str) else data
             tab.new_special_window(
-                SpecialWindow(cmd, data, _('History'), overlay_for=window.id, cwd=window.cwd_of_child),
+                SpecialWindow(cmd, bdata, title or _('History'), overlay_for=window.id, cwd=window.cwd_of_child),
                 copy_colors_from=self.active_window
                 )
 
@@ -866,7 +920,7 @@ class Boss:
         confpath = prepare_config_file_for_editing()
         # On macOS vim fails to handle SIGWINCH if it occurs early, so add a
         # small delay.
-        cmd = [kitty_exe(), '+runpy', 'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])'] + get_editor(self.opts) + [confpath]
+        cmd = [kitty_exe(), '+runpy', 'import os, sys, time; time.sleep(0.05); os.execvp(sys.argv[1], sys.argv[1:])'] + get_editor(get_options()) + [confpath]
         self.new_os_window(*cmd)
 
     def get_output(self, source_window: Window, num_lines: Optional[int] = 1) -> str:
@@ -918,7 +972,7 @@ class Boss:
                     raise ValueError('Unknown type_of_input: {}'.format(type_of_input))
             else:
                 data = input_data if isinstance(input_data, bytes) else input_data.encode('utf-8')
-            copts = common_opts_as_dict(self.opts)
+            copts = common_opts_as_dict(get_options())
             final_args: List[str] = []
             for x in args:
                 if x == '@selection':
@@ -1058,7 +1112,7 @@ class Boss:
                 if actions:
                     self.drain_actions(actions)
         if not found_action:
-            open_url(url, program or self.opts.open_url_with, cwd=cwd)
+            open_url(url, program or get_options().open_url_with, cwd=cwd)
 
     def drain_actions(self, actions: List) -> None:
 
@@ -1089,8 +1143,14 @@ class Boss:
         text = get_clipboard_string()
         self.paste_to_active_window(text)
 
+    def current_primary_selection(self) -> str:
+        return get_primary_selection() if supports_primary_selection else ''
+
+    def current_primary_selection_or_clipboard(self) -> str:
+        return get_primary_selection() if supports_primary_selection else get_clipboard_string()
+
     def paste_from_selection(self) -> None:
-        text = get_primary_selection() if supports_primary_selection else get_clipboard_string()
+        text = self.current_primary_selection_or_clipboard()
         self.paste_to_active_window(text)
 
     def set_primary_selection(self) -> None:
@@ -1099,8 +1159,8 @@ class Boss:
             text = w.text_for_selection()
             if text:
                 set_primary_selection(text)
-                if self.opts.copy_on_select:
-                    self.copy_to_buffer(self.opts.copy_on_select)
+                if get_options().copy_on_select:
+                    self.copy_to_buffer(get_options().copy_on_select)
 
     def copy_to_buffer(self, buffer_name: str) -> None:
         w = self.active_window
@@ -1147,7 +1207,10 @@ class Boss:
 
     prev_tab = previous_tab
 
-    def process_stdin_source(self, window: Optional[Window] = None, stdin: Optional[str] = None) -> Tuple[Optional[Dict[str, str]], Optional[bytes]]:
+    def process_stdin_source(
+        self, window: Optional[Window] = None,
+        stdin: Optional[str] = None, copy_pipe_data: Optional[Dict] = None
+    ) -> Tuple[Optional[Dict[str, str]], Optional[bytes]]:
         w = window or self.active_window
         if not w:
             return None, None
@@ -1161,6 +1224,8 @@ class Boss:
             if stdin is not None:
                 pipe_data = w.pipe_data(stdin, has_wrap_markers=add_wrap_markers) if w else None
                 if pipe_data:
+                    if copy_pipe_data is not None:
+                        copy_pipe_data.update(pipe_data)
                     env = {
                         'KITTY_PIPE_DATA':
                         '{scrolled_by}:{cursor_x},{cursor_y}:{lines},{columns}'.format(**pipe_data)
@@ -1203,6 +1268,11 @@ class Boss:
         cwd_from: Optional[int] = None
     ) -> None:
         import subprocess
+        env = env or None
+        if env:
+            env_ = default_env().copy()
+            env_.update(env)
+            env = env_
         if cwd_from:
             with suppress(Exception):
                 cwd = cwd_of_process(cwd_from)
@@ -1362,18 +1432,55 @@ class Boss:
             window.refresh()
 
     def patch_colors(self, spec: Dict[str, int], cursor_text_color: Union[bool, int, Color], configured: bool = False) -> None:
+        opts = get_options()
         if configured:
             for k, v in spec.items():
-                if hasattr(self.opts, k):
-                    setattr(self.opts, k, color_from_int(v))
+                if hasattr(opts, k):
+                    setattr(opts, k, color_from_int(v))
             if cursor_text_color is not False:
                 if isinstance(cursor_text_color, int):
                     cursor_text_color = color_from_int(cursor_text_color)
-                self.opts.cursor_text_color = cursor_text_color
+                opts.cursor_text_color = cursor_text_color
         for tm in self.all_tab_managers:
             tm.tab_bar.patch_colors(spec)
         patch_global_colors(spec, configured)
 
+    def apply_new_options(self, opts: Options) -> None:
+        from .fonts.box_drawing import set_scale
+
+        # Update options storage
+        set_options(opts, is_wayland(), self.args.debug_rendering, self.args.debug_font_fallback)
+        apply_options_update()
+        set_layout_options(opts)
+        set_default_env(opts.env.copy())
+        # Update font data
+        set_scale(opts.box_drawing_scale)
+        from .fonts.render import set_font_family
+        set_font_family(opts, debug_font_matching=self.args.debug_font_fallback)
+        for os_window_id, tm in self.os_window_map.items():
+            if tm is not None:
+                os_window_font_size(os_window_id, opts.font_size, True)
+                tm.resize()
+        # Update key bindings
+        self.update_keymap()
+        # Update misc options
+        for tm in self.all_tab_managers:
+            tm.apply_options()
+        # Update colors
+        for w in self.all_windows:
+            self.default_bg_changed_for(w.id)
+            w.refresh()
+
+    def load_config_file(self, *paths: str, apply_overrides: bool = True) -> None:
+        from .config import load_config
+        old_opts = get_options()
+        paths = paths or old_opts.config_paths
+        bad_lines: List[BadLine] = []
+        opts = load_config(*paths, overrides=old_opts.config_overrides if apply_overrides else None, accumulate_bad_lines=bad_lines)
+        if bad_lines:
+            self.show_bad_config_lines(bad_lines)
+        self.apply_new_options(opts)
+
     def safe_delete_temp_file(self, path: str) -> None:
         if is_path_in_temp_dir(path):
             with suppress(FileNotFoundError):
@@ -1640,8 +1747,8 @@ class Boss:
     def show_kitty_env_vars(self) -> None:
         w = self.active_window
         if w:
-            output = '\n'.join(f'{k}={v}' for k, v in os.environ.items()).encode('utf-8')
-            self.display_scrollback(w, output, ['less'])
+            output = '\n'.join(f'{k}={v}' for k, v in os.environ.items())
+            self.display_scrollback(w, output, title=_('Current kitty env vars'))
 
     def open_file(self, path: str) -> None:
         if path == ":cocoa::application launched::":
@@ -1660,3 +1767,16 @@ class Boss:
         self.new_window(path)
         if w is not None:
             tab.remove_window(w)
+
+    def debug_config(self) -> None:
+        from .debug_config import debug_config
+        w = self.active_window
+        if w is not None:
+            output = debug_config(get_options())
+            set_clipboard_string(re.sub(r'\x1b.+?m', '', output))
+            output += '\n\x1b[35mThis debug output has been copied to the clipboard\x1b[m'
+            self.display_scrollback(w, output, title=_('Current kitty options'))
+
+    def discard_event(self) -> None:
+        pass
+    mouse_discard_event = discard_event

kitty/cell_fragment.glsl

@@ -80,7 +80,7 @@ vec4 blend_onto_opaque_premul(vec3 over, float over_alpha, vec3 under) {
  *        1) Draw background for all cells
  *        2) Draw the images that are supposed to be below both the background and text, if any. This happens in the graphics shader
  *        3) Draw the background of cells that don't have the default background if any images were drawn in 2 above
-          4) Draw the images that are supposed to be below text but not background, again in graphics shader.
+ *        4) Draw the images that are supposed to be below text but not background, again in graphics shader.
  *        5) Draw the special cells (selection/cursor). Output is same as from step 1, with bg_alpha 1 for special cells and 0 otherwise
  *        6) Draw the foreground -- expected output is color with alpha which is blended using the opaque blend func
  *        7) Draw the images that are supposed to be above text again in the graphics shader

kitty/child-monitor.c

@@ -83,7 +83,7 @@ static unsigned long remove_notify[MAX_CHILDREN] = {0};
 static size_t add_queue_count = 0, remove_queue_count = 0;
 static struct pollfd fds[MAX_CHILDREN + EXTRA_FDS] = {{0}};
 static pthread_mutex_t children_lock, talk_lock;
-static bool kill_signal_received = false;
+static bool kill_signal_received = false, reload_config_signal_received = false;
 static ChildMonitor *the_monitor = NULL;
 
 typedef struct {
@@ -333,7 +333,7 @@ static bool
 parse_input(ChildMonitor *self) {
     // Parse all available input that was read in the I/O thread.
     size_t count = 0, remove_count = 0;
-    bool input_read = false;
+    bool input_read = false, reload_config_called = false;
     monotonic_t now = monotonic();
     children_mutex(lock);
     while (remove_queue_count) {
@@ -343,11 +343,17 @@ parse_input(ChildMonitor *self) {
         FREE_CHILD(remove_queue[remove_queue_count]);
     }
 
-    if (UNLIKELY(kill_signal_received)) {
-        global_state.quit_request = IMPERATIVE_CLOSE_REQUESTED;
-        global_state.has_pending_closes = true;
-        request_tick_callback();
-        kill_signal_received = false;
+    if (UNLIKELY(kill_signal_received || reload_config_signal_received)) {
+        if (kill_signal_received) {
+            global_state.quit_request = IMPERATIVE_CLOSE_REQUESTED;
+            global_state.has_pending_closes = true;
+            request_tick_callback();
+            kill_signal_received = false;
+        }
+        else if (reload_config_signal_received) {
+            reload_config_signal_received = false;
+            reload_config_called = true;
+        }
     } else {
         count = self->count;
         for (size_t i = 0; i < count; i++) {
@@ -402,6 +408,9 @@ parse_input(ChildMonitor *self) {
         }
         DECREF_CHILD(scratch[i]);
     }
+    if (reload_config_called) {
+        call_boss(load_config_file, "");
+    }
     return input_read;
 }
 
@@ -935,7 +944,8 @@ process_pending_closes(ChildMonitor *self) {
 // If we create new OS windows during wait_events(), using global menu actions
 // via the mouse causes a crash because of the way autorelease pools work in
 // glfw/cocoa. So we use a flag instead.
-static CocoaPendingAction cocoa_pending_actions = NO_COCOA_PENDING_ACTION;
+static bool cocoa_pending_actions[NUM_COCOA_PENDING_ACTIONS] = {0};
+static bool has_cocoa_pending_actions = false;
 typedef struct {
     char* wd;
     char **open_files;
@@ -955,11 +965,47 @@ set_cocoa_pending_action(CocoaPendingAction action, const char *wd) {
             cocoa_pending_actions_data.wd = strdup(wd);
         }
     }
-    cocoa_pending_actions |= action;
+    cocoa_pending_actions[action] = true;
+    has_cocoa_pending_actions = true;
     // The main loop may be blocking on the event queue, if e.g. unfocused.
     // Unjam it so the pending action is processed right now.
     wakeup_main_loop();
 }
+
+static void
+process_cocoa_pending_actions(void) {
+    if (cocoa_pending_actions[PREFERENCES_WINDOW]) { call_boss(edit_config_file, NULL); }
+    if (cocoa_pending_actions[NEW_OS_WINDOW]) { call_boss(new_os_window, NULL); }
+    if (cocoa_pending_actions[CLOSE_OS_WINDOW]) { call_boss(close_os_window, NULL); }
+    if (cocoa_pending_actions[CLOSE_TAB]) { call_boss(close_tab, NULL); }
+    if (cocoa_pending_actions[NEW_TAB]) { call_boss(new_tab, NULL); }
+    if (cocoa_pending_actions[NEXT_TAB]) { call_boss(next_tab, NULL); }
+    if (cocoa_pending_actions[PREVIOUS_TAB]) { call_boss(previous_tab, NULL); }
+    if (cocoa_pending_actions[DETACH_TAB]) { call_boss(detach_tab, NULL); }
+    if (cocoa_pending_actions[NEW_WINDOW]) { call_boss(new_window, NULL); }
+    if (cocoa_pending_actions[CLOSE_WINDOW]) { call_boss(close_window, NULL); }
+    if (cocoa_pending_actions[RESET_TERMINAL]) { call_boss(clear_terminal, "sO", "reset", Py_True ); }
+    if (cocoa_pending_actions[RELOAD_CONFIG]) { call_boss(load_config_file, NULL); }
+    if (cocoa_pending_actions_data.wd) {
+        if (cocoa_pending_actions[NEW_OS_WINDOW_WITH_WD]) { call_boss(new_os_window_with_wd, "s", cocoa_pending_actions_data.wd); }
+        if (cocoa_pending_actions[NEW_TAB_WITH_WD]) { call_boss(new_tab_with_wd, "s", cocoa_pending_actions_data.wd); }
+        free(cocoa_pending_actions_data.wd);
+        cocoa_pending_actions_data.wd = NULL;
+    }
+    if (cocoa_pending_actions_data.open_files_count) {
+        for (unsigned cpa = 0; cpa < cocoa_pending_actions_data.open_files_count; cpa++) {
+            if (cocoa_pending_actions_data.open_files[cpa]) {
+                call_boss(open_file, "s", cocoa_pending_actions_data.open_files[cpa]);
+                free(cocoa_pending_actions_data.open_files[cpa]);
+                cocoa_pending_actions_data.open_files[cpa] = NULL;
+            }
+        }
+        cocoa_pending_actions_data.open_files_count = 0;
+    }
+    memset(cocoa_pending_actions, 0, sizeof(cocoa_pending_actions));
+    has_cocoa_pending_actions = false;
+
+}
 #endif
 
 static void process_global_state(void *data);
@@ -988,33 +1034,10 @@ process_global_state(void *data) {
     if (parse_input(self)) input_read = true;
     render(now, input_read);
 #ifdef __APPLE__
-        if (cocoa_pending_actions) {
-            if (cocoa_pending_actions & PREFERENCES_WINDOW) { call_boss(edit_config_file, NULL); }
-            if (cocoa_pending_actions & NEW_OS_WINDOW) { call_boss(new_os_window, NULL); }
-            if (cocoa_pending_actions & CLOSE_OS_WINDOW) { call_boss(close_os_window, NULL); }
-            if (cocoa_pending_actions & CLOSE_TAB) { call_boss(close_tab, NULL); }
-            if (cocoa_pending_actions & NEW_TAB) { call_boss(new_tab, NULL); }
-            if (cocoa_pending_actions & NEXT_TAB) { call_boss(next_tab, NULL); }
-            if (cocoa_pending_actions & PREVIOUS_TAB) { call_boss(previous_tab, NULL); }
-            if (cocoa_pending_actions & DETACH_TAB) { call_boss(detach_tab, NULL); }
-            if (cocoa_pending_actions_data.wd) {
-                if (cocoa_pending_actions & NEW_OS_WINDOW_WITH_WD) { call_boss(new_os_window_with_wd, "s", cocoa_pending_actions_data.wd); }
-                if (cocoa_pending_actions & NEW_TAB_WITH_WD) { call_boss(new_tab_with_wd, "s", cocoa_pending_actions_data.wd); }
-                free(cocoa_pending_actions_data.wd);
-                cocoa_pending_actions_data.wd = NULL;
-            }
-            if (cocoa_pending_actions_data.open_files_count) {
-                for (unsigned cpa = 0; cpa < cocoa_pending_actions_data.open_files_count; cpa++) {
-                    if (cocoa_pending_actions_data.open_files[cpa]) {
-                        call_boss(open_file, "s", cocoa_pending_actions_data.open_files[cpa]);
-                        free(cocoa_pending_actions_data.open_files[cpa]);
-                        cocoa_pending_actions_data.open_files[cpa] = NULL;
-                    }
-                }
-                cocoa_pending_actions_data.open_files_count = 0;
-            }
-            cocoa_pending_actions = 0;
-        }
+    if (has_cocoa_pending_actions) {
+        process_cocoa_pending_actions();
+        maximum_wait = 0;  // ensure loop ticks again so that the actions side effects are performed immediately
+    }
 #endif
     report_reaped_pids();
     bool should_quit = false;
@@ -1142,7 +1165,7 @@ read_bytes(int fd, Screen *screen) {
 }
 
 
-typedef struct { bool kill_signal, child_died; } SignalSet;
+typedef struct { bool kill_signal, child_died, reload_config; } SignalSet;
 
 static void
 handle_signal(int signum, void *data) {
@@ -1155,6 +1178,9 @@ handle_signal(int signum, void *data) {
         case SIGCHLD:
             ss->child_died = true;
             break;
+        case SIGUSR1:
+            ss->reload_config = true;
+            break;
         default:
             break;
     }
@@ -1269,7 +1295,12 @@ io_loop(void *data) {
                 SignalSet ss = {0};
                 data_received = true;
                 read_signals(fds[1].fd, handle_signal, &ss);
-                if (ss.kill_signal) { children_mutex(lock); kill_signal_received = true; children_mutex(unlock); }
+                if (ss.kill_signal || ss.reload_config) {
+                    children_mutex(lock);
+                    if (ss.kill_signal) kill_signal_received = true;
+                    if (ss.reload_config) reload_config_signal_received = true;
+                    children_mutex(unlock);
+                }
                 if (ss.child_died) reap_children(self, OPT(close_on_child_death));
             }
             for (i = 0; i < self->count; i++) {

kitty/child.py

@@ -14,7 +14,6 @@ from typing import (
 import kitty.fast_data_types as fast_data_types
 
 from .constants import is_macos, shell_path, terminfo_dir
-from .options_stub import Options
 
 try:
     from typing import TypedDict
@@ -187,7 +186,6 @@ class Child:
         self,
         argv: Sequence[str],
         cwd: str,
-        opts: Options,
         stdin: Optional[bytes] = None,
         env: Optional[Dict[str, str]] = None,
         cwd_from: Optional[int] = None,
@@ -204,7 +202,6 @@ class Child:
         else:
             cwd = os.path.expandvars(os.path.expanduser(cwd or os.getcwd()))
         self.cwd = os.path.abspath(cwd)
-        self.opts = opts
         self.stdin = stdin
         self.env = env or {}
 
@@ -217,7 +214,7 @@ class Child:
                     'lc_ctype_before_python') and not getattr(default_env, 'lc_ctype_set_by_user', False):
                 del env['LC_CTYPE']
             env.update(self.env)
-            env['TERM'] = self.opts.term
+            env['TERM'] = fast_data_types.get_options().term
             env['COLORTERM'] = 'truecolor'
             if self.cwd:
                 # needed in case cwd is a symlink, in which case shells

kitty/cli.py

@@ -2,22 +2,19 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
 import re
 import sys
 from collections import deque
 from typing import (
-    Any, Callable, Dict, FrozenSet, Iterable, Iterator, List, Match, Optional,
-    Sequence, Set, Tuple, Type, TypeVar, Union, cast
+    Any, Callable, Dict, FrozenSet, Iterator, List, Match, Optional, Sequence,
+    Tuple, Type, TypeVar, Union, cast
 )
 
 from .cli_stub import CLIOptions
 from .conf.utils import resolve_config
-from .config import KeyAction
-from .constants import appname, defconf, is_macos, is_wayland, str_version
-from .options_stub import Options as OptionsStub
-from .types import SingleKey
-from .typing import BadLineType, SequenceMap, TypedDict
+from .constants import appname, defconf, is_macos, str_version
+from .options.types import Options as KittyOpts
+from .typing import BadLineType, TypedDict
 
 
 class OptionDict(TypedDict):
@@ -587,7 +584,7 @@ Override individual configuration options, can be specified multiple times.
 Syntax: :italic:`name=value`. For example: :option:`kitty -o` font_size=20
 
 
---directory -d
+--directory --working-directory -d
 default=.
 Change to the specified directory when launching
 
@@ -690,9 +687,10 @@ instead of ignoring them. Also prints out miscellaneous debug information.
 Useful when debugging rendering problems
 
 
---debug-keyboard
+--debug-input --debug-keyboard
+dest=debug_keyboard
 type=bool-set
-This option will cause kitty to print out key events as they are received
+This option will cause kitty to print out key and mouse events as they are received
 
 
 --debug-font-fallback
@@ -700,11 +698,6 @@ type=bool-set
 Print out information about the selection of fallback fonts for characters not present in the main font.
 
 
---debug-config
-type=bool-set
-Print out information about the system and kitty configuration.
-
-
 --execute -e
 type=bool-set
 !
@@ -756,107 +749,17 @@ def parse_args(
 
 
 SYSTEM_CONF = '/etc/xdg/kitty/kitty.conf'
-ShortcutMap = Dict[Tuple[SingleKey, ...], KeyAction]
 
 
-def print_shortcut(key_sequence: Iterable[SingleKey], action: KeyAction) -> None:
-    from .fast_data_types import (
-        GLFW_MOD_ALT, GLFW_MOD_CAPS_LOCK, GLFW_MOD_CONTROL, GLFW_MOD_HYPER,
-        GLFW_MOD_META, GLFW_MOD_NUM_LOCK, GLFW_MOD_SHIFT, GLFW_MOD_SUPER,
-        glfw_get_key_name
-    )
-    modmap = {'shift': GLFW_MOD_SHIFT, 'alt': GLFW_MOD_ALT, 'ctrl': GLFW_MOD_CONTROL, ('cmd' if is_macos else 'super'): GLFW_MOD_SUPER,
-              'hyper': GLFW_MOD_HYPER, 'meta': GLFW_MOD_META, 'num_lock': GLFW_MOD_NUM_LOCK, 'caps_lock': GLFW_MOD_CAPS_LOCK}
-    keys = []
-    for key_spec in key_sequence:
-        names = []
-        mods, is_native, key = key_spec
-        for name, val in modmap.items():
-            if mods & val:
-                names.append(name)
-        if key:
-            kname = glfw_get_key_name(0, key) if is_native else glfw_get_key_name(key, 0)
-            names.append(kname or f'{key}')
-        keys.append('+'.join(names))
-
-    print('\t', ' > '.join(keys), action)
-
-
-def print_shortcut_changes(defns: ShortcutMap, text: str, changes: Set[Tuple[SingleKey, ...]]) -> None:
-    if changes:
-        print(title(text))
-
-        for k in sorted(changes):
-            print_shortcut(k, defns[k])
-
-
-def compare_keymaps(final: ShortcutMap, initial: ShortcutMap) -> None:
-    added = set(final) - set(initial)
-    removed = set(initial) - set(final)
-    changed = {k for k in set(final) & set(initial) if final[k] != initial[k]}
-    print_shortcut_changes(final, 'Added shortcuts:', added)
-    print_shortcut_changes(initial, 'Removed shortcuts:', removed)
-    print_shortcut_changes(final, 'Changed shortcuts:', changed)
-
-
-def flatten_sequence_map(m: SequenceMap) -> ShortcutMap:
-    ans: Dict[Tuple[SingleKey, ...], KeyAction] = {}
-    for key_spec, rest_map in m.items():
-        for r, action in rest_map.items():
-            ans[(key_spec,) + (r)] = action
-    return ans
-
-
-def compare_opts(opts: OptionsStub) -> None:
-    from .config import defaults, load_config
-    print('\nConfig options different from defaults:')
-    default_opts = load_config()
-    changed_opts = [
-        f for f in sorted(defaults._fields)  # type: ignore
-        if f not in ('key_definitions', 'keymap', 'sequence_map') and getattr(opts, f) != getattr(defaults, f)
-    ]
-    field_len = max(map(len, changed_opts)) if changed_opts else 20
-    fmt = '{{:{:d}s}}'.format(field_len)
-    for f in changed_opts:
-        print(title(fmt.format(f)), getattr(opts, f))
-
-    final_, initial_ = opts.keymap, default_opts.keymap
-    final: ShortcutMap = {(k,): v for k, v in final_.items()}
-    initial: ShortcutMap = {(k,): v for k, v in initial_.items()}
-    final_s, initial_s = map(flatten_sequence_map, (opts.sequence_map, default_opts.sequence_map))
-    final.update(final_s)
-    initial.update(initial_s)
-    compare_keymaps(final, initial)
-
-
-def create_opts(args: CLIOptions, debug_config: bool = False, accumulate_bad_lines: Optional[List[BadLineType]] = None) -> OptionsStub:
+def create_opts(args: CLIOptions, accumulate_bad_lines: Optional[List[BadLineType]] = None) -> KittyOpts:
     from .config import load_config
     config = tuple(resolve_config(SYSTEM_CONF, defconf, args.config))
-    if debug_config:
-        print(version(add_rev=True))
-        print(' '.join(os.uname()))
-        if is_macos:
-            import subprocess
-            print(' '.join(subprocess.check_output(['sw_vers']).decode('utf-8').splitlines()).strip())
-        if os.path.exists('/etc/issue'):
-            with open('/etc/issue', encoding='utf-8', errors='replace') as f:
-                print(f.read().strip())
-        if os.path.exists('/etc/lsb-release'):
-            with open('/etc/lsb-release', encoding='utf-8', errors='replace') as f:
-                print(f.read().strip())
-        config = tuple(x for x in config if os.path.exists(x))
-        if config:
-            print(green('Loaded config files:'), ', '.join(config))
     overrides = (a.replace('=', ' ', 1) for a in args.override or ())
     opts = load_config(*config, overrides=overrides, accumulate_bad_lines=accumulate_bad_lines)
-    if debug_config:
-        if not is_macos:
-            print('Running under:', green('Wayland' if is_wayland(opts) else 'X11'))
-        compare_opts(opts)
     return opts
 
 
-def create_default_opts() -> OptionsStub:
+def create_default_opts() -> KittyOpts:
     from .config import load_config
     config = tuple(resolve_config(SYSTEM_CONF, defconf, ()))
     opts = load_config(*config)

kitty/cli_stub.py

@@ -13,12 +13,12 @@ class CLIOptions:
 LaunchCLIOptions = AskCLIOptions = ClipboardCLIOptions = DiffCLIOptions = CLIOptions
 HintsCLIOptions = IcatCLIOptions = PanelCLIOptions = ResizeCLIOptions = CLIOptions
 ErrorCLIOptions = UnicodeCLIOptions = RCOptions = RemoteFileCLIOptions = CLIOptions
-QueryTerminalCLIOptions = BroadcastCLIOptions = CLIOptions
+QueryTerminalCLIOptions = BroadcastCLIOptions = ShowKeyCLIOptions = CLIOptions
 
 
 def generate_stub() -> None:
     from .cli import parse_option_spec, as_type_stub
-    from .conf.definition import save_type_stub
+    from .conf.utils import save_type_stub
     text = 'import typing\n\n\n'
 
     def do(otext=None, cls: str = 'CLIOptions', extra_fields: Sequence[str] = ()):
@@ -42,6 +42,9 @@ def generate_stub() -> None:
     from kittens.clipboard.main import OPTIONS
     do(OPTIONS(), 'ClipboardCLIOptions')
 
+    from kittens.show_key.main import OPTIONS
+    do(OPTIONS(), 'ShowKeyCLIOptions')
+
     from kittens.diff.main import OPTIONS
     do(OPTIONS(), 'DiffCLIOptions')
 

kitty/cocoa_window.m

@@ -88,6 +88,10 @@ PENDING(close_tab, CLOSE_TAB)
 PENDING(new_tab, NEW_TAB)
 PENDING(next_tab, NEXT_TAB)
 PENDING(previous_tab, PREVIOUS_TAB)
+PENDING(new_window, NEW_WINDOW)
+PENDING(close_window, CLOSE_WINDOW)
+PENDING(reset_terminal, RESET_TERMINAL)
+PENDING(reload_config, RELOAD_CONFIG)
 
 - (void)open_kitty_website_url:(id)sender {
     (void)sender;
@@ -114,8 +118,8 @@ typedef struct {
     NSEventModifierFlags mods;
 } GlobalShortcut;
 typedef struct {
-    GlobalShortcut new_os_window, close_os_window, close_tab, edit_config_file;
-    GlobalShortcut previous_tab, next_tab, new_tab;
+    GlobalShortcut new_os_window, close_os_window, close_tab, edit_config_file, reload_config;
+    GlobalShortcut previous_tab, next_tab, new_tab, new_window, close_window, reset_terminal;
 } GlobalShortcuts;
 static GlobalShortcuts global_shortcuts;
 
@@ -129,6 +133,7 @@ cocoa_set_global_shortcut(PyObject *self UNUSED, PyObject *args) {
 #define Q(x) if (strcmp(name, #x) == 0) gs = &global_shortcuts.x
     Q(new_os_window); else Q(close_os_window); else Q(close_tab); else Q(edit_config_file);
     else Q(new_tab); else Q(next_tab); else Q(previous_tab);
+    else Q(new_window); else Q(close_window); else Q(reset_terminal); else Q(reload_config);
 #undef Q
     if (gs == NULL) { PyErr_SetString(PyExc_KeyError, "Unknown shortcut name"); return NULL; }
     int cocoa_mods;
@@ -145,7 +150,7 @@ get_dock_menu(id self UNUSED, SEL _cmd UNUSED, NSApplication *sender UNUSED) {
     if (!dockMenu) {
         GlobalMenuTarget *global_menu_target = [GlobalMenuTarget shared_instance];
         dockMenu = [[NSMenu alloc] init];
-        [[dockMenu addItemWithTitle:@"New OS window"
+        [[dockMenu addItemWithTitle:@"New OS Window"
                              action:@selector(new_os_window:)
                       keyEquivalent:@""]
                           setTarget:global_menu_target];
@@ -157,9 +162,11 @@ static PyObject *notification_activated_callback = NULL;
 
 static PyObject*
 set_notification_activated_callback(PyObject *self UNUSED, PyObject *callback) {
-    if (notification_activated_callback) Py_DECREF(notification_activated_callback);
-    notification_activated_callback = callback;
-    Py_INCREF(callback);
+    Py_CLEAR(notification_activated_callback);
+    if (callback != Py_None) {
+        notification_activated_callback = callback;
+        Py_INCREF(callback);
+    }
     Py_RETURN_NONE;
 }
 
@@ -374,7 +381,7 @@ cocoa_create_global_menu(void) {
                 keyEquivalent:@""];
     [appMenu addItem:[NSMenuItem separatorItem]];
     MENU_ITEM(appMenu, @"Preferences…", edit_config_file);
-    MENU_ITEM(appMenu, @"New OS window", new_os_window);
+    MENU_ITEM(appMenu, @"Reload preferences", reload_config);
 
     [appMenu addItemWithTitle:[NSString stringWithFormat:@"Hide %@", app_name]
                        action:@selector(hide:)
@@ -408,6 +415,23 @@ cocoa_create_global_menu(void) {
                 keyEquivalent:@"q"];
     [appMenu release];
 
+    NSMenuItem* shellMenuItem =
+        [bar addItemWithTitle:@"Shell"
+                       action:NULL
+                keyEquivalent:@""];
+    NSMenu* shellMenu = [[NSMenu alloc] initWithTitle:@"Shell"];
+    [shellMenuItem setSubmenu:shellMenu];
+    MENU_ITEM(shellMenu, @"New OS Window", new_os_window);
+    MENU_ITEM(shellMenu, @"New Tab", new_tab);
+    MENU_ITEM(shellMenu, @"New Window", new_window);
+    [shellMenu addItem:[NSMenuItem separatorItem]];
+    MENU_ITEM(shellMenu, @"Close OS Window", close_os_window);
+    MENU_ITEM(shellMenu, @"Close Tab", close_tab);
+    MENU_ITEM(shellMenu, @"Close Window", close_window);
+    [shellMenu addItem:[NSMenuItem separatorItem]];
+    MENU_ITEM(shellMenu, @"Reset", reset_terminal);
+    [shellMenu release];
+
     NSMenuItem* windowMenuItem =
         [bar addItemWithTitle:@""
                        action:NULL
@@ -427,11 +451,8 @@ cocoa_create_global_menu(void) {
                    keyEquivalent:@""];
 
     [windowMenu addItem:[NSMenuItem separatorItem]];
-    MENU_ITEM(windowMenu, @"New Tab", new_tab);
     MENU_ITEM(windowMenu, @"Show Previous Tab", previous_tab);
     MENU_ITEM(windowMenu, @"Show Next Tab", next_tab);
-    MENU_ITEM(windowMenu, @"Close Tab", close_tab);
-    MENU_ITEM(windowMenu, @"Close OS Window", close_os_window);
     [[windowMenu addItemWithTitle:@"Move Tab to New Window"
                            action:@selector(detach_tab:)
                     keyEquivalent:@""] setTarget:global_menu_target];
@@ -454,6 +475,7 @@ cocoa_create_global_menu(void) {
                          action:@selector(open_kitty_website_url:)
                   keyEquivalent:@"?"]
                       setTarget:global_menu_target];
+    [NSApp setHelpMenu:helpMenu];
     [helpMenu release];
 
     [bar release];
@@ -623,7 +645,6 @@ cleanup() {
 
     if (dockMenu) [dockMenu release];
     dockMenu = nil;
-    Py_CLEAR(notification_activated_callback);
 
 #ifndef KITTY_USE_DEPRECATED_MACOS_NOTIFICATION_API
     drain_pending_notifications(NO);

kitty/colors.c

@@ -92,11 +92,11 @@ alloc_color_profile() {
 static PyObject*
 update_ansi_color_table(ColorProfile *self, PyObject *val) {
 #define update_ansi_color_table_doc "Update the 256 basic colors"
-    if (!PyList_Check(val)) { PyErr_SetString(PyExc_TypeError, "color table must be a list"); return NULL; }
-    if (PyList_GET_SIZE(val) != arraysz(FG_BG_256)) { PyErr_SetString(PyExc_TypeError, "color table must have 256 items"); return NULL; }
+    if (!PyLong_Check(val)) { PyErr_SetString(PyExc_TypeError, "color table must be a long"); return NULL; }
+    unsigned long *color_table = PyLong_AsVoidPtr(val);
     for (size_t i = 0; i < arraysz(FG_BG_256); i++) {
-        self->color_table[i] = PyLong_AsUnsignedLong(PyList_GET_ITEM(val, i));
-        self->orig_color_table[i] = self->color_table[i];
+        self->color_table[i] = color_table[i];
+        self->orig_color_table[i] = color_table[i];
     }
     self->dirty = true;
     Py_RETURN_NONE;

kitty/complete.py

@@ -6,17 +6,23 @@ import os
 import shlex
 import sys
 from typing import (
-    Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
+    Any, Callable, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple,
+    Union
 )
 
-from kittens.runner import all_kitten_names, get_kitten_cli_docs
+from kittens.runner import (
+    all_kitten_names, get_kitten_cli_docs, get_kitten_completer
+)
 
 from .cli import (
-    OptionDict, OptionSpecSeq, options_for_completion, parse_option_spec
+    OptionDict, OptionSpecSeq, options_for_completion, parse_option_spec,
+    prettify
 )
+from .fast_data_types import truncate_point_for_length, wcswidth
 from .rc.base import all_command_names, command_for_name
 from .shell import options_for_cmd
 from .types import run_once
+from .utils import screen_size_function
 
 '''
 To add completion for a new shell, you need to:
@@ -39,7 +45,37 @@ them into something your shell will understand.
 
 parsers: Dict[str, Callable] = {}
 serializers: Dict[str, Callable] = {}
-MathGroup = Dict[str, str]
+
+
+class MatchGroup:
+
+    def __init__(
+        self, x: Union[Dict[str, str], Iterable[str]],
+        trailing_space: bool = True,
+        is_files: bool = False,
+        word_transforms: Optional[Dict[str, str]] = None,
+    ):
+        self.mdict = x if isinstance(x, dict) else dict.fromkeys(x, '')
+        self.trailing_space = trailing_space
+        self.is_files = is_files
+        self.word_transforms = word_transforms or {}
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self.mdict)
+
+    def transformed_words(self) -> Iterator[str]:
+        for w in self:
+            yield self.word_transforms.get(w, w)
+
+    def transformed_items(self) -> Iterator[Tuple[str, str]]:
+        for w, desc in self.items():
+            yield self.word_transforms.get(w, w), desc
+
+    def items(self) -> Iterator[Tuple[str, str]]:
+        return iter(self.mdict.items())
+
+    def values(self) -> Iterator[str]:
+        return iter(self.mdict.values())
 
 
 def debug(*a: Any, **kw: Any) -> None:
@@ -69,11 +105,18 @@ class Delegate:
 class Completions:
 
     def __init__(self) -> None:
-        self.match_groups: Dict[str, MathGroup] = {}
-        self.no_space_groups: Set[str] = set()
-        self.files_groups: Set[str] = set()
+        self.match_groups: Dict[str, MatchGroup] = {}
         self.delegate: Delegate = Delegate()
 
+    def add_match_group(
+        self, name: str, x: Union[Dict[str, str], Iterable[str]],
+        trailing_space: bool = True,
+        is_files: bool = False,
+        word_transforms: Optional[Dict[str, str]] = None
+    ) -> MatchGroup:
+        self.match_groups[name] = m = MatchGroup(x, trailing_space, is_files, word_transforms)
+        return m
+
 
 @run_once
 def remote_control_command_names() -> Tuple[str, ...]:
@@ -159,16 +202,55 @@ def fish_input_parser(data: str) -> ParseResult:
 @output_serializer
 def zsh_output_serializer(ans: Completions) -> str:
     lines = []
+
+    screen = screen_size_function(sys.stderr.fileno())()
+    width = screen.cols
+
+    def fmt_desc(word: str, desc: str, max_word_len: int) -> Iterator[str]:
+        if not desc:
+            yield word
+            return
+        desc = prettify(desc.splitlines()[0])
+        multiline = False
+        if wcswidth(word) > max_word_len:
+            max_desc_len = width - 2
+            multiline = True
+        else:
+            word = word.ljust(max_word_len)
+            max_desc_len = width - max_word_len - 3
+        if wcswidth(desc) > max_desc_len:
+            desc = desc[:truncate_point_for_length(desc, max_desc_len - 2)]
+            desc += '…'
+
+        word = f'\x1b[32m{word}\x1b[39m'
+        if multiline:
+            ans = f'{word}\n  {desc}'
+        else:
+            ans = f'{word}  {desc}'
+        yield ans
+
     for description, matches in ans.match_groups.items():
-        cmd = ['compadd', '-U', '-J', shlex.quote(description), '-X', shlex.quote(description)]
-        if description in ans.no_space_groups:
+        cmd = ['compadd', '-U', '-J', shlex.quote(description), '-X', shlex.quote('%B' + description + '%b')]
+        if not matches.trailing_space:
             cmd += ['-S', '""']
-        if description in ans.files_groups:
+        if matches.is_files:
             cmd.append('-f')
             common_prefix = os.path.commonprefix(tuple(matches))
             if common_prefix:
                 cmd.extend(('-p', shlex.quote(common_prefix)))
-                matches = {k[len(common_prefix):]: v for k, v in matches.items()}
+                matches = MatchGroup({k[len(common_prefix):]: v for k, v in matches.items()})
+        has_descriptions = any(matches.values())
+        if has_descriptions or matches.word_transforms:
+            lines.append('compdescriptions=(')
+            sz = max(map(wcswidth, matches.transformed_words()))
+            limit = min(16, sz)
+            for word, desc in matches.transformed_items():
+                lines.extend(map(shlex.quote, fmt_desc(word, desc, limit)))
+            lines.append(')')
+            if has_descriptions:
+                cmd.append('-l')
+            cmd.append('-d')
+            cmd.append('compdescriptions')
         cmd.append('--')
         for word in matches:
             cmd.append(shlex.quote(word))
@@ -190,9 +272,8 @@ def zsh_output_serializer(ans: Completions) -> str:
 def bash_output_serializer(ans: Completions) -> str:
     lines = []
     for description, matches in ans.match_groups.items():
-        needs_space = description not in ans.no_space_groups
         for word in matches:
-            if needs_space:
+            if matches.trailing_space:
                 word += ' '
             lines.append('COMPREPLY+=({})'.format(shlex.quote(word)))
     # debug('\n'.join(lines))
@@ -212,11 +293,11 @@ def fish_output_serializer(ans: Completions) -> str:
 
 def completions_for_first_word(ans: Completions, prefix: str, entry_points: Iterable[str], namespaced_entry_points: Iterable[str]) -> None:
     cmds = ['@' + c for c in remote_control_command_names()]
-    ans.match_groups['Entry points'] = {
+    ans.add_match_group('Entry points', {
         k: '' for k in
         list(entry_points) + cmds + ['+' + k for k in namespaced_entry_points]
         if not prefix or k.startswith(prefix)
-    }
+    })
     if prefix:
         ans.delegate = Delegate([prefix], 0)
 
@@ -229,7 +310,7 @@ def kitty_cli_opts(ans: Completions, prefix: Optional[str] = None) -> None:
         aliases = frozenset(x for x in opt['aliases'] if x.startswith(prefix)) if prefix else opt['aliases']
         for alias in aliases:
             matches[alias] = opt['help'].strip()
-    ans.match_groups['Options'] = matches
+    ans.add_match_group('Options', matches)
 
 
 def complete_kitty_cli_arg(ans: Completions, opt: Optional[OptionDict], prefix: str, unknown_args: Delegate) -> None:
@@ -242,8 +323,7 @@ def complete_kitty_cli_arg(ans: Completions, opt: Optional[OptionDict], prefix:
     if dest == 'override':
         from kitty.config import option_names_for_completion
         k = 'Config directives'
-        ans.match_groups[k] = {k+'=': '' for k in option_names_for_completion() if k.startswith(prefix)}
-        ans.no_space_groups.add(k)
+        ans.add_match_group(k, {k+'=': '' for k in option_names_for_completion() if k.startswith(prefix)}, trailing_space=False)
     elif dest == 'config':
 
         def is_conf_file(x: str) -> bool:
@@ -260,13 +340,11 @@ def complete_kitty_cli_arg(ans: Completions, opt: Optional[OptionDict], prefix:
         complete_files_and_dirs(ans, prefix, files_group_name='Directories', predicate=os.path.isdir)
     elif dest == 'start_as':
         k = 'Start as'
-        ans.match_groups[k] = {x: x for x in 'normal,fullscreen,maximized,minimized'.split(',') if x.startswith(prefix)}
-        ans.no_space_groups.add(k)
+        ans.add_match_group(k, {x: x for x in 'normal,fullscreen,maximized,minimized'.split(',') if x.startswith(prefix)}, trailing_space=False)
     elif dest == 'listen_on':
         if ':' not in prefix:
             k = 'Address type'
-            ans.match_groups[k] = {x: x for x in ('unix:', 'tcp:') if x.startswith(prefix)}
-            ans.no_space_groups.add(k)
+            ans.add_match_group(k, {x: x for x in ('unix:', 'tcp:') if x.startswith(prefix)}, trailing_space=False)
         elif prefix.startswith('unix:') and not prefix.startswith('@'):
             complete_files_and_dirs(ans, prefix[len('unix:'):], files_group_name='UNIX sockets', add_prefix='unix:')
 
@@ -295,7 +373,7 @@ def complete_alias_map(
         opt = option_map.get(w)
         if w is last_word and not new_word:
             if w.startswith('-'):
-                ans.match_groups['Options'] = {k: opt['help'] for k, opt in option_map.items() if k.startswith(last_word)}
+                ans.add_match_group('Options', {k: opt['help'] for k, opt in option_map.items() if k.startswith(last_word)})
             else:
                 if complete_args is not None:
                     complete_args(ans, None, last_word, Delegate(words, i))
@@ -312,7 +390,7 @@ def complete_alias_map(
         prefix = '' if new_word else last_word
         if complete_args is not None:
             complete_args(ans, None, prefix, Delegate())
-        ans.match_groups['Options'] = {k: opt['help'] for k, opt in option_map.items() if k.startswith(prefix)}
+        ans.add_match_group('Options', {k: opt['help'] for k, opt in option_map.items() if k.startswith(prefix)})
 
 
 def complete_cli(
@@ -336,8 +414,14 @@ def complete_remote_command(ans: Completions, cmd_name: str, words: Sequence[str
         return
     args_completer: Optional[CompleteArgsFunc] = None
     args_completion = command_for_name(cmd_name).args_completion
-    if args_completion and 'files' in args_completion:
-        args_completer = remote_files_completer(args_completion['files'])
+    if args_completion:
+        if 'files' in args_completion:
+            title, matchers = args_completion['files']
+            if isinstance(matchers, tuple):
+                args_completer = remote_files_completer(title, matchers)
+        elif 'names' in args_completion:
+            title, q = args_completion['names']
+            args_completer = remote_args_completer(title, q() if callable(q) else q)
     complete_alias_map(ans, words, new_word, alias_map, complete_args=args_completer)
 
 
@@ -379,18 +463,15 @@ def complete_files_and_dirs(
     add_prefix: Optional[str] = None
 ) -> None:
     dirs, files_ = path_completion(prefix or '')
-    files = filter(predicate, files_)
+    files: Iterable[str] = filter(predicate, files_)
     if add_prefix:
         dirs = list(add_prefix + x for x in dirs)
         files = (add_prefix + x for x in files)
 
     if dirs:
-        ans.match_groups['Directories'] = dict.fromkeys(dirs)
-        ans.files_groups.add('Directories')
-        ans.no_space_groups.add('Directories')
+        ans.add_match_group('Directories', dirs, trailing_space=False, is_files=True)
     if files:
-        ans.match_groups[files_group_name] = dict.fromkeys(files)
-        ans.files_groups.add(files_group_name)
+        ans.add_match_group(files_group_name, files, is_files=True)
 
 
 def complete_icat_args(ans: Completions, opt: Optional[OptionDict], prefix: str, unknown_args: Delegate) -> None:
@@ -406,8 +487,7 @@ def complete_icat_args(ans: Completions, opt: Optional[OptionDict], prefix: str,
         complete_files_and_dirs(ans, prefix, 'Images', icat_file_predicate)
 
 
-def remote_files_completer(spec: Tuple[str, Tuple[str, ...]]) -> CompleteArgsFunc:
-    name, matchers = spec
+def remote_files_completer(name: str, matchers: Tuple[str, ...]) -> CompleteArgsFunc:
 
     def complete_files_map(ans: Completions, opt: Optional[OptionDict], prefix: str, unknown_args: Delegate) -> None:
 
@@ -423,6 +503,16 @@ def remote_files_completer(spec: Tuple[str, Tuple[str, ...]]) -> CompleteArgsFun
     return complete_files_map
 
 
+def remote_args_completer(title: str, words: Iterable[str]) -> CompleteArgsFunc:
+    items = sorted(words)
+
+    def complete_names_for_arg(ans: Completions, opt: Optional[OptionDict], prefix: str, unknown_args: Delegate) -> None:
+        if opt is None:
+            ans.add_match_group(title, {c: '' for c in items if c.startswith(prefix)})
+
+    return complete_names_for_arg
+
+
 def config_file_predicate(filename: str) -> bool:
     return filename.endswith('.conf')
 
@@ -435,6 +525,13 @@ def complete_diff_args(ans: Completions, opt: Optional[OptionDict], prefix: str,
 
 
 def complete_kitten(ans: Completions, kitten: str, words: Sequence[str], new_word: bool) -> None:
+    try:
+        completer = get_kitten_completer(kitten)
+    except SystemExit:
+        completer = None
+    if completer is not None:
+        completer(ans, words, new_word)
+        return
     try:
         cd = get_kitten_cli_docs(kitten)
     except SystemExit:
@@ -467,24 +564,24 @@ def find_completions(words: Sequence[str], new_word: bool, entry_points: Iterabl
     if words[0] == '@':
         if len(words) == 1 or (len(words) == 2 and not new_word):
             prefix = words[1] if len(words) > 1 else ''
-            ans.match_groups['Remote control commands'] = {c: '' for c in remote_control_command_names() if c.startswith(prefix)}
+            ans.add_match_group('Remote control commands', {c: '' for c in remote_control_command_names() if c.startswith(prefix)})
         else:
             complete_remote_command(ans, words[1], words[2:], new_word)
         return ans
     if words[0].startswith('@'):
         if len(words) == 1 and not new_word:
             prefix = words[0]
-            ans.match_groups['Remote control commands'] = {'@' + c: '' for c in remote_control_command_names() if c.startswith(prefix)}
+            ans.add_match_group('Remote control commands', {'@' + c: '' for c in remote_control_command_names() if c.startswith(prefix)})
         else:
             complete_remote_command(ans, words[0][1:], words[1:], new_word)
     if words[0] == '+':
         if len(words) == 1 or (len(words) == 2 and not new_word):
             prefix = words[1] if len(words) > 1 else ''
-            ans.match_groups['Entry points'] = {c: '' for c in namespaced_entry_points if c.startswith(prefix)}
+            ans.add_match_group('Entry points', {c: '' for c in namespaced_entry_points if c.startswith(prefix)})
         else:
             if words[1] == 'kitten':
                 if len(words) == 2 or (len(words) == 3 and not new_word):
-                    ans.match_groups['Kittens'] = dict.fromkeys(k for k in all_kitten_names() if k.startswith('' if len(words) == 2 else words[2]))
+                    ans.add_match_group('Kittens', (k for k in all_kitten_names() if k.startswith('' if len(words) == 2 else words[2])))
                 else:
                     complete_kitten(ans, words[2], words[3:], new_word)
         return ans
@@ -492,13 +589,13 @@ def find_completions(words: Sequence[str], new_word: bool, entry_points: Iterabl
         if len(words) == 1:
             if new_word:
                 if words[0] == '+kitten':
-                    ans.match_groups['Kittens'] = dict.fromkeys(all_kitten_names())
+                    ans.add_match_group('Kittens', all_kitten_names())
             else:
                 prefix = words[0]
-                ans.match_groups['Entry points'] = {c: '' for c in namespaced_entry_points if c.startswith(prefix)}
+                ans.add_match_group('Entry points', (c for c in namespaced_entry_points if c.startswith(prefix)))
         else:
             if len(words) == 2 and not new_word:
-                ans.match_groups['Kittens'] = dict.fromkeys(k for k in all_kitten_names() if k.startswith(words[1]))
+                ans.add_match_group('Kittens', (k for k in all_kitten_names() if k.startswith(words[1])))
             else:
                 if words[0] == '+kitten':
                     complete_kitten(ans, words[1], words[2:], new_word)

kitty/conf/definition.py

@@ -1,354 +0,0 @@
-#!/usr/bin/env python3
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-import re
-from functools import partial
-from typing import (
-    Any, Callable, Dict, Generator, Iterable, List, Match, Optional, Sequence,
-    Set, Tuple, Union, get_type_hints
-)
-
-from .utils import Choice, to_bool
-
-
-def to_string(x: str) -> str:
-    return x
-
-
-class Group:
-
-    __slots__ = 'name', 'short_text', 'start_text', 'end_text'
-
-    def __init__(self, name: str, short_text: str, start_text: str = '', end_text: str = '') -> None:
-        self.name, self.short_text = name, short_text.strip()
-        self.start_text, self.end_text = start_text.strip(), end_text.strip()
-
-
-class Option:
-
-    __slots__ = 'name', 'group', 'long_text', 'option_type', 'defval_as_string', 'add_to_default', 'add_to_docs', 'line'
-
-    def __init__(self, name: str, group: Group, defval: str, option_type: Any, long_text: str, add_to_default: bool, add_to_docs: bool):
-        self.name, self.group = name, group
-        self.long_text, self.option_type = long_text.strip(), option_type
-        self.defval_as_string = defval
-        self.add_to_default = add_to_default
-        self.add_to_docs = add_to_docs
-        self.line = self.name + ' ' + self.defval_as_string
-
-    def type_definition(self, is_multiple: bool, imports: Set[Tuple[str, str]]) -> str:
-
-        def type_name(x: type) -> str:
-            ans = x.__name__
-            if x.__module__ and x.__module__ != 'builtins':
-                imports.add((x.__module__, x.__name__))
-            if is_multiple:
-                ans = 'typing.Dict[str, str]'
-            return ans
-
-        def option_type_as_str(x: Any) -> str:
-            if hasattr(x, '__name__'):
-                return type_name(x)
-            ans = repr(x)
-            ans = ans.replace('NoneType', 'None')
-            return ans
-
-        if type(self.option_type) is type:
-            return type_name(self.option_type)
-        if isinstance(self.option_type, Choice):
-            return 'typing.Literal[{}]'.format(','.join(f'{x!r}' for x in self.option_type.all_choices))
-        th = get_type_hints(self.option_type)
-        try:
-            rettype = th['return']
-        except KeyError:
-            raise ValueError('The Option {} has an unknown option_type: {}'.format(self.name, self.option_type))
-        return option_type_as_str(rettype)
-
-
-class Shortcut:
-
-    __slots__ = 'name', 'group', 'key', 'action_def', 'short_text', 'long_text', 'add_to_default', 'add_to_docs', 'line'
-
-    def __init__(self, name: str, group: Group, key: str, action_def: str, short_text: str, long_text: str, add_to_default: bool, add_to_docs: bool):
-        self.name, self.group, self.key, self.action_def = name, group, key, action_def
-        self.short_text, self.long_text = short_text, long_text
-        self.add_to_default = add_to_default
-        self.add_to_docs = add_to_docs
-        self.line = 'map ' + self.key + ' ' + self.action_def
-
-
-def option(
-    all_options: Dict[str, Option],
-    group: Sequence[Group],
-    name: str,
-    defval: Any,
-    long_text: str = '',
-    option_type: Callable[[str], Any] = to_string,
-    add_to_default: bool = True,
-    add_to_docs: bool = True
-) -> Option:
-    is_multiple = name.startswith('+')
-    if is_multiple:
-        name = name[1:]
-    defval_type = type(defval)
-    if defval_type is not str:
-        if option_type is to_string:
-            if defval_type is bool:
-                option_type = to_bool
-            else:
-                option_type = defval_type
-        if defval_type is bool:
-            defval = 'yes' if defval else 'no'
-        else:
-            defval = str(defval)
-
-    key = name
-    if is_multiple:
-        key = name + ' ' + defval.partition(' ')[0]
-    ans = Option(name, group[0], defval, option_type, long_text, add_to_default, add_to_docs)
-    all_options[key] = ans
-    return ans
-
-
-def shortcut(
-    all_options: Dict[str, List[Shortcut]],
-    group: Sequence[Group],
-    action_name: str,
-    key: str,
-    action_def: str,
-    short_text: str = '',
-    long_text: str = '',
-    add_to_default: bool = True,
-    add_to_docs: bool = True
-) -> Shortcut:
-    ans = Shortcut(action_name, group[0], key, action_def, short_text, long_text, add_to_default, add_to_docs)
-    key = 'sc-' + action_name
-    all_options.setdefault(key, []).append(ans)
-    return ans
-
-
-def option_func(all_options: Dict[str, Any], all_groups: Dict[str, Sequence[str]]) -> Tuple[Callable, Callable, Callable[[str], None], Dict[str, Group]]:
-    all_groups_ = {k: Group(k, *v) for k, v in all_groups.items()}
-    group: List[Optional[Group]] = [None]
-
-    def change_group(name: str) -> None:
-        group[0] = all_groups_[name]
-
-    return partial(option, all_options, group), partial(shortcut, all_options, group), change_group, all_groups_
-
-
-def merged_opts(all_options: Sequence[Union[Option, Sequence[Shortcut]]], opt: Option, i: int) -> Generator[Option, None, None]:
-    yield opt
-    for k in range(i + 1, len(all_options)):
-        q = all_options[k]
-        if not isinstance(q, Option):
-            break
-        if not q.long_text and q.add_to_docs:
-            yield q
-        else:
-            break
-
-
-def remove_markup(text: str) -> str:
-
-    def sub(m: Match) -> str:
-        if m.group(1) == 'ref':
-            return {
-                'layouts': 'https://sw.kovidgoyal.net/kitty/index.html#layouts',
-                'sessions': 'https://sw.kovidgoyal.net/kitty/index.html#sessions',
-                'functional': 'https://sw.kovidgoyal.net/kitty/keyboard-protocol.html#functional-key-definitions',
-            }[m.group(2)]
-        return str(m.group(2))
-
-    return re.sub(r':([a-zA-Z0-9]+):`(.+?)`', sub, text, flags=re.DOTALL)
-
-
-def iter_blocks(lines: Iterable[str]) -> Generator[Tuple[List[str], int], None, None]:
-    current_block: List[str] = []
-    prev_indent = 0
-    for line in lines:
-        indent_size = len(line) - len(line.lstrip())
-        if indent_size != prev_indent or not line:
-            if current_block:
-                yield current_block, prev_indent
-            current_block = []
-        prev_indent = indent_size
-        if not line:
-            yield [''], 100
-        else:
-            current_block.append(line)
-    if current_block:
-        yield current_block, indent_size
-
-
-def wrapped_block(lines: Iterable[str]) -> Generator[str, None, None]:
-    wrapper = getattr(wrapped_block, 'wrapper', None)
-    if wrapper is None:
-        import textwrap
-        wrapper = textwrap.TextWrapper(
-            initial_indent='#: ', subsequent_indent='#: ', width=70, break_long_words=False
-        )
-        setattr(wrapped_block, 'wrapper', wrapper)
-    for block, indent_size in iter_blocks(lines):
-        if indent_size > 0:
-            for line in block:
-                if not line:
-                    yield line
-                else:
-                    yield '#: ' + line
-        else:
-            for line in wrapper.wrap('\n'.join(block)):
-                yield line
-
-
-def render_block(text: str) -> str:
-    text = remove_markup(text)
-    lines = text.splitlines()
-    return '\n'.join(wrapped_block(lines))
-
-
-def as_conf_file(all_options: Iterable[Union[Option, Sequence[Shortcut]]]) -> List[str]:
-    ans = ['# vim:fileencoding=utf-8:ft=conf:foldmethod=marker', '']
-    a = ans.append
-    current_group: Optional[Group] = None
-    num_open_folds = 0
-    all_options_ = list(all_options)
-
-    def render_group(group: Group, is_shortcut: bool) -> None:
-        nonlocal num_open_folds
-        if is_shortcut or '.' not in group.name:
-            a('#: ' + group.short_text + ' {{''{')
-            num_open_folds += 1
-        a('')
-        if group.start_text:
-            a(render_block(group.start_text))
-            a('')
-
-    def handle_group_end(group: Group, new_group_name: str = '', new_group_is_shortcut: bool = False) -> None:
-        nonlocal num_open_folds
-        if group.end_text:
-            a(''), a(render_block(group.end_text))
-        is_subgroup = new_group_name.startswith(group.name + '.')
-        if not is_subgroup and num_open_folds > 0:
-            a('#: }}''}'), a('')
-            num_open_folds -= 1
-
-    def handle_group(new_group: Group, is_shortcut: bool = False) -> None:
-        nonlocal current_group
-        if new_group is not current_group:
-            if current_group:
-                handle_group_end(current_group, new_group.name, is_shortcut)
-            current_group = new_group
-            render_group(current_group, is_shortcut)
-
-    def handle_shortcut(shortcuts: Sequence[Shortcut]) -> None:
-        handle_group(shortcuts[0].group, True)
-        for sc in shortcuts:
-            if sc.add_to_default:
-                a('map {} {}'.format(sc.key, sc.action_def))
-            if sc.long_text:
-                a(''), a(render_block(sc.long_text.strip())), a('')
-
-    def handle_option(opt: Option) -> None:
-        if not opt.long_text or not opt.add_to_docs:
-            return
-        handle_group(opt.group)
-        mopts = list(merged_opts(all_options_, opt, i))
-        sz = max(len(x.name) for x in mopts)
-        for mo in mopts:
-            prefix = '' if mo.add_to_default else '# '
-            a('{}{} {}'.format(prefix, mo.name.ljust(sz), mo.defval_as_string))
-        a('')
-        a(render_block(opt.long_text))
-        a('')
-
-    for i, opt in enumerate(all_options_):
-        if isinstance(opt, Option):
-            handle_option(opt)
-        else:
-            handle_shortcut(opt)
-
-    if current_group:
-        handle_group_end(current_group)
-        while num_open_folds > 0:
-            a('# }}''}')
-            num_open_folds -= 1
-
-    map_groups = []
-    start: Optional[int] = None
-    count: Optional[int] = None
-    for i, line in enumerate(ans):
-        if line.startswith('map '):
-            if start is None:
-                start = i
-                count = 1
-            else:
-                if count is not None:
-                    count += 1
-        else:
-            if start is not None and count is not None:
-                map_groups.append((start, count))
-                start = count = None
-    for start, count in map_groups:
-        r = range(start, start + count)
-        sz = max(len(ans[i].split(' ', 3)[1]) for i in r)
-        for i in r:
-            line = ans[i]
-            parts = line.split(' ', 3)
-            parts[1] = parts[1].ljust(sz)
-            ans[i] = ' '.join(parts)
-
-    return ans
-
-
-def config_lines(
-    all_options: Dict[str, Union[Option, Sequence[Shortcut]]],
-) -> Generator[str, None, None]:
-    for opt in all_options.values():
-        if isinstance(opt, Option):
-            if opt.add_to_default:
-                yield opt.line
-        else:
-            for sc in opt:
-                if sc.add_to_default:
-                    yield sc.line
-
-
-def as_type_stub(
-    all_options: Dict[str, Union[Option, Sequence[Shortcut]]],
-    special_types: Optional[Dict[str, str]] = None,
-    preamble_lines: Union[Tuple[str, ...], List[str], Iterable[str]] = (),
-    extra_fields: Union[Tuple[Tuple[str, str], ...], List[Tuple[str, str]], Iterable[Tuple[str, str]]] = (),
-    class_name: str = 'Options'
-) -> str:
-    ans = ['import typing\n'] + list(preamble_lines) + ['', 'class {}:'.format(class_name)]
-    imports: Set[Tuple[str, str]] = set()
-    overrides = special_types or {}
-    for name, val in all_options.items():
-        if isinstance(val, Option):
-            is_multiple = ' ' in name
-            field_name = name.partition(' ')[0]
-            ans.append('    {}: {}'.format(field_name, overrides.get(field_name, val.type_definition(is_multiple, imports))))
-    for mod, name in imports:
-        ans.insert(0, 'from {} import {}'.format(mod, name))
-        ans.insert(0, 'import {}'.format(mod))
-    for field_name, type_def in extra_fields:
-        ans.append('    {}: {}'.format(field_name, type_def))
-    ans.append('    def __iter__(self) -> typing.Iterator[str]: pass')
-    ans.append('    def __len__(self) -> int: pass')
-    ans.append('    def __getitem__(self, k: typing.Union[int, str]) -> typing.Any: pass')
-    ans.append('    def _replace(self, **kw: typing.Any) -> {}: pass'.format(class_name))
-    return '\n'.join(ans) + '\n\n\n'
-
-
-def save_type_stub(text: str, fpath: str) -> None:
-    fpath += 'i'
-    preamble = '# Update this file by running: ./test.py mypy\n\n'
-    try:
-        existing = open(fpath).read()
-    except FileNotFoundError:
-        existing = ''
-    current = preamble + text
-    if existing != current:
-        open(fpath, 'w').write(current)

kitty/conf/generate.py

@@ -0,0 +1,453 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+import inspect
+import os
+import pprint
+import re
+import textwrap
+from typing import (
+    Any, Callable, Dict, Iterator, List, Set, Tuple, Union, get_type_hints
+)
+
+from kitty.conf.types import Definition, MultiOption, Option, unset
+
+
+def chunks(lst: List, n: int) -> Iterator[List]:
+    for i in range(0, len(lst), n):
+        yield lst[i:i + n]
+
+
+def atoi(text: str) -> str:
+    return f'{int(text):08d}' if text.isdigit() else text
+
+
+def natural_keys(text: str) -> Tuple[str, ...]:
+    return tuple(atoi(c) for c in re.split(r'(\d+)', text))
+
+
+def generate_class(defn: Definition, loc: str) -> Tuple[str, str]:
+    class_lines: List[str] = []
+    tc_lines: List[str] = []
+    a = class_lines.append
+    t = tc_lines.append
+    a('class Options:')
+    t('class Parser:')
+    choices = {}
+    imports: Set[Tuple[str, str]] = set()
+    tc_imports: Set[Tuple[str, str]] = set()
+
+    def type_name(x: type) -> str:
+        ans = x.__name__
+        if x.__module__ and x.__module__ != 'builtins':
+            imports.add((x.__module__, x.__name__))
+        return ans
+
+    def option_type_as_str(x: Any) -> str:
+        if hasattr(x, '__name__'):
+            return type_name(x)
+        ans = repr(x)
+        ans = ans.replace('NoneType', 'None')
+        return ans
+
+    def option_type_data(option: Union[Option, MultiOption]) -> Tuple[Callable, str]:
+        func = option.parser_func
+        if func.__module__ == 'builtins':
+            return func, func.__name__
+        th = get_type_hints(func)
+        rettype = th['return']
+        typ = option_type_as_str(rettype)
+        if isinstance(option, MultiOption):
+            typ = typ[typ.index('[') + 1:-1]
+            typ = typ.replace('Tuple', 'Dict', 1)
+        return func, typ
+
+    is_mutiple_vars = {}
+    option_names = set()
+    color_table = list(map(str, range(256)))
+
+    def parser_function_declaration(option_name: str) -> None:
+        t('')
+        t(f'    def {option_name}(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:')
+
+    for option in sorted(defn.iter_all_options(), key=lambda a: natural_keys(a.name)):
+        option_names.add(option.name)
+        parser_function_declaration(option.name)
+        if isinstance(option, MultiOption):
+            mval: Dict[str, Dict[str, Any]] = {'macos': {}, 'linux': {}, '': {}}
+            func, typ = option_type_data(option)
+            for val in option:
+                if val.add_to_default:
+                    gr = mval[val.only]
+                    for k, v in func(val.defval_as_str):
+                        gr[k] = v
+            is_mutiple_vars[option.name] = typ, mval
+            sig = inspect.signature(func)
+            tc_imports.add((func.__module__, func.__name__))
+            if len(sig.parameters) == 1:
+                t(f'        for k, v in {func.__name__}(val):')
+                t(f'            ans["{option.name}"][k] = v')
+            else:
+                t(f'        for k, v in {func.__name__}(val, ans["{option.name}"]):')
+                t(f'            ans["{option.name}"][k] = v')
+            continue
+
+        if option.choices:
+            typ = 'typing.Literal[{}]'.format(', '.join(repr(x) for x in option.choices))
+            ename = f'choices_for_{option.name}'
+            choices[ename] = typ
+            typ = ename
+            func = str
+        elif defn.has_color_table and option.is_color_table_color:
+            func, typ = option_type_data(option)
+            t(f'        ans[{option.name!r}] = {func.__name__}(val)')
+            tc_imports.add((func.__module__, func.__name__))
+            cnum = int(option.name[5:])
+            color_table[cnum] = '0x{:06x}'.format(func(option.defval_as_string).__int__())
+            continue
+        else:
+            func, typ = option_type_data(option)
+            try:
+                params = inspect.signature(func).parameters
+            except Exception:
+                params = {}
+            if 'dict_with_parse_results' in params:
+                t(f'        {func.__name__}(val, ans)')
+            else:
+                t(f'        ans[{option.name!r}] = {func.__name__}(val)')
+            if func.__module__ != 'builtins':
+                tc_imports.add((func.__module__, func.__name__))
+
+        defval = repr(func(option.defval_as_string))
+        if option.macos_defval is not unset:
+            md = repr(func(option.macos_defval))
+            defval = f'{md} if is_macos else {defval}'
+            imports.add(('kitty.constants', 'is_macos'))
+        a(f'    {option.name}: {typ} = {defval}')
+        if option.choices:
+            t('        val = val.lower()')
+            t(f'        if val not in self.choices_for_{option.name}:')
+            t(f'            raise ValueError(f"The value {{val}} is not a valid choice for {option.name}")')
+            t(f'        ans["{option.name}"] = val')
+            t('')
+            t(f'    choices_for_{option.name} = frozenset({option.choices!r})')
+
+    for option_name, (typ, mval) in is_mutiple_vars.items():
+        a(f'    {option_name}: {typ} = ' '{}')
+
+    for parser, aliases in defn.deprecations.items():
+        for alias in aliases:
+            parser_function_declaration(alias)
+            tc_imports.add((parser.__module__, parser.__name__))
+            t(f'        {parser.__name__}({alias!r}, val, ans)')
+
+    action_parsers = {}
+
+    def resolve_import(ftype: str) -> str:
+        if '.' in ftype:
+            fmod, ftype = ftype.rpartition('.')[::2]
+        else:
+            fmod = f'{loc}.options.utils'
+        imports.add((fmod, ftype))
+        return ftype
+
+    for aname, action in defn.actions.items():
+        option_names.add(aname)
+        action_parsers[aname] = func = action.parser_func
+        th = get_type_hints(func)
+        rettype = th['return']
+        typ = option_type_as_str(rettype)
+        typ = typ[typ.index('[') + 1:-1]
+        a(f'    {aname}: typing.List[{typ}] = []')
+        for imp in action.imports:
+            resolve_import(imp)
+        for fname, ftype in action.fields.items():
+            ftype = resolve_import(ftype)
+            a(f'    {fname}: {ftype} = ' '{}')
+        parser_function_declaration(aname)
+        t(f'        for k in {func.__name__}(val):')
+        t(f'            ans[{aname!r}].append(k)')
+        tc_imports.add((func.__module__, func.__name__))
+
+    if defn.has_color_table:
+        imports.add(('array', 'array'))
+        a('    color_table: array = array("L", (')
+        for grp in chunks(color_table, 8):
+            a('        ' + ', '.join(grp) + ',')
+        a('    ))')
+
+    a('    config_paths: typing.Tuple[str, ...] = ()')
+    a('    config_overrides: typing.Tuple[str, ...] = ()')
+    a('')
+    a('    def __init__(self, options_dict: typing.Optional[typing.Dict[str, typing.Any]] = None) -> None:')
+    a('        if options_dict is not None:')
+    a('            for key in option_names:')
+    a('                setattr(self, key, options_dict[key])')
+
+    a('')
+    a('    @property')
+    a('    def _fields(self) -> typing.Tuple[str, ...]:')
+    a('        return option_names')
+
+    a('')
+    a('    def __iter__(self) -> typing.Iterator[str]:')
+    a('        return iter(self._fields)')
+
+    a('')
+    a('    def __len__(self) -> int:')
+    a('        return len(self._fields)')
+
+    a('')
+    a('    def _copy_of_val(self, name: str) -> typing.Any:')
+    a('        ans = getattr(self, name)')
+    a('        if isinstance(ans, dict):\n            ans = ans.copy()')
+    a('        elif isinstance(ans, list):\n            ans = ans[:]')
+    a('        return ans')
+
+    a('')
+    a('    def _asdict(self) -> typing.Dict[str, typing.Any]:')
+    a('        return {k: self._copy_of_val(k) for k in self}')
+
+    a('')
+    a('    def _replace(self, **kw: typing.Any) -> "Options":')
+    a('        ans = Options()')
+    a('        for name in self:')
+    a('            setattr(ans, name, self._copy_of_val(name))')
+    a('        for name, val in kw.items():')
+    a('            setattr(ans, name, val)')
+    a('        return ans')
+
+    a('')
+    a('    def __getitem__(self, key: typing.Union[int, str]) -> typing.Any:')
+    a('        k = option_names[key] if isinstance(key, int) else key')
+    a('        try:')
+    a('            return getattr(self, k)')
+    a('        except AttributeError:')
+    a('            pass')
+    a('        raise KeyError(f"No option named: {k}")')
+
+    if defn.has_color_table:
+        a('')
+        a('    def __getattr__(self, key: str) -> typing.Any:')
+        a('        if key.startswith("color"):')
+        a('            q = key[5:]')
+        a('            if q.isdigit():')
+        a('                k = int(q)')
+        a('                if 0 <= k <= 255:')
+        a('                    x = self.color_table[k]')
+        a('                    return Color((x >> 16) & 255, (x >> 8) & 255, x & 255)')
+        a('        raise AttributeError(key)')
+        a('')
+        a('    def __setattr__(self, key: str, val: typing.Any) -> typing.Any:')
+        a('        if key.startswith("color"):')
+        a('            q = key[5:]')
+        a('            if q.isdigit():')
+        a('                k = int(q)')
+        a('                if 0 <= k <= 255:')
+        a('                    self.color_table[k] = int(val)')
+        a('                    return')
+        a('        object.__setattr__(self, key, val)')
+
+    a('')
+    a('')
+    a('defaults = Options()')
+    for option_name, (typ, mval) in is_mutiple_vars.items():
+        a(f'defaults.{option_name} = {mval[""]!r}')
+        if mval['macos']:
+            imports.add(('kitty.constants', 'is_macos'))
+            a('if is_macos:')
+            a(f'    defaults.{option_name}.update({mval["macos"]!r}')
+        if mval['macos']:
+            imports.add(('kitty.constants', 'is_macos'))
+            a('if not is_macos:')
+            a(f'    defaults.{option_name}.update({mval["linux"]!r}')
+
+    for aname, func in action_parsers.items():
+        a(f'defaults.{aname} = [')
+        only: Dict[str, List[Tuple[str, Callable]]] = {}
+        for sc in defn.iter_all_maps(aname):
+            if not sc.add_to_default:
+                continue
+            text = sc.parseable_text
+            if sc.only:
+                only.setdefault(sc.only, []).append((text, func))
+            else:
+                for val in func(text):
+                    a(f'    # {sc.name}')
+                    a(f'    {val!r},')
+        a(']')
+        if only:
+            imports.add(('kitty.constants', 'is_macos'))
+            for cond, items in only.items():
+                cond = 'is_macos' if cond == 'macos' else 'not is_macos'
+                a(f'if {cond}:')
+                for (text, func) in items:
+                    for val in func(text):
+                        a(f'    defaults.{aname}.append({val!r})')
+
+    t('')
+    t('')
+    t('def create_result_dict() -> typing.Dict[str, typing.Any]:')
+    t('    return {')
+    for oname in is_mutiple_vars:
+        t(f'        {oname!r}: {{}},')
+    for aname in defn.actions:
+        t(f'        {aname!r}: [],')
+    t('    }')
+
+    t('')
+    t('')
+    t(f'actions = frozenset({tuple(defn.actions)!r})')
+    t('')
+    t('')
+    t('def merge_result_dicts(defaults: typing.Dict[str, typing.Any], vals: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:')
+    t('    ans = {}')
+    t('    for k, v in defaults.items():')
+    t('        if isinstance(v, dict):')
+    t('            ans[k] = merge_dicts(v, vals.get(k, {}))')
+    t('        elif k in actions:')
+    t('            ans[k] = v + vals.get(k, [])')
+    t('        else:')
+    t('            ans[k] = vals.get(k, v)')
+    t('    return ans')
+    tc_imports.add(('kitty.conf.utils', 'merge_dicts'))
+
+    t('')
+    t('')
+    t('parser = Parser()')
+    t('')
+    t('')
+    t('def parse_conf_item(key: str, val: str, ans: typing.Dict[str, typing.Any]) -> bool:')
+    t('    func = getattr(parser, key, None)')
+    t('    if func is not None:')
+    t('        func(val, ans)')
+    t('        return True')
+    t('    return False')
+
+    preamble = ['# generated by gen-config.py DO NOT edit', '# vim:fileencoding=utf-8', '']
+    a = preamble.append
+
+    def output_imports(imports: Set, add_module_imports: bool = True) -> None:
+        a('import typing')
+        seen_mods = {'typing'}
+        mmap: Dict[str, List[str]] = {}
+        for mod, name in imports:
+            mmap.setdefault(mod, []).append(name)
+        for mod in sorted(mmap):
+            names = sorted(mmap[mod])
+            lines = textwrap.wrap(', '.join(names), 100)
+            if len(lines) == 1:
+                s = lines[0]
+            else:
+                s = '\n    '.join(lines)
+                s = f'(\n    {s}\n)'
+            a(f'from {mod} import {s}')
+            if add_module_imports and mod not in seen_mods and mod != s:
+                a(f'import {mod}')
+                seen_mods.add(mod)
+
+    output_imports(imports)
+    a('')
+    if choices:
+        a('if typing.TYPE_CHECKING:')
+        for name, cdefn in choices.items():
+            a(f'    {name} = {cdefn}')
+        a('else:')
+        for name in choices:
+            a(f'    {name} = str')
+
+    a('')
+    a('option_names = (  # {{''{')
+    a(' ' + pprint.pformat(tuple(sorted(option_names, key=natural_keys)))[1:] + '  # }}''}')
+    class_def = '\n'.join(preamble + ['', ''] + class_lines)
+
+    preamble = ['# generated by gen-config.py DO NOT edit', '# vim:fileencoding=utf-8', '']
+    a = preamble.append
+    output_imports(tc_imports, False)
+
+    return class_def, '\n'.join(preamble + ['', ''] + tc_lines)
+
+
+def generate_c_conversion(loc: str, ctypes: List[Option]) -> str:
+    lines: List[str] = []
+    basic_converters = {
+        'int': 'PyLong_AsLong', 'uint': 'PyLong_AsUnsignedLong', 'bool': 'PyObject_IsTrue',
+        'float': 'PyFloat_AsFloat', 'double': 'PyFloat_AsDouble',
+        'time': 'parse_s_double_to_monotonic_t', 'time-ms': 'parse_ms_long_to_monotonic_t'
+    }
+
+    for opt in ctypes:
+        lines.append('')
+        lines.append(f'static void\nconvert_from_python_{opt.name}(PyObject *val, Options *opts) ''{')
+        is_special = opt.ctype.startswith('!')
+        if is_special:
+            func = opt.ctype[1:]
+            lines.append(f'    {func}(val, opts);')
+        else:
+            func = basic_converters.get(opt.ctype, opt.ctype)
+            lines.append(f'    opts->{opt.name} = {func}(val);')
+        lines.append('}')
+        lines.append('')
+        lines.append(f'static void\nconvert_from_opts_{opt.name}(PyObject *py_opts, Options *opts) ''{')
+        lines.append(f'    PyObject *ret = PyObject_GetAttrString(py_opts, "{opt.name}");')
+        lines.append('    if (ret == NULL) return;')
+        lines.append(f'    convert_from_python_{opt.name}(ret, opts);')
+        lines.append('    Py_DECREF(ret);')
+        lines.append('}')
+
+    lines.append('')
+    lines.append('static bool\nconvert_opts_from_python_opts(PyObject *py_opts, Options *opts) ''{')
+    for opt in ctypes:
+        lines.append(f'    convert_from_opts_{opt.name}(py_opts, opts);')
+        lines.append('    if (PyErr_Occurred()) return false;')
+    lines.append('    return true;')
+    lines.append('}')
+
+    preamble = ['// generated by gen-config.py DO NOT edit', '// vim:fileencoding=utf-8', '#pragma once', '#include "to-c.h"']
+    return '\n'.join(preamble + ['', ''] + lines)
+
+
+def write_output(loc: str, defn: Definition) -> None:
+    cls, tc = generate_class(defn, loc)
+    with open(os.path.join(*loc.split('.'), 'options', 'types.py'), 'w') as f:
+        f.write(cls + '\n')
+    with open(os.path.join(*loc.split('.'), 'options', 'parse.py'), 'w') as f:
+        f.write(tc + '\n')
+    ctypes = []
+    for opt in defn.root_group.iter_all_non_groups():
+        if isinstance(opt, Option) and opt.ctype:
+            ctypes.append(opt)
+    if ctypes:
+        c = generate_c_conversion(loc, ctypes)
+        with open(os.path.join(*loc.split('.'), 'options', 'to-c-generated.h'), 'w') as f:
+            f.write(c + '\n')
+
+
+def main() -> None:
+    # To use run it as:
+    # kitty +runpy 'from kitty.conf.generate import main; main()' /path/to/kitten/file.py
+    import importlib
+    import sys
+
+    from kittens.runner import path_to_custom_kitten, resolved_kitten
+    from kitty.constants import config_dir
+
+    kitten = sys.argv[-1]
+    if not kitten.endswith('.py'):
+        kitten += '.py'
+    kitten = resolved_kitten(kitten)
+    path = os.path.realpath(path_to_custom_kitten(config_dir, kitten))
+    if not os.path.dirname(path):
+        raise SystemExit(f'No custom kitten named {kitten} found')
+    sys.path.insert(0, os.path.dirname(path))
+    package_name = os.path.basename(os.path.dirname(path))
+    m = importlib.import_module('kitten_options_definition')
+    defn = getattr(m, 'definition')
+    loc = package_name
+    cls, tc = generate_class(defn, loc)
+    with open(os.path.join(os.path.dirname(path), 'kitten_options_types.py'), 'w') as f:
+        f.write(cls + '\n')
+    with open(os.path.join(os.path.dirname(path), 'kitten_options_parse.py'), 'w') as f:
+        f.write(tc + '\n')

kitty/conf/types.py

@@ -0,0 +1,651 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+import builtins
+import re
+import typing
+from importlib import import_module
+from typing import (
+    Any, Callable, Dict, Iterable, Iterator, List, Match, Optional, Set, Tuple,
+    Union, cast
+)
+
+import kitty.conf.utils as generic_parsers
+
+if typing.TYPE_CHECKING:
+    Only = typing.Literal['macos', 'linux', '']
+else:
+    Only = str
+
+
+class Unset:
+    def __bool__(self) -> bool:
+        return False
+
+
+unset = Unset()
+
+
+def expand_opt_references(conf_name: str, text: str) -> str:
+    conf_name += '.'
+
+    def expand(m: Match) -> str:
+        ref = m.group(1)
+        if '<' not in ref and '.' not in ref:
+            full_ref = conf_name + ref
+            return ':opt:`{} <{}>`'.format(ref, full_ref)
+        return str(m.group())
+
+    return re.sub(r':opt:`(.+?)`', expand, text)
+
+
+def remove_markup(text: str) -> str:
+
+    def sub(m: Match) -> str:
+        if m.group(1) == 'ref':
+            return {
+                'layouts': 'https://sw.kovidgoyal.net/kitty/index.html#layouts',
+                'sessions': 'https://sw.kovidgoyal.net/kitty/index.html#sessions',
+                'functional': 'https://sw.kovidgoyal.net/kitty/keyboard-protocol.html#functional-key-definitions',
+            }[m.group(2)]
+        return str(m.group(2))
+
+    return re.sub(r':([a-zA-Z0-9]+):`(.+?)`', sub, text, flags=re.DOTALL)
+
+
+def iter_blocks(lines: Iterable[str]) -> Iterator[Tuple[List[str], int]]:
+    current_block: List[str] = []
+    prev_indent = 0
+    for line in lines:
+        indent_size = len(line) - len(line.lstrip())
+        if indent_size != prev_indent or not line:
+            if current_block:
+                yield current_block, prev_indent
+            current_block = []
+        prev_indent = indent_size
+        if not line:
+            yield [''], 100
+        else:
+            current_block.append(line)
+    if current_block:
+        yield current_block, indent_size
+
+
+def wrapped_block(lines: Iterable[str]) -> Iterator[str]:
+    wrapper = getattr(wrapped_block, 'wrapper', None)
+    if wrapper is None:
+        import textwrap
+        wrapper = textwrap.TextWrapper(
+            initial_indent='#: ', subsequent_indent='#: ', width=70, break_long_words=False
+        )
+        setattr(wrapped_block, 'wrapper', wrapper)
+    for block, indent_size in iter_blocks(lines):
+        if indent_size > 0:
+            for line in block:
+                if not line:
+                    yield line
+                else:
+                    yield '#: ' + line
+        else:
+            for line in wrapper.wrap('\n'.join(block)):
+                yield line
+
+
+def render_block(text: str) -> str:
+    text = remove_markup(text)
+    lines = text.splitlines()
+    return '\n'.join(wrapped_block(lines))
+
+
+class CoalescedIteratorData:
+
+    option_groups: Dict[int, List['Option']] = {}
+    action_groups: Dict[str, List['Mapping']] = {}
+    coalesced: Set[int] = set()
+    initialized: bool = False
+    kitty_mod: str = 'kitty_mod'
+
+    def initialize(self, root: 'Group') -> None:
+        if self.initialized:
+            return
+        self.root = root
+        option_groups = self.option_groups = {}
+        current_group: List[Option] = []
+        action_groups: Dict[str, List[Mapping]] = {}
+        self.action_groups = action_groups
+        coalesced = self.coalesced = set()
+        self.kitty_mod = 'kitty_mod'
+        for item in root.iter_all_non_groups():
+            if isinstance(item, Option):
+                if item.name == 'kitty_mod':
+                    self.kitty_mod = item.defval_as_string
+                if current_group:
+                    if item.needs_coalescing:
+                        current_group.append(item)
+                        coalesced.add(id(item))
+                        continue
+                    option_groups[id(current_group[0])] = current_group[1:]
+                    current_group = [item]
+                else:
+                    current_group.append(item)
+            elif isinstance(item, Mapping):
+                if item.name in action_groups:
+                    coalesced.add(id(item))
+                    action_groups[item.name].append(item)
+                else:
+                    action_groups[item.name] = []
+        if current_group:
+            option_groups[id(current_group[0])] = current_group[1:]
+
+    def option_group_for_option(self, opt: 'Option') -> List['Option']:
+        return self.option_groups.get(id(opt), [])
+
+    def action_group_for_action(self, ac: 'Mapping') -> List['Mapping']:
+        return self.action_groups.get(ac.name, [])
+
+
+class Option:
+
+    def __init__(
+        self, name: str, defval: str, macos_default: Union[Unset, str], parser_func: Callable,
+        long_text: str, documented: bool, group: 'Group', choices: Tuple[str, ...], ctype: str
+    ):
+        self.name = name
+        self.ctype = ctype
+        self.defval_as_string = defval
+        self.macos_defval = macos_default
+        self.long_text = long_text
+        self.documented = documented
+        self.group = group
+        self.parser_func = parser_func
+        self.choices = choices
+
+    @property
+    def needs_coalescing(self) -> bool:
+        return self.documented and not self.long_text
+
+    @property
+    def is_color_table_color(self) -> bool:
+        return self.name.startswith('color') and self.name[5:].isdigit()
+
+    def as_conf(self, commented: bool = False, level: int = 0, option_group: List['Option'] = []) -> List[str]:
+        ans: List[str] = []
+        a = ans.append
+        if not self.documented:
+            return ans
+        if option_group:
+            sz = max(len(self.name), max(len(o.name) for o in option_group))
+            a(f'{self.name.ljust(sz)} {self.defval_as_string}')
+            for o in option_group:
+                a(f'{o.name.ljust(sz)} {o.defval_as_string}')
+        else:
+            a(f'{self.name} {self.defval_as_string}')
+        if self.long_text:
+            a('')
+            a(render_block(self.long_text))
+            a('')
+        return ans
+
+    def as_rst(
+        self, conf_name: str, shortcut_slugs: Dict[str, Tuple[str, str]],
+        kitty_mod: str, level: int = 0, option_group: List['Option'] = []
+    ) -> List[str]:
+        ans: List[str] = []
+        a = ans.append
+        if not self.documented:
+            return ans
+        mopts = [self] + option_group
+        a('.. opt:: ' + ', '.join(conf_name + '.' + mo.name for mo in mopts))
+        a('.. code-block:: conf')
+        a('')
+        sz = max(len(x.name) for x in mopts)
+        for mo in mopts:
+            a(('    {:%ds} {}' % sz).format(mo.name, mo.defval_as_string))
+        a('')
+        if self.long_text:
+            a(expand_opt_references(conf_name, self.long_text))
+            a('')
+        return ans
+
+
+class MultiVal:
+
+    def __init__(self, val_as_str: str, add_to_default: bool, documented: bool, only: Only) -> None:
+        self.defval_as_str = val_as_str
+        self.documented = documented
+        self.only = only
+        self.add_to_default = add_to_default
+
+
+class MultiOption:
+
+    def __init__(self, name: str, parser_func: Callable, long_text: str, group: 'Group', ctype: str):
+        self.name = name
+        self.ctype = ctype
+        self.parser_func = parser_func
+        self.long_text = long_text
+        self.group = group
+        self.items: List[MultiVal] = []
+
+    def add_value(self, val_as_str: str, add_to_default: bool, documented: bool, only: Only) -> None:
+        self.items.append(MultiVal(val_as_str, add_to_default, documented, only))
+
+    def __iter__(self) -> Iterator[MultiVal]:
+        yield from self.items
+
+    def as_conf(self, commented: bool = False, level: int = 0) -> List[str]:
+        ans: List[str] = []
+        a = ans.append
+        for k in self.items:
+            if k.documented:
+                prefix = '' if k.add_to_default else '# '
+                a(f'{prefix}{self.name} {k.defval_as_str}')
+        if self.long_text:
+            a('')
+            a(render_block(self.long_text))
+            a('')
+        return ans
+
+    def as_rst(self, conf_name: str, shortcut_slugs: Dict[str, Tuple[str, str]], kitty_mod: str, level: int = 0) -> List[str]:
+        ans: List[str] = []
+        a = ans.append
+        a(f'.. opt:: {conf_name}.{self.name}')
+        a('.. code-block:: conf')
+        a('')
+        for k in self.items:
+            if k.documented:
+                a(f'    {self.name:s} {k.defval_as_str}')
+        a('')
+        if self.long_text:
+            a(expand_opt_references(conf_name, self.long_text))
+            a('')
+        return ans
+
+
+class Mapping:
+    add_to_default: bool
+    short_text: str
+    long_text: str
+    documented: bool
+    setting_name: str
+    name: str
+    only: Only
+
+    @property
+    def parseable_text(self) -> str:
+        return ''
+
+    @property
+    def key_text(self) -> str:
+        return ''
+
+    def as_conf(self, commented: bool = False, level: int = 0) -> List[str]:
+        ans: List[str] = []
+        if self.documented:
+            a = ans.append
+            if self.add_to_default:
+                a(self.setting_name + ' ' + self.parseable_text)
+            if self.long_text:
+                a(''), a(render_block(self.long_text.strip())), a('')
+        return ans
+
+    def as_rst(
+        self, conf_name: str, shortcut_slugs: Dict[str, Tuple[str, str]],
+        kitty_mod: str, level: int = 0, action_group: List['Mapping'] = []
+    ) -> List[str]:
+        ans: List[str] = []
+        a = ans.append
+        if not self.documented:
+            return ans
+        if not self.short_text:
+            raise ValueError(f'The shortcut for {self.name} has no short_text')
+        sc_text = f'{conf_name}.{self.short_text}'
+        shortcut_slugs[f'{conf_name}.{self.name}'] = (sc_text, self.key_text.replace('kitty_mod', kitty_mod))
+        a('.. shortcut:: ' + sc_text)
+        block_started = False
+        for sc in [self] + action_group:
+            if sc.add_to_default and sc.documented:
+                if not block_started:
+                    a('.. code-block:: conf')
+                    a('')
+                    block_started = True
+                suffix = ''
+                if sc.only == 'macos':
+                    suffix = ' 🍎'
+                elif sc.only == 'linux':
+                    suffix = ' 🐧'
+                a(f'    {sc.setting_name} {sc.parseable_text.replace("kitty_mod", kitty_mod)}{suffix}')
+        a('')
+        if self.long_text:
+            a('')
+            a(expand_opt_references(conf_name, self.long_text))
+            a('')
+
+        return ans
+
+
+class ShortcutMapping(Mapping):
+    setting_name: str = 'map'
+
+    def __init__(
+        self, name: str, key: str, action_def: str, short_text: str, long_text: str, add_to_default: bool, documented: bool, group: 'Group', only: Only
+    ):
+        self.name = name
+        self.only = only
+        self.key = key
+        self.action_def = action_def
+        self.short_text = short_text
+        self.long_text = long_text
+        self.documented = documented
+        self.add_to_default = add_to_default
+        self.group = group
+
+    @property
+    def parseable_text(self) -> str:
+        return f'{self.key} {self.action_def}'
+
+    @property
+    def key_text(self) -> str:
+        return self.key
+
+
+class MouseMapping(Mapping):
+    setting_name: str = 'mouse_map'
+
+    def __init__(
+        self, name: str, button: str, event: str, modes: str, action_def: str,
+        short_text: str, long_text: str, add_to_default: bool, documented: bool, group: 'Group', only: Only
+    ):
+        self.name = name
+        self.only = only
+        self.button = button
+        self.event = event
+        self.modes = modes
+        self.action_def = action_def
+        self.short_text = short_text
+        self.long_text = long_text
+        self.documented = documented
+        self.add_to_default = add_to_default
+        self.group = group
+
+    @property
+    def parseable_text(self) -> str:
+        return f'{self.button} {self.event} {self.modes} {self.action_def}'
+
+    @property
+    def key_text(self) -> str:
+        return self.button
+
+
+NonGroups = Union[Option, MultiOption, ShortcutMapping, MouseMapping]
+GroupItem = Union[NonGroups, 'Group']
+
+
+class Group:
+
+    def __init__(self, name: str, title: str, coalesced_iterator_data: CoalescedIteratorData, start_text: str = '', parent: Optional['Group'] = None):
+        self.name = name
+        self.coalesced_iterator_data = coalesced_iterator_data
+        self.title = title
+        self.start_text = start_text
+        self.end_text = ''
+        self.items: List[GroupItem] = []
+        self.parent = parent
+
+    def append(self, item: GroupItem) -> None:
+        self.items.append(item)
+
+    def __iter__(self) -> Iterator[GroupItem]:
+        return iter(self.items)
+
+    def __len__(self) -> int:
+        return len(self.items)
+
+    def iter_with_coalesced_options(self) -> Iterator[GroupItem]:
+        for item in self:
+            if id(item) not in self.coalesced_iterator_data.coalesced:
+                yield item
+
+    def iter_all(self) -> Iterator[GroupItem]:
+        for x in self:
+            yield x
+            if isinstance(x, Group):
+                yield from x.iter_all()
+
+    def iter_all_non_groups(self) -> Iterator[NonGroups]:
+        for x in self:
+            if isinstance(x, Group):
+                yield from x.iter_all_non_groups()
+            else:
+                yield x
+
+    def as_rst(self, conf_name: str, shortcut_slugs: Dict[str, Tuple[str, str]], kitty_mod: str = 'kitty_mod', level: int = 0) -> List[str]:
+        ans: List[str] = []
+        a = ans.append
+        if level:
+            a('')
+            a(f'.. _conf-{conf_name}-{self.name}:')
+            a('')
+            a(self.title)
+            heading_level = '+' if level > 1 else '^'
+            a(heading_level * (len(self.title) + 20))
+            a('')
+            if self.start_text:
+                a(self.start_text)
+                a('')
+        else:
+            ans.extend(('.. default-domain:: conf', ''))
+
+        kitty_mod = self.coalesced_iterator_data.kitty_mod
+        for item in self.iter_with_coalesced_options():
+            if isinstance(item, Option):
+                lines = item.as_rst(conf_name, shortcut_slugs, kitty_mod, option_group=self.coalesced_iterator_data.option_group_for_option(item))
+            elif isinstance(item, Mapping):
+                lines = item.as_rst(conf_name, shortcut_slugs, kitty_mod, level + 1, action_group=self.coalesced_iterator_data.action_group_for_action(item))
+            else:
+                lines = item.as_rst(conf_name, shortcut_slugs, kitty_mod, level + 1)
+            ans.extend(lines)
+
+        if level:
+            if self.end_text:
+                a('')
+                a(self.end_text)
+        return ans
+
+    def as_conf(self, commented: bool = False, level: int = 0) -> List[str]:
+        ans: List[str] = []
+        a = ans.append
+        if level:
+            a('#: ' + self.title + ' {{''{')
+            a('')
+            if self.start_text:
+                a(render_block(self.start_text))
+                a('')
+        else:
+            ans.extend(('# vim:fileencoding=utf-8:ft=conf:foldmethod=marker', ''))
+
+        for item in self.iter_with_coalesced_options():
+            if isinstance(item, Option):
+                lines = item.as_conf(option_group=self.coalesced_iterator_data.option_group_for_option(item))
+            else:
+                lines = item.as_conf(commented, level + 1)
+            ans.extend(lines)
+
+        if level:
+            if self.end_text:
+                a('')
+                a(render_block(self.end_text))
+            a('#: }}''}')
+            a('')
+        else:
+            map_groups = []
+            start: Optional[int] = None
+            count: Optional[int] = None
+            for i, line in enumerate(ans):
+                if line.startswith('map ') or line.startswith('mouse_map '):
+                    if start is None:
+                        start = i
+                        count = 1
+                    else:
+                        if count is not None:
+                            count += 1
+                else:
+                    if start is not None and count is not None:
+                        map_groups.append((start, count))
+                        start = count = None
+            for start, count in map_groups:
+                r = range(start, start + count)
+                sz = max(len(ans[i].split(' ', 3)[1]) for i in r)
+                for i in r:
+                    line = ans[i]
+                    parts = line.split(' ', 3)
+                    parts[1] = parts[1].ljust(sz)
+                    ans[i] = ' '.join(parts)
+
+            if commented:
+                ans = [x if x.startswith('#') or not x.strip() else ('# ' + x) for x in ans]
+
+        return ans
+
+
+def resolve_import(name: str, module: Any = None) -> Callable:
+    ans = None
+    if name.count('.') > 1:
+        m = import_module(name.rpartition('.')[0])
+        ans = getattr(m, name.rpartition('.')[2])
+    else:
+        ans = getattr(builtins, name, None)
+        if not callable(ans):
+            ans = getattr(generic_parsers, name, None)
+            if not callable(ans):
+                ans = getattr(module, name)
+    if not callable(ans):
+        raise TypeError(f'{name} is not a function')
+    return cast(Callable, ans)
+
+
+class Action:
+
+    def __init__(self, name: str, option_type: str, fields: Dict[str, str], imports: Iterable[str]):
+        self.name = name
+        self._parser_func = option_type
+        self.fields = fields
+        self.imports = frozenset(imports)
+
+    def resolve_imports(self, module: Any) -> 'Action':
+        self.parser_func = resolve_import(self._parser_func, module)
+        return self
+
+
+class Definition:
+
+    def __init__(self, package: str, *actions: Action, has_color_table: bool = False) -> None:
+        if package.startswith('!'):
+            self.module_for_parsers = import_module(package[1:])
+        else:
+            self.module_for_parsers = import_module(f'{package}.options.utils')
+        self.has_color_table = has_color_table
+        self.coalesced_iterator_data = CoalescedIteratorData()
+        self.root_group = Group('', '', self.coalesced_iterator_data)
+        self.current_group = self.root_group
+        self.option_map: Dict[str, Option] = {}
+        self.multi_option_map: Dict[str, MultiOption] = {}
+        self.shortcut_map: Dict[str, List[ShortcutMapping]] = {}
+        self.mouse_map: Dict[str, List[MouseMapping]] = {}
+        self.actions = {a.name: a.resolve_imports(self.module_for_parsers) for a in actions}
+        self.deprecations: Dict[Callable, Tuple[str, ...]] = {}
+
+    def iter_all_non_groups(self) -> Iterator[NonGroups]:
+        yield from self.root_group.iter_all_non_groups()
+
+    def iter_all_options(self) -> Iterator[Union[Option, MultiOption]]:
+        for x in self.iter_all_non_groups():
+            if isinstance(x, (Option, MultiOption)):
+                yield x
+
+    def iter_all_maps(self, which: str = 'map') -> Iterator[Union[ShortcutMapping, MouseMapping]]:
+        for x in self.iter_all_non_groups():
+            if isinstance(x, ShortcutMapping) and which == 'map':
+                yield x
+            elif isinstance(x, MouseMapping) and which == 'mouse_map':
+                yield x
+
+    def parser_func(self, name: str) -> Callable:
+        ans = getattr(builtins, name, None)
+        if callable(ans):
+            return cast(Callable, ans)
+        ans = getattr(generic_parsers, name, None)
+        if callable(ans):
+            return cast(Callable, ans)
+        ans = getattr(self.module_for_parsers, name)
+        if not callable(ans):
+            raise TypeError(f'{name} is not a function')
+        return cast(Callable, ans)
+
+    def add_group(self, name: str, title: str = '', start_text: str = '') -> None:
+        self.current_group = Group(name, title or name, self.coalesced_iterator_data, start_text.strip(), self.current_group)
+        if self.current_group.parent is not None:
+            self.current_group.parent.append(self.current_group)
+
+    def end_group(self, end_text: str = '') -> None:
+        self.current_group.end_text = end_text.strip()
+        if self.current_group.parent is not None:
+            self.current_group = self.current_group.parent
+
+    def add_option(
+        self, name: str, defval: Union[str, float, int, bool],
+        option_type: str = 'str', long_text: str = '',
+        documented: bool = True, add_to_default: bool = False,
+        only: Only = '', macos_default: Union[Unset, str] = unset,
+        choices: Tuple[str, ...] = (),
+        ctype: str = '',
+    ) -> None:
+        if isinstance(defval, bool):
+            defval = 'yes' if defval else 'no'
+        else:
+            defval = str(defval)
+        is_multiple = name.startswith('+')
+        long_text = long_text.strip()
+        if is_multiple:
+            name = name[1:]
+            if macos_default is not unset:
+                raise TypeError(f'Cannot specify macos_default for is_multiple option: {name} use only instead')
+            is_new = name not in self.multi_option_map
+            if is_new:
+                self.multi_option_map[name] = MultiOption(name, self.parser_func(option_type), long_text, self.current_group, ctype)
+            mopt = self.multi_option_map[name]
+            if is_new:
+                self.current_group.append(mopt)
+            mopt.add_value(defval, add_to_default, documented, only)
+            return
+        opt = Option(name, defval, macos_default, self.parser_func(option_type), long_text, documented, self.current_group, choices, ctype)
+        self.current_group.append(opt)
+        self.option_map[name] = opt
+
+    def add_map(
+        self, short_text: str, defn: str, long_text: str = '', add_to_default: bool = True, documented: bool = True, only: Only = ''
+    ) -> None:
+        name, key, action_def = defn.split(maxsplit=2)
+        sc = ShortcutMapping(name, key, action_def, short_text, long_text.strip(), add_to_default, documented, self.current_group, only)
+        self.current_group.append(sc)
+        self.shortcut_map.setdefault(name, []).append(sc)
+
+    def add_mouse_map(
+        self, short_text: str, defn: str, long_text: str = '', add_to_default: bool = True, documented: bool = True, only: Only = ''
+    ) -> None:
+        name, button, event, modes, action_def = defn.split(maxsplit=4)
+        mm = MouseMapping(name, button, event, modes, action_def, short_text, long_text.strip(), add_to_default, documented, self.current_group, only)
+        self.current_group.append(mm)
+        self.mouse_map.setdefault(name, []).append(mm)
+
+    def add_deprecation(self, parser_name: str, *aliases: str) -> None:
+        self.deprecations[self.parser_func(parser_name)] = aliases
+
+    def as_conf(self, commented: bool = False) -> List[str]:
+        self.coalesced_iterator_data.initialize(self.root_group)
+        return self.root_group.as_conf(commented)
+
+    def as_rst(self, conf_name: str, shortcut_slugs: Dict[str, Tuple[str, str]]) -> List[str]:
+        self.coalesced_iterator_data.initialize(self.root_group)
+        return self.root_group.as_rst(conf_name, shortcut_slugs)

kitty/conf/utils.py

@@ -6,24 +6,40 @@ import os
 import re
 import shlex
 from typing import (
-    Any, Callable, Dict, FrozenSet, Generator, Iterable, Iterator, List,
-    NamedTuple, Optional, Sequence, Tuple, Type, TypeVar, Union
+    Any, Callable, Dict, Generator, Iterable, List, NamedTuple, Optional,
+    Sequence, Set, Tuple, TypeVar, Union
 )
 
 from ..rgb import Color, to_color as as_color
-from ..types import ParsedShortcut, ConvertibleToNumbers
+from ..types import ConvertibleToNumbers, ParsedShortcut
+from ..typing import Protocol
 from ..utils import expandvars, log_error
 
 key_pat = re.compile(r'([a-zA-Z][a-zA-Z0-9_-]*)\s+(.+)$')
+ItemParser = Callable[[str, str, Dict[str, Any]], bool]
 T = TypeVar('T')
 
 
+class OptionsProtocol(Protocol):
+
+    def _asdict(self) -> Dict[str, Any]:
+        pass
+
+
 class BadLine(NamedTuple):
     number: int
     line: str
     exception: Exception
 
 
+def positive_int(x: ConvertibleToNumbers) -> int:
+    return max(0, int(x))
+
+
+def positive_float(x: ConvertibleToNumbers) -> float:
+    return max(0, float(x))
+
+
 def to_color(x: str) -> Color:
     ans = as_color(x, validate=True)
     if ans is None:  # this is only for type-checking
@@ -72,7 +88,11 @@ class ToCmdline:
         )
 
 
-to_cmdline = ToCmdline()
+to_cmdline_implementation = ToCmdline()
+
+
+def to_cmdline(x: str) -> List[str]:
+    return to_cmdline_implementation(x)
 
 
 def python_string(text: str) -> str:
@@ -90,7 +110,7 @@ class Choice:
     def __call__(self, x: str) -> str:
         x = x.lower()
         if x not in self.all_choices:
-            x = self.defval
+            raise ValueError(f'The value {x} is not a known choice')
         return x
 
 
@@ -100,10 +120,10 @@ def choices(*choices: str) -> Choice:
 
 def parse_line(
     line: str,
-    type_convert: Callable[[str, Any], Any],
-    special_handling: Callable,
-    ans: Dict[str, Any], all_keys: Optional[FrozenSet[str]],
-    base_path_for_includes: str
+    parse_conf_item: ItemParser,
+    ans: Dict[str, Any],
+    base_path_for_includes: str,
+    accumulate_bad_lines: Optional[List[BadLine]] = None
 ) -> None:
     line = line.strip()
     if not line or line.startswith('#'):
@@ -113,15 +133,13 @@ def parse_line(
         log_error('Ignoring invalid config line: {}'.format(line))
         return
     key, val = m.groups()
-    if special_handling(key, val, ans):
-        return
     if key == 'include':
         val = os.path.expandvars(os.path.expanduser(val.strip()))
         if not os.path.isabs(val):
             val = os.path.join(base_path_for_includes, val)
         try:
             with open(val, encoding='utf-8', errors='replace') as include:
-                _parse(include, type_convert, special_handling, ans, all_keys)
+                _parse(include, parse_conf_item, ans, accumulate_bad_lines)
         except FileNotFoundError:
             log_error(
                 'Could not find included config file: {}, ignoring'.
@@ -133,18 +151,14 @@ def parse_line(
                 format(val)
             )
         return
-    if all_keys is not None and key not in all_keys:
+    if not parse_conf_item(key, val, ans):
         log_error('Ignoring unknown config key: {}'.format(key))
-        return
-    ans[key] = type_convert(key, val)
 
 
 def _parse(
     lines: Iterable[str],
-    type_convert: Callable[[str, Any], Any],
-    special_handling: Callable,
+    parse_conf_item: ItemParser,
     ans: Dict[str, Any],
-    all_keys: Optional[FrozenSet[str]],
     accumulate_bad_lines: Optional[List[BadLine]] = None
 ) -> None:
     name = getattr(lines, 'name', None)
@@ -156,8 +170,7 @@ def _parse(
     for i, line in enumerate(lines):
         try:
             parse_line(
-                line, type_convert, special_handling, ans, all_keys,
-                base_path_for_includes
+                line, parse_conf_item, ans, base_path_for_includes, accumulate_bad_lines
             )
         except Exception as e:
             if accumulate_bad_lines is None:
@@ -167,64 +180,15 @@ def _parse(
 
 def parse_config_base(
     lines: Iterable[str],
-    defaults: Any,
-    type_convert: Callable[[str, Any], Any],
-    special_handling: Callable,
+    parse_conf_item: ItemParser,
     ans: Dict[str, Any],
-    check_keys: bool = True,
     accumulate_bad_lines: Optional[List[BadLine]] = None
 ) -> None:
-    all_keys: Optional[FrozenSet[str]] = defaults._asdict() if check_keys else None
     _parse(
-        lines, type_convert, special_handling, ans, all_keys, accumulate_bad_lines
+        lines, parse_conf_item, ans, accumulate_bad_lines
     )
 
 
-def create_options_class(all_keys: Iterable[str]) -> Type:
-    keys = tuple(sorted(all_keys))
-    slots = keys + ('_fields', )
-
-    def __init__(self: Any, kw: Dict[str, Any]) -> None:
-        for k, v in kw.items():
-            setattr(self, k, v)
-
-    def __iter__(self: Any) -> Iterator[str]:
-        return iter(keys)
-
-    def __len__(self: Any) -> int:
-        return len(keys)
-
-    def __getitem__(self: Any, i: Union[int, str]) -> Any:
-        if isinstance(i, int):
-            i = keys[i]
-        try:
-            return getattr(self, i)
-        except AttributeError:
-            raise KeyError('No option named: {}'.format(i))
-
-    def _asdict(self: Any) -> Dict[str, Any]:
-        return {k: getattr(self, k) for k in self._fields}
-
-    def _replace(self: Any, **kw: Dict) -> Any:
-        ans = self._asdict()
-        ans.update(kw)
-        return self.__class__(ans)
-
-    ans = type(
-        'Options', (), {
-            '__slots__': slots,
-            '__init__': __init__,
-            '_asdict': _asdict,
-            '_replace': _replace,
-            '__iter__': __iter__,
-            '__len__': __len__,
-            '__getitem__': __getitem__
-        }
-    )
-    ans._fields = keys  # type: ignore
-    return ans
-
-
 def merge_dicts(defaults: Dict, newvals: Dict) -> Dict:
     ans = defaults.copy()
     ans.update(newvals)
@@ -243,14 +207,14 @@ def resolve_config(SYSTEM_CONF: str, defconf: str, config_files_on_cmd_line: Seq
 
 
 def load_config(
-        Options: Type[T],
-        defaults: Any,
-        parse_config: Callable[[Iterable[str]], Dict[str, Any]],
-        merge_configs: Callable[[Dict, Dict], Dict],
-        *paths: str,
-        overrides: Optional[Iterable[str]] = None
-) -> T:
-    ans: Dict = defaults._asdict()
+    defaults: OptionsProtocol,
+    parse_config: Callable[[Iterable[str]], Dict[str, Any]],
+    merge_configs: Callable[[Dict, Dict], Dict],
+    *paths: str,
+    overrides: Optional[Iterable[str]] = None
+) -> Tuple[Dict[str, Any], Tuple[str, ...]]:
+    ans = defaults._asdict()
+    found_paths = []
     for path in paths:
         if not path:
             continue
@@ -259,18 +223,12 @@ def load_config(
                 vals = parse_config(f)
         except (FileNotFoundError, PermissionError):
             continue
+        found_paths.append(path)
         ans = merge_configs(ans, vals)
     if overrides is not None:
         vals = parse_config(overrides)
         ans = merge_configs(ans, vals)
-    return Options(ans)  # type: ignore
-
-
-def init_config(default_config_lines: Iterable[str], parse_config: Callable) -> Tuple[Type, Any]:
-    defaults = parse_config(default_config_lines, check_keys=False)
-    Options = create_options_class(defaults.keys())
-    defaults = Options(defaults)
-    return Options, defaults
+    return ans, tuple(found_paths)
 
 
 def key_func() -> Tuple[Callable[..., Callable], Dict[str, Callable]]:
@@ -291,14 +249,21 @@ def key_func() -> Tuple[Callable[..., Callable], Dict[str, Callable]]:
     return func_with_args, ans
 
 
-KittensKeyAction = Tuple[str, Tuple[str, ...]]
+class KeyAction(NamedTuple):
+    func: str
+    args: Tuple[Union[str, float, bool, int, None], ...] = ()
+
+    def __repr__(self) -> str:
+        if self.args:
+            return f'KeyAction({self.func!r}, {self.args!r})'
+        return f'KeyAction({self.func!r})'
 
 
-def parse_kittens_func_args(action: str, args_funcs: Dict[str, Callable]) -> KittensKeyAction:
+def parse_kittens_func_args(action: str, args_funcs: Dict[str, Callable]) -> KeyAction:
     parts = action.strip().split(' ', 1)
     func = parts[0]
     if len(parts) == 1:
-        return func, ()
+        return KeyAction(func, ())
     rest = parts[1]
 
     try:
@@ -317,15 +282,38 @@ def parse_kittens_func_args(action: str, args_funcs: Dict[str, Callable]) -> Kit
     if not isinstance(args, (list, tuple)):
         args = (args, )
 
-    return func, tuple(args)
+    return KeyAction(func, tuple(args))
+
+
+KittensKeyDefinition = Tuple[ParsedShortcut, KeyAction]
+KittensKeyMap = Dict[ParsedShortcut, KeyAction]
 
 
 def parse_kittens_key(
     val: str, funcs_with_args: Dict[str, Callable]
-) -> Optional[Tuple[KittensKeyAction, ParsedShortcut]]:
+) -> Optional[KittensKeyDefinition]:
     from ..key_encoding import parse_shortcut
     sc, action = val.partition(' ')[::2]
     if not sc or not action:
         return None
     ans = parse_kittens_func_args(action, funcs_with_args)
-    return ans, parse_shortcut(sc)
+    return parse_shortcut(sc), ans
+
+
+def uniq(vals: Iterable[T]) -> List[T]:
+    seen: Set[T] = set()
+    seen_add = seen.add
+    return [x for x in vals if x not in seen and not seen_add(x)]
+
+
+def save_type_stub(text: str, fpath: str) -> None:
+    fpath += 'i'
+    preamble = '# Update this file by running: ./test.py mypy\n\n'
+    try:
+        existing = open(fpath).read()
+    except FileNotFoundError:
+        existing = ''
+    current = preamble + text
+    if existing != current:
+        with open(fpath, 'w') as f:
+            f.write(current)

kitty/config.py

@@ -4,653 +4,34 @@
 
 import json
 import os
-import re
-import sys
 from contextlib import contextmanager, suppress
 from functools import partial
-from typing import (
-    Any, Callable, Dict, Generator, Iterable, List, NamedTuple, Optional,
-    Sequence, Set, Tuple, Type, Union
-)
+from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple
 
-from . import fast_data_types as defines
-from .conf.definition import as_conf_file, config_lines
-from .conf.utils import (
-    BadLine, init_config, key_func, load_config as _load_config, merge_dicts,
-    parse_config_base, python_string, to_bool, to_cmdline
+from .conf.utils import BadLine, load_config as _load_config, parse_config_base
+from .constants import cache_dir, defconf
+from .options.types import Options, defaults, option_names
+from .options.utils import (
+    KeyDefinition, KeyMap, MouseMap, MouseMapping, SequenceMap
 )
-from .config_data import InvalidMods, all_options, parse_shortcut, type_convert
-from .constants import cache_dir, defconf, is_macos
-from .fonts import FontFeature
-from .options_stub import Options as OptionsStub
-from .types import SingleKey
 from .typing import TypedDict
-from .utils import expandvars, log_error
+from .utils import log_error
 
-KeyMap = Dict[SingleKey, 'KeyAction']
-KeySequence = Tuple[SingleKey, ...]
-SubSequenceMap = Dict[KeySequence, 'KeyAction']
-SequenceMap = Dict[SingleKey, SubSequenceMap]
 
+def option_names_for_completion() -> Tuple[str, ...]:
+    return option_names
 
-class KeyAction(NamedTuple):
-    func: str
-    args: Sequence[str] = ()
 
-
-func_with_args, args_funcs = key_func()
-FuncArgsType = Tuple[str, Sequence[Any]]
-
-
-@func_with_args(
-    'pass_selection_to_program', 'new_window', 'new_tab', 'new_os_window',
-    'new_window_with_cwd', 'new_tab_with_cwd', 'new_os_window_with_cwd',
-    'launch'
-    )
-def shlex_parse(func: str, rest: str) -> FuncArgsType:
-    return func, to_cmdline(rest)
-
-
-@func_with_args('combine')
-def combine_parse(func: str, rest: str) -> FuncArgsType:
-    sep, rest = rest.split(maxsplit=1)
-    parts = re.split(r'\s*' + re.escape(sep) + r'\s*', rest)
-    args = tuple(map(parse_key_action, filter(None, parts)))
-    return func, args
-
-
-@func_with_args('send_text')
-def send_text_parse(func: str, rest: str) -> FuncArgsType:
-    args = rest.split(maxsplit=1)
-    mode = ''
-    data = b''
-    if len(args) > 1:
-        mode = args[0]
-        try:
-            data = parse_send_text_bytes(args[1])
-        except Exception:
-            log_error('Ignoring invalid send_text string: ' + args[1])
-    return func, [mode, data]
-
-
-@func_with_args('run_kitten', 'run_simple_kitten', 'kitten')
-def kitten_parse(func: str, rest: str) -> FuncArgsType:
-    if func == 'kitten':
-        args = rest.split(maxsplit=1)
-    else:
-        args = rest.split(maxsplit=2)[1:]
-        func = 'kitten'
-    return func, args
-
-
-@func_with_args('goto_tab')
-def goto_tab_parse(func: str, rest: str) -> FuncArgsType:
-    args = (max(0, int(rest)), )
-    return func, args
-
-
-@func_with_args('detach_window')
-def detach_window_parse(func: str, rest: str) -> FuncArgsType:
-    if rest not in ('new', 'new-tab', 'ask'):
-        log_error('Ignoring invalid detach_window argument: {}'.format(rest))
-        rest = 'new'
-    return func, (rest,)
-
-
-@func_with_args('detach_tab')
-def detach_tab_parse(func: str, rest: str) -> FuncArgsType:
-    if rest not in ('new', 'ask'):
-        log_error('Ignoring invalid detach_tab argument: {}'.format(rest))
-        rest = 'new'
-    return func, (rest,)
-
-
-@func_with_args('set_background_opacity', 'goto_layout', 'kitty_shell')
-def simple_parse(func: str, rest: str) -> FuncArgsType:
-    return func, [rest]
-
-
-@func_with_args('set_font_size')
-def float_parse(func: str, rest: str) -> FuncArgsType:
-    return func, (float(rest),)
-
-
-@func_with_args('signal_child')
-def signal_child_parse(func: str, rest: str) -> FuncArgsType:
-    import signal
-    signals = []
-    for q in rest.split():
-        try:
-            signum = getattr(signal, q.upper())
-        except AttributeError:
-            log_error(f'Unknown signal: {rest} ignoring')
-        else:
-            signals.append(signum)
-    return func, tuple(signals)
-
-
-@func_with_args('change_font_size')
-def parse_change_font_size(func: str, rest: str) -> Tuple[str, Tuple[bool, Optional[str], float]]:
-    vals = rest.strip().split(maxsplit=1)
-    if len(vals) != 2:
-        log_error('Invalid change_font_size specification: {}, treating it as default'.format(rest))
-        return func, (True, None, 0)
-    c_all = vals[0].lower() == 'all'
-    sign: Optional[str] = None
-    amt = vals[1]
-    if amt[0] in '+-':
-        sign = amt[0]
-        amt = amt[1:]
-    return func, (c_all, sign, float(amt.strip()))
-
-
-@func_with_args('clear_terminal')
-def clear_terminal(func: str, rest: str) -> FuncArgsType:
-    vals = rest.strip().split(maxsplit=1)
-    if len(vals) != 2:
-        log_error('clear_terminal needs two arguments, using defaults')
-        args: List[Union[str, bool]] = ['reset', 'active']
-    else:
-        args = [vals[0].lower(), vals[1].lower() == 'active']
-    return func, args
-
-
-@func_with_args('copy_to_buffer')
-def copy_to_buffer(func: str, rest: str) -> FuncArgsType:
-    return func, [rest]
-
-
-@func_with_args('paste_from_buffer')
-def paste_from_buffer(func: str, rest: str) -> FuncArgsType:
-    return func, [rest]
-
-
-@func_with_args('neighboring_window')
-def neighboring_window(func: str, rest: str) -> FuncArgsType:
-    rest = rest.lower()
-    rest = {'up': 'top', 'down': 'bottom'}.get(rest, rest)
-    if rest not in ('left', 'right', 'top', 'bottom'):
-        log_error('Invalid neighbor specification: {}'.format(rest))
-        rest = 'right'
-    return func, [rest]
-
-
-@func_with_args('resize_window')
-def resize_window(func: str, rest: str) -> FuncArgsType:
-    vals = rest.strip().split(maxsplit=1)
-    if len(vals) > 2:
-        log_error('resize_window needs one or two arguments, using defaults')
-        args = ['wider', 1]
-    else:
-        quality = vals[0].lower()
-        if quality not in ('taller', 'shorter', 'wider', 'narrower'):
-            log_error('Invalid quality specification: {}'.format(quality))
-            quality = 'wider'
-        increment = 1
-        if len(vals) == 2:
-            try:
-                increment = int(vals[1])
-            except Exception:
-                log_error('Invalid increment specification: {}'.format(vals[1]))
-        args = [quality, increment]
-    return func, args
-
-
-@func_with_args('move_window')
-def move_window(func: str, rest: str) -> FuncArgsType:
-    rest = rest.lower()
-    rest = {'up': 'top', 'down': 'bottom'}.get(rest, rest)
-    prest: Union[int, str] = rest
-    try:
-        prest = int(prest)
-    except Exception:
-        if prest not in ('left', 'right', 'top', 'bottom'):
-            log_error('Invalid move_window specification: {}'.format(rest))
-            prest = 0
-    return func, [prest]
-
-
-@func_with_args('pipe')
-def pipe(func: str, rest: str) -> FuncArgsType:
-    import shlex
-    r = shlex.split(rest)
-    if len(r) < 3:
-        log_error('Too few arguments to pipe function')
-        r = ['none', 'none', 'true']
-    return func, r
-
-
-@func_with_args('set_colors')
-def set_colors(func: str, rest: str) -> FuncArgsType:
-    import shlex
-    r = shlex.split(rest)
-    if len(r) < 1:
-        log_error('Too few arguments to set_colors function')
-    return func, r
-
-
-@func_with_args('remote_control')
-def remote_control(func: str, rest: str) -> FuncArgsType:
-    import shlex
-    r = shlex.split(rest)
-    if len(r) < 1:
-        log_error('Too few arguments to remote_control function')
-    return func, r
-
-
-@func_with_args('nth_window')
-def nth_window(func: str, rest: str) -> FuncArgsType:
-    try:
-        num = int(rest)
-    except Exception:
-        log_error('Invalid nth_window number: {}'.format(rest))
-        num = 1
-    return func, [num]
-
-
-@func_with_args('disable_ligatures_in')
-def disable_ligatures_in(func: str, rest: str) -> FuncArgsType:
-    parts = rest.split(maxsplit=1)
-    if len(parts) == 1:
-        where, strategy = 'active', parts[0]
-    else:
-        where, strategy = parts
-    if where not in ('active', 'all', 'tab'):
-        raise ValueError('{} is not a valid set of windows to disable ligatures in'.format(where))
-    if strategy not in ('never', 'always', 'cursor'):
-        raise ValueError('{} is not a valid disable ligatures strategy'.format(strategy))
-    return func, [where, strategy]
-
-
-@func_with_args('layout_action')
-def layout_action(func: str, rest: str) -> FuncArgsType:
-    parts = rest.split(maxsplit=1)
-    if not parts:
-        raise ValueError('layout_action must have at least one argument')
-    return func, [parts[0], tuple(parts[1:])]
-
-
-def parse_marker_spec(ftype: str, parts: Sequence[str]) -> Tuple[str, Union[str, Tuple[Tuple[int, str], ...]], int]:
-    flags = re.UNICODE
-    if ftype in ('text', 'itext', 'regex', 'iregex'):
-        if ftype.startswith('i'):
-            flags |= re.IGNORECASE
-        if not parts or len(parts) % 2 != 0:
-            raise ValueError('No color specified in marker: {}'.format(' '.join(parts)))
-        ans = []
-        for i in range(0, len(parts), 2):
-            try:
-                color = max(1, min(int(parts[i]), 3))
-            except Exception:
-                raise ValueError('color {} in marker specification is not an integer'.format(parts[i]))
-            sspec = parts[i + 1]
-            if 'regex' not in ftype:
-                sspec = re.escape(sspec)
-            ans.append((color, sspec))
-        ftype = 'regex'
-        spec: Union[str, Tuple[Tuple[int, str], ...]] = tuple(ans)
-    elif ftype == 'function':
-        spec = ' '.join(parts)
-    else:
-        raise ValueError('Unknown marker type: {}'.format(ftype))
-    return ftype, spec, flags
-
-
-@func_with_args('toggle_marker')
-def toggle_marker(func: str, rest: str) -> FuncArgsType:
-    import shlex
-    parts = rest.split(maxsplit=1)
-    if len(parts) != 2:
-        raise ValueError('{} is not a valid marker specification'.format(rest))
-    ftype, spec = parts
-    parts = shlex.split(spec)
-    return func, list(parse_marker_spec(ftype, parts))
-
-
-@func_with_args('scroll_to_mark')
-def scroll_to_mark(func: str, rest: str) -> FuncArgsType:
-    parts = rest.split()
-    if not parts or not rest:
-        return func, [True, 0]
-    if len(parts) == 1:
-        q = parts[0].lower()
-        if q in ('prev', 'previous', 'next'):
-            return func, [q != 'next', 0]
-        try:
-            return func, [True, max(0, min(int(q), 3))]
-        except Exception:
-            raise ValueError('{} is not a valid scroll_to_mark destination'.format(rest))
-    return func, [parts[0] != 'next', max(0, min(int(parts[1]), 3))]
-
-
-def parse_key_action(action: str) -> Optional[KeyAction]:
-    parts = action.strip().split(maxsplit=1)
-    func = parts[0]
-    if len(parts) == 1:
-        return KeyAction(func, ())
-    rest = parts[1]
-    parser = args_funcs.get(func)
-    if parser is not None:
-        try:
-            func, args = parser(func, rest)
-        except Exception as err:
-            log_error('Ignoring invalid key action: {} with err: {}'.format(action, err))
-        else:
-            return KeyAction(func, args)
-    return None
-
-
-all_key_actions: Set[str] = set()
-sequence_sep = '>'
-
-
-class KeyDefinition:
-
-    def __init__(self, is_sequence: bool, action: KeyAction, mods: int, is_native: bool, key: int, rest: Tuple[SingleKey, ...] = ()):
-        self.is_sequence = is_sequence
-        self.action = action
-        self.trigger = SingleKey(mods, is_native, key)
-        self.rest = rest
-
-    def resolve(self, kitty_mod: int) -> None:
-
-        def r(k: SingleKey) -> SingleKey:
-            mods = defines.resolve_key_mods(kitty_mod, k.mods)
-            key = k.key
-            is_native = k.is_native
-            return SingleKey(mods, is_native, key)
-
-        self.trigger = r(self.trigger)
-        self.rest = tuple(map(r, self.rest))
-
-    def resolve_kitten_aliases(self, aliases: Dict[str, Sequence[str]]) -> None:
-        if not self.action.args:
-            return
-        kitten = self.action.args[0]
-        rest = self.action.args[1] if len(self.action.args) > 1 else ''
-        changed = False
-        for key, expanded in aliases.items():
-            if key == kitten:
-                changed = True
-                kitten = expanded[0]
-                if len(expanded) > 1:
-                    rest = expanded[1] + ' ' + rest
-        if changed:
-            self.action = self.action._replace(args=[kitten, rest.rstrip()])
-
-
-def parse_key(val: str, key_definitions: List[KeyDefinition]) -> None:
-    parts = val.split(maxsplit=1)
-    if len(parts) != 2:
-        return
-    sc, action = parts
-    sc, action = sc.strip().strip(sequence_sep), action.strip()
-    if not sc or not action:
-        return
-    is_sequence = sequence_sep in sc
-    if is_sequence:
-        trigger: Optional[SingleKey] = None
-        restl: List[SingleKey] = []
-        for part in sc.split(sequence_sep):
-            try:
-                mods, is_native, key = parse_shortcut(part)
-            except InvalidMods:
-                return
-            if key == 0:
-                if mods is not None:
-                    log_error('Shortcut: {} has unknown key, ignoring'.format(sc))
-                return
-            if trigger is None:
-                trigger = SingleKey(mods, is_native, key)
-            else:
-                restl.append(SingleKey(mods, is_native, key))
-        rest = tuple(restl)
-    else:
-        try:
-            mods, is_native, key = parse_shortcut(sc)
-        except InvalidMods:
-            return
-        if key == 0:
-            if mods is not None:
-                log_error('Shortcut: {} has unknown key, ignoring'.format(sc))
-            return
-    try:
-        paction = parse_key_action(action)
-    except Exception:
-        log_error('Invalid shortcut action: {}. Ignoring.'.format(
-            action))
-    else:
-        if paction is not None:
-            all_key_actions.add(paction.func)
-            if is_sequence:
-                if trigger is not None:
-                    key_definitions.append(KeyDefinition(True, paction, trigger[0], trigger[1], trigger[2], rest))
-            else:
-                assert key is not None
-                key_definitions.append(KeyDefinition(False, paction, mods, is_native, key))
-
-
-def parse_symbol_map(val: str) -> Dict[Tuple[int, int], str]:
-    parts = val.split()
-    symbol_map: Dict[Tuple[int, int], str] = {}
-
-    def abort() -> Dict[Tuple[int, int], str]:
-        log_error('Symbol map: {} is invalid, ignoring'.format(
-            val))
-        return {}
-
-    if len(parts) < 2:
-        return abort()
-    family = ' '.join(parts[1:])
-
-    def to_chr(x: str) -> int:
-        if not x.startswith('U+'):
-            raise ValueError()
-        return int(x[2:], 16)
-
-    for x in parts[0].split(','):
-        a_, b_ = x.partition('-')[::2]
-        b_ = b_ or a_
-        try:
-            a, b = map(to_chr, (a_, b_))
-        except Exception:
-            return abort()
-        if b < a or max(a, b) > sys.maxunicode or min(a, b) < 1:
-            return abort()
-        symbol_map[(a, b)] = family
-    return symbol_map
-
-
-def parse_send_text_bytes(text: str) -> bytes:
-    return python_string(text).encode('utf-8')
-
-
-def parse_send_text(val: str, key_definitions: List[KeyDefinition]) -> None:
-    parts = val.split(' ')
-
-    def abort(msg: str) -> None:
-        log_error('Send text: {} is invalid ({}), ignoring'.format(
-            val, msg))
-
-    if len(parts) < 3:
-        return abort('Incomplete')
-    mode, sc = parts[:2]
-    text = ' '.join(parts[2:])
-    key_str = '{} send_text {} {}'.format(sc, mode, text)
-    parse_key(key_str, key_definitions)
-
-
-SpecialHandlerFunc = Callable[[str, str, Dict[str, Any]], None]
-special_handlers: Dict[str, SpecialHandlerFunc] = {}
-
-
-def special_handler(func: SpecialHandlerFunc) -> SpecialHandlerFunc:
-    special_handlers[func.__name__.partition('_')[2]] = func
-    return func
-
-
-def deprecated_handler(*names: str) -> Callable[[SpecialHandlerFunc], SpecialHandlerFunc]:
-    def special_handler(func: SpecialHandlerFunc) -> SpecialHandlerFunc:
-        for name in names:
-            special_handlers[name] = func
-        return func
-    return special_handler
-
-
-@special_handler
-def handle_map(key: str, val: str, ans: Dict[str, Any]) -> None:
-    parse_key(val, ans['key_definitions'])
-
-
-@special_handler
-def handle_symbol_map(key: str, val: str, ans: Dict[str, Any]) -> None:
-    ans['symbol_map'].update(parse_symbol_map(val))
-
-
-@special_handler
-def handle_font_features(key: str, val: str, ans: Dict[str, Any]) -> None:
-    if val != 'none':
-        parts = val.split()
-        if len(parts) < 2:
-            log_error("Ignoring invalid font_features {}".format(val))
-        else:
-            features = []
-            for feat in parts[1:]:
-                try:
-                    parsed = defines.parse_font_feature(feat)
-                except ValueError:
-                    log_error('Ignoring invalid font feature: {}'.format(feat))
-                else:
-                    features.append(FontFeature(feat, parsed))
-            if features:
-                ans['font_features'][parts[0]] = tuple(features)
-
-
-@special_handler
-def handle_kitten_alias(key: str, val: str, ans: Dict[str, Any]) -> None:
-    parts = val.split(maxsplit=2)
-    if len(parts) >= 2:
-        ans['kitten_aliases'][parts[0]] = parts[1:]
-
-
-@special_handler
-def handle_send_text(key: str, val: str, ans: Dict[str, Any]) -> None:
-    # For legacy compatibility
-    parse_send_text(val, ans['key_definitions'])
-
-
-@special_handler
-def handle_clear_all_shortcuts(key: str, val: str, ans: Dict[str, Any]) -> None:
-    if to_bool(val):
-        ans['key_definitions'] = [None]
-
-
-@deprecated_handler('x11_hide_window_decorations', 'macos_hide_titlebar')
-def handle_deprecated_hide_window_decorations_aliases(key: str, val: str, ans: Dict[str, Any]) -> None:
-    if not hasattr(handle_deprecated_hide_window_decorations_aliases, key):
-        setattr(handle_deprecated_hide_window_decorations_aliases, 'key', True)
-        log_error('The option {} is deprecated. Use hide_window_decorations instead.'.format(key))
-    if to_bool(val):
-        if is_macos and key == 'macos_hide_titlebar' or (not is_macos and key == 'x11_hide_window_decorations'):
-            ans['hide_window_decorations'] = True
-
-
-@deprecated_handler('macos_show_window_title_in_menubar')
-def handle_deprecated_macos_show_window_title_in_menubar_alias(key: str, val: str, ans: Dict[str, Any]) -> None:
-    if not hasattr(handle_deprecated_macos_show_window_title_in_menubar_alias, key):
-        setattr(handle_deprecated_macos_show_window_title_in_menubar_alias, 'key', True)
-        log_error('The option {} is deprecated. Use macos_show_window_title_in menubar instead.'.format(key))
-    macos_show_window_title_in = ans.get('macos_show_window_title_in', 'all')
-    if to_bool(val):
-        if macos_show_window_title_in == 'none':
-            macos_show_window_title_in = 'menubar'
-        elif macos_show_window_title_in == 'window':
-            macos_show_window_title_in = 'all'
-    else:
-        if macos_show_window_title_in == 'all':
-            macos_show_window_title_in = 'window'
-        elif macos_show_window_title_in == 'menubar':
-            macos_show_window_title_in = 'none'
-    ans['macos_show_window_title_in'] = macos_show_window_title_in
-
-
-@special_handler
-def handle_env(key: str, val: str, ans: Dict[str, Any]) -> None:
-    key, val = val.partition('=')[::2]
-    key, val = key.strip(), val.strip()
-    ans['env'][key] = expandvars(val, ans['env'])
-
-
-def special_handling(key: str, val: str, ans: Dict[str, Any]) -> bool:
-    func = special_handlers.get(key)
-    if func is not None:
-        func(key, val, ans)
-        return True
-
-
-def option_names_for_completion() -> Generator[str, None, None]:
-    yield from defaults
-    yield from special_handlers
-
-
-def parse_config(lines: Iterable[str], check_keys: bool = True, accumulate_bad_lines: Optional[List[BadLine]] = None) -> Dict[str, Any]:
-    ans: Dict[str, Any] = {
-        'symbol_map': {}, 'keymap': {}, 'sequence_map': {}, 'key_definitions': [],
-        'env': {}, 'kitten_aliases': {}, 'font_features': {}
-    }
-    defs: Optional[OptionsStub] = None
-    if check_keys:
-        defs = defaults
-
-    parse_config_base(
-        lines,
-        defs,
-        type_convert,
-        special_handling,
-        ans,
-        check_keys=check_keys,
-        accumulate_bad_lines=accumulate_bad_lines
-    )
-    return ans
-
-
-def parse_defaults(lines: Iterable[str], check_keys: bool = False) -> Dict[str, Any]:
-    ans = parse_config(lines, check_keys)
-    return ans
-
-
-xc = init_config(config_lines(all_options), parse_defaults)
-Options: Type[OptionsStub] = xc[0]
-defaults: OptionsStub = xc[1]
-actions = frozenset(all_key_actions) | frozenset(
-    'run_simple_kitten combine send_text goto_tab goto_layout set_font_size new_tab_with_cwd new_window_with_cwd new_os_window_with_cwd'.
-    split()
-)
 no_op_actions = frozenset({'noop', 'no-op', 'no_op'})
 
 
-def merge_configs(defaults: Dict, vals: Dict) -> Dict:
-    ans = {}
-    for k, v in defaults.items():
-        if isinstance(v, dict):
-            newvals = vals.get(k, {})
-            ans[k] = merge_dicts(v, newvals)
-        elif k == 'key_definitions':
-            ans['key_definitions'] = v + vals.get('key_definitions', [])
-        else:
-            ans[k] = vals.get(k, v)
-    return ans
-
-
-def build_ansi_color_table(opts: OptionsStub = defaults) -> List[int]:
-
-    def as_int(x: Tuple[int, int, int]) -> int:
-        return (x[0] << 16) | (x[1] << 8) | x[2]
-
-    def col(i: int) -> int:
-        return as_int(getattr(opts, 'color{}'.format(i)))
-
-    return list(map(col, range(256)))
+def build_ansi_color_table(opts: Optional[Options] = None) -> int:
+    if opts is None:
+        opts = defaults
+    addr, length = opts.color_table.buffer_info()
+    if length != 256 or opts.color_table.typecode != 'L':
+        raise TypeError(f'The color table has incorrect size length: {length} typecode: {opts.color_table.typecode}')
+    return addr
 
 
 def atomic_save(data: bytes, path: str) -> None:
@@ -694,12 +75,8 @@ def cached_values_for(name: str) -> Generator[Dict, None, None]:
 
 
 def commented_out_default_config() -> str:
-    ans = []
-    for line in as_conf_file(all_options.values()):
-        if line and line[0] != '#':
-            line = '# ' + line
-        ans.append(line)
-    return '\n'.join(ans)
+    from .options.definition import definition
+    return '\n'.join(definition.as_conf(commented=True))
 
 
 def prepare_config_file_for_editing() -> str:
@@ -712,18 +89,13 @@ def prepare_config_file_for_editing() -> str:
     return defconf
 
 
-def finalize_keys(opts: OptionsStub) -> None:
+def finalize_keys(opts: Options) -> None:
     defns: List[KeyDefinition] = []
-    for d in getattr(opts, 'key_definitions'):
+    for d in opts.map:
         if d is None:  # clear_all_shortcuts
-            defns = []
+            defns = []  # type: ignore
         else:
-            defns.append(d)
-    kitten_aliases: List[Dict[str, Sequence[str]]] = getattr(opts, 'kitten_aliases')
-    for d in defns:
-        d.resolve(opts.kitty_mod)
-        if kitten_aliases and d.action.func == 'kitten':
-            d.resolve_kitten_aliases(kitten_aliases)
+            defns.append(d.resolve_and_copy(opts.kitty_mod, opts.kitten_alias))
     keymap: KeyMap = {}
     sequence_map: SequenceMap = {}
 
@@ -748,15 +120,51 @@ def finalize_keys(opts: OptionsStub) -> None:
     opts.sequence_map = sequence_map
 
 
-def load_config(*paths: str, overrides: Optional[Iterable[str]] = None, accumulate_bad_lines: Optional[List[BadLine]] = None) -> OptionsStub:
-    parser = parse_config
-    if accumulate_bad_lines is not None:
-        parser = partial(parse_config, accumulate_bad_lines=accumulate_bad_lines)
-    opts = _load_config(Options, defaults, parser, merge_configs, *paths, overrides=overrides)
+def finalize_mouse_mappings(opts: Options) -> None:
+    defns: List[MouseMapping] = []
+    for d in opts.mouse_map:
+        defns.append(d.resolve_and_copy(opts.kitty_mod, opts.kitten_alias))
+
+    mousemap: MouseMap = {}
+    for defn in defns:
+        is_no_op = defn.action.func in no_op_actions
+        if is_no_op:
+            mousemap.pop(defn.trigger, None)
+        else:
+            mousemap[defn.trigger] = defn.action
+    opts.mousemap = mousemap
+
+
+def parse_config(lines: Iterable[str], accumulate_bad_lines: Optional[List[BadLine]] = None) -> Dict[str, Any]:
+    from .options.parse import create_result_dict, parse_conf_item
+    ans: Dict[str, Any] = create_result_dict()
+    parse_config_base(
+        lines,
+        parse_conf_item,
+        ans,
+        accumulate_bad_lines=accumulate_bad_lines
+    )
+    return ans
+
+
+def load_config(*paths: str, overrides: Optional[Iterable[str]] = None, accumulate_bad_lines: Optional[List[BadLine]] = None) -> Options:
+    from .options.parse import merge_result_dicts
+
+    overrides = tuple(overrides) if overrides is not None else ()
+    opts_dict, paths = _load_config(defaults, partial(parse_config, accumulate_bad_lines=accumulate_bad_lines), merge_result_dicts, *paths, overrides=overrides)
+    opts = Options(opts_dict)
+
     finalize_keys(opts)
+    finalize_mouse_mappings(opts)
+    # delete no longer needed definitions, replacing with empty placeholders
+    opts.kitten_alias = {}
+    opts.mouse_map = []
+    opts.map = []
     if opts.background_opacity < 1.0 and opts.macos_titlebar_color:
         log_error('Cannot use both macos_titlebar_color and background_opacity')
         opts.macos_titlebar_color = 0
+    opts.config_paths = paths
+    opts.config_overrides = overrides
     return opts
 
 
@@ -766,7 +174,7 @@ class KittyCommonOpts(TypedDict):
     url_prefixes: Tuple[str, ...]
 
 
-def common_opts_as_dict(opts: Optional[OptionsStub] = None) -> KittyCommonOpts:
+def common_opts_as_dict(opts: Optional[Options] = None) -> KittyCommonOpts:
     if opts is None:
         opts = defaults
     return {

kitty/constants.py

@@ -7,11 +7,13 @@ import os
 import pwd
 import sys
 from contextlib import suppress
-from typing import NamedTuple, Optional, Set
+from typing import NamedTuple, Optional, Set, TYPE_CHECKING
 
-from .options_stub import Options
 from .types import run_once
 
+if TYPE_CHECKING:
+    from .options.types import Options
+
 
 class Version(NamedTuple):
     major: int
@@ -21,7 +23,7 @@ class Version(NamedTuple):
 
 appname: str = 'kitty'
 kitty_face = '🐱'
-version: Version = Version(0, 20, 2)
+version: Version = Version(0, 21, 2)
 str_version: str = '.'.join(map(str, version))
 _plat = sys.platform.lower()
 is_macos: bool = 'darwin' in _plat
@@ -149,7 +151,7 @@ def detect_if_wayland_ok() -> bool:
     return True
 
 
-def is_wayland(opts: Optional[Options] = None) -> bool:
+def is_wayland(opts: Optional['Options'] = None) -> bool:
     if is_macos:
         return False
     if opts is None:

kitty/core_text.m

@@ -21,6 +21,8 @@
 #import <Foundation/NSString.h>
 #import <Foundation/NSDictionary.h>
 
+#define debug(...) if (global_state.debug_rendering) { fprintf(stderr, __VA_ARGS__); fflush(stderr); }
+
 typedef struct {
     PyObject_HEAD
 
@@ -45,7 +47,7 @@ convert_cfstring(CFStringRef src, int free_src) {
 }
 
 static inline void
-init_face(CTFace *self, CTFontRef font) {
+init_face(CTFace *self, CTFontRef font, FONTS_DATA_HANDLE fg UNUSED) {
     if (self->hb_font) hb_font_destroy(self->hb_font);
     self->hb_font = NULL;
     if (self->ct_font) CFRelease(self->ct_font);
@@ -60,10 +62,10 @@ init_face(CTFace *self, CTFontRef font) {
 }
 
 static inline CTFace*
-ct_face(CTFontRef font) {
+ct_face(CTFontRef font, FONTS_DATA_HANDLE fg) {
     CTFace *self = (CTFace *)CTFace_Type.tp_alloc(&CTFace_Type, 0);
     if (self) {
-        init_face(self, font);
+        init_face(self, font, fg);
         self->family_name = Py_BuildValue("s", convert_cfstring(CTFontCopyFamilyName(self->ct_font), true));
         self->full_name = Py_BuildValue("s", convert_cfstring(CTFontCopyFullName(self->ct_font), true));
         self->postscript_name = Py_BuildValue("s", convert_cfstring(CTFontCopyPostScriptName(self->ct_font), true));
@@ -247,20 +249,27 @@ find_substitute_face(CFStringRef str, CTFontRef old_font, CPUCell *cpu_cell) {
 }
 
 PyObject*
-create_fallback_face(PyObject *base_face, CPUCell* cell, bool UNUSED bold, bool UNUSED italic, bool emoji_presentation, FONTS_DATA_HANDLE fg UNUSED) {
+create_fallback_face(PyObject *base_face, CPUCell* cell, bool UNUSED bold, bool UNUSED italic, bool emoji_presentation, FONTS_DATA_HANDLE fg) {
     CTFace *self = (CTFace*)base_face;
     CTFontRef new_font;
-    if (emoji_presentation) new_font = CTFontCreateWithName((CFStringRef)@"AppleColorEmoji", self->scaled_point_sz, NULL);
-    else {
-        char text[64] = {0};
-        cell_as_utf8_for_fallback(cell, text);
-        CFStringRef str = CFStringCreateWithCString(NULL, text, kCFStringEncodingUTF8);
-        if (str == NULL) return PyErr_NoMemory();
-        new_font = find_substitute_face(str, self->ct_font, cell);
+#define search_for_fallback() \
+        char text[64] = {0}; \
+        cell_as_utf8_for_fallback(cell, text); \
+        CFStringRef str = CFStringCreateWithCString(NULL, text, kCFStringEncodingUTF8); \
+        if (str == NULL) return PyErr_NoMemory(); \
+        new_font = find_substitute_face(str, self->ct_font, cell); \
         CFRelease(str);
+
+    if (emoji_presentation) {
+        new_font = CTFontCreateWithName((CFStringRef)@"AppleColorEmoji", self->scaled_point_sz, NULL);
+        if (!new_font || !glyph_id_for_codepoint_ctfont(new_font, cell->ch)) {
+            if (new_font) CFRelease(new_font);
+            search_for_fallback();
+        }
     }
+    else { search_for_fallback(); }
     if (new_font == NULL) return NULL;
-    return (PyObject*)ct_face(new_font);
+    return (PyObject*)ct_face(new_font, fg);
 }
 
 unsigned int
@@ -299,7 +308,7 @@ set_size_for_face(PyObject *s, unsigned int UNUSED desired_height, bool force, F
     if (!force && self->scaled_point_sz == sz) return true;
     CTFontRef new_font = CTFontCreateCopyWithAttributes(self->ct_font, sz, NULL, NULL);
     if (new_font == NULL) fatal("Out of memory");
-    init_face(self, new_font);
+    init_face(self, new_font, fg);
     return true;
 }
 
@@ -314,6 +323,13 @@ harfbuzz_font_for_face(PyObject* s) {
     return self->hb_font;
 }
 
+static unsigned int
+adjust_ypos(unsigned int pos, unsigned int cell_height, int adjustment) {
+    if (adjustment >= 0) adjustment = MIN(adjustment, (int)pos - 1);
+    else adjustment = MAX(adjustment, (int)pos - (int)cell_height + 1);
+    return pos - adjustment;
+}
+
 void
 cell_metrics(PyObject *s, unsigned int* cell_width, unsigned int* cell_height, unsigned int* baseline, unsigned int* underline_position, unsigned int* underline_thickness, unsigned int* strikethrough_position, unsigned int* strikethrough_thickness) {
     // See https://developer.apple.com/library/content/documentation/StringsTextFonts/Conceptual/TextAndWebiPhoneOS/TypoFeatures/TextSystemFeatures.html
@@ -332,10 +348,7 @@ cell_metrics(PyObject *s, unsigned int* cell_width, unsigned int* cell_height, u
         }
     }
     *cell_width = MAX(1u, width);
-    *underline_position = (unsigned int)floor(self->ascent - self->underline_position + 0.5);
     *underline_thickness = (unsigned int)ceil(MAX(0.1, self->underline_thickness));
-    *baseline = (unsigned int)self->ascent;
-    *strikethrough_position = (unsigned int)floor(*baseline * 0.65);
     *strikethrough_thickness = *underline_thickness;
     // float line_height = MAX(1, floor(self->ascent + self->descent + MAX(0, self->leading) + 0.5));
     // Let CoreText's layout engine calculate the line height. Slower, but hopefully more accurate.
@@ -354,8 +367,39 @@ cell_metrics(PyObject *s, unsigned int* cell_width, unsigned int* cell_height, u
     CTFrameGetLineOrigins(test_frame, CFRangeMake(0, 1), &origin1);
     CTFrameGetLineOrigins(test_frame, CFRangeMake(1, 1), &origin2);
     CGFloat line_height = origin1.y - origin2.y;
-    CFRelease(test_frame); CFRelease(path); CFRelease(framesetter);
+    CFArrayRef lines = CTFrameGetLines(test_frame);
+    CTLineRef line = CFArrayGetValueAtIndex(lines, 0);
+    CGRect bounds = CTLineGetBoundsWithOptions(line, 0);
+    CGRect bounds_without_leading = CTLineGetBoundsWithOptions(line, kCTLineBoundsExcludeTypographicLeading);
+    CGFloat typographic_ascent, typographic_descent, typographic_leading;
+    CTLineGetTypographicBounds(line, &typographic_ascent, &typographic_descent, &typographic_leading);
     *cell_height = MAX(4u, (unsigned int)ceilf(line_height));
+    CGFloat bounds_ascent = bounds_without_leading.size.height + bounds_without_leading.origin.y;
+    int baseline_offset = 0;
+    if (OPT(adjust_baseline_px) != 0) baseline_offset = OPT(adjust_baseline_px);
+    else if (OPT(adjust_baseline_frac) != 0) baseline_offset = (int)(*cell_height * OPT(adjust_baseline_frac));
+    *baseline = (unsigned int)floor(bounds_ascent + 0.5);
+    // Not sure if we should add this to bounds ascent and then round it or add
+    // it to already rounded baseline and round again.
+    *underline_position = (unsigned int)floor(bounds_ascent - self->underline_position + 0.5);
+    *strikethrough_position = (unsigned int)floor(*baseline * 0.65);
+
+    debug("Cell height calculation:\n");
+    debug("\tline height from line origins: %f\n", line_height);
+    debug("\tline bounds: origin-y: %f height: %f\n", bounds.origin.y, bounds.size.height);
+    debug("\tline bounds-no-leading: origin-y: %f height: %f\n", bounds.origin.y, bounds.size.height);
+    debug("\tbounds metrics: ascent: %f\n", bounds_ascent);
+    debug("\tline metrics: ascent: %f descent: %f leading: %f\n", typographic_ascent, typographic_descent, typographic_leading);
+    debug("\tfont metrics: ascent: %f descent: %f leading: %f underline_position: %f\n", self->ascent, self->descent, self->leading, self->underline_position);
+    debug("\tcell_height: %u baseline: %u underline_position: %u strikethrough_position: %u\n", *cell_height, *baseline, *underline_position, *strikethrough_position);
+    if (baseline_offset) {
+        *baseline = adjust_ypos(*baseline, *cell_height, baseline_offset);
+        *underline_position = adjust_ypos(*underline_position, *cell_height, baseline_offset);
+        *strikethrough_position = adjust_ypos(*strikethrough_position, *cell_height, baseline_offset);
+    }
+
+    CFRelease(test_frame); CFRelease(path); CFRelease(framesetter);
+
 #undef count
 }
 
@@ -366,18 +410,18 @@ face_from_descriptor(PyObject *descriptor, FONTS_DATA_HANDLE fg) {
     CTFontRef font = CTFontCreateWithFontDescriptor(desc, scaled_point_sz(fg), NULL);
     CFRelease(desc); desc = NULL;
     if (!font) { PyErr_SetString(PyExc_ValueError, "Failed to create CTFont object"); return NULL; }
-    return (PyObject*) ct_face(font);
+    return (PyObject*) ct_face(font, fg);
 }
 
 PyObject*
-face_from_path(const char *path, int UNUSED index, FONTS_DATA_HANDLE fg UNUSED) {
+face_from_path(const char *path, int UNUSED index, FONTS_DATA_HANDLE fg) {
     CFStringRef s = CFStringCreateWithCString(NULL, path, kCFStringEncodingUTF8);
     CFURLRef url = CFURLCreateWithFileSystemPath(kCFAllocatorDefault, s, kCFURLPOSIXPathStyle, false);
     CGDataProviderRef dp = CGDataProviderCreateWithURL(url);
     CGFontRef cg_font = CGFontCreateWithDataProvider(dp);
     CTFontRef ct_font = CTFontCreateWithGraphicsFont(cg_font, 0.0, NULL, NULL);
     CFRelease(cg_font); CFRelease(dp); CFRelease(url); CFRelease(s);
-    return (PyObject*) ct_face(ct_font);
+    return (PyObject*) ct_face(ct_font, fg);
 }
 
 PyObject*
@@ -386,16 +430,20 @@ specialize_font_descriptor(PyObject *base_descriptor, FONTS_DATA_HANDLE fg UNUSE
     return base_descriptor;
 }
 
-static uint8_t *render_buf = NULL;
-static size_t render_buf_sz = 0;
-static CGGlyph glyphs[128];
-static CGRect boxes[128];
-static CGPoint positions[128];
-static CGSize advances[128];
+struct RenderBuffers {
+    uint8_t *render_buf;
+    size_t render_buf_sz, sz;
+    CGGlyph *glyphs;
+    CGRect *boxes;
+    CGPoint *positions;
+    CGSize *advances;
+};
+static struct RenderBuffers buffers = {0};
 
 static void
 finalize(void) {
-    free(render_buf);
+    free(buffers.render_buf); free(buffers.glyphs); free(buffers.boxes); free(buffers.positions); free(buffers.advances);
+    memset(&buffers, 0, sizeof(struct RenderBuffers));
     if (all_fonts_collection_data) CFRelease(all_fonts_collection_data);
 }
 
@@ -413,7 +461,7 @@ render_color_glyph(CTFontRef font, uint8_t *buf, int glyph_id, unsigned int widt
     CGContextSetTextDrawingMode(ctx, kCGTextFill);
     CGGlyph glyph = glyph_id;
     CGContextSetTextMatrix(ctx, transform);
-    CGContextSetTextPosition(ctx, -boxes[0].origin.x, MAX(2, height - baseline));
+    CGContextSetTextPosition(ctx, -buffers.boxes[0].origin.x, MAX(2, height - baseline));
     CGPoint p = CGPointMake(0, 0);
     CTFontDrawGlyphs(font, &glyph, &p, 1, ctx);
     CGContextRelease(ctx);
@@ -426,21 +474,30 @@ render_color_glyph(CTFontRef font, uint8_t *buf, int glyph_id, unsigned int widt
     }
 }
 
-static inline void
-ensure_render_space(size_t width, size_t height) {
-    if (render_buf_sz >= width * height) return;
-    free(render_buf);
-    render_buf_sz = width * height;
-    render_buf = malloc(render_buf_sz);
-    if (render_buf == NULL) fatal("Out of memory");
+static void
+ensure_render_space(size_t width, size_t height, size_t num_glyphs) {
+    if (buffers.render_buf_sz < width * height) {
+        free(buffers.render_buf); buffers.render_buf = NULL;
+        buffers.render_buf_sz = width * height;
+        buffers.render_buf = malloc(buffers.render_buf_sz);
+        if (buffers.render_buf == NULL) fatal("Out of memory");
+    }
+    if (buffers.sz < num_glyphs) {
+        buffers.sz = MAX(128, num_glyphs * 2);
+        buffers.advances = calloc(sizeof(buffers.advances[0]), buffers.sz);
+        buffers.boxes = calloc(sizeof(buffers.boxes[0]), buffers.sz);
+        buffers.glyphs = calloc(sizeof(buffers.glyphs[0]), buffers.sz);
+        buffers.positions = calloc(sizeof(buffers.positions[0]), buffers.sz);
+        if (!buffers.advances || !buffers.boxes || !buffers.glyphs || !buffers.positions) fatal("Out of memory");
+    }
 }
 
 static inline void
 render_glyphs(CTFontRef font, unsigned int width, unsigned int height, unsigned int baseline, unsigned int num_glyphs) {
-    memset(render_buf, 0, width * height);
+    memset(buffers.render_buf, 0, width * height);
     CGColorSpaceRef gray_color_space = CGColorSpaceCreateDeviceGray();
     if (gray_color_space == NULL) fatal("Out of memory");
-    CGContextRef render_ctx = CGBitmapContextCreate(render_buf, width, height, 8, width, gray_color_space, (kCGBitmapAlphaInfoMask & kCGImageAlphaNone));
+    CGContextRef render_ctx = CGBitmapContextCreate(buffers.render_buf, width, height, 8, width, gray_color_space, (kCGBitmapAlphaInfoMask & kCGImageAlphaNone));
     if (render_ctx == NULL) fatal("Out of memory");
     CGContextSetShouldAntialias(render_ctx, true);
     CGContextSetShouldSmoothFonts(render_ctx, true);
@@ -450,7 +507,7 @@ render_glyphs(CTFontRef font, unsigned int width, unsigned int height, unsigned
     CGContextSetTextDrawingMode(render_ctx, kCGTextFillStroke);
     CGContextSetTextMatrix(render_ctx, CGAffineTransformIdentity);
     CGContextSetTextPosition(render_ctx, 0, height - baseline);
-    CTFontDrawGlyphs(font, glyphs, positions, num_glyphs, render_ctx);
+    CTFontDrawGlyphs(font, buffers.glyphs, buffers.positions, num_glyphs, render_ctx);
     CGContextRelease(render_ctx);
     CGColorSpaceRelease(gray_color_space);
 }
@@ -463,19 +520,20 @@ render_simple_text_impl(PyObject *s, const char *text, unsigned int baseline) {
     unichar chars[num_chars];
     CGSize local_advances[num_chars];
     for (size_t i = 0; i < num_chars; i++) chars[i] = text[i];
-    CTFontGetGlyphsForCharacters(font, chars, glyphs, num_chars);
-    CTFontGetAdvancesForGlyphs(font, kCTFontOrientationDefault, glyphs, local_advances, num_chars);
-    CGRect bounding_box = CTFontGetBoundingRectsForGlyphs(font, kCTFontOrientationDefault, glyphs, boxes, num_chars);
+    ensure_render_space(0, 0, num_chars);
+    CTFontGetGlyphsForCharacters(font, chars, buffers.glyphs, num_chars);
+    CTFontGetAdvancesForGlyphs(font, kCTFontOrientationDefault, buffers.glyphs, local_advances, num_chars);
+    CGRect bounding_box = CTFontGetBoundingRectsForGlyphs(font, kCTFontOrientationDefault, buffers.glyphs, buffers.boxes, num_chars);
     CGFloat x = 0, y = 0;
     for (size_t i = 0; i < num_chars; i++) {
-        positions[i] = CGPointMake(x, y);
+        buffers.positions[i] = CGPointMake(x, y);
         x += local_advances[i].width; y += local_advances[i].height;
     }
     StringCanvas ans = { .width = (size_t)ceil(x), .height = (size_t)(2 * bounding_box.size.height) };
-    ensure_render_space(ans.width, ans.height);
+    ensure_render_space(ans.width, ans.height, num_chars);
     render_glyphs(font, ans.width, ans.height, baseline, num_chars);
     ans.canvas = malloc(ans.width * ans.height);
-    if (ans.canvas) memcpy(ans.canvas, render_buf, ans.width * ans.height);
+    if (ans.canvas) memcpy(ans.canvas, buffers.render_buf, ans.width * ans.height);
     return ans;
 }
 
@@ -483,12 +541,13 @@ render_simple_text_impl(PyObject *s, const char *text, unsigned int baseline) {
 static inline bool
 do_render(CTFontRef ct_font, bool bold, bool italic, hb_glyph_info_t *info, hb_glyph_position_t *hb_positions, unsigned int num_glyphs, pixel *canvas, unsigned int cell_width, unsigned int cell_height, unsigned int num_cells, unsigned int baseline, bool *was_colored, bool allow_resize, FONTS_DATA_HANDLE fg, bool center_glyph) {
     unsigned int canvas_width = cell_width * num_cells;
-    CGRect br = CTFontGetBoundingRectsForGlyphs(ct_font, kCTFontOrientationHorizontal, glyphs, boxes, num_glyphs);
+    ensure_render_space(canvas_width, cell_height, num_glyphs);
+    CGRect br = CTFontGetBoundingRectsForGlyphs(ct_font, kCTFontOrientationHorizontal, buffers.glyphs, buffers.boxes, num_glyphs);
     const bool debug_rendering = false;
     if (allow_resize) {
         // Resize glyphs that would bleed into neighboring cells, by scaling the font size
         float right = 0;
-        for (unsigned i=0; i < num_glyphs; i++) right = MAX(right, boxes[i].origin.x + boxes[i].size.width);
+        for (unsigned i=0; i < num_glyphs; i++) right = MAX(right, buffers.boxes[i].origin.x + buffers.boxes[i].size.width);
         if (!bold && !italic && right > canvas_width + 1) {
             if (debug_rendering) printf("resizing glyphs, right: %f canvas_width: %u\n", right, canvas_width);
             CGFloat sz = CTFontGetSize(ct_font);
@@ -500,19 +559,18 @@ do_render(CTFontRef ct_font, bool bold, bool italic, hb_glyph_info_t *info, hb_g
         }
     }
     CGFloat x = 0, y = 0;
-    CTFontGetAdvancesForGlyphs(ct_font, kCTFontOrientationDefault, glyphs, advances, num_glyphs);
+    CTFontGetAdvancesForGlyphs(ct_font, kCTFontOrientationDefault, buffers.glyphs, buffers.advances, num_glyphs);
     for (unsigned i=0; i < num_glyphs; i++) {
-        positions[i].x = x; positions[i].y = y;
-        if (debug_rendering) printf("x=%f origin=%f width=%f advance=%f\n", x, boxes[i].origin.x, boxes[i].size.width, advances[i].width);
-        x += advances[i].width; y += advances[i].height;
+        buffers.positions[i].x = x; buffers.positions[i].y = y;
+        if (debug_rendering) printf("x=%f origin=%f width=%f advance=%f\n", x, buffers.boxes[i].origin.x, buffers.boxes[i].size.width, buffers.advances[i].width);
+        x += buffers.advances[i].width; y += buffers.advances[i].height;
     }
     if (*was_colored) {
         render_color_glyph(ct_font, (uint8_t*)canvas, info[0].codepoint, cell_width * num_cells, cell_height, baseline);
     } else {
-        ensure_render_space(canvas_width, cell_height);
         render_glyphs(ct_font, canvas_width, cell_height, baseline, num_glyphs);
         Region src = {.bottom=cell_height, .right=canvas_width}, dest = {.bottom=cell_height, .right=canvas_width};
-        render_alpha_mask(render_buf, canvas, &src, &dest, canvas_width, canvas_width);
+        render_alpha_mask(buffers.render_buf, canvas, &src, &dest, canvas_width, canvas_width);
     }
     if (num_cells && (center_glyph || (num_cells == 2 && *was_colored))) {
         if (debug_rendering) printf("centering glyphs: center_glyph: %d\n", center_glyph);
@@ -528,7 +586,8 @@ do_render(CTFontRef ct_font, bool bold, bool italic, hb_glyph_info_t *info, hb_g
 bool
 render_glyphs_in_cells(PyObject *s, bool bold, bool italic, hb_glyph_info_t *info, hb_glyph_position_t *hb_positions, unsigned int num_glyphs, pixel *canvas, unsigned int cell_width, unsigned int cell_height, unsigned int num_cells, unsigned int baseline, bool *was_colored, FONTS_DATA_HANDLE fg, bool center_glyph) {
     CTFace *self = (CTFace*)s;
-    for (unsigned i=0; i < num_glyphs; i++) glyphs[i] = info[i].codepoint;
+    ensure_render_space(128, 128, num_glyphs);
+    for (unsigned i=0; i < num_glyphs; i++) buffers.glyphs[i] = info[i].codepoint;
     return do_render(self->ct_font, bold, italic, info, hb_positions, num_glyphs, canvas, cell_width, cell_height, num_cells, baseline, was_colored, true, fg, center_glyph);
 }
 

kitty/data-types.c

@@ -278,6 +278,7 @@ PyInit_fast_data_types(void) {
     PyModule_AddIntMacro(m, CURSOR_BLOCK);
     PyModule_AddIntMacro(m, CURSOR_BEAM);
     PyModule_AddIntMacro(m, CURSOR_UNDERLINE);
+    PyModule_AddIntMacro(m, NO_CURSOR_SHAPE);
     PyModule_AddIntMacro(m, DECAWM);
     PyModule_AddIntMacro(m, DECCOLM);
     PyModule_AddIntMacro(m, DECOM);

kitty/data-types.h

@@ -48,6 +48,7 @@ typedef uint16_t hyperlink_id_type;
 typedef int key_type;
 #define HYPERLINK_MAX_NUMBER UINT16_MAX
 typedef uint16_t combining_type;
+typedef uint16_t glyph_index;
 typedef uint32_t pixel;
 typedef unsigned int index_type;
 typedef uint16_t sprite_index;

kitty/debug_config.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+from functools import partial
+from typing import Callable, Dict, Generator, Iterable, Set, Tuple
+from pprint import pformat
+
+from .cli import green, title, version
+from .conf.utils import KeyAction
+from .constants import is_macos, is_wayland
+from .options.types import Options as KittyOpts, defaults
+from .options.utils import MouseMap
+from .types import MouseEvent, SingleKey
+from .typing import SequenceMap
+
+ShortcutMap = Dict[Tuple[SingleKey, ...], KeyAction]
+
+
+def mod_to_names(mods: int) -> Generator[str, None, None]:
+    from .fast_data_types import (
+        GLFW_MOD_ALT, GLFW_MOD_CAPS_LOCK, GLFW_MOD_CONTROL, GLFW_MOD_HYPER,
+        GLFW_MOD_META, GLFW_MOD_NUM_LOCK, GLFW_MOD_SHIFT, GLFW_MOD_SUPER
+    )
+    modmap = {'shift': GLFW_MOD_SHIFT, 'alt': GLFW_MOD_ALT, 'ctrl': GLFW_MOD_CONTROL, ('cmd' if is_macos else 'super'): GLFW_MOD_SUPER,
+              'hyper': GLFW_MOD_HYPER, 'meta': GLFW_MOD_META, 'num_lock': GLFW_MOD_NUM_LOCK, 'caps_lock': GLFW_MOD_CAPS_LOCK}
+    for name, val in modmap.items():
+        if mods & val:
+            yield name
+
+
+def print_shortcut(key_sequence: Iterable[SingleKey], action: KeyAction, print: Callable) -> None:
+    from .fast_data_types import glfw_get_key_name
+    keys = []
+    for key_spec in key_sequence:
+        names = []
+        mods, is_native, key = key_spec
+        names = list(mod_to_names(mods))
+        if key:
+            kname = glfw_get_key_name(0, key) if is_native else glfw_get_key_name(key, 0)
+            names.append(kname or f'{key}')
+        keys.append('+'.join(names))
+
+    print('\t' + ' > '.join(keys), action)
+
+
+def print_shortcut_changes(defns: ShortcutMap, text: str, changes: Set[Tuple[SingleKey, ...]], print: Callable) -> None:
+    if changes:
+        print(title(text))
+
+        for k in sorted(changes):
+            print_shortcut(k, defns[k], print)
+
+
+def compare_keymaps(final: ShortcutMap, initial: ShortcutMap, print: Callable) -> None:
+    added = set(final) - set(initial)
+    removed = set(initial) - set(final)
+    changed = {k for k in set(final) & set(initial) if final[k] != initial[k]}
+    print_shortcut_changes(final, 'Added shortcuts:', added, print)
+    print_shortcut_changes(initial, 'Removed shortcuts:', removed, print)
+    print_shortcut_changes(final, 'Changed shortcuts:', changed, print)
+
+
+def flatten_sequence_map(m: SequenceMap) -> ShortcutMap:
+    ans: Dict[Tuple[SingleKey, ...], KeyAction] = {}
+    for key_spec, rest_map in m.items():
+        for r, action in rest_map.items():
+            ans[(key_spec,) + (r)] = action
+    return ans
+
+
+def compare_mousemaps(final: MouseMap, initial: MouseMap, print: Callable) -> None:
+    added = set(final) - set(initial)
+    removed = set(initial) - set(final)
+    changed = {k for k in set(final) & set(initial) if final[k] != initial[k]}
+
+    def print_mouse_action(trigger: MouseEvent, action: KeyAction) -> None:
+        names = list(mod_to_names(trigger.mods)) + [f'b{trigger.button+1}']
+        when = {-1: 'repeat', 1: 'press', 2: 'doublepress', 3: 'triplepress'}.get(trigger.repeat_count, trigger.repeat_count)
+        grabbed = 'grabbed' if trigger.grabbed else 'ungrabbed'
+        print('\t', '+'.join(names), when, grabbed, action)
+
+    def print_changes(defns: MouseMap, changes: Set[MouseEvent], text: str) -> None:
+        if changes:
+            print(title(text))
+            for k in sorted(changes):
+                print_mouse_action(k, defns[k])
+
+    print_changes(final, added, 'Added mouse actions:')
+    print_changes(initial, removed, 'Removed mouse actions:')
+    print_changes(final, changed, 'Changed mouse actions:')
+
+
+def compare_opts(opts: KittyOpts, print: Callable) -> None:
+    from .config import load_config
+    print()
+    print('Config options different from defaults:')
+    default_opts = load_config()
+    ignored = ('keymap', 'sequence_map', 'mousemap', 'map', 'mouse_map')
+    changed_opts = [
+        f for f in sorted(defaults._fields)
+        if f not in ignored and getattr(opts, f) != getattr(defaults, f)
+    ]
+    field_len = max(map(len, changed_opts)) if changed_opts else 20
+    fmt = '{{:{:d}s}}'.format(field_len)
+    for f in changed_opts:
+        val = getattr(opts, f)
+        if isinstance(val, dict):
+            print(f'{title(f)}:')
+            if f == 'symbol_map':
+                for k in sorted(val):
+                    print(f'\tU+{k[0]:04x} - U+{k[1]:04x} → {val[k]}')
+            else:
+                print(pformat(val))
+        else:
+            print(title(fmt.format(f)), str(getattr(opts, f)))
+
+    compare_mousemaps(opts.mousemap, default_opts.mousemap, print)
+    final_, initial_ = opts.keymap, default_opts.keymap
+    final: ShortcutMap = {(k,): v for k, v in final_.items()}
+    initial: ShortcutMap = {(k,): v for k, v in initial_.items()}
+    final_s, initial_s = map(flatten_sequence_map, (opts.sequence_map, default_opts.sequence_map))
+    final.update(final_s)
+    initial.update(initial_s)
+    compare_keymaps(final, initial, print)
+
+
+def debug_config(opts: KittyOpts) -> str:
+    from io import StringIO
+    out = StringIO()
+    p = partial(print, file=out)
+    p(version(add_rev=True))
+    p(' '.join(os.uname()))
+    if is_macos:
+        import subprocess
+        p(' '.join(subprocess.check_output(['sw_vers']).decode('utf-8').splitlines()).strip())
+    if os.path.exists('/etc/issue'):
+        with open('/etc/issue', encoding='utf-8', errors='replace') as f:
+            p(f.read().strip())
+    if os.path.exists('/etc/lsb-release'):
+        with open('/etc/lsb-release', encoding='utf-8', errors='replace') as f:
+            p(f.read().strip())
+    if not is_macos:
+        p('Running under:' + green('Wayland' if is_wayland() else 'X11'))
+    if opts.config_paths:
+        p(green('Loaded config files:'))
+        p(' ', '\n  '.join(opts.config_paths))
+    if opts.config_overrides:
+        p(green('Loaded config overrides:'))
+        p(' ', '\n  '.join(opts.config_overrides))
+    compare_opts(opts, p)
+    return out.getvalue()

kitty/disk-cache.c

@@ -13,7 +13,7 @@
 
 #include "disk-cache.h"
 #include "safe-wrappers.h"
-#include "uthash.h"
+#include "kitty-uthash.h"
 #include "loop-utils.h"
 #include "threading.h"
 #include "cross-platform-random.h"
@@ -165,8 +165,11 @@ size_of_cache_file(DiskCache *self) {
 
 size_t
 disk_cache_size_on_disk(PyObject *self) {
-    off_t ans = size_of_cache_file((DiskCache*)self);
-    return MAX(0, ans);
+    if (((DiskCache*)self)->cache_file_fd > -1) {
+        off_t ans = size_of_cache_file((DiskCache*)self);
+        return MAX(0, ans);
+    }
+    return 0;
 }
 
 typedef struct {
@@ -362,7 +365,9 @@ write_loop(void *data) {
             continue;
         } else if (!count) {
             mutex(lock);
-            if (ftruncate(self->cache_file_fd, 0) == 0) lseek(self->cache_file_fd, 0, SEEK_END);
+            if (self->cache_file_fd > -1) {
+                if (ftruncate(self->cache_file_fd, 0) == 0) lseek(self->cache_file_fd, 0, SEEK_END);
+            }
             mutex(unlock);
         }
 

kitty/fast_data_types.pyi

@@ -6,10 +6,17 @@ from typing import (
 
 import termios
 from kitty.boss import Boss
+from kitty.fonts import FontFeature
 from kitty.fonts.render import FontObject
-from kitty.options_stub import Options
+from kitty.options.types import Options
 
 # Constants {{{
+MOUSE_SELECTION_LINE: int
+MOUSE_SELECTION_EXTEND: int
+MOUSE_SELECTION_NORMAL: int
+MOUSE_SELECTION_WORD: int
+MOUSE_SELECTION_RECTANGLE: int
+MOUSE_SELECTION_LINE_FROM_POINT: int
 KITTY_VCS_REV: str
 NO_CLOSE_REQUESTED: int
 IMPERATIVE_CLOSE_REQUESTED: int
@@ -229,6 +236,7 @@ GLFW_RELEASE: int
 GLFW_REPEAT: int
 CURSOR_BEAM: int
 CURSOR_BLOCK: int
+NO_CURSOR_SHAPE: int
 CURSOR_UNDERLINE: int
 DECAWM: int
 BGIMAGE_PROGRAM: int
@@ -508,7 +516,7 @@ def sync_os_window_title(os_window_id: int) -> None:
 
 
 def set_options(
-    opts: Options,
+    opts: Optional[Options],
     is_wayland: bool = False,
     debug_rendering: bool = False,
     debug_font_fallback: bool = False
@@ -516,6 +524,10 @@ def set_options(
     pass
 
 
+def get_options() -> Options:
+    pass
+
+
 def resolve_key_mods(kitty_mod: int, mods: int) -> int:
     pass
 
@@ -621,7 +633,7 @@ class ColorProfile:
     def reset_color(self, num: int) -> None:
         pass
 
-    def update_ansi_color_table(self, val: List[int]) -> None:
+    def update_ansi_color_table(self, val: int) -> None:
         pass
 
     def set_configured_colors(
@@ -646,7 +658,7 @@ def os_window_font_size(
     pass
 
 
-def cocoa_set_notification_activated_callback(identifier: Callable[[str], None]) -> None:
+def cocoa_set_notification_activated_callback(identifier: Optional[Callable[[str], None]]) -> None:
     pass
 
 
@@ -875,7 +887,7 @@ def set_font_data(
     descriptor_for_idx: Callable[[int], Tuple[FontObject, bool, bool]],
     bold: int, italic: int, bold_italic: int, num_symbol_fonts: int,
     symbol_maps: Tuple[Tuple[int, int, int], ...], font_sz_in_pts: float,
-    font_feature_settings: Dict[str, Tuple[str, ...]]
+    font_feature_settings: Dict[str, Tuple[FontFeature, ...]]
 ) -> None:
     pass
 
@@ -917,6 +929,8 @@ class Cursor:
     fg: int
     bold: bool
     italic: bool
+    blink: bool
+    shape: int
 
 
 class Screen:
@@ -928,6 +942,7 @@ class Screen:
     historybuf: HistoryBuf
     linebuf: LineBuf
     in_bracketed_paste_mode: bool
+    cursor_visible: bool
     scrolled_by: int
     cursor: Cursor
     disable_ligatures: int
@@ -971,6 +986,9 @@ class Screen:
     def reset_callbacks(self) -> None:
         pass
 
+    def has_selection(self) -> bool:
+        pass
+
     def text_for_selection(self) -> Tuple[str, ...]:
         pass
 
@@ -992,6 +1010,9 @@ class Screen:
     def scroll_to_next_mark(self, mark: int = 0, backwards: bool = True) -> bool:
         pass
 
+    def reverse_scroll(self, amt: int, fill_from_scrollback: bool = False) -> bool:
+        pass
+
     def clear_selection(self) -> None:
         pass
 
@@ -1153,3 +1174,15 @@ def spawn(
 
 def set_window_padding(os_window_id: int, tab_id: int, window_id: int, left: int, top: int, right: int, bottom: int) -> None:
     pass
+
+
+def click_mouse_url(os_window_id: int, tab_id: int, window_id: int) -> None:
+    pass
+
+
+def mouse_selection(os_window_id: int, tab_id: int, window_id: int, code: int, button: int) -> None:
+    pass
+
+
+def apply_options_update() -> None:
+    pass

kitty/fontconfig.c

@@ -16,6 +16,25 @@
 #define FC_COLOR "color"
 #endif
 
+
+static bool initialized = false;
+
+static void
+ensure_initialized(void) {
+    if (!initialized) {
+        if (!FcInit()) fatal("Failed to initialize fontconfig library");
+        initialized = true;
+    }
+}
+
+static void
+finalize(void) {
+    if (initialized) {
+        FcFini();
+        initialized = false;
+    }
+}
+
 static inline PyObject*
 pybool(FcBool x) { PyObject *ans = x ? Py_True: Py_False; Py_INCREF(ans); return ans; }
 
@@ -109,6 +128,7 @@ font_set(FcFontSet *fs) {
 
 static PyObject*
 fc_list(PyObject UNUSED *self, PyObject *args) {
+    ensure_initialized();
     int allow_bitmapped_fonts = 0, spacing = -1;
     PyObject *ans = NULL;
     FcObjectSet *os = NULL;
@@ -198,6 +218,7 @@ end:
 
 bool
 information_for_font_family(const char *family, bool bold, bool italic, FontConfigFace *ans) {
+    ensure_initialized();
     memset(ans, 0, sizeof(FontConfigFace));
     FcPattern *pat = FcPatternCreate();
     bool ok = false;
@@ -214,6 +235,7 @@ end:
 
 static PyObject*
 fc_match(PyObject UNUSED *self, PyObject *args) {
+    ensure_initialized();
     char *family = NULL;
     int bold = 0, italic = 0, allow_bitmapped_fonts = 0, spacing = FC_MONO;
     double size_in_pts = 0, dpi = 0;
@@ -248,6 +270,7 @@ end:
 
 static PyObject*
 fc_match_postscript_name(PyObject UNUSED *self, PyObject *args) {
+    ensure_initialized();
     const char *postscript_name = NULL;
     FcPattern *pat = NULL;
     PyObject *ans = NULL;
@@ -269,6 +292,7 @@ end:
 
 PyObject*
 specialize_font_descriptor(PyObject *base_descriptor, FONTS_DATA_HANDLE fg) {
+    ensure_initialized();
     PyObject *p = PyDict_GetItemString(base_descriptor, "path"), *ans = NULL;
     PyObject *idx = PyDict_GetItemString(base_descriptor, "index");
     if (p == NULL) { PyErr_SetString(PyExc_ValueError, "Base descriptor has no path"); return NULL; }
@@ -292,6 +316,7 @@ end:
 
 bool
 fallback_font(char_type ch, const char *family, bool bold, bool italic, bool prefer_color, FontConfigFace *ans) {
+    ensure_initialized();
     memset(ans, 0, sizeof(FontConfigFace));
     bool ok = false;
     FcPattern *pat = FcPatternCreate();
@@ -310,6 +335,7 @@ end:
 
 PyObject*
 create_fallback_face(PyObject UNUSED *base_face, CPUCell* cell, bool bold, bool italic, bool emoji_presentation, FONTS_DATA_HANDLE fg) {
+    ensure_initialized();
     PyObject *ans = NULL;
     FcPattern *pat = FcPatternCreate();
     if (pat == NULL) return PyErr_NoMemory();
@@ -336,11 +362,7 @@ static PyMethodDef module_methods[] = {
 
 bool
 init_fontconfig_library(PyObject *module) {
-    if (!FcInit()) {
-        PyErr_SetString(PyExc_RuntimeError, "Failed to initialize the fontconfig library");
-        return false;
-    }
-    register_at_exit_cleanup_func(FONTCONFIG_CLEANUP_FUNC, FcFini);
+    register_at_exit_cleanup_func(FONTCONFIG_CLEANUP_FUNC, finalize);
     if (PyModule_AddFunctions(module, module_methods) != 0) return false;
     PyModule_AddIntMacro(module, FC_WEIGHT_REGULAR);
     PyModule_AddIntMacro(module, FC_WEIGHT_MEDIUM);

kitty/fonts.c

@@ -11,10 +11,9 @@
 #include "emoji.h"
 #include "unicode-data.h"
 #include "charsets.h"
+#include "glyph-cache.h"
 
 #define MISSING_GLYPH 4
-#define MAX_NUM_EXTRA_GLYPHS 8u
-#define CELLS_IN_CANVAS ((MAX_NUM_EXTRA_GLYPHS + 1u) * 3u)
 #define MAX_NUM_EXTRA_GLYPHS_PUA 4u
 
 typedef void (*send_sprite_to_gpu_func)(FONTS_DATA_HANDLE fg, unsigned int, unsigned int, unsigned int, pixel*);
@@ -22,38 +21,16 @@ send_sprite_to_gpu_func current_send_sprite_to_gpu = NULL;
 static PyObject *python_send_to_gpu_impl = NULL;
 extern PyTypeObject Line_Type;
 
-typedef struct SpecialGlyphCache SpecialGlyphCache;
 enum {NO_FONT=-3, MISSING_FONT=-2, BLANK_FONT=-1, BOX_FONT=0};
 typedef enum {
     LIGATURE_UNKNOWN, INFINITE_LIGATURE_START, INFINITE_LIGATURE_MIDDLE, INFINITE_LIGATURE_END
 } LigatureType;
 
 
-typedef struct {
-    glyph_index data[MAX_NUM_EXTRA_GLYPHS];
-} ExtraGlyphs;
-
-typedef struct SpritePosition SpritePosition;
-struct SpritePosition {
-    SpritePosition *next;
-    bool filled, rendered, colored;
-    sprite_index x, y, z;
-    uint8_t ligature_index;
-    glyph_index glyph;
-    ExtraGlyphs extra_glyphs;
-};
-
 #define SPECIAL_FILLED_MASK 1
 #define SPECIAL_VALUE_MASK 2
 #define EMPTY_FILLED_MASK 4
 #define EMPTY_VALUE_MASK 8
-#define SPECIAL_GLYPH_CACHE_SIZE 1024
-
-struct SpecialGlyphCache {
-    SpecialGlyphCache *next;
-    glyph_index glyph;
-    uint8_t data;
-};
 
 typedef struct {
     size_t max_y;
@@ -76,19 +53,24 @@ typedef struct {
 static SymbolMap *symbol_maps = NULL;
 static size_t num_symbol_maps = 0;
 
-typedef enum { SPACER_STARTEGY_UNKNOWN, SPACERS_BEFORE, SPACERS_AFTER } SpacerStrategy;
+typedef enum { SPACER_STRATEGY_UNKNOWN, SPACERS_BEFORE, SPACERS_AFTER, SPACERS_IOSEVKA } SpacerStrategy;
 
 typedef struct {
     PyObject *face;
     // Map glyphs to sprite map co-ords
-    SpritePosition sprite_map[1024];
+    SpritePosition *sprite_position_hash_table;
     hb_feature_t* ffs_hb_features;
     size_t num_ffs_hb_features;
-    SpecialGlyphCache special_glyph_cache[SPECIAL_GLYPH_CACHE_SIZE];
+    GlyphProperties *glyph_properties_hash_table;
     bool bold, italic, emoji_presentation;
     SpacerStrategy spacer_strategy;
 } Font;
 
+typedef struct Canvas {
+    pixel *buf;
+    unsigned current_cells, alloced_cells;
+} Canvas;
+
 typedef struct {
     FONTS_DATA_HEAD
     id_type id;
@@ -96,7 +78,7 @@ typedef struct {
     size_t fonts_capacity, fonts_count, fallback_fonts_count;
     ssize_t medium_font_idx, bold_font_idx, italic_font_idx, bi_font_idx, first_symbol_font_idx, first_fallback_font_idx;
     Font *fonts;
-    pixel *canvas;
+    Canvas canvas;
     GPUSpriteTracker sprite_tracker;
 } FontGroup;
 
@@ -106,6 +88,19 @@ static size_t num_font_groups = 0;
 static id_type font_group_id_counter = 0;
 static void initialize_font_group(FontGroup *fg);
 
+static void
+ensure_canvas_can_fit(FontGroup *fg, unsigned cells) {
+    if (fg->canvas.alloced_cells < cells) {
+        free(fg->canvas.buf);
+        fg->canvas.alloced_cells = cells + 4;
+        fg->canvas.buf = malloc(sizeof(fg->canvas.buf[0]) * 3u * fg->canvas.alloced_cells * fg->cell_width * fg->cell_height);
+        if (!fg->canvas.buf) fatal("Out of memory allocating canvas");
+    }
+    fg->canvas.current_cells = cells;
+    if (fg->canvas.buf) memset(fg->canvas.buf, 0, sizeof(fg->canvas.buf[0]) * fg->canvas.current_cells * 3u * fg->cell_width * fg->cell_height);
+}
+
+
 static inline void
 save_window_font_groups(void) {
     for (size_t o = 0; o < global_state.num_os_windows; o++) {
@@ -139,21 +134,10 @@ font_group_is_unused(FontGroup *fg) {
 
 void
 free_maps(Font *font) {
-#define free_a_map(type, attr) {\
-    type *s, *t; \
-    for (size_t i = 0; i < sizeof(font->attr)/sizeof(font->attr[0]); i++) { \
-        s = font->attr[i].next; \
-        while (s) { \
-            t = s; \
-            s = s->next; \
-            free(t); \
-        } \
-    }\
-    memset(font->attr, 0, sizeof(font->attr)); \
-}
-    free_a_map(SpritePosition, sprite_map);
-    free_a_map(SpecialGlyphCache, special_glyph_cache);
-#undef free_a_map
+    free_sprite_position_hash_table(&font->sprite_position_hash_table);
+    font->sprite_position_hash_table = NULL;
+    free_glyph_properties_hash_table(&font->glyph_properties_hash_table);
+    font->glyph_properties_hash_table = NULL;
 }
 
 static inline void
@@ -166,7 +150,7 @@ del_font(Font *f) {
 
 static inline void
 del_font_group(FontGroup *fg) {
-    free(fg->canvas); fg->canvas = NULL;
+    free(fg->canvas.buf); fg->canvas.buf = NULL; fg->canvas = (Canvas){0};
     fg->sprite_map = free_sprite_map(fg->sprite_map);
     for (size_t i = 0; i < fg->fonts_count; i++) del_font(fg->fonts + i);
     free(fg->fonts); fg->fonts = NULL;
@@ -217,11 +201,6 @@ font_group_for(double font_sz_in_pts, double logical_dpi_x, double logical_dpi_y
     return fg;
 }
 
-static inline void
-clear_canvas(FontGroup *fg) {
-    if (fg->canvas) memset(fg->canvas, 0, sizeof(pixel) * CELLS_IN_CANVAS * fg->cell_width * fg->cell_height);
-}
-
 
 
 // Sprites {{{
@@ -258,64 +237,15 @@ do_increment(FontGroup *fg, int *error) {
 }
 
 
-static inline bool
-extra_glyphs_equal(ExtraGlyphs *a, ExtraGlyphs *b) {
-    for (size_t i = 0; i < MAX_NUM_EXTRA_GLYPHS; i++) {
-        if (a->data[i] != b->data[i]) return false;
-        if (a->data[i] == 0) return true;
-    }
-    return true;
-}
-
-
 static SpritePosition*
-sprite_position_for(FontGroup *fg, Font *font, glyph_index glyph, ExtraGlyphs *extra_glyphs, uint8_t ligature_index, int *error) {
-    glyph_index idx = glyph & (SPECIAL_GLYPH_CACHE_SIZE - 1);
-    SpritePosition *s = font->sprite_map + idx;
-    // Optimize for the common case of glyph under 1024 already in the cache
-    if (LIKELY(s->glyph == glyph && s->filled && extra_glyphs_equal(&s->extra_glyphs, extra_glyphs) && s->ligature_index == ligature_index)) return s;  // Cache hit
-    while(true) {
-        if (s->filled) {
-            if (s->glyph == glyph && extra_glyphs_equal(&s->extra_glyphs, extra_glyphs) && s->ligature_index == ligature_index) return s;  // Cache hit
-        } else {
-            break;
-        }
-        if (!s->next) {
-            s->next = calloc(1, sizeof(SpritePosition));
-            if (s->next == NULL) { *error = 1; return NULL; }
-        }
-        s = s->next;
+sprite_position_for(FontGroup *fg, Font *font, glyph_index *glyphs, unsigned glyph_count, uint8_t ligature_index, unsigned cell_count, int *error) {
+    bool created;
+    SpritePosition *s = find_or_create_sprite_position(&font->sprite_position_hash_table, glyphs, glyph_count, ligature_index, cell_count, &created);
+    if (!s) { *error = 1; return NULL; }
+    if (created) {
+        s->x = fg->sprite_tracker.x; s->y = fg->sprite_tracker.y; s->z = fg->sprite_tracker.z;
+        do_increment(fg, error);
     }
-    s->glyph = glyph;
-    memcpy(&s->extra_glyphs, extra_glyphs, sizeof(ExtraGlyphs));
-    s->ligature_index = ligature_index;
-    s->filled = true;
-    s->rendered = false;
-    s->colored = false;
-    s->x = fg->sprite_tracker.x; s->y = fg->sprite_tracker.y; s->z = fg->sprite_tracker.z;
-    do_increment(fg, error);
-    return s;
-}
-
-static inline SpecialGlyphCache*
-special_glyph_cache_for(Font *font, glyph_index glyph, uint8_t filled_mask) {
-    SpecialGlyphCache *s = font->special_glyph_cache + (glyph & 0x3ff);
-    // Optimize for the common case of glyph under SPECIAL_GLYPH_CACHE_SIZE already in the cache
-    if (LIKELY(s->glyph == glyph && s->data & filled_mask)) return s;  // Cache hit
-    while(true) {
-        if (s->data & filled_mask) {
-            if (s->glyph == glyph) return s;  // Cache hit
-        } else {
-            if (!s->glyph) break;  // Empty cache slot
-            else if (s->glyph == glyph) return s;  // Cache slot that contains other data than the data indicated by filled_mask
-        }
-        if (!s->next) {
-            s->next = calloc(1, sizeof(SpecialGlyphCache));
-            if (s->next == NULL) return NULL;
-        }
-        s = s->next;
-    }
-    s->glyph = glyph;
     return s;
 }
 
@@ -325,33 +255,6 @@ sprite_tracker_current_layout(FONTS_DATA_HANDLE data, unsigned int *x, unsigned
     *x = fg->sprite_tracker.xnum; *y = fg->sprite_tracker.ynum; *z = fg->sprite_tracker.z;
 }
 
-void
-clear_sprite_map(Font *font) {
-#define CLEAR(s) s->filled = false; s->rendered = false; s->colored = false; s->glyph = 0; zero_at_ptr(&s->extra_glyphs); s->x = 0; s->y = 0; s->z = 0; s->ligature_index = 0;
-    SpritePosition *s;
-    for (size_t i = 0; i < arraysz(font->sprite_map); i++) {
-        s = font->sprite_map + i;
-        CLEAR(s);
-        while ((s = s->next)) {
-            CLEAR(s);
-        }
-    }
-#undef CLEAR
-}
-
-void
-clear_special_glyph_cache(Font *font) {
-#define CLEAR(s) s->data = 0; s->glyph = 0;
-    SpecialGlyphCache *s;
-    for (size_t i = 0; i < arraysz(font->special_glyph_cache); i++) {
-        s = font->special_glyph_cache + i;
-        CLEAR(s);
-        while ((s = s->next)) {
-            CLEAR(s);
-        }
-    }
-#undef CLEAR
-}
 
 static void
 sprite_tracker_set_layout(GPUSpriteTracker *sprite_tracker, unsigned int cell_width, unsigned int cell_height) {
@@ -415,6 +318,7 @@ free_font_groups(void) {
         free(font_groups); font_groups = NULL;
         font_groups_capacity = 0; num_font_groups = 0;
     }
+    free_glyph_cache_global_resources();
 }
 
 static void
@@ -464,9 +368,7 @@ calc_cell_metrics(FontGroup *fg) {
     sprite_tracker_set_layout(&fg->sprite_tracker, cell_width, cell_height);
     fg->cell_width = cell_width; fg->cell_height = cell_height;
     fg->baseline = baseline; fg->underline_position = underline_position; fg->underline_thickness = underline_thickness, fg->strikethrough_position = strikethrough_position, fg->strikethrough_thickness = strikethrough_thickness;
-    free(fg->canvas);
-    fg->canvas = calloc((size_t)CELLS_IN_CANVAS * fg->cell_width * fg->cell_height, sizeof(pixel));
-    if (!fg->canvas) fatal("Out of memory allocating canvas for font group");
+    ensure_canvas_can_fit(fg, 8);
 }
 
 static inline bool
@@ -658,10 +560,10 @@ END_ALLOW_CASE_RANGE
 static PyObject* box_drawing_function = NULL, *prerender_function = NULL, *descriptor_for_idx = NULL;
 
 void
-render_alpha_mask(uint8_t *alpha_mask, pixel* dest, Region *src_rect, Region *dest_rect, size_t src_stride, size_t dest_stride) {
+render_alpha_mask(const uint8_t *alpha_mask, pixel* dest, Region *src_rect, Region *dest_rect, size_t src_stride, size_t dest_stride) {
     for (size_t sr = src_rect->top, dr = dest_rect->top; sr < src_rect->bottom && dr < dest_rect->bottom; sr++, dr++) {
         pixel *d = dest + dest_stride * dr;
-        uint8_t *s = alpha_mask + src_stride * sr;
+        const uint8_t *s = alpha_mask + src_stride * sr;
         for(size_t sc = src_rect->left, dc = dest_rect->left; sc < src_rect->right && dc < dest_rect->right; sc++, dc++) {
             uint8_t src_alpha = d[dc] & 0xff;
             uint8_t alpha = s[sc];
@@ -674,8 +576,7 @@ static void
 render_box_cell(FontGroup *fg, CPUCell *cpu_cell, GPUCell *gpu_cell) {
     int error = 0;
     glyph_index glyph = box_glyph_id(cpu_cell->ch);
-    static ExtraGlyphs extra_glyphs = {{0}};
-    SpritePosition *sp = sprite_position_for(fg, &fg->fonts[BOX_FONT], glyph, &extra_glyphs, false, &error);
+    SpritePosition *sp = sprite_position_for(fg, &fg->fonts[BOX_FONT], &glyph, 1, 0, 1, &error);
     if (sp == NULL) {
         sprite_map_set_error(error); PyErr_Print();
         set_sprite(gpu_cell, 0, 0, 0);
@@ -688,10 +589,10 @@ render_box_cell(FontGroup *fg, CPUCell *cpu_cell, GPUCell *gpu_cell) {
     PyObject *ret = PyObject_CallFunction(box_drawing_function, "IIId", cpu_cell->ch, fg->cell_width, fg->cell_height, (fg->logical_dpi_x + fg->logical_dpi_y) / 2.0);
     if (ret == NULL) { PyErr_Print(); return; }
     uint8_t *alpha_mask = PyLong_AsVoidPtr(PyTuple_GET_ITEM(ret, 0));
-    clear_canvas(fg);
+    ensure_canvas_can_fit(fg, 1);
     Region r = { .right = fg->cell_width, .bottom = fg->cell_height };
-    render_alpha_mask(alpha_mask, fg->canvas, &r, &r, fg->cell_width, fg->cell_width);
-    current_send_sprite_to_gpu((FONTS_DATA_HANDLE)fg, sp->x, sp->y, sp->z, fg->canvas);
+    render_alpha_mask(alpha_mask, fg->canvas.buf, &r, &r, fg->cell_width, fg->cell_width);
+    current_send_sprite_to_gpu((FONTS_DATA_HANDLE)fg, sp->x, sp->y, sp->z, fg->canvas.buf);
     Py_DECREF(ret);
 }
 
@@ -717,46 +618,62 @@ load_hb_buffer(CPUCell *first_cpu_cell, GPUCell *first_gpu_cell, index_type num_
 
 
 static inline void
-set_cell_sprite(GPUCell *cell, SpritePosition *sp) {
+set_cell_sprite(GPUCell *cell, const SpritePosition *sp) {
     cell->sprite_x = sp->x; cell->sprite_y = sp->y; cell->sprite_z = sp->z;
     if (sp->colored) cell->sprite_z |= 0x4000;
 }
 
 static inline pixel*
 extract_cell_from_canvas(FontGroup *fg, unsigned int i, unsigned int num_cells) {
-    pixel *ans = fg->canvas + (fg->cell_width * fg->cell_height * (CELLS_IN_CANVAS - 1)), *dest = ans, *src = fg->canvas + (i * fg->cell_width);
+    pixel *ans = fg->canvas.buf + (fg->cell_width * fg->cell_height * (fg->canvas.current_cells - 1)), *dest = ans, *src = fg->canvas.buf + (i * fg->cell_width);
     unsigned int stride = fg->cell_width * num_cells;
-    for (unsigned int r = 0; r < fg->cell_height; r++, dest += fg->cell_width, src += stride) memcpy(dest, src, fg->cell_width * sizeof(pixel));
+    for (unsigned int r = 0; r < fg->cell_height; r++, dest += fg->cell_width, src += stride) memcpy(dest, src, fg->cell_width * sizeof(fg->canvas.buf[0]));
     return ans;
 }
 
+typedef struct GlyphRenderScratch {
+    SpritePosition* *sprite_positions;
+    glyph_index *glyphs;
+    size_t sz;
+} GlyphRenderScratch;
+static GlyphRenderScratch global_glyph_render_scratch = {0};
+
 static inline void
-render_group(FontGroup *fg, unsigned int num_cells, unsigned int num_glyphs, CPUCell *cpu_cells, GPUCell *gpu_cells, hb_glyph_info_t *info, hb_glyph_position_t *positions, Font *font, glyph_index glyph, ExtraGlyphs *extra_glyphs, bool center_glyph) {
-    static SpritePosition* sprite_position[16];
+render_group(FontGroup *fg, unsigned int num_cells, unsigned int num_glyphs, CPUCell *cpu_cells, GPUCell *gpu_cells, hb_glyph_info_t *info, hb_glyph_position_t *positions, Font *font, glyph_index *glyphs, unsigned glyph_count, bool center_glyph) {
+#define sp global_glyph_render_scratch.sprite_positions
     int error = 0;
-    num_cells = MIN(arraysz(sprite_position), num_cells);
-    for (unsigned int i = 0; i < num_cells; i++) {
-        sprite_position[i] = sprite_position_for(fg, font, glyph, extra_glyphs, (uint8_t)i, &error);
+    bool all_rendered = true;
+    bool is_infinite_ligature = num_cells > 9 && num_glyphs == num_cells;
+    for (unsigned i = 0, ligature_index = 0; i < num_cells; i++) {
+        bool is_repeat_glyph = is_infinite_ligature && i > 1 && i + 1 < num_cells && glyphs[i] == glyphs[i-1] && glyphs[i] == glyphs[i-2] && glyphs[i] == glyphs[i+1];
+        if (is_repeat_glyph) {
+            sp[i] = sp[i-1];
+        } else {
+            sp[i] = sprite_position_for(fg, font, glyphs, glyph_count, ligature_index++, num_cells, &error);
+        }
         if (error != 0) { sprite_map_set_error(error); PyErr_Print(); return; }
+        if (!sp[i]->rendered) all_rendered = false;
     }
-    if (sprite_position[0]->rendered) {
-        for (unsigned int i = 0; i < num_cells; i++) { set_cell_sprite(gpu_cells + i, sprite_position[i]); }
+    if (all_rendered) {
+        for (unsigned i = 0; i < num_cells; i++) { set_cell_sprite(gpu_cells + i, sp[i]); }
         return;
     }
 
-    clear_canvas(fg);
+    ensure_canvas_can_fit(fg, num_cells + 1);
     bool was_colored = (gpu_cells->attrs & WIDTH_MASK) == 2 && is_emoji(cpu_cells->ch);
-    render_glyphs_in_cells(font->face, font->bold, font->italic, info, positions, num_glyphs, fg->canvas, fg->cell_width, fg->cell_height, num_cells, fg->baseline, &was_colored, (FONTS_DATA_HANDLE)fg, center_glyph);
+    render_glyphs_in_cells(font->face, font->bold, font->italic, info, positions, num_glyphs, fg->canvas.buf, fg->cell_width, fg->cell_height, num_cells, fg->baseline, &was_colored, (FONTS_DATA_HANDLE)fg, center_glyph);
     if (PyErr_Occurred()) PyErr_Print();
 
-    for (unsigned int i = 0; i < num_cells; i++) {
-        sprite_position[i]->rendered = true;
-        sprite_position[i]->colored = was_colored;
-        set_cell_sprite(gpu_cells + i, sprite_position[i]);
-        pixel *buf = num_cells == 1 ? fg->canvas : extract_cell_from_canvas(fg, i, num_cells);
-        current_send_sprite_to_gpu((FONTS_DATA_HANDLE)fg, sprite_position[i]->x, sprite_position[i]->y, sprite_position[i]->z, buf);
+    for (unsigned i = 0; i < num_cells; i++) {
+        if (!sp[i]->rendered) {
+            sp[i]->rendered = true;
+            sp[i]->colored = was_colored;
+            pixel *buf = num_cells == 1 ? fg->canvas.buf : extract_cell_from_canvas(fg, i, num_cells);
+            current_send_sprite_to_gpu((FONTS_DATA_HANDLE)fg, sp[i]->x, sp[i]->y, sp[i]->z, buf);
+        }
+        set_cell_sprite(gpu_cells + i, sp[i]);
     }
-
+#undef sp
 }
 
 typedef struct {
@@ -834,7 +751,7 @@ static inline bool
 is_special_glyph(glyph_index glyph_id, Font *font, CellData* cell_data) {
     // A glyph is special if the codepoint it corresponds to matches a
     // different glyph in the font
-    SpecialGlyphCache *s = special_glyph_cache_for(font, glyph_id, SPECIAL_FILLED_MASK);
+    GlyphProperties *s = find_or_create_glyph_properties(&font->glyph_properties_hash_table, glyph_id);
     if (s == NULL) return false;
     if (!(s->data & SPECIAL_FILLED_MASK)) {
         bool is_special = cell_data->current_codepoint ? (
@@ -850,7 +767,7 @@ is_special_glyph(glyph_index glyph_id, Font *font, CellData* cell_data) {
 static inline bool
 is_empty_glyph(glyph_index glyph_id, Font *font) {
     // A glyph is empty if its metrics have a width of zero
-    SpecialGlyphCache *s = special_glyph_cache_for(font, glyph_id, EMPTY_FILLED_MASK);
+    GlyphProperties *s = find_or_create_glyph_properties(&font->glyph_properties_hash_table, glyph_id);
     if (s == NULL) return false;
     if (!(s->data & EMPTY_FILLED_MASK)) {
         uint8_t val = is_glyph_empty(font->face, glyph_id) ? EMPTY_VALUE_MASK : 0;
@@ -890,12 +807,20 @@ check_cell_consumed(CellData *cell_data, CPUCell *last_cpu_cell) {
 }
 
 static inline LigatureType
-ligature_type_from_glyph_name(const char *glyph_name) {
-    const char *p = strrchr(glyph_name, '_');
-    if (!p) return LIGATURE_UNKNOWN;
-    if (strcmp(p, "_start.seq") == 0) return INFINITE_LIGATURE_START;
-    if (strcmp(p, "_middle.seq") == 0) return INFINITE_LIGATURE_MIDDLE;
-    if (strcmp(p, "_end.seq") == 0) return INFINITE_LIGATURE_END;
+ligature_type_from_glyph_name(const char *glyph_name, SpacerStrategy strategy) {
+    const char *p, *m, *s, *e;
+    if (strategy == SPACERS_IOSEVKA) {
+        p = strrchr(glyph_name, '.');
+        m = ".join-m"; s = ".join-l"; e = ".join-r";
+    } else {
+        p = strrchr(glyph_name, '_');
+        m = "_middle.seq"; s = "_start.seq"; e = "_end.seq";
+    }
+    if (p) {
+        if (strcmp(p, m) == 0) return INFINITE_LIGATURE_MIDDLE;
+        if (strcmp(p, s) == 0) return INFINITE_LIGATURE_START;
+        if (strcmp(p, e) == 0) return INFINITE_LIGATURE_END;
+    }
     return LIGATURE_UNKNOWN;
 }
 
@@ -913,19 +838,116 @@ detect_spacer_strategy(hb_font_t *hbf, Font *font) {
         bool is_empty = is_special && is_empty_glyph(glyph_id, font);
         if (is_empty) font->spacer_strategy = SPACERS_AFTER;
     }
+    shape(cpu_cells, gpu_cells, 2, hbf, font, false);
+    if (G(num_glyphs)) {
+        char glyph_name[128]; glyph_name[arraysz(glyph_name)-1] = 0;
+        for (unsigned i = 0; i < G(num_glyphs); i++) {
+            glyph_index glyph_id = G(info)[i].codepoint;
+            hb_font_glyph_to_string(hbf, glyph_id, glyph_name, arraysz(glyph_name)-1);
+            char *dot = strrchr(glyph_name, '.');
+            if (dot && (!strcmp(dot, ".join-l") || !strcmp(dot, ".join-r") || !strcmp(dot, ".join-m"))) {
+                font->spacer_strategy = SPACERS_IOSEVKA;
+                break;
+            }
+        }
+    }
 }
 
-static inline void
-shape_run(CPUCell *first_cpu_cell, GPUCell *first_gpu_cell, index_type num_cells, Font *font, bool disable_ligature) {
-    hb_font_t *hbf = harfbuzz_font_for_face(font->face);
-    if (font->spacer_strategy == SPACER_STARTEGY_UNKNOWN) detect_spacer_strategy(hbf, font);
-    shape(first_cpu_cell, first_gpu_cell, num_cells, hbf, font, disable_ligature);
-#if 0
-        static char dbuf[1024];
-        // You can also generate this easily using hb-shape --show-extents --cluster-level=1 --shapers=ot /path/to/font/file text
-        hb_buffer_serialize_glyphs(harfbuzz_buffer, 0, group_state.num_glyphs, dbuf, sizeof(dbuf), NULL, harfbuzz_font_for_face(font->face), HB_BUFFER_SERIALIZE_FORMAT_TEXT, HB_BUFFER_SERIALIZE_FLAG_DEFAULT | HB_BUFFER_SERIALIZE_FLAG_GLYPH_EXTENTS);
-        printf("\n%s\n", dbuf);
-#endif
+static LigatureType
+ligature_type_for_glyph(hb_font_t *hbf, glyph_index glyph_id, SpacerStrategy strategy) {
+    static char glyph_name[128]; glyph_name[arraysz(glyph_name)-1] = 0;
+    hb_font_glyph_to_string(hbf, glyph_id, glyph_name, arraysz(glyph_name)-1);
+    return ligature_type_from_glyph_name(glyph_name, strategy);
+}
+
+#define L INFINITE_LIGATURE_START
+#define M INFINITE_LIGATURE_MIDDLE
+#define R INFINITE_LIGATURE_END
+#define I LIGATURE_UNKNOWN
+static bool
+is_iosevka_lig_starter(LigatureType before, LigatureType current, LigatureType after) {
+    return (current == R || (current == I && (after == L || after == M))) \
+                     && \
+           !(before == R || before == M);
+}
+
+static bool
+is_iosevka_lig_ender(LigatureType before, LigatureType current, LigatureType after) {
+    return (current == L || (current == I && (before == R || before == M))) \
+                     && \
+            !(after == L || after == M);
+}
+#undef L
+#undef M
+#undef R
+#undef I
+
+static LigatureType *ligature_types = NULL;
+static size_t ligature_types_sz = 0;
+
+static void
+group_iosevka(Font *font, hb_font_t *hbf) {
+    // Group as per algorithm discussed in: https://github.com/be5invis/Iosevka/issues/1007
+    if (ligature_types_sz <= G(num_glyphs)) {
+        ligature_types_sz = G(num_glyphs) + 16;
+        ligature_types = realloc(ligature_types, ligature_types_sz * sizeof(ligature_types[0]));
+        if (!ligature_types) fatal("Out of memory allocating ligature types array");
+    }
+    for (size_t i = G(glyph_idx); i < G(num_glyphs); i++) {
+        ligature_types[i] = ligature_type_for_glyph(hbf, G(info)[i].codepoint, font->spacer_strategy);
+    }
+
+    uint32_t cluster, next_cluster;
+    while (G(glyph_idx) < G(num_glyphs) && G(cell_idx) < G(num_cells)) {
+        cluster = G(info)[G(glyph_idx)].cluster;
+        uint32_t num_codepoints_used_by_glyph = 0;
+        bool is_last_glyph = G(glyph_idx) == G(num_glyphs) - 1;
+        Group *current_group = G(groups) + G(group_idx);
+        if (is_last_glyph) {
+            num_codepoints_used_by_glyph = UINT32_MAX;
+            next_cluster = 0;
+        } else {
+            next_cluster = G(info)[G(glyph_idx) + 1].cluster;
+            // RTL languages like Arabic have decreasing cluster numbers
+            if (next_cluster != cluster) num_codepoints_used_by_glyph = cluster > next_cluster ? cluster - next_cluster : next_cluster - cluster;
+        }
+        const LigatureType before = G(glyph_idx) ? ligature_types[G(glyph_idx - 1)] : LIGATURE_UNKNOWN;
+        const LigatureType current = ligature_types[G(glyph_idx)];
+        const LigatureType after = is_last_glyph ? LIGATURE_UNKNOWN : ligature_types[G(glyph_idx + 1)];
+        bool end_current_group = false;
+        if (current_group->num_glyphs && is_iosevka_lig_ender(before, current, after)) {
+            end_current_group = true;
+        }
+        if (!current_group->num_glyphs++) {
+            if (is_iosevka_lig_starter(before, current, after)) current_group->has_special_glyph = true;
+            else end_current_group = true;
+            current_group->first_glyph_idx = G(glyph_idx);
+            current_group->first_cell_idx = G(cell_idx);
+        }
+        if (is_last_glyph) {
+            // soak up all remaining cells
+            if (G(cell_idx) < G(num_cells)) {
+                unsigned int num_left = G(num_cells) - G(cell_idx);
+                current_group->num_cells += num_left;
+                G(cell_idx) += num_left;
+            }
+        } else {
+            unsigned int num_cells_consumed = 0;
+            while (num_codepoints_used_by_glyph && G(cell_idx) < G(num_cells)) {
+                unsigned int w = check_cell_consumed(&G(current_cell_data), G(last_cpu_cell));
+                G(cell_idx) += w;
+                num_cells_consumed += w;
+                num_codepoints_used_by_glyph--;
+            }
+            current_group->num_cells += num_cells_consumed;
+        }
+        if (end_current_group) G(group_idx)++;
+        G(glyph_idx)++;
+    }
+}
+
+static void
+group_normal(Font *font, hb_font_t *hbf) {
     /* Now distribute the glyphs into groups of cells
      * Considerations to keep in mind:
      * Group sizes should be as small as possible for best performance
@@ -946,13 +968,10 @@ shape_run(CPUCell *first_cpu_cell, GPUCell *first_gpu_cell, index_type num_cells
      */
     uint32_t cluster, next_cluster;
     bool add_to_current_group;
-    char glyph_name[128]; glyph_name[arraysz(glyph_name)-1] = 0;
     bool prev_glyph_was_inifinte_ligature_end = false;
-#define MAX_GLYPHS_IN_GROUP (MAX_NUM_EXTRA_GLYPHS + 1u)
     while (G(glyph_idx) < G(num_glyphs) && G(cell_idx) < G(num_cells)) {
         glyph_index glyph_id = G(info)[G(glyph_idx)].codepoint;
-        hb_font_glyph_to_string(hbf, glyph_id, glyph_name, arraysz(glyph_name)-1);
-        LigatureType ligature_type = ligature_type_from_glyph_name(glyph_name);
+        LigatureType ligature_type = ligature_type_for_glyph(hbf, glyph_id, font->spacer_strategy);
         cluster = G(info)[G(glyph_idx)].cluster;
         bool is_special = is_special_glyph(glyph_id, font, &G(current_cell_data));
         bool is_empty = is_special && is_empty_glyph(glyph_id, font);
@@ -982,16 +1001,14 @@ shape_run(CPUCell *first_cpu_cell, GPUCell *first_gpu_cell, index_type num_cells
                 add_to_current_group = !G(prev_was_special) || !current_group->num_cells;
             }
         }
-        static const bool debug_grouping = false;
-        if (debug_grouping) {
-            char ch[8] = {0};
-            encode_utf8(G(current_cell_data).current_codepoint, ch);
-            printf("\x1b[32m→ %s\x1b[m glyph_idx: %zu glyph_id: %u (%s) group_idx: %zu cluster: %u -> %u is_special: %d\n"
-                    "  num_codepoints_used_by_glyph: %u current_group: (%u cells, %u glyphs) add_to_current_group: %d\n",
-                    ch, G(glyph_idx), glyph_id, glyph_name, G(group_idx), cluster, next_cluster, is_special,
-                    num_codepoints_used_by_glyph, current_group->num_cells, current_group->num_glyphs, add_to_current_group);
-        }
-        if (current_group->num_glyphs >= MAX_GLYPHS_IN_GROUP || current_group->num_cells >= MAX_GLYPHS_IN_GROUP) add_to_current_group = false;
+#if 0
+        char ch[8] = {0};
+        encode_utf8(G(current_cell_data).current_codepoint, ch);
+        printf("\x1b[32m→ %s\x1b[m glyph_idx: %zu glyph_id: %u (%s) group_idx: %zu cluster: %u -> %u is_special: %d\n"
+                "  num_codepoints_used_by_glyph: %u current_group: (%u cells, %u glyphs) add_to_current_group: %d\n",
+                ch, G(glyph_idx), glyph_id, glyph_name, G(group_idx), cluster, next_cluster, is_special,
+                num_codepoints_used_by_glyph, current_group->num_cells, current_group->num_glyphs, add_to_current_group);
+#endif
 
         if (!add_to_current_group) { current_group = G(groups) + ++G(group_idx); }
         if (!current_group->num_glyphs++) {
@@ -1011,13 +1028,11 @@ shape_run(CPUCell *first_cpu_cell, GPUCell *first_gpu_cell, index_type num_cells
             // soak up all remaining cells
             if (G(cell_idx) < G(num_cells)) {
                 unsigned int num_left = G(num_cells) - G(cell_idx);
-                if (current_group->num_cells + num_left > MAX_GLYPHS_IN_GROUP) MOVE_GLYPH_TO_NEXT_GROUP(G(cell_idx));
                 current_group->num_cells += num_left;
-                if (current_group->num_cells > MAX_GLYPHS_IN_GROUP) current_group->num_cells = MAX_GLYPHS_IN_GROUP;  // leave any trailing cells empty
                 G(cell_idx) += num_left;
             }
         } else {
-            unsigned int num_cells_consumed = 0, start_cell_idx = G(cell_idx);
+            unsigned int num_cells_consumed = 0;
             while (num_codepoints_used_by_glyph && G(cell_idx) < G(num_cells)) {
                 unsigned int w = check_cell_consumed(&G(current_cell_data), G(last_cpu_cell));
                 G(cell_idx) += w;
@@ -1025,24 +1040,9 @@ shape_run(CPUCell *first_cpu_cell, GPUCell *first_gpu_cell, index_type num_cells
                 num_codepoints_used_by_glyph--;
             }
             if (num_cells_consumed) {
-                if (num_cells_consumed > MAX_GLYPHS_IN_GROUP) {
-                    // Nasty, a single glyph used more than MAX_GLYPHS_IN_GROUP cells, we cannot render this case correctly
-                    log_error("The glyph: %u needs more than %u cells, cannot render it", glyph_id, MAX_GLYPHS_IN_GROUP);
-                    current_group->num_glyphs--;
-                    while (num_cells_consumed) {
-                        G(group_idx)++; current_group = G(groups) + G(group_idx);
-                        current_group->num_glyphs = 1; current_group->first_glyph_idx = G(glyph_idx);
-                        current_group->num_cells = MIN(num_cells_consumed, MAX_GLYPHS_IN_GROUP);
-                        current_group->first_cell_idx = start_cell_idx;
-                        start_cell_idx += current_group->num_cells;
-                        num_cells_consumed -= current_group->num_cells;
-                    }
-                } else {
-                    if (num_cells_consumed + current_group->num_cells > MAX_GLYPHS_IN_GROUP) MOVE_GLYPH_TO_NEXT_GROUP(start_cell_idx);
-                    current_group->num_cells += num_cells_consumed;
-                    if (!is_special) {  // not a ligature, end the group
-                        G(group_idx)++; current_group = G(groups) + G(group_idx);
-                    }
+                current_group->num_cells += num_cells_consumed;
+                if (!is_special) {  // not a ligature, end the group
+                    G(group_idx)++; current_group = G(groups) + G(group_idx);
                 }
             }
         }
@@ -1053,8 +1053,22 @@ shape_run(CPUCell *first_cpu_cell, GPUCell *first_gpu_cell, index_type num_cells
         prev_glyph_was_inifinte_ligature_end = ligature_type == INFINITE_LIGATURE_END;
         G(glyph_idx)++;
     }
-#undef MOVE_GLYPH_TO_NEXT_GROUP
-#undef MAX_GLYPHS_IN_GROUP
+}
+
+
+static void
+shape_run(CPUCell *first_cpu_cell, GPUCell *first_gpu_cell, index_type num_cells, Font *font, bool disable_ligature) {
+    hb_font_t *hbf = harfbuzz_font_for_face(font->face);
+    if (font->spacer_strategy == SPACER_STRATEGY_UNKNOWN) detect_spacer_strategy(hbf, font);
+    shape(first_cpu_cell, first_gpu_cell, num_cells, hbf, font, disable_ligature);
+    if (font->spacer_strategy == SPACERS_IOSEVKA) group_iosevka(font, hbf);
+    else group_normal(font, hbf);
+#if 0
+        static char dbuf[1024];
+        // You can also generate this easily using hb-shape --show-extents --cluster-level=1 --shapers=ot /path/to/font/file text
+        hb_buffer_serialize_glyphs(harfbuzz_buffer, 0, group_state.num_glyphs, dbuf, sizeof(dbuf), NULL, harfbuzz_font_for_face(font->face), HB_BUFFER_SERIALIZE_FORMAT_TEXT, HB_BUFFER_SERIALIZE_FLAG_DEFAULT | HB_BUFFER_SERIALIZE_FLAG_GLYPH_EXTENTS);
+        printf("\n%s\n", dbuf);
+#endif
 }
 
 static inline void
@@ -1063,12 +1077,13 @@ merge_groups_for_pua_space_ligature(void) {
         Group *g = G(groups), *g1 = G(groups) + 1;
         g->num_cells += g1->num_cells;
         g->num_glyphs += g1->num_glyphs;
-        g->num_glyphs = MIN(g->num_glyphs, MAX_NUM_EXTRA_GLYPHS + 1);
         G(group_idx)--;
     }
     G(groups)->is_space_ligature = true;
 }
 
+#undef MOVE_GLYPH_TO_NEXT_GROUP
+
 static inline bool
 is_group_calt_ligature(const Group *group) {
     GPUCell *first_cell = G(first_gpu_cell) + group->first_cell_idx;
@@ -1095,22 +1110,26 @@ split_run_at_offset(index_type cursor_offset, index_type *left, index_type *righ
 static inline void
 render_groups(FontGroup *fg, Font *font, bool center_glyph) {
     unsigned idx = 0;
-    ExtraGlyphs ed;
     while (idx <= G(group_idx)) {
         Group *group = G(groups) + idx;
         if (!group->num_cells) break;
         /* printf("Group: idx: %u num_cells: %u num_glyphs: %u first_glyph_idx: %u first_cell_idx: %u total_num_glyphs: %zu\n", */
         /*         idx, group->num_cells, group->num_glyphs, group->first_glyph_idx, group->first_cell_idx, group_state.num_glyphs); */
-        glyph_index primary = group->num_glyphs ? G(info)[group->first_glyph_idx].codepoint : 0;
-        unsigned int i;
-        int last = -1;
-        for (i = 1; i < MIN(arraysz(ed.data) + 1, group->num_glyphs); i++) { last = i - 1; ed.data[last] = G(info)[group->first_glyph_idx + i].codepoint; }
-        if ((size_t)(last + 1) < arraysz(ed.data)) ed.data[last + 1] = 0;
-        // We dont want to render the spaces in a space ligature because
-        // there exist stupid fonts like Powerline that have no space glyph,
-        // so special case it: https://github.com/kovidgoyal/kitty/issues/1225
-        unsigned int num_glyphs = group->is_space_ligature ? 1 : group->num_glyphs;
-        render_group(fg, group->num_cells, num_glyphs, G(first_cpu_cell) + group->first_cell_idx, G(first_gpu_cell) + group->first_cell_idx, G(info) + group->first_glyph_idx, G(positions) + group->first_glyph_idx, font, primary, &ed, center_glyph);
+        if (group->num_glyphs) {
+            size_t sz = MAX(group->num_glyphs, group->num_cells) + 16;
+            if (global_glyph_render_scratch.sz < sz) {
+#define a(what) free(global_glyph_render_scratch.what); global_glyph_render_scratch.what = malloc(sz * sizeof(global_glyph_render_scratch.what[0])); if (!global_glyph_render_scratch.what) fatal("Out of memory");
+                a(glyphs); a(sprite_positions);
+#undef a
+                global_glyph_render_scratch.sz = sz;
+            }
+            for (unsigned i = 0; i < group->num_glyphs; i++) global_glyph_render_scratch.glyphs[i] = G(info)[group->first_glyph_idx + i].codepoint;
+            // We dont want to render the spaces in a space ligature because
+            // there exist stupid fonts like Powerline that have no space glyph,
+            // so special case it: https://github.com/kovidgoyal/kitty/issues/1225
+            unsigned int num_glyphs = group->is_space_ligature ? 1 : group->num_glyphs;
+            render_group(fg, group->num_cells, num_glyphs, G(first_cpu_cell) + group->first_cell_idx, G(first_gpu_cell) + group->first_cell_idx, G(info) + group->first_glyph_idx, G(positions) + group->first_glyph_idx, font, global_glyph_render_scratch.glyphs, num_glyphs, center_glyph);
+        }
         idx++;
     }
 }
@@ -1145,8 +1164,8 @@ test_shape(PyObject UNUSED *self, PyObject *args) {
         if (!group->num_cells) break;
         first_glyph = group->num_glyphs ? G(info)[group->first_glyph_idx].codepoint : 0;
 
-        PyObject *eg = PyTuple_New(MAX_NUM_EXTRA_GLYPHS);
-        for (size_t g = 0; g < MAX_NUM_EXTRA_GLYPHS; g++) PyTuple_SET_ITEM(eg, g, Py_BuildValue("H", g + 1 < group->num_glyphs ? G(info)[group->first_glyph_idx + g].codepoint : 0));
+        PyObject *eg = PyTuple_New(group->num_glyphs);
+        for (size_t g = 0; g < group->num_glyphs; g++) PyTuple_SET_ITEM(eg, g, Py_BuildValue("H", G(info)[group->first_glyph_idx + g].codepoint));
         PyList_Append(ans, Py_BuildValue("IIHN", group->num_cells, group->num_glyphs, first_glyph, eg));
         idx++;
     }
@@ -1294,7 +1313,7 @@ set_font_data(PyObject UNUSED *m, PyObject *args) {
     if (!PyArg_ParseTuple(args, "OOOIIIIO!dO",
                 &box_drawing_function, &prerender_function, &descriptor_for_idx,
                 &descriptor_indices.bold, &descriptor_indices.italic, &descriptor_indices.bi, &descriptor_indices.num_symbol_fonts,
-                &PyTuple_Type, &sm, &global_state.font_sz_in_pts, &font_feature_settings)) return NULL;
+                &PyTuple_Type, &sm, &OPT(font_size), &font_feature_settings)) return NULL;
     Py_INCREF(box_drawing_function); Py_INCREF(prerender_function); Py_INCREF(descriptor_for_idx); Py_INCREF(font_feature_settings);
     free_font_groups();
     clear_symbol_maps();
@@ -1315,8 +1334,8 @@ send_prerendered_sprites(FontGroup *fg) {
     int error = 0;
     sprite_index x = 0, y = 0, z = 0;
     // blank cell
-    clear_canvas(fg);
-    current_send_sprite_to_gpu((FONTS_DATA_HANDLE)fg, x, y, z, fg->canvas);
+    ensure_canvas_can_fit(fg, 1);
+    current_send_sprite_to_gpu((FONTS_DATA_HANDLE)fg, x, y, z, fg->canvas.buf);
     do_increment(fg, &error);
     if (error != 0) { sprite_map_set_error(error); PyErr_Print(); fatal("Failed"); }
     PyObject *args = PyObject_CallFunction(prerender_function, "IIIIIIIffdd", fg->cell_width, fg->cell_height, fg->baseline, fg->underline_position, fg->underline_thickness, fg->strikethrough_position, fg->strikethrough_thickness, OPT(cursor_beam_thickness), OPT(cursor_underline_thickness), fg->logical_dpi_x, fg->logical_dpi_y);
@@ -1327,10 +1346,10 @@ send_prerendered_sprites(FontGroup *fg) {
         do_increment(fg, &error);
         if (error != 0) { sprite_map_set_error(error); PyErr_Print(); fatal("Failed"); }
         uint8_t *alpha_mask = PyLong_AsVoidPtr(PyTuple_GET_ITEM(args, i));
-        clear_canvas(fg);
+        ensure_canvas_can_fit(fg, 1);  // clear canvas
         Region r = { .right = fg->cell_width, .bottom = fg->cell_height };
-        render_alpha_mask(alpha_mask, fg->canvas, &r, &r, fg->cell_width, fg->cell_width);
-        current_send_sprite_to_gpu((FONTS_DATA_HANDLE)fg, x, y, z, fg->canvas);
+        render_alpha_mask(alpha_mask, fg->canvas.buf, &r, &r, fg->cell_width, fg->cell_width);
+        current_send_sprite_to_gpu((FONTS_DATA_HANDLE)fg, x, y, z, fg->canvas.buf);
     }
     Py_CLEAR(args);
 }
@@ -1399,8 +1418,12 @@ finalize(void) {
     Py_CLEAR(descriptor_for_idx);
     Py_CLEAR(font_feature_settings);
     free_font_groups();
+    free(ligature_types);
     if (harfbuzz_buffer) { hb_buffer_destroy(harfbuzz_buffer); harfbuzz_buffer = NULL; }
     free(group_state.groups); group_state.groups = NULL; group_state.groups_capacity = 0;
+    free(global_glyph_render_scratch.glyphs);
+    free(global_glyph_render_scratch.sprite_positions);
+    global_glyph_render_scratch = (GlyphRenderScratch){0};
 }
 
 static PyObject*
@@ -1414,13 +1437,19 @@ sprite_map_set_layout(PyObject UNUSED *self, PyObject *args) {
 
 static PyObject*
 test_sprite_position_for(PyObject UNUSED *self, PyObject *args) {
-    glyph_index glyph;
-    ExtraGlyphs extra_glyphs = {{0}};
-    if (!PyArg_ParseTuple(args, "H|H", &glyph, &extra_glyphs.data)) return NULL;
     int error;
+    FREE_AFTER_FUNCTION glyph_index *glyphs = calloc(PyTuple_GET_SIZE(args), sizeof(glyph_index));
+    for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(args); i++) {
+        if (!PyLong_Check(PyTuple_GET_ITEM(args, i))) {
+            PyErr_SetString(PyExc_TypeError, "glyph indices must be integers");
+            return NULL;
+        }
+        glyphs[i] = (glyph_index)PyLong_AsUnsignedLong(PyTuple_GET_ITEM(args, i));
+        if (PyErr_Occurred()) return NULL;
+    }
     FontGroup *fg = font_groups;
     if (!num_font_groups) { PyErr_SetString(PyExc_RuntimeError, "must create font group first"); return NULL; }
-    SpritePosition *pos = sprite_position_for(fg, &fg->fonts[fg->medium_font_idx], glyph, &extra_glyphs, 0, &error);
+    SpritePosition *pos = sprite_position_for(fg, &fg->fonts[fg->medium_font_idx], glyphs, PyTuple_GET_SIZE(args), 0, 1, &error);
     if (pos == NULL) { sprite_map_set_error(error); return NULL; }
     return Py_BuildValue("HHH", pos->x, pos->y, pos->z);
 }

kitty/fonts.h

@@ -19,7 +19,6 @@ typedef struct {
 } StringCanvas;
 
 // API that font backends need to implement
-typedef uint16_t glyph_index;
 unsigned int glyph_id_for_codepoint(PyObject *, char_type);
 int get_glyph_width(PyObject *, glyph_index);
 bool is_glyph_empty(PyObject *, glyph_index);
@@ -34,7 +33,7 @@ PyObject* face_from_descriptor(PyObject*, FONTS_DATA_HANDLE);
 const char* postscript_name_for_face(const PyObject*);
 
 void sprite_tracker_current_layout(FONTS_DATA_HANDLE data, unsigned int *x, unsigned int *y, unsigned int *z);
-void render_alpha_mask(uint8_t *alpha_mask, pixel* dest, Region *src_rect, Region *dest_rect, size_t src_stride, size_t dest_stride);
+void render_alpha_mask(const uint8_t *alpha_mask, pixel* dest, Region *src_rect, Region *dest_rect, size_t src_stride, size_t dest_stride);
 void render_line(FONTS_DATA_HANDLE, Line *line, index_type lnum, Cursor *cursor, DisableLigature);
 void sprite_tracker_set_limits(size_t max_texture_size, size_t max_array_len);
 typedef void (*free_extra_data_func)(void*);

kitty/fonts/__init__.py

@@ -11,9 +11,13 @@ class ListedFont(TypedDict):
     is_monospace: bool
 
 
-class FontFeature(str):
+class FontFeature:
 
-    def __new__(cls, name: str, parsed: bytes) -> 'FontFeature':
-        ans: FontFeature = str.__new__(cls, name)
-        ans.parsed = parsed  # type: ignore
-        return ans
+    __slots__ = 'name', 'parsed'
+
+    def __init__(self, name: str, parsed: bytes):
+        self.name = name
+        self.parsed = parsed
+
+    def __repr__(self) -> str:
+        return repr(self.name)

kitty/fonts/core_text.py

@@ -6,7 +6,8 @@ import re
 from typing import Dict, Generator, Iterable, List, Optional, Tuple
 
 from kitty.fast_data_types import coretext_all_fonts
-from kitty.options_stub import Options
+from kitty.fonts import FontFeature
+from kitty.options.types import Options
 from kitty.typing import CoreTextFont
 from kitty.utils import log_error
 
@@ -50,7 +51,7 @@ def list_fonts() -> Generator[ListedFont, None, None]:
             yield {'family': f, 'full_name': fn, 'postscript_name': fd['postscript_name'] or '', 'is_monospace': is_mono}
 
 
-def find_font_features(postscript_name: str) -> Tuple[str, ...]:
+def find_font_features(postscript_name: str) -> Tuple[FontFeature, ...]:
     """Not Implemented"""
     return ()
 

kitty/fonts/fontconfig.py

@@ -11,7 +11,7 @@ from kitty.fast_data_types import (
     FC_WEIGHT_REGULAR, FC_WIDTH_NORMAL, fc_list, fc_match as fc_match_impl,
     fc_match_postscript_name, parse_font_feature
 )
-from kitty.options_stub import Options
+from kitty.options.types import Options
 from kitty.typing import FontConfigPattern
 from kitty.utils import log_error
 
@@ -45,7 +45,9 @@ def all_fonts_map(monospaced: bool = True) -> FontMap:
     if monospaced:
         ans = fc_list(FC_DUAL) + fc_list(FC_MONO)
     else:
-        ans = fc_list()
+        # allow non-monospaced and bitmapped fonts as these are used for
+        # symbol_map
+        ans = fc_list(-1, True)
     return create_font_map(ans)
 
 
@@ -71,7 +73,7 @@ def fc_match(family: str, bold: bool, italic: bool, spacing: int = FC_MONO) -> F
     return fc_match_impl(family, bold, italic, spacing)
 
 
-def find_font_features(postscript_name: str) -> Tuple[str, ...]:
+def find_font_features(postscript_name: str) -> Tuple[FontFeature, ...]:
     pat = fc_match_postscript_name(postscript_name)
 
     if pat.get('postscript_name') != postscript_name or 'fontfeatures' not in pat:

kitty/fonts/render.py

@@ -10,7 +10,6 @@ from typing import (
     Any, Callable, Dict, Generator, List, Optional, Tuple, Union, cast
 )
 
-from kitty.config import defaults
 from kitty.constants import is_macos
 from kitty.fast_data_types import (
     Screen, create_test_font_group, get_fallback_font, set_font_data,
@@ -20,7 +19,7 @@ from kitty.fast_data_types import (
 from kitty.fonts.box_drawing import (
     BufType, render_box_char, render_missing_glyph
 )
-from kitty.options_stub import Options as OptionsStub
+from kitty.options.types import Options, defaults
 from kitty.typing import CoreTextFont, FontConfigPattern
 from kitty.utils import log_error
 
@@ -33,7 +32,7 @@ FontObject = Union[CoreTextFont, FontConfigPattern]
 current_faces: List[Tuple[FontObject, bool, bool]] = []
 
 
-def get_font_files(opts: OptionsStub) -> Dict[str, Any]:
+def get_font_files(opts: Options) -> Dict[str, Any]:
     if is_macos:
         return get_font_files_coretext(opts)
     return get_font_files_fontconfig(opts)
@@ -136,7 +135,7 @@ def coalesce_symbol_maps(maps: Dict[Tuple[int, int], str]) -> Dict[Tuple[int, in
     return dict(ans)
 
 
-def create_symbol_map(opts: OptionsStub) -> Tuple[Tuple[int, int, int], ...]:
+def create_symbol_map(opts: Options) -> Tuple[Tuple[int, int, int], ...]:
     val = coalesce_symbol_maps(opts.symbol_map)
     family_map: Dict[str, int] = {}
     count = 0
@@ -174,7 +173,7 @@ def dump_faces(ftypes: List[str], indices: Dict[str, int]) -> None:
             log_error(face_str(face))
 
 
-def set_font_family(opts: Optional[OptionsStub] = None, override_font_size: Optional[float] = None, debug_font_matching: bool = False) -> None:
+def set_font_family(opts: Optional[Options] = None, override_font_size: Optional[float] = None, debug_font_matching: bool = False) -> None:
     global current_faces
     opts = opts or defaults
     sz = override_font_size or opts.font_size
@@ -236,7 +235,15 @@ def add_dline(buf: ctypes.Array, cell_width: int, position: int, thickness: int,
 def add_curl(buf: ctypes.Array, cell_width: int, position: int, thickness: int, cell_height: int) -> None:
     max_x, max_y = cell_width - 1, cell_height - 1
     xfactor = 2.0 * pi / max_x
-    half_height = max(thickness // 2, 1)
+    thickness = max(1, thickness)
+    if thickness < 3:
+        half_height = thickness
+        thickness -= 1
+    elif thickness == 3:
+        half_height = thickness = 2
+    else:
+        half_height = thickness // 2
+        thickness -= 2
 
     def add_intensity(x: int, y: int, val: int) -> None:
         y += position
@@ -244,20 +251,22 @@ def add_curl(buf: ctypes.Array, cell_width: int, position: int, thickness: int,
         idx = cell_width * y + x
         buf[idx] = min(255, buf[idx] + val)
 
-    # Ensure all space at bottom of cell is used
-    if position + half_height < max_y:
-        position += max_y - (position + half_height)
+    # Ensure curve doesn't exceed cell boundary at the bottom
+    position += half_height * 2
     if position + half_height > max_y:
-        position -= position + half_height - max_y
+        position = max_y - half_height
 
     # Use the Wu antialias algorithm to draw the curve
     # cosine waves always have slope <= 1 so are never steep
     for x in range(cell_width):
         y = half_height * cos(x * xfactor)
-        y1, y2 = floor(y), ceil(y)
-        i1 = int(255 * abs(y - y1))
-        add_intensity(x, y1, 255 - i1)
-        add_intensity(x, y2, i1)
+        y1, y2 = floor(y - thickness), ceil(y)
+        i1 = int(255 * abs(y - floor(y)))
+        add_intensity(x, y1, 255 - i1)  # upper bound
+        add_intensity(x, y2, i1)  # lower bound
+        # fill between upper and lower bound
+        for t in range(1, thickness + 1):
+            add_intensity(x, y1 + t, 255)
 
 
 def render_special(
@@ -269,7 +278,9 @@ def render_special(
     underline_position: int = 0,
     underline_thickness: int = 0,
     strikethrough_position: int = 0,
-    strikethrough_thickness: int = 0
+    strikethrough_thickness: int = 0,
+    dpi_x: float = 96.,
+    dpi_y: float = 96.,
 ) -> ctypes.Array:
     underline_position = min(underline_position, cell_height - underline_thickness)
     CharTexture = ctypes.c_ubyte * (cell_width * cell_height)
@@ -356,7 +367,9 @@ def prerender_function(
     f = partial(
         render_special, cell_width=cell_width, cell_height=cell_height, baseline=baseline,
         underline_position=underline_position, underline_thickness=underline_thickness,
-        strikethrough_position=strikethrough_position, strikethrough_thickness=strikethrough_thickness)
+        strikethrough_position=strikethrough_position, strikethrough_thickness=strikethrough_thickness,
+        dpi_x=dpi_x, dpi_y=dpi_y
+    )
     c = partial(
         render_cursor, cursor_beam_thickness=cursor_beam_thickness,
         cursor_underline_thickness=cursor_underline_thickness, cell_width=cell_width,

kitty/freetype.c

@@ -302,11 +302,22 @@ calc_cell_width(Face *self) {
     return ans;
 }
 
+
+static unsigned int
+adjust_ypos(unsigned int pos, unsigned int cell_height, int adjustment) {
+    if (adjustment >= 0) adjustment = MIN(adjustment, (int)pos - 1);
+    else adjustment = MAX(adjustment, (int)pos - (int)cell_height + 1);
+    return pos - adjustment;
+}
+
 void
 cell_metrics(PyObject *s, unsigned int* cell_width, unsigned int* cell_height, unsigned int* baseline, unsigned int* underline_position, unsigned int* underline_thickness, unsigned int* strikethrough_position, unsigned int* strikethrough_thickness) {
     Face *self = (Face*)s;
     *cell_width = calc_cell_width(self);
     *cell_height = calc_cell_height(self, true);
+    int baseline_offset = 0;
+    if (OPT(adjust_baseline_px) != 0) baseline_offset = OPT(adjust_baseline_px);
+    else if (OPT(adjust_baseline_frac) != 0) baseline_offset = (int)(*cell_height * OPT(adjust_baseline_frac));
     *baseline = font_units_to_pixels_y(self, self->ascender);
     *underline_position = MIN(*cell_height - 1, (unsigned int)font_units_to_pixels_y(self, MAX(0, self->ascender - self->underline_position)));
     *underline_thickness = MAX(1, font_units_to_pixels_y(self, self->underline_thickness));
@@ -321,6 +332,11 @@ cell_metrics(PyObject *s, unsigned int* cell_width, unsigned int* cell_height, u
     } else {
       *strikethrough_thickness = *underline_thickness;
     }
+    if (baseline_offset) {
+        *baseline = adjust_ypos(*baseline, *cell_height, baseline_offset);
+        *underline_position = adjust_ypos(*underline_position, *cell_height, baseline_offset);
+        *strikethrough_position = adjust_ypos(*strikethrough_position, *cell_height, baseline_offset);
+    }
 }
 
 unsigned int
@@ -576,7 +592,7 @@ place_bitmap_in_canvas(pixel *cell, ProcessedBitmap *bm, size_t cell_width, size
         dest.top = baseline - yoff;
     }
 
-    /* printf("x_offset: %d bearing_x: %f y_offset: %d bearing_y: %f src_start_row: %u src_start_column: %u dest_start_row: %u dest_start_column: %u bm_width: %lu bitmap_rows: %lu\n", xoff, bearing_x, yoff, bearing_y, src.top, src.left, dest.top, dest.left, bm->width, bm->rows); */
+    /* printf("x_offset: %d y_offset: %d src_start_row: %u src_start_column: %u dest_start_row: %u dest_start_column: %u bm_width: %lu bitmap_rows: %lu\n", xoff, yoff, src.top, src.left, dest.top, dest.left, bm->width, bm->rows); */
 
     if (bm->pixel_mode == FT_PIXEL_MODE_BGRA) {
         copy_color_bitmap(bm->buf, cell, &src, &dest, bm->stride, cell_width);

kitty/glfw-wrapper.h

@@ -261,18 +261,20 @@ typedef enum {
  *
  *  @ingroup input
  *  @{ */
-#define GLFW_MOUSE_BUTTON_1         0
-#define GLFW_MOUSE_BUTTON_2         1
-#define GLFW_MOUSE_BUTTON_3         2
-#define GLFW_MOUSE_BUTTON_4         3
-#define GLFW_MOUSE_BUTTON_5         4
-#define GLFW_MOUSE_BUTTON_6         5
-#define GLFW_MOUSE_BUTTON_7         6
-#define GLFW_MOUSE_BUTTON_8         7
-#define GLFW_MOUSE_BUTTON_LAST      GLFW_MOUSE_BUTTON_8
-#define GLFW_MOUSE_BUTTON_LEFT      GLFW_MOUSE_BUTTON_1
-#define GLFW_MOUSE_BUTTON_RIGHT     GLFW_MOUSE_BUTTON_2
-#define GLFW_MOUSE_BUTTON_MIDDLE    GLFW_MOUSE_BUTTON_3
+typedef enum GLFWMouseButton {
+    GLFW_MOUSE_BUTTON_1 = 0,
+    GLFW_MOUSE_BUTTON_LEFT = 0,
+    GLFW_MOUSE_BUTTON_2 = 1,
+    GLFW_MOUSE_BUTTON_RIGHT = 1,
+    GLFW_MOUSE_BUTTON_3 = 2,
+    GLFW_MOUSE_BUTTON_MIDDLE = 2,
+    GLFW_MOUSE_BUTTON_4 = 3,
+    GLFW_MOUSE_BUTTON_5 = 4,
+    GLFW_MOUSE_BUTTON_6 = 5,
+    GLFW_MOUSE_BUTTON_7 = 6,
+    GLFW_MOUSE_BUTTON_8 = 7,
+    GLFW_MOUSE_BUTTON_LAST = 7
+} GLFWMouseButton;
 /*! @} */
 
 /*! @defgroup joysticks Joysticks

kitty/glfw.c

@@ -32,6 +32,18 @@ static GLFWcursor *standard_cursor = NULL, *click_cursor = NULL, *arrow_cursor =
 static void set_os_window_dpi(OSWindow *w);
 
 
+void
+get_platform_dependent_config_values(void *glfw_window) {
+    if (OPT(click_interval) < 0) OPT(click_interval) = glfwGetDoubleClickInterval(glfw_window);
+    if (OPT(cursor_blink_interval) < 0) {
+        OPT(cursor_blink_interval) = ms_to_monotonic_t(500ll);
+#ifdef __APPLE__
+        monotonic_t cbi = cocoa_cursor_blink_interval();
+        if (cbi >= 0) OPT(cursor_blink_interval) = cbi / 2;
+#endif
+    }
+}
+
 void
 request_tick_callback(void) {
     glfwPostEmptyEvent();
@@ -653,7 +665,7 @@ create_os_window(PyObject UNUSED *self, PyObject *args) {
     float xscale, yscale;
     double xdpi, ydpi;
     get_window_content_scale(temp_window, &xscale, &yscale, &xdpi, &ydpi);
-    FONTS_DATA_HANDLE fonts_data = load_fonts_data(global_state.font_sz_in_pts, xdpi, ydpi);
+    FONTS_DATA_HANDLE fonts_data = load_fonts_data(OPT(font_size), xdpi, ydpi);
     PyObject *ret = PyObject_CallFunction(get_window_size, "IIddff", fonts_data->cell_width, fonts_data->cell_height, fonts_data->logical_dpi_x, fonts_data->logical_dpi_y, xscale, yscale);
     if (ret == NULL) return NULL;
     int width = PyLong_AsLong(PyTuple_GET_ITEM(ret, 0)), height = PyLong_AsLong(PyTuple_GET_ITEM(ret, 1));
@@ -693,7 +705,7 @@ create_os_window(PyObject UNUSED *self, PyObject *args) {
         if (n_xdpi != xdpi || n_ydpi != ydpi) {
             // this can happen if the window is moved by the OS to a different monitor when shown
             xdpi = n_xdpi; ydpi = n_ydpi;
-            fonts_data = load_fonts_data(global_state.font_sz_in_pts, xdpi, ydpi);
+            fonts_data = load_fonts_data(OPT(font_size), xdpi, ydpi);
         }
 #endif
     }
@@ -706,16 +718,9 @@ create_os_window(PyObject UNUSED *self, PyObject *args) {
         dest##_cursor = glfwCreateStandardCursor(GLFW_##shape##_CURSOR); \
         if (dest##_cursor == NULL) { log_error("Failed to create the %s mouse cursor, using default cursor.", #shape); } \
 }}
-    CC(standard, IBEAM); CC(click, HAND); CC(arrow, ARROW);
+        CC(standard, IBEAM); CC(click, HAND); CC(arrow, ARROW);
 #undef CC
-        if (OPT(click_interval) < 0) OPT(click_interval) = glfwGetDoubleClickInterval(glfw_window);
-        if (OPT(cursor_blink_interval) < 0) {
-            OPT(cursor_blink_interval) = ms_to_monotonic_t(500ll);
-#ifdef __APPLE__
-            monotonic_t cbi = cocoa_cursor_blink_interval();
-            if (cbi >= 0) OPT(cursor_blink_interval) = cbi / 2;
-#endif
-        }
+        get_platform_dependent_config_values(glfw_window);
         is_first_window = false;
     }
     OSWindow *w = add_os_window();
@@ -730,7 +735,7 @@ create_os_window(PyObject UNUSED *self, PyObject *args) {
     w->fonts_data = fonts_data;
     w->shown_once = true;
     w->last_focused_counter = ++focus_counter;
-    if (OPT(resize_in_steps)) os_window_update_size_increments(w);
+    os_window_update_size_increments(w);
 #ifdef __APPLE__
     if (OPT(macos_option_as_alt)) glfwSetCocoaTextInputFilter(glfw_window, filter_option);
     glfwSetCocoaToggleFullscreenIntercept(glfw_window, intercept_cocoa_fullscreen);
@@ -783,8 +788,13 @@ create_os_window(PyObject UNUSED *self, PyObject *args) {
 
 void
 os_window_update_size_increments(OSWindow *window) {
-    if (window->handle && window->fonts_data) glfwSetWindowSizeIncrements(
-            window->handle, window->fonts_data->cell_width, window->fonts_data->cell_height);
+    if (OPT(resize_in_steps)) {
+        if (window->handle && window->fonts_data) glfwSetWindowSizeIncrements(
+                window->handle, window->fonts_data->cell_width, window->fonts_data->cell_height);
+    } else {
+        if (window->handle) glfwSetWindowSizeIncrements(
+                window->handle, GLFW_DONT_CARE, GLFW_DONT_CARE);
+    }
 }
 
 #ifdef __APPLE__

kitty/glyph-cache.c

@@ -0,0 +1,91 @@
+/*
+ * glyph-cache.c
+ * Copyright (C) 2021 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "glyph-cache.h"
+#include "kitty-uthash.h"
+
+
+typedef struct SpritePosItem {
+    SpritePositionHead
+    UT_hash_handle hh;
+    glyph_index key[];
+} SpritePosItem;
+
+static glyph_index *scratch = NULL;
+static unsigned scratch_sz = 0;
+
+
+void
+free_glyph_cache_global_resources(void) {
+    free(scratch);
+    scratch = NULL; scratch_sz = 0;
+}
+
+
+static unsigned
+key_size_for_glyph_count(unsigned count) { return count + 3; }
+
+
+SpritePosition*
+find_or_create_sprite_position(SpritePosition **head_, glyph_index *glyphs, glyph_index count, glyph_index ligature_index, glyph_index cell_count, bool *created) {
+    SpritePosItem **head = (SpritePosItem**)head_, *p;
+    const unsigned key_sz = key_size_for_glyph_count(count);
+    if (key_sz > scratch_sz) {
+        scratch = realloc(scratch, sizeof(glyph_index) * (key_sz + 16));
+        if (!scratch) return NULL;
+        scratch_sz = key_sz + 16;
+    }
+    const unsigned key_sz_bytes = key_sz * sizeof(glyph_index);
+    scratch[0] = count; scratch[1] = ligature_index; scratch[2] = cell_count;
+    memcpy(scratch + 3, glyphs, count * sizeof(glyph_index));
+    HASH_FIND(hh, *head, scratch, key_sz_bytes, p);
+    if (p) { *created = false; return (SpritePosition*)p; }
+
+    p = calloc(1, sizeof(SpritePosItem) + key_sz_bytes);
+    if (!p) return NULL;
+    memcpy(p->key, scratch, key_sz_bytes);
+    HASH_ADD(hh, *head, key, key_sz_bytes, p);
+    *created = true;
+    return (SpritePosition*)p;
+}
+
+void
+free_sprite_position_hash_table(SpritePosition **head_) {
+    SpritePosItem **head = (SpritePosItem**)head_, *s, *tmp;
+    HASH_ITER(hh, *head, s, tmp) {
+        HASH_DEL(*head, s);
+        free(s);
+    }
+}
+
+typedef struct GlyphPropertiesItem {
+    GlyphPropertiesHead
+    UT_hash_handle hh;
+    unsigned key;
+} GlyphPropertiesItem;
+
+
+GlyphProperties*
+find_or_create_glyph_properties(GlyphProperties **head_, unsigned glyph) {
+    GlyphPropertiesItem **head = (GlyphPropertiesItem**)head_, *p;
+    HASH_FIND_INT(*head, &glyph, p);
+    if (p) return (GlyphProperties*)p;
+    p = calloc(1, sizeof(GlyphPropertiesItem));
+    if (!p) return NULL;
+    p->key = glyph;
+    HASH_ADD_INT(*head, key, p);
+    return (GlyphProperties*)p;
+}
+
+void
+free_glyph_properties_hash_table(GlyphProperties **head_) {
+    GlyphPropertiesItem **head = (GlyphPropertiesItem**)head_, *s, *tmp;
+    HASH_ITER(hh, *head, s, tmp) {
+        HASH_DEL(*head, s);
+        free(s);
+    }
+}

kitty/glyph-cache.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2021 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#pragma once
+
+#include "data-types.h"
+
+void free_glyph_cache_global_resources(void);
+
+#define SpritePositionHead \
+    bool rendered, colored; \
+    sprite_index x, y, z; \
+
+typedef struct SpritePosition {
+    SpritePositionHead
+} SpritePosition;
+
+
+void free_sprite_position_hash_table(SpritePosition **head);
+SpritePosition*
+find_or_create_sprite_position(SpritePosition **head, glyph_index *glyphs, glyph_index count, glyph_index ligature_index, glyph_index cell_count, bool *created);
+
+#define GlyphPropertiesHead \
+    uint8_t data;
+
+typedef struct GlyphProperties {
+    GlyphPropertiesHead
+} GlyphProperties;
+
+void free_glyph_properties_hash_table(GlyphProperties **head);
+GlyphProperties*
+find_or_create_glyph_properties(GlyphProperties **head, unsigned glyph);

kitty/graphics.c

@@ -306,7 +306,7 @@ png_path_to_bitmap(const char* path, uint8_t** data, unsigned int* width, unsign
     unsigned char *buf = malloc(capacity);
     if (!buf) { log_error("Out of memory reading PNG file at: %s", path); fclose(fp); return false; }
     while (!feof(fp)) {
-        if (pos - capacity < 1024) {
+        if (capacity - pos < 1024) {
             capacity *= 2;
             unsigned char *new_buf = realloc(buf, capacity);
             if (!new_buf) {

kitty/history.c

@@ -536,7 +536,8 @@ HistoryBuf *alloc_historybuf(unsigned int lines, unsigned int columns, unsigned
 
 #include "rewrap.h"
 
-void historybuf_rewrap(HistoryBuf *self, HistoryBuf *other, ANSIBuf *as_ansi_buf) {
+void
+historybuf_rewrap(HistoryBuf *self, HistoryBuf *other, ANSIBuf *as_ansi_buf) {
     while(other->num_segments < self->num_segments) add_segment(other);
     if (other->xnum == self->xnum && other->ynum == self->ynum) {
         // Fast path
@@ -551,9 +552,8 @@ void historybuf_rewrap(HistoryBuf *self, HistoryBuf *other, ANSIBuf *as_ansi_buf
     if (other->pagerhist && other->xnum != self->xnum && ringbuf_bytes_used(other->pagerhist->ringbuf))
         other->pagerhist->rewrap_needed = true;
     other->count = 0; other->start_of_data = 0;
-    index_type x = 0, y = 0;
     if (self->count > 0) {
-        rewrap_inner(self, other, self->count, NULL, &x, &y, as_ansi_buf);
+        rewrap_inner(self, other, self->count, NULL, NULL, as_ansi_buf);
         for (index_type i = 0; i < other->count; i++) *attrptr(other, (other->start_of_data + i) % other->ynum) |= TEXT_DIRTY_MASK;
     }
 }

kitty/hyperlink.c

@@ -6,14 +6,12 @@
  */
 
 #include "hyperlink.h"
-#include "uthash.h"
+#include "kitty-uthash.h"
 #include <string.h>
 
 #define MAX_KEY_LEN 2048
 #define MAX_ID_LEN 256
 #define MAX_ADDS_BEFORE_GC 256
-#undef uthash_fatal
-#define uthash_fatal(msg) fatal(msg)
 
 typedef struct {
     const char *key;

kitty/key_encoding.c

@@ -176,15 +176,20 @@ encode_function_key(const KeyEvent *ev, char *output) {
             case GLFW_FKEY_KP_BEGIN: SIMPLE("\x1bOE");
             case GLFW_FKEY_END: SIMPLE("\x1bOF");
             case GLFW_FKEY_HOME: SIMPLE("\x1bOH");
-            case GLFW_FKEY_F1: SIMPLE("\x1bOP");
-            case GLFW_FKEY_F2: SIMPLE("\x1bOQ");
-            case GLFW_FKEY_F3: SIMPLE("\x1bOR");
-            case GLFW_FKEY_F4: SIMPLE("\x1bOS");
             default: break;
         }
     }
     if (!ev->mods.value) {
         if (!ev->disambiguate && !ev->report_text && key_number == GLFW_FKEY_ESCAPE) SIMPLE("\x1b");
+        if (legacy_mode) {
+            switch(key_number) {
+                case GLFW_FKEY_F1: SIMPLE("\x1bOP");
+                case GLFW_FKEY_F2: SIMPLE("\x1bOQ");
+                case GLFW_FKEY_F3: SIMPLE("\x1bOR");
+                case GLFW_FKEY_F4: SIMPLE("\x1bOS");
+                default: break;
+            }
+        }
         if (!ev->report_text) {
             switch(key_number) {
                 case GLFW_FKEY_ENTER: SIMPLE("\r");

kitty/key_encoding.py

@@ -216,7 +216,7 @@ class KeyEvent(NamedTuple):
     num_lock: bool = False
 
     def matches(self, spec: Union[str, ParsedShortcut], types: int = EventType.PRESS | EventType.REPEAT) -> bool:
-        mods = self.mods & ~(NUM_LOCK | CAPS_LOCK)
+        mods = self.mods_without_locks
         if not self.type & types:
             return False
         if isinstance(spec, str):
@@ -228,6 +228,14 @@ class KeyEvent(NamedTuple):
             return True
         return False
 
+    @property
+    def mods_without_locks(self) -> int:
+        return self.mods & ~(NUM_LOCK | CAPS_LOCK)
+
+    @property
+    def has_mods(self) -> bool:
+        return bool(self.mods_without_locks)
+
     def as_window_system_event(self) -> WindowSystemKeyEvent:
         action = defines.GLFW_PRESS
         if self.type is EventType.REPEAT: