version 0.22.1

This matches the behavior of GNOME based clients and in any case sway
seems to get very confused if CSD is drawn.

Fixes #3888

.github/workflows/ci.yml

@@ -76,7 +76,7 @@ jobs:
                 python-version: 3.8
 
             - name: Install build-only deps
-              run: pip install flake8 mypy types-requests types-docutils sphinx
+              run: pip install -r docs/requirements.txt flake8 mypy types-requests types-docutils
 
             - name: Run pyflakes
               run: python -m flake8 --count .

CHANGELOG.rst

@@ -1 +1 @@
-See https://sw.kovidgoyal.net/kitty/changelog.html
+See https://sw.kovidgoyal.net/kitty/changelog/

CONTRIBUTING.md

@@ -10,9 +10,9 @@ config to reproduce the issue with).
 
 ### Contributing code
 
-Install [the dependencies](https://sw.kovidgoyal.net/kitty/build.html#dependencies)
+Install [the dependencies](https://sw.kovidgoyal.net/kitty/build/#dependencies)
 using your favorite package manager. Build and run kitty [from
-source](https://sw.kovidgoyal.net/kitty/build.html#install-and-run-from-source).
+source](https://sw.kovidgoyal.net/kitty/build/#install-and-run-from-source).
 
 Make a fork, submit your Pull Request. If it's a large/controversial change, open an issue
 beforehand to discuss it, so that you don't waste your time making a pull

INSTALL.md

@@ -1,3 +1,3 @@
-To build from source: <https://sw.kovidgoyal.net/kitty/build.html>
+To build from source: <https://sw.kovidgoyal.net/kitty/build/>
 
-Pre-built binaries: <https://sw.kovidgoyal.net/kitty/binary.html>
+Pre-built binaries: <https://sw.kovidgoyal.net/kitty/binary/>

Makefile

@@ -40,4 +40,11 @@ html:
 linkcheck:
 	$(MAKE) FAIL_WARN=$(FAIL_WARN) -C docs linkcheck
 
+website:
+	./publish.py --only website
+
 docs: man html
+
+
+develop-docs:
+	$(MAKE) -C docs develop-docs

README.asciidoc

@@ -4,7 +4,7 @@ See https://sw.kovidgoyal.net/kitty/[the kitty website].
 
 image:https://github.com/kovidgoyal/kitty/workflows/CI/badge.svg["Build status", link="https://github.com/kovidgoyal/kitty/actions?query=workflow%3ACI"]
 
-https://sw.kovidgoyal.net/kitty/faq.html[Frequently Asked Questions]
+https://sw.kovidgoyal.net/kitty/faq/[Frequently Asked Questions]
 
 To ask other questions about kitty usage, use either the https://github.com/kovidgoyal/kitty/discussions/[discussions on GitHub] or the
 https://www.reddit.com/r/KittyTerminal[Reddit community]

__main__.py

@@ -49,13 +49,22 @@ def complete(args: List[str]) -> None:
 def launch(args: List[str]) -> None:
     import runpy
     sys.argv = args[1:]
-    exe = args[1]
+    try:
+        exe = args[1]
+    except IndexError:
+        raise SystemExit(
+            'usage: kitty +launch script.py [arguments to be passed to script.py ...]\n\n'
+            'script.py will be run with full access to kitty code. If script.py is '
+            'prefixed with a : it will be searched for in PATH'
+        )
     if exe.startswith(':'):
         import shutil
         q = shutil.which(exe[1:])
         if not q:
-            raise SystemExit('{} not found in PATH'.format(args[1][1:]))
+            raise SystemExit(f'{exe[1:]} not found in PATH')
         exe = q
+    if not os.path.exists(exe):
+        raise SystemExit(f'{exe} does not exist')
     runpy.run_path(exe, run_name='__main__')
 
 
@@ -80,8 +89,14 @@ def edit_config_file(args: List[str]) -> None:
 
 
 def namespaced(args: List[str]) -> None:
-    func = namespaced_entry_points[args[1]]
-    func(args[1:])
+    try:
+        func = namespaced_entry_points[args[1]]
+    except KeyError:
+        pass
+    else:
+        func(args[1:])
+        return
+    raise SystemExit(f'{args[1]} is not a known entry point. Choices are: ' + ', '.join(namespaced_entry_points))
 
 
 entry_points = {

bypy/linux/__main__.py

@@ -88,6 +88,8 @@ def copy_libs(env):
     for x in binary_includes():
         dest = env.bin_dir if '/bin/' in x else env.lib_dir
         shutil.copy2(x, dest)
+        dest = os.path.join(dest, os.path.basename(x))
+        subprocess.check_call(['chrpath', '-d', dest])
 
 
 def copy_python(env):

docs/Makefile

@@ -2,7 +2,7 @@
 #
 
 # You can set these variables from the command line.
-SPHINXOPTS    = -j auto -T $(FAIL_WARN)
+SPHINXOPTS    = -n -q -j auto -T $(FAIL_WARN) $(OPTS)
 SPHINXBUILD   = sphinx-build
 SPHINXPROJ    = kitty
 SOURCEDIR     = .
@@ -18,3 +18,7 @@ help:
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
 	$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+
+develop-docs:
+	sphinx-autobuild --ignore "$(abspath $(SOURCEDIR))/generated/*" --watch ../kitty --watch ../kittens -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS)

docs/_static/custom.css

@@ -5,122 +5,15 @@
  * Distributed under terms of the MIT license.
  */
 
-.float-left-img {
-    float: left;
-    margin-right: 1em;
-    margin-bottom: 1em;
-}
-
-.float-right-img { float: right; margin-left: 1em; margin-bottom: 1em }
-
-.half-with-img { max-width: 50% }
-
-.fit-img { max-width: 95% }
-
-div.body p, div.body dd, div.body li, div.body blockquote {
-    text-align: justify;
-}
-
-div.body {
-    min-width: 200px;
-}
-
-pre {
-    white-space: pre-wrap;
-}
-
-pre.pre {
-    white-space: pre;
-}
-
-a[href], input[type="submit"] { cursor: pointer; }
-
-img[style] {
-    /* Used for :scale: images to have them render properly but still popup when clicked */
-    width: auto !important;
-    height: auto !important;
-}
-
-a {
-    text-decoration: none !important;
-    border-bottom: none !important;
-}
-
-body div.document {
-    margin-top: 1ex;
+.sidebar-logo {
+    max-height: 128px;
 }
 
 .major-features li {
-    margin-top: 0.75ex;
     margin-bottom: 0.75ex;
-}
-
-.support-form input[type=submit] {
-    border-radius: 6px;
-    box-shadow: rgb(255, 246, 175) 0px 1px 0px 0px;
-    background: linear-gradient(rgb(255, 236, 100) 5%, rgb(255, 171, 35) 100%) rgb(255, 236, 100);
-    border: 1px solid rgb(255, 170, 34);
-    display: inline-block;
-    color: rgb(51, 51, 51);
-    font-family: Arial;
-    font-size: 15px;
-    font-weight: bold;
-    padding: 6px 24px;
-    text-decoration: none;
-    text-shadow: rgb(255, 238, 102) 0px 1px 0px;
-}
-
-.support-form input[type=submit]:hover {
-    background: linear-gradient(rgb(255, 171, 35) 5%, rgb(255, 236, 100) 100%) rgb(255, 171, 35);
-}
-
-.support-form input[type=submit]:focus {
-    outline: 0;
-}
-
-
-div.sphinxsidebar {
-    font-size: inherit;
-    line-height: inherit;
-    max-height: 100%;
-    overflow-y: auto;
-}
-
-#sidebartoc li {
     margin-top: 0.75ex;
-    margin-bottom: 0.75ex;
 }
 
-#sidebartoc ul {
-    list-style: none !important;
-}
-
-#sidebartoc a[href]:hover {
-    color: red;
-}
-
-
-.green {
-    color: green;
-}
-
-.cyan {
-    color: blue;
-}
-
-.env {
+.sidebar-tree a.current {
     font-style: italic;
 }
-
-.italic {
-    font-style: italic;
-}
-
-.bold {
-    font-weight: bold;
-}
-
-.title {
-    font-size: larger;
-    font-weight: bold
-}

docs/_static/custom.js

@@ -0,0 +1,57 @@
+/* vim:fileencoding=utf-8
+ *
+ * Copyright (C) 2021 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPLv3 license
+ */
+
+(function() {
+"use strict";
+
+function get_sidebar_tree() {
+    return document.querySelector('.sidebar-tree');
+}
+
+function scroll_sidebar_node_into_view(a) {
+    var ss = get_sidebar_tree().closest('.sidebar-scroll');
+    if (!ss || !a) return;
+    ss.style.position = 'relative';
+    var pos = 0;
+    while (true) {
+        pos += a.offsetTop;
+        a = a.offsetParent;
+        if (!a || a == ss) break;
+    }
+    ss.scrollTop = pos;
+}
+
+function mark_current_link(sidebar_tree, a, onload) {
+    var li = a.closest('li.has-children');
+    while (li) {
+        li.querySelector('input[type=checkbox]').setAttribute('checked', 'checked');
+        li = li.parentNode.closest('li.has-children');
+    }
+    sidebar_tree.querySelectorAll('.current').forEach(function (elem) {
+        elem.classList.remove('current');
+    });
+    if (onload) scroll_sidebar_node_into_view(a);
+    a.classList.add('current');
+}
+
+function show_hash_in_sidebar(onload) {
+    var sidebar_tree = document.querySelector('.sidebar-tree');
+    if (document.location.hash.length > 1) {
+        var a = sidebar_tree.querySelector('a[href="' + document.location.hash + '"]');
+        if (a) mark_current_link(sidebar_tree, a, onload);
+    } else {
+        if (onload) scroll_sidebar_node_into_view(sidebar_tree.querySelector('.current-page a'));
+    }
+}
+
+document.addEventListener("DOMContentLoaded", function() {
+    show_hash_in_sidebar(true);
+    window.addEventListener('hashchange', show_hash_in_sidebar.bind(null, false));
+});
+
+}());
+

docs/_templates/base.html

@@ -0,0 +1,22 @@
+{% extends "!base.html" %}
+{% block extrahead %}
+
+  {{ super() }}
+
+  {%- if analytics_id %}
+      <script type="text/javascript">
+      var _gaq = _gaq || [];
+      _gaq.push(['_setAccount', '{{ analytics_id }}']);
+      _gaq.push(['_setDomainName', 'none']);
+      _gaq.push(['_setAllowLinker', true]);
+      _gaq.push(['_trackPageview']);
+
+      (function() {
+        var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+        ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+        var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+      })();
+    </script>
+  {% endif -%}
+
+{% endblock %}

docs/_templates/layout.html

@@ -1,6 +0,0 @@
-{% extends "!layout.html" %}
-
-{%- block extrahead %}
-<!-- kitty analytics placeholder -->
-  {{ super() }}
-{% endblock %}

docs/_templates/localtoc.html

@@ -1,6 +0,0 @@
-{%- if display_toc %}
-<div>&nbsp;</div>
-<div id="sidebartoc">
-  {{ toc }}
-</div>
-{%- endif %}

docs/_templates/searchbox.html

@@ -1,22 +0,0 @@
-{#
-    basic/searchbox.html
-    ~~~~~~~~~~~~~~~~~~~~
-
-    Sphinx sidebar template: quick search box.
-
-    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-#}
-{%- if pagename != "search" and builder != "singlehtml" %}
-<div id="searchbox" style="display: none" role="search">
-    <div class="searchformwrapper">
-    <form class="search" action="{{ pathto('search') }}" method="get">
-      <input type="text" name="q" placeholder="{{ _('Search') }}" />
-      <input type="submit" value="{{ _('Go') }}" style="cursor: pointer" />
-      <input type="hidden" name="check_keywords" value="yes" />
-      <input type="hidden" name="area" value="default" />
-    </form>
-    </div>
-</div>
-<script type="text/javascript">$('#searchbox').show(0);</script>
-{%- endif %}

docs/_templates/support.html

@@ -1,7 +0,0 @@
-{% if pagename != "support" %}
-<div id="support" style="text-align: center">
-    <form class="support-form" action="{{ pathto('support') }}" title="{{ _('Donate to support kitty development') }}">
-        <input type="submit" value="{{ _('Support kitty') }}">
-    </form>
-</div>
-{% endif %}

docs/actions.rst

@@ -0,0 +1,9 @@
+Mappable actions
+-----------------------
+
+.. highlight:: conf
+
+The actions described below can be mapped to any key press or mouse action
+using the ``map`` and ``mouse_map`` directives in :file:`kitty.conf`.
+
+.. include:: /generated/actions.rst

docs/basic.rst

@@ -83,6 +83,10 @@ move it to another tab or another OS window::
     map ctrl+f2 detach_window
     # moves the window into a new Tab
     map ctrl+f3 detach_window new-tab
+    # moves the window into the previously active tab
+    map ctrl+f3 detach_window tab-prev
+    # moves the window into the tab at the left of the active tab
+    map ctrl+f3 detach_window tab-left
     # asks which tab to move the window into
     map ctrl+f4 detach_window ask
 
@@ -102,6 +106,9 @@ the currently active window::
 Other keyboard shortcuts
 ----------------------------------
 
+The full list of actions that can be mapped to key presses is available
+:doc:`here </actions>`.
+
 ==================================  =======================
 Action                              Shortcut
 ==================================  =======================

docs/binary.rst

@@ -1,6 +1,9 @@
-kitty - Binary install
+Install kitty
 ========================
 
+Binary install
+----------------
+
 .. |ins| replace:: curl -L :literal:`https://sw.kovidgoyal.net/kitty/installer.sh` | sh /dev/stdin
 
 .. highlight:: sh

docs/build.rst

@@ -1,5 +1,5 @@
-Building kitty from source
-==============================
+Build from source
+==================
 
 .. image:: https://github.com/kovidgoyal/kitty/workflows/CI/badge.svg
   :alt: Build status
@@ -25,24 +25,27 @@ Dependencies
 
 Run-time dependencies:
 
-    * python >= 3.6
-    * harfbuzz >= 2.2.0
-    * zlib
-    * libpng
-    * liblcms2
-    * freetype (not needed on macOS)
-    * fontconfig (not needed on macOS)
-    * libcanberra (not needed on macOS)
-    * ImageMagick (optional, needed to use the ``kitty +kitten icat`` tool to display images in the terminal)
-    * pygments (optional, need for syntax highlighting in ``kitty +kitten diff``)
+* ``python`` >= 3.6
+* ``harfbuzz`` >= 2.2.0
+* ``zlib``
+* ``libpng``
+* ``liblcms2``
+* ``freetype`` (not needed on macOS)
+* ``fontconfig`` (not needed on macOS)
+* ``libcanberra`` (not needed on macOS)
+* ``ImageMagick`` (optional, needed to use the ``kitty +kitten icat`` tool to display images in the terminal)
+* ``pygments`` (optional, needed for syntax highlighting in ``kitty +kitten diff``)
+
 
 Build-time dependencies:
 
-    * gcc or clang
-    * pkg-config
-    * For building on Linux in addition to the above dependencies you might also need to install the ``-dev`` packages for:
-      ``libdbus-1-dev``, ``libxcursor-dev``, ``libxrandr-dev``, ``libxi-dev``, ``libxinerama-dev``, ``libgl1-mesa-dev``, ``libxkbcommon-x11-dev``, ``libfontconfig-dev``, ``libx11-xcb-dev``, ``liblcms2-dev``, and ``libpython3-dev``,
-      if they are not already installed by your distro.
+* ``gcc`` or ``clang``
+* ``pkg-config``
+* For building on Linux in addition to the above dependencies you might also need to install the ``-dev`` packages for:
+  ``libdbus-1-dev``, ``libxcursor-dev``, ``libxrandr-dev``, ``libxi-dev``, ``libxinerama-dev``,
+  ``libgl1-mesa-dev``, ``libxkbcommon-x11-dev``, ``libfontconfig-dev``, ``libx11-xcb-dev``,
+  ``liblcms2-dev``, and ``libpython3-dev``, if they are not already installed by your distro.
+
 
 Install and run from source
 ------------------------------
@@ -78,8 +81,8 @@ you might have to rebuild the app.
 .. note::
    The released :file:`kitty.dmg` includes all dependencies, unlike the
    :file:`kitty.app` built above and is built automatically by using the
-   <https://github.com/kovidgoyal/bypy>`_ however, that is designed to
-   run on Linux and is not for the faint of heart.
+   `bypy framework <https://github.com/kovidgoyal/bypy>`_ however, that is
+   designed to run on Linux and is not for the faint of heart.
 
 
 .. note::
@@ -128,18 +131,19 @@ This allows users to install the terminfo file on servers into which they ssh,
 without needing to install all of |kitty|.
 
 .. note::
-        You need a couple of extra dependencies to build linux-package.
-        :file:`tic` to compile terminfo files, usually found in the
-        development package of :file:`ncurses`. Also, if you are building from
-        a git checkout instead of the released source code tarball, you will
-        need :file:`sphinx-build` from the `Sphinx documentation generator
-        <https://www.sphinx-doc.org/>`_.
+   You need a couple of extra dependencies to build linux-package.
+   :file:`tic` to compile terminfo files, usually found in the
+   development package of :file:`ncurses`. Also, if you are building from
+   a git checkout instead of the released source code tarball, you will
+   need to install the dependencies from ``docs/requirements.txt`` to
+   build the kitty documentation. They can be installed most easily with
+   ``python -m pip -r docs/requirements.txt``.
 
 This applies to creating packages for |kitty| for macOS package managers such as
 brew or MacPorts as well.
 
 
 .. note::
-        |kitty| has its own update check mechanism, if you would like to turn
-        it off for your package, use
-        ``python3 setup.py linux-package --update-check-interval=0``
+   |kitty| has its own update check mechanism, if you would like to turn
+   it off for your package, use
+   ``python3 setup.py linux-package --update-check-interval=0``

docs/changelog.rst

@@ -4,6 +4,142 @@ Changelog
 |kitty| is a feature-rich, cross-platform, *fast*, GPU based terminal.
 To update |kitty|, :doc:`follow the instructions <binary>`.
 
+0.22.1 [2021-07-31]
+----------------------
+
+- Fix a regression in the previous release that broke ``kitty --help`` (:iss:`3869`)
+
+- Graphics protocol: Fix composing onto currently displayed frame not updating the frame on the GPU (:iss:`3874`)
+
+- Fix switching to previously active tab after detaching a tab not working (:pull:`3871`)
+
+- macOS: Fix an error on Apple silicon when enumerating monitors (:pull:`3875`)
+
+- detach_window: Allow specifying the previously active tab or the tab to the left/right of
+  the active tab (:disc:`3877`)
+
+- broadcast kitten: Fix a regression in ``0.20.0`` that broke sending of some
+  keys, such as backspace
+
+- Linux binary: Remove any RPATH build artifacts from bundled libraries
+
+- Wayland: If the compositor turns off server side decorations after turning
+  them on do not draw client side decorations (:iss:`3888`)
+
+
+0.22.0 [2021-07-26]
+----------------------
+
+- Add a new :ref:`action-toggle_layout` action to easily zoom/unzoom a window
+
+- When right clicking to extend a selection, move the nearest selection
+  boundary rather than the end of the selection. To restore previous behavior
+  use ``mouse_map right press ungrabbed mouse_selection move-end``.
+
+- When opening hyperlinks, allow defining open actions for directories
+  (:pull:`3836`)
+
+- When using the OSC 52 escape code to copy to clipboard allow large
+  copies (up to 8MB) without needing a kitty specific chunking protocol.
+  Note that if you used the chunking protocol in the past, it will no longer
+  work and you should switch to using the unmodified protocol which has the
+  advantage of working with all terminal emulators.
+
+- Fix a bug in the implementation of the synchronized updates escape code that
+  could cause incorrect parsing if either the pending buffer capacity or the
+  pending timeout were exceeded (:iss:`3779`)
+
+- A new remote control command to :program:`resize the OS Window <kitty @
+  resize-os-window>`
+
+- Graphics protocol: Add support for composing rectangles from one animation
+  frame onto another (:iss:`3809`)
+
+- diff kitten: Remove limit on max line length of 4096 characters (:iss:`3806`)
+
+- Fix turning off cursor blink via escape codes not working (:iss:`3808`)
+
+- Allow using neighboring window operations in the stack layout. The previous
+  window is considered the left and top neighbor and the next window is
+  considered the bottom and right neighbor (:iss:`3778`)
+
+- macOS: Render colors in the sRGB colorspace to match other macOS terminal
+  applications (:iss:`2249`)
+
+- Add a new variable ``{num_window_groups}`` for the :opt:`tab_title_template`
+  (:iss:`3837`)
+
+- Wayland: Fix :opt:`initial_window_width/height <remember_window_size>` specified
+  in cells not working on High DPI screens (:iss:`3834`)
+
+- A new theme for the kitty website with support for dark mode.
+
+- Render ┄ ┅ ┆ ┇ ┈ ┉ ┊ ┋ with spaces at the edges. Matches rendering in
+  most other programs and allows long chains of them to look better
+  (:iss:`3844`)
+
+- hints kitten: Detect paths and hashes that appear over multiple lines.
+  Note that this means that all line breaks in the text are no longer \n
+  soft breaks are instead \r. If you use a custom regular expression that
+  is meant to match over line breaks, you will need to match over both.
+  (:iss:`3845`)
+
+- Allow leading or trailing spaces in :opt:`tab_activity_symbol`
+
+- Fix mouse actions not working when caps lock or num lock are engaged
+  (:iss:`3859`)
+
+- macOS: Fix automatic detection of bold/italic faces for fonts that
+  use the family name as the full face name of the regular font not working
+  (:iss:`3861`)
+
+- clipboard kitten: fix copies to clipboard not working without the
+  :option:`kitty +kitten clipboard --wait-for-completion` option
+
+
+0.21.2 [2021-06-28]
+----------------------
+
+- A new :opt:`adjust_baseline` option to adjust the vertical alignment of text
+  inside a line (:pull:`3734`)
+
+- A new :opt:`url_excluded_characters` option to exclude additional characters
+  when detecting URLs under the mouse (:pull:`3738`)
+
+- Fix a regression in 0.21.0 that broke rendering of private use Unicode symbols followed
+  by spaces, when they also exist not followed by spaces (:iss:`3729`)
+
+- ssh kitten: Support systems where the login shell is a non-POSIX shell
+  (:iss:`3405`)
+
+- ssh kitten: Add completion (:iss:`3760`)
+
+- ssh kitten: Fix "Connection closed" message being printed by ssh when running
+  remote commands
+
+- Add support for the XTVERSION escape code
+
+- macOS: Fix a regression in 0.21.0 that broke middle-click to paste from clipboard (:iss:`3730`)
+
+- macOS: Fix shortcuts in the global menu bar responding slowly when cursor blink
+  is disabled/timed out (:iss:`3693`)
+
+- When displaying scrollback ensure that the window does not quit if the amount
+  of scrollback is less than a screen and the user has the ``--quit-if-one-screen``
+  option enabled for less (:iss:`3740`)
+
+- Linux: Fix Emoji/bitmapped fonts not use able in symbol_map
+
+- query terminal kitten: Allow querying font face and size information
+  (:iss:`3756`)
+
+- hyperlinked grep kitten: Fix context options not generating contextual output (:iss:`3759`)
+
+- Allow using superscripts in tab titles (:iss:`3763`)
+
+- Unicode input kitten: Fix searching when a word has more than 1024 matches (:iss:`3773`)
+
+
 0.21.1 [2021-06-14]
 ----------------------
 
@@ -27,7 +163,7 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
   :kbd:`ctrl+shift`
 
 - Allow remapping all mouse button press/release events to perform arbitrary
-  actions. :ref:`See details <conf-kitty-mouse.mousemap>`.
+  actions. :ref:`See details <conf-kitty-mouse.mousemap>` (:iss:`1033`)
 
 - Support infinite length ligatures (:iss:`3504`)
 
@@ -1639,8 +1775,7 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
   using standard keyboards) via `IBus
   <https://github.com/ibus/ibus/wiki/ReadMe>`_ (:iss:`469`)
 
-- Implement completion for the kitty command in bash and zsh. See
-  :ref:`completion`.
+- Implement completion for the kitty command in bash and zsh
 
 - Render the text under the cursor in a fixed color, configurable via
   the option :opt:`cursor_text_color` (:iss:`126`)

docs/color-stack.rst

@@ -0,0 +1,26 @@
+Saving and restoring colors
+==============================
+
+It is often useful for a full screen application with its own color themes to
+set the default foreground, background, selection and cursor colors and the
+ANSI color table. This allows for various performance optimizations when
+drawing the screen. The problem is that if the user previously used the escape
+codes to change these colors herself, then running the full screen application
+will lose her changes even after it exits. To avoid this, kitty introduces a
+new pair of *OSC* escape codes to push and pop the current color values from a
+stack::
+
+    <ESC>]30001<ESC>\  # push onto stack
+    <ESC>]30101<ESC>\  # pop from stack
+
+These escape codes save/restore the colors, default
+background, default foreground, selection background, selection foreground and
+cursor color and the 256 colors of the ANSI color table.
+
+.. note:: In July 2020, after several years, XTerm copied this protocol
+   extension, without acknowledgement, and using incompatible escape codes
+   (XTPUSHCOLORS, XTPOPCOLORS, XTREPORTCOLORS). And they decided to save not
+   just the dynamic colors but the entire ANSI color table. In the interests of
+   promoting interoperability, kitty added support for XTerm's escape codes as
+   well, and changed this extension to also save/restore the entire ANSI color
+   table.

docs/conf.py

@@ -22,7 +22,6 @@ from pygments.token import (  # type: ignore
     Comment, Keyword, Literal, Name, Number, String, Whitespace
 )
 from sphinx import addnodes, version_info  # type: ignore
-from sphinx.environment.adapters.toctree import TocTree  # type: ignore
 from sphinx.util.logging import getLogger  # type: ignore
 
 kitty_src = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -30,7 +29,7 @@ if kitty_src not in sys.path:
     sys.path.insert(0, kitty_src)
 
 from kitty.conf.types import Definition  # noqa
-from kitty.constants import str_version  # noqa
+from kitty.constants import str_version, website_url  # noqa
 
 # config {{{
 # -- Project information -----------------------------------------------------
@@ -60,8 +59,14 @@ extensions = [
     'sphinx.ext.ifconfig',
     'sphinx.ext.viewcode',
     'sphinx.ext.githubpages',
+    'sphinx_copybutton',
+    'sphinx_inline_tabs',
+    "sphinxext.opengraph",
 ]
 
+# URL for OpenGraph tags
+ogp_site_url = website_url()
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -85,13 +90,10 @@ language: Optional[str] = None
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path .
 exclude_patterns = [
-    '_build', 'Thumbs.db', '.DS_Store',
-    'generated/cli-*.rst', 'generated/conf-*.rst'
+    '_build', 'Thumbs.db', '.DS_Store', 'basic.rst',
+    'generated/cli-*.rst', 'generated/conf-*.rst', 'generated/actions.rst'
 ]
 
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
 rst_prolog = '''
 .. |kitty| replace:: *kitty*
 .. |version| replace:: VERSION
@@ -101,7 +103,6 @@ rst_prolog = '''
 .. role:: bold
 .. role:: cyan
 .. role:: title
-.. role:: env
 
 '''.replace('VERSION', str_version)
 
@@ -111,32 +112,26 @@ rst_prolog = '''
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = 'furo'
+html_title = 'kitty'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-html_theme_options = {
-    'logo': 'kitty.png',
-    'show_powered_by': False,
-    'fixed_sidebar': True,
-    'sidebar_collapse': True,
-    'github_button': False,
-    'github_banner': True,
-    'github_user': 'kovidgoyal',
-    'github_repo': 'kitty',
-    # increase contrast of link color with text color
-    'link': '#00587d',
-    'link_hover': 'green',
+html_theme_options: Dict[str, Any] = {
+    'sidebar_hide_name': True,
+    'navigation_with_keys': True,
 }
 
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static', '../logo/kitty.png']
-html_favicon = '../logo/kitty.png'
+html_static_path = ['_static']
+html_favicon = html_logo = '../logo/kitty.svg'
+html_css_files = ['custom.css']
+html_js_files = ['custom.js']
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -146,17 +141,9 @@ html_favicon = '../logo/kitty.png'
 # default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
 # 'searchbox.html']``.
 #
-html_sidebars = {
-    '**': [
-        'about.html',
-        'support.html',
-        'searchbox.html',
-        'localtoc.html',
-        'relations.html',
-    ]
-}
 html_show_sourcelink = False
-
+html_show_sphinx = False
+manpages_url = 'https://man7.org/linux/man-pages/man{section}/{page}.{section}.html'
 
 # -- Options for manual page output ------------------------------------------
 
@@ -220,28 +207,6 @@ def commit_role(name: str, rawtext: str, text: str, lineno: int, inliner: Any, o
 # }}}
 
 
-# Sidebar ToC {{{
-def create_toc(app: Any, pagename: str) -> Optional[Any]:
-    tt = TocTree(app.env)
-    toctree = tt.get_toc_for(pagename, app.builder)
-    if toctree is not None:
-        subtree = toctree[toctree.first_child_matching_class(nodes.list_item)]
-        bl = subtree.first_child_matching_class(nodes.bullet_list)
-        if bl is None:
-            return None  # Empty ToC
-        subtree = subtree[bl]
-        # for li in subtree.traverse(nodes.list_item):
-        #     modify_li(li)
-        # subtree['ids'] = [ID]
-        return app.builder.render_partial(subtree)['fragment']
-
-
-def add_html_context(app: Any, pagename: str, templatename: str, context: Any, *args: Any) -> None:
-    if 'toc' in context:
-        context['toc'] = create_toc(app, pagename) or context['toc']
-# }}}
-
-
 # CLI docs {{{
 def write_cli_docs(all_kitten_names: Iterable[str]) -> None:
     from kitty.cli import option_spec_as_rst
@@ -281,6 +246,11 @@ if you specify a program-to-run you can use the special placeholder
             with open(f'generated/cli-kitten-{kitten}.rst', 'w') as f:
                 p = partial(print, file=f)
                 p('.. program::', f'kitty +kitten {kitten}')
+                p(f'\nSource code for {kitten}')
+                p('-' * 72)
+                p(f'\nThe source code for this kitten is `available on GitHub <https://github.com/kovidgoyal/kitty/tree/master/kittens/{kitten}>`_.')
+                p('\nCommand Line Interface')
+                p('-' * 72, file=f)
                 p('\n\n' + option_spec_as_rst(
                     data['options'], message=data['help_text'], usage=data['usage'], appname=f'kitty +kitten {kitten}',
                     heading_char='^'))
@@ -347,6 +317,8 @@ class ConfLexer(RegexLexer):
             (r'(include)(\s+)(.+?)$', bygroups(Comment.Preproc, Whitespace, Name.Namespace)),
             (r'(map)(\s+)(\S+)(\s+)', bygroups(
                 Keyword.Declaration, Whitespace, String, Whitespace), 'action'),
+            (r'(mouse_map)(\s+)(\S+)(\s+)(\S+)(\s+)(\S+)(\s+)', bygroups(
+                Keyword.Declaration, Whitespace, String, Whitespace, Name.Variable, Whitespace, String, Whitespace), 'action'),
             (r'(symbol_map)(\s+)(\S+)(\s+)(.+?)$', bygroups(
                 Keyword.Declaration, Whitespace, String, Whitespace, Literal)),
             (r'([a-zA-Z_0-9]+)(\s+)', bygroups(
@@ -510,9 +482,27 @@ def write_conf_docs(app: Any, all_kitten_names: Iterable[str]) -> None:
         definition = get_kitten_conf_docs(kitten)
         if definition:
             generate_default_config(definition, f'kitten-{kitten}')
+
+    from kitty.actions import as_rst
+    with open('generated/actions.rst', 'w', encoding='utf-8') as f:
+        f.write(as_rst())
 # }}}
 
 
+def add_html_context(app: Any, pagename: str, templatename: str, context: Any, doctree: Any, *args: Any) -> None:
+    context['analytics_id'] = app.config.analytics_id
+    if 'toctree' in context:
+        # this is needed with furo to use all titles from pages
+        # in the sidebar (global) toc
+        original_toctee_function = context['toctree']
+
+        def include_sub_headings(**kwargs: Any) -> Any:
+            kwargs['titles_only'] = False
+            return original_toctee_function(**kwargs)
+
+        context['toctree'] = include_sub_headings
+
+
 def setup(app: Any) -> None:
     os.makedirs('generated/conf', exist_ok=True)
     from kittens.runner import all_kitten_names
@@ -520,10 +510,11 @@ def setup(app: Any) -> None:
     write_cli_docs(kn)
     write_remote_control_protocol_docs()
     write_conf_docs(app, kn)
-    app.add_css_file('custom.css')
+    app.add_config_value('analytics_id', '', 'env')
+    app.connect('html-page-context', add_html_context)
     app.add_lexer('session', SessionLexer() if version_info[0] < 3 else SessionLexer)
     app.add_role('link', link_role)
     app.add_role('iss', partial(num_role, 'issues'))
     app.add_role('pull', partial(num_role, 'pull'))
+    app.add_role('disc', partial(num_role, 'discussions'))
     app.add_role('commit', commit_role)
-    app.connect('html-page-context', add_html_context)

docs/conf.rst

@@ -1,7 +1,5 @@
-:tocdepth: 2
-
-Configuring kitty
-===============================
+kitty.conf
+-----------------------
 
 .. highlight:: conf
 
@@ -34,11 +32,15 @@ expanded, so :code:`${USER}.conf` becomes :file:`name.conf` if
      include other.conf
 
 
+.. note:: Syntax highlighting for :file:`kitty.conf` in vim is available via
+   `vim-kitty <https://github.com/fladson/vim-kitty>`_.
+
+
 .. include:: /generated/conf-kitty.rst
 
 
 Sample kitty.conf
-^^^^^^^^^^^^^^^^^^^^^
+--------------------
 
 .. only:: html
 
@@ -53,3 +55,14 @@ Sample kitty.conf
    file with full documentation and all settings commented out. If you
    have a pre-existing kitty.conf, then that will be used instead, delete
    it to see the sample file.
+
+
+All mappable actions
+------------------------
+
+See the :doc:`list of all the things you can make kitty can do </actions>`.
+
+.. toctree::
+   :hidden:
+
+   actions

docs/deccara.rst

@@ -0,0 +1,21 @@
+Setting text styles/colors in arbitrary regions of the screen
+------------------------------------------------------------------
+
+There already exists an escape code to set *some* text attributes in arbitrary
+regions of the screen, `DECCARA
+<https://vt100.net/docs/vt510-rm/DECCARA.html>`_.  However, it is limited to
+only a few attributes. |kitty| extends this to work with *all* SGR attributes.
+So, for example, this can be used to set the background color in an arbitrary
+region of the screen.
+
+The motivation for this extension is the various problems with the existing
+solution for erasing to background color, namely the *background color erase
+(bce)* capability. See
+`this discussion <https://github.com/kovidgoyal/kitty/issues/160#issuecomment-346470545>`_
+and `this FAQ <https://invisible-island.net/ncurses/ncurses.faq.html#bce_mismatches>`_
+for a summary of problems with *bce*.
+
+For example, to set the background color to blue in a
+rectangular region of the screen from (3, 4) to (10, 11), you use::
+
+    <ESC>[2*x<ESC>[4;3;11;10;44$r<ESC>[*x

docs/desktop-notifications.rst

@@ -0,0 +1,122 @@
+.. _desktop_notifications:
+
+
+Desktop notifications
+=======================
+
+|kitty| implements an extensible escape code (OSC 99) to show desktop
+notifications. It is easy to use from shell scripts and fully extensible to
+show title and body.  Clicking on the notification can optionally focus the
+window it came from, and/or send an escape code back to the application running
+in that window.
+
+The design of the escape code is partially based on the discussion in
+the defunct
+`terminal-wg <https://gitlab.freedesktop.org/terminal-wg/specifications/-/issues/13>`_
+
+The escape code has the form::
+
+    <OSC> 99 ; metadata ; payload <terminator>
+
+Here ``<OSC>`` is :code:`<ESC>]` and ``<terminator>`` is
+:code:`<ESC><backslash>`.  The metadata is a section of colon separated
+:code:`key=value` pairs. Every key must be a single character from the set
+:code:`a-zA-Z` and every value must be a word consisting of characters from
+the set :code:`a-zA-Z0-9-_/\+.,(){}[]*&^%$#@!`~`. The payload must be
+interpreted based on the metadata section. The two semi-colons *must* always be
+present even when no metadata is present.
+
+Before going into details, lets see how one can display a simple, single line
+notification from a shell script::
+
+    printf '\x1b]99;;Hello world\x1b\\'
+
+To show a message with a title and a body::
+
+    printf '\x1b]99;i=1:d=0;Hello world\x1b\\'
+    printf '\x1b]99;i=1:d=1:p=body;This is cool\x1b\\'
+
+The most important key in the metadata is the ``p`` key, it controls how the
+payload is interpreted. A value of ``title`` means the payload is setting the
+title for the notification. A value of ``body`` means it is setting the body,
+and so on, see the table below for full details.
+
+The design of the escape code is fundamentally chunked, this is because
+different terminal emulators have different limits on how large a single escape
+code can be. Chunking is accomplished by the ``i`` and ``d`` keys. The ``i``
+key is the *notification id* which can be any string containing the characters
+``[a-zA-Z0-9_-+.]``. The ``d`` key stands for *done* and
+can only take the values ``0`` and ``1``. A value of ``0`` means the
+notification is not yet done and the terminal emulator should hold off
+displaying it. A value of ``1`` means the notification is done, and should be
+displayed. You can specify the title or body multiple times and the terminal
+emulator will concatenate them, thereby allowing arbitrarily long text
+(terminal emulators are free to impose a sensible limit to avoid
+Denial-of-Service attacks).
+
+Both the ``title`` and ``body`` payloads must be either UTF-8 encoded plain
+text with no embedded escape codes, or UTF-8 text that is base64 encoded, in
+which case there must be an ``e=1`` key in the metadata to indicate the payload
+is base64 encoded.
+
+When the user clicks the notification, a couple of things can happen, the
+terminal emulator can focus the window from which the notification came, and/or
+it can send back an escape code to the application indicating the notification
+was activated. This is controlled by the ``a`` key which takes a comma
+separated set of values, ``report`` and ``focus``. The value ``focus`` means
+focus the window from which the notification was issued and is the default.
+``report`` means send an escape code back to the application. The format of the
+returned escape code is::
+
+    <OSC> 99 ; i=identifier ; <terminator>
+
+The value of ``identifier`` comes from the ``i`` key in the escape code sent by
+the application. If the application sends no identifier, then the terminal
+*must* use ``i=0``. Actions can be preceded by a negative sign to turn them
+off, so for example if you do not want any action, turn off the default
+``focus`` action with::
+
+    a=-focus
+
+Complete specification of all the metadata keys is in the table below. If a
+terminal emulator encounters a key in the metadata it does not understand,
+the key *must* be ignored, to allow for future extensibility of this escape
+code. Similarly if values for known keys are unknown, the terminal emulator
+*should* either ignore the entire escape code or perform a best guess effort
+to display it based on what it does understand.
+
+.. note::
+   It is possible to extend this escape code to allow specifying an icon for
+   the notification, however, given that some platforms, such as macOS, dont
+   allow displaying custom icons on a notification, at all, it was decided to
+   leave it out of the spec for the time being.
+
+   Similarly, features such as scheduled notifications could be added in future
+   revisions.
+
+
+=======  ====================  =========  =================
+Key      Value                 Default    Description
+=======  ====================  =========  =================
+``a``    Comma separated list  ``focus``  What action to perform when the
+         of ``report``,                   notification is clicked
+         ``focus``, with
+         optional leading
+         ``-``
+
+``d``    ``0`` or ``1``        ``1``      Indicates if the notification is
+                                          complete or not.
+
+``e``    ``0`` or ``1``        ``0``      If set to ``1`` means the payload is base64 encoded UTF-8,
+                                          otherwise it is plain UTF-8 text with no C0 control codes in it
+
+``i``    ``[a-zA-Z0-9-_+.]``   ``0``      Identifier for the notification
+
+``p``    One of ``title`` or   ``title``  Whether the payload is the notification title or body. If a
+         ``body``.                        notification has no title, the body will be used as title.
+=======  ====================  =========  =================
+
+
+.. note::
+   |kitty| also supports the legacy OSC 9 protocol developed by iTerm2 for
+   desktop notifications.

docs/faq.rst

@@ -3,9 +3,6 @@ Frequently Asked Questions
 
 .. highlight:: sh
 
-.. contents::
-   :local:
-
 Some special symbols are rendered small/truncated in kitty?
 -----------------------------------------------------------
 
@@ -36,7 +33,7 @@ it by adding the following to your vimrc::
 
     let &t_ut=''
 
-See :ref:`here <ext_styles>` for why |kitty| does not support background color erase.
+See :doc:`here <deccara>` for why |kitty| does not support background color erase.
 
 
 I get errors about the terminal being unknown or opening the terminal failing when SSHing into a different computer?
@@ -55,7 +52,8 @@ type it each time::
     alias ssh="kitty +kitten ssh"
 
 If for some reason that does not work (typically because the server is using a
-non POSIX compliant shell), you can try using it with python instead::
+non POSIX compliant shell as ``/bin/sh``), you can try using it with ``python``
+instead::
 
     kitty +kitten ssh use-python myserver
 
@@ -120,7 +118,7 @@ How do I change the colors in a running kitty instance?
 ------------------------------------------------------------
 
 You can either use the
-`OSC terminal escape codes <https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h2-Operating-System-Commands>`_
+`OSC terminal escape codes <https://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h3-Operating-System-Commands>`_
 to set colors or you can define keyboard shortcuts to set colors, for example::
 
     map f1 set_colors --configured /path/to/some/config/file/colors.conf
@@ -306,10 +304,6 @@ terminal and then switch to another and these terminals have different TERM
 variables, tmux will break. You will need to restart it as tmux does not
 support multiple terminfo definitions.
 
-Copying to clipboard via OSC 52 will not work, because tmux does not support
-the extended version of that protocol, you will need to add ``no-append`` to
-:opt:`clipboard_control` in kitty.conf.
-
 If you use any of the advanced features that kitty has innovated, such as
 styled underlines, desktop notifications, extended keyboard support, etc.
 they may or may not work, depending on the whims of tmux's maintainer, your

docs/glossary.rst

@@ -0,0 +1,122 @@
+:orphan:
+
+Glossary
+=========
+
+.. glossary::
+
+   os_window
+     kitty has two kinds of windows. Operating System windows, refered to as :term:`OS
+     Window <os_window>`, and *kitty windows*. An OS Window consists of one or more kitty
+     :term:`tabs <tab>`. Each tab in turn consists of one or more *kitty
+     windows* organized in a :term:`layout`.
+
+   tab
+     A *tab* refers to a group of :term:`kitty windows <window>`, organized in
+     a :term:`layout`. Every :term:`OS Window <os_window>` contains one or more tabs.
+
+   layout
+     A *layout* is a system of organizing :term:`kitty windows <window>` in
+     groups inside a tab. The layout automatically maintains the size and
+     position of the windows, think of a layout as a tiling window manager for
+     the terminal. See :doc:`layouts` for details.
+
+   window
+     kitty has two kinds of windows. Operating System windows, refered to as :term:`OS
+     Window <os_window>`, and *kitty windows*. An OS Window consists of one or more kitty
+     :term:`tabs <tab>`. Each tab in turn consists of one or more *kitty
+     windows* organized in a :term:`layout`.
+
+   overlay
+      An *overlay window* is a :term:`kitty window <window>` that is placed on
+      top of an existing kitty window, entirely covering it. Overlays are used
+      throught kitty, for example, to display the :ref:`the scrollback buffer <scrollback>`,
+      to display :doc:`hints </kittens/hints>`, for :doc:`unicode input
+      </kittens/unicode-input>` etc.
+
+   hyperlinks
+      Terminals can have hyperlinks, just like the internet. In kitty you can
+      :doc:`control exactly what happens <open_actions>` when clicking on a
+      hyperlink, based on the type of link and its URL.
+
+.. _env_vars:
+
+Environment variables
+------------------------
+
+Variables that influence kitty behavior
+
+.. envvar:: KITTY_CONFIG_DIRECTORY
+
+   Controls where kitty looks for :file:`kitty.conf` and other configuration
+   files. Defaults to :file:`~/.config/kitty`. For full details of the config
+   directory lookup mechanism see, :option:`kitty --config`.
+
+
+.. envvar:: VISUAL
+
+   The terminal editor (such as ``vi`` or ``nano``) kitty uses, when, for
+   instance, opening :file:`kitty.conf` in response to :sc:`edit_config_file`.
+
+
+.. envvar:: EDITOR
+
+   Same as :envvar:`VISUAL`. Used if :envvar:`VISUAL` is not set.
+
+
+Variables that kitty sets when running child programs
+
+.. envvar:: KITTY_WINDOW_ID
+
+   An integer that is the id for the kitty :term:`window` the program is running in.
+   Can be used with the :doc:`kitty remote control facility <remote-control>`.
+
+
+.. envvar:: WINDOWID
+
+   The id for the :term:`OS Window <os_window>` the program is running in. Only available
+   on platforms that have ids for their windows, such as X11 and macOS.
+
+
+.. envvar:: TERM
+
+   The name of the terminal, defaults to ``xterm-kitty``. See :opt:`term`.
+
+
+.. envvar:: TERMINFO
+
+   Path to a directory containing the kitty terminfo database.
+
+
+.. envvar:: COLORTERM
+
+   Set to the value ``truecolor`` to indicate that kitty supports 16 million
+   colors.
+
+
+.. envvar:: KITTY_LISTEN_ON
+
+   Set when the :doc:`remote control <remote-control>` facility is enabled and
+   the a socket is used for control via :option:`kitty --listen-on` or :opt:`listen_on`.
+   Contains the path to the socket. Avoids needs to use :option:`kitty @ --to` when
+   issuing remote control commands.
+
+
+.. envvar:: KITTY_PIPE_DATA
+
+   Set to data describing the layout of the screen when running child
+   programs using :option:`launch --stdin-source` with the contents of the
+   screen/scrollback piped to them.
+
+
+.. envvar:: KITTY_CHILD_CMDLINE
+
+   Set to the command line of the child process running in the kitty
+   window when calling the notification callback program on terminal bell, see
+   :opt:`command_on_bell`.
+
+
+.. envvar:: KITTY_COMMON_OPTS
+
+   Set with the values of some common kitty options when running
+   kittens, so kittens can use them without needing to load kitty.conf.

docs/graphics-protocol.rst

@@ -1,18 +1,16 @@
-:tocdepth: 3
-
-The terminal graphics protocol
-==================================
+Terminal graphics protocol
+=================================
 
 The goal of this specification is to create a flexible and performant protocol
 that allows the program running in the terminal, hereafter called the *client*,
 to render arbitrary pixel (raster) graphics to the screen of the terminal
-emulator. The major design goals are
+emulator. The major design goals are:
 
- * Should not require terminal emulators to understand image formats.
- * Should allow specifying graphics to be drawn at individual pixel positions.
- * The graphics should integrate with the text, in particular it should be possible to draw graphics
-   below as well as above the text, with alpha blending. The graphics should also scroll with the text, automatically.
- * Should use optimizations when the client is running on the same computer as the terminal emulator.
+* Should not require terminal emulators to understand image formats.
+* Should allow specifying graphics to be drawn at individual pixel positions.
+* The graphics should integrate with the text, in particular it should be possible to draw graphics
+  below as well as above the text, with alpha blending. The graphics should also scroll with the text, automatically.
+* Should use optimizations when the client is running on the same computer as the terminal emulator.
 
 For some discussion regarding the design choices, see `#33
 <https://github.com/kovidgoyal/kitty/issues/33>`_.
@@ -30,23 +28,19 @@ alpha-blending and text over graphics.
 
 Some programs and libraries that use the kitty graphics protocol:
 
- * `termpdf.py <https://github.com/dsanson/termpdf.py>`_ - a terminal PDF/DJVU/CBR viewer
- * `ranger <https://github.com/ranger/ranger>`_ - a terminal file manager, with
-   image previews, see this `PR <https://github.com/ranger/ranger/pull/1077>`_
- * :doc:`kitty-diff <kittens/diff>` - a side-by-side terminal diff program with support for images
- * `pixcat <https://github.com/mirukana/pixcat>`_ - a third party CLI and python library that wraps the graphics protocol
- * `neofetch <https://github.com/dylanaraps/neofetch>`_ - A command line system
-   information tool
- * `viu <https://github.com/atanunq/viu>`_ - a terminal image viewer
- * `glkitty <https://github.com/michaeljclark/glkitty>`_ - C library to draw OpenGL shaders in the terminal with a glgears demo
- * `ctx.graphics <https://ctx.graphics/>`_ - Library for drawing graphics
- * `timg <https://github.com/hzeller/timg>`_ - a terminal image and video viewer
- * `notcurses <https://github.com/dankamongmen/notcurses>`_ - C library for terminal graphics with bindings for C++, Rust and Python
- * `rasterm <https://github.com/BourgeoisBear/rasterm>`_  - Go library to display images in the the terminal
-
-
-.. contents::
-   :local:
+* `termpdf.py <https://github.com/dsanson/termpdf.py>`_ - a terminal PDF/DJVU/CBR viewer
+* `ranger <https://github.com/ranger/ranger>`_ - a terminal file manager, with
+  image previews, see this `PR <https://github.com/ranger/ranger/pull/1077>`_
+* :doc:`kitty-diff <kittens/diff>` - a side-by-side terminal diff program with support for images
+* `pixcat <https://github.com/mirukana/pixcat>`_ - a third party CLI and python library that wraps the graphics protocol
+* `neofetch <https://github.com/dylanaraps/neofetch>`_ - A command line system
+  information tool
+* `viu <https://github.com/atanunq/viu>`_ - a terminal image viewer
+* `glkitty <https://github.com/michaeljclark/glkitty>`_ - C library to draw OpenGL shaders in the terminal with a glgears demo
+* `ctx.graphics <https://ctx.graphics/>`_ - Library for drawing graphics
+* `timg <https://github.com/hzeller/timg>`_ - a terminal image and video viewer
+* `notcurses <https://github.com/dankamongmen/notcurses>`_ - C library for terminal graphics with bindings for C++, Rust and Python
+* `rasterm <https://github.com/BourgeoisBear/rasterm>`_  - Go library to display images in the the terminal
 
 
 Getting the window size
@@ -57,28 +51,33 @@ client must be able to get the window size in pixels and the number of cells
 per row and column. This can be done by using the ``TIOCGWINSZ`` ioctl.  Some
 code to demonstrate its use
 
-In C:
+.. tab:: C
 
-.. code-block:: c
+    .. code-block:: c
 
-    #include <stdio.h>
-    #include <sys/ioctl.h>
+        #include <stdio.h>
+        #include <sys/ioctl.h>
 
-    int main(int argc, char **argv) {
-        struct winsize sz;
-        ioctl(0, TIOCGWINSZ, &sz);
-        printf("number of rows: %i, number of columns: %i, screen width: %i, screen height: %i\n", sz.ws_row, sz.ws_col, sz.ws_xpixel, sz.ws_ypixel);
-        return 0;
-    }
+        int main(int argc, char **argv) {
+            struct winsize sz;
+            ioctl(0, TIOCGWINSZ, &sz);
+            printf(
+                "number of rows: %i, number of columns: %i, screen width: %i, screen height: %i\n",
+                sz.ws_row, sz.ws_col, sz.ws_xpixel, sz.ws_ypixel);
+            return 0;
+        }
 
-In Python:
 
-.. code-block:: python
+.. tab:: Python
 
-    import array, fcntl, sys, termios
-    buf = array.array('H', [0, 0, 0, 0])
-    fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, buf)
-    print('number of rows: {}, number of columns: {}, screen width: {}, screen height: {}'.format(*buf))
+    .. code-block:: python
+
+        import array, fcntl, sys, termios
+        buf = array.array('H', [0, 0, 0, 0])
+        fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, buf)
+        print((
+            'number of rows: {} number of columns: {}'
+            'screen width: {} screen height: {}').format(*buf))
 
 Note that some terminals return ``0`` for the width and height values. Such
 terminals should be modified to return the correct values.  Examples of
@@ -98,32 +97,36 @@ features of the graphics protocol:
 
 .. code-block:: python
 
-   import sys
-   from base64 import standard_b64encode
+    import sys
+    from base64 import standard_b64encode
 
-   def serialize_gr_command(cmd, payload=None):
-      cmd = ','.join('{}={}'.format(k, v) for k, v in cmd.items())
-      ans = []
-      w = ans.append
-      w(b'\033_G'), w(cmd.encode('ascii'))
-      if payload:
-         w(b';')
-         w(payload)
-      w(b'\033\\')
-      return b''.join(ans)
 
-   def write_chunked(cmd, data):
-      data = standard_b64encode(data)
-      while data:
-         chunk, data = data[:4096], data[4096:]
-         m = 1 if data else 0
-         cmd['m'] = m
-         sys.stdout.buffer.write(serialize_gr_command(cmd, chunk))
-         sys.stdout.flush()
-         cmd.clear()
+    def serialize_gr_command(**cmd):
+        payload = cmd.pop('payload', None)
+        cmd = ','.join('{}={}'.format(k, v) for k, v in cmd.items())
+        ans = []
+        w = ans.append
+        w(b'\033_G'), w(cmd.encode('ascii'))
+        if payload:
+            w(b';')
+            w(payload)
+        w(b'\033\\')
+        return b''.join(ans)
 
-   with open(sys.argv[-1], 'rb') as f:
-      write_chunked({'a': 'T', 'f': 100}, f.read())
+
+    def write_chunked(**cmd):
+        data = standard_b64encode(cmd.pop('data'))
+        while data:
+            chunk, data = data[:4096], data[4096:]
+            m = 1 if data else 0
+            sys.stdout.buffer.write(serialize_gr_command(payload=chunk, m=m,
+                                                        **cmd))
+            sys.stdout.flush()
+            cmd.clear()
+
+
+    with open(sys.argv[-1], 'rb') as f:
+        write_chunked(a='T', f=100, data=f.read())
 
 
 Save this script as :file:`png.py`, then you can use it to display any PNG
@@ -226,8 +229,9 @@ Value of `t`          Meaning
                       is in a known temporary directory, such as :file:`/tmp`,
                       :file:`/dev/shm`, :file:`TMPDIR env var if present` and any platform
                       specific temporary directories.
-``s``                 A *shared memory object*, which on POSIX systems is a `POSIX shared memory object
-                      <http://man7.org/linux/man-pages/man7/shm_overview.7.html>`_ and on Windows is a
+``s``                 A *shared memory object*, which on POSIX systems is a
+                      `POSIX shared memory object <https://pubs.opengroup.org/onlinepubs/9699919799/functions/shm_open.html>`_
+                      and on Windows is a
                       `Named shared memory object <https://docs.microsoft.com/en-us/windows/win32/memory/creating-named-shared-memory>`_.
                       The terminal emulator must read the data from the memory
                       object and then unlink and close it on POSIX and just
@@ -590,7 +594,7 @@ Clients can control animations by using the ``a=a`` key in the escape code sent
 to the terminal.
 
 The simplest is client driven animations, where the client transmits the frame
-data and the also instructs the terminal to make a particular frame the current
+data and then also instructs the terminal to make a particular frame the current
 frame.  To change the current frame, use the ``c`` key::
 
     <ESC>_Ga=a,i=3,c=7<ESC>\
@@ -632,6 +636,45 @@ static background.
 In particular, the first frame or *root frame* is created with the base image
 data and has no gap, so its gap must be set using this control code.
 
+Composing animation frames
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 0.22.0
+   Support for frame composition
+
+Clients can *compose* animation frames, this means that they can compose pixels
+in rectangular regions from one frame onto another frame. This allows for fast
+and low band-width modification of frames.
+
+To achieve this use the ``a=c`` key. The source frame is specified with
+``r=frame number`` and the destination frame as ``c=frame number``. The size of
+the rectangle is specified as ``w=width,h=height`` pixels. If unspecified, the
+full image width and height are used. The offset of the rectangle from the
+top-left corner for the source frame is specified by the ``x,y`` keys and the
+destination frame by the ``X,Y`` keys. The composition operation is specified
+by the ``C`` key with the default being to alpha blend the source rectangle
+onto the destination rectangle. With ``C=1`` it will be a simple replacement
+of pixels. For example::
+
+    <ESC>_Gi=1,r=7,c=9,w=23,h=27,X=4,Y=8,x=1,y=3<ESC>\
+
+Will compose a ``23x27`` rectangle located at ``(4, 8)`` in the ``7th frame``
+onto the rectangle located at ``(1, 3)`` in the ``9th frame``. These will be
+in the image with ``id=1``.
+
+If the frames or the image are not found the terminal emulator must
+respond with `ENOENT`. If the rectangles go out of bounds of the image
+the terminal must respond with `EINVAL`. If the source and destination frames are
+the same and the rectangles overlap, the terminal must respond with `EINVAL`.
+
+
+.. note::
+   In kitty, doing a composition will cause a frame to be *fully rendered*
+   potentially increasing its storage requirements, when the frame was previously
+   stored as a set of operations on other frames. If this happens and there
+   is not enough storage space, kitty will respond with ENOSPC.
+
+
 Image persistence and storage quotas
 -----------------------------------------
 
@@ -654,10 +697,10 @@ take, and the default value they take when missing. All integers are 32-bit.
 Key      Value                 Default    Description
 =======  ====================  =========  =================
 ``a``    Single character.     ``t``      The overall action this graphics command is performing.
-         ``(t, T, q, p, d)``              ``t`` - transmit data, ``T`` - transmit data and display image,
-                                          ``q`` - query terminal, ``p`` - put (display) previous transmitted image,
+         ``(a, c, d, f, ``                ``t`` - transmit data, ``T`` - transmit data and display image,
+         ``p, q, t, T)``                  ``q`` - query terminal, ``p`` - put (display) previous transmitted image,
                                           ``d`` - delete image, ``f`` - transmit data for animation frames,
-                                          ``a`` - control animation
+                                          ``a`` - control animation, ``c`` - compose animation frames
 
 ``q``    ``0, 1, 2``           ``0``      Suppress responses from the terminal to this graphics command.
 
@@ -711,6 +754,20 @@ Key      Value                 Default    Description
 ``Y``    Positive integer      ``0``      The background color for pixels not
                                           specified in the frame data. Must be in 32-bit RGBA format
 
+**Keys for animation frame composition**
+-----------------------------------------------------------
+
+``c``    Positive integer      ``0``      The 1-based frame number of the frame whose image data serves as the overlaid data
+``r``    Positive integer      ``0``      The 1-based frame number of the frame that is being edited.
+``x``    Positive integer      ``0``      The left edge (in pixels) of the destination rectangle
+``y``    Positive integer      ``0``      The top edge (in pixels) of the destination rectangle
+``w``    Positive integer      ``0``      The width (in pixels) of the source and destination rectangles. By default, the entire width is used
+``h``    Positive integer      ``0``      The height (in pixels) of the source and destination rectangles. By default, the entire height is used
+``X``    Positive integer      ``0``      The left edge (in pixels) of the source rectangle
+``Y``    Positive integer      ``0``      The top edge (in pixels) of the source rectangle
+``C``    Positive integer      ``0``      The composition mode for blending
+                                          pixels. Default is full alpha blending. ``1`` means a simple overwrite.
+
 
 **Keys for animation control**
 -----------------------------------------------------------

docs/index.rst

@@ -1,421 +1,66 @@
-:tocdepth: 2
-
-==========================================================
-kitty - the fast, featureful, GPU based terminal emulator
+kitty
 ==========================================================
 
-.. container:: major-features
+*The fast, feature-rich, GPU based terminal emulator*
 
-    * Offloads rendering to the GPU for :doc:`lower system load <performance>` and
-      buttery smooth scrolling.  Uses threaded rendering to minimize input latency.
+.. toctree::
+    :hidden:
 
-    * Supports all modern terminal features: :doc:`graphics (images)
-      <graphics-protocol>`, unicode, `true-color
-      <https://gist.github.com/XVilka/8346728>`_,
-      OpenType ligatures, mouse protocol, :doc:`hyperlinks <open_actions>`,
-      focus tracking, `bracketed paste <https://cirw.in/blog/bracketed-paste>`_
-      and several :doc:`new terminal protocol extensions
-      <protocol-extensions>`.
+    quickstart
+    overview
+    faq
+    support
+    performance
+    changelog
+    integrations
+    protocol-extensions
 
-    * Supports tiling multiple terminal windows side by side in different
-      :ref:`layouts <layouts>` without needing to use an extra program like tmux
 
-    * Can be :doc:`controlled from scripts or the shell prompt <remote-control>`,
-      even over SSH.
+.. tab:: Fast
 
-    * Has a framework for :ref:`kittens`, small terminal programs that can be used to
-      extend |kitty|'s functionality.  For example, they are used for
-      :doc:`Unicode input <kittens/unicode-input>`, :doc:`Hints <kittens/hints>` and
-      :doc:`Side-by-side diff <kittens/diff>`.
+   * Offloads rendering to the GPU for :doc:`lower system load <performance>`
+   * Uses threaded rendering for absolutely minimal latency
+   * Performance tradeoffs can be :ref:`tuned <conf-kitty-performance>`
 
-    * Supports :ref:`startup sessions <sessions>` which allow you to specify
-      the window/tab layout, working directories and programs to run on startup.
+.. tab:: Capable
 
-    * Cross-platform: |kitty| works on Linux and macOS, but because it uses only
-      OpenGL for rendering, it should be trivial to port to other Unix-like platforms.
+   * Graphics, with :doc:`images and animations <graphics-protocol>`
+   * Ligatures and emoji, with :opt:`per glyph font substitution <symbol_map>`
+   * :term:`Hyperlinks<hyperlinks>`, with :doc:`configurable actions <open_actions>`
 
-    * Allows you to open :ref:`the scrollback buffer <scrollback>` in a
-      separate window using arbitrary programs of your choice. This is useful for
-      browsing the history comfortably in a pager or editor.
+.. tab:: Scriptable
 
-    * Has :ref:`multiple copy/paste buffers <cpbuf>`, like vim.
+   * Control from :doc:`scripts or the shell <remote-control>`
+   * Extend with :ref:`kittens <kittens>` using the Python language
+   * Use :ref:`startup sessions <sessions>` to specify working environments
+
+.. tab:: Composable
+
+   * Programmble tabs, :ref:`splits <splits_layout>` and multiple :doc:`layouts <layouts>` to manage windows
+   * Browse the :ref:`entire history <scrollback>` or the output from the last command comfortably in pagers and editors
+   * Edit or download :doc:`remote files <kittens/remote_file>` in an existing SSH session
+
+.. tab:: Cross-platform
+
+   * Linux
+   * macOS
+   * Various BSDs
+
+.. tab:: Innovative
+
+   Pioneered various extensions to move the entire terminal ecosystem forward
+
+   * :doc:`graphics-protocol`
+   * :doc:`keyboard-protocol`
+   * Lots more in :doc:`protocol-extensions`
 
 
 .. figure:: screenshots/screenshot.png
     :alt: Screenshot, showing three programs in the 'Tall' layout
     :align: center
-    :scale: 100%
+    :width: 100%
 
     Screenshot, showing vim, tig and git running in |kitty| with the 'Tall' layout
 
 
-
-.. contents::
-   :local:
-   :depth: 1
-
-
-.. _quickstart:
-
-Quickstart
---------------
-
-Pre-built binaries of |kitty| are available for both macOS and Linux.
-See the :doc:`binary install instructions </binary>`. You can also
-:doc:`build from source </build>`.
-
-Additionally, you can use your favorite package manager to install the |kitty|
-package, but note that some Linux distribution packages are woefully outdated.
-|kitty| is available in a vast number of package repositories for macOS
-and Linux.
-
-.. image:: https://repology.org/badge/tiny-repos/kitty.svg
-   :target: https://repology.org/project/kitty/versions
-   :alt: Number of repositories kitty is available in
-
-See :doc:`Configuring kitty <conf>` for help on configuring |kitty| and
-:doc:`Invocation <invocation>` for the command line arguments |kitty| supports.
-
-
-Design philosophy
--------------------
-
-|kitty| is designed for power keyboard users. To that end all its controls
-work with the keyboard (although it fully supports mouse interactions as
-well). Its configuration is a simple, human editable, single file for
-easy reproducibility (I like to store configuration in source control).
-
-The code in |kitty| is designed to be simple, modular and hackable. It is
-written in a mix of C (for performance sensitive parts) and Python (for
-easy hackability of the UI). It does not depend on any large and complex
-UI toolkit, using only OpenGL for rendering everything.
-
-Finally, |kitty| is designed from the ground up to support all modern
-terminal features, such as unicode, true color, bold/italic fonts, text
-formatting, etc. It even extends existing text formatting escape codes,
-to add support for features not available elsewhere, such as colored and
-styled (curly) underlines. One of the design goals of |kitty| is to be
-easily extensible so that new features can be added in the future with
-relatively little effort.
-
-.. include:: basic.rst
-
-.. _layouts:
-
-Layouts
-----------
-
-A layout is an arrangement of multiple kitty *windows* inside a top-level OS window. You can create a new window
-using the :sc:`new_window` key combination.
-
-Currently, there are seven layouts available:
-
-* **Fat** -- One (or optionally more) windows are shown full width on the top, the rest of the windows are shown side-by-side on the bottom
-* **Grid** -- All windows are shown in a grid
-* **Horizontal** -- All windows are shown side-by-side
-* **Splits** -- Windows arranged in arbitrary patterns created using horizontal and vertical splits
-* **Stack** -- Only a single maximized window is shown at a time
-* **Tall** -- One (or optionally more) windows are shown full height on the left, the rest of the windows are shown one below the other on the right
-* **Vertical** -- All windows are shown one below the other
-
-By default, all layouts are enabled and you can switch between layouts using
-the :sc:`next_layout` key combination. You can also create shortcuts to select
-particular layouts, and choose which layouts you want to enable/disable, see
-:ref:`conf-kitty-shortcuts.layout` for examples. The first layout listed in
-:opt:`enabled_layouts` becomes the default layout.
-
-For more details on the layouts and how to use them see :doc:`layouts`.
-
-.. _kittens:
-
-Kittens
-------------------
-
-|kitty| has a framework for easily creating terminal programs that make use of
-its advanced features. These programs are called kittens. They are used both
-to add features to |kitty| itself and to create useful standalone programs.
-Some prominent kittens:
-
-:doc:`icat <kittens/icat>`
-    Display images in the terminal
-
-
-:doc:`diff <kittens/diff>`
-    A fast, side-by-side diff for the terminal with syntax highlighting and
-    images
-
-
-:doc:`Unicode Input <kittens/unicode-input>`
-    Easily input arbitrary unicode characters in |kitty| by name or hex code.
-
-
-:doc:`Hints <kittens/hints>`
-    Select and open/paste/insert arbitrary text snippets such as URLs,
-    filenames, words, lines, etc. from the terminal screen.
-
-
-:doc:`Remote file <kittens/remote_file>`
-    Edit, open, or download remote files over SSH easily, by simply clicking on
-    the filename.
-
-
-:doc:`Hyperlinked grep <kittens/hyperlinked_grep>`
-    Search your files using `ripgrep <https://github.com/BurntSushi/ripgrep>`_
-    and open the results directly in your favorite editor in the terminal,
-    at the line containing the search result, simply by clicking on the result you want.
-
-
-:doc:`Broadcast <kittens/broadcast>`
-    Type in one kitty window and have it broadcast to all (or a subset) of
-    other kitty windows.
-
-
-:doc:`Panel <kittens/panel>`
-    Draw a GPU accelerated dock panel on your desktop showing the output
-    from an arbitrary terminal program.
-
-
-:doc:`Clipboard <kittens/clipboard>`
-    Copy/paste to the clipboard from shell scripts, even over SSH.
-
-You can also :doc:`Learn to create your own kittens <kittens/custom>`.
-
-
-Configuring kitty
--------------------
-
-|kitty| is highly configurable, everything from keyboard shortcuts to
-painting frames-per-second. Press :sc:`edit_config_file` in kitty
-to open its fully commented sample config file in your text editor.
-For details see the :doc:`configuration docs <conf>`.
-
-
-Remote control
-------------------
-
-|kitty| has a very powerful system that allows you to control it from the
-:doc:`shell prompt, even over SSH <remote-control>`. You can change colors,
-fonts, open new windows, tabs, set their titles, change window layout, get text
-from one window and send text to another, etc, etc. The possibilities are
-endless. See the :doc:`tutorial <remote-control>` to get started.
-
-.. _sessions:
-
-Startup Sessions
-------------------
-
-You can control the tabs, window layout, working directory, startup programs,
-etc. by creating a "session" file and using the :option:`kitty --session`
-command line flag or the :opt:`startup_session` option in :file:`kitty.conf`.
-For example:
-
-.. code-block:: session
-
-    # Set the window layout for the current tab
-    layout tall
-    # Set the working directory for windows in the current tab
-    cd ~
-    # Create a window and run the specified command in it
-    launch zsh
-    # Create a window with some environment variables set and run
-    # vim in it
-    launch --env FOO=BAR vim
-    # Set the title for the next window
-    launch --title "Chat with x" irssi --profile x
-
-    # Create a new tab (the part after new_tab is the optional tab
-    # name which will be displayed in the tab bar, if omitted, the
-    # title of the active window will be used instead)
-    new_tab my tab
-    cd ~/somewhere
-    # Set the layouts allowed in this tab
-    enabled_layouts tall, stack
-    # Set the current layout
-    layout stack
-    launch zsh
-
-    # Create a new OS window
-    new_os_window
-    # set new window size to 80x25 cells
-    os_window_size 80c 25c
-    # set the --class for the new OS window
-    os_window_class mywindow
-    launch sh
-    # Make the current window the active (focused) window
-    focus
-    launch emacs
-
-.. note::
-    The :doc:`launch <launch>` command when used in a session file
-    cannot create new OS windows, or tabs.
-
-
-Mouse features
--------------------
-
-* You can click on a URL to open it in a browser.
-* You can double click to select a word and then drag to select more words.
-* You can triple click to select a line and then drag to select more lines.
-* You can triple click while holding :kbd:`ctrl+alt` to select from clicked
-  point to end of line.
-* You can right click to extend a previous selection.
-* You can hold down :kbd:`ctrl+alt` and drag with the mouse to select in
-  columns.
-* Selecting text automatically copies it to the primary clipboard (on
-  platforms with a primary clipboard).
-* You can middle click to paste from the primary clipboard (on platforms
-  with a primary clipboard).
-* You can select text with kitty even when a terminal program has grabbed
-  the mouse by holding down the :kbd:`shift` key.
-
-All these actions can be customized in :file:`kitty.conf` as described
-:ref:`here <conf-kitty-mouse.mousemap>`.
-
-
-Font control
------------------
-
-|kitty| has extremely flexible and powerful font selection features. You can
-specify individual families for the regular, bold, italic and bold+italic
-fonts. You can even specify specific font families for specific ranges of
-unicode characters. This allows precise control over text rendering. It can
-come in handy for applications like powerline, without the need to use patched
-fonts. See the various font related configuration directives in
-:ref:`conf-kitty-fonts`.
-
-
-.. _scrollback:
-
-The scrollback buffer
------------------------
-
-|kitty| supports scrolling back to view history, just like most terminals. You
-can use either keyboard shortcuts or the mouse scroll wheel to do so.  However,
-|kitty| has an extra, neat feature. Sometimes you need to explore the
-scrollback buffer in more detail, maybe search for some text or refer to it
-side-by-side while typing in a follow-up command. |kitty| allows you to do this
-by pressing the :sc:`show_scrollback` key-combination, which will open the
-scrollback buffer in your favorite pager program (which is ``less`` by default).
-Colors and text formatting are preserved. You can explore the scrollback buffer
-comfortably within the pager.
-
-Additionally, you can pipe the contents of the scrollback buffer to an
-arbitrary, command running in a new window, tab or overlay, for example::
-
-   map f1 launch --stdin-source=@screen_scrollback --stdin-add-formatting less +G -R
-
-Would open the scrollback buffer in a new window when you press the :kbd:`F1`
-key. See :sc:`show_scrollback` for details.
-
-If you want to use it with an editor such as vim to get more powerful features,
-you can see tips for doing so, in
-`this thread <https://github.com/kovidgoyal/kitty/issues/719>`_.
-
-If you wish to store very large amounts of scrollback to view using the piping or
-:sc:`show_scrollback` features, you can use the :opt:`scrollback_pager_history_size`
-option.
-
-.. _cpbuf:
-
-Multiple copy/paste buffers
------------------------------
-
-In addition to being able to copy/paste from the system clipboard, in |kitty| you
-can also setup an arbitrary number of copy paste buffers. To do so, simply add
-something like the following to your :file:`kitty.conf`::
-
-   map f1 copy_to_buffer a
-   map f2 paste_from_buffer a
-
-This will allow you to press :kbd:`F1` to copy the current selection to an
-internal buffer named ``a`` and :kbd:`F2` to paste from that buffer. The buffer
-names are arbitrary strings, so you can define as many such buffers as you
-need.
-
-Marks
--------------
-
-kitty has the ability to mark text on the screen based on regular expressions.
-This can be useful to highlight words or phrases when browsing output from long
-running programs or similar. To learn how this feature works, see :doc:`marks`.
-
-
-Frequently Asked Questions
----------------------------------
-
-The list of Frequently Asked Questions (*FAQ*) is :doc:`available here <faq>`.
-
-.. _completion:
-
-
-Cool integrations for kitty with other CLI tools
---------------------------------------------------
-
-kitty provides extremely powerful interfaces such as :doc:`remote-control` and
-:doc:`kittens/custom` and :doc:`kittens/icat`
-that allow it to be integrated with other tools seamlessly. For a list of such
-user created integrations, see: :doc:`integrations`.
-
-Completion for kitty
----------------------------------
-
-|kitty| comes with completion for the ``kitty`` command for popular shells.
-
-
-bash
-~~~~~~~~
-
-Add the following to your :file:`~/.bashrc`
-
-.. code-block:: sh
-
-   source <(kitty + complete setup bash)
-
-Older versions of bash (for example, v3.2) do not support
-process substitution with the source command, in which
-case you can try an alternative:
-
-.. code-block:: sh
-
-    source /dev/stdin <<<"$(kitty + complete setup bash)"
-
-
-zsh
-~~~~~~~~~
-
-Add the following to your :file:`~/.zshrc`
-
-.. code-block:: sh
-
-    autoload -Uz compinit
-    compinit
-    # Completion for kitty
-    kitty + complete setup zsh | source /dev/stdin
-
-The important thing above is to make sure the call to |kitty| to load the zsh
-completions happens after the call to :file:`compinit`.
-
-
-fish
-~~~~~~~~
-
-For versions of fish earlier than 3.0.0, add the following to your
-:file:`~/.config/fish/config.fish`. Later versions source completions by default.
-
-.. code-block:: sh
-
-   kitty + complete setup fish | source
-
-
-Changelog
-------------------
-
-See :doc:`changelog`.
-
-.. toctree::
-    :hidden:
-    :glob:
-
-    *
-    kittens/*
-    generated/rc
+To get started see :doc:`quickstart`.

docs/integrations.rst

@@ -1,3 +1,5 @@
+:tocdepth: 2
+
 Integrations with other tools
 ================================
 
@@ -12,149 +14,245 @@ Image and document viewers
 Powered by kitty's :doc:`graphics-protocol` there exist many tools for viewing
 images and other types of documents directly in your terminal, even over SSH.
 
+.. _tool_termpdf:
+
 `termpdf.py <https://github.com/dsanson/termpdf.py>`_
-    a terminal PDF/DJVU/CBR viewer
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+A terminal PDF/DJVU/CBR viewer
+
+.. _tool_mdcat:
 
 `mdcat <https://github.com/lunaryorn/mdcat>`_
-    Display markdown files nicely formatted with images in the terminal
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Display markdown files nicely formatted with images in the terminal
+
+.. _tool_ranger:
 
 `ranger <https://github.com/ranger/ranger>`_
-    a terminal file manager, with previews of file contents powered by kitty's graphics protocol.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+A terminal file manager, with previews of file contents powered by kitty's graphics protocol.
+
+.. _tool_nnn:
 
 `nnn <https://github.com/jarun/nnn/>`_
-    another terminal file manager, with previews of file contents powered by kitty's graphics protocol.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Another terminal file manager, with previews of file contents powered by kitty's graphics protocol.
+
+.. _tool_hunter:
 
 `hunter <https://github.com/rabite0/hunter>`_
-    another terminal file manager, with previews of file contents powered by kitty's graphics protocol.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Another terminal file manager, with previews of file contents powered by kitty's graphics protocol.
+
+.. _tool_koneko:
 
 `koneko <https://github.com/twenty5151/koneko>`_
-    browse images from the pixiv artist community directly in kitty.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Browse images from the pixiv artist community directly in kitty.
+
+.. _tool_viu:
 
 `viu <https://github.com/atanunq/viu>`_
-    view images in the terminal, similar to kitty's icat.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+View images in the terminal, similar to kitty's icat.
+
+.. _tool_nb:
+
 
 `nb <https://github.com/xwmx/nb>`_
-    command line and local web note-taking, bookmarking, archiving, and
-    knowledge base application that uses kitty's graphics protocol for images.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+command line and local web note-taking, bookmarking, archiving, and
+knowledge base application that uses kitty's graphics protocol for images.
+
+.. _tool_w3m:
 
 `w3m <https://github.com/tats/w3m>`_
-    A text mode WWW browser that supports kitty's graphics protocol to display
-    images.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+A text mode WWW browser that supports kitty's graphics protocol to display
+images.
+
+.. _tool_timg:
 
 `timg <https://github.com/hzeller/timg>`_
-    A terminal image and video viewer, that displays static and animated
-    images or plays videos. Fast multi-threaded loading, JPEG exif rotation,
-    grid view and connecting to the webcam make it a versatile terminal utility.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+A terminal image and video viewer, that displays static and animated
+images or plays videos. Fast multi-threaded loading, JPEG exif rotation,
+grid view and connecting to the webcam make it a versatile terminal utility.
 
 
 System and data visualisation tools
 ---------------------------------------
 
+.. _tool_neofetch:
+
 `neofetch <https://github.com/dylanaraps/neofetch>`_
-    A command line system information tool that shows images using kitty's graphics protocol
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+A command line system information tool that shows images using kitty's graphics protocol
+
+.. _tool_matplotlib:
 
 `matplotlib <https://github.com/jktr/matplotlib-backend-kitty>`_
-    show matplotlib plots directly in kitty
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Show matplotlib plots directly in kitty
+
+.. _tool_KittyTerminalImage:
 
 `KittyTerminalImages.jl <https://github.com/simonschoelly/KittyTerminalImages.jl>`_
-    show images from Julia directly in kitty
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Show images from Julia directly in kitty
+
+.. _tool_euporie:
 
 `euporie <https://github.com/joouha/euporie>`_
-    a text-based user interface for running and editing Jupyter notebooks,
-    powered by kitty's graphics protocol for displaying plots
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+A text-based user interface for running and editing Jupyter notebooks,
+powered by kitty's graphics protocol for displaying plots
+
+.. _tool_gnuplot:
 
 `gnuplot <http://www.gnuplot.info/>`_
-    a graphing and data visualization tool that can be made to display its
-    output in kitty with the following bash snippet::
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-        function iplot {
-            cat <<EOF | gnuplot
-            set terminal pngcairo enhanced font 'Fira Sans,10'
-            set autoscale
-            set samples 1000
-            set output '|kitty +kitten icat --stdin yes'
-            set object 1 rectangle from screen 0,0 to screen 1,1 fillcolor rgb"#fdf6e3" behind
-            plot $@
-            set output '/dev/null'
-            EOF
-        }
+A graphing and data visualization tool that can be made to display its
+output in kitty with the following bash snippet:
 
-   Add this to bashrc and then to plot a function, simply do::
+.. code-block:: sh
 
-        iplot 'sin(x*3)*exp(x*.2)'
+    function iplot {
+        cat <<EOF | gnuplot
+        set terminal pngcairo enhanced font 'Fira Sans,10'
+        set autoscale
+        set samples 1000
+        set output '|kitty +kitten icat --stdin yes'
+        set object 1 rectangle from screen 0,0 to screen 1,1 fillcolor rgb"#fdf6e3" behind
+        plot $@
+        set output '/dev/null'
+        EOF
+    }
+
+Add this to bashrc and then to plot a function, simply do:
+
+.. code-block:: sh
+
+    iplot 'sin(x*3)*exp(x*.2)'
+
+.. tool_onefetch:
 
 `onefetch <https://github.com/o2sh/onefetch>`_
-    a tool to fetch information about your git repositories
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+A tool to fetch information about your git repositories
+
+.. tool_patat:
 
 `patat <https://github.com/jaspervdj/patat>`_
-    terminal based presentations using pandoc and kitty's image protocol for
-    images
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Terminal based presentations using pandoc and kitty's image protocol for
+images
+
+.. tool_wttr:
 
 `wttr.in <https://github.com/chubin/wttr.in>`_
-    a tool to display weather information in your terminal with curl
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+A tool to display weather information in your terminal with curl
+
+.. tool_wl_clipboard:
 
 `wl-clipboard-manager <https://github.com/maximbaz/wl-clipboard-manager>`_
-    view and manage the system clipboard under Wayland in your kitty terminal
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+View and manage the system clipboard under Wayland in your kitty terminal
+
+.. tool_dmenu_term:
 
 `dmenu-term <https://github.com/maximbaz/dmenu-term>`_
-    run applications on your system with fuzzy find inside a kitty window
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Run applications on your system with fuzzy find inside a kitty window
 
 
 Editor integration
 -----------------------
 
-kitty can be integrated into many different terminal editors to add features
+|kitty| can be integrated into many different terminal editors to add features
 such a split windows, previews, REPLs etc.
 
+.. tool_kakoune:
 
 `kakoune <https://kakoune.org/>`_
-    integrates with kitty to use native kitty windows for its windows/panels and REPLs.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+integrates with kitty to use native kitty windows for its windows/panels and REPLs.
+
+.. tool_vim_slime:
 
 `vim-slime <https://github.com/jpalardy/vim-slime#kitty>`_
-    uses kitty remote control for a Lisp REPL.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+uses kitty remote control for a Lisp REPL.
+
+.. tool_vim_kitty_navigator:
 
 `vim-kitty-navigator <https://github.com/knubie/vim-kitty-navigator>`_
-    allows you to navigate seamlessly between vim and kitty splits using a consistent set of hotkeys.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+allows you to navigate seamlessly between vim and kitty splits using a consistent set of hotkeys.
+
+.. tool_vim_test:
 
 `vim-test <https://github.com/vim-test/vim-test>`_
-    allows easily running tests in a terminal window
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Allows easily running tests in a terminal window
+
+.. tool_hologram:
 
 `hologram.nvim <https://github.com/edluffy/hologram.nvim>`_
-    terminal image viewer for nvim
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Terminal image viewer for nvim
 
 
 Scrollback manipulation
 -------------------------
 
+.. tool_kitty_search:
+
 `kitty-search <https://github.com/trygveaa/kitty-kitten-search>`_
-    Live incremental search of the scrollback buffer.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Live incremental search of the scrollback buffer.
+
+.. tool_kitty_grab:
 
 `kitty-grab <https://github.com/yurikhan/kitty_grab>`_
-    keyboard based text selection for the kitty scrollback buffer.
-
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Keyboard based text selection for the kitty scrollback buffer.
 
 
 Miscellaneous
 ------------------
 
+.. tool_kitty_smart_tab:
+
 `kitty-smart-tab <https://github.com/yurikhan/kitty-smart-tab>`_
-    use keys to either control tabs or pass them onto running applications if
-    no tabs are present
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+use keys to either control tabs or pass them onto running applications if
+no tabs are present
+
+.. tool_kitty_smart_scroll:
 
 `kitty-smart-scroll <https://github.com/yurikhan/kitty-smart-scroll>`_
-    use keys to either scroll or pass them onto running applications if
-    no scrollback buffer is present
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+use keys to either scroll or pass them onto running applications if
+no scrollback buffer is present
 
-`reload keybindings <https://github.com/kovidgoyal/kitty/issues/1292#issuecomment-582388769>`_
-    reload key bindings from :file:`kitty.conf` without needing to restart
-    kitty
+.. tool_kitti3:
 
 `kitti3 <https://github.com/LandingEllipse/kitti3>`_
-    allow using kitty as a drop-down terminal under the i3 window manager
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+allow using kitty as a drop-down terminal under the i3 window manager
+
+.. tool_weechat_hints:
 
 `weechat-hints <https://github.com/GermainZ/kitty-weechat-hints>`_
-    URL hints kitten for WeeChat that works without having to use WeeChat's
-    raw-mode.
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+URL hints kitten for WeeChat that works without having to use WeeChat's
+raw-mode.
+
+.. tool_glkitty:
 
 `glkitty <https://github.com/michaeljclark/glkitty>`_
-    C library to draw OpenGL shaders in the terminal with a glgears demo
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+C library to draw OpenGL shaders in the terminal with a glgears demo

docs/invocation.rst

@@ -1,3 +1,5 @@
+:orphan:
+
 The kitty command line interface
 ====================================
 

docs/keyboard-protocol.rst

@@ -1,5 +1,5 @@
-A protocol for comprehensive keyboard handling in terminals
-=================================================================
+Comprehensive keyboard handling in terminals
+==============================================
 
 There are various problems with the current state of keyboard handling in
 terminals. They include:
@@ -41,12 +41,14 @@ If you are an application or library developer just interested in using this
 protocol to make keyboard handling simpler and more robust in your application,
 without too many changes, do the following:
 
-#. Emit the escape code ``CSI > 1 u`` at application startup or when entering
-   alternate screen mode
+#. Emit the escape code ``CSI > 1 u`` at application startup if using the main
+   screen or when entering alternate screen mode, if using the alternate
+   screen.
 #. All key events will now be sent in only a few forms to your application,
    that are easy to parse unambiguously.
-#. Emit the escape sequence ``CSI < u`` at application exit or just before
-   leaving alternate screen mode to restore the previously used keyboard mode.
+#. Emit the escape sequence ``CSI < u`` at application exit if using the main
+   screen or just before leaving alternate screen mode if using the alternate screen,
+   to restore the previously used keyboard mode.
 
 Key events will all be delivered to your application either as plain UTF-8
 text, or using the following escape codes, for those keys that do not produce
@@ -266,6 +268,15 @@ and alternate screens. If a pop request is received that empties the stack,
 all flags are reset. If a push request is received and the stack is full, the
 oldest entry from the stack must be evicted.
 
+.. note:: The main and alternate screens in the terminal emulator must maintain
+   their own, independent, keyboard mode stacks. This is so that a program that
+   uses the alternate screen such as an editor, can change the keyboard mode
+   in the alternate screen only, without affecting the mode in the main screen
+   or even knowing what that mode is. Without this, and if no stack is
+   implemented for keyboard modes (such as in some legacy terminal emulators)
+   the editor would have to somehow know what the keyboard mode of the main
+   screen is and restore to that mode on exit.
+
 .. note:: In the interests of interoperation, the XTerm specific sequences
    `CSI > 4; 1 m` and `CSI > 4; 0 m` are treated as `CSI > 1 u` and `CSI < 1 u`.
    These codes cause XTerm to use the CSI u encoding for more keys and are therefore

docs/kittens/broadcast.rst

@@ -1,6 +1,8 @@
-broadcast - type text in all kitty windows
+broadcast
 ==================================================
 
+*Type text in all kitty windows simultaneously*
+
 The ``broadcast`` kitten can be used to type text simultaneously in
 all kitty windows (or a subset as desired).
 
@@ -17,7 +19,4 @@ are selected.
 .. program:: kitty +kitten broadcast
 
 
-Command Line Interface
---------------------------
-
 .. include:: /generated/cli-kitten-broadcast.rst

docs/kittens/clipboard.rst

@@ -1,6 +1,8 @@
-clipboard - copy/paste to the system clipboard
+clipboard
 ==================================================
 
+*Copy/paste to the system clipboard from shell scripts*
+
 .. highlight:: sh
 
 
@@ -21,7 +23,4 @@ use::
 .. program:: kitty +kitten clipboard
 
 
-Command Line Interface
---------------------------
-
 .. include:: /generated/cli-kitten-clipboard.rst

docs/kittens/custom.rst

@@ -103,7 +103,8 @@ terminal program, you can tell the kittens system to run the
 ``handle_result()`` function without first running the ``main()`` function.
 
 For example, here is a kitten that "zooms/unzooms" the current terminal window
-by switching to the stack layout or back to the previous layout.
+by switching to the stack layout or back to the previous layout. This is
+equivalent to the builtin :ref:`action-toggle_layout` action.
 
 Create a file in the kitty config folder, :file:`~/.config/kitty/zoom_toggle.py`
 
@@ -185,6 +186,104 @@ The output of print statements will go to the ``STDOUT`` of the kitty process.
 So if you run kitty from another kitty instance, the output will be visible
 in the first kitty instance.
 
+Adding options to kittens
+----------------------------
+
+If you would like to use kitty's config framework to make your kittens
+configurable, you will need some boilerplate. In the directory
+of your kitten make the following files.
+
+:file:`kitten_options_definition.py`
+
+.. code-block:: python
+
+    from kitty.conf.types import Action, Definition
+
+    definition = Definition(
+        '!kitten_options_utils',
+        Action(
+            'map', 'parse_map',
+            {'key_definitions': 'kitty.conf.utils.KittensKeyMap'},
+            ['kitty.types.ParsedShortcut', 'kitty.conf.utils.KeyAction']
+        ),
+    )
+
+    agr = definition.add_group
+    egr = definition.end_group
+    opt = definition.add_option
+    map = definition.add_map
+
+    # main options {{{
+    agr('main', 'Main')
+
+    opt('some_option', '33',
+        option_type='some_option_parser',
+        long_text='''
+    Help text for this option
+    '''
+        )
+    egr()  # }}}
+
+    # shortcuts {{{
+    agr('shortcuts', 'Keyboard shortcuts')
+
+    map('Quit', 'quit q quit')
+    egr()  # }}}
+
+
+:file:`kitten_options_utils.py`
+
+.. code-block:: python
+
+    from kitty.conf.utils import KittensKeyDefinition, key_func, parse_kittens_key
+
+    func_with_args, args_funcs = key_func()
+    FuncArgsType = Tuple[str, Sequence[Any]]
+
+    def some_option_parser(val: str) -> int:
+        return int(val) + 3000
+
+    def parse_map(val: str) -> Iterable[KittensKeyDefinition]:
+        x = parse_kittens_key(val, args_funcs)
+        if x is not None:
+            yield x
+
+Then run::
+
+    kitty +runpy 'from kitty.conf.generate import main; main()' /path/to/kitten_options_definition.py
+
+You can parse and read the options in your kitten using the following code:
+
+.. code-block:: python
+
+    from .kitten_options_types import Options, defaults
+    from kitty.conf.utils import load_config as _load_config, parse_config_base
+    from typing import Optional, Iterable, Dict, Any
+
+    def load_config(*paths: str, overrides: Optional[Iterable[str]] = None) -> Options:
+        from .kitten_options_parse import  (
+            create_result_dict, merge_result_dicts, parse_conf_item
+        )
+
+        def parse_config(lines: Iterable[str]) -> Dict[str, Any]:
+            ans: Dict[str, Any] = create_result_dict()
+            parse_config_base(
+                lines,
+                parse_conf_item,
+                ans,
+            )
+            return ans
+
+        overrides = tuple(overrides) if overrides is not None else ()
+        opts_dict, paths = _load_config(defaults, parse_config, merge_result_dicts, *paths, overrides=overrides)
+        opts = Options(opts_dict)
+        opts.config_paths = paths
+        opts.config_overrides = overrides
+        return opts
+
+See the code for the builtin diff kitten for examples of creating more options
+and keyboard shortcuts.
+
 .. _external_kittens:
 
 Kittens created by kitty users

docs/kittens/diff.rst

@@ -1,6 +1,8 @@
-kitty-diff - A fast side-by-side diff tool with syntax highlighting and images
+kitty-diff
 ================================================================================
 
+*A fast side-by-side diff tool with syntax highlighting and images*
+
 .. highlight:: sh
 
 Major Features
@@ -21,13 +23,10 @@ Major Features
 .. figure:: ../screenshots/diff.png
    :alt: Screenshot, showing a sample diff
    :align: center
-   :scale: 100%
+   :width: 100%
 
    Screenshot, showing a sample diff
 
-.. contents::
-   :local:
-
 
 Installation
 ---------------
@@ -35,7 +34,7 @@ Installation
 Simply :ref:`install kitty <quickstart>`.  You also need
 to have either the `git <https://git-scm.com/>`_ program or the ``diff`` program
 installed. Additionally, for syntax highlighting to work,
-`pygments <http://pygments.org/>`_ must be installed (note that pygments is
+`pygments <https://pygments.org/>`_ must be installed (note that pygments is
 included in the macOS kitty app).
 
 
@@ -121,8 +120,8 @@ Why does this work only in kitty?
 
 The diff kitten makes use of various features that are :doc:`kitty only
 </protocol-extensions>`,  such as the :doc:`kitty graphics protocol
-</graphics-protocol>`, the :ref:`extended keyboard protocol
-<extended-key-protocol>`, etc. It also leverages terminal program
+</graphics-protocol>`, the :doc:`extended keyboard protocol
+</keyboard-protocol>`, etc. It also leverages terminal program
 infrastructure I created for all of kitty's other kittens to reduce the amount
 of code needed (the entire implementation is under 2000 lines of code).
 
@@ -143,9 +142,6 @@ configuration directives.
 .. include:: /generated/conf-kitten-diff.rst
 
 
-Command Line Interface
--------------------------
-
 .. include:: /generated/cli-kitten-diff.rst
 
 

docs/kittens/hints.rst

@@ -9,7 +9,7 @@ browser.
 .. figure:: ../screenshots/hints_mode.png
     :alt: URL hints mode
     :align: center
-    :scale: 100%
+    :width: 100%
 
     URL hints mode
 
@@ -88,8 +88,14 @@ Now run kitty with::
 When you press the :kbd:`F1` key you will be able to select a word to
 look it up in the Google dictionary.
 
-
-Command Line Interface
--------------------------
-
 .. include:: ../generated/cli-kitten-hints.rst
+
+.. note::
+
+    To avoid having to specify the same command line options on ever invocation,
+    you can use the :opt:`kitten_alias` option in :file:`kitty.conf` to create aliases
+    that have common sets of options. For example::
+
+        kitten_alias myhints hints --alphabet qfjdkslaureitywovmcxzpq1234567890
+
+    Documentation for the full set of options is below.

docs/kittens/hyperlinked_grep.rst

@@ -58,7 +58,7 @@ Then, for example, for ZSH, add the following to :file:`.zshrc`::
     compdef _rg hg
 
 To learn more about kitty's powerful framework for customizing URL click
-actions, :doc:`see here <../open_actions>`.
+actions, :doc:`see here </open_actions>`.
 
 Hopefully, someday this functionality will make it into some `upstream grep
 <https://github.com/BurntSushi/ripgrep/issues/665>`_

docs/kittens/icat.rst

@@ -1,6 +1,8 @@
-icat - Display images in the terminal
+icat
 ========================================
 
+*Display images in the terminal*
+
 The ``icat`` kitten can be used to display arbitrary images in the |kitty|
 terminal. Using it is as simple as::
 
@@ -35,7 +37,4 @@ The ``icat`` kitten has various command line arguments to allow it to be used
 from inside other programs to display images. In particular, :option:`--place`,
 :option:`--detect-support` and :option:`--print-window-size`.
 
-Command Line Interface
---------------------------
-
 .. include:: /generated/cli-kitten-icat.rst

docs/kittens/panel.rst

@@ -14,7 +14,7 @@ using terminal programs instead of GUI toolkits.
 .. figure:: ../screenshots/panel.png
    :alt: Screenshot, showing a sample panel
    :align: center
-   :scale: 100%
+   :width: 100%
 
    Screenshot, showing a sample panel
 
@@ -37,7 +37,4 @@ print out ``Hello, world!``. You can make the terminal program as complex as
 you like, as demonstrated in the screenshot above.
 
 
-Command Line Interface
--------------------------
-
 .. include:: ../generated/cli-kitten-panel.rst

docs/kittens/query_terminal.rst

@@ -15,7 +15,4 @@ for *XTGETTCAP* to see the syntax for the escape code and read the source
 of this kitten to find the values of the keys for the various queries.
 
 
-Command Line Interface
--------------------------
-
 .. include:: ../generated/cli-kitten-query_terminal.rst

docs/kittens/remote_file.rst

@@ -11,15 +11,16 @@ Then hold down :kbd:`ctrl+shift` and click the name of the file.
 .. figure:: ../screenshots/remote_file.png
     :alt: Remote file actions
     :align: center
-    :scale: 100%
+    :width: 100%
 
     Remote file actions
 
 |kitty| will ask you what you want to do with the remote file. You can choose
 to *Edit* it in which case kitty will download it and open it locally in your
-``EDITOR``. As you make changes to the file, they are automatically transferred
-to the remote computer. Note that this happens without needing to install *any*
-special software on the server, beyond ``ls`` that supports hyperlinks.
+:envvar:`EDITOR`. As you make changes to the file, they are automatically
+transferred to the remote computer. Note that this happens without needing
+to install *any* special software on the server, beyond ``ls`` that supports
+hyperlinks.
 
 .. versionadded:: 0.19.0
 

docs/kittens/unicode-input.rst

@@ -7,7 +7,7 @@ Press :sc:`input_unicode_character` to start the unicode input widget, shown bel
 .. figure:: ../screenshots/unicode.png
     :alt: A screenshot of the unicode input widget
     :align: center
-    :scale: 100%
+    :width: 100%
 
     A screenshot of the unicode input widget
 
@@ -28,7 +28,4 @@ You can switch between modes using either the function keys or by pressing
 :kbd:`Ctrl+Shift+Tab`.
 
 
-Command Line Interface
--------------------------
-
 .. include:: ../generated/cli-kitten-unicode_input.rst

docs/kittens_intro.rst

@@ -0,0 +1,66 @@
+.. _kittens:
+
+Extend with kittens
+-----------------------
+
+.. toctree::
+   :hidden:
+   :glob:
+
+   kittens/icat
+   kittens/diff
+   kittens/unicode-input
+   kittens/hints
+   kittens/remote_file
+   kittens/hyperlinked_grep
+   kittens/custom
+   kittens/*
+
+|kitty| has a framework for easily creating terminal programs that make use of
+its advanced features. These programs are called kittens. They are used both
+to add features to |kitty| itself and to create useful standalone programs.
+Some prominent kittens:
+
+:doc:`icat <kittens/icat>`
+    Display images in the terminal
+
+
+:doc:`diff <kittens/diff>`
+    A fast, side-by-side diff for the terminal with syntax highlighting and
+    images
+
+
+:doc:`Unicode Input <kittens/unicode-input>`
+    Easily input arbitrary unicode characters in |kitty| by name or hex code.
+
+
+:doc:`Hints <kittens/hints>`
+    Select and open/paste/insert arbitrary text snippets such as URLs,
+    filenames, words, lines, etc. from the terminal screen.
+
+
+:doc:`Remote file <kittens/remote_file>`
+    Edit, open, or download remote files over SSH easily, by simply clicking on
+    the filename.
+
+
+:doc:`Hyperlinked grep <kittens/hyperlinked_grep>`
+    Search your files using `ripgrep <https://github.com/BurntSushi/ripgrep>`_
+    and open the results directly in your favorite editor in the terminal,
+    at the line containing the search result, simply by clicking on the result you want.
+
+
+:doc:`Broadcast <kittens/broadcast>`
+    Type in one :term:`kitty window <window>` and have it broadcast to all (or a subset) of
+    other :term:`kitty windows <window>`.
+
+
+:doc:`Panel <kittens/panel>`
+    Draw a GPU accelerated dock panel on your desktop showing the output
+    from an arbitrary terminal program.
+
+
+:doc:`Clipboard <kittens/clipboard>`
+    Copy/paste to the clipboard from shell scripts, even over SSH.
+
+You can also :doc:`Learn to create your own kittens <kittens/custom>`.

docs/launch.rst

@@ -1,5 +1,5 @@
-Launching programs in new windows/tabs
-========================================
+The :command:`launch` command
+--------------------------------
 
 .. program:: launch
 
@@ -42,7 +42,7 @@ The piping environment
 --------------------------
 
 When using :option:`launch --stdin-source`, the program to which the data is
-piped has a special environment variable declared, ``KITTY_PIPE_DATA`` whose
+piped has a special environment variable declared, :envvar:`KITTY_PIPE_DATA` whose
 contents are::
 
    KITTY_PIPE_DATA={scrolled_by}:{cursor_x},{cursor_y}:{lines},{columns}

docs/layouts.rst

@@ -1,5 +1,5 @@
-Layouts
-============
+Arrange windows
+-------------------
 
 kitty has the ability to define its own windows that can be tiled next to each
 other in arbitrary arrangements, based on *Layouts*, see below for examples:
@@ -8,7 +8,7 @@ other in arbitrary arrangements, based on *Layouts*, see below for examples:
 .. figure:: screenshots/screenshot.png
     :alt: Screenshot, showing three programs in the 'Tall' layout
     :align: center
-    :scale: 100%
+    :width: 100%
 
     Screenshot, showing vim, tig and git running in |kitty| with the 'Tall' layout
 
@@ -16,7 +16,7 @@ other in arbitrary arrangements, based on *Layouts*, see below for examples:
 .. figure:: screenshots/splits.png
     :alt: Screenshot, showing windows in the 'Splits' layout
     :align: center
-    :scale: 100%
+    :width: 100%
 
     Screenshot, showing windows with arbitrary arrangement in the 'Splits'
     layout
@@ -27,9 +27,6 @@ you can switch layouts using :sc:`next_layout`. To control which layouts
 are available use :opt:`enabled_layouts`, the first listed layout becomes
 the default. Individual layouts and how to use them are described below.
 
-.. contents::
-   :local:
-
 
 The Stack Layout
 ------------------

docs/marks.rst

@@ -1,11 +1,14 @@
-Marks
-=================
+Mark text on screen
+---------------------
 
 
 kitty has the ability to mark text on the screen based on regular expressions.
 This can be useful to highlight words or phrases when browsing output from long
 running programs or similar. Lets start with a few examples:
 
+Examples
+----------
+
 Suppose we want to be able to highlight the word ERROR in the current window.
 Add the following to :file:`kitty.conf`::
 
@@ -89,12 +92,11 @@ The syntax of the :code:`toggle_marker` command is::
 
 Here :code:`marker-type` is one of:
 
-    * :code:`text` - simple substring matching
-    * :code:`itext` - case-insensitive substring matching
-    * :code:`regex` - A python regular expression
-    * :code:`iregex` - A case-insensitive python regular expression
-    * :code:`function` - An arbitrary function defined in a python file, see
-      :ref:`marker_funcs`.
+* :code:`text` - simple substring matching
+* :code:`itext` - case-insensitive substring matching
+* :code:`regex` - A python regular expression
+* :code:`iregex` - A case-insensitive python regular expression
+* :code:`function` - An arbitrary function defined in a python file, see :ref:`marker_funcs`.
 
 .. _marker_funcs:
 

docs/open_actions.rst

@@ -1,10 +1,11 @@
-Customizing the actions taken when clicking on links
+Scripting the mouse click
 ======================================================
 
-|kitty| has support for `terminal hyperlinks <https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda>`_. These
+|kitty| has support for `terminal hyperlinks
+<https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda>`_. These
 are generated by many terminal programs, such as ``ls``, ``gcc``, ``systemd``,
-``mdcat``, etc. You can customize exactly what happens when clicking on these hyperlinks
-in |kitty|.
+:ref:`tool_mdcat`, etc. You can customize exactly what happens when clicking on these
+hyperlinks in |kitty|.
 
 You can tell kitty to take arbitrarily many, complex actions
 when a link is clicked. Let us illustrate with some examples, first. Create
@@ -85,7 +86,7 @@ lines. The various available criteria are:
     :file:`mime.types` in the kitty configuration directory. Useful if your
     system MIME database does not have definitions you need. This file is
     in the standard format of one definition per line, like: ``text/plain rst
-    md``.
+    md``. Note that the MIME type for directories is ``inode/directory``.
 
 ``ext``
     A comma separated list of file extensions, for example: ``jpeg, tar.gz``

docs/overview.rst

@@ -0,0 +1,283 @@
+Overview
+==============
+
+Design philosophy
+-------------------
+
+|kitty| is designed for power keyboard users. To that end all its controls
+work with the keyboard (although it fully supports mouse interactions as
+well). Its configuration is a simple, human editable, single file for
+easy reproducibility (I like to store configuration in source control).
+
+The code in |kitty| is designed to be simple, modular and hackable. It is
+written in a mix of C (for performance sensitive parts) and Python (for
+easy hackability of the UI). It does not depend on any large and complex
+UI toolkit, using only OpenGL for rendering everything.
+
+Finally, |kitty| is designed from the ground up to support all modern
+terminal features, such as unicode, true color, bold/italic fonts, text
+formatting, etc. It even extends existing text formatting escape codes,
+to add support for features not available elsewhere, such as colored and
+styled (curly) underlines. One of the design goals of |kitty| is to be
+easily extensible so that new features can be added in the future with
+relatively little effort.
+
+.. include:: basic.rst
+
+
+Configuring kitty
+-------------------
+
+|kitty| is highly configurable, everything from keyboard shortcuts to
+painting frames-per-second. Press :sc:`edit_config_file` in kitty
+to open its fully commented sample config file in your text editor.
+For details see the :doc:`configuration docs <conf>`.
+
+.. toctree::
+   :hidden:
+
+   conf
+
+
+.. _layouts:
+
+Layouts
+----------
+
+A :term:`layout` is an arrangement of multiple :term:`kitty windows <window>`
+inside a top-level :term:`OS window <os_window>`. The layout manages all its
+windows automatically, resizing and moving them as needed. You can create a new
+:term:`window` using the :sc:`new_window` key combination.
+
+Currently, there are seven layouts available:
+
+* **Fat** -- One (or optionally more) windows are shown full width on the top, the rest of the windows are shown side-by-side on the bottom
+* **Grid** -- All windows are shown in a grid
+* **Horizontal** -- All windows are shown side-by-side
+* **Splits** -- Windows arranged in arbitrary patterns created using horizontal and vertical splits
+* **Stack** -- Only a single maximized window is shown at a time
+* **Tall** -- One (or optionally more) windows are shown full height on the left, the rest of the windows are shown one below the other on the right
+* **Vertical** -- All windows are shown one below the other
+
+By default, all layouts are enabled and you can switch between layouts using
+the :sc:`next_layout` key combination. You can also create shortcuts to select
+particular layouts, and choose which layouts you want to enable/disable, see
+:ref:`conf-kitty-shortcuts.layout` for examples. The first layout listed in
+:opt:`enabled_layouts` becomes the default layout.
+
+For more details on the layouts and how to use them see :doc:`the documentation
+<layouts>`.
+
+.. toctree::
+   :hidden:
+
+   layouts
+
+Extending kitty
+------------------
+
+kitty has a powerful framework for scripting. You can create small terminal
+programs called :doc:`kittens <kittens_intro>`. These can used to add features
+to kitty, for example, :doc:`editing remote files <kittens/remote_file>` or
+:doc:`inputting unicode characters <kittens/unicode-input>`. They can also be
+used to create programs that leverage kitty's powerful features, for example,
+:doc:`viewing images <kittens/icat>` or :doc:`diffing files with images
+<kittens/diff>`.
+
+You can :doc:`create your own kittens to scratch your own itches
+<kittens/custom>`.
+
+For a list of all the builtin kittens, :ref:`see here <kittens>`.
+
+.. toctree::
+   :hidden:
+
+   kittens_intro
+
+
+Remote control
+------------------
+
+|kitty| has a very powerful system that allows you to control it from the
+:doc:`shell prompt, even over SSH <remote-control>`. You can change colors,
+fonts, open new :term:`windows <window>`, :term:`tabs <tab>`, set their titles,
+change window layout, get text
+from one window and send text to another, etc, etc. The possibilities are
+endless. See the :doc:`tutorial <remote-control>` to get started.
+
+.. toctree::
+   :hidden:
+
+   remote-control
+
+
+.. _sessions:
+
+Startup Sessions
+------------------
+
+You can control the :term:`tabs <tab>`, `:term:`kitty window <window>` layout,
+working directory, startup programs,
+etc. by creating a "session" file and using the :option:`kitty --session`
+command line flag or the :opt:`startup_session` option in :file:`kitty.conf`.
+For example:
+
+.. code-block:: session
+
+    # Set the layout for the current tab
+    layout tall
+    # Set the working directory for windows in the current tab
+    cd ~
+    # Create a window and run the specified command in it
+    launch zsh
+    # Create a window with some environment variables set and run
+    # vim in it
+    launch --env FOO=BAR vim
+    # Set the title for the next window
+    launch --title "Chat with x" irssi --profile x
+
+    # Create a new tab (the part after new_tab is the optional tab
+    # name which will be displayed in the tab bar, if omitted, the
+    # title of the active window will be used instead)
+    new_tab my tab
+    cd ~/somewhere
+    # Set the layouts allowed in this tab
+    enabled_layouts tall, stack
+    # Set the current layout
+    layout stack
+    launch zsh
+
+    # Create a new OS window
+    new_os_window
+    # set new window size to 80x25 cells
+    os_window_size 80c 25c
+    # set the --class for the new OS window
+    os_window_class mywindow
+    launch sh
+    # Make the current window the active (focused) window
+    focus
+    launch emacs
+
+.. note::
+    The :doc:`launch <launch>` command when used in a session file
+    cannot create new OS windows, or tabs.
+
+
+Creating tabs/windows
+-------------------------------
+
+kitty can be told to run arbitrary programs in new :term:`tabs <tab>`,
+:term:`windows <window>` or :term:`overlays <overlay>` at a keypress.
+To learn how to do this, see :doc:`here <launch>`.
+
+.. toctree::
+   :hidden:
+
+   launch
+
+
+Mouse features
+-------------------
+
+* You can click on a URL to open it in a browser.
+* You can double click to select a word and then drag to select more words.
+* You can triple click to select a line and then drag to select more lines.
+* You can triple click while holding :kbd:`ctrl+alt` to select from clicked
+  point to end of line.
+* You can right click to extend a previous selection.
+* You can hold down :kbd:`ctrl+alt` and drag with the mouse to select in
+  columns.
+* Selecting text automatically copies it to the primary clipboard (on
+  platforms with a primary clipboard).
+* You can middle click to paste from the primary clipboard (on platforms
+  with a primary clipboard).
+* You can select text with kitty even when a terminal program has grabbed
+  the mouse by holding down the :kbd:`shift` key.
+
+All these actions can be customized in :file:`kitty.conf` as described
+:ref:`here <conf-kitty-mouse.mousemap>`.
+
+You can also customize what happens when clicking on :term:`hyperlinks` in kitty,
+having it open files in your editor, download remote files, open things
+in your browser, etc.
+
+For details, see :doc:`here <open_actions>`.
+
+.. toctree::
+   :hidden:
+
+   open_actions
+
+Font control
+-----------------
+
+|kitty| has extremely flexible and powerful font selection features. You can
+specify individual families for the regular, bold, italic and bold+italic
+fonts. You can even specify specific font families for specific ranges of
+unicode characters. This allows precise control over text rendering. It can
+come in handy for applications like powerline, without the need to use patched
+fonts. See the various font related configuration directives in
+:ref:`conf-kitty-fonts`.
+
+
+.. _scrollback:
+
+The scrollback buffer
+-----------------------
+
+|kitty| supports scrolling back to view history, just like most terminals. You
+can use either keyboard shortcuts or the mouse scroll wheel to do so.  However,
+|kitty| has an extra, neat feature. Sometimes you need to explore the
+scrollback buffer in more detail, maybe search for some text or refer to it
+side-by-side while typing in a follow-up command. |kitty| allows you to do this
+by pressing the :sc:`show_scrollback` key-combination, which will open the
+scrollback buffer in your favorite pager program (which is ``less`` by default).
+Colors and text formatting are preserved. You can explore the scrollback buffer
+comfortably within the pager.
+
+Additionally, you can pipe the contents of the scrollback buffer to an
+arbitrary, command running in a new :term:`window`, :term:`tab` or :term:`overlay`,
+for example::
+
+   map f1 launch --stdin-source=@screen_scrollback --stdin-add-formatting less +G -R
+
+Would open the scrollback buffer in a new :term:`window` when you press the :kbd:`F1`
+key. See :sc:`show_scrollback` for details.
+
+If you want to use it with an editor such as vim to get more powerful features,
+you can see tips for doing so, in
+`this thread <https://github.com/kovidgoyal/kitty/issues/719>`_.
+
+If you wish to store very large amounts of scrollback to view using the piping or
+:sc:`show_scrollback` features, you can use the :opt:`scrollback_pager_history_size`
+option.
+
+.. _cpbuf:
+
+Multiple copy/paste buffers
+-----------------------------
+
+In addition to being able to copy/paste from the system clipboard, in |kitty| you
+can also setup an arbitrary number of copy paste buffers. To do so, simply add
+something like the following to your :file:`kitty.conf`::
+
+   map f1 copy_to_buffer a
+   map f2 paste_from_buffer a
+
+This will allow you to press :kbd:`F1` to copy the current selection to an
+internal buffer named ``a`` and :kbd:`F2` to paste from that buffer. The buffer
+names are arbitrary strings, so you can define as many such buffers as you
+need.
+
+
+Marks
+-------------
+
+kitty has the ability to mark text on the screen based on regular expressions.
+This can be useful to highlight words or phrases when browsing output from long
+running programs or similar. To learn how this feature works, see :doc:`marks`.
+
+.. toctree::
+   :hidden:
+
+   marks

docs/performance.rst

@@ -1,4 +1,4 @@
-|kitty| Performance
+Performance
 ===================
 
 The main goals for |kitty| performance are user perceived latency while typing

docs/pipe.rst

@@ -1,3 +1,5 @@
+:orphan:
+
 Working with the screen and history buffer contents
 ======================================================
 

docs/protocol-extensions.rst

@@ -1,7 +1,7 @@
-Extensions to the xterm protocol
+Terminal protocol extensions
 ===================================
 
-|kitty| has a few extensions to the xterm protocol, to enable advanced features.
+|kitty| has extensions to the legacy terminal protocol, to enable advanced features.
 These are typically in the form of new or re-purposed escape codes. While these
 extensions are currently |kitty| specific, it would be nice to get some of them
 adopted more broadly, to push the state of terminal emulators forward.
@@ -19,301 +19,14 @@ If you wish to discuss these extensions, propose additions/changes to them
 please do so by opening issues in the `GitHub
 <https://github.com/kovidgoyal/kitty/issues>`_ bug tracker.
 
-.. contents::
-   :local:
 
-Colored and styled underlines
--------------------------------
+.. toctree::
+   :maxdepth: 1
 
-|kitty| supports colored and styled (wavy) underlines. This is of particular
-use in terminal editors such as vim and emacs to display red, wavy underlines
-under mis-spelled words and/or syntax errors. This is done by re-purposing some
-SGR escape codes that are not used in modern terminals (`CSI codes
-<https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences>`_)
-
-To set the underline style::
-
-    <ESC>[4:0m  # this is no underline
-    <ESC>[4:1m  # this is a straight underline
-    <ESC>[4:2m  # this is a double underline
-    <ESC>[4:3m  # this is a curly underline
-    <ESC>[4:4m  # this is a dotted underline (not implemented in kitty)
-    <ESC>[4:5m  # this is a dashed underline (not implemented in kitty)
-    <ESC>[4m    # this is a straight underline (for backwards compat)
-    <ESC>[24m   # this is no underline (for backwards compat)
-
-To set the underline color (this is reserved and as far as I can tell not actually used for anything)::
-
-    <ESC>[58...m
-
-This works exactly like the codes ``38, 48`` that are used to set foreground and
-background color respectively.
-
-To reset the underline color (also previously reserved and unused)::
-
-    <ESC>[59m
-
-The underline color must remain the same under reverse video, if it has a
-color, if not, it should follow the foreground color.
-
-To detect support for this feature in a terminal emulator, query the terminfo database
-for the ``Su`` boolean capability.
-
-Graphics rendering
----------------------
-
-See :doc:`/graphics-protocol` for a description
-of this protocol to enable drawing of arbitrary raster images in the terminal.
-
-
-.. _extended-key-protocol:
-
-Keyboard handling
--------------------
-
-kitty has a :doc:`keyboard protocol <keyboard-protocol>` for reporting key
-presses to terminal applications that solves all key handling issues in
-terminal applications.
-
-.. _ext_styles:
-
-Setting text styles/colors in arbitrary regions of the screen
-------------------------------------------------------------------
-
-There already exists an escape code to set *some* text attributes in arbitrary
-regions of the screen, `DECCARA
-<https://vt100.net/docs/vt510-rm/DECCARA.html>`_.  However, it is limited to
-only a few attributes. |kitty| extends this to work with *all* SGR attributes.
-So, for example, this can be used to set the background color in an arbitrary
-region of the screen.
-
-The motivation for this extension is the various problems with the existing
-solution for erasing to background color, namely the *background color erase
-(bce)* capability. See
-`this discussion <https://github.com/kovidgoyal/kitty/issues/160#issuecomment-346470545>`_
-and `this FAQ <https://invisible-island.net/ncurses/ncurses.faq.html#bce_mismatches>`_
-for a summary of problems with *bce*.
-
-For example, to set the background color to blue in a
-rectangular region of the screen from (3, 4) to (10, 11), you use::
-
-    <ESC>[2*x<ESC>[4;3;11;10;44$r<ESC>[*x
-
-
-Saving and restoring colors
----------------------------------------------------------------------------------
-
-It is often useful for a full screen application with its own color themes to
-set the default foreground, background, selection and cursor colors and the
-ANSI color table. This allows for various performance optimizations when
-drawing the screen. The problem is that if the user previously used the escape
-codes to change these colors herself, then running the full screen application
-will lose her changes even after it exits. To avoid this, kitty introduces a
-new pair of *OSC* escape codes to push and pop the current color values from a
-stack::
-
-    <ESC>]30001<ESC>\  # push onto stack
-    <ESC>]30101<ESC>\  # pop from stack
-
-These escape codes save/restore the colors, default
-background, default foreground, selection background, selection foreground and
-cursor color and the 256 colors of the ANSI color table.
-
-.. note:: In July 2020, after several years, XTerm copied this protocol
-   extension, without acknowledgement, and using incompatible escape codes
-   (XTPUSHCOLORS, XTPOPCOLORS, XTREPORTCOLORS). And they decided to save not
-   just the dynamic colors but the entire ANSI color table. In the interests of
-   promoting interoperability, kitty added support for XTerm's escape codes as
-   well, and changed this extension to also save/restore the entire ANSI color
-   table.
-
-
-Pasting to clipboard
-----------------------
-
-|kitty| implements the OSC 52 escape code protocol to get/set the clipboard
-contents (controlled via the :opt:`clipboard_control` setting). There is one
-difference in kitty's implementation compared to some other terminal emulators.
-|kitty| allows sending arbitrary amounts of text to the clipboard. It does so
-by modifying the protocol slightly. Successive OSC 52 escape codes to set the
-clipboard will concatenate, so::
-
-    <ESC>]52;c;<payload1><ESC>\
-    <ESC>]52;c;<payload2><ESC>\
-
-will result in the clipboard having the contents ``payload1 + payload2``. To
-send a new string to the clipboard send an OSC 52 sequence with an invalid payload
-first, for example::
-
-    <ESC>]52;c;!<ESC>\
-
-Here ``!`` is not valid base64 encoded text, so it clears the clipboard.
-Further, since it is invalid, it should be ignored by terminal emulators
-that do not support this extension, thereby making it safe to use, simply
-always send it before starting a new OSC 52 paste, even if you aren't chunking
-up large pastes, that way kitty won't concatenate your paste, and it will have
-no ill-effects in other terminal emulators.
-
-In case you're using software that can't be easily adapted to this
-protocol extension, it can be disabled by specifying ``no-append`` to the
-:opt:`clipboard_control` setting.
-
-
-.. _unscroll:
-
-Unscrolling the screen
------------------------
-
-This is a small extension to the `SD (Pan up) escape code
-<https://vt100.net/docs/vt510-rm/SD.html>`_ from the VT-420 terminal. The
-``SD`` escape code normally causes the text on screen to scroll down by the
-specified number of lines, with empty lines appearing at the top of the screen.
-This extension allows the new lines to be filled in from the scrollback buffer
-instead of being blank.
-
-The motivation for this is that many modern shells will show completions in a
-block of lines under the cursor, this causes some of the on-screen text to be
-lost even after the completion is completed, because it has scrolled off
-screen. This escape code allows that text to be restored.
-
-If the scrollback buffer is empty or there is no scrollback buffer, such as for
-the alternate screen, then the newly inserted lines must be empty, just as with
-the original ``SD`` escape code. The maximum number of lines that can be
-scrolled down is implementation defined, but must be at least one screen worth.
-
-The syntax of the escape code is identical to that of ``SD`` except that it has
-a trailing ``+`` modifier. This is legal under the `ECMA 48 standard
-<https://www.ecma-international.org/publications-and-standards/standards/ecma-48/>`_
-and unused for any other purpose as far as I can tell. So for example, to
-unscroll three lines, the escape code would be::
-
-    CSI 3 + T
-
-See `discussion here
-<https://gitlab.freedesktop.org/terminal-wg/specifications/-/issues/30>`_.
-
-.. versionadded:: 0.20.2
-
-
-.. _desktop_notifications:
-
-
-Desktop notifications
----------------------------------
-
-|kitty| implements an extensible escape code (OSC 99) to show desktop
-notifications. It is easy to use from shell scripts and fully extensible to
-show title and body.  Clicking on the notification can optionally focus the
-window it came from, and/or send an escape code back to the application running
-in that window.
-
-The design of the escape code is partially based on the discussion in
-the defunct
-`terminal-wg <https://gitlab.freedesktop.org/terminal-wg/specifications/-/issues/13>`_
-
-The escape code has the form::
-
-    <OSC> 99 ; metadata ; payload <terminator>
-
-Here ``<OSC>`` is :code:`<ESC>]` and ``<terminator>`` is
-:code:`<ESC><backslash>`.  The metadata is a section of colon separated
-:code:`key=value` pairs. Every key must be a single character from the set
-:code:`a-zA-Z` and every value must be a word consisting of characters from
-the set :code:`a-zA-Z0-9-_/\+.,(){}[]*&^%$#@!`~`. The payload must be
-interpreted based on the metadata section. The two semi-colons *must* always be
-present even when no metadata is present.
-
-Before going into details, lets see how one can display a simple, single line
-notification from a shell script::
-
-    printf '\x1b]99;;Hello world\x1b\\'
-
-To show a message with a title and a body::
-
-    printf '\x1b]99;i=1:d=0;Hello world\x1b\\'
-    printf '\x1b]99;i=1:d=1:p=body;This is cool\x1b\\'
-
-The most important key in the metadata is the ``p`` key, it controls how the
-payload is interpreted. A value of ``title`` means the payload is setting the
-title for the notification. A value of ``body`` means it is setting the body,
-and so on, see the table below for full details.
-
-The design of the escape code is fundamentally chunked, this is because
-different terminal emulators have different limits on how large a single escape
-code can be. Chunking is accomplished by the ``i`` and ``d`` keys. The ``i``
-key is the *notification id* which can be any string containing the characters
-``[a-zA-Z0-9_-+.]``. The ``d`` key stands for *done* and
-can only take the values ``0`` and ``1``. A value of ``0`` means the
-notification is not yet done and the terminal emulator should hold off
-displaying it. A value of ``1`` means the notification is done, and should be
-displayed. You can specify the title or body multiple times and the terminal
-emulator will concatenate them, thereby allowing arbitrarily long text
-(terminal emulators are free to impose a sensible limit to avoid
-Denial-of-Service attacks).
-
-Both the ``title`` and ``body`` payloads must be either UTF-8 encoded plain
-text with no embedded escape codes, or UTF-8 text that is base64 encoded, in
-which case there must be an ``e=1`` key in the metadata to indicate the payload
-is base64 encoded.
-
-When the user clicks the notification, a couple of things can happen, the
-terminal emulator can focus the window from which the notification came, and/or
-it can send back an escape code to the application indicating the notification
-was activated. This is controlled by the ``a`` key which takes a comma
-separated set of values, ``report`` and ``focus``. The value ``focus`` means
-focus the window from which the notification was issued and is the default.
-``report`` means send an escape code back to the application. The format of the
-returned escape code is::
-
-    <OSC> 99 ; i=identifier ; <terminator>
-
-The value of ``identifier`` comes from the ``i`` key in the escape code sent by
-the application. If the application sends no identifier, then the terminal
-*must* use ``i=0``. Actions can be preceded by a negative sign to turn them
-off, so for example if you do not want any action, turn off the default
-``focus`` action with::
-
-    a=-focus
-
-Complete specification of all the metadata keys is in the table below. If a
-terminal emulator encounters a key in the metadata it does not understand,
-the key *must* be ignored, to allow for future extensibility of this escape
-code. Similarly if values for known keys are unknown, the terminal emulator
-*should* either ignore the entire escape code or perform a best guess effort
-to display it based on what it does understand.
-
-.. note::
-   It is possible to extend this escape code to allow specifying an icon for
-   the notification, however, given that some platforms, such as macOS, dont
-   allow displaying custom icons on a notification, at all, it was decided to
-   leave it out of the spec for the time being.
-
-   Similarly, features such as scheduled notifications could be added in future
-   revisions.
-
-
-=======  ====================  =========  =================
-Key      Value                 Default    Description
-=======  ====================  =========  =================
-``a``    Comma separated list  ``focus``  What action to perform when the
-         of ``report``,                   notification is clicked
-         ``focus``, with
-         optional leading
-         ``-``
-
-``d``    ``0`` or ``1``        ``1``      Indicates if the notification is
-                                          complete or not.
-
-``e``    ``0`` or ``1``        ``0``      If set to ``1`` means the payload is base64 encoded UTF-8,
-                                          otherwise it is plain UTF-8 text with no C0 control codes in it
-
-``i``    ``[a-zA-Z0-9-_+.]``   ``0``      Identifier for the notification
-
-``p``    One of ``title`` or   ``title``  Whether the payload is the notification title or body. If a
-         ``body``.                        notification has no title, the body will be used as title.
-=======  ====================  =========  =================
-
-
-.. note::
-   |kitty| also supports the legacy OSC 9 protocol developed by iTerm2 for
-   desktop notifications.
+   underlines
+   graphics-protocol
+   keyboard-protocol
+   desktop-notifications
+   unscroll
+   color-stack
+   deccara

docs/quickstart.rst

@@ -0,0 +1,28 @@
+.. _quickstart:
+
+Quickstart
+===========
+
+.. toctree::
+   :hidden:
+
+   binary
+   build
+
+Pre-built binaries of |kitty| are available for both macOS and Linux.
+See the :doc:`binary install instructions </binary>`. You can also
+:doc:`build from source </build>`.
+
+Additionally, you can use your favorite package manager to install the |kitty|
+package, but note that some Linux distribution packages are woefully outdated.
+|kitty| is available in a vast number of package repositories for macOS
+and Linux.
+
+.. image:: https://repology.org/badge/tiny-repos/kitty.svg
+   :target: https://repology.org/project/kitty/versions
+   :alt: Number of repositories kitty is available in
+
+See :doc:`Configuring kitty <conf>` for help on configuring |kitty| and
+:doc:`Invocation <invocation>` for the command line arguments |kitty| supports.
+
+For a tour of kitty's design and features, see the :doc:`/overview`.

docs/rc_protocol.rst

@@ -1,5 +1,5 @@
-Documentation for the kitty remote control protocol
-======================================================
+The kitty remote control protocol
+==================================
 
 The kitty remote control protocol is a simple protocol that involves sending
 data to kitty in the form of JSON. Any individual command of kitty has the
@@ -37,5 +37,4 @@ with the following command line::
 
     echo -en '\eP@kitty-cmd{"cmd":"ls","version":[0,14,2]}\e\' | socat - unix:/tmp/test | awk '{ print substr($0, 13, length($0) - 14) }' | jq -c '.data | fromjson' | jq .
 
-
 .. include:: generated/rc.rst

docs/remote-control.rst

@@ -1,17 +1,15 @@
-:tocdepth: 2
-
-Controlling kitty from scripts or the shell
-==============================================
+Control kitty from scripts
+----------------------------
 
 .. highlight:: sh
 
-Tutorial
-----------
-
 |kitty| can be controlled from scripts or the shell prompt. You can open new
 windows, send arbitrary text input to any window, name windows and tabs, etc.
 Let's walk through a few examples of controlling |kitty|.
 
+Tutorial
+------------
+
 Start by running |kitty| as::
 
     kitty -o allow_remote_control=yes -o enabled_layouts=tall
@@ -142,6 +140,8 @@ running on other computers (for example, over ssh) or as other users.
    kitty, as if you were running with ``allow_remote_control`` turned on.
 
 
+.. _rc_mapping:
+
 Mapping key presses to remote control commands
 --------------------------------------------------
 
@@ -172,10 +172,15 @@ Now press, F1 and start typing, what you type will be sent to all windows,
 live, as you type it.
 
 
-Documentation for the remote control protocol
+The remote control protocol
 -----------------------------------------------
 
 If you wish to develop your own client to talk to |kitty|, you
-can use the :doc:`rc_protocol`.
+can use the :doc:`protocol specification <rc_protocol>`.
+
+.. toctree::
+   :hidden:
+
+   rc_protocol
 
 .. include:: generated/cli-kitty-at.rst

docs/requirements.txt

@@ -0,0 +1,5 @@
+sphinx
+furo
+sphinx-copybutton
+sphinxext-opengraph
+sphinx-inline-tabs

docs/underlines.rst

@@ -0,0 +1,36 @@
+Colored and styled underlines
+================================
+
+|kitty| supports colored and styled (wavy) underlines. This is of particular
+use in terminal editors such as vim and emacs to display red, wavy underlines
+under mis-spelled words and/or syntax errors. This is done by re-purposing some
+SGR escape codes that are not used in modern terminals (`CSI codes
+<https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_(Control_Sequence_Introducer)_sequences>`_)
+
+To set the underline style::
+
+    <ESC>[4:0m  # this is no underline
+    <ESC>[4:1m  # this is a straight underline
+    <ESC>[4:2m  # this is a double underline
+    <ESC>[4:3m  # this is a curly underline
+    <ESC>[4:4m  # this is a dotted underline (not implemented in kitty)
+    <ESC>[4:5m  # this is a dashed underline (not implemented in kitty)
+    <ESC>[4m    # this is a straight underline (for backwards compat)
+    <ESC>[24m   # this is no underline (for backwards compat)
+
+To set the underline color (this is reserved and as far as I can tell not actually used for anything)::
+
+    <ESC>[58...m
+
+This works exactly like the codes ``38, 48`` that are used to set foreground and
+background color respectively.
+
+To reset the underline color (also previously reserved and unused)::
+
+    <ESC>[59m
+
+The underline color must remain the same under reverse video, if it has a
+color, if not, it should follow the foreground color.
+
+To detect support for this feature in a terminal emulator, query the terminfo database
+for the ``Su`` boolean capability.

docs/unscroll.rst

@@ -0,0 +1,34 @@
+.. _unscroll:
+
+Unscrolling the screen
+========================
+
+This is a small extension to the `SD (Pan up) escape code
+<https://vt100.net/docs/vt510-rm/SD.html>`_ from the VT-420 terminal. The
+``SD`` escape code normally causes the text on screen to scroll down by the
+specified number of lines, with empty lines appearing at the top of the screen.
+This extension allows the new lines to be filled in from the scrollback buffer
+instead of being blank.
+
+The motivation for this is that many modern shells will show completions in a
+block of lines under the cursor, this causes some of the on-screen text to be
+lost even after the completion is completed, because it has scrolled off
+screen. This escape code allows that text to be restored.
+
+If the scrollback buffer is empty or there is no scrollback buffer, such as for
+the alternate screen, then the newly inserted lines must be empty, just as with
+the original ``SD`` escape code. The maximum number of lines that can be
+scrolled down is implementation defined, but must be at least one screen worth.
+
+The syntax of the escape code is identical to that of ``SD`` except that it has
+a trailing ``+`` modifier. This is legal under the `ECMA 48 standard
+<https://www.ecma-international.org/publications-and-standards/standards/ecma-48/>`_
+and unused for any other purpose as far as I can tell. So for example, to
+unscroll three lines, the escape code would be::
+
+    CSI 3 + T
+
+See `discussion here
+<https://gitlab.freedesktop.org/terminal-wg/specifications/-/issues/30>`_.
+
+.. versionadded:: 0.20.2

gen-apc-parsers.py

@@ -250,7 +250,7 @@ def write_header(text: str, path: str) -> None:
 def graphics_parser() -> None:
     flag = frozenset
     keymap: KeymapType = {
-        'a': ('action', flag('tTqpdfa')),
+        'a': ('action', flag('tTqpdfac')),
         'd': ('delete_action', flag('aAiIcCfFnNpPqQxXyYzZ')),
         't': ('transmission_type', flag('dfts')),
         'o': ('compressed', flag('z')),

gen-config.py

@@ -2,426 +2,8 @@
 # vim:fileencoding=utf-8
 # License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
 
-import inspect
-import os
-import pprint
-import re
-import textwrap
-from typing import (
-    Any, Callable, Dict, Iterator, List, Set, Tuple, Union, get_type_hints
-)
 
-from kitty.conf.types import Definition, MultiOption, Option, unset
-
-
-def chunks(lst: List, n: int) -> Iterator[List]:
-    for i in range(0, len(lst), n):
-        yield lst[i:i + n]
-
-
-def atoi(text: str) -> str:
-    return f'{int(text):08d}' if text.isdigit() else text
-
-
-def natural_keys(text: str) -> Tuple[str, ...]:
-    return tuple(atoi(c) for c in re.split(r'(\d+)', text))
-
-
-def generate_class(defn: Definition, loc: str) -> Tuple[str, str]:
-    class_lines: List[str] = []
-    tc_lines: List[str] = []
-    a = class_lines.append
-    t = tc_lines.append
-    a('class Options:')
-    t('class Parser:')
-    choices = {}
-    imports: Set[Tuple[str, str]] = set()
-    tc_imports: Set[Tuple[str, str]] = set()
-
-    def type_name(x: type) -> str:
-        ans = x.__name__
-        if x.__module__ and x.__module__ != 'builtins':
-            imports.add((x.__module__, x.__name__))
-        return ans
-
-    def option_type_as_str(x: Any) -> str:
-        if hasattr(x, '__name__'):
-            return type_name(x)
-        ans = repr(x)
-        ans = ans.replace('NoneType', 'None')
-        return ans
-
-    def option_type_data(option: Union[Option, MultiOption]) -> Tuple[Callable, str]:
-        func = option.parser_func
-        if func.__module__ == 'builtins':
-            return func, func.__name__
-        th = get_type_hints(func)
-        rettype = th['return']
-        typ = option_type_as_str(rettype)
-        if isinstance(option, MultiOption):
-            typ = typ[typ.index('[') + 1:-1]
-            typ = typ.replace('Tuple', 'Dict', 1)
-        return func, typ
-
-    is_mutiple_vars = {}
-    option_names = set()
-    color_table = list(map(str, range(256)))
-
-    def parser_function_declaration(option_name: str) -> None:
-        t('')
-        t(f'    def {option_name}(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:')
-
-    for option in sorted(defn.iter_all_options(), key=lambda a: natural_keys(a.name)):
-        option_names.add(option.name)
-        parser_function_declaration(option.name)
-        if isinstance(option, MultiOption):
-            mval: Dict[str, Dict[str, Any]] = {'macos': {}, 'linux': {}, '': {}}
-            func, typ = option_type_data(option)
-            for val in option:
-                if val.add_to_default:
-                    gr = mval[val.only]
-                    for k, v in func(val.defval_as_str):
-                        gr[k] = v
-            is_mutiple_vars[option.name] = typ, mval
-            sig = inspect.signature(func)
-            tc_imports.add((func.__module__, func.__name__))
-            if len(sig.parameters) == 1:
-                t(f'        for k, v in {func.__name__}(val):')
-                t(f'            ans["{option.name}"][k] = v')
-            else:
-                t(f'        for k, v in {func.__name__}(val, ans["{option.name}"]):')
-                t(f'            ans["{option.name}"][k] = v')
-            continue
-
-        if option.choices:
-            typ = 'typing.Literal[{}]'.format(', '.join(repr(x) for x in option.choices))
-            ename = f'choices_for_{option.name}'
-            choices[ename] = typ
-            typ = ename
-            func = str
-        elif defn.has_color_table and option.is_color_table_color:
-            func, typ = option_type_data(option)
-            t(f'        ans[{option.name!r}] = {func.__name__}(val)')
-            tc_imports.add((func.__module__, func.__name__))
-            cnum = int(option.name[5:])
-            color_table[cnum] = '0x{:06x}'.format(func(option.defval_as_string).__int__())
-            continue
-        else:
-            func, typ = option_type_data(option)
-            try:
-                params = inspect.signature(func).parameters
-            except Exception:
-                params = {}
-            if 'dict_with_parse_results' in params:
-                t(f'        {func.__name__}(val, ans)')
-            else:
-                t(f'        ans[{option.name!r}] = {func.__name__}(val)')
-            if func.__module__ != 'builtins':
-                tc_imports.add((func.__module__, func.__name__))
-
-        defval = repr(func(option.defval_as_string))
-        if option.macos_defval is not unset:
-            md = repr(func(option.macos_defval))
-            defval = f'{md} if is_macos else {defval}'
-            imports.add(('kitty.constants', 'is_macos'))
-        a(f'    {option.name}: {typ} = {defval}')
-        if option.choices:
-            t('        val = val.lower()')
-            t(f'        if val not in self.choices_for_{option.name}:')
-            t(f'            raise ValueError(f"The value {{val}} is not a valid choice for {option.name}")')
-            t(f'        ans["{option.name}"] = val')
-            t('')
-            t(f'    choices_for_{option.name} = frozenset({option.choices!r})')
-
-    for option_name, (typ, mval) in is_mutiple_vars.items():
-        a(f'    {option_name}: {typ} = ' '{}')
-
-    for parser, aliases in defn.deprecations.items():
-        for alias in aliases:
-            parser_function_declaration(alias)
-            tc_imports.add((parser.__module__, parser.__name__))
-            t(f'        {parser.__name__}({alias!r}, val, ans)')
-
-    action_parsers = {}
-
-    def resolve_import(ftype: str) -> str:
-        if '.' in ftype:
-            fmod, ftype = ftype.rpartition('.')[::2]
-        else:
-            fmod = f'{loc}.options.utils'
-        imports.add((fmod, ftype))
-        return ftype
-
-    for aname, action in defn.actions.items():
-        option_names.add(aname)
-        action_parsers[aname] = func = action.parser_func
-        th = get_type_hints(func)
-        rettype = th['return']
-        typ = option_type_as_str(rettype)
-        typ = typ[typ.index('[') + 1:-1]
-        a(f'    {aname}: typing.List[{typ}] = []')
-        for imp in action.imports:
-            resolve_import(imp)
-        for fname, ftype in action.fields.items():
-            ftype = resolve_import(ftype)
-            a(f'    {fname}: {ftype} = ' '{}')
-        parser_function_declaration(aname)
-        t(f'        for k in {func.__name__}(val):')
-        t(f'            ans[{aname!r}].append(k)')
-        tc_imports.add((func.__module__, func.__name__))
-
-    if defn.has_color_table:
-        imports.add(('array', 'array'))
-        a('    color_table: array = array("L", (')
-        for grp in chunks(color_table, 8):
-            a('        ' + ', '.join(grp) + ',')
-        a('    ))')
-
-    a('    config_paths: typing.Tuple[str, ...] = ()')
-    a('    config_overrides: typing.Tuple[str, ...] = ()')
-    a('')
-    a('    def __init__(self, options_dict: typing.Optional[typing.Dict[str, typing.Any]] = None) -> None:')
-    a('        if options_dict is not None:')
-    a('            for key in option_names:')
-    a('                setattr(self, key, options_dict[key])')
-
-    a('')
-    a('    @property')
-    a('    def _fields(self) -> typing.Tuple[str, ...]:')
-    a('        return option_names')
-
-    a('')
-    a('    def __iter__(self) -> typing.Iterator[str]:')
-    a('        return iter(self._fields)')
-
-    a('')
-    a('    def __len__(self) -> int:')
-    a('        return len(self._fields)')
-
-    a('')
-    a('    def _copy_of_val(self, name: str) -> typing.Any:')
-    a('        ans = getattr(self, name)')
-    a('        if isinstance(ans, dict):\n            ans = ans.copy()')
-    a('        elif isinstance(ans, list):\n            ans = ans[:]')
-    a('        return ans')
-
-    a('')
-    a('    def _asdict(self) -> typing.Dict[str, typing.Any]:')
-    a('        return {k: self._copy_of_val(k) for k in self}')
-
-    a('')
-    a('    def _replace(self, **kw: typing.Any) -> "Options":')
-    a('        ans = Options()')
-    a('        for name in self:')
-    a('            setattr(ans, name, self._copy_of_val(name))')
-    a('        for name, val in kw.items():')
-    a('            setattr(ans, name, val)')
-    a('        return ans')
-
-    a('')
-    a('    def __getitem__(self, key: typing.Union[int, str]) -> typing.Any:')
-    a('        k = option_names[key] if isinstance(key, int) else key')
-    a('        try:')
-    a('            return getattr(self, k)')
-    a('        except AttributeError:')
-    a('            pass')
-    a('        raise KeyError(f"No option named: {k}")')
-
-    if defn.has_color_table:
-        a('')
-        a('    def __getattr__(self, key: str) -> typing.Any:')
-        a('        if key.startswith("color"):')
-        a('            q = key[5:]')
-        a('            if q.isdigit():')
-        a('                k = int(q)')
-        a('                if 0 <= k <= 255:')
-        a('                    x = self.color_table[k]')
-        a('                    return Color((x >> 16) & 255, (x >> 8) & 255, x & 255)')
-        a('        raise AttributeError(key)')
-        a('')
-        a('    def __setattr__(self, key: str, val: typing.Any) -> typing.Any:')
-        a('        if key.startswith("color"):')
-        a('            q = key[5:]')
-        a('            if q.isdigit():')
-        a('                k = int(q)')
-        a('                if 0 <= k <= 255:')
-        a('                    self.color_table[k] = int(val)')
-        a('                    return')
-        a('        object.__setattr__(self, key, val)')
-
-    a('')
-    a('')
-    a('defaults = Options()')
-    for option_name, (typ, mval) in is_mutiple_vars.items():
-        a(f'defaults.{option_name} = {mval[""]!r}')
-        if mval['macos']:
-            imports.add(('kitty.constants', 'is_macos'))
-            a('if is_macos:')
-            a(f'    defaults.{option_name}.update({mval["macos"]!r}')
-        if mval['macos']:
-            imports.add(('kitty.constants', 'is_macos'))
-            a('if not is_macos:')
-            a(f'    defaults.{option_name}.update({mval["linux"]!r}')
-
-    for aname, func in action_parsers.items():
-        a(f'defaults.{aname} = [')
-        only: Dict[str, List[Tuple[str, Callable]]] = {}
-        for sc in defn.iter_all_maps(aname):
-            if not sc.add_to_default:
-                continue
-            text = sc.parseable_text
-            if sc.only:
-                only.setdefault(sc.only, []).append((text, func))
-            else:
-                for val in func(text):
-                    a(f'    # {sc.name}')
-                    a(f'    {val!r},')
-        a(']')
-        if only:
-            imports.add(('kitty.constants', 'is_macos'))
-            for cond, items in only.items():
-                cond = 'is_macos' if cond == 'macos' else 'not is_macos'
-                a(f'if {cond}:')
-                for (text, func) in items:
-                    for val in func(text):
-                        a(f'    defaults.{aname}.append({val!r})')
-
-    t('')
-    t('')
-    t('def create_result_dict() -> typing.Dict[str, typing.Any]:')
-    t('    return {')
-    for oname in is_mutiple_vars:
-        t(f'        {oname!r}: {{}},')
-    for aname in defn.actions:
-        t(f'        {aname!r}: [],')
-    t('    }')
-
-    t('')
-    t('')
-    t(f'actions = frozenset({tuple(defn.actions)!r})')
-    t('')
-    t('')
-    t('def merge_result_dicts(defaults: typing.Dict[str, typing.Any], vals: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:')
-    t('    ans = {}')
-    t('    for k, v in defaults.items():')
-    t('        if isinstance(v, dict):')
-    t('            ans[k] = merge_dicts(v, vals.get(k, {}))')
-    t('        elif k in actions:')
-    t('            ans[k] = v + vals.get(k, [])')
-    t('        else:')
-    t('            ans[k] = vals.get(k, v)')
-    t('    return ans')
-    tc_imports.add(('kitty.conf.utils', 'merge_dicts'))
-
-    t('')
-    t('')
-    t('parser = Parser()')
-    t('')
-    t('')
-    t('def parse_conf_item(key: str, val: str, ans: typing.Dict[str, typing.Any]) -> bool:')
-    t('    func = getattr(parser, key, None)')
-    t('    if func is not None:')
-    t('        func(val, ans)')
-    t('        return True')
-    t('    return False')
-
-    preamble = ['# generated by gen-config.py DO NOT edit', '# vim:fileencoding=utf-8', '']
-    a = preamble.append
-
-    def output_imports(imports: Set, add_module_imports: bool = True) -> None:
-        a('import typing')
-        seen_mods = {'typing'}
-        mmap: Dict[str, List[str]] = {}
-        for mod, name in imports:
-            mmap.setdefault(mod, []).append(name)
-        for mod in sorted(mmap):
-            names = sorted(mmap[mod])
-            lines = textwrap.wrap(', '.join(names), 100)
-            if len(lines) == 1:
-                s = lines[0]
-            else:
-                s = '\n    '.join(lines)
-                s = f'(\n    {s}\n)'
-            a(f'from {mod} import {s}')
-            if add_module_imports and mod not in seen_mods and mod != s:
-                a(f'import {mod}')
-                seen_mods.add(mod)
-
-    output_imports(imports)
-    a('')
-    if choices:
-        a('if typing.TYPE_CHECKING:')
-        for name, cdefn in choices.items():
-            a(f'    {name} = {cdefn}')
-        a('else:')
-        for name in choices:
-            a(f'    {name} = str')
-
-    a('')
-    a('option_names = (  # {{''{')
-    a(' ' + pprint.pformat(tuple(sorted(option_names, key=natural_keys)))[1:] + '  # }}''}')
-    class_def = '\n'.join(preamble + ['', ''] + class_lines)
-
-    preamble = ['# generated by gen-config.py DO NOT edit', '# vim:fileencoding=utf-8', '']
-    a = preamble.append
-    output_imports(tc_imports, False)
-
-    return class_def, '\n'.join(preamble + ['', ''] + tc_lines)
-
-
-def generate_c_conversion(loc: str, ctypes: List[Option]) -> str:
-    lines: List[str] = []
-    basic_converters = {
-        'int': 'PyLong_AsLong', 'uint': 'PyLong_AsUnsignedLong', 'bool': 'PyObject_IsTrue',
-        'float': 'PyFloat_AsFloat', 'double': 'PyFloat_AsDouble',
-        'time': 'parse_s_double_to_monotonic_t', 'time-ms': 'parse_ms_long_to_monotonic_t'
-    }
-
-    for opt in ctypes:
-        lines.append('')
-        lines.append(f'static void\nconvert_from_python_{opt.name}(PyObject *val, Options *opts) ''{')
-        is_special = opt.ctype.startswith('!')
-        if is_special:
-            func = opt.ctype[1:]
-            lines.append(f'    {func}(val, opts);')
-        else:
-            func = basic_converters.get(opt.ctype, opt.ctype)
-            lines.append(f'    opts->{opt.name} = {func}(val);')
-        lines.append('}')
-        lines.append('')
-        lines.append(f'static void\nconvert_from_opts_{opt.name}(PyObject *py_opts, Options *opts) ''{')
-        lines.append(f'    PyObject *ret = PyObject_GetAttrString(py_opts, "{opt.name}");')
-        lines.append('    if (ret == NULL) return;')
-        lines.append(f'    convert_from_python_{opt.name}(ret, opts);')
-        lines.append('    Py_DECREF(ret);')
-        lines.append('}')
-
-    lines.append('')
-    lines.append('static bool\nconvert_opts_from_python_opts(PyObject *py_opts, Options *opts) ''{')
-    for opt in ctypes:
-        lines.append(f'    convert_from_opts_{opt.name}(py_opts, opts);')
-        lines.append('    if (PyErr_Occurred()) return false;')
-    lines.append('    return true;')
-    lines.append('}')
-
-    preamble = ['// generated by gen-config.py DO NOT edit', '// vim:fileencoding=utf-8', '#pragma once', '#include "to-c.h"']
-    return '\n'.join(preamble + ['', ''] + lines)
-
-
-def write_output(loc: str, defn: Definition) -> None:
-    cls, tc = generate_class(defn, loc)
-    with open(os.path.join(*loc.split('.'), 'options', 'types.py'), 'w') as f:
-        f.write(cls + '\n')
-    with open(os.path.join(*loc.split('.'), 'options', 'parse.py'), 'w') as f:
-        f.write(tc + '\n')
-    ctypes = []
-    for opt in defn.root_group.iter_all_non_groups():
-        if isinstance(opt, Option) and opt.ctype:
-            ctypes.append(opt)
-    if ctypes:
-        c = generate_c_conversion(loc, ctypes)
-        with open(os.path.join(*loc.split('.'), 'options', 'to-c-generated.h'), 'w') as f:
-            f.write(c + '\n')
+from kitty.conf.generate import write_output
 
 
 def main() -> None:

gen-wcwidth.py

@@ -388,7 +388,8 @@ def gen_ucd() -> None:
                 rmap[0xfe0e], rmap[0xfe0f]
             ))
     with open('kittens/hints/url_regex.py', 'w') as f:
-        f.write("url_delimiters = '{}'  # noqa".format(''.join(classes_to_regex(cz, exclude='\n'))))
+        f.write('# generated by gen-wcwidth.py, do not edit\n\n')
+        f.write("url_delimiters = '{}'  # noqa".format(''.join(classes_to_regex(cz, exclude='\n\r'))))
 
 
 def gen_names() -> None:

glfw/backend_utils.c

@@ -375,7 +375,7 @@ GLFWAPI char* utf_8_strndup(const char* source, size_t max_length) {
 int createAnonymousFile(off_t size) {
     int ret, fd = -1, shm_anon = 0;
 #ifdef HAS_MEMFD_CREATE
-    fd = memfd_create("glfw-shared", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+    fd = glfw_memfd_create("glfw-shared", MFD_CLOEXEC | MFD_ALLOW_SEALING);
     if (fd < 0) return -1;
     // We can add this seal before calling posix_fallocate(), as the file
     // is currently zero-sized anyway.

glfw/cocoa_monitor.m

@@ -41,8 +41,20 @@
 
 // Get the name of the specified display, or NULL
 //
-static char* getDisplayName(CGDirectDisplayID displayID)
+static char* getDisplayName(CGDirectDisplayID displayID, NSScreen* screen)
 {
+    // IOKit doesn't work on Apple Silicon anymore
+    // Luckily, 10.15 introduced -[NSScreen localizedName].
+    // Use it if available, and fall back to IOKit otherwise.
+    if (screen)
+    {
+        if ([screen respondsToSelector:@selector(localizedName)])
+        {
+            NSString* name = [screen valueForKey:@"localizedName"];
+            if (name)
+                return _glfw_strdup([name UTF8String]);
+        }
+    }
     io_iterator_t it;
     io_service_t service;
     CFDictionaryRef info;
@@ -52,7 +64,7 @@ static char* getDisplayName(CGDirectDisplayID displayID)
                                      &it) != 0)
     {
         // This may happen if a desktop Mac is running headless
-        return NULL;
+        return _glfw_strdup("Display");
     }
 
     while ((service = IOIteratorNext(it)) != 0)
@@ -89,8 +101,8 @@ static char* getDisplayName(CGDirectDisplayID displayID)
     if (!service)
     {
         _glfwInputError(GLFW_PLATFORM_ERROR,
-                        "Cocoa: Failed to find service port for display, cannot get its name, using Unknown");
-        return NULL;
+                        "Cocoa: Failed to find service port for display");
+        return _glfw_strdup("Display");
     }
 
     CFDictionaryRef names =
@@ -103,7 +115,7 @@ static char* getDisplayName(CGDirectDisplayID displayID)
     {
         // This may happen if a desktop Mac is running headless
         CFRelease(info);
-        return NULL;
+        return _glfw_strdup("Display");
     }
 
     const CFIndex size =
@@ -381,28 +393,46 @@ void _glfwPollMonitorsNS(void)
         if (CGDisplayIsAsleep(displays[i]))
             continue;
 
+        const uint32_t unitNumber = CGDisplayUnitNumber(displays[i]);
+        NSScreen* screen = nil;
+
+        for (screen in [NSScreen screens])
+        {
+            NSNumber* screenNumber = [screen deviceDescription][@"NSScreenNumber"];
+
+            // HACK: Compare unit numbers instead of display IDs to work around
+            //       display replacement on machines with automatic graphics
+            //       switching
+            if (CGDisplayUnitNumber([screenNumber unsignedIntValue]) == unitNumber)
+                break;
+        }
+
         // HACK: Compare unit numbers instead of display IDs to work around
         //       display replacement on machines with automatic graphics
         //       switching
-        const uint32_t unitNumber = CGDisplayUnitNumber(displays[i]);
-
-        for (uint32_t j = 0;  j < disconnectedCount;  j++)
+        uint32_t j;
+        for (j = 0;  j < disconnectedCount;  j++)
         {
             if (disconnected[j] && disconnected[j]->ns.unitNumber == unitNumber)
             {
+                disconnected[j]->ns.screen = screen;
                 disconnected[j] = NULL;
                 break;
             }
         }
 
+        if (j < disconnectedCount)
+            continue;
+
         const CGSize size = CGDisplayScreenSize(displays[i]);
-        char* name = getDisplayName(displays[i]);
+        char* name = getDisplayName(displays[i], screen);
         if (!name)
-            name = _glfw_strdup("Unknown");
+            continue;
 
         _GLFWmonitor* monitor = _glfwAllocMonitor(name, (int)size.width, (int)size.height);
         monitor->ns.displayID  = displays[i];
         monitor->ns.unitNumber = unitNumber;
+        monitor->ns.screen     = screen;
         createDisplayLink(monitor->ns.displayID);
 
         free(name);
@@ -554,15 +584,16 @@ GLFWvidmode* _glfwPlatformGetVideoModes(_GLFWmonitor* monitor, int* count)
 
         const GLFWvidmode mode =
             vidmodeFromCGDisplayMode(dm, monitor->ns.fallbackRefreshRate);
+        CFIndex j;
 
-        for (CFIndex j = 0;  j < *count;  j++)
+        for (j = 0;  j < *count;  j++)
         {
             if (_glfwCompareVideoModes(result + j, &mode) == 0)
                 break;
         }
 
         // Skip duplicate modes
-        if (i < *count)
+        if (j < *count)
             continue;
 
         (*count)++;

glfw/cocoa_window.m

@@ -1712,6 +1712,7 @@ int _glfwPlatformCreateWindow(_GLFWwindow* window,
 
     if (!createNativeWindow(window, wndconfig, fbconfig))
         return false;
+    [window->ns.object setColorSpace:[NSColorSpace sRGBColorSpace]];
 
     if (ctxconfig->client != GLFW_NO_API)
     {

glfw/glfw.py

@@ -222,6 +222,7 @@ def generate_wrappers(glfw_header: str) -> None:
     unsigned long long glfwDBusUserNotify(const char *app_name, const char* icon, const char *summary, const char *body, \
 const char *action_text, int32_t timeout, GLFWDBusnotificationcreatedfun callback, void *data)
     void glfwDBusSetUserNotificationHandler(GLFWDBusnotificationactivatedfun handler)
+    int glfwSetX11LaunchCommand(GLFWwindow *handle, char **argv, int argc)
 '''.splitlines():
         if line:
             functions.append(Function(line.strip(), check_fail=False))

glfw/memfd.h

@@ -10,8 +10,8 @@
 
 #include <unistd.h>
 #include <sys/syscall.h>
-static inline int memfd_create(const char *name, unsigned int flags) {
-    return syscall(__NR_memfd_create, name, flags);
+static inline int glfw_memfd_create(const char *name, unsigned int flags) {
+    return (int)syscall(__NR_memfd_create, name, flags);
 }
 
 #ifndef F_LINUX_SPECIFIC_BASE

glfw/wl_init.c

@@ -29,7 +29,6 @@
 #define _GNU_SOURCE
 #include "internal.h"
 #include "backend_utils.h"
-#include "wl_client_side_decorations.h"
 #include "linux_desktop_settings.h"
 #include "../kitty/monotonic.h"
 

glfw/wl_window.c

@@ -255,13 +255,22 @@ dispatchChangesAfterConfigure(_GLFWwindow *window, int32_t width, int32_t height
 }
 
 
-static void xdgDecorationHandleConfigure(void* data,
+static void
+xdgDecorationHandleConfigure(void* data,
                                          struct zxdg_toplevel_decoration_v1* decoration UNUSED,
                                          uint32_t mode)
 {
     _GLFWwindow* window = data;
 
-    window->wl.decorations.serverSide = (mode == ZXDG_TOPLEVEL_DECORATION_V1_MODE_SERVER_SIDE);
+    bool has_server_side_decorations = (mode == ZXDG_TOPLEVEL_DECORATION_V1_MODE_SERVER_SIDE);
+    debug("XDG decoration configure event received: Has server side decorations: %d\n", has_server_side_decorations);
+    if (!has_server_side_decorations && window->wl.decorations.serverSide) {
+        // this can happen for example on sway where it has a "border toggle" function
+        // that turns on/off the server side decorations. In such a case, we dont turn
+        // on client side decorations, as that causes things to break.
+        return;
+    }
+    window->wl.decorations.serverSide = has_server_side_decorations;
     ensure_csd_resources(window);
 }
 
@@ -396,7 +405,16 @@ _glfwPlatformToggleFullscreen(_GLFWwindow *window, unsigned int flags UNUSED) {
     return !already_fullscreen;
 }
 
-static void xdgToplevelHandleConfigure(void* data,
+static void
+inform_compositor_of_window_geometry(_GLFWwindow *window, const char *event) {
+#define geometry window->wl.decorations.geometry
+    debug("Setting window geometry in %s event: x=%d y=%d %dx%d\n", event, geometry.x, geometry.y, geometry.width, geometry.height);
+    xdg_surface_set_window_geometry(window->wl.xdg.surface, geometry.x, geometry.y, geometry.width, geometry.height);
+#undef geometry
+}
+
+static void
+xdgToplevelHandleConfigure(void* data,
                                        struct xdg_toplevel* toplevel UNUSED,
                                        int32_t width,
                                        int32_t height,
@@ -452,10 +470,7 @@ static void xdgToplevelHandleConfigure(void* data,
     _glfwInputWindowFocus(window, window->wl.toplevel_states & TOPLEVEL_STATE_ACTIVATED);
     ensure_csd_resources(window);
     wl_surface_commit(window->wl.surface);
-#define geometry window->wl.decorations.geometry
-    debug("Setting window geometry: x=%d y=%d %dx%d\n", geometry.x, geometry.y, geometry.width, geometry.height);
-    xdg_surface_set_window_geometry(window->wl.xdg.surface, geometry.x, geometry.y, geometry.width, geometry.height);
-#undef geometry
+    inform_compositor_of_window_geometry(window, "configure");
     if (live_resize_done) _glfwInputLiveResize(window, false);
 }
 
@@ -482,7 +497,8 @@ static const struct xdg_surface_listener xdgSurfaceListener = {
     xdgSurfaceHandleConfigure
 };
 
-static void setXdgDecorations(_GLFWwindow* window)
+static void
+setXdgDecorations(_GLFWwindow* window)
 {
     if (_glfw.wl.decorationManager)
     {
@@ -504,7 +520,8 @@ static void setXdgDecorations(_GLFWwindow* window)
     }
 }
 
-static bool createXdgSurface(_GLFWwindow* window)
+static bool
+createXdgSurface(_GLFWwindow* window)
 {
     window->wl.xdg.surface = xdg_wm_base_get_xdg_surface(_glfw.wl.wmBase,
                                                          window->wl.surface);
@@ -872,6 +889,7 @@ void _glfwPlatformSetWindowSize(_GLFWwindow* window, int width, int height)
         resizeFramebuffer(window);
         ensure_csd_resources(window);
         wl_surface_commit(window->wl.surface);
+        inform_compositor_of_window_geometry(window, "SetWindowSize");
     }
 }
 

glfw/x11_window.c

@@ -3122,3 +3122,10 @@ GLFWAPI unsigned long long glfwDBusUserNotify(const char *app_name, const char*
 GLFWAPI void glfwDBusSetUserNotificationHandler(GLFWDBusnotificationactivatedfun handler) {
     glfw_dbus_set_user_notification_activated_handler(handler);
 }
+
+GLFWAPI int glfwSetX11LaunchCommand(GLFWwindow *handle, char **argv, int argc)
+{
+    _GLFW_REQUIRE_INIT_OR_RETURN(0);
+    _GLFWwindow* window = (_GLFWwindow*) handle;
+    return XSetCommand(_glfw.x11.display, window->x11.handle, argv, argc);
+}

kittens/clipboard/main.py

@@ -19,6 +19,7 @@ class Clipboard(Handler):
         self.args = args
         self.clipboard_contents: Optional[str] = None
         self.data_to_send = data_to_send
+        self.quit_on_write = False
 
     def initialize(self) -> None:
         if self.data_to_send is not None:
@@ -30,10 +31,14 @@ class Clipboard(Handler):
                 self.print('\x1bP+q544e\x1b\\', end='')
                 self.print('Waiting for completion...')
                 return
-            self.quit_loop(0)
+            self.quit_on_write = True
             return
         self.cmd.request_from_clipboard(self.args.use_primary)
 
+    def on_writing_finished(self) -> None:
+        if self.quit_on_write:
+            self.quit_loop(0)
+
     def on_clipboard_response(self, text: str, from_primary: bool = False) -> None:
         self.clipboard_contents = text
         self.quit_loop(0)

kittens/diff/README.asciidoc

@@ -1 +1 @@
-See https://sw.kovidgoyal.net/kitty/kittens/diff.html
+See https://sw.kovidgoyal.net/kitty/kittens/diff/

kittens/diff/collect.py

@@ -155,7 +155,7 @@ def sanitize(text: str) -> str:
 
 @lru_cache(maxsize=1024)
 def mime_type_for_path(path: str) -> str:
-    return guess_type(path) or 'application/octet-stream'
+    return guess_type(path, allow_filesystem_access=True) or 'application/octet-stream'
 
 
 @lru_cache(maxsize=1024)

kittens/diff/speedup.c

@@ -71,28 +71,48 @@ next_segment(SegmentPointer *s, PyObject *highlights) {
     return true;
 }
 
-static inline bool
-insert_code(PyObject *code, Py_UCS4 *buf, size_t bufsz, unsigned int *buf_pos) {
-    unsigned int csz = PyUnicode_GET_LENGTH(code);
-    if (*buf_pos + csz >= bufsz) return false;
-    for (unsigned int s = 0; s < csz; s++) buf[(*buf_pos)++] = PyUnicode_READ(PyUnicode_KIND(code), PyUnicode_DATA(code), s);
+typedef struct LineBuffer {
+    Py_UCS4 *buf;
+    size_t pos, capacity;
+} LineBuffer;
+
+
+static bool
+ensure_space(LineBuffer *b, size_t num) {
+    if (b->pos + num >= b->capacity) {
+        size_t new_cap = MAX(b->capacity * 2, 4096u);
+        new_cap = MAX(b->pos + num + 1024u, new_cap);
+        b->buf = realloc(b->buf, new_cap * sizeof(b->buf[0]));
+        if (!b->buf) { PyErr_NoMemory(); return false; }
+        b->capacity = new_cap;
+    }
     return true;
 }
 
 static inline bool
-add_line(Segment *bg_segment, Segment *fg_segment, Py_UCS4 *buf, size_t bufsz, unsigned int *buf_pos, PyObject *ans) {
-    bool bg_is_active = bg_segment->current_pos == bg_segment->end_pos, fg_is_active = fg_segment->current_pos == fg_segment->end_pos;
-    if (bg_is_active) { if(!insert_code(bg_segment->end_code, buf, bufsz, buf_pos)) return false; }
-    if (fg_is_active) { if(!insert_code(fg_segment->end_code, buf, bufsz, buf_pos)) return false; }
-    PyObject *wl = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, *buf_pos);
-    if (!wl) return false;
-    int ret = PyList_Append(ans, wl); Py_DECREF(wl); if (ret != 0) return false;
-    *buf_pos = 0;
-    if (bg_is_active) { if(!insert_code(bg_segment->start_code, buf, bufsz, buf_pos)) return false; }
-    if (fg_is_active) { if(!insert_code(fg_segment->start_code, buf, bufsz, buf_pos)) return false; }
+insert_code(PyObject *code, LineBuffer *b) {
+    unsigned int csz = PyUnicode_GET_LENGTH(code);
+    if (!ensure_space(b, csz)) return false;
+    for (unsigned int s = 0; s < csz; s++) b->buf[b->pos++] = PyUnicode_READ(PyUnicode_KIND(code), PyUnicode_DATA(code), s);
     return true;
 }
 
+static inline bool
+add_line(Segment *bg_segment, Segment *fg_segment, LineBuffer *b, PyObject *ans) {
+    bool bg_is_active = bg_segment->current_pos == bg_segment->end_pos, fg_is_active = fg_segment->current_pos == fg_segment->end_pos;
+    if (bg_is_active) { if(!insert_code(bg_segment->end_code, b)) return false; }
+    if (fg_is_active) { if(!insert_code(fg_segment->end_code, b)) return false; }
+    PyObject *wl = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, b->buf, b->pos);
+    if (!wl) return false;
+    int ret = PyList_Append(ans, wl); Py_DECREF(wl); if (ret != 0) return false;
+    b->pos = 0;
+    if (bg_is_active) { if(!insert_code(bg_segment->start_code, b)) return false; }
+    if (fg_is_active) { if(!insert_code(fg_segment->start_code, b)) return false; }
+    return true;
+}
+
+static LineBuffer b;
+
 static PyObject*
 split_with_highlights(PyObject *self UNUSED, PyObject *args) {
     PyObject *line, *truncate_points_py, *fg_highlights, *bg_highlight;
@@ -106,19 +126,19 @@ split_with_highlights(PyObject *self UNUSED, PyObject *args) {
     }
     SegmentPointer fg_segment = { .sg = EMPTY_SEGMENT, .num = PyList_GET_SIZE(fg_highlights)}, bg_segment = { .sg = EMPTY_SEGMENT };
     if (bg_highlight != Py_None) { if (!convert_segment(bg_highlight, &bg_segment.sg)) { Py_CLEAR(ans); return NULL; }; bg_segment.num = 1; }
-#define CHECK_CALL(func, ...) if (!func(__VA_ARGS__)) { Py_CLEAR(ans); if (!PyErr_Occurred()) PyErr_SetString(PyExc_ValueError, "line too long"); return NULL; }
+#define CHECK_CALL(func, ...) if (!func(__VA_ARGS__)) { Py_CLEAR(ans); if (!PyErr_Occurred()) PyErr_SetString(PyExc_ValueError, "unknown error while processing line"); return NULL; }
     CHECK_CALL(next_segment, &fg_segment, fg_highlights);
 
 #define NEXT_TRUNCATE_POINT truncate_point = (truncate_pos < num_truncate_pts) ? truncate_points[truncate_pos++] : UINT_MAX
     NEXT_TRUNCATE_POINT;
 
-#define INSERT_CODE(x) { CHECK_CALL(insert_code, x, buf, arraysz(buf), &buf_pos); }
+#define INSERT_CODE(x) { CHECK_CALL(insert_code, x, &b); }
 
-#define ADD_LINE CHECK_CALL(add_line, &bg_segment.sg, &fg_segment.sg, buf, arraysz(buf), &buf_pos, ans);
+#define ADD_LINE CHECK_CALL(add_line, &bg_segment.sg, &fg_segment.sg, &b, ans);
 
 #define ADD_CHAR(x) { \
-    buf[buf_pos++] = x; \
-    if (buf_pos >= arraysz(buf)) { Py_CLEAR(ans); PyErr_SetString(PyExc_ValueError, "line too long"); return NULL; } \
+    if (!ensure_space(&b, 1)) { Py_CLEAR(ans); return NULL; } \
+    b.buf[b.pos++] = x; \
 }
 #define CHECK_SEGMENT(sgp, is_fg) { \
     if (i == sgp.sg.current_pos) { \
@@ -137,15 +157,15 @@ split_with_highlights(PyObject *self UNUSED, PyObject *args) {
 }
 
     const unsigned int line_sz = PyUnicode_GET_LENGTH(line);
-    static Py_UCS4 buf[4096];
-    unsigned int i = 0, buf_pos = 0;
+    b.pos = 0;
+    unsigned int i = 0;
     for (; i < line_sz; i++) {
         if (i == truncate_point) { ADD_LINE; NEXT_TRUNCATE_POINT; }
         CHECK_SEGMENT(bg_segment, false);
         CHECK_SEGMENT(fg_segment, true)
         ADD_CHAR(PyUnicode_READ(PyUnicode_KIND(line), PyUnicode_DATA(line), i));
     }
-    if (buf_pos) ADD_LINE;
+    if (b.pos) ADD_LINE;
     return ans;
 #undef INSERT_CODE
 #undef CHECK_SEGMENT
@@ -155,6 +175,11 @@ split_with_highlights(PyObject *self UNUSED, PyObject *args) {
 #undef NEXT_TRUNCATE_POINT
 }
 
+static void
+free_resources(void) {
+    free(b.buf); b.buf = NULL; b.capacity = 0; b.pos = 0;
+}
+
 static PyMethodDef module_methods[] = {
     {"changed_center", (PyCFunction)changed_center, METH_VARARGS, ""},
     {"split_with_highlights", (PyCFunction)split_with_highlights, METH_VARARGS, ""},
@@ -175,5 +200,6 @@ PyInit_diff_speedup(void) {
 
     m = PyModule_Create(&module);
     if (m == NULL) return NULL;
+    Py_AtExit(free_resources);
     return m;
 }

kittens/hints/main.py

@@ -16,6 +16,7 @@ from typing import (
 
 from kitty.cli import parse_args
 from kitty.cli_stub import HintsCLIOptions
+from kitty.constants import website_url
 from kitty.fast_data_types import set_clipboard_string
 from kitty.key_encoding import KeyEvent
 from kitty.typing import BossType, KittyCommonOpts
@@ -94,6 +95,11 @@ def highlight_mark(m: Mark, text: str, current_input: str, alphabet: str, colors
     )
 
 
+def debug(*a: Any, **kw: Any) -> None:
+    from ..tui.loop import debug as d
+    d(*a, **kw)
+
+
 def render(text: str, current_input: str, all_marks: Sequence[Mark], ignore_mark_indices: Set[int], alphabet: str, colors: Dict[str, str]) -> str:
     for mark in reversed(all_marks):
         if mark.index in ignore_mark_indices:
@@ -102,8 +108,7 @@ def render(text: str, current_input: str, all_marks: Sequence[Mark], ignore_mark
         text = text[:mark.start] + mtext + text[mark.end:]
 
     text = text.replace('\0', '')
-
-    return text.replace('\n', '\r\n').rstrip()
+    return re.sub('[\r\n]', '\r\n', text).rstrip()
 
 
 class Hints(Handler):
@@ -306,7 +311,7 @@ def mark(pattern: str, post_processors: Iterable[PostprocessorFunc], text: str,
         except InvalidMatch:
             continue
 
-        mark_text = text[s:e].replace('\n', '').replace('\0', '')
+        mark_text = re.sub('[\r\n\0]', '', text[s:e])
         yield Mark(idx, s, e, mark_text, groupdict)
 
 
@@ -342,12 +347,12 @@ def functions_for(args: HintsCLIOptions) -> Tuple[str, List[PostprocessorFunc]]:
         )
         post_processors.append(url)
     elif args.type == 'path':
-        pattern = r'(?:\S*/\S+)|(?:\S+[.][a-zA-Z0-9]{2,7})'
+        pattern = r'(?:\S*?/[\r\S]+)|(?:\S[\r\S]*\.[a-zA-Z0-9\r]{2,7})'
         post_processors.extend((brackets, quotes))
     elif args.type == 'line':
         pattern = '(?m)^\\s*(.+)[\\s\0]*$'
     elif args.type == 'hash':
-        pattern = '[0-9a-f]{7,128}'
+        pattern = '[0-9a-f][0-9a-f\r]{6,127}'
     elif args.type == 'ip':
         pattern = (
             # # IPv4 with no validation
@@ -370,16 +375,22 @@ def functions_for(args: HintsCLIOptions) -> Tuple[str, List[PostprocessorFunc]]:
 
 def convert_text(text: str, cols: int) -> str:
     lines: List[str] = []
-    empty_line = '\0' * cols
+    empty_line = '\0' * cols + '\n'
     for full_line in text.split('\n'):
         if full_line:
             if not full_line.rstrip('\r'):  # empty lines
                 lines.extend(repeat(empty_line, len(full_line)))
                 continue
+            appended = False
             for line in full_line.split('\r'):
                 if line:
                     lines.append(line.ljust(cols, '\0'))
-    return '\n'.join(lines)
+                    lines.append('\r')
+                    appended = True
+            if appended:
+                lines[-1] = '\n'
+    rstripped = re.sub('[\r\n]+$', '', ''.join(lines))
+    return rstripped
 
 
 def parse_input(text: str) -> str:
@@ -630,7 +641,7 @@ The foreground color for text pointed to by the hints
 --customize-processing
 Name of a python file in the kitty config directory which will be imported to provide
 custom implementations for pattern finding and performing actions
-on selected matches. See https://sw.kovidgoyal.net/kitty/kittens/hints.html
+on selected matches. See {hints_url}
 for details. You can also specify absolute paths to load the script from elsewhere.
 
 
@@ -639,7 +650,8 @@ The window title for the hints window, default title is selected based on
 the type of text being hinted.
 '''.format(
     default_regex=DEFAULT_REGEX,
-    line='{{line}}', path='{{path}}'
+    line='{{line}}', path='{{path}}',
+    hints_url=website_url('kittens/hints'),
 ).format
 help_text = 'Select text from the screen using the keyboard. Defaults to searching for URLs.'
 usage = ''

kittens/hints/url_regex.py

@@ -1 +1,3 @@
-url_delimiters = '\x00-\x09\x0b-\x20\x7f-\xa0\xad\u0600-\u0605\u061c\u06dd\u070f\u08e2\u1680\u180e\u2000-\u200f\u2028-\u202f\u205f-\u2064\u2066-\u206f\u3000\ud800-\uf8ff\ufeff\ufff9-\ufffb\U000110bd\U000110cd\U00013430-\U00013438\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f\U000f0000-\U000ffffd\U00100000-\U0010fffd'  # noqa
+# generated by gen-wcwidth.py, do not edit
+
+url_delimiters = '\x00-\x09\x0b-\x0c\x0e-\x20\x7f-\xa0\xad\u0600-\u0605\u061c\u06dd\u070f\u08e2\u1680\u180e\u2000-\u200f\u2028-\u202f\u205f-\u2064\u2066-\u206f\u3000\ud800-\uf8ff\ufeff\ufff9-\ufffb\U000110bd\U000110cd\U00013430-\U00013438\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f\U000f0000-\U000ffffd\U00100000-\U0010fffd'  # noqa

kittens/hyperlinked_grep/main.py

@@ -29,7 +29,7 @@ def main() -> None:
     write: Callable[[bytes], None] = cast(Callable[[bytes], None], sys.stdout.buffer.write)
     sgr_pat = re.compile(br'\x1b\[.*?m')
     osc_pat = re.compile(b'\x1b\\].*?\x1b\\\\')
-    num_pat = re.compile(b'^(\\d+):')
+    num_pat = re.compile(br'^(\d+)[:-]')
 
     in_result: bytes = b''
     hostname = socket.gethostname().encode('utf-8')
@@ -46,6 +46,8 @@ def main() -> None:
                 m = num_pat.match(clean_line)
                 if m is not None:
                     write_hyperlink(write, in_result, line, frag=m.group(1))
+                else:
+                    write(line)
             else:
                 if line.strip():
                     path = quote_from_bytes(os.path.abspath(clean_line)).encode('utf-8')
@@ -55,8 +57,9 @@ def main() -> None:
                     write(line)
     except KeyboardInterrupt:
         p.send_signal(signal.SIGINT)
-        p.stdout.close()
     except EOFError:
+        pass
+    finally:
         p.stdout.close()
     raise SystemExit(p.wait())
 

kittens/query_terminal/main.py

@@ -4,22 +4,27 @@
 
 import re
 import sys
-from binascii import unhexlify, hexlify
+from binascii import hexlify, unhexlify
 from contextlib import suppress
-from typing import Dict, Iterable, List, Type
+from typing import Dict, Iterable, List, Type, Optional
 
 from kitty.cli import parse_args
 from kitty.cli_stub import QueryTerminalCLIOptions
-from kitty.constants import appname
-from kitty.utils import TTYIO
+from kitty.constants import appname, str_version
+from kitty.options.types import Options
 from kitty.terminfo import names
+from kitty.utils import TTYIO
 
 
 class Query:
     name: str = ''
     ans: str = ''
-    query_name: str = ''
     help_text: str = ''
+    override_query_name: str = ''
+
+    @property
+    def query_name(self) -> str:
+        return self.override_query_name or f'kitty-query-{self.name}'
 
     def __init__(self) -> None:
         self.encoded_query_name = hexlify(self.query_name.encode('utf-8')).decode('ascii')
@@ -45,6 +50,10 @@ class Query:
     def output_line(self) -> str:
         return self.ans
 
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        raise NotImplementedError()
+
 
 all_queries: Dict[str, Type[Query]] = {}
 
@@ -57,22 +66,108 @@ def query(cls: Type[Query]) -> Type[Query]:
 @query
 class TerminalName(Query):
     name: str = 'name'
-    query_name: str = 'TN'
+    override_query_name: str = 'name'
     help_text: str = f'Terminal name ({names[0]})'
 
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        return appname
+
 
 @query
 class TerminalVersion(Query):
     name: str = 'version'
-    query_name: str = 'kitty-query-version'
     help_text: str = 'Terminal version, for e.g.: 0.19.2'
 
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        return str_version
+
 
 @query
 class AllowHyperlinks(Query):
     name: str = 'allow_hyperlinks'
-    query_name: str = 'kitty-query-allow_hyperlinks'
-    help_text: str = 'yes, no or ask'
+    help_text: str = 'The :opt:`setting <allow_hyperlinks>` for allowing hyperlinks can be yes, no or ask'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        return 'ask' if opts.allow_hyperlinks == 0b11 else ('yes' if opts.allow_hyperlinks else 'no')
+
+
+@query
+class FontFamily(Query):
+    name: str = 'font_family'
+    help_text: str = 'The current font\'s PostScript name'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        from kitty.fast_data_types import current_fonts
+        cf = current_fonts()
+        return str(cf['medium'].display_name())
+
+
+@query
+class BoldFont(Query):
+    name: str = 'bold_font'
+    help_text: str = 'The current bold font\'s PostScript name'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        from kitty.fast_data_types import current_fonts
+        cf = current_fonts()
+        return str(cf['bold'].display_name())
+
+
+@query
+class ItalicFont(Query):
+    name: str = 'italic_font'
+    help_text: str = 'The current italic font\'s PostScript name'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        from kitty.fast_data_types import current_fonts
+        cf = current_fonts()
+        return str(cf['italic'].display_name())
+
+
+@query
+class BiFont(Query):
+    name: str = 'bold_italic_font'
+    help_text: str = 'The current bold-italic font\'s PostScript name'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        from kitty.fast_data_types import current_fonts
+        cf = current_fonts()
+        return str(cf['bi'].display_name())
+
+
+@query
+class FontSize(Query):
+    name: str = 'font_size'
+    help_text: str = 'The current overall font size (individual windows can have different per window font sizes)'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        return f'{opts.font_size:g}'
+
+
+@query
+class ClipboardControl(Query):
+    name: str = 'clipboard_control'
+    help_text: str = 'The :opt:`setting <clipboard_control>` for allowing reads/writes to/from the clipboard'
+
+    @staticmethod
+    def get_result(opts: Options) -> str:
+        return ' '.join(opts.clipboard_control)
+
+
+def get_result(name: str) -> Optional[str]:
+    from kitty.fast_data_types import get_options
+    q = all_queries.get(name)
+    if q is None:
+        return None
+    return q.get_result(get_options())
 
 
 def do_queries(queries: Iterable, cli_opts: QueryTerminalCLIOptions) -> Dict[str, str]:
@@ -124,11 +219,12 @@ Note that when calling this from another program, be very
 careful not to perform any I/O on the terminal device
 until the kitten exits.
 
-Available queries are::
+Available queries are:
 
 {}
-'''.format('  ' + '\n  '.join(
-    f'{name}: {c.help_text}' for name, c in all_queries.items()))
+
+'''.format('\n'.join(
+    f'``{name}``\n  {c.help_text}\n' for name, c in all_queries.items()))
 usage = '[query1 query2 ...]'
 
 

kittens/runner.py

@@ -6,8 +6,9 @@
 import importlib
 import os
 import sys
+from contextlib import contextmanager
 from functools import partial
-from typing import Any, Dict, FrozenSet, List, TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, Any, Dict, FrozenSet, Generator, List, cast
 
 from kitty.types import run_once
 
@@ -19,7 +20,10 @@ else:
 
 
 def resolved_kitten(k: str) -> str:
-    return aliases.get(k, k).replace('-', '_')
+    ans = aliases.get(k, k)
+    head, tail = os.path.split(ans)
+    tail = tail.replace('-', '_')
+    return os.path.join(head, tail)
 
 
 def path_to_custom_kitten(config_dir: str, kitten: str) -> str:
@@ -30,21 +34,29 @@ def path_to_custom_kitten(config_dir: str, kitten: str) -> str:
     return path
 
 
+@contextmanager
+def preserve_sys_path() -> Generator[None, None, None]:
+    orig = sys.path[:]
+    try:
+        yield
+    finally:
+        if sys.path != orig:
+            del sys.path[:]
+            sys.path.extend(orig)
+
+
 def import_kitten_main_module(config_dir: str, kitten: str) -> Dict[str, Any]:
     if kitten.endswith('.py'):
-        path_modified = False
-        path = path_to_custom_kitten(config_dir, kitten)
-        if os.path.dirname(path):
-            sys.path.insert(0, os.path.dirname(path))
-            path_modified = True
-        with open(path) as f:
-            src = f.read()
-        code = compile(src, path, 'exec')
-        g = {'__name__': 'kitten'}
-        exec(code, g)
-        hr = g.get('handle_result', lambda *a, **kw: None)
-        if path_modified:
-            del sys.path[0]
+        with preserve_sys_path():
+            path = path_to_custom_kitten(config_dir, kitten)
+            if os.path.dirname(path):
+                sys.path.insert(0, os.path.dirname(path))
+            with open(path) as f:
+                src = f.read()
+            code = compile(src, path, 'exec')
+            g = {'__name__': 'kitten'}
+            exec(code, g)
+            hr = g.get('handle_result', lambda *a, **kw: None)
         return {'start': g['main'], 'end': hr}
 
     kitten = resolved_kitten(kitten)
@@ -155,6 +167,14 @@ def get_kitten_cli_docs(kitten: str) -> Any:
         return ans
 
 
+def get_kitten_completer(kitten: str) -> Any:
+    run_kitten(kitten, run_name='__completer__')
+    ans = getattr(sys, 'kitten_completer', None)
+    if ans is not None:
+        delattr(sys, 'kitten_completer')
+    return ans
+
+
 def get_kitten_conf_docs(kitten: str) -> Definition:
     setattr(sys, 'options_definition', None)
     run_kitten(kitten, run_name='__conf__')

kittens/ssh/completion.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+import re
+import subprocess
+from typing import Callable, Dict, Iterable, Iterator, Sequence, Tuple
+
+from kitty.complete import Completions, complete_files_and_dirs, debug
+from kitty.types import run_once
+
+debug
+
+
+def lines_from_file(path: str) -> Iterator[str]:
+    try:
+        f = open(os.path.expanduser(path))
+    except OSError:
+        pass
+    else:
+        yield from f
+
+
+def lines_from_command(*cmd: str) -> Iterator[str]:
+    try:
+        output = subprocess.check_output(cmd).decode('utf-8')
+    except Exception:
+        return
+    yield from output.splitlines()
+
+
+def parts_yielder(lines: Iterable[str], pfilter: Callable[[str], Iterator[str]]) -> Iterator[str]:
+    for line in lines:
+        yield from pfilter(line)
+
+
+def hosts_from_config_lines(line: str) -> Iterator[str]:
+    parts = line.strip().split()
+    if len(parts) > 1 and parts[0] == 'Host':
+        yield parts[1]
+
+
+def hosts_from_known_hosts(line: str) -> Iterator[str]:
+    parts = line.strip().split()
+    if parts:
+        yield re.sub(r':\d+$', '', parts[0])
+
+
+def hosts_from_hosts(line: str) -> Iterator[str]:
+    line = line.strip()
+    if not line.startswith('#'):
+        parts = line.split()
+        if parts:
+            yield parts[0]
+        if len(parts) > 1:
+            yield parts[1]
+        if len(parts) > 2:
+            yield parts[2]
+
+
+def iter_known_hosts() -> Iterator[str]:
+    yield from parts_yielder(lines_from_file('~/.ssh/config'), hosts_from_config_lines)
+    yield from parts_yielder(lines_from_file('~/.ssh/known_hosts'), hosts_from_known_hosts)
+    yield from parts_yielder(lines_from_file('/etc/ssh/ssh_known_hosts'), hosts_from_known_hosts)
+    yield from parts_yielder(lines_from_file('/etc/hosts'), hosts_from_hosts)
+    yield from parts_yielder(lines_from_command('getent', 'hosts'), hosts_from_hosts)
+
+
+@run_once
+def known_hosts() -> Tuple[str, ...]:
+    return tuple(sorted(filter(lambda x: '*' not in x and '[' not in x, set(iter_known_hosts()))))
+
+
+@run_once
+def ssh_options() -> Dict[str, str]:
+    stderr = subprocess.Popen(['ssh'], stderr=subprocess.PIPE).stderr
+    assert stderr is not None
+    raw = stderr.read().decode('utf-8')
+    ans: Dict[str, str] = {}
+    pos = 0
+    while True:
+        pos = raw.find('[', pos)
+        if pos < 0:
+            break
+        num = 1
+        epos = pos
+        while num > 0:
+            epos += 1
+            if raw[epos] not in '[]':
+                continue
+            num += 1 if raw[epos] == '[' else -1
+        q = raw[pos+1:epos]
+        pos = epos
+        if len(q) < 2 or q[0] != '-':
+            continue
+        if ' ' in q:
+            opt, desc = q.split(' ', 1)
+            ans[opt[1:]] = desc
+        else:
+            ans.update(dict.fromkeys(q[1:], ''))
+    return ans
+
+
+# option help {{{
+@run_once
+def option_help_map() -> Dict[str, str]:
+    ans: Dict[str, str] = {}
+    lines = '''
+-4  -- force ssh to use IPv4 addresses only
+-6  -- force ssh to use IPv6 addresses only
+-a  -- disable forwarding of authentication agent connection
+-A  -- enable forwarding of the authentication agent connection
+-B  -- bind to specified interface before attempting to connect
+-b  -- specify interface to transmit on
+-C  -- compress data
+-c  -- select encryption cipher
+-D  -- specify a dynamic port forwarding
+-E  -- append log output to file instead of stderr
+-e  -- set escape character
+-f  -- go to background
+-F  -- specify alternate config file
+-g  -- allow remote hosts to connect to local forwarded ports
+-G  -- output configuration and exit
+-i  -- select identity file
+-I  -- specify smartcard device
+-J  -- connect via a jump host
+-k  -- disable forwarding of GSSAPI credentials
+-K  -- enable GSSAPI-based authentication and forwarding
+-L  -- specify local port forwarding
+-l  -- specify login name
+-M  -- master mode for connection sharing
+-m  -- specify mac algorithms
+-N  -- don't execute a remote command
+-n  -- redirect stdin from /dev/null
+-O  -- control an active connection multiplexing master process
+-o  -- specify extra options
+-p  -- specify port on remote host
+-P  -- use non privileged port
+-Q  -- query parameters
+-q  -- quiet operation
+-R  -- specify remote port forwarding
+-s  -- invoke subsystem
+-S  -- specify location of control socket for connection sharing
+-T  -- disable pseudo-tty allocation
+-t  -- force pseudo-tty allocation
+-V  -- show version number
+-v  -- verbose mode (multiple increase verbosity, up to 3)
+-W  -- forward standard input and output to host
+-w  -- request tunnel device forwarding
+-x  -- disable X11 forwarding
+-X  -- enable (untrusted) X11 forwarding
+-Y  -- enable trusted X11 forwarding
+-y  -- send log info via syslog instead of stderr
+'''.splitlines()
+    for line in lines:
+        line = line.strip()
+        if line:
+            parts = line.split(maxsplit=2)
+            ans[parts[0]] = parts[2]
+    return ans
+# }}}
+
+
+# option names {{{
+@run_once
+def option_names() -> Tuple[str, ...]:
+    return tuple(filter(None, (
+        line.strip() for line in '''
+AddKeysToAgent
+AddressFamily
+BatchMode
+BindAddress
+CanonicalDomains
+CanonicalizeFallbackLocal
+CanonicalizeHostname
+CanonicalizeMaxDots
+CanonicalizePermittedCNAMEs
+CASignatureAlgorithms
+CertificateFile
+ChallengeResponseAuthentication
+CheckHostIP
+Ciphers
+ClearAllForwardings
+Compression
+ConnectionAttempts
+ConnectTimeout
+ControlMaster
+ControlPath
+ControlPersist
+DynamicForward
+EscapeChar
+ExitOnForwardFailure
+FingerprintHash
+ForwardAgent
+ForwardX11
+ForwardX11Timeout
+ForwardX11Trusted
+GatewayPorts
+GlobalKnownHostsFile
+GSSAPIAuthentication
+GSSAPIDelegateCredentials
+HashKnownHosts
+Host
+HostbasedAcceptedAlgorithms
+HostbasedAuthentication
+HostKeyAlgorithms
+HostKeyAlias
+Hostname
+IdentitiesOnly
+IdentityAgent
+IdentityFile
+IPQoS
+KbdInteractiveAuthentication
+KbdInteractiveDevices
+KexAlgorithms
+KnownHostsCommand
+LocalCommand
+LocalForward
+LogLevel
+MACs
+Match
+NoHostAuthenticationForLocalhost
+NumberOfPasswordPrompts
+PasswordAuthentication
+PermitLocalCommand
+PermitRemoteOpen
+PKCS11Provider
+Port
+PreferredAuthentications
+ProxyCommand
+ProxyJump
+ProxyUseFdpass
+PubkeyAcceptedAlgorithms
+PubkeyAuthentication
+RekeyLimit
+RemoteCommand
+RemoteForward
+RequestTTY
+SendEnv
+ServerAliveInterval
+ServerAliveCountMax
+SetEnv
+StreamLocalBindMask
+StreamLocalBindUnlink
+StrictHostKeyChecking
+TCPKeepAlive
+Tunnel
+TunnelDevice
+UpdateHostKeys
+User
+UserKnownHostsFile
+VerifyHostKeyDNS
+VisualHostKey
+XAuthLocation
+'''.splitlines())))
+# }}}
+
+
+def complete_choices(ans: Completions, prefix: str, title: str, choices: Iterable[str], comma_separated: bool = False) -> None:
+    matches: Dict[str, str] = {}
+    word_transforms = {}
+    effective_prefix = prefix
+    hidden_prefix = ''
+    if comma_separated:
+        effective_prefix = prefix.split(',')[-1]
+        hidden_prefix = ','.join(prefix.split(',')[:-1])
+        if hidden_prefix:
+            hidden_prefix += ','
+    for q in choices:
+        if q.startswith(effective_prefix):
+            if comma_separated:
+                tq = q
+                q = hidden_prefix + q + ','
+                word_transforms[q] = tq
+            matches[q] = ''
+    ans.add_match_group(title, matches, trailing_space=not comma_separated, word_transforms=word_transforms)
+
+
+def complete_q_choices(ans: Completions, prefix: str, title: str, key: str, comma_separated: bool) -> None:
+    choices = (line.strip() for line in lines_from_command('ssh', '-Q', key))
+    complete_choices(ans, prefix, title, choices, comma_separated)
+
+
+def complete_arg(ans: Completions, option_flag: str, prefix: str = '') -> None:
+    options = ssh_options()
+    option_name = options.get(option_flag[1:])
+    if option_name.endswith('file') or option_name.endswith('path'):
+        return complete_files_and_dirs(ans, prefix, option_name)
+    choices = {
+        'mac_spec': ('MAC algorithm', 'mac', True),
+        'cipher_spec': ('encryption cipher', 'cipher', True),
+        'query_option': ('query option', 'help', False),
+    }
+    if option_name in choices:
+        return complete_q_choices(ans, prefix, *choices[option_name])
+    if option_name == 'destination':
+        return complete_destination(ans, prefix)
+    if option_name == 'ctl_cmd':
+        return complete_choices(ans, prefix, 'control command', ('check', 'forward', 'cancel', 'exit'))
+    if option_name == 'option':
+        matches = (x+'=' for x in option_names() if x.startswith(prefix))
+        word_transforms = {x+'=': x for x in option_names()}
+        ans.add_match_group('configure file option', matches, trailing_space=False, word_transforms=word_transforms)
+
+
+def complete_destination(ans: Completions, prefix: str = '') -> None:
+    result = (k for k in known_hosts() if k.startswith(prefix))
+    ans.add_match_group('remote host name', result)
+
+
+def complete_option(ans: Completions, prefix: str = '-') -> None:
+    hm = option_help_map()
+    if len(prefix) <= 1:
+        result = {k: v for k, v in hm.items() if k.startswith(prefix)}
+        ans.add_match_group('option', result)
+    else:
+        ans.add_match_group('option', {prefix: ''})
+
+
+def complete(ans: Completions, words: Sequence[str], new_word: bool) -> None:
+    options = ssh_options()
+    expecting_arg = False
+    types = ['' for i in range(len(words))]
+    for i, word in enumerate(words):
+        if expecting_arg:
+            types[i] = 'arg'
+            expecting_arg = False
+            continue
+        if word.startswith('-'):
+            types[i] = 'option'
+            if len(word) == 2 and options.get(word[1]):
+                expecting_arg = True
+            continue
+        types[i] = 'destination'
+        break
+    if new_word:
+        if words:
+            if expecting_arg:
+                return complete_arg(ans, words[-1])
+        return complete_destination(ans)
+    if words:
+        if types[-1] == 'arg' and len(words) > 1:
+            return complete_arg(ans, words[-2], words[-1])
+        if types[-1] == 'destination':
+            return complete_destination(ans, words[-1])
+        if types[-1] == 'option':
+            return complete_option(ans, words[-1])

kittens/ssh/main.py

@@ -3,12 +3,12 @@
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
 import os
-import re
 import shlex
 import subprocess
 import sys
 from contextlib import suppress
 from typing import List, NoReturn, Optional, Set, Tuple
+from .completion import ssh_options, complete
 
 from kitty.utils import SSHConnectionData
 
@@ -21,27 +21,83 @@ cat >$tmp << 'TERMEOF'
 TERMINFO
 TERMEOF
 
-tic_out=$(tic -x -o ~/.terminfo $tmp 2>&1)
+tic_out=$(tic -x -o $HOME/.terminfo $tmp 2>&1)
 rc=$?
 rm $tmp
 if [ "$rc" != "0" ]; then echo "$tic_out"; exit 1; fi
 if [ -z "$USER" ]; then export USER=$(whoami); fi
-shell_name=$(basename $0)
+export TERMINFO="$HOME/.terminfo"
+login_shell=""
+python=""
+
+login_shell_is_ok() {
+    if [ -z "$login_shell" ] || [ ! -x "$login_shell" ]; then return 1; fi
+    case "$login_shell" in
+        *sh) return 0;
+    esac
+    return 1;
+}
+
+detect_python() {
+    python=$(command -v python3)
+    if [ -z "$python" ]; then python=$(command -v python2); fi
+    if [ -z "$python" ]; then python=python; fi
+}
+
+using_getent() {
+    cmd=$(command -v getent)
+    if [ -z "$cmd" ]; then return; fi
+    output=$($cmd passwd $USER 2>/dev/null)
+    if [ $? = 0 ]; then login_shell=$(echo $output | cut -d: -f7); fi
+}
+
+using_id() {
+    cmd=$(command -v id)
+    if [ -z "$cmd" ]; then return; fi
+    output=$($cmd -P $USER 2>/dev/null)
+    if [ $? = 0 ]; then login_shell=$(echo $output | cut -d: -f7); fi
+}
+
+using_passwd() {
+    cmd=$(command -v grep)
+    if [ -z "$cmd" ]; then return; fi
+    output=$($cmd "^$USER:" /etc/passwd 2>/dev/null)
+    if [ $? = 0 ]; then login_shell=$(echo $output | cut -d: -f7); fi
+}
+
+using_python() {
+    detect_python
+    if [ ! -x "$python" ]; then return; fi
+    output=$($python -c "import pwd, os; print(pwd.getpwuid(os.geteuid()).pw_shell)")
+    if [ $? = 0 ]; then login_shell=$output; fi
+}
+
+execute_with_python() {
+    detect_python
+    exec $python -c "import os; os.execl('$login_shell', '-' '$shell_name')"
+}
+
+die() { echo "$*" 1>&2 ; exit 1; }
+
+using_getent
+if ! login_shell_is_ok; then using_id; fi
+if ! login_shell_is_ok; then using_python; fi
+if ! login_shell_is_ok; then using_passwd; fi
+if ! login_shell_is_ok; then die "Could not detect login shell"; fi
+
+
+# If a command was passed to SSH execute it here
 EXEC_CMD
 
 # We need to pass the first argument to the executed program with a leading -
 # to make sure the shell executes as a login shell. Note that not all shells
 # support exec -a so we use the below to try to detect such shells
-
-case "dash" in
-    *$shell_name*)
-        python=$(command -v python3)
-        if [ -z "$python" ]; then python=$(command -v python2); fi
-        if [ -z "$python" ]; then python=python; fi
-        exec $python -c "import os; os.execlp('$0', '-' '$shell_name')"
-    ;;
-esac
-exec -a "-$shell_name" "$0"
+shell_name=$(basename $login_shell)
+if [ -z "$PIPESTATUS" ]; then
+    # the dash shell does not support exec -a and also does not define PIPESTATUS
+    execute_with_python
+fi
+exec -a "-$shell_name" $login_shell
 '''
 
 
@@ -73,20 +129,15 @@ os.execlp(shell_path, shell_name)
 
 
 def get_ssh_cli() -> Tuple[Set[str], Set[str]]:
-    other_ssh_args: List[str] = []
-    boolean_ssh_args: List[str] = []
-    stderr = subprocess.Popen(['ssh'], stderr=subprocess.PIPE).stderr
-    assert stderr is not None
-    raw = stderr.read().decode('utf-8')
-    for m in re.finditer(r'\[(.+?)\]', raw):
-        q = m.group(1)
-        if len(q) < 2 or q[0] != '-':
-            continue
-        if ' ' in q:
-            other_ssh_args.append(q[1])
+    other_ssh_args: Set[str] = set()
+    boolean_ssh_args: Set[str] = set()
+    for k, v in ssh_options().items():
+        k = '-' + k
+        if v:
+            other_ssh_args.add(k)
         else:
-            boolean_ssh_args.extend(q[1:])
-    return set('-' + x for x in boolean_ssh_args), set('-' + x for x in other_ssh_args)
+            boolean_ssh_args.add(k)
+    return boolean_ssh_args, other_ssh_args
 
 
 def get_connection_data(args: List[str]) -> Optional[SSHConnectionData]:
@@ -124,6 +175,18 @@ def get_connection_data(args: List[str]) -> Optional[SSHConnectionData]:
         return SSHConnectionData(found_ssh, arg, port)
 
 
+class InvalidSSHArgs(ValueError):
+
+    def __init__(self, msg: str = ''):
+        super().__init__(msg)
+        self.err_msg = msg
+
+    def system_exit(self) -> None:
+        if self.err_msg:
+            print(self.err_msg, file=sys.stderr)
+        os.execlp('ssh', 'ssh')
+
+
 def parse_ssh_args(args: List[str]) -> Tuple[List[str], List[str], bool]:
     boolean_ssh_args, other_ssh_args = get_ssh_cli()
     passthrough_args = {'-' + x for x in 'Nnf'}
@@ -131,11 +194,16 @@ def parse_ssh_args(args: List[str]) -> Tuple[List[str], List[str], bool]:
     server_args: List[str] = []
     expecting_option_val = False
     passthrough = False
+    stop_option_processing = False
     for arg in args:
-        if len(server_args) > 1:
+        if len(server_args) > 1 or stop_option_processing:
             server_args.append(arg)
             continue
         if arg.startswith('-') and not expecting_option_val:
+            if arg == '--':
+                stop_option_processing = True
+                continue
+            # could be a multi-character option
             all_args = arg[1:]
             for i, arg in enumerate(all_args):
                 arg = '-' + arg
@@ -152,7 +220,7 @@ def parse_ssh_args(args: List[str]) -> Tuple[List[str], List[str], bool]:
                     else:
                         expecting_option_val = True
                     break
-                raise SystemExit('Unknown option: {}'.format(arg))
+                raise InvalidSSHArgs('unknown option -- {}'.format(arg[1:]))
             continue
         if expecting_option_val:
             ssh_args.append(arg)
@@ -160,7 +228,7 @@ def parse_ssh_args(args: List[str]) -> Tuple[List[str], List[str], bool]:
             continue
         server_args.append(arg)
     if not server_args:
-        raise SystemExit('Must specify server to connect to')
+        raise InvalidSSHArgs()
     return ssh_args, server_args, passthrough
 
 
@@ -184,9 +252,9 @@ def get_posix_cmd(terminfo: str, remote_args: List[str]) -> List[str]:
         # line 1129 of ssh.c and on the remote side sshd.c runs the
         # concatenated command as shell -c cmd
         args = [c.replace("'", """'"'"'""") for c in remote_args]
-        command_to_execute = "exec $0 -c '{}'".format(' '.join(args))
+        command_to_execute = "exec $login_shell -c '{}'".format(' '.join(args))
     sh_script = sh_script.replace('EXEC_CMD', command_to_execute)
-    return [sh_script] + remote_args
+    return [f'sh -c {shlex.quote(sh_script)}']
 
 
 def get_python_cmd(terminfo: str, command_to_execute: List[str]) -> List[str]:
@@ -204,13 +272,19 @@ def main(args: List[str]) -> NoReturn:
     if args and args[0] == 'use-python':
         args = args[1:]
         use_posix = False
-    ssh_args, server_args, passthrough = parse_ssh_args(args)
+    try:
+        ssh_args, server_args, passthrough = parse_ssh_args(args)
+    except InvalidSSHArgs as e:
+        e.system_exit()
     cmd = ['ssh'] + ssh_args
     if passthrough:
         cmd += server_args
     else:
         hostname, remote_args = server_args[0], server_args[1:]
-        cmd += ['-t', hostname]
+        if not remote_args:
+            cmd.append('-t')
+        cmd.append('--')
+        cmd.append(hostname)
         terminfo = subprocess.check_output(['infocmp', '-a']).decode('utf-8')
         f = get_posix_cmd if use_posix else get_python_cmd
         cmd += f(terminfo, remote_args)
@@ -219,3 +293,5 @@ def main(args: List[str]) -> NoReturn:
 
 if __name__ == '__main__':
     main(sys.argv)
+elif __name__ == '__completer__':
+    setattr(sys, 'kitten_completer', complete)

kittens/tui/handler.py

@@ -107,6 +107,9 @@ class Handler:
     def on_eot(self) -> None:
         pass
 
+    def on_writing_finished(self) -> None:
+        pass
+
     def on_kitty_cmd_response(self, response: Dict) -> None:
         pass
 

kittens/tui/images.py

@@ -11,14 +11,15 @@ from contextlib import suppress
 from enum import IntEnum
 from itertools import count
 from typing import (
-    Any, Callable, DefaultDict, Deque, Dict, Iterator, List, Optional,
-    Sequence, Tuple, Union
+    Any, Callable, ClassVar, DefaultDict, Deque, Dict, Generic, Iterator, List,
+    Optional, Sequence, Tuple, Type, TypeVar, Union, cast
 )
 
 from kitty.conf.utils import positive_float, positive_int
 from kitty.fast_data_types import create_canvas
 from kitty.typing import (
-    CompletedProcess, GRT_a, GRT_d, GRT_f, GRT_m, GRT_o, GRT_t, HandlerType
+    GRT_C, CompletedProcess, GRT_a, GRT_d, GRT_f, GRT_m, GRT_o, GRT_t,
+    HandlerType
 )
 from kitty.utils import ScreenSize, find_exe, fit_image
 
@@ -298,49 +299,75 @@ def can_display_images() -> bool:
 
 ImageKey = Tuple[str, int, int]
 SentImageKey = Tuple[int, int, int]
+T = TypeVar('T')
+
+
+class Alias(Generic[T]):
+
+    currently_processing: ClassVar[str] = ''
+
+    def __init__(self, defval: T) -> None:
+        self.name = ''
+        self.defval = defval
+
+    def __get__(self, instance: Optional['GraphicsCommand'], cls: Optional[Type['GraphicsCommand']] = None) -> T:
+        if instance is None:
+            return self.defval
+        return cast(T, instance._actual_values.get(self.name, self.defval))
+
+    def __set__(self, instance: 'GraphicsCommand', val: T) -> None:
+        if val == self.defval:
+            instance._actual_values.pop(self.name, None)
+        else:
+            instance._actual_values[self.name] = val
+
+    def __set_name__(self, owner: Type['GraphicsCommand'], name: str) -> None:
+        if len(name) == 1:
+            Alias.currently_processing = name
+        self.name = Alias.currently_processing
 
 
 class GraphicsCommand:
-    a: GRT_a = 't'  # action
-    q: int = 0      # suppress responses
-    f: GRT_f = 32   # image data format
-    t: GRT_t = 'd'  # transmission medium
-    s: int = 0        # sent image width
-    v: int = 0        # sent image height
-    S: int = 0        # size of data to read from file
-    O: int = 0        # offset of data to read from file
-    i: int = 0        # image id
-    I: int = 0        # image number
-    p: int = 0        # placement id
-    o: Optional[GRT_o] = None  # type of compression
-    m: GRT_m = 0    # 0 or 1 whether there is more chunked data
-    x: int = 0        # left edge of image area to display
-    y: int = 0        # top edge of image area to display
-    w: int = 0        # image width to display
-    h: int = 0        # image height to display
-    X: int = 0        # X-offset within cell
-    Y: int = 0        # Y-offset within cell
-    c: int = 0        # number of cols to display image over
-    r: int = 0        # number of rows to display image over
-    z: int = 0        # z-index
-    d: GRT_d = 'a'  # what to delete
+    a = action = Alias(cast(GRT_a, 't'))
+    q = quiet = Alias(0)
+    f = format = Alias(32)
+    t = transmission_type = Alias(cast(GRT_t, 'd'))
+    s = data_width = animation_state = Alias(0)
+    v = data_height = loop_count = Alias(0)
+    S = data_size = Alias(0)
+    O = data_offset = Alias(0)  # noqa
+    i = image_id = Alias(0)
+    I = image_number = Alias(0)  # noqa
+    p = placement_id = Alias(0)
+    o = compression = Alias(cast(Optional[GRT_o], None))
+    m = more = Alias(cast(GRT_m, 0))
+    x = left_edge = Alias(0)
+    y = top_edge = Alias(0)
+    w = width = Alias(0)
+    h = height = Alias(0)
+    X = cell_x_offset = blend_mode = Alias(0)
+    Y = cell_y_offset = bgcolor = Alias(0)
+    c = columns = other_frame_number = dest_frame = Alias(0)
+    r = rows = frame_number = source_frame = Alias(0)
+    z = z_index = gap = Alias(0)
+    C = cursor_movement = compose_mode = Alias(cast(GRT_C, 0))
+    d = delete_action = Alias(cast(GRT_d, 'a'))
+
+    def __init__(self) -> None:
+        self._actual_values: Dict[str, Any] = {}
 
     def __repr__(self) -> str:
         return self.serialize().decode('ascii').replace('\033', '^]')
 
     def clone(self) -> 'GraphicsCommand':
         ans = GraphicsCommand()
-        for k in GraphicsCommand.__annotations__:
-            setattr(ans, k, getattr(self, k))
+        ans._actual_values = self._actual_values.copy()
         return ans
 
     def serialize(self, payload: Union[bytes, str] = b'') -> bytes:
         items = []
-        for k in GraphicsCommand.__annotations__:
-            val: Union[str, None, int] = getattr(self, k)
-            defval: Union[str, None, int] = getattr(GraphicsCommand, k)
-            if val != defval and val is not None:
-                items.append('{}={}'.format(k, val))
+        for k, val in self._actual_values.items():
+            items.append('{}={}'.format(k, val))
 
         ans: List[bytes] = []
         w = ans.append
@@ -355,9 +382,27 @@ class GraphicsCommand:
         return b''.join(ans)
 
     def clear(self) -> None:
-        for k in GraphicsCommand.__annotations__:
-            defval: Union[str, None, int] = getattr(GraphicsCommand, k)
-            setattr(self, k, defval)
+        self._actual_values = {}
+
+    def iter_transmission_chunks(self, data: Optional[bytes] = None, level: int = -1, compression_threshold: int = 1024) -> Iterator[bytes]:
+        if data is None:
+            yield self.serialize()
+            return
+        gc = self.clone()
+        gc.S = len(data)
+        if level and len(data) >= compression_threshold:
+            import zlib
+            compressed = zlib.compress(data, level)
+            if len(compressed) < len(data):
+                gc.o = 'z'
+                data = compressed
+                gc.S = len(data)
+        data = standard_b64encode(data)
+        while data:
+            chunk, data = data[:4096], data[4096:]
+            gc.m = 1 if data else 0
+            yield gc.serialize(chunk)
+            gc.clear()
 
 
 class Placement:

kittens/tui/loop.py

@@ -39,20 +39,27 @@ class BinaryWrite(Protocol):
         pass
 
 
+def debug_write(*a: Any, **kw: Any) -> None:
+    from base64 import standard_b64encode
+    fobj = kw.pop('file', sys.stderr.buffer)
+    buf = io.StringIO()
+    kw['file'] = buf
+    print(*a, **kw)
+    stext = buf.getvalue()
+    for i in range(0, len(stext), 256):
+        chunk = stext[i:i + 256]
+        text = b'\x1bP@kitty-print|' + standard_b64encode(chunk.encode('utf-8')) + b'\x1b\\'
+        fobj.write(text)
+    fobj.flush()
+
+
 class Debug:
 
     fobj: Optional[BinaryWrite] = None
 
     def __call__(self, *a: Any, **kw: Any) -> None:
-        from base64 import standard_b64encode
-        buf = io.StringIO()
-        kw['file'] = buf
-        print(*a, **kw)
-        stext = buf.getvalue()
-        text = b'\x1bP@kitty-print|' + standard_b64encode(stext.encode('utf-8')) + b'\x1b\\'
-        fobj = self.fobj or sys.stdout.buffer
-        fobj.write(text)
-        fobj.flush()
+        kw['file'] = self.fobj or sys.stdout.buffer
+        debug_write(*a, **kw)
 
 
 debug = Debug()
@@ -336,6 +343,7 @@ class Loop:
             self.write_buf: List[bytes] = []
             self.asycio_loop.remove_writer(fd)
             self.waiting_for_writes = False
+            handler.on_writing_finished()
         else:
             consumed = 0
             for i, buf in enumerate(self.write_buf):

kittens/tui/operations.py

@@ -352,24 +352,17 @@ def set_default_colors(
 
 @cmd
 def write_to_clipboard(data: Union[str, bytes], use_primary: bool = False) -> str:
-    if isinstance(data, str):
-        data = data.encode('utf-8')
     from base64 import standard_b64encode
     fmt = 'p' if use_primary else 'c'
-
-    def esc(chunk: str) -> str:
-        return '\x1b]52;{};{}\x07'.format(fmt, chunk)
-
-    ans = esc('!')  # clear clipboard buffer
-    for chunk in (data[i:i+512] for i in range(0, len(data), 512)):
-        s = standard_b64encode(chunk).decode('ascii')
-        ans += esc(s)
-    return ans
+    if isinstance(data, str):
+        data = data.encode('utf-8')
+    payload = standard_b64encode(data).decode('ascii')
+    return f'\x1b]52;{fmt};{payload}\a'
 
 
 @cmd
 def request_from_clipboard(use_primary: bool = False) -> str:
-    return '\x1b]52;{};?\x07'.format('p' if use_primary else 'c')
+    return '\x1b]52;{};?\a'.format('p' if use_primary else 'c')
 
 
 # Boilerplate to make operations available via Handler.cmd  {{{

kittens/unicode_input/main.py

@@ -67,7 +67,7 @@ def name(cp: Union[int, str]) -> str:
 
 
 @lru_cache(maxsize=256)
-def codepoints_matching_search(parts: Sequence[str]) -> List[int]:
+def codepoints_matching_search(parts: Tuple[str, ...]) -> List[int]:
     ans = []
     if parts and parts[0] and len(parts[0]) > 1:
         codepoints = points_for_word(parts[0])

kittens/unicode_input/names.h

@@ -1,4 +1,4 @@
-// unicode data, built from the unicode standard on: 2021-04-02
+// unicode data, built from the unicode standard on: 2021-07-19
 // see gen-wcwidth.py
 #pragma once
 #include "data-types.h"

kittens/unicode_input/unicode_names.c

@@ -19,27 +19,31 @@ all_words(PYNOARG) {
     return ans;
 }
 
-static inline void
-add_matches(const word_trie *wt, char_type *codepoints, size_t *pos, const size_t sz) {
+static void
+add_matches(const word_trie *wt, PyObject *ans) {
     size_t num = mark_groups[wt->match_offset];
-    for (size_t i = wt->match_offset + 1; i < wt->match_offset + 1 + num && *pos < sz; i++, (*pos)++) {
-        codepoints[*pos] = mark_to_cp[mark_groups[i]];
+    for (size_t i = wt->match_offset + 1; i < wt->match_offset + 1 + num; i++) {
+        PyObject *t = PyLong_FromUnsignedLong(mark_to_cp[mark_groups[i]]);
+        if (!t) return;
+        int ret = PySet_Add(ans, t);
+        Py_DECREF(t);
+        if (ret != 0) return;
     }
 }
 
 static void
-process_trie_node(const word_trie *wt, char_type *codepoints, size_t *pos, const size_t sz) {
-    if (wt->match_offset) add_matches(wt, codepoints, pos, sz);
+process_trie_node(const word_trie *wt, PyObject *ans) {
+    if (wt->match_offset) { add_matches(wt, ans); if (PyErr_Occurred()) return; }
     size_t num_children = children_array[wt->children_offset];
     if (!num_children) return;
     for (size_t c = wt->children_offset + 1; c < wt->children_offset + 1 + num_children; c++) {
-        if (*pos > sz) return;
         uint32_t x = children_array[c];
-        process_trie_node(&all_trie_nodes[x >> 8], codepoints, pos, sz);
+        process_trie_node(&all_trie_nodes[x >> 8], ans);
+        if (PyErr_Occurred()) return;
     }
 }
 
-static inline PyObject*
+static PyObject*
 codepoints_for_word(const char *word, size_t len) {
     const word_trie *wt = all_trie_nodes;
     for (size_t i = 0; i < len; i++) {
@@ -57,14 +61,10 @@ codepoints_for_word(const char *word, size_t len) {
         }
         if (!found) return PyFrozenSet_New(NULL);
     }
-    static char_type codepoints[1024];
-    size_t cpos = 0;
-    process_trie_node(wt, codepoints, &cpos, arraysz(codepoints));
-    PyObject *ans = PyFrozenSet_New(NULL); if (ans == NULL) return NULL;
-    for (size_t i = 0; i < cpos; i++) {
-        PyObject *t = PyLong_FromUnsignedLong(codepoints[i]); if (t == NULL) { Py_DECREF(ans); return NULL; }
-        int ret = PySet_Add(ans, t); Py_DECREF(t); if (ret != 0) { Py_DECREF(ans); return NULL; }
-    }
+    PyObject *ans = PyFrozenSet_New(NULL);
+    if (!ans) return NULL;
+    process_trie_node(wt, ans);
+    if (PyErr_Occurred()) return NULL;
     return ans;
 }
 

kitty/actions.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+import inspect
+from typing import Dict, List, NamedTuple
+
+from .boss import Boss
+from .tabs import Tab
+from .types import run_once
+from .window import Window
+
+
+class Action(NamedTuple):
+    name: str
+    group: str
+    short_help: str
+    long_help: str
+
+
+groups = {
+    'cp': 'Copy/paste',
+    'sc': 'Scrolling',
+    'win': 'Window management',
+    'tab': 'Tab management',
+    'mouse': 'Mouse actions',
+    'mk': 'Marks',
+    'lay': 'Layouts',
+    'misc': 'Miscellaneous',
+}
+group_title = groups.__getitem__
+
+
+@run_once
+def get_all_actions() -> Dict[str, List[Action]]:
+
+    ans: Dict[str, List[Action]] = {}
+
+    def is_action(x: object) -> bool:
+        doc = getattr(x, '__doc__', '')
+        return bool(doc and doc.strip().startswith('@ac:'))
+
+    def as_action(x: object) -> Action:
+        doc = inspect.cleandoc(x.__doc__ or '')
+        lines = doc.splitlines()
+        first = lines.pop(0)
+        parts = first.split(':', 2)
+        grp = parts[1].strip()
+        short_help = parts[2].strip()
+        long_help = '\n'.join(lines).strip()
+        return Action(getattr(x, '__name__'), grp, short_help, long_help)
+
+    seen = set()
+    for cls in (Window, Tab, Boss):
+        for (name, func) in inspect.getmembers(cls, is_action):
+            ac = as_action(func)
+            if ac.name not in seen:
+                ans.setdefault(ac.group, []).append(ac)
+                seen.add(ac.name)
+    for i, which in enumerate('first second third fourth fifth sixth seventh eighth ninth tenth'.split()):
+        name = f'{which}_window'
+        if name not in seen:
+            seen.add(name)
+            ans['win'].append(Action(name, 'win', f'Focus the {which} window', ''))
+
+    return ans
+
+
+def dump() -> None:
+    from pprint import pprint
+    pprint(get_all_actions())
+
+
+def as_rst() -> str:
+    from .options.definition import definition
+    from .conf.types import Mapping
+    allg = get_all_actions()
+    lines: List[str] = []
+    a = lines.append
+    maps: Dict[str, List[Mapping]] = {}
+    for m in definition.iter_all_maps():
+        if m.documented:
+            func = m.action_def.split()[0]
+            maps.setdefault(func, []).append(m)
+
+    for group in sorted(allg, key=lambda x: group_title(x).lower()):
+        title = group_title(group)
+        a('')
+        a(f'.. _action-group-{group}:')
+        a('')
+        a(title)
+        a('-' * len(title))
+        a('')
+
+        for action in allg[group]:
+            a('')
+            a(f'.. _action-{action.name}:')
+            a('')
+            a(action.name)
+            a('+' * len(action.name))
+            a('')
+            a(action.short_help)
+            a('')
+            if action.long_help:
+                a(action.long_help)
+            if action.name in maps:
+                a('')
+                a('Default shortcuts using this action:')
+                scs = {f':sc:`kitty.{m.name}`' for m in maps[action.name]}
+                a(', '.join(sorted(scs)))
+    return '\n'.join(lines)

kitty/boss.py

@@ -24,7 +24,7 @@ from .conf.utils import BadLine, KeyAction, to_cmdline
 from .config import common_opts_as_dict, prepare_config_file_for_editing
 from .constants import (
     appname, config_dir, is_macos, is_wayland, kitty_exe,
-    supports_primary_selection
+    supports_primary_selection, website_url
 )
 from .fast_data_types import (
     CLOSE_BEING_CONFIRMED, IMPERATIVE_CLOSE_REQUESTED, NO_CLOSE_REQUESTED,
@@ -32,11 +32,12 @@ from .fast_data_types import (
     background_opacity_of, change_background_opacity, change_os_window_state,
     cocoa_set_menubar_title, create_os_window,
     current_application_quit_request, current_os_window, destroy_global_data,
-    focus_os_window, get_clipboard_string, get_options, global_font_size,
-    mark_os_window_for_close, os_window_font_size, patch_global_colors,
-    safe_pipe, set_application_quit_request, set_background_image, set_boss,
-    set_clipboard_string, set_in_sequence_mode, set_options, thread_write,
-    toggle_fullscreen, toggle_maximized
+    focus_os_window, get_clipboard_string, get_options, get_os_window_size,
+    global_font_size, mark_os_window_for_close, os_window_font_size,
+    patch_global_colors, safe_pipe, set_application_quit_request,
+    set_background_image, set_boss, set_clipboard_string, set_in_sequence_mode,
+    set_options, set_os_window_size, thread_write, toggle_fullscreen,
+    toggle_maximized
 )
 from .keys import get_shortcut, shortcut_matches
 from .layout.base import set_layout_options
@@ -52,10 +53,11 @@ from .tabs import (
 from .types import SingleKey
 from .typing import PopenType, TypedDict
 from .utils import (
-    func_name, get_editor, get_primary_selection, is_path_in_temp_dir,
-    log_error, open_url, parse_address_spec, parse_uri_list,
-    platform_window_id, read_shell_environment, remove_socket_file, safe_print,
-    set_primary_selection, single_instance, startup_notification_handler
+    func_name, get_editor, get_new_os_window_size, get_primary_selection,
+    is_path_in_temp_dir, log_error, open_url, parse_address_spec,
+    parse_uri_list, platform_window_id, read_shell_environment,
+    remove_socket_file, safe_print, set_primary_selection, single_instance,
+    startup_notification_handler
 )
 from .window import MatchPatternType, Window
 
@@ -332,6 +334,7 @@ class Boss:
         return self.add_os_window(startup_session)
 
     def new_os_window(self, *args: str) -> None:
+        '@ac:win: New OS Window'
         self._new_os_window(args)
 
     @property
@@ -341,6 +344,7 @@ class Boss:
             return t.active_window_for_cwd
 
     def new_os_window_with_cwd(self, *args: str) -> None:
+        '@ac:win: New OS Window with the same working directory as the currently active window'
         w = self.active_window_for_cwd
         cwd_from = w.child.pid_for_cwd if w is not None else None
         self._new_os_window(args, cwd_from)
@@ -377,6 +381,15 @@ class Boss:
         return response
 
     def remote_control(self, *args: str) -> None:
+        '''
+        @ac:misc: Run a remote control command
+
+        For example::
+
+            map F1 remote_control set-spacing margin=30
+
+        See :ref:`rc_mapping` for details.
+        '''
         from .rc.base import (
             PayloadGetter, command_for_name, parse_subcommand_cli
         )
@@ -489,6 +502,7 @@ class Boss:
             self.child_monitor.mark_for_close(window.id)
 
     def close_tab(self, tab: Optional[Tab] = None) -> None:
+        '@ac:tab: Close the current tab'
         tab = tab or self.active_tab
         if tab:
             self.confirm_tab_close(tab)
@@ -520,11 +534,13 @@ class Boss:
         for window in tab:
             self.close_window(window)
 
-    def toggle_fullscreen(self) -> None:
-        toggle_fullscreen()
+    def toggle_fullscreen(self, os_window_id: int = 0) -> None:
+        '@ac:win: Toggle the fullscreen status of the active OS Window'
+        toggle_fullscreen(os_window_id)
 
-    def toggle_maximized(self) -> None:
-        toggle_maximized()
+    def toggle_maximized(self, os_window_id: int = 0) -> None:
+        '@ac:win: Toggle the maximized status of the active OS Window'
+        toggle_maximized(os_window_id)
 
     def start(self, first_os_window_id: int) -> None:
         if not getattr(self, 'io_thread_started', False):
@@ -551,6 +567,21 @@ class Boss:
                 tm.resize()
 
     def clear_terminal(self, action: str, only_active: bool) -> None:
+        '''
+        @ac:misc: Clear the terminal
+
+        See :sc:`reset_terminal` for details. For example::
+
+            # Reset the terminal
+            map kitty_mod+f9 clear_terminal reset active
+            # Clear the terminal screen by erasing all contents
+            map kitty_mod+f10 clear_terminal clear active
+            # Clear the terminal scrollback by erasing it
+            map kitty_mod+f11 clear_terminal scrollback active
+            # Scroll the contents of the screen into the scrollback
+            map kitty_mod+f12 clear_terminal scroll active
+
+        '''
         if only_active:
             windows = []
             w = self.active_window
@@ -585,6 +616,11 @@ class Boss:
         self.change_font_size(True, None, new_size)
 
     def change_font_size(self, all_windows: bool, increment_operation: Optional[str], amt: float) -> None:
+        '''
+        @ac:win: Change the font size for the current or all OS Windows
+
+        See :ref:`conf-kitty-shortcuts.fonts` for details.
+        '''
         def calc_new_size(old_size: float) -> float:
             new_size = old_size
             if amt == 0:
@@ -641,6 +677,15 @@ class Boss:
         change_background_opacity(os_window_id, max(0.1, min(opacity, 1.0)))
 
     def set_background_opacity(self, opacity: str) -> None:
+        '''
+        @ac:win: Set the background opacity for the active OS Window
+
+        For example::
+
+            map f1 set_background_opacity +0.1
+            map f2 set_background_opacity -0.1
+            map f3 set_background_opacity 0.5
+        '''
         window = self.active_window
         if window is None or not opacity:
             return
@@ -717,6 +762,11 @@ class Boss:
                 self.dispatch_action(matched_action)
 
     def start_resizing_window(self) -> None:
+        '''
+        @ac:win: Resize the active window interactively
+
+        See :ref:`window_resizing` for details.
+        '''
         w = self.active_window
         if w is None:
             return
@@ -736,6 +786,16 @@ class Boss:
             return None
         return tab.resize_window_by(window.id, increment, is_horizontal)
 
+    def resize_os_window(self, os_window_id: int, width: int, height: int, unit: str, incremental: bool = False) -> None:
+        if not incremental and (width < 0 or height < 0):
+            return
+        metrics = get_os_window_size(os_window_id)
+        if metrics is None:
+            return
+        has_window_scaling = is_macos or is_wayland()
+        w, h = get_new_os_window_size(metrics, width, height, unit, incremental, has_window_scaling)
+        set_os_window_size(os_window_id, w, h)
+
     def default_bg_changed_for(self, window_id: int) -> None:
         w = self.window_id_map.get(window_id)
         if w is not None:
@@ -756,7 +816,8 @@ class Boss:
 
         def report_match(f: Callable) -> None:
             if self.args.debug_keyboard:
-                print(f'\x1b[35m{dispatch_type}\x1b[m matched action:', func_name(f), flush=True)
+                prefix = '\n' if dispatch_type == 'KeyPress' else ''
+                print(f'{prefix}\x1b[35m{dispatch_type}\x1b[m matched action:', func_name(f), flush=True)
 
         if key_action is not None:
             f = getattr(self, key_action.func, None)
@@ -783,6 +844,17 @@ class Boss:
         return False
 
     def combine(self, *actions: KeyAction) -> None:
+        '''
+        @ac:misc: Combine multiple actions and map to a single keypress
+
+        The syntax is::
+
+            map key combine <separator> action1 <separator> action2 <separator> action3 ...
+
+        For example::
+
+            map kitty_mod+e combine : new_window : next_layout
+        '''
         for key_action in actions:
             self.dispatch_action(key_action)
 
@@ -818,6 +890,7 @@ class Boss:
                 w.paste(text)
 
     def close_os_window(self) -> None:
+        '@ac:win: Close the currently active OS Window'
         tm = self.active_tab_manager
         if tm is not None:
             self.confirm_os_window_close(tm.os_window_id)
@@ -857,6 +930,7 @@ class Boss:
             action()
 
     def quit(self, *args: Any) -> None:
+        '@ac:win: Quit, closing all windows'
         tm = self.active_tab
         num = 0
         for q in self.os_window_map.values():
@@ -906,6 +980,8 @@ class Boss:
                     if exe:
                         cmd[0] = exe
 
+        if os.path.basename(cmd[0]) == 'less':
+            cmd.append('-+F')  # reset --quit-if-one-screen
         tab = self.active_tab
         if tab is not None:
             bdata = data.encode('utf-8') if isinstance(data, str) else data
@@ -915,6 +991,7 @@ class Boss:
                 )
 
     def edit_config_file(self, *a: Any) -> None:
+        '@ac:misc: Edit the kitty.conf config file in your favorite text editor'
         confpath = prepare_config_file_for_editing()
         # On macOS vim fails to handle SIGWINCH if it occurs early, so add a
         # small delay.
@@ -1005,6 +1082,7 @@ class Boss:
             return overlay_window
 
     def kitten(self, kitten: str, *args: str) -> None:
+        '@ac:misc: Run the specified kitten. See :doc:`/kittens/custom` for details'
         import shlex
         cmdline = args[0] if args else ''
         kargs = shlex.split(cmdline) if cmdline else []
@@ -1021,9 +1099,11 @@ class Boss:
             end_kitten(data, target_window_id, self)
 
     def input_unicode_character(self) -> None:
+        '@ac:misc: Input an arbitrary unicode character. See :doc:`/kittens/unicode-input` for details.'
         self._run_kitten('unicode_input')
 
     def set_tab_title(self) -> None:
+        '@ac:tab: Change the title of the active tab'
         tab = self.active_tab
         if tab:
             args = ['--name=tab-title', '--message', _('Enter the new title for this tab below.'), 'do_set_tab_title', str(tab.id)]
@@ -1042,6 +1122,7 @@ class Boss:
         self._run_kitten('show_error', args=['--title', title], input_data=msg)
 
     def create_marker(self) -> None:
+        '@ac:mk: Create a new marker'
         w = self.active_window
         if w:
             spec = None
@@ -1060,11 +1141,12 @@ class Boss:
 
             self._run_kitten('ask', [
                 '--name=create-marker', '--message',
-                _('Create marker, for example:\ntext 1 ERROR\nSee https://sw.kovidgoyal.net/kitty/marks.html\n')
+                _('Create marker, for example:\ntext 1 ERROR\nSee {}\n').format(website_url('marks'))
                 ],
                 custom_callback=done, action_on_removal=done2)
 
-    def kitty_shell(self, window_type: str) -> None:
+    def kitty_shell(self, window_type: str = 'window') -> None:
+        '@ac:misc: Run the kitty shell to control kitty with commands'
         kw: Dict[str, Any] = {}
         cmd = [kitty_exe(), '@']
         aw = self.active_window
@@ -1112,6 +1194,10 @@ class Boss:
         if not found_action:
             open_url(url, program or get_options().open_url_with, cwd=cwd)
 
+    def open_url_with_hints(self) -> None:
+        '@ac:misc: Click a URL using the keyboard'
+        self._run_kitten('hints')
+
     def drain_actions(self, actions: List) -> None:
 
         def callback(timer_id: Optional[int]) -> None:
@@ -1138,6 +1224,7 @@ class Boss:
                 w.paste(text)
 
     def paste_from_clipboard(self) -> None:
+        '@ac:cp: Paste from the clipboard to the active window'
         text = get_clipboard_string()
         self.paste_to_active_window(text)
 
@@ -1148,6 +1235,7 @@ class Boss:
         return get_primary_selection() if supports_primary_selection else get_clipboard_string()
 
     def paste_from_selection(self) -> None:
+        '@ac:cp: Paste from the clipboard to the active window'
         text = self.current_primary_selection_or_clipboard()
         self.paste_to_active_window(text)
 
@@ -1161,6 +1249,11 @@ class Boss:
                     self.copy_to_buffer(get_options().copy_on_select)
 
     def copy_to_buffer(self, buffer_name: str) -> None:
+        '''
+        @ac:cp: Copy the selection from the active window to the specified buffer
+
+        See :ref:`cpbuf` for details.
+        '''
         w = self.active_window
         if w is not None and not w.destroyed:
             text = w.text_for_selection()
@@ -1173,6 +1266,11 @@ class Boss:
                     self.clipboard_buffers[buffer_name] = text
 
     def paste_from_buffer(self, buffer_name: str) -> None:
+        '''
+        @ac:cp: Paste from the specified buffer to the active window
+
+        See :ref:`cpbuf` for details.
+        '''
         if buffer_name == 'clipboard':
             text: Optional[str] = get_clipboard_string()
         elif buffer_name == 'primary':
@@ -1183,6 +1281,11 @@ class Boss:
             self.paste_to_active_window(text)
 
     def goto_tab(self, tab_num: int) -> None:
+        '''
+        @ac:tab: Go to the specified tab, by number, starting with 1
+
+        Zero and negative numbers go to previously active tabs
+        '''
         tm = self.active_tab_manager
         if tm is not None:
             tm.goto_tab(tab_num - 1)
@@ -1194,11 +1297,13 @@ class Boss:
         return False
 
     def next_tab(self) -> None:
+        '@ac:tab: Make the next tab active'
         tm = self.active_tab_manager
         if tm is not None:
             tm.next_tab()
 
     def previous_tab(self) -> None:
+        '@ac:tab: Make the previous tab active'
         tm = self.active_tab_manager
         if tm is not None:
             tm.next_tab(-1)
@@ -1358,9 +1463,11 @@ class Boss:
         self._new_tab(args, as_neighbor=as_neighbor, cwd_from=cwd_from)
 
     def new_tab(self, *args: str) -> None:
+        '@ac:tab: Create a new tab'
         self._create_tab(list(args))
 
     def new_tab_with_cwd(self, *args: str) -> None:
+        '@ac:tab: Create a new tab with working directory for the window in it set to the same as the active window'
         w = self.active_window_for_cwd
         cwd_from = w.child.pid_for_cwd if w is not None else None
         self._create_tab(list(args), cwd_from=cwd_from)
@@ -1388,9 +1495,11 @@ class Boss:
                 return tab.new_window(cwd_from=cwd_from, location=location, allow_remote_control=allow_remote_control)
 
     def new_window(self, *args: str) -> None:
+        '@ac:win: Create a new window'
         self._new_window(list(args))
 
     def new_window_with_cwd(self, *args: str) -> None:
+        '@ac:win: Create a new window with working directory same as that of the active window'
         w = self.active_window_for_cwd
         if w is None:
             return self.new_window(*args)
@@ -1398,21 +1507,33 @@ class Boss:
         self._new_window(list(args), cwd_from=cwd_from)
 
     def launch(self, *args: str) -> None:
+        '''
+        @ac:misc: Launch the specified program in a new window/tab/etc.
+
+        See :doc:`launch` for details
+        '''
         from kitty.launch import launch, parse_launch_args
         opts, args_ = parse_launch_args(args)
         launch(self, opts, args_)
 
     def move_tab_forward(self) -> None:
+        '@ac:tab: Move the active tab forward'
         tm = self.active_tab_manager
         if tm is not None:
             tm.move_tab(1)
 
     def move_tab_backward(self) -> None:
+        '@ac:tab: Move the active tab backward'
         tm = self.active_tab_manager
         if tm is not None:
             tm.move_tab(-1)
 
     def disable_ligatures_in(self, where: Union[str, Iterable[Window]], strategy: int) -> None:
+        '''
+        @ac:misc: Turn on/off ligatures in the specified window
+
+        See :opt:`disable_ligatures` for details
+        '''
         if isinstance(where, str):
             windows: List[Window] = []
             if where == 'active':
@@ -1470,6 +1591,15 @@ class Boss:
             w.refresh()
 
     def load_config_file(self, *paths: str, apply_overrides: bool = True) -> None:
+        '''
+        @ac:misc: Reload the config file
+
+        If mapped without arguments reloads the default config file, otherwise loads
+        the specified config files, in order. Loading a config file *replaces* all
+        config options. For example::
+
+            map f5 load_config_file /path/to/some/kitty.conf
+        '''
         from .config import load_config
         old_opts = get_options()
         paths = paths or old_opts.config_paths
@@ -1527,6 +1657,13 @@ class Boss:
         self.show_error(_('Errors in kitty.conf'), msg)
 
     def set_colors(self, *args: str) -> None:
+        '''
+        @ac:misc: Change colors in the specified windows
+
+        For details, see :ref:`at_set-colors`. For example::
+
+            map f5 set_colors --configured /path/to/some/config/file/colors.conf
+        '''
         from kitty.rc.base import (
             PayloadGetter, command_for_name, parse_subcommand_cli
         )
@@ -1554,14 +1691,17 @@ class Boss:
             target_tab = tm.new_tab(empty_tab=True)
         else:
             target_os_window_id = target_os_window_id or current_os_window()
-            if target_tab_id == 'new':
+            if isinstance(target_tab_id, str):
                 if not isinstance(target_os_window_id, int):
                     q = self.active_tab_manager
                     assert q is not None
                     tm = q
                 else:
                     tm = self.os_window_map[target_os_window_id]
-                target_tab = tm.new_tab(empty_tab=True)
+                if target_tab_id == 'new':
+                    target_tab = tm.new_tab(empty_tab=True)
+                else:
+                    target_tab = tm.tab_at_location(target_tab_id) or tm.new_tab(empty_tab=True)
             else:
                 for tab in self.all_tabs:
                     if tab.id == target_tab_id:
@@ -1589,6 +1729,7 @@ class Boss:
         target_tab.make_active()
 
     def select_tab(self) -> None:
+        '@ac:tab: Interactively select a tab to switch to'
         title = 'Choose a tab to switch to'
         lines = [title, '']
         fmt = ': {1}'
@@ -1621,10 +1762,16 @@ class Boss:
         )
 
     def detach_window(self, *args: str) -> None:
+        '''
+        @ac:win: Detach a window, moving it to another tab or OS Window
+
+        See :ref:`detaching windows <detach_window>` for details.
+        '''
         if not args or args[0] == 'new':
             return self._move_window_to(target_os_window_id='new')
-        if args[0] == 'new-tab':
-            return self._move_window_to(target_tab_id='new')
+        if args[0] in ('new-tab', 'tab-prev', 'tab-left', 'tab-right'):
+            where = 'new' if args[0] == 'new-tab' else args[0][4:]
+            return self._move_window_to(target_tab_id=where)
         title = 'Choose a tab to move the window to'
         lines = [title, '']
         fmt = ': {1}'
@@ -1671,6 +1818,11 @@ class Boss:
         )
 
     def detach_tab(self, *args: str) -> None:
+        '''
+        @ac:tab: Detach a tab, moving it to another OS Window
+
+        See :ref:`detaching windows <detach_window>` for details.
+        '''
         if not args or args[0] == 'new':
             return self._move_tab_to()
 
@@ -1743,6 +1895,7 @@ class Boss:
             w.report_notification_activated(identifier)
 
     def show_kitty_env_vars(self) -> None:
+        '@ac:misc: Show the environment variables that the kitty process sees'
         w = self.active_window
         if w:
             output = '\n'.join(f'{k}={v}' for k, v in os.environ.items())
@@ -1767,6 +1920,7 @@ class Boss:
             tab.remove_window(w)
 
     def debug_config(self) -> None:
+        '@ac:misc: Show the effective configuration kitty is running with'
         from .debug_config import debug_config
         w = self.active_window
         if w is not None:
@@ -1774,3 +1928,8 @@ class Boss:
             set_clipboard_string(re.sub(r'\x1b.+?m', '', output))
             output += '\n\x1b[35mThis debug output has been copied to the clipboard\x1b[m'
             self.display_scrollback(w, output, title=_('Current kitty options'))
+
+    def discard_event(self) -> None:
+        '@ac:misc: Discard this event completely ignoring it'
+        pass
+    mouse_discard_event = discard_event

kitty/child-monitor.c

@@ -523,7 +523,7 @@ collect_cursor_info(CursorRenderInfo *ans, Window *w, monotonic_t now, OSWindow
     ans->is_visible = false;
     if (rd->screen->scrolled_by || !screen_is_cursor_visible(rd->screen)) return;
     monotonic_t time_since_start_blink = now - os_window->cursor_blink_zero_time;
-    bool cursor_blinking = OPT(cursor_blink_interval) > 0 && os_window->is_focused && (OPT(cursor_stop_blinking_after) == 0 || time_since_start_blink <= OPT(cursor_stop_blinking_after));
+    bool cursor_blinking = OPT(cursor_blink_interval) > 0 && !cursor->non_blinking && os_window->is_focused && (OPT(cursor_stop_blinking_after) == 0 || time_since_start_blink <= OPT(cursor_stop_blinking_after));
     bool do_draw_cursor = true;
     if (cursor_blinking) {
         int t = monotonic_t_to_ms(time_since_start_blink);
@@ -971,6 +971,41 @@ set_cocoa_pending_action(CocoaPendingAction action, const char *wd) {
     // Unjam it so the pending action is processed right now.
     wakeup_main_loop();
 }
+
+static void
+process_cocoa_pending_actions(void) {
+    if (cocoa_pending_actions[PREFERENCES_WINDOW]) { call_boss(edit_config_file, NULL); }
+    if (cocoa_pending_actions[NEW_OS_WINDOW]) { call_boss(new_os_window, NULL); }
+    if (cocoa_pending_actions[CLOSE_OS_WINDOW]) { call_boss(close_os_window, NULL); }
+    if (cocoa_pending_actions[CLOSE_TAB]) { call_boss(close_tab, NULL); }
+    if (cocoa_pending_actions[NEW_TAB]) { call_boss(new_tab, NULL); }
+    if (cocoa_pending_actions[NEXT_TAB]) { call_boss(next_tab, NULL); }
+    if (cocoa_pending_actions[PREVIOUS_TAB]) { call_boss(previous_tab, NULL); }
+    if (cocoa_pending_actions[DETACH_TAB]) { call_boss(detach_tab, NULL); }
+    if (cocoa_pending_actions[NEW_WINDOW]) { call_boss(new_window, NULL); }
+    if (cocoa_pending_actions[CLOSE_WINDOW]) { call_boss(close_window, NULL); }
+    if (cocoa_pending_actions[RESET_TERMINAL]) { call_boss(clear_terminal, "sO", "reset", Py_True ); }
+    if (cocoa_pending_actions[RELOAD_CONFIG]) { call_boss(load_config_file, NULL); }
+    if (cocoa_pending_actions_data.wd) {
+        if (cocoa_pending_actions[NEW_OS_WINDOW_WITH_WD]) { call_boss(new_os_window_with_wd, "s", cocoa_pending_actions_data.wd); }
+        if (cocoa_pending_actions[NEW_TAB_WITH_WD]) { call_boss(new_tab_with_wd, "s", cocoa_pending_actions_data.wd); }
+        free(cocoa_pending_actions_data.wd);
+        cocoa_pending_actions_data.wd = NULL;
+    }
+    if (cocoa_pending_actions_data.open_files_count) {
+        for (unsigned cpa = 0; cpa < cocoa_pending_actions_data.open_files_count; cpa++) {
+            if (cocoa_pending_actions_data.open_files[cpa]) {
+                call_boss(open_file, "s", cocoa_pending_actions_data.open_files[cpa]);
+                free(cocoa_pending_actions_data.open_files[cpa]);
+                cocoa_pending_actions_data.open_files[cpa] = NULL;
+            }
+        }
+        cocoa_pending_actions_data.open_files_count = 0;
+    }
+    memset(cocoa_pending_actions, 0, sizeof(cocoa_pending_actions));
+    has_cocoa_pending_actions = false;
+
+}
 #endif
 
 static void process_global_state(void *data);
@@ -999,38 +1034,10 @@ process_global_state(void *data) {
     if (parse_input(self)) input_read = true;
     render(now, input_read);
 #ifdef __APPLE__
-        if (has_cocoa_pending_actions) {
-            if (cocoa_pending_actions[PREFERENCES_WINDOW]) { call_boss(edit_config_file, NULL); }
-            if (cocoa_pending_actions[NEW_OS_WINDOW]) { call_boss(new_os_window, NULL); }
-            if (cocoa_pending_actions[CLOSE_OS_WINDOW]) { call_boss(close_os_window, NULL); }
-            if (cocoa_pending_actions[CLOSE_TAB]) { call_boss(close_tab, NULL); }
-            if (cocoa_pending_actions[NEW_TAB]) { call_boss(new_tab, NULL); }
-            if (cocoa_pending_actions[NEXT_TAB]) { call_boss(next_tab, NULL); }
-            if (cocoa_pending_actions[PREVIOUS_TAB]) { call_boss(previous_tab, NULL); }
-            if (cocoa_pending_actions[DETACH_TAB]) { call_boss(detach_tab, NULL); }
-            if (cocoa_pending_actions[NEW_WINDOW]) { call_boss(new_window, NULL); }
-            if (cocoa_pending_actions[CLOSE_WINDOW]) { call_boss(close_window, NULL); }
-            if (cocoa_pending_actions[RESET_TERMINAL]) { call_boss(clear_terminal, "sO", "reset", Py_True ); }
-            if (cocoa_pending_actions[RELOAD_CONFIG]) { call_boss(load_config_file, NULL); }
-            if (cocoa_pending_actions_data.wd) {
-                if (cocoa_pending_actions[NEW_OS_WINDOW_WITH_WD]) { call_boss(new_os_window_with_wd, "s", cocoa_pending_actions_data.wd); }
-                if (cocoa_pending_actions[NEW_TAB_WITH_WD]) { call_boss(new_tab_with_wd, "s", cocoa_pending_actions_data.wd); }
-                free(cocoa_pending_actions_data.wd);
-                cocoa_pending_actions_data.wd = NULL;
-            }
-            if (cocoa_pending_actions_data.open_files_count) {
-                for (unsigned cpa = 0; cpa < cocoa_pending_actions_data.open_files_count; cpa++) {
-                    if (cocoa_pending_actions_data.open_files[cpa]) {
-                        call_boss(open_file, "s", cocoa_pending_actions_data.open_files[cpa]);
-                        free(cocoa_pending_actions_data.open_files[cpa]);
-                        cocoa_pending_actions_data.open_files[cpa] = NULL;
-                    }
-                }
-                cocoa_pending_actions_data.open_files_count = 0;
-            }
-            memset(cocoa_pending_actions, 0, sizeof(cocoa_pending_actions));
-            has_cocoa_pending_actions = false;
-        }
+    if (has_cocoa_pending_actions) {
+        process_cocoa_pending_actions();
+        maximum_wait = 0;  // ensure loop ticks again so that the actions side effects are performed immediately
+    }
 #endif
     report_reaped_pids();
     bool should_quit = false;

kitty/cli.py

@@ -12,7 +12,7 @@ from typing import (
 
 from .cli_stub import CLIOptions
 from .conf.utils import resolve_config
-from .constants import appname, defconf, is_macos, str_version
+from .constants import appname, defconf, is_macos, str_version, website_url
 from .options.types import Options as KittyOpts
 from .typing import BadLineType, TypedDict
 
@@ -39,7 +39,7 @@ If this option is not specified, config files are searched for in the order:
 :file:`$XDG_CONFIG_DIRS/{appname}/{conf_name}.conf`. The first one that exists is used as the
 config file.
 
-If the environment variable :env:`KITTY_CONFIG_DIRECTORY` is specified, that
+If the environment variable :envvar:`KITTY_CONFIG_DIRECTORY` is specified, that
 directory is always used and the above searching does not happen.
 
 If :file:`/etc/xdg/{appname}/{conf_name}.conf` exists it is merged before (i.e. with lower
@@ -57,42 +57,60 @@ def surround(x: str, start: int, end: int) -> str:
     return x
 
 
+role_map: Dict[str, Callable[[str], str]] = {}
+
+
+def role(func: Callable[[str], str]) -> Callable[[str], str]:
+    role_map[func.__name__] = func
+    return func
+
+
+@role
 def emph(x: str) -> str:
     return surround(x, 91, 39)
 
 
+@role
 def cyan(x: str) -> str:
     return surround(x, 96, 39)
 
 
+@role
 def green(x: str) -> str:
     return surround(x, 32, 39)
 
 
+@role
 def blue(x: str) -> str:
     return surround(x, 34, 39)
 
 
+@role
 def yellow(x: str) -> str:
     return surround(x, 93, 39)
 
 
+@role
 def italic(x: str) -> str:
     return surround(x, 3, 23)
 
 
+@role
 def bold(x: str) -> str:
     return surround(x, 1, 22)
 
 
+@role
 def title(x: str) -> str:
     return blue(bold(x))
 
 
+@role
 def opt(text: str) -> str:
     return text
 
 
+@role
 def option(x: str) -> str:
     idx = x.rfind('--')
     if idx < 0:
@@ -103,24 +121,32 @@ def option(x: str) -> str:
     return ' '.join(parts)
 
 
+@role
 def code(x: str) -> str:
     return x
 
 
+@role
 def kbd(x: str) -> str:
     return x
 
 
+@role
 def env(x: str) -> str:
     return italic(x)
 
 
+role_map['envvar'] = role_map['env']
+
+
+@role
 def file(x: str) -> str:
     return italic(x)
 
 
+@role
 def doc(x: str) -> str:
-    return f'https://sw.kovidgoyal.net/kitty/{x}.html'
+    return website_url(x)
 
 
 OptionSpecSeq = List[Union[str, OptionDict]]
@@ -197,11 +223,13 @@ def parse_option_spec(spec: Optional[str] = None) -> Tuple[OptionSpecSeq, Option
 
 
 def prettify(text: str) -> str:
-    role_map = globals()
+
+    def identity(x: str) -> str:
+        return x
 
     def sub(m: Match) -> str:
         role, text = m.group(1, 2)
-        return str(role_map[role](text))
+        return role_map.get(role, identity)(text)
 
     text = re.sub(r':([a-z]+):`([^`]+)`', sub, text)
     return text
@@ -265,7 +293,8 @@ Run the :italic:`{appname}` terminal emulator. You can also specify the :italic:
 to run inside :italic:`{appname}` as normal arguments following the :italic:`options`.
 For example: {appname} sh -c "echo hello, world. Press ENTER to quit; read"
 
-For comprehensive documentation for kitty, please see: https://sw.kovidgoyal.net/kitty/''').format(appname=appname)
+For comprehensive documentation for kitty, please see: {url}''').format(
+    appname=appname, url=website_url())
 
 
 class PrintHelpForSeq:

kitty/client.py

@@ -208,10 +208,27 @@ def write_osc(code: int, string: str = '') -> None:
 
 
 set_dynamic_color = set_color_table_color = process_cwd_notification = write_osc
+clipboard_control_pending: str = ''
+
+
+def clipboard_control(payload: str) -> None:
+    global clipboard_control_pending
+    code, data = payload.split(';', 1)
+    if code == '-52':
+        if clipboard_control_pending:
+            clipboard_control_pending += data.lstrip(';')
+        else:
+            clipboard_control_pending = payload
+        return
+    if clipboard_control_pending:
+        clipboard_control_pending += data.lstrip(';')
+        payload = clipboard_control_pending
+        clipboard_control_pending = ''
+    write(OSC + payload + '\x07')
 
 
 def replay(raw: str) -> None:
-    specials = {'draw', 'set_title', 'set_icon', 'set_dynamic_color', 'set_color_table_color', 'process_cwd_notification'}
+    specials = {'draw', 'set_title', 'set_icon', 'set_dynamic_color', 'set_color_table_color', 'process_cwd_notification', 'clipboard_control'}
     for line in raw.splitlines():
         if line.strip() and not line.startswith('#'):
             cmd, rest = line.partition(' ')[::2]

kitty/complete.py

@@ -6,17 +6,23 @@ import os
 import shlex
 import sys
 from typing import (
-    Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
+    Any, Callable, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple,
+    Union
 )
 
-from kittens.runner import all_kitten_names, get_kitten_cli_docs
+from kittens.runner import (
+    all_kitten_names, get_kitten_cli_docs, get_kitten_completer
+)
 
 from .cli import (
-    OptionDict, OptionSpecSeq, options_for_completion, parse_option_spec
+    OptionDict, OptionSpecSeq, options_for_completion, parse_option_spec,
+    prettify
 )
+from .fast_data_types import truncate_point_for_length, wcswidth
 from .rc.base import all_command_names, command_for_name
 from .shell import options_for_cmd
 from .types import run_once
+from .utils import screen_size_function
 
 '''
 To add completion for a new shell, you need to:
@@ -39,12 +45,42 @@ them into something your shell will understand.
 
 parsers: Dict[str, Callable] = {}
 serializers: Dict[str, Callable] = {}
-MathGroup = Dict[str, str]
+
+
+class MatchGroup:
+
+    def __init__(
+        self, x: Union[Dict[str, str], Iterable[str]],
+        trailing_space: bool = True,
+        is_files: bool = False,
+        word_transforms: Optional[Dict[str, str]] = None,
+    ):
+        self.mdict = x if isinstance(x, dict) else dict.fromkeys(x, '')
+        self.trailing_space = trailing_space
+        self.is_files = is_files
+        self.word_transforms = word_transforms or {}
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self.mdict)
+
+    def transformed_words(self) -> Iterator[str]:
+        for w in self:
+            yield self.word_transforms.get(w, w)
+
+    def transformed_items(self) -> Iterator[Tuple[str, str]]:
+        for w, desc in self.items():
+            yield self.word_transforms.get(w, w), desc
+
+    def items(self) -> Iterator[Tuple[str, str]]:
+        return iter(self.mdict.items())
+
+    def values(self) -> Iterator[str]:
+        return iter(self.mdict.values())
 
 
 def debug(*a: Any, **kw: Any) -> None:
-    kw['file'] = sys.stderr
-    print(*a, **kw)
+    from kittens.tui.loop import debug_write
+    debug_write(*a, **kw)
 
 
 class Delegate:
@@ -69,11 +105,18 @@ class Delegate:
 class Completions:
 
     def __init__(self) -> None:
-        self.match_groups: Dict[str, MathGroup] = {}
-        self.no_space_groups: Set[str] = set()
-        self.files_groups: Set[str] = set()
+        self.match_groups: Dict[str, MatchGroup] = {}
         self.delegate: Delegate = Delegate()
 
+    def add_match_group(
+        self, name: str, x: Union[Dict[str, str], Iterable[str]],
+        trailing_space: bool = True,
+        is_files: bool = False,
+        word_transforms: Optional[Dict[str, str]] = None
+    ) -> MatchGroup:
+        self.match_groups[name] = m = MatchGroup(x, trailing_space, is_files, word_transforms)
+        return m
+
 
 @run_once
 def remote_control_command_names() -> Tuple[str, ...]:
@@ -159,16 +202,54 @@ def fish_input_parser(data: str) -> ParseResult:
 @output_serializer
 def zsh_output_serializer(ans: Completions) -> str:
     lines = []
+
+    screen = screen_size_function(sys.stderr.fileno())()
+    width = screen.cols
+
+    def fmt_desc(word: str, desc: str, max_word_len: int) -> Iterator[str]:
+        if not desc:
+            yield word
+            return
+        desc = prettify(desc.splitlines()[0])
+        multiline = False
+        if wcswidth(word) > max_word_len:
+            max_desc_len = width - 2
+            multiline = True
+        else:
+            word = word.ljust(max_word_len)
+            max_desc_len = width - max_word_len - 3
+        if wcswidth(desc) > max_desc_len:
+            desc = desc[:truncate_point_for_length(desc, max_desc_len - 2)]
+            desc += '…'
+
+        if multiline:
+            ans = f'{word}\n  {desc}'
+        else:
+            ans = f'{word}  {desc}'
+        yield ans
+
     for description, matches in ans.match_groups.items():
-        cmd = ['compadd', '-U', '-J', shlex.quote(description), '-X', shlex.quote(description)]
-        if description in ans.no_space_groups:
+        cmd = ['compadd', '-U', '-J', shlex.quote(description), '-X', shlex.quote('%B' + description + '%b')]
+        if not matches.trailing_space:
             cmd += ['-S', '""']
-        if description in ans.files_groups:
+        if matches.is_files:
             cmd.append('-f')
             common_prefix = os.path.commonprefix(tuple(matches))
             if common_prefix:
                 cmd.extend(('-p', shlex.quote(common_prefix)))
-                matches = {k[len(common_prefix):]: v for k, v in matches.items()}
+                matches = MatchGroup({k[len(common_prefix):]: v for k, v in matches.items()})
+        has_descriptions = any(matches.values())
+        if has_descriptions or matches.word_transforms:
+            lines.append('compdescriptions=(')
+            sz = max(map(wcswidth, matches.transformed_words()))
+            limit = min(16, sz)
+            for word, desc in matches.transformed_items():
+                lines.extend(map(shlex.quote, fmt_desc(word, desc, limit)))
+            lines.append(')')
+            if has_descriptions:
+                cmd.append('-l')
+            cmd.append('-d')
+            cmd.append('compdescriptions')
         cmd.append('--')
         for word in matches:
             cmd.append(shlex.quote(word))
@@ -182,17 +263,17 @@ def zsh_output_serializer(ans: Completions) -> str:
                 lines.append('shift words')
                 lines.append('(( CURRENT-- ))')
             lines.append(f'_normal -p "{ans.delegate.precommand}"')
-    # debug('\n'.join(lines))
-    return '\n'.join(lines)
+    result = '\n'.join(lines)
+    # debug(result)
+    return result
 
 
 @output_serializer
 def bash_output_serializer(ans: Completions) -> str:
     lines = []
     for description, matches in ans.match_groups.items():
-        needs_space = description not in ans.no_space_groups
         for word in matches:
-            if needs_space:
+            if matches.trailing_space:
                 word += ' '
             lines.append('COMPREPLY+=({})'.format(shlex.quote(word)))
     # debug('\n'.join(lines))
@@ -212,11 +293,11 @@ def fish_output_serializer(ans: Completions) -> str:
 
 def completions_for_first_word(ans: Completions, prefix: str, entry_points: Iterable[str], namespaced_entry_points: Iterable[str]) -> None:
     cmds = ['@' + c for c in remote_control_command_names()]
-    ans.match_groups['Entry points'] = {
+    ans.add_match_group('Entry points', {
         k: '' for k in
         list(entry_points) + cmds + ['+' + k for k in namespaced_entry_points]
         if not prefix or k.startswith(prefix)
-    }
+    })
     if prefix:
         ans.delegate = Delegate([prefix], 0)
 
@@ -229,7 +310,7 @@ def kitty_cli_opts(ans: Completions, prefix: Optional[str] = None) -> None:
         aliases = frozenset(x for x in opt['aliases'] if x.startswith(prefix)) if prefix else opt['aliases']
         for alias in aliases:
             matches[alias] = opt['help'].strip()
-    ans.match_groups['Options'] = matches
+    ans.add_match_group('Options', matches)
 
 
 def complete_kitty_cli_arg(ans: Completions, opt: Optional[OptionDict], prefix: str, unknown_args: Delegate) -> None:
@@ -242,8 +323,7 @@ def complete_kitty_cli_arg(ans: Completions, opt: Optional[OptionDict], prefix:
     if dest == 'override':
         from kitty.config import option_names_for_completion
         k = 'Config directives'
-        ans.match_groups[k] = {k+'=': '' for k in option_names_for_completion() if k.startswith(prefix)}
-        ans.no_space_groups.add(k)
+        ans.add_match_group(k, {k+'=': '' for k in option_names_for_completion() if k.startswith(prefix)}, trailing_space=False)
     elif dest == 'config':
 
         def is_conf_file(x: str) -> bool:
@@ -260,13 +340,11 @@ def complete_kitty_cli_arg(ans: Completions, opt: Optional[OptionDict], prefix:
         complete_files_and_dirs(ans, prefix, files_group_name='Directories', predicate=os.path.isdir)
     elif dest == 'start_as':
         k = 'Start as'
-        ans.match_groups[k] = {x: x for x in 'normal,fullscreen,maximized,minimized'.split(',') if x.startswith(prefix)}
-        ans.no_space_groups.add(k)
+        ans.add_match_group(k, {x: x for x in 'normal,fullscreen,maximized,minimized'.split(',') if x.startswith(prefix)}, trailing_space=False)
     elif dest == 'listen_on':
         if ':' not in prefix:
             k = 'Address type'
-            ans.match_groups[k] = {x: x for x in ('unix:', 'tcp:') if x.startswith(prefix)}
-            ans.no_space_groups.add(k)
+            ans.add_match_group(k, {x: x for x in ('unix:', 'tcp:') if x.startswith(prefix)}, trailing_space=False)
         elif prefix.startswith('unix:') and not prefix.startswith('@'):
             complete_files_and_dirs(ans, prefix[len('unix:'):], files_group_name='UNIX sockets', add_prefix='unix:')
 
@@ -295,7 +373,7 @@ def complete_alias_map(
         opt = option_map.get(w)
         if w is last_word and not new_word:
             if w.startswith('-'):
-                ans.match_groups['Options'] = {k: opt['help'] for k, opt in option_map.items() if k.startswith(last_word)}
+                ans.add_match_group('Options', {k: opt['help'] for k, opt in option_map.items() if k.startswith(last_word)})
             else:
                 if complete_args is not None:
                     complete_args(ans, None, last_word, Delegate(words, i))
@@ -312,7 +390,7 @@ def complete_alias_map(
         prefix = '' if new_word else last_word
         if complete_args is not None:
             complete_args(ans, None, prefix, Delegate())
-        ans.match_groups['Options'] = {k: opt['help'] for k, opt in option_map.items() if k.startswith(prefix)}
+        ans.add_match_group('Options', {k: opt['help'] for k, opt in option_map.items() if k.startswith(prefix)})
 
 
 def complete_cli(
@@ -391,19 +469,16 @@ def complete_files_and_dirs(
         files = (add_prefix + x for x in files)
 
     if dirs:
-        ans.match_groups['Directories'] = dict.fromkeys(dirs)
-        ans.files_groups.add('Directories')
-        ans.no_space_groups.add('Directories')
+        ans.add_match_group('Directories', dirs, trailing_space=False, is_files=True)
     if files:
-        ans.match_groups[files_group_name] = dict.fromkeys(files)
-        ans.files_groups.add(files_group_name)
+        ans.add_match_group(files_group_name, files, is_files=True)
 
 
 def complete_icat_args(ans: Completions, opt: Optional[OptionDict], prefix: str, unknown_args: Delegate) -> None:
     from .guess_mime_type import guess_type
 
     def icat_file_predicate(filename: str) -> bool:
-        mt = guess_type(filename)
+        mt = guess_type(filename, allow_filesystem_access=True)
         if mt and mt.startswith('image/'):
             return True
         return False
@@ -433,7 +508,7 @@ def remote_args_completer(title: str, words: Iterable[str]) -> CompleteArgsFunc:
 
     def complete_names_for_arg(ans: Completions, opt: Optional[OptionDict], prefix: str, unknown_args: Delegate) -> None:
         if opt is None:
-            ans.match_groups[title] = {c: '' for c in items if c.startswith(prefix)}
+            ans.add_match_group(title, {c: '' for c in items if c.startswith(prefix)})
 
     return complete_names_for_arg
 
@@ -450,6 +525,13 @@ def complete_diff_args(ans: Completions, opt: Optional[OptionDict], prefix: str,
 
 
 def complete_kitten(ans: Completions, kitten: str, words: Sequence[str], new_word: bool) -> None:
+    try:
+        completer = get_kitten_completer(kitten)
+    except SystemExit:
+        completer = None
+    if completer is not None:
+        completer(ans, words, new_word)
+        return
     try:
         cd = get_kitten_cli_docs(kitten)
     except SystemExit:
@@ -482,24 +564,24 @@ def find_completions(words: Sequence[str], new_word: bool, entry_points: Iterabl
     if words[0] == '@':
         if len(words) == 1 or (len(words) == 2 and not new_word):
             prefix = words[1] if len(words) > 1 else ''
-            ans.match_groups['Remote control commands'] = {c: '' for c in remote_control_command_names() if c.startswith(prefix)}
+            ans.add_match_group('Remote control commands', {c: '' for c in remote_control_command_names() if c.startswith(prefix)})
         else:
             complete_remote_command(ans, words[1], words[2:], new_word)
         return ans
     if words[0].startswith('@'):
         if len(words) == 1 and not new_word:
             prefix = words[0]
-            ans.match_groups['Remote control commands'] = {'@' + c: '' for c in remote_control_command_names() if c.startswith(prefix)}
+            ans.add_match_group('Remote control commands', {'@' + c: '' for c in remote_control_command_names() if c.startswith(prefix)})
         else:
             complete_remote_command(ans, words[0][1:], words[1:], new_word)
     if words[0] == '+':
         if len(words) == 1 or (len(words) == 2 and not new_word):
             prefix = words[1] if len(words) > 1 else ''
-            ans.match_groups['Entry points'] = {c: '' for c in namespaced_entry_points if c.startswith(prefix)}
+            ans.add_match_group('Entry points', {c: '' for c in namespaced_entry_points if c.startswith(prefix)})
         else:
             if words[1] == 'kitten':
                 if len(words) == 2 or (len(words) == 3 and not new_word):
-                    ans.match_groups['Kittens'] = dict.fromkeys(k for k in all_kitten_names() if k.startswith('' if len(words) == 2 else words[2]))
+                    ans.add_match_group('Kittens', (k for k in all_kitten_names() if k.startswith('' if len(words) == 2 else words[2])))
                 else:
                     complete_kitten(ans, words[2], words[3:], new_word)
         return ans
@@ -507,13 +589,13 @@ def find_completions(words: Sequence[str], new_word: bool, entry_points: Iterabl
         if len(words) == 1:
             if new_word:
                 if words[0] == '+kitten':
-                    ans.match_groups['Kittens'] = dict.fromkeys(all_kitten_names())
+                    ans.add_match_group('Kittens', all_kitten_names())
             else:
                 prefix = words[0]
-                ans.match_groups['Entry points'] = {c: '' for c in namespaced_entry_points if c.startswith(prefix)}
+                ans.add_match_group('Entry points', (c for c in namespaced_entry_points if c.startswith(prefix)))
         else:
             if len(words) == 2 and not new_word:
-                ans.match_groups['Kittens'] = dict.fromkeys(k for k in all_kitten_names() if k.startswith(words[1]))
+                ans.add_match_group('Kittens', (k for k in all_kitten_names() if k.startswith(words[1])))
             else:
                 if words[0] == '+kitten':
                     complete_kitten(ans, words[1], words[2:], new_word)

kitty/conf/generate.py

@@ -0,0 +1,453 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+import inspect
+import os
+import pprint
+import re
+import textwrap
+from typing import (
+    Any, Callable, Dict, Iterator, List, Set, Tuple, Union, get_type_hints
+)
+
+from kitty.conf.types import Definition, MultiOption, Option, unset
+
+
+def chunks(lst: List, n: int) -> Iterator[List]:
+    for i in range(0, len(lst), n):
+        yield lst[i:i + n]
+
+
+def atoi(text: str) -> str:
+    return f'{int(text):08d}' if text.isdigit() else text
+
+
+def natural_keys(text: str) -> Tuple[str, ...]:
+    return tuple(atoi(c) for c in re.split(r'(\d+)', text))
+
+
+def generate_class(defn: Definition, loc: str) -> Tuple[str, str]:
+    class_lines: List[str] = []
+    tc_lines: List[str] = []
+    a = class_lines.append
+    t = tc_lines.append
+    a('class Options:')
+    t('class Parser:')
+    choices = {}
+    imports: Set[Tuple[str, str]] = set()
+    tc_imports: Set[Tuple[str, str]] = set()
+
+    def type_name(x: type) -> str:
+        ans = x.__name__
+        if x.__module__ and x.__module__ != 'builtins':
+            imports.add((x.__module__, x.__name__))
+        return ans
+
+    def option_type_as_str(x: Any) -> str:
+        if hasattr(x, '__name__'):
+            return type_name(x)
+        ans = repr(x)
+        ans = ans.replace('NoneType', 'None')
+        return ans
+
+    def option_type_data(option: Union[Option, MultiOption]) -> Tuple[Callable, str]:
+        func = option.parser_func
+        if func.__module__ == 'builtins':
+            return func, func.__name__
+        th = get_type_hints(func)
+        rettype = th['return']
+        typ = option_type_as_str(rettype)
+        if isinstance(option, MultiOption):
+            typ = typ[typ.index('[') + 1:-1]
+            typ = typ.replace('Tuple', 'Dict', 1)
+        return func, typ
+
+    is_mutiple_vars = {}
+    option_names = set()
+    color_table = list(map(str, range(256)))
+
+    def parser_function_declaration(option_name: str) -> None:
+        t('')
+        t(f'    def {option_name}(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:')
+
+    for option in sorted(defn.iter_all_options(), key=lambda a: natural_keys(a.name)):
+        option_names.add(option.name)
+        parser_function_declaration(option.name)
+        if isinstance(option, MultiOption):
+            mval: Dict[str, Dict[str, Any]] = {'macos': {}, 'linux': {}, '': {}}
+            func, typ = option_type_data(option)
+            for val in option:
+                if val.add_to_default:
+                    gr = mval[val.only]
+                    for k, v in func(val.defval_as_str):
+                        gr[k] = v
+            is_mutiple_vars[option.name] = typ, mval
+            sig = inspect.signature(func)
+            tc_imports.add((func.__module__, func.__name__))
+            if len(sig.parameters) == 1:
+                t(f'        for k, v in {func.__name__}(val):')
+                t(f'            ans["{option.name}"][k] = v')
+            else:
+                t(f'        for k, v in {func.__name__}(val, ans["{option.name}"]):')
+                t(f'            ans["{option.name}"][k] = v')
+            continue
+
+        if option.choices:
+            typ = 'typing.Literal[{}]'.format(', '.join(repr(x) for x in option.choices))
+            ename = f'choices_for_{option.name}'
+            choices[ename] = typ
+            typ = ename
+            func = str
+        elif defn.has_color_table and option.is_color_table_color:
+            func, typ = option_type_data(option)
+            t(f'        ans[{option.name!r}] = {func.__name__}(val)')
+            tc_imports.add((func.__module__, func.__name__))
+            cnum = int(option.name[5:])
+            color_table[cnum] = '0x{:06x}'.format(func(option.defval_as_string).__int__())
+            continue
+        else:
+            func, typ = option_type_data(option)
+            try:
+                params = inspect.signature(func).parameters
+            except Exception:
+                params = {}
+            if 'dict_with_parse_results' in params:
+                t(f'        {func.__name__}(val, ans)')
+            else:
+                t(f'        ans[{option.name!r}] = {func.__name__}(val)')
+            if func.__module__ != 'builtins':
+                tc_imports.add((func.__module__, func.__name__))
+
+        defval = repr(func(option.defval_as_string))
+        if option.macos_defval is not unset:
+            md = repr(func(option.macos_defval))
+            defval = f'{md} if is_macos else {defval}'
+            imports.add(('kitty.constants', 'is_macos'))
+        a(f'    {option.name}: {typ} = {defval}')
+        if option.choices:
+            t('        val = val.lower()')
+            t(f'        if val not in self.choices_for_{option.name}:')
+            t(f'            raise ValueError(f"The value {{val}} is not a valid choice for {option.name}")')
+            t(f'        ans["{option.name}"] = val')
+            t('')
+            t(f'    choices_for_{option.name} = frozenset({option.choices!r})')
+
+    for option_name, (typ, mval) in is_mutiple_vars.items():
+        a(f'    {option_name}: {typ} = ' '{}')
+
+    for parser, aliases in defn.deprecations.items():
+        for alias in aliases:
+            parser_function_declaration(alias)
+            tc_imports.add((parser.__module__, parser.__name__))
+            t(f'        {parser.__name__}({alias!r}, val, ans)')
+
+    action_parsers = {}
+
+    def resolve_import(ftype: str) -> str:
+        if '.' in ftype:
+            fmod, ftype = ftype.rpartition('.')[::2]
+        else:
+            fmod = f'{loc}.options.utils'
+        imports.add((fmod, ftype))
+        return ftype
+
+    for aname, action in defn.actions.items():
+        option_names.add(aname)
+        action_parsers[aname] = func = action.parser_func
+        th = get_type_hints(func)
+        rettype = th['return']
+        typ = option_type_as_str(rettype)
+        typ = typ[typ.index('[') + 1:-1]
+        a(f'    {aname}: typing.List[{typ}] = []')
+        for imp in action.imports:
+            resolve_import(imp)
+        for fname, ftype in action.fields.items():
+            ftype = resolve_import(ftype)
+            a(f'    {fname}: {ftype} = ' '{}')
+        parser_function_declaration(aname)
+        t(f'        for k in {func.__name__}(val):')
+        t(f'            ans[{aname!r}].append(k)')
+        tc_imports.add((func.__module__, func.__name__))
+
+    if defn.has_color_table:
+        imports.add(('array', 'array'))
+        a('    color_table: array = array("L", (')
+        for grp in chunks(color_table, 8):
+            a('        ' + ', '.join(grp) + ',')
+        a('    ))')
+
+    a('    config_paths: typing.Tuple[str, ...] = ()')
+    a('    config_overrides: typing.Tuple[str, ...] = ()')
+    a('')
+    a('    def __init__(self, options_dict: typing.Optional[typing.Dict[str, typing.Any]] = None) -> None:')
+    a('        if options_dict is not None:')
+    a('            for key in option_names:')
+    a('                setattr(self, key, options_dict[key])')
+
+    a('')
+    a('    @property')
+    a('    def _fields(self) -> typing.Tuple[str, ...]:')
+    a('        return option_names')
+
+    a('')
+    a('    def __iter__(self) -> typing.Iterator[str]:')
+    a('        return iter(self._fields)')
+
+    a('')
+    a('    def __len__(self) -> int:')
+    a('        return len(self._fields)')
+
+    a('')
+    a('    def _copy_of_val(self, name: str) -> typing.Any:')
+    a('        ans = getattr(self, name)')
+    a('        if isinstance(ans, dict):\n            ans = ans.copy()')
+    a('        elif isinstance(ans, list):\n            ans = ans[:]')
+    a('        return ans')
+
+    a('')
+    a('    def _asdict(self) -> typing.Dict[str, typing.Any]:')
+    a('        return {k: self._copy_of_val(k) for k in self}')
+
+    a('')
+    a('    def _replace(self, **kw: typing.Any) -> "Options":')
+    a('        ans = Options()')
+    a('        for name in self:')
+    a('            setattr(ans, name, self._copy_of_val(name))')
+    a('        for name, val in kw.items():')
+    a('            setattr(ans, name, val)')
+    a('        return ans')
+
+    a('')
+    a('    def __getitem__(self, key: typing.Union[int, str]) -> typing.Any:')
+    a('        k = option_names[key] if isinstance(key, int) else key')
+    a('        try:')
+    a('            return getattr(self, k)')
+    a('        except AttributeError:')
+    a('            pass')
+    a('        raise KeyError(f"No option named: {k}")')
+
+    if defn.has_color_table:
+        a('')
+        a('    def __getattr__(self, key: str) -> typing.Any:')
+        a('        if key.startswith("color"):')
+        a('            q = key[5:]')
+        a('            if q.isdigit():')
+        a('                k = int(q)')
+        a('                if 0 <= k <= 255:')
+        a('                    x = self.color_table[k]')
+        a('                    return Color((x >> 16) & 255, (x >> 8) & 255, x & 255)')
+        a('        raise AttributeError(key)')
+        a('')
+        a('    def __setattr__(self, key: str, val: typing.Any) -> typing.Any:')
+        a('        if key.startswith("color"):')
+        a('            q = key[5:]')
+        a('            if q.isdigit():')
+        a('                k = int(q)')
+        a('                if 0 <= k <= 255:')
+        a('                    self.color_table[k] = int(val)')
+        a('                    return')
+        a('        object.__setattr__(self, key, val)')
+
+    a('')
+    a('')
+    a('defaults = Options()')
+    for option_name, (typ, mval) in is_mutiple_vars.items():
+        a(f'defaults.{option_name} = {mval[""]!r}')
+        if mval['macos']:
+            imports.add(('kitty.constants', 'is_macos'))
+            a('if is_macos:')
+            a(f'    defaults.{option_name}.update({mval["macos"]!r}')
+        if mval['macos']:
+            imports.add(('kitty.constants', 'is_macos'))
+            a('if not is_macos:')
+            a(f'    defaults.{option_name}.update({mval["linux"]!r}')
+
+    for aname, func in action_parsers.items():
+        a(f'defaults.{aname} = [')
+        only: Dict[str, List[Tuple[str, Callable]]] = {}
+        for sc in defn.iter_all_maps(aname):
+            if not sc.add_to_default:
+                continue
+            text = sc.parseable_text
+            if sc.only:
+                only.setdefault(sc.only, []).append((text, func))
+            else:
+                for val in func(text):
+                    a(f'    # {sc.name}')
+                    a(f'    {val!r},')
+        a(']')
+        if only:
+            imports.add(('kitty.constants', 'is_macos'))
+            for cond, items in only.items():
+                cond = 'is_macos' if cond == 'macos' else 'not is_macos'
+                a(f'if {cond}:')
+                for (text, func) in items:
+                    for val in func(text):
+                        a(f'    defaults.{aname}.append({val!r})')
+
+    t('')
+    t('')
+    t('def create_result_dict() -> typing.Dict[str, typing.Any]:')
+    t('    return {')
+    for oname in is_mutiple_vars:
+        t(f'        {oname!r}: {{}},')
+    for aname in defn.actions:
+        t(f'        {aname!r}: [],')
+    t('    }')
+
+    t('')
+    t('')
+    t(f'actions = frozenset({tuple(defn.actions)!r})')
+    t('')
+    t('')
+    t('def merge_result_dicts(defaults: typing.Dict[str, typing.Any], vals: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:')
+    t('    ans = {}')
+    t('    for k, v in defaults.items():')
+    t('        if isinstance(v, dict):')
+    t('            ans[k] = merge_dicts(v, vals.get(k, {}))')
+    t('        elif k in actions:')
+    t('            ans[k] = v + vals.get(k, [])')
+    t('        else:')
+    t('            ans[k] = vals.get(k, v)')
+    t('    return ans')
+    tc_imports.add(('kitty.conf.utils', 'merge_dicts'))
+
+    t('')
+    t('')
+    t('parser = Parser()')
+    t('')
+    t('')
+    t('def parse_conf_item(key: str, val: str, ans: typing.Dict[str, typing.Any]) -> bool:')
+    t('    func = getattr(parser, key, None)')
+    t('    if func is not None:')
+    t('        func(val, ans)')
+    t('        return True')
+    t('    return False')
+
+    preamble = ['# generated by gen-config.py DO NOT edit', '# vim:fileencoding=utf-8', '']
+    a = preamble.append
+
+    def output_imports(imports: Set, add_module_imports: bool = True) -> None:
+        a('import typing')
+        seen_mods = {'typing'}
+        mmap: Dict[str, List[str]] = {}
+        for mod, name in imports:
+            mmap.setdefault(mod, []).append(name)
+        for mod in sorted(mmap):
+            names = sorted(mmap[mod])
+            lines = textwrap.wrap(', '.join(names), 100)
+            if len(lines) == 1:
+                s = lines[0]
+            else:
+                s = '\n    '.join(lines)
+                s = f'(\n    {s}\n)'
+            a(f'from {mod} import {s}')
+            if add_module_imports and mod not in seen_mods and mod != s:
+                a(f'import {mod}')
+                seen_mods.add(mod)
+
+    output_imports(imports)
+    a('')
+    if choices:
+        a('if typing.TYPE_CHECKING:')
+        for name, cdefn in choices.items():
+            a(f'    {name} = {cdefn}')
+        a('else:')
+        for name in choices:
+            a(f'    {name} = str')
+
+    a('')
+    a('option_names = (  # {{''{')
+    a(' ' + pprint.pformat(tuple(sorted(option_names, key=natural_keys)))[1:] + '  # }}''}')
+    class_def = '\n'.join(preamble + ['', ''] + class_lines)
+
+    preamble = ['# generated by gen-config.py DO NOT edit', '# vim:fileencoding=utf-8', '']
+    a = preamble.append
+    output_imports(tc_imports, False)
+
+    return class_def, '\n'.join(preamble + ['', ''] + tc_lines)
+
+
+def generate_c_conversion(loc: str, ctypes: List[Option]) -> str:
+    lines: List[str] = []
+    basic_converters = {
+        'int': 'PyLong_AsLong', 'uint': 'PyLong_AsUnsignedLong', 'bool': 'PyObject_IsTrue',
+        'float': 'PyFloat_AsFloat', 'double': 'PyFloat_AsDouble',
+        'time': 'parse_s_double_to_monotonic_t', 'time-ms': 'parse_ms_long_to_monotonic_t'
+    }
+
+    for opt in ctypes:
+        lines.append('')
+        lines.append(f'static void\nconvert_from_python_{opt.name}(PyObject *val, Options *opts) ''{')
+        is_special = opt.ctype.startswith('!')
+        if is_special:
+            func = opt.ctype[1:]
+            lines.append(f'    {func}(val, opts);')
+        else:
+            func = basic_converters.get(opt.ctype, opt.ctype)
+            lines.append(f'    opts->{opt.name} = {func}(val);')
+        lines.append('}')
+        lines.append('')
+        lines.append(f'static void\nconvert_from_opts_{opt.name}(PyObject *py_opts, Options *opts) ''{')
+        lines.append(f'    PyObject *ret = PyObject_GetAttrString(py_opts, "{opt.name}");')
+        lines.append('    if (ret == NULL) return;')
+        lines.append(f'    convert_from_python_{opt.name}(ret, opts);')
+        lines.append('    Py_DECREF(ret);')
+        lines.append('}')
+
+    lines.append('')
+    lines.append('static bool\nconvert_opts_from_python_opts(PyObject *py_opts, Options *opts) ''{')
+    for opt in ctypes:
+        lines.append(f'    convert_from_opts_{opt.name}(py_opts, opts);')
+        lines.append('    if (PyErr_Occurred()) return false;')
+    lines.append('    return true;')
+    lines.append('}')
+
+    preamble = ['// generated by gen-config.py DO NOT edit', '// vim:fileencoding=utf-8', '#pragma once', '#include "to-c.h"']
+    return '\n'.join(preamble + ['', ''] + lines)
+
+
+def write_output(loc: str, defn: Definition) -> None:
+    cls, tc = generate_class(defn, loc)
+    with open(os.path.join(*loc.split('.'), 'options', 'types.py'), 'w') as f:
+        f.write(cls + '\n')
+    with open(os.path.join(*loc.split('.'), 'options', 'parse.py'), 'w') as f:
+        f.write(tc + '\n')
+    ctypes = []
+    for opt in defn.root_group.iter_all_non_groups():
+        if isinstance(opt, Option) and opt.ctype:
+            ctypes.append(opt)
+    if ctypes:
+        c = generate_c_conversion(loc, ctypes)
+        with open(os.path.join(*loc.split('.'), 'options', 'to-c-generated.h'), 'w') as f:
+            f.write(c + '\n')
+
+
+def main() -> None:
+    # To use run it as:
+    # kitty +runpy 'from kitty.conf.generate import main; main()' /path/to/kitten/file.py
+    import importlib
+    import sys
+
+    from kittens.runner import path_to_custom_kitten, resolved_kitten
+    from kitty.constants import config_dir
+
+    kitten = sys.argv[-1]
+    if not kitten.endswith('.py'):
+        kitten += '.py'
+    kitten = resolved_kitten(kitten)
+    path = os.path.realpath(path_to_custom_kitten(config_dir, kitten))
+    if not os.path.dirname(path):
+        raise SystemExit(f'No custom kitten named {kitten} found')
+    sys.path.insert(0, os.path.dirname(path))
+    package_name = os.path.basename(os.path.dirname(path))
+    m = importlib.import_module('kitten_options_definition')
+    defn = getattr(m, 'definition')
+    loc = package_name
+    cls, tc = generate_class(defn, loc)
+    with open(os.path.join(os.path.dirname(path), 'kitten_options_types.py'), 'w') as f:
+        f.write(cls + '\n')
+    with open(os.path.join(os.path.dirname(path), 'kitten_options_parse.py'), 'w') as f:
+        f.write(tc + '\n')

kitty/conf/types.py

@@ -12,6 +12,7 @@ from typing import (
 )
 
 import kitty.conf.utils as generic_parsers
+from kitty.constants import website_url
 
 if typing.TYPE_CHECKING:
     Only = typing.Literal['macos', 'linux', '']
@@ -41,14 +42,15 @@ def expand_opt_references(conf_name: str, text: str) -> str:
 
 
 def remove_markup(text: str) -> str:
+    ref_map = {
+        'layouts': f'{website_url("overview")}#layouts',
+        'sessions': f'{website_url("overview")}#layouts',
+        'functional': f'{website_url("keyboard-protocol")}#functional-key-definitions',
+    }
 
     def sub(m: Match) -> str:
         if m.group(1) == 'ref':
-            return {
-                'layouts': 'https://sw.kovidgoyal.net/kitty/index.html#layouts',
-                'sessions': 'https://sw.kovidgoyal.net/kitty/index.html#sessions',
-                'functional': 'https://sw.kovidgoyal.net/kitty/keyboard-protocol.html#functional-key-definitions',
-            }[m.group(2)]
+            return ref_map[m.group(2)]
         return str(m.group(2))
 
     return re.sub(r':([a-zA-Z0-9]+):`(.+?)`', sub, text, flags=re.DOTALL)
@@ -428,7 +430,7 @@ class Group:
             a(f'.. _conf-{conf_name}-{self.name}:')
             a('')
             a(self.title)
-            heading_level = '+' if level > 1 else '^'
+            heading_level = '+' if level > 1 else '-'
             a(heading_level * (len(self.title) + 20))
             a('')
             if self.start_text:
@@ -463,7 +465,7 @@ class Group:
                 a(render_block(self.start_text))
                 a('')
         else:
-            ans.extend(('# vim:fileencoding=utf-8:ft=conf:foldmethod=marker', ''))
+            ans.extend(('# vim:fileencoding=utf-8:foldmethod=marker', ''))
 
         for item in self.iter_with_coalesced_options():
             if isinstance(item, Option):
@@ -541,9 +543,11 @@ class Action:
 class Definition:
 
     def __init__(self, package: str, *actions: Action, has_color_table: bool = False) -> None:
-        self.module_for_parsers = import_module(f'{package}.options.utils')
+        if package.startswith('!'):
+            self.module_for_parsers = import_module(package[1:])
+        else:
+            self.module_for_parsers = import_module(f'{package}.options.utils')
         self.has_color_table = has_color_table
-        self.package = package
         self.coalesced_iterator_data = CoalescedIteratorData()
         self.root_group = Group('', '', self.coalesced_iterator_data)
         self.current_group = self.root_group
@@ -564,9 +568,9 @@ class Definition:
 
     def iter_all_maps(self, which: str = 'map') -> Iterator[Union[ShortcutMapping, MouseMapping]]:
         for x in self.iter_all_non_groups():
-            if isinstance(x, ShortcutMapping) and which == 'map':
+            if isinstance(x, ShortcutMapping) and which in ('map', '*'):
                 yield x
-            elif isinstance(x, MouseMapping) and which == 'mouse_map':
+            elif isinstance(x, MouseMapping) and which in ('mouse_map', '*'):
                 yield x
 
     def parser_func(self, name: str) -> Callable:

kitty/conf/utils.py

@@ -96,8 +96,8 @@ def to_cmdline(x: str) -> List[str]:
 
 
 def python_string(text: str) -> str:
-    import ast
-    ans: str = ast.literal_eval("'''" + text.replace("'''", "'\\''") + "'''")
+    from ast import literal_eval
+    ans: str = literal_eval("'''" + text.replace("'''", "'\\''") + "'''")
     return ans
 
 
@@ -310,7 +310,8 @@ def save_type_stub(text: str, fpath: str) -> None:
     fpath += 'i'
     preamble = '# Update this file by running: ./test.py mypy\n\n'
     try:
-        existing = open(fpath).read()
+        with open(fpath) as fs:
+            existing = fs.read()
     except FileNotFoundError:
         existing = ''
     current = preamble + text

kitty/constants.py

@@ -23,7 +23,7 @@ class Version(NamedTuple):
 
 appname: str = 'kitty'
 kitty_face = '🐱'
-version: Version = Version(0, 21, 1)
+version: Version = Version(0, 22, 1)
 str_version: str = '.'.join(map(str, version))
 _plat = sys.platform.lower()
 is_macos: bool = 'darwin' in _plat
@@ -186,3 +186,11 @@ def read_kitty_resource(name: str) -> bytes:
     except ImportError:
         from importlib_resources import read_binary  # type: ignore
     return read_binary('kitty', name)
+
+
+def website_url(doc_name: str = '') -> str:
+    if doc_name:
+        doc_name = doc_name.rstrip('/')
+        if doc_name:
+            doc_name += '/'
+    return f'https://sw.kovidgoyal.net/kitty/{doc_name}'

kitty/core_text.m

@@ -323,6 +323,13 @@ harfbuzz_font_for_face(PyObject* s) {
     return self->hb_font;
 }
 
+static unsigned int
+adjust_ypos(unsigned int pos, unsigned int cell_height, int adjustment) {
+    if (adjustment >= 0) adjustment = MIN(adjustment, (int)pos - 1);
+    else adjustment = MAX(adjustment, (int)pos - (int)cell_height + 1);
+    return pos - adjustment;
+}
+
 void
 cell_metrics(PyObject *s, unsigned int* cell_width, unsigned int* cell_height, unsigned int* baseline, unsigned int* underline_position, unsigned int* underline_thickness, unsigned int* strikethrough_position, unsigned int* strikethrough_thickness) {
     // See https://developer.apple.com/library/content/documentation/StringsTextFonts/Conceptual/TextAndWebiPhoneOS/TypoFeatures/TextSystemFeatures.html
@@ -366,9 +373,12 @@ cell_metrics(PyObject *s, unsigned int* cell_width, unsigned int* cell_height, u
     CGRect bounds_without_leading = CTLineGetBoundsWithOptions(line, kCTLineBoundsExcludeTypographicLeading);
     CGFloat typographic_ascent, typographic_descent, typographic_leading;
     CTLineGetTypographicBounds(line, &typographic_ascent, &typographic_descent, &typographic_leading);
-    CGFloat bounds_ascent = bounds_without_leading.size.height + bounds_without_leading.origin.y;
-    *baseline = (unsigned int)floor(bounds_ascent + 0.5);
     *cell_height = MAX(4u, (unsigned int)ceilf(line_height));
+    CGFloat bounds_ascent = bounds_without_leading.size.height + bounds_without_leading.origin.y;
+    int baseline_offset = 0;
+    if (OPT(adjust_baseline_px) != 0) baseline_offset = OPT(adjust_baseline_px);
+    else if (OPT(adjust_baseline_frac) != 0) baseline_offset = (int)(*cell_height * OPT(adjust_baseline_frac));
+    *baseline = (unsigned int)floor(bounds_ascent + 0.5);
     // Not sure if we should add this to bounds ascent and then round it or add
     // it to already rounded baseline and round again.
     *underline_position = (unsigned int)floor(bounds_ascent - self->underline_position + 0.5);
@@ -382,6 +392,11 @@ cell_metrics(PyObject *s, unsigned int* cell_width, unsigned int* cell_height, u
     debug("\tline metrics: ascent: %f descent: %f leading: %f\n", typographic_ascent, typographic_descent, typographic_leading);
     debug("\tfont metrics: ascent: %f descent: %f leading: %f underline_position: %f\n", self->ascent, self->descent, self->leading, self->underline_position);
     debug("\tcell_height: %u baseline: %u underline_position: %u strikethrough_position: %u\n", *cell_height, *baseline, *underline_position, *strikethrough_position);
+    if (baseline_offset) {
+        *baseline = adjust_ypos(*baseline, *cell_height, baseline_offset);
+        *underline_position = adjust_ypos(*underline_position, *cell_height, baseline_offset);
+        *strikethrough_position = adjust_ypos(*strikethrough_position, *cell_height, baseline_offset);
+    }
 
     CFRelease(test_frame); CFRelease(path); CFRelease(framesetter);
 

kitty/cursor.c

@@ -24,7 +24,7 @@ dealloc(Cursor* self) {
 
 #define EQ(x) (a->x == b->x)
 static int __eq__(Cursor *a, Cursor *b) {
-    return EQ(bold) && EQ(italic) && EQ(strikethrough) && EQ(dim) && EQ(reverse) && EQ(decoration) && EQ(fg) && EQ(bg) && EQ(decoration_fg) && EQ(x) && EQ(y) && EQ(shape) && EQ(blink);
+    return EQ(bold) && EQ(italic) && EQ(strikethrough) && EQ(dim) && EQ(reverse) && EQ(decoration) && EQ(fg) && EQ(bg) && EQ(decoration_fg) && EQ(x) && EQ(y) && EQ(shape) && EQ(non_blinking);
 }
 
 static const char* cursor_names[NUM_OF_CURSOR_SHAPES] = { "NO_SHAPE", "BLOCK", "BEAM", "UNDERLINE" };
@@ -35,7 +35,7 @@ repr(Cursor *self) {
     return PyUnicode_FromFormat(
         "Cursor(x=%u, y=%u, shape=%s, blink=%R, fg=#%08x, bg=#%08x, bold=%R, italic=%R, reverse=%R, strikethrough=%R, dim=%R, decoration=%d, decoration_fg=#%08x)",
         self->x, self->y, (self->shape < NUM_OF_CURSOR_SHAPES ? cursor_names[self->shape] : "INVALID"),
-        BOOL(self->blink), self->fg, self->bg, BOOL(self->bold), BOOL(self->italic), BOOL(self->reverse), BOOL(self->strikethrough), BOOL(self->dim), self->decoration, self->decoration_fg
+        BOOL(!self->non_blinking), self->fg, self->bg, BOOL(self->bold), BOOL(self->italic), BOOL(self->reverse), BOOL(self->strikethrough), BOOL(self->dim), self->decoration, self->decoration_fg
     );
 }
 
@@ -232,12 +232,12 @@ reset_display_attrs(Cursor *self, PyObject *a UNUSED) {
 void cursor_reset(Cursor *self) {
     cursor_reset_display_attrs(self);
     self->x = 0; self->y = 0;
-    self->shape = NO_CURSOR_SHAPE; self->blink = false;
+    self->shape = NO_CURSOR_SHAPE; self->non_blinking = false;
 }
 
 void cursor_copy_to(Cursor *src, Cursor *dest) {
 #define CCY(x) dest->x = src->x;
-    CCY(x); CCY(y); CCY(shape); CCY(blink);
+    CCY(x); CCY(y); CCY(shape); CCY(non_blinking);
     CCY(bold); CCY(italic); CCY(strikethrough); CCY(dim); CCY(reverse); CCY(decoration); CCY(fg); CCY(bg); CCY(decoration_fg);
 }
 
@@ -252,7 +252,11 @@ BOOL_GETSET(Cursor, italic)
 BOOL_GETSET(Cursor, reverse)
 BOOL_GETSET(Cursor, strikethrough)
 BOOL_GETSET(Cursor, dim)
-BOOL_GETSET(Cursor, blink)
+
+static PyObject* blink_get(Cursor *self, void UNUSED *closure) { PyObject *ans = !self->non_blinking ? Py_True : Py_False; Py_INCREF(ans); return ans; }
+
+static int blink_set(Cursor *self, PyObject *value, void UNUSED *closure) { if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Cannot delete attribute"); return -1; } self->non_blinking = PyObject_IsTrue(value) ? false : true; return 0; }
+
 
 static PyMemberDef members[] = {
     {"x", T_UINT, offsetof(Cursor, x), 0, "x"},