Update patches from https://github.com/KittyPatch/kitty to work on current kitty

Note that we loose unicode char matching for --type=word because of
https://github.com/dlclark/regexp2/issues/65 and of course user regexps
cant use \p{N} escapes any more. Hopefully regexp2 will add support for
these soon-ish. IMO lookaround is more important than \p.

Fixes #6265

.editorconfig

@@ -1,12 +1,12 @@
 root = true
 
 [*]
-indent_style = spaces
+indent_style = space
 indent_size = 4
 end_of_line = lf
 trim_trailing_whitespace = true
 
-[{Makefile,*.terminfo}]
+[{Makefile,*.terminfo,*.go}]
 indent_style = tab
 
 # Autogenerated files with tabs below this line.

.gitattributes

@@ -3,7 +3,9 @@ kitty/emoji.h linguist-generated=true
 kitty/charsets.c linguist-generated=true
 kitty/key_encoding.py linguist-generated=true
 kitty/unicode-data.c linguist-generated=true
+kitty/rowcolumn-diacritics.c linguist-generated=true
 kitty/rgb.py linguist-generated=true
+kitty/srgb_gamma.c linguist-generated=true
 kitty/gl-wrapper.* linguist-generated=true
 kitty/glfw-wrapper.* linguist-generated=true
 kitty/parse-graphics-command.h linguist-generated=true
@@ -16,6 +18,7 @@ glfw/*.c linguist-vendored=true
 glfw/*.h linguist-vendored=true
 kittens/unicode_input/names.h linguist-generated=true
 tools/wcswidth/std.go linguist-generated=true
+tools/unicode_names/names.txt linguist-generated=true
 
 *.py  text diff=python
 *.m  text diff=objc

.github/FUNDING.yml

@@ -1,4 +1,2 @@
-github: kovidgoyal
+custom: https://my.fsf.org/donate
-patreon: kovidgoyal
-liberapay: kovidgoyal
 custom: https://sw.kovidgoyal.net/kitty/support.html

.github/workflows/ci.yml

@@ -5,7 +5,6 @@ env:
     ASAN_OPTIONS: leak_check_at_exit=0
     LC_ALL: en_US.UTF-8
     LANG: en_US.UTF-8
-    GO_INSTALL_VERSION: ">=1.19.0"
 
 permissions:
   contents: read # to fetch code (actions/checkout)
@@ -51,14 +50,14 @@ jobs:
               fetch-depth: 10
 
           - name: Set up Python ${{ matrix.pyver }}
-            uses: actions/setup-python@v3
+            uses: actions/setup-python@v4
             with:
               python-version: ${{ matrix.pyver }}
 
           - name: Install Go
             uses: actions/setup-go@v3
             with:
-              go-version: ${{ env.GO_INSTALL_VERSION }}
+              go-version-file: go.mod
 
           - name: Build kitty
             run: python .github/workflows/ci.py build
@@ -81,14 +80,14 @@ jobs:
             run: if grep -Inr '\s$' kitty kitty_tests kittens docs *.py *.asciidoc *.rst *.go .gitattributes .gitignore; then echo Trailing whitespace found, aborting.; exit 1; fi
 
           - name: Set up Python
-            uses: actions/setup-python@v3
+            uses: actions/setup-python@v4
             with:
               python-version: "3.10"
 
           - name: Install Go
             uses: actions/setup-go@v3
             with:
-              go-version: ${{ env.GO_INSTALL_VERSION }}
+              go-version-file: go.mod
 
           - name: Install build-only deps
             run: python -m pip install -r docs/requirements.txt ruff mypy types-requests types-docutils
@@ -105,7 +104,7 @@ jobs:
           - name: Build kitty
             run: python setup.py build --debug
 
-          - name: Build static kitty-tool
+          - name: Build static kitten
             run: python setup.py build-static-binaries
 
           - name: Run mypy
@@ -130,14 +129,14 @@ jobs:
             KITTY_BUNDLE: 1
         steps:
           - name: Checkout source code
-            uses: actions/checkout@master
+            uses: actions/checkout@v3
             with:
               fetch-depth: 10
 
           - name: Install Go
             uses: actions/setup-go@v3
             with:
-              go-version: ${{ env.GO_INSTALL_VERSION }}
+              go-version-file: go.mod
 
           - name: Build kitty
             run: which python3 && python3 .github/workflows/ci.py build
@@ -150,19 +149,19 @@ jobs:
         runs-on: macos-latest
         steps:
           - name: Checkout source code
-            uses: actions/checkout@master
+            uses: actions/checkout@v3
             with:
               fetch-depth: 0  # needed for :commit: docs role
 
           - name: Set up Python
-            uses: actions/setup-python@v3
+            uses: actions/setup-python@v4
             with:
               python-version: "3.10"
 
           - name: Install Go
             uses: actions/setup-go@v3
             with:
-              go-version: ${{ env.GO_INSTALL_VERSION }}
+              go-version-file: go.mod
 
           - name: Build kitty
             run: python3 .github/workflows/ci.py build

.github/workflows/codeql-analysis.yml

@@ -23,11 +23,6 @@ jobs:
 
     steps:
 
-    - name: Install Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: ">=1.19.0"
-
     - name: Checkout repository
       uses: actions/checkout@v3
       with:
@@ -35,6 +30,11 @@ jobs:
         # a pull request then we can checkout the head.
         fetch-depth: 2
 
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version-file: go.mod
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2
@@ -46,4 +46,4 @@ jobs:
       run: python3 .github/workflows/ci.py build
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.gitignore

@@ -1,6 +1,7 @@
 *.so
 *.pyc
 *.pyo
+*.bin
 *_stub.pyi
 *_generated.go
 *_generated.h
@@ -11,14 +12,13 @@
 /kitty.app/
 /glad/out/
 /kitty/launcher/kitt*
-/tools/cmd/at/*_generated.go
-*_generated.go
 /*.dSYM/
 __pycache__/
 /glfw/wayland-*-client-protocol.[ch]
 /docs/_build/
 /docs/generated/
 /.mypy_cache
+/.ruff_cache
 .DS_Store
 .cache
 bypy/b

bypy/init_env.py

@@ -106,6 +106,8 @@ def build_c_extensions(ext_dir, args):
     cmd = SETUP_CMD + ['macos-freeze' if ismacos else 'linux-freeze']
     if args.dont_strip:
         cmd.append('--debug')
+    if args.extra_program_data:
+        cmd.append(f'--vcs-rev={args.extra_program_data}')
     dest = kitty_constants['appname'] + ('.app' if ismacos else '')
     dest = build_frozen_launcher.prefix = os.path.join(ext_dir, dest)
     cmd += ['--prefix', dest, '--full']

bypy/sources.json

@@ -163,15 +163,6 @@
         }
     },
 
-    {
-        "name": "pygments",
-        "unix": {
-            "filename": "Pygments-2.11.2.tar.gz",
-            "hash": "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a",
-            "urls": ["pypi"]
-        }
-    },
-
     {
         "name": "libpng",
         "unix": {

count-lines-of-code

@@ -2,36 +2,20 @@
 
 import subprocess
 
-files_to_exclude = '''\
-kitty/wcwidth-std.h
-kitty/charsets.c
-kitty/unicode-data.c
-kitty/key_encoding.py
-kitty/rgb.py
-kitty/gl.h
-kitty/gl-wrapper.h
-kitty/gl-wrapper.c
-kitty/glfw-wrapper.h
-kitty/glfw-wrapper.c
-kitty/emoji.h
-kittens/unicode_input/names.h
-kitty/parse-graphics-command.h
-kitty/options/types.py
-kitty/options/parse.py
-kitty/options/to-c-generated.h
-kittens/diff/options/types.py
-kittens/diff/options/parse.py
-tools/wcswidth/std.go
-'''
-
 ignored = []
 for line in subprocess.check_output(['git', 'status', '--ignored', '--porcelain']).decode().splitlines():
     if line.startswith('!! '):
         ignored.append(line[3:])
-files_to_exclude += '\n'.join(ignored)
+files_to_exclude = '\n'.join(ignored)
+
+cp = subprocess.run(['git', 'check-attr', 'linguist-generated', '--stdin'],
+                    check=True, stdout=subprocess.PIPE, input=subprocess.check_output([ 'git', 'ls-files']))
+for line in cp.stdout.decode().splitlines():
+    if line.endswith(' true'):
+        files_to_exclude += '\n' + line.split(':')[0]
 
 p = subprocess.Popen([
-    'cloc', '--exclude-list-file', '/dev/stdin', 'kitty', 'kittens', 'tools',
+    'cloc', '--exclude-list-file', '/dev/stdin', 'kitty', 'kittens', 'tools', 'kitty_tests', 'docs',
 ], stdin=subprocess.PIPE)
 p.communicate(files_to_exclude.encode('utf-8'))
 raise SystemExit(p.wait())

docs/basic.rst

@@ -58,6 +58,7 @@ Action                      Shortcut
 New window                  :sc:`new_window` (also :kbd:`⌘+↩` on macOS)
 New OS window               :sc:`new_os_window` (also :kbd:`⌘+n` on macOS)
 Close window                :sc:`close_window` (also :kbd:`⇧+⌘+d` on macOS)
+Resize window               :sc:`start_resizing_window` (also :kbd:`⌘+r` on macOS)
 Next window                 :sc:`next_window`
 Previous window             :sc:`previous_window`
 Move window forward         :sc:`move_window_forward`

docs/binary.rst

@@ -22,7 +22,8 @@ simply re-run the command.
 .. warning::
    **Do not** copy the kitty binary out of the installation folder. If you want
    to add it to your :envvar:`PATH`, create a symlink in :file:`~/.local/bin` or
-   :file:`/usr/bin` or wherever.
+   :file:`/usr/bin` or wherever. You should create a symlink for the :file:`kitten`
+   binary as well.
 
 
 Manually installing
@@ -30,7 +31,7 @@ Manually installing
 
 If something goes wrong or you simply do not want to run the installer, you can
 manually download and install |kitty| from the `GitHub releases page
-<https://github.com/kovidgoyal/kitty/releases>`__. If you are on macOS, download
+<https://gitea.rexy712.xyz/KittyPatch/kitty/releases>`__. If you are on macOS, download
 the :file:`.dmg` and install as normal. If you are on Linux, download the
 tarball and extract it into a directory. The |kitty| executable will be in the
 :file:`bin` sub-directory.
@@ -46,9 +47,9 @@ particular desktop, but it should work for most major desktop environments.
 
 .. code-block:: sh
 
-    # Create a symbolic link to add kitty to PATH (assuming ~/.local/bin is in
+    # Create symbolic links to add kitty and kitten to PATH (assuming ~/.local/bin is in
     # your system-wide PATH)
-    ln -s ~/.local/kitty.app/bin/kitty ~/.local/bin/
+    ln -sf ~/.local/kitty.app/bin/kitty ~/.local/kitty.app/bin/kitten ~/.local/bin/
     # Place the kitty.desktop file somewhere it can be found by the OS
     cp ~/.local/kitty.app/share/applications/kitty.desktop ~/.local/share/applications/
     # If you want to open text files and images in kitty via your file manager also add the kitty-open.desktop file

docs/build.rst

@@ -1,9 +1,9 @@
 Build from source
 ==================
 
-.. image:: https://github.com/kovidgoyal/kitty/workflows/CI/badge.svg
+.. image:: https://gitea.rexy712.xyz/KittyPatch/kitty/workflows/CI/badge.svg
   :alt: Build status
-  :target: https://github.com/kovidgoyal/kitty/actions?query=workflow%3ACI
+  :target: https://gitea.rexy712.xyz/KittyPatch/kitty/actions?query=workflow%3ACI
 
 .. highlight:: sh
 
@@ -40,13 +40,12 @@ Run-time dependencies:
 * ``fontconfig`` (not needed on macOS)
 * ``libcanberra`` (not needed on macOS)
 * ``ImageMagick`` (optional, needed to display uncommon image formats in the terminal)
-* ``pygments`` (optional, needed for syntax highlighting in ``kitty +kitten diff``)
 
 
 Build-time dependencies:
 
 * ``gcc`` or ``clang``
-* ``go >= 1.19`` (see :file:`go.mod` for go packages used during building)
+* ``go`` >= _build_go_version (see :file:`go.mod` for go packages used during building)
 * ``pkg-config``
 * For building on Linux in addition to the above dependencies you might also
   need to install the following packages, if they are not already installed by
@@ -62,6 +61,7 @@ Build-time dependencies:
   - ``libfontconfig-dev``
   - ``libx11-xcb-dev``
   - ``liblcms2-dev``
+  - ``libssl-dev``
   - ``libpython3-dev``
   - ``librsync-dev``
 
@@ -71,7 +71,7 @@ Install and run from source
 
 .. code-block:: sh
 
-    git clone https://github.com/kovidgoyal/kitty && cd kitty
+    git clone https://gitea.rexy712.xyz/KittyPatch/kitty && cd kitty
 
 Now build the native code parts of |kitty| with the following command::
 
@@ -106,7 +106,7 @@ dependencies you might have to rebuild the app.
 .. note::
    The released :file:`kitty.dmg` includes all dependencies, unlike the
    :file:`kitty.app` built above and is built automatically by using the
-   `bypy framework <https://github.com/kovidgoyal/bypy>`__ however, that is
+   `bypy framework <https://gitea.rexy712.xyz/KittyPatch/bypy>`__ however, that is
    designed to run on Linux and is not for the faint of heart.
 
 .. note::
@@ -155,7 +155,7 @@ Notes for Linux/macOS packagers
 ----------------------------------
 
 The released |kitty| source code is available as a `tarball`_ from
-`the GitHub releases page <https://github.com/kovidgoyal/kitty/releases>`__.
+`the GitHub releases page <https://gitea.rexy712.xyz/KittyPatch/kitty/releases>`__.
 
 While |kitty| does use Python, it is not a traditional Python package, so please
 do not install it in site-packages.

docs/changelog.rst

@@ -35,6 +35,124 @@ mouse anywhere in the current command to move the cursor there. See
 Detailed list of changes
 -------------------------------------
 
+0.28.2 [future]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- A new escape code ``<ESC>[22J`` that moves the current contents of the screen into the scrollback before clearing it
+
+- unicode_input kitten: Fix a regression in 0.28.0 that caused the order of recent and favorites entries to not be respected (:iss:`6214`)
+
+- unicode_input kitten: Fix a regression in 0.28.0 that caused editing of favorites to sometimes hang
+
+- clipboard kitten: Fix a bug causing the last MIME type available on the clipboard not being recognized when pasting
+
+- Fix regression in 0.28.0 causing color fringing when rendering in transparent windows on light backgrounds (:iss:`6209`)
+
+- show_key kitten: In kitty mode show the actual bytes sent by the terminal rather than a re-encoding of the parsed key event
+
+- hints kitten: Fix a regression in 0.28.0 that broke using sub-groups in regexp captures (:iss:`6228`)
+
+- hints kitten: Fix a regression in 0.28.0 that broke using lookahead/lookbehind in regexp captures (:iss:`6265`)
+
+- diff kitten: Fix a regression in 0.28.0 that broke using relative paths as arguments to the kitten (:iss:`6325`)
+
+- Fix re-using the image id of an animated image for a still image causing a crash (:iss:`6244`)
+
+- kitty +open: Ask for permission before executing script files that are not marked as executable. This prevents accidental execution
+  of script files via MIME type association from programs that unconditionally "open" attachments/downloaded files
+
+- edit-in-kitty: Fix running edit-in-kitty with elevated privileges to edit a restricted file not working (:disc:`6245`)
+
+- ssh kitten: Fix a regression in 0.28.0 that caused interrupt during setup to not be handled gracefully (:iss:`6254`)
+
+0.28.1 [2023-04-21]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Fix a regression in the previous release that broke the remote file kitten (:iss:`6186`)
+
+- Fix a regression in the previous release that broke handling of some keyboard shortcuts in some kittens on some keyboard layouts (:iss:`6189`)
+
+- Fix a regression in the previous release that broke usage of custom themes (:iss:`6191`)
+
+0.28.0 [2023-04-15]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- **Text rendering change**: Use sRGB correct linear gamma blending for nicer font
+  rendering and better color accuracy with transparent windows.
+  See the option :opt:`text_composition_strategy` for details.
+  The obsolete :opt:`macos_thicken_font` will make the font too thick and needs to be removed manually
+  if it is configured. (:pull:`5969`)
+
+- icat kitten: Support display of images inside tmux >= 3.3 (:pull:`5664`)
+
+- Graphics protocol: Add support for displaying images inside programs that do not support the protocol such as vim and tmux (:pull:`5664`)
+
+- diff kitten: Add support for selecting multi-line text with the mouse
+
+- Fix a regression in 0.27.0 that broke ``kitty @ set-font-size 0`` (:iss:`5992`)
+
+- launch: When using ``--cwd=current`` for a remote system support running non shell commands as well (:disc:`5987`)
+
+- When changing the cursor color via escape codes or remote control to a fixed color, do not reset cursor_text_color (:iss:`5994`)
+
+- Input Method Extensions: Fix incorrect rendering of IME in-progress and committed text in some situations (:pull:`6049`, :pull:`6087`)
+
+- Linux: Reduce minimum required OpenGL version from 3.3 to 3.1 + extensions (:iss:`2790`)
+
+- Fix a regression that broke drawing of images below cell backgrounds (:iss:`6061`)
+
+- macOS: Fix the window buttons not being hidden after exiting the traditional full screen (:iss:`6009`)
+
+- When reloading configuration, also reload custom MIME types from :file:`mime.types` config file (:pull:`6012`)
+
+- launch: Allow specifying the state (full screen/maximized/minimized) for newly created OS Windows (:iss:`6026`)
+
+- Sessions: Allow specifying the OS window state via the ``os_window_state`` directive (:iss:`5863`)
+
+- macOS: Display the newly created OS window in specified state to avoid or reduce the window transition animations (:pull:`6035`)
+
+- macOS: Fix the maximized window not taking up full space when the title bar is hidden or when :opt:`resize_in_steps` is configured (:iss:`6021`)
+
+- Linux: A new option :opt:`linux_bell_theme` to control which sound theme is used for the bell sound (:pull:`4858`)
+
+- ssh kitten: Change the syntax of glob patterns slightly to match common usage
+  elsewhere. Now the syntax is the same as "extendedglob" in most shells.
+
+- hints kitten: Allow copying matches to named buffers (:disc:`6073`)
+
+- Fix overlay windows not inheriting the per-window padding and margin settings
+  of their parents (:iss:`6063`)
+
+- Wayland KDE: Fix selecting in un-focused OS window not working correctly (:iss:`6095`)
+
+- Linux X11: Fix a crash if the X server requests clipboard data after we have relinquished the clipboard (:iss:`5650`)
+
+- Allow stopping of URL detection at newlines via :opt:`url_excluded_characters` (:iss:`6122`)
+
+- Linux Wayland: Fix animated images not being animated continuously (:iss:`6126`)
+
+- Keyboard input: Fix text not being reported as unicode codepoints for multi-byte characters in the kitty keyboard protocol (:iss:`6167`)
+
+
+0.27.1 [2023-02-07]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Fix :opt:`modify_font` not working for strikethrough position (:iss:`5946`)
+
+- Fix a regression causing the ``edit-in-kitty`` command not working if :file:`kitten` is not added
+  to PATH (:iss:`5956`)
+
+- icat kitten: Fix a regression that broke display of animated GIFs over SSH (:iss:`5958`)
+
+- Wayland GNOME: Fix for ibus not working when using XWayland (:iss:`5967`)
+
+- Fix regression in previous release that caused incorrect entries in terminfo for modifier+F3 key combinations (:pull:`5970`)
+
+- Bring back the deprecated and removed ``kitty +complete`` and delegate it to :program:`kitten` for backward compatibility (:pull:`5977`)
+
+- Bump the version of Go needed to build kitty to ``1.20`` so we can use the Go stdlib ecdh package for crypto.
+
+
 0.27.0 [2023-01-31]
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -67,7 +185,7 @@ Detailed list of changes
 
 - A new option :opt:`undercurl_style` to control the rendering of undercurls (:pull:`5883`)
 
-- Bash integration: Fix ``clone-in-kitty`` not working on bash >= 5.2 if an environment variable values contain newlines or other special characters (:iss:`5629`)
+- Bash integration: Fix ``clone-in-kitty`` not working on bash >= 5.2 if environment variable values contain newlines or other special characters (:iss:`5629`)
 
 - A new :ac:`sleep` action useful in combine based mappings to make kitty sleep before executing the next action
 

docs/conf.py

@@ -33,8 +33,8 @@ from kitty.constants import str_version, website_url  # noqa
 # -- Project information -----------------------------------------------------
 
 project = 'kitty'
-copyright = time.strftime('%Y, Kovid Goyal')
+copyright = time.strftime('%Y, Kovid Goyal, KittyPatch')
-author = 'Kovid Goyal'
+author = 'Kovid Goyal, KittyPatch'
 building_man_pages = 'man' in sys.argv
 
 # The short X.Y version
@@ -65,6 +65,10 @@ extensions = [
 
 # URL for OpenGraph tags
 ogp_site_url = website_url()
+# OGP needs a PNG image because of: https://github.com/wpilibsuite/sphinxext-opengraph/issues/96
+ogp_social_cards = {
+    'image': '../logo/kitty.png'
+}
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -96,14 +100,23 @@ exclude_patterns = [
 rst_prolog = '''
 .. |kitty| replace:: *kitty*
 .. |version| replace:: VERSION
-.. _tarball: https://github.com/kovidgoyal/kitty/releases/download/vVERSION/kitty-VERSION.tar.xz
+.. _tarball: https://gitea.rexy712.xyz/KittyPatch/kitty/releases/download/vVERSION/kitty-VERSION.tar.xz
 .. role:: italic
 
 '''.replace('VERSION', str_version)
 smartquotes_action = 'qe'  # educate quotes and ellipses but not dashes
 
+def go_version(go_mod_path: str) -> str:  # {{{
+    with open(go_mod_path) as f:
+        for line in f:
+            if line.startswith('go '):
+                return line.strip().split()[1]
+    raise SystemExit(f'No Go version in {go_mod_path}')
+# }}}
+
 string_replacements = {
     '_kitty_install_cmd': 'curl -L https://sw.kovidgoyal.net/kitty/installer.sh | sh /dev/stdin',
+    '_build_go_version': go_version('../go.mod'),
 }
 
 
@@ -202,7 +215,7 @@ def commit_role(
             f'GitHub commit id "{text}" not recognized.', line=lineno)
         prb = inliner.problematic(rawtext, rawtext, msg)
         return [prb], [msg]
-    url = f'https://github.com/kovidgoyal/kitty/commit/{commit_id}'
+    url = f'https://gitea.rexy712.xyz/KittyPatch/kitty/commit/{commit_id}'
     set_classes(options)
     short_id = subprocess.check_output(
         f'git rev-list --max-count=1 --abbrev-commit --skip=# {commit_id}'.split()).decode('utf-8').strip()
@@ -213,15 +226,16 @@ def commit_role(
 
 # CLI docs {{{
 def write_cli_docs(all_kitten_names: Iterable[str]) -> None:
-    from kittens.ssh.copy import option_text
+    from kittens.ssh.main import copy_message, option_text
-    from kittens.ssh.options.definition import copy_message
     from kitty.cli import option_spec_as_rst
-    from kitty.launch import options_spec as launch_options_spec
     with open('generated/ssh-copy.rst', 'w') as f:
         f.write(option_spec_as_rst(
             appname='copy', ospec=option_text, heading_char='^',
             usage='file-or-dir-to-copy ...', message=copy_message
         ))
+    del sys.modules['kittens.ssh.main']
+
+    from kitty.launch import options_spec as launch_options_spec
     with open('generated/launch.rst', 'w') as f:
         f.write(option_spec_as_rst(
             appname='launch', ospec=launch_options_spec, heading_char='_',
@@ -253,7 +267,6 @@ if you specify a program-to-run you can use the special placeholder
             p('.. program::', 'kitty @', func.name)
             p('\n\n' + as_rst(*cli_params_for(func)))
     from kittens.runner import get_kitten_cli_docs
-    from kitty.fast_data_types import wrapped_kitten_names
 
     for kitten in all_kitten_names:
         data = get_kitten_cli_docs(kitten)
@@ -263,10 +276,7 @@ if you specify a program-to-run you can use the special placeholder
                 p('.. program::', 'kitty +kitten', kitten)
                 p('\nSource code for', kitten)
                 p('-' * 72)
-                if kitten in wrapped_kitten_names():
+                scurl = f'https://github.com/kovidgoyal/kitty/tree/master/kittens/{kitten}'
-                    scurl = f'https://github.com/kovidgoyal/kitty/tree/master/tools/cmd/{kitten}'
-                else:
-                    scurl = f'https://github.com/kovidgoyal/kitty/tree/master/kittens/{kitten}'
                 p(f'\nThe source code for this kitten is `available on GitHub <{scurl}>`_.')
                 p('\nCommand Line Interface')
                 p('-' * 72)
@@ -504,7 +514,7 @@ def write_conf_docs(app: Any, all_kitten_names: Iterable[str]) -> None:
 
         conf_name = re.sub(r'^kitten-', '', name) + '.conf'
         with open(f'generated/conf/{conf_name}', 'w', encoding='utf-8') as f:
-            text = '\n'.join(definition.as_conf())
+            text = '\n'.join(definition.as_conf(commented=True))
             print(text, file=f)
 
     from kitty.options.definition import definition
@@ -512,9 +522,9 @@ def write_conf_docs(app: Any, all_kitten_names: Iterable[str]) -> None:
 
     from kittens.runner import get_kitten_conf_docs
     for kitten in all_kitten_names:
-        definition = get_kitten_conf_docs(kitten)
+        defn = get_kitten_conf_docs(kitten)
-        if definition:
+        if defn is not None:
-            generate_default_config(definition, f'kitten-{kitten}')
+            generate_default_config(defn, f'kitten-{kitten}')
 
     from kitty.actions import as_rst
     with open('generated/actions.rst', 'w', encoding='utf-8') as f:

docs/conf.rst

@@ -68,6 +68,11 @@ Sample kitty.conf
    pre-existing :file:`kitty.conf`, then that will be used instead, delete it to
    see the sample file.
 
+A default configuration file can also be generated by running::
+
+    kitty +runpy 'from kitty.config import *; print(commented_out_default_config())'
+
+This will print the commented out default config file to :file:`STDOUT`.
 
 All mappable actions
 ------------------------

docs/faq.rst

@@ -260,9 +260,9 @@ fonts to be freely resizable, so it does not support bitmapped fonts.
    symbols from it automatically, and you can tell it to do so explicitly in
    case it doesn't with the :opt:`symbol_map` directive::
 
-        # Nerd Fonts v2.2.2
+        # Nerd Fonts v2.3.3
 
-        symbol_map U+23FB-U+23FE,U+2665,U+26A1,U+2B58,U+E000-U+E00A,U+E0A0-U+E0A3,U+E0B0-U+E0C8,U+E0CA,U+E0CC-U+E0D2,U+E0D4,U+E200-U+E2A9,U+E300-U+E3E3,U+E5FA-U+E634,U+E700-U+E7C5,U+EA60-U+EBEB,U+F000-U+F2E0,U+F300-U+F32F,U+F400-U+F4A9,U+F500-U+F8FF Symbols Nerd Font Mono
+        symbol_map U+23FB-U+23FE,U+2665,U+26A1,U+2B58,U+E000-U+E00A,U+E0A0-U+E0A3,U+E0B0-U+E0D4,U+E200-U+E2A9,U+E300-U+E3E3,U+E5FA-U+E6AA,U+E700-U+E7C5,U+EA60-U+EBEB,U+F000-U+F2E0,U+F300-U+F32F,U+F400-U+F4A9,U+F500-U+F8FF,U+F0001-U+F1AF0 Symbols Nerd Font Mono
 
    Those Unicode symbols beyond the ``E000-F8FF`` Unicode private use area are
    not included.
@@ -314,7 +314,7 @@ I do not like the kitty icon!
 There are many alternate icons available, click on an icon to visit its
 homepage:
 
-.. image:: https://github.com/k0nserv/kitty-icon/raw/main/icon_512x512.png
+.. image:: https://github.com/k0nserv/kitty-icon/raw/main/kitty.iconset/icon_256x256.png
    :target: https://github.com/k0nserv/kitty-icon
    :width: 256
 
@@ -338,6 +338,14 @@ homepage:
    :target: https://github.com/samholmes/whiskers
    :width: 256
 
+.. image:: https://github.com/eccentric-j/eccentric-icons/raw/main/icons/kitty-terminal/2d/kitty-preview.png
+   :target: https://github.com/eccentric-j/eccentric-icons
+   :width: 256
+
+.. image:: https://github.com/eccentric-j/eccentric-icons/raw/main/icons/kitty-terminal/3d/kitty-preview.png
+   :target: https://github.com/eccentric-j/eccentric-icons
+   :width: 256
+
 On macOS you can put :file:`kitty.app.icns` or :file:`kitty.app.png` in the
 :ref:`kitty configuration directory <confloc>`, and this icon will be applied
 automatically at startup. Unfortunately, Apple's Dock does not change its

docs/glossary.rst

@@ -45,6 +45,12 @@ Glossary
       hyperlink, based on the type of link and its URL. See also `Hyperlinks in terminal
       emulators <https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda>`__.
 
+   kittens
+      Small, independent statically compiled command line programs that are designed to run
+      inside kitty windows and provide it with lots of powerful and flexible
+      features such as viewing images, connecting conveniently to remote
+      computers, transferring files, inputting unicode characters, etc.
+
 .. _env_vars:
 
 Environment variables

docs/graphics-protocol.rst

@@ -44,6 +44,7 @@ Some programs and libraries that use the kitty graphics protocol:
 * `hologram.nvim <https://github.com/edluffy/hologram.nvim>`_  - view images inside nvim
 * `term-image <https://github.com/AnonymouX47/term-image>`_  - A Python library, CLI and TUI to display and browse images in the terminal
 * `glkitty <https://github.com/michaeljclark/glkitty>`_ - C library to draw OpenGL shaders in the terminal with a glgears demo
+* `twitch-tui <https://github.com/Xithrius/twitch-tui>`_ - Twitch chat in the terminal
 
 Other terminals that have implemented the graphics protocol:
 
@@ -57,7 +58,8 @@ Getting the window size
 
 In order to know what size of images to display and how to position them, the
 client must be able to get the window size in pixels and the number of cells
-per row and column. This can be done by using the ``TIOCGWINSZ`` ioctl.  Some
+per row and column. The cell width is then simply the window size divided by the
+number of rows. This can be done by using the ``TIOCGWINSZ`` ioctl. Some
 code to demonstrate its use
 
 .. tab:: C
@@ -98,6 +100,20 @@ code to demonstrate its use
         fmt.Println("rows: %v columns: %v width: %v height %v", sz.Row, sz.Col, sz.Xpixel, sz.Ypixel)
 
 
+.. tab:: Bash
+
+    .. code-block:: sh
+
+        #!/bin/bash
+
+        # This uses the kitten standalone binary from kitty to get the pixel sizes
+        # since we cant do IOCTLs directly. Fortunately, kitten is a static exe
+        # pre-built for every Unix like OS under the sun.
+
+        builtin read -r rows cols < <(command stty size)
+        IFS=x builtin read -r width height < <(command kitten icat --print-window-size); builtin unset IFS
+        builtin echo "number of rows: $rows number of columns: $cols screen width: $width screen height: $height"
+
 
 Note that some terminals return ``0`` for the width and height values. Such
 terminals should be modified to return the correct values.  Examples of
@@ -330,12 +346,13 @@ sequence of escape codes to the terminal emulator::
     <ESC>_Gm=0;<encoded pixel data last chunk><ESC>\
 
 Note that only the first escape code needs to have the full set of control
-codes such as width, height, format etc. Subsequent chunks **must** have
+codes such as width, height, format, etc. Subsequent chunks **must** have only
-only the ``m`` key. The client **must** finish sending all chunks for a single image
+the ``m`` and optionally ``q`` keys. When sending animation frame data, subsequent
-before sending any other graphics related escape codes. Note that the cursor
+chunks **must** also specify the ``a=f`` key. The client **must** finish sending
-position used to display the image **must** be the position when the final chunk is
+all chunks for a single image before sending any other graphics related escape
-received. Finally, terminals must not display anything, until the entire sequence is
+codes. Note that the cursor position used to display the image **must** be the
-received and validated.
+position when the final chunk is received. Finally, terminals must not display
+anything, until the entire sequence is received and validated.
 
 
 Querying support and available transmission mediums
@@ -471,6 +488,132 @@ z-index and the same id, then the behavior is undefined.
    Support for the C=1 cursor movement policy
 
 
+.. _graphics_unicode_placeholders:
+
+Unicode placeholders
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 0.28.0
+   Support for image display via Unicode placeholders
+
+You can also use a special Unicode character ``U+10EEEE`` as a placeholder for
+an image. This approach is less flexible, but it allows using images inside
+any host application that supports Unicode and foreground colors (tmux, vim, weechat, etc.)
+and has a way to pass escape codes through to the underlying terminal.
+
+The central idea is that we use a single *Private Use* Unicode character as a
+*placeholder* to indicate to the terminal that an image is supposed to be
+displayed at that cell. Since this character is just normal text, Unicode aware
+application will move it around as needed when they redraw their screens,
+thereby automatically moving the displayed image as well, even though they know
+nothing about the graphics protocol. So an image is first created using the
+normal graphics protocol escape codes (albeit in quiet mode (``q=2``) so that there are
+no responses from the terminal that could confuse the host application). Then,
+the actual image is displayed by getting the host application to emit normal
+text consisting of ``U+10EEEE`` and various diacritics (Unicode combining
+characters) and colors.
+
+To use it, first create an image as you would normally with the graphics
+protocol with (``q=2``), but do not create a placement for it, that is, do not
+display it. Then, create a *virtual image placement* by specifying ``U=1`` and
+the desired number of lines and columns::
+
+    <ESC>_Ga=p,U=1,i=<image_id>,c=<columns>,r=<rows><ESC>\
+
+The creation of the placement need not be a separate escape code, it can be
+combined with ``a=T`` to both transmit and create the virtual placement with a
+single code.
+
+The image will eventually be fit to the specified rectangle, its aspect ratio
+preserved. Finally, the image can be actually displayed by using the
+placeholder character, encoding the image ID in its foreground color. The row
+and column values are specified with diacritics listed in
+:download:`rowcolumn-diacritics.txt <../rowcolumn-diacritics.txt>`.  For
+example, here is how you can print a ``2x2`` placeholder for image ID ``42``:
+
+.. code-block:: sh
+
+    printf "\e[38;5;42m\U10EEEE\U0305\U0305\U10EEEE\U0305\U030D\e[39m\n"
+    printf "\e[38;5;42m\U10EEEE\U030D\U0305\U10EEEE\U030D\U030D\e[39m\n"
+
+Here, ``U+305`` is the diacritic corresponding to the number ``0``
+and ``U+30D`` corresponds to ``1``. So these two commands create the following
+``2x2`` placeholder:
+
+========== ==========
+(0, 0)     (1, 0)
+(1, 0)     (1, 1)
+========== ==========
+
+This will cause the image with ID ``42`` to be displayed in a ``2x2`` grid.
+Ideally, you would print out as many cells as the number of rows and columns
+specified when creating the virtual placement, but in case of a mismatch only
+part of the image will be displayed.
+
+By using only the foreground color for image ID you are limited to either 8-bit IDs in 256 color
+mode or 24-bit IDs in true color mode. Since IDs are in a global namespace
+there can easily be collisions. If you need more bits for the image
+ID, you can specify the most significant byte via a third diacritic. For
+example, this is the placeholder for the image ID ``33554474 = 42 + (2 << 24)``:
+
+.. code-block:: sh
+
+    printf "\e[38;5;42m\U10EEEE\U0305\U0305\U030E\U10EEEE\U0305\U030D\U030E\n"
+    printf "\e[38;5;42m\U10EEEE\U030D\U0305\U030E\U10EEEE\U030D\U030D\U030E\n"
+
+Here, ``U+30E`` is the diacritic corresponding to the number ``2``.
+
+You can also specify a placement ID using the underline color (if it's omitted
+or zero, the terminal may choose any virtual placement of the given image). The
+background color is interpreted as the background color, visible if the image is
+transparent. Other text attributes are reserved for future use.
+
+Row, column and most significant byte diacritics may also be omitted, in which
+case the placeholder cell will inherit the missing values from the placeholder
+cell to the left, following the algorithm:
+
+- If no diacritics are present, and the previous placeholder cell has the same
+  foreground and underline colors, then the row of the current cell will be the
+  row of the cell to the left, the column will be the column of the cell to the
+  left plus one, and the most significant image ID byte will be the most
+  significant image ID byte of the cell to the left.
+- If only the row diacritic is present, and the previous placeholder cell has
+  the same row and the same foreground and underline colors, then the column of
+  the current cell will be the column of the cell to the left plus one, and the
+  most significant image ID byte will be the most significant image ID byte of
+  the cell to the left.
+- If only the row and column diacritics are present, and the previous
+  placeholder cell has the same row, the same foreground and underline colors,
+  and its column is one less than the current column, then the most significant
+  image ID byte of the current cell will be the most significant image ID byte
+  of the cell to the left.
+
+These rules are applied left-to-right, which allows specifying only row
+diacritics of the first column, i.e. here is a 2 rows by 3 columns placeholder:
+
+.. code-block:: sh
+
+    printf "\e[38;5;42m\U10EEEE\U0305\U10EEEE\U10EEEE\n"
+    printf "\e[38;5;42m\U10EEEE\U030D\U10EEEE\U10EEEE\n"
+
+This will not work for horizontal scrolling and overlapping images since the two
+given rules will fail to guess the missing information. In such cases, the
+terminal may apply other heuristics (but it doesn't have to).
+
+It is important to distinguish between virtual image placements and real images
+displayed on top of Unicode placeholders. Virtual placements are invisible and only play
+the role of prototypes for real images. Virtual placements can be deleted by a
+deletion command only when the `d` key is equal to ``i``, ``I``, ``n`` or ``N``.
+The key values ``a``, ``c``, ``p``, ``q``, ``x``, ``y``, ``z`` and their capital
+variants never affect virtual placements because they do not have a physical
+location on the screen.
+
+Real images displayed on top of Unicode placeholders are not considered
+placements from the protocol perspective. They cannot be manipulated using
+graphics commands, instead they should be moved, deleted, or modified by
+manipulating the underlying Unicode placeholder as normal text.
+
+
 Deleting images
 ---------------------
 
@@ -789,6 +932,8 @@ Key      Value                 Default    Description
 ``r``    Positive integer      ``0``      The number of rows to display the image over
 ``C``    Positive integer      ``0``      Cursor movement policy. ``0`` is the default, to move the cursor to after the image.
                                           ``1`` is to not move the cursor at all when placing the image.
+``U``    Positive integer      ``0``      Set to ``1`` to create a virtual placement for a Unicode placeholder.
+                                          ``1`` is to not move the cursor at all when placing the image.
 ``z``    32-bit integer        ``0``      The *z-index* vertical stacking order of the image
 
 **Keys for animation frame loading**

docs/installer.sh

@@ -46,7 +46,7 @@ detect_os() {
         'Linux')
             OS="linux"
             case "$(command uname -m)" in
-                x86_64) arch="x86_64";;
+                amd64|x86_64) arch="x86_64";;
                 aarch64*) arch="arm64";;
                 armv8*) arch="arm64";;
                 i386) arch="i686";;
@@ -114,36 +114,38 @@ get_download_url() {
     esac
 }
 
-linux_install() {
+download_installer() {
-    if [ "$installer_is_file" = "y" ]; then
+    tdir=$(command mktemp -d "/tmp/kitty-install-XXXXXXXXXXXX")
-        command tar -C "$dest" "-xJof" "$installer"
+    [ "$installer_is_file" != "y" ] && {
-    else
         printf '%s\n\n' "Downloading from: $url"
-        fetch "$url" | command tar -C "$dest" "-xJof" "-"
+        if [ "$OS" = "macos" ]; then
-    fi
+            installer="$tdir/kitty.dmg"
+        else
+            installer="$tdir/kitty.txz"
+        fi
+        fetch "$url" > "$installer" || die "Failed to download: $url"
+        installer_is_file="y"
+    }
+}
+
+linux_install() {
+    command mkdir "$tdir/mp"
+    command tar -C "$tdir/mp" "-xJof" "$installer" || die "Failed to extract kitty tarball"
+    printf "%s\n" "Installing to $dest"
+    command rm -rf "$dest" || die "Failed to delete $dest"
+    command mv "$tdir/mp" "$dest" || die "Failed to move kitty.app to $dest"
 }
 
 macos_install() {
-    tdir=$(command mktemp -d "/tmp/kitty-install-XXXXXXXXXXXX")
-    [ "$installer_is_file" != "y" ] && {
-        installer="$tdir/kitty.dmg"
-        printf '%s\n\n' "Downloading from: $url"
-        fetch "$url" > "$installer" || die "Failed to download: $url"
-    }
     command mkdir "$tdir/mp"
     command hdiutil attach "$installer" "-mountpoint" "$tdir/mp" || die "Failed to mount kitty.dmg"
-    command ditto -v "$tdir/mp/kitty.app" "$dest"
-    rc="$?"
-    command hdiutil detach "$tdir/mp"
-    command rm -rf "$tdir"
-    tdir=''
-    [ "$rc" != "0" ] && die "Failed to copy kitty.app from mounted dmg"
-}
-
-prepare_install_dest() {
     printf "%s\n" "Installing to $dest"
     command rm -rf "$dest"
     command mkdir -p "$dest" || die "Failed to create the directory: $dest"
+    command ditto -v "$tdir/mp/kitty.app" "$dest"
+    rc="$?"
+    command hdiutil detach "$tdir/mp"
+    [ "$rc" != "0" ] && die "Failed to copy kitty.app from mounted dmg"
 }
 
 exec_kitty() {
@@ -160,12 +162,13 @@ main() {
     parse_args "$@"
     detect_network_tool
     get_download_url
-    prepare_install_dest
+    download_installer
     if [ "$OS" = "macos" ]; then
         macos_install
     else
         linux_install
     fi
+    cleanup
     [ "$launch" = "y" ] && exec_kitty
     exit 0
 }

docs/integrations.rst

@@ -80,6 +80,11 @@ base application that uses kitty's graphics protocol for images.
 A text mode WWW browser that supports kitty's graphics protocol to display
 images.
 
+`awrit <https://github.com/chase/awrit>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+A full Chromium based web browser running in the terminal using kitty's
+graphics protocol.
+
 .. _tool_mpv:
 
 `mpv <https://github.com/mpv-player/mpv/commit/874e28f4a41a916bb567a882063dd2589e9234e1>`_
@@ -143,7 +148,7 @@ kitty with the following bash snippet:
         set object 1 rectangle from screen 0,0 to screen 1,1 fillcolor rgb"#fdf6e3" behind
         plot $@
         set output '/dev/null'
-        EOF
+    EOF
     }
 
 Add this to bashrc and then to plot a function, simply do:
@@ -220,7 +225,8 @@ Allows easily running tests in a terminal window
 
 `hologram.nvim <https://github.com/edluffy/hologram.nvim>`_
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Terminal image viewer for Neovim
+Terminal image viewer for Neovim. For a bit of fun, you can even have `cats
+running around inside nvim <https://github.com/giusgad/pets.nvim>`__.
 
 
 Scrollback manipulation

docs/keyboard-protocol.rst

@@ -41,9 +41,12 @@ In addition to kitty, this protocol is also implemented in:
 * The `crossterm library
   <https://github.com/crossterm-rs/crossterm/pull/688>`__
 * The `Vim text editor <https://github.com/vim/vim/commit/63a2e360cca2c70ab0a85d14771d3259d4b3aafa>`__
+* The `Emacs text editor via the kkp package <https://github.com/benjaminor/kkp>`__
 * The `Neovim text editor <https://github.com/neovim/neovim/pull/18181>`__
 * The `kakoune text editor <https://github.com/mawww/kakoune/issues/4103>`__
 * The `dte text editor <https://gitlab.com/craigbarnes/dte/-/issues/138>`__
+* The `Helix text editor <https://github.com/helix-editor/helix/pull/4939>`__
+* The `far2l file manager <https://github.com/elfmz/far2l/commit/e1f2ee0ef2b8332e5fa3ad7f2e4afefe7c96fc3b>`__
 
 .. versionadded:: 0.20.0
 

docs/kittens/diff.rst

@@ -31,11 +31,7 @@ Major Features
 Installation
 ---------------
 
-Simply :ref:`install kitty <quickstart>`.  You also need to have either the `git
+Simply :ref:`install kitty <quickstart>`.
-<https://git-scm.com/>`__ program or the :program:`diff` program installed.
-Additionally, for syntax highlighting to work, `pygments
-<https://pygments.org/>`__ must be installed (note that pygments is included in
-the official kitty binary builds).
 
 
 Usage
@@ -65,30 +61,32 @@ directory contents.
 Keyboard controls
 ----------------------
 
-=========================   ===========================
+===========================       ===========================
-Action                      Shortcut
+Action                            Shortcut
-=========================   ===========================
+===========================       ===========================
-Quit                        :kbd:`Q`, :kbd:`Ctrl+C`, :kbd:`Esc`
+Quit                              :kbd:`Q`, :kbd:`Esc`
-Scroll line up              :kbd:`K`, :kbd:`Up`
+Scroll line up                    :kbd:`K`, :kbd:`Up`
-Scroll line down            :kbd:`J`, :kbd:`Down`
+Scroll line down                  :kbd:`J`, :kbd:`Down`
-Scroll page up              :kbd:`PgUp`
+Scroll page up                    :kbd:`PgUp`
-Scroll page down            :kbd:`PgDn`
+Scroll page down                  :kbd:`PgDn`
-Scroll to top               :kbd:`Home`
+Scroll to top                     :kbd:`Home`
-Scroll to bottom            :kbd:`End`
+Scroll to bottom                  :kbd:`End`
-Scroll to next page         :kbd:`Space`, :kbd:`PgDn`
+Scroll to next page               :kbd:`Space`, :kbd:`PgDn`
-Scroll to previous page     :kbd:`PgUp`
+Scroll to previous page           :kbd:`PgUp`
-Scroll to next change       :kbd:`N`
+Scroll to next change             :kbd:`N`
-Scroll to previous change   :kbd:`P`
+Scroll to previous change         :kbd:`P`
-Increase lines of context   :kbd:`+`
+Increase lines of context         :kbd:`+`
-Decrease lines of context   :kbd:`-`
+Decrease lines of context         :kbd:`-`
-All lines of context        :kbd:`A`
+All lines of context              :kbd:`A`
-Restore default context     :kbd:`=`
+Restore default context           :kbd:`=`
-Search forwards             :kbd:`/`
+Search forwards                   :kbd:`/`
-Search backwards            :kbd:`?`
+Search backwards                  :kbd:`?`
-Clear search                :kbd:`Esc`
+Clear search                      :kbd:`Esc`
-Scroll to next match        :kbd:`>`, :kbd:`.`
+Scroll to next match              :kbd:`>`, :kbd:`.`
-Scroll to previous match    :kbd:`<`, :kbd:`,`
+Scroll to previous match          :kbd:`<`, :kbd:`,`
-=========================   ===========================
+Copy selection to clipboard       :kbd:`y`
+Copy selection or exit            :kbd:`Ctrl+C`
+===========================       ===========================
 
 
 Integrating with git
@@ -124,7 +122,7 @@ The diff kitten makes use of various features that are :doc:`kitty only
 </graphics-protocol>`, the :doc:`extended keyboard protocol
 </keyboard-protocol>`, etc. It also leverages terminal program infrastructure
 I created for all of kitty's other kittens to reduce the amount of code needed
-(the entire implementation is under 2000 lines of code).
+(the entire implementation is under 3000 lines of code).
 
 And fundamentally, it's kitty only because I wrote it for myself, and I am
 highly unlikely to use any other terminals :)

docs/kittens/hints.rst

@@ -72,7 +72,8 @@ the :ref:`kitty config directory <confloc>` with the following contents:
             start, end = m.span()
             mark_text = text[start:end].replace('\n', '').replace('\0', '')
             # The empty dictionary below will be available as groupdicts
-            # in handle_result() and can contain arbitrary data.
+            # in handle_result() and can contain string keys and arbitrary JSON
+            # serializable values.
             yield Mark(idx, start, end, mark_text, {})
 
 

docs/kittens/themes.rst

@@ -44,7 +44,8 @@ You can also create your own themes as :file:`.conf` files. Put them in the
 usually, :file:`~/.config/kitty/themes`. The kitten will automatically add them
 to the list of themes. You can use this to modify the builtin themes, by giving
 the conf file the name :file:`Some theme name.conf` to override the builtin
-theme of that name. Note that after doing so you have to run the kitten and
+theme of that name. Here, ``Some theme name`` is the actual builtin theme name, not
+its file name. Note that after doing so you have to run the kitten and
 choose that theme once for your changes to be applied.
 
 

docs/overview.rst

@@ -10,8 +10,9 @@ configuration is a simple, human editable, single file for easy reproducibility
 (I like to store configuration in source control).
 
 The code in |kitty| is designed to be simple, modular and hackable. It is
-written in a mix of C (for performance sensitive parts) and Python (for easy
+written in a mix of C (for performance sensitive parts), Python (for easy
-hackability of the UI). It does not depend on any large and complex UI toolkit,
+extensibility and flexibility of the UI) and Go (for the command line
+:term:`kittens`).  It does not depend on any large and complex UI toolkit,
 using only OpenGL for rendering everything.
 
 Finally, |kitty| is designed from the ground up to support all modern terminal
@@ -160,6 +161,8 @@ option in :file:`kitty.conf`. An example, showing all available commands:
     os_window_size 80c 24c
     # Set the --class for the new OS window
     os_window_class mywindow
+    # Change the OS window state to normal, fullscreen, maximized or minimized
+    os_window_state normal
     launch sh
     # Resize the current window (see the resize_window action for details)
     resize_window wider 2
@@ -232,9 +235,10 @@ Font control
 |kitty| has extremely flexible and powerful font selection features. You can
 specify individual families for the regular, bold, italic and bold+italic fonts.
 You can even specify specific font families for specific ranges of Unicode
-characters. This allows precise control over text rendering. It can comein handy
+characters. This allows precise control over text rendering. It can come in
-for applications like powerline, without the need to use patched fonts. See the
+handy for applications like powerline, without the need to use patched fonts.
-various font related configuration directives in :ref:`conf-kitty-fonts`.
+See the various font related configuration directives in
+:ref:`conf-kitty-fonts`.
 
 
 .. _scrollback:

docs/rc_protocol.rst

@@ -83,5 +83,25 @@ is created and transmitted that contains the fields:
         "encrypted": "The original command encrypted and base85 encoded"
     }
 
+Async and streaming requests
+---------------------------------
+
+Some remote control commands require asynchronous communication, that is, the
+response from the terminal can happen after an arbitrary amount of time. For
+example, the :code:`select-window` command requires the user to select a window
+before a response can be sent. Such command must set the field :code:`async`
+in the JSON block above to a random string that serves as a unique id. The
+client can cancel an async request in flight by adding the :code:`cancel_async`
+field to the JSON block. A async response remains in flight until the terminal
+sends a response to the request. Note that cancellation requests dont need to
+be encrypted as users must not be prompted for these and the worst a malicious
+cancellation request can do is prevent another sync request from getting a
+response.
+
+Similar to async requests are *streaming* requests. In these the client has to
+send a large amount of data to the terminal and so the request is split into
+chunks. In every chunk the JSON block must contain the field ``stream`` set to
+``true`` and ``stream_id`` set to a random long string, that should be the same for
+all chunks in a request. End of data is indicated by sending a chunk with no data.
 
 .. include:: generated/rc.rst

docs/support.rst

@@ -1,5 +1,21 @@
+A message from us at KittyPatch
+===============================
+
+KittyPatch was created as a home for useful features that are unavailable
+in the kitty main branch.  To this end, we do not accept donations directly.
+
+If you wish to support KittyPatch: share your ideas and spread the word.
+
+If you still wish to donate, please `support the Free Software Foundation
+<https://my.fsf.org/donate>`.
+
+
+
+A message from the maintainer of Kitty
+======================================
 Support kitty development ❤️
 ==============================
+>>>>>>> upstream/master
 
 My goal with |kitty| is to move the stagnant terminal ecosystem forward.  To that
 end kitty has many foundational features, such as: :doc:`image support

gen-apc-parsers.py

@@ -274,6 +274,7 @@ def graphics_parser() -> None:
         'Y': ('cell_y_offset', 'uint'),
         'z': ('z_index', 'int'),
         'C': ('cursor_movement', 'uint'),
+        'U': ('unicode_placement', 'uint'),
     }
     text = generate('parse_graphics_code', 'screen_handle_graphics_command', 'graphics_command', keymap, 'GraphicsCommand')
     write_header(text, 'kitty/parse-graphics-command.h')

gen-config.py

@@ -47,12 +47,7 @@ def main() -> None:
                 all_colors.append(opt.name)
     patch_color_list('kitty/rc/set_colors.py', nullable_colors, 'NULLABLE')
     patch_color_list('tools/cmd/at/set_colors.go', nullable_colors, 'NULLABLE')
-    patch_color_list('kittens/themes/collection.py', all_colors, 'ALL', ' ' * 8)
+    patch_color_list('tools/themes/collection.go', all_colors, 'ALL')
-
-    from kittens.diff.options.definition import definition as kd
-    write_output('kittens.diff', kd)
-    from kittens.ssh.options.definition import definition as sd
-    write_output('kittens.ssh', sd)
 
 
 if __name__ == '__main__':

gen-go-code.py

@@ -1,14 +1,31 @@
 #!./kitty/launcher/kitty +launch
 # License: GPLv3 Copyright: 2022, Kovid Goyal <kovid at kovidgoyal.net>
 
+import bz2
 import io
 import json
 import os
+import re
+import struct
 import subprocess
 import sys
+import tarfile
 from contextlib import contextmanager, suppress
 from functools import lru_cache
-from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple, Union
+from itertools import chain
+from typing import (
+    Any,
+    BinaryIO,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    TextIO,
+    Tuple,
+    Union,
+)
 
 import kitty.constants as kc
 from kittens.tui.operations import Mode
@@ -20,7 +37,9 @@ from kitty.cli import (
     parse_option_spec,
     serialize_as_go_string,
 )
-from kitty.guess_mime_type import text_mimes
+from kitty.conf.generate import gen_go_code
+from kitty.conf.types import Definition
+from kitty.guess_mime_type import known_extensions, text_mimes
 from kitty.key_encoding import config_mod_map
 from kitty.key_names import character_key_name_aliases, functional_key_name_aliases
 from kitty.options.types import Options
@@ -31,6 +50,19 @@ from kitty.rgb import color_names
 changed: List[str] = []
 
 
+def newer(dest: str, *sources: str) -> bool:
+    try:
+        dtime = os.path.getmtime(dest)
+    except OSError:
+        return True
+    for s in chain(sources, (__file__,)):
+        with suppress(FileNotFoundError):
+            if os.path.getmtime(s) >= dtime:
+                return True
+    return False
+
+
+
 # Utils {{{
 
 def serialize_go_dict(x: Union[Dict[str, int], Dict[int, str], Dict[int, int], Dict[str, str]]) -> str:
@@ -303,9 +335,46 @@ def wrapped_kittens() -> Sequence[str]:
     raise Exception('Failed to read wrapped kittens from kitty wrapper script')
 
 
+def generate_conf_parser(kitten: str, defn: Definition) -> None:
+    with replace_if_needed(f'kittens/{kitten}/conf_generated.go'):
+        print(f'package {kitten}')
+        print(gen_go_code(defn))
+
+
+def generate_extra_cli_parser(name: str, spec: str) -> None:
+    print('import "kitty/tools/cli"')
+    go_opts = tuple(go_options_for_seq(parse_option_spec(spec)[0]))
+    print(f'type {name}_options struct ''{')
+    for opt in go_opts:
+        print(opt.struct_declaration())
+    print('}')
+    print(f'func parse_{name}_args(args []string) (*{name}_options, []string, error) ''{')
+    print(f'root := cli.Command{{Name: `{name}` }}')
+    for opt in go_opts:
+        print(opt.as_option('root'))
+    print('cmd, err := root.ParseArgs(args)')
+    print('if err != nil { return nil, nil, err }')
+    print(f'var opts {name}_options')
+    print('err = cmd.GetOptionValues(&opts)')
+    print('if err != nil { return nil, nil, err }')
+    print('return &opts, cmd.Args, nil')
+    print('}')
+
+
 def kitten_clis() -> None:
+    from kittens.runner import get_kitten_conf_docs, get_kitten_extra_cli_parsers
     for kitten in wrapped_kittens():
-        with replace_if_needed(f'tools/cmd/{kitten}/cli_generated.go'):
+        defn = get_kitten_conf_docs(kitten)
+        if defn is not None:
+            generate_conf_parser(kitten, defn)
+        ecp = get_kitten_extra_cli_parsers(kitten)
+        if ecp:
+            for name, spec in ecp.items():
+                with replace_if_needed(f'kittens/{kitten}/{name}_cli_generated.go'):
+                    print(f'package {kitten}')
+                    generate_extra_cli_parser(name, spec)
+
+        with replace_if_needed(f'kittens/{kitten}/cli_generated.go'):
             od = []
             kcd = kitten_cli_docs(kitten)
             has_underscore = '_' in kitten
@@ -314,10 +383,11 @@ def kitten_clis() -> None:
             print('func create_cmd(root *cli.Command, run_func func(*cli.Command, *Options, []string)(int, error)) {')
             print('ans := root.AddSubCommand(&cli.Command{')
             print(f'Name: "{kitten}",')
-            print(f'ShortDescription: "{serialize_as_go_string(kcd["short_desc"])}",')
+            if kcd:
-            if kcd['usage']:
+                print(f'ShortDescription: "{serialize_as_go_string(kcd["short_desc"])}",')
-                print(f'Usage: "[options] {serialize_as_go_string(kcd["usage"])}",')
+                if kcd['usage']:
-            print(f'HelpText: "{serialize_as_go_string(kcd["help_text"])}",')
+                    print(f'Usage: "[options] {serialize_as_go_string(kcd["usage"])}",')
+                print(f'HelpText: "{serialize_as_go_string(kcd["help_text"])}",')
             print('Run: func(cmd *cli.Command, args []string) (int, error) {')
             print('opts := Options{}')
             print('err := cmd.GetOptionValues(&opts)')
@@ -336,6 +406,8 @@ def kitten_clis() -> None:
                 print("clone := root.AddClone(ans.Group, ans)")
                 print('clone.Hidden = false')
                 print(f'clone.Name = "{serialize_as_go_string(kitten.replace("_", "-"))}"')
+            if not kcd:
+                print('specialize_command(ans)')
             print('}')
             print('type Options struct {')
             print('\n'.join(od))
@@ -368,11 +440,24 @@ def generate_spinners() -> str:
 
 
 def generate_color_names() -> str:
+    selfg = "" if Options.selection_foreground is None else Options.selection_foreground.as_sharp
+    selbg = "" if Options.selection_background is None else Options.selection_background.as_sharp
+    cursor = "" if Options.cursor is None else Options.cursor.as_sharp
     return 'package style\n\nvar ColorNames = map[string]RGBA{' + '\n'.join(
         f'\t"{name}": RGBA{{ Red:{val.red}, Green:{val.green}, Blue:{val.blue} }},'
         for name, val in color_names.items()
     ) + '\n}' + '\n\nvar ColorTable = [256]uint32{' + ', '.join(
-        f'{x}' for x in Options.color_table) + '}\n'
+        f'{x}' for x in Options.color_table) + '}\n' + f'''
+var DefaultColors = struct {{
+Foreground, Background, Cursor, SelectionFg, SelectionBg string
+}}{{
+Foreground: "{Options.foreground.as_sharp}",
+Background: "{Options.background.as_sharp}",
+Cursor: "{cursor}",
+SelectionFg: "{selfg}",
+SelectionBg: "{selbg}",
+}}
+'''
 
 
 def load_ref_map() -> Dict[str, Dict[str, str]]:
@@ -384,8 +469,17 @@ def load_ref_map() -> Dict[str, Dict[str, str]]:
 
 
 def generate_constants() -> str:
+    from kittens.hints.main import DEFAULT_REGEX
+    from kitty.options.types import Options
+    from kitty.options.utils import allowed_shell_integration_values
+    del sys.modules['kittens.hints.main']
     ref_map = load_ref_map()
+    with open('kitty/data-types.h') as dt:
+        m = re.search(r'^#define IMAGE_PLACEHOLDER_CHAR (\S+)', dt.read(), flags=re.M)
+        assert m is not None
+        placeholder_char = int(m.group(1), 16)
     dp = ", ".join(map(lambda x: f'"{serialize_as_go_string(x)}"', kc.default_pager_for_help))
+    url_prefixes = ','.join(f'"{x}"' for x in Options.url_prefixes)
     return f'''\
 package kitty
 
@@ -394,11 +488,14 @@ type VersionType struct {{
 }}
 const VersionString string = "{kc.str_version}"
 const WebsiteBaseURL string = "{kc.website_base_url}"
+const ImagePlaceholderChar rune = {placeholder_char}
 const VCSRevision string = ""
+const SSHControlMasterTemplate = "{kc.ssh_control_master_template}"
 const RC_ENCRYPTION_PROTOCOL_VERSION string = "{kc.RC_ENCRYPTION_PROTOCOL_VERSION}"
 const IsFrozenBuild bool = false
 const IsStandaloneBuild bool = false
 const HandleTermiosSignals = {Mode.HANDLE_TERMIOS_SIGNALS.value[0]}
+const HintsDefaultRegex = `{DEFAULT_REGEX}`
 var Version VersionType = VersionType{{Major: {kc.version.major}, Minor: {kc.version.minor}, Patch: {kc.version.patch},}}
 var DefaultPager []string = []string{{ {dp} }}
 var FunctionalKeyNameAliases = map[string]string{serialize_go_dict(functional_key_name_aliases)}
@@ -406,6 +503,15 @@ var CharacterKeyNameAliases = map[string]string{serialize_go_dict(character_key_
 var ConfigModMap = map[string]uint16{serialize_go_dict(config_mod_map)}
 var RefMap = map[string]string{serialize_go_dict(ref_map['ref'])}
 var DocTitleMap = map[string]string{serialize_go_dict(ref_map['doc'])}
+var AllowedShellIntegrationValues = []string{{ {str(sorted(allowed_shell_integration_values))[1:-1].replace("'", '"')} }}
+var KittyConfigDefaults = struct {{
+Term, Shell_integration, Select_by_word_characters string
+Wheel_scroll_multiplier int
+Url_prefixes []string
+}}{{
+Term: "{Options.term}", Shell_integration: "{' '.join(Options.shell_integration)}", Url_prefixes: []string{{ {url_prefixes} }},
+Select_by_word_characters: `{Options.select_by_word_characters}`, Wheel_scroll_multiplier: {Options.wheel_scroll_multiplier},
+}}
 '''  # }}}
 
 
@@ -580,9 +686,59 @@ def generate_textual_mimetypes() -> str:
     for k in text_mimes:
         ans.append(f'  "{serialize_as_go_string(k)}": true,')
     ans.append('}')
+    ans.append('var KnownExtensions = map[string]string{')
+    for k, v in known_extensions.items():
+        ans.append(f'  ".{serialize_as_go_string(k)}": "{serialize_as_go_string(v)}",')
+    ans.append('}')
     return '\n'.join(ans)
 
 
+def write_compressed_data(data: bytes, d: BinaryIO) -> None:
+    d.write(struct.pack('<I', len(data)))
+    d.write(bz2.compress(data))
+
+
+def generate_unicode_names(src: TextIO, dest: BinaryIO) -> None:
+    num_names, num_of_words = map(int, next(src).split())
+    gob = io.BytesIO()
+    gob.write(struct.pack('<II', num_names, num_of_words))
+    for line in src:
+        line = line.strip()
+        if line:
+            a, aliases = line.partition('\t')[::2]
+            cp, name = a.partition(' ')[::2]
+            ename = name.encode()
+            record = struct.pack('<IH', int(cp), len(ename)) + ename
+            if aliases:
+                record += aliases.encode()
+            gob.write(struct.pack('<H', len(record)) + record)
+    write_compressed_data(gob.getvalue(), dest)
+
+
+def generate_ssh_kitten_data() -> None:
+    files = {
+        'terminfo/kitty.terminfo', 'terminfo/x/xterm-kitty',
+    }
+    for dirpath, dirnames, filenames in os.walk('shell-integration'):
+        for f in filenames:
+            path = os.path.join(dirpath, f)
+            files.add(path.replace(os.sep, '/'))
+    dest = 'kittens/ssh/data_generated.bin'
+
+    def normalize(t: tarfile.TarInfo) -> tarfile.TarInfo:
+        t.uid = t.gid = 0
+        t.uname = t.gname = ''
+        return t
+
+    if newer(dest, *files):
+        buf = io.BytesIO()
+        with tarfile.open(fileobj=buf, mode='w') as tf:
+            for f in sorted(files):
+                tf.add(f, filter=normalize)
+        with open(dest, 'wb') as d:
+            write_compressed_data(buf.getvalue(), d)
+
+
 def main() -> None:
     with replace_if_needed('constants_generated.go') as f:
         f.write(generate_constants())
@@ -596,6 +752,10 @@ def main() -> None:
         f.write(generate_mimetypes())
     with replace_if_needed('tools/utils/mimetypes_textual_generated.go') as f:
         f.write(generate_textual_mimetypes())
+    if newer('tools/unicode_names/data_generated.bin', 'tools/unicode_names/names.txt'):
+        with open('tools/unicode_names/data_generated.bin', 'wb') as dest, open('tools/unicode_names/names.txt') as src:
+            generate_unicode_names(src, dest)
+    generate_ssh_kitten_data()
 
     update_completion()
     update_at_commands()

gen-srgb-lut.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# vim:fileencoding=utf-8
+
+import os
+from typing import List
+
+
+def to_linear(a: float) -> float:
+    if a <= 0.04045:
+        return a / 12.92
+    else:
+        return float(pow((a + 0.055) / 1.055, 2.4))
+
+
+def generate_srgb_lut(line_prefix: str = '') -> List[str]:
+    values: List[str] = []
+    lines: List[str] = []
+
+    for i in range(256):
+        values.append('{:1.5f}f'.format(to_linear(i / 255.0)))
+
+    for i in range(16):
+        lines.append(line_prefix + ', '.join(values[i * 16:(i + 1) * 16]) + ',')
+
+    return lines
+
+
+def generate_srgb_gamma_c() -> str:
+    lines: List[str] = []
+
+    lines.append('// Generated by gen-srgb-lut.py DO NOT edit')
+    lines.append('#include "srgb_gamma.h"')
+    lines.append('')
+    lines.append('const GLfloat srgb_lut[256] = {')
+    lines += generate_srgb_lut('    ')
+    lines.append('};')
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    c = generate_srgb_gamma_c()
+    with open(os.path.join('kitty', 'srgb_gamma.c'), 'w') as f:
+        f.write(f'{c}\n')
+
+
+if __name__ == '__main__':
+    main()

gen-wcwidth.py

@@ -369,19 +369,25 @@ def codepoint_to_mark_map(p: Callable[..., None], mark_map: List[int]) -> Dict[i
     return rmap
 
 
-def classes_to_regex(classes: Iterable[str], exclude: str = '') -> Iterable[str]:
+def classes_to_regex(classes: Iterable[str], exclude: str = '', for_go: bool = True) -> Iterable[str]:
     chars: Set[int] = set()
     for c in classes:
         chars |= class_maps[c]
     for x in map(ord, exclude):
         chars.discard(x)
 
-    def as_string(codepoint: int) -> str:
+    if for_go:
-        if codepoint < 256:
+        def as_string(codepoint: int) -> str:
-            return fr'\x{codepoint:02x}'
+            if codepoint < 256:
-        if codepoint <= 0xffff:
+                return fr'\x{codepoint:02x}'
-            return fr'\u{codepoint:04x}'
+            return fr'\x{{{codepoint:x}}}'
-        return fr'\U{codepoint:08x}'
+    else:
+        def as_string(codepoint: int) -> str:
+            if codepoint < 256:
+                return fr'\x{codepoint:02x}'
+            if codepoint <= 0xffff:
+                return fr'\u{codepoint:04x}'
+            return fr'\U{codepoint:08x}'
 
     for spec in get_ranges(list(chars)):
         if isinstance(spec, tuple):
@@ -438,110 +444,30 @@ def gen_ucd() -> None:
             f.truncate()
             f.write(raw)
 
-    with open('kittens/hints/url_regex.py', 'w') as f:
+    chars = ''.join(classes_to_regex(cz, exclude='\n\r'))
-        f.write('# generated by gen-wcwidth.py, do not edit\n\n')
+    with open('tools/cmd/hints/url_regex.go', 'w') as f:
-        f.write("url_delimiters = '{}'  # noqa".format(''.join(classes_to_regex(cz, exclude='\n\r'))))
+        f.write('// generated by gen-wcwidth.py, do not edit\n\n')
+        f.write('package hints\n\n')
+        f.write(f'const URL_DELIMITERS = `{chars}`\n')
 
 
 def gen_names() -> None:
-    with create_header('kittens/unicode_input/names.h') as p:
+    aliases_map: Dict[int, Set[str]] = {}
-        mark_to_cp = list(sorted(name_map))
+    for word, codepoints in word_search_map.items():
-        cp_to_mark = {cp: m for m, cp in enumerate(mark_to_cp)}
+        for cp in codepoints:
-        # Mapping of mark to codepoint name
+            aliases_map.setdefault(cp, set()).add(word)
-        p(f'static const char* name_map[{len(mark_to_cp)}] = {{' ' // {{{')
+    if len(name_map) > 0xffff:
-        for cp in mark_to_cp:
+        raise Exception('Too many named codepoints')
-            w = name_map[cp].replace('"', '\\"')
+    with open('tools/unicode_names/names.txt', 'w') as f:
-            p(f'\t"{w}",')
+        print(len(name_map), len(word_search_map), file=f)
-        p("}; // }}}\n")
+        for cp in sorted(name_map):
-
+            name = name_map[cp]
-        # Mapping of mark to codepoint
+            words = name.lower().split()
-        p(f'static const char_type mark_to_cp[{len(mark_to_cp)}] = {{' ' // {{{')
+            aliases = aliases_map.get(cp, set()) - set(words)
-        p(', '.join(map(str, mark_to_cp)))
+            end = '\n'
-        p('}; // }}}\n')
+            if aliases:
-
+                end = '\t' + ' '.join(sorted(aliases)) + end
-        # Function to get mark number for codepoint
+            print(cp, *words, end=end, file=f)
-        p('static char_type mark_for_codepoint(char_type c) {')
-        codepoint_to_mark_map(p, mark_to_cp)
-        p('}\n')
-        p('static inline const char* name_for_codepoint(char_type cp) {')
-        p('\tchar_type m = mark_for_codepoint(cp); if (m == 0) return NULL;')
-        p('\treturn name_map[m];')
-        p('}\n')
-
-        # Array of all words
-        word_map = tuple(sorted(word_search_map))
-        word_rmap = {w: i for i, w in enumerate(word_map)}
-        p(f'static const char* all_words_map[{len(word_map)}] = {{' ' // {{{')
-        cwords = (w.replace('"', '\\"') for w in word_map)
-        p(', '.join(f'"{w}"' for w in cwords))
-        p('}; // }}}\n')
-
-        # Array of sets of marks for each word
-        word_to_marks = {word_rmap[w]: frozenset(map(cp_to_mark.__getitem__, cps)) for w, cps in word_search_map.items()}
-        all_mark_groups = frozenset(word_to_marks.values())
-        array = [0]
-        mg_to_offset = {}
-        for mg in all_mark_groups:
-            mg_to_offset[mg] = len(array)
-            array.append(len(mg))
-            array.extend(sorted(mg))
-        p(f'static const char_type mark_groups[{len(array)}] = {{' ' // {{{')
-        p(', '.join(map(str, array)))
-        p('}; // }}}\n')
-        offsets_array = []
-        for wi, w in enumerate(word_map):
-            mg = word_to_marks[wi]
-            offsets_array.append(mg_to_offset[mg])
-        p(f'static const char_type mark_to_offset[{len(offsets_array)}] = {{' ' // {{{')
-        p(', '.join(map(str, offsets_array)))
-        p('}; // }}}\n')
-
-        # The trie
-        p('typedef struct { uint32_t children_offset; uint32_t match_offset; } word_trie;\n')
-        all_trie_nodes: List['TrieNode'] = []
-
-        class TrieNode:
-
-            def __init__(self) -> None:
-                self.match_offset = 0
-                self.children_offset = 0
-                self.children: Dict[int, int] = {}
-
-            def add_letter(self, letter: int) -> int:
-                if letter not in self.children:
-                    self.children[letter] = len(all_trie_nodes)
-                    all_trie_nodes.append(TrieNode())
-                return self.children[letter]
-
-            def __str__(self) -> str:
-                return f'{{ .children_offset={self.children_offset}, .match_offset={self.match_offset} }}'
-
-        root = TrieNode()
-        all_trie_nodes.append(root)
-
-        def add_word(word_idx: int, word: str) -> None:
-            parent = root
-            for letter in map(ord, word):
-                idx = parent.add_letter(letter)
-                parent = all_trie_nodes[idx]
-            parent.match_offset = offsets_array[word_idx]
-
-        for i, word in enumerate(word_map):
-            add_word(i, word)
-        children_array = [0]
-        for node in all_trie_nodes:
-            if node.children:
-                node.children_offset = len(children_array)
-                children_array.append(len(node.children))
-                for letter, child_offset in node.children.items():
-                    children_array.append((child_offset << 8) | (letter & 0xff))
-
-        p(f'static const word_trie all_trie_nodes[{len(all_trie_nodes)}] = {{' ' // {{{')
-        p(',\n'.join(map(str, all_trie_nodes)))
-        p('\n}; // }}}\n')
-        p(f'static const uint32_t children_array[{len(children_array)}] = {{' ' // {{{')
-        p(', '.join(map(str, children_array)))
-        p('}; // }}}\n')
 
 
 def gen_wcwidth() -> None:
@@ -611,6 +537,53 @@ def gen_wcwidth() -> None:
     subprocess.check_call(['gofmt', '-w', '-s', gof.name])
 
 
+def gen_rowcolumn_diacritics() -> None:
+    # codes of all row/column diacritics
+    codes = []
+    with open("./rowcolumn-diacritics.txt") as file:
+        for line in file.readlines():
+            if line.startswith('#'):
+                continue
+            code = int(line.split(";")[0], 16)
+            codes.append(code)
+
+    go_file = 'tools/utils/images/rowcolumn_diacritics.go'
+    with create_header('kitty/rowcolumn-diacritics.c') as p, create_header(go_file, include_data_types=False) as g:
+        p('#include "unicode-data.h"')
+        p('int diacritic_to_num(char_type code) {')
+        p('\tswitch (code) {')
+        g('package images')
+        g(f'var NumberToDiacritic = [{len(codes)}]rune''{')
+        g(', '.join(f'0x{x:x}' for x in codes) + ',')
+        g('}')
+
+        range_start_num = 1
+        range_start = 0
+        range_end = 0
+
+        def print_range() -> None:
+            if range_start >= range_end:
+                return
+            write_case((range_start, range_end), p)
+            p('\t\treturn code - ' + hex(range_start) + ' + ' +
+              str(range_start_num) + ';')
+
+        for code in codes:
+            if range_end == code:
+                range_end += 1
+            else:
+                print_range()
+                range_start_num += range_end - range_start
+                range_start = code
+                range_end = code + 1
+        print_range()
+
+        p('\t}')
+        p('\treturn 0;')
+        p('}')
+    subprocess.check_call(['gofmt', '-w', '-s', go_file])
+
+
 parse_ucd()
 parse_prop_list()
 parse_emoji()
@@ -619,3 +592,4 @@ gen_ucd()
 gen_wcwidth()
 gen_emoji()
 gen_names()
+gen_rowcolumn_diacritics()

glad/generate.py

@@ -9,8 +9,8 @@ import shutil
 import subprocess
 
 cmdline = (
-    'glad --out-path {dest} --api gl:core=3.3 '
+    'glad --out-path {dest} --api gl:core=3.1 '
-    ' --extensions GL_ARB_texture_storage,GL_ARB_copy_image,GL_ARB_multisample,GL_ARB_robustness,GL_KHR_debug '
+    ' --extensions GL_ARB_texture_storage,GL_ARB_copy_image,GL_ARB_multisample,GL_ARB_robustness,GL_ARB_instanced_arrays,GL_KHR_debug '
     'c --header-only --debug'
 )
 

glfw/cocoa_init.m

@@ -1014,7 +1014,7 @@ static pthread_t main_thread;
 static NSLock *tick_lock = NULL;
 
 
-void _glfwDispatchTickCallback() {
+void _glfwDispatchTickCallback(void) {
     if (tick_lock && tick_callback) {
         [tick_lock lock];
         while(tick_callback_requested) {
@@ -1026,7 +1026,7 @@ void _glfwDispatchTickCallback() {
 }
 
 static void
-request_tick_callback() {
+request_tick_callback(void) {
     if (!tick_callback_requested) {
         tick_callback_requested = true;
         [NSApp performSelectorOnMainThread:@selector(tick_callback) withObject:nil waitUntilDone:NO];

glfw/cocoa_monitor.m

@@ -323,7 +323,7 @@ static double getFallbackRefreshRate(CGDirectDisplayID displayID)
 //////                       GLFW internal API                      //////
 //////////////////////////////////////////////////////////////////////////
 
-void _glfwClearDisplayLinks() {
+void _glfwClearDisplayLinks(void) {
     for (size_t i = 0; i < _glfw.ns.displayLinks.count; i++) {
         if (_glfw.ns.displayLinks.entries[i].displayLink) {
             CVDisplayLinkStop(_glfw.ns.displayLinks.entries[i].displayLink);

glfw/cocoa_window.m

@@ -1010,6 +1010,18 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
     _glfwInputCursorEnter(window, true);
 }
 
+- (void)viewDidChangeEffectiveAppearance
+{
+    static int appearance = 0;
+    if (_glfw.callbacks.system_color_theme_change) {
+        int new_appearance = glfwGetCurrentSystemColorTheme();
+        if (new_appearance != appearance) {
+            appearance = new_appearance;
+            _glfw.callbacks.system_color_theme_change(appearance);
+        }
+    }
+}
+
 - (void)viewDidChangeBackingProperties
 {
     if (!window) return;
@@ -1454,15 +1466,11 @@ is_ascii_control_char(char x) {
 }
 
 void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
-    [w->ns.view updateIMEStateFor: ev->type focused:(bool)ev->focused left:(CGFloat)ev->cursor.left top:(CGFloat)ev->cursor.top cellWidth:(CGFloat)ev->cursor.width cellHeight:(CGFloat)ev->cursor.height];
+    [w->ns.view updateIMEStateFor: ev->type focused:(bool)ev->focused];
 }
 
 - (void)updateIMEStateFor:(GLFWIMEUpdateType)which
                   focused:(bool)focused
-                     left:(CGFloat)left
-                      top:(CGFloat)top
-                cellWidth:(CGFloat)cellWidth
-               cellHeight:(CGFloat)cellHeight
 {
     if (which == GLFW_IME_UPDATE_FOCUS && !focused && [self hasMarkedText] && window) {
         [input_context discardMarkedText];
@@ -1472,16 +1480,7 @@ void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
         _glfw.ns.text[0] = 0;
     }
     if (which != GLFW_IME_UPDATE_CURSOR_POSITION) return;
-    left /= window->ns.xscale;
+
-    top /= window->ns.yscale;
-    cellWidth /= window->ns.xscale;
-    cellHeight /= window->ns.yscale;
-    debug_key("updateIMEPosition: left=%f, top=%f, width=%f, height=%f\n", left, top, cellWidth, cellHeight);
-    const NSRect frame = [window->ns.view frame];
-    const NSRect rectInView = NSMakeRect(left,
-                                         frame.size.height - top - cellHeight,
-                                         cellWidth, cellHeight);
-    markedRect = [window->ns.object convertRectToScreen: rectInView];
     if (_glfwPlatformWindowFocused(window)) [[window->ns.view inputContext] invalidateCharacterCoordinates];
 }
 
@@ -1507,6 +1506,21 @@ void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
                          actualRange:(NSRangePointer)actualRange
 {
     (void)range; (void)actualRange;
+    if (_glfw.callbacks.get_ime_cursor_position) {
+        GLFWIMEUpdateEvent ev = { .type = GLFW_IME_UPDATE_CURSOR_POSITION };
+        if (_glfw.callbacks.get_ime_cursor_position((GLFWwindow*)window, &ev)) {
+            const CGFloat left = (CGFloat)ev.cursor.left / window->ns.xscale;
+            const CGFloat top = (CGFloat)ev.cursor.top / window->ns.yscale;
+            const CGFloat cellWidth = (CGFloat)ev.cursor.width / window->ns.xscale;
+            const CGFloat cellHeight = (CGFloat)ev.cursor.height / window->ns.yscale;
+            debug_key("updateIMEPosition: left=%f, top=%f, width=%f, height=%f\n", left, top, cellWidth, cellHeight);
+            const NSRect frame = [window->ns.view frame];
+            const NSRect rectInView = NSMakeRect(left,
+                                                 frame.size.height - top - cellHeight,
+                                                 cellWidth, cellHeight);
+            markedRect = [window->ns.object convertRectToScreen: rectInView];
+        }
+    }
     return markedRect;
 }
 
@@ -1581,29 +1595,38 @@ void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
 // Support services receiving "public.utf8-plain-text" and "NSStringPboardType"
 - (id)validRequestorForSendType:(NSString *)sendType returnType:(NSString *)returnType
 {
-    if ([sendType isEqual:NSPasteboardTypeString] || [sendType isEqual:@"NSStringPboardType"]) {
+    if (
-        return self;
+        (!sendType || [sendType isEqual:NSPasteboardTypeString] || [sendType isEqual:@"NSStringPboardType"]) &&
+        (!returnType || [returnType isEqual:NSPasteboardTypeString] || [returnType isEqual:@"NSStringPboardType"])
+    ) {
+        if (_glfw.callbacks.has_current_selection && _glfw.callbacks.has_current_selection()) return self;
     }
-    return nil;
+    return [super validRequestorForSendType:sendType returnType:returnType];
 }
 
 // Selected text as input to be sent to Services
+// For example, after selecting an absolute path, open the global menu bar kitty->Services and click `Show in Finder`.
 - (BOOL)writeSelectionToPasteboard:(NSPasteboard *)pboard types:(NSArray *)types
 {
-    NSString *text = [self accessibilitySelectedText];
+    if (!_glfw.callbacks.get_current_selection) return NO;
-    if (text && [text length] > 0) {
+    char *text = _glfw.callbacks.get_current_selection();
+    if (!text) return NO;
+    BOOL ans = NO;
+    if (text[0]) {
         if ([types containsObject:NSPasteboardTypeString] == YES) {
             [pboard declareTypes:@[NSPasteboardTypeString] owner:self];
-            return [pboard setString:text forType:NSPasteboardTypeString];
+            ans = [pboard setString:@(text) forType:NSPasteboardTypeString];
         } else if ([types containsObject:@"NSStringPboardType"] == YES) {
             [pboard declareTypes:@[@"NSStringPboardType"] owner:self];
-            return [pboard setString:text forType:NSPasteboardTypeString];
+            ans = [pboard setString:@(text) forType:@"NSStringPboardType"];
         }
+        free(text);
     }
-    return NO;
+    return ans;
 }
 
 // Service output to be handled
+// For example, open System Settings->Keyboard->Keyboard Shortcuts->Services->Text, enable `Convert Text to Full Width`, select some text and execute the service.
 - (BOOL)readSelectionFromPasteboard:(NSPasteboard *)pboard
 {
     NSString* text = nil;
@@ -1616,17 +1639,17 @@ void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
         return NO;
     }
     if (text && [text length] > 0) {
-        // Terminal.app inserts the output, do the same
+        // The service wants us to replace the selection, but we can't replace anything but insert text.
         const char *utf8 = polymorphic_string_as_utf8(text);
+        debug_key("Sending text received in readSelectionFromPasteboard as key event\n");
+        GLFWkeyevent glfw_keyevent = {.text=utf8, .ime_state=GLFW_IME_COMMIT_TEXT};
+        _glfwInputKeyboard(window, &glfw_keyevent);
+        // Restore pre-edit text after inserting the received text
         if ([self hasMarkedText]) {
-            [self unmarkText];
+            glfw_keyevent.text = [[markedText string] UTF8String];
-            debug_key("Clearing pre-edit because insertText called from readSelectionFromPasteboard\n");
+            glfw_keyevent.ime_state = GLFW_IME_PREEDIT_CHANGED;
-            GLFWkeyevent glfw_keyevent = {.ime_state = GLFW_IME_PREEDIT_CHANGED};
             _glfwInputKeyboard(window, &glfw_keyevent);
         }
-        debug_key("Sending text received in readSelectionFromPasteboard as key event\n");
-        GLFWkeyevent glfw_keyevent = {.text=utf8};
-        _glfwInputKeyboard(window, &glfw_keyevent);
         return YES;
     }
     return NO;
@@ -1708,6 +1731,18 @@ void _glfwPlatformUpdateIMEState(_GLFWwindow *w, const GLFWIMEUpdateEvent *ev) {
     if (glfw_window && !glfw_window->decorated && glfw_window->ns.view) [self makeFirstResponder:glfw_window->ns.view];
 }
 
+- (void)zoom:(id)sender
+{
+    if (![self isZoomed]) {
+        const NSSize original = [self resizeIncrements];
+        [self setResizeIncrements:NSMakeSize(1.0, 1.0)];
+        [super zoom:sender];
+        [self setResizeIncrements:original];
+    } else {
+        [super zoom:sender];
+    }
+}
+
 @end
 // }}}
 
@@ -1861,8 +1896,9 @@ int _glfwPlatformCreateWindow(_GLFWwindow* window,
 
     if (window->monitor)
     {
-        _glfwPlatformShowWindow(window);
+        // Do not show the window here until after setting the window size, maximized state, and full screen
-        _glfwPlatformFocusWindow(window);
+        // _glfwPlatformShowWindow(window);
+        // _glfwPlatformFocusWindow(window);
         acquireMonitor(window);
     }
 
@@ -2054,8 +2090,9 @@ void _glfwPlatformRestoreWindow(_GLFWwindow* window)
 
 void _glfwPlatformMaximizeWindow(_GLFWwindow* window)
 {
-    if (![window->ns.object isZoomed])
+    if (![window->ns.object isZoomed]) {
         [window->ns.object zoom:nil];
+    }
 }
 
 void _glfwPlatformShowWindow(_GLFWwindow* window)
@@ -2561,6 +2598,19 @@ bool _glfwPlatformToggleFullscreen(_GLFWwindow* w, unsigned int flags) {
         if (in_fullscreen) made_fullscreen = false;
         [window toggleFullScreen: nil];
     }
+    // Update window button visibility
+    if (w->ns.titlebar_hidden) {
+        // The hidden buttons might be automatically reset to be visible after going full screen
+        // to show up in the auto-hide title bar, so they need to be set back to hidden.
+        BOOL button_hidden = YES;
+        // When title bar is configured to be hidden, it should be shown with buttons (auto-hide) after going to full screen.
+        if (!traditional) {
+            button_hidden = (BOOL) !made_fullscreen;
+        }
+        [[window standardWindowButton: NSWindowCloseButton] setHidden:button_hidden];
+        [[window standardWindowButton: NSWindowMiniaturizeButton] setHidden:button_hidden];
+        [[window standardWindowButton: NSWindowZoomButton] setHidden:button_hidden];
+    }
     return made_fullscreen;
 }
 
@@ -2919,6 +2969,19 @@ GLFWAPI void glfwCocoaRequestRenderFrame(GLFWwindow *w, GLFWcocoarenderframefun
     requestRenderFrame((_GLFWwindow*)w, callback);
 }
 
+GLFWAPI int glfwGetCurrentSystemColorTheme(void) {
+    int theme_type = 0;
+    NSAppearance *changedAppearance = NSApp.effectiveAppearance;
+    NSAppearanceName newAppearance = [changedAppearance bestMatchFromAppearancesWithNames:@[NSAppearanceNameAqua, NSAppearanceNameDarkAqua]];
+    if([newAppearance isEqualToString:NSAppearanceNameDarkAqua]){
+        theme_type = 1;
+    } else {
+        theme_type = 2;
+    }
+    return theme_type;
+}
+
+
 GLFWAPI uint32_t
 glfwGetCocoaKeyEquivalent(uint32_t glfw_key, int glfw_mods, int *cocoa_mods) {
     *cocoa_mods = 0;

glfw/dbus_glfw.c

@@ -174,7 +174,7 @@ glfw_dbus_dispatch(DBusConnection *conn) {
 }
 
 void
-glfw_dbus_session_bus_dispatch() {
+glfw_dbus_session_bus_dispatch(void) {
     if (session_bus) glfw_dbus_dispatch(session_bus);
 }
 
@@ -344,7 +344,7 @@ glfw_dbus_connect_to_session_bus(void) {
 }
 
 DBusConnection *
-glfw_dbus_session_bus() {
+glfw_dbus_session_bus(void) {
     if (!session_bus) glfw_dbus_connect_to_session_bus();
     return session_bus;
 }

glfw/glfw.py

@@ -41,6 +41,7 @@ class Env:
     library_paths: Dict[str, List[str]] = {}
     ldpaths: List[str] = []
     ccver: Tuple[int, int]
+    vcs_rev: str = ''
 
     # glfw stuff
     all_headers: List[str] = []
@@ -52,11 +53,13 @@ class Env:
 
     def __init__(
         self, cc: List[str] = [], cppflags: List[str] = [], cflags: List[str] = [], ldflags: List[str] = [],
-        library_paths: Dict[str, List[str]] = {}, ldpaths: Optional[List[str]] = None, ccver: Tuple[int, int] = (0, 0)
+        library_paths: Dict[str, List[str]] = {}, ldpaths: Optional[List[str]] = None, ccver: Tuple[int, int] = (0, 0),
+        vcs_rev: str = ''
     ):
         self.cc, self.cppflags, self.cflags, self.ldflags, self.library_paths = cc, cppflags, cflags, ldflags, library_paths
         self.ldpaths = ldpaths or []
         self.ccver = ccver
+        self.vcs_rev = vcs_rev
 
     def copy(self) -> 'Env':
         ans = Env(self.cc, list(self.cppflags), list(self.cflags), list(self.ldflags), dict(self.library_paths), list(self.ldpaths), self.ccver)
@@ -66,6 +69,7 @@ class Env:
         ans.wayland_scanner = self.wayland_scanner
         ans.wayland_scanner_code = self.wayland_scanner_code
         ans.wayland_protocols = self.wayland_protocols
+        ans.vcs_rev = self.vcs_rev
         return ans
 
 

glfw/glfw3.h

@@ -1368,6 +1368,22 @@ typedef void (* GLFWwindowclosefun)(GLFWwindow*);
  */
 typedef void (* GLFWapplicationclosefun)(int);
 
+/*! @brief The function pointer type for system color theme change callbacks.
+ *
+ *  This is the function pointer type for system color theme changes.
+ *  @code
+ *  void function_name(int theme_type)
+ *  @endcode
+ *
+ *  @param[in] theme_type 0 for unknown, 1 for dark and 2 for light
+ *
+ *  @sa @ref glfwSetSystemColorThemeChangeCallback
+ *
+ *  @ingroup window
+ */
+typedef void (* GLFWsystemcolorthemechangefun)(int);
+
+
 /*! @brief The function pointer type for window content refresh callbacks.
  *
  *  This is the function pointer type for window content refresh callbacks.
@@ -1719,6 +1735,7 @@ typedef void (* GLFWtickcallback)(void*);
 typedef void (* GLFWactivationcallback)(GLFWwindow *window, const char *token, void *data);
 typedef bool (* GLFWdrawtextfun)(GLFWwindow *window, const char *text, uint32_t fg, uint32_t bg, uint8_t *output_buf, size_t width, size_t height, float x_offset, float y_offset, size_t right_margin);
 typedef char* (* GLFWcurrentselectionfun)(void);
+typedef bool (* GLFWhascurrentselectionfun)(void);
 typedef void (* GLFWclipboarddatafreefun)(void* data);
 typedef struct GLFWDataChunk {
     const char *data;
@@ -1731,6 +1748,7 @@ typedef enum {
 } GLFWClipboardType;
 typedef GLFWDataChunk (* GLFWclipboarditerfun)(const char *mime_type, void *iter, GLFWClipboardType ctype);
 typedef bool (* GLFWclipboardwritedatafun)(void *object, const char *data, size_t sz);
+typedef bool (* GLFWimecursorpositionfun)(GLFWwindow *window, GLFWIMEUpdateEvent *ev);
 
 /*! @brief Video mode type.
  *
@@ -1889,6 +1907,8 @@ GLFWAPI void glfwUpdateTimer(unsigned long long timer_id, monotonic_t interval,
 GLFWAPI void glfwRemoveTimer(unsigned long long);
 GLFWAPI GLFWdrawtextfun glfwSetDrawTextFunction(GLFWdrawtextfun function);
 GLFWAPI GLFWcurrentselectionfun glfwSetCurrentSelectionCallback(GLFWcurrentselectionfun callback);
+GLFWAPI GLFWhascurrentselectionfun glfwSetHasCurrentSelectionCallback(GLFWhascurrentselectionfun callback);
+GLFWAPI GLFWimecursorpositionfun glfwSetIMECursorPositionCallback(GLFWimecursorpositionfun callback);
 
 /*! @brief Terminates the GLFW library.
  *
@@ -3902,6 +3922,8 @@ GLFWAPI GLFWwindowsizefun glfwSetWindowSizeCallback(GLFWwindow* window, GLFWwind
  */
 GLFWAPI GLFWwindowclosefun glfwSetWindowCloseCallback(GLFWwindow* window, GLFWwindowclosefun callback);
 GLFWAPI GLFWapplicationclosefun glfwSetApplicationCloseCallback(GLFWapplicationclosefun callback);
+GLFWAPI GLFWsystemcolorthemechangefun glfwSetSystemColorThemeChangeCallback(GLFWsystemcolorthemechangefun callback);
+GLFWAPI int glfwGetCurrentSystemColorTheme(void);
 
 /*! @brief Sets the refresh callback for the specified window.
  *

glfw/ibus_glfw.c

@@ -283,29 +283,35 @@ static const char*
 get_ibus_address_file_name(void) {
     const char *addr;
     static char ans[PATH_MAX];
+    static char display[64] = {0};
     addr = getenv("IBUS_ADDRESS");
     int offset = 0;
     if (addr && addr[0]) {
         memcpy(ans, addr, GLFW_MIN(strlen(addr), sizeof(ans)));
         return ans;
     }
-
+    const char* disp_num = NULL;
-    const char *de = getenv("DISPLAY");
+    const char *host = "unix";
-    if (!de || !de[0]) de = ":0.0";
+    // See https://github.com/ibus/ibus/commit/8ce25208c3f4adfd290a032c6aa739d2b7580eb1 for why we need this dance.
-    char *display = _glfw_strdup(de);
+    const char *de = getenv("WAYLAND_DISPLAY");
-    const char *host = display;
+    if (de) {
-    char *disp_num  = strrchr(display, ':');
+        disp_num = de;
-    char *screen_num = strrchr(display, '.');
+    } else {
-
+        const char *de = getenv("DISPLAY");
-    if (!disp_num) {
+        if (!de || !de[0]) de = ":0.0";
-        _glfwInputError(GLFW_PLATFORM_ERROR, "Could not get IBUS address file name as DISPLAY env var has no colon");
+        strncpy(display, de, sizeof(display) - 1);
-        free(display);
+        char *dnum = strrchr(display, ':');
-        return NULL;
+        if (!dnum) {
+            _glfwInputError(GLFW_PLATFORM_ERROR, "Could not get IBUS address file name as DISPLAY env var has no colon");
+            return NULL;
+        }
+        char *screen_num = strrchr(display, '.');
+        *dnum = 0;
+        dnum++;
+        if (screen_num) *screen_num = 0;
+        if (*display) host = display;
+        disp_num = dnum;
     }
-    *disp_num = 0;
-    disp_num++;
-    if (screen_num) *screen_num = 0;
-    if (!*host) host = "unix";
 
     memset(ans, 0, sizeof(ans));
     const char *conf_env = getenv("XDG_CONFIG_HOME");
@@ -315,7 +321,6 @@ get_ibus_address_file_name(void) {
         conf_env = getenv("HOME");
         if (!conf_env || !conf_env[0]) {
             _glfwInputError(GLFW_PLATFORM_ERROR, "Could not get IBUS address file name as no HOME env var is set");
-            free(display);
             return NULL;
         }
         offset = snprintf(ans, sizeof(ans), "%s/.config", conf_env);
@@ -323,7 +328,6 @@ get_ibus_address_file_name(void) {
     char *key = dbus_get_local_machine_id();
     snprintf(ans + offset, sizeof(ans) - offset, "/ibus/bus/%s-%s-%s", key, host, disp_num);
     dbus_free(key);
-    free(display);
     return ans;
 }
 

glfw/init.c

@@ -382,6 +382,14 @@ GLFWAPI GLFWapplicationclosefun glfwSetApplicationCloseCallback(GLFWapplicationc
     return cbfun;
 }
 
+GLFWAPI GLFWapplicationclosefun glfwSetSystemColorThemeChangeCallback(GLFWsystemcolorthemechangefun cbfun)
+{
+    _GLFW_REQUIRE_INIT_OR_RETURN(NULL);
+    _GLFW_SWAP_POINTERS(_glfw.callbacks.system_color_theme_change, cbfun);
+    return cbfun;
+}
+
+
 GLFWAPI GLFWdrawtextfun glfwSetDrawTextFunction(GLFWdrawtextfun cbfun)
 {
     _GLFW_REQUIRE_INIT_OR_RETURN(NULL);
@@ -395,3 +403,17 @@ GLFWAPI GLFWcurrentselectionfun glfwSetCurrentSelectionCallback(GLFWcurrentselec
     _GLFW_SWAP_POINTERS(_glfw.callbacks.get_current_selection, cbfun);
     return cbfun;
 }
+
+GLFWAPI GLFWhascurrentselectionfun glfwSetHasCurrentSelectionCallback(GLFWhascurrentselectionfun cbfun)
+{
+    _GLFW_REQUIRE_INIT_OR_RETURN(NULL);
+    _GLFW_SWAP_POINTERS(_glfw.callbacks.has_current_selection, cbfun);
+    return cbfun;
+}
+
+GLFWAPI GLFWimecursorpositionfun glfwSetIMECursorPositionCallback(GLFWimecursorpositionfun cbfun)
+{
+    _GLFW_REQUIRE_INIT_OR_RETURN(NULL);
+    _GLFW_SWAP_POINTERS(_glfw.callbacks.get_ime_cursor_position, cbfun);
+    return cbfun;
+}

glfw/internal.h

@@ -632,11 +632,13 @@ struct _GLFWlibrary
         GLFWmonitorfun  monitor;
         GLFWjoystickfun joystick;
         GLFWapplicationclosefun application_close;
+        GLFWsystemcolorthemechangefun system_color_theme_change;
         GLFWdrawtextfun draw_text;
         GLFWcurrentselectionfun get_current_selection;
+        GLFWhascurrentselectionfun has_current_selection;
+        GLFWimecursorpositionfun get_ime_cursor_position;
     } callbacks;
 
-
     // This is defined in the window API's platform.h
     _GLFW_PLATFORM_LIBRARY_WINDOW_STATE;
     // This is defined in the context API's context.h

glfw/linux_desktop_settings.c

@@ -24,6 +24,11 @@ static uint32_t appearance = 0;
 static bool is_gnome = false;
 static bool cursor_theme_changed = false;
 
+int
+glfw_current_system_color_theme(void) {
+    return appearance;
+}
+
 #define HANDLER(name) static void name(DBusMessage *msg, const char* errmsg, void *data) { \
     (void)data; \
     if (errmsg) { \
@@ -155,6 +160,9 @@ on_color_scheme_change(DBusMessage *message) {
                 if (val > 2) val = 0;
                 if (val != appearance) {
                     appearance = val;
+                    if (_glfw.callbacks.system_color_theme_change) {
+                        _glfw.callbacks.system_color_theme_change(appearance);
+                    }
                 }
             }
             break;

glfw/linux_desktop_settings.h

@@ -12,3 +12,4 @@
 
 void glfw_initialize_desktop_settings(void);
 void glfw_current_cursor_theme(const char **theme, int *size);
+int glfw_current_system_color_theme(void);

glfw/wl_init.c

@@ -789,6 +789,10 @@ glfwWaylandCheckForServerSideDecorations(void) {
     return has_ssd ? "YES" : "NO";
 }
 
+GLFWAPI int glfwGetCurrentSystemColorTheme(void) {
+    return glfw_current_system_color_theme();
+}
+
 //////////////////////////////////////////////////////////////////////////
 //////                       GLFW platform API                      //////
 //////////////////////////////////////////////////////////////////////////

glfw/wl_window.c

@@ -1952,12 +1952,12 @@ primary_selection_copy_callback_done(void *data, struct wl_callback *callback, u
     wl_callback_destroy(callback);
 }
 
-void _glfwSetupWaylandDataDevice() {
+void _glfwSetupWaylandDataDevice(void) {
     _glfw.wl.dataDevice = wl_data_device_manager_get_data_device(_glfw.wl.dataDeviceManager, _glfw.wl.seat);
     if (_glfw.wl.dataDevice) wl_data_device_add_listener(_glfw.wl.dataDevice, &data_device_listener, NULL);
 }
 
-void _glfwSetupWaylandPrimarySelectionDevice() {
+void _glfwSetupWaylandPrimarySelectionDevice(void) {
     _glfw.wl.primarySelectionDevice = zwp_primary_selection_device_manager_v1_get_device(_glfw.wl.primarySelectionDeviceManager, _glfw.wl.seat);
     if (_glfw.wl.primarySelectionDevice) zwp_primary_selection_device_v1_add_listener(_glfw.wl.primarySelectionDevice, &primary_selection_device_listener, NULL);
 }

glfw/x11_init.c

@@ -614,6 +614,10 @@ Cursor _glfwCreateCursorX11(const GLFWimage* image, int xhot, int yhot)
 //////                       GLFW platform API                      //////
 //////////////////////////////////////////////////////////////////////////
 
+GLFWAPI int glfwGetCurrentSystemColorTheme(void) {
+    return 0;
+}
+
 int _glfwPlatformInit(void)
 {
     XInitThreads();

glfw/x11_window.c

@@ -719,6 +719,7 @@ static bool createNativeWindow(_GLFWwindow* window,
 static size_t
 get_clipboard_data(const _GLFWClipboardData *cd, const char *mime, char **data) {
     *data = NULL;
+    if (cd->get_data == NULL) { return 0; }
     GLFWDataChunk chunk = cd->get_data(mime, NULL, cd->ctype);
     char *buf = NULL;
     size_t sz = 0, cap = 0;

go.mod

@@ -1,18 +1,30 @@
 module kitty
 
-go 1.19
+go 1.20
 
 require (
 	github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924
+	github.com/alecthomas/chroma/v2 v2.7.0
+	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/disintegration/imaging v1.6.2
-	github.com/google/go-cmp v0.5.8
+	github.com/dlclark/regexp2 v1.9.0
+	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/jamesruan/go-rfc1924 v0.0.0-20170108144916-2767ca7c638f
-	github.com/seancfoley/ipaddress-go v1.2.1
+	github.com/seancfoley/ipaddress-go v1.5.4
-	golang.org/x/crypto v0.0.0-20220924013350-4ba4fb4dd9e7
+	github.com/shirou/gopsutil/v3 v3.23.3
-	golang.org/x/exp v0.0.0-20220921164117-439092de6870
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
-	golang.org/x/image v0.2.0
+	golang.org/x/image v0.7.0
-	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
+	golang.org/x/sys v0.7.0
 )
 
-require github.com/seancfoley/bintree v1.1.0 // indirect
+require (
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect
+	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
+	github.com/seancfoley/bintree v1.2.1 // indirect
+	github.com/shoenig/go-m1cpu v0.1.5 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+)

go.sum

@@ -1,47 +1,104 @@
 github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924 h1:DG4UyTVIujioxwJc8Zj8Nabz1L1wTgQ/xNBSQDfdP3I=
 github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924/go.mod h1:+NaH2gLeY6RPBPPQf4aRotPPStg+eXc8f9ZaE4vRfD4=
+github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink=
+github.com/alecthomas/chroma/v2 v2.7.0 h1:hm1rY6c/Ob4eGclpQ7X/A3yhqBOZNUTk9q+yhyLIViI=
+github.com/alecthomas/chroma/v2 v2.7.0/go.mod h1:yrkMI9807G1ROx13fhe1v6PN2DDeaR73L3d+1nmYQtw=
+github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
+github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc=
+github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/dlclark/regexp2 v1.9.0 h1:pTK/l/3qYIKaRXuHnEnIf7Y5NxfRPfpb7dis6/gdlVI=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/dlclark/regexp2 v1.9.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/jamesruan/go-rfc1924 v0.0.0-20170108144916-2767ca7c638f h1:Ko4+g6K16vSyUrtd/pPXuQnWsiHe5BYptEtTxfwYwCc=
 github.com/jamesruan/go-rfc1924 v0.0.0-20170108144916-2767ca7c638f/go.mod h1:eHzfhOKbTGJEGPSdMHzU6jft192tHHt2Bu2vIZArvC0=
-github.com/seancfoley/bintree v1.1.0 h1:6J0rj9hLNLIcWSsfYdZ4ZHkMHokaK/PHkak8qyBO/mc=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
-github.com/seancfoley/bintree v1.1.0/go.mod h1:CtE6qO6/n9H3V2CAGEC0lpaYr6/OijhNaMG/dt7P70c=
+github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik=
-github.com/seancfoley/ipaddress-go v1.2.1 h1:yEZxnyC6NQEDDPflyQm4KkWozffx1vHWsx+knKBr/n0=
+github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE=
-github.com/seancfoley/ipaddress-go v1.2.1/go.mod h1:/UEVHyrBg1ASVap2ffdY2cq5UMYIX9f3QW3uWSVqpbo=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig=
+github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/seancfoley/bintree v1.2.1 h1:Z/iNjRKkXnn0CTW7jDQYtjW5fz2GH1yWvOTJ4MrMvdo=
+github.com/seancfoley/bintree v1.2.1/go.mod h1:hIUabL8OFYyFVTQ6azeajbopogQc2l5C/hiXMcemWNU=
+github.com/seancfoley/ipaddress-go v1.5.4 h1:ZdjewWC1J2y5ruQjWHwK6rA1tInWB6mz1ftz6uTm+Uw=
+github.com/seancfoley/ipaddress-go v1.5.4/go.mod h1:fpvVPC+Jso+YEhNcNiww8HQmBgKP8T4T6BTp1SLxxIo=
+github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE=
+github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU=
+github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ=
+github.com/shoenig/go-m1cpu v0.1.5 h1:LF57Z/Fpb/WdGLjt2HZilNnmZOxg/q2bSKTQhgbrLrQ=
+github.com/shoenig/go-m1cpu v0.1.5/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ=
+github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c=
+github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
+github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
+github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
+github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
+github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220924013350-4ba4fb4dd9e7 h1:WJywXQVIb56P2kAvXeMGTIgQ1ZHQxR60+F9dLsodECc=
+golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
-golang.org/x/crypto v0.0.0-20220924013350-4ba4fb4dd9e7/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
-golang.org/x/exp v0.0.0-20220921164117-439092de6870 h1:j8b6j9gzSigH28O5SjSpQSSh9lFd6f5D/q0aHjNTulc=
-golang.org/x/exp v0.0.0-20220921164117-439092de6870/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/image v0.2.0 h1:/DcQ0w3VHKCC5p0/P2B0JpAZ9Z++V2KOo2fyU89CXBQ=
+golang.org/x/image v0.7.0 h1:gzS29xtG1J5ybQlv0PuyfE3nmc6R4qB73m6LUUmvFuw=
-golang.org/x/image v0.2.0/go.mod h1:la7oBXb9w3YFjBqaAwtynVioc1ZvOnNteUNrifGNmAI=
+golang.org/x/image v0.7.0/go.mod h1:nd/q4ef1AKKYl/4kft7g+6UyGbdiqWqTP1ZAbRoV7Rg=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

kittens/ask/choices.go

@@ -0,0 +1,433 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package ask
+
+import (
+	"fmt"
+	"io"
+	"kitty/tools/cli/markup"
+	"kitty/tools/tui/loop"
+	"kitty/tools/utils"
+	"kitty/tools/utils/style"
+	"kitty/tools/wcswidth"
+	"os"
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+var _ = fmt.Print
+
+type Choice struct {
+	text          string
+	idx           int
+	color, letter string
+}
+
+func (self Choice) prefix() string {
+	return string([]rune(self.text)[:self.idx])
+}
+
+func (self Choice) display_letter() string {
+	return string([]rune(self.text)[self.idx])
+}
+
+func (self Choice) suffix() string {
+	return string([]rune(self.text)[self.idx+1:])
+}
+
+type Range struct {
+	start, end, y int
+}
+
+func (self *Range) has_point(x, y int) bool {
+	return y == self.y && self.start <= x && x <= self.end
+}
+
+func truncate_at_space(text string, width int) (string, string) {
+	truncated, p := wcswidth.TruncateToVisualLengthWithWidth(text, width)
+	if len(truncated) == len(text) {
+		return text, ""
+	}
+	i := strings.LastIndexByte(truncated, ' ')
+	if i > 0 && p-i < 12 {
+		p = i + 1
+	}
+	return text[:p], text[p:]
+}
+
+func extra_for(width, screen_width int) int {
+	return utils.Max(0, screen_width-width)/2 + 1
+}
+
+func GetChoices(o *Options) (response string, err error) {
+	response = ""
+	lp, err := loop.New()
+	if err != nil {
+		return "", err
+	}
+	lp.MouseTrackingMode(loop.BUTTONS_ONLY_MOUSE_TRACKING)
+
+	prefix_style_pat := regexp.MustCompile("^(?:\x1b\\[[^m]*?m)+")
+	choice_order := make([]Choice, 0, len(o.Choices))
+	clickable_ranges := make(map[string][]Range, 16)
+	allowed := utils.NewSet[string](utils.Max(2, len(o.Choices)))
+	response_on_accept := o.Default
+	switch o.Type {
+	case "yesno":
+		allowed.AddItems("y", "n")
+		if !allowed.Has(response_on_accept) {
+			response_on_accept = "y"
+		}
+	case "choices":
+		first_choice := ""
+		for i, x := range o.Choices {
+			letter, text, _ := strings.Cut(x, ":")
+			color := ""
+			if strings.Contains(letter, ";") {
+				letter, color, _ = strings.Cut(letter, ";")
+			}
+			letter = strings.ToLower(letter)
+			idx := strings.Index(strings.ToLower(text), letter)
+			idx = len([]rune(strings.ToLower(text)[:idx]))
+			allowed.Add(letter)
+			c := Choice{text: text, idx: idx, color: color, letter: letter}
+			choice_order = append(choice_order, c)
+			if i == 0 {
+				first_choice = letter
+			}
+		}
+		if !allowed.Has(response_on_accept) {
+			response_on_accept = first_choice
+		}
+	}
+	message := o.Message
+	hidden_text_start_pos := -1
+	hidden_text_end_pos := -1
+	hidden_text := ""
+	m := markup.New(true)
+	replacement_text := fmt.Sprintf("Press %s or click to show", m.Green(o.UnhideKey))
+	replacement_range := Range{-1, -1, -1}
+	if message != "" && o.HiddenTextPlaceholder != "" {
+		hidden_text_start_pos = strings.Index(message, o.HiddenTextPlaceholder)
+		if hidden_text_start_pos > -1 {
+			raw, err := io.ReadAll(os.Stdin)
+			if err != nil {
+				return "", fmt.Errorf("Failed to read hidden text from STDIN: %w", err)
+			}
+			hidden_text = strings.TrimRightFunc(utils.UnsafeBytesToString(raw), unicode.IsSpace)
+			hidden_text_end_pos = hidden_text_start_pos + len(replacement_text)
+			suffix := message[hidden_text_start_pos+len(o.HiddenTextPlaceholder):]
+			message = message[:hidden_text_start_pos] + replacement_text + suffix
+		}
+	}
+
+	draw_long_text := func(screen_width int, text string, msg_lines []string) []string {
+		if text == "" {
+			msg_lines = append(msg_lines, "")
+		} else {
+			width := screen_width - 2
+			prefix := prefix_style_pat.FindString(text)
+			for text != "" {
+				var t string
+				t, text = truncate_at_space(text, width)
+				t = strings.TrimSpace(t)
+				msg_lines = append(msg_lines, strings.Repeat(" ", extra_for(wcswidth.Stringwidth(t), width))+m.Bold(prefix+t))
+			}
+		}
+		return msg_lines
+	}
+
+	ctx := style.Context{AllowEscapeCodes: true}
+
+	draw_choice_boxes := func(y, screen_width, screen_height int, choices ...Choice) {
+		clickable_ranges = map[string][]Range{}
+		width := screen_width - 2
+		current_line_length := 0
+		type Item struct{ letter, text string }
+		type Line = []Item
+		var current_line Line
+		lines := make([]Line, 0, 32)
+		sep := "  "
+		sep_sz := len(sep) + 2 // for the borders
+
+		for _, choice := range choices {
+			clickable_ranges[choice.letter] = make([]Range, 0, 4)
+			text := " " + choice.prefix()
+			color := choice.color
+			if choice.color == "" {
+				color = "green"
+			}
+			text += ctx.SprintFunc("fg=" + color)(choice.display_letter())
+			text += choice.suffix() + " "
+			sz := wcswidth.Stringwidth(text)
+			if sz+sep_sz+current_line_length > width {
+				lines = append(lines, current_line)
+				current_line = nil
+				current_line_length = 0
+			}
+			current_line = append(current_line, Item{choice.letter, text})
+			current_line_length += sz + sep_sz
+		}
+		if len(current_line) > 0 {
+			lines = append(lines, current_line)
+		}
+
+		highlight := func(text string) string {
+			return m.Yellow(text)
+		}
+
+		top := func(text string, highlight_frame bool) (ans string) {
+			ans = "╭" + strings.Repeat("─", wcswidth.Stringwidth(text)) + "╮"
+			if highlight_frame {
+				ans = highlight(ans)
+			}
+			return
+		}
+
+		middle := func(text string, highlight_frame bool) (ans string) {
+			f := "│"
+			if highlight_frame {
+				f = highlight(f)
+			}
+			return f + text + f
+		}
+
+		bottom := func(text string, highlight_frame bool) (ans string) {
+			ans = "╰" + strings.Repeat("─", wcswidth.Stringwidth(text)) + "╯"
+			if highlight_frame {
+				ans = highlight(ans)
+			}
+			return
+		}
+
+		print_line := func(add_borders func(string, bool) string, is_last bool, items ...Item) {
+			type Position struct {
+				letter  string
+				x, size int
+			}
+			texts := make([]string, 0, 8)
+			positions := make([]Position, 0, 8)
+			x := 0
+			for _, item := range items {
+				text := item.text
+				positions = append(positions, Position{item.letter, x, wcswidth.Stringwidth(text) + 2})
+				text = add_borders(text, item.letter == response_on_accept)
+				text += sep
+				x += wcswidth.Stringwidth(text)
+				texts = append(texts, text)
+			}
+			line := strings.TrimRightFunc(strings.Join(texts, ""), unicode.IsSpace)
+			offset := extra_for(wcswidth.Stringwidth(line), width)
+			for _, pos := range positions {
+				x = pos.x
+				x += offset
+				clickable_ranges[pos.letter] = append(clickable_ranges[pos.letter], Range{x, x + pos.size - 1, y})
+			}
+			end := "\r\n"
+			if is_last {
+				end = ""
+			}
+			lp.QueueWriteString(strings.Repeat(" ", offset) + line + end)
+			y++
+		}
+		lp.AllowLineWrapping(false)
+		defer func() { lp.AllowLineWrapping(true) }()
+		for i, boxed_line := range lines {
+			print_line(top, false, boxed_line...)
+			print_line(middle, false, boxed_line...)
+			is_last := i == len(lines)-1
+			print_line(bottom, is_last, boxed_line...)
+		}
+	}
+
+	draw_yesno := func(y, screen_width, screen_height int) {
+		yes := m.Green("Y") + "es"
+		no := m.BrightRed("N") + "o"
+		if y+3 <= screen_height {
+			draw_choice_boxes(y, screen_width, screen_height, Choice{"Yes", 0, "green", "y"}, Choice{"No", 0, "red", "n"})
+		} else {
+			sep := strings.Repeat(" ", 3)
+			text := yes + sep + no
+			w := wcswidth.Stringwidth(text)
+			x := extra_for(w, screen_width-2)
+			nx := x + wcswidth.Stringwidth(yes) + len(sep)
+			clickable_ranges = map[string][]Range{
+				"y": {{x, x + wcswidth.Stringwidth(yes) - 1, y}},
+				"n": {{nx, nx + wcswidth.Stringwidth(no) - 1, y}},
+			}
+			lp.QueueWriteString(strings.Repeat(" ", x) + text)
+		}
+	}
+
+	draw_choice := func(y, screen_width, screen_height int) {
+		if y+3 <= screen_height {
+			draw_choice_boxes(y, screen_width, screen_height, choice_order...)
+			return
+		}
+		clickable_ranges = map[string][]Range{}
+		current_line := ""
+		current_ranges := map[string]int{}
+		width := screen_width - 2
+
+		commit_line := func(add_newline bool) {
+			x := extra_for(wcswidth.Stringwidth(current_line), width)
+			text := strings.Repeat(" ", x) + current_line
+			if add_newline {
+				lp.Println(text)
+			} else {
+				lp.QueueWriteString(text)
+			}
+			for letter, sz := range current_ranges {
+				clickable_ranges[letter] = []Range{{x, x + sz - 3, y}}
+				x += sz
+			}
+			current_ranges = map[string]int{}
+			y++
+			current_line = ""
+		}
+		for _, choice := range choice_order {
+			text := choice.prefix()
+			spec := ""
+			if choice.color != "" {
+				spec = "fg=" + choice.color
+			} else {
+				spec = "fg=green"
+			}
+			if choice.letter == response_on_accept {
+				spec += " u=straight"
+			}
+			text += ctx.SprintFunc(spec)(choice.display_letter())
+			text += choice.suffix()
+			text += "  "
+			sz := wcswidth.Stringwidth(text)
+			if sz+wcswidth.Stringwidth(current_line) >= width {
+				commit_line(true)
+			}
+			current_line += text
+			current_ranges[choice.letter] = sz
+		}
+		if current_line != "" {
+			commit_line(false)
+		}
+	}
+
+	draw_screen := func() error {
+		lp.StartAtomicUpdate()
+		defer lp.EndAtomicUpdate()
+		lp.ClearScreen()
+		msg_lines := make([]string, 0, 8)
+		sz, err := lp.ScreenSize()
+		if err != nil {
+			return err
+		}
+		if message != "" {
+			scanner := utils.NewLineScanner(message)
+			for scanner.Scan() {
+				msg_lines = draw_long_text(int(sz.WidthCells), scanner.Text(), msg_lines)
+			}
+		}
+		y := int(sz.HeightCells) - len(msg_lines)
+		y = utils.Max(0, (y/2)-2)
+		lp.QueueWriteString(strings.Repeat("\r\n", y))
+		for _, line := range msg_lines {
+			if replacement_text != "" {
+				idx := strings.Index(line, replacement_text)
+				if idx > -1 {
+					x := wcswidth.Stringwidth(line[:idx])
+					replacement_range = Range{x, x + wcswidth.Stringwidth(replacement_text), y}
+				}
+			}
+			lp.Println(line)
+			y++
+		}
+		if sz.HeightCells > 2 {
+			lp.Println()
+			y++
+		}
+		switch o.Type {
+		case "yesno":
+			draw_yesno(y, int(sz.WidthCells), int(sz.HeightCells))
+		case "choices":
+			draw_choice(y, int(sz.WidthCells), int(sz.HeightCells))
+		}
+		return nil
+	}
+
+	unhide := func() {
+		if hidden_text != "" && message != "" {
+			message = message[:hidden_text_start_pos] + hidden_text + message[hidden_text_end_pos:]
+			hidden_text = ""
+			draw_screen()
+		}
+	}
+
+	lp.OnInitialize = func() (string, error) {
+		lp.SetCursorVisible(false)
+		return "", draw_screen()
+	}
+
+	lp.OnFinalize = func() string {
+		lp.SetCursorVisible(true)
+		return ""
+	}
+
+	lp.OnText = func(text string, from_key_event, in_bracketed_paste bool) error {
+		text = strings.ToLower(text)
+		if allowed.Has(text) {
+			response = text
+			lp.Quit(0)
+		} else if hidden_text != "" && text == o.UnhideKey {
+			unhide()
+		} else if o.Type == "yesno" {
+			lp.Quit(1)
+		}
+		return nil
+	}
+
+	lp.OnKeyEvent = func(ev *loop.KeyEvent) error {
+		if ev.MatchesPressOrRepeat("esc") || ev.MatchesPressOrRepeat("ctrl+c") {
+			ev.Handled = true
+			lp.Quit(1)
+		} else if ev.MatchesPressOrRepeat("enter") {
+			ev.Handled = true
+			response = response_on_accept
+			lp.Quit(0)
+		}
+		return nil
+	}
+
+	lp.OnMouseEvent = func(ev *loop.MouseEvent) error {
+		if ev.Event_type == loop.MOUSE_CLICK {
+			for letter, ranges := range clickable_ranges {
+				for _, r := range ranges {
+					if r.has_point(ev.Cell.X, ev.Cell.Y) {
+						response = letter
+						lp.Quit(0)
+						return nil
+					}
+				}
+			}
+			if hidden_text != "" && replacement_range.has_point(ev.Cell.X, ev.Cell.Y) {
+				unhide()
+			}
+		}
+		return nil
+	}
+
+	lp.OnResize = func(old, news loop.ScreenSize) error {
+		return draw_screen()
+	}
+
+	err = lp.Run()
+	if err != nil {
+		return "", err
+	}
+	ds := lp.DeathSignalName()
+	if ds != "" {
+		fmt.Println("Killed by signal: ", ds)
+		lp.KillIfSignalled()
+		return "", fmt.Errorf("Filled by signal: %s", ds)
+	}
+	return response, nil
+}

kittens/ask/get_line.go

@@ -0,0 +1,92 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package ask
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"kitty/tools/tui/loop"
+	"kitty/tools/tui/readline"
+	"kitty/tools/utils"
+)
+
+var _ = fmt.Print
+
+func get_line(o *Options) (result string, err error) {
+	lp, err := loop.New(loop.NoAlternateScreen, loop.NoRestoreColors)
+	if err != nil {
+		return
+	}
+	cwd, _ := os.Getwd()
+	ropts := readline.RlInit{Prompt: o.Prompt}
+	if o.Name != "" {
+		base := filepath.Join(utils.CacheDir(), "ask")
+		ropts.HistoryPath = filepath.Join(base, o.Name+".history.json")
+		os.MkdirAll(base, 0o755)
+	}
+	rl := readline.New(lp, ropts)
+	if o.Default != "" {
+		rl.SetText(o.Default)
+	}
+	lp.OnInitialize = func() (string, error) {
+		rl.Start()
+		return "", nil
+	}
+	lp.OnFinalize = func() string { rl.End(); return "" }
+
+	lp.OnResumeFromStop = func() error {
+		rl.Start()
+		return nil
+	}
+
+	lp.OnResize = rl.OnResize
+
+	lp.OnKeyEvent = func(event *loop.KeyEvent) error {
+		if event.MatchesPressOrRepeat("ctrl+c") {
+			return fmt.Errorf("Canceled by user")
+		}
+		err := rl.OnKeyEvent(event)
+		if err != nil {
+			if err == io.EOF {
+				lp.Quit(0)
+				return nil
+			}
+			if err == readline.ErrAcceptInput {
+				hi := readline.HistoryItem{Timestamp: time.Now(), Cmd: rl.AllText(), ExitCode: 0, Cwd: cwd}
+				rl.AddHistoryItem(hi)
+				result = rl.AllText()
+				lp.Quit(0)
+				return nil
+			}
+			return err
+		}
+		if event.Handled {
+			rl.Redraw()
+			return nil
+		}
+		return nil
+	}
+
+	lp.OnText = func(text string, from_key_event, in_bracketed_paste bool) error {
+		err := rl.OnText(text, from_key_event, in_bracketed_paste)
+		if err == nil {
+			rl.Redraw()
+		}
+		return err
+	}
+
+	err = lp.Run()
+	rl.Shutdown()
+	if err != nil {
+		return "", err
+	}
+	ds := lp.DeathSignalName()
+	if ds != "" {
+		return "", fmt.Errorf("Killed by signal: %s", ds)
+	}
+	return
+}

kittens/ask/main.go

@@ -0,0 +1,73 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package ask
+
+import (
+	"errors"
+	"fmt"
+
+	"kitty/tools/cli"
+	"kitty/tools/cli/markup"
+	"kitty/tools/tui"
+)
+
+var _ = fmt.Print
+
+type Response struct {
+	Items    []string `json:"items"`
+	Response string   `json:"response"`
+}
+
+func show_message(msg string) {
+	if msg != "" {
+		m := markup.New(true)
+		fmt.Println(m.Bold(msg))
+	}
+}
+
+func main(_ *cli.Command, o *Options, args []string) (rc int, err error) {
+	output := tui.KittenOutputSerializer()
+	result := &Response{Items: args}
+	if len(o.Prompt) > 2 && o.Prompt[0] == o.Prompt[len(o.Prompt)-1] && (o.Prompt[0] == '"' || o.Prompt[0] == '\'') {
+		o.Prompt = o.Prompt[1 : len(o.Prompt)-1]
+	}
+	switch o.Type {
+	case "yesno", "choices":
+		result.Response, err = GetChoices(o)
+		if err != nil {
+			return 1, err
+		}
+	case "password":
+		show_message(o.Message)
+		pw, err := tui.ReadPassword(o.Prompt, false)
+		if err != nil {
+			if errors.Is(err, tui.Canceled) {
+				pw = ""
+			} else {
+				return 1, err
+			}
+		}
+		result.Response = pw
+	case "line":
+		show_message(o.Message)
+		result.Response, err = get_line(o)
+		if err != nil {
+			return 1, err
+		}
+	default:
+		return 1, fmt.Errorf("Unknown type: %s", o.Type)
+	}
+	s, err := output(result)
+	if err != nil {
+		return 1, err
+	}
+	_, err = fmt.Println(s)
+	if err != nil {
+		return 1, err
+	}
+	return
+}
+
+func EntryPoint(parent *cli.Command) {
+	create_cmd(parent, main)
+}

kittens/ask/main.py

@@ -1,84 +1,15 @@
 #!/usr/bin/env python3
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
-import re
 import sys
-from contextlib import suppress
 from typing import (
-    TYPE_CHECKING,
-    Callable,
-    Dict,
-    Iterator,
     List,
-    NamedTuple,
     Optional,
-    Tuple,
 )
 
-from kitty.cli import parse_args
+from kitty.typing import BossType, TypedDict
-from kitty.cli_stub import AskCLIOptions
-from kitty.constants import cache_dir
-from kitty.fast_data_types import truncate_point_for_length, wcswidth
-from kitty.types import run_once
-from kitty.typing import BossType, KeyEventType, TypedDict
-from kitty.utils import ScreenSize
 
-from ..tui.handler import Handler, result_handler
+from ..tui.handler import result_handler
-from ..tui.loop import Loop, MouseEvent, debug
-from ..tui.operations import MouseTracking, alternate_screen, styled
-
-if TYPE_CHECKING:
-    import readline
-    debug
-else:
-    readline = None
-
-
-def get_history_items() -> List[str]:
-    return list(map(readline.get_history_item, range(1, readline.get_current_history_length() + 1)))
-
-
-def sort_key(item: str) -> Tuple[int, str]:
-    return len(item), item.lower()
-
-
-class HistoryCompleter:
-
-    def __init__(self, name: Optional[str] = None):
-        self.matches: List[str] = []
-        self.history_path = None
-        if name:
-            ddir = os.path.join(cache_dir(), 'ask')
-            with suppress(FileExistsError):
-                os.makedirs(ddir)
-            self.history_path = os.path.join(ddir, name)
-
-    def complete(self, text: str, state: int) -> Optional[str]:
-        response = None
-        if state == 0:
-            history_values = get_history_items()
-            if text:
-                self.matches = sorted(
-                        (h for h in history_values if h and h.startswith(text)), key=sort_key)
-            else:
-                self.matches = []
-        try:
-            response = self.matches[state]
-        except IndexError:
-            response = None
-        return response
-
-    def __enter__(self) -> 'HistoryCompleter':
-        if self.history_path:
-            with suppress(Exception):
-                readline.read_history_file(self.history_path)
-            readline.set_completer(self.complete)
-        return self
-
-    def __exit__(self, *a: object) -> None:
-        if self.history_path:
-            readline.write_history_file(self.history_path)
 
 
 def option_text() -> str:
@@ -134,397 +65,8 @@ class Response(TypedDict):
     items: List[str]
     response: Optional[str]
 
-
-class Choice(NamedTuple):
-    text: str
-    idx: int
-    color: str
-    letter: str
-
-
-class Range(NamedTuple):
-    start: int
-    end: int
-    y: int
-
-    def has_point(self, x: int, y: int) -> bool:
-        return y == self.y and self.start <= x <= self.end
-
-
-def truncate_at_space(text: str, width: int) -> Tuple[str, str]:
-    p = truncate_point_for_length(text, width)
-    if p < len(text):
-        i = text.rfind(' ', 0, p + 1)
-        if i > 0 and p - i < 12:
-            p = i + 1
-    return text[:p], text[p:]
-
-
-def extra_for(width: int, screen_width: int) -> int:
-    return max(0, screen_width - width) // 2 + 1
-
-
-class Password(Handler):
-
-    def __init__(self, cli_opts: AskCLIOptions, prompt: str, is_password: bool = True, initial_text: str = '') -> None:
-        self.cli_opts = cli_opts
-        self.prompt = prompt
-        self.initial_text = initial_text
-        from kittens.tui.line_edit import LineEdit
-        self.line_edit = LineEdit(is_password=is_password)
-
-    def initialize(self) -> None:
-        self.cmd.set_cursor_shape('beam')
-        if self.initial_text:
-            self.line_edit.on_text(self.initial_text, True)
-        self.draw_screen()
-
-    @Handler.atomic_update
-    def draw_screen(self) -> None:
-        self.cmd.clear_screen()
-        if self.cli_opts.message:
-            for line in self.cli_opts.message.splitlines():
-                self.print(line)
-            self.print()
-        self.line_edit.write(self.write, self.prompt)
-
-    def on_text(self, text: str, in_bracketed_paste: bool = False) -> None:
-        self.line_edit.on_text(text, in_bracketed_paste)
-        self.draw_screen()
-
-    def on_key(self, key_event: KeyEventType) -> None:
-        if self.line_edit.on_key(key_event):
-            self.draw_screen()
-            return
-        if key_event.matches('enter'):
-            self.quit_loop(0)
-        if key_event.matches('esc'):
-            self.quit_loop(1)
-
-    def on_resize(self, screen_size: ScreenSize) -> None:
-        self.screen_size = screen_size
-        self.draw_screen()
-
-    def on_interrupt(self) -> None:
-        self.quit_loop(1)
-    on_eot = on_interrupt
-
-    @property
-    def response(self) -> str:
-        if self._tui_loop.return_code == 0:
-            return self.line_edit.current_input
-        return ''
-
-
-class Choose(Handler):  # {{{
-    mouse_tracking = MouseTracking.buttons_only
-
-    def __init__(self, cli_opts: AskCLIOptions) -> None:
-        self.prefix_style_pat = re.compile(r'(?:\x1b\[[^m]*?m)+')
-        self.cli_opts = cli_opts
-        self.choices: Dict[str, Choice] = {}
-        self.clickable_ranges: Dict[str, List[Range]] = {}
-        if cli_opts.type == 'yesno':
-            self.allowed = frozenset('yn')
-        else:
-            allowed = []
-            for choice in cli_opts.choices:
-                letter, text = choice.split(':', maxsplit=1)
-                color = ''
-                if ';' in letter:
-                    letter, color = letter.split(';', maxsplit=1)
-                letter = letter.lower()
-                idx = text.lower().index(letter)
-                allowed.append(letter)
-                self.choices[letter] = Choice(text, idx, color, letter)
-            self.allowed = frozenset(allowed)
-        self.response = ''
-        self.response_on_accept = cli_opts.default or ''
-        if cli_opts.type in ('yesno', 'choices') and self.response_on_accept not in self.allowed:
-            self.response_on_accept = 'y' if cli_opts.type == 'yesno' else tuple(self.choices.keys())[0]
-        self.message = cli_opts.message
-        self.hidden_text_start_pos = self.hidden_text_end_pos = -1
-        self.hidden_text = ''
-        self.replacement_text = t = f'Press {styled(self.cli_opts.unhide_key, fg="green")} or click to show'
-        self.replacement_range = Range(-1, -1, -1)
-        if self.message and self.cli_opts.hidden_text_placeholder:
-            self.hidden_text_start_pos = self.message.find(self.cli_opts.hidden_text_placeholder)
-            if self.hidden_text_start_pos > -1:
-                self.hidden_text = sys.stdin.read().rstrip()
-                self.hidden_text_end_pos = self.hidden_text_start_pos + len(t)
-                suffix = self.message[self.hidden_text_start_pos + len(self.cli_opts.hidden_text_placeholder):]
-                self.message = self.message[:self.hidden_text_start_pos] + t + suffix
-
-    def initialize(self) -> None:
-        self.cmd.set_cursor_visible(False)
-        self.draw_screen()
-
-    def finalize(self) -> None:
-        self.cmd.set_cursor_visible(True)
-
-    def draw_long_text(self, text: str) -> Iterator[str]:
-        if not text:
-            yield ''
-            return
-        width = self.screen_size.cols - 2
-        m = self.prefix_style_pat.match(text)
-        prefix = m.group() if m else ''
-        while text:
-            t, text = truncate_at_space(text, width)
-            t = t.strip()
-            yield ' ' * extra_for(wcswidth(t), width) + styled(prefix + t, bold=True)
-
-    @Handler.atomic_update
-    def draw_screen(self) -> None:
-        self.cmd.clear_screen()
-        msg_lines: List[str] = []
-        if self.message:
-            for line in self.message.splitlines():
-                msg_lines.extend(self.draw_long_text(line))
-        y = self.screen_size.rows - len(msg_lines)
-        y = max(0, (y // 2) - 2)
-        self.print(end='\r\n'*y)
-        for line in msg_lines:
-            if self.replacement_text in line:
-                idx = line.find(self.replacement_text)
-                x = wcswidth(line[:idx])
-                self.replacement_range = Range(x, x + wcswidth(self.replacement_text), y)
-            self.print(line)
-            y += 1
-        if self.screen_size.rows > 2:
-            self.print()
-            y += 1
-        if self.cli_opts.type == 'yesno':
-            self.draw_yesno(y)
-        else:
-            self.draw_choice(y)
-
-    def draw_choice_boxes(self, y: int, *choices: Choice) -> None:
-        self.clickable_ranges.clear()
-        width = self.screen_size.cols - 2
-        current_line_length = 0
-        current_line: List[Tuple[str, str]] = []
-        lines: List[List[Tuple[str, str]]] = []
-        sep = '  '
-        sep_sz = len(sep) + 2  # for the borders
-
-        for choice in choices:
-            self.clickable_ranges[choice.letter] = []
-            text = ' ' + choice.text[:choice.idx]
-            text += styled(choice.text[choice.idx], fg=choice.color or 'green')
-            text += choice.text[choice.idx + 1:] + ' '
-            sz = wcswidth(text)
-            if sz + sep_sz + current_line_length > width:
-                lines.append(current_line)
-                current_line = []
-                current_line_length = 0
-            current_line.append((choice.letter, text))
-            current_line_length += sz + sep_sz
-        if current_line:
-            lines.append(current_line)
-
-        def top(text: str) -> str:
-            return '╭' + '─' * wcswidth(text) + '╮'
-
-        def middle(text: str) -> str:
-            return f'│{text}│'
-
-        def bottom(text: str) -> str:
-            return '╰' + '─' * wcswidth(text) + '╯'
-
-        def highlight(text: str, only_edges: bool = False) -> str:
-            if only_edges:
-                return styled(text[0], fg='yellow') + text[1:-1] + styled(text[-1], fg='yellow')
-            return styled(text, fg='yellow')
-
-        def print_line(add_borders: Callable[[str], str], *items: Tuple[str, str], is_last: bool = False) -> None:
-            nonlocal y
-            texts = []
-            positions = []
-            x = 0
-            for (letter, text) in items:
-                positions.append((letter, x, wcswidth(text) + 2))
-                text = add_borders(text)
-                if letter == self.response_on_accept:
-                    text = highlight(text, only_edges=add_borders is middle)
-                text += sep
-                x += wcswidth(text)
-                texts.append(text)
-            line = ''.join(texts).rstrip()
-            offset = extra_for(wcswidth(line), width)
-            for (letter, x, sz) in positions:
-                x += offset
-                self.clickable_ranges[letter].append(Range(x, x + sz - 1, y))
-            self.print(' ' * offset, line, sep='', end='' if is_last else '\r\n')
-            y += 1
-
-        self.cmd.set_line_wrapping(False)
-        for boxed_line in lines:
-            print_line(top, *boxed_line)
-            print_line(middle, *boxed_line)
-            print_line(bottom, *boxed_line, is_last=boxed_line is lines[-1])
-        self.cmd.set_line_wrapping(True)
-
-    def draw_choice(self, y: int) -> None:
-        if y + 3 <= self.screen_size.rows:
-            self.draw_choice_boxes(y, *self.choices.values())
-            return
-        self.clickable_ranges.clear()
-        current_line = ''
-        current_ranges: Dict[str, int] = {}
-        width = self.screen_size.cols - 2
-
-        def commit_line(end: str = '\r\n') -> None:
-            nonlocal current_line, y
-            x = extra_for(wcswidth(current_line), width)
-            self.print(' ' * x + current_line, end=end)
-            for letter, sz in current_ranges.items():
-                self.clickable_ranges[letter] = [Range(x, x + sz - 3, y)]
-                x += sz
-            current_ranges.clear()
-            y += 1
-            current_line = ''
-
-        for letter, choice in self.choices.items():
-            text = choice.text[:choice.idx]
-            text += styled(choice.text[choice.idx], fg=choice.color or 'green', underline='straight' if letter == self.response_on_accept else None)
-            text += choice.text[choice.idx + 1:]
-            text += '  '
-            sz = wcswidth(text)
-            if sz + wcswidth(current_line) >= width:
-                commit_line()
-            current_line += text
-            current_ranges[letter] = sz
-        if current_line:
-            commit_line(end='')
-
-    def draw_yesno(self, y: int) -> None:
-        yes = styled('Y', fg='green') + 'es'
-        no = styled('N', fg='red') + 'o'
-        if y + 3 <= self.screen_size.rows:
-            self.draw_choice_boxes(y, Choice('Yes', 0, 'green', 'y'), Choice('No', 0, 'red', 'n'))
-            return
-        sep = ' ' * 3
-        text = yes + sep + no
-        w = wcswidth(text)
-        x = extra_for(w, self.screen_size.cols - 2)
-        nx = x + wcswidth(yes) + len(sep)
-        self.clickable_ranges = {'y': [Range(x, x + wcswidth(yes) - 1, y)], 'n': [Range(nx, nx + wcswidth(no) - 1, y)]}
-        self.print(' ' * x + text, end='')
-
-    def on_text(self, text: str, in_bracketed_paste: bool = False) -> None:
-        text = text.lower()
-        if text in self.allowed:
-            self.response = text
-            self.quit_loop(0)
-        elif self.cli_opts.type == 'yesno':
-            self.on_interrupt()
-        elif self.hidden_text and text == self.cli_opts.unhide_key:
-            self.unhide()
-
-    def unhide(self) -> None:
-        if self.hidden_text and self.message:
-            self.message = self.message[:self.hidden_text_start_pos] + self.hidden_text + self.message[self.hidden_text_end_pos:]
-            self.hidden_text = ''
-            self.draw_screen()
-
-    def on_key(self, key_event: KeyEventType) -> None:
-        if key_event.matches('esc'):
-            self.on_interrupt()
-        elif key_event.matches('enter'):
-            self.response = self.response_on_accept
-            self.quit_loop(0)
-
-    def on_click(self, ev: MouseEvent) -> None:
-        for letter, ranges in self.clickable_ranges.items():
-            for r in ranges:
-                if r.has_point(ev.cell_x, ev.cell_y):
-                    self.response = letter
-                    self.quit_loop(0)
-                    return
-        if self.hidden_text and self.replacement_range.has_point(ev.cell_x, ev.cell_y):
-            self.unhide()
-
-    def on_resize(self, screen_size: ScreenSize) -> None:
-        self.screen_size = screen_size
-        self.draw_screen()
-
-    def on_interrupt(self) -> None:
-        self.quit_loop(1)
-    on_eot = on_interrupt
-# }}}
-
-
-@run_once
-def init_readline() -> None:
-    import readline
-    with suppress(OSError):
-        readline.read_init_file()
-    if 'libedit' in readline.__doc__:
-        readline.parse_and_bind("bind ^I rl_complete")
-    else:
-        readline.parse_and_bind('tab: complete')
-
-
 def main(args: List[str]) -> Response:
-    # For some reason importing readline in a key handler in the main kitty process
+    raise SystemExit('This must be run as kitten ask')
-    # causes a crash of the python interpreter, probably because of some global
-    # lock
-    global readline
-    msg = 'Ask the user for input'
-    try:
-        cli_opts, items = parse_args(args[1:], option_text, '', msg, 'kitty +kitten ask', result_class=AskCLIOptions)
-    except SystemExit as e:
-        if e.code != 0:
-            print(e.args[0])
-            input('Press Enter to quit')
-        raise SystemExit(e.code)
-
-    if cli_opts.type in ('yesno', 'choices'):
-        loop = Loop()
-        handler = Choose(cli_opts)
-        loop.loop(handler)
-        return {'items': items, 'response': handler.response}
-
-    prompt = cli_opts.prompt
-    if prompt[0] == prompt[-1] and prompt[0] in '\'"':
-        prompt = prompt[1:-1]
-    if cli_opts.type == 'password':
-        loop = Loop()
-        phandler = Password(cli_opts, prompt)
-        loop.loop(phandler)
-        return {'items': items, 'response': phandler.response}
-
-    # we do this file descriptor dance to get readline to work even when STDOUT
-    # is redirected
-    orig_stdout = os.dup(sys.stdout.fileno())
-    try:
-        with open(os.ctermid(), 'r') as tty:
-            os.dup2(tty.fileno(), sys.stdin.fileno())
-        with open(os.ctermid(), 'w') as tty:
-            os.dup2(tty.fileno(), sys.stdout.fileno())
-        import readline as rl
-        readline = rl
-        init_readline()
-        response = None
-
-        with alternate_screen(), HistoryCompleter(cli_opts.name), suppress(KeyboardInterrupt, EOFError):
-            if cli_opts.message:
-                print(styled(cli_opts.message, bold=True))
-            if cli_opts.default:
-                def prefill_text() -> None:
-                    readline.insert_text(cli_opts.default or '')
-                    readline.redisplay()
-                readline.set_pre_input_hook(prefill_text)
-                response = input(prompt)
-                readline.set_pre_input_hook()
-            else:
-                response = input(prompt)
-        sys.stdout.flush()
-        os.dup2(orig_stdout, sys.stdout.fileno())
-    finally:
-        os.close(orig_stdout)
-    return {'items': items, 'response': response}
 
 
 @result_handler()
@@ -535,7 +77,10 @@ def handle_result(args: List[str], data: Response, target_window_id: int, boss:
 
 
 if __name__ == '__main__':
-    ans = main(sys.argv)
+    main(sys.argv)
-    if ans:
+elif __name__ == '__doc__':
-        import json
+    cd = sys.cli_docs  # type: ignore
-        print(json.dumps(ans))
+    cd['usage'] = ''
+    cd['options'] = option_text
+    cd['help_text'] = 'Ask the user for input'
+    cd['short_desc'] = 'Ask the user for input'

kittens/choose/choose-data-types.h

@@ -1,86 +0,0 @@
-/*
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#pragma once
-#include "data-types.h"
-#if defined(_MSC_VER)
-#define ISWINDOWS
-#define STDCALL __stdcall
-#ifndef ssize_t
-#include <BaseTsd.h>
-typedef SSIZE_T ssize_t;
-#ifndef SSIZE_MAX
-#if defined(_WIN64)
-    #define SSIZE_MAX _I64_MAX
-#else
-    #define SSIZE_MAX LONG_MAX
-#endif
-#endif
-#endif
-#else
-#define STDCALL
-#endif
-
-#include "vector.h"
-
-typedef uint8_t len_t;
-typedef uint32_t text_t;
-
-#define LEN_MAX UINT8_MAX
-#define IS_LOWERCASE(x) (x) >= 'a' && (x) <= 'z'
-#define IS_UPPERCASE(x) (x) >= 'A' && (x) <= 'Z'
-#define LOWERCASE(x) ((IS_UPPERCASE(x)) ? (x) + 32 : (x))
-#define arraysz(x) (sizeof(x)/sizeof(x[0]))
-
-typedef struct {
-    text_t* src;
-    ssize_t src_sz;
-    len_t haystack_len;
-    len_t *positions;
-    double score;
-    ssize_t idx;
-} Candidate;
-
-typedef struct {
-    Candidate *haystack;
-    size_t haystack_count;
-    text_t level1[LEN_MAX], level2[LEN_MAX], level3[LEN_MAX], needle[LEN_MAX];
-    len_t level1_len, level2_len, level3_len, needle_len;
-    size_t haystack_size;
-    text_t *output;
-    size_t output_sz, output_pos;
-    int oom;
-} GlobalData;
-
-typedef struct {
-    bool output_positions;
-    size_t limit;
-    int num_threads;
-    text_t mark_before[128], mark_after[128], delimiter[128];
-    size_t mark_before_sz, mark_after_sz, delimiter_sz;
-} Options;
-
-VECTOR_OF(len_t, Positions)
-VECTOR_OF(text_t, Chars)
-VECTOR_OF(Candidate, Candidates)
-
-
-void output_results(GlobalData *, Candidate *haystack, size_t count, Options *opts, len_t needle_len);
-void* alloc_workspace(len_t max_haystack_len, GlobalData*);
-void* free_workspace(void *v);
-double score_item(void *v, text_t *haystack, len_t haystack_len, len_t *match_positions);
-unsigned int encode_codepoint(text_t ch, char* dest);
-size_t unescape(const char *src, char *dest, size_t destlen);
-int cpu_count(void);
-void* alloc_threads(size_t num_threads);
-#ifdef ISWINDOWS
-bool start_thread(void* threads, size_t i, unsigned int (STDCALL *start_routine) (void *), void *arg);
-ssize_t getdelim(char **lineptr, size_t *n, int delim, FILE *stream);
-#else
-bool start_thread(void* threads, size_t i, void *(*start_routine) (void *), void *arg);
-#endif
-void wait_for_thread(void *threads, size_t i);
-void free_threads(void *threads);

kittens/choose/main.c

@@ -1,244 +0,0 @@
-/*
- * main.c
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "choose-data-types.h"
-#include "charsets.h"
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-#include <fcntl.h>
-#ifndef ISWINDOWS
-#include <unistd.h>
-#endif
-
-typedef struct {
-    size_t start, count;
-    void *workspace;
-    len_t max_haystack_len;
-    bool started;
-    GlobalData *global;
-} JobData;
-
-
-static unsigned int STDCALL
-run_scoring(JobData *job_data) {
-    GlobalData *global = job_data->global;
-    for (size_t i = job_data->start; i < job_data->start + job_data->count; i++) {
-        global->haystack[i].score = score_item(job_data->workspace, global->haystack[i].src, global->haystack[i].haystack_len, global->haystack[i].positions);
-    }
-    return 0;
-}
-
-static void*
-run_scoring_pthreads(void *job_data) {
-    run_scoring((JobData*)job_data);
-    return NULL;
-}
-#ifdef ISWINDOWS
-#define START_FUNC run_scoring
-#else
-#define START_FUNC run_scoring_pthreads
-#endif
-
-static JobData*
-create_job(size_t i, size_t blocksz, GlobalData *global) {
-    JobData *ans = (JobData*)calloc(1, sizeof(JobData));
-    if (ans == NULL) return NULL;
-    ans->start = i * blocksz;
-    if (ans->start >= global->haystack_count) ans->count = 0;
-    else ans->count = global->haystack_count - ans->start;
-    ans->max_haystack_len = 0;
-    for (size_t j = ans->start; j < ans->start + ans->count; j++) ans->max_haystack_len = MAX(ans->max_haystack_len, global->haystack[j].haystack_len);
-    if (ans->count > 0) {
-        ans->workspace = alloc_workspace(ans->max_haystack_len, global);
-        if (!ans->workspace) { free(ans); return NULL; }
-    }
-    ans->global = global;
-    return ans;
-}
-
-static JobData*
-free_job(JobData *job) {
-    if (job) {
-        if (job->workspace) free_workspace(job->workspace);
-        free(job);
-    }
-    return NULL;
-}
-
-
-static int
-run_threaded(int num_threads_asked, GlobalData *global) {
-    int ret = 0;
-    size_t i, blocksz;
-    size_t num_threads = MAX(1, num_threads_asked > 0 ? num_threads_asked : cpu_count());
-    if (global->haystack_size < 10000) num_threads = 1;
-    /* printf("num_threads: %lu asked: %d sysconf: %ld\n", num_threads, num_threads_asked, sysconf(_SC_NPROCESSORS_ONLN)); */
-
-    void *threads = alloc_threads(num_threads);
-    JobData **job_data = calloc(num_threads, sizeof(JobData*));
-    if (threads == NULL || job_data == NULL) { ret = 1; goto end; }
-
-    blocksz = global->haystack_count / num_threads + global->haystack_count % num_threads;
-
-    for (i = 0; i < num_threads; i++) {
-        job_data[i] = create_job(i, blocksz, global);
-        if (job_data[i] == NULL) { ret = 1; goto end; }
-    }
-
-    if (num_threads == 1) {
-        run_scoring(job_data[0]);
-    } else {
-        for (i = 0; i < num_threads; i++) {
-            job_data[i]->started = false;
-            if (job_data[i]->count > 0) {
-                if (!start_thread(threads, i, START_FUNC, job_data[i])) ret = 1;
-                else job_data[i]->started = true;
-            }
-        }
-    }
-
-end:
-    if (num_threads > 1 && job_data) {
-        for (i = 0; i < num_threads; i++) {
-            if (job_data[i] && job_data[i]->started) wait_for_thread(threads, i);
-        }
-    }
-    if (job_data) { for (i = 0; i < num_threads; i++) job_data[i] = free_job(job_data[i]); }
-    free(job_data);
-    free_threads(threads);
-    return ret;
-}
-
-
-static int
-run_search(Options *opts, GlobalData *global, const char * const *lines, const size_t* sizes, size_t num_lines) {
-    const char *linebuf = NULL;
-    size_t idx = 0;
-    ssize_t sz = 0;
-    int ret = 0;
-    Candidates candidates = {0};
-    Chars chars = {0};
-
-    ALLOC_VEC(text_t, chars, 8192 * 20);
-    if (chars.data == NULL) return 1;
-    ALLOC_VEC(Candidate, candidates, 8192);
-    if (candidates.data == NULL) { FREE_VEC(chars); return 1; }
-
-    for (size_t i = 0; i < num_lines; i++) {
-        sz = sizes[i];
-        linebuf = lines[i];
-        if (sz > 0) {
-            ENSURE_SPACE(text_t, chars, sz);
-            ENSURE_SPACE(Candidate, candidates, 1);
-            sz = decode_utf8_string(linebuf, sz, &(NEXT(chars)));
-            NEXT(candidates).src_sz = sz;
-            NEXT(candidates).haystack_len = (len_t)(MIN(LEN_MAX, sz));
-            global->haystack_size += NEXT(candidates).haystack_len;
-            NEXT(candidates).idx = idx++;
-            INC(candidates, 1); INC(chars, sz);
-        }
-    }
-
-    // Prepare the haystack allocating space for positions arrays and settings
-    // up the src pointers to point to the correct locations
-    Candidate *haystack = &ITEM(candidates, 0);
-    len_t *positions = (len_t*)calloc(SIZE(candidates), sizeof(len_t) * global->needle_len);
-    if (positions) {
-        text_t *cdata = &ITEM(chars, 0);
-        for (size_t i = 0, off = 0; i < SIZE(candidates); i++) {
-            haystack[i].positions = positions + (i * global->needle_len);
-            haystack[i].src = cdata + off;
-            off += haystack[i].src_sz;
-        }
-        global->haystack = haystack;
-        global->haystack_count = SIZE(candidates);
-        ret = run_threaded(opts->num_threads, global);
-        if (ret == 0) output_results(global, haystack, SIZE(candidates), opts, global->needle_len);
-        else { REPORT_OOM; }
-    } else { ret = 1; REPORT_OOM; }
-
-    FREE_VEC(chars); free(positions); FREE_VEC(candidates);
-    return ret;
-}
-
-static size_t
-copy_unicode_object(PyObject *src, text_t *dest, size_t dest_sz) {
-    PyUnicode_READY(src);
-    int kind = PyUnicode_KIND(src);
-    void *data = PyUnicode_DATA(src);
-    size_t len = PyUnicode_GetLength(src);
-    for (size_t i = 0; i < len && i < dest_sz; i++) {
-        dest[i] = PyUnicode_READ(kind, data, i);
-    }
-    return len;
-}
-
-static PyObject*
-match(PyObject *self, PyObject *args) {
-    (void)(self);
-    int output_positions;
-    unsigned long limit;
-    PyObject *lines, *levels, *needle, *mark_before, *mark_after, *delimiter;
-    Options opts = {0};
-    GlobalData global = {0};
-    if (!PyArg_ParseTuple(args, "O!O!UpkiUUU",
-            &PyList_Type, &lines, &PyTuple_Type, &levels, &needle,
-            &output_positions, &limit, &opts.num_threads,
-            &mark_before, &mark_after, &delimiter
-    )) return NULL;
-    opts.output_positions = output_positions ? true : false;
-    opts.limit = limit;
-    global.level1_len = copy_unicode_object(PyTuple_GET_ITEM(levels, 0), global.level1, arraysz(global.level1));
-    global.level2_len = copy_unicode_object(PyTuple_GET_ITEM(levels, 1), global.level2, arraysz(global.level2));
-    global.level3_len = copy_unicode_object(PyTuple_GET_ITEM(levels, 2), global.level3, arraysz(global.level3));
-    global.needle_len = copy_unicode_object(needle, global.needle, arraysz(global.needle));
-    opts.mark_before_sz = copy_unicode_object(mark_before, opts.mark_before, arraysz(opts.mark_before));
-    opts.mark_after_sz = copy_unicode_object(mark_after, opts.mark_after, arraysz(opts.mark_after));
-    opts.delimiter_sz = copy_unicode_object(delimiter, opts.delimiter, arraysz(opts.delimiter));
-    size_t num_lines = PyList_GET_SIZE(lines);
-    char **clines = malloc(sizeof(char*) * num_lines);
-    if (!clines) { return PyErr_NoMemory(); }
-    size_t *sizes = malloc(sizeof(size_t) * num_lines);
-    if (!sizes) { free(clines); clines = NULL; return PyErr_NoMemory(); }
-    for (size_t i = 0; i < num_lines; i++) {
-        clines[i] = PyBytes_AS_STRING(PyList_GET_ITEM(lines, i));
-        sizes[i] = PyBytes_GET_SIZE(PyList_GET_ITEM(lines, i));
-    }
-    Py_BEGIN_ALLOW_THREADS;
-    run_search(&opts, &global, (const char* const *)clines, sizes, num_lines);
-    Py_END_ALLOW_THREADS;
-    free(clines); free(sizes);
-    if (global.oom) { free(global.output); return PyErr_NoMemory(); }
-    if (global.output) {
-        PyObject *ans = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, global.output, global.output_pos);
-        free(global.output);
-        return ans;
-    }
-    Py_RETURN_NONE;
-}
-
-static PyMethodDef module_methods[] = {
-    {"match", match, METH_VARARGS, ""},
-    {NULL, NULL, 0, NULL}        /* Sentinel */
-};
-
-static struct PyModuleDef module = {
-    .m_base = PyModuleDef_HEAD_INIT,
-    .m_name = "subseq_matcher",   /* name of module */
-    .m_doc = NULL,
-    .m_size = -1,
-    .m_methods = module_methods
-};
-
-EXPORTED PyMODINIT_FUNC
-PyInit_subseq_matcher(void) {
-    return PyModule_Create(&module);
-}

kittens/choose/main.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-import sys
-from typing import List
-
-from kitty.key_encoding import KeyEvent
-
-from ..tui.handler import Handler
-from ..tui.loop import Loop
-
-
-class ChooseHandler(Handler):
-
-    def initialize(self) -> None:
-        pass
-
-    def on_text(self, text: str, in_bracketed_paste: bool = False) -> None:
-        pass
-
-    def on_key(self, key_event: KeyEvent) -> None:
-        pass
-
-    def on_interrupt(self) -> None:
-        self.quit_loop(1)
-
-    def on_eot(self) -> None:
-        self.quit_loop(1)
-
-
-def main(args: List[str]) -> None:
-    loop = Loop()
-    handler = ChooseHandler()
-    loop.loop(handler)
-    raise SystemExit(loop.return_code)
-
-
-if __name__ == '__main__':
-    main(sys.argv)

kittens/choose/match.py

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
-
-from typing import Iterable, List, Union
-
-from . import subseq_matcher
-
-
-def match(
-    input_data: Union[str, bytes, Iterable[Union[str, bytes]]],
-    query: str,
-    threads: int = 0,
-    positions: bool = False,
-    level1: str = '/',
-    level2: str = '-_0123456789',
-    level3: str = '.',
-    limit: int = 0,
-    mark_before: str = '',
-    mark_after: str = '',
-    delimiter: str = '\n'
-) -> List[str]:
-    if isinstance(input_data, str):
-        idata = [x.encode('utf-8') for x in input_data.split(delimiter)]
-    elif isinstance(input_data, bytes):
-        idata = input_data.split(delimiter.encode('utf-8'))
-    else:
-        idata = [x.encode('utf-8') if isinstance(x, str) else x for x in input_data]
-    query = query.lower()
-    level1 = level1.lower()
-    level2 = level2.lower()
-    level3 = level3.lower()
-    data = subseq_matcher.match(
-        idata, (level1, level2, level3), query,
-        positions, limit, threads,
-        mark_before, mark_after, delimiter)
-    if data is None:
-        return []
-    return list(filter(None, data.split(delimiter or '\n')))

kittens/choose/output.c

@@ -1,101 +0,0 @@
-/*
- * output.c
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "choose-data-types.h"
-#include "../../kitty/iqsort.h"
-#include <string.h>
-#include <ctype.h>
-#include <stdlib.h>
-#include <stdio.h>
-#ifdef ISWINDOWS
-#include <io.h>
-#define STDOUT_FILENO 1
-static ssize_t ms_write(int fd, const void* buf, size_t count) { return _write(fd, buf, (unsigned int)count); }
-#define write ms_write
-#else
-#include <unistd.h>
-#endif
-#include <errno.h>
-
-
-#define FIELD(x, which) (((Candidate*)(x))->which)
-
-static bool
-ensure_space(GlobalData *global, size_t sz) {
-    if (global->output_sz < sz + global->output_pos || !global->output) {
-        size_t before = global->output_sz;
-        global->output_sz += MAX(sz, (64u * 1024u));
-        global->output = realloc(global->output, sizeof(text_t) * global->output_sz);
-        if (!global->output) {
-            global->output_sz = before;
-            return false;
-        }
-    }
-    return true;
-}
-
-static void
-output_text(GlobalData *global, const text_t *data, size_t sz) {
-    if (ensure_space(global, sz)) {
-        memcpy(global->output + global->output_pos, data, sizeof(text_t) * sz);
-        global->output_pos += sz;
-    }
-}
-
-static void
-output_with_marks(GlobalData *global, Options *opts, text_t *src, size_t src_sz, len_t *positions, len_t poslen) {
-    size_t pos, i = 0;
-    for (pos = 0; pos < poslen; pos++, i++) {
-        output_text(global, src + i, MIN(src_sz, positions[pos]) - i);
-        i = positions[pos];
-        if (i < src_sz) {
-            if (opts->mark_before_sz > 0) output_text(global, opts->mark_before, opts->mark_before_sz);
-            output_text(global, src + i, 1);
-            if (opts->mark_after_sz > 0) output_text(global, opts->mark_after, opts->mark_after_sz);
-        }
-    }
-    i = positions[poslen - 1];
-    if (i + 1 < src_sz) output_text(global, src + i + 1, src_sz - i - 1);
-}
-
-static void
-output_positions(GlobalData *global, len_t *positions, len_t num) {
-    wchar_t buf[128];
-    for (len_t i = 0; i < num; i++) {
-        int pnum = swprintf(buf, arraysz(buf), L"%u", positions[i]);
-        if (pnum > 0 && ensure_space(global, pnum + 1)) {
-            for (int k = 0; k < pnum; k++) global->output[global->output_pos++] = buf[k];
-            global->output[global->output_pos++] = (i == num - 1) ? ':' : ',';
-        }
-    }
-}
-
-
-static void
-output_result(GlobalData *global, Candidate *c, Options *opts, len_t needle_len) {
-    if (opts->output_positions) output_positions(global, c->positions, needle_len);
-    if (opts->mark_before_sz > 0 || opts->mark_after_sz > 0) {
-        output_with_marks(global, opts, c->src, c->src_sz, c->positions, needle_len);
-    } else {
-        output_text(global, c->src, c->src_sz);
-    }
-    output_text(global, opts->delimiter, opts->delimiter_sz);
-}
-
-
-void
-output_results(GlobalData *global, Candidate *haystack, size_t count, Options *opts, len_t needle_len) {
-    Candidate *c;
-#define lt(b, a) ( (a)->score < (b)->score || ((a)->score == (b)->score && (a->idx < b->idx)) )
-    QSORT(Candidate, haystack, count, lt);
-#undef lt
-    size_t left = opts->limit > 0 ? opts->limit : count;
-    for (size_t i = 0; i < left; i++) {
-        c = haystack + i;
-        if (c->score > 0) output_result(global, c, opts, needle_len);
-    }
-}

kittens/choose/score.c

@@ -1,182 +0,0 @@
-/*
- * score.c
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "choose-data-types.h"
-#include <stdlib.h>
-#include <string.h>
-#include <float.h>
-#include <stdio.h>
-
-typedef struct {
-    len_t *positions_buf;  // buffer to store positions for every char in needle
-    len_t **positions;  // Array of pointers into positions_buf
-    len_t *positions_count; // Array of counts for positions
-    len_t needle_len;  // Length of the needle
-    len_t max_haystack_len;  // Max length of a string in the haystack
-    len_t haystack_len; // Length of the current string in the haystack
-    len_t *address; // Array of offsets into the positions array
-    double max_score_per_char;
-    uint8_t *level_factors;  // Array of score factors for every character in the current haystack that matches a character in the needle
-    text_t *level1, *level2, *level3;  // The characters in the levels
-    len_t level1_len, level2_len, level3_len;
-    text_t *needle;  // The current needle
-    text_t *haystack; //The current haystack
-} WorkSpace;
-
-void*
-alloc_workspace(len_t max_haystack_len, GlobalData *global) {
-    WorkSpace *ans = calloc(1, sizeof(WorkSpace));
-    if (ans == NULL) return NULL;
-    ans->positions_buf = (len_t*) calloc(global->needle_len, sizeof(len_t) * max_haystack_len);
-    ans->positions = (len_t**)calloc(global->needle_len, sizeof(len_t*));
-    ans->positions_count = (len_t*)calloc(2*global->needle_len, sizeof(len_t));
-    ans->level_factors = (uint8_t*)calloc(max_haystack_len, sizeof(uint8_t));
-    if (ans->positions == NULL || ans->positions_buf == NULL || ans->positions_count == NULL || ans->level_factors == NULL) { free_workspace(ans); return NULL; }
-    ans->needle = global->needle;
-    ans->needle_len = global->needle_len;
-    ans->max_haystack_len = max_haystack_len;
-    ans->level1 = global->level1; ans->level2 = global->level2; ans->level3 = global->level3;
-    ans->level1_len = global->level1_len; ans->level2_len = global->level2_len; ans->level3_len = global->level3_len;
-    ans->address = ans->positions_count + sizeof(len_t) * global->needle_len;
-    for (len_t i = 0; i < global->needle_len; i++) ans->positions[i] = ans->positions_buf + i * max_haystack_len;
-    return ans;
-}
-
-#define NUKE(x) free(x); x = NULL;
-
-void*
-free_workspace(void *v) {
-    WorkSpace *w = (WorkSpace*)v;
-    NUKE(w->positions_buf);
-    NUKE(w->positions);
-    NUKE(w->positions_count);
-    NUKE(w->level_factors);
-    free(w);
-    return NULL;
-}
-
-static bool
-has_char(text_t *text, len_t sz, text_t ch) {
-    for(len_t i = 0; i < sz; i++) {
-        if(text[i] == ch) return true;
-    }
-    return false;
-}
-
-static uint8_t
-level_factor_for(text_t current, text_t last, WorkSpace *w) {
-    text_t lch = LOWERCASE(last);
-    if (has_char(w->level1, w->level1_len, lch)) return 90;
-    if (has_char(w->level2, w->level2_len, lch)) return 80;
-    if (IS_LOWERCASE(last) && IS_UPPERCASE(current)) return 80; // CamelCase
-    if (has_char(w->level3, w->level3_len, lch)) return 70;
-    return 0;
-}
-
-static void
-init_workspace(WorkSpace *w, text_t *haystack, len_t haystack_len) {
-    // Calculate the positions and level_factors arrays for the specified haystack
-    bool level_factor_calculated = false;
-    memset(w->positions_count, 0, sizeof(*(w->positions_count)) * 2 * w->needle_len);
-    memset(w->level_factors, 0, sizeof(*(w->level_factors)) * w->max_haystack_len);
-    for (len_t i = 0; i < haystack_len; i++) {
-        level_factor_calculated = false;
-        for (len_t j = 0; j < w->needle_len; j++) {
-            if (w->needle[j] == LOWERCASE(haystack[i])) {
-                if (!level_factor_calculated) {
-                    level_factor_calculated = true;
-                    w->level_factors[i] = i > 0 ? level_factor_for(haystack[i], haystack[i-1], w) : 0;
-                }
-                w->positions[j][w->positions_count[j]++] = i;
-            }
-        }
-    }
-    w->haystack = haystack;
-    w->haystack_len = haystack_len;
-    w->max_score_per_char = (1.0 / haystack_len + 1.0 / w->needle_len) / 2.0;
-}
-
-
-static bool
-has_atleast_one_match(WorkSpace *w) {
-    int p = -1;
-    bool found;
-    for (len_t i = 0; i < w->needle_len; i++) {
-        if (w->positions_count[i] == 0) return false;  // All characters of the needle are not present in the haystack
-        found = false;
-        for (len_t j = 0; j < w->positions_count[i]; j++) {
-            if (w->positions[i][j] > p) { p = w->positions[i][j]; found = true; break; }
-        }
-        if (!found) return false; // Characters of needle not present in sequence in haystack
-    }
-    return true;
-}
-
-#define POSITION(x) w->positions[x][w->address[x]]
-
-static bool
-increment_address(WorkSpace *w) {
-    len_t pos = w->needle_len - 1;
-    while(true) {
-        w->address[pos]++;
-        if (w->address[pos] < w->positions_count[pos]) return true;
-        if (pos == 0) break;
-        w->address[pos--] = 0;
-    }
-    return false;
-}
-
-static bool
-address_is_monotonic(WorkSpace *w) {
-    // Check if the character positions pointed to by the current address are monotonic
-    for (len_t i = 1; i < w->needle_len; i++) {
-        if (POSITION(i) <= POSITION(i-1)) return false;
-    }
-    return true;
-}
-
-static double
-calc_score(WorkSpace *w) {
-    double ans = 0;
-    len_t distance, pos;
-    for (len_t i = 0; i < w->needle_len; i++) {
-        pos = POSITION(i);
-        if (i == 0) distance = pos < LEN_MAX ? pos + 1 : LEN_MAX;
-        else {
-            distance = pos - POSITION(i-1);
-            if (distance < 2) {
-                ans += w->max_score_per_char; // consecutive characters
-                continue;
-            }
-        }
-        if (w->level_factors[pos]) ans += (100 * w->max_score_per_char) / w->level_factors[pos];  // at a special location
-        else ans += (0.75 * w->max_score_per_char) / distance;
-    }
-    return ans;
-}
-
-static double
-process_item(WorkSpace *w, len_t *match_positions) {
-    double highscore = 0, score;
-    do {
-        if (!address_is_monotonic(w)) continue;
-        score = calc_score(w);
-        if (score > highscore) {
-            highscore = score;
-            for (len_t i = 0; i < w->needle_len; i++) match_positions[i] = POSITION(i);
-        }
-    } while(increment_address(w));
-    return highscore;
-}
-
-double
-score_item(void *v, text_t *haystack, len_t haystack_len, len_t *match_positions) {
-    WorkSpace *w = (WorkSpace*)v;
-    init_workspace(w, haystack, haystack_len);
-    if (!has_atleast_one_match(w)) return 0;
-    return process_item(w, match_positions);
-}

kittens/choose/subseq_matcher.pyi

@@ -1,8 +0,0 @@
-from typing import List, Optional, Tuple
-
-def match(
-    lines: List[bytes], levels: Tuple[str, str, str], needle: str,
-    output_positions: bool, limit: int, num_threads: int, mark_before: str,
-    mark_after: str, delimiter: str
-) -> Optional[str]:
-    pass

kittens/choose/unix_compat.c

@@ -1,50 +0,0 @@
-/*
- * unix_compat.c
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "choose-data-types.h"
-#include <unistd.h>
-#include <pthread.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-
-#ifdef __APPLE__
-#ifndef _SC_NPROCESSORS_ONLN
-#define _SC_NPROCESSORS_ONLN 58
-#endif
-#endif
-
-int
-cpu_count() {
-    return sysconf(_SC_NPROCESSORS_ONLN);
-}
-
-
-void*
-alloc_threads(size_t num_threads) {
-    return calloc(num_threads, sizeof(pthread_t));
-}
-
-bool
-start_thread(void* threads, size_t i, void *(*start_routine) (void *), void *arg) {
-    int rc;
-    if ((rc = pthread_create(((pthread_t*)threads) + i, NULL, start_routine, arg))) {
-        fprintf(stderr, "Failed to create thread, with error: %s\n", strerror(rc));
-        return false;
-    }
-    return true;
-}
-
-void
-wait_for_thread(void *threads, size_t i) {
-    pthread_join(((pthread_t*)(threads))[i], NULL);
-}
-
-void
-free_threads(void *threads) {
-    free(threads);
-}

kittens/choose/vector.h

@@ -1,42 +0,0 @@
-/*
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#pragma once
-
-#include "data-types.h"
-
-#define REPORT_OOM global->oom = 1;
-
-#define VECTOR_OF(TYPE, NAME) typedef struct { \
-    TYPE *data; \
-    size_t size; \
-    size_t capacity; \
-} NAME;
-
-#define ALLOC_VEC(TYPE, vec, cap) \
-    vec.size = 0; vec.capacity = cap; \
-    vec.data = (TYPE*)malloc(vec.capacity * sizeof(TYPE)); \
-    if (vec.data == NULL) { REPORT_OOM; }
-
-#define FREE_VEC(vec) \
-    if (vec.data) { free(vec.data); vec.data = NULL; } \
-    vec.size = 0; vec.capacity = 0;
-
-#define ENSURE_SPACE(TYPE, vec, amt) \
-    if (vec.size + amt >= vec.capacity) { \
-        vec.capacity = MAX(vec.capacity * 2, vec.size + amt); \
-        void *temp = realloc(vec.data, sizeof(TYPE) * vec.capacity); \
-        if (temp == NULL) { REPORT_OOM; ret = 1; free(vec.data); vec.data = NULL; vec.size = 0; vec.capacity = 0; break; }  \
-        else vec.data = temp; \
-    }
-
-#define NEXT(vec) (vec.data[vec.size])
-
-#define INC(vec, amt) vec.size += amt;
-
-#define SIZE(vec) (vec.size)
-
-#define ITEM(vec, n) (vec.data[n])

kittens/choose/windows_compat.c

@@ -1,107 +0,0 @@
-/*
- * windows_compat.c
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "choose-data-types.h"
-
-#include <windows.h>
-#include <process.h>
-#include <stdio.h>
-#include <errno.h>
-
-int
-cpu_count() {
-    SYSTEM_INFO sysinfo;
-    GetSystemInfo(&sysinfo);
-    return sysinfo.dwNumberOfProcessors;
-}
-
-void*
-alloc_threads(size_t num_threads) {
-    return calloc(num_threads, sizeof(uintptr_t));
-}
-
-bool
-start_thread(void* vt, size_t i, unsigned int (STDCALL *start_routine) (void *), void *arg) {
-    uintptr_t *threads = (uintptr_t*)vt;
-    errno = 0;
-    threads[i] = _beginthreadex(NULL, 0, start_routine, arg, 0, NULL);
-    if (threads[i] == 0) {
-        perror("Failed to create thread, with error");
-        return false;
-    }
-    return true;
-}
-
-void
-wait_for_thread(void *vt, size_t i) {
-    uintptr_t *threads = vt;
-    WaitForSingleObject((HANDLE)threads[i], INFINITE);
-    CloseHandle((HANDLE)threads[i]);
-    threads[i] = 0;
-}
-
-void
-free_threads(void *threads) {
-    free(threads);
-}
-
-ssize_t
-getdelim(char **lineptr, size_t *n, int delim, FILE *stream) {
-    char c, *cur_pos, *new_lineptr;
-    size_t new_lineptr_len;
-
-    if (lineptr == NULL || n == NULL || stream == NULL) {
-        errno = EINVAL;
-        return -1;
-    }
-
-    if (*lineptr == NULL) {
-        *n = 8192; /* init len */
-        if ((*lineptr = (char *)malloc(*n)) == NULL) {
-            errno = ENOMEM;
-            return -1;
-        }
-    }
-
-    cur_pos = *lineptr;
-    for (;;) {
-        c = getc(stream);
-
-        if (ferror(stream) || (c == EOF && cur_pos == *lineptr))
-            return -1;
-
-        if (c == EOF)
-            break;
-
-        if ((*lineptr + *n - cur_pos) < 2) {
-            if (SSIZE_MAX / 2 < *n) {
-#ifdef EOVERFLOW
-                errno = EOVERFLOW;
-#else
-                errno = ERANGE; /* no EOVERFLOW defined */
-#endif
-                return -1;
-            }
-            new_lineptr_len = *n * 2;
-
-            if ((new_lineptr = (char *)realloc(*lineptr, new_lineptr_len)) == NULL) {
-                errno = ENOMEM;
-                return -1;
-            }
-            *lineptr = new_lineptr;
-            *n = new_lineptr_len;
-        }
-
-        *cur_pos++ = c;
-
-        if (c == delim)
-            break;
-    }
-
-    *cur_pos = '\0';
-    return (ssize_t)(cur_pos - *lineptr);
-}

kittens/clipboard/read.go

@@ -16,6 +16,9 @@ import (
 	"kitty/tools/tui/loop"
 	"kitty/tools/utils"
 	"kitty/tools/utils/images"
+
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 )
 
 var _ = fmt.Print
@@ -144,13 +147,13 @@ func (self *Output) assign_mime_type(available_mimes []string, aliases map[strin
 		self.remote_mime_type = "."
 		return
 	}
-	if utils.Contains(available_mimes, self.mime_type) {
+	if slices.Contains(available_mimes, self.mime_type) {
 		self.remote_mime_type = self.mime_type
 		return
 	}
 	if len(aliases[self.mime_type]) > 0 {
 		for _, alias := range aliases[self.mime_type] {
-			if utils.Contains(available_mimes, alias) {
+			if slices.Contains(available_mimes, alias) {
 				self.remote_mime_type = alias
 				return
 			}
@@ -269,7 +272,7 @@ func parse_escape_code(etype loop.EscapeCodeType, data []byte) (metadata map[str
 func parse_aliases(raw []string) (map[string][]string, error) {
 	ans := make(map[string][]string, len(raw))
 	for _, x := range raw {
-		k, v, found := utils.Cut(x, "=")
+		k, v, found := strings.Cut(x, "=")
 		if !found {
 			return nil, fmt.Errorf("%s is not valid MIME alias specification", x)
 		}
@@ -340,7 +343,7 @@ func run_get_loop(opts *Options, args []string) (err error) {
 		if reading_available_mimes {
 			switch metadata["status"] {
 			case "DATA":
-				available_mimes = strings.Split(utils.UnsafeBytesToString(payload), " ")
+				available_mimes = utils.Map(strings.TrimSpace, strings.Split(utils.UnsafeBytesToString(payload), " "))
 			case "OK":
 			case "DONE":
 				reading_available_mimes = false
@@ -361,7 +364,7 @@ func run_get_loop(opts *Options, args []string) (err error) {
 					}
 				}
 				if len(requested_mimes) > 0 {
-					lp.QueueWriteString(encode(basic_metadata, strings.Join(utils.Keys(requested_mimes), " ")))
+					lp.QueueWriteString(encode(basic_metadata, strings.Join(maps.Keys(requested_mimes), " ")))
 				} else {
 					lp.Quit(0)
 				}

kittens/diff/README.asciidoc

@@ -1 +0,0 @@
-See https://sw.kovidgoyal.net/kitty/kittens/diff/

kittens/diff/__init__.py

@@ -1,8 +1,9 @@
-class GlobalData:
+from typing import Dict
-
-    def __init__(self) -> None:
-        self.title = ''
-        self.cmd = ''
 
 
-global_data = GlobalData
+def syntax_aliases(x: str) -> Dict[str, str]:
+    ans = {}
+    for x in x.split():
+        k, _, v = x.partition(':')
+        ans[k] = v
+    return ans

kittens/diff/collect.go

@@ -0,0 +1,390 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"crypto/md5"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"unicode/utf8"
+
+	"kitty/tools/utils"
+)
+
+var _ = fmt.Print
+var path_name_map, remote_dirs map[string]string
+
+var mimetypes_cache, data_cache, hash_cache *utils.LRUCache[string, string]
+var size_cache *utils.LRUCache[string, int64]
+var lines_cache *utils.LRUCache[string, []string]
+var highlighted_lines_cache *utils.LRUCache[string, []string]
+var is_text_cache *utils.LRUCache[string, bool]
+
+func init_caches() {
+	path_name_map = make(map[string]string, 32)
+	remote_dirs = make(map[string]string, 32)
+	const sz = 4096
+	size_cache = utils.NewLRUCache[string, int64](sz)
+	mimetypes_cache = utils.NewLRUCache[string, string](sz)
+	data_cache = utils.NewLRUCache[string, string](sz)
+	is_text_cache = utils.NewLRUCache[string, bool](sz)
+	lines_cache = utils.NewLRUCache[string, []string](sz)
+	highlighted_lines_cache = utils.NewLRUCache[string, []string](sz)
+	hash_cache = utils.NewLRUCache[string, string](sz)
+}
+
+func add_remote_dir(val string) {
+	x := filepath.Base(val)
+	idx := strings.LastIndex(x, "-")
+	if idx > -1 {
+		x = x[idx+1:]
+	} else {
+		x = ""
+	}
+	remote_dirs[val] = x
+}
+
+func mimetype_for_path(path string) string {
+	return mimetypes_cache.MustGetOrCreate(path, func(path string) string {
+		mt := utils.GuessMimeTypeWithFileSystemAccess(path)
+		if mt == "" {
+			mt = "application/octet-stream"
+		}
+		if utils.KnownTextualMimes[mt] {
+			if _, a, found := strings.Cut(mt, "/"); found {
+				mt = "text/" + a
+			}
+		}
+		return mt
+	})
+}
+
+func data_for_path(path string) (string, error) {
+	return data_cache.GetOrCreate(path, func(path string) (string, error) {
+		ans, err := os.ReadFile(path)
+		return utils.UnsafeBytesToString(ans), err
+	})
+}
+
+func size_for_path(path string) (int64, error) {
+	return size_cache.GetOrCreate(path, func(path string) (int64, error) {
+		s, err := os.Stat(path)
+		if err != nil {
+			return 0, err
+		}
+		return s.Size(), nil
+	})
+}
+
+func is_image(path string) bool {
+	return strings.HasPrefix(mimetype_for_path(path), "image/")
+}
+
+func is_path_text(path string) bool {
+	return is_text_cache.MustGetOrCreate(path, func(path string) bool {
+		if is_image(path) {
+			return false
+		}
+		s1, err := os.Stat(path)
+		if err == nil {
+			s2, err := os.Stat("/dev/null")
+			if err == nil && os.SameFile(s1, s2) {
+				return false
+			}
+		}
+		d, err := data_for_path(path)
+		if err != nil {
+			return false
+		}
+		return utf8.ValidString(d)
+	})
+}
+
+func hash_for_path(path string) (string, error) {
+	return hash_cache.GetOrCreate(path, func(path string) (string, error) {
+		ans, err := data_for_path(path)
+		if err != nil {
+			return "", err
+		}
+		hash := md5.Sum(utils.UnsafeStringToBytes(ans))
+		return utils.UnsafeBytesToString(hash[:]), err
+	})
+
+}
+
+// Remove all control codes except newlines
+func sanitize_control_codes(x string) string {
+	pat := utils.MustCompile("[\x00-\x09\x0b-\x1f\x7f\u0080-\u009f]")
+	return pat.ReplaceAllLiteralString(x, "░")
+}
+
+func sanitize_tabs_and_carriage_returns(x string) string {
+	return strings.NewReplacer("\t", conf.Replace_tab_by, "\r", "⏎").Replace(x)
+}
+
+func sanitize(x string) string {
+	return sanitize_control_codes(sanitize_tabs_and_carriage_returns(x))
+}
+
+func text_to_lines(text string) []string {
+	lines := make([]string, 0, 512)
+	splitlines_like_git(text, false, func(line string) { lines = append(lines, line) })
+	return lines
+}
+
+func lines_for_path(path string) ([]string, error) {
+	return lines_cache.GetOrCreate(path, func(path string) ([]string, error) {
+		ans, err := data_for_path(path)
+		if err != nil {
+			return nil, err
+		}
+		return text_to_lines(sanitize(ans)), nil
+	})
+}
+
+func highlighted_lines_for_path(path string) ([]string, error) {
+	plain_lines, err := lines_for_path(path)
+	if err != nil {
+		return nil, err
+	}
+	if ans, found := highlighted_lines_cache.Get(path); found && len(ans) == len(plain_lines) {
+		return ans, nil
+	}
+	return plain_lines, nil
+}
+
+type Collection struct {
+	changes, renames, type_map map[string]string
+	adds, removes              *utils.Set[string]
+	all_paths                  []string
+	paths_to_highlight         *utils.Set[string]
+	added_count, removed_count int
+}
+
+func (self *Collection) add_change(left, right string) {
+	self.changes[left] = right
+	self.all_paths = append(self.all_paths, left)
+	self.paths_to_highlight.Add(left)
+	self.paths_to_highlight.Add(right)
+	self.type_map[left] = `diff`
+}
+
+func (self *Collection) add_rename(left, right string) {
+	self.renames[left] = right
+	self.all_paths = append(self.all_paths, left)
+	self.type_map[left] = `rename`
+}
+
+func (self *Collection) add_add(right string) {
+	self.adds.Add(right)
+	self.all_paths = append(self.all_paths, right)
+	self.paths_to_highlight.Add(right)
+	self.type_map[right] = `add`
+	if is_path_text(right) {
+		num, _ := lines_for_path(right)
+		self.added_count += len(num)
+	}
+}
+
+func (self *Collection) add_removal(left string) {
+	self.removes.Add(left)
+	self.all_paths = append(self.all_paths, left)
+	self.paths_to_highlight.Add(left)
+	self.type_map[left] = `removal`
+	if is_path_text(left) {
+		num, _ := lines_for_path(left)
+		self.removed_count += len(num)
+	}
+}
+
+func (self *Collection) finalize() {
+	utils.StableSortWithKey(self.all_paths, func(path string) string {
+		return path_name_map[path]
+	})
+}
+
+func (self *Collection) Len() int { return len(self.all_paths) }
+
+func (self *Collection) Items() int { return len(self.all_paths) }
+
+func (self *Collection) Apply(f func(path, typ, changed_path string) error) error {
+	for _, path := range self.all_paths {
+		typ := self.type_map[path]
+		changed_path := ""
+		switch typ {
+		case "diff":
+			changed_path = self.changes[path]
+		case "rename":
+			changed_path = self.renames[path]
+		}
+		if err := f(path, typ, changed_path); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func allowed(path string, patterns ...string) bool {
+	name := filepath.Base(path)
+	for _, pat := range patterns {
+		if matched, err := filepath.Match(pat, name); err == nil && matched {
+			return false
+		}
+	}
+	return true
+}
+
+func remote_hostname(path string) (string, string) {
+	for q, val := range remote_dirs {
+		if strings.HasPrefix(path, q) {
+			return q, val
+		}
+	}
+	return "", ""
+}
+
+func resolve_remote_name(path, defval string) string {
+	remote_dir, rh := remote_hostname(path)
+	if remote_dir != "" && rh != "" {
+		r, err := filepath.Rel(remote_dir, path)
+		if err == nil {
+			return rh + ":" + r
+		}
+	}
+	return defval
+}
+
+func walk(base string, patterns []string, names *utils.Set[string], pmap, path_name_map map[string]string) error {
+	base, err := filepath.Abs(base)
+	if err != nil {
+		return err
+	}
+	return filepath.WalkDir(base, func(path string, d fs.DirEntry, err error) error {
+		is_allowed := allowed(path, patterns...)
+		if !is_allowed {
+			if d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		if d.IsDir() {
+			return nil
+		}
+		path, err = filepath.Abs(path)
+		if err != nil {
+			return err
+		}
+		name, err := filepath.Rel(base, path)
+		if err != nil {
+			return err
+		}
+		if name != "." {
+			path_name_map[path] = name
+			names.Add(name)
+			pmap[name] = path
+		}
+		return nil
+	})
+}
+
+func (self *Collection) collect_files(left, right string) error {
+	left_names, right_names := utils.NewSet[string](16), utils.NewSet[string](16)
+	left_path_map, right_path_map := make(map[string]string, 16), make(map[string]string, 16)
+	err := walk(left, conf.Ignore_name, left_names, left_path_map, path_name_map)
+	if err != nil {
+		return err
+	}
+	err = walk(right, conf.Ignore_name, right_names, right_path_map, path_name_map)
+	common_names := left_names.Intersect(right_names)
+	changed_names := utils.NewSet[string](common_names.Len())
+	for n := range common_names.Iterable() {
+		ld, err := data_for_path(left_path_map[n])
+		var rd string
+		if err == nil {
+			rd, err = data_for_path(right_path_map[n])
+		}
+		if err != nil {
+			return err
+		}
+		if ld != rd {
+			changed_names.Add(n)
+			self.add_change(left_path_map[n], right_path_map[n])
+		}
+	}
+	removed := left_names.Subtract(common_names)
+	added := right_names.Subtract(common_names)
+	ahash, rhash := make(map[string]string, added.Len()), make(map[string]string, removed.Len())
+	for a := range added.Iterable() {
+		ahash[a], err = hash_for_path(right_path_map[a])
+		if err != nil {
+			return err
+		}
+	}
+	for r := range removed.Iterable() {
+		rhash[r], err = hash_for_path(left_path_map[r])
+		if err != nil {
+			return err
+		}
+	}
+	for name, rh := range rhash {
+		found := false
+		for n, ah := range ahash {
+			if ah == rh {
+				ld, _ := data_for_path(left_path_map[name])
+				rd, _ := data_for_path(right_path_map[n])
+				if ld == rd {
+					self.add_rename(left_path_map[name], right_path_map[n])
+					added.Discard(n)
+					found = true
+					break
+				}
+			}
+		}
+		if !found {
+			self.add_removal(left_path_map[name])
+		}
+	}
+	for name := range added.Iterable() {
+		self.add_add(right_path_map[name])
+	}
+	return nil
+}
+
+func create_collection(left, right string) (ans *Collection, err error) {
+	ans = &Collection{
+		changes:            make(map[string]string),
+		renames:            make(map[string]string),
+		type_map:           make(map[string]string),
+		adds:               utils.NewSet[string](32),
+		removes:            utils.NewSet[string](32),
+		paths_to_highlight: utils.NewSet[string](32),
+		all_paths:          make([]string, 0, 32),
+	}
+	left_stat, err := os.Stat(left)
+	if err != nil {
+		return nil, err
+	}
+	if left_stat.IsDir() {
+		err = ans.collect_files(left, right)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		pl, err := filepath.Abs(left)
+		if err != nil {
+			return nil, err
+		}
+		pr, err := filepath.Abs(right)
+		if err != nil {
+			return nil, err
+		}
+		path_name_map[pl] = resolve_remote_name(pl, left)
+		path_name_map[pr] = resolve_remote_name(pr, right)
+		ans.add_change(pl, pr)
+	}
+	ans.finalize()
+	return ans, err
+}

kittens/diff/collect.py

@@ -1,233 +0,0 @@
-#!/usr/bin/env python3
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-import os
-from contextlib import suppress
-from fnmatch import fnmatch
-from functools import lru_cache
-from hashlib import md5
-from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Sequence, Set, Tuple, Union
-
-from kitty.guess_mime_type import guess_type
-from kitty.utils import control_codes_pat
-
-if TYPE_CHECKING:
-    from .highlight import DiffHighlight
-
-
-path_name_map: Dict[str, str] = {}
-remote_dirs: Dict[str, str] = {}
-
-
-def add_remote_dir(val: str) -> None:
-    remote_dirs[val] = os.path.basename(val).rpartition('-')[-1]
-
-
-class Segment:
-
-    __slots__ = ('start', 'end', 'start_code', 'end_code')
-
-    def __init__(self, start: int, start_code: str):
-        self.start = start
-        self.start_code = start_code
-        self.end: Optional[int] = None
-        self.end_code: Optional[str] = None
-
-    def __repr__(self) -> str:
-        return f'Segment(start={self.start!r}, start_code={self.start_code!r}, end={self.end!r}, end_code={self.end_code!r})'
-
-
-class Collection:
-
-    ignore_names: Tuple[str, ...] = ()
-
-    def __init__(self) -> None:
-        self.changes: Dict[str, str] = {}
-        self.renames: Dict[str, str] = {}
-        self.adds: Set[str] = set()
-        self.removes: Set[str] = set()
-        self.all_paths: List[str] = []
-        self.type_map: Dict[str, str] = {}
-        self.added_count = self.removed_count = 0
-
-    def add_change(self, left_path: str, right_path: str) -> None:
-        self.changes[left_path] = right_path
-        self.all_paths.append(left_path)
-        self.type_map[left_path] = 'diff'
-
-    def add_rename(self, left_path: str, right_path: str) -> None:
-        self.renames[left_path] = right_path
-        self.all_paths.append(left_path)
-        self.type_map[left_path] = 'rename'
-
-    def add_add(self, right_path: str) -> None:
-        self.adds.add(right_path)
-        self.all_paths.append(right_path)
-        self.type_map[right_path] = 'add'
-        if isinstance(data_for_path(right_path), str):
-            self.added_count += len(lines_for_path(right_path))
-
-    def add_removal(self, left_path: str) -> None:
-        self.removes.add(left_path)
-        self.all_paths.append(left_path)
-        self.type_map[left_path] = 'removal'
-        if isinstance(data_for_path(left_path), str):
-            self.removed_count += len(lines_for_path(left_path))
-
-    def finalize(self) -> None:
-        def key(x: str) -> str:
-            return path_name_map.get(x, '')
-
-        self.all_paths.sort(key=key)
-
-    def __iter__(self) -> Iterator[Tuple[str, str, Optional[str]]]:
-        for path in self.all_paths:
-            typ = self.type_map[path]
-            if typ == 'diff':
-                data: Optional[str] = self.changes[path]
-            elif typ == 'rename':
-                data = self.renames[path]
-            else:
-                data = None
-            yield path, typ, data
-
-    def __len__(self) -> int:
-        return len(self.all_paths)
-
-
-def remote_hostname(path: str) -> Tuple[Optional[str], Optional[str]]:
-    for q in remote_dirs:
-        if path.startswith(q):
-            return q, remote_dirs[q]
-    return None, None
-
-
-def resolve_remote_name(path: str, default: str) -> str:
-    remote_dir, rh = remote_hostname(path)
-    if remote_dir and rh:
-        return f'{rh}:{os.path.relpath(path, remote_dir)}'
-    return default
-
-
-def allowed_items(items: Sequence[str], ignore_patterns: Sequence[str]) -> Iterator[str]:
-    for name in items:
-        for pat in ignore_patterns:
-            if fnmatch(name, pat):
-                break
-        else:
-            yield name
-
-
-def walk(base: str, names: Set[str], pmap: Dict[str, str], ignore_names: Tuple[str, ...]) -> None:
-    for dirpath, dirnames, filenames in os.walk(base):
-        dirnames[:] = allowed_items(dirnames, ignore_names)
-        for filename in allowed_items(filenames, ignore_names):
-            path = os.path.abspath(os.path.join(dirpath, filename))
-            path_name_map[path] = name = os.path.relpath(path, base)
-            names.add(name)
-            pmap[name] = path
-
-
-def collect_files(collection: Collection, left: str, right: str) -> None:
-    left_names: Set[str] = set()
-    right_names: Set[str] = set()
-    left_path_map: Dict[str, str] = {}
-    right_path_map: Dict[str, str] = {}
-
-    walk(left, left_names, left_path_map, collection.ignore_names)
-    walk(right, right_names, right_path_map, collection.ignore_names)
-
-    common_names = left_names & right_names
-    changed_names = {n for n in common_names if data_for_path(left_path_map[n]) != data_for_path(right_path_map[n])}
-    for n in changed_names:
-        collection.add_change(left_path_map[n], right_path_map[n])
-
-    removed = left_names - common_names
-    added = right_names - common_names
-    ahash = {a: hash_for_path(right_path_map[a]) for a in added}
-    rhash = {r: hash_for_path(left_path_map[r]) for r in removed}
-    for name, rh in rhash.items():
-        for n, ah in ahash.items():
-            if ah == rh and data_for_path(left_path_map[name]) == data_for_path(right_path_map[n]):
-                collection.add_rename(left_path_map[name], right_path_map[n])
-                added.discard(n)
-                break
-        else:
-            collection.add_removal(left_path_map[name])
-
-    for name in added:
-        collection.add_add(right_path_map[name])
-
-
-def sanitize(text: str) -> str:
-    ntext = text.replace('\r\n', '⏎\n')
-    return control_codes_pat().sub('░', ntext)
-
-
-@lru_cache(maxsize=1024)
-def mime_type_for_path(path: str) -> str:
-    return guess_type(path, allow_filesystem_access=True) or 'application/octet-stream'
-
-
-@lru_cache(maxsize=1024)
-def raw_data_for_path(path: str) -> bytes:
-    with open(path, 'rb') as f:
-        return f.read()
-
-
-def is_image(path: Optional[str]) -> bool:
-    return mime_type_for_path(path).startswith('image/') if path else False
-
-
-@lru_cache(maxsize=1024)
-def data_for_path(path: str) -> Union[str, bytes]:
-    raw_bytes = raw_data_for_path(path)
-    if not is_image(path) and not os.path.samefile(path, os.devnull):
-        with suppress(UnicodeDecodeError):
-            return raw_bytes.decode('utf-8')
-    return raw_bytes
-
-
-class LinesForPath:
-
-    replace_tab_by = ' ' * 4
-
-    @lru_cache(maxsize=1024)
-    def __call__(self, path: str) -> Tuple[str, ...]:
-        data = data_for_path(path)
-        assert isinstance(data, str)
-        data = data.replace('\t', self.replace_tab_by)
-        return tuple(sanitize(data).splitlines())
-
-
-lines_for_path = LinesForPath()
-
-
-@lru_cache(maxsize=1024)
-def hash_for_path(path: str) -> bytes:
-    return md5(raw_data_for_path(path)).digest()
-
-
-def create_collection(left: str, right: str) -> Collection:
-    collection = Collection()
-    if os.path.isdir(left):
-        collect_files(collection, left, right)
-    else:
-        pl, pr = os.path.abspath(left), os.path.abspath(right)
-        path_name_map[pl] = resolve_remote_name(pl, left)
-        path_name_map[pr] = resolve_remote_name(pr, right)
-        collection.add_change(pl, pr)
-    collection.finalize()
-    return collection
-
-
-highlight_data: Dict[str, 'DiffHighlight'] = {}
-
-
-def set_highlight_data(data: Dict[str, 'DiffHighlight']) -> None:
-    global highlight_data
-    highlight_data = data
-
-
-def highlights_for_path(path: str) -> 'DiffHighlight':
-    return highlight_data.get(path, [])

kittens/diff/collect_test.go

@@ -0,0 +1,53 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"kitty/tools/utils"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+var _ = fmt.Print
+
+func TestDiffCollectWalk(t *testing.T) {
+	tdir := t.TempDir()
+	j := func(x ...string) string { return filepath.Join(append([]string{tdir}, x...)...) }
+	os.MkdirAll(j("a", "b"), 0o700)
+	os.WriteFile(j("a/b/c"), nil, 0o600)
+	os.WriteFile(j("b"), nil, 0o600)
+	os.WriteFile(j("d"), nil, 0o600)
+	os.WriteFile(j("e"), nil, 0o600)
+	os.WriteFile(j("#d#"), nil, 0o600)
+	os.WriteFile(j("e~"), nil, 0o600)
+	os.MkdirAll(j("f"), 0o700)
+	os.WriteFile(j("f/g"), nil, 0o600)
+	os.WriteFile(j("h space"), nil, 0o600)
+
+	expected_names := utils.NewSetWithItems("d", "e", "f/g", "h space")
+	expected_pmap := map[string]string{
+		"d":       j("d"),
+		"e":       j("e"),
+		"f/g":     j("f/g"),
+		"h space": j("h space"),
+	}
+	names := utils.NewSet[string](16)
+	pmap := make(map[string]string, 16)
+	if err := walk(tdir, []string{"*~", "#*#", "b"}, names, pmap, map[string]string{}); err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(
+		utils.Sort(expected_names.AsSlice(), func(a, b string) bool { return a < b }),
+		utils.Sort(names.AsSlice(), func(a, b string) bool { return a < b }),
+	); diff != "" {
+		t.Fatal(diff)
+	}
+	if diff := cmp.Diff(expected_pmap, pmap); diff != "" {
+		t.Fatal(diff)
+	}
+}

kittens/diff/config.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python3
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-import os
-from typing import Any, Dict, Iterable, Optional
-
-from kitty.cli_stub import DiffCLIOptions
-from kitty.conf.utils import load_config as _load_config
-from kitty.conf.utils import parse_config_base, resolve_config
-from kitty.constants import config_dir
-from kitty.rgb import color_as_sgr
-
-from .options.types import Options as DiffOptions
-from .options.types import defaults
-
-formats: Dict[str, str] = {
-    'title': '',
-    'margin': '',
-    'text': '',
-}
-
-
-def set_formats(opts: DiffOptions) -> None:
-    formats['text'] = '48' + color_as_sgr(opts.background)
-    formats['title'] = '38' + color_as_sgr(opts.title_fg) + ';48' + color_as_sgr(opts.title_bg) + ';1'
-    formats['margin'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.margin_bg)
-    formats['added_margin'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.added_margin_bg)
-    formats['removed_margin'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.removed_margin_bg)
-    formats['added'] = '48' + color_as_sgr(opts.added_bg)
-    formats['removed'] = '48' + color_as_sgr(opts.removed_bg)
-    formats['filler'] = '48' + color_as_sgr(opts.filler_bg)
-    formats['margin_filler'] = '48' + color_as_sgr(opts.margin_filler_bg or opts.filler_bg)
-    formats['hunk_margin'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.hunk_margin_bg)
-    formats['hunk'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.hunk_bg)
-    formats['removed_highlight'] = '48' + color_as_sgr(opts.highlight_removed_bg)
-    formats['added_highlight'] = '48' + color_as_sgr(opts.highlight_added_bg)
-
-
-SYSTEM_CONF = '/etc/xdg/kitty/diff.conf'
-defconf = os.path.join(config_dir, 'diff.conf')
-
-
-def load_config(*paths: str, overrides: Optional[Iterable[str]] = None) -> DiffOptions:
-    from .options.parse import create_result_dict, merge_result_dicts, parse_conf_item
-
-    def parse_config(lines: Iterable[str]) -> Dict[str, Any]:
-        ans: Dict[str, Any] = create_result_dict()
-        parse_config_base(
-            lines,
-            parse_conf_item,
-            ans,
-        )
-        return ans
-
-    overrides = tuple(overrides) if overrides is not None else ()
-    opts_dict, paths = _load_config(defaults, parse_config, merge_result_dicts, *paths, overrides=overrides)
-    opts = DiffOptions(opts_dict)
-    opts.config_paths = paths
-    opts.config_overrides = overrides
-    return opts
-
-
-def init_config(args: DiffCLIOptions) -> DiffOptions:
-    config = tuple(resolve_config(SYSTEM_CONF, defconf, args.config))
-    overrides = (a.replace('=', ' ', 1) for a in args.override or ())
-    opts = load_config(*config, overrides=overrides)
-    set_formats(opts)
-    for (sc, action) in opts.map:
-        opts.key_definitions[sc] = action
-    return opts

kittens/diff/diff.go

@@ -0,0 +1,264 @@
+// Copied from the Go stdlib, with modifications.
+//https://github.com/golang/go/raw/master/src/internal/diff/diff.go
+
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package diff
+
+import (
+	"bytes"
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// A pair is a pair of values tracked for both the x and y side of a diff.
+// It is typically a pair of line indexes.
+type pair struct{ x, y int }
+
+// Diff returns an anchored diff of the two texts old and new
+// in the “unified diff” format. If old and new are identical,
+// Diff returns a nil slice (no output).
+//
+// Unix diff implementations typically look for a diff with
+// the smallest number of lines inserted and removed,
+// which can in the worst case take time quadratic in the
+// number of lines in the texts. As a result, many implementations
+// either can be made to run for a long time or cut off the search
+// after a predetermined amount of work.
+//
+// In contrast, this implementation looks for a diff with the
+// smallest number of “unique” lines inserted and removed,
+// where unique means a line that appears just once in both old and new.
+// We call this an “anchored diff” because the unique lines anchor
+// the chosen matching regions. An anchored diff is usually clearer
+// than a standard diff, because the algorithm does not try to
+// reuse unrelated blank lines or closing braces.
+// The algorithm also guarantees to run in O(n log n) time
+// instead of the standard O(n²) time.
+//
+// Some systems call this approach a “patience diff,” named for
+// the “patience sorting” algorithm, itself named for a solitaire card game.
+// We avoid that name for two reasons. First, the name has been used
+// for a few different variants of the algorithm, so it is imprecise.
+// Second, the name is frequently interpreted as meaning that you have
+// to wait longer (to be patient) for the diff, meaning that it is a slower algorithm,
+// when in fact the algorithm is faster than the standard one.
+func Diff(oldName, old, newName, new string, num_of_context_lines int) []byte {
+	if old == new {
+		return nil
+	}
+	x := lines(old)
+	y := lines(new)
+
+	// Print diff header.
+	var out bytes.Buffer
+	fmt.Fprintf(&out, "diff %s %s\n", oldName, newName)
+	fmt.Fprintf(&out, "--- %s\n", oldName)
+	fmt.Fprintf(&out, "+++ %s\n", newName)
+
+	// Loop over matches to consider,
+	// expanding each match to include surrounding lines,
+	// and then printing diff chunks.
+	// To avoid setup/teardown cases outside the loop,
+	// tgs returns a leading {0,0} and trailing {len(x), len(y)} pair
+	// in the sequence of matches.
+	var (
+		done  pair     // printed up to x[:done.x] and y[:done.y]
+		chunk pair     // start lines of current chunk
+		count pair     // number of lines from each side in current chunk
+		ctext []string // lines for current chunk
+	)
+	for _, m := range tgs(x, y) {
+		if m.x < done.x {
+			// Already handled scanning forward from earlier match.
+			continue
+		}
+
+		// Expand matching lines as far possible,
+		// establishing that x[start.x:end.x] == y[start.y:end.y].
+		// Note that on the first (or last) iteration we may (or definitey do)
+		// have an empty match: start.x==end.x and start.y==end.y.
+		start := m
+		for start.x > done.x && start.y > done.y && x[start.x-1] == y[start.y-1] {
+			start.x--
+			start.y--
+		}
+		end := m
+		for end.x < len(x) && end.y < len(y) && x[end.x] == y[end.y] {
+			end.x++
+			end.y++
+		}
+
+		// Emit the mismatched lines before start into this chunk.
+		// (No effect on first sentinel iteration, when start = {0,0}.)
+		for _, s := range x[done.x:start.x] {
+			ctext = append(ctext, "-"+s)
+			count.x++
+		}
+		for _, s := range y[done.y:start.y] {
+			ctext = append(ctext, "+"+s)
+			count.y++
+		}
+
+		// If we're not at EOF and have too few common lines,
+		// the chunk includes all the common lines and continues.
+		C := num_of_context_lines // number of context lines
+		if (end.x < len(x) || end.y < len(y)) &&
+			(end.x-start.x < C || (len(ctext) > 0 && end.x-start.x < 2*C)) {
+			for _, s := range x[start.x:end.x] {
+				ctext = append(ctext, " "+s)
+				count.x++
+				count.y++
+			}
+			done = end
+			continue
+		}
+
+		// End chunk with common lines for context.
+		if len(ctext) > 0 {
+			n := end.x - start.x
+			if n > C {
+				n = C
+			}
+			for _, s := range x[start.x : start.x+n] {
+				ctext = append(ctext, " "+s)
+				count.x++
+				count.y++
+			}
+			done = pair{start.x + n, start.y + n}
+
+			// Format and emit chunk.
+			// Convert line numbers to 1-indexed.
+			// Special case: empty file shows up as 0,0 not 1,0.
+			if count.x > 0 {
+				chunk.x++
+			}
+			if count.y > 0 {
+				chunk.y++
+			}
+			fmt.Fprintf(&out, "@@ -%d,%d +%d,%d @@\n", chunk.x, count.x, chunk.y, count.y)
+			for _, s := range ctext {
+				out.WriteString(s)
+			}
+			count.x = 0
+			count.y = 0
+			ctext = ctext[:0]
+		}
+
+		// If we reached EOF, we're done.
+		if end.x >= len(x) && end.y >= len(y) {
+			break
+		}
+
+		// Otherwise start a new chunk.
+		chunk = pair{end.x - C, end.y - C}
+		for _, s := range x[chunk.x:end.x] {
+			ctext = append(ctext, " "+s)
+			count.x++
+			count.y++
+		}
+		done = end
+	}
+
+	return out.Bytes()
+}
+
+// lines returns the lines in the file x, including newlines.
+// If the file does not end in a newline, one is supplied
+// along with a warning about the missing newline.
+func lines(x string) []string {
+	l := strings.SplitAfter(x, "\n")
+	if l[len(l)-1] == "" {
+		l = l[:len(l)-1]
+	} else {
+		// Treat last line as having a message about the missing newline attached,
+		// using the same text as BSD/GNU diff (including the leading backslash).
+		l[len(l)-1] += "\n\\ No newline at end of file\n"
+	}
+	return l
+}
+
+// tgs returns the pairs of indexes of the longest common subsequence
+// of unique lines in x and y, where a unique line is one that appears
+// once in x and once in y.
+//
+// The longest common subsequence algorithm is as described in
+// Thomas G. Szymanski, “A Special Case of the Maximal Common
+// Subsequence Problem,” Princeton TR #170 (January 1975),
+// available at https://research.swtch.com/tgs170.pdf.
+func tgs(x, y []string) []pair {
+	// Count the number of times each string appears in a and b.
+	// We only care about 0, 1, many, counted as 0, -1, -2
+	// for the x side and 0, -4, -8 for the y side.
+	// Using negative numbers now lets us distinguish positive line numbers later.
+	m := make(map[string]int)
+	for _, s := range x {
+		if c := m[s]; c > -2 {
+			m[s] = c - 1
+		}
+	}
+	for _, s := range y {
+		if c := m[s]; c > -8 {
+			m[s] = c - 4
+		}
+	}
+
+	// Now unique strings can be identified by m[s] = -1+-4.
+	//
+	// Gather the indexes of those strings in x and y, building:
+	//	xi[i] = increasing indexes of unique strings in x.
+	//	yi[i] = increasing indexes of unique strings in y.
+	//	inv[i] = index j such that x[xi[i]] = y[yi[j]].
+	var xi, yi, inv []int
+	for i, s := range y {
+		if m[s] == -1+-4 {
+			m[s] = len(yi)
+			yi = append(yi, i)
+		}
+	}
+	for i, s := range x {
+		if j, ok := m[s]; ok && j >= 0 {
+			xi = append(xi, i)
+			inv = append(inv, j)
+		}
+	}
+
+	// Apply Algorithm A from Szymanski's paper.
+	// In those terms, A = J = inv and B = [0, n).
+	// We add sentinel pairs {0,0}, and {len(x),len(y)}
+	// to the returned sequence, to help the processing loop.
+	J := inv
+	n := len(xi)
+	T := make([]int, n)
+	L := make([]int, n)
+	for i := range T {
+		T[i] = n + 1
+	}
+	for i := 0; i < n; i++ {
+		k := sort.Search(n, func(k int) bool {
+			return T[k] >= J[i]
+		})
+		T[k] = J[i]
+		L[i] = k + 1
+	}
+	k := 0
+	for _, v := range L {
+		if k < v {
+			k = v
+		}
+	}
+	seq := make([]pair, 2+k)
+	seq[1+k] = pair{len(x), len(y)} // sentinel at end
+	lastj := n
+	for i := n - 1; i >= 0; i-- {
+		if L[i] == k && J[i] < lastj {
+			seq[k] = pair{xi[i], yi[J[i]]}
+			k--
+		}
+	}
+	seq[0] = pair{0, 0} // sentinel at start
+	return seq
+}

kittens/diff/diff_speedup.pyi

@@ -1,13 +0,0 @@
-from typing import List, Optional, Tuple
-
-from .collect import Segment
-
-def split_with_highlights(
-    line: str, truncate_points: List[int], fg_highlights: List[Segment],
-    bg_highlight: Optional[Segment]
-) -> List[str]:
-    pass
-
-
-def changed_center(left_prefix: str, right_postfix: str) -> Tuple[int, int]:
-    pass

kittens/diff/highlight.go

@@ -0,0 +1,207 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"kitty/tools/utils"
+	"kitty/tools/utils/images"
+
+	"github.com/alecthomas/chroma/v2"
+	"github.com/alecthomas/chroma/v2/lexers"
+	"github.com/alecthomas/chroma/v2/styles"
+)
+
+var _ = fmt.Print
+var _ = os.WriteFile
+
+var ErrNoLexer = errors.New("No lexer available for this format")
+var DefaultStyle = (&utils.Once[*chroma.Style]{Run: func() *chroma.Style {
+	// Default style generated by python style.py default pygments.styles.default.DefaultStyle
+	// with https://raw.githubusercontent.com/alecthomas/chroma/master/_tools/style.py
+	return styles.Register(chroma.MustNewStyle("default", chroma.StyleEntries{
+		chroma.TextWhitespace:        "#bbbbbb",
+		chroma.Comment:               "italic #3D7B7B",
+		chroma.CommentPreproc:        "noitalic #9C6500",
+		chroma.Keyword:               "bold #008000",
+		chroma.KeywordPseudo:         "nobold",
+		chroma.KeywordType:           "nobold #B00040",
+		chroma.Operator:              "#666666",
+		chroma.OperatorWord:          "bold #AA22FF",
+		chroma.NameBuiltin:           "#008000",
+		chroma.NameFunction:          "#0000FF",
+		chroma.NameClass:             "bold #0000FF",
+		chroma.NameNamespace:         "bold #0000FF",
+		chroma.NameException:         "bold #CB3F38",
+		chroma.NameVariable:          "#19177C",
+		chroma.NameConstant:          "#880000",
+		chroma.NameLabel:             "#767600",
+		chroma.NameEntity:            "bold #717171",
+		chroma.NameAttribute:         "#687822",
+		chroma.NameTag:               "bold #008000",
+		chroma.NameDecorator:         "#AA22FF",
+		chroma.LiteralString:         "#BA2121",
+		chroma.LiteralStringDoc:      "italic",
+		chroma.LiteralStringInterpol: "bold #A45A77",
+		chroma.LiteralStringEscape:   "bold #AA5D1F",
+		chroma.LiteralStringRegex:    "#A45A77",
+		chroma.LiteralStringSymbol:   "#19177C",
+		chroma.LiteralStringOther:    "#008000",
+		chroma.LiteralNumber:         "#666666",
+		chroma.GenericHeading:        "bold #000080",
+		chroma.GenericSubheading:     "bold #800080",
+		chroma.GenericDeleted:        "#A00000",
+		chroma.GenericInserted:       "#008400",
+		chroma.GenericError:          "#E40000",
+		chroma.GenericEmph:           "italic",
+		chroma.GenericStrong:         "bold",
+		chroma.GenericPrompt:         "bold #000080",
+		chroma.GenericOutput:         "#717171",
+		chroma.GenericTraceback:      "#04D",
+		chroma.Error:                 "border:#FF0000",
+		chroma.Background:            " bg:#f8f8f8",
+	}))
+}}).Get
+
+// Clear the background colour.
+func clear_background(style *chroma.Style) *chroma.Style {
+	builder := style.Builder()
+	bg := builder.Get(chroma.Background)
+	bg.Background = 0
+	bg.NoInherit = true
+	builder.AddEntry(chroma.Background, bg)
+	style, _ = builder.Build()
+	return style
+}
+
+func ansi_formatter(w io.Writer, style *chroma.Style, it chroma.Iterator) error {
+	const SGR_PREFIX = "\033["
+	const SGR_SUFFIX = "m"
+	style = clear_background(style)
+	before, after := make([]byte, 0, 64), make([]byte, 0, 64)
+	nl := []byte{'\n'}
+	write_sgr := func(which []byte) {
+		if len(which) > 1 {
+			w.Write(utils.UnsafeStringToBytes(SGR_PREFIX))
+			w.Write(which[:len(which)-1])
+			w.Write(utils.UnsafeStringToBytes(SGR_SUFFIX))
+		}
+	}
+	write := func(text string) {
+		write_sgr(before)
+		w.Write(utils.UnsafeStringToBytes(text))
+		write_sgr(after)
+	}
+
+	for token := it(); token != chroma.EOF; token = it() {
+		entry := style.Get(token.Type)
+		before, after = before[:0], after[:0]
+		if !entry.IsZero() {
+			if entry.Bold == chroma.Yes {
+				before = append(before, '1', ';')
+				after = append(after, '2', '2', '1', ';')
+			}
+			if entry.Underline == chroma.Yes {
+				before = append(before, '4', ';')
+				after = append(after, '2', '4', ';')
+			}
+			if entry.Italic == chroma.Yes {
+				before = append(before, '3', ';')
+				after = append(after, '2', '3', ';')
+			}
+			if entry.Colour.IsSet() {
+				before = append(before, fmt.Sprintf("38:2:%d:%d:%d;", entry.Colour.Red(), entry.Colour.Green(), entry.Colour.Blue())...)
+				after = append(after, '3', '9', ';')
+			}
+		}
+		// independently format each line in a multiline token, needed for the diff kitten highlighting to work, also
+		// pagers like less reset SGR formatting at line boundaries
+		text := sanitize(token.Value)
+		for text != "" {
+			idx := strings.IndexByte(text, '\n')
+			if idx < 0 {
+				write(text)
+				break
+			}
+			write(text[:idx])
+			w.Write(nl)
+			text = text[idx+1:]
+		}
+	}
+	return nil
+}
+
+func highlight_file(path string) (highlighted string, err error) {
+	filename_for_detection := filepath.Base(path)
+	ext := filepath.Ext(filename_for_detection)
+	if ext != "" {
+		ext = strings.ToLower(ext[1:])
+		r := conf.Syntax_aliases[ext]
+		if r != "" {
+			filename_for_detection = "file." + r
+		}
+	}
+	text, err := data_for_path(path)
+	if err != nil {
+		return "", err
+	}
+	lexer := lexers.Match(filename_for_detection)
+	if lexer == nil {
+		if err == nil {
+			lexer = lexers.Analyse(text)
+		}
+	}
+	if lexer == nil {
+		return "", fmt.Errorf("Cannot highlight %#v: %w", path, ErrNoLexer)
+	}
+	lexer = chroma.Coalesce(lexer)
+	name := conf.Pygments_style
+	var style *chroma.Style
+	if name == "default" {
+		style = DefaultStyle()
+	} else {
+		style = styles.Get(name)
+	}
+	if style == nil {
+		if conf.Background.IsDark() && !conf.Foreground.IsDark() {
+			style = styles.Get("monokai")
+			if style == nil {
+				style = styles.Get("github-dark")
+			}
+		} else {
+			style = DefaultStyle()
+		}
+		if style == nil {
+			style = styles.Fallback
+		}
+	}
+	iterator, err := lexer.Tokenise(nil, text)
+	if err != nil {
+		return "", err
+	}
+	formatter := chroma.FormatterFunc(ansi_formatter)
+	w := strings.Builder{}
+	w.Grow(len(text) * 2)
+	err = formatter.Format(&w, style, iterator)
+	// os.WriteFile(filepath.Base(path+".highlighted"), []byte(w.String()), 0o600)
+	return w.String(), err
+}
+
+func highlight_all(paths []string) {
+	ctx := images.Context{}
+	ctx.Parallel(0, len(paths), func(nums <-chan int) {
+		for i := range nums {
+			path := paths[i]
+			raw, err := highlight_file(path)
+			if err == nil {
+				highlighted_lines_cache.Set(path, text_to_lines(raw))
+			}
+		}
+	})
+}

kittens/diff/highlight.py

@@ -1,185 +0,0 @@
-#!/usr/bin/env python3
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-import concurrent
-import os
-import re
-from concurrent.futures import ProcessPoolExecutor
-from typing import IO, Dict, Iterable, Iterator, List, Optional, Tuple, Union, cast
-
-from pygments import highlight  # type: ignore
-from pygments.formatter import Formatter  # type: ignore
-from pygments.lexers import get_lexer_for_filename  # type: ignore
-from pygments.util import ClassNotFound  # type: ignore
-
-from kitty.multiprocessing import get_process_pool_executor
-from kitty.rgb import color_as_sgr, parse_sharp
-
-from .collect import Collection, Segment, data_for_path, lines_for_path
-
-
-class StyleNotFound(Exception):
-    pass
-
-
-class DiffFormatter(Formatter):  # type: ignore
-
-    def __init__(self, style: str = 'default') -> None:
-        try:
-            Formatter.__init__(self, style=style)
-            initialized = True
-        except ClassNotFound:
-            initialized = False
-        if not initialized:
-            raise StyleNotFound(f'pygments style "{style}" not found')
-
-        self.styles: Dict[str, Tuple[str, str]] = {}
-        for token, token_style in self.style:
-            start = []
-            end = []
-            fstart = fend = ''
-            # a style item is a tuple in the following form:
-            # colors are readily specified in hex: 'RRGGBB'
-            col = token_style['color']
-            if col:
-                pc = parse_sharp(col)
-                if pc is not None:
-                    start.append('38' + color_as_sgr(pc))
-                    end.append('39')
-            if token_style['bold']:
-                start.append('1')
-                end.append('22')
-            if token_style['italic']:
-                start.append('3')
-                end.append('23')
-            if token_style['underline']:
-                start.append('4')
-                end.append('24')
-            if start:
-                fstart = '\033[{}m'.format(';'.join(start))
-                fend = '\033[{}m'.format(';'.join(end))
-            self.styles[token] = fstart, fend
-
-    def format(self, tokensource: Iterable[Tuple[str, str]], outfile: IO[str]) -> None:
-        for ttype, value in tokensource:
-            not_found = True
-            if value.rstrip('\n'):
-                while ttype and not_found:
-                    tok = self.styles.get(ttype)
-                    if tok is None:
-                        ttype = ttype[:-1]
-                    else:
-                        on, off = tok
-                        lines = value.split('\n')
-                        for line in lines:
-                            if line:
-                                outfile.write(on + line + off)
-                            if line is not lines[-1]:
-                                outfile.write('\n')
-                        not_found = False
-
-            if not_found:
-                outfile.write(value)
-
-
-formatter: Optional[DiffFormatter] = None
-
-
-def initialize_highlighter(style: str = 'default') -> None:
-    global formatter
-    formatter = DiffFormatter(style)
-
-
-def highlight_data(code: str, filename: str, aliases: Optional[Dict[str, str]] = None) -> Optional[str]:
-    if aliases:
-        base, ext = os.path.splitext(filename)
-        alias = aliases.get(ext[1:])
-        if alias is not None:
-            filename = f'{base}.{alias}'
-    try:
-        lexer = get_lexer_for_filename(filename, stripnl=False)
-    except ClassNotFound:
-        return None
-    return cast(str, highlight(code, lexer, formatter))
-
-
-split_pat = re.compile(r'(\033\[.*?m)')
-
-
-def highlight_line(line: str) -> List[Segment]:
-    ans: List[Segment] = []
-    current: Optional[Segment] = None
-    pos = 0
-    for x in split_pat.split(line):
-        if x.startswith('\033'):
-            if current is None:
-                current = Segment(pos, x)
-            else:
-                current.end = pos
-                current.end_code = x
-                ans.append(current)
-                current = None
-        else:
-            pos += len(x)
-    return ans
-
-
-DiffHighlight = List[List[Segment]]
-
-
-def highlight_for_diff(path: str, aliases: Dict[str, str]) -> DiffHighlight:
-    ans: DiffHighlight = []
-    lines = lines_for_path(path)
-    hd = highlight_data('\n'.join(lines), path, aliases)
-    if hd is not None:
-        for line in hd.splitlines():
-            ans.append(highlight_line(line))
-    return ans
-
-
-process_pool_executor: Optional[ProcessPoolExecutor] = None
-
-
-def get_highlight_processes() -> Iterator[int]:
-    if process_pool_executor is None:
-        return
-    for pid in process_pool_executor._processes:
-        yield pid
-
-
-def highlight_collection(collection: Collection, aliases: Optional[Dict[str, str]] = None) -> Union[str, Dict[str, DiffHighlight]]:
-    global process_pool_executor
-    jobs = {}
-    ans: Dict[str, DiffHighlight] = {}
-    with get_process_pool_executor(prefer_fork=True) as executor:
-        process_pool_executor = executor
-        for path, item_type, other_path in collection:
-            if item_type != 'rename':
-                for p in (path, other_path):
-                    if p:
-                        is_binary = isinstance(data_for_path(p), bytes)
-                        if not is_binary:
-                            jobs[executor.submit(highlight_for_diff, p, aliases or {})] = p
-        for future in concurrent.futures.as_completed(jobs):
-            path = jobs[future]
-            try:
-                highlights = future.result()
-            except Exception as e:
-                import traceback
-                tb = traceback.format_exc()
-                return f'Running syntax highlighting for {path} generated an exception: {e} with traceback:\n{tb}'
-            ans[path] = highlights
-    return ans
-
-
-def main() -> None:
-    # kitty +runpy "from kittens.diff.highlight import main; main()" file
-    import sys
-
-    from .options.types import defaults
-    initialize_highlighter()
-    with open(sys.argv[-1]) as f:
-        highlighted = highlight_data(f.read(), f.name, defaults.syntax_aliases)
-    if highlighted is None:
-        raise SystemExit(f'Unknown filetype: {sys.argv[-1]}')
-    print(highlighted)

kittens/diff/main.go

@@ -0,0 +1,177 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"kitty/kittens/ssh"
+	"kitty/tools/cli"
+	"kitty/tools/config"
+	"kitty/tools/tui/loop"
+	"kitty/tools/utils"
+)
+
+var _ = fmt.Print
+
+func load_config(opts *Options) (ans *Config, err error) {
+	ans = NewConfig()
+	p := config.ConfigParser{LineHandler: ans.Parse}
+	err = p.LoadConfig("diff.conf", opts.Config, opts.Override)
+	if err != nil {
+		return nil, err
+	}
+	ans.KeyboardShortcuts = config.ResolveShortcuts(ans.KeyboardShortcuts)
+	return ans, nil
+}
+
+var conf *Config
+var opts *Options
+var lp *loop.Loop
+
+func isdir(path string) bool {
+	if s, err := os.Stat(path); err == nil {
+		return s.IsDir()
+	}
+	return false
+}
+
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func get_ssh_file(hostname, rpath string) (string, error) {
+	tdir, err := os.MkdirTemp("", "*-"+hostname)
+	if err != nil {
+		return "", err
+	}
+	add_remote_dir(tdir)
+	is_abs := strings.HasPrefix(rpath, "/")
+	for strings.HasPrefix(rpath, "/") {
+		rpath = rpath[1:]
+	}
+	cmd := []string{ssh.SSHExe(), hostname, "tar", "-c", "-f", "-"}
+	if is_abs {
+		cmd = append(cmd, "-C", "/")
+	}
+	cmd = append(cmd, rpath)
+	c := exec.Command(cmd[0], cmd[1:]...)
+	c.Stdin, c.Stderr = os.Stdin, os.Stderr
+	stdout, err := c.Output()
+	if err != nil {
+		return "", fmt.Errorf("Failed to ssh into remote host %s to get file %s with error: %w", hostname, rpath, err)
+	}
+	tf := tar.NewReader(bytes.NewReader(stdout))
+	count, err := utils.ExtractAllFromTar(tf, tdir)
+	if err != nil {
+		return "", fmt.Errorf("Failed to untar data from remote host %s to get file %s with error: %w", hostname, rpath, err)
+	}
+	ans := filepath.Join(tdir, rpath)
+	if count == 1 {
+		filepath.WalkDir(tdir, func(path string, d fs.DirEntry, err error) error {
+			if !d.IsDir() {
+				ans = path
+				return fs.SkipAll
+			}
+			return nil
+		})
+	}
+	return ans, nil
+}
+
+func get_remote_file(path string) (string, error) {
+	if strings.HasPrefix(path, "ssh:") {
+		parts := strings.SplitN(path, ":", 3)
+		if len(parts) == 3 {
+			return get_ssh_file(parts[1], parts[2])
+		}
+	}
+	return path, nil
+}
+
+func main(_ *cli.Command, opts_ *Options, args []string) (rc int, err error) {
+	opts = opts_
+	conf, err = load_config(opts)
+	if err != nil {
+		return 1, err
+	}
+	if len(args) != 2 {
+		return 1, fmt.Errorf("You must specify exactly two files/directories to compare")
+	}
+	if err = set_diff_command(conf.Diff_cmd); err != nil {
+		return 1, err
+	}
+	init_caches()
+	create_formatters()
+	defer func() {
+		for tdir := range remote_dirs {
+			os.RemoveAll(tdir)
+		}
+	}()
+	left, err := get_remote_file(args[0])
+	if err != nil {
+		return 1, err
+	}
+	right, err := get_remote_file(args[1])
+	if err != nil {
+		return 1, err
+	}
+	if isdir(left) != isdir(right) {
+		return 1, fmt.Errorf("The items to be diffed should both be either directories or files. Comparing a directory to a file is not valid.'")
+	}
+	if !exists(left) {
+		return 1, fmt.Errorf("%s does not exist", left)
+	}
+	if !exists(right) {
+		return 1, fmt.Errorf("%s does not exist", right)
+	}
+	lp, err = loop.New()
+	loop.MouseTrackingMode(lp, loop.BUTTONS_AND_DRAG_MOUSE_TRACKING)
+	if err != nil {
+		return 1, err
+	}
+	h := Handler{left: left, right: right, lp: lp}
+	lp.OnInitialize = func() (string, error) {
+		lp.SetCursorVisible(false)
+		lp.SetCursorShape(loop.BAR_CURSOR, true)
+		lp.AllowLineWrapping(false)
+		lp.SetWindowTitle(fmt.Sprintf("%s vs. %s", left, right))
+		h.initialize()
+		return "", nil
+	}
+	lp.OnWakeup = h.on_wakeup
+	lp.OnFinalize = func() string {
+		lp.SetCursorVisible(true)
+		lp.SetCursorShape(loop.BLOCK_CURSOR, true)
+		h.finalize()
+		return ""
+	}
+	lp.OnResize = h.on_resize
+	lp.OnKeyEvent = h.on_key_event
+	lp.OnText = h.on_text
+	lp.OnMouseEvent = h.on_mouse_event
+	err = lp.Run()
+	if err != nil {
+		return 1, err
+	}
+	ds := lp.DeathSignalName()
+	if ds != "" {
+		fmt.Println("Killed by signal: ", ds)
+		lp.KillIfSignalled()
+		return 1, nil
+	}
+
+	return
+}
+
+func EntryPoint(parent *cli.Command) {
+	create_cmd(parent, main)
+}

kittens/diff/main.py

@@ -1,591 +1,276 @@
 #!/usr/bin/env python3
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
-import atexit
-import os
-import signal
-import subprocess
 import sys
-import tempfile
-import warnings
-from collections import defaultdict
-from contextlib import suppress
-from enum import Enum, auto
 from functools import partial
-from gettext import gettext as _
+from typing import List
-from typing import (
-    Any,
-    DefaultDict,
-    Dict,
-    Iterable,
-    Iterator,
-    List,
-    Optional,
-    Tuple,
-    Union,
-)
 
-from kitty.cli import CONFIG_HELP, CompletionSpec, parse_args
+from kitty.cli import CONFIG_HELP, CompletionSpec
-from kitty.cli_stub import DiffCLIOptions
+from kitty.conf.types import Definition
-from kitty.conf.utils import KeyAction
 from kitty.constants import appname
-from kitty.fast_data_types import wcswidth
-from kitty.key_encoding import EventType, KeyEvent
-from kitty.utils import ScreenSize, extract_all_from_tarfile_safely
 
-from ..tui.handler import Handler
-from ..tui.images import ImageManager, Placement
-from ..tui.line_edit import LineEdit
-from ..tui.loop import Loop
-from ..tui.operations import styled
-from . import global_data
-from .collect import (
-    Collection,
-    add_remote_dir,
-    create_collection,
-    data_for_path,
-    lines_for_path,
-    sanitize,
-    set_highlight_data,
-)
-from .config import init_config
-from .options.types import Options as DiffOptions
-from .patch import Differ, Patch, set_diff_command, worker_processes
-from .render import (
-    ImagePlacement,
-    ImageSupportWarning,
-    Line,
-    LineRef,
-    Reference,
-    render_diff,
-)
-from .search import BadRegex, Search
 
-try:
+def main(args: List[str]) -> None:
-    from .highlight import (
+    raise SystemExit('Must be run as kitten diff')
-        DiffHighlight,
+
-        get_highlight_processes,
+definition = Definition(
-        highlight_collection,
+    '!kittens.diff',
-        initialize_highlighter,
+)
+
+agr = definition.add_group
+egr = definition.end_group
+opt = definition.add_option
+map = definition.add_map
+mma = definition.add_mouse_map
+
+# diff {{{
+agr('diff', 'Diffing')
+
+opt('syntax_aliases', 'pyj:py pyi:py recipe:py', ctype='strdict_ _:', option_type='syntax_aliases',
+    long_text='''
+File extension aliases for syntax highlight. For example, to syntax highlight
+:file:`file.xyz` as :file:`file.abc` use a setting of :code:`xyz:abc`.
+Multiple aliases must be separated by spaces.
+'''
     )
-    has_highlighter = True
-    DiffHighlight
-except ImportError:
-    has_highlighter = False
 
-    def highlight_collection(collection: 'Collection', aliases: Optional[Dict[str, str]] = None) -> Union[str, Dict[str, 'DiffHighlight']]:
+opt('num_context_lines', '3', option_type='positive_int',
-        return ''
+    long_text='The number of lines of context to show around each change.'
+    )
 
-    def get_highlight_processes() -> Iterator[int]:
+opt('diff_cmd', 'auto',
-        if has_highlighter:
+    long_text='''
-            yield -1
+The diff command to use. Must contain the placeholder :code:`_CONTEXT_` which
+will be replaced by the number of lines of context. A few special values are allowed:
+:code:`auto` will automatically pick an available diff implementation. :code:`builtin`
+will use the anchored diff algorithm from the Go standard library. :code:`git` will
+use the git command to do the diffing. :code:`diff` will use the diff command to
+do the diffing.
+'''
+    )
 
+opt('replace_tab_by', '\\x20\\x20\\x20\\x20', option_type='python_string',
+    long_text='The string to replace tabs with. Default is to use four spaces.'
+    )
 
-class State(Enum):
+opt('+ignore_name', '', ctype='string',
-    initializing = auto()
+    add_to_default=False,
-    collected = auto()
+    long_text='''
-    diffed = auto()
+A glob pattern that is matched against only the filename of files and directories. Matching
-    command = auto()
+files and directories are ignored when scanning the filesystem to look for files to diff.
-    message = auto()
+Can be specified multiple times to use multiple patterns. For example::
 
+    ignore_name .git
+    ignore_name *~
+    ignore_name *.pyc
+''',
+    )
 
-class BackgroundWork(Enum):
+egr()  # }}}
-    none = auto()
-    collecting = auto()
-    diffing = auto()
-    highlighting = auto()
 
+# colors {{{
+agr('colors', 'Colors')
 
-def generate_diff(collection: Collection, context: int) -> Union[str, Dict[str, Patch]]:
+opt('pygments_style', 'default',
-    d = Differ()
+    long_text='''
+The pygments color scheme to use for syntax highlighting. See :link:`pygments
+builtin styles <https://pygments.org/styles/>` for a list of schemes. Note that
+this **does not** change the colors used for diffing,
+only the colors used for syntax highlighting. To change the general colors use the settings below.
+'''
+    )
 
-    for path, item_type, changed_path in collection:
+opt('foreground', 'black',
-        if item_type == 'diff':
+    option_type='to_color',
-            is_binary = isinstance(data_for_path(path), bytes) or isinstance(data_for_path(changed_path), bytes)
+    long_text='Basic colors'
-            if not is_binary:
+    )
-                assert changed_path is not None
-                d.add_diff(path, changed_path)
 
-    return d(context)
+opt('background', 'white',
+    option_type='to_color',
+    )
 
+opt('title_fg', 'black',
+    option_type='to_color',
+    long_text='Title colors'
+    )
 
-class DiffHandler(Handler):
+opt('title_bg', 'white',
+    option_type='to_color',
+    )
 
-    image_manager_class = ImageManager
+opt('margin_bg', '#fafbfc',
+    option_type='to_color',
+    long_text='Margin colors'
+    )
 
-    def __init__(self, args: DiffCLIOptions, opts: DiffOptions, left: str, right: str) -> None:
+opt('margin_fg', '#aaaaaa',
-        self.state = State.initializing
+    option_type='to_color',
-        self.message = ''
+    )
-        self.current_search_is_regex = True
-        self.current_search: Optional[Search] = None
-        self.line_edit = LineEdit()
-        self.opts = opts
-        self.left, self.right = left, right
-        self.report_traceback_on_exit: Union[str, Dict[str, Patch], None] = None
-        self.args = args
-        self.scroll_pos = self.max_scroll_pos = 0
-        self.current_context_count = self.original_context_count = self.args.context
-        if self.current_context_count < 0:
-            self.current_context_count = self.original_context_count = self.opts.num_context_lines
-        self.highlighting_done = False
-        self.doing_background_work = BackgroundWork.none
-        self.restore_position: Optional[Reference] = None
-        for key_def, action in self.opts.key_definitions.items():
-            self.add_shortcut(action, key_def)
 
-    def terminate(self, return_code: int = 0) -> None:
+opt('removed_bg', '#ffeef0',
-        self.quit_loop(return_code)
+    option_type='to_color',
+    long_text='Removed text backgrounds'
+    )
 
-    def perform_action(self, action: KeyAction) -> None:
+opt('highlight_removed_bg', '#fdb8c0',
-        func, args = action
+    option_type='to_color',
-        if func == 'quit':
+    )
-            self.terminate()
-            return
-        if self.state.value <= State.diffed.value:
-            if func == 'scroll_by':
-                return self.scroll_lines(int(args[0] or 0))
-            if func == 'scroll_to':
-                where = str(args[0])
-                if 'change' in where:
-                    return self.scroll_to_next_change(backwards='prev' in where)
-                if 'match' in where:
-                    return self.scroll_to_next_match(backwards='prev' in where)
-                if 'page' in where:
-                    amt = self.num_lines * (1 if 'next' in where else -1)
-                else:
-                    amt = len(self.diff_lines) * (1 if 'end' in where else -1)
-                return self.scroll_lines(amt)
-            if func == 'change_context':
-                new_ctx = self.current_context_count
-                to = args[0]
-                if to == 'all':
-                    new_ctx = 100000
-                elif to == 'default':
-                    new_ctx = self.original_context_count
-                else:
-                    new_ctx += int(to or 0)
-                return self.change_context_count(new_ctx)
-            if func == 'start_search':
-                self.start_search(bool(args[0]), bool(args[1]))
-                return
 
-    def create_collection(self) -> None:
+opt('removed_margin_bg', '#ffdce0',
+    option_type='to_color',
+    )
 
-        def collect_done(collection: Collection) -> None:
+opt('added_bg', '#e6ffed',
-            self.doing_background_work = BackgroundWork.none
+    option_type='to_color',
-            self.collection = collection
+    long_text='Added text backgrounds'
-            self.state = State.collected
+    )
-            self.generate_diff()
 
-        def collect(left: str, right: str) -> None:
+opt('highlight_added_bg', '#acf2bd',
-            collection = create_collection(left, right)
+    option_type='to_color',
-            self.asyncio_loop.call_soon_threadsafe(collect_done, collection)
+    )
 
-        self.asyncio_loop.run_in_executor(None, collect, self.left, self.right)
+opt('added_margin_bg', '#cdffd8',
-        self.doing_background_work = BackgroundWork.collecting
+    option_type='to_color',
+    )
 
-    def generate_diff(self) -> None:
+opt('filler_bg', '#fafbfc',
+    option_type='to_color',
+    long_text='Filler (empty) line background'
+    )
 
-        def diff_done(diff_map: Union[str, Dict[str, Patch]]) -> None:
+opt('margin_filler_bg', 'none',
-            self.doing_background_work = BackgroundWork.none
+    option_type='to_color_or_none',
-            if isinstance(diff_map, str):
+    long_text='Filler (empty) line background in margins, defaults to the filler background'
-                self.report_traceback_on_exit = diff_map
+    )
-                self.terminate(1)
-                return
-            self.state = State.diffed
-            self.diff_map = diff_map
-            self.calculate_statistics()
-            self.render_diff()
-            self.scroll_pos = 0
-            if self.restore_position is not None:
-                self.current_position = self.restore_position
-                self.restore_position = None
-            self.draw_screen()
-            if has_highlighter and not self.highlighting_done:
-                from .highlight import StyleNotFound
-                self.highlighting_done = True
-                try:
-                    initialize_highlighter(self.opts.pygments_style)
-                except StyleNotFound as e:
-                    self.report_traceback_on_exit = str(e)
-                    self.terminate(1)
-                    return
-                self.syntax_highlight()
 
-        def diff(collection: Collection, current_context_count: int) -> None:
+opt('hunk_margin_bg', '#dbedff',
-            diff_map = generate_diff(collection, current_context_count)
+    option_type='to_color',
-            self.asyncio_loop.call_soon_threadsafe(diff_done, diff_map)
+    long_text='Hunk header colors'
+    )
 
-        self.asyncio_loop.run_in_executor(None, diff, self.collection, self.current_context_count)
+opt('hunk_bg', '#f1f8ff',
-        self.doing_background_work = BackgroundWork.diffing
+    option_type='to_color',
+    )
 
-    def syntax_highlight(self) -> None:
+opt('search_bg', '#444',
+    option_type='to_color',
+    long_text='Highlighting'
+    )
 
-        def highlighting_done(hdata: Union[str, Dict[str, 'DiffHighlight']]) -> None:
+opt('search_fg', 'white',
-            self.doing_background_work = BackgroundWork.none
+    option_type='to_color',
-            if isinstance(hdata, str):
+    )
-                self.report_traceback_on_exit = hdata
-                self.terminate(1)
-                return
-            set_highlight_data(hdata)
-            self.render_diff()
-            self.draw_screen()
 
-        def highlight(collection: Collection, aliases: Optional[Dict[str, str]] = None) -> None:
+opt('select_bg', '#b4d5fe',
-            result = highlight_collection(collection, aliases)
+    option_type='to_color',
-            self.asyncio_loop.call_soon_threadsafe(highlighting_done, result)
+    )
 
-        self.asyncio_loop.run_in_executor(None, highlight, self.collection, self.opts.syntax_aliases)
+opt('select_fg', 'black',
-        self.doing_background_work = BackgroundWork.highlighting
+    option_type='to_color_or_none',
+    )
+egr()  # }}}
 
-    def calculate_statistics(self) -> None:
+# shortcuts {{{
-        self.added_count = self.collection.added_count
+agr('shortcuts', 'Keyboard shortcuts')
-        self.removed_count = self.collection.removed_count
-        for patch in self.diff_map.values():
-            self.added_count += patch.added_count
-            self.removed_count += patch.removed_count
 
-    def render_diff(self) -> None:
+map('Quit',
-        self.diff_lines: Tuple[Line, ...] = tuple(render_diff(self.collection, self.diff_map, self.args, self.screen_size.cols, self.image_manager))
+    'quit q quit',
-        self.margin_size = render_diff.margin_size
+    )
-        self.ref_path_map: DefaultDict[str, List[Tuple[int, Reference]]] = defaultdict(list)
+map('Quit',
-        for i, dl in enumerate(self.diff_lines):
+    'quit esc quit',
-            self.ref_path_map[dl.ref.path].append((i, dl.ref))
+    )
-        self.max_scroll_pos = len(self.diff_lines) - self.num_lines
-        if self.current_search is not None:
-            self.current_search(self.diff_lines, self.margin_size, self.screen_size.cols)
 
-    @property
+map('Scroll down',
-    def current_position(self) -> Reference:
+    'scroll_down j scroll_by 1',
-        return self.diff_lines[min(len(self.diff_lines) - 1, self.scroll_pos)].ref
+    )
+map('Scroll down',
+    'scroll_down down scroll_by 1',
+    )
 
-    @current_position.setter
+map('Scroll up',
-    def current_position(self, ref: Reference) -> None:
+    'scroll_up k scroll_by -1',
-        num = None
+    )
-        if isinstance(ref.extra, LineRef):
+map('Scroll up',
-            sln = ref.extra.src_line_number
+    'scroll_up up scroll_by -1',
-            for i, q in self.ref_path_map[ref.path]:
+    )
-                if isinstance(q.extra, LineRef):
-                    if q.extra.src_line_number >= sln:
-                        if q.extra.src_line_number == sln:
-                            num = i
-                        break
-                    num = i
-        if num is None:
-            for i, q in self.ref_path_map[ref.path]:
-                num = i
-                break
 
-        if num is not None:
+map('Scroll to top',
-            self.scroll_pos = max(0, min(num, self.max_scroll_pos))
+    'scroll_top home scroll_to start',
+    )
 
-    @property
+map('Scroll to bottom',
-    def num_lines(self) -> int:
+    'scroll_bottom end scroll_to end',
-        return self.screen_size.rows - 1
+    )
 
-    def scroll_to_next_change(self, backwards: bool = False) -> None:
+map('Scroll to next page',
-        if backwards:
+    'scroll_page_down page_down scroll_to next-page',
-            r = range(self.scroll_pos - 1, -1, -1)
+    )
-        else:
+map('Scroll to next page',
-            r = range(self.scroll_pos + 1, len(self.diff_lines))
+    'scroll_page_down space scroll_to next-page',
-        for i in r:
+    )
-            line = self.diff_lines[i]
-            if line.is_change_start:
-                self.scroll_lines(i - self.scroll_pos)
-                return
-        self.cmd.bell()
 
-    def scroll_to_next_match(self, backwards: bool = False, include_current: bool = False) -> None:
+map('Scroll to previous page',
-        if self.current_search is not None:
+    'scroll_page_up page_up scroll_to prev-page',
-            offset = 0 if include_current else 1
+    )
-            if backwards:
-                r = range(self.scroll_pos - offset, -1, -1)
-            else:
-                r = range(self.scroll_pos + offset, len(self.diff_lines))
-            for i in r:
-                if i in self.current_search:
-                    self.scroll_lines(i - self.scroll_pos)
-                    return
-        self.cmd.bell()
 
-    def set_scrolling_region(self) -> None:
+map('Scroll to next change',
-        self.cmd.set_scrolling_region(self.screen_size, 0, self.num_lines - 2)
+    'next_change n scroll_to next-change',
+    )
 
-    def scroll_lines(self, amt: int = 1) -> None:
+map('Scroll to previous change',
-        new_pos = max(0, min(self.scroll_pos + amt, self.max_scroll_pos))
+    'prev_change p scroll_to prev-change',
-        amt = new_pos - self.scroll_pos
+    )
-        if new_pos == self.scroll_pos:
-            self.cmd.bell()
-            return
-        if abs(amt) >= self.num_lines - 1:
-            self.scroll_pos = new_pos
-            self.draw_screen()
-            return
-        self.enforce_cursor_state()
-        self.cmd.scroll_screen(amt)
-        self.scroll_pos = new_pos
-        if amt < 0:
-            self.cmd.set_cursor_position(0, 0)
-            self.draw_lines(-amt)
-        else:
-            self.cmd.set_cursor_position(0, self.num_lines - amt)
-            self.draw_lines(amt, self.num_lines - amt)
-        self.draw_status_line()
 
-    def init_terminal_state(self) -> None:
+map('Show all context',
-        self.cmd.set_line_wrapping(False)
+    'all_context a change_context all',
-        self.cmd.set_window_title(global_data.title)
+    )
-        self.cmd.set_default_colors(
-            fg=self.opts.foreground, bg=self.opts.background,
-            cursor=self.opts.foreground, select_fg=self.opts.select_fg,
-            select_bg=self.opts.select_bg)
-        self.cmd.set_cursor_shape('beam')
 
-    def finalize(self) -> None:
+map('Show default context',
-        self.cmd.set_default_colors()
+    'default_context = change_context default',
-        self.cmd.set_cursor_visible(True)
+    )
-        self.cmd.set_scrolling_region()
 
-    def initialize(self) -> None:
+map('Increase context',
-        self.init_terminal_state()
+    'increase_context + change_context 5',
-        self.set_scrolling_region()
+    )
-        self.draw_screen()
-        self.create_collection()
 
-    def enforce_cursor_state(self) -> None:
+map('Decrease context',
-        self.cmd.set_cursor_visible(self.state is State.command)
+    'decrease_context - change_context -5',
+    )
 
-    def draw_lines(self, num: int, offset: int = 0) -> None:
+map('Search forward',
-        offset += self.scroll_pos
+    'search_forward / start_search regex forward',
-        image_involved = False
+    )
-        limit = len(self.diff_lines)
-        for i in range(num):
-            lpos = offset + i
-            if lpos >= limit:
-                text = ''
-            else:
-                line = self.diff_lines[lpos]
-                text = line.text
-                if line.image_data is not None:
-                    image_involved = True
-            self.write(f'\r\x1b[K{text}\x1b[0m')
-            if self.current_search is not None:
-                self.current_search.highlight_line(self.write, lpos)
-            if i < num - 1:
-                self.write('\n')
-        if image_involved:
-            self.place_images()
 
-    def update_image_placement_for_resend(self, image_id: int, pl: Placement) -> bool:
+map('Search backward',
-        offset = self.scroll_pos
+    'search_backward ? start_search regex backward',
-        limit = len(self.diff_lines)
+    )
-        in_image = False
 
-        def adjust(row: int, candidate: ImagePlacement, is_left: bool) -> bool:
+map('Scroll to next search match',
-            if candidate.image.image_id == image_id:
+    'next_match . scroll_to next-match',
-                q = self.xpos_for_image(row, candidate, is_left)
+    )
-                if q is not None:
+map('Scroll to next search match',
-                    pl.x = q[0]
+    'next_match > scroll_to next-match',
-                    pl.y = row
+    )
-                    return True
-            return False
 
-        for row in range(self.num_lines):
+map('Scroll to previous search match',
-            lpos = offset + row
+    'prev_match , scroll_to prev-match',
-            if lpos >= limit:
+    )
-                break
+map('Scroll to previous search match',
-            line = self.diff_lines[lpos]
+    'prev_match < scroll_to prev-match',
-            if in_image:
+    )
-                if line.image_data is None:
-                    in_image = False
-                continue
-            if line.image_data is not None:
-                left_placement, right_placement = line.image_data
-                if left_placement is not None:
-                    if adjust(row, left_placement, True):
-                        return True
-                    in_image = True
-                if right_placement is not None:
-                    if adjust(row, right_placement, False):
-                        return True
-                    in_image = True
-        return False
 
-    def place_images(self) -> None:
+map('Search forward (no regex)',
-        self.image_manager.update_image_placement_for_resend = self.update_image_placement_for_resend
+    'search_forward_simple f start_search substring forward',
-        self.cmd.clear_images_on_screen()
+    )
-        offset = self.scroll_pos
-        limit = len(self.diff_lines)
-        in_image = False
-        for row in range(self.num_lines):
-            lpos = offset + row
-            if lpos >= limit:
-                break
-            line = self.diff_lines[lpos]
-            if in_image:
-                if line.image_data is None:
-                    in_image = False
-                continue
-            if line.image_data is not None:
-                left_placement, right_placement = line.image_data
-                if left_placement is not None:
-                    self.place_image(row, left_placement, True)
-                    in_image = True
-                if right_placement is not None:
-                    self.place_image(row, right_placement, False)
-                    in_image = True
 
-    def xpos_for_image(self, row: int, placement: ImagePlacement, is_left: bool) -> Optional[Tuple[int, float]]:
+map('Search backward (no regex)',
-        xpos = (0 if is_left else (self.screen_size.cols // 2)) + placement.image.margin_size
+    'search_backward_simple b start_search substring backward',
-        image_height_in_rows = placement.image.rows
+    )
-        topmost_visible_row = placement.row
-        num_visible_rows = image_height_in_rows - topmost_visible_row
-        visible_frac = min(num_visible_rows / image_height_in_rows, 1)
-        if visible_frac <= 0:
-            return None
-        return xpos, visible_frac
 
-    def place_image(self, row: int, placement: ImagePlacement, is_left: bool) -> None:
+map('Copy selection to clipboard', 'copy_to_clipboard y copy_to_clipboard')
-        q = self.xpos_for_image(row, placement, is_left)
+map('Copy selection to clipboard or exit if no selection is present', 'copy_to_clipboard_or_exit ctrl+c copy_to_clipboard_or_exit')
-        if q is not None:
-            xpos, visible_frac = q
-            height = int(visible_frac * placement.image.height)
-            top = placement.image.height - height
-            self.image_manager.show_image(placement.image.image_id, xpos, row, src_rect=(
-                0, top, placement.image.width, height))
-
-    def draw_screen(self) -> None:
-        self.enforce_cursor_state()
-        if self.state.value < State.diffed.value:
-            self.cmd.clear_screen()
-            self.write(_('Calculating diff, please wait...'))
-            return
-        self.cmd.clear_images_on_screen()
-        self.cmd.set_cursor_position(0, 0)
-        self.draw_lines(self.num_lines)
-        self.draw_status_line()
-
-    def draw_status_line(self) -> None:
-        if self.state.value < State.diffed.value:
-            return
-        self.enforce_cursor_state()
-        self.cmd.set_cursor_position(0, self.num_lines)
-        self.cmd.clear_to_eol()
-        if self.state is State.command:
-            self.line_edit.write(self.write)
-        elif self.state is State.message:
-            self.cmd.styled(self.message, reverse=True)
-        else:
-            sp = f'{self.scroll_pos/self.max_scroll_pos:.0%}' if self.scroll_pos and self.max_scroll_pos else '0%'
-            scroll_frac = styled(sp, fg=self.opts.margin_fg)
-            if self.current_search is None:
-                counts = '{}{}{}'.format(
-                        styled(str(self.added_count), fg=self.opts.highlight_added_bg),
-                        styled(',', fg=self.opts.margin_fg),
-                        styled(str(self.removed_count), fg=self.opts.highlight_removed_bg)
-                )
-            else:
-                counts = styled(f'{len(self.current_search)} matches', fg=self.opts.margin_fg)
-            suffix = f'{counts}  {scroll_frac}'
-            prefix = styled(':', fg=self.opts.margin_fg)
-            filler = self.screen_size.cols - wcswidth(prefix) - wcswidth(suffix)
-            text = '{}{}{}'.format(prefix, ' ' * filler, suffix)
-            self.write(text)
-
-    def change_context_count(self, new_ctx: int) -> None:
-        new_ctx = max(0, new_ctx)
-        if new_ctx != self.current_context_count:
-            self.current_context_count = new_ctx
-            self.state = State.collected
-            self.generate_diff()
-            self.restore_position = self.current_position
-            self.draw_screen()
-
-    def start_search(self, is_regex: bool, is_backward: bool) -> None:
-        if self.state is not State.diffed:
-            self.cmd.bell()
-            return
-        self.state = State.command
-        self.line_edit.clear()
-        self.line_edit.add_text('?' if is_backward else '/')
-        self.current_search_is_regex = is_regex
-        self.draw_status_line()
-
-    def do_search(self) -> None:
-        self.current_search = None
-        query = self.line_edit.current_input
-        if len(query) < 2:
-            return
-        try:
-            self.current_search = Search(self.opts, query[1:], self.current_search_is_regex, query[0] == '?')
-        except BadRegex:
-            self.state = State.message
-            self.message = sanitize(_('Bad regex: {}').format(query[1:]))
-            self.cmd.bell()
-        else:
-            if self.current_search(self.diff_lines, self.margin_size, self.screen_size.cols):
-                self.scroll_to_next_match(include_current=True)
-            else:
-                self.state = State.message
-                self.message = sanitize(_('No matches found'))
-                self.cmd.bell()
-
-    def on_key_event(self, key_event: KeyEvent, in_bracketed_paste: bool = False) -> None:
-        if key_event.text:
-            if self.state is State.command:
-                self.line_edit.on_text(key_event.text, in_bracketed_paste)
-                self.draw_status_line()
-                return
-            if self.state is State.message:
-                self.state = State.diffed
-                self.draw_status_line()
-                return
-        else:
-            if self.state is State.message:
-                if key_event.type is not EventType.RELEASE:
-                    self.state = State.diffed
-                    self.draw_status_line()
-                return
-            if self.state is State.command:
-                if self.line_edit.on_key(key_event):
-                    if not self.line_edit.current_input:
-                        self.state = State.diffed
-                    self.draw_status_line()
-                    return
-                if key_event.matches('enter'):
-                    self.state = State.diffed
-                    self.do_search()
-                    self.line_edit.clear()
-                    self.draw_screen()
-                    return
-                if key_event.matches('esc'):
-                    self.state = State.diffed
-                    self.draw_status_line()
-                    return
-            if self.state.value >= State.diffed.value and self.current_search is not None and key_event.matches('esc'):
-                self.current_search = None
-                self.draw_screen()
-                return
-            if key_event.type is EventType.RELEASE:
-                return
-        action = self.shortcut_action(key_event)
-        if action is not None:
-            return self.perform_action(action)
-
-    def on_resize(self, screen_size: ScreenSize) -> None:
-        self.screen_size = screen_size
-        self.set_scrolling_region()
-        if self.state.value > State.collected.value:
-            self.image_manager.delete_all_sent_images()
-            self.render_diff()
-        self.draw_screen()
-
-    def on_interrupt(self) -> None:
-        self.terminate(1)
-
-    def on_eot(self) -> None:
-        self.terminate(1)
 
+egr()  # }}}
 
 OPTIONS = partial('''\
 --context
@@ -607,99 +292,10 @@ Override individual configuration options, can be specified multiple times.
 Syntax: :italic:`name=value`. For example: :italic:`-o background=gray`
 
 '''.format, config_help=CONFIG_HELP.format(conf_name='diff', appname=appname))
-
-
-class ShowWarning:
-
-    def __init__(self) -> None:
-        self.warnings: List[str] = []
-
-    def __call__(self, message: Any, category: Any, filename: str, lineno: int, file: object = None, line: object = None) -> None:
-        if category is ImageSupportWarning and isinstance(message, str):
-            showwarning.warnings.append(message)
-
-
-showwarning = ShowWarning()
 help_text = 'Show a side-by-side diff of the specified files/directories. You can also use :italic:`ssh:hostname:remote-file-path` to diff remote files.'
 usage = 'file_or_directory_left file_or_directory_right'
 
 
-def terminate_processes(processes: Iterable[int]) -> None:
-    for pid in processes:
-        with suppress(Exception):
-            os.kill(pid, signal.SIGKILL)
-
-
-def get_ssh_file(hostname: str, rpath: str) -> str:
-    import io
-    import shutil
-    import tarfile
-    tdir = tempfile.mkdtemp(suffix=f'-{hostname}')
-    add_remote_dir(tdir)
-    atexit.register(shutil.rmtree, tdir)
-    is_abs = rpath.startswith('/')
-    rpath = rpath.lstrip('/')
-    cmd = ['ssh', hostname, 'tar', '-c', '-f', '-']
-    if is_abs:
-        cmd.extend(('-C', '/'))
-    cmd.append(rpath)
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-    assert p.stdout is not None
-    raw = p.stdout.read()
-    if p.wait() != 0:
-        raise SystemExit(p.returncode)
-    with tarfile.open(fileobj=io.BytesIO(raw), mode='r:') as tf:
-        members = tf.getmembers()
-        extract_all_from_tarfile_safely(tf, tdir)
-        if len(members) == 1:
-            for root, dirs, files in os.walk(tdir):
-                if files:
-                    return os.path.join(root, files[0])
-        return os.path.abspath(os.path.join(tdir, rpath))
-
-
-def get_remote_file(path: str) -> str:
-    if path.startswith('ssh:'):
-        parts = path.split(':', 2)
-        if len(parts) == 3:
-            return get_ssh_file(parts[1], parts[2])
-    return path
-
-
-def main(args: List[str]) -> None:
-    warnings.showwarning = showwarning
-    cli_opts, items = parse_args(args[1:], OPTIONS, usage, help_text, 'kitty +kitten diff', result_class=DiffCLIOptions)
-    if len(items) != 2:
-        raise SystemExit('You must specify exactly two files/directories to compare')
-    left, right = items
-    global_data.title = _('{} vs. {}').format(left, right)
-    opts = init_config(cli_opts)
-    set_diff_command(opts.diff_cmd)
-    lines_for_path.replace_tab_by = opts.replace_tab_by
-    Collection.ignore_names = tuple(opts.ignore_name)
-    left, right = map(get_remote_file, (left, right))
-    if os.path.isdir(left) != os.path.isdir(right):
-        raise SystemExit('The items to be diffed should both be either directories or files. Comparing a directory to a file is not valid.')
-    for f in left, right:
-        if not os.path.exists(f):
-            raise SystemExit(f'{f} does not exist')
-
-    loop = Loop()
-    handler = DiffHandler(cli_opts, opts, left, right)
-    loop.loop(handler)
-    for message in showwarning.warnings:
-        from kitty.utils import safe_print
-        safe_print(message, file=sys.stderr)
-    if handler.doing_background_work is BackgroundWork.highlighting:
-        terminate_processes(tuple(get_highlight_processes()))
-    elif handler.doing_background_work == BackgroundWork.diffing:
-        terminate_processes(tuple(worker_processes))
-    if loop.return_code != 0:
-        if handler.report_traceback_on_exit:
-            print(handler.report_traceback_on_exit, file=sys.stderr)
-            input('Press Enter to quit.')
-        raise SystemExit(loop.return_code)
-
 
 if __name__ == '__main__':
     main(sys.argv)
@@ -708,7 +304,7 @@ elif __name__ == '__doc__':
     cd['usage'] = usage
     cd['options'] = OPTIONS
     cd['help_text'] = help_text
+    cd['short_desc'] = 'Pretty, side-by-side diffing of files and images'
     cd['args_completion'] = CompletionSpec.from_string('type:file mime:text/* mime:image/* group:"Text and image files"')
 elif __name__ == '__conf__':
-    from .options.definition import definition
     sys.options_definition = definition  # type: ignore

kittens/diff/mouse.go

@@ -0,0 +1,210 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"kitty"
+	"kitty/tools/config"
+	"kitty/tools/tui"
+	"kitty/tools/tui/loop"
+	"kitty/tools/utils"
+	"kitty/tools/wcswidth"
+)
+
+var _ = fmt.Print
+
+type KittyOpts struct {
+	Wheel_scroll_multiplier int
+	Copy_on_select          bool
+}
+
+func read_relevant_kitty_opts(path string) KittyOpts {
+	ans := KittyOpts{Wheel_scroll_multiplier: kitty.KittyConfigDefaults.Wheel_scroll_multiplier}
+	handle_line := func(key, val string) error {
+		switch key {
+		case "wheel_scroll_multiplier":
+			v, err := strconv.Atoi(val)
+			if err == nil {
+				ans.Wheel_scroll_multiplier = v
+			}
+		case "copy_on_select":
+			ans.Copy_on_select = strings.ToLower(val) == "clipboard"
+		}
+		return nil
+	}
+	cp := config.ConfigParser{LineHandler: handle_line}
+	cp.ParseFiles(path)
+	return ans
+}
+
+var RelevantKittyOpts = (&utils.Once[KittyOpts]{Run: func() KittyOpts {
+	return read_relevant_kitty_opts(filepath.Join(utils.ConfigDir(), "kitty.conf"))
+}}).Get
+
+func (self *Handler) handle_wheel_event(up bool) {
+	amt := RelevantKittyOpts().Wheel_scroll_multiplier
+	if up {
+		amt *= -1
+	}
+	self.dispatch_action(`scroll_by`, strconv.Itoa(amt))
+}
+
+type line_pos struct {
+	min_x, max_x int
+	y            ScrollPos
+}
+
+func (self *line_pos) MinX() int { return self.min_x }
+func (self *line_pos) MaxX() int { return self.max_x }
+func (self *line_pos) Equal(other tui.LinePos) bool {
+	if o, ok := other.(*line_pos); ok {
+		return self.y == o.y
+	}
+	return false
+}
+func (self *line_pos) LessThan(other tui.LinePos) bool {
+	if o, ok := other.(*line_pos); ok {
+		return self.y.Less(o.y)
+	}
+	return false
+}
+
+func (self *Handler) line_pos_from_pos(x int, pos ScrollPos) *line_pos {
+	ans := line_pos{min_x: self.logical_lines.margin_size, y: pos}
+	available_cols := self.logical_lines.columns / 2
+	if x >= available_cols {
+		ans.min_x += available_cols
+		ans.max_x = utils.Max(ans.min_x, ans.min_x+self.logical_lines.ScreenLineAt(pos).right.wcswidth()-1)
+	} else {
+		ans.max_x = utils.Max(ans.min_x, ans.min_x+self.logical_lines.ScreenLineAt(pos).left.wcswidth()-1)
+	}
+	return &ans
+}
+
+func (self *Handler) start_mouse_selection(ev *loop.MouseEvent) {
+	available_cols := self.logical_lines.columns / 2
+	if ev.Cell.Y >= self.screen_size.num_lines || ev.Cell.X < self.logical_lines.margin_size || (ev.Cell.X >= available_cols && ev.Cell.X < available_cols+self.logical_lines.margin_size) {
+		return
+	}
+	pos := self.scroll_pos
+	self.logical_lines.IncrementScrollPosBy(&pos, ev.Cell.Y)
+	ll := self.logical_lines.At(pos.logical_line)
+	if ll.line_type == EMPTY_LINE || ll.line_type == IMAGE_LINE {
+		return
+	}
+	self.mouse_selection.StartNewSelection(ev, self.line_pos_from_pos(ev.Cell.X, pos), 0, self.screen_size.num_lines-1, self.screen_size.cell_width, self.screen_size.cell_height)
+}
+
+func (self *Handler) drag_scroll_tick(timer_id loop.IdType) error {
+	return self.mouse_selection.DragScrollTick(timer_id, self.lp, self.drag_scroll_tick, func(amt int, ev *loop.MouseEvent) error {
+		if self.scroll_lines(amt) != 0 {
+			self.do_update_mouse_selection(ev)
+			self.draw_screen()
+		}
+		return nil
+	})
+}
+
+func (self *Handler) update_mouse_selection(ev *loop.MouseEvent) {
+	if !self.mouse_selection.IsActive() {
+		return
+	}
+	if self.mouse_selection.OutOfVerticalBounds(ev) {
+		self.mouse_selection.DragScroll(ev, self.lp, self.drag_scroll_tick)
+		return
+	}
+	self.do_update_mouse_selection(ev)
+}
+
+func (self *Handler) do_update_mouse_selection(ev *loop.MouseEvent) {
+	pos := self.scroll_pos
+	y := ev.Cell.Y
+	y = utils.Max(0, utils.Min(y, self.screen_size.num_lines-1))
+	self.logical_lines.IncrementScrollPosBy(&pos, y)
+	x := self.mouse_selection.StartLine().MinX()
+	self.mouse_selection.Update(ev, self.line_pos_from_pos(x, pos))
+	self.draw_screen()
+}
+
+func (self *Handler) clear_mouse_selection() {
+	self.mouse_selection.Clear()
+}
+
+func (self *Handler) text_for_current_mouse_selection() string {
+	if self.mouse_selection.IsEmpty() {
+		return ""
+	}
+	text := make([]byte, 0, 2048)
+	start_pos, end_pos := *self.mouse_selection.StartLine().(*line_pos), *self.mouse_selection.EndLine().(*line_pos)
+	start, end := start_pos.y, end_pos.y
+	is_left := start_pos.min_x == self.logical_lines.margin_size
+
+	line_for_pos := func(pos ScrollPos) string {
+		if is_left {
+			return self.logical_lines.ScreenLineAt(pos).left.marked_up_text
+		}
+		return self.logical_lines.ScreenLineAt(pos).right.marked_up_text
+	}
+
+	for pos, prev_ll_idx := start, start.logical_line; pos.Less(end) || pos == end; self.logical_lines.IncrementScrollPosBy(&pos, 1) {
+		ll := self.logical_lines.At(pos.logical_line)
+		var line string
+		switch ll.line_type {
+		case EMPTY_LINE:
+		case IMAGE_LINE:
+			if pos.screen_line < ll.image_lines_offset {
+				line = line_for_pos(pos)
+			}
+		default:
+			line = line_for_pos(pos)
+		}
+		line = wcswidth.StripEscapeCodes(line)
+		s, e := self.mouse_selection.LineBounds(self.line_pos_from_pos(start_pos.min_x, pos))
+		s -= start_pos.min_x
+		e -= start_pos.min_x
+		line = wcswidth.TruncateToVisualLength(line, e+1)
+		if s > 0 {
+			prefix := wcswidth.TruncateToVisualLength(line, s)
+			line = line[len(prefix):]
+		}
+		// TODO: look at the original line from the source and handle leading tabs as per it
+		if pos.logical_line > prev_ll_idx {
+			line = "\n" + line
+		}
+		prev_ll_idx = pos.logical_line
+		if line != "" {
+			text = append(text, line...)
+		}
+	}
+	return utils.UnsafeBytesToString(text)
+}
+
+func (self *Handler) finish_mouse_selection(ev *loop.MouseEvent) {
+	if !self.mouse_selection.IsActive() {
+		return
+	}
+	self.update_mouse_selection(ev)
+	self.mouse_selection.Finish()
+	text := self.text_for_current_mouse_selection()
+	if text != "" {
+		if RelevantKittyOpts().Copy_on_select {
+			self.lp.CopyTextToClipboard(text)
+		} else {
+			self.lp.CopyTextToPrimarySelection(text)
+		}
+	}
+}
+
+func (self *Handler) add_mouse_selection_to_line(line_pos ScrollPos, y int) string {
+	if self.mouse_selection.IsEmpty() {
+		return ""
+	}
+	selection_sgr := format_as_sgr.selection
+	x := self.mouse_selection.StartLine().MinX()
+	return self.mouse_selection.LineFormatSuffix(self.line_pos_from_pos(x, line_pos), selection_sgr[2:len(selection_sgr)-1], y)
+}

kittens/diff/options/definition.py

@@ -1,262 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=utf-8
-# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
-
-# After editing this file run ./gen-config.py to apply the changes
-
-from kitty.conf.types import Action, Definition
-
-definition = Definition(
-    'kittens.diff',
-    Action('map', 'parse_map', {'key_definitions': 'kitty.conf.utils.KittensKeyMap'}, ['kitty.types.ParsedShortcut', 'kitty.conf.utils.KeyAction']),
-)
-
-agr = definition.add_group
-egr = definition.end_group
-opt = definition.add_option
-map = definition.add_map
-mma = definition.add_mouse_map
-
-# diff {{{
-agr('diff', 'Diffing')
-
-opt('syntax_aliases', 'pyj:py pyi:py recipe:py',
-    option_type='syntax_aliases',
-    long_text='''
-File extension aliases for syntax highlight. For example, to syntax highlight
-:file:`file.xyz` as :file:`file.abc` use a setting of :code:`xyz:abc`.
-'''
-    )
-
-opt('num_context_lines', '3',
-    option_type='positive_int',
-    long_text='The number of lines of context to show around each change.'
-    )
-
-opt('diff_cmd', 'auto',
-    long_text='''
-The diff command to use. Must contain the placeholder :code:`_CONTEXT_` which
-will be replaced by the number of lines of context. The default special value
-:code:`auto` is to search the system for either :program:`git` or
-:program:`diff` and use that, if found.
-'''
-    )
-
-opt('replace_tab_by', '\\x20\\x20\\x20\\x20',
-    option_type='python_string',
-    long_text='The string to replace tabs with. Default is to use four spaces.'
-    )
-
-opt('+ignore_name', '',
-    option_type='store_multiple',
-    add_to_default=False,
-    long_text='''
-A glob pattern that is matched against only the filename of files and directories. Matching
-files and directories are ignored when scanning the filesystem to look for files to diff.
-Can be specified multiple times to use multiple patterns. For example::
-
-    ignore_name .git
-    ignore_name *~
-    ignore_name *.pyc
-''',
-    )
-
-egr()  # }}}
-
-# colors {{{
-agr('colors', 'Colors')
-
-opt('pygments_style', 'default',
-    long_text='''
-The pygments color scheme to use for syntax highlighting. See :link:`pygments
-builtin styles <https://pygments.org/styles/>` for a list of schemes.
-'''
-    )
-
-opt('foreground', 'black',
-    option_type='to_color',
-    long_text='Basic colors'
-    )
-
-opt('background', 'white',
-    option_type='to_color',
-    )
-
-opt('title_fg', 'black',
-    option_type='to_color',
-    long_text='Title colors'
-    )
-
-opt('title_bg', 'white',
-    option_type='to_color',
-    )
-
-opt('margin_bg', '#fafbfc',
-    option_type='to_color',
-    long_text='Margin colors'
-    )
-
-opt('margin_fg', '#aaaaaa',
-    option_type='to_color',
-    )
-
-opt('removed_bg', '#ffeef0',
-    option_type='to_color',
-    long_text='Removed text backgrounds'
-    )
-
-opt('highlight_removed_bg', '#fdb8c0',
-    option_type='to_color',
-    )
-
-opt('removed_margin_bg', '#ffdce0',
-    option_type='to_color',
-    )
-
-opt('added_bg', '#e6ffed',
-    option_type='to_color',
-    long_text='Added text backgrounds'
-    )
-
-opt('highlight_added_bg', '#acf2bd',
-    option_type='to_color',
-    )
-
-opt('added_margin_bg', '#cdffd8',
-    option_type='to_color',
-    )
-
-opt('filler_bg', '#fafbfc',
-    option_type='to_color',
-    long_text='Filler (empty) line background'
-    )
-
-opt('margin_filler_bg', 'none',
-    option_type='to_color_or_none',
-    long_text='Filler (empty) line background in margins, defaults to the filler background'
-    )
-
-opt('hunk_margin_bg', '#dbedff',
-    option_type='to_color',
-    long_text='Hunk header colors'
-    )
-
-opt('hunk_bg', '#f1f8ff',
-    option_type='to_color',
-    )
-
-opt('search_bg', '#444',
-    option_type='to_color',
-    long_text='Highlighting'
-    )
-
-opt('search_fg', 'white',
-    option_type='to_color',
-    )
-
-opt('select_bg', '#b4d5fe',
-    option_type='to_color',
-    )
-
-opt('select_fg', 'black',
-    option_type='to_color_or_none',
-    )
-egr()  # }}}
-
-# shortcuts {{{
-agr('shortcuts', 'Keyboard shortcuts')
-
-map('Quit',
-    'quit q quit',
-    )
-map('Quit',
-    'quit esc quit',
-    )
-
-map('Scroll down',
-    'scroll_down j scroll_by 1',
-    )
-map('Scroll down',
-    'scroll_down down scroll_by 1',
-    )
-
-map('Scroll up',
-    'scroll_up k scroll_by -1',
-    )
-map('Scroll up',
-    'scroll_up up scroll_by -1',
-    )
-
-map('Scroll to top',
-    'scroll_top home scroll_to start',
-    )
-
-map('Scroll to bottom',
-    'scroll_bottom end scroll_to end',
-    )
-
-map('Scroll to next page',
-    'scroll_page_down page_down scroll_to next-page',
-    )
-map('Scroll to next page',
-    'scroll_page_down space scroll_to next-page',
-    )
-
-map('Scroll to previous page',
-    'scroll_page_up page_up scroll_to prev-page',
-    )
-
-map('Scroll to next change',
-    'next_change n scroll_to next-change',
-    )
-
-map('Scroll to previous change',
-    'prev_change p scroll_to prev-change',
-    )
-
-map('Show all context',
-    'all_context a change_context all',
-    )
-
-map('Show default context',
-    'default_context = change_context default',
-    )
-
-map('Increase context',
-    'increase_context + change_context 5',
-    )
-
-map('Decrease context',
-    'decrease_context - change_context -5',
-    )
-
-map('Search forward',
-    'search_forward / start_search regex forward',
-    )
-
-map('Search backward',
-    'search_backward ? start_search regex backward',
-    )
-
-map('Scroll to next search match',
-    'next_match . scroll_to next-match',
-    )
-map('Scroll to next search match',
-    'next_match > scroll_to next-match',
-    )
-
-map('Scroll to previous search match',
-    'prev_match , scroll_to prev-match',
-    )
-map('Scroll to previous search match',
-    'prev_match < scroll_to prev-match',
-    )
-
-map('Search forward (no regex)',
-    'search_forward_simple f start_search substring forward',
-    )
-
-map('Search backward (no regex)',
-    'search_backward_simple b start_search substring backward',
-    )
-egr()  # }}}

kittens/diff/options/parse.py

@@ -1,125 +0,0 @@
-# generated by gen-config.py DO NOT edit
-
-# isort: skip_file
-import typing
-from kittens.diff.options.utils import parse_map, store_multiple, syntax_aliases
-from kitty.conf.utils import merge_dicts, positive_int, python_string, to_color, to_color_or_none
-
-
-class Parser:
-
-    def added_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['added_bg'] = to_color(val)
-
-    def added_margin_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['added_margin_bg'] = to_color(val)
-
-    def background(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['background'] = to_color(val)
-
-    def diff_cmd(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['diff_cmd'] = str(val)
-
-    def filler_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['filler_bg'] = to_color(val)
-
-    def foreground(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['foreground'] = to_color(val)
-
-    def highlight_added_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['highlight_added_bg'] = to_color(val)
-
-    def highlight_removed_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['highlight_removed_bg'] = to_color(val)
-
-    def hunk_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['hunk_bg'] = to_color(val)
-
-    def hunk_margin_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['hunk_margin_bg'] = to_color(val)
-
-    def ignore_name(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        for k, v in store_multiple(val, ans["ignore_name"]):
-            ans["ignore_name"][k] = v
-
-    def margin_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['margin_bg'] = to_color(val)
-
-    def margin_fg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['margin_fg'] = to_color(val)
-
-    def margin_filler_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['margin_filler_bg'] = to_color_or_none(val)
-
-    def num_context_lines(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['num_context_lines'] = positive_int(val)
-
-    def pygments_style(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['pygments_style'] = str(val)
-
-    def removed_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['removed_bg'] = to_color(val)
-
-    def removed_margin_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['removed_margin_bg'] = to_color(val)
-
-    def replace_tab_by(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['replace_tab_by'] = python_string(val)
-
-    def search_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['search_bg'] = to_color(val)
-
-    def search_fg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['search_fg'] = to_color(val)
-
-    def select_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['select_bg'] = to_color(val)
-
-    def select_fg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['select_fg'] = to_color_or_none(val)
-
-    def syntax_aliases(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['syntax_aliases'] = syntax_aliases(val)
-
-    def title_bg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['title_bg'] = to_color(val)
-
-    def title_fg(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        ans['title_fg'] = to_color(val)
-
-    def map(self, val: str, ans: typing.Dict[str, typing.Any]) -> None:
-        for k in parse_map(val):
-            ans['map'].append(k)
-
-
-def create_result_dict() -> typing.Dict[str, typing.Any]:
-    return {
-        'ignore_name': {},
-        'map': [],
-    }
-
-
-actions: typing.FrozenSet[str] = frozenset(('map',))
-
-
-def merge_result_dicts(defaults: typing.Dict[str, typing.Any], vals: typing.Dict[str, typing.Any]) -> typing.Dict[str, typing.Any]:
-    ans = {}
-    for k, v in defaults.items():
-        if isinstance(v, dict):
-            ans[k] = merge_dicts(v, vals.get(k, {}))
-        elif k in actions:
-            ans[k] = v + vals.get(k, [])
-        else:
-            ans[k] = vals.get(k, v)
-    return ans
-
-
-parser = Parser()
-
-
-def parse_conf_item(key: str, val: str, ans: typing.Dict[str, typing.Any]) -> bool:
-    func = getattr(parser, key, None)
-    if func is not None:
-        func(val, ans)
-        return True
-    return False

kittens/diff/options/types.py

@@ -1,174 +0,0 @@
-# generated by gen-config.py DO NOT edit
-
-# isort: skip_file
-import typing
-from kitty.conf.utils import KeyAction, KittensKeyMap
-import kitty.conf.utils
-from kitty.fast_data_types import Color
-import kitty.fast_data_types
-from kitty.types import ParsedShortcut
-import kitty.types
-
-
-option_names = (  # {{{
- 'added_bg',
- 'added_margin_bg',
- 'background',
- 'diff_cmd',
- 'filler_bg',
- 'foreground',
- 'highlight_added_bg',
- 'highlight_removed_bg',
- 'hunk_bg',
- 'hunk_margin_bg',
- 'ignore_name',
- 'map',
- 'margin_bg',
- 'margin_fg',
- 'margin_filler_bg',
- 'num_context_lines',
- 'pygments_style',
- 'removed_bg',
- 'removed_margin_bg',
- 'replace_tab_by',
- 'search_bg',
- 'search_fg',
- 'select_bg',
- 'select_fg',
- 'syntax_aliases',
- 'title_bg',
- 'title_fg')  # }}}
-
-
-class Options:
-    added_bg: Color = Color(230, 255, 237)
-    added_margin_bg: Color = Color(205, 255, 216)
-    background: Color = Color(255, 255, 255)
-    diff_cmd: str = 'auto'
-    filler_bg: Color = Color(250, 251, 252)
-    foreground: Color = Color(0, 0, 0)
-    highlight_added_bg: Color = Color(172, 242, 189)
-    highlight_removed_bg: Color = Color(253, 184, 192)
-    hunk_bg: Color = Color(241, 248, 255)
-    hunk_margin_bg: Color = Color(219, 237, 255)
-    margin_bg: Color = Color(250, 251, 252)
-    margin_fg: Color = Color(170, 170, 170)
-    margin_filler_bg: typing.Optional[kitty.fast_data_types.Color] = None
-    num_context_lines: int = 3
-    pygments_style: str = 'default'
-    removed_bg: Color = Color(255, 238, 240)
-    removed_margin_bg: Color = Color(255, 220, 224)
-    replace_tab_by: str = '    '
-    search_bg: Color = Color(68, 68, 68)
-    search_fg: Color = Color(255, 255, 255)
-    select_bg: Color = Color(180, 213, 254)
-    select_fg: typing.Optional[kitty.fast_data_types.Color] = Color(0, 0, 0)
-    syntax_aliases: typing.Dict[str, str] = {'pyj': 'py', 'pyi': 'py', 'recipe': 'py'}
-    title_bg: Color = Color(255, 255, 255)
-    title_fg: Color = Color(0, 0, 0)
-    ignore_name: typing.Dict[str, str] = {}
-    map: typing.List[typing.Tuple[kitty.types.ParsedShortcut, kitty.conf.utils.KeyAction]] = []
-    key_definitions: KittensKeyMap = {}
-    config_paths: typing.Tuple[str, ...] = ()
-    config_overrides: typing.Tuple[str, ...] = ()
-
-    def __init__(self, options_dict: typing.Optional[typing.Dict[str, typing.Any]] = None) -> None:
-        if options_dict is not None:
-            null = object()
-            for key in option_names:
-                val = options_dict.get(key, null)
-                if val is not null:
-                    setattr(self, key, val)
-
-    @property
-    def _fields(self) -> typing.Tuple[str, ...]:
-        return option_names
-
-    def __iter__(self) -> typing.Iterator[str]:
-        return iter(self._fields)
-
-    def __len__(self) -> int:
-        return len(self._fields)
-
-    def _copy_of_val(self, name: str) -> typing.Any:
-        ans = getattr(self, name)
-        if isinstance(ans, dict):
-            ans = ans.copy()
-        elif isinstance(ans, list):
-            ans = ans[:]
-        return ans
-
-    def _asdict(self) -> typing.Dict[str, typing.Any]:
-        return {k: self._copy_of_val(k) for k in self}
-
-    def _replace(self, **kw: typing.Any) -> "Options":
-        ans = Options()
-        for name in self:
-            setattr(ans, name, self._copy_of_val(name))
-        for name, val in kw.items():
-            setattr(ans, name, val)
-        return ans
-
-    def __getitem__(self, key: typing.Union[int, str]) -> typing.Any:
-        k = option_names[key] if isinstance(key, int) else key
-        try:
-            return getattr(self, k)
-        except AttributeError:
-            pass
-        raise KeyError(f"No option named: {k}")
-
-
-defaults = Options()
-defaults.ignore_name = {}
-defaults.map = [
-    # quit
-    (ParsedShortcut(mods=0, key_name='q'), KeyAction('quit')), 
-    # quit
-    (ParsedShortcut(mods=0, key_name='ESCAPE'), KeyAction('quit')), 
-    # scroll_down
-    (ParsedShortcut(mods=0, key_name='j'), KeyAction('scroll_by', (1,))), 
-    # scroll_down
-    (ParsedShortcut(mods=0, key_name='DOWN'), KeyAction('scroll_by', (1,))), 
-    # scroll_up
-    (ParsedShortcut(mods=0, key_name='k'), KeyAction('scroll_by', (-1,))), 
-    # scroll_up
-    (ParsedShortcut(mods=0, key_name='UP'), KeyAction('scroll_by', (-1,))), 
-    # scroll_top
-    (ParsedShortcut(mods=0, key_name='HOME'), KeyAction('scroll_to', ('start',))), 
-    # scroll_bottom
-    (ParsedShortcut(mods=0, key_name='END'), KeyAction('scroll_to', ('end',))), 
-    # scroll_page_down
-    (ParsedShortcut(mods=0, key_name='PAGE_DOWN'), KeyAction('scroll_to', ('next-page',))), 
-    # scroll_page_down
-    (ParsedShortcut(mods=0, key_name=' '), KeyAction('scroll_to', ('next-page',))), 
-    # scroll_page_up
-    (ParsedShortcut(mods=0, key_name='PAGE_UP'), KeyAction('scroll_to', ('prev-page',))), 
-    # next_change
-    (ParsedShortcut(mods=0, key_name='n'), KeyAction('scroll_to', ('next-change',))), 
-    # prev_change
-    (ParsedShortcut(mods=0, key_name='p'), KeyAction('scroll_to', ('prev-change',))), 
-    # all_context
-    (ParsedShortcut(mods=0, key_name='a'), KeyAction('change_context', ('all',))), 
-    # default_context
-    (ParsedShortcut(mods=0, key_name='='), KeyAction('change_context', ('default',))), 
-    # increase_context
-    (ParsedShortcut(mods=0, key_name='+'), KeyAction('change_context', (5,))), 
-    # decrease_context
-    (ParsedShortcut(mods=0, key_name='-'), KeyAction('change_context', (-5,))), 
-    # search_forward
-    (ParsedShortcut(mods=0, key_name='/'), KeyAction('start_search', (True, False))), 
-    # search_backward
-    (ParsedShortcut(mods=0, key_name='?'), KeyAction('start_search', (True, True))), 
-    # next_match
-    (ParsedShortcut(mods=0, key_name='.'), KeyAction('scroll_to', ('next-match',))), 
-    # next_match
-    (ParsedShortcut(mods=0, key_name='>'), KeyAction('scroll_to', ('next-match',))), 
-    # prev_match
-    (ParsedShortcut(mods=0, key_name=','), KeyAction('scroll_to', ('prev-match',))), 
-    # prev_match
-    (ParsedShortcut(mods=0, key_name='<'), KeyAction('scroll_to', ('prev-match',))), 
-    # search_forward_simple
-    (ParsedShortcut(mods=0, key_name='f'), KeyAction('start_search', (False, False))), 
-    # search_backward_simple
-    (ParsedShortcut(mods=0, key_name='b'), KeyAction('start_search', (False, True))), 
-]

kittens/diff/options/utils.py

@@ -1,68 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=utf-8
-# License: GPLv3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
-
-
-from typing import Any, Container, Dict, Iterable, Tuple, Union
-
-from kitty.conf.utils import KeyFuncWrapper, KittensKeyDefinition, parse_kittens_key
-
-ReturnType = Tuple[str, Any]
-func_with_args = KeyFuncWrapper[ReturnType]()
-
-
-@func_with_args('scroll_by')
-def parse_scroll_by(func: str, rest: str) -> Tuple[str, int]:
-    try:
-        return func, int(rest)
-    except Exception:
-        return func, 1
-
-
-@func_with_args('scroll_to')
-def parse_scroll_to(func: str, rest: str) -> Tuple[str, str]:
-    rest = rest.lower()
-    if rest not in {'start', 'end', 'next-change', 'prev-change', 'next-page', 'prev-page', 'next-match', 'prev-match'}:
-        rest = 'start'
-    return func, rest
-
-
-@func_with_args('change_context')
-def parse_change_context(func: str, rest: str) -> Tuple[str, Union[int, str]]:
-    rest = rest.lower()
-    if rest in {'all', 'default'}:
-        return func, rest
-    try:
-        amount = int(rest)
-    except Exception:
-        amount = 5
-    return func, amount
-
-
-@func_with_args('start_search')
-def parse_start_search(func: str, rest: str) -> Tuple[str, Tuple[bool, bool]]:
-    rest_ = rest.lower().split()
-    is_regex = bool(rest_ and rest_[0] == 'regex')
-    is_backward = bool(len(rest_) > 1 and rest_[1] == 'backward')
-    return func, (is_regex, is_backward)
-
-
-def syntax_aliases(raw: str) -> Dict[str, str]:
-    ans = {}
-    for x in raw.split():
-        a, b = x.partition(':')[::2]
-        if a and b:
-            ans[a.lower()] = b
-    return ans
-
-
-def store_multiple(val: str, current_val: Container[str]) -> Iterable[Tuple[str, str]]:
-    val = val.strip()
-    if val not in current_val:
-        yield val, val
-
-
-def parse_map(val: str) -> Iterable[KittensKeyDefinition]:
-    x = parse_kittens_key(val, func_with_args.args_funcs)
-    if x is not None:
-        yield x

kittens/diff/patch.go

@@ -0,0 +1,376 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"kitty/tools/utils"
+	"kitty/tools/utils/images"
+	"kitty/tools/utils/shlex"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+var _ = fmt.Print
+
+const GIT_DIFF = `git diff --no-color --no-ext-diff --exit-code -U_CONTEXT_ --no-index --`
+const DIFF_DIFF = `diff -p -U _CONTEXT_ --`
+
+var diff_cmd []string
+
+var GitExe = (&utils.Once[string]{Run: func() string {
+	return utils.FindExe("git")
+}}).Get
+
+var DiffExe = (&utils.Once[string]{Run: func() string {
+	return utils.FindExe("diff")
+}}).Get
+
+func find_differ() {
+	if GitExe() != "git" && exec.Command(GitExe(), "--help").Run() == nil {
+		diff_cmd, _ = shlex.Split(GIT_DIFF)
+	} else if DiffExe() != "diff" && exec.Command(DiffExe(), "--help").Run() == nil {
+		diff_cmd, _ = shlex.Split(DIFF_DIFF)
+	} else {
+		diff_cmd = []string{}
+	}
+}
+
+func set_diff_command(q string) error {
+	switch q {
+	case "auto":
+		find_differ()
+	case "builtin", "":
+		diff_cmd = []string{}
+	case "diff":
+		diff_cmd, _ = shlex.Split(DIFF_DIFF)
+	case "git":
+		diff_cmd, _ = shlex.Split(GIT_DIFF)
+	default:
+		c, err := shlex.Split(q)
+		if err != nil {
+			return err
+		}
+		diff_cmd = c
+	}
+	return nil
+}
+
+type Center struct{ offset, left_size, right_size int }
+
+type Chunk struct {
+	is_context              bool
+	left_start, right_start int
+	left_count, right_count int
+	centers                 []Center
+}
+
+func (self *Chunk) add_line() {
+	self.right_count++
+}
+
+func (self *Chunk) remove_line() {
+	self.left_count++
+}
+
+func (self *Chunk) context_line() {
+	self.left_count++
+	self.right_count++
+}
+
+func changed_center(left, right string) (ans Center) {
+	if len(left) > 0 && len(right) > 0 {
+		ll, rl := len(left), len(right)
+		ml := utils.Min(ll, rl)
+		for ; ans.offset < ml && left[ans.offset] == right[ans.offset]; ans.offset++ {
+		}
+		suffix_count := 0
+		for ; suffix_count < ml && left[ll-1-suffix_count] == right[rl-1-suffix_count]; suffix_count++ {
+		}
+		ans.left_size = ll - suffix_count - ans.offset
+		ans.right_size = rl - suffix_count - ans.offset
+	}
+	return
+}
+
+func (self *Chunk) finalize(left_lines, right_lines []string) {
+	if !self.is_context && self.left_count == self.right_count {
+		for i := 0; i < self.left_count; i++ {
+			self.centers = append(self.centers, changed_center(left_lines[self.left_start+i], right_lines[self.right_start+i]))
+		}
+	}
+}
+
+type Hunk struct {
+	left_start, left_count     int
+	right_start, right_count   int
+	title                      string
+	added_count, removed_count int
+	chunks                     []*Chunk
+	current_chunk              *Chunk
+	largest_line_number        int
+}
+
+func (self *Hunk) new_chunk(is_context bool) *Chunk {
+	left_start, right_start := self.left_start, self.right_start
+	if len(self.chunks) > 0 {
+		c := self.chunks[len(self.chunks)-1]
+		left_start = c.left_start + c.left_count
+		right_start = c.right_start + c.right_count
+	}
+	return &Chunk{is_context: is_context, left_start: left_start, right_start: right_start}
+}
+
+func (self *Hunk) ensure_diff_chunk() {
+	if self.current_chunk == nil || self.current_chunk.is_context {
+		if self.current_chunk != nil {
+			self.chunks = append(self.chunks, self.current_chunk)
+		}
+		self.current_chunk = self.new_chunk(false)
+	}
+}
+
+func (self *Hunk) ensure_context_chunk() {
+	if self.current_chunk == nil || !self.current_chunk.is_context {
+		if self.current_chunk != nil {
+			self.chunks = append(self.chunks, self.current_chunk)
+		}
+		self.current_chunk = self.new_chunk(true)
+	}
+}
+
+func (self *Hunk) add_line() {
+	self.ensure_diff_chunk()
+	self.current_chunk.add_line()
+	self.added_count++
+}
+
+func (self *Hunk) remove_line() {
+	self.ensure_diff_chunk()
+	self.current_chunk.remove_line()
+	self.removed_count++
+}
+
+func (self *Hunk) context_line() {
+	self.ensure_context_chunk()
+	self.current_chunk.context_line()
+}
+
+func (self *Hunk) finalize(left_lines, right_lines []string) error {
+	if self.current_chunk != nil {
+		self.chunks = append(self.chunks, self.current_chunk)
+	}
+	// Sanity check
+	c := self.chunks[len(self.chunks)-1]
+	if c.left_start+c.left_count != self.left_start+self.left_count {
+		return fmt.Errorf("Left side line mismatch %d != %d", c.left_start+c.left_count, self.left_start+self.left_count)
+	}
+	if c.right_start+c.right_count != self.right_start+self.right_count {
+		return fmt.Errorf("Right side line mismatch %d != %d", c.right_start+c.right_count, self.right_start+self.right_count)
+	}
+	for _, c := range self.chunks {
+		c.finalize(left_lines, right_lines)
+	}
+	return nil
+}
+
+type Patch struct {
+	all_hunks                                       []*Hunk
+	largest_line_number, added_count, removed_count int
+}
+
+func (self *Patch) Len() int { return len(self.all_hunks) }
+
+func splitlines_like_git(raw string, strip_trailing_lines bool, process_line func(string)) {
+	sz := len(raw)
+	if strip_trailing_lines {
+		for sz > 0 && (raw[sz-1] == '\n' || raw[sz-1] == '\r') {
+			sz--
+		}
+	}
+	start := 0
+	for i := 0; i < sz; i++ {
+		switch raw[i] {
+		case '\n':
+			process_line(raw[start:i])
+			start = i + 1
+		case '\r':
+			process_line(raw[start:i])
+			start = i + 1
+			if start < sz && raw[start] == '\n' {
+				i++
+				start++
+			}
+		}
+	}
+	if start < sz {
+		process_line(raw[start:sz])
+	}
+}
+
+func parse_range(x string) (start, count int) {
+	s, c, found := strings.Cut(x, ",")
+	start, _ = strconv.Atoi(s)
+	if start < 0 {
+		start = -start
+	}
+	count = 1
+	if found {
+		count, _ = strconv.Atoi(c)
+	}
+	return
+}
+
+func parse_hunk_header(line string) *Hunk {
+	parts := strings.SplitN(line, "@@", 3)
+	linespec := strings.TrimSpace(parts[1])
+	title := ""
+	if len(parts) == 3 {
+		title = strings.TrimSpace(parts[2])
+	}
+	left, right, _ := strings.Cut(linespec, " ")
+	ls, lc := parse_range(left)
+	rs, rc := parse_range(right)
+	return &Hunk{
+		title: title, left_start: ls - 1, left_count: lc, right_start: rs - 1, right_count: rc,
+		largest_line_number: utils.Max(ls-1+lc, rs-1+rc),
+	}
+}
+
+func parse_patch(raw string, left_lines, right_lines []string) (ans *Patch, err error) {
+	ans = &Patch{all_hunks: make([]*Hunk, 0, 32)}
+	var current_hunk *Hunk
+	splitlines_like_git(raw, true, func(line string) {
+		if strings.HasPrefix(line, "@@ ") {
+			current_hunk = parse_hunk_header(line)
+			ans.all_hunks = append(ans.all_hunks, current_hunk)
+		} else if current_hunk != nil {
+			var ch byte
+			if len(line) > 0 {
+				ch = line[0]
+			}
+			switch ch {
+			case '+':
+				current_hunk.add_line()
+			case '-':
+				current_hunk.remove_line()
+			case '\\':
+			default:
+				current_hunk.context_line()
+			}
+		}
+	})
+	for _, h := range ans.all_hunks {
+		err = h.finalize(left_lines, right_lines)
+		if err != nil {
+			return
+		}
+		ans.added_count += h.added_count
+		ans.removed_count += h.removed_count
+	}
+	if len(ans.all_hunks) > 0 {
+		ans.largest_line_number = ans.all_hunks[len(ans.all_hunks)-1].largest_line_number
+	}
+	return
+}
+
+func run_diff(file1, file2 string, num_of_context_lines int) (ok, is_different bool, patch string, err error) {
+	// we resolve symlinks because git diff does not follow symlinks, while diff
+	// does. We want consistent behavior, also for integration with git difftool
+	// we always want symlinks to be followed.
+	path1, err := filepath.EvalSymlinks(file1)
+	if err != nil {
+		return
+	}
+	path2, err := filepath.EvalSymlinks(file2)
+	if err != nil {
+		return
+	}
+	if len(diff_cmd) == 0 {
+		data1, err := data_for_path(path1)
+		if err != nil {
+			return false, false, "", err
+		}
+		data2, err := data_for_path(path2)
+		if err != nil {
+			return false, false, "", err
+		}
+		patchb := Diff(path1, data1, path2, data2, num_of_context_lines)
+		if patchb == nil {
+			return true, false, "", nil
+		}
+		return true, len(patchb) > 0, utils.UnsafeBytesToString(patchb), nil
+	} else {
+		context := strconv.Itoa(num_of_context_lines)
+		cmd := utils.Map(func(x string) string {
+			return strings.ReplaceAll(x, "_CONTEXT_", context)
+		}, diff_cmd)
+
+		cmd = append(cmd, path1, path2)
+		c := exec.Command(cmd[0], cmd[1:]...)
+		stdout, stderr := bytes.Buffer{}, bytes.Buffer{}
+		c.Stdout, c.Stderr = &stdout, &stderr
+		err = c.Run()
+		if err != nil {
+			var e *exec.ExitError
+			if errors.As(err, &e) && e.ExitCode() == 1 {
+				return true, true, stdout.String(), nil
+			}
+			return false, false, stderr.String(), err
+		}
+		return true, false, stdout.String(), nil
+	}
+}
+
+func do_diff(file1, file2 string, context_count int) (ans *Patch, err error) {
+	ok, _, raw, err := run_diff(file1, file2, context_count)
+	if !ok {
+		return nil, fmt.Errorf("Failed to diff %s vs. %s with errors:\n%s", file1, file2, raw)
+	}
+	if err != nil {
+		return
+	}
+	left_lines, err := lines_for_path(file1)
+	if err != nil {
+		return
+	}
+	right_lines, err := lines_for_path(file2)
+	if err != nil {
+		return
+	}
+	ans, err = parse_patch(raw, left_lines, right_lines)
+	return
+}
+
+type diff_job struct{ file1, file2 string }
+
+func diff(jobs []diff_job, context_count int) (ans map[string]*Patch, err error) {
+	ans = make(map[string]*Patch)
+	ctx := images.Context{}
+	type result struct {
+		file1, file2 string
+		err          error
+		patch        *Patch
+	}
+	results := make(chan result, len(jobs))
+	ctx.Parallel(0, len(jobs), func(nums <-chan int) {
+		for i := range nums {
+			job := jobs[i]
+			r := result{file1: job.file1, file2: job.file2}
+			r.patch, r.err = do_diff(job.file1, job.file2, context_count)
+			results <- r
+		}
+	})
+	close(results)
+	for r := range results {
+		if r.err != nil {
+			return nil, r.err
+		}
+		ans[r.file1] = r.patch
+	}
+	return ans, nil
+}

kittens/diff/patch.py

@@ -1,257 +0,0 @@
-#!/usr/bin/env python3
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-import concurrent.futures
-import os
-import shlex
-import shutil
-import subprocess
-from typing import Dict, Iterator, List, Optional, Sequence, Tuple, Union
-
-from . import global_data
-from .collect import lines_for_path
-from .diff_speedup import changed_center
-
-left_lines: Tuple[str, ...] = ()
-right_lines: Tuple[str, ...] = ()
-GIT_DIFF = 'git diff --no-color --no-ext-diff --exit-code -U_CONTEXT_ --no-index --'
-DIFF_DIFF = 'diff -p -U _CONTEXT_ --'
-worker_processes: List[int] = []
-
-
-def find_differ() -> Optional[str]:
-    if shutil.which('git') and subprocess.Popen(['git', '--help'], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL).wait() == 0:
-        return GIT_DIFF
-    if shutil.which('diff'):
-        return DIFF_DIFF
-    return None
-
-
-def set_diff_command(opt: str) -> None:
-    if opt == 'auto':
-        cmd = find_differ()
-        if cmd is None:
-            raise SystemExit('Failed to find either the git or diff programs on your system')
-    else:
-        cmd = opt
-    global_data.cmd = cmd
-
-
-def run_diff(file1: str, file2: str, context: int = 3) -> Tuple[bool, Union[int, bool], str]:
-    # returns: ok, is_different, patch
-    cmd = shlex.split(global_data.cmd.replace('_CONTEXT_', str(context)))
-    # we resolve symlinks because git diff does not follow symlinks, while diff
-    # does. We want consistent behavior, also for integration with git difftool
-    # we always want symlinks to be followed.
-    path1 = os.path.realpath(file1)
-    path2 = os.path.realpath(file2)
-    p = subprocess.Popen(
-            cmd + [path1, path2],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL)
-    worker_processes.append(p.pid)
-    stdout, stderr = p.communicate()
-    returncode = p.wait()
-    worker_processes.remove(p.pid)
-    if returncode in (0, 1):
-        return True, returncode == 1, stdout.decode('utf-8')
-    return False, returncode, stderr.decode('utf-8')
-
-
-class Chunk:
-
-    __slots__ = ('is_context', 'left_start', 'right_start', 'left_count', 'right_count', 'centers')
-
-    def __init__(self, left_start: int, right_start: int, is_context: bool = False) -> None:
-        self.is_context = is_context
-        self.left_start = left_start
-        self.right_start = right_start
-        self.left_count = self.right_count = 0
-        self.centers: Optional[Tuple[Tuple[int, int], ...]] = None
-
-    def add_line(self) -> None:
-        self.right_count += 1
-
-    def remove_line(self) -> None:
-        self.left_count += 1
-
-    def context_line(self) -> None:
-        self.left_count += 1
-        self.right_count += 1
-
-    def finalize(self) -> None:
-        if not self.is_context and self.left_count == self.right_count:
-            self.centers = tuple(
-                changed_center(left_lines[self.left_start + i], right_lines[self.right_start + i])
-                for i in range(self.left_count)
-            )
-
-    def __repr__(self) -> str:
-        return 'Chunk(is_context={}, left_start={}, left_count={}, right_start={}, right_count={})'.format(
-                self.is_context, self.left_start, self.left_count, self.right_start, self.right_count)
-
-
-class Hunk:
-
-    def __init__(self, title: str, left: Tuple[int, int], right: Tuple[int, int]) -> None:
-        self.left_start, self.left_count = left
-        self.right_start, self.right_count = right
-        self.left_start -= 1  # 0-index
-        self.right_start -= 1  # 0-index
-        self.title = title
-        self.added_count = self.removed_count = 0
-        self.chunks: List[Chunk] = []
-        self.current_chunk: Optional[Chunk] = None
-        self.largest_line_number = max(self.left_start + self.left_count, self.right_start + self.right_count)
-
-    def new_chunk(self, is_context: bool = False) -> Chunk:
-        if self.chunks:
-            c = self.chunks[-1]
-            left_start = c.left_start + c.left_count
-            right_start = c.right_start + c.right_count
-        else:
-            left_start = self.left_start
-            right_start = self.right_start
-        return Chunk(left_start, right_start, is_context)
-
-    def ensure_diff_chunk(self) -> None:
-        if self.current_chunk is None:
-            self.current_chunk = self.new_chunk(is_context=False)
-        elif self.current_chunk.is_context:
-            self.chunks.append(self.current_chunk)
-            self.current_chunk = self.new_chunk(is_context=False)
-
-    def ensure_context_chunk(self) -> None:
-        if self.current_chunk is None:
-            self.current_chunk = self.new_chunk(is_context=True)
-        elif not self.current_chunk.is_context:
-            self.chunks.append(self.current_chunk)
-            self.current_chunk = self.new_chunk(is_context=True)
-
-    def add_line(self) -> None:
-        self.ensure_diff_chunk()
-        if self.current_chunk is not None:
-            self.current_chunk.add_line()
-        self.added_count += 1
-
-    def remove_line(self) -> None:
-        self.ensure_diff_chunk()
-        if self.current_chunk is not None:
-            self.current_chunk.remove_line()
-        self.removed_count += 1
-
-    def context_line(self) -> None:
-        self.ensure_context_chunk()
-        if self.current_chunk is not None:
-            self.current_chunk.context_line()
-
-    def finalize(self) -> None:
-        if self.current_chunk is not None:
-            self.chunks.append(self.current_chunk)
-        del self.current_chunk
-        # Sanity check
-        c = self.chunks[-1]
-        if c.left_start + c.left_count != self.left_start + self.left_count:
-            raise ValueError(f'Left side line mismatch {c.left_start + c.left_count} != {self.left_start + self.left_count}')
-        if c.right_start + c.right_count != self.right_start + self.right_count:
-            raise ValueError(f'Right side line mismatch {c.right_start + c.right_count} != {self.right_start + self.right_count}')
-        for c in self.chunks:
-            c.finalize()
-
-
-def parse_range(x: str) -> Tuple[int, int]:
-    parts = x[1:].split(',', 1)
-    start = abs(int(parts[0]))
-    count = 1 if len(parts) < 2 else int(parts[1])
-    return start, count
-
-
-def parse_hunk_header(line: str) -> Hunk:
-    parts: Tuple[str, ...] = tuple(filter(None, line.split('@@', 2)))
-    linespec = parts[0].strip()
-    title = ''
-    if len(parts) == 2:
-        title = parts[1].strip()
-    left, right = map(parse_range, linespec.split())
-    return Hunk(title, left, right)
-
-
-class Patch:
-
-    def __init__(self, all_hunks: Sequence[Hunk]):
-        self.all_hunks = all_hunks
-        self.largest_line_number = self.all_hunks[-1].largest_line_number if self.all_hunks else 0
-        self.added_count = sum(h.added_count for h in all_hunks)
-        self.removed_count = sum(h.removed_count for h in all_hunks)
-
-    def __iter__(self) -> Iterator[Hunk]:
-        return iter(self.all_hunks)
-
-    def __len__(self) -> int:
-        return len(self.all_hunks)
-
-
-def parse_patch(raw: str) -> Patch:
-    all_hunks = []
-    current_hunk = None
-    for line in raw.splitlines():
-        if line.startswith('@@ '):
-            current_hunk = parse_hunk_header(line)
-            all_hunks.append(current_hunk)
-        else:
-            if current_hunk is None:
-                continue
-            q = line[0] if line else ''
-            if q == '+':
-                current_hunk.add_line()
-            elif q == '-':
-                current_hunk.remove_line()
-            elif q == '\\':
-                continue
-            else:
-                current_hunk.context_line()
-    for h in all_hunks:
-        h.finalize()
-    return Patch(all_hunks)
-
-
-class Differ:
-
-    diff_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
-
-    def __init__(self) -> None:
-        self.jmap: Dict[str, str] = {}
-        self.jobs: List[str] = []
-        if Differ.diff_executor is None:
-            Differ.diff_executor = self.diff_executor = concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count())
-
-    def add_diff(self, file1: str, file2: str) -> None:
-        self.jmap[file1] = file2
-        self.jobs.append(file1)
-
-    def __call__(self, context: int = 3) -> Union[str, Dict[str, Patch]]:
-        global left_lines, right_lines
-        ans: Dict[str, Patch] = {}
-        executor = self.diff_executor
-        assert executor is not None
-        jobs = {executor.submit(run_diff, key, self.jmap[key], context): key for key in self.jobs}
-        for future in concurrent.futures.as_completed(jobs):
-            key = jobs[future]
-            left_path, right_path = key, self.jmap[key]
-            try:
-                ok, returncode, output = future.result()
-            except FileNotFoundError as err:
-                return f'Could not find the {err.filename} executable. Is it in your PATH?'
-            except Exception as e:
-                return f'Running git diff for {left_path} vs. {right_path} generated an exception: {e}'
-            if not ok:
-                return f'{output}\nRunning git diff for {left_path} vs. {right_path} failed'
-            left_lines = lines_for_path(left_path)
-            right_lines = lines_for_path(right_path)
-            try:
-                patch = parse_patch(output)
-            except Exception:
-                import traceback
-                return f'{traceback.format_exc()}\nParsing diff for {left_path} vs. {right_path} failed'
-            else:
-                ans[key] = patch
-        return ans

kittens/diff/render.go

@@ -0,0 +1,763 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"kitty/tools/tui/graphics"
+	"kitty/tools/tui/loop"
+	"kitty/tools/tui/sgr"
+	"kitty/tools/utils"
+	"kitty/tools/utils/style"
+	"kitty/tools/wcswidth"
+)
+
+var _ = fmt.Print
+
+type LineType int
+
+const (
+	TITLE_LINE LineType = iota
+	CHANGE_LINE
+	CONTEXT_LINE
+	HUNK_TITLE_LINE
+	IMAGE_LINE
+	EMPTY_LINE
+)
+
+type Reference struct {
+	path    string
+	linenum int // 1 based
+}
+
+type HalfScreenLine struct {
+	marked_up_margin_text string
+	marked_up_text        string
+	is_filler             bool
+	cached_wcswidth       int
+}
+
+func (self *HalfScreenLine) wcswidth() int {
+	if self.cached_wcswidth == 0 && self.marked_up_text != "" {
+		self.cached_wcswidth = wcswidth.Stringwidth(self.marked_up_text)
+	}
+	return self.cached_wcswidth
+}
+
+type ScreenLine struct {
+	left, right HalfScreenLine
+}
+
+type LogicalLine struct {
+	line_type                       LineType
+	screen_lines                    []*ScreenLine
+	is_full_width                   bool
+	is_change_start                 bool
+	left_reference, right_reference Reference
+	left_image, right_image         struct {
+		key   string
+		count int
+	}
+	image_lines_offset int
+}
+
+func (self *LogicalLine) render_screen_line(n int, lp *loop.Loop, margin_size, columns int) {
+	if n >= len(self.screen_lines) || n < 0 {
+		return
+	}
+	sl := self.screen_lines[n]
+	available_cols := columns/2 - margin_size
+	if self.is_full_width {
+		available_cols = columns - margin_size
+	}
+	left_margin := place_in(sl.left.marked_up_margin_text, margin_size)
+	left_text := place_in(sl.left.marked_up_text, available_cols)
+	if sl.left.is_filler {
+		left_margin = format_as_sgr.margin_filler + left_margin
+		left_text = format_as_sgr.filler + left_text
+	} else {
+		switch self.line_type {
+		case CHANGE_LINE, IMAGE_LINE:
+			left_margin = format_as_sgr.removed_margin + left_margin
+			left_text = format_as_sgr.removed + left_text
+		case HUNK_TITLE_LINE:
+			left_margin = format_as_sgr.hunk_margin + left_margin
+			left_text = format_as_sgr.hunk + left_text
+		case TITLE_LINE:
+		default:
+			left_margin = format_as_sgr.margin + left_margin
+		}
+	}
+	lp.QueueWriteString(left_margin + "\x1b[m")
+	lp.QueueWriteString(left_text)
+	if self.is_full_width {
+		return
+	}
+	right_margin := place_in(sl.right.marked_up_margin_text, margin_size)
+	right_text := place_in(sl.right.marked_up_text, available_cols)
+	if sl.right.is_filler {
+		right_margin = format_as_sgr.margin_filler + right_margin
+		right_text = format_as_sgr.filler + right_text
+	} else {
+		switch self.line_type {
+		case CHANGE_LINE, IMAGE_LINE:
+			right_margin = format_as_sgr.added_margin + right_margin
+			right_text = format_as_sgr.added + right_text
+		case HUNK_TITLE_LINE:
+			right_margin = format_as_sgr.hunk_margin + right_margin
+			right_text = format_as_sgr.hunk + right_text
+		case TITLE_LINE:
+		default:
+			right_margin = format_as_sgr.margin + right_margin
+		}
+	}
+	lp.QueueWriteString("\x1b[m\r")
+	lp.MoveCursorHorizontally(available_cols + margin_size)
+	lp.QueueWriteString(right_margin + "\x1b[m")
+	lp.QueueWriteString(right_text)
+}
+
+func (self *LogicalLine) IncrementScrollPosBy(pos *ScrollPos, amt int) (delta int) {
+	if len(self.screen_lines) > 0 {
+		npos := utils.Max(0, utils.Min(pos.screen_line+amt, len(self.screen_lines)-1))
+		delta = npos - pos.screen_line
+		pos.screen_line = npos
+	}
+	return
+}
+
+func fit_in(text string, count int) string {
+	truncated := wcswidth.TruncateToVisualLength(text, count)
+	if len(truncated) >= len(text) {
+		return text
+	}
+	if count > 1 {
+		truncated = wcswidth.TruncateToVisualLength(text, count-1)
+	}
+	return truncated + `…`
+}
+
+func fill_in(text string, sz int) string {
+	w := wcswidth.Stringwidth(text)
+	if w < sz {
+		text += strings.Repeat(` `, (sz - w))
+	}
+	return text
+}
+
+func place_in(text string, sz int) string {
+	return fill_in(fit_in(text, sz), sz)
+}
+
+var format_as_sgr struct {
+	title, margin, added, removed, added_margin, removed_margin, filler, margin_filler, hunk_margin, hunk, selection, search string
+}
+
+var statusline_format, added_count_format, removed_count_format, message_format, selection_format func(...any) string
+
+func create_formatters() {
+	ctx := style.Context{AllowEscapeCodes: true}
+	only_open := func(x string) string {
+		ans := ctx.SprintFunc(x)("|")
+		ans, _, _ = strings.Cut(ans, "|")
+		return ans
+	}
+	format_as_sgr.filler = only_open("bg=" + conf.Filler_bg.AsRGBSharp())
+	if conf.Margin_filler_bg.IsSet {
+		format_as_sgr.margin_filler = only_open("bg=" + conf.Margin_filler_bg.Color.AsRGBSharp())
+	} else {
+		format_as_sgr.margin_filler = only_open("bg=" + conf.Filler_bg.AsRGBSharp())
+	}
+	format_as_sgr.added = only_open("bg=" + conf.Added_bg.AsRGBSharp())
+	format_as_sgr.added_margin = only_open(fmt.Sprintf("fg=%s bg=%s", conf.Margin_fg.AsRGBSharp(), conf.Added_margin_bg.AsRGBSharp()))
+	format_as_sgr.removed = only_open("bg=" + conf.Removed_bg.AsRGBSharp())
+	format_as_sgr.removed_margin = only_open(fmt.Sprintf("fg=%s bg=%s", conf.Margin_fg.AsRGBSharp(), conf.Removed_margin_bg.AsRGBSharp()))
+	format_as_sgr.title = only_open(fmt.Sprintf("fg=%s bg=%s bold", conf.Title_fg.AsRGBSharp(), conf.Title_bg.AsRGBSharp()))
+	format_as_sgr.margin = only_open(fmt.Sprintf("fg=%s bg=%s", conf.Margin_fg.AsRGBSharp(), conf.Margin_bg.AsRGBSharp()))
+	format_as_sgr.hunk = only_open(fmt.Sprintf("fg=%s bg=%s", conf.Margin_fg.AsRGBSharp(), conf.Hunk_bg.AsRGBSharp()))
+	format_as_sgr.hunk_margin = only_open(fmt.Sprintf("fg=%s bg=%s", conf.Margin_fg.AsRGBSharp(), conf.Hunk_margin_bg.AsRGBSharp()))
+	format_as_sgr.search = only_open(fmt.Sprintf("fg=%s bg=%s", conf.Search_fg.AsRGBSharp(), conf.Search_bg.AsRGBSharp()))
+	statusline_format = ctx.SprintFunc(fmt.Sprintf("fg=%s", conf.Margin_fg.AsRGBSharp()))
+	added_count_format = ctx.SprintFunc(fmt.Sprintf("fg=%s", conf.Highlight_added_bg.AsRGBSharp()))
+	removed_count_format = ctx.SprintFunc(fmt.Sprintf("fg=%s", conf.Highlight_removed_bg.AsRGBSharp()))
+	message_format = ctx.SprintFunc("bold")
+	if conf.Select_fg.IsSet {
+		format_as_sgr.selection = only_open(fmt.Sprintf("fg=%s bg=%s", conf.Select_fg.Color.AsRGBSharp(), conf.Select_bg.AsRGBSharp()))
+	} else {
+		format_as_sgr.selection = only_open("bg=" + conf.Select_bg.AsRGBSharp())
+	}
+}
+
+func center_span(ltype string, offset, size int) *sgr.Span {
+	ans := sgr.NewSpan(offset, size)
+	switch ltype {
+	case "add":
+		ans.SetBackground(conf.Highlight_added_bg).SetClosingBackground(conf.Added_bg)
+	case "remove":
+		ans.SetBackground(conf.Highlight_removed_bg).SetClosingBackground(conf.Removed_bg)
+	}
+	return ans
+}
+
+func title_lines(left_path, right_path string, columns, margin_size int, ans []*LogicalLine) []*LogicalLine {
+	left_name, right_name := path_name_map[left_path], path_name_map[right_path]
+	available_cols := columns/2 - margin_size
+	ll := LogicalLine{
+		line_type:      TITLE_LINE,
+		left_reference: Reference{path: left_path}, right_reference: Reference{path: right_path},
+	}
+	sl := ScreenLine{}
+	if right_name != "" && right_name != left_name {
+		sl.left.marked_up_text = format_as_sgr.title + fit_in(sanitize(left_name), available_cols)
+		sl.right.marked_up_text = format_as_sgr.title + fit_in(sanitize(right_name), available_cols)
+	} else {
+		sl.left.marked_up_text = format_as_sgr.title + fit_in(sanitize(left_name), columns-margin_size)
+		ll.is_full_width = true
+	}
+	l2 := ll
+	l2.line_type = EMPTY_LINE
+	ll.screen_lines = append(ll.screen_lines, &sl)
+	sl2 := ScreenLine{}
+	sl2.left.marked_up_margin_text = "\x1b[m" + strings.Repeat("━", margin_size)
+	sl2.left.marked_up_text = strings.Repeat("━", columns-margin_size)
+	l2.is_full_width = true
+	l2.screen_lines = append(l2.screen_lines, &sl2)
+	return append(ans, &ll, &l2)
+}
+
+type LogicalLines struct {
+	lines                []*LogicalLine
+	margin_size, columns int
+}
+
+func (self *LogicalLines) At(i int) *LogicalLine { return self.lines[i] }
+
+func (self *LogicalLines) ScreenLineAt(pos ScrollPos) *ScreenLine {
+	if pos.logical_line < len(self.lines) && pos.logical_line >= 0 {
+		line := self.lines[pos.logical_line]
+		if pos.screen_line < len(line.screen_lines) && pos.screen_line >= 0 {
+			return self.lines[pos.logical_line].screen_lines[pos.screen_line]
+		}
+	}
+	return nil
+}
+func (self *LogicalLines) Len() int { return len(self.lines) }
+
+func (self *LogicalLines) NumScreenLinesTo(a ScrollPos) (ans int) {
+	return self.Minus(a, ScrollPos{})
+}
+
+// a - b in terms of number of screen lines between the positions
+func (self *LogicalLines) Minus(a, b ScrollPos) (delta int) {
+	if a.logical_line == b.logical_line {
+		return a.screen_line - b.screen_line
+	}
+	amt := 1
+	if a.Less(b) {
+		amt = -1
+	} else {
+		a, b = b, a
+	}
+	for i := a.logical_line; i < utils.Min(len(self.lines), b.logical_line+1); i++ {
+		line := self.lines[i]
+		switch i {
+		case a.logical_line:
+			delta += utils.Max(0, len(line.screen_lines)-a.screen_line)
+		case b.logical_line:
+			delta += b.screen_line
+		default:
+			delta += len(line.screen_lines)
+		}
+	}
+	return delta * amt
+}
+
+func (self *LogicalLines) IncrementScrollPosBy(pos *ScrollPos, amt int) (delta int) {
+	if pos.logical_line < 0 || pos.logical_line >= len(self.lines) || amt == 0 {
+		return
+	}
+	one := 1
+	if amt < 0 {
+		one = -1
+	}
+	for amt != 0 {
+		line := self.lines[pos.logical_line]
+		d := line.IncrementScrollPosBy(pos, amt)
+		if d == 0 {
+			nlp := pos.logical_line + one
+			if nlp < 0 || nlp >= len(self.lines) {
+				break
+			}
+			pos.logical_line = nlp
+			if one > 0 {
+				pos.screen_line = 0
+			} else {
+				pos.screen_line = len(self.lines[nlp].screen_lines) - 1
+			}
+			delta += one
+			amt -= one
+		} else {
+			amt -= d
+			delta += d
+		}
+	}
+	return
+}
+
+func human_readable(size int64) string {
+	divisor, suffix := 1, "B"
+	for i, candidate := range []string{"B", "KB", "MB", "GB", "TB", "PB", "EB"} {
+		if size < (1 << ((i + 1) * 10)) {
+			divisor, suffix = (1 << (i * 10)), candidate
+			break
+		}
+	}
+	fs := float64(size) / float64(divisor)
+	s := strconv.FormatFloat(fs, 'f', 2, 64)
+	if idx := strings.Index(s, "."); idx > -1 {
+		s = s[:idx+2]
+	}
+	if strings.HasSuffix(s, ".0") || strings.HasSuffix(s, ".00") {
+		idx := strings.IndexByte(s, '.')
+		s = s[:idx]
+	}
+	return s + " " + suffix
+}
+
+func image_lines(left_path, right_path string, screen_size screen_size, margin_size int, image_size graphics.Size, ans []*LogicalLine) ([]*LogicalLine, error) {
+	columns := screen_size.columns
+	available_cols := columns/2 - margin_size
+	ll, err := first_binary_line(left_path, right_path, columns, margin_size, func(path string) (string, error) {
+		sz, err := size_for_path(path)
+		if err != nil {
+			return "", err
+		}
+		text := fmt.Sprintf("Size: %s", human_readable(sz))
+		res := image_collection.ResolutionOf(path)
+		if res.Width > -1 {
+			text = fmt.Sprintf("Dimensions: %dx%d %s", res.Width, res.Height, text)
+		}
+		return text, nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	ll.image_lines_offset = len(ll.screen_lines)
+
+	do_side := func(path string) []string {
+		if path == "" {
+			return nil
+		}
+		sz, err := image_collection.GetSizeIfAvailable(path, image_size)
+		if err == nil {
+			count := int(math.Ceil(float64(sz.Height) / float64(screen_size.cell_height)))
+			return utils.Repeat("", count)
+		}
+		if errors.Is(err, graphics.ErrNotFound) {
+			return splitlines("Loading image...", available_cols)
+		}
+		return splitlines(fmt.Sprintf("Failed to load image: %s", err), available_cols)
+	}
+	left_lines := do_side(left_path)
+	if ll.left_image.count = len(left_lines); ll.left_image.count > 0 {
+		ll.left_image.key = left_path
+	}
+	right_lines := do_side(right_path)
+	if ll.right_image.count = len(right_lines); ll.right_image.count > 0 {
+		ll.right_image.key = right_path
+	}
+	for i := 0; i < utils.Max(len(left_lines), len(right_lines)); i++ {
+		sl := ScreenLine{}
+		if i < len(left_lines) {
+			sl.left.marked_up_text = left_lines[i]
+		} else {
+			sl.left.is_filler = true
+		}
+		if i < len(right_lines) {
+			sl.right.marked_up_text = right_lines[i]
+		} else {
+			sl.right.is_filler = true
+		}
+		ll.screen_lines = append(ll.screen_lines, &sl)
+	}
+	ll.line_type = IMAGE_LINE
+	return append(ans, ll), nil
+}
+
+type formatter = func(...any) string
+
+func first_binary_line(left_path, right_path string, columns, margin_size int, renderer func(path string) (string, error)) (*LogicalLine, error) {
+	available_cols := columns/2 - margin_size
+	ll := LogicalLine{
+		is_change_start: true, line_type: CHANGE_LINE,
+		left_reference: Reference{path: left_path}, right_reference: Reference{path: right_path},
+	}
+	if left_path == "" {
+		line, err := renderer(right_path)
+		if err != nil {
+			return nil, err
+		}
+		for _, x := range splitlines(line, available_cols) {
+			sl := ScreenLine{}
+			sl.right.marked_up_text = x
+			sl.left.is_filler = true
+			ll.screen_lines = append(ll.screen_lines, &sl)
+		}
+	} else if right_path == "" {
+		line, err := renderer(left_path)
+		if err != nil {
+			return nil, err
+		}
+		for _, x := range splitlines(line, available_cols) {
+			sl := ScreenLine{}
+			sl.right.is_filler = true
+			sl.left.marked_up_text = x
+			ll.screen_lines = append(ll.screen_lines, &sl)
+		}
+	} else {
+		l, err := renderer(left_path)
+		if err != nil {
+			return nil, err
+		}
+		r, err := renderer(right_path)
+		if err != nil {
+			return nil, err
+		}
+		left_lines, right_lines := splitlines(l, available_cols), splitlines(r, available_cols)
+		for i := 0; i < utils.Max(len(left_lines), len(right_lines)); i++ {
+			sl := ScreenLine{}
+			if i < len(left_lines) {
+				sl.left.marked_up_text = left_lines[i]
+			}
+			if i < len(right_lines) {
+				sl.right.marked_up_text = right_lines[i]
+			}
+			ll.screen_lines = append(ll.screen_lines, &sl)
+		}
+	}
+	return &ll, nil
+}
+
+func binary_lines(left_path, right_path string, columns, margin_size int, ans []*LogicalLine) (ans2 []*LogicalLine, err error) {
+	ll, err := first_binary_line(left_path, right_path, columns, margin_size, func(path string) (string, error) {
+		sz, err := size_for_path(path)
+		if err != nil {
+			return "", err
+		}
+		return fmt.Sprintf("Binary file: %s", human_readable(sz)), nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	return append(ans, ll), nil
+}
+
+type DiffData struct {
+	left_path, right_path       string
+	available_cols, margin_size int
+
+	left_lines, right_lines []string
+}
+
+func hunk_title(hunk *Hunk) string {
+	return fmt.Sprintf("@@ -%d,%d +%d,%d @@ %s", hunk.left_start+1, hunk.left_count, hunk.right_start+1, hunk.right_count, hunk.title)
+}
+
+func lines_for_context_chunk(data *DiffData, hunk_num int, chunk *Chunk, chunk_num int, ans []*LogicalLine) []*LogicalLine {
+	for i := 0; i < chunk.left_count; i++ {
+		left_line_number := chunk.left_start + i
+		right_line_number := chunk.right_start + i
+		ll := LogicalLine{line_type: CONTEXT_LINE,
+			left_reference:  Reference{path: data.left_path, linenum: left_line_number + 1},
+			right_reference: Reference{path: data.right_path, linenum: right_line_number + 1},
+		}
+		left_line_number_s := strconv.Itoa(left_line_number + 1)
+		right_line_number_s := strconv.Itoa(right_line_number + 1)
+		for _, text := range splitlines(data.left_lines[left_line_number], data.available_cols) {
+			left_line := HalfScreenLine{marked_up_margin_text: left_line_number_s, marked_up_text: text}
+			right_line := left_line
+			if right_line_number_s != left_line_number_s {
+				right_line = HalfScreenLine{marked_up_margin_text: right_line_number_s, marked_up_text: text}
+			}
+			ll.screen_lines = append(ll.screen_lines, &ScreenLine{left_line, right_line})
+			left_line_number_s, right_line_number_s = "", ""
+		}
+		ans = append(ans, &ll)
+	}
+	return ans
+}
+
+func splitlines(text string, width int) []string {
+	return style.WrapTextAsLines(text, width, style.WrapOptions{})
+}
+
+func render_half_line(line_number int, line, ltype string, available_cols int, center Center, ans []HalfScreenLine) []HalfScreenLine {
+	size := center.left_size
+	if ltype != "remove" {
+		size = center.right_size
+	}
+	if size > 0 {
+		span := center_span(ltype, center.offset, size)
+		line = sgr.InsertFormatting(line, span)
+	}
+	lnum := strconv.Itoa(line_number + 1)
+	for _, sc := range splitlines(line, available_cols) {
+		ans = append(ans, HalfScreenLine{marked_up_margin_text: lnum, marked_up_text: sc})
+		lnum = ""
+	}
+	return ans
+}
+
+func lines_for_diff_chunk(data *DiffData, hunk_num int, chunk *Chunk, chunk_num int, ans []*LogicalLine) []*LogicalLine {
+	common := utils.Min(chunk.left_count, chunk.right_count)
+	ll, rl := make([]HalfScreenLine, 0, 32), make([]HalfScreenLine, 0, 32)
+	for i := 0; i < utils.Max(chunk.left_count, chunk.right_count); i++ {
+		ll, rl = ll[:0], rl[:0]
+		var center Center
+		left_lnum, right_lnum := 0, 0
+		if i < len(chunk.centers) {
+			center = chunk.centers[i]
+		}
+		if i < chunk.left_count {
+			left_lnum = chunk.left_start + i
+			ll = render_half_line(left_lnum, data.left_lines[left_lnum], "remove", data.available_cols, center, ll)
+			left_lnum++
+		}
+
+		if i < chunk.right_count {
+			right_lnum = chunk.right_start + i
+			rl = render_half_line(right_lnum, data.right_lines[right_lnum], "add", data.available_cols, center, rl)
+			right_lnum++
+		}
+
+		if i < common {
+			extra := len(ll) - len(rl)
+			if extra < 0 {
+				ll = append(ll, utils.Repeat(HalfScreenLine{}, -extra)...)
+			} else if extra > 0 {
+				rl = append(rl, utils.Repeat(HalfScreenLine{}, extra)...)
+			}
+		} else {
+			if len(ll) > 0 {
+				rl = append(rl, utils.Repeat(HalfScreenLine{is_filler: true}, len(ll))...)
+			} else if len(rl) > 0 {
+				ll = append(ll, utils.Repeat(HalfScreenLine{is_filler: true}, len(rl))...)
+			}
+		}
+		logline := LogicalLine{
+			line_type: CHANGE_LINE, is_change_start: i == 0,
+			left_reference:  Reference{path: data.left_path, linenum: left_lnum},
+			right_reference: Reference{path: data.left_path, linenum: right_lnum},
+		}
+		for l := 0; l < len(ll); l++ {
+			logline.screen_lines = append(logline.screen_lines, &ScreenLine{left: ll[l], right: rl[l]})
+		}
+		ans = append(ans, &logline)
+	}
+	return ans
+}
+
+func lines_for_diff(left_path string, right_path string, patch *Patch, columns, margin_size int, ans []*LogicalLine) (result []*LogicalLine, err error) {
+	ht := LogicalLine{
+		line_type:      HUNK_TITLE_LINE,
+		left_reference: Reference{path: left_path}, right_reference: Reference{path: right_path},
+		is_full_width: true,
+	}
+	if patch.Len() == 0 {
+		for _, line := range splitlines("The files are identical", columns-margin_size) {
+			sl := ScreenLine{}
+			sl.left.marked_up_text = line
+			ht.screen_lines = append(ht.screen_lines, &sl)
+		}
+		ht.line_type = EMPTY_LINE
+		ht.is_full_width = true
+		return append(ans, &ht), nil
+	}
+	available_cols := columns/2 - margin_size
+	data := DiffData{left_path: left_path, right_path: right_path, available_cols: available_cols, margin_size: margin_size}
+	if left_path != "" {
+		data.left_lines, err = highlighted_lines_for_path(left_path)
+		if err != nil {
+			return
+		}
+	}
+	if right_path != "" {
+		data.right_lines, err = highlighted_lines_for_path(right_path)
+		if err != nil {
+			return
+		}
+	}
+
+	for hunk_num, hunk := range patch.all_hunks {
+		htl := ht
+		htl.left_reference.linenum = hunk.left_start + 1
+		htl.right_reference.linenum = hunk.right_start + 1
+		for _, line := range splitlines(hunk_title(hunk), columns-margin_size) {
+			sl := ScreenLine{}
+			sl.left.marked_up_text = line
+			htl.screen_lines = append(htl.screen_lines, &sl)
+		}
+		ans = append(ans, &htl)
+		for cnum, chunk := range hunk.chunks {
+			if chunk.is_context {
+				ans = lines_for_context_chunk(&data, hunk_num, chunk, cnum, ans)
+			} else {
+				ans = lines_for_diff_chunk(&data, hunk_num, chunk, cnum, ans)
+			}
+		}
+	}
+	return ans, nil
+}
+
+func all_lines(path string, columns, margin_size int, is_add bool, ans []*LogicalLine) ([]*LogicalLine, error) {
+	available_cols := columns/2 - margin_size
+	ltype := `add`
+	ll := LogicalLine{line_type: CHANGE_LINE}
+	if !is_add {
+		ltype = `remove`
+		ll.left_reference.path = path
+	} else {
+		ll.right_reference.path = path
+	}
+	lines, err := highlighted_lines_for_path(path)
+	if err != nil {
+		return nil, err
+	}
+	var msg_lines []string
+	if is_add {
+		msg_lines = splitlines(`This file was added`, available_cols)
+	} else {
+		msg_lines = splitlines(`This file was removed`, available_cols)
+	}
+	for line_number, line := range lines {
+		hlines := make([]HalfScreenLine, 0, 8)
+		hlines = render_half_line(line_number, line, ltype, available_cols, Center{}, hlines)
+		l := ll
+		if is_add {
+			l.right_reference.linenum = line_number + 1
+		} else {
+			l.left_reference.linenum = line_number + 1
+		}
+		l.is_change_start = line_number == 0
+		for i, hl := range hlines {
+			sl := ScreenLine{}
+			if is_add {
+				sl.right = hl
+				if len(msg_lines) > 0 {
+					sl.left.marked_up_text = msg_lines[i]
+					sl.left.is_filler = true
+					msg_lines = msg_lines[1:]
+				} else {
+					sl.left.is_filler = true
+				}
+			} else {
+				sl.left = hl
+				if len(msg_lines) > 0 {
+					sl.right.marked_up_text = msg_lines[i]
+					sl.right.is_filler = true
+					msg_lines = msg_lines[1:]
+				} else {
+					sl.right.is_filler = true
+				}
+			}
+			l.screen_lines = append(l.screen_lines, &sl)
+		}
+		ans = append(ans, &l)
+	}
+	return ans, nil
+}
+
+func rename_lines(path, other_path string, columns, margin_size int, ans []*LogicalLine) ([]*LogicalLine, error) {
+	ll := LogicalLine{
+		left_reference: Reference{path: path}, right_reference: Reference{path: other_path},
+		line_type: CHANGE_LINE, is_change_start: true, is_full_width: true}
+	for _, line := range splitlines(fmt.Sprintf(`The file %s was renamed to %s`, sanitize(path_name_map[path]), sanitize(path_name_map[other_path])), columns-margin_size) {
+		sl := ScreenLine{}
+		sl.right.marked_up_text = line
+		ll.screen_lines = append(ll.screen_lines, &sl)
+	}
+	return append(ans, &ll), nil
+}
+
+func render(collection *Collection, diff_map map[string]*Patch, screen_size screen_size, largest_line_number int, image_size graphics.Size) (result *LogicalLines, err error) {
+	margin_size := utils.Max(3, len(strconv.Itoa(largest_line_number))+1)
+	ans := make([]*LogicalLine, 0, 1024)
+	columns := screen_size.columns
+	err = collection.Apply(func(path, item_type, changed_path string) error {
+		ans = title_lines(path, changed_path, columns, margin_size, ans)
+		defer func() {
+			ans = append(ans, &LogicalLine{line_type: EMPTY_LINE, screen_lines: []*ScreenLine{{}}})
+		}()
+
+		is_binary := !is_path_text(path)
+		if !is_binary && item_type == `diff` && !is_path_text(changed_path) {
+			is_binary = true
+		}
+		is_img := is_binary && is_image(path) || (item_type == `diff` && is_image(changed_path))
+		_ = is_img
+		switch item_type {
+		case "diff":
+			if is_binary {
+				if is_img {
+					ans, err = image_lines(path, changed_path, screen_size, margin_size, image_size, ans)
+				} else {
+					ans, err = binary_lines(path, changed_path, columns, margin_size, ans)
+				}
+			} else {
+				ans, err = lines_for_diff(path, changed_path, diff_map[path], columns, margin_size, ans)
+			}
+			if err != nil {
+				return err
+			}
+		case "add":
+			if is_binary {
+				if is_img {
+					ans, err = image_lines("", path, screen_size, margin_size, image_size, ans)
+				} else {
+					ans, err = binary_lines("", path, columns, margin_size, ans)
+				}
+			} else {
+				ans, err = all_lines(path, columns, margin_size, true, ans)
+			}
+			if err != nil {
+				return err
+			}
+		case "removal":
+			if is_binary {
+				if is_img {
+					ans, err = image_lines(path, "", screen_size, margin_size, image_size, ans)
+				} else {
+					ans, err = binary_lines(path, "", columns, margin_size, ans)
+				}
+			} else {
+				ans, err = all_lines(path, columns, margin_size, false, ans)
+			}
+			if err != nil {
+				return err
+			}
+		case "rename":
+			ans, err = rename_lines(path, changed_path, columns, margin_size, ans)
+			if err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("Unknown change type: %#v", item_type)
+		}
+		return nil
+	})
+	return &LogicalLines{lines: ans[:len(ans)-1], margin_size: margin_size, columns: columns}, err
+}
+
+func (self *LogicalLines) num_of_screen_lines() (ans int) {
+	for _, l := range self.lines {
+		ans += len(l.screen_lines)
+	}
+	return
+}

kittens/diff/render.py

@@ -1,546 +0,0 @@
-#!/usr/bin/env python3
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-import warnings
-from gettext import gettext as _
-from itertools import repeat, zip_longest
-from math import ceil
-from typing import Callable, Dict, Generator, Iterable, List, Optional, Tuple
-
-from kitty.cli_stub import DiffCLIOptions
-from kitty.fast_data_types import truncate_point_for_length, wcswidth
-from kitty.types import run_once
-from kitty.utils import ScreenSize
-
-from ..tui.images import ImageManager, can_display_images
-from .collect import Collection, Segment, data_for_path, highlights_for_path, is_image, lines_for_path, path_name_map, sanitize
-from .config import formats
-from .diff_speedup import split_with_highlights as _split_with_highlights
-from .patch import Chunk, Hunk, Patch
-
-
-class ImageSupportWarning(Warning):
-    pass
-
-
-@run_once
-def images_supported() -> bool:
-    ans = can_display_images()
-    if not ans:
-        warnings.warn('ImageMagick not found images cannot be displayed', ImageSupportWarning)
-    return ans
-
-
-class Ref:
-
-    __slots__: Tuple[str, ...] = ()
-
-    def __setattr__(self, name: str, value: object) -> None:
-        raise AttributeError("can't set attribute")
-
-    def __repr__(self) -> str:
-        return '{}({})'.format(self.__class__.__name__, ', '.join(
-            f'{n}={getattr(self, n)}' for n in self.__slots__ if n != '_hash'))
-
-
-class LineRef(Ref):
-
-    __slots__ = ('src_line_number', 'wrapped_line_idx')
-    src_line_number: int
-    wrapped_line_idx: int
-
-    def __init__(self, sln: int, wli: int = 0) -> None:
-        object.__setattr__(self, 'src_line_number', sln)
-        object.__setattr__(self, 'wrapped_line_idx', wli)
-
-
-class Reference(Ref):
-
-    __slots__ = ('path', 'extra')
-    path: str
-    extra: Optional[LineRef]
-
-    def __init__(self, path: str, extra: Optional[LineRef] = None) -> None:
-        object.__setattr__(self, 'path', path)
-        object.__setattr__(self, 'extra', extra)
-
-
-class Line:
-
-    __slots__ = ('text', 'ref', 'is_change_start', 'image_data')
-
-    def __init__(
-        self,
-        text: str,
-        ref: Reference,
-        change_start: bool = False,
-        image_data: Optional[Tuple[Optional['ImagePlacement'], Optional['ImagePlacement']]] = None
-    ) -> None:
-        self.text = text
-        self.ref = ref
-        self.is_change_start = change_start
-        self.image_data = image_data
-
-
-def yield_lines_from(iterator: Iterable[str], reference: Reference, is_change_start: bool = True) -> Generator[Line, None, None]:
-    for text in iterator:
-        yield Line(text, reference, is_change_start)
-        is_change_start = False
-
-
-def human_readable(size: int, sep: str = ' ') -> str:
-    """ Convert a size in bytes into a human readable form """
-    divisor, suffix = 1, "B"
-    for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
-        if size < (1 << ((i + 1) * 10)):
-            divisor, suffix = (1 << (i * 10)), candidate
-            break
-    s = str(float(size)/divisor)
-    if s.find(".") > -1:
-        s = s[:s.find(".")+2]
-    if s.endswith('.0'):
-        s = s[:-2]
-    return f'{s}{sep}{suffix}'
-
-
-def fit_in(text: str, count: int) -> str:
-    p = truncate_point_for_length(text, count)
-    if p >= len(text):
-        return text
-    if count > 1:
-        p = truncate_point_for_length(text, count - 1)
-    return f'{text[:p]}…'
-
-
-def fill_in(text: str, sz: int) -> str:
-    w = wcswidth(text)
-    if w < sz:
-        text += ' ' * (sz - w)
-    return text
-
-
-def place_in(text: str, sz: int) -> str:
-    return fill_in(fit_in(text, sz), sz)
-
-
-def format_func(which: str) -> Callable[[str], str]:
-    def formatted(text: str) -> str:
-        fmt = formats[which]
-        return f'\x1b[{fmt}m{text}\x1b[0m'
-    formatted.__name__ = f'{which}_format'
-    return formatted
-
-
-text_format = format_func('text')
-title_format = format_func('title')
-margin_format = format_func('margin')
-added_format = format_func('added')
-removed_format = format_func('removed')
-removed_margin_format = format_func('removed_margin')
-added_margin_format = format_func('added_margin')
-filler_format = format_func('filler')
-margin_filler_format = format_func('margin_filler')
-hunk_margin_format = format_func('hunk_margin')
-hunk_format = format_func('hunk')
-highlight_map = {'remove': ('removed_highlight', 'removed'), 'add': ('added_highlight', 'added')}
-
-
-def highlight_boundaries(ltype: str) -> Tuple[str, str]:
-    s, e = highlight_map[ltype]
-    start = f'\x1b[{formats[s]}m'
-    stop = f'\x1b[{formats[e]}m'
-    return start, stop
-
-
-def title_lines(left_path: Optional[str], right_path: Optional[str], args: DiffCLIOptions, columns: int, margin_size: int) -> Generator[str, None, None]:
-    m = ' ' * margin_size
-    left_name = path_name_map.get(left_path) if left_path else None
-    right_name = path_name_map.get(right_path) if right_path else None
-    if right_name and right_name != left_name:
-        n1 = fit_in(m + sanitize(left_name or ''), columns // 2 - margin_size)
-        n1 = place_in(n1, columns // 2)
-        n2 = fit_in(m + sanitize(right_name), columns // 2 - margin_size)
-        n2 = place_in(n2, columns // 2)
-        name = n1 + n2
-    else:
-        name = place_in(m + sanitize(left_name or ''), columns)
-    yield title_format(place_in(name, columns))
-    yield title_format('━' * columns)
-
-
-def binary_lines(path: Optional[str], other_path: Optional[str], columns: int, margin_size: int) -> Generator[str, None, None]:
-    template = _('Binary file: {}')
-    available_cols = columns // 2 - margin_size
-
-    def fl(path: str, fmt: Callable[[str], str]) -> str:
-        text = template.format(human_readable(len(data_for_path(path))))
-        text = place_in(text, available_cols)
-        return margin_format(' ' * margin_size) + fmt(text)
-
-    if path is None:
-        filler = render_diff_line('', '', 'filler', margin_size, available_cols)
-        assert other_path is not None
-        yield filler + fl(other_path, added_format)
-    elif other_path is None:
-        filler = render_diff_line('', '', 'filler', margin_size, available_cols)
-        yield fl(path, removed_format) + filler
-    else:
-        yield fl(path, removed_format) + fl(other_path, added_format)
-
-
-def split_to_size(line: str, width: int) -> Generator[str, None, None]:
-    if not line:
-        yield line
-    while line:
-        p = truncate_point_for_length(line, width)
-        yield line[:p]
-        line = line[p:]
-
-
-def truncate_points(line: str, width: int) -> Generator[int, None, None]:
-    pos = 0
-    sz = len(line)
-    while True:
-        pos = truncate_point_for_length(line, width, pos)
-        if pos < sz:
-            yield pos
-        else:
-            break
-
-
-def split_with_highlights(line: str, width: int, highlights: List[Segment], bg_highlight: Optional[Segment] = None) -> List[str]:
-    truncate_pts = list(truncate_points(line, width))
-    return _split_with_highlights(line, truncate_pts, highlights, bg_highlight)
-
-
-margin_bg_map = {'filler': margin_filler_format, 'remove': removed_margin_format, 'add': added_margin_format, 'context': margin_format}
-text_bg_map = {'filler': filler_format, 'remove': removed_format, 'add': added_format, 'context': text_format}
-
-
-class DiffData:
-
-    def __init__(self, left_path: str, right_path: str, available_cols: int, margin_size: int):
-        self.left_path, self.right_path = left_path, right_path
-        self.available_cols = available_cols
-        self.margin_size = margin_size
-        self.left_lines, self.right_lines = map(lines_for_path, (left_path, right_path))
-        self.filler_line = render_diff_line('', '', 'filler', margin_size, available_cols)
-        self.left_filler_line = render_diff_line('', '', 'remove', margin_size, available_cols)
-        self.right_filler_line = render_diff_line('', '', 'add', margin_size, available_cols)
-        self.left_hdata = highlights_for_path(left_path)
-        self.right_hdata = highlights_for_path(right_path)
-
-    def left_highlights_for_line(self, line_num: int) -> List[Segment]:
-        if line_num < len(self.left_hdata):
-            return self.left_hdata[line_num]
-        return []
-
-    def right_highlights_for_line(self, line_num: int) -> List[Segment]:
-        if line_num < len(self.right_hdata):
-            return self.right_hdata[line_num]
-        return []
-
-
-def render_diff_line(number: Optional[str], text: str, ltype: str, margin_size: int, available_cols: int) -> str:
-    margin = margin_bg_map[ltype](place_in(number or '', margin_size))
-    content = text_bg_map[ltype](fill_in(text or '', available_cols))
-    return margin + content
-
-
-def render_diff_pair(
-    left_line_number: Optional[str], left: str, left_is_change: bool,
-    right_line_number: Optional[str], right: str, right_is_change: bool,
-    is_first: bool, margin_size: int, available_cols: int
-) -> str:
-    ltype = 'filler' if left_line_number is None else ('remove' if left_is_change else 'context')
-    rtype = 'filler' if right_line_number is None else ('add' if right_is_change else 'context')
-    return (
-            render_diff_line(left_line_number if is_first else None, left, ltype, margin_size, available_cols) +
-            render_diff_line(right_line_number if is_first else None, right, rtype, margin_size, available_cols)
-    )
-
-
-def hunk_title(hunk_num: int, hunk: Hunk, margin_size: int, available_cols: int) -> str:
-    m = hunk_margin_format(' ' * margin_size)
-    t = f'@@ -{hunk.left_start + 1},{hunk.left_count} +{hunk.right_start + 1},{hunk.right_count} @@ {hunk.title}'
-    return m + hunk_format(place_in(t, available_cols))
-
-
-def render_half_line(
-    line_number: int,
-    line: str,
-    highlights: List[Segment],
-    ltype: str,
-    margin_size: int,
-    available_cols: int,
-    changed_center: Optional[Tuple[int, int]] = None
-) -> Generator[str, None, None]:
-    bg_highlight: Optional[Segment] = None
-    if changed_center is not None and changed_center[0]:
-        prefix_count, suffix_count = changed_center
-        line_sz = len(line)
-        if prefix_count + suffix_count < line_sz:
-            start, stop = highlight_boundaries(ltype)
-            seg = Segment(prefix_count, start)
-            seg.end = line_sz - suffix_count
-            seg.end_code = stop
-            bg_highlight = seg
-    if highlights or bg_highlight:
-        lines: Iterable[str] = split_with_highlights(line, available_cols, highlights, bg_highlight)
-    else:
-        lines = split_to_size(line, available_cols)
-    lnum = str(line_number + 1)
-    for line in lines:
-        yield render_diff_line(lnum, line, ltype, margin_size, available_cols)
-        lnum = ''
-
-
-def lines_for_chunk(data: DiffData, hunk_num: int, chunk: Chunk, chunk_num: int) -> Generator[Line, None, None]:
-    if chunk.is_context:
-        for i in range(chunk.left_count):
-            left_line_number = line_ref = chunk.left_start + i
-            right_line_number = chunk.right_start + i
-            highlights = data.left_highlights_for_line(left_line_number)
-            if highlights:
-                lines: Iterable[str] = split_with_highlights(data.left_lines[left_line_number], data.available_cols, highlights)
-            else:
-                lines = split_to_size(data.left_lines[left_line_number], data.available_cols)
-            left_line_number_s = str(left_line_number + 1)
-            right_line_number_s = str(right_line_number + 1)
-            for wli, text in enumerate(lines):
-                line = render_diff_line(left_line_number_s, text, 'context', data.margin_size, data.available_cols)
-                if right_line_number_s == left_line_number_s:
-                    r = line
-                else:
-                    r = render_diff_line(right_line_number_s, text, 'context', data.margin_size, data.available_cols)
-                ref = Reference(data.left_path, LineRef(line_ref, wli))
-                yield Line(line + r, ref)
-                left_line_number_s = right_line_number_s = ''
-    else:
-        common = min(chunk.left_count, chunk.right_count)
-        for i in range(max(chunk.left_count, chunk.right_count)):
-            ll: List[str] = []
-            rl: List[str] = []
-            if i < chunk.left_count:
-                rln = ref_ln = chunk.left_start + i
-                ll.extend(render_half_line(
-                    rln, data.left_lines[rln], data.left_highlights_for_line(rln),
-                    'remove', data.margin_size, data.available_cols,
-                    None if chunk.centers is None else chunk.centers[i]))
-                ref_path = data.left_path
-            if i < chunk.right_count:
-                rln = ref_ln = chunk.right_start + i
-                rl.extend(render_half_line(
-                    rln, data.right_lines[rln], data.right_highlights_for_line(rln),
-                    'add', data.margin_size, data.available_cols,
-                    None if chunk.centers is None else chunk.centers[i]))
-                ref_path = data.right_path
-            if i < common:
-                extra = len(ll) - len(rl)
-                if extra != 0:
-                    if extra < 0:
-                        x, fl = ll, data.left_filler_line
-                        extra = -extra
-                    else:
-                        x, fl = rl, data.right_filler_line
-                    x.extend(repeat(fl, extra))
-            else:
-                if ll:
-                    x, count = rl, len(ll)
-                else:
-                    x, count = ll, len(rl)
-                x.extend(repeat(data.filler_line, count))
-            for wli, (left_line, right_line) in enumerate(zip(ll, rl)):
-                ref = Reference(ref_path, LineRef(ref_ln, wli))
-                yield Line(left_line + right_line, ref, i == 0 and wli == 0)
-
-
-def lines_for_diff(left_path: str, right_path: str, hunks: Iterable[Hunk], args: DiffCLIOptions, columns: int, margin_size: int) -> Generator[Line, None, None]:
-    available_cols = columns // 2 - margin_size
-    data = DiffData(left_path, right_path, available_cols, margin_size)
-
-    for hunk_num, hunk in enumerate(hunks):
-        yield Line(hunk_title(hunk_num, hunk, margin_size, columns - margin_size), Reference(left_path, LineRef(hunk.left_start)))
-        for cnum, chunk in enumerate(hunk.chunks):
-            yield from lines_for_chunk(data, hunk_num, chunk, cnum)
-
-
-def all_lines(path: str, args: DiffCLIOptions, columns: int, margin_size: int, is_add: bool = True) -> Generator[Line, None, None]:
-    available_cols = columns // 2 - margin_size
-    ltype = 'add' if is_add else 'remove'
-    lines = lines_for_path(path)
-    filler = render_diff_line('', '', 'filler', margin_size, available_cols)
-    msg_written = False
-    hdata = highlights_for_path(path)
-
-    def highlights(num: int) -> List[Segment]:
-        return hdata[num] if num < len(hdata) else []
-
-    for line_number, line in enumerate(lines):
-        h = render_half_line(line_number, line, highlights(line_number), ltype, margin_size, available_cols)
-        for i, hl in enumerate(h):
-            ref = Reference(path, LineRef(line_number, i))
-            empty = filler
-            if not msg_written:
-                msg_written = True
-                empty = render_diff_line(
-                        '', _('This file was added') if is_add else _('This file was removed'),
-                        'filler', margin_size, available_cols)
-            text = (empty + hl) if is_add else (hl + empty)
-            yield Line(text, ref, line_number == 0 and i == 0)
-
-
-def rename_lines(path: str, other_path: str, args: DiffCLIOptions, columns: int, margin_size: int) -> Generator[str, None, None]:
-    m = ' ' * margin_size
-    for line in split_to_size(_('The file {0} was renamed to {1}').format(
-            sanitize(path_name_map[path]), sanitize(path_name_map[other_path])), columns - margin_size):
-        yield m + line
-
-
-class Image:
-
-    def __init__(self, image_id: int, width: int, height: int, margin_size: int, screen_size: ScreenSize) -> None:
-        self.image_id = image_id
-        self.width, self.height = width, height
-        self.rows = int(ceil(self.height / screen_size.cell_height))
-        self.columns = int(ceil(self.width / screen_size.cell_width))
-        self.margin_size = margin_size
-
-
-class ImagePlacement:
-
-    def __init__(self, image: Image, row: int) -> None:
-        self.image = image
-        self.row = row
-
-
-def render_image(
-    path: str,
-    is_left: bool,
-    available_cols: int, margin_size: int,
-    image_manager: ImageManager
-) -> Generator[Tuple[str, Reference, Optional[ImagePlacement]], None, None]:
-    lnum = 0
-    margin_fmt = removed_margin_format if is_left else added_margin_format
-    m = margin_fmt(' ' * margin_size)
-    fmt = removed_format if is_left else added_format
-
-    def yield_split(text: str) -> Generator[Tuple[str, Reference, Optional[ImagePlacement]], None, None]:
-        nonlocal lnum
-        for i, line in enumerate(split_to_size(text, available_cols)):
-            yield m + fmt(place_in(line, available_cols)), Reference(path, LineRef(lnum, i)), None
-        lnum += 1
-
-    try:
-        image_id, width, height = image_manager.send_image(path, available_cols - margin_size, image_manager.screen_size.rows - 2)
-    except Exception as e:
-        yield from yield_split(_('Failed to render image, with error:'))
-        yield from yield_split(' '.join(str(e).splitlines()))
-        return
-    meta = _('Dimensions: {0}x{1} pixels Size: {2}').format(
-            width, height, human_readable(len(data_for_path(path))))
-    yield from yield_split(meta)
-    bg_line = m + fmt(' ' * available_cols)
-    img = Image(image_id, width, height, margin_size, image_manager.screen_size)
-    for r in range(img.rows):
-        yield bg_line, Reference(path, LineRef(lnum)), ImagePlacement(img, r)
-        lnum += 1
-
-
-def image_lines(
-    left_path: Optional[str],
-    right_path: Optional[str],
-    columns: int,
-    margin_size: int,
-    image_manager: ImageManager
-) -> Generator[Line, None, None]:
-    available_cols = columns // 2 - margin_size
-    left_lines: Iterable[Tuple[str, Reference, Optional[ImagePlacement]]] = iter(())
-    right_lines: Iterable[Tuple[str, Reference, Optional[ImagePlacement]]] = iter(())
-    if left_path is not None:
-        left_lines = render_image(left_path, True, available_cols, margin_size, image_manager)
-    if right_path is not None:
-        right_lines = render_image(right_path, False, available_cols, margin_size, image_manager)
-    filler = ' ' * (available_cols + margin_size)
-    is_change_start = True
-    for left, right in zip_longest(left_lines, right_lines):
-        left_placement = right_placement = None
-        if left is None:
-            left = filler
-            right, ref, right_placement = right
-        elif right is None:
-            right = filler
-            left, ref, left_placement = left
-        else:
-            right, ref, right_placement = right
-            left, ref, left_placement = left
-        image_data = (left_placement, right_placement) if left_placement or right_placement else None
-        yield Line(left + right, ref, is_change_start, image_data)
-        is_change_start = False
-
-
-class RenderDiff:
-
-    margin_size: int = 0
-
-    def __call__(
-        self,
-        collection: Collection,
-        diff_map: Dict[str, Patch],
-        args: DiffCLIOptions,
-        columns: int,
-        image_manager: ImageManager
-    ) -> Generator[Line, None, None]:
-        largest_line_number = 0
-        for path, item_type, other_path in collection:
-            if item_type == 'diff':
-                patch = diff_map.get(path)
-                if patch is not None:
-                    largest_line_number = max(largest_line_number, patch.largest_line_number)
-
-        margin_size = self.margin_size = max(3, len(str(largest_line_number)) + 1)
-        last_item_num = len(collection) - 1
-
-        for i, (path, item_type, other_path) in enumerate(collection):
-            item_ref = Reference(path)
-            is_binary = isinstance(data_for_path(path), bytes)
-            if not is_binary and item_type == 'diff' and isinstance(data_for_path(other_path), bytes):
-                is_binary = True
-            is_img = is_binary and (is_image(path) or is_image(other_path)) and images_supported()
-            yield from yield_lines_from(title_lines(path, other_path, args, columns, margin_size), item_ref, False)
-            if item_type == 'diff':
-                if is_binary:
-                    if is_img:
-                        ans = image_lines(path, other_path, columns, margin_size, image_manager)
-                    else:
-                        ans = yield_lines_from(binary_lines(path, other_path, columns, margin_size), item_ref)
-                else:
-                    assert other_path is not None
-                    ans = lines_for_diff(path, other_path, diff_map[path], args, columns, margin_size)
-            elif item_type == 'add':
-                if is_binary:
-                    if is_img:
-                        ans = image_lines(None, path, columns, margin_size, image_manager)
-                    else:
-                        ans = yield_lines_from(binary_lines(None, path, columns, margin_size), item_ref)
-                else:
-                    ans = all_lines(path, args, columns, margin_size, is_add=True)
-            elif item_type == 'removal':
-                if is_binary:
-                    if is_img:
-                        ans = image_lines(path, None, columns, margin_size, image_manager)
-                    else:
-                        ans = yield_lines_from(binary_lines(path, None, columns, margin_size), item_ref)
-                else:
-                    ans = all_lines(path, args, columns, margin_size, is_add=False)
-            elif item_type == 'rename':
-                assert other_path is not None
-                ans = yield_lines_from(rename_lines(path, other_path, args, columns, margin_size), item_ref)
-            else:
-                raise ValueError(f'Unsupported item type: {item_type}')
-            yield from ans
-            if i < last_item_num:
-                yield Line('', item_ref)
-
-
-render_diff = RenderDiff()

kittens/diff/search.go

@@ -0,0 +1,149 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+
+	"kitty/tools/tui"
+	"kitty/tools/utils"
+	"kitty/tools/utils/images"
+	"kitty/tools/wcswidth"
+
+	"golang.org/x/exp/slices"
+)
+
+var _ = fmt.Print
+
+type Search struct {
+	pat     *regexp.Regexp
+	matches map[ScrollPos][]Span
+}
+
+func (self *Search) Len() int { return len(self.matches) }
+
+func (self *Search) find_matches_in_lines(clean_lines []string, origin int, send_result func(screen_line, offset, size int)) {
+	lengths := utils.Map(func(x string) int { return len(x) }, clean_lines)
+	offsets := make([]int, len(clean_lines))
+	cell_lengths := utils.Map(wcswidth.Stringwidth, clean_lines)
+	cell_offsets := make([]int, len(clean_lines))
+	for i := range clean_lines {
+		if i > 0 {
+			offsets[i] = offsets[i-1] + lengths[i-1]
+			cell_offsets[i] = cell_offsets[i-1] + cell_lengths[i-1]
+		}
+	}
+	joined_text := strings.Join(clean_lines, "")
+	matches := self.pat.FindAllStringIndex(joined_text, -1)
+	pos := 0
+
+	find_pos := func(start int) int {
+		for i := pos; i < len(clean_lines); i++ {
+			if start < offsets[i]+lengths[i] {
+				pos = i
+				return pos
+			}
+
+		}
+		return -1
+	}
+	for _, m := range matches {
+		start, end := m[0], m[1]
+		total_size := end - start
+		if total_size < 1 {
+			continue
+		}
+		start_line := find_pos(start)
+		if start_line > -1 {
+			end_line := find_pos(end)
+			if end_line > -1 {
+				for i := start_line; i <= end_line; i++ {
+					cell_start := 0
+					if i == start_line {
+						byte_offset := start - offsets[i]
+						cell_start = wcswidth.Stringwidth(clean_lines[i][:byte_offset])
+					}
+					cell_end := cell_lengths[i]
+					if i == end_line {
+						byte_offset := end - offsets[i]
+						cell_end = wcswidth.Stringwidth(clean_lines[i][:byte_offset])
+					}
+					send_result(i, origin+cell_start, cell_end-cell_start)
+				}
+			}
+		}
+	}
+
+}
+
+func (self *Search) find_matches_in_line(line *LogicalLine, margin_size, cols int, send_result func(screen_line, offset, size int)) {
+	half_width := cols / 2
+	right_offset := half_width + margin_size
+	left_clean_lines, right_clean_lines := make([]string, len(line.screen_lines)), make([]string, len(line.screen_lines))
+	for i, sl := range line.screen_lines {
+		if line.is_full_width {
+			left_clean_lines[i] = wcswidth.StripEscapeCodes(sl.left.marked_up_text)
+		} else {
+			left_clean_lines[i] = wcswidth.StripEscapeCodes(sl.left.marked_up_text)
+			right_clean_lines[i] = wcswidth.StripEscapeCodes(sl.right.marked_up_text)
+		}
+	}
+	self.find_matches_in_lines(left_clean_lines, margin_size, send_result)
+	self.find_matches_in_lines(right_clean_lines, right_offset, send_result)
+}
+
+func (self *Search) Has(pos ScrollPos) bool {
+	return len(self.matches[pos]) > 0
+}
+
+type Span struct{ start, end int }
+
+func (self *Search) search(logical_lines *LogicalLines) {
+	margin_size := logical_lines.margin_size
+	cols := logical_lines.columns
+	self.matches = make(map[ScrollPos][]Span)
+	ctx := images.Context{}
+	mutex := sync.Mutex{}
+	ctx.Parallel(0, logical_lines.Len(), func(nums <-chan int) {
+		for i := range nums {
+			line := logical_lines.At(i)
+			if line.line_type == EMPTY_LINE || line.line_type == IMAGE_LINE {
+				continue
+			}
+			self.find_matches_in_line(line, margin_size, cols, func(screen_line, offset, size int) {
+				if size > 0 {
+					mutex.Lock()
+					defer mutex.Unlock()
+					pos := ScrollPos{i, screen_line}
+					self.matches[pos] = append(self.matches[pos], Span{offset, offset + size - 1})
+				}
+			})
+		}
+	})
+	for _, spans := range self.matches {
+		slices.SortFunc(spans, func(a, b Span) bool { return a.start < b.start })
+	}
+}
+
+func (self *Search) markup_line(pos ScrollPos, y int) string {
+	spans := self.matches[pos]
+	if spans == nil {
+		return ""
+	}
+	sgr := format_as_sgr.search[2:]
+	sgr = sgr[:len(sgr)-1]
+	ans := make([]byte, 0, 32)
+	for _, span := range spans {
+		ans = append(ans, tui.FormatPartOfLine(sgr, span.start, span.end, y)...)
+	}
+	return utils.UnsafeBytesToString(ans)
+}
+
+func do_search(pat *regexp.Regexp, logical_lines *LogicalLines) *Search {
+	ans := &Search{pat: pat, matches: make(map[ScrollPos][]Span)}
+	ans.search(logical_lines)
+	return ans
+}

kittens/diff/search.py

@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
-
-import re
-from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Tuple
-
-from kitty.fast_data_types import wcswidth
-
-from ..tui.operations import styled
-from .options.types import Options as DiffOptions
-
-if TYPE_CHECKING:
-    from .render import Line
-    Line
-
-
-class BadRegex(ValueError):
-    pass
-
-
-class Search:
-
-    def __init__(self, opts: DiffOptions, query: str, is_regex: bool, is_backward: bool):
-        self.matches: Dict[int, List[Tuple[int, str]]] = {}
-        self.count = 0
-        self.style = styled('|', fg=opts.search_fg, bg=opts.search_bg).split('|', 1)[0]
-        if not is_regex:
-            query = re.escape(query)
-        try:
-            self.pat = re.compile(query, flags=re.UNICODE | re.IGNORECASE)
-        except Exception:
-            raise BadRegex(f'Not a valid regex: {query}')
-
-    def __call__(self, diff_lines: Iterable['Line'], margin_size: int, cols: int) -> bool:
-        self.matches = {}
-        self.count = 0
-        half_width = cols // 2
-        strip_pat = re.compile('\033[[].*?m')
-        right_offset = half_width + 1 + margin_size
-        find = self.pat.finditer
-        for i, line in enumerate(diff_lines):
-            text = strip_pat.sub('', line.text)
-            left, right = text[margin_size:half_width + 1], text[right_offset:]
-            matches = []
-
-            def add(which: str, offset: int) -> None:
-                for m in find(which):
-                    before = which[:m.start()]
-                    matches.append((wcswidth(before) + offset, m.group()))
-                    self.count += 1
-
-            add(left, margin_size)
-            add(right, right_offset)
-            if matches:
-                self.matches[i] = matches
-        return bool(self.matches)
-
-    def __contains__(self, i: int) -> bool:
-        return i in self.matches
-
-    def __len__(self) -> int:
-        return self.count
-
-    def highlight_line(self, write: Callable[[str], None], line_num: int) -> bool:
-        highlights = self.matches.get(line_num)
-        if not highlights:
-            return False
-        write(self.style)
-        for start, text in highlights:
-            write(f'\r\x1b[{start}C{text}')
-        write('\x1b[m')
-        return True

kittens/diff/speedup.c

@@ -1,205 +0,0 @@
-/*
- * speedup.c
- * Copyright (C) 2018 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "data-types.h"
-
-static PyObject*
-changed_center(PyObject *self UNUSED, PyObject *args) {
-    unsigned int prefix_count = 0, suffix_count = 0;
-    PyObject *lp, *rp;
-    if (!PyArg_ParseTuple(args, "UU", &lp, &rp)) return NULL;
-    const size_t left_len = PyUnicode_GET_LENGTH(lp), right_len = PyUnicode_GET_LENGTH(rp);
-
-#define R(which, index) PyUnicode_READ(PyUnicode_KIND(which), PyUnicode_DATA(which), index)
-    while(prefix_count < MIN(left_len, right_len)) {
-        if (R(lp, prefix_count) != R(rp, prefix_count)) break;
-        prefix_count++;
-    }
-    if (left_len && right_len && prefix_count < MIN(left_len, right_len)) {
-        while(suffix_count < MIN(left_len - prefix_count, right_len - prefix_count)) {
-            if(R(lp, left_len - 1 - suffix_count) != R(rp, right_len - 1 - suffix_count)) break;
-            suffix_count++;
-        }
-    }
-#undef R
-    return Py_BuildValue("II", prefix_count, suffix_count);
-}
-
-typedef struct {
-    unsigned int start_pos, end_pos, current_pos;
-    PyObject *start_code, *end_code;
-} Segment;
-
-typedef struct {
-    Segment sg;
-    unsigned int num, pos;
-} SegmentPointer;
-
-static const Segment EMPTY_SEGMENT = { .current_pos = UINT_MAX };
-
-static bool
-convert_segment(PyObject *highlight, Segment *dest) {
-    PyObject *val = NULL;
-#define I
-#define A(x, d, c) { \
-    val = PyObject_GetAttrString(highlight, #x); \
-    if (val == NULL) return false; \
-    dest->d = c(val); Py_DECREF(val); \
-}
-    A(start, start_pos, PyLong_AsUnsignedLong);
-    A(end, end_pos, PyLong_AsUnsignedLong);
-    dest->current_pos = dest->start_pos;
-    A(start_code, start_code, I);
-    A(end_code, end_code, I);
-    if (!PyUnicode_Check(dest->start_code)) { PyErr_SetString(PyExc_TypeError, "start_code is not a string"); return false; }
-    if (!PyUnicode_Check(dest->end_code)) { PyErr_SetString(PyExc_TypeError, "end_code is not a string"); return false; }
-#undef A
-#undef I
-    return true;
-}
-
-static bool
-next_segment(SegmentPointer *s, PyObject *highlights) {
-    if (s->pos < s->num) {
-        if (!convert_segment(PyList_GET_ITEM(highlights, s->pos), &s->sg)) return false;
-        s->pos++;
-    } else s->sg.current_pos = UINT_MAX;
-    return true;
-}
-
-typedef struct LineBuffer {
-    Py_UCS4 *buf;
-    size_t pos, capacity;
-} LineBuffer;
-
-
-static bool
-ensure_space(LineBuffer *lb, size_t num) {
-    if (lb->pos + num >= lb->capacity) {
-        size_t new_cap = MAX(lb->capacity * 2, 4096u);
-        new_cap = MAX(lb->pos + num + 1024u, new_cap);
-        lb->buf = realloc(lb->buf, new_cap * sizeof(lb->buf[0]));
-        if (!lb->buf) { PyErr_NoMemory(); return false; }
-        lb->capacity = new_cap;
-    }
-    return true;
-}
-
-static bool
-insert_code(PyObject *code, LineBuffer *lb) {
-    unsigned int csz = PyUnicode_GET_LENGTH(code);
-    if (!ensure_space(lb, csz)) return false;
-    for (unsigned int s = 0; s < csz; s++) lb->buf[lb->pos++] = PyUnicode_READ(PyUnicode_KIND(code), PyUnicode_DATA(code), s);
-    return true;
-}
-
-static bool
-add_line(Segment *bg_segment, Segment *fg_segment, LineBuffer *lb, PyObject *ans) {
-    bool bg_is_active = bg_segment->current_pos == bg_segment->end_pos, fg_is_active = fg_segment->current_pos == fg_segment->end_pos;
-    if (bg_is_active) { if(!insert_code(bg_segment->end_code, lb)) return false; }
-    if (fg_is_active) { if(!insert_code(fg_segment->end_code, lb)) return false; }
-    PyObject *wl = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, lb->buf, lb->pos);
-    if (!wl) return false;
-    int ret = PyList_Append(ans, wl); Py_DECREF(wl); if (ret != 0) return false;
-    lb->pos = 0;
-    if (bg_is_active) { if(!insert_code(bg_segment->start_code, lb)) return false; }
-    if (fg_is_active) { if(!insert_code(fg_segment->start_code, lb)) return false; }
-    return true;
-}
-
-static LineBuffer line_buffer;
-
-static PyObject*
-split_with_highlights(PyObject *self UNUSED, PyObject *args) {
-    PyObject *line, *truncate_points_py, *fg_highlights, *bg_highlight;
-    if (!PyArg_ParseTuple(args, "UO!O!O", &line, &PyList_Type, &truncate_points_py, &PyList_Type, &fg_highlights, &bg_highlight)) return NULL;
-    PyObject *ans = PyList_New(0);
-    if (!ans) return NULL;
-    static unsigned int truncate_points[256];
-    unsigned int num_truncate_pts = PyList_GET_SIZE(truncate_points_py), truncate_pos = 0, truncate_point;
-    for (unsigned int i = 0; i < MIN(num_truncate_pts, arraysz(truncate_points)); i++) {
-        truncate_points[i] = PyLong_AsUnsignedLong(PyList_GET_ITEM(truncate_points_py, i));
-    }
-    SegmentPointer fg_segment = { .sg = EMPTY_SEGMENT, .num = PyList_GET_SIZE(fg_highlights)}, bg_segment = { .sg = EMPTY_SEGMENT };
-    if (bg_highlight != Py_None) { if (!convert_segment(bg_highlight, &bg_segment.sg)) { Py_CLEAR(ans); return NULL; }; bg_segment.num = 1; }
-#define CHECK_CALL(func, ...) if (!func(__VA_ARGS__)) { Py_CLEAR(ans); if (!PyErr_Occurred()) PyErr_SetString(PyExc_ValueError, "unknown error while processing line"); return NULL; }
-    CHECK_CALL(next_segment, &fg_segment, fg_highlights);
-
-#define NEXT_TRUNCATE_POINT truncate_point = (truncate_pos < num_truncate_pts) ? truncate_points[truncate_pos++] : UINT_MAX
-    NEXT_TRUNCATE_POINT;
-
-#define INSERT_CODE(x) { CHECK_CALL(insert_code, x, &line_buffer); }
-
-#define ADD_LINE CHECK_CALL(add_line, &bg_segment.sg, &fg_segment.sg, &line_buffer, ans);
-
-#define ADD_CHAR(x) { \
-    if (!ensure_space(&line_buffer, 1)) { Py_CLEAR(ans); return NULL; } \
-    line_buffer.buf[line_buffer.pos++] = x; \
-}
-#define CHECK_SEGMENT(sgp, is_fg) { \
-    if (i == sgp.sg.current_pos) { \
-        INSERT_CODE(sgp.sg.current_pos == sgp.sg.start_pos ? sgp.sg.start_code : sgp.sg.end_code); \
-        if (sgp.sg.current_pos == sgp.sg.start_pos) sgp.sg.current_pos = sgp.sg.end_pos; \
-        else { \
-            if (is_fg) { \
-                CHECK_CALL(next_segment, &fg_segment, fg_highlights); \
-                if (sgp.sg.current_pos == i) { \
-                    INSERT_CODE(sgp.sg.start_code); \
-                    sgp.sg.current_pos = sgp.sg.end_pos; \
-                } \
-            } else sgp.sg.current_pos = UINT_MAX; \
-        } \
-    }\
-}
-
-    const unsigned int line_sz = PyUnicode_GET_LENGTH(line);
-    line_buffer.pos = 0;
-    unsigned int i = 0;
-    for (; i < line_sz; i++) {
-        if (i == truncate_point) { ADD_LINE; NEXT_TRUNCATE_POINT; }
-        CHECK_SEGMENT(bg_segment, false);
-        CHECK_SEGMENT(fg_segment, true)
-        ADD_CHAR(PyUnicode_READ(PyUnicode_KIND(line), PyUnicode_DATA(line), i));
-    }
-    if (line_buffer.pos) ADD_LINE;
-    return ans;
-#undef INSERT_CODE
-#undef CHECK_SEGMENT
-#undef CHECK_CALL
-#undef ADD_CHAR
-#undef ADD_LINE
-#undef NEXT_TRUNCATE_POINT
-}
-
-static void
-free_resources(void) {
-    free(line_buffer.buf); line_buffer.buf = NULL; line_buffer.capacity = 0; line_buffer.pos = 0;
-}
-
-static PyMethodDef module_methods[] = {
-    {"changed_center", (PyCFunction)changed_center, METH_VARARGS, ""},
-    {"split_with_highlights", (PyCFunction)split_with_highlights, METH_VARARGS, ""},
-    {NULL, NULL, 0, NULL}        /* Sentinel */
-};
-
-static struct PyModuleDef module = {
-    .m_base = PyModuleDef_HEAD_INIT,
-    .m_name = "diff_speedup",   /* name of module */
-    .m_doc = NULL,
-    .m_size = -1,
-    .m_methods = module_methods
-};
-
-EXPORTED PyMODINIT_FUNC
-PyInit_diff_speedup(void) {
-    PyObject *m;
-
-    m = PyModule_Create(&module);
-    if (m == NULL) return NULL;
-    Py_AtExit(free_resources);
-    return m;
-}

kittens/diff/ui.go

@@ -0,0 +1,688 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"kitty/tools/config"
+	"kitty/tools/tui"
+	"kitty/tools/tui/graphics"
+	"kitty/tools/tui/loop"
+	"kitty/tools/tui/readline"
+	"kitty/tools/utils"
+	"kitty/tools/wcswidth"
+)
+
+var _ = fmt.Print
+
+type ResultType int
+
+const (
+	COLLECTION ResultType = iota
+	DIFF
+	HIGHLIGHT
+	IMAGE_LOAD
+	IMAGE_RESIZE
+)
+
+type ScrollPos struct {
+	logical_line, screen_line int
+}
+
+func (self ScrollPos) Less(other ScrollPos) bool {
+	return self.logical_line < other.logical_line || (self.logical_line == other.logical_line && self.screen_line < other.screen_line)
+}
+
+func (self ScrollPos) Add(other ScrollPos) ScrollPos {
+	return ScrollPos{self.logical_line + other.logical_line, self.screen_line + other.screen_line}
+}
+
+type AsyncResult struct {
+	err        error
+	rtype      ResultType
+	collection *Collection
+	diff_map   map[string]*Patch
+	page_size  graphics.Size
+}
+
+var image_collection *graphics.ImageCollection
+
+type screen_size struct{ rows, columns, num_lines, cell_width, cell_height int }
+type Handler struct {
+	async_results                                       chan AsyncResult
+	mouse_selection                                     tui.MouseSelection
+	image_count                                         int
+	shortcut_tracker                                    config.ShortcutTracker
+	left, right                                         string
+	collection                                          *Collection
+	diff_map                                            map[string]*Patch
+	logical_lines                                       *LogicalLines
+	lp                                                  *loop.Loop
+	current_context_count, original_context_count       int
+	added_count, removed_count                          int
+	screen_size                                         screen_size
+	scroll_pos, max_scroll_pos                          ScrollPos
+	restore_position                                    *ScrollPos
+	inputting_command                                   bool
+	statusline_message                                  string
+	rl                                                  *readline.Readline
+	current_search                                      *Search
+	current_search_is_regex, current_search_is_backward bool
+	largest_line_number                                 int
+	images_resized_to                                   graphics.Size
+}
+
+func (self *Handler) calculate_statistics() {
+	self.added_count, self.removed_count = self.collection.added_count, self.collection.removed_count
+	self.largest_line_number = 0
+	for _, patch := range self.diff_map {
+		self.added_count += patch.added_count
+		self.removed_count += patch.removed_count
+		self.largest_line_number = utils.Max(patch.largest_line_number, self.largest_line_number)
+	}
+}
+
+func (self *Handler) update_screen_size(sz loop.ScreenSize) {
+	self.screen_size.rows = int(sz.HeightCells)
+	self.screen_size.columns = int(sz.WidthCells)
+	self.screen_size.num_lines = self.screen_size.rows - 1
+	self.screen_size.cell_height = int(sz.CellHeight)
+	self.screen_size.cell_width = int(sz.CellWidth)
+}
+
+func (self *Handler) on_escape_code(etype loop.EscapeCodeType, payload []byte) error {
+	switch etype {
+	case loop.APC:
+		gc := graphics.GraphicsCommandFromAPC(payload)
+		if gc != nil {
+			if !image_collection.HandleGraphicsCommand(gc) {
+				self.draw_screen()
+			}
+		}
+	}
+	return nil
+}
+
+func (self *Handler) finalize() {
+	image_collection.Finalize(self.lp)
+}
+
+func (self *Handler) initialize() {
+	self.rl = readline.New(self.lp, readline.RlInit{DontMarkPrompts: true, Prompt: "/"})
+	self.lp.OnEscapeCode = self.on_escape_code
+	image_collection = graphics.NewImageCollection()
+	self.current_context_count = opts.Context
+	if self.current_context_count < 0 {
+		self.current_context_count = int(conf.Num_context_lines)
+	}
+	sz, _ := self.lp.ScreenSize()
+	self.update_screen_size(sz)
+	self.original_context_count = self.current_context_count
+	self.lp.SetDefaultColor(loop.FOREGROUND, conf.Foreground)
+	self.lp.SetDefaultColor(loop.CURSOR, conf.Foreground)
+	self.lp.SetDefaultColor(loop.BACKGROUND, conf.Background)
+	self.lp.SetDefaultColor(loop.SELECTION_BG, conf.Select_bg)
+	if conf.Select_fg.IsSet {
+		self.lp.SetDefaultColor(loop.SELECTION_FG, conf.Select_fg.Color)
+	}
+	self.async_results = make(chan AsyncResult, 32)
+	go func() {
+		r := AsyncResult{}
+		r.collection, r.err = create_collection(self.left, self.right)
+		self.async_results <- r
+		self.lp.WakeupMainThread()
+	}()
+	self.draw_screen()
+}
+
+func (self *Handler) generate_diff() {
+	self.diff_map = nil
+	jobs := make([]diff_job, 0, 32)
+	self.collection.Apply(func(path, typ, changed_path string) error {
+		if typ == "diff" {
+			if is_path_text(path) && is_path_text(changed_path) {
+				jobs = append(jobs, diff_job{path, changed_path})
+			}
+		}
+		return nil
+	})
+	go func() {
+		r := AsyncResult{rtype: DIFF}
+		r.diff_map, r.err = diff(jobs, self.current_context_count)
+		self.async_results <- r
+		self.lp.WakeupMainThread()
+	}()
+}
+
+func (self *Handler) on_wakeup() error {
+	var r AsyncResult
+	for {
+		select {
+		case r = <-self.async_results:
+			if r.err != nil {
+				return r.err
+			}
+			r.err = self.handle_async_result(r)
+			if r.err != nil {
+				return r.err
+			}
+		default:
+			return nil
+		}
+	}
+}
+
+func (self *Handler) highlight_all() {
+	text_files := utils.Filter(self.collection.paths_to_highlight.AsSlice(), is_path_text)
+	go func() {
+		r := AsyncResult{rtype: HIGHLIGHT}
+		highlight_all(text_files)
+		self.async_results <- r
+		self.lp.WakeupMainThread()
+	}()
+
+}
+
+func (self *Handler) load_all_images() {
+	self.collection.Apply(func(path, item_type, changed_path string) error {
+		if path != "" && is_image(path) {
+			image_collection.AddPaths(path)
+			self.image_count++
+		}
+		if changed_path != "" && is_image(changed_path) {
+			image_collection.AddPaths(changed_path)
+			self.image_count++
+		}
+		return nil
+	})
+	if self.image_count > 0 {
+		image_collection.Initialize(self.lp)
+		go func() {
+			r := AsyncResult{rtype: IMAGE_LOAD}
+			image_collection.LoadAll()
+			self.async_results <- r
+			self.lp.WakeupMainThread()
+		}()
+	}
+}
+
+func (self *Handler) resize_all_images_if_needed() {
+	if self.logical_lines == nil {
+		return
+	}
+	margin_size := self.logical_lines.margin_size
+	columns := self.logical_lines.columns
+	available_cols := columns/2 - margin_size
+	sz := graphics.Size{
+		Width:  available_cols * self.screen_size.cell_width,
+		Height: self.screen_size.num_lines * 2 * self.screen_size.cell_height,
+	}
+	if sz != self.images_resized_to && self.image_count > 0 {
+		go func() {
+			image_collection.ResizeForPageSize(sz.Width, sz.Height)
+			r := AsyncResult{rtype: IMAGE_RESIZE, page_size: sz}
+			self.async_results <- r
+			self.lp.WakeupMainThread()
+		}()
+	}
+}
+
+func (self *Handler) rerender_diff() error {
+	if self.diff_map != nil && self.collection != nil {
+		err := self.render_diff()
+		if err != nil {
+			return err
+		}
+		self.draw_screen()
+	}
+	return nil
+}
+
+func (self *Handler) handle_async_result(r AsyncResult) error {
+	switch r.rtype {
+	case COLLECTION:
+		self.collection = r.collection
+		self.generate_diff()
+		self.highlight_all()
+		self.load_all_images()
+	case DIFF:
+		self.diff_map = r.diff_map
+		self.calculate_statistics()
+		self.clear_mouse_selection()
+		err := self.render_diff()
+		if err != nil {
+			return err
+		}
+		self.scroll_pos = ScrollPos{}
+		if self.restore_position != nil {
+			self.scroll_pos = *self.restore_position
+			if self.max_scroll_pos.Less(self.scroll_pos) {
+				self.scroll_pos = self.max_scroll_pos
+			}
+			self.restore_position = nil
+		}
+		self.draw_screen()
+	case IMAGE_RESIZE:
+		self.images_resized_to = r.page_size
+		return self.rerender_diff()
+	case IMAGE_LOAD, HIGHLIGHT:
+		return self.rerender_diff()
+	}
+	return nil
+}
+
+func (self *Handler) on_resize(old_size, new_size loop.ScreenSize) error {
+	self.clear_mouse_selection()
+	self.update_screen_size(new_size)
+	if self.diff_map != nil && self.collection != nil {
+		err := self.render_diff()
+		if err != nil {
+			return err
+		}
+		if self.max_scroll_pos.Less(self.scroll_pos) {
+			self.scroll_pos = self.max_scroll_pos
+		}
+	}
+	self.draw_screen()
+	return nil
+}
+
+func (self *Handler) render_diff() (err error) {
+	if self.screen_size.columns < 8 {
+		return fmt.Errorf("Screen too narrow, need at least 8 columns")
+	}
+	if self.screen_size.rows < 2 {
+		return fmt.Errorf("Screen too short, need at least 2 rows")
+	}
+	self.logical_lines, err = render(self.collection, self.diff_map, self.screen_size, self.largest_line_number, self.images_resized_to)
+	if err != nil {
+		return err
+	}
+	last := self.logical_lines.Len() - 1
+	self.max_scroll_pos.logical_line = last
+	if last > -1 {
+		self.max_scroll_pos.screen_line = len(self.logical_lines.At(last).screen_lines) - 1
+	} else {
+		self.max_scroll_pos.screen_line = 0
+	}
+	self.logical_lines.IncrementScrollPosBy(&self.max_scroll_pos, -self.screen_size.num_lines+1)
+	if self.current_search != nil {
+		self.current_search.search(self.logical_lines)
+	}
+	return nil
+}
+
+func (self *Handler) draw_image(key string, num_rows, starting_row int) {
+	image_collection.PlaceImageSubRect(self.lp, key, self.images_resized_to, 0, self.screen_size.cell_height*starting_row, -1, -1)
+}
+
+func (self *Handler) draw_image_pair(ll *LogicalLine, starting_row int) {
+	if ll.left_image.key == "" && ll.right_image.key == "" {
+		return
+	}
+	defer self.lp.QueueWriteString("\r")
+	if ll.left_image.key != "" {
+		self.lp.QueueWriteString("\r")
+		self.lp.MoveCursorHorizontally(self.logical_lines.margin_size)
+		self.draw_image(ll.left_image.key, ll.left_image.count, starting_row)
+	}
+	if ll.right_image.key != "" {
+		self.lp.QueueWriteString("\r")
+		self.lp.MoveCursorHorizontally(self.logical_lines.margin_size + self.logical_lines.columns/2)
+		self.draw_image(ll.right_image.key, ll.right_image.count, starting_row)
+	}
+}
+
+func (self *Handler) draw_screen() {
+	self.lp.StartAtomicUpdate()
+	defer self.lp.EndAtomicUpdate()
+	if self.image_count > 0 {
+		self.resize_all_images_if_needed()
+		image_collection.DeleteAllVisiblePlacements(self.lp)
+	}
+	lp.MoveCursorTo(1, 1)
+	lp.ClearToEndOfScreen()
+	if self.logical_lines == nil || self.diff_map == nil || self.collection == nil {
+		lp.Println(`Calculating diff, please wait...`)
+		return
+	}
+	pos := self.scroll_pos
+	seen_images := utils.NewSet[int]()
+	for num_written := 0; num_written < self.screen_size.num_lines; num_written++ {
+		ll := self.logical_lines.At(pos.logical_line)
+		if ll == nil || self.logical_lines.ScreenLineAt(pos) == nil {
+			num_written--
+		} else {
+			is_image := ll.line_type == IMAGE_LINE
+			ll.render_screen_line(pos.screen_line, lp, self.logical_lines.margin_size, self.logical_lines.columns)
+			if is_image && !seen_images.Has(pos.logical_line) && pos.screen_line >= ll.image_lines_offset {
+				seen_images.Add(pos.logical_line)
+				self.draw_image_pair(ll, pos.screen_line-ll.image_lines_offset)
+			}
+			if self.current_search != nil {
+				if mkp := self.current_search.markup_line(pos, num_written); mkp != "" {
+					lp.QueueWriteString(mkp)
+				}
+			}
+			if mkp := self.add_mouse_selection_to_line(pos, num_written); mkp != "" {
+				lp.QueueWriteString(mkp)
+			}
+			lp.MoveCursorVertically(1)
+			lp.QueueWriteString("\x1b[m\r")
+		}
+		if self.logical_lines.IncrementScrollPosBy(&pos, 1) == 0 {
+			break
+		}
+	}
+	self.draw_status_line()
+}
+
+func (self *Handler) draw_status_line() {
+	if self.logical_lines == nil || self.diff_map == nil {
+		return
+	}
+	self.lp.MoveCursorTo(1, self.screen_size.rows)
+	self.lp.ClearToEndOfLine()
+	self.lp.SetCursorVisible(self.inputting_command)
+	if self.inputting_command {
+		self.rl.RedrawNonAtomic()
+	} else if self.statusline_message != "" {
+		self.lp.QueueWriteString(message_format(wcswidth.TruncateToVisualLength(sanitize(self.statusline_message), self.screen_size.columns)))
+	} else {
+		num := self.logical_lines.NumScreenLinesTo(self.scroll_pos)
+		den := self.logical_lines.NumScreenLinesTo(self.max_scroll_pos)
+		var frac int
+		if den > 0 {
+			frac = int((float64(num) * 100.0) / float64(den))
+		}
+		sp := statusline_format(fmt.Sprintf("%d%%", frac))
+		var counts string
+		if self.current_search == nil {
+			counts = added_count_format(strconv.Itoa(self.added_count)) + statusline_format(`,`) + removed_count_format(strconv.Itoa(self.removed_count))
+		} else {
+			counts = statusline_format(fmt.Sprintf("%d matches", self.current_search.Len()))
+		}
+		suffix := counts + "  " + sp
+		prefix := statusline_format(":")
+		filler := strings.Repeat(" ", utils.Max(0, self.screen_size.columns-wcswidth.Stringwidth(prefix)-wcswidth.Stringwidth(suffix)))
+		self.lp.QueueWriteString(prefix + filler + suffix)
+	}
+}
+
+func (self *Handler) on_text(text string, a, b bool) error {
+	if self.inputting_command {
+		defer self.draw_status_line()
+		return self.rl.OnText(text, a, b)
+	}
+	if self.statusline_message != "" {
+		self.statusline_message = ""
+		self.draw_status_line()
+		return nil
+	}
+	return nil
+}
+
+func (self *Handler) do_search(query string) {
+	self.current_search = nil
+	if len(query) < 2 {
+		return
+	}
+	if !self.current_search_is_regex {
+		query = regexp.QuoteMeta(query)
+	}
+	pat, err := regexp.Compile(`(?i)` + query)
+	if err != nil {
+		self.statusline_message = fmt.Sprintf("Bad regex: %s", err)
+		self.lp.Beep()
+		return
+	}
+	self.current_search = do_search(pat, self.logical_lines)
+	if self.current_search.Len() == 0 {
+		self.current_search = nil
+		self.statusline_message = fmt.Sprintf("No matches for: %#v", query)
+		self.lp.Beep()
+	} else {
+		if self.scroll_to_next_match(false, true) {
+			self.draw_screen()
+		} else {
+			self.lp.Beep()
+		}
+	}
+}
+
+func (self *Handler) on_key_event(ev *loop.KeyEvent) error {
+	if self.inputting_command {
+		defer self.draw_status_line()
+		if ev.MatchesPressOrRepeat("esc") {
+			self.inputting_command = false
+			ev.Handled = true
+			return nil
+		}
+		if ev.MatchesPressOrRepeat("enter") {
+			self.inputting_command = false
+			ev.Handled = true
+			self.do_search(self.rl.AllText())
+			self.draw_screen()
+			return nil
+		}
+		return self.rl.OnKeyEvent(ev)
+	}
+	if self.statusline_message != "" {
+		if ev.Type != loop.RELEASE {
+			ev.Handled = true
+			self.statusline_message = ""
+			self.draw_status_line()
+		}
+		return nil
+	}
+	if self.current_search != nil && ev.MatchesPressOrRepeat("esc") {
+		self.current_search = nil
+		self.draw_screen()
+		return nil
+	}
+	ac := self.shortcut_tracker.Match(ev, conf.KeyboardShortcuts)
+	if ac != nil {
+		ev.Handled = true
+		return self.dispatch_action(ac.Name, ac.Args)
+	}
+	return nil
+}
+
+func (self *Handler) scroll_lines(amt int) (delta int) {
+	before := self.scroll_pos
+	delta = self.logical_lines.IncrementScrollPosBy(&self.scroll_pos, amt)
+	if delta > 0 && self.max_scroll_pos.Less(self.scroll_pos) {
+		self.scroll_pos = self.max_scroll_pos
+		delta = self.logical_lines.Minus(self.scroll_pos, before)
+	}
+	return
+}
+
+func (self *Handler) scroll_to_next_change(backwards bool) bool {
+	if backwards {
+		for i := self.scroll_pos.logical_line - 1; i >= 0; i-- {
+			line := self.logical_lines.At(i)
+			if line.is_change_start {
+				self.scroll_pos = ScrollPos{i, 0}
+				return true
+			}
+		}
+	} else {
+		for i := self.scroll_pos.logical_line + 1; i < self.logical_lines.Len(); i++ {
+			line := self.logical_lines.At(i)
+			if line.is_change_start {
+				self.scroll_pos = ScrollPos{i, 0}
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func (self *Handler) scroll_to_next_match(backwards, include_current_match bool) bool {
+	if self.current_search == nil {
+		return false
+	}
+	if self.current_search_is_backward {
+		backwards = !backwards
+	}
+	offset, delta := 1, 1
+	if include_current_match {
+		offset = 0
+	}
+	if backwards {
+		offset *= -1
+		delta *= -1
+	}
+	pos := self.scroll_pos
+	if offset != 0 && self.logical_lines.IncrementScrollPosBy(&pos, offset) == 0 {
+		return false
+	}
+	for {
+		if self.current_search.Has(pos) {
+			self.scroll_pos = pos
+			self.draw_screen()
+			return true
+		}
+		if self.logical_lines.IncrementScrollPosBy(&pos, delta) == 0 || self.max_scroll_pos.Less(pos) {
+			break
+		}
+	}
+	return false
+}
+
+func (self *Handler) change_context_count(val int) bool {
+	val = utils.Max(0, val)
+	if val == self.current_context_count {
+		return false
+	}
+	self.current_context_count = val
+	p := self.scroll_pos
+	self.restore_position = &p
+	self.clear_mouse_selection()
+	self.generate_diff()
+	self.draw_screen()
+	return true
+}
+
+func (self *Handler) start_search(is_regex, is_backward bool) {
+	if self.inputting_command {
+		self.lp.Beep()
+		return
+	}
+	self.inputting_command = true
+	self.current_search_is_regex = is_regex
+	self.current_search_is_backward = is_backward
+	self.rl.SetText(``)
+	self.draw_status_line()
+}
+
+func (self *Handler) dispatch_action(name, args string) error {
+	switch name {
+	case `quit`:
+		self.lp.Quit(0)
+	case `copy_to_clipboard`:
+		text := self.text_for_current_mouse_selection()
+		if text == "" {
+			self.lp.Beep()
+		} else {
+			self.lp.CopyTextToClipboard(text)
+		}
+	case `copy_to_clipboard_or_exit`:
+		text := self.text_for_current_mouse_selection()
+		if text == "" {
+			self.lp.Quit(0)
+		} else {
+			self.lp.CopyTextToClipboard(text)
+		}
+	case `scroll_by`:
+		if args == "" {
+			args = "1"
+		}
+		amt, err := strconv.Atoi(args)
+		if err == nil {
+			if self.scroll_lines(amt) == 0 {
+				self.lp.Beep()
+			} else {
+				self.draw_screen()
+			}
+		} else {
+			self.lp.Beep()
+		}
+	case `scroll_to`:
+		done := false
+		switch {
+		case strings.Contains(args, `change`):
+			done = self.scroll_to_next_change(strings.Contains(args, `prev`))
+		case strings.Contains(args, `match`):
+			done = self.scroll_to_next_match(strings.Contains(args, `prev`), false)
+		case strings.Contains(args, `page`):
+			amt := self.screen_size.num_lines
+			if strings.Contains(args, `prev`) {
+				amt *= -1
+			}
+			done = self.scroll_lines(amt) != 0
+		default:
+			npos := self.scroll_pos
+			if strings.Contains(args, `end`) {
+				npos = self.max_scroll_pos
+			} else {
+				npos = ScrollPos{}
+			}
+			done = npos != self.scroll_pos
+			self.scroll_pos = npos
+		}
+		if done {
+			self.draw_screen()
+		} else {
+			self.lp.Beep()
+		}
+	case `change_context`:
+		new_ctx := self.current_context_count
+		switch args {
+		case `all`:
+			new_ctx = 100000
+		case `default`:
+			new_ctx = self.original_context_count
+		default:
+			delta, _ := strconv.Atoi(args)
+			new_ctx += delta
+		}
+		if !self.change_context_count(new_ctx) {
+			self.lp.Beep()
+		}
+	case `start_search`:
+		if self.diff_map != nil && self.logical_lines != nil {
+			a, b, _ := strings.Cut(args, " ")
+			self.start_search(config.StringToBool(a), config.StringToBool(b))
+		}
+	}
+	return nil
+}
+
+func (self *Handler) on_mouse_event(ev *loop.MouseEvent) error {
+	if self.logical_lines == nil {
+		return nil
+	}
+	if ev.Event_type == loop.MOUSE_PRESS && ev.Buttons&(loop.MOUSE_WHEEL_UP|loop.MOUSE_WHEEL_DOWN) != 0 {
+		self.handle_wheel_event(ev.Buttons&(loop.MOUSE_WHEEL_UP) != 0)
+		return nil
+	}
+	if ev.Event_type == loop.MOUSE_PRESS && ev.Buttons&loop.LEFT_MOUSE_BUTTON != 0 {
+		self.start_mouse_selection(ev)
+		return nil
+	}
+	if ev.Event_type == loop.MOUSE_MOVE {
+		self.update_mouse_selection(ev)
+		return nil
+	}
+	if ev.Event_type == loop.MOUSE_RELEASE && ev.Buttons&loop.LEFT_MOUSE_BUTTON != 0 {
+		self.finish_mouse_selection(ev)
+		return nil
+	}
+	return nil
+}

kittens/hints/main.go

@@ -0,0 +1,328 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package hints
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"unicode"
+
+	"kitty/tools/cli"
+	"kitty/tools/tty"
+	"kitty/tools/tui"
+	"kitty/tools/tui/loop"
+	"kitty/tools/utils"
+	"kitty/tools/utils/style"
+	"kitty/tools/wcswidth"
+)
+
+var _ = fmt.Print
+
+func convert_text(text string, cols int) string {
+	lines := make([]string, 0, 64)
+	empty_line := strings.Repeat("\x00", cols) + "\n"
+	s1 := utils.NewLineScanner(text)
+	for s1.Scan() {
+		full_line := s1.Text()
+		if full_line == "" {
+			lines = append(lines, empty_line)
+			continue
+		}
+		if strings.TrimRight(full_line, "\r") == "" {
+			for i := 0; i < len(full_line); i++ {
+				lines = append(lines, empty_line)
+			}
+			continue
+		}
+		appended := false
+		s2 := utils.NewSeparatorScanner(full_line, "\r")
+		for s2.Scan() {
+			line := s2.Text()
+			if line != "" {
+				line_sz := wcswidth.Stringwidth(line)
+				extra := cols - line_sz
+				if extra > 0 {
+					line += strings.Repeat("\x00", extra)
+				}
+				lines = append(lines, line)
+				lines = append(lines, "\r")
+				appended = true
+			}
+		}
+		if appended {
+			lines[len(lines)-1] = "\n"
+		}
+	}
+	ans := strings.Join(lines, "")
+	return strings.TrimRight(ans, "\r\n")
+}
+
+func parse_input(text string) string {
+	cols, err := strconv.Atoi(os.Getenv("OVERLAID_WINDOW_COLS"))
+	if err == nil {
+		return convert_text(text, cols)
+	}
+	term, err := tty.OpenControllingTerm()
+	if err == nil {
+		sz, err := term.GetSize()
+		term.Close()
+		if err == nil {
+			return convert_text(text, int(sz.Col))
+		}
+	}
+	return convert_text(text, 80)
+}
+
+type Result struct {
+	Match                []string         `json:"match"`
+	Programs             []string         `json:"programs"`
+	Multiple_joiner      string           `json:"multiple_joiner"`
+	Customize_processing string           `json:"customize_processing"`
+	Type                 string           `json:"type"`
+	Groupdicts           []map[string]any `json:"groupdicts"`
+	Extra_cli_args       []string         `json:"extra_cli_args"`
+	Linenum_action       string           `json:"linenum_action"`
+	Cwd                  string           `json:"cwd"`
+}
+
+func encode_hint(num int, alphabet string) (res string) {
+	runes := []rune(alphabet)
+	d := len(runes)
+	for res == "" || num > 0 {
+		res = string(runes[num%d]) + res
+		num /= d
+	}
+	return
+}
+
+func decode_hint(x string, alphabet string) (ans int) {
+	base := len(alphabet)
+	index_map := make(map[rune]int, len(alphabet))
+	for i, c := range alphabet {
+		index_map[c] = i
+	}
+	for _, char := range x {
+		ans = ans*base + index_map[char]
+	}
+	return
+}
+
+func main(_ *cli.Command, o *Options, args []string) (rc int, err error) {
+	output := tui.KittenOutputSerializer()
+	if tty.IsTerminal(os.Stdin.Fd()) {
+		tui.ReportError(fmt.Errorf("You must pass the text to be hinted on STDIN"))
+		return 1, nil
+	}
+	stdin, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		tui.ReportError(fmt.Errorf("Failed to read from STDIN with error: %w", err))
+		return 1, nil
+	}
+	if len(args) > 0 && o.CustomizeProcessing == "" && o.Type != "linenum" {
+		tui.ReportError(fmt.Errorf("Extra command line arguments present: %s", strings.Join(args, " ")))
+		return 1, nil
+	}
+	input_text := parse_input(utils.UnsafeBytesToString(stdin))
+	text, all_marks, index_map, err := find_marks(input_text, o, os.Args[2:]...)
+	if err != nil {
+		tui.ReportError(err)
+		return 1, nil
+	}
+
+	result := Result{
+		Programs: o.Program, Multiple_joiner: o.MultipleJoiner, Customize_processing: o.CustomizeProcessing, Type: o.Type,
+		Extra_cli_args: args, Linenum_action: o.LinenumAction,
+	}
+	result.Cwd, _ = os.Getwd()
+	alphabet := o.Alphabet
+	if alphabet == "" {
+		alphabet = DEFAULT_HINT_ALPHABET
+	}
+	ignore_mark_indices := utils.NewSet[int](8)
+	window_title := o.WindowTitle
+	if window_title == "" {
+		switch o.Type {
+		case "url":
+			window_title = "Choose URL"
+		default:
+			window_title = "Choose text"
+		}
+	}
+	current_text := ""
+	current_input := ""
+	match_suffix := ""
+	switch o.AddTrailingSpace {
+	case "always":
+		match_suffix = " "
+	case "never":
+	default:
+		if o.Multiple {
+			match_suffix = " "
+		}
+	}
+	chosen := []*Mark{}
+	lp, err := loop.New(loop.NoAlternateScreen) // no alternate screen reduces flicker on exit
+	if err != nil {
+		return
+	}
+	fctx := style.Context{AllowEscapeCodes: true}
+	faint := fctx.SprintFunc("dim")
+	hint_style := fctx.SprintFunc(fmt.Sprintf("fg=%s bg=%s bold", o.HintsForegroundColor, o.HintsBackgroundColor))
+	text_style := fctx.SprintFunc(fmt.Sprintf("fg=bright-%s bold", o.HintsTextColor))
+
+	highlight_mark := func(m *Mark, mark_text string) string {
+		hint := encode_hint(m.Index, alphabet)
+		if current_input != "" && !strings.HasPrefix(hint, current_input) {
+			return faint(mark_text)
+		}
+		hint = hint[len(current_input):]
+		if hint == "" {
+			hint = " "
+		}
+		mark_text = mark_text[len(hint):]
+		return hint_style(hint) + text_style(mark_text)
+	}
+
+	render := func() string {
+		ans := text
+		for i := len(all_marks) - 1; i >= 0; i-- {
+			mark := &all_marks[i]
+			if ignore_mark_indices.Has(mark.Index) {
+				continue
+			}
+			mtext := highlight_mark(mark, ans[mark.Start:mark.End])
+			ans = ans[:mark.Start] + mtext + ans[mark.End:]
+		}
+		ans = strings.ReplaceAll(ans, "\x00", "")
+		return strings.TrimRightFunc(strings.NewReplacer("\r", "\r\n", "\n", "\r\n").Replace(ans), unicode.IsSpace)
+	}
+
+	draw_screen := func() {
+		lp.StartAtomicUpdate()
+		defer lp.EndAtomicUpdate()
+		if current_text == "" {
+			current_text = render()
+		}
+		lp.ClearScreen()
+		lp.QueueWriteString(current_text)
+	}
+	reset := func() {
+		current_input = ""
+		current_text = ""
+	}
+
+	lp.OnInitialize = func() (string, error) {
+		lp.SendOverlayReady()
+		lp.SetCursorVisible(false)
+		lp.SetWindowTitle(window_title)
+		lp.AllowLineWrapping(false)
+		draw_screen()
+		return "", nil
+	}
+	lp.OnFinalize = func() string {
+		lp.SetCursorVisible(true)
+		return ""
+	}
+	lp.OnResize = func(old_size, new_size loop.ScreenSize) error {
+		draw_screen()
+		return nil
+	}
+	lp.OnText = func(text string, _, _ bool) error {
+		changed := false
+		for _, ch := range text {
+			if strings.ContainsRune(alphabet, ch) {
+				current_input += string(ch)
+				changed = true
+			}
+		}
+		if changed {
+			matches := []*Mark{}
+			for idx, m := range index_map {
+				if eh := encode_hint(idx, alphabet); strings.HasPrefix(eh, current_input) {
+					matches = append(matches, m)
+				}
+			}
+			if len(matches) == 1 {
+				chosen = append(chosen, matches[0])
+				if o.Multiple {
+					ignore_mark_indices.Add(matches[0].Index)
+					reset()
+				} else {
+					lp.Quit(0)
+					return nil
+				}
+			}
+			current_text = ""
+			draw_screen()
+		}
+		return nil
+	}
+
+	lp.OnKeyEvent = func(ev *loop.KeyEvent) error {
+		if ev.MatchesPressOrRepeat("backspace") {
+			ev.Handled = true
+			r := []rune(current_input)
+			if len(r) > 0 {
+				r = r[:len(r)-1]
+				current_input = string(r)
+				current_text = ""
+			}
+			draw_screen()
+		} else if ev.MatchesPressOrRepeat("enter") || ev.MatchesPressOrRepeat("space") {
+			ev.Handled = true
+			if current_input != "" {
+				idx := decode_hint(current_input, alphabet)
+				if m := index_map[idx]; m != nil {
+					chosen = append(chosen, m)
+					ignore_mark_indices.Add(idx)
+					if o.Multiple {
+						reset()
+						draw_screen()
+					} else {
+						lp.Quit(0)
+					}
+				} else {
+					current_input = ""
+					current_text = ""
+					draw_screen()
+				}
+			}
+		} else if ev.MatchesPressOrRepeat("esc") {
+			if o.Multiple {
+				lp.Quit(0)
+			} else {
+				lp.Quit(1)
+			}
+		}
+		return nil
+	}
+
+	err = lp.Run()
+	if err != nil {
+		return 1, err
+	}
+	ds := lp.DeathSignalName()
+	if ds != "" {
+		fmt.Println("Killed by signal: ", ds)
+		lp.KillIfSignalled()
+		return 1, nil
+	}
+	if lp.ExitCode() != 0 {
+		return lp.ExitCode(), nil
+	}
+	result.Match = make([]string, len(chosen))
+	result.Groupdicts = make([]map[string]any, len(chosen))
+	for i, m := range chosen {
+		result.Match[i] = m.Text + match_suffix
+		result.Groupdicts[i] = m.Groupdict
+	}
+	fmt.Println(output(result))
+	return
+}
+
+func EntryPoint(parent *cli.Command) {
+	create_cmd(parent, main)
+}

kittens/hints/main.py

@@ -1,46 +1,31 @@
 #!/usr/bin/env python3
 # License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
-import re
-import string
 import sys
 from functools import lru_cache
-from gettext import gettext as _
+from typing import Any, Dict, List, Optional, Sequence, Tuple
-from itertools import repeat
-from typing import Any, Callable, Dict, Generator, Iterable, Iterator, List, Optional, Pattern, Sequence, Set, Tuple, Type, cast
 
-from kitty.cli import parse_args
 from kitty.cli_stub import HintsCLIOptions
 from kitty.clipboard import set_clipboard_string, set_primary_selection
 from kitty.constants import website_url
-from kitty.fast_data_types import get_options, wcswidth
+from kitty.fast_data_types import get_options
-from kitty.key_encoding import KeyEvent
+from kitty.typing import BossType
-from kitty.typing import BossType, KittyCommonOpts
+from kitty.utils import resolve_custom_file
-from kitty.utils import ScreenSize, kitty_ansi_sanitizer_pat, resolve_custom_file, screen_size_function
 
-from ..tui.handler import Handler, result_handler
+from ..tui.handler import result_handler
-from ..tui.loop import Loop
-from ..tui.operations import faint, styled
-from ..tui.utils import report_error, report_unhandled_error
 
-
-@lru_cache()
-def kitty_common_opts() -> KittyCommonOpts:
-    import json
-    v = os.environ.get('KITTY_COMMON_OPTS')
-    if v:
-        return cast(KittyCommonOpts, json.loads(v))
-    from kitty.config import common_opts_as_dict
-    return common_opts_as_dict()
-
-
-DEFAULT_HINT_ALPHABET = string.digits + string.ascii_lowercase
 DEFAULT_REGEX = r'(?m)^\s*(.+)\s*$'
-FILE_EXTENSION = r'\.(?:[a-zA-Z0-9]{2,7}|[ahcmo])(?!\.)'
-PATH_REGEX = fr'(?:\S*?/[\r\S]+)|(?:\S[\r\S]*{FILE_EXTENSION})\b'
-DEFAULT_LINENUM_REGEX = fr'(?P<path>{PATH_REGEX}):(?P<line>\d+)'
 
+def load_custom_processor(customize_processing: str) -> Any:
+    if customize_processing.startswith('::import::'):
+        import importlib
+        m = importlib.import_module(customize_processing[len('::import::'):])
+        return {k: getattr(m, k) for k in dir(m)}
+    if customize_processing == '::linenum::':
+        return {'handle_result': linenum_handle_result}
+    custom_path = resolve_custom_file(customize_processing)
+    import runpy
+    return runpy.run_path(custom_path, run_name='__main__')
 
 class Mark:
 
@@ -60,476 +45,32 @@ class Mark:
         self.is_hyperlink = is_hyperlink
         self.group_id = group_id
 
-    def __repr__(self) -> str:
+    def as_dict(self) -> Dict[str, Any]:
-        return (f'Mark(index={self.index!r}, start={self.start!r}, end={self.end!r},'
-                f' text={self.text!r}, groupdict={self.groupdict!r}, is_hyperlink={self.is_hyperlink!r}, group_id={self.group_id!r})')
-
-
-@lru_cache(maxsize=2048)
-def encode_hint(num: int, alphabet: str) -> str:
-    res = ''
-    d = len(alphabet)
-    while not res or num > 0:
-        num, i = divmod(num, d)
-        res = alphabet[i] + res
-    return res
-
-
-def decode_hint(x: str, alphabet: str = DEFAULT_HINT_ALPHABET) -> int:
-    base = len(alphabet)
-    index_map = {c: i for i, c in enumerate(alphabet)}
-    i = 0
-    for char in x:
-        i = i * base + index_map[char]
-    return i
-
-
-def highlight_mark(m: Mark, text: str, current_input: str, alphabet: str, colors: Dict[str, str]) -> str:
-    hint = encode_hint(m.index, alphabet)
-    if current_input and not hint.startswith(current_input):
-        return faint(text)
-    hint = hint[len(current_input):] or ' '
-    text = text[len(hint):]
-    return styled(
-        hint,
-        fg=colors['foreground'],
-        bg=colors['background'],
-        bold=True
-    ) + styled(
-        text, fg=colors['text'], fg_intense=True, bold=True
-    )
-
-
-def debug(*a: Any, **kw: Any) -> None:
-    from ..tui.loop import debug as d
-    d(*a, **kw)
-
-
-def render(text: str, current_input: str, all_marks: Sequence[Mark], ignore_mark_indices: Set[int], alphabet: str, colors: Dict[str, str]) -> str:
-    for mark in reversed(all_marks):
-        if mark.index in ignore_mark_indices:
-            continue
-        mtext = highlight_mark(mark, text[mark.start:mark.end], current_input, alphabet, colors)
-        text = text[:mark.start] + mtext + text[mark.end:]
-
-    text = text.replace('\0', '')
-    return re.sub('[\r\n]', '\r\n', text).rstrip()
-
-
-class Hints(Handler):
-
-    use_alternate_screen = False  # disabled to avoid screen being blanked at exit causing flicker
-    overlay_ready_report_needed = True
-
-    def __init__(self, text: str, all_marks: Sequence[Mark], index_map: Dict[int, Mark], args: HintsCLIOptions):
-        self.text, self.index_map = text, index_map
-        self.alphabet = args.alphabet or DEFAULT_HINT_ALPHABET
-        self.colors = {'foreground': args.hints_foreground_color,
-                       'background': args.hints_background_color,
-                       'text': args.hints_text_color}
-        self.all_marks = all_marks
-        self.ignore_mark_indices: Set[int] = set()
-        self.args = args
-        self.window_title = args.window_title or (_('Choose URL') if args.type == 'url' else _('Choose text'))
-        self.multiple = args.multiple
-        self.match_suffix = self.get_match_suffix(args)
-        self.chosen: List[Mark] = []
-        self.reset()
-
-    @property
-    def text_matches(self) -> List[str]:
-        return [m.text + self.match_suffix for m in self.chosen]
-
-    @property
-    def groupdicts(self) -> List[Any]:
-        return [m.groupdict for m in self.chosen]
-
-    def get_match_suffix(self, args: HintsCLIOptions) -> str:
-        if args.add_trailing_space == 'always':
-            return ' '
-        if args.add_trailing_space == 'never':
-            return ''
-        return ' ' if args.multiple else ''
-
-    def reset(self) -> None:
-        self.current_input = ''
-        self.current_text: Optional[str] = None
-
-    def init_terminal_state(self) -> None:
-        self.cmd.set_cursor_visible(False)
-        self.cmd.set_window_title(self.window_title)
-        self.cmd.set_line_wrapping(False)
-
-    def initialize(self) -> None:
-        self.init_terminal_state()
-        self.draw_screen()
-
-    def on_text(self, text: str, in_bracketed_paste: bool = False) -> None:
-        changed = False
-        for c in text:
-            if c in self.alphabet:
-                self.current_input += c
-                changed = True
-        if changed:
-            matches = [
-                m for idx, m in self.index_map.items()
-                if encode_hint(idx, self.alphabet).startswith(self.current_input)
-            ]
-            if len(matches) == 1:
-                self.chosen.append(matches[0])
-                if self.multiple:
-                    self.ignore_mark_indices.add(matches[0].index)
-                    self.reset()
-                else:
-                    self.quit_loop(0)
-                    return
-            self.current_text = None
-            self.draw_screen()
-
-    def on_key(self, key_event: KeyEvent) -> None:
-        if key_event.matches('backspace'):
-            self.current_input = self.current_input[:-1]
-            self.current_text = None
-            self.draw_screen()
-        elif (key_event.matches('enter') or key_event.matches('space')) and self.current_input:
-            try:
-                idx = decode_hint(self.current_input, self.alphabet)
-                self.chosen.append(self.index_map[idx])
-                self.ignore_mark_indices.add(idx)
-            except Exception:
-                self.current_input = ''
-                self.current_text = None
-                self.draw_screen()
-            else:
-                if self.multiple:
-                    self.reset()
-                    self.draw_screen()
-                else:
-                    self.quit_loop(0)
-        elif key_event.matches('esc'):
-            self.quit_loop(0 if self.multiple else 1)
-
-    def on_interrupt(self) -> None:
-        self.quit_loop(1)
-
-    def on_eot(self) -> None:
-        self.quit_loop(1)
-
-    def on_resize(self, new_size: ScreenSize) -> None:
-        self.draw_screen()
-
-    def draw_screen(self) -> None:
-        if self.current_text is None:
-            self.current_text = render(self.text, self.current_input, self.all_marks, self.ignore_mark_indices, self.alphabet, self.colors)
-        self.cmd.clear_screen()
-        self.write(self.current_text)
-
-
-def regex_finditer(pat: 'Pattern[str]', minimum_match_length: int, text: str) -> Iterator[Tuple[int, int, 're.Match[str]']]:
-    has_named_groups = bool(pat.groupindex)
-    for m in pat.finditer(text):
-        s, e = m.span(0 if has_named_groups else pat.groups)
-        while e > s + 1 and text[e-1] == '\0':
-            e -= 1
-        if e - s >= minimum_match_length:
-            yield s, e, m
-
-
-closing_bracket_map = {'(': ')', '[': ']', '{': '}', '<': '>', '*': '*', '"': '"', "'": "'", "“": "”", "‘": "’"}
-opening_brackets = ''.join(closing_bracket_map)
-PostprocessorFunc = Callable[[str, int, int], Tuple[int, int]]
-postprocessor_map: Dict[str, PostprocessorFunc] = {}
-
-
-def postprocessor(func: PostprocessorFunc) -> PostprocessorFunc:
-    postprocessor_map[func.__name__] = func
-    return func
-
-
-class InvalidMatch(Exception):
-    """Raised when a match turns out to be invalid."""
-    pass
-
-
-@postprocessor
-def url(text: str, s: int, e: int) -> Tuple[int, int]:
-    if s > 4 and text[s - 5:s] == 'link:':  # asciidoc URLs
-        url = text[s:e]
-        idx = url.rfind('[')
-        if idx > -1:
-            e -= len(url) - idx
-    while text[e - 1] in '.,?!' and e > 1:  # remove trailing punctuation
-        e -= 1
-    # truncate url at closing bracket/quote
-    if s > 0 and e <= len(text) and text[s-1] in opening_brackets:
-        q = closing_bracket_map[text[s-1]]
-        idx = text.find(q, s)
-        if idx > s:
-            e = idx
-    # Restructured Text URLs
-    if e > 3 and text[e-2:e] == '`_':
-        e -= 2
-
-    return s, e
-
-
-@postprocessor
-def brackets(text: str, s: int, e: int) -> Tuple[int, int]:
-    # Remove matching brackets
-    if s < e <= len(text):
-        before = text[s]
-        if before in '({[<':
-            q = closing_bracket_map[before]
-            if text[e-1] == q:
-                s += 1
-                e -= 1
-            elif text[e:e+1] == q:
-                s += 1
-    return s, e
-
-
-@postprocessor
-def quotes(text: str, s: int, e: int) -> Tuple[int, int]:
-    # Remove matching quotes
-    if s < e <= len(text):
-        before = text[s]
-        if before in '\'"“‘':
-            q = closing_bracket_map[before]
-            if text[e-1] == q:
-                s += 1
-                e -= 1
-            elif text[e:e+1] == q:
-                s += 1
-    return s, e
-
-
-@postprocessor
-def ip(text: str, s: int, e: int) -> Tuple[int, int]:
-    from ipaddress import ip_address
-
-    # Check validity of IPs (or raise InvalidMatch)
-    ip = text[s:e]
-
-    try:
-        ip_address(ip)
-    except Exception:
-        raise InvalidMatch("Invalid IP")
-
-    return s, e
-
-
-def mark(pattern: str, post_processors: Iterable[PostprocessorFunc], text: str, args: HintsCLIOptions) -> Iterator[Mark]:
-    pat = re.compile(pattern)
-    sanitize_pat = re.compile('[\r\n\0]')
-    for idx, (s, e, match_object) in enumerate(regex_finditer(pat, args.minimum_match_length, text)):
-        try:
-            for func in post_processors:
-                s, e = func(text, s, e)
-        except InvalidMatch:
-            continue
-        groupdict = match_object.groupdict()
-        for group_name in groupdict:
-            group_idx = pat.groupindex[group_name]
-            gs, ge = match_object.span(group_idx)
-            gs, ge = max(gs, s), min(ge, e)
-            groupdict[group_name] = sanitize_pat.sub('', text[gs:ge])
-        mark_text = sanitize_pat.sub('', text[s:e])
-        yield Mark(idx, s, e, mark_text, groupdict)
-
-
-def run_loop(args: HintsCLIOptions, text: str, all_marks: Sequence[Mark], index_map: Dict[int, Mark], extra_cli_args: Sequence[str] = ()) -> Dict[str, Any]:
-    loop = Loop()
-    handler = Hints(text, all_marks, index_map, args)
-    loop.loop(handler)
-    if handler.chosen and loop.return_code == 0:
         return {
-            'match': handler.text_matches, 'programs': args.program,
+            'index': self.index, 'start': self.start, 'end': self.end,
-            'multiple_joiner': args.multiple_joiner, 'customize_processing': args.customize_processing,
+            'text': self.text, 'groupdict': {str(k):v for k, v in (self.groupdict or {}).items()},
-            'type': args.type, 'groupdicts': handler.groupdicts, 'extra_cli_args': extra_cli_args,
+            'group_id': self.group_id or '', 'is_hyperlink': self.is_hyperlink
-            'linenum_action': args.linenum_action,
-            'cwd': os.getcwd(),
         }
-    raise SystemExit(loop.return_code)
 
 
-def escape(chars: str) -> str:
+def parse_hints_args(args: List[str]) -> Tuple[HintsCLIOptions, List[str]]:
-    return chars.replace('\\', '\\\\').replace('-', r'\-').replace(']', r'\]')
+    from kitty.cli import parse_args
+    return parse_args(args, OPTIONS, usage, help_text, 'kitty +kitten hints', result_class=HintsCLIOptions)
 
 
-def functions_for(args: HintsCLIOptions) -> Tuple[str, List[PostprocessorFunc]]:
+def custom_marking() -> None:
-    post_processors = []
+    import json
-    if args.type == 'url':
+    text = sys.stdin.read()
-        if args.url_prefixes == 'default':
+    sys.stdin.close()
-            url_prefixes = kitty_common_opts()['url_prefixes']
+    opts, extra_cli_args = parse_hints_args(sys.argv[1:])
-        else:
+    m = load_custom_processor(opts.customize_processing or '::impossible::')
-            url_prefixes = tuple(args.url_prefixes.split(','))
+    if 'mark' not in m:
-        from .url_regex import url_delimiters
+        raise SystemExit(2)
-        pattern = '(?:{})://[^{}]{{3,}}'.format(
+    all_marks = tuple(x.as_dict() for x in m['mark'](text, opts, Mark, extra_cli_args))
-            '|'.join(url_prefixes), url_delimiters
+    sys.stdout.write(json.dumps(all_marks))
-        )
+    raise SystemExit(0)
-        post_processors.append(url)
-    elif args.type == 'path':
-        pattern = PATH_REGEX
-        post_processors.extend((brackets, quotes))
-    elif args.type == 'line':
-        pattern = '(?m)^\\s*(.+)[\\s\0]*$'
-    elif args.type == 'hash':
-        pattern = '[0-9a-f][0-9a-f\r]{6,127}'
-    elif args.type == 'ip':
-        pattern = (
-            # # IPv4 with no validation
-            r"((?:\d{1,3}\.){3}\d{1,3}"
-            r"|"
-            # # IPv6 with no validation
-            r"(?:[a-fA-F0-9]{0,4}:){2,7}[a-fA-F0-9]{1,4})"
-        )
-        post_processors.append(ip)
-    elif args.type == 'word':
-        chars = args.word_characters
-        if chars is None:
-            chars = kitty_common_opts()['select_by_word_characters']
-        pattern = fr'(?u)[{escape(chars)}\w]{{{args.minimum_match_length},}}'
-        post_processors.extend((brackets, quotes))
-    else:
-        pattern = args.regex
-    return pattern, post_processors
 
 
-def convert_text(text: str, cols: int) -> str:
-    lines: List[str] = []
-    empty_line = '\0' * cols + '\n'
-    for full_line in text.split('\n'):
-        if full_line:
-            if not full_line.rstrip('\r'):  # empty lines
-                lines.extend(repeat(empty_line, len(full_line)))
-                continue
-            appended = False
-            for line in full_line.split('\r'):
-                if line:
-                    line_sz = wcswidth(line)
-                    if line_sz < cols:
-                        line += '\0' * (cols - line_sz)
-                    lines.append(line)
-                    lines.append('\r')
-                    appended = True
-            if appended:
-                lines[-1] = '\n'
-    rstripped = re.sub('[\r\n]+$', '', ''.join(lines))
-    return rstripped
-
-
-def parse_input(text: str) -> str:
-    try:
-        cols = int(os.environ['OVERLAID_WINDOW_COLS'])
-    except KeyError:
-        cols = screen_size_function()().cols
-    return convert_text(text, cols)
-
-
-def linenum_marks(text: str, args: HintsCLIOptions, Mark: Type[Mark], extra_cli_args: Sequence[str], *a: Any) -> Generator[Mark, None, None]:
-    regex = args.regex
-    if regex == DEFAULT_REGEX:
-        regex = DEFAULT_LINENUM_REGEX
-    yield from mark(regex, [brackets, quotes], text, args)
-
-
-def load_custom_processor(customize_processing: str) -> Any:
-    if customize_processing.startswith('::import::'):
-        import importlib
-        m = importlib.import_module(customize_processing[len('::import::'):])
-        return {k: getattr(m, k) for k in dir(m)}
-    if customize_processing == '::linenum::':
-        return {'mark': linenum_marks, 'handle_result': linenum_handle_result}
-    custom_path = resolve_custom_file(customize_processing)
-    import runpy
-    return runpy.run_path(custom_path, run_name='__main__')
-
-
-def process_escape_codes(text: str) -> Tuple[str, Tuple[Mark, ...]]:
-    hyperlinks: List[Mark] = []
-    removed_size = idx = 0
-    active_hyperlink_url: Optional[str] = None
-    active_hyperlink_id: Optional[str] = None
-    active_hyperlink_start_offset = 0
-
-    def add_hyperlink(end: int) -> None:
-        nonlocal idx, active_hyperlink_url, active_hyperlink_id, active_hyperlink_start_offset
-        assert active_hyperlink_url is not None
-        hyperlinks.append(Mark(
-            idx, active_hyperlink_start_offset, end,
-            active_hyperlink_url,
-            groupdict={},
-            is_hyperlink=True, group_id=active_hyperlink_id
-        ))
-        active_hyperlink_url = active_hyperlink_id = None
-        active_hyperlink_start_offset = 0
-        idx += 1
-
-    def process_hyperlink(m: 're.Match[str]') -> str:
-        nonlocal removed_size, active_hyperlink_url, active_hyperlink_id, active_hyperlink_start_offset
-        raw = m.group()
-        if not raw.startswith('\x1b]8'):
-            removed_size += len(raw)
-            return ''
-        start = m.start() - removed_size
-        removed_size += len(raw)
-        if active_hyperlink_url is not None:
-            add_hyperlink(start)
-        raw = raw[4:-2]
-        parts = raw.split(';', 1)
-        if len(parts) == 2 and parts[1]:
-            active_hyperlink_url = parts[1]
-            active_hyperlink_start_offset = start
-            if parts[0]:
-                for entry in parts[0].split(':'):
-                    if entry.startswith('id=') and len(entry) > 3:
-                        active_hyperlink_id = entry[3:]
-                        break
-
-        return ''
-
-    text = kitty_ansi_sanitizer_pat().sub(process_hyperlink, text)
-    if active_hyperlink_url is not None:
-        add_hyperlink(len(text))
-    return text, tuple(hyperlinks)
-
-
-def run(args: HintsCLIOptions, text: str, extra_cli_args: Sequence[str] = ()) -> Optional[Dict[str, Any]]:
-    try:
-        text = parse_input(text)
-        text, hyperlinks = process_escape_codes(text)
-        pattern, post_processors = functions_for(args)
-        if args.type == 'linenum':
-            args.customize_processing = '::linenum::'
-        if args.type == 'hyperlink':
-            all_marks = hyperlinks
-        elif args.customize_processing:
-            m = load_custom_processor(args.customize_processing)
-            if 'mark' in m:
-                all_marks = tuple(m['mark'](text, args, Mark, extra_cli_args))
-            else:
-                all_marks = tuple(mark(pattern, post_processors, text, args))
-        else:
-            all_marks = tuple(mark(pattern, post_processors, text, args))
-        if not all_marks:
-            none_of = {'url': 'URLs', 'hyperlink': 'hyperlinks'}.get(args.type, 'matches')
-            report_error(_('No {} found.').format(none_of))
-            return None
-
-        largest_index = all_marks[-1].index
-        offset = max(0, args.hints_offset)
-        for m in all_marks:
-            if args.ascending:
-                m.index += offset
-            else:
-                m.index = largest_index - m.index + offset
-        index_map = {m.index: m for m in all_marks}
-    except Exception:
-        report_unhandled_error()
-    return run_loop(args, text, all_marks, index_map, extra_cli_args)
-
-
-# CLI {{{
 OPTIONS = r'''
 --program
 type=list
@@ -545,8 +86,12 @@ for the operating system. Various special values are supported:
 :code:`*`
     copy the match to the primary selection (on systems that support primary selections)
 
+:code:`@NAME`
+    copy the match to the specified buffer, e.g. :code:`@a`
+
 :code:`default`
-    run the default open program.
+    run the default open program. Note that when using the hyperlink :code:`--type`
+    the default is to use the kitty :doc:`hyperlink handling </open_actions>` facilities.
 
 :code:`launch`
     run :doc:`/launch` to open the program in a new kitty tab, window, overlay, etc.
@@ -592,7 +137,7 @@ example:
 :code:`kitty +kitten hints --type=linenum --linenum-action=tab vim +{line} {path}`
 will open the matched path at the matched line number in vim in
 a new kitty tab. Note that in order to use :option:`--program` to copy or paste
-text, you need to use the special value :code:`self`.
+the provided arguments, you need to use the special value :code:`self`.
 
 
 --url-prefixes
@@ -695,43 +240,15 @@ help_text = 'Select text from the screen using the keyboard. Defaults to searchi
 usage = ''
 
 
-def parse_hints_args(args: List[str]) -> Tuple[HintsCLIOptions, List[str]]:
-    return parse_args(args, OPTIONS, usage, help_text, 'kitty +kitten hints', result_class=HintsCLIOptions)
-
-
 def main(args: List[str]) -> Optional[Dict[str, Any]]:
-    text = ''
+    raise SystemExit('Should be run as kitten hints')
-    if sys.stdin.isatty():
-        if '--help' not in args and '-h' not in args:
-            report_unhandled_error('You must pass the text to be hinted on STDIN')
-    else:
-        text = sys.stdin.buffer.read().decode('utf-8')
-        sys.stdin = open(os.ctermid())
-    try:
-        opts, items = parse_hints_args(args[1:])
-    except SystemExit as e:
-        if e.code != 0:
-            report_unhandled_error(e.args[0])
-        return None
-    if items and not (opts.customize_processing or opts.type == 'linenum'):
-        report_unhandled_error('Extra command line arguments present: {}'.format(' '.join(items)))
-    try:
-        return run(opts, text, items)
-    except Exception:
-        report_unhandled_error()
-    return None
 
 
 def linenum_process_result(data: Dict[str, Any]) -> Tuple[str, int]:
-    lnum_pat = re.compile(r'(:\d+)$')
     for match, g in zip(data['match'], data['groupdicts']):
         path, line = g['path'], g['line']
         if path and line:
-            m = lnum_pat.search(path)
+            return path, int(line)
-            if m is not None:
-                line = m.group(1)[1:]
-                path = path.rpartition(':')[0]
-            return os.path.expanduser(path), int(line)
     return '', -1
 
 
@@ -746,15 +263,24 @@ def linenum_handle_result(args: List[str], data: Dict[str, Any], target_window_i
 
     if action == 'self':
         if w is not None:
-            is_copy_action = cmd[0] in ('-', '@', '*')
+            def is_copy_action(s: str) -> bool:
-            if is_copy_action:
+                return s in ('-', '@', '*') or s.startswith('@')
-                text = ' '.join(cmd[1:])
+
-                if cmd[0] == '-':
+            programs = list(filter(is_copy_action, data['programs'] or ()))
-                    w.paste_bytes(text)
+            # keep for backward compatibility, previously option `--program` does not need to be specified to perform copy actions
-                elif cmd[0] == '@':
+            if is_copy_action(cmd[0]):
-                    set_clipboard_string(text)
+                programs.append(cmd.pop(0))
-                elif cmd[0] == '*':
+            if programs:
-                    set_primary_selection(text)
+                text = ' '.join(cmd)
+                for program in programs:
+                    if program == '-':
+                        w.paste_bytes(text)
+                    elif program == '@':
+                        set_clipboard_string(text)
+                    elif program == '*':
+                        set_primary_selection(text)
+                    elif program.startswith('@'):
+                        boss.set_clipboard_buffer(program[1:], text)
             else:
                 import shlex
                 text = ' '.join(shlex.quote(arg) for arg in cmd)
@@ -768,10 +294,13 @@ def linenum_handle_result(args: List[str], data: Dict[str, Any], target_window_i
             }[action])(*cmd)
 
 
-@result_handler(type_of_input='screen-ansi', has_ready_notification=Hints.overlay_ready_report_needed)
+@result_handler(type_of_input='screen-ansi', has_ready_notification=True)
 def handle_result(args: List[str], data: Dict[str, Any], target_window_id: int, boss: BossType) -> None:
-    if data['customize_processing']:
+    cp = data['customize_processing']
-        m = load_custom_processor(data['customize_processing'])
+    if data['type'] == 'linenum':
+        cp = '::linenum::'
+    if cp:
+        m = load_custom_processor(cp)
         if 'handle_result' in m:
             m['handle_result'](args, data, target_window_id, boss, data['extra_cli_args'])
             return None
@@ -811,15 +340,19 @@ def handle_result(args: List[str], data: Dict[str, Any], target_window_id: int,
             w = boss.window_id_map.get(target_window_id)
             if w is not None:
                 w.paste_text(joined_text())
-        elif program == '@':
-            set_clipboard_string(joined_text())
         elif program == '*':
             set_primary_selection(joined_text())
+        elif program.startswith('@'):
+            if program == '@':
+                set_clipboard_string(joined_text())
+            else:
+                boss.set_clipboard_buffer(program[1:], joined_text())
         else:
             from kitty.conf.utils import to_cmdline
             cwd = data['cwd']
-            program = get_options().open_url_with if program == 'default' else program
+            is_default_program = program == 'default'
-            if text_type == 'hyperlink':
+            program = get_options().open_url_with if is_default_program else program
+            if text_type == 'hyperlink' and is_default_program:
                 w = boss.window_id_map.get(target_window_id)
                 for m in matches:
                     if w is not None:
@@ -849,6 +382,7 @@ if __name__ == '__main__':
 elif __name__ == '__doc__':
     cd = sys.cli_docs  # type: ignore
     cd['usage'] = usage
+    cd['short_desc'] = 'Select text from screen with keyboard'
     cd['options'] = OPTIONS
     cd['help_text'] = help_text
 # }}}

kittens/hints/marks.go

@@ -0,0 +1,590 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package hints
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"kitty"
+	"kitty/tools/config"
+	"kitty/tools/utils"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/dlclark/regexp2"
+	"github.com/seancfoley/ipaddress-go/ipaddr"
+	"golang.org/x/exp/slices"
+)
+
+var _ = fmt.Print
+
+const (
+	DEFAULT_HINT_ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyz"
+	FILE_EXTENSION        = `\.(?:[a-zA-Z0-9]{2,7}|[ahcmo])(?:\b|[^.])`
+)
+
+func path_regex() string {
+	return fmt.Sprintf(`(?:\S*?/[\r\S]+)|(?:\S[\r\S]*%s)\b`, FILE_EXTENSION)
+}
+
+func default_linenum_regex() string {
+	return fmt.Sprintf(`(?P<path>%s):(?P<line>\d+)`, path_regex())
+}
+
+type Mark struct {
+	Index        int            `json:"index"`
+	Start        int            `json:"start"`
+	End          int            `json:"end"`
+	Text         string         `json:"text"`
+	Group_id     string         `json:"group_id"`
+	Is_hyperlink bool           `json:"is_hyperlink"`
+	Groupdict    map[string]any `json:"groupdict"`
+}
+
+func process_escape_codes(text string) (ans string, hyperlinks []Mark) {
+	removed_size, idx := 0, 0
+	active_hyperlink_url := ""
+	active_hyperlink_id := ""
+	active_hyperlink_start_offset := 0
+
+	add_hyperlink := func(end int) {
+		hyperlinks = append(hyperlinks, Mark{
+			Index: idx, Start: active_hyperlink_start_offset, End: end, Text: active_hyperlink_url, Is_hyperlink: true, Group_id: active_hyperlink_id})
+		active_hyperlink_url, active_hyperlink_id = "", ""
+		active_hyperlink_start_offset = 0
+		idx++
+	}
+
+	ans = utils.ReplaceAll(utils.MustCompile("\x1b(?:\\[[0-9;:]*?m|\\].*?\x1b\\\\)"), text, func(raw string, groupdict map[string]utils.SubMatch) string {
+		if !strings.HasPrefix(raw, "\x1b]8") {
+			removed_size += len(raw)
+			return ""
+		}
+		start := groupdict[""].Start - removed_size
+		removed_size += len(raw)
+		if active_hyperlink_url != "" {
+			add_hyperlink(start)
+		}
+		raw = raw[4 : len(raw)-2]
+		if metadata, url, found := strings.Cut(raw, ";"); found && url != "" {
+			active_hyperlink_url = url
+			active_hyperlink_start_offset = start
+			if metadata != "" {
+				for _, entry := range strings.Split(metadata, ":") {
+					if strings.HasPrefix(entry, "id=") && len(entry) > 3 {
+						active_hyperlink_id = entry[3:]
+					}
+				}
+			}
+		}
+		return ""
+	})
+	if active_hyperlink_url != "" {
+		add_hyperlink(len(ans))
+	}
+	return
+}
+
+type PostProcessorFunc = func(string, int, int) (int, int)
+type GroupProcessorFunc = func(map[string]string)
+
+func is_punctuation(b string) bool {
+	switch b {
+	case ",", ".", "?", "!":
+		return true
+	}
+	return false
+}
+
+func closing_bracket_for(ch string) string {
+	switch ch {
+	case "(":
+		return ")"
+	case "[":
+		return "]"
+	case "{":
+		return "}"
+	case "<":
+		return ">"
+	case "*":
+		return "*"
+	case `"`:
+		return `"`
+	case "'":
+		return "'"
+	case "“":
+		return "”"
+	case "‘":
+		return "’"
+	}
+	return ""
+}
+
+func char_at(s string, i int) string {
+	ans, _ := utf8.DecodeRuneInString(s[i:])
+	if ans == utf8.RuneError {
+		return ""
+	}
+	return string(ans)
+}
+
+func matching_remover(openers ...string) PostProcessorFunc {
+	return func(text string, s, e int) (int, int) {
+		if s < e && e <= len(text) {
+			before := char_at(text, s)
+			if slices.Index(openers, before) > -1 {
+				q := closing_bracket_for(before)
+				if e > 0 && char_at(text, e-1) == q {
+					s++
+					e--
+				} else if char_at(text, e) == q {
+					s++
+				}
+			}
+		}
+		return s, e
+	}
+}
+
+func linenum_group_processor(gd map[string]string) {
+	pat := utils.MustCompile(`:\d+$`)
+	gd[`path`] = pat.ReplaceAllStringFunc(gd["path"], func(m string) string {
+		gd["line"] = m[1:]
+		return ``
+	})
+	gd[`path`] = utils.Expanduser(gd[`path`])
+}
+
+var PostProcessorMap = (&utils.Once[map[string]PostProcessorFunc]{Run: func() map[string]PostProcessorFunc {
+	return map[string]PostProcessorFunc{
+		"url": func(text string, s, e int) (int, int) {
+			if s > 4 && text[s-5:s] == "link:" { // asciidoc URLs
+				url := text[s:e]
+				idx := strings.LastIndex(url, "[")
+				if idx > -1 {
+					e -= len(url) - idx
+				}
+			}
+			for e > 1 && is_punctuation(char_at(text, e)) { // remove trailing punctuation
+				e--
+			}
+			// truncate url at closing bracket/quote
+			if s > 0 && e <= len(text) && closing_bracket_for(char_at(text, s-1)) != "" {
+				q := closing_bracket_for(char_at(text, s-1))
+				idx := strings.Index(text[s:], q)
+				if idx > 0 {
+					e = s + idx
+				}
+			}
+			// reStructuredText URLs
+			if e > 3 && text[e-2:e] == "`_" {
+				e -= 2
+			}
+			return s, e
+		},
+
+		"brackets": matching_remover("(", "{", "[", "<"),
+		"quotes":   matching_remover("'", `"`, "“", "‘"),
+		"ip": func(text string, s, e int) (int, int) {
+			addr := ipaddr.NewHostName(text[s:e])
+			if !addr.IsAddress() {
+				return -1, -1
+			}
+			return s, e
+		},
+	}
+}}).Get
+
+type KittyOpts struct {
+	Url_prefixes              *utils.Set[string]
+	Select_by_word_characters string
+}
+
+func read_relevant_kitty_opts(path string) KittyOpts {
+	ans := KittyOpts{Select_by_word_characters: kitty.KittyConfigDefaults.Select_by_word_characters}
+	handle_line := func(key, val string) error {
+		switch key {
+		case "url_prefixes":
+			ans.Url_prefixes = utils.NewSetWithItems(strings.Split(val, " ")...)
+		case "select_by_word_characters":
+			ans.Select_by_word_characters = strings.TrimSpace(val)
+		}
+		return nil
+	}
+	cp := config.ConfigParser{LineHandler: handle_line}
+	cp.ParseFiles(path)
+	if ans.Url_prefixes == nil {
+		ans.Url_prefixes = utils.NewSetWithItems(kitty.KittyConfigDefaults.Url_prefixes...)
+	}
+	return ans
+}
+
+var RelevantKittyOpts = (&utils.Once[KittyOpts]{Run: func() KittyOpts {
+	return read_relevant_kitty_opts(filepath.Join(utils.ConfigDir(), "kitty.conf"))
+}}).Get
+
+func functions_for(opts *Options) (pattern string, post_processors []PostProcessorFunc, group_processors []GroupProcessorFunc) {
+	switch opts.Type {
+	case "url":
+		var url_prefixes *utils.Set[string]
+		if opts.UrlPrefixes == "default" {
+			url_prefixes = RelevantKittyOpts().Url_prefixes
+		} else {
+			url_prefixes = utils.NewSetWithItems(strings.Split(opts.UrlPrefixes, ",")...)
+		}
+		pattern = fmt.Sprintf(`(?:%s)://[^%s]{3,}`, strings.Join(url_prefixes.AsSlice(), "|"), URL_DELIMITERS)
+		post_processors = append(post_processors, PostProcessorMap()["url"])
+	case "path":
+		pattern = path_regex()
+		post_processors = append(post_processors, PostProcessorMap()["brackets"], PostProcessorMap()["quotes"])
+	case "line":
+		pattern = "(?m)^\\s*(.+)[\\s\x00]*$"
+	case "hash":
+		pattern = "[0-9a-f][0-9a-f\r]{6,127}"
+	case "ip":
+		pattern = (
+		// IPv4 with no validation
+		`((?:\d{1,3}\.){3}\d{1,3}` + "|" +
+			// IPv6 with no validation
+			`(?:[a-fA-F0-9]{0,4}:){2,7}[a-fA-F0-9]{1,4})`)
+		post_processors = append(post_processors, PostProcessorMap()["ip"])
+	default:
+		pattern = opts.Regex
+		if opts.Type == "linenum" {
+			if pattern == kitty.HintsDefaultRegex {
+				pattern = default_linenum_regex()
+			}
+			post_processors = append(post_processors, PostProcessorMap()["brackets"], PostProcessorMap()["quotes"])
+			group_processors = append(group_processors, linenum_group_processor)
+		}
+	}
+	return
+}
+
+type Capture struct {
+	Text          string
+	Text_as_runes []rune
+	Byte_Offsets  struct {
+		Start, End int
+	}
+	Rune_Offsets struct {
+		Start, End int
+	}
+}
+
+func (self Capture) String() string {
+	return fmt.Sprintf("Capture(start=%d, end=%d, %#v)", self.Byte_Offsets.Start, self.Byte_Offsets.End, self.Text)
+}
+
+type Group struct {
+	Name     string
+	IsNamed  bool
+	Captures []Capture
+}
+
+func (self Group) LastCapture() Capture {
+	if len(self.Captures) == 0 {
+		return Capture{}
+	}
+	return self.Captures[len(self.Captures)-1]
+}
+
+func (self Group) String() string {
+	return fmt.Sprintf("Group(name=%#v, captures=%v)", self.Name, self.Captures)
+}
+
+type Match struct {
+	Groups []Group
+}
+
+func (self Match) HasNamedGroups() bool {
+	for _, g := range self.Groups {
+		if g.IsNamed {
+			return true
+		}
+	}
+	return false
+}
+
+func find_all_matches(re *regexp2.Regexp, text string) (ans []Match, err error) {
+	m, err := re.FindStringMatch(text)
+	if err != nil {
+		return
+	}
+	rune_to_bytes := utils.RuneOffsetsToByteOffsets(text)
+	get_byte_offset_map := func(groups []regexp2.Group) (ans map[int]int, err error) {
+		ans = make(map[int]int, len(groups)*2)
+		rune_offsets := make([]int, 0, len(groups)*2)
+		for _, g := range groups {
+			for _, c := range g.Captures {
+				if _, found := ans[c.Index]; !found {
+					rune_offsets = append(rune_offsets, c.Index)
+					ans[c.Index] = -1
+				}
+				end := c.Index + c.Length
+				if _, found := ans[end]; !found {
+					rune_offsets = append(rune_offsets, end)
+					ans[end] = -1
+				}
+			}
+		}
+		slices.Sort(rune_offsets)
+		for _, pos := range rune_offsets {
+			if ans[pos] = rune_to_bytes(pos); ans[pos] < 0 {
+				return nil, fmt.Errorf("Matches are not monotonic cannot map rune offsets to byte offsets")
+			}
+		}
+		return
+	}
+
+	for m != nil {
+		groups := m.Groups()
+		bom, err := get_byte_offset_map(groups)
+		if err != nil {
+			return nil, err
+		}
+		match := Match{Groups: make([]Group, len(groups))}
+		for i, g := range m.Groups() {
+			match.Groups[i].Name = g.Name
+			match.Groups[i].IsNamed = g.Name != "" && g.Name != strconv.Itoa(i)
+			for _, c := range g.Captures {
+				cn := Capture{Text: c.String(), Text_as_runes: c.Runes()}
+				cn.Rune_Offsets.End = c.Index + c.Length
+				cn.Rune_Offsets.Start = c.Index
+				cn.Byte_Offsets.Start, cn.Byte_Offsets.End = bom[c.Index], bom[cn.Rune_Offsets.End]
+				match.Groups[i].Captures = append(match.Groups[i].Captures, cn)
+			}
+		}
+		ans = append(ans, match)
+		m, _ = re.FindNextMatch(m)
+	}
+	return
+}
+
+func mark(r *regexp2.Regexp, post_processors []PostProcessorFunc, group_processors []GroupProcessorFunc, text string, opts *Options) (ans []Mark) {
+	sanitize_pat := regexp.MustCompile("[\r\n\x00]")
+	all_matches, _ := find_all_matches(r, text)
+	for i, m := range all_matches {
+		full_capture := m.Groups[0].LastCapture()
+		match_start, match_end := full_capture.Byte_Offsets.Start, full_capture.Byte_Offsets.End
+		for match_end > match_start+1 && text[match_end-1] == 0 {
+			match_end--
+		}
+		full_match := text[match_start:match_end]
+		if len([]rune(full_match)) < opts.MinimumMatchLength {
+			continue
+		}
+		for _, f := range post_processors {
+			match_start, match_end = f(text, match_start, match_end)
+			if match_start < 0 {
+				break
+			}
+		}
+		if match_start < 0 {
+			continue
+		}
+		full_match = sanitize_pat.ReplaceAllLiteralString(text[match_start:match_end], "")
+		gd := make(map[string]string, len(m.Groups))
+		for idx, g := range m.Groups {
+			if idx > 0 && g.IsNamed {
+				c := g.LastCapture()
+				if s, e := c.Byte_Offsets.Start, c.Byte_Offsets.End; s > -1 && e > -1 {
+					s = utils.Max(s, match_start)
+					e = utils.Min(e, match_end)
+					gd[g.Name] = sanitize_pat.ReplaceAllLiteralString(text[s:e], "")
+				}
+			}
+		}
+		for _, f := range group_processors {
+			f(gd)
+		}
+		gd2 := make(map[string]any, len(gd))
+		for k, v := range gd {
+			gd2[k] = v
+		}
+		if opts.Type == "regex" && len(m.Groups) > 1 && !m.HasNamedGroups() {
+			cp := m.Groups[1].LastCapture()
+			ms, me := cp.Byte_Offsets.Start, cp.Byte_Offsets.End
+			match_start = utils.Max(match_start, ms)
+			match_end = utils.Min(match_end, me)
+			full_match = sanitize_pat.ReplaceAllLiteralString(text[match_start:match_end], "")
+		}
+		if full_match != "" {
+			ans = append(ans, Mark{
+				Index: i, Start: match_start, End: match_end, Text: full_match, Groupdict: gd2,
+			})
+		}
+	}
+	return
+}
+
+type ErrNoMatches struct{ Type string }
+
+func is_word_char(ch rune, current_chars []rune) bool {
+	return unicode.IsLetter(ch) || unicode.IsNumber(ch) || (unicode.IsMark(ch) && len(current_chars) > 0 && unicode.IsLetter(current_chars[len(current_chars)-1]))
+}
+
+func mark_words(text string, opts *Options) (ans []Mark) {
+	left := text
+	var current_run struct {
+		chars       []rune
+		start, size int
+	}
+	chars := opts.WordCharacters
+	if chars == "" {
+		chars = RelevantKittyOpts().Select_by_word_characters
+	}
+	allowed_chars := make(map[rune]bool, len(chars))
+	for _, ch := range chars {
+		allowed_chars[ch] = true
+	}
+	pos := 0
+	post_processors := []PostProcessorFunc{PostProcessorMap()["brackets"], PostProcessorMap()["quotes"]}
+
+	commit_run := func() {
+		if len(current_run.chars) >= opts.MinimumMatchLength {
+			match_start, match_end := current_run.start, current_run.start+current_run.size
+			for _, f := range post_processors {
+				match_start, match_end = f(text, match_start, match_end)
+				if match_start < 0 {
+					break
+				}
+			}
+			if match_start > -1 && match_end > match_start {
+				full_match := text[match_start:match_end]
+				if len([]rune(full_match)) >= opts.MinimumMatchLength {
+					ans = append(ans, Mark{
+						Index: len(ans), Start: match_start, End: match_end, Text: full_match,
+					})
+				}
+			}
+		}
+		current_run.chars = nil
+		current_run.start = 0
+		current_run.size = 0
+	}
+
+	for {
+		ch, size := utf8.DecodeRuneInString(left)
+		if ch == utf8.RuneError {
+			break
+		}
+		if allowed_chars[ch] || is_word_char(ch, current_run.chars) {
+			if len(current_run.chars) == 0 {
+				current_run.start = pos
+			}
+			current_run.chars = append(current_run.chars, ch)
+			current_run.size += size
+		} else {
+			commit_run()
+		}
+		left = left[size:]
+		pos += size
+	}
+	commit_run()
+	return
+}
+
+func adjust_python_offsets(text string, marks []Mark) error {
+	// python returns rune based offsets (unicode chars not utf-8 bytes)
+	adjust := utils.RuneOffsetsToByteOffsets(text)
+	for i := range marks {
+		mark := &marks[i]
+		if mark.End < mark.Start {
+			return fmt.Errorf("The end of a mark must not be before its start")
+		}
+		s, e := adjust(mark.Start), adjust(mark.End)
+		if s < 0 || e < 0 {
+			return fmt.Errorf("Overlapping marks are not supported")
+		}
+		mark.Start, mark.End = s, e
+	}
+	return nil
+}
+
+func (self *ErrNoMatches) Error() string {
+	none_of := "matches"
+	switch self.Type {
+	case "urls":
+		none_of = "URLs"
+	case "hyperlinks":
+		none_of = "hyperlinks"
+	}
+	return fmt.Sprintf("No %s found", none_of)
+}
+
+func find_marks(text string, opts *Options, cli_args ...string) (sanitized_text string, ans []Mark, index_map map[int]*Mark, err error) {
+	sanitized_text, hyperlinks := process_escape_codes(text)
+
+	run_basic_matching := func() error {
+		pattern, post_processors, group_processors := functions_for(opts)
+		r, err := regexp2.Compile(pattern, regexp2.RE2)
+		if err != nil {
+			return fmt.Errorf("Failed to compile the regex pattern: %#v with error: %w", pattern, err)
+		}
+		ans = mark(r, post_processors, group_processors, sanitized_text, opts)
+		return nil
+	}
+
+	if opts.CustomizeProcessing != "" {
+		cmd := exec.Command(utils.KittyExe(), append([]string{"+runpy", "from kittens.hints.main import custom_marking; custom_marking()"}, cli_args...)...)
+		cmd.Stdin = strings.NewReader(sanitized_text)
+		stdout, stderr := bytes.Buffer{}, bytes.Buffer{}
+		cmd.Stdout, cmd.Stderr = &stdout, &stderr
+		err = cmd.Run()
+		if err != nil {
+			var e *exec.ExitError
+			if errors.As(err, &e) && e.ExitCode() == 2 {
+				err = run_basic_matching()
+				if err != nil {
+					return
+				}
+				goto process_answer
+			} else {
+				return "", nil, nil, fmt.Errorf("Failed to run custom processor %#v with error: %w\n%s", opts.CustomizeProcessing, err, stderr.String())
+			}
+		}
+		ans = make([]Mark, 0, 32)
+		err = json.Unmarshal(stdout.Bytes(), &ans)
+		if err != nil {
+			return "", nil, nil, fmt.Errorf("Failed to load output from custom processor %#v with error: %w", opts.CustomizeProcessing, err)
+		}
+		err = adjust_python_offsets(sanitized_text, ans)
+		if err != nil {
+			return "", nil, nil, fmt.Errorf("Custom processor %#v produced invalid mark output with error: %w", opts.CustomizeProcessing, err)
+		}
+	} else if opts.Type == "hyperlink" {
+		ans = hyperlinks
+	} else if opts.Type == "word" {
+		ans = mark_words(text, opts)
+	} else {
+		err = run_basic_matching()
+		if err != nil {
+			return
+		}
+	}
+process_answer:
+	if len(ans) == 0 {
+		return "", nil, nil, &ErrNoMatches{Type: opts.Type}
+	}
+	largest_index := ans[len(ans)-1].Index
+	offset := utils.Max(0, opts.HintsOffset)
+	index_map = make(map[int]*Mark, len(ans))
+	for i := range ans {
+		m := &ans[i]
+		if opts.Ascending {
+			m.Index += offset
+		} else {
+			m.Index = largest_index - m.Index + offset
+		}
+		index_map[m.Index] = m
+	}
+	return
+}

kittens/hints/marks_test.go

@@ -0,0 +1,143 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package hints
+
+import (
+	"errors"
+	"fmt"
+	"kitty"
+	"kitty/tools/utils"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+var _ = fmt.Print
+
+func TestHintMarking(t *testing.T) {
+
+	var opts *Options
+	cols := 20
+	cli_args := []string{}
+
+	reset := func() {
+		opts = &Options{Type: "url", UrlPrefixes: "default", Regex: kitty.HintsDefaultRegex}
+		cols = 20
+		cli_args = []string{}
+	}
+
+	r := func(text string, url ...string) (marks []Mark) {
+		ptext := convert_text(text, cols)
+		ptext, marks, _, err := find_marks(ptext, opts, cli_args...)
+		if err != nil {
+			var e *ErrNoMatches
+			if len(url) != 0 || !errors.As(err, &e) {
+				t.Fatalf("%#v failed with error: %s", text, err)
+			}
+			return
+		}
+		actual := utils.Map(func(m Mark) string { return m.Text }, marks)
+		if diff := cmp.Diff(url, actual); diff != "" {
+			t.Fatalf("%#v failed:\n%s", text, diff)
+		}
+		for _, m := range marks {
+			q := strings.NewReplacer("\n", "", "\r", "", "\x00", "").Replace(ptext[m.Start:m.End])
+			if diff := cmp.Diff(m.Text, q); diff != "" {
+				t.Fatalf("Mark start (%d) and end (%d) dont point to correct offset in text for %#v\n%s", m.Start, m.End, text, diff)
+			}
+		}
+		return
+	}
+
+	reset()
+	u := `http://test.me/`
+	r(u, u)
+	r(`"`+u+`"`, u)
+	r("("+u+")", u)
+	cols = len(u)
+	r(u+"\nxxx", u+"xxx")
+	cols = 20
+	r("link:"+u+"[xxx]", u)
+	r("`xyz <"+u+">`_.", u)
+	r(`<a href="`+u+`">moo`, u)
+	r("\x1b[mhttp://test.me/1234\n\x1b[mx", "http://test.me/1234")
+	r("\x1b[mhttp://test.me/12345\r\x1b[m6\n\x1b[mx", "http://test.me/123456")
+
+	opts.Type = "linenum"
+	m := func(text, path string, line int) {
+		ptext := convert_text(text, cols)
+		_, marks, _, err := find_marks(ptext, opts, cli_args...)
+		if err != nil {
+			t.Fatalf("%#v failed with error: %s", text, err)
+		}
+		gd := map[string]any{"path": path, "line": strconv.Itoa(line)}
+		if diff := cmp.Diff(marks[0].Groupdict, gd); diff != "" {
+			t.Fatalf("%#v failed:\n%s", text, diff)
+		}
+	}
+	m("file.c:23", "file.c", 23)
+	m("file.c:23:32", "file.c", 23)
+	m("file.cpp:23:1", "file.cpp", 23)
+	m("a/file.c:23", "a/file.c", 23)
+	m("a/file.c:23:32", "a/file.c", 23)
+	m("~/file.c:23:32", utils.Expanduser("~/file.c"), 23)
+
+	reset()
+	opts.Type = "path"
+	r("file.c", "file.c")
+	r("file.c.", "file.c")
+	r("file.epub.", "file.epub")
+	r("(file.epub)", "file.epub")
+	r("some/path", "some/path")
+
+	reset()
+	cols = 60
+	opts.Type = "ip"
+	r(`100.64.0.0`, `100.64.0.0`)
+	r(`2001:0db8:0000:0000:0000:ff00:0042:8329`, `2001:0db8:0000:0000:0000:ff00:0042:8329`)
+	r(`2001:db8:0:0:0:ff00:42:8329`, `2001:db8:0:0:0:ff00:42:8329`)
+	r(`2001:db8::ff00:42:8329`, `2001:db8::ff00:42:8329`)
+	r(`2001:DB8::FF00:42:8329`, `2001:DB8::FF00:42:8329`)
+	r(`0000:0000:0000:0000:0000:0000:0000:0001`, `0000:0000:0000:0000:0000:0000:0000:0001`)
+	r(`::1`, `::1`)
+	r(`255.255.255.256`)
+	r(`:1`)
+
+	reset()
+	opts.Type = "regex"
+	opts.Regex = `(?ms)^[*]?\s(\S+)`
+	r(`* 2b687c2 - test1`, `2b687c2`)
+	opts.Regex = `(?<=got:    )sha256.{4}`
+	r(`got:    sha256-L8=`, `sha256-L8=`)
+
+	reset()
+	opts.Type = "word"
+	r(`#one (two) 😍 a-1b `, `#one`, `two`, `a-1b`)
+	r("fōtiz час a\u0310b ", `fōtiz`, `час`, "a\u0310b")
+
+	reset()
+	tdir := t.TempDir()
+	simple := filepath.Join(tdir, "simple.py")
+	cli_args = []string{"--customize-processing", simple, "extra1"}
+	os.WriteFile(simple, []byte(`
+def mark(text, args, Mark, extra_cli_args, *a):
+    import re
+    for idx, m in enumerate(re.finditer(r'\w+', text)):
+        start, end = m.span()
+        mark_text = text[start:end].replace('\n', '').replace('\0', '')
+        yield Mark(idx, start, end, mark_text, {"idx": idx, "args": extra_cli_args})
+`), 0o600)
+	opts.Type = "regex"
+	opts.CustomizeProcessing = simple
+	marks := r("漢字 b", `漢字`, `b`)
+	if diff := cmp.Diff(marks[0].Groupdict, map[string]any{"idx": float64(0), "args": []any{"extra1"}}); diff != "" {
+		t.Fatalf("Did not get expected groupdict from custom processor:\n%s", diff)
+	}
+	opts.Regex = "b"
+	os.WriteFile(simple, []byte(""), 0o600)
+	r("a b", `b`)
+}

kittens/hints/url_regex.go

@@ -0,0 +1,5 @@
+// generated by gen-wcwidth.py, do not edit
+
+package hints
+
+const URL_DELIMITERS = `\x00-\x09\x0b-\x0c\x0e-\x20\x7f-\xa0\xad\x{600}-\x{605}\x{61c}\x{6dd}\x{70f}\x{890}-\x{891}\x{8e2}\x{1680}\x{180e}\x{2000}-\x{200f}\x{2028}-\x{202f}\x{205f}-\x{2064}\x{2066}-\x{206f}\x{3000}\x{d800}-\x{f8ff}\x{feff}\x{fff9}-\x{fffb}\x{110bd}\x{110cd}\x{13430}-\x{1343f}\x{1bca0}-\x{1bca3}\x{1d173}-\x{1d17a}\x{e0001}\x{e0020}-\x{e007f}\x{f0000}-\x{ffffd}\x{100000}-\x{10fffd}`