version 0.12.0

Fixes #830

.gitignore

@@ -13,3 +13,6 @@ kitty-profile
 dev
 __pycache__
 glfw/wayland-*-client-protocol.[ch]
+docs/_build
+docs/generated
+.DS_Store

.ignore

@@ -0,0 +1 @@
+kittens/unicode_input/names.h

.travis.yml

@@ -1,12 +1,30 @@
+git:
+    depth: false
 matrix:
     fast_finish: true
     include:
+        - os: linux
+          dist: trusty
+          group: beta
+          language: python
+          sudo: false
+          env: USE_BUNDLE=1 SW=$HOME/sw PKG_CONFIG_PATH=$SW/lib/pkgconfig LD_LIBRARY_PATH=$SW/lib PYTHON=$SW/bin/python3
+          addons:
+            apt:
+              packages:
+                - libxi-dev
+                - libxrandr-dev
+                - libxinerama-dev
+                - libxcursor-dev
+                - libxcb-xkb-dev
+                - libdbus-1-dev
+
         - os: linux
           dist: trusty
           group: beta
           sudo: false
           env:
-            - CC=gcc SANITIZE_ARG=--sanitize
+            - CC=gcc CFLAGS=-funsigned-char SANITIZE_ARG=--sanitize
           language: python
           python: "3.5"
           addons:
@@ -18,9 +36,9 @@ matrix:
                 - libxrandr-dev
                 - libxinerama-dev
                 - libxcursor-dev
-                - libunistring-dev
                 - libxcb-xkb-dev
                 - libpng-dev
+                - libdbus-1-dev
                 - python3-pil
 
         - os: linux
@@ -40,9 +58,9 @@ matrix:
                 - libxrandr-dev
                 - libxinerama-dev
                 - libxcursor-dev
-                - libunistring-dev
                 - libxcb-xkb-dev
                 - libpng-dev
+                - libdbus-1-dev
                 - python3-pil
 
         - os: linux
@@ -50,7 +68,7 @@ matrix:
           group: beta
           sudo: false
           env:
-            - RUN_FLAKE=1 BUILD_PKG=1
+            - RUN_FLAKE=1 BUILD_PKG=linux-package
           language: python
           python: "3.6"
           addons:
@@ -62,9 +80,9 @@ matrix:
                 - libxrandr-dev
                 - libxinerama-dev
                 - libxcursor-dev
-                - libunistring-dev
                 - libxcb-xkb-dev
                 - libpng-dev
+                - libdbus-1-dev
                 - python3-pil
                 - clang
         - os: osx
@@ -73,7 +91,7 @@ matrix:
 
         - os: osx
           language: generic
-          env: USE_BREW=1 BUILD_PKG=1
+          env: USE_BREW=1 BUILD_PKG=osx-bundle
 
 env:
     global:
@@ -89,27 +107,41 @@ install: |
         if [[ "$USE_BREW" == "1" ]]; then
             brew update;
             brew upgrade python;
-            brew install harfbuzz --without-glib --without-gobject-introspection --without-graphite2 --without-icu4c --without-freetype;
+            brew install harfbuzz --without-graphite2 --without-icu4c --without-freetype;
+            brew install imagemagick
+            brew install optipng
+            brew install librsvg
+            logo/make.py
         else
             mkdir -p $SW;
             curl https://download.calibre-ebook.com/travis/kitty/osx.tar.xz | tar xJ -C $SW;
         fi
     else
-        pushd /tmp
-        wget http://xkbcommon.org/download/libxkbcommon-0.6.1.tar.xz
-        tar xf libxkbcommon-0.6.1.tar.xz
-        cd libxkbcommon-0.6.1
-        ./configure --prefix=$HOME/harfbuzz --disable-dependency-tracking --disable-static
-        make -j2
-        make install
-        wget https://github.com/behdad/harfbuzz/releases/download/1.6.3/harfbuzz-1.6.3.tar.bz2
-        tar xf harfbuzz-1.6.3.tar.bz2
-        cd harfbuzz-1.6.3
-        ./configure --prefix=$HOME/harfbuzz --disable-dependency-tracking --disable-static --with-glib=no --with-gobject=no --with-cairo=no --with-fontconfig=no --with-icu=no --with-directwrite=no --with-freetype=yes
-        make -j2
-        make install
-        cd ..
-        popd
+        if [[ "$USE_BUNDLE" == "1" ]]; then
+            mkdir -p $SW;
+            curl https://download.calibre-ebook.com/travis/kitty/linux-64.tar.xz | tar xJ -C $SW;
+            find $SW -type f -name '*.pc' -exec sed -i "s,/sw/sw,$SW,g" {} \;;
+            find $SW -type f -name '_sysconfig*.py' -exec sed -i "s,/sw/sw,$SW,g" {} \;;
+        else
+            pushd /tmp
+            wget --no-check-certificate http://xkbcommon.org/download/libxkbcommon-0.6.1.tar.xz
+            tar xf libxkbcommon-0.6.1.tar.xz
+            cd libxkbcommon-0.6.1
+            ./configure --prefix=$HOME/harfbuzz --disable-dependency-tracking --disable-static
+            make -j2
+            make install
+            wget https://github.com/behdad/harfbuzz/releases/download/1.6.3/harfbuzz-1.6.3.tar.bz2
+            tar xf harfbuzz-1.6.3.tar.bz2
+            cd harfbuzz-1.6.3
+            ./configure --prefix=$HOME/harfbuzz --disable-dependency-tracking --disable-static --with-glib=no --with-gobject=no --with-cairo=no --with-fontconfig=no --with-icu=no --with-directwrite=no --with-freetype=yes
+            make -j2
+            make install
+            cd ..
+            popd
+        fi
+        if [[ "$BUILD_PKG" == "linux-package" ]]; then
+            pip install sphinx
+        fi
     fi
     pkg-config --cflags harfbuzz
     if [[ "$TRAVIS_OS_NAME" != 'osx' ]]; then
@@ -119,11 +151,20 @@ install: |
     set +e
 
 before_script:
-    - echo $LD_LIBRARY_PATH
+    - echo LD_LIBRARY_PATH=$LD_LIBRARY_PATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH
     - $PYTHON setup.py build --debug --verbose $SANITIZE_ARG;
 
-script:
-    - if grep -Inr '\s$' kitty kitty_tests *.py *.asciidoc .gitattributes .gitignore; then echo Trailing whitespace found, aborting.; exit 1; fi
-    - if [[ -z $SANITIZE_ARG ]]; then $PYTHON test.py; else ./asan-launcher test.py; fi
-    - if [[ "$RUN_FLAKE" == "1" ]]; then flake8 --count .; fi
-    - if [[ "$BUILD_PKG" == "1" ]]; then $PYTHON setup.py linux-package; fi
+script: |
+    set -e
+
+    if grep -Inr '\s$' kitty kitty_tests kittens docs *.py *.asciidoc *.rst .gitattributes .gitignore; then echo Trailing whitespace found, aborting.; exit 1; fi
+
+    if [[ -z "$SANITIZE_ARG" ]]; then $PYTHON test.py; else ./asan-launcher test.py; fi
+
+    if [[ "$RUN_FLAKE" == "1" ]]; then flake8 --count .; fi
+
+    if [[ "$BUILD_PKG" == "linux-package" ]]; then make FAIL_WARN=-W man && make FAIL_WARN=-W html; fi
+
+    if [[ ! -z "$BUILD_PKG" ]]; then $PYTHON setup.py $BUILD_PKG; fi
+
+    set +e

=template.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: %YEAR%, %USER% <%MAIL%>
-
-%HERE%

CHANGELOG.rst

@@ -1,418 +1 @@
-Changelog
-==============
-
-kitty is a feature full, cross-platform, *fast*, GPU based terminal emulator.
-
-version 0.9.0 [2018-04-15]
-------------------------------
-
-- A new kitty command shell to allow controlling kitty via commands. Press
-  `ctrl+shift+escape` to run the shell.
-
-- The hints kitten has become much more powerful. Now in addition to URLs you
-  can use it to select word, paths, filenames, lines, etc. from the screen.
-  These can be inserted into the terminal, copied to clipboard or sent to
-  external programs.
-
-- Linux: Switch to libxkbcommon for keyboard handling. It allows kitty to
-  support XCompose and dead keys and also react to keyboard remapping/layout
-  change without needing a restart.
-
-- Add support for multiple-key-sequence shortcuts
-
-- A new remote control command `set-colors` to change the current and/or
-  configured colors.
-
-- When double-clicking to select a word, select words that continue onto the
-  next/prev line as well.
-
-- Add an `include` directive for the config files to read multiple config files
-
-- Improve mouse selection for windows with padding. Moving the mouse into the
-  padding area now acts as if the mouse is over the nearest cell.
-
-- Allow setting all 256 terminal colors in the config file
-
-- Fix using `kitty --single-instance` to open a new window in a running kitty
-  instance, not respecting the `--directory` flag
-
-- URL hints: Exclude trailing punctuation from URLs
-
-- URL hints: Launch the browser from the kitty parent process rather than the
-  hints kitten. Fixes launching on some systems where xdg-open doesn't like
-  being run from a kitten.
-
-- Allow using rectangle select mode by pressing shift in addition to the
-  rectangle select modifiers even when the terminal program has grabbed the
-  mouse.
-
-
-version 0.8.4 [2018-03-31]
------------------------------
-
-- Fix presence of XDG_CONFIG_DIRS and absence of XDG_CONFIG_HOME preventing
-  kitty from starting
-
-- Revert change in last release to cell width calculation. Instead just clip
-  the right edges of characters that overflow the cell by at most two pixels
-
-
-version 0.8.3 [2018-03-29]
------------------------------
-
-- Fix a regression that broke the visual bell and invert screen colors escape
-  code
-
-- Allow double-click and triple-click + drag to extend selections word at a
-  time or line at a time
-
-- Add a keyboard shortcut to set the tab title
-
-- Fix setting window title to empty via OSC escape code not working correctly
-
-- Linux: Fix cell width calculation incorrect for some fonts (cell widths are
-  now calculated by actually rendering bitmaps, which is slower but more
-  accurate)
-
-- Allow specifying a system wide kitty config file, for all users
-
-- Add a --debug-config command line flag to output data about the system and
-  kitty configuration.
-
-- Wayland: Fix auto-repeat of keys not working
-
-
-version 0.8.2 [2018-03-17]
------------------------------
-
-- Allow extending existing selections by right clicking
-
-- Add a configurable keyboard shortcut and remote command to set the font size to a specific value
-
-- Add an option to have kitty close the window when the main processes running in it exits, even if there are still background processes writing to that terminal
-
-- Add configurable keyboard shortcuts to switch to a specific layout
-
-- Add a keyboard shortcut to edit the kitty config file easily 
-
-- macOS: Fix restoring of window size not correct on Retina screens
-
-- macOS: Add a facility to specify command line arguments when running kitty from the GUI 
-
-- Add a focus-tab remote command
-
-- Fix screen not being refreshed immediately after moving a window.
-
-- Fix a crash when getting the contents of the scrollback buffer as text
-
-version 0.8.1 [2018-03-09]
------------------------------
-
-- Extend kitty's remote control feature to work over both UNIX and TCP sockets,
-  so now you can control kitty from across the internet, if you want to.
-
-- Render private use unicode characters that are followed by a space as a two
-  character ligature. This fixes rendering for applications that misuse
-  private-use characters to display square symbols.
-
-- Fix Unicode emoji presentation variant selector causing new a fallback font
-  instance to be created
-
-- Fix a rare error that prevented the Unicode input kitten from working
-  sometimes
-
-- Allow using Ctrl+Alt+letter in legacy keyboard modes by outputting them as Ctrl+letter and Alt+letter.
-  This matches other terminals' behavior.
-
-- Fix cursor position off-by-one on horizontal axis when resizing the terminal
-
-- Wayland: Fix auto-repeat of keys not working
-
-- Wayland: Add support for window decorations provided by the Wayland shell
-
-- macOS: Fix URL hints not working 
-
-- macOS: Fix shell not starting in login mode on some computers
-
-- macOS: Output errors into console.app when running as a bundle
-
-
-version 0.8.0 [2018-02-24]
------------------------------
-
-- A framework for kittens, that is, small terminal programs designed to run
-  inside kitty and extend its capabilities. Examples include unicode input and
-  selecting URLs with the keyboard.
-
-- Input arbitrary unicode characters by pressing Ctrl+Shift+u. You can choose
-  characters by name, by hex code, by recently used, etc. There is even and
-  editable Favorites list.
-
-- Open URLs using only the keyboard. kitty has a new "hints mode". Press
-  Ctrl+Shift+e and all detected URLs on the screen are highlighted with a key
-  to press to open them. The facility is customizable so you can change
-  what is detected as a URL and which program is used to open it.
-
-- Add an option to change the titlebar color of kitty windows on macOS
-
-- Only consider Emoji characters with default Emoji presentation to be two
-  cells wide. This matches the standard. Also add support for the Unicode Emoji
-  variation presentation selector.
-
-- Prevent video tearing during high speed scrolling by syncing draws
-  to the monitor's refresh rate. There is a new configuration option to
-  control this ``sync_to_monitor``.
-
-- When displaying only a single window, use the default background color of the
-  window (which can be changed via escape codes) as the color for the margin
-  and padding of the window.
-
-- Add some non standard terminfo capabilities used by neovim and tmux.
-
-- Fix large drop in performance when using multiple top-level windows on macOS
-
-- Fix save/restore of window sizes not working correctly.
-
-- Remove option to use system wcwidth(). Now always use a wcwidth() based on
-  the Unicode standard. Only sane way.
-
-- Fix a regression that caused a few ligature glyphs to not render correctly in
-  rare circumstances.
-
-- Browsing the scrollback buffer now happens in an overlay window instead of a
-  new window/tab.
-
-version 0.7.1 [2018-01-31]
----------------------------
-
-- Add an option to adjust the width of character cells
-
-- Fix selecting text with the mouse in the scrollback buffer selecting text
-  from the line above the actually selected line
-
-- Fix some italic fonts having the right edge of characters cut-off,
-  unnecessarily
-
-
-version 0.7.0 [2018-01-24]
----------------------------
-
-- Allow controlling kitty from the shell prompt/scripts. You can
-  open/close/rename windows and tabs and even send input to specific windows.
-  See the README for details.
-
-- Add option to put tab bar at the top instead of the bottom
-
-- Add option to override the default shell
-
-- Add "Horizontal" and "Vertical" window layouts
-
-- Sessions: Allow setting titles and working directories for individual windows 
-
-- Option to copy to clipboard on mouse select
-
-- Fix incorrect reporting of mouse move events when using the SGR protocol
-
-- Make alt+backspace delete the previous word
-
-- Take the mouse wheel multiplier option in to account when generating fake key
-  scroll events
-
-- macOS: Fix closing top-level window does not transfer focus to other
-  top-level windows.
-
-- macOS: Fix alt+arrow keys not working when disabling the macos_option_as_alt
-  config option.
-
-- kitty icat: Workaround for bug in ImageMagick that would cause some images
-  to fail to display at certain sizes.
-
-- Fix rendering of text with ligature fonts that do not use dummy glyphs
-
-- Fix a regression that caused copying of the selection to clipboard to only
-  copy the visible part of the selection
-
-- Fix incorrect handling of some unicode combining marks that are not re-ordered
-
-- Fix handling on non-BMP combining characters
-
-- Drop the dependency on libunistring
-
-
-version 0.6.1 [2017-12-28]
----------------------------
-
-- Add an option to fade the text in inactive windows
-
-- Add new actions to open windows/tabs/etc. with the working directory set to 
-  the working directory of the current window.
-
-- Automatically adjust cell size when DPI changes, for example when kitty is
-  moved from one monitor to another with a different DPI
-
-- Ensure underlines are rendered even for fonts with very poor metrics
-
-- Fix some emoji glyphs not colored on Linux
-
-- Internal wcwidth() implementation is now auto-generated from the unicode
-  standard database
-
-- Allow configuring the modifiers to use for rectangular selection with the
-  mouse.
-
-- Fix incorrect minimum wayland version in the build script
-
-- Fix a crash when detecting a URL that ends at the end of the line
-
-- Fix regression that broke drawing of hollow cursor when window loses focus
-
-
-version 0.6.0 [2017-12-18]
----------------------------
-
-- Support background transparency via the background_opacity option. Provided
-  that your OS/window manager supports transparency, you can now have kitty
-  render pixels that have only the default background color as
-  semi-transparent.
-
-- Support multiple top level (OS) windows. These windows all share the sprite
-  texture cache on the GPU, further reducing overall resource usage. Use
-  the shortcut `ctrl+shift+n` to open a new top-level window.
-
-- Add support for a *daemon* mode using the `--single-instance` command line
-  option. With this option you can have only a single kitty instance running.
-  All future invocations simply open new top-level windows in the existing
-  instance.
-
-- Support colored emoji
-
-- Use CoreText instead of FreeType to render text on macOS
-
-- Support running on the "low power" GPU on dual GPU macOS machines
-
-- Add a new "grid" window layout
-
-- Drop the dependency on glfw (kitty now uses a modified, bundled copy of glfw)
-
-- Add an option to control the audio bell volume on X11 systems
-
-- Add a command line switch to set the name part of the WM_CLASS window
-  property independently.
-
-- Add a command line switch to set the window title.
-
-- Add more options to customize the tab-bar's appearance (font styles and
-  separator)
-
-- Allow drag and drop of files into kitty. On drop kitty will paste the
-  file path to the running program.
-
-- Add an option to control the underline style for URL highlighting on hover
-
-- X11: Set the WINDOWID environment variable 
-
-- Fix middle and right buttons swapped when sending mouse events to child
-  processes
-
-- Allow selecting in a rectangle by holding down Ctrl+Alt while dragging with
-  the mouse.
-
-
-version 0.5.1 [2017-12-01]
----------------------------
-
-- Add an option to control the thickness of lines in box drawing characters
-
-- Increase max. allowed ligature length to nine characters
-
-- Fix text not vertically centered when adjusting line height
-
-- Fix unicode block characters not being rendered properly
-
-- Fix shift+up/down not generating correct escape codes
-
-- Image display: Fix displaying images taller than two screen heights not
-  scrolling properly
-
-
-version 0.5.0 [2017-11-19]
----------------------------
-
-- Add support for ligature fonts such as Fira Code, Hasklig, etc. kitty now
-  uses harfbuzz for text shaping which allow it to support advanced OpenType
-  features such as contextual alternates/ligatures/combining glyphs/etc.
-
-- Make it easy to select fonts by allowing listing of monospace fonts using:
-  kitty list-fonts
-
-- Add an option to have window focus follow mouse
-
-- Add a keyboard shortcut (ctrl+shift+f11) to toggle fullscreen mode
-
-- macOS: Fix handling of option key. It now behaves just like the alt key on
-  Linux. There is an option to make it type unicode characters instead.
-
-- Linux: Add support for startup notification on X11 desktops. kitty will
-  now inform the window manager when its startup is complete.
-
-- Fix shell prompt being duplicated when window is resized
-
-- Fix crash when displaying more than 64 images in the same session
-
-- Add support for colons in SGR color codes. These are generated by some
-  applications such as neovim when they mistakenly identify kitty as a libvte
-  based terminal.
-
-- Fix mouse interaction not working in apps using obsolete mouse interaction
-  protocols
-
-- Linux: no longer require glew as a dependency
-
-
-version 0.4.2 [2017-10-23]
----------------------------
-
-- Fix a regression in 0.4.0 that broke custom key mappings
-
-- Fix a regression in 0.4.0 that broke support for non-QWERTY keyboard layouts 
-
-- Avoid using threads to reap zombie child processes. Also prevent kitty from
-  hanging if the open program hangs when clicking on a URL.
-
-
-version 0.4.0 [2017-10-22]
----------------------------
-
-- Support for drawing arbitrary raster graphics (images) in the terminal via a
-  new graphics protocol. kitty can draw images with full 32-bit color using both
-  ssh connections and files/shared memory (when available) for better
-  performance. The drawing primitives support alpha blending and z-index.
-  Images can be drawn both above and below text. See
-  https://github.com/kovidgoyal/kitty/blob/master/graphics-protocol.asciidoc
-  for details. 
-
-- Refactor kitty's internals to make it even faster and more efficient. The CPU
-  usage of kitty + X server while doing intensive tasks such as scrolling a
-  file continuously in less has been reduced by 50%. There are now two
-  configuration options ``repaint_delay`` and ``input_delay`` you can use to
-  fine tune kitty's performance vs CPU usage profile. The CPU usage of kitty +
-  X when scrolling in less is now significantly better than most (all?) other
-  terminals. See https://github.com/kovidgoyal/kitty#performance
-
-- Hovering over URLs with the mouse now underlines them to indicate they
-  can be clicked. Hold down Ctrl+Shift while clicking to open the URL.
-
-- Selection using the mouse is now more intelligent. It does not add
-  blank cells (i.e. cells that have no content) after the end of text in a
-  line to the selection.
-
-- The block cursor in now fully opaque but renders the character under it in
-  the background color, for enhanced visibility.
-
-- Allow combining multiple independent actions into a single shortcut
-
-- Add a new shortcut to pass the current selection to an external program
-
-- Allow creating shortcuts to open new windows running arbitrary commands. You
-  can also pass the current selection to the command as an arguments and the
-  contents of the screen + scrollback buffer as stdin to the command.
+See https://sw.kovidgoyal.net/kitty/changelog.html

INSTALL.md

@@ -0,0 +1,7 @@
+To build from source:
+
+    https://sw.kovidgoyal.net/kitty/build.html
+
+Pre-built binaries:
+
+    https://sw.kovidgoyal.net/kitty/binary.html

Makefile

@@ -32,3 +32,14 @@ rendered_logo: logo/kitty.iconset/icon_256x256.png
 
 app: rendered_logo
 	python3 setup.py kitty.app $(VVAL)
+
+man:
+	$(MAKE) FAIL_WARN=$(FAIL_WARN) -C docs man
+
+html:
+	$(MAKE) FAIL_WARN=$(FAIL_WARN) -C docs html
+
+linkcheck:
+	$(MAKE) FAIL_WARN=$(FAIL_WARN) -C docs linkcheck
+
+docs: man html

README.asciidoc

@@ -1,568 +1,5 @@
-= kitty - A terminal emulator
-:toc:
-:toc-placement!:
-// START_SHORTCUT_BLOCK
-:sc_close_tab: pass:quotes[`ctrl+shift+q`]
-:sc_close_window: pass:quotes[`ctrl+shift+w`]
-:sc_copy_to_clipboard: pass:quotes[`ctrl+shift+c`]
-:sc_decrease_font_size: pass:quotes[`ctrl+shift+minus`]
-:sc_edit_config_file: pass:quotes[`ctrl+shift+f2`]
-:sc_eighth_window: pass:quotes[`ctrl+shift+8`]
-:sc_fifth_window: pass:quotes[`ctrl+shift+5`]
-:sc_first_window: pass:quotes[`ctrl+shift+1`]
-:sc_fourth_window: pass:quotes[`ctrl+shift+4`]
-:sc_increase_font_size: pass:quotes[`ctrl+shift+equal`]
-:sc_input_unicode_character: pass:quotes[`ctrl+shift+u`]
-:sc_kitty_shell_window: pass:quotes[`ctrl+shift+escape`]
-:sc_move_tab_backward: pass:quotes[`ctrl+shift+,`]
-:sc_move_tab_forward: pass:quotes[`ctrl+shift+.`]
-:sc_move_window_backward: pass:quotes[`ctrl+shift+b`]
-:sc_move_window_forward: pass:quotes[`ctrl+shift+f`]
-:sc_move_window_to_top: pass:quotes[`ctrl+shift+``]
-:sc_new_os_window: pass:quotes[`ctrl+shift+n`]
-:sc_new_tab: pass:quotes[`ctrl+shift+t`]
-:sc_new_window: pass:quotes[`ctrl+shift+enter`]
-:sc_next_layout: pass:quotes[`ctrl+shift+l`]
-:sc_next_tab: pass:quotes[`ctrl+shift+right`]
-:sc_next_window: pass:quotes[`ctrl+shift+]`]
-:sc_ninth_window: pass:quotes[`ctrl+shift+9`]
-:sc_pass_selection_to_program: pass:quotes[`ctrl+shift+o`]
-:sc_paste_from_clipboard: pass:quotes[`ctrl+shift+v`]
-:sc_paste_from_selection: pass:quotes[`ctrl+shift+s` or `shift+insert`]
-:sc_previous_tab: pass:quotes[`ctrl+shift+left`]
-:sc_previous_window: pass:quotes[`ctrl+shift+[`]
-:sc_restore_font_size: pass:quotes[`ctrl+shift+backspace`]
-:sc_run_kitten_text_hints: pass:quotes[`ctrl+shift+e`]
-:sc_run_kitten_text_hints_type_line_program: pass:quotes[`ctrl+shift+p → l`]
-:sc_run_kitten_text_hints_type_path: pass:quotes[`ctrl+shift+p → shift+f`]
-:sc_run_kitten_text_hints_type_path_program: pass:quotes[`ctrl+shift+p → f`]
-:sc_run_kitten_text_hints_type_word_program: pass:quotes[`ctrl+shift+p → w`]
-:sc_scroll_end: pass:quotes[`ctrl+shift+end`]
-:sc_scroll_home: pass:quotes[`ctrl+shift+home`]
-:sc_scroll_line_down: pass:quotes[`ctrl+shift+down` or `ctrl+shift+j`]
-:sc_scroll_line_up: pass:quotes[`ctrl+shift+up` or `ctrl+shift+k`]
-:sc_scroll_page_down: pass:quotes[`ctrl+shift+page_down`]
-:sc_scroll_page_up: pass:quotes[`ctrl+shift+page_up`]
-:sc_second_window: pass:quotes[`ctrl+shift+2`]
-:sc_set_tab_title: pass:quotes[`ctrl+shift+alt+t`]
-:sc_seventh_window: pass:quotes[`ctrl+shift+7`]
-:sc_show_scrollback: pass:quotes[`ctrl+shift+h`]
-:sc_sixth_window: pass:quotes[`ctrl+shift+6`]
-:sc_tenth_window: pass:quotes[`ctrl+shift+0`]
-:sc_third_window: pass:quotes[`ctrl+shift+3`]
-:sc_toggle_fullscreen: pass:quotes[`ctrl+shift+f11`]
-// END_SHORTCUT_BLOCK
+= kitty - the fast, featureful, GPU based, terminal emulator
+
+See https://sw.kovidgoyal.net/kitty
 
-++++
-<img align="left" role="left" src="logo/kitty.png?raw=true"/>
-++++
 image::https://travis-ci.org/kovidgoyal/kitty.svg?branch=master[Build status, link=https://travis-ci.org/kovidgoyal/kitty]
-
-== Major Features
-
-* Uses OpenGL for rendering, offloads rendering to the GPU for link:#performance[lower system
-load] and buttery smooth scrolling. Uses threaded rendering to minimize input
-latency.
-
-* Supports all modern terminal features: link:graphics-protocol.asciidoc[graphics (images)], unicode,
-true-color, OpenType ligatures, mouse protocol, focus tracking, bracketed paste and so on.
-
-* Supports tiling multiple terminal windows side by side in different
-link:#layouts[layouts] without needing to use an extra program like tmux
-
-* Can be link:remote-control.asciidoc[controlled from scripts or the shell prompt], even over SSH.
-
-* Has a framework for _kittens_, small terminal programs that can be used to extend kitty's functionality.
-For example, they are used for link:#unicode-input[Unicode input] and link:#url-hints[Hints].
-
-* Supports link:#startup-sessions[startup sessions] which allow you to specify the window/tab layout,
-working directories and programs to run on startup.
-
-* Cross-platform support: kitty currently works on Linux and macOS, but because
-it uses only OpenGL for rendering, it should be trivial to port to other
-platforms.
-
-* Allows you to open link:#the-scrollback-buffer[the scrollback buffer] in a separate
-window using arbitrary programs of your choice. This is useful for browsing
-the history comfortably in a pager or editor.
-
-
-image::screenshot.png?raw=true[Screenshot, showing three programs in the "Tall" layout]
-
-toc::[]
-
-
-== Installation
-
-kitty is designed to run from source, for easy hackability. Make sure
-the following dependencies are installed first.
-
-=== Dependencies
-
-* python >= 3.5
-* harfbuzz >= 1.5.0
-* zlib
-* libpng
-* freetype (not needed on macOS)
-* fontconfig (not needed on macOS)
-* ImageMagick (optional, needed to use the `kitty icat` tool to display images in the terminal)
-* gcc or clang (required only for building)
-* pkg-config (required only for building)
-* For building on Linux in addition to the above dependencies you might also need to install the `-dev` packages for `xcursor`, `xrandr`, `xinerama`, `libgl1-mesa` and `xkbcommon-x11`, if they are not already installed by your distro.
-
-=== Install and run from source
-
-....
-git clone https://github.com/kovidgoyal/kitty && cd kitty
-....
-
-Now build the C parts of kitty with the following command:
-
-....
-make
-....
-
-You can run kitty, as:
-
-....
-python3 .
-....
-
-=== Linux packages
-
-* Arch Linux: AUR git package https://aur.archlinux.org/packages/kitty-git/
-
-* Arch Linux: AUR stable package https://aur.archlinux.org/packages/kitty/
-
-* NixOS / nixpkgs: https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/kitty/default.nix
-
-* Gentoo: https://packages.gentoo.org/packages/x11-terms/kitty
-
-* openSUSE: https://build.opensuse.org/package/show/X11:terminals/kitty
-
-=== macOS packages
-
-kitty is available as a macOS `dmg` file for easy installation from the
-link:../../releases[releases page].  You can also run kitty directly from
-source using the above install from source instructions, after installing its
-dependencies using http://brew.sh/[brew] or a similar package manager.
-
-
-== Design philosophy
-
-kitty is designed for power keyboard users. To that end all its controls
-work with the keyboard (although it fully supports mouse interactions as
-well). Its configuration is a simple, human editable, single file for
-easy reproducibility (I like to store config files in source control).
-
-The code in kitty is designed to be simple, modular and hackable. It is
-written in a mix of C (for performance sensitive parts) and Python (for
-easy hackability of the UI). It does not depend on any large and complex
-UI toolkit, using only OpenGL for rendering everything.
-
-Finally, kitty is designed from the ground up to support all modern
-terminal features, such as unicode, true color, bold/italic fonts, text
-formatting, etc. It even extends existing text formatting escape codes,
-to add support for features not available elsewhere, such as colored and
-styled (curly) underlines. One of the design goals of kitty is to be
-easily extensible so that new features can be added in the future with
-relatively less effort.
-
-== Tabs and Windows
-
-kitty is capable of running multiple programs organized into tabs and
-windows. The top level of organization is the _Tab_. Each tab consists
-of one or more _windows_. The windows can be arranged in multiple
-different layouts, like windows are organized in a tiling window
-manager. The keyboard controls (which are all customizable) for tabs and
-windows are:
-
-[[scrolling-shortcuts]]
-[options="header"]
-
-=== Scrolling
-
-|===
-|Action |Shortcut
-
-|Scroll line up   | {sc_scroll_line_up}
-|Scroll line down | {sc_scroll_line_down}
-|Scroll page up   | {sc_scroll_page_up}
-|Scroll page down | {sc_scroll_page_down}
-|Scroll to top    | {sc_scroll_home}
-|Scroll to bottom | {sc_scroll_end}
-
-|===
-
-[options="header"]
-
-=== Tabs
-
-|===
-|Action |Shortcut
-
-|New tab   | {sc_new_tab}
-|Close tab | {sc_close_tab}
-|Next tab  | {sc_next_tab}
-|Previous tab | {sc_previous_tab}
-|Next layout | {sc_next_layout}
-|Move tab forward | {sc_move_tab_forward}
-|Move tab backward | {sc_move_tab_backward}
-|Set tab title | {sc_set_tab_title}
-
-|===
-
-
-[options="header"]
-
-=== Windows
-
-|===
-|Action |Shortcut
-
-|New window   | {sc_new_window}
-|New OS window| {sc_new_os_window}
-|Close window | {sc_close_window}
-|Next window  | {sc_next_window}
-|Previous window | {sc_previous_window}
-|Move window forward | {sc_move_window_forward}
-|Move window backward | {sc_move_window_backward}
-|Move window to top | {sc_move_window_to_top}
-|Focus specific window (windows are counted clockwise from the top left corner) | {sc_first_window}, {sc_second_window} ... {sc_tenth_window}
-
-|===
-
-=== Miscellaneous
-
-|===
-|Action |Shortcut
-
-|Increase font size   | {sc_increase_font_size}
-|Decrease font size   | {sc_decrease_font_size}
-|Restore font size    | {sc_restore_font_size}
-|Toggle fullscreen    | {sc_toggle_fullscreen}
-|Input unicode character | {sc_input_unicode_character}
-|Click URL using the keyboard | {sc_run_simple_kitten_text_url_hints}
-|Pass current selection to program | {sc_pass_selection_to_program}
-|Edit kitty config file | {sc_edit_config_file}
-|Open a kitty shell | {sc_kitty_shell}
-|===
-
-
-== Controlling kitty from scripts
-
-You can control kitty from the command line/scripts by sending it messages.
-You can tell kitty to open/close/rename tabs and windows. You can even send
-arbitrary input/text to any specified window. Messages can be sent using `kitty
-@`. Note that you must set `allow_remote_control yes` in your
-link:kitty/kitty.conf[kitty.conf] to use this feature.  It even works over SSH
-connections.  This feature is best illustrated with a
-link:remote-control.asciidoc[tutorial].
-
-
-== The scrollback buffer
-
-kitty supports scrolling back to view history, just like most terminals. You
-can use either the <<scrolling-shortcuts,keyboard shortcuts>> or the mouse
-scroll wheel to do so.  However, kitty has an extra, neat feature. Sometimes
-you need to explore the scrollback buffer in more detail, maybe search for some
-text or refer to it side-by-side while typing in a follow-up command. kitty
-allows you to do this by pressing the {sc_show_scrollback} key-combination,
-which will open the scrollback buffer in your favorite pager program (which is
-`less` by default). Colors and text formatting are preserved. You can explore
-the scrollback pager comfortably within the pager.
-
-
-== Unicode input
-
-You can input unicode characters by name, hex code, recently used and even an editable favorites list.
-Press {sc_input_unicode_character} to start the unicode input widget, shown below.
-
-image::unicode.png?raw=true[Unicode input widget, showing selection of unicode characters by name]
-
-In Code mode, you enter a unicode character by typing in the hex code for the
-character and pressing enter, for example, type in 2716 and press enter to get
-✖. You can also choose a character from the list of recently used characters by
-typing a leading period and then the two character index and pressing Enter.
-
-In Name mode you instead type words from the character name and use the arrow
-keys/tab to select the character from the displayed matches. You can also type
-a leading period and the index for the match if you dont like to use arrow
-keys.
-
-== Hints
-
-kitty has a _hints mode_ to select and act on arbitrary text snippets currently
-visible on the screen.  For example, you can press {sc_run_kitten_text_hints}
-to choose any URL visible on the screen and then open it using your system
-browser.
-
-image::hints_mode.png?raw=true[URL hints mode]
-
-Similarly, you can press {sc_run_kitten_text_hints_type_path_program} to
-select anything that looks like a path or filename and then insert it into the
-terminal, very useful for picking files from the output of a git or ls command and
-adding them to the command line for the next command.
-
-The hints kitten is very powerful to see more detailed help on its various
-options and modes of operation, use: `kitty +kitten hints --help`.
-
-== Miscellaneous features
-
-* You can also hold down `ctrl+shift` and click on a URL to open it in a browser.
-* You can double click to select a word and triple click to select a line.
-* You can right click to extend a previous selection
-
-
-== Layouts
-
-Currently, there are five layouts available,
-
-* Stack -- Only a single maximized window is shown at a time
-* Tall -- One window is shown full height on the left, the rest of the windows are shown one below the other on the right
-* Grid -- All windows are shown in a grid
-* Horizontal -- All windows are shown side-by-side
-* Vertical -- All windows are shown one below the other
-
-You can switch between layouts using the {sc_next_layout} key combination. You can
-also create shortcuts to select particular layouts, and choose which layouts
-you want to enable/disable, see link:kitty/kitty.conf[kitty.conf] for examples.
-
-Writing a new layout only requires about fifty lines of code, so if there is
-some layout you want, take a look at link:kitty/layout.py[layout.py] and submit
-a pull request!
-
-
-== Configuration
-
-kitty is highly customizable, everything from keyboard shortcuts, to painting
-frames-per-second. See the heavily commented link:kitty/kitty.conf[default
-config file] for an overview of all customization possibilities.
-
-By default kitty looks for a config file in the OS config directories (usually
-`~/.config/kitty/kitty.conf` and additionally
-`~/Library/Preferences/kitty/kitty.conf` on macOS) but you can pass a specific
-path via the `--config` option or use the `KITTY_CONFIG_DIRECTORY` environment
-variable. See the help for the `--config` option in `kitty --help` for full
-details. You can also dump the current configuration using the `--debug-config`
-option.
-
-
-== Startup Sessions
-
-You can control the tabs, window layout, working directory, startup
-programs, etc. by creating a "session" file and using the `--session`
-command line flag. For example:
-
-....
-# Set the window layout for the current tab
-layout tall
-# Set the working directory for windows in the current tab
-cd ~
-# Create a window and run the specified command in it
-launch zsh
-launch vim
-# Set the title for the next window
-title Chat with x
-launch irssi --profile x
-
-# Create a new tab (the part after new_tab is the optional tab name which will
-# be displayed in the tab bar, if omitted, the title of the active window will
-# be used instead)
-new_tab my tab
-cd ~/somewhere
-# Set the layouts allowed in this tab
-enabled_layouts tall, stack
-# Set the current layout
-layout stack
-launch zsh
-# Make the current window the active (focused) window
-focus
-launch emacs
-....
-
-== Protocol Extensions
-
-kitty has a few extensions to the xterm protocol, to enable advanced features,
-see link:protocol-extensions.asciidoc[Protocol Extensions].
-
-
-== Font control
-
-kitty has extremely flexible and powerful font selection features. You can
-specify individual families for the regular, bold, italic and bold+italic
-fonts. You can even specify specific font families for specific ranges of
-unicode characters. This allows precise control over text rendering. It can
-come in handy for applications like powerline, without the need to use patched
-fonts. See the various font related configuration directives in the
-link:kitty/kitty.conf[config file].
-
-== Performance
-
-The main goals for kitty performance are user perceived latency while typing
-and "smoothness" while scrolling as well as CPU usage. kitty tries hard to find
-an optimum balance for these. To that end it keeps a cache of each rendered
-glyph in video RAM so that font rendering is not a bottleneck.  Interaction
-with child programs takes place in a separate thread from rendering, to improve
-smoothness.
-
-There are two parameters you can tune to adjust the performance. ``repaint_delay``
-and ``input_delay``. These control the artificial delays introduced into the
-render loop to reduce CPU usage. See the link:kitty/kitty.conf[config file] for details.
-See also the ``sync_to_monitor`` option to further decrease latency at the cost
-of some link:https://en.wikipedia.org/wiki/Screen_tearing[tearing] while scrolling.
-
-You can generate detailed per-function performance data using
-link:https://github.com/gperftools/gperftools[gperftools]. Build kitty with
-`make profile` which will create an executable called `kitty-profile`.  Run
-that and perform the task you want to analyse, for example, scrolling a large
-file with `less`. After you quit, function call statistics will be printed to
-`stdout` and you can use tools like *kcachegrind* for more detailed analysis.
-
-Here are some CPU usage numbers for the task of scrolling a file continuously in less.
-The CPU usage is for the terminal process and X together and is measured using htop.
-The measurements are taken at the same font and window size for all terminals on a
-`Intel(R) Core(TM) i7-4820K CPU @ 3.70GHz` CPU with a
-`Advanced Micro Devices, Inc. [AMD/ATI] Cape Verde XT [Radeon HD 7770/8760 / R7 250X]` GPU.
-
-|===
-| Terminal | CPU usage (X + terminal)
-
-| kitty    | 6 - 8%
-| xterm    | 5 - 7% (but scrolling was extremely janky)
-| termite  | 10 - 13%
-| urxvt    | 12 - 14%
-| gnome-terminal | 15 - 17%
-| konsole  | 29 - 31%
-
-|===
-
-As you can see, kitty uses much less CPU than all terminals, except xterm, but
-its scrolling "smoothness" is much better than that of xterm (at least to my,
-admittedly biased, eyes).
-
-== Note for Linux/macOS packagers
-
-While kitty does use python, it is not a traditional python package, so please do not install it in site-packages.
-Instead run,
-
-```
-python3 setup.py linux-package
-```
-
-This will install kitty into the directory `linux-package`. You can run kitty
-with `linux-package/bin/kitty`.  All the files needed to run kitty will be in
-`linux-package/lib/kitty`. The terminfo file will be installed into
-`linux-package/share/terminfo`. Simply copy these files into `/usr` to install
-kitty. In other words, `linux-package` is the staging area into which kitty is
-installed. You can choose a different staging area, by passing the `--prefix`
-argument to `setup.py`.
-
-You should probably split kitty into two packages, `kitty-terminfo` that
-installs the terminfo file and `kitty` that installs the main program.
-This allows users to install the terminfo file on servers into which they ssh,
-without needing to install all of kitty.
-
-You also need `tic` to compile the terminfo files, it is usually found in the development package of `ncurses`
-
-This applies to creating packages for kitty for macOS package managers such as
-brew or MacPorts as well.
-
-
-== Frequently Asked Questions
-
-=== Some special symbols are rendered small/truncated in kitty?
-
-The number of cells a unicode character takes up are controlled by the unicode
-standard.  All characters are rendered in a single cell unless the unicode
-standard says they should be rendered in two cells. When a symbol does not fit,
-it will either be rescaled to be smaller or truncated (depending on how much
-extra space it needs). This is often different from other terminals which just
-let the character overflow into neighboring cells, which is fine if the
-neighboring cell is empty, but looks terrible if it is not.
-
-Some programs, like powerline, vim with fancy gutter symbols/status-bar, etc.
-misuse unicode characters from the private use area to represent symbols. Often
-these symbols are square and should be rendered in two cells.  However, since
-private use area symbols all have their width set to one in the unicode
-standard, kitty renders them either smaller or truncated. The exception is if
-these characters are followed by a space or empty cell in which case kitty
-makes use of the extra cell to render them in two cells.
-
-=== How do I build kitty.app on macOS?
-
-Install `imagemagick`, `optipng` and `librsvg` using `brew` or similar (needed for the logo generation step).
-
-And run:
-```
-make app
-```
-
-This `kitty.app` unlike the released one does not include its own copy of
-python and the other dependencies. So if you ever un-install/upgrade those dependencies
-you might have to rebuild the app.
-
-Note that the released kitty.dmg includes all dependencies, unlike the
-`kitty.app` built above and is built automatically by using the `kitty` branch of
-link:https://github.com/kovidgoyal/build-calibre[build-calibre] however, that
-is designed to run on Linux and is not for the faint of heart.
-
-
-=== Using a color theme with a background color does not work well in vim?
-
-First make sure you have not changed the TERM environment variable, it should
-be `xterm-kitty`. vim uses *background color erase* even if the terminfo file
-does not contain the `bce` capability. This is a bug in vim. You can work around
-it by adding the following to your vimrc:
-
-....
-let &t_ut=''
-....
-
-See link:https://github.com/kovidgoyal/kitty/blob/master/protocol-extensions.asciidoc#setting-text-styles-colors-in-arbitrary-regions-of-the-screen[here]
-for why kitty does not support background color erase.
-
-=== I get errors about the terminal being unknown or opening the terminal failing when SSHing into a different computer?
-
-This happens because the kitty terminfo files are not available on the server.
-Either install kitty on the server, or simply copy over the terminfo files,
-using:
-
-....
-infocmp xterm-kitty | ssh myserver tic -x -o \~/.terminfo /dev/stdin
-....
-
-Really, the correct solution for this is to convince the OpenSSH maintainers to
-have ssh do this automatically when connecting to a server, so that all
-terminals work transparently.
-
-
-=== How do I change the colors in a running kitty instance?
-
-You can either use the
-link:http://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h2-Operating-System-Commands[OSC
-terminal escape codes] to set colors or you can enable link:remote-control.asciidoc[remote control]
-for kitty and use `kitty @ set-colors --help`.
-
-
-=== How do I specify command line options for kitty on macOS?
-
-Apple does not want you to use command line options with GUI applications. To
-workaround that limitation, kitty will read command line options from the file
-`<kitty config dir>/macos-launch-services-cmdline` when it is launched
-from the GUI, i.e. by clicking the kitty application icon or using `open -a kitty`.
-Note that this file is *only read* when running via the GUI.
-
-You can, of course, also run kitty from a terminal with command line options, using:
-`/Applications/kitty.app/Contents/MacOS/kitty`.
-
-And within kitty itself, you can always run kitty using just `kitty` as it
-cleverly adds itself to the PATH.
-
-== Resources on terminal behavior
-
-http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
-
-https://en.wikipedia.org/wiki/C0_and_C1_control_codes
-
-https://vt100.net/

__main__.py

@@ -3,10 +3,11 @@
 # License: GPL v3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
 
 import sys
+import os
 
 
 def icat(args):
-    from kitty.icat import main
+    from kittens.icat.main import main
     main(args)
 
 
@@ -25,6 +26,18 @@ def runpy(args):
     exec(args[1])
 
 
+def hold(args):
+    import subprocess
+    ret = subprocess.Popen(args[1:]).wait()
+    sys.stdin.read()
+    raise SystemExit(ret)
+
+
+def complete(args):
+    from kitty.complete import main
+    main(args[1:], entry_points, namespaced_entry_points)
+
+
 def launch(args):
     import runpy
     sys.argv = args[1:]
@@ -32,7 +45,12 @@ def launch(args):
 
 
 def run_kitten(args):
-    kitten = args[1]
+    try:
+        kitten = args[1]
+    except IndexError:
+        from kittens.runner import list_kittens
+        list_kittens()
+        raise SystemExit(1)
     sys.argv = args[1:]
     from kittens.runner import run_kitten
     run_kitten(kitten)
@@ -55,9 +73,25 @@ entry_points = {
     '+': namespaced,
 }
 namespaced_entry_points = {k: v for k, v in entry_points.items() if k[0] not in '+@'}
+namespaced_entry_points['hold'] = hold
+namespaced_entry_points['complete'] = complete
+
+
+def setup_openssl_environment():
+    # Workaround for Linux distros that have still failed to get their heads
+    # out of their asses and implement a common location for SSL certificates.
+    # It's not that hard people, there exists a wonderful tool called the symlink
+    # See http://www.mobileread.com/forums/showthread.php?t=256095
+    if 'SSL_CERT_FILE' not in os.environ and 'SSL_CERT_DIR' not in os.environ:
+        if os.access('/etc/pki/tls/certs/ca-bundle.crt', os.R_OK):
+            os.environ['SSL_CERT_FILE'] = '/etc/pki/tls/certs/ca-bundle.crt'
+        elif os.path.isdir('/etc/ssl/certs'):
+            os.environ['SSL_CERT_DIR'] = '/etc/ssl/certs'
 
 
 def main():
+    if getattr(sys, 'frozen', False) and 'darwin' not in sys.platform.lower():
+        setup_openssl_environment()
     first_arg = '' if len(sys.argv) < 2 else sys.argv[1]
     func = entry_points.get(first_arg)
     if func is None:

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    = -T $(FAIL_WARN)
+SPHINXBUILD   = python3 .. +launch $(shell which sphinx-build)
+SPHINXPROJ    = kitty
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/custom.css

@@ -0,0 +1,120 @@
+/*
+ * custom.css
+ * Copyright (C) 2018 Kovid Goyal
+ *
+ * Distributed under terms of the MIT license.
+ */
+
+.float-left-img {
+    float: left;
+    margin-right: 1em;
+    margin-bottom: 1em;
+}
+
+.float-right-img { float: right; margin-left: 1em; margin-bottom: 1em }
+
+.half-with-img { max-width: 50% }
+
+.fit-img { max-width: 95% }
+
+div.body p, div.body dd, div.body li, div.body blockquote {
+    text-align: justify;
+}
+
+pre {
+    white-space: pre-wrap;
+}
+
+pre.pre {
+    white-space: pre;
+}
+
+a[href], input[type="submit"] { cursor: pointer; }
+
+img[style] {
+    /* Used for :scale: images to have them render properly but still popup when clicked */
+    width: auto !important;
+    height: auto !important;
+}
+
+a {
+    text-decoration: none !important;
+    border-bottom: none !important;
+}
+
+body div.document {
+    margin-top: 1ex;
+}
+
+.major-features li {
+    margin-top: 0.75ex;
+    margin-bottom: 0.75ex;
+}
+
+.support-form input[type=submit] {
+    border-radius: 6px;
+    box-shadow: rgb(255, 246, 175) 0px 1px 0px 0px;
+    background: linear-gradient(rgb(255, 236, 100) 5%, rgb(255, 171, 35) 100%) rgb(255, 236, 100);
+    border: 1px solid rgb(255, 170, 34);
+    display: inline-block;
+    color: rgb(51, 51, 51);
+    font-family: Arial;
+    font-size: 15px;
+    font-weight: bold;
+    padding: 6px 24px;
+    text-decoration: none;
+    text-shadow: rgb(255, 238, 102) 0px 1px 0px;
+}
+
+.support-form input[type=submit]:hover {
+    background: linear-gradient(rgb(255, 171, 35) 5%, rgb(255, 236, 100) 100%) rgb(255, 171, 35);
+}
+
+.support-form input[type=submit]:focus {
+    outline: 0;
+}
+
+
+div.sphinxsidebar {
+    font-size: inherit;
+    line-height: inherit;
+}
+
+#sidebartoc li {
+    margin-top: 0.75ex;
+    margin-bottom: 0.75ex;
+}
+
+#sidebartoc ul {
+    list-style: none !important;
+}
+
+#sidebartoc a[href]:hover {
+    color: red;
+}
+
+
+.green {
+    color: green;
+}
+
+.cyan {
+    color: blue;
+}
+
+.env {
+    font-style: italic;
+}
+
+.italic {
+    font-style: italic;
+}
+
+.bold {
+    font-weight: bold;
+}
+
+.title {
+    font-size: larger;
+    font-weight: bold
+}

docs/_templates/layout.html

@@ -0,0 +1,13 @@
+{% extends "!layout.html" %}
+
+{%- block extrahead %}
+<!-- Google Analytics -->
+<script>
+window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
+ga('create', 'UA-20736318-2', 'auto');
+ga('send', 'pageview');
+</script>
+<script async="async" src='https://www.google-analytics.com/analytics.js'></script>
+<!-- End Google Analytics -->
+  {{ super() }}
+{% endblock %}

docs/_templates/localtoc.html

@@ -0,0 +1,6 @@
+{%- if display_toc %}
+<div>&nbsp;</div>
+<div id="sidebartoc">
+  {{ toc }}
+</div>
+{%- endif %}

docs/_templates/searchbox.html

@@ -0,0 +1,22 @@
+{#
+    basic/searchbox.html
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Sphinx sidebar template: quick search box.
+
+    :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+#}
+{%- if pagename != "search" and builder != "singlehtml" %}
+<div id="searchbox" style="display: none" role="search">
+    <div class="searchformwrapper">
+    <form class="search" action="{{ pathto('search') }}" method="get">
+      <input type="text" name="q" placeholder="{{ _('Search') }}" />
+      <input type="submit" value="{{ _('Go') }}" style="cursor: pointer" />
+      <input type="hidden" name="check_keywords" value="yes" />
+      <input type="hidden" name="area" value="default" />
+    </form>
+    </div>
+</div>
+<script type="text/javascript">$('#searchbox').show(0);</script>
+{%- endif %}

docs/_templates/support.html

@@ -0,0 +1,7 @@
+{% if pagename != "support" %}
+<div id="support" style="text-align: center">
+    <form class="support-form" action="{{ pathto('support') }}" title="{{ _('Donate to support kitty development') }}">
+        <input type="submit" value="{{ _('Support kitty') }}">
+    </form>
+</div>
+{% endif %}

docs/binary.rst

@@ -0,0 +1,65 @@
+kitty - Binary install
+========================
+
+.. |ins| replace:: curl -L :literal:`https://sw.kovidgoyal.net/kitty/installer.sh` | sh /dev/stdin
+
+.. highlight:: sh
+
+You can install pre-built binaries of |kitty| if you are on macOS or Linux using
+the following simple command:
+
+.. parsed-literal::
+    :class: pre
+
+    |ins|
+
+
+The binaries will be installed in the standard location for your OS,
+:file:`/Applications/kitty.app` on macOS and :file:`~/.local/kitty.app` on
+Linux. The installer only touches files in that directory.
+
+
+Manually installing
+---------------------
+
+If something goes wrong or you simply do not want to run the installer, you can
+manually download and install |kitty| from the `GitHub releases page
+<https://github.com/kovidgoyal/kitty/releases>`_. If you are on macOS, download
+the :file:`.dmg` and install as normal. If you are on Linux, download the tarball
+and extract it into a directory. The |kitty| executable will be in the
+:file:`bin` sub-directory.
+
+
+Customizing the installation
+--------------------------------
+
+* You can specify a different install location, with ``dest``:
+
+  .. parsed-literal::
+     :class: pre
+
+     |ins| \\
+         dest=/some/other/location
+
+* You can tell the installer not to launch |kitty| after installing it with
+  ``launch=n``:
+
+  .. parsed-literal::
+     :class: pre
+
+     |ins| \\
+         launch=n
+
+* You can use a previously downloaded dmg/tarball, with ``installer``:
+
+  .. parsed-literal::
+     :class: pre
+
+     |ins| \\
+         installer=/path/to/dmg or tarball
+
+
+Building from source
+------------------------
+
+|kitty| is easy to build from source, follow the :doc:`instructions <build>`.

docs/build.rst

@@ -0,0 +1,94 @@
+Building kitty from source
+==============================
+
+.. image:: https://travis-ci.org/kovidgoyal/kitty.svg?branch=master
+  :alt: Build status
+  :target: https://travis-ci.org/kovidgoyal/kitty
+
+
+|kitty| is designed to run from source, for easy hackability. Make sure
+the following dependencies are installed first.
+
+Dependencies
+----------------
+
+Run-time dependencies:
+
+    * python >= 3.5
+    * harfbuzz >= 1.5.0
+    * zlib
+    * libpng
+    * freetype (not needed on macOS)
+    * fontconfig (not needed on macOS)
+    * ImageMagick (optional, needed to use the ``kitty icat`` tool to display images in the terminal)
+    * pygments (optional, need for syntax highlighting in ``kitty +kitten diff``)
+
+Build-time dependencies:
+
+    * gcc or clang
+    * pkg-config
+    * For building on Linux in addition to the above dependencies you might also need to install the ``-dev`` packages for:
+      ``libdbus-1-dev``, ``libxcursor-dev``, ``libxrandr-dev``, ``libxi-dev``, ``libxinerama-dev``, ``libgl1-mesa-dev`` and ``libxkbcommon-x11-dev``,
+      if they are not already installed by your distro.
+
+Install and run from source
+------------------------------
+
+.. code-block:: sh
+
+    git clone https://github.com/kovidgoyal/kitty && cd kitty
+
+Now build the native code parts of |kitty| with the following command::
+
+    make
+
+You can run |kitty|, as::
+
+    python3 .
+
+If that works, you can create a script to launch |kitty|:
+
+.. code-block:: sh
+
+    #!/usr/bin/env python3
+    import runpy
+    runpy.run_path('/path/to/kitty/dir', run_name='__main__')
+
+And place it in :file:`~/bin` or :file:`/usr/bin` so that you can run |kitty| using
+just ``kitty``.
+
+Note for Linux/macOS packagers
+----------------------------------
+
+The released |kitty| source code is available as a `tarball`_ from
+`the GitHub releases page <https://github.com/kovidgoyal/kitty/releases>`_.
+
+While |kitty| does use python, it is not a traditional python package, so please
+do not install it in site-packages.
+Instead run::
+
+    python3 setup.py linux-package
+
+This will install |kitty| into the directory :file:`linux-package`. You can run |kitty|
+with :file:`linux-package/bin/kitty`.  All the files needed to run kitty will be in
+:file:`linux-package/lib/kitty`. The terminfo file will be installed into
+:file:`linux-package/share/terminfo`. Simply copy these files into :file:`/usr` to install
+|kitty|. In other words, :file:`linux-package` is the staging area into which |kitty| is
+installed. You can choose a different staging area, by passing the ``--prefix``
+argument to :file:`setup.py`.
+
+You should probably split |kitty| into two packages, :file:`kitty-terminfo` that
+installs the terminfo file and :file:`kitty` that installs the main program.
+This allows users to install the terminfo file on servers into which they ssh,
+without needing to install all of |kitty|.
+
+.. note::
+        You need a couple of extra dependencies to build linux-package.
+        :file:`tic` to compile terminfo files, usually found in the
+        development package of :file:`ncurses`. Also, if you are building from
+        a git checkout instead of the released source code tarball, you will
+        need :file:`sphinx-build` from the `Sphinx documentation generator
+        <http://www.sphinx-doc.org/>`_.
+
+This applies to creating packages for |kitty| for macOS package managers such as
+brew or MacPorts as well.

docs/changelog.rst

@@ -0,0 +1,822 @@
+Changelog
+==============
+
+|kitty| is a feature full, cross-platform, *fast*, GPU based terminal emulator.
+
+0.12.0 [2018-09-01]
+------------------------------
+
+- Preserve the mouse selection even when the contents of the screen are
+  scrolled or overwritten provided the new text does not intersect the
+  selected lines.
+
+- Linux: Implement support for Input Method Extensions (multilingual input
+  using standard keyboards) via `IBus
+  <https://github.com/ibus/ibus/wiki/ReadMe>`_ (:iss:`469`)
+
+- Implement completion for the kitty command in bash and zsh. See
+  :ref:`completion`.
+
+- Render the text under the cursor in a fixed color, configurable via
+  the option :opt:`cursor_text_color` (:iss:`126`)
+
+- Add an option :opt:`env` to set environment variables in child processes
+  from kitty.conf
+
+- Implement high precision scrolling with the trackpad on platforms such as
+  macOS and Wayland that implement it. (:pull:`819`)
+
+- macOS: Allow scrolling window contents using mouse wheel/trackpad even when the
+  window is not the active window (:iss:`729`)
+
+- Remote control: Allow changing the current window layout with a new
+  :ref:`at_goto-layout` command (:iss:`845`)
+
+- Remote control: Allow matching windows by the environment variables of their
+  child process as well
+
+- Allow running kittens via the remote control system (:iss:`738`)
+
+- Allow enabling remote control in only some kitty windows
+
+- Add a keyboard shortcut to reset the terminal (:sc:`reset_terminal`). It
+  takes parameters so you can define your own shortcuts to clear the
+  screen/scrollback also (:iss:`747`)
+
+- Fix one-pixel line appearing at window edges at some window sizes when
+  displaying images with background opacity enabled (:iss:`741`)
+
+- diff kitten: Fix error when right hand side file is binary and left hand side
+  file is text (:pull:`752`)
+
+- kitty @ new-window: Add a new option :option:`kitty @ new-window --window-type`
+  to create top-level OS windows (:iss:`770`)
+
+- macOS: The :opt:`focus_follows_mouse` option now also works across top-level kitty OS windows
+  (:iss:`754`)
+
+- Fix detection of URLs in HTML source code (URLs inside quotes) (:iss:`785`)
+
+- Implement support for emoji skin tone modifiers (:iss:`787`)
+
+- Round-trip the zwj unicode character. Rendering of sequences containing zwj
+  is still not implemented, since it can cause the collapse of an unbounded
+  number of characters into a single cell. However, kitty at least preserves
+  the zwj by storing it as a combining character.
+
+- macOS: Disable the custom mouse cursor. Using a custom cursor fails on dual
+  GPU machines. I give up, Apple users will just have to live with the
+  limitations of their choice of OS. (:iss:`794`)
+
+- macOS: Fix control+tab key combination not working (:iss:`801`)
+
+- Linux: Fix slow startup on some systems caused by GLFW searching for
+  joysticks. Since kitty does not use joysticks, disable joystick support.
+  (:iss:`830`)
+
+
+0.11.3 [2018-07-10]
+------------------------------
+
+- Draw only the minimum borders needed for inactive windows. That is only the borders
+  that separate the inactive window from a neighbor. Note that setting
+  a non-zero window margin overrides this and causes all borders to be drawn.
+  The old behavior of drawing all borders can be restored via the
+  :opt:`draw_minimal_borders` setting in kitty.conf. (:iss:`699`)
+
+- macOS: Add an option :opt:`macos_window_resizable` to control if kitty
+  top-level windows are resizable using the mouse or not (:iss:`698`)
+
+- macOS: Use a custom mouse cursor that shows up well on both light and dark backgrounds
+  (:iss:`359`)
+
+- macOS: Workaround for switching from fullscreen to windowed mode with the
+  titlebar hidden causing window resizing to not work. (:iss:`711`)
+
+- Fix triple-click to select line not working when the entire line is filled
+  (:iss:`703`)
+
+- When dragging to select with the mouse "grab" the mouse so that if it strays
+  into neighboring windows, the selection is still updated (:pull:`624`)
+
+- When clicking in the margin/border area of a window, map the click to the
+  nearest cell in the window. Avoids selection with the mouse failing when
+  starting the selection just outside the window.
+
+- When drag-scrolling stop the scroll when the mouse button is released.
+
+- Fix a regression in the previous release that caused pasting large amounts
+  of text to be duplicated (:iss:`709`)
+
+
+0.11.2 [2018-07-01]
+------------------------------
+
+- Linux: Allow using XKB key names to bind shortcuts to keys not supported by GLFW (:pull:`665`)
+
+- kitty shell: Ignore failure to read readline history file. Happens if the
+  user migrates their kitty cache directory between systems with incompatible
+  readline implementations.
+
+- macOS: Fix an error in remote control when using --listen-on (:iss:`679`)
+
+- hints kitten: Add a :option:`kitty +kitten hints --multiple` option to select
+  multiple items (:iss:`687`)
+
+- Fix pasting large amounts of text very slow (:iss:`682`)
+
+- Add an option :opt:`single_window_margin_width` to allow different margins
+  when only a single window is visible in the layout (:iss:`688`)
+
+- Add a :option:`kitty --hold` command line option to stay open after the child process exits (:iss:`667`)
+
+- diff kitten: When triggering a search scroll to the first match automatically
+
+- :option:`kitty --debug-font-fallback` also prints out what basic fonts were matched
+
+- When closing a kitty window reset the mouse cursor to its default shape and ensure it is visible (:iss:`655`).
+
+- Remote control: Speed-up reading of command responses
+
+- Linux installer: Fix installer failing on systems with python < 3.5
+
+- Support "-T" as an alias for "--title" (:pull:`659`)
+
+- Fix a regression in the previous release that broke using :option:`kitty
+  --debug-config` with custom key mappings (:iss:`695`)
+
+
+0.11.1 [2018-06-17]
+------------------------------
+
+- diff kitten: Implement searching for text in the diff (:iss:`574`)
+
+- Add an option :opt:`startup_session` to :file:`kitty.conf` to specify a
+  default startup session (:iss:`641`)
+
+- Add a command line option :option:`kitty --wait-for-single-instance-window-close`
+  to make :option:`kitty --single-instance` wait for the closing of the newly opened
+  window before quitting (:iss:`630`)
+
+- diff kitten: Allow theming the selection background/foreground as well
+
+- diff kitten: Display CRLF line endings using the unicode return symbol
+  instead of <d> as it is less intrusive (:iss:`638`)
+
+- diff kitten: Fix default foreground/background colors not being restored when
+  kitten quits (:iss:`637`)
+
+- Fix :option:`kitty @ set-colors --all` not working when more than one window
+  present (:iss:`632`)
+
+- Fix a regression that broke the legacy increase/decrease_font_size actions
+
+- Clear scrollback on reset (:iss:`631`)
+
+
+0.11.0 [2018-06-12]
+------------------------------
+
+- A new tab bar style "fade" in which each tab's edges fade into the background.
+  See :opt:`tab_bar_style` and :opt:`tab_fade` for details. The old look can be
+  restored by setting :opt:`tab_bar_style` to :code:`separator`.
+
+- :doc:`Pre-compiled binaries <binary>` with all bundled dependencies for Linux
+  (:iss:`595`)
+
+- A :doc:`new kitten <kittens/panel>` to create dock panels on X11 desktops
+  showing the output from arbitrary terminal programs.
+
+- Reduce data sent to the GPU per render by 30% (:commit:`8dea5b3`)
+
+- Implement changing the font size for individual top level (OS) windows
+  (:iss:`408`)
+
+- When viewing the scrollback in less using :sc:`show_scrollback` and kitty
+  is currently scrolled, position the scrollback in less to match kitty's
+  scroll position. (:iss:`148`)
+
+- ssh kitten: Support all SSH options. It can now be aliased directly to ssh
+  for convenience. (:pull:`591`)
+
+- icat kitten: Add :option:`kitty +kitten icat --print-window-size` to easily
+  detect the window size in pixels from scripting languages (:iss:`581`)
+
+- hints kitten: Allow selecting hashes from the terminal with
+  :sc:`insert_selected_hash` useful for git commits. (:pull:`604`)
+
+- Allow specifying initial window size in number of cells in addition to pixels
+  (:iss:`436`)
+
+- Add a setting to control the margins to the left and right of the tab-bar
+  (:iss:`584`)
+
+- When closing a tab switch to the last active tab instead of the right-most
+  tab (:iss:`585`)
+
+- Wayland: Fix kitty not starting when using wl_roots based compositors
+  (:iss:`157`)
+
+- Wayland: Fix mouse wheel/touchpad scrolling in opposite direction to other apps (:iss:`594`)
+
+- macOS: Fix the new OS window keyboard shortcut (:sc:`new_os_window`) not
+  working if no kitty window currently has focus. (:iss:`524`)
+
+- macOS: Keep kitty running even when the last window is closed. This is in
+  line with how applications are supposed to behave on macOS (:iss:`543`).
+  There is a new option (:opt:`macos_quit_when_last_window_closed`) to control
+  this.
+
+- macOS: Add macOS standard shortcuts for copy, paste and new OS window
+  (⌘+C, ⌘+V, ⌘+N)
+
+- Add a config option (:opt:`editor`) to set the EDITOR kitty uses (:iss:`580`)
+
+- Add a config option (:opt:`x11_hide_window_decorations`) to hide window
+  decorations under X11/Wayland (:iss:`607`)
+
+- Add an option to @set-window-title to make the title change non-permanent
+  (:iss:`592`)
+
+- Add support for the CSI t escape code to query window and cell sizes
+  (:iss:`581`)
+
+- Linux: When using layouts that map the keys to non-ascii characters,
+  map shortcuts using the ascii equivalents, from the default layout.
+  (:iss:`606`)
+
+- Linux: Fix fonts not being correctly read from TrueType Collection
+  (.ttc) files (:iss:`577`)
+
+- Fix :opt:`inactive_text_alpha` also applying to the tab bar (:iss:`612`)
+
+- :doc:`hints kitten <kittens/hints>`: Fix a regression that caused some blank lines to be not
+  be displayed.
+
+- Linux: Include a man page and the HTML docs when building the linux-package
+
+- Remote control: Fix kitty @ sometimes failing to read the response from
+  kitty. (:iss:`614`)
+
+- Fix `kitty @ set-colors` not working with the window border colors.
+  (:iss:`623`)
+
+- Fix a regression in 0.10 that caused incorrect rendering of the status bar in
+  irssi when used inside screen. (:iss:`621`)
+
+
+0.10.1 [2018-05-24]
+------------------------------
+
+- Add a kitten to easily ssh into servers that automatically copies the
+  terminfo files over. ``kitty +kitten ssh myserver``.
+
+- diff kitten: Make the keyboard shortcuts configurable (:iss:`563`)
+
+- Allow controlling *background_opacity* via either keyboard shortcuts or
+  remote control. Note that you must set *dynamic_background_opacity yes* in
+  kitty.conf first. (:iss:`569`)
+
+- diff kitten: Add keybindings to scroll by page
+
+- diff kitten: Fix incorrect syntax highlighting for a few file formats such as
+  yaml
+
+- macOS: Fix regression that caused the *macos_option_as_alt* setting to always
+  be disabled for all OS windows in a kitty instance after the first window
+  (:iss:`571`)
+
+- Fix Ctrl+Alt+Space not working in normal and application keyboard modes
+  (:iss:`562`)
+
+
+0.10.0 [2018-05-21]
+------------------------------
+
+- A diff kitten to show side-by-side diffs with syntax highlighting and support
+  for images. See :doc:`diff kitten <kittens/diff>`.
+
+- Make windows in the various kitty layouts manually resizable. See
+  :ref:`layouts` for details.
+
+- Implement support for the SGR *faint* escape code to make text blend
+  into the background (:iss:`446`).
+
+- Make the hints kitten a little smarter (:commit:`ad1109b`)
+  so that URLs that stretch over multiple lines are detected. Also improve
+  detection of surrounding brackets/quotes.
+
+- Make the kitty window id available as the environment variable
+  ``KITTY_WINDOW_ID`` (:iss:`532`).
+
+- Add a "fat" layout that is similar to the "tall" layout but vertically
+  oriented.
+
+- Expand environment variables in config file include directives
+
+- Allow programs running in kitty to read/write from the clipboard (:commit:`889ca77`).
+  By default only writing is allowed. This feature is supported in many
+  terminals, search for `OSC 52 clipboard` to find out more about using it.
+
+- Fix moving cursor outside a defined page area incorrectly causing the cursor
+  to be placed inside the page area. Caused incorrect rendering in neovim, in
+  some situations (:iss:`542`).
+
+- Render a couple more powerline symbols directly, bypassing the font
+  (:iss:`550`).
+
+- Fix ctrl+alt+<special> not working in normal and application keyboard (:iss:`548`).
+
+- Partial fix for rendering Right-to-left languages like Arabic. Rendering of
+  Arabic is never going to be perfect, but now it is at least readable.
+
+- Fix Ctrl+backspace acting as plain backspace in normal and application
+  keyboard modes (:iss:`538`).
+
+- Have the paste_from_selection action paste from the clipboard on platforms
+  that do not have a primary selection such as Wayland and macOS
+  (:iss:`529`)
+
+- Fix cursor_stop_blinking_after=0 not working (:iss:`530`)
+
+
+0.9.1 [2018-05-05]
+------------------------------
+
+- Show a bell symbol on the tab if a bell occurs in one of the windows in the tab and
+  the window is not the currently focused window
+
+- Change the window border color if a bell occurs in an unfocused window. Can
+  be disabled by setting the bell_border_color to be the same as the
+  inactive_border_color.
+
+- macOS: Add support for dead keys
+
+- Unicode input: When searching by name search for prefix matches as well as
+  whole word matches
+
+- Dynamically allocate the memory used for the scrollback history buffer.
+  Reduces startup memory consumption when using very large scrollback
+  buffer sizes.
+
+- Add an option to not request window attention on bell.
+
+- Remote control: Allow matching windows by number (visible position).
+
+- macOS: Fix changing tab title and kitty shell not working
+
+- When triple-clicking select all wrapped lines belonging to a single logical line.
+
+- hints kitten: Detect bracketed URLs and don't include the closing bracket in the URL.
+
+- When calling pass_selection_to_program use the current directory of the child
+  process as the cwd of the program.
+
+- Add macos_hide_from_tasks option to hide kitty from the macOS task switcher
+
+- macOS: When the macos_titlebar_color is set to background change the titlebar
+  colors to match the current background color of the active kitty window
+
+- Add a setting to clear all shortcuts defined up to that point in the config
+  file(s)
+
+- Add a setting (kitty_mod) to change the modifier used by all the default
+  kitty shortcuts, globally
+
+- Fix Shift+function key not working
+
+- Support the F13 to F25 function keys
+
+- Don't fail to start if the user deletes the hintstyle key from their
+  fontconfig configuration.
+
+- When rendering a private use unicode codepoint and a space as a two cell
+  ligature, set the foreground colors of the space cell to match the colors of
+  the first cell. Works around applications like powerline that use different
+  colors for the two cells.
+
+- Fix passing @text to other programs such as when viewing the scrollback
+  buffer not working correctly if kitty is itself scrolled up.
+
+- Fix window focus gained/lost events not being reported to child programs when
+  switching windows/tabs using the various keyboard shortcuts.
+
+- Fix tab title not changing to reflect the window title when switching between different windows in a tab
+
+- Ignore -e if it is specified on the command line. This is for compatibility
+  with broken software that assumes terminals should run with an -e option to
+  execute commands instead of just passing the commands as arguments.
+
+
+0.9.0 [2018-04-15]
+------------------------------
+
+- A new kitty command shell to allow controlling kitty via commands. Press
+  `ctrl+shift+escape` to run the shell.
+
+- The hints kitten has become much more powerful. Now in addition to URLs you
+  can use it to select word, paths, filenames, lines, etc. from the screen.
+  These can be inserted into the terminal, copied to clipboard or sent to
+  external programs.
+
+- Linux: Switch to libxkbcommon for keyboard handling. It allows kitty to
+  support XCompose and dead keys and also react to keyboard remapping/layout
+  change without needing a restart.
+
+- Add support for multiple-key-sequence shortcuts
+
+- A new remote control command `set-colors` to change the current and/or
+  configured colors.
+
+- When double-clicking to select a word, select words that continue onto the
+  next/prev line as well.
+
+- Add an `include` directive for the config files to read multiple config files
+
+- Improve mouse selection for windows with padding. Moving the mouse into the
+  padding area now acts as if the mouse is over the nearest cell.
+
+- Allow setting all 256 terminal colors in the config file
+
+- Fix using `kitty --single-instance` to open a new window in a running kitty
+  instance, not respecting the `--directory` flag
+
+- URL hints: Exclude trailing punctuation from URLs
+
+- URL hints: Launch the browser from the kitty parent process rather than the
+  hints kitten. Fixes launching on some systems where xdg-open doesn't like
+  being run from a kitten.
+
+- Allow using rectangle select mode by pressing shift in addition to the
+  rectangle select modifiers even when the terminal program has grabbed the
+  mouse.
+
+
+0.8.4 [2018-03-31]
+-----------------------------
+
+- Fix presence of XDG_CONFIG_DIRS and absence of XDG_CONFIG_HOME preventing
+  kitty from starting
+
+- Revert change in last release to cell width calculation. Instead just clip
+  the right edges of characters that overflow the cell by at most two pixels
+
+
+0.8.3 [2018-03-29]
+-----------------------------
+
+- Fix a regression that broke the visual bell and invert screen colors escape
+  code
+
+- Allow double-click and triple-click + drag to extend selections word at a
+  time or line at a time
+
+- Add a keyboard shortcut to set the tab title
+
+- Fix setting window title to empty via OSC escape code not working correctly
+
+- Linux: Fix cell width calculation incorrect for some fonts (cell widths are
+  now calculated by actually rendering bitmaps, which is slower but more
+  accurate)
+
+- Allow specifying a system wide kitty config file, for all users
+
+- Add a --debug-config command line flag to output data about the system and
+  kitty configuration.
+
+- Wayland: Fix auto-repeat of keys not working
+
+
+0.8.2 [2018-03-17]
+-----------------------------
+
+- Allow extending existing selections by right clicking
+
+- Add a configurable keyboard shortcut and remote command to set the font size to a specific value
+
+- Add an option to have kitty close the window when the main processes running in it exits, even if there are still background processes writing to that terminal
+
+- Add configurable keyboard shortcuts to switch to a specific layout
+
+- Add a keyboard shortcut to edit the kitty config file easily
+
+- macOS: Fix restoring of window size not correct on Retina screens
+
+- macOS: Add a facility to specify command line arguments when running kitty from the GUI
+
+- Add a focus-tab remote command
+
+- Fix screen not being refreshed immediately after moving a window.
+
+- Fix a crash when getting the contents of the scrollback buffer as text
+
+0.8.1 [2018-03-09]
+-----------------------------
+
+- Extend kitty's remote control feature to work over both UNIX and TCP sockets,
+  so now you can control kitty from across the internet, if you want to.
+
+- Render private use unicode characters that are followed by a space as a two
+  character ligature. This fixes rendering for applications that misuse
+  private-use characters to display square symbols.
+
+- Fix Unicode emoji presentation variant selector causing new a fallback font
+  instance to be created
+
+- Fix a rare error that prevented the Unicode input kitten from working
+  sometimes
+
+- Allow using Ctrl+Alt+letter in legacy keyboard modes by outputting them as Ctrl+letter and Alt+letter.
+  This matches other terminals' behavior.
+
+- Fix cursor position off-by-one on horizontal axis when resizing the terminal
+
+- Wayland: Fix auto-repeat of keys not working
+
+- Wayland: Add support for window decorations provided by the Wayland shell
+
+- macOS: Fix URL hints not working
+
+- macOS: Fix shell not starting in login mode on some computers
+
+- macOS: Output errors into console.app when running as a bundle
+
+
+0.8.0 [2018-02-24]
+-----------------------------
+
+- A framework for kittens, that is, small terminal programs designed to run
+  inside kitty and extend its capabilities. Examples include unicode input and
+  selecting URLs with the keyboard.
+
+- Input arbitrary unicode characters by pressing Ctrl+Shift+u. You can choose
+  characters by name, by hex code, by recently used, etc. There is even and
+  editable Favorites list.
+
+- Open URLs using only the keyboard. kitty has a new "hints mode". Press
+  Ctrl+Shift+e and all detected URLs on the screen are highlighted with a key
+  to press to open them. The facility is customizable so you can change
+  what is detected as a URL and which program is used to open it.
+
+- Add an option to change the titlebar color of kitty windows on macOS
+
+- Only consider Emoji characters with default Emoji presentation to be two
+  cells wide. This matches the standard. Also add support for the Unicode Emoji
+  variation presentation selector.
+
+- Prevent video tearing during high speed scrolling by syncing draws
+  to the monitor's refresh rate. There is a new configuration option to
+  control this ``sync_to_monitor``.
+
+- When displaying only a single window, use the default background color of the
+  window (which can be changed via escape codes) as the color for the margin
+  and padding of the window.
+
+- Add some non standard terminfo capabilities used by neovim and tmux.
+
+- Fix large drop in performance when using multiple top-level windows on macOS
+
+- Fix save/restore of window sizes not working correctly.
+
+- Remove option to use system wcwidth(). Now always use a wcwidth() based on
+  the Unicode standard. Only sane way.
+
+- Fix a regression that caused a few ligature glyphs to not render correctly in
+  rare circumstances.
+
+- Browsing the scrollback buffer now happens in an overlay window instead of a
+  new window/tab.
+
+0.7.1 [2018-01-31]
+---------------------------
+
+- Add an option to adjust the width of character cells
+
+- Fix selecting text with the mouse in the scrollback buffer selecting text
+  from the line above the actually selected line
+
+- Fix some italic fonts having the right edge of characters cut-off,
+  unnecessarily
+
+
+0.7.0 [2018-01-24]
+---------------------------
+
+- Allow controlling kitty from the shell prompt/scripts. You can
+  open/close/rename windows and tabs and even send input to specific windows.
+  See the README for details.
+
+- Add option to put tab bar at the top instead of the bottom
+
+- Add option to override the default shell
+
+- Add "Horizontal" and "Vertical" window layouts
+
+- Sessions: Allow setting titles and working directories for individual windows
+
+- Option to copy to clipboard on mouse select
+
+- Fix incorrect reporting of mouse move events when using the SGR protocol
+
+- Make alt+backspace delete the previous word
+
+- Take the mouse wheel multiplier option in to account when generating fake key
+  scroll events
+
+- macOS: Fix closing top-level window does not transfer focus to other
+  top-level windows.
+
+- macOS: Fix alt+arrow keys not working when disabling the macos_option_as_alt
+  config option.
+
+- kitty icat: Workaround for bug in ImageMagick that would cause some images
+  to fail to display at certain sizes.
+
+- Fix rendering of text with ligature fonts that do not use dummy glyphs
+
+- Fix a regression that caused copying of the selection to clipboard to only
+  copy the visible part of the selection
+
+- Fix incorrect handling of some unicode combining marks that are not re-ordered
+
+- Fix handling on non-BMP combining characters
+
+- Drop the dependency on libunistring
+
+
+0.6.1 [2017-12-28]
+---------------------------
+
+- Add an option to fade the text in inactive windows
+
+- Add new actions to open windows/tabs/etc. with the working directory set to
+  the working directory of the current window.
+
+- Automatically adjust cell size when DPI changes, for example when kitty is
+  moved from one monitor to another with a different DPI
+
+- Ensure underlines are rendered even for fonts with very poor metrics
+
+- Fix some emoji glyphs not colored on Linux
+
+- Internal wcwidth() implementation is now auto-generated from the unicode
+  standard database
+
+- Allow configuring the modifiers to use for rectangular selection with the
+  mouse.
+
+- Fix incorrect minimum wayland version in the build script
+
+- Fix a crash when detecting a URL that ends at the end of the line
+
+- Fix regression that broke drawing of hollow cursor when window loses focus
+
+
+0.6.0 [2017-12-18]
+---------------------------
+
+- Support background transparency via the background_opacity option. Provided
+  that your OS/window manager supports transparency, you can now have kitty
+  render pixels that have only the default background color as
+  semi-transparent.
+
+- Support multiple top level (OS) windows. These windows all share the sprite
+  texture cache on the GPU, further reducing overall resource usage. Use
+  the shortcut `ctrl+shift+n` to open a new top-level window.
+
+- Add support for a *daemon* mode using the `--single-instance` command line
+  option. With this option you can have only a single kitty instance running.
+  All future invocations simply open new top-level windows in the existing
+  instance.
+
+- Support colored emoji
+
+- Use CoreText instead of FreeType to render text on macOS
+
+- Support running on the "low power" GPU on dual GPU macOS machines
+
+- Add a new "grid" window layout
+
+- Drop the dependency on glfw (kitty now uses a modified, bundled copy of glfw)
+
+- Add an option to control the audio bell volume on X11 systems
+
+- Add a command line switch to set the name part of the WM_CLASS window
+  property independently.
+
+- Add a command line switch to set the window title.
+
+- Add more options to customize the tab-bar's appearance (font styles and
+  separator)
+
+- Allow drag and drop of files into kitty. On drop kitty will paste the
+  file path to the running program.
+
+- Add an option to control the underline style for URL highlighting on hover
+
+- X11: Set the WINDOWID environment variable
+
+- Fix middle and right buttons swapped when sending mouse events to child
+  processes
+
+- Allow selecting in a rectangle by holding down Ctrl+Alt while dragging with
+  the mouse.
+
+
+0.5.1 [2017-12-01]
+---------------------------
+
+- Add an option to control the thickness of lines in box drawing characters
+
+- Increase max. allowed ligature length to nine characters
+
+- Fix text not vertically centered when adjusting line height
+
+- Fix unicode block characters not being rendered properly
+
+- Fix shift+up/down not generating correct escape codes
+
+- Image display: Fix displaying images taller than two screen heights not
+  scrolling properly
+
+
+0.5.0 [2017-11-19]
+---------------------------
+
+- Add support for ligature fonts such as Fira Code, Hasklig, etc. kitty now
+  uses harfbuzz for text shaping which allow it to support advanced OpenType
+  features such as contextual alternates/ligatures/combining glyphs/etc.
+
+- Make it easy to select fonts by allowing listing of monospace fonts using:
+  kitty list-fonts
+
+- Add an option to have window focus follow mouse
+
+- Add a keyboard shortcut (ctrl+shift+f11) to toggle fullscreen mode
+
+- macOS: Fix handling of option key. It now behaves just like the alt key on
+  Linux. There is an option to make it type unicode characters instead.
+
+- Linux: Add support for startup notification on X11 desktops. kitty will
+  now inform the window manager when its startup is complete.
+
+- Fix shell prompt being duplicated when window is resized
+
+- Fix crash when displaying more than 64 images in the same session
+
+- Add support for colons in SGR color codes. These are generated by some
+  applications such as neovim when they mistakenly identify kitty as a libvte
+  based terminal.
+
+- Fix mouse interaction not working in apps using obsolete mouse interaction
+  protocols
+
+- Linux: no longer require glew as a dependency
+
+
+0.4.2 [2017-10-23]
+---------------------------
+
+- Fix a regression in 0.4.0 that broke custom key mappings
+
+- Fix a regression in 0.4.0 that broke support for non-QWERTY keyboard layouts
+
+- Avoid using threads to reap zombie child processes. Also prevent kitty from
+  hanging if the open program hangs when clicking on a URL.
+
+
+0.4.0 [2017-10-22]
+---------------------------
+
+- Support for drawing arbitrary raster graphics (images) in the terminal via a
+  new graphics protocol. kitty can draw images with full 32-bit color using both
+  ssh connections and files/shared memory (when available) for better
+  performance. The drawing primitives support alpha blending and z-index.
+  Images can be drawn both above and below text. See :doc:`graphics-protocol`.
+  for details.
+
+- Refactor kitty's internals to make it even faster and more efficient. The CPU
+  usage of kitty + X server while doing intensive tasks such as scrolling a
+  file continuously in less has been reduced by 50%. There are now two
+  configuration options ``repaint_delay`` and ``input_delay`` you can use to
+  fine tune kitty's performance vs CPU usage profile. The CPU usage of kitty +
+  X when scrolling in less is now significantly better than most (all?) other
+  terminals. See :doc:`performance`.
+
+- Hovering over URLs with the mouse now underlines them to indicate they
+  can be clicked. Hold down Ctrl+Shift while clicking to open the URL.
+
+- Selection using the mouse is now more intelligent. It does not add
+  blank cells (i.e. cells that have no content) after the end of text in a
+  line to the selection.
+
+- The block cursor in now fully opaque but renders the character under it in
+  the background color, for enhanced visibility.
+
+- Allow combining multiple independent actions into a single shortcut
+
+- Add a new shortcut to pass the current selection to an external program
+
+- Allow creating shortcuts to open new windows running arbitrary commands. You
+  can also pass the current selection to the command as an arguments and the
+  contents of the screen + scrollback buffer as stdin to the command.

docs/conf.py

@@ -0,0 +1,538 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+import os
+import re
+import subprocess
+import sys
+from functools import partial
+
+from docutils import nodes
+from docutils.parsers.rst.roles import set_classes
+from pygments.lexer import RegexLexer, bygroups
+from pygments.token import (
+    Comment, Keyword, Literal, Name, Number, String, Whitespace
+)
+from sphinx import addnodes
+from sphinx.util.logging import getLogger
+
+from kitty.constants import str_version
+
+# config {{{
+# -- Project information -----------------------------------------------------
+
+project = 'kitty'
+copyright = '2018, Kovid Goyal'
+author = 'Kovid Goyal'
+building_man_pages = 'man' in sys.argv
+
+# The short X.Y version
+version = str_version
+# The full version, including alpha/beta/rc tags
+release = str_version
+logger = getLogger(__name__)
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+needs_sphinx = '1.7'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.githubpages',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = [
+    '_build', 'Thumbs.db', '.DS_Store',
+    'generated/cli-*.rst', 'generated/conf-*.rst'
+]
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+rst_prolog = '''
+.. |kitty| replace:: *kitty*
+.. |version| replace:: VERSION
+.. _tarball: https://github.com/kovidgoyal/kitty/releases/download/vVERSION/kitty-VERSION.tar.xz
+.. role:: green
+.. role:: italic
+.. role:: bold
+.. role:: cyan
+.. role:: title
+.. role:: env
+
+'''.replace('VERSION', str_version)
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+html_theme_options = {
+    'logo': 'kitty.png',
+    'show_powered_by': False,
+    'fixed_sidebar': True,
+    'sidebar_collapse': True,
+    'github_button': False,
+    'github_banner': True,
+    'github_user': 'kovidgoyal',
+    'github_repo': 'kitty',
+    # increase contrast of link color with text color
+    'link': '#00587d',
+    'link_hover': 'green',
+}
+
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static', '../logo/kitty.png']
+html_context = {
+    'css_files': ['_static/custom.css']
+}
+html_favicon = '../logo/kitty.png'
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+html_sidebars = {
+    '**': [
+        'about.html',
+        'support.html',
+        'searchbox.html',
+        'localtoc.html',
+        'relations.html',
+    ]
+}
+html_show_sourcelink = False
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('invocation', 'kitty', 'kitty Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'kitty', 'kitty Documentation',
+     author, 'kitty', 'A cross-platform, fast, feature full, GPU based terminal emulator',
+     'Miscellaneous'),
+]
+# }}}
+
+
+# GitHub linking inlne roles {{{
+
+def num_role(which, name, rawtext, text, lineno, inliner, options={}, content=[]):
+    ' Link to a github issue '
+    try:
+        issue_num = int(text)
+        if issue_num <= 0:
+            raise ValueError
+    except ValueError:
+        msg = inliner.reporter.error(
+            'GitHub issue number must be a number greater than or equal to 1; '
+            '"%s" is invalid.' % text, line=lineno)
+        prb = inliner.problematic(rawtext, rawtext, msg)
+        return [prb], [msg]
+    url = f'https://github.com/kovidgoyal/kitty/{which}/{issue_num}'
+    set_classes(options)
+    node = nodes.reference(rawtext, f'#{issue_num}', refuri=url, **options)
+    return [node], []
+
+
+def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
+    ' Link to a github commit '
+    try:
+        commit_id = subprocess.check_output(f'git rev-list --max-count=1 --skip=# {text}'.split()).decode('utf-8').strip()
+    except Exception:
+        msg = inliner.reporter.error(
+            f'GitHub commit id "{text}" not recognized.', line=lineno)
+        prb = inliner.problematic(rawtext, rawtext, msg)
+        return [prb], [msg]
+    url = f'https://github.com/kovidgoyal/kitty/commit/{commit_id}'
+    set_classes(options)
+    short_id = subprocess.check_output(f'git rev-list --max-count=1 --abbrev-commit --skip=# {commit_id}'.split()).decode('utf-8').strip()
+    node = nodes.reference(rawtext, f'commit: {short_id}', refuri=url, **options)
+    return [node], []
+# }}}
+
+
+# Sidebar ToC {{{
+def create_toc(app, pagename):
+    toctree = app.env.get_toc_for(pagename, app.builder)
+    if toctree is not None:
+        subtree = toctree[toctree.first_child_matching_class(nodes.list_item)]
+        bl = subtree.first_child_matching_class(nodes.bullet_list)
+        if bl is None:
+            return  # Empty ToC
+        subtree = subtree[bl]
+        # for li in subtree.traverse(nodes.list_item):
+        #     modify_li(li)
+        # subtree['ids'] = [ID]
+        return app.builder.render_partial(subtree)['fragment']
+
+
+def add_html_context(app, pagename, templatename, context, *args):
+    if 'toc' in context:
+        context['toc'] = create_toc(app, pagename) or context['toc']
+# }}}
+
+
+# CLI docs {{{
+def write_cli_docs(all_kitten_names):
+    from kitty.cli import option_spec_as_rst
+    with open('generated/cli-kitty.rst', 'w') as f:
+        f.write(option_spec_as_rst(appname='kitty').replace(
+            'kitty --to', 'kitty @ --to'))
+    as_rst = partial(option_spec_as_rst, heading_char='_')
+    from kitty.remote_control import global_options_spec, cli_msg, cmap, all_commands
+    with open('generated/cli-kitty-at.rst', 'w') as f:
+        p = partial(print, file=f)
+        p('kitty @\n' + '-' * 80)
+        p('.. program::', 'kitty @')
+        p('\n\n' + as_rst(
+            global_options_spec, message=cli_msg, usage='command ...', appname='kitty @'))
+        from kitty.cmds import cli_params_for
+        for cmd_name in all_commands:
+            func = cmap[cmd_name]
+            p(f'.. _at_{func.name}:\n')
+            p('kitty @', func.name + '\n' + '-' * 120)
+            p('.. program::', 'kitty @', func.name)
+            p('\n\n' + as_rst(*cli_params_for(func)))
+    from kittens.runner import get_kitten_cli_docs
+    for kitten in all_kitten_names:
+        data = get_kitten_cli_docs(kitten)
+        if data:
+            with open(f'generated/cli-kitten-{kitten}.rst', 'w') as f:
+                p = partial(print, file=f)
+                p('.. program::', f'kitty +kitten {kitten}')
+                p('\n\n' + option_spec_as_rst(
+                    data['options'], message=data['help_text'], usage=data['usage'], appname=f'kitty +kitten {kitten}',
+                    heading_char='^'))
+
+# }}}
+
+
+# config file docs {{{
+
+class ConfLexer(RegexLexer):
+    name = 'Conf'
+    aliases = ['conf']
+    filenames = ['*.conf']
+
+    tokens = {
+        'root': [
+            (r'#.*?$', Comment.Single),
+            (r'\s+$', Whitespace),
+            (r'\s+', Whitespace),
+            (r'(include)(\s+)(.+?)$', bygroups(Comment.Preproc, Whitespace, Name.Namespace)),
+            (r'(map)(\s+)(\S+)(\s+)', bygroups(
+                Keyword.Declaration, Whitespace, String, Whitespace), 'action'),
+            (r'(symbol_map)(\s+)(\S+)(\s+)(.+?)$', bygroups(
+                Keyword.Declaration, Whitespace, String, Whitespace, Literal)),
+            (r'([a-zA-Z_0-9]+)(\s+)', bygroups(
+                Name.Variable, Whitespace), 'args'),
+        ],
+        'action': [
+            (r'[a-z_0-9]+$', Name.Function, 'root'),
+            (r'[a-z_0-9]+', Name.Function, 'args'),
+        ],
+        'args': [
+            (r'\s+', Whitespace, 'args'),
+            (r'\b(yes|no)\b$', Number.Bin, 'root'),
+            (r'\b(yes|no)\b', Number.Bin, 'args'),
+            (r'[+-]?[0-9]+\s*$', Number.Integer, 'root'),
+            (r'[+-]?[0-9.]+\s*$', Number.Float, 'root'),
+            (r'[+-]?[0-9]+', Number.Integer, 'args'),
+            (r'[+-]?[0-9.]+', Number.Float, 'args'),
+            (r'#[a-fA-F0-9]{3,6}\s*$', String, 'root'),
+            (r'#[a-fA-F0-9]{3,6}\s*', String, 'args'),
+            (r'.+', String, 'root'),
+        ],
+    }
+
+
+class SessionLexer(RegexLexer):
+    name = 'Session'
+    aliases = ['session']
+    filenames = ['*.session']
+
+    tokens = {
+        'root': [
+            (r'#.*?$', Comment.Single),
+            (r'[a-z][a-z0-9_]+', Name.Function, 'args'),
+        ],
+        'args': [
+            (r'.*?$', Literal, 'root'),
+        ]
+    }
+
+
+def link_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
+    m = re.match(r'(.+)\s+<(.+?)>', text)
+    if m is None:
+        msg = inliner.reporter.error(f'link "{text}" not recognized', line=lineno)
+        prb = inliner.problematic(rawtext, rawtext, msg)
+        return [prb], [msg]
+    text, url = m.group(1, 2)
+    set_classes(options)
+    node = nodes.reference(rawtext, text, refuri=url, **options)
+    return [node], []
+
+
+def expand_opt_references(conf_name, text):
+    conf_name += '.'
+
+    def expand(m):
+        ref = m.group(1)
+        if '<' not in ref and '.' not in ref:
+            full_ref = conf_name + ref
+            return ':opt:`{} <{}>`'.format(ref, full_ref)
+        return m.group()
+
+    return re.sub(r':opt:`(.+?)`', expand, text)
+
+
+opt_aliases = {}
+shortcut_slugs = {}
+
+
+def parse_opt_node(env, sig, signode):
+    """Transform an option description into RST nodes."""
+    count = 0
+    firstname = ''
+    for potential_option in sig.split(', '):
+        optname = potential_option.strip()
+        if count:
+            signode += addnodes.desc_addname(', ', ', ')
+        text = optname.split('.', 1)[-1]
+        signode += addnodes.desc_name(text, text)
+        if not count:
+            firstname = optname
+            signode['allnames'] = [optname]
+        else:
+            signode['allnames'].append(optname)
+            opt_aliases[optname] = firstname
+        count += 1
+    if not firstname:
+        raise ValueError('{} is not a valid opt'.format(sig))
+    return firstname
+
+
+def parse_shortcut_node(env, sig, signode):
+    """Transform a shortcut description into RST nodes."""
+    conf_name, text = sig.split('.', 1)
+    signode += addnodes.desc_name(text, text)
+    return sig
+
+
+def render_conf(conf_name, all_options):
+    from kitty.conf.definition import merged_opts, Option
+    ans = ['.. default-domain:: conf', '']
+    a = ans.append
+    current_group = None
+    all_options = list(all_options)
+    kitty_mod = 'kitty_mod'
+
+    def render_group(group):
+        a('')
+        a(f'.. _conf-{conf_name}-{group.name}:')
+        a('')
+        a(group.short_text)
+        heading_level = '+' if '.' in group.name else '^'
+        a(heading_level * (len(group.short_text) + 20))
+        a('')
+        if group.start_text:
+            a(group.start_text)
+            a('')
+
+    def handle_group_end(group):
+        if group.end_text:
+            a(''), a(current_group.end_text)
+
+    def handle_group(new_group, new_group_is_shortcut=False):
+        nonlocal current_group
+        if new_group is not current_group:
+            if current_group:
+                handle_group_end(current_group)
+            current_group = new_group
+            render_group(current_group)
+
+    def handle_option(i, opt):
+        nonlocal kitty_mod
+        if not opt.long_text or not opt.add_to_docs:
+            return
+        handle_group(opt.group)
+        if opt.name == 'kitty_mod':
+            kitty_mod = opt.defval_as_string
+        mopts = list(merged_opts(all_options, opt, i))
+        a('.. opt:: ' + ', '.join(conf_name + '.' + mo.name for mo in mopts))
+        a('.. code-block:: conf')
+        a('')
+        sz = max(len(x.name) for x in mopts)
+        for mo in mopts:
+            a(('    {:%ds} {}' % sz).format(mo.name, mo.defval_as_string))
+        a('')
+        if opt.long_text:
+            a(expand_opt_references(conf_name, opt.long_text))
+            a('')
+
+    def handle_shortcuts(shortcuts):
+        sc = shortcuts[0]
+        handle_group(sc.group, True)
+        sc_text = f'{conf_name}.{sc.short_text}'
+        a('.. shortcut:: ' + sc_text)
+        shortcuts = [s for s in shortcuts if s.add_to_default]
+        shortcut_slugs[f'{conf_name}.{sc.name}'] = (sc_text, sc.key.replace('kitty_mod', kitty_mod))
+        if shortcuts:
+            a('.. code-block:: conf')
+            a('')
+            for x in shortcuts:
+                if x.add_to_default:
+                    a('    map {} {}'.format(x.key.replace('kitty_mod', kitty_mod), x.action_def))
+        a('')
+        if sc.long_text:
+            a(expand_opt_references(conf_name, sc.long_text))
+            a('')
+
+    for i, opt in enumerate(all_options):
+        if isinstance(opt, Option):
+            handle_option(i, opt)
+        else:
+            handle_shortcuts(opt)
+
+    if current_group:
+        handle_group_end(current_group)
+    return '\n'.join(ans)
+
+
+def process_opt_link(env, refnode, has_explicit_title, title, target):
+    conf_name, opt = target.partition('.')[::2]
+    if not opt:
+        conf_name, opt = 'kitty', conf_name
+    full_name = conf_name + '.' + opt
+    return title, opt_aliases.get(full_name, full_name)
+
+
+def process_shortcut_link(env, refnode, has_explicit_title, title, target):
+    conf_name, slug = target.partition('.')[::2]
+    if not slug:
+        conf_name, slug = 'kitty', conf_name
+    full_name = conf_name + '.' + slug
+    try:
+        target, title = shortcut_slugs[full_name]
+    except KeyError:
+        logger.warning('Unknown shortcut: {}'.format(target), location=refnode)
+    return title, target
+
+
+def write_conf_docs(app, all_kitten_names):
+    app.add_lexer('conf', ConfLexer())
+    app.add_object_type(
+        'opt', 'opt',
+        indextemplate="pair: %s; Config Setting",
+        parse_node=parse_opt_node,
+    )
+    # Warn about opt references that could not be resolved
+    opt_role = app.registry.domain_roles['std']['opt']
+    opt_role.warn_dangling = True
+    opt_role.process_link = process_opt_link
+
+    app.add_object_type(
+        'shortcut', 'sc',
+        indextemplate="pair: %s; Keyboard Shortcut",
+        parse_node=parse_shortcut_node,
+    )
+    sc_role = app.registry.domain_roles['std']['sc']
+    sc_role.warn_dangling = True
+    sc_role.process_link = process_shortcut_link
+
+    def generate_default_config(all_options, name):
+        from kitty.conf.definition import as_conf_file
+        with open(f'generated/conf-{name}.rst', 'w', encoding='utf-8') as f:
+            print('.. highlight:: conf\n', file=f)
+            f.write(render_conf(name, all_options.values()))
+
+        conf_name = re.sub(r'^kitten-', '', name) + '.conf'
+        with open(f'generated/conf/{conf_name}', 'w', encoding='utf-8') as f:
+            text = '\n'.join(as_conf_file(all_options.values()))
+            print(text, file=f)
+
+    from kitty.config_data import all_options
+    generate_default_config(all_options, 'kitty')
+
+    from kittens.runner import get_kitten_conf_docs
+    for kitten in all_kitten_names:
+        all_options = get_kitten_conf_docs(kitten)
+        if all_options:
+            generate_default_config(all_options, f'kitten-{kitten}')
+# }}}
+
+
+def setup(app):
+    os.makedirs('generated/conf', exist_ok=True)
+    from kittens.runner import all_kitten_names
+    all_kitten_names = all_kitten_names()
+    write_cli_docs(all_kitten_names)
+    write_conf_docs(app, all_kitten_names)
+    app.add_lexer('session', SessionLexer())
+    app.add_role('link', link_role)
+    app.add_role('iss', partial(num_role, 'issues'))
+    app.add_role('pull', partial(num_role, 'pull'))
+    app.add_role('commit', commit_role)
+    app.connect('html-page-context', add_html_context)

docs/conf.rst

@@ -0,0 +1,40 @@
+:tocdepth: 2
+
+Configuring kitty
+===============================
+
+.. highlight:: conf
+
+|kitty| is highly customizable, everything from keyboard shortcuts, to painting
+frames-per-second. See below for an overview of all customization
+possibilities.
+
+You can open the config file within kitty by pressing :sc:`edit_config_file`.
+You can also display the current configuration by running ``kitty
+--debug-config``.
+
+.. _confloc:
+
+|kitty| looks for a config file in the OS config directories (usually
+:file:`~/.config/kitty/kitty.conf`) but you can pass a specific path via the
+:option:`kitty --config` option or use the ``KITTY_CONFIG_DIRECTORY``
+environment variable. See the :option:`kitty --config` option for full details.
+
+You can include secondary config files via the :code:`include` directive.  If
+you use a relative path for include, it is resolved with respect to the
+location of the current config file. Note that environment variables are
+expanded, so :code:`${USER}.conf` becomes :file:`name.conf` if
+:code:`USER=name`.  For example::
+
+     include other.conf
+
+
+.. include:: /generated/conf-kitty.rst
+
+
+Sample kitty.conf
+^^^^^^^^^^^^^^^^^^^^^
+
+You can download a sample :file:`kitty.conf` file with all default settings and
+comments describing each setting by clicking: :download:`sample kitty.conf
+</generated/conf/kitty.conf>`.

docs/faq.rst

@@ -0,0 +1,115 @@
+Frequently Asked Questions
+==============================
+
+.. highlight:: sh
+
+.. contents::
+
+Some special symbols are rendered small/truncated in kitty?
+-----------------------------------------------------------------
+
+The number of cells a unicode character takes up are controlled by the unicode
+standard.  All characters are rendered in a single cell unless the unicode
+standard says they should be rendered in two cells. When a symbol does not fit,
+it will either be rescaled to be smaller or truncated (depending on how much
+extra space it needs). This is often different from other terminals which just
+let the character overflow into neighboring cells, which is fine if the
+neighboring cell is empty, but looks terrible if it is not.
+
+Some programs, like powerline, vim with fancy gutter symbols/status-bar, etc.
+misuse unicode characters from the private use area to represent symbols. Often
+these symbols are square and should be rendered in two cells.  However, since
+private use area symbols all have their width set to one in the unicode
+standard, |kitty| renders them either smaller or truncated. The exception is if
+these characters are followed by a space or empty cell in which case kitty
+makes use of the extra cell to render them in two cells.
+
+
+How do I build kitty.app on macOS?
+----------------------------------------
+
+Install `imagemagick`, `optipng` and `librsvg` using `brew` or similar (needed
+for the logo generation step). And run::
+
+    make app
+
+This :file:`kitty.app` unlike the released one does not include its own copy of
+python and the other dependencies. So if you ever un-install/upgrade those dependencies
+you might have to rebuild the app.
+
+Note that the released :file:`kitty.dmg` includes all dependencies, unlike the
+:file:`kitty.app` built above and is built automatically by using the :file:`kitty` branch of
+`build-calibre <https://github.com/kovidgoyal/build-calibre>`_ however, that
+is designed to run on Linux and is not for the faint of heart.
+
+
+Using a color theme with a background color does not work well in vim?
+-----------------------------------------------------------------------
+
+First make sure you have not changed the TERM environment variable, it should
+be ``xterm-kitty``. vim uses *background color erase* even if the terminfo file
+does not contain the ``bce`` capability. This is a bug in vim. You can work around
+it by adding the following to your vimrc::
+
+    let &t_ut=''
+
+See :ref:`here <ext_styles>` for why |kitty| does not support background color erase.
+
+
+I get errors about the terminal being unknown or opening the terminal failing when SSHing into a different computer?
+-----------------------------------------------------------------------------------------------------------------------
+
+This happens because the |kitty| terminfo files are not available on the server.
+You can ssh in using the following command which will automatically copy the
+terminfo files to the server::
+
+    kitty +kitten ssh myserver
+
+If for some reason that does not work (typically because the server is using a
+a non POSIX compliant shell), you can use the following one-liner instead (it
+is slower as it needs to ssh into the server twice, but will work with most
+servers)::
+
+    infocmp xterm-kitty | ssh myserver tic -x -o \~/.terminfo /dev/stdin
+
+Really, the correct solution for this is to convince the OpenSSH maintainers to
+have ssh do this automatically when connecting to a server, so that all
+terminals work transparently.
+
+
+How do I change the colors in a running kitty instance?
+------------------------------------------------------------
+
+You can either use the
+`OSC terminal escape codes <http://invisible-island.net/xterm/ctlseqs/ctlseqs.html#h2-Operating-System-Commands>`_
+to set colors or you can enable :doc:`remote control <remote-control>`
+for |kitty| and use :ref:`at_set-colors`.
+
+
+How do I specify command line options for kitty on macOS?
+---------------------------------------------------------------
+
+Apple does not want you to use command line options with GUI applications. To
+workaround that limitation, |kitty| will read command line options from the file
+:file:`<kitty config dir>/macos-launch-services-cmdline` when it is launched
+from the GUI, i.e. by clicking the |kitty| application icon or using ``open -a kitty``.
+Note that this file is *only read* when running via the GUI.
+
+You can, of course, also run |kitty| from a terminal with command line options, using:
+:file:`/Applications/kitty.app/Contents/MacOS/kitty`.
+
+And within |kitty| itself, you can always run |kitty| using just `kitty` as it
+cleverly adds itself to the ``PATH``.
+
+
+kitty is not able to use my favorite font?
+---------------------------------------------
+
+|kitty| achieves its stellar performance by caching alpha masks of each rendered
+character on the GPU, so that every character needs to be rendered only once.
+This means it is a strictly character cell based display.  As such it can use
+only monospace fonts, since every cell in the grid has to be the same size. If
+your font is not listed in ``kitty list-fonts`` it means that it is not
+monospace. On Linux you can list all monospace fonts with::
+
+    fc-list : family spacing | grep spacing=100

docs/graphics-protocol.rst

@@ -0,0 +1,398 @@
+:tocdepth: 3
+
+The terminal graphics protocol
+==================================
+
+The goal of this specification is to create a flexible and performant protocol
+that allows the program running in the terminal, hereafter called the *client*,
+to render arbitrary pixel (raster) graphics to the screen of the terminal
+emulator. The major design goals are
+
+ * Should not require terminal emulators to understand image formats.
+ * Should allow specifying graphics to be drawn at individual pixel positions.
+ * The graphics should integrate with the text, in particular it should be possible to draw graphics
+   below as well as above the text, with alpha blending. The graphics should also scroll with the text, automatically.
+ * Should use optimizations when the client is running on the same computer as the terminal emulator.
+
+For some discussion regarding the design choices, see `#33
+<https://github.com/kovidgoyal/kitty/issues/33>`_.
+
+To see a quick demo, inside a |kitty| terminal run::
+
+    kitty icat path/to/some/image.png
+
+You can also see a screenshot with more sophisticated features such as
+alpha-blending and text over graphics.
+
+.. image:: https://user-images.githubusercontent.com/1308621/31647475-1188ab66-b326-11e7-8d26-24b937f1c3e8.png
+    :alt: Demo of graphics rendering in kitty
+    :align: center
+
+Some programs that use the kitty graphics protocol:
+
+ * `termpdf <https://github.com/dsanson/termpdf>`_ - a terminal PDF/DJVU/CBR viewer
+ * `ranger <https://github.com/ranger/ranger>`_ - a terminal file manager, with
+   image previews, see this `PR <https://github.com/ranger/ranger/pull/1077>`_
+ * :doc:`kitty-diff <kittens/diff>` - a side-by-side terminal diff program with support for images
+ * `neofetch <https://github.com/dylanaraps/neofetch>`_ - A command line system
+   information tool
+
+
+.. contents::
+
+
+Getting the window size
+-------------------------
+
+In order to know what size of images to display and how to position them, the
+client must be able to get the window size in pixels and the number of cells
+per row and column. This can be done by using the ``TIOCGWINSZ`` ioctl.  Some
+code to demonstrate its use
+
+In C:
+
+.. code-block:: c
+
+    struct ttysize ts;
+    ioctl(0, TIOCGWINSZ, &ts);
+    printf("number of columns: %i, number of rows: %i, screen width: %i, screen height: %i\n", sz.ws_col, sz.ws_row, sz.ws_xpixel, sz.ws_ypixel);
+
+In Python:
+
+.. code-block:: python
+
+    import array, fcntl, termios
+    buf = array.array('H', [0, 0, 0, 0])
+    fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, buf)
+    print('number of columns: {}, number of rows: {}, screen width: {}, screen height: {}'.format(*buf))
+
+Note that some terminals return ``0`` for the width and height values. Such
+terminals should be modified to return the correct values.  Examples of
+terminals that return correct values: ``kitty, xterm``
+
+You can also use the *CSI t* escape code to get the screen size. Send
+``<ESC>[14t`` to *stdout* and kitty will reply on *stdin* with
+``<ESC>[4;<height>;<width>t`` where *height* and *width* are the window size in
+pixels. This escape code is supported in many terminals, not just kitty.
+
+The graphics escape code
+---------------------------
+
+All graphics escape codes are of the form::
+
+    <ESC>_G<control data>;<payload><ESC>\
+
+This is a so-called *Application Programming Command (APC)*. Most terminal
+emulators ignore APC codes, making it safe to use.
+
+The control data is a comma-separated list of ``key=value`` pairs.  The payload
+is arbitrary binary data, base64-encoded to prevent interoperation problems
+with legacy terminals that get confused by control codes within an APC code.
+The meaning of the payload is interpreted based on the control data.
+
+The first step is to transmit the actual image data.
+
+Transferring pixel data
+--------------------------
+
+The first consideration when transferring data between the client and the
+terminal emulator is the format in which to do so. Since there is a vast and
+growing number of image formats in existence, it does not make sense to have
+every terminal emulator implement support for them. Instead, the client should
+send simple pixel data to the terminal emulator. The obvious downside to this
+is performance, especially when the client is running on a remote machine.
+Techniques for remedying this limitation are discussed later. The terminal
+emulator must understand pixel data in three formats, 24-bit RGB, 32-bit RGBA and
+PNG. This is specified using the ``f`` key in the control data. ``f=32`` (which is the
+default) indicates 32-bit RGBA data and ``f=24`` indicates 24-bit RGB data and ``f=100``
+indicates PNG data. The PNG format is supported for convenience and a compact way
+of transmitting paletted images.
+
+RGB and RGBA data
+~~~~~~~~~~~~~~~~~~~
+
+In these formats the pixel data is stored directly as 3 or 4 bytes per pixel, respectively.
+When specifying images in this format, the image dimensions **must** be sent in the control data.
+For example::
+
+    <ESC>_Gf=24,s=10,v=20;<payload><ESC>\
+
+Here the width and height are specified using the ``s`` and ``v`` keys respectively. Since
+``f=24`` there are three bytes per pixel and therefore the pixel data must be ``3 * 10 * 20 = 600``
+bytes.
+
+PNG data
+~~~~~~~~~~~~~~~
+
+In this format any PNG image can be transmitted directly.  For example::
+
+    <ESC>_Gf=100;<payload><ESC>\
+
+
+The PNG format is specified using the ``f=100`` key. The width and height of
+the image will be read from the PNG data itself. Note that if you use both PNG and
+compression, then you must provide the ``S`` key with the size of the PNG data.
+
+
+Compression
+~~~~~~~~~~~~~
+
+The client can send compressed image data to the terminal emulator, by specifying the
+``o`` key. Currently, only zlib based deflate compression is supported, which is specified using
+``o=z``. For example::
+
+    <ESC>_Gf=24,s=10,v=20,o=z;<payload><ESC>\
+
+This is the same as the example from the RGB data section, except that the
+payload is now compressed using deflate. The terminal emulator will decompress
+it before rendering. You can specify compression for any format. The terminal
+emulator will decompress before interpreting the pixel data.
+
+
+The transmission medium
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The transmission medium is specified using the ``t`` key. The ``t`` key defaults to ``d``
+and can take the values:
+
+==================    ============
+Value of `t`          Meaning
+==================    ============
+``d``                 Direct (the data is transmitted within the escape code itself)
+``f``                 A simple file
+``t``                 A temporary file, the terminal emulator will delete the file after reading the pixel data
+``s``                 A `POSIX shared memory object <http://man7.org/linux/man-pages/man7/shm_overview.7.html>`_.
+                      The terminal emulator will delete it after reading the pixel data
+==================    ============
+
+Local client
+^^^^^^^^^^^^^^
+
+First let us consider the local client techniques (files and shared memory). Some examples::
+
+    <ESC>_Gf=100,t=f;<encoded /path/to/file.png><ESC>\
+
+Here we tell the terminal emulator to read PNG data from the specified file of
+the specified size::
+
+    <ESC>_Gs=10,v=2,t=s,o=z;<encoded /some-shared-memory-name><ESC>\
+
+Here we tell the terminal emulator to read compressed image data from
+the specified shared memory object.
+
+The client can also specify a size and offset to tell the terminal emulator
+to only read a part of the specified file. The is done using the ``S`` and ``O``
+keys respectively. For example::
+
+    <ESC>_Gs=10,v=2,t=s,S=80,O=10;<encoded /some-shared-memory-name><ESC>\
+
+This tells the terminal emulator to read ``80`` bytes starting from the offset ``10``
+inside the specified shared memory buffer.
+
+
+Remote client
+^^^^^^^^^^^^^^^^
+
+Remote clients, those that are unable to use the filesystem/shared memory to
+transmit data, must send the pixel data directly using escape codes. Since
+escape codes are of limited maximum length, the data will need to be chunked up
+for transfer. This is done using the ``m`` key. The pixel data must first be
+base64 encoded then chunked up into chunks no larger than ``4096`` bytes. The client
+then sends the graphics escape code as usual, with the addition of an ``m`` key that
+must have the value ``1`` for all but the last chunk, where it must be ``0``. For example,
+if the data is split into three chunks, the client would send the following
+sequence of escape codes to the terminal emulator::
+
+    <ESC>_Gs=100,v=30,m=1;<encoded pixel data first chunk><ESC>\
+    <ESC>_Gm=1;<encoded pixel data second chunk><ESC>\
+    <ESC>_Gm=0;<encoded pixel data last chunk><ESC>\
+
+Note that only the first escape code needs to have the full set of control
+codes such as width, height, format etc. Subsequent chunks must have
+only the ``m`` key. The client **must** finish sending all chunks for a single image
+before sending any other graphics related escape codes.
+
+
+Detecting available transmission mediums
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Since a client has no a-priori knowledge of whether it shares a filesystem/shared memory
+with the terminal emulator, it can send an id with the control data, using the ``i`` key
+(which can be an arbitrary positive integer up to 4294967295, it must not be zero).
+If it does so, the terminal emulator will reply after trying to load the image, saying
+whether loading was successful or not. For example::
+
+    <ESC>_Gi=31,s=10,v=2,t=s;<encoded /some-shared-memory-name><ESC>\
+
+to which the terminal emulator will reply (after trying to load the data)::
+
+    <ESC>_Gi=31;error message or OK<ESC>\
+
+Here the ``i`` value will be the same as was sent by the client in the original
+request.  The message data will be a ASCII encoded string containing only
+printable characters and spaces. The string will be ``OK`` if reading the pixel
+data succeeded or an error message.
+
+Sometimes, using an id is not appropriate, for example, if you do not want to
+replace a previously sent image with the same id, or if you are sending a dummy
+image and do not want it stored by the terminal emulator. In that case, you can
+use the *query action*, set ``a=q``. Then the terminal emulator will try to load
+the image and respond with either OK or an error, as above, but it will not
+replace an existing image with the same id, nor will it store the image.
+
+
+Display images on screen
+-----------------------------
+
+Every transmitted image can be displayed an arbitrary number of times on the
+screen, in different locations, using different parts of the source image, as
+needed. You can either simultaneously transmit and display an image using the
+action ``a=T``, or first transmit the image with a id, such as ``i=10`` and then display
+it with ``a=p,i=10`` which will display the previously transmitted image at the current
+cursor position. When specifying an image id, the terminal emulator will reply with an
+acknowledgement code, which will be either::
+
+    <ESC>_Gi=<id>;OK<ESC>\
+
+when the image referred to by id was found, or::
+
+    <ESC>_Gi=<id>;ENOENT:<some detailed error msg><ESC>\
+
+when the image with the specified id was not found. This is similar to the
+scheme described above for querying available transmission media, except that
+here we are querying if the image with the specified id is available or needs to
+be re-transmitted.
+
+Controlling displayed image layout
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The image is rendered at the current cursor position, from the upper left corner of
+the current cell. You can also specify extra ``X=3`` and ``Y=4`` pixel offsets to display from
+a different origin within the cell. Note that the offsets must be smaller that the size of the cell.
+
+By default, the entire image will be displayed (images wider than the available
+width will be truncated on the right edge). You can choose a source rectangle (in pixels)
+as the part of the image to display. This is done with the keys: ``x, y, w, h`` which specify
+the top-left corner, width and height of the source rectangle.
+
+You can also ask the terminal emulator to display the image in a specified rectangle
+(num of columns / num of lines), using the control codes ``c,r``. ``c`` is the number of columns
+and `r` the number of rows. The image will be scaled (enlarged/shrunk) as needed to fit
+the specified area. Note that if you specify a start cell offset via the ``X,Y`` keys, it is not
+added to the number of rows/columns.
+
+Finally, you can specify the image *z-index*, i.e. the vertical stacking order. Images
+placed in the same location with different z-index values will be blended if
+they are semi-transparent. You can specify z-index values using the ``z`` key.
+Negative z-index values mean that the images will be drawn under the text. This
+allows rendering of text on top of images.
+
+Deleting images
+---------------------
+
+Images can be deleted by using the delete action ``a=d``. If specified without any
+other keys, it will delete all images visible on screen. To delete specific images,
+use the `d` key as described in the table below. Note that each value of d has
+both a lowercase and an uppercase variant. The lowercase variant only deletes the
+images without necessarily freeing up the stored image data, so that the images can be
+re-displayed without needing to resend the data. The uppercase variants will delete
+the image data as well, provided that the image is not referenced elsewhere, such as in the
+scrollback buffer. The values of the ``x`` and ``y`` keys are the same as cursor positions (i.e.
+``x=1, y=1`` is the top left cell).
+
+=================    ============
+Value of ``d``       Meaning
+=================    ============
+``a`` or ``A``       Delete all images visible on screen
+``i`` or ``I``       Delete all images with the specified id, specified using the ``i`` key.
+``c`` or ``C``       Delete all images that intersect with the current cursor position.
+``p`` or ``P``       Delete all images that intersect a specific cell, the cell is specified using the ``x`` and ``y`` keys
+``q`` or ``Q``       Delete all images that intersect a specific cell having a specific z-index. The cell and z-index is specified using the ``x``, ``y`` and ``z`` keys.
+``x`` or ``X``       Delete all images that intersect the specified column, specified using the ``x`` key.
+``y`` or ``Y``       Delete all images that intersect the specified row, specified using the ``y`` key.
+``z`` or ``Z``       Delete all images that have the specified z-index, specified using the ``z`` key.
+=================    ============
+
+
+
+Some examples::
+
+    <ESC>_Ga=d<ESC>\         # delete all visible images
+    <ESC>_Ga=d,i=10<ESC>\    # delete the image with id=10
+    <ESC>_Ga=Z,z=-1<ESC>\    # delete the images with z-index -1, also freeing up image data
+    <ESC>_Ga=P,x=3,y=4<ESC>\ # delete all images that intersect the cell at (3, 4)
+
+Image persistence and storage quotas
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to avoid *Denial-of-Service* attacks, terminal emulators should have a
+maximum storage quota for image data. It should allow at least a few full
+screen images.  For example the quota in kitty is 320MB per buffer. When adding
+a new image, if the total size exceeds the quota, the terminal emulator should
+delete older images to make space for the new one.
+
+
+Control data reference
+---------------------------
+
+The table below shows all the control data keys as well as what values they can
+take, and the default value they take when missing. All integers are 32-bit.
+
+=======  ====================  =========  =================
+Key      Value                 Default    Description
+=======  ====================  =========  =================
+``a``    Single character.     ``t``      The overall action this graphics command is performing.
+         ``(t, T, q, p, d)``
+**Keys for image transmission**
+-----------------------------------------------------------
+``f``    Positive integer.     ``32``     The format in which the image data is sent.
+         ``(24, 32, 100)``.
+``t``    Single character.     ``d``      The transmission medium used.
+         ``(d, f, t, s)``.
+``s``    Positive integer.     ``0``      The width of the image being sent.
+``v``    Positive integer.     ``0``      The height of the image being sent.
+``S``    Positive integer.     ``0``      The size of data to read from a file.
+``O``    Positive integer.     ``0``      The offset from which to read data from a file.
+``i``    Positive integer.
+         ``(0 - 4294967295)``  ``0``      The image id
+``o``    Single character.     ``null``   The type of data compression.
+         ``only z``
+``m``    zero or one           ``0``      Whether there is more chunked data available.
+**Keys for image display**
+-----------------------------------------------------------
+``x``    Positive integer      ``0``      The left edge (in pixels) of the image area to display
+``y``    Positive integer      ``0``      The top edge (in pixels) of the image area to display
+``w``    Positive integer      ``0``      The width (in pixels) of the image area to display. By default, the entire width is used.
+``h``    Positive integer      ``0``      The height (in pixels) of the image area to display. By default, the entire height is used
+``X``    Positive integer      ``0``      The x-offset within the first cell at which to start displaying the image
+``Y``    Positive integer      ``0``      The y-offset within the first cell at which to start displaying the image
+``c``    Positive integer      ``0``      The number of columns to display the image over
+``r``    Positive integer      ``0``      The number of rows to display the image over
+``z``    Integer               ``0``      The *z-index* vertical stacking order of the image
+**Keys for deleting images**
+-----------------------------------------------------------
+``d``    Single character.     ``a``      What to delete.
+         ``(a, A, c, C, i,
+         I, p, P, q, Q, x, X,
+         y, Y, z, Z)``.
+=======  ====================  =========  =================
+
+
+Interaction with other terminal actions
+--------------------------------------------
+
+When resetting the terminal, all images that are visible on the screen must be
+cleared.  When switching from the main screen to the alternate screen buffer
+(1049 private mode) all images in the alternate screen must be cleared, just as
+all text is cleared. The clear screen escape code (usually ``<ESC>[2J``) should
+also clear all images. This is so that the clear command works.
+
+The other commands to erase text must have no effect on graphics.
+The dedicated delete graphics commands must be used for those.
+
+When scrolling the screen (such as when using index cursor movement commands,
+or scrolling through the history buffer), images must be scrolled along with
+text. When page margins are defined and the index commands are used, only
+images that are entirely within the page area (between the margins) must be
+scrolled. When scrolling them would cause them to extend outside the page area,
+they must be clipped.

docs/index.rst

@@ -0,0 +1,397 @@
+:tocdepth: 2
+
+==========================================================
+kitty - the fast, featureful, GPU based terminal emulator
+==========================================================
+
+.. container:: major-features
+
+    * Offloads rendering to the GPU for :doc:`lower system load <performance>` and
+      buttery smooth scrolling.  Uses threaded rendering to minimize input latency.
+
+    * Supports all modern terminal features: :doc:`graphics (images)
+      <graphics-protocol>`, unicode, true-color, OpenType ligatures, mouse
+      protocol, focus tracking, `bracketed paste
+      <https://cirw.in/blog/bracketed-paste>`_ and several :doc:`new terminal
+      protocol extensions <protocol-extensions>`.
+
+    * Supports tiling multiple terminal windows side by side in different
+      :ref:`layouts <layouts>` without needing to use an extra program like tmux
+
+    * Can be :doc:`controlled from scripts or the shell prompt <remote-control>`,
+      even over SSH.
+
+    * Has a framework for :ref:`kittens`, small terminal programs that can be used to
+      extend |kitty|'s functionality.  For example, they are used for
+      :doc:`Unicode input <kittens/unicode-input>`, :doc:`Hints <kittens/hints>` and
+      :doc:`Side-by-side diff <kittens/diff>`.
+
+    * Supports :ref:`startup sessions <sessions>` which allow you to specify
+      the window/tab layout, working directories and programs to run on startup.
+
+    * Cross-platform: |kitty| works on Linux and macOS, but because it uses only
+      OpenGL for rendering, it should be trivial to port to other Unix-like platforms.
+
+    * Allows you to open :ref:`the scrollback buffer <scrollback>` in a
+      separate window using arbitrary programs of your choice. This is useful for
+      browsing the history comfortably in a pager or editor.
+
+
+.. figure:: screenshots/screenshot.png
+    :alt: Screenshot, showing three programs in the 'Tall' layout
+    :align: center
+    :scale: 100%
+
+    Screenshot, showing vim, tig and git running in |kitty| with the 'Tall' layout
+
+
+.. _quickstart:
+
+Quickstart
+--------------
+
+Pre-built binaries of |kitty| are available for both macOS and Linux.
+See the :doc:`binary install instructions </binary>`.
+
+If you are on Linux, you can also use your distribution's |kitty| package.
+|kitty| packages are available for:
+`Debian <https://packages.debian.org/buster/kitty>`_,
+`openSUSE <https://build.opensuse.org/package/show/X11:terminals/kitty>`_,
+`Arch Linux <https://www.archlinux.org/packages/community/x86_64/kitty/>`_,
+`NixOS <https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/kitty/default.nix>`_,
+`Gentoo <https://packages.gentoo.org/packages/x11-terms/kitty>`_,
+`Fedora <https://copr.fedorainfracloud.org/coprs/oleastre/kitty-terminal/>`_.
+
+See :doc:`Configuring kitty <conf>` for help on configuring |kitty| and
+:doc:`Invocation <invocation>` for the command line arguments |kitty| supports.
+
+
+.. contents::
+
+
+Design philosophy
+-------------------
+
+|kitty| is designed for power keyboard users. To that end all its controls
+work with the keyboard (although it fully supports mouse interactions as
+well). Its configuration is a simple, human editable, single file for
+easy reproducibility (I like to store configuration in source control).
+
+The code in |kitty| is designed to be simple, modular and hackable. It is
+written in a mix of C (for performance sensitive parts) and Python (for
+easy hackability of the UI). It does not depend on any large and complex
+UI toolkit, using only OpenGL for rendering everything.
+
+Finally, |kitty| is designed from the ground up to support all modern
+terminal features, such as unicode, true color, bold/italic fonts, text
+formatting, etc. It even extends existing text formatting escape codes,
+to add support for features not available elsewhere, such as colored and
+styled (curly) underlines. One of the design goals of |kitty| is to be
+easily extensible so that new features can be added in the future with
+relatively less effort.
+
+Tabs and Windows
+-------------------
+
+|kitty| is capable of running multiple programs organized into tabs and
+windows. The top level of organization is the *Tab*. Each tab consists
+of one or more *windows*. The windows can be arranged in multiple
+different layouts, like windows are organized in a tiling window
+manager. The keyboard controls (which are all customizable) for tabs and
+windows are:
+
+Scrolling
+~~~~~~~~~~~~~~
+
+========================    =======================
+Action                      Shortcut
+========================    =======================
+Scroll line up              :sc:`scroll_line_up`
+Scroll line down            :sc:`scroll_line_down`
+Scroll page up              :sc:`scroll_page_up`
+Scroll page down            :sc:`scroll_page_down`
+Scroll to top               :sc:`scroll_home`
+Scroll to bottom            :sc:`scroll_end`
+========================    =======================
+
+Tabs
+~~~~~~~~~~~
+
+========================    =======================
+Action                      Shortcut
+========================    =======================
+New tab                     :sc:`new_tab`
+Close tab                   :sc:`close_tab`
+Next tab                    :sc:`next_tab` (also :kbd:`control+tab` on macOS)
+Previous tab                :sc:`previous_tab`
+Next layout                 :sc:`next_layout`
+Move tab forward            :sc:`move_tab_forward`
+Move tab backward           :sc:`move_tab_backward`
+Set tab title               :sc:`set_tab_title`
+========================    =======================
+
+
+Windows
+~~~~~~~~~~~~~~~~~~
+
+========================    =======================
+Action                      Shortcut
+========================    =======================
+New window                  :sc:`new_window`
+New OS window               :sc:`new_os_window` (also :kbd:`⌘+n` on macOS)
+Close window                :sc:`close_window`
+Next window                 :sc:`next_window`
+Previous window             :sc:`previous_window`
+Move window forward         :sc:`move_window_forward`
+Move window backward        :sc:`move_window_backward`
+Move window to top          :sc:`move_window_to_top`
+Focus specific window       :sc:`first_window`, :sc:`second_window` ... :sc:`tenth_window`
+                            (clockwise from the top-left)
+========================    =======================
+
+
+Other keyboard shortcuts
+----------------------------------
+
+==================================  =======================
+Action                              Shortcut
+==================================  =======================
+Copy to clipboard                   :sc:`copy_to_clipboard` (also :kbd:`⌘+c` on macOS)
+Paste from clipboard                :sc:`paste_from_clipboard` (also :kbd:`⌘+v` on macOS)
+Paste from selection                :sc:`paste_from_selection`
+Increase font size                  :sc:`increase_font_size`
+Decrease font size                  :sc:`decrease_font_size`
+Restore font size                   :sc:`reset_font_size`
+Toggle fullscreen                   :sc:`toggle_fullscreen`
+Input unicode character             :sc:`input_unicode_character`
+Click URL using the keyboard        :sc:`open_url`
+Reset the terminal                  :sc:`reset_terminal`
+Pass current selection to program   :sc:`pass_selection_to_program`
+Edit |kitty| config file            :sc:`edit_config_file`
+Open a |kitty| shell                :sc:`kitty_shell`
+Increase background opacity         :sc:`increase_background_opacity`
+Decrease background opacity         :sc:`decrease_background_opacity`
+Full background opacity             :sc:`full_background_opacity`
+Reset background opacity            :sc:`reset_background_opacity`
+==================================  =======================
+
+
+.. _layouts:
+
+Layouts
+----------
+
+Currently, there are five layouts available,
+
+* **Stack** -- Only a single maximized window is shown at a time
+* **Tall** -- One window is shown full height on the left, the rest of the windows are shown one below the other on the right
+* **Fat** -- One window is shown full width on the top, the rest of the windows are shown side-by-side on the bottom
+* **Grid** -- All windows are shown in a grid
+* **Horizontal** -- All windows are shown side-by-side
+* **Vertical** -- All windows are shown one below the other
+
+You can switch between layouts using the :sc:`next_layout` key combination. You can
+also create shortcuts to select particular layouts, and choose which layouts
+you want to enable/disable, see :ref:`conf-kitty-shortcuts.layout` for examples.
+
+You can resize windows inside layouts. Press :sc:`start_resizing_window` to
+enter resizing mode and follow the on-screen instructions.  In a given window
+layout only some operations may be possible for a particular window. For
+example, in the Tall layout you can make the first window wider/narrower, but
+not taller/shorter. Note that what you are resizing is actually not a window,
+but a row/column in the layout, all windows in that row/column will be resized.
+
+Some layouts take options to control their behavior. For example, the ``fat``
+and ``tall`` layouts accept the ``bias`` option to control how the available
+space is split up. To specify the option, in :opt:`kitty.conf <enabled_layouts>` use::
+
+    enabled_layouts tall:bias=70
+
+This will make the tall window occupy ``70%`` of available width. ``bias`` can be
+any number between 10 and 90.
+
+Writing a new layout only requires about a hundred lines of code, so if there
+is some layout you want, take a look at `layout.py
+<https://github.com/kovidgoyal/kitty/blob/master/kitty/layout.py>`_  and submit
+a pull request!
+
+.. _kittens:
+
+Kittens
+------------------
+
+|kitty| has a framework for easily creating terminal programs that make use of
+its advanced features. These programs are called kittens. They are used both
+to add features to |kitty| itself and to create useful standalone programs.
+Some prominent kittens:
+
+:doc:`icat <kittens/icat>`
+    Display images in the terminal
+
+
+:doc:`diff <kittens/diff>`
+    A fast, side-by-side diff for the terminal with syntax highlighting and
+    images
+
+
+:doc:`Unicode Input <kittens/unicode-input>`
+    Easily input arbitrary unicode characters in |kitty| by name or hex code.
+
+
+:doc:`Hints <kittens/hints>`
+    Select and open/paste/insert arbitrary text snippets such as URLs,
+    filenames, words, lines, etc from the terminal screen.
+
+
+:doc:`Panel <kittens/panel>`
+    Draw a GPU accelerated dock panel on your desktop showing the output
+    from an arbitrary terminal program.
+
+:doc:`Clipboard <kittens/clipboard>`
+    Copy/paste to the clipboard from shell scripts, even over SSH.
+
+
+Configuring kitty
+-------------------
+
+|kitty| is highly configurable, everything from keyboard shortcuts to
+painting frames-per-second. For details and a sample :file:`kitty.conf`,
+see the :doc:`configuration docs <conf>`.
+
+.. _sessions:
+
+Remote control
+------------------
+
+|kitty| has a very powerful system that allows you to control it from the
+:doc:`shell prompt, even over SSH <remote-control>`. You can change colors,
+fonts, open new windows, tabs, set their titles, change window layout, get text
+from one window and send text to another, etc, etc. The possibilities are
+endless. See the :doc:`tutorial <remote-control>` to get started.
+
+Startup Sessions
+------------------
+
+You can control the tabs, window layout, working directory, startup programs,
+etc. by creating a "session" file and using the :option:`kitty --session`
+command line flag or the :opt:`startup_session` option in :file:`kitty.conf`.
+For example:
+
+.. code-block:: session
+
+    # Set the window layout for the current tab
+    layout tall
+    # Set the working directory for windows in the current tab
+    cd ~
+    # Create a window and run the specified command in it
+    launch zsh
+    # Create a window with some environment variables set and run
+    # vim in it
+    launch env FOO=BAR vim
+    # Set the title for the next window
+    title Chat with x
+    launch irssi --profile x
+
+    # Create a new tab (the part after new_tab is the optional tab
+    # name which will be displayed in the tab bar, if omitted, the
+    # title of the active window will be used instead)
+    new_tab my tab
+    cd ~/somewhere
+    # Set the layouts allowed in this tab
+    enabled_layouts tall, stack
+    # Set the current layout
+    layout stack
+    launch zsh
+    # Make the current window the active (focused) window
+    focus
+    launch emacs
+
+
+Mouse features
+-------------------
+
+* You can also hold down :kbd:`ctrl+shift` and click on a URL to open it in a browser.
+* You can double click to select a word and triple click to select a line.
+* You can right click to extend a previous selection
+* You can hold down :kbd:`ctrl+alt` and drag with the mouse to select in
+  columns
+
+
+Font control
+-----------------
+
+|kitty| has extremely flexible and powerful font selection features. You can
+specify individual families for the regular, bold, italic and bold+italic
+fonts. You can even specify specific font families for specific ranges of
+unicode characters. This allows precise control over text rendering. It can
+come in handy for applications like powerline, without the need to use patched
+fonts. See the various font related configuration directives in
+:ref:`conf-kitty-fonts`.
+
+
+.. _scrollback:
+
+The scrollback buffer
+-----------------------
+
+|kitty| supports scrolling back to view history, just like most terminals. You
+can use either keyboard shortcuts or the mouse scroll wheel to do so.  However,
+|kitty| has an extra, neat feature. Sometimes you need to explore the
+scrollback buffer in more detail, maybe search for some text or refer to it
+side-by-side while typing in a follow-up command. |kitty| allows you to do this
+by pressing the :sc:`show_scrollback` key-combination, which will open the
+scrollback buffer in your favorite pager program (which is ``less`` by default).
+Colors and text formatting are preserved. You can explore the scrollback buffer
+comfortably within the pager.
+
+
+Frequently Asked Questions
+---------------------------------
+
+The list of Frequently Asked Questions (*FAQ*) is :doc:`available here <faq>`.
+
+.. _completion:
+
+Completion for kitty
+---------------------------------
+
+|kitty| comes with completion for the ``kitty`` command for popular shells.
+
+
+bash
+~~~~~~~~
+
+Add the following to your :file:`~/.bashrc`
+
+.. code-block:: sh
+
+   source <(kitty + complete setup bash)
+
+
+zsh
+~~~~~~~~~
+
+Add the following to your :file:`~/.zshrc`
+
+.. code-block:: sh
+
+    autoload -Uz compinit
+    compinit
+    # Completion for kitty
+    kitty + complete setup zsh | source /dev/stdin
+
+The important thing above is to make sure the call to |kitty| to load the zsh
+completions happens after the call to :file:`compinit`.
+
+
+Changelog
+------------------
+
+See :doc:`changelog`.
+
+.. toctree::
+    :hidden:
+    :glob:
+
+    *
+    kittens/*

docs/installer.py

@@ -0,0 +1,241 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import (
+    absolute_import, division, print_function, unicode_literals
+)
+
+import atexit
+import json
+import os
+import platform
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+
+py3 = sys.version_info[0] > 2
+is64bit = platform.architecture()[0] == '64bit'
+is_macos = 'darwin' in sys.platform.lower()
+
+try:
+    __file__
+    from_file = True
+except NameError:
+    from_file = False
+
+if py3:
+    unicode = str
+    raw_input = input
+    import urllib.request as urllib
+
+    def encode_for_subprocess(x):
+        return x
+else:
+    from future_builtins import map
+    import urllib2 as urllib
+
+    def encode_for_subprocess(x):
+        if isinstance(x, unicode):
+            x = x.encode('utf-8')
+        return x
+
+
+def run(*args):
+    if len(args) == 1:
+        args = shlex.split(args[0])
+    args = list(map(encode_for_subprocess, args))
+    ret = subprocess.Popen(args).wait()
+    if ret != 0:
+        raise SystemExit(ret)
+
+
+class Reporter:  # {{{
+
+    def __init__(self, fname):
+        self.fname = fname
+        self.last_percent = 0
+
+    def __call__(self, blocks, block_size, total_size):
+        percent = (blocks*block_size)/float(total_size)
+        report = '\rDownloaded {:.1%}         '.format(percent)
+        if percent - self.last_percent > 0.05:
+            self.last_percent = percent
+            print(report, end='')
+            sys.stdout.flush()
+# }}}
+
+
+def get_latest_release_data():
+    print('Checking for latest release on GitHub...')
+    req = urllib.Request('https://api.github.com/repos/kovidgoyal/kitty/releases/latest', headers={'Accept': 'application/vnd.github.v3+json'})
+    res = urllib.urlopen(req).read().decode('utf-8')
+    data = json.loads(res)
+    html_url = data['html_url'].replace('/tag/', '/download/').rstrip('/')
+    for asset in data.get('assets', ()):
+        name = asset['name']
+        if is_macos:
+            if name.endswith('.dmg'):
+                return html_url + '/' + name, asset['size']
+        else:
+            if name.endswith('.txz'):
+                if is64bit:
+                    if name.endswith('-x86_64.txz'):
+                        return html_url + '/' + name, asset['size']
+                else:
+                    if name.endswith('-i686.txz'):
+                        return html_url + '/' + name, asset['size']
+    raise SystemExit('Failed to find the installer package on github')
+
+
+def do_download(url, size, dest):
+    print('Will download and install', os.path.basename(dest))
+    reporter = Reporter(os.path.basename(dest))
+
+    # Get content length and check if range is supported
+    rq = urllib.urlopen(url)
+    headers = rq.info()
+    sent_size = int(headers['content-length'])
+    if sent_size != size:
+        raise SystemExit('Failed to download from {} Content-Length ({}) != {}'.format(url, sent_size, size))
+    with open(dest, 'wb') as f:
+        while f.tell() < size:
+            raw = rq.read(8192)
+            if not raw:
+                break
+            f.write(raw)
+            reporter(f.tell(), 1, size)
+    rq.close()
+    if os.path.getsize(dest) < size:
+        raise SystemExit('Download failed, try again later')
+    print('\rDownloaded {} bytes'.format(os.path.getsize(dest)))
+
+
+def clean_cache(cache, fname):
+    for x in os.listdir(cache):
+        if fname not in x:
+            os.remove(os.path.join(cache, x))
+
+
+def download_installer(url, size):
+    fname = url.rpartition('/')[-1]
+    tdir = tempfile.gettempdir()
+    cache = os.path.join(tdir, 'kitty-installer-cache')
+    if not os.path.exists(cache):
+        os.makedirs(cache)
+    clean_cache(cache, fname)
+    dest = os.path.join(cache, fname)
+    if os.path.exists(dest) and os.path.getsize(dest) == size:
+        print('Using previously downloaded', fname)
+        return dest
+    if os.path.exists(dest):
+        os.remove(dest)
+    do_download(url, size, dest)
+    return dest
+
+
+def macos_install(dmg, dest='/Applications', launch=True):
+    mp = tempfile.mkdtemp()
+    atexit.register(shutil.rmtree, mp)
+    run('hdiutil', 'attach', dmg, '-mountpoint', mp)
+    try:
+        os.chdir(mp)
+        app = 'kitty.app'
+        d = os.path.join(dest, app)
+        if os.path.exists(d):
+            shutil.rmtree(d)
+        dest = os.path.join(dest, app)
+        run('ditto', '-v', app, dest)
+        print('Successfully installed kitty into', dest)
+        if launch:
+            run('open', dest)
+    finally:
+        os.chdir('/')
+        run('hdiutil', 'detach', mp)
+
+
+def linux_install(installer, dest=os.path.expanduser('~/.local'), launch=True):
+    dest = os.path.join(dest, 'kitty.app')
+    if os.path.exists(dest):
+        shutil.rmtree(dest)
+    os.makedirs(dest)
+    print('Extracting tarball...')
+    run('tar', '-C', dest, '-xJof', installer)
+    print('kitty successfully installed to', dest)
+    kitty = os.path.join(dest, 'bin', 'kitty')
+    print('Use', kitty, 'to run kitty')
+    if launch:
+        run(kitty, '--detach')
+
+
+def main(dest=None, launch=True, installer=None):
+    if not dest:
+        if is_macos:
+            dest = '/Applications'
+        else:
+            dest = os.path.expanduser('~/.local')
+    machine = os.uname()[4]
+    if machine and machine.lower().startswith('arm'):
+        raise SystemExit(
+            'You are running on an ARM system. The kitty binaries are only'
+            ' available for x86 systems. You will have to build from'
+            ' source.')
+    if not installer:
+        url, size = get_latest_release_data()
+        installer = download_installer(url, size)
+    else:
+        installer = os.path.abspath(installer)
+        if not os.access(installer, os.R_OK):
+            raise SystemExit('Could not read from: {}'.format(installer))
+    if is_macos:
+        macos_install(installer, dest=dest, launch=launch)
+    else:
+        linux_install(installer, dest=dest, launch=launch)
+
+
+def script_launch():
+    # To test: python3 -c "import runpy; runpy.run_path('installer.py', run_name='script_launch')"
+    def path(x):
+        return os.path.expandvars(os.path.expanduser(x))
+
+    def to_bool(x):
+        return x.lower() in {'y', 'yes', '1', 'true'}
+
+    type_map = {x: path for x in 'dest installer'.split()}
+    type_map['launch'] = to_bool
+    kwargs = {}
+
+    for arg in sys.argv[1:]:
+        if arg:
+            m = re.match('([a-z_]+)=(.+)', arg)
+            if m is None:
+                raise SystemExit('Unrecognized command line argument: ' + arg)
+            k = m.group(1)
+            if k not in type_map:
+                raise SystemExit('Unrecognized command line argument: ' + arg)
+            kwargs[k] = type_map[k](m.group(2))
+    main(**kwargs)
+
+
+def update_intaller_wrapper():
+    # To run: python3 -c "import runpy; runpy.run_path('installer.py', run_name='update_wrapper')" installer.sh
+    src = open(__file__, 'rb').read().decode('utf-8')
+    wrapper = sys.argv[-1]
+    with open(wrapper, 'r+b') as f:
+        raw = f.read().decode('utf-8')
+        nraw = re.sub(r'^# HEREDOC_START.+^# HEREDOC_END', lambda m: '# HEREDOC_START\n{}\n# HEREDOC_END'.format(src), raw, flags=re.MULTILINE | re.DOTALL)
+        if 'update_intaller_wrapper()' not in nraw:
+            raise SystemExit('regex substitute of HEREDOC failed')
+        f.seek(0), f.truncate()
+        f.write(nraw.encode('utf-8'))
+
+
+if __name__ == '__main__' and from_file:
+    main()
+elif __name__ == 'update_wrapper':
+    update_intaller_wrapper()
+elif __name__ == 'script_launch':
+    script_launch()

docs/installer.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+#
+# installer.sh
+# Copyright (C) 2018 Kovid Goyal <kovid at kovidgoyal.net>
+#
+# Distributed under terms of the GPLv3 license.
+#
+
+python=$(command -v python3)
+if [ -z "$python" ]; then
+    python=$(command -v python2)
+fi
+if [ -z "$python" ]; then
+    python=$(command -v python2.7)
+fi
+if [ -z "$python" ]; then
+    python=$(command -v python)
+fi
+if [ -z "$python" ]; then
+    python=python
+fi
+
+echo Using python executable: $python
+
+$python -c "import sys; script_launch=lambda:sys.exit('Download of installer failed!'); exec(sys.stdin.read()); script_launch()" "$@" <<'INSTALLER_HEREDOC'
+# {{{
+# HEREDOC_START
+# HEREDOC_END
+# }}}
+INSTALLER_HEREDOC

docs/invocation.rst

@@ -0,0 +1,6 @@
+The kitty command line interface
+====================================
+
+.. program:: kitty
+
+.. include:: generated/cli-kitty.rst

docs/key-encoding.rst

@@ -0,0 +1,128 @@
+Key encoding for extended keyboard protocol
+===============================================
+
+See :ref:`extended-key-protocol` for more information and `this table in JSON
+format <https://github.com/kovidgoyal/kitty/blob/master/key_encoding.json>`_.
+
+=====================   ======================
+Name                    Encoded representation
+0                       ``G``
+1                       ``H``
+2                       ``I``
+3                       ``J``
+4                       ``K``
+5                       ``L``
+6                       ``M``
+7                       ``N``
+8                       ``O``
+9                       ``P``
+A                       ``S``
+APOSTROPHE              ``B``
+B                       ``T``
+BACKSLASH               ``t``
+BACKSPACE               ``1``
+C                       ``U``
+CAPS LOCK               ``:``
+COMMA                   ``C``
+D                       ``V``
+DELETE                  ``3``
+DOWN                    ``6``
+E                       ``W``
+END                     ``-``
+ENTER                   ``z``
+EQUAL                   ``R``
+ESCAPE                  ``y``
+F                       ``X``
+F1                      ``/``
+F10                     ``]``
+F11                     ``{``
+F12                     ``}``
+F13                     ``@``
+F14                     ``%``
+F15                     ``$``
+F16                     ``#``
+F17                     ``BA``
+F18                     ``BB``
+F19                     ``BC``
+F2                      ``*``
+F20                     ``BD``
+F21                     ``BE``
+F22                     ``BF``
+F23                     ``BG``
+F24                     ``BH``
+F25                     ``BI``
+F3                      ``?``
+F4                      ``&``
+F5                      ``<``
+F6                      ``>``
+F7                      ``(``
+F8                      ``)``
+F9                      ``[``
+G                       ``Y``
+GRAVE ACCENT            ``v``
+H                       ``Z``
+HOME                    ``.``
+I                       ``a``
+INSERT                  ``2``
+J                       ``b``
+K                       ``c``
+KP 0                    ``BJ``
+KP 1                    ``BK``
+KP 2                    ``BL``
+KP 3                    ``BM``
+KP 4                    ``BN``
+KP 5                    ``BO``
+KP 6                    ``BP``
+KP 7                    ``BQ``
+KP 8                    ``BR``
+KP 9                    ``BS``
+KP ADD                  ``BX``
+KP DECIMAL              ``BT``
+KP DIVIDE               ``BU``
+KP ENTER                ``BY``
+KP EQUAL                ``BZ``
+KP MULTIPLY             ``BV``
+KP SUBTRACT             ``BW``
+L                       ``d``
+LEFT                    ``5``
+LEFT ALT                ``Bc``
+LEFT BRACKET            ``s``
+LEFT CONTROL            ``Bb``
+LEFT SHIFT              ``Ba``
+LEFT SUPER              ``Bd``
+M                       ``e``
+MINUS                   ``D``
+N                       ``f``
+NUM LOCK                ``=``
+O                       ``g``
+P                       ``h``
+PAGE DOWN               ``9``
+PAGE UP                 ``8``
+PAUSE                   ``!``
+PERIOD                  ``E``
+PRINT SCREEN            ``^``
+Q                       ``i``
+R                       ``j``
+RIGHT                   ``4``
+RIGHT ALT               ``Bg``
+RIGHT BRACKET           ``u``
+RIGHT CONTROL           ``Bf``
+RIGHT SHIFT             ``Be``
+RIGHT SUPER             ``Bh``
+S                       ``k``
+SCROLL LOCK             ``+``
+SEMICOLON               ``Q``
+SLASH                   ``F``
+SPACE                   ``A``
+T                       ``l``
+TAB                     ``0``
+U                       ``m``
+UP                      ``7``
+V                       ``n``
+W                       ``o``
+WORLD 1                 ``w``
+WORLD 2                 ``x``
+X                       ``p``
+Y                       ``q``
+Z                       ``r``
+=====================   ======================

docs/kittens/clipboard.rst

@@ -0,0 +1,27 @@
+clipboard - copy/paste to the system clipboard
+==================================================
+
+.. highlight:: sh
+
+
+The ``clipboard`` kitten can be used to read or write to the system clipboard
+from the shell. It even works over SSH. Using it is as simple as::
+
+    echo hooray | kitty +kitten clipboard
+
+All text received on :file:`stdin` is copied to the clipboard.
+
+To get text from the clipboard you have to enable reading of the clipboard
+in :opt:`clipboard_control` in :file:`kitty.conf`. Once you do that, you can
+use::
+
+    kitty +kitten clipboard --get-clipboard
+
+
+.. program:: kitty +kitten clipboard
+
+
+Command Line Interface
+--------------------------
+
+.. include:: /generated/cli-kitten-clipboard.rst

docs/kittens/diff.rst

@@ -0,0 +1,157 @@
+kitty-diff - A fast side-by-side diff tool with syntax highlighting and images
+================================================================================
+
+.. highlight:: sh
+
+Major Features
+-----------------
+
+.. container:: major-features
+
+    * Displays diffs side-by-side in the kitty terminal
+
+    * Does syntax highlighting of the displayed diffs, asynchronously, for maximum
+      speed
+
+    * Displays images as well as text diffs, even over SSH
+
+    * Does recursive directory diffing
+
+
+.. figure:: ../screenshots/diff.png
+   :alt: Screenshot, showing a sample diff
+   :align: center
+   :scale: 100%
+
+   Screenshot, showing a sample diff
+
+.. contents::
+
+
+Installation
+---------------
+
+Simply :ref:`install kitty <quickstart>`.  You also need
+to have either the `git <https://git-scm.com/>`_ program or the ``diff`` program
+installed. Additionally, for syntax highlighting to work,
+`pygments <http://pygments.org/>`_ must be installed (note that pygments is
+included in the macOS kitty app).
+
+
+Usage
+--------
+
+In the kitty terminal, run::
+
+    kitty +kitten diff file1 file2
+
+to see the diff between file1 and file2.
+
+Create an alias in your shell's startup file to shorten the command, for example:
+
+.. code-block:: sh
+
+    alias d="kitty +kitten diff"
+
+Now all you need to do to diff two files is::
+
+    d file1 file2
+
+You can also pass directories instead of files to see the recursive diff of the
+directory contents.
+
+
+Keyboard controls
+----------------------
+
+=========================   ===========================
+Action                      Shortcut
+=========================   ===========================
+Quit                        :kbd:`q, Ctrl+c, Esc`
+Scroll line up              :kbd:`k, up`
+Scroll line down            :kbd:`j, down`
+Scroll page up              :kbd:`PgUp`
+Scroll page down            :kbd:`PgDn`
+Scroll to top               :kbd:`Home`
+Scroll to bottom            :kbd:`End`
+Scroll to next page         :kbd:`Space, PgDn`
+Scroll to previous page     :kbd:`PgUp`
+Scroll to next change       :kbd:`n`
+Scroll to previous change   :kbd:`p`
+Increase lines of context   :kbd:`+`
+Decrease lines of context   :kbd:`-`
+All lines of context        :kbd:`a`
+Restore default context     :kbd:`=`
+Search forwards             :kbd:`/`
+Search backwards            :kbd:`?`
+Clear search                :kbd:`Esc`
+Scroll to next match        :kbd:`>, .`
+Scroll to previous match    :kbd:`<, ,`
+=========================   ===========================
+
+
+Integrating with git
+-----------------------
+
+Add the following to `~/.gitconfig`:
+
+.. code-block:: ini
+
+    [diff]
+        tool = kitty
+        guitool = kitty.gui
+    [difftool]
+        prompt = false
+        trustExitCode = true
+    [difftool "kitty"]
+        cmd = kitty +kitten diff $LOCAL $REMOTE
+    [difftool "kitty.gui"]
+        cmd = kitty kitty +kitten diff $LOCAL $REMOTE
+
+Now to use kitty-diff to view git diffs, you can simply do::
+
+    git difftool --no-symlinks --dir-diff
+
+Once again, creating an alias for this command is useful.
+
+
+Why does this work only in kitty?
+----------------------------------------
+
+The diff kitten makes use of various features that are :doc:`kitty only
+</protocol-extensions>`,  such as the :doc:`kitty graphics protocol
+</graphics-protocol>`, the :ref:`extended keyboard protocol
+<extended-key-protocol>`, etc. It also leverages terminal program
+infrastructure I created for all of kitty's other kittens to reduce the amount
+of code needed (the entire implementation is under 2000 lines of code).
+
+And fundamentally, it's kitty only because I wrote it for myself, and I am
+highly unlikely to use any other terminals :)
+
+
+
+Configuration
+------------------------
+
+You can configure the colors used, keyboard shortcuts, the diff implementation,
+the default lines of context, etc.  by creating a :file:`diff.conf` file in
+your :ref:`kitty config folder <confloc>`. See below for the supported
+configuration directives.
+
+
+.. include:: /generated/conf-kitten-diff.rst
+
+
+Command Line Interface
+-------------------------
+
+.. include:: /generated/cli-kitten-diff.rst
+
+
+
+Sample diff.conf
+-----------------
+
+You can download a sample :file:`diff.conf` file with all default settings and
+comments describing each setting by clicking: :download:`sample diff.conf
+</generated/conf/diff.conf>`.

docs/kittens/hints.rst

@@ -0,0 +1,27 @@
+Hints
+==========
+
+|kitty| has a *hints mode* to select and act on arbitrary text snippets currently
+visible on the screen.  For example, you can press :sc:`open_url`
+to choose any URL visible on the screen and then open it using your system
+browser.
+
+.. figure:: ../screenshots/hints_mode.png
+    :alt: URL hints mode
+    :align: center
+    :scale: 100%
+
+    URL hints mode
+
+Similarly, you can press :sc:`insert_selected_path` to
+select anything that looks like a path or filename and then insert it into the
+terminal, very useful for picking files from the output of a ``git`` or ``ls`` command and
+adding them to the command line for the next command.
+
+The hints kitten is very powerful to see more detailed help on its various
+options and modes of operation, see below.
+
+Command Line Interface
+-------------------------
+
+.. include:: ../generated/cli-kitten-hints.rst

docs/kittens/icat.rst

@@ -0,0 +1,29 @@
+icat - Display images in the terminal
+========================================
+
+The ``icat`` kitten can be used to display arbitrary images in the |kitty|
+terminal. Using it is as simple as::
+
+    kitty +kitten icat image.jpeg
+
+It supports all image types supported by `ImageMagick
+<https://www.imagemagick.org>`_. It even works over SSH. For details, see
+the :doc:`kitty graphics protocol </graphics-protocol>`.
+
+.. note::
+
+    `ImageMagick <https://www.imagemagick.org>`_ must be installed for ``icat`` to
+    work.
+
+
+.. program:: kitty +kitten icat
+
+
+The ``icat`` kitten has various command line arguments to allow it to be used
+from inside other programs to display images. In particular, :option:`--place`,
+:option:`--detect-support` and :option:`--print-window-size`.
+
+Command Line Interface
+--------------------------
+
+.. include:: /generated/cli-kitten-icat.rst

docs/kittens/panel.rst

@@ -0,0 +1,43 @@
+Draw a GPU accelerated dock panel on your desktop
+====================================================================================================
+
+.. highlight:: sh
+
+
+You can use this kitten to draw a GPU accelerated panel on the edge
+of your screen, that shows the output from an arbitrary terminal program.
+
+It is useful for showing status information or notifications on your desktop
+using terminal programs instead of GUI toolkits.
+
+
+.. figure:: ../screenshots/panel.png
+   :alt: Screenshot, showing a sample panel
+   :align: center
+   :scale: 100%
+
+   Screenshot, showing a sample panel
+
+
+The screenshot above shows a sample panel that displays the current desktop and
+window title as well as miscellaneous system information such as network
+activity, CPU load, date/time, etc.
+
+.. note::
+
+    This kitten currently only works on X11 desktops
+
+Using this kitten is simple, for example::
+
+    kitty +kitten panel sh -c 'printf "\n\n\nHello, world."; sleep 5s'
+
+This will show ``Hello, world.`` at the top edge of your screen for five
+seconds. Here the terminal program we are running is ``sh`` with a script to
+print out ``Hello, world!``. You can make the terminal program as complex as
+you like, as demonstrated in the screenshot above.
+
+
+Command Line Interface
+-------------------------
+
+.. include:: ../generated/cli-kitten-panel.rst

docs/kittens/unicode-input.rst

@@ -0,0 +1,22 @@
+Unicode input
+================
+
+You can input unicode characters by name, hex code, recently used and even an editable favorites list.
+Press :sc:`input_unicode_character` to start the unicode input widget, shown below.
+
+.. figure:: ../screenshots/unicode.png
+    :alt: A screenshot of the unicode input widget
+    :align: center
+    :scale: 100%
+
+    A screenshot of the unicode input widget
+
+In :guilabel:`Code` mode, you enter a unicode character by typing in the hex code for the
+character and pressing enter, for example, type in ``2716`` and press enter to get
+✖. You can also choose a character from the list of recently used characters by
+typing a leading period and then the two character index and pressing Enter.
+
+In :guilabel:`Name` mode you instead type words from the character name and use the arrow
+keys/tab to select the character from the displayed matches. You can also type
+a leading period and the index for the match if you don't like to use arrow
+keys.

docs/performance.rst

@@ -0,0 +1,59 @@
+|kitty| Performance
+===================
+
+The main goals for |kitty| performance are user perceived latency while typing
+and "smoothness" while scrolling as well as CPU usage. |kitty| tries hard to find
+an optimum balance for these. To that end it keeps a cache of each rendered
+glyph in video RAM so that font rendering is not a bottleneck.  Interaction
+with child programs takes place in a separate thread from rendering, to improve
+smoothness.
+
+There are two parameters you can tune to adjust the performance. ``repaint_delay``
+and ``input_delay``. These control the artificial delays introduced into the
+render loop to reduce CPU usage. See :ref:`conf-kitty-performance` for details.
+See also the ``sync_to_monitor`` option to further decrease latency at the cost
+of some `tearing <https://en.wikipedia.org/wiki/Screen_tearing>`_ while scrolling.
+
+You can generate detailed per-function performance data using
+`gperftools <https://github.com/gperftools/gperftools>`_. Build |kitty| with
+`make profile` which will create an executable called `kitty-profile`.  Run
+that and perform the task you want to analyse, for example, scrolling a large
+file with `less`. After you quit, function call statistics will be printed to
+`stdout` and you can use tools like *kcachegrind* for more detailed analysis.
+
+Here are some CPU usage numbers for the task of scrolling a file continuously
+in less.  The CPU usage is for the terminal process and X together and is
+measured using htop.  The measurements are taken at the same font and window
+size for all terminals on a ``Intel(R) Core(TM) i7-4820K CPU @ 3.70GHz`` CPU
+with a ``Advanced Micro Devices, Inc. [AMD/ATI] Cape Verde XT [Radeon HD
+7770/8760 / R7 250X]`` GPU.
+
+==============   =========================
+Terminal         CPU usage (X + terminal)
+==============   =========================
+|kitty|          6 - 8%
+xterm            5 - 7% (but scrolling was extremely janky)
+termite          10 - 13%
+urxvt            12 - 14%
+gnome-terminal   15 - 17%
+konsole          29 - 31%
+==============   =========================
+
+
+As you can see, |kitty| uses much less CPU than all terminals, except xterm, but
+its scrolling "smoothness" is much better than that of xterm (at least to my,
+admittedly biased, eyes).
+
+
+.. _perf-cat:
+
+.. note::
+
+    Some people have asked why kitty does not perform better than terminal XXX in
+    the test of sinking large amounts of data, such as catting a large text
+    file. The answer is because this is not a goal for kitty. kitty
+    deliberately throttles input parsing and output rendering to minimize
+    resource usage while still being able to sink output faster than any real
+    world program can produce it. Reducing CPU usage, and hence battery drain
+    while achieving instant response times and smooth scrolling to a human eye
+    is a far more important goal.

docs/protocol-extensions.rst

@@ -0,0 +1,208 @@
+Extensions to the xterm protocol
+===================================
+
+|kitty| has a few extensions to the xterm protocol, to enable advanced features.
+These are typically in the form of new or re-purposed escape codes. While these
+extensions are currently |kitty| specific, it would be nice to get some of them
+adopted more broadly, to push the state of terminal emulators forward.
+
+The goal of these extensions is to be as small an unobtrusive as possible,
+while filling in some gaps in the existing xterm protocol. In particular, one
+of the goals of this specification is explicitly not to "re-imagine" the tty.
+The tty should remain what it is -- a device for efficiently processing text
+received as a simple byte stream. Another objective is to only move the minimum
+possible amount of extra functionality into the terminal program itself. This
+is to make it as easy to implement these protocol extensions as possible,
+thereby hopefully encouraging their widespread adoption.
+
+If you wish to discuss these extensions, propose additions/changes to them
+please do so by opening issues in the github bug tracker.
+
+.. contents::
+
+Colored and styled underlines
+-------------------------------
+
+|kitty| supports colored and styled (wavy) underlines. This is of particular
+use in terminal editors such as vim and emacs to display red, wavy underlines
+under mis-spelled words and/or syntax errors. This is done by re-purposing some
+SGR escape codes that are not used in modern terminals (`CSI codes
+<https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences>`_)
+
+To set the underline style::
+
+    <ESC>[4:0m  # this is no underline
+    <ESC>[4:1m  # this is a straight underline
+    <ESC>[4:2m  # this is a double underline
+    <ESC>[4:3m  # this is a curly underline
+    <ESC>[4:4m  # this is a dotted underline (not implemented in kitty)
+    <ESC>[4:5m  # this is a dashed underline (not implemented in kitty)
+    <ESC>[4m    # this is a straight underline (for backwards compat)
+    <ESC>[24m   # this is no underline (for backwards compat)
+
+To set the underline color (this is reserved and as far as I can tell not actually used for anything)::
+
+    <ESC>[58...m
+
+This works exactly like the codes ``38, 48`` that are used to set foreground and
+background color respectively.
+
+To reset the underline color (also previously reserved and unused)::
+
+    <ESC>[59m
+
+To detect support for this feature in a terminal emulator, query the terminfo database
+for the ``Su`` boolean capability.
+
+Graphics rendering
+---------------------
+
+See :doc:`/graphics-protocol` for a description
+of this protocol to enable drawing of arbitrary raster images in the terminal.
+
+
+.. _extended-key-protocol:
+
+Keyboard handling
+-------------------
+
+There are various problems with the current state of keyboard handling. They
+include:
+
+* No way to use modifiers other than ``Ctrl`` and ``Alt``
+
+* No way to reliably use multiple modifier keys, other than, ``Shift+Alt``.
+
+* No way to handle different types of keyboard events, such as press, release or repeat
+
+* No reliable way to distinguish single ``Esc`` keypresses from the start of a
+  escape sequence. Currently, client programs use fragile timing related hacks
+  for this, leading to bugs, for example:
+  `neovim #2035 <https://github.com/neovim/neovim/issues/2035>`_.
+
+There are already two distinct keyboard handling modes, *normal mode* and
+*application mode*. These modes generate different escape sequences for the
+various special keys (arrow keys, function keys, home/end etc.) Most terminals
+start out in normal mode, however, most shell programs like ``bash`` switch them to
+application mode. We propose adding a third mode, named *full mode* that addresses
+the shortcomings listed above.
+
+Switching to the new *full mode* is accomplished using the standard private
+mode DECSET escape sequence::
+
+    <ESC>[?2017h
+
+and to leave *full mode*, use DECRST::
+
+    <ESC>[?2017l
+
+The number ``2017`` above is not used for any existing modes, as far as I know.
+Client programs can query if the terminal emulator is in *full mode* by using
+the standard `DECRQM <https://vt100.net/docs/vt510-rm/DECRQM.html>`_ escape sequence.
+
+The new mode works as follows:
+
+  * All printable key presses without modifier keys are sent just as in the
+    *normal mode*. This means all printable ASCII characters and in addition,
+    ``Enter``, ``Space`` and ``Backspace``. Also any unicode characters generated by
+    platform specific extended input modes, such as using the ``AltGr`` key. This
+    is done so that client programs that are not aware of this mode can still
+    handle basic text entry, so if a *full mode* using program crashes and does
+    not reset, the user can still issue a ``reset`` command in the shell to restore
+    normal key handling. Note that this includes pressing the ``Shift`` modifier
+    and printable keys. Note that this means there are no repeat and release
+    events for these keys and also for the left and right shift keys.
+
+  * For non printable keys and key combinations including one or more modifiers,
+    an escape sequence encoding the key event is sent. For details on the
+    escape sequence, see below.
+
+The escape sequence encodes the following properties:
+
+  * Type of event: ``press,repeat,release``
+  * Modifiers pressed at the time of the event
+  * The actual key being pressed
+
+Schematically::
+
+    <ESC>_K<type><modifiers><key><ESC>\
+
+Where ``<type>`` is one of ``p`` -- press, ``r`` -- release and ``t`` -- repeat.
+Modifiers is a bitmask represented as a single base64 digit.  Shift -- ``0x1``,
+Alt -- ``0x2``, Control -- ``0x4`` and Super -- ``0x8``.  ``<key>`` is a number
+(encoded in base85) corresponding to the key pressed. The key name to number
+mapping is defined in :doc:`this table <key-encoding>`.
+
+For example::
+
+    <ESC>_KpGp<ESC>\  is  <Ctrl>+<Alt>+x (press)
+    <ESC>_KrP8<ESC>\  is  <Ctrl>+<Alt>+<Shift>+<Super>+PageUp (release)
+
+This encoding means each key event is represented by 8 or 9 printable ascii
+only bytes, for maximum robustness.
+
+
+.. _ext_styles:
+
+Setting text styles/colors in arbitrary regions of the screen
+------------------------------------------------------------------
+
+There already exists an escape code to set *some* text attributes in arbitrary
+regions of the screen, `DECCARA
+<https://vt100.net/docs/vt510-rm/DECCARA.html>`_.  However, it is limited to
+only a few attributes. |kitty| extends this to work with *all* SGR attributes.
+So, for example, this can be used to set the background color in an arbitrary
+region of the screen.
+
+The motivation for this extension is the various problems with the existing
+solution for erasing to background color, namely the *background color erase
+(bce)* capability. See
+`this discussion <https://github.com/kovidgoyal/kitty/issues/160#issuecomment-346470545>`_
+and `this FAQ <http://invisible-island.net/ncurses/ncurses.faq.html#bce_mismatches>`_
+for a summary of problems with *bce*.
+
+For example, to set the background color to blue in a
+rectangular region of the screen from (3, 4) to (10, 11), you use::
+
+    <ESC>[2*x<ESC>[4;3;11;10;44$r<ESC>[*x
+
+
+Saving and restoring the default foreground/background/selection/cursor colors
+---------------------------------------------------------------------------------
+
+It is often useful for a full screen application with its own color themes
+to set the default foreground, background, selection and cursor colors. This
+allows for various performance optimizations when drawing the screen. The
+problem is that if the user previously used the escape codes to change these
+colors herself, then running the full screen application will lose her
+changes even after it exits. To avoid this, kitty introduces a new pair of
+*OSC* escape codes to push and pop the current color values from a stack::
+
+    <ESC>]30001<ESC>\  # push onto stack
+    <ESC>]30101<ESC>\  # pop from stack
+
+These escape codes save/restore the so called *dynamic colors*, default
+background, default foreground, selection background, selection foreground and
+cursor color.
+
+
+Pasting to clipboard
+----------------------
+
+|kitty| implements the OSC 52 escape code protocol to get/set the clipboard
+contents (controlled via the :opt:`clipboard_control` setting). There is one
+difference in kitty's implementation compared to some other terminal emulators.
+|kitty| allows sending arbitrary amounts of text to the clipboard. It does so
+by modifying the protocol slightly. Successive OSC 52 escape codes to set the
+clipboard will concatenate, so::
+
+    <ESC>]52;c;<payload1><ESC>\
+    <ESC>]52;c;<payload2><ESC>\
+
+will result in the clipboard having the contents ``payload1 + payload2``. To
+send a new string to the clipboard send an OSC 52 sequence with an invalid payload
+first, for example::
+
+    <ESC>]52;c;!<ESC>\
+
+Here ``!`` is not valid base64 encoded text, so it clears the clipboard.

docs/remote-control.rst

@@ -0,0 +1,138 @@
+:tocdepth: 2
+
+Controlling kitty from scripts or the shell
+==============================================
+
+.. highlight:: sh
+
+Tutorial
+----------
+
+|kitty| can be controlled from scripts or the shell prompt. You can open new
+windows, send arbitrary text input to any window, name windows and tabs, etc.
+Let's walk through a few examples of controlling |kitty|.
+
+Start by running |kitty| as::
+
+    kitty -o allow_remote_control=yes --window-layout tall
+
+In order for control to work, :opt:`allow_remote_control` must be enabled in
+:file:`kitty.conf`. Here we turn it on explicitly at the command line.
+
+Now, in the new |kitty| window, enter the command::
+
+    kitty @ new-window --title Output --keep-focus cat
+
+This will open a new window, running the ``cat`` program that will appear next
+to the current window.
+
+Let's send some text to this new window::
+
+    kitty @ send-text --match cmdline:cat Hello, World
+
+This will make ``Hello, World`` show up in the window running the ``cat`` program.
+The :option:`kitty @ send-text --match` option is very powerful, it allows selecting windows by their
+titles, the command line of the program running in the window, the working
+directory of the program running in the window, etc.  See ``kitty @ send-text
+--help`` for details.
+
+More usefully, you can pipe the output of a command running in one window to
+another window, for example::
+
+    ls | kitty @ send-text --match title:Output --stdin
+
+This will show the output of ls in the output window instead of the current
+window. You can use this technique to, for example, show the output of running
+``make`` in your editor in a different window. The possibilities are endless.
+
+You can even have things you type show up in a different window. Run::
+
+    kitty @ send-text --match title:Output --stdin
+
+And type some text, it will show up in the output window, instead of the current
+window. Type ``Ctrl+D`` when you are ready to stop.
+
+Now, let's open a new tab::
+
+   kitty @ new-window --new-tab --tab-title "My Tab" --keep-focus bash
+
+This will open a new tab running the bash shell with the title "My Tab".
+We can change the title of the tab with::
+
+   kitty @ set-tab-title --match title:My  New Title
+
+Let's change the title of the current tab::
+
+   kitty @ set-tab-title Master Tab
+
+Now lets switch to the newly opened tab::
+
+   kitty @ focus-tab --match title:New
+
+Similarly, to focus the previously opened output window (which will also switch
+back to the old tab, automatically)::
+
+   kitty @ focus-window --match title:Output
+
+You can get a listing of available tabs and windows, by running::
+
+   kitty @ ls
+
+This outputs a tree of data in JSON format. The top level of the tree is all
+operating system kitty windows. Each OS window has an id and a list of tabs.
+Each tab has its own id, a title and a list of windows. Each window has an id,
+title, current working directory, process id (PID) and command-line of the
+process running in the window. You can use this information with :option:`kitty @ focus-window --match`
+to control individual windows.
+
+As you can see, it is very easy to control |kitty| using the
+``kitty @`` messaging system. This tutorial touches only the
+surface of what is possible. See ``kitty @ --help`` for more details.
+
+Note that in the example's above, ``kitty @`` messaging works only when run inside a |kitty| window,
+not anywhere. But, within a |kitty| window it even works over SSH. If you want to control
+|kitty| from programs/scripts not running inside a |kitty| window, you have to implement a couple of
+extra steps. First start |kitty| as::
+
+    kitty -o allow_remote_control=yes --listen-on unix:/tmp/mykitty
+
+The :option:`kitty --listen-on` option tells |kitty| to listen for control messages at the
+specified path. See ``kitty --help`` for details. Now you can control this
+instance of |kitty| using the :option:`kitty @ --to` command line argument to ``kitty @``. For example::
+
+    kitty @ --to unix:/tmp/mykitty ls
+
+
+Note that if all you want to do is run a single |kitty| "daemon" and have subsequent
+|kitty| invocations appear as new top-level windows, you can use the simpler :option:`kitty --single-instance`
+option, see ``kitty --help`` for that.
+
+The builtin kitty shell
+--------------------------
+
+You can explore the |kitty| command language more easily using the builtin |kitty|
+shell. Run ``kitty @`` with no arguments and you will be dropped into the |kitty|
+shell with completion for |kitty| command names and options.
+
+You can even open the |kitty| shell inside a running |kitty| using a simple
+keyboard shortcut (:sc:`kitty_shell` by default). This has the added
+advantage that you don't need to use ``allow_remote_control`` to make it work.
+
+
+Allowing only some windows to control kitty
+----------------------------------------------
+
+If you do not want to allow all programs running in |kitty| to control it, you can selectively
+enable remote control for only some |kitty| windows. Simply create a shortcut
+such as::
+
+    map ctrl+k new_window @ some_program
+
+Then programs running in windows created with that shortcut can use ``kitty @``
+to control kitty. Note that any program with the right level of permissions can
+still write to the pipes of any other program on the same computer and
+therefore can control |kitty|. It can, however, be useful to block programs
+running on other computers (for example, over ssh) or as other users.
+
+
+.. include:: generated/cli-kitty-at.rst

docs/support.html

@@ -0,0 +1,69 @@
+<style>
+.support-button {
+    border-radius: 6px;
+    box-shadow: rgb(255, 246, 175) 0px 1px 0px 0px;
+    display: inline-block;
+    cursor: pointer;
+    color: rgb(51, 51, 51);
+    font-size: inherit;
+    font-family: Arial;
+    font-weight: bold;
+    padding: 8px 24px;
+    text-decoration: none;
+}
+
+.support-button:hover {
+    transform: scale(1.5);
+}
+
+.support-button {
+    outline: 0;
+}
+.support-button:visited {
+    color: rgb(51, 51, 51);
+    outline: 0;
+}
+
+#support-buttons > div {
+    margin-top: 3ex;
+    text-align: center;
+}
+
+#paypal input[type=submit] {
+    background: linear-gradient(rgb(255, 236, 100) 5%, rgb(255, 171, 35) 100%) rgb(255, 236, 100);
+    border: 1px solid rgb(255, 171, 35);
+}
+
+#liberapay a {
+    background: linear-gradient(rgb(92, 184, 92) 5%, rgb(62, 142, 62) 100%) rgb(92, 184, 92);
+    border: 1px solid rgb(62, 142, 62);
+    color: rgb(51, 51, 51);
+}
+
+#patreon a {
+    background: linear-gradient(rgb(11, 100, 163) 5%, rgb(5, 45, 73) 100%) rgb(11, 100, 163);
+    border: 1px solid rgb(5, 45, 73);
+    color: #eee;
+}
+</style>
+
+<div id="support-buttons">
+<div id="patreon">
+    <a class="support-button" href="https://www.patreon.com/bePatron?u=917933">Patronage via Patreon</a>
+</div>
+
+<div>
+    <form id="paypal" action="https://www.paypal.com/cgi-bin/webscr" method="post" title="Contribute to support calibre development">
+        <input type="hidden" name="cmd" value="_s-xclick" />
+        <input type="hidden" name="hosted_button_id" value="FEELASHWEL5HS" />
+        <input class="support-button" type="submit" name="submit" value="One time support via PayPal" />
+        <img alt="" border="0" src="https://www.paypalobjects.com/en_GB/i/scr/pixel.gif" width="1" height="1" />
+    </form>
+</div>
+
+<div id="liberapay">
+    <a class="support-button" href="https://liberapay.com/kovidgoyal/donate">Patronage via Liberapay</a>
+</div>
+
+</div>
+

docs/support.rst

@@ -0,0 +1,23 @@
+Support kitty development
+===========================
+
+My goal with |kitty| is to move the stagnant terminal ecosystem forward.  To that
+end kitty has many foundational features, such as: :doc:`image support
+<graphics-protocol>`, :doc:`superlative performance <performance>`,
+:doc:`various enhancements to the terminal protocol <protocol-extensions>`,
+etc. These features allow the development of rich terminal applications, such
+as :doc:`Side-by-side diff <kittens/diff>` and :doc:`Unicode input
+<kittens/unicode-input>`.
+
+If you wish to support this mission and see the terminal ecosystem evolve,
+consider donating so that I can devote more time to |kitty| development.
+I have personally written `almost all kitty code
+<https://github.com/kovidgoyal/kitty/graphs/contributors>`_.
+
+You can choose to make either a one-time payment via PayPal or become a
+*patron* of kitty development via the Patreon or Liberapay services
+below:
+
+
+.. raw:: html
+    :file: support.html

gen-apc-parsers.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import subprocess
+from collections import defaultdict
+
+
+def resolve_keys(keymap):
+    ans = defaultdict(list)
+    for ch, (attr, atype) in keymap.items():
+        if atype not in ('int', 'uint'):
+            atype = 'flag'
+        ans[atype].append(ch)
+    return ans
+
+
+def enum(keymap):
+    lines = []
+    for ch, (attr, atype) in keymap.items():
+        lines.append(f"{attr}='{ch}'")
+    return '''
+    enum KEYS {{
+        {}
+    }};
+    '''.format(',\n'.join(lines))
+
+
+def parse_key(keymap):
+    lines = []
+    for attr, atype in keymap.values():
+        vs = atype.upper() if atype in ('uint', 'int') else 'FLAG'
+        lines.append(f'case {attr}: value_state = {vs}; break;')
+    return '        \n'.join(lines)
+
+
+def parse_flag(keymap, type_map, command_class):
+    lines = []
+    for ch in type_map['flag']:
+        attr, allowed_values = keymap[ch]
+        q = ' && '.join(f"g.{attr} != '{x}'" for x in allowed_values)
+        lines.append(f'''
+            case {attr}: {{
+                g.{attr} = screen->parser_buf[pos++] & 0xff;
+                if ({q}) {{
+                    REPORT_ERROR("Malformed {command_class} control block, unknown flag value for {attr}: 0x%x", g.{attr});
+                    return;
+                }};
+            }}
+            break;
+        ''')
+    return '        \n'.join(lines)
+
+
+def parse_number(keymap):
+    int_keys = [f'I({attr})' for attr, atype in keymap.values() if atype == 'int']
+    uint_keys = [f'U({attr})' for attr, atype in keymap.values() if atype == 'uint']
+    return '; '.join(int_keys), '; '.join(uint_keys)
+
+
+def cmd_for_report(report_name, keymap, type_map, payload_allowed):
+    def group(atype, conv):
+        flag_fmt, flag_attrs = [], []
+        cv = {'flag': 'c', 'int': 'i', 'uint': 'I'}[atype]
+        for ch in type_map[atype]:
+            flag_fmt.append('s' + cv)
+            attr = keymap[ch][0]
+            flag_attrs.append(f'"{attr}", {conv}g.{attr}')
+        return ' '.join(flag_fmt), ', '.join(flag_attrs)
+
+    flag_fmt, flag_attrs = group('flag', '')
+    int_fmt, int_attrs = group('int', '(int)')
+    uint_fmt, uint_attrs = group('uint', '(unsigned int)')
+
+    fmt = f'{flag_fmt} {uint_fmt} {int_fmt}'
+    if payload_allowed:
+        ans = [f'REPORT_VA_COMMAND("s {{{fmt} sI}} y#", "{report_name}",']
+    else:
+        ans = [f'REPORT_VA_COMMAND("s {{{fmt}}}", "{report_name}",']
+    ans.append(',\n     '.join((flag_attrs, uint_attrs, int_attrs)))
+    if payload_allowed:
+        ans.append(', "payload_sz", g.payload_sz, payload, g.payload_sz')
+    ans.append(');')
+    return '\n'.join(ans)
+
+
+def generate(function_name, callback_name, report_name, keymap, command_class, initial_key='a', payload_allowed=True):
+    type_map = resolve_keys(keymap)
+    keys_enum = enum(keymap)
+    handle_key = parse_key(keymap)
+    flag_keys = parse_flag(keymap, type_map, command_class)
+    int_keys, uint_keys = parse_number(keymap)
+    report_cmd = cmd_for_report(report_name, keymap, type_map, payload_allowed)
+    if payload_allowed:
+        payload_after_value = "case ';': state = PAYLOAD; break;"
+        payload = payload = ', PAYLOAD'
+        parr = 'static uint8_t payload[4096];'
+        payload_case = f'''
+            case PAYLOAD: {{
+                sz = screen->parser_buf_pos - pos;
+                const char *err = base64_decode(screen->parser_buf + pos, sz, payload, sizeof(payload), &g.payload_sz);
+                if (err != NULL) {{ REPORT_ERROR("Failed to parse {command_class} command payload with error: %s", err); return; }}
+                pos = screen->parser_buf_pos;
+                }}
+                break;
+        '''
+        callback = f'{callback_name}(screen, &g, payload)'
+    else:
+        payload_after_value = payload = parr = payload_case = ''
+        callback = f'{callback_name}(screen, &g)'
+
+    return f'''
+static inline void
+{function_name}(Screen *screen, PyObject UNUSED *dump_callback) {{
+    unsigned int pos = 1;
+    enum PARSER_STATES {{ KEY, EQUAL, UINT, INT, FLAG, AFTER_VALUE {payload} }};
+    enum PARSER_STATES state = KEY, value_state = FLAG;
+    static {command_class} g;
+    unsigned int i, code;
+    uint64_t lcode;
+    bool is_negative;
+    memset(&g, 0, sizeof(g));
+    size_t sz;
+    {parr}
+    {keys_enum}
+    enum KEYS key = '{initial_key}';
+
+    while (pos < screen->parser_buf_pos) {{
+        switch(state) {{
+            case KEY:
+                key = screen->parser_buf[pos++];
+                state = EQUAL;
+                switch(key) {{
+                    {handle_key}
+                    default:
+                        REPORT_ERROR("Malformed {command_class} control block, invalid key character: 0x%x", key);
+                        return;
+                }}
+                break;
+
+            case EQUAL:
+                if (screen->parser_buf[pos++] != '=') {{
+                    REPORT_ERROR("Malformed {command_class} control block, no = after key, found: 0x%x instead", screen->parser_buf[pos-1]);
+                    return;
+                }}
+                state = value_state;
+                break;
+
+            case FLAG:
+                switch(key) {{
+                    {flag_keys}
+                    default:
+                        break;
+                }}
+                state = AFTER_VALUE;
+                break;
+
+            case INT:
+#define READ_UINT \\
+                for (i = pos; i < MIN(screen->parser_buf_pos, pos + 10); i++) {{ \\
+                    if (screen->parser_buf[i] < '0' || screen->parser_buf[i] > '9') break; \\
+                }} \\
+                if (i == pos) {{ REPORT_ERROR("Malformed {command_class} control block, expecting an integer value for key: %c", key & 0xFF); return; }} \\
+                lcode = utoi(screen->parser_buf + pos, i - pos); pos = i; \\
+                if (lcode > UINT32_MAX) {{ REPORT_ERROR("Malformed {command_class} control block, number is too large"); return; }} \\
+                code = lcode;
+
+                is_negative = false;
+                if(screen->parser_buf[pos] == '-') {{ is_negative = true; pos++; }}
+#define I(x) case x: g.x = is_negative ? 0 - (int32_t)code : (int32_t)code; break
+                READ_UINT;
+                switch(key) {{
+                    {int_keys};
+                    default: break;
+                }}
+                state = AFTER_VALUE;
+                break;
+#undef I
+            case UINT:
+                READ_UINT;
+#define U(x) case x: g.x = code; break
+                switch(key) {{
+                    {uint_keys};
+                    default: break;
+                }}
+                state = AFTER_VALUE;
+                break;
+#undef U
+#undef READ_UINT
+
+            case AFTER_VALUE:
+                switch (screen->parser_buf[pos++]) {{
+                    default:
+                        REPORT_ERROR("Malformed {command_class} control block, expecting a comma or semi-colon after a value, found: 0x%x",
+                                     screen->parser_buf[pos - 1]);
+                        return;
+                    case ',':
+                        state = KEY;
+                        break;
+                    {payload_after_value}
+                }}
+                break;
+
+            {payload_case}
+
+        }} // end switch
+    }} // end while
+
+    switch(state) {{
+        case EQUAL:
+            REPORT_ERROR("Malformed {command_class} control block, no = after key"); return;
+        case INT:
+        case UINT:
+            REPORT_ERROR("Malformed {command_class} control block, expecting an integer value"); return;
+        case FLAG:
+            REPORT_ERROR("Malformed {command_class} control block, expecting a flag value"); return;
+        default:
+            break;
+    }}
+
+    {report_cmd}
+
+    {callback};
+}}
+    '''
+
+
+def write_header(text, path):
+    with open(path, 'w') as f:
+        print(f'// This file is generated by {__file__} do not edit!', file=f, end='\n\n')
+        print('#pragma once', file=f)
+        print(text, file=f)
+    subprocess.check_call(['clang-format', '-i', path])
+
+
+def graphics_parser():
+    flag = frozenset
+    keymap = {
+        'a': ('action', flag('tTqpd')),
+        'd': ('delete_action', flag('aAiIcCpPqQxXyYzZ')),
+        't': ('transmission_type', flag('dfts')),
+        'o': ('compressed', flag('z')),
+        'f': ('format', 'uint'),
+        'm': ('more', 'uint'),
+        'i': ('id', 'uint'),
+        'w': ('width', 'uint'),
+        'h': ('height', 'uint'),
+        'x': ('x_offset', 'uint'),
+        'y': ('y_offset', 'uint'),
+        'v': ('data_height', 'uint'),
+        's': ('data_width', 'uint'),
+        'S': ('data_sz', 'uint'),
+        'O': ('data_offset', 'uint'),
+        'c': ('num_cells', 'uint'),
+        'r': ('num_lines', 'uint'),
+        'X': ('cell_x_offset', 'uint'),
+        'Y': ('cell_y_offset', 'uint'),
+        'z': ('z_index', 'int'),
+    }
+    text = generate('parse_graphics_code', 'screen_handle_graphics_command', 'graphics_command', keymap, 'GraphicsCommand')
+    write_header(text, 'kitty/parse-graphics-command.h')
+
+
+graphics_parser()

gen-wcwidth.py

@@ -15,6 +15,13 @@ from urllib.request import urlopen
 
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
+non_characters = frozenset(range(0xfffe, 0x10ffff, 0x10000))
+non_characters |= frozenset(range(0xffff, 0x10ffff + 1, 0x10000))
+non_characters |= frozenset(range(0xfdd0, 0xfdf0))
+if len(non_characters) != 66:
+    raise SystemExit('non_characters table incorrect')
+emoji_skin_tone_modifiers = frozenset(range(0x1f3fb, 0x1F3FF + 1))
+
 
 def get_data(fname, folder='UCD'):
     url = f'https://www.unicode.org/Public/{folder}/latest/{fname}'
@@ -35,7 +42,8 @@ def get_data(fname, folder='UCD'):
 class_maps = {}
 name_map = {}
 word_search_map = defaultdict(set)
-marks = set()
+zwj = 0x200d
+marks = set(emoji_skin_tone_modifiers) | {zwj}
 not_assigned = set(range(0, sys.maxunicode))
 
 
@@ -44,7 +52,8 @@ def parse_ucd():
     def add_word(w, c):
         if c <= 32 or c == 127 or 128 <= c <= 159:
             return
-        word_search_map[w.lower()].add(c)
+        if len(w) > 1:
+            word_search_map[w.lower()].add(c)
 
     first = None
     for word, c in html5.items():
@@ -54,7 +63,9 @@ def parse_ucd():
     for line in get_data('ucd/UnicodeData.txt'):
         parts = [x.strip() for x in line.split(';')]
         codepoint = int(parts[0], 16)
-        name = parts[1]
+        name = parts[1] or parts[10]
+        if name == '<control>':
+            name = parts[10]
         if name:
             name_map[codepoint] = name
             for word in name.lower().split():
@@ -186,11 +197,13 @@ def gen_emoji():
         p('\treturn false;\n}')
 
 
-def category_test(name, p, classes, comment, static=False):
+def category_test(name, p, classes, comment, static=False, extra_chars=frozenset(), exclude=frozenset()):
     static = 'static inline ' if static else ''
     chars = set()
     for c in classes:
         chars |= class_maps[c]
+    chars |= extra_chars
+    chars -= exclude
     p(f'{static}bool\n{name}(char_type code) {{')
     p(f'\t// {comment} ({len(chars)} codepoints)' + ' {{' '{')
     p('\tswitch(code) {')
@@ -207,7 +220,8 @@ def codepoint_to_mark_map(p, mark_map):
     for spec in get_ranges(mark_map):
         if isinstance(spec, tuple):
             s = rmap[spec[0]]
-            p(f'\t\tcase {spec[0]} ... {spec[1]}: return {s} + c - {spec[0]};')
+            cases = ' '.join(f'case {i}:' for i in range(spec[0], spec[1]+1))
+            p(f'\t\t{cases} return {s} + c - {spec[0]};')
         else:
             p(f'\t\tcase {spec}: return {rmap[spec]};')
     p('default: return 0;')
@@ -215,10 +229,12 @@ def codepoint_to_mark_map(p, mark_map):
     return rmap
 
 
-def classes_to_regex(classes):
+def classes_to_regex(classes, exclude=''):
     chars = set()
     for c in classes:
         chars |= class_maps[c]
+    for c in map(ord, exclude):
+        chars.discard(c)
 
     def as_string(codepoint):
         if codepoint < 256:
@@ -238,8 +254,16 @@ def gen_ucd():
     cz = {c for c in class_maps if c[0] in 'CZ'}
     with create_header('kitty/unicode-data.c') as p:
         p('#include "unicode-data.h"')
-        category_test('is_combining_char', p, {c for c in class_maps if c.startswith('M')}, 'M category (marks)')
-        category_test('is_ignored_char', p, 'Cc Cf Cs'.split(), 'Control characters (Cc Cf Cs)')
+        category_test(
+                'is_combining_char', p,
+                {c for c in class_maps if c.startswith('M')},
+                'M category (marks)',
+                # See https://github.com/harfbuzz/harfbuzz/issues/169
+                extra_chars=emoji_skin_tone_modifiers | {zwj}
+        )
+        category_test(
+            'is_ignored_char', p, 'Cc Cf Cs'.split(),
+            'Control characters and non-characters', extra_chars=non_characters, exclude={zwj})
         category_test('is_word_char', p, {c for c in class_maps if c[0] in 'LN'}, 'L and N categories')
         category_test('is_CZ_category', p, cz, 'C and Z categories')
         category_test('is_P_category', p, {c for c in class_maps if c[0] == 'P'}, 'P category (punctuation)')
@@ -252,57 +276,115 @@ def gen_ucd():
         p('combining_type mark_for_codepoint(char_type c) {')
         rmap = codepoint_to_mark_map(p, mark_map)
         p('}\n')
-        if rmap[0xfe0e] != 1275:
+        if rmap[0xfe0e] != 1281:
             raise ValueError('The mark for 0xfe0e has changed, you have to update VS15 to {} and VS16 to {} in unicode-data.h'.format(
                 rmap[0xfe0e], rmap[0xfe0f]
             ))
-    with open('kittens/url_hints/url_regex.py', 'w') as f:
-        f.write("url_delimiters = '{}'  # noqa".format(''.join(classes_to_regex(cz))))
+    with open('kittens/hints/url_regex.py', 'w') as f:
+        f.write("url_delimiters = '{}'  # noqa".format(''.join(classes_to_regex(cz, exclude='\n'))))
 
 
 def gen_names():
-    words = tuple(sorted(word_search_map))
-
     with create_header('kittens/unicode_input/names.h') as p:
-        cp_map = list(sorted(name_map))
-        p(f'static const char* name_map[{len(cp_map)}] = {{' ' // {{{')
-        for cp in cp_map:
+        mark_to_cp = list(sorted(name_map))
+        cp_to_mark = {cp: m for m, cp in enumerate(mark_to_cp)}
+        # Mapping of mark to codepoint name
+        p(f'static const char* name_map[{len(mark_to_cp)}] = {{' ' // {{{')
+        for cp in mark_to_cp:
             w = name_map[cp].replace('"', '\\"')
             p(f'\t"{w}",')
         p("}; // }}}\n")
 
-        p(f'static const char* idx_to_word[{len(words)}] = ' '{ // {{{')
-        for s in words:
-            s = s.replace('"', '\\"')
-            p(f'\t"{s}",')
-        p("}; // }}}\n")
-
-        first_letters = {ord(w[0]) for w in words if ord(w[0]) < 256}
-        wmap = {w: i for i, w in enumerate(words)}
-        p(f'static const unsigned short* words_for_first_letter[256] = ' '{ // {{{')
-        for fl in range(0, 256):
-            if fl in first_letters:
-                winds = [str(wmap[w]) for w in words if w.startswith(chr(fl))]
-                p(f'\t(const unsigned short[{len(winds) + 1}]){{{len(winds)}, ', ', '.join(winds), '},')
-            else:
-                p('NULL,')
-        p("}; // }}}\n")
-
-        p(f'static const char_type* codepoints_for_word_idx[{len(words)}] = ' '{ // {{{')
-        for s in words:
-            cps = word_search_map[s]
-            a = ', '.join(map(str, cps))
-            p(f'\t(const char_type[{len(cps) + 1}]){{{len(cps)}, ', a, '},')
-        p("}; // }}}\n")
+        # Mapping of mark to codepoint
+        p(f'static const char_type mark_to_cp[{len(mark_to_cp)}] = {{' ' // {{{')
+        p(', '.join(map(str, mark_to_cp)))
+        p('}; // }}}\n')
 
+        # Function to get mark number for codepoint
         p('static char_type mark_for_codepoint(char_type c) {')
-        codepoint_to_mark_map(p, cp_map)
+        codepoint_to_mark_map(p, mark_to_cp)
         p('}\n')
         p('static inline const char* name_for_codepoint(char_type cp) {')
         p('\tchar_type m = mark_for_codepoint(cp); if (m == 0) return NULL;')
         p('\treturn name_map[m];')
         p('}\n')
 
+        # Array of all words
+        word_map = tuple(sorted(word_search_map))
+        word_rmap = {w: i for i, w in enumerate(word_map)}
+        p(f'static const char* all_words_map[{len(word_map)}] = {{' ' // {{{')
+        cwords = (w.replace('"', '\\"') for w in word_map)
+        p(', '.join(f'"{w}"' for w in cwords))
+        p('}; // }}}\n')
+
+        # Array of sets of marks for each word
+        word_to_marks = {word_rmap[w]: frozenset(map(cp_to_mark.__getitem__, cps)) for w, cps in word_search_map.items()}
+        all_mark_groups = frozenset(word_to_marks.values())
+        array = [0]
+        mg_to_offset = {}
+        for mg in all_mark_groups:
+            mg_to_offset[mg] = len(array)
+            array.append(len(mg))
+            array.extend(sorted(mg))
+        p(f'static const char_type mark_groups[{len(array)}] = {{' ' // {{{')
+        p(', '.join(map(str, array)))
+        p('}; // }}}\n')
+        offsets_array = []
+        for wi, w in enumerate(word_map):
+            mg = word_to_marks[wi]
+            offsets_array.append(mg_to_offset[mg])
+        p(f'static const char_type mark_to_offset[{len(offsets_array)}] = {{' ' // {{{')
+        p(', '.join(map(str, offsets_array)))
+        p('}; // }}}\n')
+
+        # The trie
+        p(f'typedef struct {{ uint32_t children_offset; uint32_t match_offset; }} word_trie;\n')
+        all_trie_nodes = []
+
+        class TrieNode:
+
+            def __init__(self):
+                self.match_offset = 0
+                self.children_offset = 0
+                self.children = {}
+
+            def add_letter(self, letter):
+                if letter not in self.children:
+                    self.children[letter] = len(all_trie_nodes)
+                    all_trie_nodes.append(TrieNode())
+                return self.children[letter]
+
+            def __str__(self):
+                return f'{{ .children_offset={self.children_offset}, .match_offset={self.match_offset} }}'
+
+        root = TrieNode()
+        all_trie_nodes.append(root)
+
+        def add_word(word_idx):
+            word = word_map[word_idx]
+            parent = root
+            for letter in map(ord, word):
+                idx = parent.add_letter(letter)
+                parent = all_trie_nodes[idx]
+            parent.match_offset = offsets_array[word_idx]
+
+        for i in range(len(word_map)):
+            add_word(i)
+        children_array = [0]
+        for node in all_trie_nodes:
+            if node.children:
+                node.children_offset = len(children_array)
+                children_array.append(len(node.children))
+                for letter, child_offset in node.children.items():
+                    children_array.append((child_offset << 8) | (letter & 0xff))
+
+        p(f'static const word_trie all_trie_nodes[{len(all_trie_nodes)}] = {{' ' // {{{')
+        p(',\n'.join(map(str, all_trie_nodes)))
+        p('\n}; // }}}\n')
+        p(f'static const uint32_t children_array[{len(children_array)}] = {{' ' // {{{')
+        p(', '.join(map(str, children_array)))
+        p('}; // }}}\n')
+
 
 def gen_wcwidth():
     seen = set()
@@ -329,7 +411,7 @@ def gen_wcwidth():
         add(p, 'East Asian double width', doublewidth, 2)
         add(p, 'Emoji Presentation', emoji_categories['Emoji_Presentation'], 2)
 
-        add(p, 'Not assigned in the unicode character database', not_assigned, -1)
+        add(p, 'Not assigned in the unicode character database', not_assigned, -4)
 
         p('\t\tdefault: return 1;')
         p('\t}')

glad/generate.py

@@ -27,6 +27,10 @@ def regenerate():
     )
 
 
+def strip_trailing_whitespace(c):
+    return re.sub(r'\s+$', '', c, flags=re.MULTILINE) + '\n'
+
+
 def export():
     c = open('out/src/glad.c', 'rb').read().decode('utf-8')
     functions = []
@@ -48,12 +52,13 @@ init_glad(GLADloadproc load, int debug) {
     }
     return ret;
 }'''.replace('SUB', ';\n        '.join(switch), 1)
-    open('../kitty/gl-wrapper.c', 'w').write(c)
+    open('../kitty/gl-wrapper.c', 'w').write(strip_trailing_whitespace(c))
     raw = open('out/include/glad/glad.h').read()
     raw = raw.replace('<KHR/khrplatform.h>', '"khrplatform.h"')
     raw += '\nint init_glad(GLADloadproc, int);\n'
-    open('../kitty/gl-wrapper.h', 'w').write(raw)
-    shutil.copy2('out/include/KHR/khrplatform.h', '../kitty')
+    open('../kitty/gl-wrapper.h', 'w').write(strip_trailing_whitespace(raw))
+    raw = open('out/include/KHR/khrplatform.h', 'rb').read().decode('utf-8')
+    open('../kitty/khrplatform.h', 'w').write(strip_trailing_whitespace(raw))
 
 
 if __name__ == '__main__':

glfw/backend_utils.c

@@ -0,0 +1,263 @@
+/*
+ * backend_utils.c
+ * Copyright (C) 2018 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#define _GNU_SOURCE
+#include "backend_utils.h"
+#include "internal.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <float.h>
+#include <time.h>
+#include <stdio.h>
+
+#ifdef __NetBSD__
+#define ppoll pollts
+#endif
+
+static inline double
+monotonic() {
+    struct timespec ts = {0};
+#ifdef CLOCK_HIGHRES
+	clock_gettime(CLOCK_HIGHRES, &ts);
+#elif CLOCK_MONOTONIC_RAW
+	clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
+#else
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+#endif
+	return (((double)ts.tv_nsec) / 1e9) + (double)ts.tv_sec;
+}
+
+void
+update_fds(EventLoopData *eld) {
+    for (nfds_t i = 0; i < eld->watches_count; i++) {
+        Watch *w = eld->watches + i;
+        eld->fds[i].fd = w->fd;
+        eld->fds[i].events = w->enabled ? w->events : 0;
+    }
+}
+
+static id_type watch_counter = 0;
+
+id_type
+addWatch(EventLoopData *eld, const char* name, int fd, int events, int enabled, watch_callback_func cb, void *cb_data) {
+    if (eld->watches_count >= sizeof(eld->watches)/sizeof(eld->watches[0])) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Too many watches added");
+        return 0;
+    }
+    Watch *w = eld->watches + eld->watches_count++;
+    w->name = name;
+    w->fd = fd; w->events = events; w->enabled = enabled;
+    w->callback = cb;
+    w->callback_data = cb_data;
+    w->id = ++watch_counter;
+    update_fds(eld);
+    return w->id;
+}
+
+#define removeX(which, item_id, update_func) {\
+    for (nfds_t i = 0; i < eld->which##_count; i++) { \
+        if (eld->which[i].id == item_id) { \
+            eld->which##_count--; \
+            if (i < eld->which##_count) { \
+                memmove(eld->which + i, eld->which + i + 1, sizeof(eld->which[0]) * (eld->which##_count - i)); \
+            } \
+            update_func(eld); break; \
+}}}
+
+void
+removeWatch(EventLoopData *eld, id_type watch_id) {
+    removeX(watches, watch_id, update_fds);
+}
+
+void
+toggleWatch(EventLoopData *eld, id_type watch_id, int enabled) {
+    for (nfds_t i = 0; i < eld->watches_count; i++) {
+        if (eld->watches[i].id == watch_id) {
+            if (eld->watches[i].enabled != enabled) {
+                eld->watches[i].enabled = enabled;
+                update_fds(eld);
+            }
+            break;
+        }
+    }
+}
+
+static id_type timer_counter = 0;
+
+static int
+compare_timers(const void *a_, const void *b_) {
+    const Timer *a = (const Timer*)a_, *b = (const Timer*)b_;
+    return (a->trigger_at > b->trigger_at) ? 1 : (a->trigger_at < b->trigger_at) ? -1 : 0;
+}
+
+static inline void
+update_timers(EventLoopData *eld) {
+    if (eld->timers_count > 1) qsort(eld->timers, eld->timers_count, sizeof(eld->timers[0]), compare_timers);
+}
+
+id_type
+addTimer(EventLoopData *eld, const char *name, double interval, int enabled, timer_callback_func cb, void *cb_data) {
+    if (eld->timers_count >= sizeof(eld->timers)/sizeof(eld->timers[0])) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Too many timers added");
+        return 0;
+    }
+    Timer *t = eld->timers + eld->timers_count++;
+    t->interval = interval;
+    t->name = name;
+    t->trigger_at = enabled ? monotonic() + interval : DBL_MAX;
+    t->callback = cb;
+    t->callback_data = cb_data;
+    t->id = ++timer_counter;
+    update_timers(eld);
+    return t->id;
+}
+
+void
+removeTimer(EventLoopData *eld, id_type timer_id) {
+    removeX(timers, timer_id, update_timers);
+}
+
+void
+toggleTimer(EventLoopData *eld, id_type timer_id, int enabled) {
+    for (nfds_t i = 0; i < eld->timers_count; i++) {
+        if (eld->timers[i].id == timer_id) {
+            double trigger_at = enabled ? (monotonic() + eld->timers[i].interval) : DBL_MAX;
+            if (trigger_at != eld->timers[i].trigger_at) {
+                eld->timers[i].trigger_at = trigger_at;
+                update_timers(eld);
+            }
+            break;
+        }
+    }
+}
+
+void
+changeTimerInterval(EventLoopData *eld, id_type timer_id, double interval) {
+    for (nfds_t i = 0; i < eld->timers_count; i++) {
+        if (eld->timers[i].id == timer_id) {
+            eld->timers[i].interval = interval;
+            break;
+        }
+    }
+}
+
+
+double
+prepareForPoll(EventLoopData *eld, double timeout) {
+    for (nfds_t i = 0; i < eld->watches_count; i++) eld->fds[i].revents = 0;
+    if (!eld->timers_count || eld->timers[0].trigger_at == DBL_MAX) return timeout;
+    double now = monotonic(), next_repeat_at = eld->timers[0].trigger_at;
+    if (timeout < 0 || now + timeout > next_repeat_at) {
+        timeout = next_repeat_at <= now ? 0 : next_repeat_at - now;
+    }
+    return timeout;
+}
+
+int
+pollWithTimeout(struct pollfd *fds, nfds_t nfds, double timeout) {
+    const long seconds = (long) timeout;
+    const long nanoseconds = (long) ((timeout - seconds) * 1e9);
+    struct timespec tv = { seconds, nanoseconds };
+    return ppoll(fds, nfds, &tv, NULL);
+}
+
+static void
+dispatchEvents(EventLoopData *eld) {
+    for (nfds_t i = 0; i < eld->watches_count; i++) {
+        Watch *ww = eld->watches + i;
+        struct pollfd *pfd = eld->fds + i;
+        if (pfd->revents & ww->events) {
+            ww->ready = 1;
+            if (ww->callback) ww->callback(ww->fd, pfd->revents, ww->callback_data);
+        } else ww->ready = 0;
+    }
+}
+
+unsigned
+dispatchTimers(EventLoopData *eld) {
+    if (!eld->timers_count || eld->timers[0].trigger_at == DBL_MAX) return 0;
+    static struct { timer_callback_func func; id_type id; void* data; } dispatches[sizeof(eld->timers)/sizeof(eld->timers[0])];
+    unsigned num_dispatches = 0;
+    double now = monotonic();
+    for (nfds_t i = 0; i < eld->timers_count && eld->timers[i].trigger_at <= now; i++) {
+        eld->timers[i].trigger_at = now + eld->timers[i].interval;
+        dispatches[num_dispatches].func = eld->timers[i].callback;
+        dispatches[num_dispatches].id = eld->timers[i].id;
+        dispatches[num_dispatches].data = eld->timers[i].callback_data;
+        num_dispatches++;
+    }
+    // we dispatch separately so that the callbacks can modify timers
+    for (unsigned i = 0; i < num_dispatches; i++) {
+        dispatches[i].func(dispatches[i].id, dispatches[i].data);
+    }
+    if (num_dispatches) update_timers(eld);
+    return num_dispatches;
+}
+
+static void
+drain_wakeup_fd(int fd, int events, void* data) {
+    static char drain_buf[64];
+    while(read(fd, drain_buf, sizeof(drain_buf)) < 0 && errno == EINTR);
+}
+
+void
+initPollData(EventLoopData *eld, int wakeup_fd, int display_fd) {
+    addWatch(eld, "display", display_fd, POLLIN, 1, NULL, NULL);
+    addWatch(eld, "wakeup", wakeup_fd, POLLIN, 1, drain_wakeup_fd, NULL);
+}
+
+
+int
+pollForEvents(EventLoopData *eld, double timeout) {
+    int read_ok = 0;
+    timeout = prepareForPoll(eld, timeout);
+    int result;
+    double end_time = monotonic() + timeout;
+
+    while(1) {
+        if (timeout >= 0) {
+            errno = 0;
+            result = pollWithTimeout(eld->fds, eld->watches_count, timeout);
+            int saved_errno = errno;
+            dispatchTimers(eld);
+            if (result > 0) {
+                dispatchEvents(eld);
+                read_ok = eld->watches[0].ready;
+                break;
+            }
+            timeout = end_time - monotonic();
+            if (timeout <= 0) break;
+            if (result < 0 && (saved_errno == EINTR || saved_errno == EAGAIN)) continue;
+            break;
+        } else {
+            errno = 0;
+            result = poll(eld->fds, eld->watches_count, -1);
+            int saved_errno = errno;
+            dispatchTimers(eld);
+            if (result > 0) {
+                dispatchEvents(eld);
+                read_ok = eld->watches[0].ready;
+            }
+            if (result < 0 && (saved_errno == EINTR || saved_errno == EAGAIN)) continue;
+            break;
+        }
+    }
+    return read_ok;
+}
+
+void
+closeFds(int *fds, size_t count) {
+    while(count--) {
+        if (*fds > 0) {
+            close(*fds);
+            *fds = -1;
+        }
+        fds++;
+    }
+}

glfw/backend_utils.h

@@ -0,0 +1,73 @@
+//========================================================================
+// GLFW 3.3 Wayland - www.glfw.org
+//------------------------------------------------------------------------
+// Copyright (c) 2014 Jonas Ådahl <jadahl@gmail.com>
+//
+// This software is provided 'as-is', without any express or implied
+// warranty. In no event will the authors be held liable for any damages
+// arising from the use of this software.
+//
+// Permission is granted to anyone to use this software for any purpose,
+// including commercial applications, and to alter it and redistribute it
+// freely, subject to the following restrictions:
+//
+// 1. The origin of this software must not be misrepresented; you must not
+//    claim that you wrote the original software. If you use this software
+//    in a product, an acknowledgment in the product documentation would
+//    be appreciated but is not required.
+//
+// 2. Altered source versions must be plainly marked as such, and must not
+//    be misrepresented as being the original software.
+//
+// 3. This notice may not be removed or altered from any source
+//    distribution.
+//
+//========================================================================
+
+#pragma once
+#include <poll.h>
+#include <unistd.h>
+
+typedef unsigned long long id_type;
+typedef void(*watch_callback_func)(int, int, void*);
+typedef void(*timer_callback_func)(id_type, void*);
+
+typedef struct {
+    int fd, events, enabled, ready;
+    watch_callback_func callback;
+    void *callback_data;
+    id_type id;
+    const char *name;
+} Watch;
+
+typedef struct {
+    id_type id;
+    double interval, trigger_at;
+    timer_callback_func callback;
+    void *callback_data;
+    const char *name;
+} Timer;
+
+
+typedef struct {
+    struct pollfd fds[32];
+    int wakeupFds[2];
+    nfds_t watches_count, timers_count;
+    Watch watches[32];
+    Timer timers[128];
+} EventLoopData;
+
+
+id_type addWatch(EventLoopData *eld, const char *name, int fd, int events, int enabled, watch_callback_func cb, void *cb_data);
+void removeWatch(EventLoopData *eld, id_type watch_id);
+void toggleWatch(EventLoopData *eld, id_type watch_id, int enabled);
+id_type addTimer(EventLoopData *eld, const char *name, double interval, int enabled, timer_callback_func cb, void *cb_data);
+void removeTimer(EventLoopData *eld, id_type timer_id);
+void toggleTimer(EventLoopData *eld, id_type timer_id, int enabled);
+void changeTimerInterval(EventLoopData *eld, id_type timer_id, double interval);
+double prepareForPoll(EventLoopData *eld, double timeout);
+int pollWithTimeout(struct pollfd *fds, nfds_t nfds, double timeout);
+int pollForEvents(EventLoopData *eld, double timeout);
+unsigned dispatchTimers(EventLoopData *eld);
+void closeFds(int *fds, size_t count);
+void initPollData(EventLoopData *eld, int wakeup_fd, int display_fd);

glfw/cocoa_init.m

@@ -69,7 +69,6 @@ static void changeToResourcesDirectory(void)
 static void createKeyTables(void)
 {
     int scancode;
-    _glfw.ns.debug_keyboard = getenv("GLFW_DEBUG_KEYBOARD") != NULL;
 
     memset(_glfw.ns.keycodes, -1, sizeof(_glfw.ns.keycodes));
     memset(_glfw.ns.scancodes, -1, sizeof(_glfw.ns.scancodes));
@@ -208,6 +207,9 @@ static GLFWbool updateUnicodeDataNS(void)
         _glfw.ns.unicodeData = nil;
     }
 
+    for (_GLFWwindow *window = _glfw.windowListHead;  window;  window = window->next)
+        window->ns.deadKeyState = 0;
+
     _glfw.ns.inputSource = TISCopyCurrentKeyboardLayoutInputSource();
     if (!_glfw.ns.inputSource)
     {

glfw/cocoa_platform.h

@@ -37,6 +37,8 @@ typedef void* id;
 #endif
 
 typedef VkFlags VkMacOSSurfaceCreateFlagsMVK;
+typedef int (* GLFWcocoatextinputfilterfun)(int,int,int);
+typedef int (* GLFWapplicationshouldhandlereopenfun)(int);
 
 typedef struct VkMacOSSurfaceCreateInfoMVK
 {
@@ -98,6 +100,10 @@ typedef struct _GLFWwindowNS
     // This is kept to counteract Cocoa doing the same internally
     double          cursorWarpDeltaX, cursorWarpDeltaY;
 
+    // The text input filter callback
+    GLFWcocoatextinputfilterfun textInputFilterCallback;
+    // Dead key state
+    UInt32 deadKeyState;
 } _GLFWwindowNS;
 
 // Cocoa-specific global data
@@ -115,7 +121,6 @@ typedef struct _GLFWlibraryNS
 
     char                keyName[64];
     char                text[256];
-    GLFWbool            debug_keyboard;
     short int           keycodes[256];
     short int           scancodes[GLFW_KEY_LAST + 1];
     char*               clipboardString;

glfw/cocoa_window.m

@@ -128,6 +128,32 @@ static void updateCursorImage(_GLFWwindow* window)
         hideCursor(window);
 }
 
+// Apply chosen cursor mode to a focused window
+//
+static void updateCursorMode(_GLFWwindow* window)
+{
+    if (window->cursorMode == GLFW_CURSOR_DISABLED)
+    {
+        _glfw.ns.disabledCursorWindow = window;
+        _glfwPlatformGetCursorPos(window,
+                                  &_glfw.ns.restoreCursorPosX,
+                                  &_glfw.ns.restoreCursorPosY);
+        centerCursor(window);
+        CGAssociateMouseAndMouseCursorPosition(false);
+    }
+    else if (_glfw.ns.disabledCursorWindow == window)
+    {
+        _glfw.ns.disabledCursorWindow = NULL;
+        CGAssociateMouseAndMouseCursorPosition(true);
+        _glfwPlatformSetCursorPos(window,
+                                  _glfw.ns.restoreCursorPosX,
+                                  _glfw.ns.restoreCursorPosY);
+    }
+
+    if (cursorInClientArea(window))
+        updateCursorImage(window);
+}
+
 // Transforms the specified y-coordinate between the CG display and NS screen
 // coordinate systems
 //
@@ -183,7 +209,7 @@ static int translateFlags(NSUInteger flags)
     return mods;
 }
 
-#define debug_key(...) if (_glfw.ns.debug_keyboard) NSLog(__VA_ARGS__)
+#define debug_key(...) if (_glfw.hints.init.debugKeyboard) NSLog(__VA_ARGS__)
 
 static inline const char*
 format_mods(int mods) {
@@ -423,7 +449,7 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
         centerCursor(window);
 
     _glfwInputWindowFocus(window, GLFW_TRUE);
-    _glfwPlatformSetCursorMode(window, window->cursorMode);
+    updateCursorMode(window);
 }
 
 - (void)windowDidResignKey:(NSNotification *)notification
@@ -456,6 +482,15 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
     return NSTerminateCancel;
 }
 
+static GLFWapplicationshouldhandlereopenfun handle_reopen_callback = NULL;
+
+- (BOOL)applicationShouldHandleReopen:(NSApplication *)sender hasVisibleWindows:(BOOL)flag
+{
+    if (!handle_reopen_callback) return YES;
+    if (handle_reopen_callback(flag)) return YES;
+    return NO;
+}
+
 - (void)applicationDidChangeScreenParameters:(NSNotification *) notification
 {
     _GLFWwindow* window;
@@ -704,7 +739,7 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
     }
 
     const NSTrackingAreaOptions options = NSTrackingMouseEnteredAndExited |
-                                          NSTrackingActiveInKeyWindow |
+                                          NSTrackingActiveAlways |
                                           NSTrackingEnabledDuringMouseDrag |
                                           NSTrackingCursorUpdate |
                                           NSTrackingInVisibleRect |
@@ -719,18 +754,98 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
     [super updateTrackingAreas];
 }
 
+static inline UInt32
+convert_cocoa_to_carbon_modifiers(NSUInteger flags) {
+    UInt32 mods = 0;
+    if (flags & NSEventModifierFlagShift)
+        mods |= shiftKey;
+    if (flags & NSEventModifierFlagControl)
+        mods |= controlKey;
+    if (flags & NSEventModifierFlagOption)
+        mods |= optionKey;
+    if (flags & NSEventModifierFlagCommand)
+        mods |= cmdKey;
+    if (flags & NSEventModifierFlagCapsLock)
+        mods |= alphaLock;
+
+    return (mods >> 8) & 0xFF;
+}
+
+static inline void
+convert_utf16_to_utf8(UniChar *src, UniCharCount src_length, char *dest, size_t dest_sz) {
+    CFStringRef string = CFStringCreateWithCharactersNoCopy(kCFAllocatorDefault,
+                                                            src,
+                                                            src_length,
+                                                            kCFAllocatorNull);
+    CFStringGetCString(string,
+                       dest,
+                       dest_sz,
+                       kCFStringEncodingUTF8);
+    CFRelease(string);
+}
+
+static inline GLFWbool
+is_ascii_control_char(char x) {
+    return x == 0 || (1 <= x && x <= 31) || x == 127;
+}
+
 - (void)keyDown:(NSEvent *)event
 {
     const unsigned int scancode = [event keyCode];
+    const NSUInteger flags = [event modifierFlags];
+    const int mods = translateFlags(flags);
     const int key = translateKey(scancode, GLFW_TRUE);
-    const int mods = translateFlags([event modifierFlags]);
+    const GLFWbool process_text = !window->ns.textInputFilterCallback || window->ns.textInputFilterCallback(key, mods, scancode) != 1;
     _glfw.ns.text[0] = 0;
-    // this will call insertText with the text for this event, if any
-    [self interpretKeyEvents:[NSArray arrayWithObject:event]];
-    if ((1 <= _glfw.ns.text[0] && _glfw.ns.text[0] <= 31) || (unsigned)_glfw.ns.text[0] == 127) _glfw.ns.text[0] = 0;  // dont send text for ascii control codes
-    debug_key(@"scancode: 0x%x (%s) mods: %stext: %s glfw_key: %s\n",
+    if (!_glfw.ns.unicodeData) {
+        // Using the cocoa API for key handling is disabled, as there is no
+        // reliable way to handle dead keys using it. Only use it if the
+        // keyboard unicode data is not available.
+        if (process_text) {
+            // this will call insertText with the text for this event, if any
+            [self interpretKeyEvents:[NSArray arrayWithObject:event]];
+        }
+    } else {
+        static UniChar text[256];
+        UniCharCount char_count = 0;
+        if (UCKeyTranslate(
+                    [(NSData*) _glfw.ns.unicodeData bytes],
+                    scancode,
+                    kUCKeyActionDown,
+                    convert_cocoa_to_carbon_modifiers(flags),
+                    LMGetKbdType(),
+                    (process_text ? 0 : kUCKeyTranslateNoDeadKeysMask),
+                    &(window->ns.deadKeyState),
+                    sizeof(text)/sizeof(text[0]),
+                    &char_count,
+                    text
+                    ) != noErr) {
+            debug_key(@"UCKeyTranslate failed for scancode: 0x%x (%s) %s\n", scancode, safe_name_for_scancode(scancode), format_mods(mods));
+            window->ns.deadKeyState = 0;
+            return;
+        }
+        if (process_text) {
+            // We check if cocoa wants to insert text, as UCKeyTranslate
+            // inserts text even when the cmd key is pressed. For instance,
+            // cmd+a will result in the text a.
+            [self interpretKeyEvents:[NSArray arrayWithObject:event]];
+            debug_key(@"char_count: %lu cocoa text: %s\n", char_count, format_text(_glfw.ns.text));
+            GLFWbool cocoa_wants_to_insert_text = !is_ascii_control_char(_glfw.ns.text[0]);
+            _glfw.ns.text[0] = 0;
+            if (char_count && cocoa_wants_to_insert_text) convert_utf16_to_utf8(text, char_count, _glfw.ns.text, sizeof(_glfw.ns.text));
+        } else {
+            window->ns.deadKeyState = 0;
+        }
+        if (window->ns.deadKeyState && char_count == 0) {
+            debug_key(@"Ignoring dead key. deadKeyState: 0x%x scancode: 0x%x (%s) %s\n",
+                    window->ns.deadKeyState, scancode, safe_name_for_scancode(scancode), format_mods(mods));
+            return;
+        }
+    }
+    debug_key(@"scancode: 0x%x (%s) %stext: %s glfw_key: %s\n",
             scancode, safe_name_for_scancode(scancode), format_mods(mods),
             format_text(_glfw.ns.text), _glfwGetKeyName(key));
+    if (is_ascii_control_char(_glfw.ns.text[0])) _glfw.ns.text[0] = 0;  // don't send text for ascii control codes
     _glfwInputKeyboard(window, key, scancode, GLFW_PRESS, mods, _glfw.ns.text, 0);
 }
 
@@ -769,15 +884,10 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
 
     deltaX = [event scrollingDeltaX];
     deltaY = [event scrollingDeltaY];
-
-    if ([event hasPreciseScrollingDeltas])
-    {
-        deltaX *= 0.1;
-        deltaY *= 0.1;
-    }
+    int flags = [event hasPreciseScrollingDeltas] ? 1 : 0;
 
     if (fabs(deltaX) > 0.0 || fabs(deltaY) > 0.0)
-        _glfwInputScroll(window, deltaX, deltaY);
+        _glfwInputScroll(window, deltaX, deltaY, flags);
 }
 
 - (NSDragOperation)draggingEntered:(id <NSDraggingInfo>)sender
@@ -946,18 +1056,39 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
 
 @implementation GLFWApplication
 
-// From http://cocoadev.com/index.pl?GameKeyboardHandlingAlmost
-// This works around an AppKit bug, where key up events while holding
-// down the command key don't get sent to the key window.
 - (void)sendEvent:(NSEvent *)event
 {
-    if ([event type] == NSEventTypeKeyUp &&
-        ([event modifierFlags] & NSEventModifierFlagCommand))
-    {
-        [[self keyWindow] sendEvent:event];
+    NSEventType etype = [event type];
+    NSEventModifierFlags flags;
+    switch(etype) {
+        case NSEventTypeKeyUp:
+            flags = [event modifierFlags] & NSEventModifierFlagDeviceIndependentFlagsMask;
+            if (flags & NSEventModifierFlagCommand) {
+                // From http://cocoadev.com/index.pl?GameKeyboardHandlingAlmost
+                // This works around an AppKit bug, where key up events while holding
+                // down the command key don't get sent to the key window.
+                [[self keyWindow] sendEvent:event];
+                return;
+            }
+            if (event.keyCode == kVK_Tab && (flags == NSEventModifierFlagControl || flags == (NSEventModifierFlagControl | NSEventModifierFlagShift))) {
+                // Cocoa swallows Ctrl+Tab to cycle between views
+                [[self keyWindow].contentView keyUp:event];
+                return;
+            }
+            break;
+        case NSEventTypeKeyDown:
+            flags = [event modifierFlags] & NSEventModifierFlagDeviceIndependentFlagsMask;
+            if (event.keyCode == kVK_Tab && (flags == NSEventModifierFlagControl || flags == (NSEventModifierFlagControl | NSEventModifierFlagShift))) {
+                // Cocoa swallows Ctrl+Tab to cycle between views
+                [[self keyWindow].contentView keyDown:event];
+                return;
+            }
+            break;
+        default:
+            break;
     }
-    else
-        [super sendEvent:event];
+
+    [super sendEvent:event];
 }
 
 
@@ -968,7 +1099,7 @@ static const NSRange kEmptyRange = { NSNotFound, 0 };
 }
 
 - (void)loadMainMenu
-{ // removed by Kovid as it generated compiler warnings 
+{ // removed by Kovid as it generated compiler warnings
 }
 
 @end
@@ -1233,6 +1364,7 @@ int _glfwPlatformCreateWindow(_GLFWwindow* window,
                               const _GLFWctxconfig* ctxconfig,
                               const _GLFWfbconfig* fbconfig)
 {
+    window->ns.deadKeyState = 0;
     if (!initializeAppKit())
         return GLFW_FALSE;
 
@@ -1732,26 +1864,8 @@ void _glfwPlatformSetCursorPos(_GLFWwindow* window, double x, double y)
 
 void _glfwPlatformSetCursorMode(_GLFWwindow* window, int mode)
 {
-    if (mode == GLFW_CURSOR_DISABLED)
-    {
-        _glfw.ns.disabledCursorWindow = window;
-        _glfwPlatformGetCursorPos(window,
-                                  &_glfw.ns.restoreCursorPosX,
-                                  &_glfw.ns.restoreCursorPosY);
-        centerCursor(window);
-        CGAssociateMouseAndMouseCursorPosition(false);
-    }
-    else if (_glfw.ns.disabledCursorWindow == window)
-    {
-        _glfw.ns.disabledCursorWindow = NULL;
-        CGAssociateMouseAndMouseCursorPosition(true);
-        _glfwPlatformSetCursorPos(window,
-                                  _glfw.ns.restoreCursorPosX,
-                                  _glfw.ns.restoreCursorPosY);
-    }
-
-    if (cursorInClientArea(window))
-        updateCursorImage(window);
+    if (_glfwPlatformWindowFocused(window))
+        updateCursorMode(window);
 }
 
 const char* _glfwPlatformGetScancodeName(int scancode)
@@ -1777,16 +1891,7 @@ const char* _glfwPlatformGetScancodeName(int scancode)
     if (!characterCount)
         return NULL;
 
-    CFStringRef string = CFStringCreateWithCharactersNoCopy(kCFAllocatorDefault,
-                                                            characters,
-                                                            characterCount,
-                                                            kCFAllocatorNull);
-    CFStringGetCString(string,
-                       _glfw.ns.keyName,
-                       sizeof(_glfw.ns.keyName),
-                       kCFStringEncodingUTF8);
-    CFRelease(string);
-
+    convert_utf16_to_utf8(characters, characterCount, _glfw.ns.keyName, sizeof(_glfw.ns.keyName));
     return _glfw.ns.keyName;
 }
 
@@ -1797,44 +1902,44 @@ int _glfwPlatformGetKeyScancode(int key)
 
 int _glfwPlatformCreateCursor(_GLFWcursor* cursor,
                               const GLFWimage* image,
-                              int xhot, int yhot)
+                              int xhot, int yhot, int count)
 {
     NSImage* native;
     NSBitmapImageRep* rep;
 
     if (!initializeAppKit())
         return GLFW_FALSE;
-
-    rep = [[NSBitmapImageRep alloc]
-        initWithBitmapDataPlanes:NULL
-                      pixelsWide:image->width
-                      pixelsHigh:image->height
-                   bitsPerSample:8
-                 samplesPerPixel:4
-                        hasAlpha:YES
-                        isPlanar:NO
-                  colorSpaceName:NSCalibratedRGBColorSpace
-                    bitmapFormat:NSAlphaNonpremultipliedBitmapFormat
-                     bytesPerRow:image->width * 4
-                    bitsPerPixel:32];
-
-    if (rep == nil)
+    native = [[NSImage alloc] initWithSize:NSMakeSize(image->width, image->height)];
+    if (native == nil)
         return GLFW_FALSE;
 
-    memcpy([rep bitmapData], image->pixels, image->width * image->height * 4);
+    for (int i = 0; i < count; i++) {
+        const GLFWimage *src = image + i;
+        rep = [[NSBitmapImageRep alloc]
+            initWithBitmapDataPlanes:NULL
+                        pixelsWide:src->width
+                        pixelsHigh:src->height
+                    bitsPerSample:8
+                    samplesPerPixel:4
+                            hasAlpha:YES
+                            isPlanar:NO
+                    colorSpaceName:NSCalibratedRGBColorSpace
+                        bitmapFormat:NSAlphaNonpremultipliedBitmapFormat
+                        bytesPerRow:src->width * 4
+                        bitsPerPixel:32];
+        if (rep == nil)
+            return GLFW_FALSE;
 
-    native = [[NSImage alloc] initWithSize:NSMakeSize(image->width, image->height)];
-    [native addRepresentation:rep];
+        memcpy([rep bitmapData], src->pixels, src->width * src->height * 4);
+        [native addRepresentation:rep];
+        [rep release];
+    }
 
     cursor->ns.object = [[NSCursor alloc] initWithImage:native
                                                 hotSpot:NSMakePoint(xhot, yhot)];
-
     [native release];
-    [rep release];
-
     if (cursor->ns.object == nil)
         return GLFW_FALSE;
-
     return GLFW_TRUE;
 }
 
@@ -2000,3 +2105,110 @@ GLFWAPI id glfwGetCocoaWindow(GLFWwindow* handle)
     _GLFW_REQUIRE_INIT_OR_RETURN(nil);
     return window->ns.object;
 }
+
+GLFWAPI GLFWcocoatextinputfilterfun glfwSetCocoaTextInputFilter(GLFWwindow *handle, GLFWcocoatextinputfilterfun callback) {
+    _GLFWwindow* window = (_GLFWwindow*) handle;
+    _GLFW_REQUIRE_INIT_OR_RETURN(nil);
+    GLFWcocoatextinputfilterfun previous = window->ns.textInputFilterCallback;
+    window->ns.textInputFilterCallback = callback;
+    return previous;
+}
+
+GLFWAPI GLFWapplicationshouldhandlereopenfun glfwSetApplicationShouldHandleReopen(GLFWapplicationshouldhandlereopenfun callback) {
+    GLFWapplicationshouldhandlereopenfun previous = handle_reopen_callback;
+    handle_reopen_callback = callback;
+    return previous;
+}
+
+GLFWAPI void glfwGetCocoaKeyEquivalent(int glfw_key, int glfw_mods, unsigned short *cocoa_key, int *cocoa_mods) {
+    *cocoa_key = 0;
+    *cocoa_mods = 0;
+
+    if (glfw_mods & GLFW_MOD_SHIFT)
+        *cocoa_mods |= NSEventModifierFlagShift;
+    if (glfw_mods & GLFW_MOD_CONTROL)
+        *cocoa_mods |= NSEventModifierFlagControl;
+    if (glfw_mods & GLFW_MOD_ALT)
+        *cocoa_mods |= NSEventModifierFlagOption;
+    if (glfw_mods & GLFW_MOD_SUPER)
+        *cocoa_mods |= NSEventModifierFlagCommand;
+    if (glfw_mods & GLFW_MOD_CAPS_LOCK)
+        *cocoa_mods |= NSEventModifierFlagCapsLock;
+
+    switch(glfw_key) {
+#define K(ch, name) case GLFW_KEY_##name: *cocoa_key = ch; break;
+        K('a', A);
+        K('b', B);
+        K('c', C);
+        K('d', D);
+        K('e', E);
+        K('f', F);
+        K('g', G);
+        K('h', H);
+        K('i', I);
+        K('j', J);
+        K('k', K);
+        K('l', L);
+        K('m', M);
+        K('n', N);
+        K('o', O);
+        K('p', P);
+        K('q', Q);
+        K('r', R);
+        K('s', S);
+        K('t', T);
+        K('u', U);
+        K('v', V);
+        K('w', W);
+        K('x', X);
+        K('y', Y);
+        K('z', Z);
+        K('0', 0);
+        K('1', 1);
+        K('2', 2);
+        K('3', 3);
+        K('5', 5);
+        K('6', 6);
+        K('7', 7);
+        K('8', 8);
+        K('9', 9);
+        K('\'', APOSTROPHE);
+        K(',', COMMA);
+        K('.', PERIOD);
+        K('/', SLASH);
+        K('-', MINUS);
+        K('=', EQUAL);
+        K(';', SEMICOLON);
+        K('[', LEFT_BRACKET);
+        K(']', RIGHT_BRACKET);
+        K('`', GRAVE_ACCENT);
+        K('\\', BACKSLASH);
+
+        K(0x35, ESCAPE);
+        K('\r', ENTER);
+        K('\t', TAB);
+        K(NSBackspaceCharacter, BACKSPACE);
+        K(NSInsertFunctionKey, INSERT);
+        K(NSDeleteCharacter, DELETE);
+        K(NSLeftArrowFunctionKey, LEFT);
+        K(NSRightArrowFunctionKey, RIGHT);
+        K(NSUpArrowFunctionKey, UP);
+        K(NSDownArrowFunctionKey, DOWN);
+        K(NSPageUpFunctionKey, PAGE_UP);
+        K(NSPageDownFunctionKey, PAGE_DOWN);
+        K(NSHomeFunctionKey, HOME);
+        K(NSEndFunctionKey, END);
+        K(NSPrintFunctionKey, PRINT_SCREEN);
+        case GLFW_KEY_F1 ... GLFW_KEY_F24:
+            *cocoa_key = NSF1FunctionKey + (glfw_key - GLFW_KEY_F1); break;
+        case GLFW_KEY_KP_0 ... GLFW_KEY_KP_9:
+            *cocoa_key = NSEventModifierFlagNumericPad | (0x52 + (glfw_key - GLFW_KEY_KP_0)); break;
+        K((unichar)(0x41|NSEventModifierFlagNumericPad), KP_DECIMAL);
+        K((unichar)(0x43|NSEventModifierFlagNumericPad), KP_MULTIPLY);
+        K((unichar)(0x45|NSEventModifierFlagNumericPad), KP_ADD);
+        K((unichar)(0x4B|NSEventModifierFlagNumericPad), KP_DIVIDE);
+        K((unichar)(0x4E|NSEventModifierFlagNumericPad), KP_SUBTRACT);
+        K((unichar)(0x51|NSEventModifierFlagNumericPad), KP_EQUAL);
+#undef K
+    }
+}

glfw/dbus_glfw.c

@@ -0,0 +1,295 @@
+//========================================================================
+// GLFW 3.3 XKB - www.glfw.org
+//------------------------------------------------------------------------
+// Copyright (c) 2018 Kovid Goyal <kovid@kovidgoyal.net>
+//
+// This software is provided 'as-is', without any express or implied
+// warranty. In no event will the authors be held liable for any damages
+// arising from the use of this software.
+//
+// Permission is granted to anyone to use this software for any purpose,
+// including commercial applications, and to alter it and redistribute it
+// freely, subject to the following restrictions:
+//
+// 1. The origin of this software must not be misrepresented; you must not
+//    claim that you wrote the original software. If you use this software
+//    in a product, an acknowledgment in the product documentation would
+//    be appreciated but is not required.
+//
+// 2. Altered source versions must be plainly marked as such, and must not
+//    be misrepresented as being the original software.
+//
+// 3. This notice may not be removed or altered from any source
+//    distribution.
+//
+//========================================================================
+
+
+#include "internal.h"
+#include "dbus_glfw.h"
+#include <stdlib.h>
+#include <string.h>
+
+static inline void
+report_error(DBusError *err, const char *fmt, ...) {
+    static char buf[1024];
+    va_list args;
+    va_start(args, fmt);
+    int n = vsnprintf(buf, sizeof(buf), fmt, args);
+    va_end(args);
+    snprintf(buf + n, sizeof(buf), ". DBUS error: %s", err->message);
+    _glfwInputError(GLFW_PLATFORM_ERROR, "%s", buf);
+    dbus_error_free(err);
+}
+
+static _GLFWDBUSData *dbus_data = NULL;
+
+GLFWbool
+glfw_dbus_init(_GLFWDBUSData *dbus, EventLoopData *eld) {
+    dbus->eld = eld;
+    dbus_data = dbus;
+    return GLFW_TRUE;
+}
+
+static void
+on_dbus_watch_ready(int fd, int events, void *data) {
+    DBusWatch *watch = (DBusWatch*)data;
+    unsigned int flags = 0;
+    if (events & POLLERR) flags |= DBUS_WATCH_ERROR;
+    if (events & POLLHUP) flags |= DBUS_WATCH_HANGUP;
+    if (events & POLLIN) flags |= DBUS_WATCH_READABLE;
+    if (events & POLLOUT) flags |= DBUS_WATCH_WRITABLE;
+    dbus_watch_handle(watch, flags);
+}
+
+static inline int
+events_for_watch(DBusWatch *watch) {
+    int events = 0;
+    unsigned int flags = dbus_watch_get_flags(watch);
+    if (flags & DBUS_WATCH_READABLE) events |= POLLIN;
+    if (flags & DBUS_WATCH_WRITABLE) events |= POLLOUT;
+    return events;
+}
+
+static dbus_bool_t
+add_dbus_watch(DBusWatch *watch, void *data) {
+    id_type watch_id = addWatch(dbus_data->eld, data, dbus_watch_get_unix_fd(watch), events_for_watch(watch), dbus_watch_get_enabled(watch), on_dbus_watch_ready, watch);
+    if (!watch_id) return FALSE;
+    id_type *idp = malloc(sizeof(id_type));
+    if (!idp) return FALSE;
+    *idp = watch_id;
+    dbus_watch_set_data(watch, idp, free);
+    return TRUE;
+}
+
+static void
+remove_dbus_watch(DBusWatch *watch, void *data) {
+    id_type *idp = dbus_watch_get_data(watch);
+    if (idp) removeWatch(dbus_data->eld, *idp);
+}
+
+static void
+toggle_dbus_watch(DBusWatch *watch, void *data) {
+    id_type *idp = dbus_watch_get_data(watch);
+    if (idp) toggleWatch(dbus_data->eld, *idp, dbus_watch_get_enabled(watch));
+}
+
+static void
+on_dbus_timer_ready(id_type timer_id, void *data) {
+    if (data) {
+        DBusTimeout *t = (DBusTimeout*)data;
+        dbus_timeout_handle(t);
+    }
+}
+
+
+static dbus_bool_t
+add_dbus_timeout(DBusTimeout *timeout, void *data) {
+    int enabled = dbus_timeout_get_enabled(timeout) ? 1 : 0;
+    double interval = ((double)dbus_timeout_get_interval(timeout)) / 1000.0;
+    if (interval < 0) return FALSE;
+    id_type timer_id = addTimer(dbus_data->eld, data, interval, enabled, on_dbus_timer_ready, timeout);
+    if (!timer_id) return FALSE;
+    id_type *idp = malloc(sizeof(id_type));
+    if (!idp) {
+        removeTimer(dbus_data->eld, timer_id);
+        return FALSE;
+    }
+    *idp = timer_id;
+    dbus_timeout_set_data(timeout, idp, free);
+    return TRUE;
+
+}
+
+static void
+remove_dbus_timeout(DBusTimeout *timeout, void *data) {
+    id_type *idp = dbus_timeout_get_data(timeout);
+    if (idp) removeTimer(dbus_data->eld, *idp);
+}
+
+static void
+toggle_dbus_timeout(DBusTimeout *timeout, void *data) {
+    id_type *idp = dbus_timeout_get_data(timeout);
+    if (idp) toggleTimer(dbus_data->eld, *idp, dbus_timeout_get_enabled(timeout));
+}
+
+
+DBusConnection*
+glfw_dbus_connect_to(const char *path, const char* err_msg, const char *name, GLFWbool register_on_bus) {
+    DBusError err;
+    dbus_error_init(&err);
+    DBusConnection *ans = dbus_connection_open_private(path, &err);
+    if (!ans) {
+        report_error(&err, err_msg);
+        return NULL;
+    }
+    dbus_connection_set_exit_on_disconnect(ans, FALSE);
+    dbus_error_free(&err);
+    if (register_on_bus) {
+        if (!dbus_bus_register(ans, &err)) {
+            report_error(&err, err_msg);
+            return NULL;
+        }
+    }
+    if (!dbus_connection_set_watch_functions(ans, add_dbus_watch, remove_dbus_watch, toggle_dbus_watch, (void*)name, NULL)) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to set DBUS watches on connection to: %s", path);
+        dbus_connection_close(ans);
+        dbus_connection_unref(ans);
+        return NULL;
+    }
+    if (!dbus_connection_set_timeout_functions(ans, add_dbus_timeout, remove_dbus_timeout, toggle_dbus_timeout, (void*)name, NULL)) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to set DBUS timeout functions on connection to: %s", path);
+        dbus_connection_close(ans);
+        dbus_connection_unref(ans);
+        return NULL;
+    }
+    return ans;
+}
+
+void
+glfw_dbus_dispatch(DBusConnection *conn) {
+    while(dbus_connection_dispatch(conn) == DBUS_DISPATCH_DATA_REMAINS);
+}
+
+void
+glfw_dbus_terminate(_GLFWDBUSData *dbus) {
+    if (dbus_data) {
+        dbus_data->eld = NULL;
+        dbus_data = NULL;
+    }
+}
+
+void
+glfw_dbus_close_connection(DBusConnection *conn) {
+    dbus_connection_close(conn);
+    dbus_connection_unref(conn);
+}
+
+GLFWbool
+glfw_dbus_get_args(DBusMessage *msg, const char *failmsg, ...) {
+    DBusError err;
+    dbus_error_init(&err);
+    va_list ap;
+    va_start(ap, failmsg);
+    int firstarg = va_arg(ap, int);
+    GLFWbool ret = dbus_message_get_args_valist(msg, &err, firstarg, ap) ? GLFW_TRUE : GLFW_FALSE;
+    va_end(ap);
+    if (!ret) report_error(&err, failmsg);
+    return ret;
+}
+
+typedef struct {
+    dbus_pending_callback callback;
+    void *user_data;
+} MethodResponse;
+
+static const char*
+format_message_error(DBusError *err) {
+    static char buf[1024];
+    snprintf(buf, sizeof(buf), "[%s] %s", err->name ? err->name : "", err->message);
+    return buf;
+}
+
+static void
+method_reply_received(DBusPendingCall *pending, void *user_data) {
+    MethodResponse *res = (MethodResponse*)user_data;
+    DBusMessage *msg = dbus_pending_call_steal_reply(pending);
+    if (msg) {
+        DBusError err;
+        dbus_error_init(&err);
+        if (dbus_set_error_from_message(&err, msg)) res->callback(NULL, format_message_error(&err), res->user_data);
+        else res->callback(msg, NULL, res->user_data);
+        dbus_message_unref(msg);
+    }
+}
+
+static GLFWbool
+call_method(DBusConnection *conn, const char *node, const char *path, const char *interface, const char *method, int timeout, dbus_pending_callback callback, void *user_data, va_list ap) {
+    if (!conn) return GLFW_FALSE;
+    DBusMessage *msg = dbus_message_new_method_call(node, path, interface, method);
+    if (!msg) return GLFW_FALSE;
+    GLFWbool retval = GLFW_FALSE;
+    MethodResponse *res = malloc(sizeof(MethodResponse));
+    if (!res) { dbus_message_unref(msg); return GLFW_FALSE; }
+    res->callback = callback;
+    res->user_data = user_data;
+
+    int firstarg = va_arg(ap, int);
+    if ((firstarg == DBUS_TYPE_INVALID) || dbus_message_append_args_valist(msg, firstarg, ap)) {
+        if (callback) {
+            DBusPendingCall *pending = NULL;
+            if (dbus_connection_send_with_reply(conn, msg, &pending, timeout)) {
+                dbus_pending_call_set_notify(pending, method_reply_received, res, free);
+                retval = GLFW_TRUE;
+            } else {
+                _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to call DBUS method: %s on node: %s and interface: %s out of memory", method, node, interface);
+            }
+        } else {
+            if (dbus_connection_send(conn, msg, NULL)) {
+                retval = GLFW_TRUE;
+            } else {
+                _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to call DBUS method: %s on node: %s and interface: %s out of memory", method, node, interface);
+            }
+        }
+    } else {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to call DBUS method: %s on node: %s and interface: %s could not add arguments", method, node, interface);
+    }
+    dbus_message_unref(msg);
+
+    return retval;
+}
+
+GLFWbool
+glfw_dbus_call_method_with_reply(DBusConnection *conn, const char *node, const char *path, const char *interface, const char *method, int timeout, dbus_pending_callback callback, void* user_data, ...) {
+    GLFWbool retval;
+    va_list ap;
+    va_start(ap, user_data);
+    retval = call_method(conn, node, path, interface, method, timeout, callback, user_data, ap);
+    va_end(ap);
+    return retval;
+}
+
+GLFWbool
+glfw_dbus_call_method_no_reply(DBusConnection *conn, const char *node, const char *path, const char *interface, const char *method, ...) {
+    GLFWbool retval;
+    va_list ap;
+    va_start(ap, method);
+    retval = call_method(conn, node, path, interface, method, DBUS_TIMEOUT_USE_DEFAULT, NULL, NULL, ap);
+    va_end(ap);
+    return retval;
+}
+
+int
+glfw_dbus_match_signal(DBusMessage *msg, const char *interface, ...) {
+    va_list ap;
+    va_start(ap, interface);
+    int ans = -1, num = -1;
+    while(1) {
+        num++;
+        const char *name = va_arg(ap, const char*);
+        if (!name) break;
+        if (dbus_message_is_signal(msg, interface, name)) { ans = num; break; }
+    }
+    va_end(ap);
+    return ans;
+}

glfw/dbus_glfw.h

@@ -0,0 +1,50 @@
+//========================================================================
+// GLFW 3.3 XKB - www.glfw.org
+//------------------------------------------------------------------------
+// Copyright (c) 2018 Kovid Goyal <kovid@kovidgoyal.net>
+//
+// This software is provided 'as-is', without any express or implied
+// warranty. In no event will the authors be held liable for any damages
+// arising from the use of this software.
+//
+// Permission is granted to anyone to use this software for any purpose,
+// including commercial applications, and to alter it and redistribute it
+// freely, subject to the following restrictions:
+//
+// 1. The origin of this software must not be misrepresented; you must not
+//    claim that you wrote the original software. If you use this software
+//    in a product, an acknowledgment in the product documentation would
+//    be appreciated but is not required.
+//
+// 2. Altered source versions must be plainly marked as such, and must not
+//    be misrepresented as being the original software.
+//
+// 3. This notice may not be removed or altered from any source
+//    distribution.
+//
+//========================================================================
+
+
+#pragma once
+
+#include <dbus/dbus.h>
+#include "backend_utils.h"
+
+typedef void(*dbus_pending_callback)(DBusMessage *msg, const char* err, void* data);
+
+typedef struct {
+    EventLoopData* eld;
+} _GLFWDBUSData;
+
+
+GLFWbool glfw_dbus_init(_GLFWDBUSData *dbus, EventLoopData *eld);
+void glfw_dbus_terminate(_GLFWDBUSData *dbus);
+DBusConnection* glfw_dbus_connect_to(const char *path, const char* err_msg, const char* name, GLFWbool register_on_bus);
+void glfw_dbus_close_connection(DBusConnection *conn);
+GLFWbool
+glfw_dbus_call_method_no_reply(DBusConnection *conn, const char *node, const char *path, const char *interface, const char *method, ...);
+GLFWbool
+glfw_dbus_call_method_with_reply(DBusConnection *conn, const char *node, const char *path, const char *interface, const char *method, int timeout_ms, dbus_pending_callback callback, void *user_data, ...);
+void glfw_dbus_dispatch(DBusConnection *);
+GLFWbool glfw_dbus_get_args(DBusMessage *msg, const char *failmsg, ...);
+int glfw_dbus_match_signal(DBusMessage *msg, const char *interface, ...);

glfw/glfw.py

@@ -43,14 +43,19 @@ def init_env(env, pkg_config, at_least_version, module='x11'):
     else:
         ans.ldpaths.extend('-lrt -lm -ldl'.split())
     sinfo = json.load(open(os.path.join(base, 'source-info.json')))
-    ans.sources = sinfo['common']['sources'] + sinfo[module]['sources']
+    module_sources = list(sinfo[module]['sources'])
+    if module in ('x11', 'wayland'):
+        remove = 'linux_joystick.c' if is_bsd else 'null_joystick.c'
+        module_sources.remove(remove)
+
+    ans.sources = sinfo['common']['sources'] + module_sources
     ans.all_headers = [x for x in os.listdir(base) if x.endswith('.h')]
 
     if module in ('x11', 'wayland'):
         at_least_version('xkbcommon', 0, 5)
 
     if module == 'x11':
-        for dep in 'x11 xrandr xinerama xcursor xkbcommon xkbcommon-x11'.split():
+        for dep in 'x11 xrandr xinerama xcursor xkbcommon xkbcommon-x11 x11-xcb dbus-1'.split():
             ans.cflags.extend(pkg_config(dep, '--cflags-only-I'))
             ans.ldpaths.extend(pkg_config(dep, '--libs'))
 
@@ -62,11 +67,13 @@ def init_env(env, pkg_config, at_least_version, module='x11'):
         at_least_version('wayland-protocols', *sinfo['wayland_protocols'])
         ans.wayland_packagedir = os.path.abspath(pkg_config('wayland-protocols', '--variable=pkgdatadir')[0])
         ans.wayland_scanner = os.path.abspath(pkg_config('wayland-scanner', '--variable=wayland_scanner')[0])
+        scanner_version = tuple(map(int, pkg_config('wayland-scanner', '--modversion')[0].strip().split('.')))
+        ans.wayland_scanner_code = 'private-code' if scanner_version >= (1, 14, 91) else 'code'
         ans.wayland_protocols = tuple(sinfo[module]['protocols'])
         for p in ans.wayland_protocols:
             ans.sources.append(wayland_protocol_file_name(p))
             ans.all_headers.append(wayland_protocol_file_name(p, 'h'))
-        for dep in 'wayland-egl wayland-client wayland-cursor xkbcommon'.split():
+        for dep in 'wayland-egl wayland-client wayland-cursor xkbcommon dbus-1'.split():
             ans.cflags.extend(pkg_config(dep, '--cflags-only-I'))
             ans.ldpaths.extend(pkg_config(dep, '--libs'))
 
@@ -82,7 +89,7 @@ def build_wayland_protocols(env, run_tool, emphasis, newer, dest_dir):
             dest = wayland_protocol_file_name(src, ext)
             dest = os.path.join(dest_dir, dest)
             if newer(dest, src):
-                q = 'client-header' if ext == 'h' else 'code'
+                q = 'client-header' if ext == 'h' else env.wayland_scanner_code
                 run_tool([env.wayland_scanner, q, src, dest],
                          desc='Generating {} ...'.format(emphasis(os.path.basename(dest))))
 
@@ -93,27 +100,27 @@ def collect_source_information():
 
     def extract_sources(group, start_pos=0):
         for which in 'HEADERS SOURCES'.split():
-            yield which.lower(), filter(
+            yield which.lower(), list(filter(
                 lambda x: x[0] not in '"$',
                 re.search(
                     r'{0}_{1}\s+([^)]+?)[)]'.format(group, which),
                     raw[start_pos:]
                 ).group(1).strip().split()
-            )
+            ))
 
-    wayland_protocols = re.search('WaylandProtocols\s+(\S+)\s+', mraw).group(1)
+    wayland_protocols = re.search(r'WaylandProtocols\s+(\S+)\s+', mraw).group(1)
     wayland_protocols = list(map(int, wayland_protocols.split('.')))
     ans = {
         'common': dict(extract_sources('common')),
         'wayland_protocols': wayland_protocols,
     }
-    joystick = 'null' if is_bsd else 'linux'
     for group in 'cocoa win32 x11 wayland osmesa'.split():
         m = re.search('_GLFW_' + group.upper(), raw)
         ans[group] = dict(extract_sources('glfw', m.start()))
         if group in ('x11', 'wayland'):
-            ans[group]['headers'].append('{}_joystick.h'.format(joystick))
-            ans[group]['sources'].append('{}_joystick.c'.format(joystick))
+            for joystick in ('linux', 'null'):
+                ans[group]['headers'].append('{}_joystick.h'.format(joystick))
+                ans[group]['sources'].append('{}_joystick.c'.format(joystick))
         if group == 'wayland':
             ans[group]['protocols'] = p = []
             for m in re.finditer(r'WAYLAND_PROTOCOLS_PKGDATADIR\}/(.+?)"?$', raw, flags=re.M):
@@ -196,10 +203,14 @@ def generate_wrappers(glfw_header, glfw_native_header):
     for line in '''\
     void* glfwGetCocoaWindow(GLFWwindow* window)
     uint32_t glfwGetCocoaMonitor(GLFWmonitor* monitor)
+    GLFWcocoatextinputfilterfun glfwSetCocoaTextInputFilter(GLFWwindow* window, GLFWcocoatextinputfilterfun callback)
+    GLFWapplicationshouldhandlereopenfun glfwSetApplicationShouldHandleReopen(GLFWapplicationshouldhandlereopenfun callback)
+    void glfwGetCocoaKeyEquivalent(int glfw_key, int glfw_mods, void* cocoa_key, void* cocoa_mods)
     void* glfwGetX11Display(void)
     int32_t glfwGetX11Window(GLFWwindow* window)
     void glfwSetX11SelectionString(const char* string)
     const char* glfwGetX11SelectionString(void)
+    int glfwGetXKBScancode(const char* key_name, int case_sensitive)
 '''.splitlines():
         if line:
             functions.append(Function(line.strip(), check_fail=False))
@@ -212,6 +223,9 @@ def generate_wrappers(glfw_header, glfw_native_header):
 #pragma once
 #include <stddef.h>
 #include <stdint.h>
+typedef int (* GLFWcocoatextinputfilterfun)(int,int,unsigned int);
+typedef int (* GLFWapplicationshouldhandlereopenfun)(int);
+
 {}
 
 {}
@@ -222,9 +236,9 @@ const char* load_glfw(const char* path);
         f.write(header)
 
     code = '''
-#include <dlfcn.h>
 #include "data-types.h"
 #include "glfw-wrapper.h"
+#include <dlfcn.h>
 
 static void* handle = NULL;
 
@@ -251,8 +265,8 @@ unload_glfw() {
         f.write(code)
 
 
-def main():
-    os.chdir(sys.argv[-1])
+def from_glfw(glfw_dir):
+    os.chdir(glfw_dir)
     sinfo = collect_source_information()
     files_to_copy = set()
     for x in sinfo.values():
@@ -269,7 +283,6 @@ def main():
     for src in files_to_copy:
         shutil.copy2(src, '.')
     shutil.copy2(glfw_header, '.')
-    patch_in_file('internal.h', lambda x: x.replace('../include/GLFW/', ''))
     patch_in_file('cocoa_window.m', lambda x: re.sub(
         r'[(]void[)]loadMainMenu.+?}', '(void)loadMainMenu\n{ // removed by Kovid as it generated compiler warnings \n}\n', x, flags=re.DOTALL))
     json.dump(
@@ -282,5 +295,26 @@ def main():
     generate_wrappers(glfw_header, glfw_native_header)
 
 
+def to_glfw(glfw_dir):
+    src = base
+    for x in os.listdir(src):
+        if x in ('glfw.py', 'glfw3.h', '__pycache__', 'source-info.json') or x.startswith('wayland-'):
+            continue
+        xp = os.path.join(src, x)
+        shutil.copyfile(xp, os.path.join(glfw_dir, 'src', x))
+    shutil.copyfile(os.path.join(src, 'glfw3.h'), os.path.join(glfw_dir, 'include/GLFW/glfw3.h'))
+
+
+def main():
+    glfw_dir = os.path.abspath(os.path.join(base, '../../glfw'))
+    q = sys.argv[1].lower().replace('_', '-')
+    if q == 'from-glfw':
+        from_glfw(glfw_dir)
+    elif q == 'to-glfw':
+        to_glfw(glfw_dir)
+    else:
+        raise SystemExit('First argument must be one of to-glfw or from-glfw')
+
+
 if __name__ == '__main__':
     main()

glfw/glfw3.h

@@ -819,6 +819,12 @@ extern "C" {
  *  Mouse cursor hover [window attribute](@ref GLFW_HOVERED_attrib).
  */
 #define GLFW_HOVERED                0x0002000B
+/*! @brief Input focus on calling show window hint and attribute
+ *
+ *  Input focus [window hint](@ref GLFW_FOCUS_ON_SHOW_hint) or
+ *  [window attribute](@ref GLFW_FOCUS_ON_SHOW_attrib).
+ */
+#define GLFW_FOCUS_ON_SHOW          0x0002000C
 
 /*! @brief Framebuffer bit depth hint.
  *
@@ -1051,6 +1057,8 @@ extern "C" {
 /*! @addtogroup init
  *  @{ */
 #define GLFW_JOYSTICK_HAT_BUTTONS   0x00050001
+#define GLFW_DEBUG_KEYBOARD         0x00050002
+#define GLFW_ENABLE_JOYSTICKS       0x00050003
 
 #define GLFW_COCOA_CHDIR_RESOURCES  0x00051001
 #define GLFW_COCOA_MENUBAR          0x00051002
@@ -1365,15 +1373,17 @@ typedef void (* GLFWcursorenterfun)(GLFWwindow*,int);
  *  @param[in] window The window that received the event.
  *  @param[in] xoffset The scroll offset along the x-axis.
  *  @param[in] yoffset The scroll offset along the y-axis.
+ *  @param[in] flags A bit-mask providing extra data about the event. flags & 1 will be true if and only if the offset values are "high-precision". Typically pixel values. Otherwise the offset values are number of lines.
  *
  *  @sa @ref scrolling
  *  @sa @ref glfwSetScrollCallback
  *
  *  @since Added in version 3.0.  Replaces `GLFWmousewheelfun`.
+ *  @since Changed in version 4.0.  Added `flags` parameter.
  *
  *  @ingroup input
  */
-typedef void (* GLFWscrollfun)(GLFWwindow*,double,double);
+typedef void (* GLFWscrollfun)(GLFWwindow*,double,double,int);
 
 /*! @brief The function signature for key callbacks.
  *
@@ -1392,7 +1402,9 @@ typedef void (* GLFWscrollfun)(GLFWwindow*,double,double);
  *  @param[in] mods Bit field describing which [modifier keys](@ref mods) were
  *  held down.
  *  @param[in] text UTF-8 encoded text generated by this key event or empty string.
- *  @param[in] reserved Reserved for future use.
+ *  @param[in] Used for Input Method events. Zero for normal key events.
+ *      A value of 1 means the pre-edit text for the input event has been changed.
+ *      A value of 2 means the text should be committed.
  *
  *  @note On X11/Wayland if a modifier other than the modifiers GLFW reports
  *  (ctrl/shift/alt/super) is used, GLFW will report the shifted key rather
@@ -3057,6 +3069,11 @@ GLFWAPI void glfwMaximizeWindow(GLFWwindow* window);
  *  hidden.  If the window is already visible or is in full screen mode, this
  *  function does nothing.
  *
+ *  By default, windowed mode windows are focused when shown
+ *  Set the [GLFW_FOCUS_ON_SHOW](@ref GLFW_FOCUS_ON_SHOW_hint) window hint
+ *  to change this behavior for all newly created windows, or change the
+ *  behavior for an existing window with @ref glfwSetWindowAttrib.
+ *
  *  @param[in] window The window to make visible.
  *
  *  @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
@@ -3104,6 +3121,10 @@ GLFWAPI void glfwHideWindow(GLFWwindow* window);
  *  initially created.  Set the [GLFW_FOCUSED](@ref GLFW_FOCUSED_hint) to
  *  disable this behavior.
  *
+ *  Also by default, windowed mode windows are focused when shown
+ *  with @ref glfwShowWindow. Set the
+ *  [GLFW_FOCUS_ON_SHOW](@ref GLFW_FOCUS_ON_SHOW_hint) to disable this behavior.
+ *
  *  __Do not use this function__ to steal focus from other applications unless
  *  you are certain that is what the user wants.  Focus stealing can be
  *  extremely disruptive.
@@ -3301,8 +3322,9 @@ GLFWAPI int glfwGetWindowAttrib(GLFWwindow* window, int attrib);
  *
  *  The supported attributes are [GLFW_DECORATED](@ref GLFW_DECORATED_attrib),
  *  [GLFW_RESIZABLE](@ref GLFW_RESIZABLE_attrib),
- *  [GLFW_FLOATING](@ref GLFW_FLOATING_attrib) and
- *  [GLFW_AUTO_ICONIFY](@ref GLFW_AUTO_ICONIFY_attrib).
+ *  [GLFW_FLOATING](@ref GLFW_FLOATING_attrib),
+ *  [GLFW_AUTO_ICONIFY](@ref GLFW_AUTO_ICONIFY_attrib) and
+ *  [GLFW_FOCUS_ON_SHOW](@ref GLFW_FOCUS_ON_SHOW_attrib).
  *
  *  Some of these attributes are ignored for full screen windows.  The new
  *  value will take effect if the window is later made windowed.
@@ -3990,8 +4012,8 @@ GLFWAPI int glfwGetKey(GLFWwindow* window, int key);
  *  `GLFW_RELEASE`.
  *
  *  If the @ref GLFW_STICKY_MOUSE_BUTTONS input mode is enabled, this function
- *  `GLFW_PRESS` the first time you call it for a mouse button that was pressed,
- *  even if that mouse button has already been released.
+ *  returns `GLFW_PRESS` the first time you call it for a mouse button that was
+ *  pressed, even if that mouse button has already been released.
  *
  *  @param[in] window The desired window.
  *  @param[in] button The desired [mouse button](@ref buttons).
@@ -4106,6 +4128,7 @@ GLFWAPI void glfwSetCursorPos(GLFWwindow* window, double xpos, double ypos);
  *  @param[in] image The desired cursor image.
  *  @param[in] xhot The desired x-coordinate, in pixels, of the cursor hotspot.
  *  @param[in] yhot The desired y-coordinate, in pixels, of the cursor hotspot.
+ *  @param[in] count The number of images. Used on Cocoa for retina cursors. The first image should be the 1:1 scale image.
  *  @return The handle of the created cursor, or `NULL` if an
  *  [error](@ref error_handling) occurred.
  *
@@ -4121,11 +4144,11 @@ GLFWAPI void glfwSetCursorPos(GLFWwindow* window, double xpos, double ypos);
  *  @sa @ref glfwDestroyCursor
  *  @sa @ref glfwCreateStandardCursor
  *
- *  @since Added in version 3.1.
+ *  @since Added in version 3.1. Changed in 4.0 to add the count parameter.
  *
  *  @ingroup input
  */
-GLFWAPI GLFWcursor* glfwCreateCursor(const GLFWimage* image, int xhot, int yhot);
+GLFWAPI GLFWcursor* glfwCreateCursor(const GLFWimage* image, int xhot, int yhot, int count);
 
 /*! @brief Creates a cursor with a standard shape.
  *
@@ -4210,6 +4233,23 @@ GLFWAPI void glfwSetCursor(GLFWwindow* window, GLFWcursor* cursor);
  */
 GLFWAPI GLFWkeyboardfun glfwSetKeyboardCallback(GLFWwindow* window, GLFWkeyboardfun cbfun);
 
+/*! @brief Notifies the OS Input Method Event system of changes to application input state
+ *
+ * Used to notify the IME system of changes in state such as focus gained/lost
+ * and text cursor position.
+ *
+ * @param which: What data to notify. 1 means focus and 2 means cursor position.
+ * @param a, b, c, d: Interpreted based on the value of which. When which is 1
+ * a is interpreted as a boolean indicating focus gained/lost. When which is 2
+ * a, b, c, d are the cursor x, y, width and height values (in the window co-ordinate
+ * system).
+ *
+ *  @ingroup input
+ *  @since Added in version 4.0
+ */
+GLFWAPI void glfwUpdateIMEState(GLFWwindow* window, int which, int a, int b, int c, int d);
+
+
 /*! @brief Sets the mouse button callback.
  *
  *  This function sets the mouse button callback of the specified window, which
@@ -4582,7 +4622,7 @@ GLFWAPI const char* glfwGetJoystickGUID(int jid);
  *  This function may be called from the joystick callback, even for a joystick
  *  that is being disconnected.
  *
- *  @param[in] joystick The joystick whose pointer to set.
+ *  @param[in] jid The joystick whose pointer to set.
  *  @param[in] pointer The new value.
  *
  *  @errors Possible errors include @ref GLFW_NOT_INITIALIZED.
@@ -4607,7 +4647,7 @@ GLFWAPI void glfwSetJoystickUserPointer(int jid, void* pointer);
  *  This function may be called from the joystick callback, even for a joystick
  *  that is being disconnected.
  *
- *  @param[in] joystick The joystick whose pointer to return.
+ *  @param[in] jid The joystick whose pointer to return.
  *
  *  @errors Possible errors include @ref GLFW_NOT_INITIALIZED.
  *
@@ -4770,6 +4810,8 @@ GLFWAPI const char* glfwGetGamepadName(int jid);
  *  @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
  *  GLFW_INVALID_ENUM.
  *
+ *  @thread_safety This function must only be called from the main thread.
+ *
  *  @sa @ref gamepad
  *  @sa @ref glfwUpdateGamepadMappings
  *  @sa @ref glfwJoystickIsGamepad

glfw/ibus_glfw.c

@@ -0,0 +1,427 @@
+//========================================================================
+// GLFW 3.3 XKB - www.glfw.org
+//------------------------------------------------------------------------
+// Copyright (c) 2018 Kovid Goyal <kovid@kovidgoyal.net>
+//
+// This software is provided 'as-is', without any express or implied
+// warranty. In no event will the authors be held liable for any damages
+// arising from the use of this software.
+//
+// Permission is granted to anyone to use this software for any purpose,
+// including commercial applications, and to alter it and redistribute it
+// freely, subject to the following restrictions:
+//
+// 1. The origin of this software must not be misrepresented; you must not
+//    claim that you wrote the original software. If you use this software
+//    in a product, an acknowledgment in the product documentation would
+//    be appreciated but is not required.
+//
+// 2. Altered source versions must be plainly marked as such, and must not
+//    be misrepresented as being the original software.
+//
+// 3. This notice may not be removed or altered from any source
+//    distribution.
+//
+//========================================================================
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <limits.h>
+
+#include "internal.h"
+#include "ibus_glfw.h"
+
+#define debug(...) if (_glfw.hints.init.debugKeyboard) printf(__VA_ARGS__);
+static const char IBUS_SERVICE[]         = "org.freedesktop.IBus";
+static const char IBUS_PATH[]            = "/org/freedesktop/IBus";
+static const char IBUS_INTERFACE[]       = "org.freedesktop.IBus";
+static const char IBUS_INPUT_INTERFACE[] = "org.freedesktop.IBus.InputContext";
+enum Capabilities {
+    IBUS_CAP_PREEDIT_TEXT       = 1 << 0,
+    IBUS_CAP_AUXILIARY_TEXT     = 1 << 1,
+    IBUS_CAP_LOOKUP_TABLE       = 1 << 2,
+    IBUS_CAP_FOCUS              = 1 << 3,
+    IBUS_CAP_PROPERTY           = 1 << 4,
+    IBUS_CAP_SURROUNDING_TEXT   = 1 << 5
+};
+
+
+static inline GLFWbool
+test_env_var(const char *name, const char *val) {
+    const char *q = getenv(name);
+    return (q && strcmp(q, val) == 0) ? GLFW_TRUE : GLFW_FALSE;
+}
+
+static inline GLFWbool
+MIN(size_t a, size_t b) {
+    return a < b ? a : b;
+}
+
+static const char*
+get_ibus_text_from_message(DBusMessage *msg) {
+    /* The message structure is (from dbus-monitor)
+       variant       struct {
+         string "IBusText"
+         array [
+         ]
+         string "ash "
+         variant             struct {
+               string "IBusAttrList"
+               array [
+               ]
+               array [
+               ]
+            }
+      }
+    */
+    const char *text = NULL;
+    const char *struct_id = NULL;
+    DBusMessageIter iter, sub1, sub2;
+    dbus_message_iter_init(msg, &iter);
+
+    if (dbus_message_iter_get_arg_type(&iter) != DBUS_TYPE_VARIANT) return NULL;
+
+    dbus_message_iter_recurse(&iter, &sub1);
+
+    if (dbus_message_iter_get_arg_type(&sub1) != DBUS_TYPE_STRUCT) return NULL;
+
+    dbus_message_iter_recurse(&sub1, &sub2);
+
+    if (dbus_message_iter_get_arg_type(&sub2) != DBUS_TYPE_STRING) return NULL;
+
+    dbus_message_iter_get_basic(&sub2, &struct_id);
+    if (!struct_id || strncmp(struct_id, "IBusText", sizeof("IBusText")) != 0) return NULL;
+
+    dbus_message_iter_next(&sub2);
+    dbus_message_iter_next(&sub2);
+
+    if (dbus_message_iter_get_arg_type(&sub2) != DBUS_TYPE_STRING) return NULL;
+
+    dbus_message_iter_get_basic(&sub2, &text);
+
+    return text;
+}
+
+static inline void
+send_text(const char *text, int state) {
+    _GLFWwindow *w = _glfwFocusedWindow();
+    if (w && w->callbacks.keyboard) {
+        w->callbacks.keyboard((GLFWwindow*) w, GLFW_KEY_UNKNOWN, 0, GLFW_PRESS, 0, text, state);
+    }
+}
+
+// Connection handling {{{
+
+static DBusHandlerResult
+message_handler(DBusConnection *conn, DBusMessage *msg, void *user_data) {
+    // To monitor signals from IBUS, use
+    //  dbus-monitor --address `ibus address` "type='signal',interface='org.freedesktop.IBus.InputContext'"
+    _GLFWIBUSData *ibus = (_GLFWIBUSData*)user_data;
+    (void)ibus;
+    const char *text;
+    switch(glfw_dbus_match_signal(msg, IBUS_INPUT_INTERFACE, "CommitText", "UpdatePreeditText", "HidePreeditText", "ShowPreeditText", NULL)) {
+        case 0:
+            text = get_ibus_text_from_message(msg);
+            debug("IBUS: CommitText: '%s'\n", text ? text : "(nil)");
+            send_text(text, 2);
+            break;
+        case 1:
+            text = get_ibus_text_from_message(msg);
+            send_text(text, 1);
+            debug("IBUS: UpdatePreeditText: '%s'\n", text ? text : "(nil)");
+            break;
+        case 2:
+            debug("IBUS: HidePreeditText\n");
+            break;
+        case 3:
+            debug("IBUS: ShowPreeditText\n");
+            break;
+    }
+    return DBUS_HANDLER_RESULT_NOT_YET_HANDLED;
+}
+
+static inline const char*
+get_ibus_address_file_name(void) {
+    const char *addr;
+    static char ans[PATH_MAX];
+    addr = getenv("IBUS_ADDRESS");
+    int offset = 0;
+    if (addr && addr[0]) {
+        memcpy(ans, addr, MIN(strlen(addr), sizeof(ans)));
+        return ans;
+    }
+
+    const char *de = getenv("DISPLAY");
+    if (!de || !de[0]) de = ":0.0";
+    char *display = _glfw_strdup(de);
+    const char *host = display;
+    char *disp_num  = strrchr(display, ':');
+    char *screen_num = strrchr(display, '.');
+
+    if (!disp_num) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Could not get IBUS address file name as DISPLAY env var has no colon");
+        free(display);
+        return NULL;
+    }
+    *disp_num = 0;
+    disp_num++;
+    if (screen_num) *screen_num = 0;
+    if (!*host) host = "unix";
+
+    memset(ans, 0, sizeof(ans));
+    const char *conf_env = getenv("XDG_CONFIG_HOME");
+    if (conf_env && conf_env[0]) {
+        offset = snprintf(ans, sizeof(ans), "%s", conf_env);
+    } else {
+        conf_env = getenv("HOME");
+        if (!conf_env || !conf_env[0]) {
+            _glfwInputError(GLFW_PLATFORM_ERROR, "Could not get IBUS address file name as no HOME env var is set");
+            free(display);
+            return NULL;
+        }
+        offset = snprintf(ans, sizeof(ans), "%s/.config", conf_env);
+    }
+    char *key = dbus_get_local_machine_id();
+    snprintf(ans + offset, sizeof(ans) - offset, "/ibus/bus/%s-%s-%s", key, host, disp_num);
+    dbus_free(key);
+    free(display);
+    return ans;
+}
+
+
+static inline GLFWbool
+read_ibus_address(_GLFWIBUSData *ibus) {
+    static char buf[1024];
+    struct stat s;
+    FILE *addr_file = fopen(ibus->address_file_name, "r");
+    if (!addr_file) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to open IBUS address file: %s with error: %s", ibus->address_file_name, strerror(errno));
+        return GLFW_FALSE;
+    }
+    int stat_result = fstat(fileno(addr_file), &s);
+    GLFWbool found = GLFW_FALSE;
+    while (fgets(buf, sizeof(buf), addr_file)) {
+        if (strncmp(buf, "IBUS_ADDRESS=", sizeof("IBUS_ADDRESS=")-1) == 0) {
+            size_t sz = strlen(buf);
+            if (buf[sz-1] == '\n') buf[sz-1] = 0;
+            if (buf[sz-2] == '\r') buf[sz-2] = 0;
+            found = GLFW_TRUE;
+            break;
+        }
+    }
+    fclose(addr_file); addr_file = NULL;
+    if (stat_result != 0) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to stat IBUS address file: %s with error: %s", ibus->address_file_name, strerror(errno));
+        return GLFW_FALSE;
+    }
+    ibus->address_file_mtime = s.st_mtime;
+    if (found) {
+        free((void*)ibus->address);
+        ibus->address = _glfw_strdup(buf + sizeof("IBUS_ADDRESS=") - 1);
+        return GLFW_TRUE;
+    }
+    _glfwInputError(GLFW_PLATFORM_ERROR, "Could not find IBUS_ADDRESS in %s", ibus->address_file_name);
+    return GLFW_FALSE;
+}
+
+void
+input_context_created(DBusMessage *msg, const char* errmsg, void *data) {
+    if (errmsg) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "IBUS: Failed to create input context with error: %s", errmsg);
+        return;
+    }
+    const char *path = NULL;
+    if (!glfw_dbus_get_args(msg, "Failed to get IBUS context path from reply", DBUS_TYPE_OBJECT_PATH, &path, DBUS_TYPE_INVALID)) return;
+    _GLFWIBUSData *ibus = (_GLFWIBUSData*)data;
+    free((void*)ibus->input_ctx_path);
+    ibus->input_ctx_path = _glfw_strdup(path);
+    if (!ibus->input_ctx_path) return;
+    dbus_bus_add_match(ibus->conn, "type='signal',interface='org.freedesktop.IBus.InputContext'", NULL);
+    DBusObjectPathVTable ibus_vtable = {.message_function = message_handler};
+    dbus_connection_try_register_object_path(ibus->conn, ibus->input_ctx_path, &ibus_vtable, ibus, NULL);
+    enum Capabilities caps = IBUS_CAP_FOCUS | IBUS_CAP_PREEDIT_TEXT;
+    if (!glfw_dbus_call_method_no_reply(ibus->conn, IBUS_SERVICE, ibus->input_ctx_path, IBUS_INPUT_INTERFACE, "SetCapabilities", DBUS_TYPE_UINT32, &caps, DBUS_TYPE_INVALID)) return;
+    ibus->ok = GLFW_TRUE;
+    glfw_ibus_set_focused(ibus, GLFW_FALSE);
+    glfw_ibus_set_cursor_geometry(ibus, 0, 0, 0, 0);
+    debug("Connected to IBUS daemon for IME input management\n");
+}
+
+GLFWbool
+setup_connection(_GLFWIBUSData *ibus) {
+    const char *client_name = "GLFW_Application";
+    const char *address_file_name = get_ibus_address_file_name();
+    ibus->ok = GLFW_FALSE;
+    if (!address_file_name) return GLFW_FALSE;
+    free((void*)ibus->address_file_name);
+    ibus->address_file_name = _glfw_strdup(address_file_name);
+    if (!read_ibus_address(ibus)) return GLFW_FALSE;
+    if (ibus->conn) {
+        glfw_dbus_close_connection(ibus->conn);
+        ibus->conn = NULL;
+    }
+    debug("Connecting to IBUS daemon @ %s for IME input management\n", ibus->address);
+    ibus->conn = glfw_dbus_connect_to(ibus->address, "Failed to connect to the IBUS daemon, with error", "ibus", GLFW_FALSE);
+    if (!ibus->conn) return GLFW_FALSE;
+    free((void*)ibus->input_ctx_path); ibus->input_ctx_path = NULL;
+    if (!glfw_dbus_call_method_with_reply(
+            ibus->conn, IBUS_SERVICE, IBUS_PATH, IBUS_INTERFACE, "CreateInputContext", DBUS_TIMEOUT_USE_DEFAULT, input_context_created, ibus,
+            DBUS_TYPE_STRING, &client_name, DBUS_TYPE_INVALID)) {
+        return GLFW_FALSE;
+    }
+    return GLFW_TRUE;
+}
+
+
+void
+glfw_connect_to_ibus(_GLFWIBUSData *ibus) {
+    if (ibus->inited) return;
+    if (!test_env_var("XMODIFIERS", "@im=ibus") && !test_env_var("GTK_IM_MODULE", "ibus") && !test_env_var("QT_IM_MODULE", "ibus") && !test_env_var("GLFW_IM_MODULE", "ibus")) return;
+    if (getenv("GLFW_IM_MODULE") && !test_env_var("GLFW_IM_MODULE", "ibus")) return;
+    ibus->inited = GLFW_TRUE;
+    setup_connection(ibus);
+}
+
+void
+glfw_ibus_terminate(_GLFWIBUSData *ibus) {
+    if (ibus->conn) {
+        glfw_dbus_close_connection(ibus->conn);
+        ibus->conn = NULL;
+    }
+#define F(x) if (ibus->x) { free((void*)ibus->x); ibus->x = NULL; }
+    F(input_ctx_path);
+    F(address);
+    F(address_file_name);
+#undef F
+
+    ibus->ok = GLFW_FALSE;
+}
+
+static GLFWbool
+check_connection(_GLFWIBUSData *ibus) {
+    if (!ibus->inited) return GLFW_FALSE;
+    if (ibus->conn && dbus_connection_get_is_connected(ibus->conn)) {
+        return ibus->ok;
+    }
+    struct stat s;
+    if (stat(ibus->address_file_name, &s) != 0 || s.st_mtime != ibus->address_file_mtime) {
+        if (!read_ibus_address(ibus)) return GLFW_FALSE;
+        setup_connection(ibus);
+    }
+    return GLFW_FALSE;
+}
+
+
+void
+glfw_ibus_dispatch(_GLFWIBUSData *ibus) {
+    if (ibus->conn) glfw_dbus_dispatch(ibus->conn);
+}
+// }}}
+
+static void
+simple_message(_GLFWIBUSData *ibus, const char *method) {
+    if (check_connection(ibus)) {
+        glfw_dbus_call_method_no_reply(ibus->conn, IBUS_SERVICE, ibus->input_ctx_path, IBUS_INPUT_INTERFACE, method, DBUS_TYPE_INVALID);
+    }
+}
+
+void
+glfw_ibus_set_focused(_GLFWIBUSData *ibus, GLFWbool focused) {
+    simple_message(ibus, focused ? "FocusIn" : "FocusOut");
+}
+
+void
+glfw_ibus_set_cursor_geometry(_GLFWIBUSData *ibus, int x, int y, int w, int h) {
+    if (check_connection(ibus)) {
+        glfw_dbus_call_method_no_reply(ibus->conn, IBUS_SERVICE, ibus->input_ctx_path, IBUS_INPUT_INTERFACE, "SetCursorLocation",
+                DBUS_TYPE_INT32, &x, DBUS_TYPE_INT32, &y, DBUS_TYPE_INT32, &w, DBUS_TYPE_INT32, &h, DBUS_TYPE_INVALID);
+    }
+}
+
+typedef enum
+{
+    IBUS_SHIFT_MASK    = 1 << 0,
+    IBUS_LOCK_MASK     = 1 << 1,
+    IBUS_CONTROL_MASK  = 1 << 2,
+    IBUS_MOD1_MASK     = 1 << 3,
+    IBUS_MOD2_MASK     = 1 << 4,
+    IBUS_MOD3_MASK     = 1 << 5,
+    IBUS_MOD4_MASK     = 1 << 6,
+    IBUS_MOD5_MASK     = 1 << 7,
+    IBUS_BUTTON1_MASK  = 1 << 8,
+    IBUS_BUTTON2_MASK  = 1 << 9,
+    IBUS_BUTTON3_MASK  = 1 << 10,
+    IBUS_BUTTON4_MASK  = 1 << 11,
+    IBUS_BUTTON5_MASK  = 1 << 12,
+
+    /* The next few modifiers are used by XKB, so we skip to the end.
+     * Bits 15 - 23 are currently unused. Bit 29 is used internally.
+     */
+
+    /* ibus mask */
+    IBUS_HANDLED_MASK  = 1 << 24,
+    IBUS_FORWARD_MASK  = 1 << 25,
+    IBUS_IGNORED_MASK  = IBUS_FORWARD_MASK,
+
+    IBUS_SUPER_MASK    = 1 << 26,
+    IBUS_HYPER_MASK    = 1 << 27,
+    IBUS_META_MASK     = 1 << 28,
+
+    IBUS_RELEASE_MASK  = 1 << 30,
+
+    IBUS_MODIFIER_MASK = 0x5f001fff
+} IBusModifierType;
+
+
+static inline uint32_t
+ibus_key_state(unsigned int glfw_modifiers, int action) {
+    uint32_t ans = action == GLFW_RELEASE ? IBUS_RELEASE_MASK : 0;
+#define M(g, i) if(glfw_modifiers & GLFW_MOD_##g) ans |= i
+    M(SHIFT, IBUS_SHIFT_MASK);
+    M(CAPS_LOCK, IBUS_LOCK_MASK);
+    M(CONTROL, IBUS_CONTROL_MASK);
+    M(ALT, IBUS_MOD1_MASK);
+    M(NUM_LOCK, IBUS_MOD2_MASK);
+    M(SUPER, IBUS_MOD4_MASK);
+#undef M
+    return ans;
+}
+
+void
+key_event_processed(DBusMessage *msg, const char* errmsg, void *data) {
+    uint32_t handled = 0;
+    KeyEvent *ev = (KeyEvent*)data;
+    GLFWbool is_release = ev->action == GLFW_RELEASE;
+    GLFWbool failed = GLFW_FALSE;
+    if (errmsg) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "IBUS: Failed to process key with error: %s", errmsg);
+        failed = GLFW_TRUE;
+    } else {
+        glfw_dbus_get_args(msg, "Failed to get IBUS handled key from reply", DBUS_TYPE_BOOLEAN, &handled, DBUS_TYPE_INVALID);
+        debug("IBUS processed scancode: 0x%x release: %d handled: %u\n", ev->keycode, is_release, handled);
+    }
+    glfw_xkb_key_from_ime(ev, handled ? GLFW_TRUE : GLFW_FALSE, failed);
+    free(ev);
+}
+
+GLFWbool
+ibus_process_key(const KeyEvent *ev_, _GLFWIBUSData *ibus) {
+    if (!check_connection(ibus)) return GLFW_FALSE;
+    KeyEvent *ev = malloc(sizeof(KeyEvent));
+    if (!ev) return GLFW_FALSE;
+    memcpy(ev, ev_, sizeof(KeyEvent));
+    uint32_t state = ibus_key_state(ev->glfw_modifiers, ev->action);
+    if (!glfw_dbus_call_method_with_reply(
+            ibus->conn, IBUS_SERVICE, ibus->input_ctx_path, IBUS_INPUT_INTERFACE, "ProcessKeyEvent",
+            3000, key_event_processed, ev,
+            DBUS_TYPE_UINT32, &ev->ibus_sym, DBUS_TYPE_UINT32, &ev->ibus_keycode, DBUS_TYPE_UINT32,
+            &state, DBUS_TYPE_INVALID)) {
+        free(ev);
+        return GLFW_FALSE;
+    }
+    return GLFW_TRUE;
+}

glfw/ibus_glfw.h

@@ -0,0 +1,55 @@
+//========================================================================
+// GLFW 3.3 XKB - www.glfw.org
+//------------------------------------------------------------------------
+// Copyright (c) 2018 Kovid Goyal <kovid@kovidgoyal.net>
+//
+// This software is provided 'as-is', without any express or implied
+// warranty. In no event will the authors be held liable for any damages
+// arising from the use of this software.
+//
+// Permission is granted to anyone to use this software for any purpose,
+// including commercial applications, and to alter it and redistribute it
+// freely, subject to the following restrictions:
+//
+// 1. The origin of this software must not be misrepresented; you must not
+//    claim that you wrote the original software. If you use this software
+//    in a product, an acknowledgment in the product documentation would
+//    be appreciated but is not required.
+//
+// 2. Altered source versions must be plainly marked as such, and must not
+//    be misrepresented as being the original software.
+//
+// 3. This notice may not be removed or altered from any source
+//    distribution.
+//
+//========================================================================
+
+
+#pragma once
+
+#include "dbus_glfw.h"
+#include <xkbcommon/xkbcommon.h>
+
+typedef struct {
+    GLFWbool ok, inited;
+    time_t address_file_mtime;
+    DBusConnection *conn;
+    const char *input_ctx_path, *address_file_name, *address;
+} _GLFWIBUSData;
+
+typedef struct {
+    xkb_keycode_t keycode, ibus_keycode;
+    xkb_keysym_t keysym, ibus_sym;
+    unsigned int glfw_modifiers;
+    int action;
+    GLFWid window_id;
+    int glfw_keycode;
+    char text[64];
+} KeyEvent;
+
+void glfw_connect_to_ibus(_GLFWIBUSData *ibus);
+void glfw_ibus_terminate(_GLFWIBUSData *ibus);
+void glfw_ibus_set_focused(_GLFWIBUSData *ibus, GLFWbool focused);
+void glfw_ibus_dispatch(_GLFWIBUSData *ibus);
+GLFWbool ibus_process_key(const KeyEvent *ev_, _GLFWIBUSData *ibus);
+void glfw_ibus_set_cursor_geometry(_GLFWIBUSData *ibus, int x, int y, int w, int h);

glfw/init.c

@@ -51,43 +51,14 @@ static GLFWerrorfun _glfwErrorCallback;
 static _GLFWinitconfig _glfwInitHints =
 {
     GLFW_TRUE,      // hat buttons
+    GLFW_FALSE,     // debug keyboard
+    GLFW_TRUE,      // enable joystick
     {
         GLFW_TRUE,  // macOS menu bar
         GLFW_TRUE   // macOS bundle chdir
     }
 };
 
-// Returns a generic string representation of the specified error
-//
-static const char* getErrorString(int code)
-{
-    switch (code)
-    {
-        case GLFW_NOT_INITIALIZED:
-            return "The GLFW library is not initialized";
-        case GLFW_NO_CURRENT_CONTEXT:
-            return "There is no current context";
-        case GLFW_INVALID_ENUM:
-            return "Invalid argument for enum parameter";
-        case GLFW_INVALID_VALUE:
-            return "Invalid value for parameter";
-        case GLFW_OUT_OF_MEMORY:
-            return "Out of memory";
-        case GLFW_API_UNAVAILABLE:
-            return "The requested API is unavailable";
-        case GLFW_VERSION_UNAVAILABLE:
-            return "The requested API version is unavailable";
-        case GLFW_PLATFORM_ERROR:
-            return "An undocumented platform-specific error occurred";
-        case GLFW_FORMAT_UNAVAILABLE:
-            return "The requested format is unavailable";
-        case GLFW_NO_WINDOW_CONTEXT:
-            return "The specified window has no context";
-        default:
-            return "ERROR: UNKNOWN GLFW ERROR";
-    }
-}
-
 // Terminate the library
 //
 static void terminate(void)
@@ -145,8 +116,9 @@ static void terminate(void)
 char* _glfw_strdup(const char* source)
 {
     const size_t length = strlen(source);
-    char* result = calloc(length + 1, 1);
-    strcpy(result, source);
+    char* result = malloc(length + 1);
+    memcpy(result, source, length);
+    result[length] = 0;
     return result;
 }
 
@@ -173,7 +145,30 @@ void _glfwInputError(int code, const char* format, ...)
         description[sizeof(description) - 1] = '\0';
     }
     else
-        strcpy(description, getErrorString(code));
+    {
+        if (code == GLFW_NOT_INITIALIZED)
+            strcpy(description, "The GLFW library is not initialized");
+        else if (code == GLFW_NO_CURRENT_CONTEXT)
+            strcpy(description, "There is no current context");
+        else if (code == GLFW_INVALID_ENUM)
+            strcpy(description, "Invalid argument for enum parameter");
+        else if (code == GLFW_INVALID_VALUE)
+            strcpy(description, "Invalid value for parameter");
+        else if (code == GLFW_OUT_OF_MEMORY)
+            strcpy(description, "Out of memory");
+        else if (code == GLFW_API_UNAVAILABLE)
+            strcpy(description, "The requested API is unavailable");
+        else if (code == GLFW_VERSION_UNAVAILABLE)
+            strcpy(description, "The requested API version is unavailable");
+        else if (code == GLFW_PLATFORM_ERROR)
+            strcpy(description, "A platform-specific error occurred");
+        else if (code == GLFW_FORMAT_UNAVAILABLE)
+            strcpy(description, "The requested format is unavailable");
+        else if (code == GLFW_NO_WINDOW_CONTEXT)
+            strcpy(description, "The specified window has no context");
+        else
+            strcpy(description, "ERROR: UNKNOWN GLFW ERROR");
+    }
 
     if (_glfw.initialized)
     {
@@ -260,9 +255,15 @@ GLFWAPI void glfwInitHint(int hint, int value)
 {
     switch (hint)
     {
+        case GLFW_ENABLE_JOYSTICKS:
+            _glfwInitHints.enableJoysticks = value;
+            return;
         case GLFW_JOYSTICK_HAT_BUTTONS:
             _glfwInitHints.hatButtons = value;
             return;
+        case GLFW_DEBUG_KEYBOARD:
+            _glfwInitHints.debugKeyboard = value;
+            return;
         case GLFW_COCOA_CHDIR_RESOURCES:
             _glfwInitHints.ns.chdir = value;
             return;
@@ -319,4 +320,3 @@ GLFWAPI GLFWerrorfun glfwSetErrorCallback(GLFWerrorfun cbfun)
     _GLFW_SWAP_POINTERS(_glfwErrorCallback, cbfun);
     return cbfun;
 }
-

glfw/input.c

@@ -32,7 +32,6 @@
 #include <math.h>
 #include <stdlib.h>
 #include <string.h>
-#include <ctype.h>
 
 // Internal key state used for sticky keys
 #define _GLFW_STICK 3
@@ -287,10 +286,10 @@ void _glfwInputKeyboard(_GLFWwindow* window, int key, int scancode, int action,
 
 // Notifies shared code of a scroll event
 //
-void _glfwInputScroll(_GLFWwindow* window, double xoffset, double yoffset)
+void _glfwInputScroll(_GLFWwindow* window, double xoffset, double yoffset, int flags)
 {
     if (window->callbacks.scroll)
-        window->callbacks.scroll((GLFWwindow*) window, xoffset, yoffset);
+        window->callbacks.scroll((GLFWwindow*) window, xoffset, yoffset, flags);
 }
 
 // Notifies shared code of a mouse button click event
@@ -619,11 +618,7 @@ GLFWAPI void glfwSetInputMode(GLFWwindow* handle, int mode, int value)
         _glfwPlatformGetCursorPos(window,
                                   &window->virtualCursorPosX,
                                   &window->virtualCursorPosY);
-
-#ifdef __APPLE__
-        if (_glfwPlatformWindowFocused(window))
-#endif
-            _glfwPlatformSetCursorMode(window, value);
+        _glfwPlatformSetCursorMode(window, value);
     }
     else if (mode == GLFW_STICKY_KEYS)
     {
@@ -804,11 +799,12 @@ GLFWAPI void glfwSetCursorPos(GLFWwindow* handle, double xpos, double ypos)
     }
 }
 
-GLFWAPI GLFWcursor* glfwCreateCursor(const GLFWimage* image, int xhot, int yhot)
+GLFWAPI GLFWcursor* glfwCreateCursor(const GLFWimage* image, int xhot, int yhot, int count)
 {
     _GLFWcursor* cursor;
 
     assert(image != NULL);
+    assert(count > 0);
 
     _GLFW_REQUIRE_INIT_OR_RETURN(NULL);
 
@@ -816,7 +812,7 @@ GLFWAPI GLFWcursor* glfwCreateCursor(const GLFWimage* image, int xhot, int yhot)
     cursor->next = _glfw.cursorListHead;
     _glfw.cursorListHead = cursor;
 
-    if (!_glfwPlatformCreateCursor(cursor, image, xhot, yhot))
+    if (!_glfwPlatformCreateCursor(cursor, image, xhot, yhot, count))
     {
         glfwDestroyCursor((GLFWcursor*) cursor);
         return NULL;
@@ -913,6 +909,18 @@ GLFWAPI GLFWkeyboardfun glfwSetKeyboardCallback(GLFWwindow* handle, GLFWkeyboard
     return cbfun;
 }
 
+GLFWAPI void glfwUpdateIMEState(GLFWwindow* handle, int which, int a, int b, int c, int d) {
+    _GLFWwindow* window = (_GLFWwindow*) handle;
+    assert(window != NULL);
+
+    _GLFW_REQUIRE_INIT();
+#if defined(_GLFW_X11) || defined(_GLFW_WAYLAND)
+    _glfwPlatformUpdateIMEState(window, which, a, b, c, d);
+#else
+    (void)window; (void)which; (void)a; (void)b; (void)c; (void)d;
+#endif
+}
+
 GLFWAPI GLFWmousebuttonfun glfwSetMouseButtonCallback(GLFWwindow* handle,
                                                       GLFWmousebuttonfun cbfun)
 {
@@ -1180,7 +1188,9 @@ GLFWAPI int glfwUpdateGamepadMappings(const char* string)
 
     while (*c)
     {
-        if (isxdigit(*c))
+        if ((*c >= '0' && *c <= '9') ||
+            (*c >= 'a' && *c <= 'f') ||
+            (*c >= 'A' && *c <= 'F'))
         {
             char line[1024];
 

glfw/internal.h

@@ -59,6 +59,7 @@
 #define _GLFW_MESSAGE_SIZE      1024
 
 typedef int GLFWbool;
+typedef unsigned long long GLFWid;
 
 typedef struct _GLFWerror       _GLFWerror;
 typedef struct _GLFWinitconfig  _GLFWinitconfig;
@@ -242,6 +243,8 @@ struct _GLFWerror
 struct _GLFWinitconfig
 {
     GLFWbool      hatButtons;
+    GLFWbool      debugKeyboard;
+    GLFWbool      enableJoysticks;
     struct {
         GLFWbool  menubar;
         GLFWbool  chdir;
@@ -267,6 +270,7 @@ struct _GLFWwndconfig
     GLFWbool      floating;
     GLFWbool      maximized;
     GLFWbool      centerCursor;
+    GLFWbool      focusOnShow;
     struct {
         GLFWbool  retina;
         char      frameName[256];
@@ -372,8 +376,10 @@ struct _GLFWwindow
     GLFWbool            decorated;
     GLFWbool            autoIconify;
     GLFWbool            floating;
+    GLFWbool            focusOnShow;
     GLFWbool            shouldClose;
     void*               userPointer;
+    GLFWid              id;
     GLFWvidmode         videoMode;
     _GLFWmonitor*       monitor;
     _GLFWcursor*        cursor;
@@ -522,6 +528,7 @@ struct _GLFWlibrary
     _GLFWerror*         errorListHead;
     _GLFWcursor*        cursorListHead;
     _GLFWwindow*        windowListHead;
+    GLFWid              focusedWindowId;
 
     _GLFWmonitor**      monitors;
     int                 monitorCount;
@@ -597,7 +604,7 @@ void _glfwPlatformGetCursorPos(_GLFWwindow* window, double* xpos, double* ypos);
 void _glfwPlatformSetCursorPos(_GLFWwindow* window, double xpos, double ypos);
 void _glfwPlatformSetCursorMode(_GLFWwindow* window, int mode);
 int _glfwPlatformCreateCursor(_GLFWcursor* cursor,
-                              const GLFWimage* image, int xhot, int yhot);
+                              const GLFWimage* image, int xhot, int yhot, int count);
 int _glfwPlatformCreateStandardCursor(_GLFWcursor* cursor, int shape);
 void _glfwPlatformDestroyCursor(_GLFWcursor* cursor);
 void _glfwPlatformSetCursor(_GLFWwindow* window, _GLFWcursor* cursor);
@@ -667,6 +674,7 @@ void _glfwPlatformSetWindowResizable(_GLFWwindow* window, GLFWbool enabled);
 void _glfwPlatformSetWindowDecorated(_GLFWwindow* window, GLFWbool enabled);
 void _glfwPlatformSetWindowFloating(_GLFWwindow* window, GLFWbool enabled);
 void _glfwPlatformSetWindowOpacity(_GLFWwindow* window, float opacity);
+void _glfwPlatformUpdateIMEState(_GLFWwindow *w, int which, int a, int b, int c, int d);
 
 void _glfwPlatformPollEvents(void);
 void _glfwPlatformWaitEvents(void);
@@ -710,7 +718,7 @@ void _glfwInputWindowCloseRequest(_GLFWwindow* window);
 void _glfwInputWindowMonitor(_GLFWwindow* window, _GLFWmonitor* monitor);
 
 void _glfwInputKeyboard(_GLFWwindow* window, int key, int scancode, int action, int mods, const char* text, int state);
-void _glfwInputScroll(_GLFWwindow* window, double xoffset, double yoffset);
+void _glfwInputScroll(_GLFWwindow* window, double xoffset, double yoffset, int flags);
 void _glfwInputMouseClick(_GLFWwindow* window, int button, int action, int mods);
 void _glfwInputCursorPos(_GLFWwindow* window, double xpos, double ypos);
 void _glfwInputCursorEnter(_GLFWwindow* window, GLFWbool entered);
@@ -763,5 +771,7 @@ const char* _glfwGetKeyName(int key);
 GLFWbool _glfwInitVulkan(int mode);
 void _glfwTerminateVulkan(void);
 const char* _glfwGetVulkanResultString(VkResult result);
+_GLFWwindow* _glfwFocusedWindow();
+_GLFWwindow* _glfwWindowForId(GLFWid id);
 
 char* _glfw_strdup(const char* source);

glfw/null_window.c

@@ -267,7 +267,7 @@ void _glfwPlatformSetCursorMode(_GLFWwindow* window, int mode)
 
 int _glfwPlatformCreateCursor(_GLFWcursor* cursor,
                               const GLFWimage* image,
-                              int xhot, int yhot)
+                              int xhot, int yhot, int count)
 {
     return GLFW_TRUE;
 }

glfw/source-info.json

@@ -1,132 +1,148 @@
 {
   "cocoa": {
     "headers": [
-      "cocoa_platform.h", 
-      "cocoa_joystick.h", 
-      "posix_thread.h", 
-      "nsgl_context.h", 
-      "egl_context.h", 
+      "cocoa_platform.h",
+      "cocoa_joystick.h",
+      "posix_thread.h",
+      "nsgl_context.h",
+      "egl_context.h",
       "osmesa_context.h"
-    ], 
+    ],
     "sources": [
-      "cocoa_init.m", 
-      "cocoa_joystick.m", 
-      "cocoa_monitor.m", 
-      "cocoa_window.m", 
-      "cocoa_time.c", 
-      "posix_thread.c", 
-      "nsgl_context.m", 
-      "egl_context.c", 
+      "cocoa_init.m",
+      "cocoa_joystick.m",
+      "cocoa_monitor.m",
+      "cocoa_window.m",
+      "cocoa_time.c",
+      "posix_thread.c",
+      "nsgl_context.m",
+      "egl_context.c",
       "osmesa_context.c"
     ]
-  }, 
+  },
   "common": {
     "headers": [
-      "internal.h", 
+      "internal.h",
       "mappings.h"
-    ], 
+    ],
     "sources": [
-      "context.c", 
-      "init.c", 
-      "input.c", 
-      "monitor.c", 
-      "vulkan.c", 
+      "context.c",
+      "init.c",
+      "input.c",
+      "monitor.c",
+      "vulkan.c",
       "window.c"
     ]
-  }, 
+  },
   "osmesa": {
     "headers": [
-      "null_platform.h", 
-      "null_joystick.h", 
-      "posix_time.h", 
-      "posix_thread.h", 
+      "null_platform.h",
+      "null_joystick.h",
+      "posix_time.h",
+      "posix_thread.h",
       "osmesa_context.h"
-    ], 
+    ],
     "sources": [
-      "null_init.c", 
-      "null_monitor.c", 
-      "null_window.c", 
-      "null_joystick.c", 
-      "posix_time.c", 
-      "posix_thread.c", 
+      "null_init.c",
+      "null_monitor.c",
+      "null_window.c",
+      "null_joystick.c",
+      "posix_time.c",
+      "posix_thread.c",
       "osmesa_context.c"
     ]
-  }, 
+  },
   "wayland": {
     "headers": [
-      "wl_platform.h", 
-      "posix_time.h", 
-      "posix_thread.h", 
-      "xkb_glfw.h", 
-      "egl_context.h", 
-      "osmesa_context.h", 
-      "linux_joystick.h"
-    ], 
+      "wl_platform.h",
+      "posix_time.h",
+      "posix_thread.h",
+      "xkb_glfw.h",
+      "dbus_glfw.h",
+      "ibus_glfw.h",
+      "backend_utils.h",
+      "egl_context.h",
+      "osmesa_context.h",
+      "linux_joystick.h",
+      "null_joystick.h"
+    ],
     "protocols": [
-      "stable/xdg-shell/xdg-shell.xml", 
-      "stable/viewporter/viewporter.xml", 
-      "unstable/relative-pointer/relative-pointer-unstable-v1.xml", 
-      "unstable/pointer-constraints/pointer-constraints-unstable-v1.xml", 
+      "stable/xdg-shell/xdg-shell.xml",
+      "stable/viewporter/viewporter.xml",
+      "unstable/relative-pointer/relative-pointer-unstable-v1.xml",
+      "unstable/pointer-constraints/pointer-constraints-unstable-v1.xml",
       "unstable/idle-inhibit/idle-inhibit-unstable-v1.xml"
-    ], 
+    ],
     "sources": [
-      "wl_init.c", 
-      "wl_monitor.c", 
-      "wl_window.c", 
-      "posix_time.c", 
-      "posix_thread.c", 
-      "xkb_glfw.c", 
-      "egl_context.c", 
-      "osmesa_context.c", 
-      "linux_joystick.c"
+      "wl_init.c",
+      "wl_monitor.c",
+      "wl_window.c",
+      "posix_time.c",
+      "posix_thread.c",
+      "xkb_glfw.c",
+      "dbus_glfw.c",
+      "ibus_glfw.c",
+      "egl_context.c",
+      "osmesa_context.c",
+      "backend_utils.c",
+      "linux_joystick.c",
+      "null_joystick.c"
     ]
-  }, 
+  },
   "wayland_protocols": [
-    1, 
+    1,
     12
-  ], 
+  ],
   "win32": {
     "headers": [
-      "win32_platform.h", 
-      "win32_joystick.h", 
-      "wgl_context.h", 
-      "egl_context.h", 
+      "win32_platform.h",
+      "win32_joystick.h",
+      "wgl_context.h",
+      "egl_context.h",
       "osmesa_context.h"
-    ], 
+    ],
     "sources": [
-      "win32_init.c", 
-      "win32_joystick.c", 
-      "win32_monitor.c", 
-      "win32_time.c", 
-      "win32_thread.c", 
-      "win32_window.c", 
-      "wgl_context.c", 
-      "egl_context.c", 
+      "win32_init.c",
+      "win32_joystick.c",
+      "win32_monitor.c",
+      "win32_time.c",
+      "win32_thread.c",
+      "win32_window.c",
+      "wgl_context.c",
+      "egl_context.c",
       "osmesa_context.c"
     ]
-  }, 
+  },
   "x11": {
     "headers": [
-      "x11_platform.h", 
-      "xkb_glfw.h", 
-      "posix_time.h", 
-      "posix_thread.h", 
-      "glx_context.h", 
-      "egl_context.h", 
-      "osmesa_context.h", 
-      "linux_joystick.h"
-    ], 
+      "x11_platform.h",
+      "xkb_glfw.h",
+      "dbus_glfw.h",
+      "ibus_glfw.h",
+      "backend_utils.h",
+      "posix_time.h",
+      "posix_thread.h",
+      "glx_context.h",
+      "egl_context.h",
+      "osmesa_context.h",
+      "linux_joystick.h",
+      "null_joystick.h"
+    ],
     "sources": [
-      "x11_init.c", 
-      "x11_monitor.c", 
-      "x11_window.c", 
-      "xkb_glfw.c", 
-      "posix_time.c", 
-      "posix_thread.c", 
-      "glx_context.c", 
-      "egl_context.c", 
-      "osmesa_context.c", 
-      "linux_joystick.c"
+      "x11_init.c",
+      "x11_monitor.c",
+      "x11_window.c",
+      "xkb_glfw.c",
+      "dbus_glfw.c",
+      "ibus_glfw.c",
+      "posix_time.c",
+      "posix_thread.c",
+      "glx_context.c",
+      "egl_context.c",
+      "osmesa_context.c",
+      "backend_utils.c",
+      "linux_joystick.c",
+      "null_joystick.c"
     ]
   }
 }

glfw/win32_platform.h

@@ -120,6 +120,8 @@ typedef struct
     HRGN hRgnBlur;
     BOOL fTransitionOnMaximized;
 } DWM_BLURBEHIND;
+#else
+ #include <dwmapi.h>
 #endif /*Windows Vista*/
 
 #ifndef DPI_ENUMS_DECLARED
@@ -140,6 +142,9 @@ typedef enum
 
 // HACK: Define versionhelpers.h functions manually as MinGW lacks the header
 BOOL IsWindowsVersionOrGreater(WORD major, WORD minor, WORD sp);
+#define IsWindowsXPOrGreater()                                 \
+    IsWindowsVersionOrGreater(HIBYTE(_WIN32_WINNT_WINXP),      \
+                              LOBYTE(_WIN32_WINNT_WINXP), 0)
 #define IsWindowsVistaOrGreater()                              \
     IsWindowsVersionOrGreater(HIBYTE(_WIN32_WINNT_VISTA),      \
                               LOBYTE(_WIN32_WINNT_VISTA), 0)
@@ -240,7 +245,9 @@ typedef VkBool32 (APIENTRY *PFN_vkGetPhysicalDeviceWin32PresentationSupportKHR)(
 #include "egl_context.h"
 #include "osmesa_context.h"
 
-#define _GLFW_WNDCLASSNAME L"GLFW30"
+#if !defined(_GLFW_WNDCLASSNAME)
+ #define _GLFW_WNDCLASSNAME L"GLFW30"
+#endif
 
 #define _glfw_dlopen(name) LoadLibraryA(name)
 #define _glfw_dlclose(handle) FreeLibrary((HMODULE) handle)
@@ -296,6 +303,7 @@ typedef struct _GLFWlibraryWin32
     _GLFWwindow*        disabledCursorWindow;
     RAWINPUT*           rawInput;
     int                 rawInputSize;
+    UINT                mouseTrailSize;
 
     struct {
         HINSTANCE                       instance;

glfw/win32_window.c

@@ -235,26 +235,6 @@ static void centerCursor(_GLFWwindow* window)
     _glfwPlatformSetCursorPos(window, width / 2.0, height / 2.0);
 }
 
-// Returns whether the cursor is in the client area of the specified window
-//
-static GLFWbool cursorInClientArea(_GLFWwindow* window)
-{
-    RECT area;
-    POINT pos;
-
-    if (!GetCursorPos(&pos))
-        return GLFW_FALSE;
-
-    if (WindowFromPoint(pos) != window->win32.handle)
-        return GLFW_FALSE;
-
-    GetClientRect(window->win32.handle, &area);
-    ClientToScreen(window->win32.handle, (POINT*) &area.left);
-    ClientToScreen(window->win32.handle, (POINT*) &area.right);
-
-    return PtInRect(&area, pos);
-}
-
 // Updates the cursor image according to its cursor mode
 //
 static void updateCursorImage(_GLFWwindow* window)
@@ -286,6 +266,67 @@ static void updateClipRect(_GLFWwindow* window)
         ClipCursor(NULL);
 }
 
+// Apply disabled cursor mode to a focused window
+//
+static void disableCursor(_GLFWwindow* window)
+{
+    const RAWINPUTDEVICE rid = { 0x01, 0x02, 0, window->win32.handle };
+
+    _glfw.win32.disabledCursorWindow = window;
+    _glfwPlatformGetCursorPos(window,
+                              &_glfw.win32.restoreCursorPosX,
+                              &_glfw.win32.restoreCursorPosY);
+    updateCursorImage(window);
+    centerCursor(window);
+    updateClipRect(window);
+
+    if (!RegisterRawInputDevices(&rid, 1, sizeof(rid)))
+    {
+        _glfwInputErrorWin32(GLFW_PLATFORM_ERROR,
+                             "Win32: Failed to register raw input device");
+    }
+}
+
+// Exit disabled cursor mode for the specified window
+//
+static void enableCursor(_GLFWwindow* window)
+{
+    const RAWINPUTDEVICE rid = { 0x01, 0x02, RIDEV_REMOVE, NULL };
+
+    _glfw.win32.disabledCursorWindow = NULL;
+    updateClipRect(NULL);
+    _glfwPlatformSetCursorPos(window,
+                              _glfw.win32.restoreCursorPosX,
+                              _glfw.win32.restoreCursorPosY);
+    updateCursorImage(window);
+
+    if (!RegisterRawInputDevices(&rid, 1, sizeof(rid)))
+    {
+        _glfwInputErrorWin32(GLFW_PLATFORM_ERROR,
+                             "Win32: Failed to remove raw input device");
+    }
+}
+
+// Returns whether the cursor is in the client area of the specified window
+//
+static GLFWbool cursorInClientArea(_GLFWwindow* window)
+{
+    RECT area;
+    POINT pos;
+
+    if (!GetCursorPos(&pos))
+        return GLFW_FALSE;
+
+    if (WindowFromPoint(pos) != window->win32.handle)
+        return GLFW_FALSE;
+
+    GetClientRect(window->win32.handle, &area);
+    ClientToScreen(window->win32.handle, (POINT*) &area.left);
+    ClientToScreen(window->win32.handle, (POINT*) &area.right);
+
+    return PtInRect(&area, pos);
+}
+
 // Update native window styles to match attributes
 //
 static void updateWindowStyles(const _GLFWwindow* window)
@@ -488,7 +529,18 @@ static void fitToMonitor(_GLFWwindow* window)
 static void acquireMonitor(_GLFWwindow* window)
 {
     if (!_glfw.win32.acquiredMonitorCount)
+    {
         SetThreadExecutionState(ES_CONTINUOUS | ES_DISPLAY_REQUIRED);
+
+        // HACK: When mouse trails are enabled the cursor becomes invisible when
+        //       the OpenGL ICD switches to page flipping
+        if (IsWindowsXPOrGreater())
+        {
+            SystemParametersInfo(SPI_GETMOUSETRAILS, 0, &_glfw.win32.mouseTrailSize, 0);
+            SystemParametersInfo(SPI_SETMOUSETRAILS, 0, 0, 0);
+        }
+    }
+
     if (!window->monitor->window)
         _glfw.win32.acquiredMonitorCount++;
 
@@ -505,8 +557,14 @@ static void releaseMonitor(_GLFWwindow* window)
 
     _glfw.win32.acquiredMonitorCount--;
     if (!_glfw.win32.acquiredMonitorCount)
+    {
         SetThreadExecutionState(ES_CONTINUOUS);
 
+        // HACK: Restore mouse trail length saved in acquireMonitor
+        if (IsWindowsXPOrGreater())
+            SystemParametersInfo(SPI_SETMOUSETRAILS, _glfw.win32.mouseTrailSize, 0, 0);
+    }
+
     _glfwInputMonitorWindow(window->monitor, NULL);
     _glfwRestoreVideoModeWin32(window->monitor);
 }
@@ -575,7 +633,7 @@ static LRESULT CALLBACK windowProc(HWND hWnd, UINT uMsg,
             if (lParam == 0 && window->win32.frameAction)
             {
                 if (window->cursorMode == GLFW_CURSOR_DISABLED)
-                    _glfwPlatformSetCursorMode(window, GLFW_CURSOR_DISABLED);
+                    disableCursor(window);
 
                 window->win32.frameAction = GLFW_FALSE;
             }
@@ -593,7 +651,7 @@ static LRESULT CALLBACK windowProc(HWND hWnd, UINT uMsg,
                 break;
 
             if (window->cursorMode == GLFW_CURSOR_DISABLED)
-                _glfwPlatformSetCursorMode(window, GLFW_CURSOR_DISABLED);
+                disableCursor(window);
 
             return 0;
         }
@@ -601,7 +659,7 @@ static LRESULT CALLBACK windowProc(HWND hWnd, UINT uMsg,
         case WM_KILLFOCUS:
         {
             if (window->cursorMode == GLFW_CURSOR_DISABLED)
-                _glfwPlatformSetCursorMode(window, GLFW_CURSOR_NORMAL);
+                enableCursor(window);
 
             if (window->monitor && window->autoIconify)
                 _glfwPlatformIconifyWindow(window);
@@ -857,10 +915,10 @@ static LRESULT CALLBACK windowProc(HWND hWnd, UINT uMsg,
         case WM_ENTERSIZEMOVE:
         case WM_ENTERMENULOOP:
         {
-            // HACK: Postpone cursor disabling while the user is moving or
-            //       resizing the window or using the menu
+            // HACK: Enable the cursor while the user is moving or
+            //       resizing the window or using the window menu
             if (window->cursorMode == GLFW_CURSOR_DISABLED)
-                _glfwPlatformSetCursorMode(window, GLFW_CURSOR_NORMAL);
+                enableCursor(window);
 
             break;
         }
@@ -871,7 +929,7 @@ static LRESULT CALLBACK windowProc(HWND hWnd, UINT uMsg,
             // HACK: Disable the cursor once the user is done moving or
             //       resizing the window or using the menu
             if (window->cursorMode == GLFW_CURSOR_DISABLED)
-                _glfwPlatformSetCursorMode(window, GLFW_CURSOR_DISABLED);
+                disableCursor(window);
 
             break;
         }
@@ -1777,39 +1835,12 @@ void _glfwPlatformSetCursorMode(_GLFWwindow* window, int mode)
 {
     if (mode == GLFW_CURSOR_DISABLED)
     {
-        const RAWINPUTDEVICE rid = { 0x01, 0x02, 0, window->win32.handle };
-
-        _glfw.win32.disabledCursorWindow = window;
-        _glfwPlatformGetCursorPos(window,
-                                  &_glfw.win32.restoreCursorPosX,
-                                  &_glfw.win32.restoreCursorPosY);
-        centerCursor(window);
-        updateClipRect(window);
-
-        if (!RegisterRawInputDevices(&rid, 1, sizeof(rid)))
-        {
-            _glfwInputErrorWin32(GLFW_PLATFORM_ERROR,
-                                 "Win32: Failed to register raw input device");
-        }
+        if (_glfwPlatformWindowFocused(window))
+            disableCursor(window);
     }
     else if (_glfw.win32.disabledCursorWindow == window)
-    {
-        const RAWINPUTDEVICE rid = { 0x01, 0x02, RIDEV_REMOVE, NULL };
-
-        _glfw.win32.disabledCursorWindow = NULL;
-        updateClipRect(NULL);
-        _glfwPlatformSetCursorPos(window,
-                                  _glfw.win32.restoreCursorPosX,
-                                  _glfw.win32.restoreCursorPosY);
-
-        if (!RegisterRawInputDevices(&rid, 1, sizeof(rid)))
-        {
-            _glfwInputErrorWin32(GLFW_PLATFORM_ERROR,
-                                 "Win32: Failed to remove raw input device");
-        }
-    }
-
-    if (cursorInClientArea(window))
+        enableCursor(window);
+    else if (cursorInClientArea(window))
         updateCursorImage(window);
 }
 
@@ -1825,7 +1856,7 @@ int _glfwPlatformGetKeyScancode(int key)
 
 int _glfwPlatformCreateCursor(_GLFWcursor* cursor,
                               const GLFWimage* image,
-                              int xhot, int yhot)
+                              int xhot, int yhot, int count)
 {
     cursor->win32.handle = (HCURSOR) createIcon(image, xhot, yhot, GLFW_FALSE);
     if (!cursor->win32.handle)

glfw/window.c

@@ -48,6 +48,7 @@ void _glfwInputWindowFocus(_GLFWwindow* window, GLFWbool focused)
     if (!focused)
     {
         int key, button;
+        _glfw.focusedWindowId = 0;
 
         for (key = 0;  key <= GLFW_KEY_LAST;  key++)
         {
@@ -63,7 +64,28 @@ void _glfwInputWindowFocus(_GLFWwindow* window, GLFWbool focused)
             if (window->mouseButtons[button] == GLFW_PRESS)
                 _glfwInputMouseClick(window, button, GLFW_RELEASE, 0);
         }
+    } else
+        _glfw.focusedWindowId = window->id;
+}
+
+_GLFWwindow* _glfwFocusedWindow() {
+    if (_glfw.focusedWindowId) {
+        _GLFWwindow *w = _glfw.windowListHead;
+        while (w) {
+            if (w->id == _glfw.focusedWindowId) return w;
+            w = w->next;
+        }
     }
+    return NULL;
+}
+
+_GLFWwindow* _glfwWindowForId(GLFWid id) {
+    _GLFWwindow *w = _glfw.windowListHead;
+    while (w) {
+        if (w->id == _glfw.focusedWindowId) return w;
+        w = w->next;
+    }
+    return NULL;
 }
 
 // Notifies shared code that a window has moved
@@ -185,8 +207,10 @@ GLFWAPI GLFWwindow* glfwCreateWindow(int width, int height,
     if (!_glfwIsValidContextConfig(&ctxconfig))
         return NULL;
 
+    static GLFWid windowIdCounter = 0;
     window = calloc(1, sizeof(_GLFWwindow));
     window->next = _glfw.windowListHead;
+    window->id = ++windowIdCounter;
     _glfw.windowListHead = window;
 
     window->videoMode.width       = width;
@@ -201,6 +225,7 @@ GLFWAPI GLFWwindow* glfwCreateWindow(int width, int height,
     window->decorated   = wndconfig.decorated;
     window->autoIconify = wndconfig.autoIconify;
     window->floating    = wndconfig.floating;
+    window->focusOnShow = wndconfig.focusOnShow;
     window->cursorMode  = GLFW_CURSOR_NORMAL;
 
     window->minwidth    = GLFW_DONT_CARE;
@@ -267,6 +292,7 @@ void glfwDefaultWindowHints(void)
     _glfw.hints.window.focused      = GLFW_TRUE;
     _glfw.hints.window.autoIconify  = GLFW_TRUE;
     _glfw.hints.window.centerCursor = GLFW_TRUE;
+    _glfw.hints.window.focusOnShow  = GLFW_TRUE;
 
     // The default is 24 bits of color, 24 bits of depth and 8 bits of stencil,
     // double buffered
@@ -370,6 +396,9 @@ GLFWAPI void glfwWindowHint(int hint, int value)
         case GLFW_CENTER_CURSOR:
             _glfw.hints.window.centerCursor = value ? GLFW_TRUE : GLFW_FALSE;
             return;
+        case GLFW_FOCUS_ON_SHOW:
+            _glfw.hints.window.focusOnShow = value ? GLFW_TRUE : GLFW_FALSE;
+            return;
         case GLFW_CLIENT_API:
             _glfw.hints.context.client = value;
             return;
@@ -755,7 +784,9 @@ GLFWAPI void glfwShowWindow(GLFWwindow* handle)
         return;
 
     _glfwPlatformShowWindow(window);
-    _glfwPlatformFocusWindow(window);
+
+    if (window->focusOnShow)
+        _glfwPlatformFocusWindow(window);
 }
 
 GLFWAPI void glfwRequestWindowAttention(GLFWwindow* handle)
@@ -820,6 +851,8 @@ GLFWAPI int glfwGetWindowAttrib(GLFWwindow* handle, int attrib)
             return _glfwPlatformWindowMaximized(window);
         case GLFW_HOVERED:
             return _glfwPlatformWindowHovered(window);
+        case GLFW_FOCUS_ON_SHOW:
+            return window->focusOnShow;
         case GLFW_TRANSPARENT_FRAMEBUFFER:
             return _glfwPlatformFramebufferTransparent(window);
         case GLFW_RESIZABLE:
@@ -896,6 +929,8 @@ GLFWAPI void glfwSetWindowAttrib(GLFWwindow* handle, int attrib, int value)
         if (!window->monitor)
             _glfwPlatformSetWindowFloating(window, value);
     }
+    else if (attrib == GLFW_FOCUS_ON_SHOW)
+        window->focusOnShow = value;
     else
         _glfwInputError(GLFW_INVALID_ENUM, "Invalid window attribute 0x%08X", attrib);
 }

glfw/wl_init.c

@@ -24,7 +24,9 @@
 //
 //========================================================================
 
+#define _GNU_SOURCE
 #include "internal.h"
+#include "backend_utils.h"
 
 #include <assert.h>
 #include <linux/input.h>
@@ -33,6 +35,7 @@
 #include <string.h>
 #include <sys/mman.h>
 #include <unistd.h>
+#include <fcntl.h>
 #include <wayland-client.h>
 
 
@@ -161,7 +164,7 @@ static void pointerHandleMotion(void* data,
                                 wl_fixed_t sy)
 {
     _GLFWwindow* window = _glfw.wl.pointerFocus;
-    const char* cursorName;
+    const char* cursorName = NULL;
 
     if (!window)
         return;
@@ -305,7 +308,7 @@ static void pointerHandleButton(void* data,
                          state == WL_POINTER_BUTTON_STATE_PRESSED
                                 ? GLFW_PRESS
                                 : GLFW_RELEASE,
-                         _glfw.wl.xkb.modifiers);
+                         _glfw.wl.xkb.states.modifiers);
 }
 
 static void pointerHandleAxis(void* data,
@@ -316,11 +319,6 @@ static void pointerHandleAxis(void* data,
 {
     _GLFWwindow* window = _glfw.wl.pointerFocus;
     double x = 0.0, y = 0.0;
-    // Wayland scroll events are in pointer motion coordinate space (think two
-    // finger scroll).  The factor 10 is commonly used to convert to "scroll
-    // step means 1.0.
-    const double scrollFactor = 1.0 / 10.0;
-
     if (!window)
         return;
 
@@ -328,11 +326,11 @@ static void pointerHandleAxis(void* data,
            axis == WL_POINTER_AXIS_VERTICAL_SCROLL);
 
     if (axis == WL_POINTER_AXIS_HORIZONTAL_SCROLL)
-        x = wl_fixed_to_double(value) * scrollFactor;
+        x = wl_fixed_to_double(value) * -1;
     else if (axis == WL_POINTER_AXIS_VERTICAL_SCROLL)
-        y = wl_fixed_to_double(value) * scrollFactor;
+        y = wl_fixed_to_double(value) * -1;
 
-    _glfwInputScroll(window, x, y);
+    _glfwInputScroll(window, x, y, 1);
 }
 
 static const struct wl_pointer_listener pointerListener = {
@@ -404,6 +402,15 @@ static void keyboardHandleLeave(void* data,
     _glfwInputWindowFocus(window, GLFW_FALSE);
 }
 
+static void
+dispatchPendingKeyRepeats(id_type timer_id, void *data) {
+    if (_glfw.wl.keyRepeatInfo.keyboardFocus != _glfw.wl.keyboardFocus || _glfw.wl.keyboardRepeatRate == 0) return;
+    glfw_xkb_handle_key_event(_glfw.wl.keyRepeatInfo.keyboardFocus, &_glfw.wl.xkb, _glfw.wl.keyRepeatInfo.key, GLFW_REPEAT);
+    changeTimerInterval(&_glfw.wl.eventLoopData, _glfw.wl.keyRepeatInfo.keyRepeatTimer, ((double)_glfw.wl.keyboardRepeatRate) / 1000.0);
+    toggleTimer(&_glfw.wl.eventLoopData, _glfw.wl.keyRepeatInfo.keyRepeatTimer, 1);
+}
+
+
 static void keyboardHandleKey(void* data,
                               struct wl_keyboard* keyboard,
                               uint32_t serial,
@@ -416,14 +423,18 @@ static void keyboardHandleKey(void* data,
         return;
     int action = state == WL_KEYBOARD_KEY_STATE_PRESSED ? GLFW_PRESS : GLFW_RELEASE;
     glfw_xkb_handle_key_event(window, &_glfw.wl.xkb, key, action);
-    _glfw.wl.keyRepeatInfo.nextRepeatAt = 0;
+    GLFWbool repeatable = GLFW_FALSE;
 
     if (action == GLFW_PRESS && _glfw.wl.keyboardRepeatRate > 0 && glfw_xkb_should_repeat(&_glfw.wl.xkb, key))
     {
         _glfw.wl.keyRepeatInfo.key = key;
-        _glfw.wl.keyRepeatInfo.nextRepeatAt = glfwGetTime() + (double)(_glfw.wl.keyboardRepeatDelay) / 1000.0;
+        repeatable = GLFW_TRUE;
         _glfw.wl.keyRepeatInfo.keyboardFocus = window;
     }
+    if (repeatable) {
+        changeTimerInterval(&_glfw.wl.eventLoopData, _glfw.wl.keyRepeatInfo.keyRepeatTimer, (double)(_glfw.wl.keyboardRepeatDelay) / 1000.0);
+    }
+    toggleTimer(&_glfw.wl.eventLoopData, _glfw.wl.keyRepeatInfo.keyRepeatTimer, repeatable ? 1 : 0);
 }
 
 static void keyboardHandleModifiers(void* data,
@@ -615,6 +626,13 @@ static const struct wl_registry_listener registryListener = {
 
 int _glfwPlatformInit(void)
 {
+    if (pipe2(_glfw.wl.eventLoopData.wakeupFds, O_CLOEXEC | O_NONBLOCK) != 0)
+    {
+        _glfwInputError(GLFW_PLATFORM_ERROR,
+                "Wayland: failed to create self pipe");
+        return GLFW_FALSE;
+    }
+
     _glfw.wl.cursor.handle = _glfw_dlopen("libwayland-cursor.so.0");
     if (!_glfw.wl.cursor.handle)
     {
@@ -654,6 +672,9 @@ int _glfwPlatformInit(void)
                         "Wayland: Failed to connect to display");
         return GLFW_FALSE;
     }
+    initPollData(&_glfw.wl.eventLoopData, _glfw.wl.eventLoopData.wakeupFds[0], wl_display_get_fd(_glfw.wl.display));
+    glfw_dbus_init(&_glfw.wl.dbus, &_glfw.wl.eventLoopData);
+    _glfw.wl.keyRepeatInfo.keyRepeatTimer = addTimer(&_glfw.wl.eventLoopData, "wayland-keyrepeat", 0.5, 0, dispatchPendingKeyRepeats, NULL);
 
     _glfw.wl.registry = wl_display_get_registry(_glfw.wl.display);
     wl_registry_add_listener(_glfw.wl.registry, &registryListener, NULL);
@@ -667,8 +688,10 @@ int _glfwPlatformInit(void)
     wl_display_roundtrip(_glfw.wl.display);
 
 #ifdef __linux__
-    if (!_glfwInitJoysticksLinux())
-        return GLFW_FALSE;
+    if (_glfw.hints.init.enableJoysticks) {
+        if (!_glfwInitJoysticksLinux())
+            return GLFW_FALSE;
+    }
 #endif
 
     _glfwInitTimerPOSIX();
@@ -702,6 +725,7 @@ void _glfwPlatformTerminate(void)
     }
 
     glfw_xkb_release(&_glfw.wl.xkb);
+    glfw_dbus_terminate(&_glfw.wl.dbus);
 
     if (_glfw.wl.cursorTheme)
         wl_cursor_theme_destroy(_glfw.wl.cursorTheme);
@@ -744,6 +768,7 @@ void _glfwPlatformTerminate(void)
         wl_display_flush(_glfw.wl.display);
         wl_display_disconnect(_glfw.wl.display);
     }
+    closeFds(_glfw.wl.eventLoopData.wakeupFds, sizeof(_glfw.wl.eventLoopData.wakeupFds)/sizeof(_glfw.wl.eventLoopData.wakeupFds[0]));
 }
 
 const char* _glfwPlatformGetVersionString(void)

glfw/wl_platform.h

@@ -26,6 +26,7 @@
 
 #include <wayland-client.h>
 #include <dlfcn.h>
+#include <poll.h>
 
 typedef VkFlags VkWaylandSurfaceCreateFlagsKHR;
 
@@ -48,6 +49,7 @@ typedef VkBool32 (APIENTRY *PFN_vkGetPhysicalDeviceWaylandPresentationSupportKHR
 #else
 #include "null_joystick.h"
 #endif
+#include "backend_utils.h"
 #include "xkb_glfw.h"
 #include "egl_context.h"
 #include "osmesa_context.h"
@@ -203,10 +205,11 @@ typedef struct _GLFWlibraryWayland
     int32_t                     keyboardRepeatDelay;
     struct {
         uint32_t                key;
-        double                  nextRepeatAt;
+        id_type                 keyRepeatTimer;
         _GLFWwindow*            keyboardFocus;
     } keyRepeatInfo;
     _GLFWXKBData                xkb;
+    _GLFWDBUSData               dbus;
 
     _GLFWwindow*                pointerFocus;
     _GLFWwindow*                keyboardFocus;
@@ -228,6 +231,8 @@ typedef struct _GLFWlibraryWayland
         PFN_wl_egl_window_resize window_resize;
     } egl;
 
+    EventLoopData eventLoopData;
+
 } _GLFWlibraryWayland;
 
 // Wayland-specific per-monitor data

glfw/wl_window.c

@@ -27,15 +27,14 @@
 #define _GNU_SOURCE
 
 #include "internal.h"
+#include "backend_utils.h"
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <errno.h>
-#include <unistd.h>
 #include <string.h>
 #include <fcntl.h>
 #include <sys/mman.h>
-#include <poll.h>
 
 
 static void handlePing(void* data,
@@ -690,34 +689,13 @@ static GLFWbool createXdgSurface(_GLFWwindow* window)
 }
 
 static void
-dispatchPendingKeyRepeats() {
-    if (_glfw.wl.keyRepeatInfo.nextRepeatAt <= 0 || _glfw.wl.keyRepeatInfo.keyboardFocus != _glfw.wl.keyboardFocus || _glfw.wl.keyboardRepeatRate == 0) return;
-    double now = glfwGetTime();
-    while (_glfw.wl.keyRepeatInfo.nextRepeatAt <= now) {
-        glfw_xkb_handle_key_event(_glfw.wl.keyRepeatInfo.keyboardFocus, &_glfw.wl.xkb, _glfw.wl.keyRepeatInfo.key, GLFW_REPEAT);
-        _glfw.wl.keyRepeatInfo.nextRepeatAt += 1.0 / _glfw.wl.keyboardRepeatRate;
-        now = glfwGetTime();
-    }
-}
-
-static int
-adjustTimeoutForKeyRepeat(int timeout) {
-    if (_glfw.wl.keyRepeatInfo.nextRepeatAt <= 0 || _glfw.wl.keyRepeatInfo.keyboardFocus != _glfw.wl.keyboardFocus || _glfw.wl.keyboardRepeatRate == 0) return timeout;
-    double now = glfwGetTime();
-    if (timeout < 0 || now + timeout / 1000. > _glfw.wl.keyRepeatInfo.nextRepeatAt) {
-        timeout = _glfw.wl.keyRepeatInfo.nextRepeatAt <= now ? 0 : ( (_glfw.wl.keyRepeatInfo.nextRepeatAt - now) * 1000 + 1 );
-    }
-    return timeout;
-}
-
-static void
-handleEvents(int timeout)
+handleEvents(double timeout)
 {
     struct wl_display* display = _glfw.wl.display;
-    struct pollfd pfd = { wl_display_get_fd(display), POLLIN };
 
-    while (wl_display_prepare_read(display) != 0)
+    while (wl_display_prepare_read(display) != 0) {
         wl_display_dispatch_pending(display);
+    }
 
     // If an error different from EAGAIN happens, we have likely been
     // disconnected from the Wayland session, try to handle that the best we
@@ -734,27 +712,16 @@ handleEvents(int timeout)
         return;
     }
 
-    dispatchPendingKeyRepeats();
-    timeout = adjustTimeoutForKeyRepeat(timeout);
-
-    if (poll(&pfd, 1, timeout) > 0)
-    {
-        if (pfd.revents & POLLIN)
-        {
-            wl_display_read_events(display);
-            wl_display_dispatch_pending(display);
-        }
-        else
-        {
-            wl_display_cancel_read(display);
-        }
-
+    GLFWbool display_read_ok = pollForEvents(&_glfw.wl.eventLoopData, timeout);
+    if (display_read_ok) {
+        wl_display_read_events(display);
+        wl_display_dispatch_pending(display);
     }
     else
     {
         wl_display_cancel_read(display);
     }
-    dispatchPendingKeyRepeats();
+    glfw_ibus_dispatch(&_glfw.wl.xkb.ibus);
 }
 
 // Translates a GLFW standard cursor to a theme cursor name
@@ -1090,7 +1057,8 @@ void _glfwPlatformRequestWindowAttention(_GLFWwindow* window)
 int _glfwPlatformWindowBell(_GLFWwindow* window)
 {
     // TODO: Use an actual Wayland API to implement this when one becomes available
-    int fd = open("/dev/tty", O_WRONLY | O_CLOEXEC);
+    static char tty[L_ctermid + 1];
+    int fd = open(ctermid(tty), O_WRONLY | O_CLOEXEC);
     if (fd > -1) {
         int ret = write(fd, "\x07", 1) == 1 ? GLFW_TRUE : GLFW_FALSE;
         close(fd);
@@ -1196,22 +1164,26 @@ void _glfwPlatformSetWindowOpacity(_GLFWwindow* window, float opacity)
 
 void _glfwPlatformPollEvents(void)
 {
+    wl_display_dispatch_pending(_glfw.wl.display);
     handleEvents(0);
 }
 
 void _glfwPlatformWaitEvents(void)
 {
-    handleEvents(-1);
+    double timeout = wl_display_dispatch_pending(_glfw.wl.display) > 0 ? 0 : -1;
+    handleEvents(timeout);
 }
 
 void _glfwPlatformWaitEventsTimeout(double timeout)
 {
-    handleEvents((int) (timeout * 1e3));
+    if (wl_display_dispatch_pending(_glfw.wl.display) > 0) timeout = 0;
+    handleEvents(timeout);
 }
 
 void _glfwPlatformPostEmptyEvent(void)
 {
     wl_display_sync(_glfw.wl.display);
+    while (write(_glfw.wl.eventLoopData.wakeupFds[1], "w", 1) < 0 && errno == EINTR);
 }
 
 void _glfwPlatformGetCursorPos(_GLFWwindow* window, double* xpos, double* ypos)
@@ -1252,7 +1224,7 @@ int _glfwPlatformGetKeyScancode(int key)
 
 int _glfwPlatformCreateCursor(_GLFWcursor* cursor,
                               const GLFWimage* image,
-                              int xhot, int yhot)
+                              int xhot, int yhot, int count)
 {
     cursor->wl.buffer = createShmBuffer(image);
     cursor->wl.width = image->width;
@@ -1541,6 +1513,11 @@ VkResult _glfwPlatformCreateWindowSurface(VkInstance instance,
     return err;
 }
 
+void
+_glfwPlatformUpdateIMEState(_GLFWwindow *w, int which, int a, int b, int c, int d) {
+    glfw_xkb_update_ime_state(w, &_glfw.wl.xkb, which, a, b, c, d);
+}
+
 
 //////////////////////////////////////////////////////////////////////////
 //////                        GLFW native API                       //////
@@ -1558,3 +1535,7 @@ GLFWAPI struct wl_surface* glfwGetWaylandWindow(GLFWwindow* handle)
     _GLFW_REQUIRE_INIT_OR_RETURN(NULL);
     return window->wl.surface;
 }
+
+GLFWAPI int glfwGetXKBScancode(const char* keyName, GLFWbool caseSensitive) {
+    return glfw_xkb_keysym_from_name(keyName, caseSensitive);
+}

glfw/x11_init.c

@@ -25,7 +25,9 @@
 //
 //========================================================================
 
+#define _GNU_SOURCE
 #include "internal.h"
+#include "backend_utils.h"
 
 #include <X11/Xresource.h>
 
@@ -34,6 +36,8 @@
 #include <limits.h>
 #include <stdio.h>
 #include <locale.h>
+#include <fcntl.h>
+#include <unistd.h>
 
 
 // Check whether the specified atom is supported
@@ -166,7 +170,11 @@ static GLFWbool initExtensions(void)
                                       &_glfw.x11.vidmode.errorBase);
     }
 
+#if defined(__CYGWIN__)
+    _glfw.x11.xi.handle = _glfw_dlopen("libXi-6.so");
+#else
     _glfw.x11.xi.handle = _glfw_dlopen("libXi.so.6");
+#endif
     if (_glfw.x11.xi.handle)
     {
         _glfw.x11.xi.QueryVersion = (PFN_XIQueryVersion)
@@ -192,7 +200,11 @@ static GLFWbool initExtensions(void)
         }
     }
 
+#if defined(__CYGWIN__)
+    _glfw.x11.randr.handle = _glfw_dlopen("libXrandr-2.so");
+#else
     _glfw.x11.randr.handle = _glfw_dlopen("libXrandr.so.2");
+#endif
     if (_glfw.x11.randr.handle)
     {
         _glfw.x11.randr.AllocGamma = (PFN_XRRAllocGamma)
@@ -280,7 +292,11 @@ static GLFWbool initExtensions(void)
                        RROutputChangeNotifyMask);
     }
 
+#if defined(__CYGWIN__)
+    _glfw.x11.xcursor.handle = _glfw_dlopen("libXcursor-1.so");
+#else
     _glfw.x11.xcursor.handle = _glfw_dlopen("libXcursor.so.1");
+#endif
     if (_glfw.x11.xcursor.handle)
     {
         _glfw.x11.xcursor.ImageCreate = (PFN_XcursorImageCreate)
@@ -291,7 +307,11 @@ static GLFWbool initExtensions(void)
             _glfw_dlsym(_glfw.x11.xcursor.handle, "XcursorImageLoadCursor");
     }
 
+#if defined(__CYGWIN__)
+    _glfw.x11.xinerama.handle = _glfw_dlopen("libXinerama-1.so");
+#else
     _glfw.x11.xinerama.handle = _glfw_dlopen("libXinerama.so.1");
+#endif
     if (_glfw.x11.xinerama.handle)
     {
         _glfw.x11.xinerama.IsActive = (PFN_XineramaIsActive)
@@ -310,16 +330,11 @@ static GLFWbool initExtensions(void)
         }
     }
 
-    _glfw.x11.x11xcb.handle = _glfw_dlopen("libX11-xcb.so.1");
-    if (!_glfw.x11.x11xcb.handle)
-    {
-        _glfwInputError(GLFW_PLATFORM_ERROR, "X11: Failed to load libX11-xcb");
-        return GLFW_FALSE;
-    }
-    _glfw.x11.x11xcb.GetXCBConnection = (PFN_XGetXCBConnection)
-        _glfw_dlsym(_glfw.x11.x11xcb.handle, "XGetXCBConnection");
-
+#if defined(__CYGWIN__)
+    _glfw.x11.xrender.handle = _glfw_dlopen("libXrender-1.so");
+#else
     _glfw.x11.xrender.handle = _glfw_dlopen("libXrender.so.1");
+#endif
     if (_glfw.x11.xrender.handle)
     {
         _glfw.x11.xrender.QueryExtension = (PFN_XRenderQueryExtension)
@@ -443,7 +458,7 @@ static GLFWbool initExtensions(void)
 
 // Retrieve system content scale via folklore heuristics
 //
-static void getSystemContentScale(float* xscale, float* yscale)
+void _glfwGetSystemContentScaleX11(float* xscale, float* yscale, GLFWbool bypass_cache)
 {
     // NOTE: Default to the display-wide DPI as we don't currently have a policy
     //       for which monitor a window is considered to be on
@@ -455,7 +470,20 @@ static void getSystemContentScale(float* xscale, float* yscale)
     // NOTE: Basing the scale on Xft.dpi where available should provide the most
     //       consistent user experience (matches Qt, Gtk, etc), although not
     //       always the most accurate one
-    char* rms = XResourceManagerString(_glfw.x11.display);
+    char* rms = NULL;
+    char* owned_rms = NULL;
+
+    if (bypass_cache)
+    {
+        _glfwGetWindowPropertyX11(_glfw.x11.root,
+                                  _glfw.x11.RESOURCE_MANAGER,
+                                  XA_STRING,
+                                  (unsigned char**) &owned_rms);
+        rms = owned_rms;
+    } else {
+        rms = XResourceManagerString(_glfw.x11.display);
+    }
+
     if (rms)
     {
         XrmDatabase db = XrmGetStringDatabase(rms);
@@ -472,6 +500,7 @@ static void getSystemContentScale(float* xscale, float* yscale)
 
             XrmDestroyDatabase(db);
         }
+        XFree(owned_rms);
     }
 
     *xscale = xdpi / 96.f;
@@ -578,7 +607,6 @@ Cursor _glfwCreateCursorX11(const GLFWimage* image, int xhot, int yhot)
     return cursor;
 }
 
-
 //////////////////////////////////////////////////////////////////////////
 //////                       GLFW platform API                      //////
 //////////////////////////////////////////////////////////////////////////
@@ -588,6 +616,13 @@ int _glfwPlatformInit(void)
     XInitThreads();
     XrmInitialize();
 
+    if (pipe2(_glfw.x11.eventLoopData.wakeupFds, O_CLOEXEC | O_NONBLOCK) != 0)
+    {
+        _glfwInputError(GLFW_PLATFORM_ERROR,
+                "X11: failed to create self pipe");
+        return GLFW_FALSE;
+    }
+
     _glfw.x11.display = XOpenDisplay(NULL);
     if (!_glfw.x11.display)
     {
@@ -606,11 +641,16 @@ int _glfwPlatformInit(void)
         return GLFW_FALSE;
     }
 
+    initPollData(&_glfw.x11.eventLoopData, _glfw.x11.eventLoopData.wakeupFds[0], ConnectionNumber(_glfw.x11.display));
+    glfw_dbus_init(&_glfw.x11.dbus, &_glfw.x11.eventLoopData);
+
     _glfw.x11.screen = DefaultScreen(_glfw.x11.display);
     _glfw.x11.root = RootWindow(_glfw.x11.display, _glfw.x11.screen);
     _glfw.x11.context = XUniqueContext();
+    _glfw.x11.RESOURCE_MANAGER = XInternAtom(_glfw.x11.display, "RESOURCE_MANAGER", True);
+    XSelectInput(_glfw.x11.display, _glfw.x11.root, PropertyChangeMask);
 
-    getSystemContentScale(&_glfw.x11.contentScaleX, &_glfw.x11.contentScaleY);
+    _glfwGetSystemContentScaleX11(&_glfw.x11.contentScaleX, &_glfw.x11.contentScaleY, GLFW_FALSE);
 
     if (!initExtensions())
         return GLFW_FALSE;
@@ -619,8 +659,12 @@ int _glfwPlatformInit(void)
     _glfw.x11.hiddenCursorHandle = createHiddenCursor();
 
 #if defined(__linux__)
-    if (!_glfwInitJoysticksLinux())
-        return GLFW_FALSE;
+    if (_glfw.hints.init.enableJoysticks) {
+        if (!_glfwInitJoysticksLinux())
+            return GLFW_FALSE;
+        if (_glfw.linjs.inotify > 0)
+            addWatch(&_glfw.x11.eventLoopData, "joystick", _glfw.linjs.inotify, POLLIN, 1, NULL, NULL);
+    }
 #endif
 
     _glfwInitTimerPOSIX();
@@ -650,6 +694,7 @@ void _glfwPlatformTerminate(void)
     }
 
     glfw_xkb_release(&_glfw.x11.xkb);
+    glfw_dbus_terminate(&_glfw.x11.dbus);
     free(_glfw.x11.primarySelectionString);
     free(_glfw.x11.clipboardString);
 
@@ -657,12 +702,7 @@ void _glfwPlatformTerminate(void)
     {
         XCloseDisplay(_glfw.x11.display);
         _glfw.x11.display = NULL;
-    }
-
-    if (_glfw.x11.x11xcb.handle)
-    {
-        _glfw_dlclose(_glfw.x11.x11xcb.handle);
-        _glfw.x11.x11xcb.handle = NULL;
+        _glfw.x11.eventLoopData.fds[0].fd = -1;
     }
 
     if (_glfw.x11.xcursor.handle)
@@ -683,6 +723,24 @@ void _glfwPlatformTerminate(void)
         _glfw.x11.xinerama.handle = NULL;
     }
 
+    if (_glfw.x11.xrender.handle)
+    {
+        _glfw_dlclose(_glfw.x11.xrender.handle);
+        _glfw.x11.xrender.handle = NULL;
+    }
+
+    if (_glfw.x11.vidmode.handle)
+    {
+        _glfw_dlclose(_glfw.x11.vidmode.handle);
+        _glfw.x11.vidmode.handle = NULL;
+    }
+
+    if (_glfw.x11.xi.handle)
+    {
+        _glfw_dlclose(_glfw.x11.xi.handle);
+        _glfw.x11.xi.handle = NULL;
+    }
+
     // NOTE: These need to be unloaded after XCloseDisplay, as they register
     //       cleanup callbacks that get called by that function
     _glfwTerminateEGL();
@@ -691,6 +749,7 @@ void _glfwPlatformTerminate(void)
 #if defined(__linux__)
     _glfwTerminateJoysticksLinux();
 #endif
+    closeFds(_glfw.x11.eventLoopData.wakeupFds, sizeof(_glfw.x11.eventLoopData.wakeupFds)/sizeof(_glfw.x11.eventLoopData.wakeupFds[0]));
 }
 
 const char* _glfwPlatformGetVersionString(void)

glfw/x11_monitor.c

@@ -152,6 +152,11 @@ void _glfwPollMonitorsX11(void)
             }
 
             ci = XRRGetCrtcInfo(_glfw.x11.display, sr, oi->crtc);
+            if (!ci)
+            {
+                XRRFreeOutputInfo(oi);
+                continue;
+            }
             if (ci->rotation == RR_Rotate_90 || ci->rotation == RR_Rotate_270)
             {
                 widthMM  = oi->mm_height;
@@ -321,13 +326,15 @@ void _glfwPlatformGetMonitorPos(_GLFWmonitor* monitor, int* xpos, int* ypos)
 
         sr = XRRGetScreenResourcesCurrent(_glfw.x11.display, _glfw.x11.root);
         ci = XRRGetCrtcInfo(_glfw.x11.display, sr, monitor->x11.crtc);
+        if (ci)
+        {
+            if (xpos)
+                *xpos = ci->x;
+            if (ypos)
+                *ypos = ci->y;
 
-        if (xpos)
-            *xpos = ci->x;
-        if (ypos)
-            *ypos = ci->y;
-
-        XRRFreeCrtcInfo(ci);
+            XRRFreeCrtcInfo(ci);
+        }
         XRRFreeScreenResources(sr);
     }
 }
@@ -406,9 +413,13 @@ void _glfwPlatformGetVideoMode(_GLFWmonitor* monitor, GLFWvidmode* mode)
         sr = XRRGetScreenResourcesCurrent(_glfw.x11.display, _glfw.x11.root);
         ci = XRRGetCrtcInfo(_glfw.x11.display, sr, monitor->x11.crtc);
 
-        *mode = vidmodeFromModeInfo(getModeInfo(sr, ci->mode), ci);
+        if (ci) {
+            const XRRModeInfo* mi = getModeInfo(sr, ci->mode);
+            if (mi)  // mi can be NULL if the monitor has been disconnected
+                *mode = vidmodeFromModeInfo(mi, ci);
 
-        XRRFreeCrtcInfo(ci);
+            XRRFreeCrtcInfo(ci);
+        }
         XRRFreeScreenResources(sr);
     }
     else
@@ -501,4 +512,3 @@ GLFWAPI RROutput glfwGetX11Monitor(GLFWmonitor* handle)
     _GLFW_REQUIRE_INIT_OR_RETURN(None);
     return monitor->x11.output;
 }
-

glfw/x11_platform.h

@@ -29,12 +29,16 @@
 #include <signal.h>
 #include <stdint.h>
 #include <dlfcn.h>
+#include <poll.h>
 
 #include <X11/Xlib.h>
 #include <X11/keysym.h>
 #include <X11/Xatom.h>
 #include <X11/Xcursor/Xcursor.h>
 
+// The xcb library is needed to work with libxkb
+#include <X11/Xlib-xcb.h>
+
 // The XRandR extension provides mode setting and gamma control
 #include <X11/extensions/Xrandr.h>
 
@@ -47,7 +51,9 @@
 // The XInput extension provides raw mouse motion input
 #include <X11/extensions/XInput2.h>
 
+// The libxkb library is used for improved keyboard support
 #include "xkb_glfw.h"
+#include "backend_utils.h"
 
 typedef XRRCrtcGamma* (* PFN_XRRAllocGamma)(int);
 typedef void (* PFN_XRRFreeCrtcInfo)(XRRCrtcInfo*);
@@ -98,10 +104,6 @@ typedef XineramaScreenInfo* (* PFN_XineramaQueryScreens)(Display*,int*);
 #define XineramaQueryExtension _glfw.x11.xinerama.QueryExtension
 #define XineramaQueryScreens _glfw.x11.xinerama.QueryScreens
 
-typedef struct xcb_connection_t xcb_connection_t;
-typedef xcb_connection_t* (* PFN_XGetXCBConnection)(Display*);
-#define XGetXCBConnection _glfw.x11.x11xcb.GetXCBConnection
-
 typedef Bool (* PFN_XF86VidModeQueryExtension)(Display*,int*,int*);
 typedef Bool (* PFN_XF86VidModeGetGammaRamp)(Display*,int,int,unsigned short*,unsigned short*,unsigned short*);
 typedef Bool (* PFN_XF86VidModeSetGammaRamp)(Display*,int,int,unsigned short*,unsigned short*,unsigned short*);
@@ -280,6 +282,9 @@ typedef struct _GLFWlibraryX11
     Atom            ATOM_PAIR;
     Atom            GLFW_SELECTION;
 
+    // XRM database atom
+    Atom            RESOURCE_MANAGER;
+
     struct {
         GLFWbool    available;
         void*       handle;
@@ -309,6 +314,7 @@ typedef struct _GLFWlibraryX11
     } randr;
 
     _GLFWXKBData xkb;
+    _GLFWDBUSData dbus;
 
     struct {
         int         count;
@@ -341,11 +347,6 @@ typedef struct _GLFWlibraryX11
         PFN_XineramaQueryScreens QueryScreens;
     } xinerama;
 
-    struct {
-        void*       handle;
-        PFN_XGetXCBConnection GetXCBConnection;
-    } x11xcb;
-
     struct {
         GLFWbool    available;
         void*       handle;
@@ -381,6 +382,8 @@ typedef struct _GLFWlibraryX11
         PFN_XRenderFindVisualFormat FindVisualFormat;
     } xrender;
 
+    EventLoopData eventLoopData;
+
 } _GLFWlibraryX11;
 
 // X11-specific per-monitor data
@@ -405,7 +408,6 @@ typedef struct _GLFWcursorX11
 
 } _GLFWcursorX11;
 
-
 void _glfwPollMonitorsX11(void);
 void _glfwSetVideoModeX11(_GLFWmonitor* monitor, const GLFWvidmode* desired);
 void _glfwRestoreVideoModeX11(_GLFWmonitor* monitor);
@@ -422,4 +424,5 @@ void _glfwGrabErrorHandlerX11(void);
 void _glfwReleaseErrorHandlerX11(void);
 void _glfwInputErrorX11(int error, const char* message);
 
+void _glfwGetSystemContentScaleX11(float* xscale, float* yscale, GLFWbool bypass_cache);
 void _glfwPushSelectionToManagerX11(void);

glfw/x11_window.c

@@ -25,13 +25,13 @@
 //
 //========================================================================
 
+#define _GNU_SOURCE
 #include "internal.h"
+#include "backend_utils.h"
 
 #include <X11/cursorfont.h>
 #include <X11/Xmd.h>
 
-#include <sys/select.h>
-
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -50,50 +50,37 @@
 
 #define _GLFW_XDND_VERSION 5
 
-
-// Wait for data to arrive using select
+// Wait for data to arrive using poll
 // This avoids blocking other threads via the per-display Xlib lock that also
 // covers GLX functions
 //
-static GLFWbool waitForEvent(double* timeout)
-{
-    fd_set fds;
-    const int fd = ConnectionNumber(_glfw.x11.display);
-    int count = fd + 1;
+GLFWbool _glfwDispatchX11Events(void);
 
-#if defined(__linux__)
-    if (_glfw.linjs.inotify > fd)
-        count = _glfw.linjs.inotify + 1;
-#endif
-    for (;;)
-    {
-        FD_ZERO(&fds);
-        FD_SET(fd, &fds);
-#if defined(__linux__)
-        if (_glfw.linjs.inotify > 0)
-            FD_SET(_glfw.linjs.inotify, &fds);
-#endif
+static void
+handleEvents(double timeout) {
+    int display_read_ok = pollForEvents(&_glfw.x11.eventLoopData, timeout);
+    if (display_read_ok) _glfwDispatchX11Events();
+    glfw_ibus_dispatch(&_glfw.x11.xkb.ibus);
+}
 
-        if (timeout)
-        {
-            const long seconds = (long) *timeout;
-            const long microseconds = (long) ((*timeout - seconds) * 1e6);
-            struct timeval tv = { seconds, microseconds };
-            const uint64_t base = _glfwPlatformGetTimerValue();
-
-            const int result = select(count, &fds, NULL, NULL, &tv);
-            const int error = errno;
-
-            *timeout -= (_glfwPlatformGetTimerValue() - base) /
-                (double) _glfwPlatformGetTimerFrequency();
-
-            if (result > 0)
-                return GLFW_TRUE;
-            if ((result == -1 && error == EINTR) || *timeout <= 0.0)
-                return GLFW_FALSE;
+static GLFWbool
+waitForX11Event(double timeout) {
+    // returns true iff there is X11 data waiting to be read, does not run watches and timers
+    double end_time = glfwGetTime() + timeout;
+    while(GLFW_TRUE) {
+        if (timeout >= 0) {
+            const int result = pollWithTimeout(_glfw.x11.eventLoopData.fds, 1, timeout);
+            if (result > 0) return GLFW_TRUE;
+            timeout = end_time - glfwGetTime();
+            if (timeout <= 0) return GLFW_FALSE;
+            if (result < 0 && (errno == EINTR || errno == EAGAIN)) continue;
+            return GLFW_FALSE;
+        } else {
+            const int result = poll(_glfw.x11.eventLoopData.fds, 1, -1);
+            if (result > 0) return GLFW_TRUE;
+            if (result < 0 && (errno == EINTR || errno == EAGAIN)) continue;
+            return GLFW_FALSE;
         }
-        else if (select(count, &fds, NULL, NULL, NULL) != -1 || errno != EINTR)
-            return GLFW_TRUE;
     }
 }
 
@@ -103,14 +90,13 @@ static GLFWbool waitForEvent(double* timeout)
 static GLFWbool waitForVisibilityNotify(_GLFWwindow* window)
 {
     XEvent dummy;
-    double timeout = 0.1;
 
     while (!XCheckTypedWindowEvent(_glfw.x11.display,
                                    window->x11.handle,
                                    VisibilityNotify,
                                    &dummy))
     {
-        if (!waitForEvent(&timeout))
+        if (!waitForX11Event(0.1))
             return GLFW_FALSE;
     }
 
@@ -535,6 +521,61 @@ static void updateCursorImage(_GLFWwindow* window)
     }
 }
 
+// Apply disabled cursor mode to a focused window
+//
+static void disableCursor(_GLFWwindow* window)
+{
+    if (_glfw.x11.xi.available)
+    {
+        XIEventMask em;
+        unsigned char mask[XIMaskLen(XI_RawMotion)] = { 0 };
+
+        em.deviceid = XIAllMasterDevices;
+        em.mask_len = sizeof(mask);
+        em.mask = mask;
+        XISetMask(mask, XI_RawMotion);
+
+        XISelectEvents(_glfw.x11.display, _glfw.x11.root, &em, 1);
+    }
+
+    _glfw.x11.disabledCursorWindow = window;
+    _glfwPlatformGetCursorPos(window,
+                              &_glfw.x11.restoreCursorPosX,
+                              &_glfw.x11.restoreCursorPosY);
+    updateCursorImage(window);
+    centerCursor(window);
+    XGrabPointer(_glfw.x11.display, window->x11.handle, True,
+                 ButtonPressMask | ButtonReleaseMask | PointerMotionMask,
+                 GrabModeAsync, GrabModeAsync,
+                 window->x11.handle,
+                 _glfw.x11.hiddenCursorHandle,
+                 CurrentTime);
+}
+
+// Exit disabled cursor mode for the specified window
+//
+static void enableCursor(_GLFWwindow* window)
+{
+    if (_glfw.x11.xi.available)
+    {
+        XIEventMask em;
+        unsigned char mask[] = { 0 };
+
+        em.deviceid = XIAllMasterDevices;
+        em.mask_len = sizeof(mask);
+        em.mask = mask;
+
+        XISelectEvents(_glfw.x11.display, _glfw.x11.root, &em, 1);
+    }
+
+    _glfw.x11.disabledCursorWindow = NULL;
+    XUngrabPointer(_glfw.x11.display, CurrentTime);
+    _glfwPlatformSetCursorPos(window,
+                              _glfw.x11.restoreCursorPosX,
+                              _glfw.x11.restoreCursorPosY);
+    updateCursorImage(window);
+}
+
 // Create the X11 window (and its colormap)
 //
 static GLFWbool createNativeWindow(_GLFWwindow* window,
@@ -927,7 +968,7 @@ static const char* getSelectionString(Atom selection)
                                        SelectionNotify,
                                        &notification))
         {
-            waitForEvent(NULL);
+            waitForX11Event(-1);
         }
 
         if (notification.xselection.property == None)
@@ -963,7 +1004,7 @@ static const char* getSelectionString(Atom selection)
                                       isSelPropNewValueNotify,
                                       (XPointer) &notification))
                 {
-                    waitForEvent(NULL);
+                    waitForX11Event(-1);
                 }
 
                 XFree(data);
@@ -1087,6 +1128,24 @@ static void releaseMonitor(_GLFWwindow* window)
     }
 }
 
+static void onConfigChange()
+{
+    float xscale, yscale;
+    _glfwGetSystemContentScaleX11(&xscale, &yscale, GLFW_TRUE);
+
+    if (xscale != _glfw.x11.contentScaleX || yscale != _glfw.x11.contentScaleY)
+    {
+        _GLFWwindow* window = _glfw.windowListHead;
+        _glfw.x11.contentScaleX = xscale;
+        _glfw.x11.contentScaleY = yscale;
+        while (window)
+        {
+            _glfwInputWindowContentScale(window, xscale, yscale);
+            window = window->next;
+        }
+    }
+}
+
 // Process the specified X event
 //
 static void processEvent(XEvent *event)
@@ -1104,6 +1163,14 @@ static void processEvent(XEvent *event)
         }
     }
 
+    if (event->type == PropertyNotify &&
+        event->xproperty.window == _glfw.x11.root &&
+        event->xproperty.atom == _glfw.x11.RESOURCE_MANAGER)
+    {
+        onConfigChange();
+        return;
+    }
+
     if (event->type == GenericEvent)
     {
         if (_glfw.x11.xi.available)
@@ -1242,13 +1309,13 @@ static void processEvent(XEvent *event)
 
             // Modern X provides scroll events as mouse button presses
             else if (event->xbutton.button == Button4)
-                _glfwInputScroll(window, 0.0, 1.0);
+                _glfwInputScroll(window, 0.0, 1.0, 0);
             else if (event->xbutton.button == Button5)
-                _glfwInputScroll(window, 0.0, -1.0);
+                _glfwInputScroll(window, 0.0, -1.0, 0);
             else if (event->xbutton.button == Button6)
-                _glfwInputScroll(window, 1.0, 0.0);
+                _glfwInputScroll(window, 1.0, 0.0, 0);
             else if (event->xbutton.button == Button7)
-                _glfwInputScroll(window, -1.0, 0.0);
+                _glfwInputScroll(window, -1.0, 0.0, 0);
 
             else
             {
@@ -1306,7 +1373,7 @@ static void processEvent(XEvent *event)
             // HACK: This is a workaround for WMs (KWM, Fluxbox) that otherwise
             //       ignore the defined cursor for hidden cursor mode
             if (window->cursorMode == GLFW_CURSOR_HIDDEN)
-                _glfwPlatformSetCursorMode(window, GLFW_CURSOR_HIDDEN);
+                updateCursorImage(window);
 
             _glfwInputCursorEnter(window, GLFW_TRUE);
             return;
@@ -1599,7 +1666,7 @@ static void processEvent(XEvent *event)
         case FocusIn:
         {
             if (window->cursorMode == GLFW_CURSOR_DISABLED)
-                _glfwPlatformSetCursorMode(window, GLFW_CURSOR_DISABLED);
+                disableCursor(window);
 
             if (event->xfocus.mode == NotifyGrab ||
                 event->xfocus.mode == NotifyUngrab)
@@ -1616,7 +1683,7 @@ static void processEvent(XEvent *event)
         case FocusOut:
         {
             if (window->cursorMode == GLFW_CURSOR_DISABLED)
-                _glfwPlatformSetCursorMode(window, GLFW_CURSOR_NORMAL);
+                enableCursor(window);
 
             if (event->xfocus.mode == NotifyGrab ||
                 event->xfocus.mode == NotifyUngrab)
@@ -1769,7 +1836,7 @@ void _glfwPushSelectionToManagerX11(void)
             }
         }
 
-        waitForEvent(NULL);
+        waitForX11Event(-1);
     }
 }
 
@@ -2059,7 +2126,6 @@ void _glfwPlatformGetWindowFrameSize(_GLFWwindow* window,
         _glfw.x11.NET_REQUEST_FRAME_EXTENTS)
     {
         XEvent event;
-        double timeout = 0.5;
 
         // Ensure _NET_FRAME_EXTENTS is set, allowing glfwGetWindowFrameSize to
         // function before the window is mapped
@@ -2076,7 +2142,7 @@ void _glfwPlatformGetWindowFrameSize(_GLFWwindow* window,
                               isFrameExtentsEvent,
                               (XPointer) window))
         {
-            if (!waitForEvent(&timeout))
+            if (!waitForX11Event(0.5))
             {
                 _glfwInputError(GLFW_PLATFORM_ERROR,
                                 "X11: The window manager has a broken _NET_REQUEST_FRAME_EXTENTS implementation; please report this issue");
@@ -2477,19 +2543,22 @@ void _glfwPlatformSetWindowOpacity(_GLFWwindow* window, float opacity)
                     PropModeReplace, (unsigned char*) &value, 1);
 }
 
-void _glfwPlatformPollEvents(void)
-{
+GLFWbool
+_glfwDispatchX11Events(void) {
     _GLFWwindow* window;
+    GLFWbool dispatched = GLFW_FALSE;
 
 #if defined(__linux__)
     _glfwDetectJoystickConnectionLinux();
 #endif
-    int count = XPending(_glfw.x11.display);
-    while (count--)
+    XPending(_glfw.x11.display);
+
+    while (XQLength(_glfw.x11.display))
     {
         XEvent event;
         XNextEvent(_glfw.x11.display, &event);
         processEvent(&event);
+        dispatched = GLFW_TRUE;
     }
 
     window = _glfw.x11.disabledCursorWindow;
@@ -2508,25 +2577,25 @@ void _glfwPlatformPollEvents(void)
     }
 
     XFlush(_glfw.x11.display);
+    return dispatched;
+}
+
+void _glfwPlatformPollEvents(void)
+{
+    _glfwDispatchX11Events();
+    handleEvents(0);
 }
 
 void _glfwPlatformWaitEvents(void)
 {
-    while (!XPending(_glfw.x11.display))
-        waitForEvent(NULL);
-
-    _glfwPlatformPollEvents();
+    double timeout = _glfwDispatchX11Events() ? 0 : -1;
+    handleEvents(timeout);
 }
 
 void _glfwPlatformWaitEventsTimeout(double timeout)
 {
-    while (!XPending(_glfw.x11.display))
-    {
-        if (!waitForEvent(&timeout))
-            break;
-    }
-
-    _glfwPlatformPollEvents();
+    if (_glfwDispatchX11Events()) timeout = 0;
+    handleEvents(timeout);
 }
 
 void _glfwPlatformPostEmptyEvent(void)
@@ -2541,6 +2610,7 @@ void _glfwPlatformPostEmptyEvent(void)
 
     XSendEvent(_glfw.x11.display, _glfw.x11.helperWindowHandle, False, 0, &event);
     XFlush(_glfw.x11.display);
+    while (write(_glfw.x11.eventLoopData.wakeupFds[1], "w", 1) < 0 && errno == EINTR);
 }
 
 void _glfwPlatformGetCursorPos(_GLFWwindow* window, double* xpos, double* ypos)
@@ -2575,53 +2645,14 @@ void _glfwPlatformSetCursorMode(_GLFWwindow* window, int mode)
 {
     if (mode == GLFW_CURSOR_DISABLED)
     {
-        if (_glfw.x11.xi.available)
-        {
-            XIEventMask em;
-            unsigned char mask[XIMaskLen(XI_RawMotion)] = { 0 };
-
-            em.deviceid = XIAllMasterDevices;
-            em.mask_len = sizeof(mask);
-            em.mask = mask;
-            XISetMask(mask, XI_RawMotion);
-
-            XISelectEvents(_glfw.x11.display, _glfw.x11.root, &em, 1);
-        }
-
-        _glfw.x11.disabledCursorWindow = window;
-        _glfwPlatformGetCursorPos(window,
-                                  &_glfw.x11.restoreCursorPosX,
-                                  &_glfw.x11.restoreCursorPosY);
-        centerCursor(window);
-        XGrabPointer(_glfw.x11.display, window->x11.handle, True,
-                     ButtonPressMask | ButtonReleaseMask | PointerMotionMask,
-                     GrabModeAsync, GrabModeAsync,
-                     window->x11.handle,
-                     _glfw.x11.hiddenCursorHandle,
-                     CurrentTime);
+        if (_glfwPlatformWindowFocused(window))
+            disableCursor(window);
     }
     else if (_glfw.x11.disabledCursorWindow == window)
-    {
-        if (_glfw.x11.xi.available)
-        {
-            XIEventMask em;
-            unsigned char mask[] = { 0 };
+        enableCursor(window);
+    else
+        updateCursorImage(window);
 
-            em.deviceid = XIAllMasterDevices;
-            em.mask_len = sizeof(mask);
-            em.mask = mask;
-
-            XISelectEvents(_glfw.x11.display, _glfw.x11.root, &em, 1);
-        }
-
-        _glfw.x11.disabledCursorWindow = NULL;
-        XUngrabPointer(_glfw.x11.display, CurrentTime);
-        _glfwPlatformSetCursorPos(window,
-                                  _glfw.x11.restoreCursorPosX,
-                                  _glfw.x11.restoreCursorPosY);
-    }
-
-    updateCursorImage(window);
     XFlush(_glfw.x11.display);
 }
 
@@ -2638,7 +2669,7 @@ int _glfwPlatformGetKeyScancode(int key)
 
 int _glfwPlatformCreateCursor(_GLFWcursor* cursor,
                               const GLFWimage* image,
-                              int xhot, int yhot)
+                              int xhot, int yhot, int count)
 {
     cursor->x11.handle = _glfwCreateCursorX11(image, xhot, yhot);
     if (!cursor->x11.handle)
@@ -2704,7 +2735,7 @@ void _glfwPlatformGetRequiredInstanceExtensions(char** extensions)
     if (!_glfw.vk.KHR_surface)
         return;
 
-    if (!_glfw.vk.KHR_xcb_surface || !_glfw.x11.x11xcb.handle)
+    if (!_glfw.vk.KHR_xcb_surface)
     {
         if (!_glfw.vk.KHR_xlib_surface)
             return;
@@ -2714,7 +2745,7 @@ void _glfwPlatformGetRequiredInstanceExtensions(char** extensions)
 
     // NOTE: VK_KHR_xcb_surface is preferred due to some early ICDs exposing but
     //       not correctly implementing VK_KHR_xlib_surface
-    if (_glfw.vk.KHR_xcb_surface && _glfw.x11.x11xcb.handle)
+    if (_glfw.vk.KHR_xcb_surface)
         extensions[1] = "VK_KHR_xcb_surface";
     else
         extensions[1] = "VK_KHR_xlib_surface";
@@ -2727,7 +2758,7 @@ int _glfwPlatformGetPhysicalDevicePresentationSupport(VkInstance instance,
     VisualID visualID = XVisualIDFromVisual(DefaultVisual(_glfw.x11.display,
                                                           _glfw.x11.screen));
 
-    if (_glfw.vk.KHR_xcb_surface && _glfw.x11.x11xcb.handle)
+    if (_glfw.vk.KHR_xcb_surface)
     {
         PFN_vkGetPhysicalDeviceXcbPresentationSupportKHR
             vkGetPhysicalDeviceXcbPresentationSupportKHR =
@@ -2778,7 +2809,7 @@ VkResult _glfwPlatformCreateWindowSurface(VkInstance instance,
                                           const VkAllocationCallbacks* allocator,
                                           VkSurfaceKHR* surface)
 {
-    if (_glfw.vk.KHR_xcb_surface && _glfw.x11.x11xcb.handle)
+    if (_glfw.vk.KHR_xcb_surface)
     {
         VkResult err;
         VkXcbSurfaceCreateInfoKHR sci;
@@ -2848,6 +2879,10 @@ VkResult _glfwPlatformCreateWindowSurface(VkInstance instance,
     }
 }
 
+void
+_glfwPlatformUpdateIMEState(_GLFWwindow *w, int which, int a, int b, int c, int d) {
+    glfw_xkb_update_ime_state(w, &_glfw.x11.xkb, which, a, b, c, d);
+}
 
 //////////////////////////////////////////////////////////////////////////
 //////                        GLFW native API                       //////
@@ -2891,3 +2926,7 @@ GLFWAPI const char* glfwGetX11SelectionString(void)
     _GLFW_REQUIRE_INIT_OR_RETURN(NULL);
     return getSelectionString(_glfw.x11.PRIMARY);
 }
+
+GLFWAPI int glfwGetXKBScancode(const char* keyName, GLFWbool caseSensitive) {
+    return glfw_xkb_keysym_from_name(keyName, caseSensitive);
+}

glfw/xkb_glfw.c

@@ -30,8 +30,8 @@
 #include "internal.h"
 #include "xkb_glfw.h"
 
-static GLFWbool debug_keyboard = GLFW_FALSE;
-#define debug(...) if (debug_keyboard) printf(__VA_ARGS__);
+#define debug(...) if (_glfw.hints.init.debugKeyboard) printf(__VA_ARGS__);
+
 
 #define map_key(key) { \
     switch(key) { \
@@ -171,129 +171,150 @@ glfw_xkb_update_x11_keyboard_id(_GLFWXKBData *xkb) {
 
 #endif
 
+static void
+release_keyboard_data(_GLFWXKBData *xkb) {
+#define US(group, state, unref) if (xkb->group.state) {  unref(xkb->group.state); xkb->group.state = NULL; }
+#define UK(keymap) if(xkb->keymap) { xkb_keymap_unref(xkb->keymap); xkb->keymap = NULL; }
+    US(states, composeState, xkb_compose_state_unref);
+    UK(keymap);
+    UK(default_keymap);
+    US(states, state, xkb_state_unref);
+    US(states, clean_state, xkb_state_unref);
+    US(states, default_state, xkb_state_unref);
+#undef US
+#undef UK
+
+}
+
 void
 glfw_xkb_release(_GLFWXKBData *xkb) {
-    if (xkb->composeState) {
-        xkb_compose_state_unref(xkb->composeState);
-        xkb->composeState = NULL;
-    }
-    if (xkb->keymap) {
-        xkb_keymap_unref(xkb->keymap);
-        xkb->keymap = NULL;
-    }
-    if (xkb->state) {
-        xkb_state_unref(xkb->state);
-        xkb->state = NULL;
-    }
-    if (xkb->clean_state) {
-        xkb_state_unref(xkb->clean_state);
-        xkb->clean_state = NULL;
-    }
+    release_keyboard_data(xkb);
     if (xkb->context) {
         xkb_context_unref(xkb->context);
         xkb->context = NULL;
     }
+    glfw_ibus_terminate(&xkb->ibus);
 }
 
 GLFWbool
 glfw_xkb_create_context(_GLFWXKBData *xkb) {
     xkb->context = xkb_context_new(0);
-    debug_keyboard = getenv("GLFW_DEBUG_KEYBOARD") != NULL;
     if (!xkb->context)
     {
         _glfwInputError(GLFW_PLATFORM_ERROR,
                         "Failed to initialize XKB context");
         return GLFW_FALSE;
     }
+    glfw_connect_to_ibus(&xkb->ibus);
     return GLFW_TRUE;
 }
 
+static const char*
+load_keymaps(_GLFWXKBData *xkb, const char *map_str) {
+    (void)(map_str);  // not needed on X11
+    xkb_glfw_load_keymap(xkb->keymap, map_str);
+    if (!xkb->keymap) return "Failed to compile XKB keymap";
+    // The system default keymap, can be overridden by the XKB_DEFAULT_RULES
+    // env var, see
+    // https://xkbcommon.org/doc/current/structxkb__rule__names.html
+    static struct xkb_rule_names default_rule_names = {0};
+    xkb->default_keymap = xkb_keymap_new_from_names(xkb->context, &default_rule_names, XKB_KEYMAP_COMPILE_NO_FLAGS);
+    if (!xkb->default_keymap) return "Failed to create default XKB keymap";
+    return NULL;
+}
+
+static const char*
+load_states(_GLFWXKBData *xkb) {
+    xkb_glfw_load_state(xkb->keymap, xkb->states.state);
+    xkb->states.clean_state = xkb_state_new(xkb->keymap);
+    xkb->states.default_state = xkb_state_new(xkb->default_keymap);
+    if (!xkb->states.state || !xkb->states.clean_state || !xkb->states.default_state) return "Failed to create XKB state";
+    return NULL;
+}
+
+static void
+load_compose_tables(_GLFWXKBData *xkb) {
+    /* Look up the preferred locale, falling back to "C" as default. */
+    struct xkb_compose_table* compose_table = NULL;
+    const char *locale = getenv("LC_ALL");
+    if (!locale) locale = getenv("LC_CTYPE");
+    if (!locale) locale = getenv("LANG");
+    if (!locale) locale = "C";
+    compose_table = xkb_compose_table_new_from_locale(xkb->context, locale, XKB_COMPOSE_COMPILE_NO_FLAGS);
+    if (!compose_table) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to create XKB compose table for locale %s", locale);
+        return;
+    }
+    xkb->states.composeState = xkb_compose_state_new(compose_table, XKB_COMPOSE_STATE_NO_FLAGS);
+    if (!xkb->states.composeState) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to create XKB compose state");
+    }
+}
+
 GLFWbool
 glfw_xkb_compile_keymap(_GLFWXKBData *xkb, const char *map_str) {
-    const char* locale = NULL;
-    struct xkb_state* state = NULL, *clean_state = NULL;
-    struct xkb_keymap* keymap = NULL;
-    struct xkb_compose_table* compose_table = NULL;
-    struct xkb_compose_state* compose_state = NULL;
-    (void)(map_str);  // not needed on X11
-    GLFWbool ok = GLFW_FALSE;
-
-    xkb_glfw_load_keymap(keymap, map_str);
-    if (!keymap) _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to compile XKB keymap");
-    else {
-        xkb_glfw_load_state(keymap, state);
-        clean_state = xkb_state_new(keymap);
-        if (!state || !clean_state) {
-            _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to create XKB state");
-            xkb_keymap_unref(keymap); keymap = NULL;
-        } else {
-            ok = GLFW_TRUE;
-            /* Look up the preferred locale, falling back to "C" as default. */
-            locale = getenv("LC_ALL");
-            if (!locale) locale = getenv("LC_CTYPE");
-            if (!locale) locale = getenv("LANG");
-            if (!locale) locale = "C";
-            compose_table = xkb_compose_table_new_from_locale(xkb->context, locale, XKB_COMPOSE_COMPILE_NO_FLAGS);
-            if (!compose_table) {
-                _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to create XKB compose table");
-            } else {
-                compose_state = xkb_compose_state_new(compose_table, XKB_COMPOSE_STATE_NO_FLAGS);
-                xkb_compose_table_unref(compose_table); compose_table = NULL;
-                if (!compose_state) {
-                    _glfwInputError(GLFW_PLATFORM_ERROR, "Failed to create XKB compose state");
-                }
-            }
-        }
+    const char *err;
+    release_keyboard_data(xkb);
+    err = load_keymaps(xkb, map_str);
+    if (err) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "%s", err);
+        release_keyboard_data(xkb);
+        return GLFW_FALSE;
     }
-    if (keymap && state && clean_state) {
-        if (xkb->composeState) xkb_compose_state_unref(xkb->composeState);
-        xkb->composeState = compose_state;
-        if (xkb->keymap) xkb_keymap_unref(xkb->keymap);
-        xkb->keymap = keymap;
-        if (xkb->state) xkb_state_unref(xkb->state);
-        xkb->state = state;
-        if (xkb->clean_state) xkb_state_unref(xkb->clean_state);
-        xkb->clean_state = clean_state;
+    err = load_states(xkb);
+    if (err) {
+        _glfwInputError(GLFW_PLATFORM_ERROR, "%s", err);
+        release_keyboard_data(xkb);
+        return GLFW_FALSE;
     }
-    if (xkb->keymap) {
+    load_compose_tables(xkb);
 #define S(a, n) xkb->a##Idx = xkb_keymap_mod_get_index(xkb->keymap, n); xkb->a##Mask = 1 << xkb->a##Idx;
-        S(control, XKB_MOD_NAME_CTRL);
-        S(alt, XKB_MOD_NAME_ALT);
-        S(shift, XKB_MOD_NAME_SHIFT);
-        S(super, XKB_MOD_NAME_LOGO);
-        S(capsLock, XKB_MOD_NAME_CAPS);
-        S(numLock, XKB_MOD_NAME_NUM);
-        size_t capacity = sizeof(xkb->unknownModifiers)/sizeof(xkb->unknownModifiers[0]), j = 0;
-        for (xkb_mod_index_t i = 0; i < capacity; i++) xkb->unknownModifiers[i] = XKB_MOD_INVALID;
-        for (xkb_mod_index_t i = 0; i < xkb_keymap_num_mods(xkb->keymap) && j < capacity - 1; i++) {
-            if (i != xkb->controlIdx && i != xkb->altIdx && i != xkb->shiftIdx && i != xkb->superIdx && i != xkb->capsLockIdx && i != xkb->numLockIdx) xkb->unknownModifiers[j++] = i;
-        }
+    S(control, XKB_MOD_NAME_CTRL);
+    S(alt, XKB_MOD_NAME_ALT);
+    S(shift, XKB_MOD_NAME_SHIFT);
+    S(super, XKB_MOD_NAME_LOGO);
+    S(capsLock, XKB_MOD_NAME_CAPS);
+    S(numLock, XKB_MOD_NAME_NUM);
 #undef S
+    size_t capacity = sizeof(xkb->unknownModifiers)/sizeof(xkb->unknownModifiers[0]), j = 0;
+    for (xkb_mod_index_t i = 0; i < capacity; i++) xkb->unknownModifiers[i] = XKB_MOD_INVALID;
+    for (xkb_mod_index_t i = 0; i < xkb_keymap_num_mods(xkb->keymap) && j < capacity - 1; i++) {
+        if (i != xkb->controlIdx && i != xkb->altIdx && i != xkb->shiftIdx && i != xkb->superIdx && i != xkb->capsLockIdx && i != xkb->numLockIdx) xkb->unknownModifiers[j++] = i;
     }
-    return ok;
+    xkb->states.modifiers = 0;
+    xkb->states.activeUnknownModifiers = 0;
+    return GLFW_TRUE;
 }
 
 static inline xkb_mod_mask_t
-active_unknown_modifiers(_GLFWXKBData *xkb) {
+active_unknown_modifiers(_GLFWXKBData *xkb, struct xkb_state *state) {
     size_t i = 0;
     xkb_mod_mask_t ans = 0;
     while (xkb->unknownModifiers[i] != XKB_MOD_INVALID) {
-        if (xkb_state_mod_index_is_active(xkb->state, xkb->unknownModifiers[i], XKB_STATE_MODS_EFFECTIVE)) ans |= (1 << xkb->unknownModifiers[i]);
+        if (xkb_state_mod_index_is_active(state, xkb->unknownModifiers[i], XKB_STATE_MODS_EFFECTIVE)) ans |= (1 << xkb->unknownModifiers[i]);
         i++;
     }
     return ans;
 }
 
+static void
+update_modifiers(_GLFWXKBData *xkb, XKBStateGroup *group) {
+#define S(attr, name) if (xkb_state_mod_index_is_active(group->state, xkb->attr##Idx, XKB_STATE_MODS_EFFECTIVE)) group->modifiers |= GLFW_MOD_##name
+    S(control, CONTROL); S(alt, ALT); S(shift, SHIFT); S(super, SUPER); S(capsLock, CAPS_LOCK); S(numLock, NUM_LOCK);
+#undef S
+    xkb->states.activeUnknownModifiers = active_unknown_modifiers(xkb, xkb->states.state);
+
+}
 
 void
 glfw_xkb_update_modifiers(_GLFWXKBData *xkb, xkb_mod_mask_t depressed, xkb_mod_mask_t latched, xkb_mod_mask_t locked, xkb_layout_index_t base_group, xkb_layout_index_t latched_group, xkb_layout_index_t locked_group) {
     if (!xkb->keymap) return;
-    xkb->modifiers = 0;
-    xkb_state_update_mask(xkb->state, depressed, latched, locked, base_group, latched_group, locked_group);
-#define S(attr, name) if (xkb_state_mod_index_is_active(xkb->state, xkb->attr##Idx, XKB_STATE_MODS_EFFECTIVE)) xkb->modifiers |= GLFW_MOD_##name
-    S(control, CONTROL); S(alt, ALT); S(shift, SHIFT); S(super, SUPER); S(capsLock, CAPS_LOCK); S(numLock, NUM_LOCK);
-#undef S
-    xkb->activeUnknownModifiers = active_unknown_modifiers(xkb);
+    xkb->states.modifiers = 0;
+    xkb_state_update_mask(xkb->states.state, depressed, latched, locked, base_group, latched_group, locked_group);
+    // We have to update the groups in clean_state, as they change for
+    // different keyboard layouts, see https://github.com/kovidgoyal/kitty/issues/488
+    xkb_state_update_mask(xkb->states.clean_state, 0, 0, 0, base_group, latched_group, locked_group);
+    update_modifiers(xkb, &xkb->states);
 }
 
 GLFWbool
@@ -305,16 +326,16 @@ glfw_xkb_should_repeat(_GLFWXKBData *xkb, xkb_keycode_t scancode) {
 }
 
 
-static char text[256];
+static KeyEvent key_event = {};
 
 static inline xkb_keysym_t
-compose_symbol(_GLFWXKBData *xkb, xkb_keysym_t sym) {
-    if (sym == XKB_KEY_NoSymbol || !xkb->composeState) return sym;
-    if (xkb_compose_state_feed(xkb->composeState, sym) != XKB_COMPOSE_FEED_ACCEPTED) return sym;
-    switch (xkb_compose_state_get_status(xkb->composeState)) {
+compose_symbol(struct xkb_compose_state *composeState, xkb_keysym_t sym) {
+    if (sym == XKB_KEY_NoSymbol || !composeState) return sym;
+    if (xkb_compose_state_feed(composeState, sym) != XKB_COMPOSE_FEED_ACCEPTED) return sym;
+    switch (xkb_compose_state_get_status(composeState)) {
         case XKB_COMPOSE_COMPOSED:
-            xkb_compose_state_get_utf8(xkb->composeState, text, sizeof(text));
-            return xkb_compose_state_get_one_sym(xkb->composeState);
+            xkb_compose_state_get_utf8(composeState, key_event.text, sizeof(key_event.text));
+            return xkb_compose_state_get_one_sym(composeState);
         case XKB_COMPOSE_COMPOSING:
         case XKB_COMPOSE_CANCELLED:
             return XKB_KEY_NoSymbol;
@@ -333,6 +354,10 @@ glfw_xkb_keysym_name(xkb_keysym_t sym) {
     return name;
 }
 
+int
+glfw_xkb_keysym_from_name(const char *name, GLFWbool case_sensitive) {
+    return (int)xkb_keysym_from_name(name, case_sensitive ? XKB_KEYSYM_NO_FLAGS : XKB_KEYSYM_CASE_INSENSITIVE);
+}
 
 static inline const char*
 format_mods(unsigned int mods) {
@@ -372,29 +397,75 @@ format_xkb_mods(_GLFWXKBData *xkb, const char* name, xkb_mod_mask_t mods) {
     return buf;
 }
 
+void
+glfw_xkb_update_ime_state(_GLFWwindow *w, _GLFWXKBData *xkb, int which, int a, int b, int c, int d) {
+    int x = 0, y = 0;
+    switch(which) {
+        case 1:
+            glfw_ibus_set_focused(&xkb->ibus, a ? GLFW_TRUE : GLFW_FALSE);
+            break;
+        case 2:
+            _glfwPlatformGetWindowPos(w, &x, &y);
+            x += a; y += b;
+            glfw_ibus_set_cursor_geometry(&xkb->ibus, x, y, c, d);
+            break;
+    }
+}
+
+void
+glfw_xkb_key_from_ime(KeyEvent *ev, GLFWbool handled_by_ime, GLFWbool failed) {
+    _GLFWwindow *window = _glfwWindowForId(ev->window_id);
+    if (failed && window && window->callbacks.keyboard) {
+        // notify application to remove any existing pre-edit text
+        window->callbacks.keyboard((GLFWwindow*) window, GLFW_KEY_UNKNOWN, 0, GLFW_PRESS, 0, "", 1);
+    }
+    static xkb_keycode_t last_handled_press_keycode = 0;
+    // We filter out release events that correspond to the last press event
+    // handled by the IME system. This wont fix the case of multiple key
+    // presses before a release, but is better than nothing. For that case
+    // you'd need to implement a ring buffer to store pending key presses.
+    xkb_keycode_t prev_handled_press = last_handled_press_keycode;
+    last_handled_press_keycode = 0;
+    GLFWbool is_release = ev->action == GLFW_RELEASE;
+    debug("From IBUS: scancode: 0x%x name: %s is_release: %d\n", ev->keycode, glfw_xkb_keysym_name(ev->keysym), is_release);
+    if (window && !handled_by_ime && !(is_release && ev->keycode == prev_handled_press)) {
+        debug("↳ to application: glfw_keycode: 0x%x (%s) keysym: 0x%x (%s) action: %s %s text: %s\n",
+            ev->glfw_keycode, _glfwGetKeyName(ev->glfw_keycode), ev->keysym, glfw_xkb_keysym_name(ev->keysym),
+            (ev->action == GLFW_RELEASE ? "RELEASE" : (ev->action == GLFW_PRESS ? "PRESS" : "REPEAT")),
+            format_mods(ev->glfw_modifiers), key_event.text
+        );
+        _glfwInputKeyboard(window, ev->glfw_keycode, ev->keysym, ev->action, ev->glfw_modifiers, key_event.text, 0);
+    } else debug("↳ discarded\n");
+    if (!is_release && handled_by_ime) last_handled_press_keycode = ev->keycode;
+}
+
 void
 glfw_xkb_handle_key_event(_GLFWwindow *window, _GLFWXKBData *xkb, xkb_keycode_t scancode, int action) {
-    const xkb_keysym_t *syms, *clean_syms;
+    const xkb_keysym_t *syms, *clean_syms, *default_syms;
     xkb_keysym_t glfw_sym;
     xkb_keycode_t code_for_sym = scancode;
+    key_event.ibus_keycode = scancode;
 #ifdef _GLFW_WAYLAND
     code_for_sym += 8;
+#else
+    key_event.ibus_keycode -= 8;
 #endif
-    debug("scancode: 0x%x release: %d ", scancode, action == GLFW_RELEASE);
-    int num_syms = xkb_state_key_get_syms(xkb->state, code_for_sym, &syms);
-    int num_clean_syms = xkb_state_key_get_syms(xkb->clean_state, code_for_sym, &clean_syms);
-    text[0] = 0;
+    debug("%s scancode: 0x%x ", action == GLFW_RELEASE ? "Release" : "Press", scancode);
+    XKBStateGroup *sg = &xkb->states;
+    int num_syms = xkb_state_key_get_syms(sg->state, code_for_sym, &syms);
+    int num_clean_syms = xkb_state_key_get_syms(sg->clean_state, code_for_sym, &clean_syms);
+    key_event.text[0] = 0;
     // According to the documentation of xkb_compose_state_feed it does not
     // support multi-sym events, so we ignore them
     if (num_syms != 1 || num_clean_syms != 1) {
-        debug("scancode: 0x%x num_syms: %d num_clean_syms: %d ignoring event\n", scancode, num_syms, num_clean_syms);
+        debug("num_syms: %d num_clean_syms: %d ignoring event\n", num_syms, num_clean_syms);
         return;
     }
     glfw_sym = clean_syms[0];
     debug("clean_sym: %s ", glfw_xkb_keysym_name(clean_syms[0]));
     if (action == GLFW_PRESS || action == GLFW_REPEAT) {
         const char *text_type = "composed_text";
-        glfw_sym = compose_symbol(xkb, syms[0]);
+        glfw_sym = compose_symbol(sg->composeState, syms[0]);
         if (glfw_sym == XKB_KEY_NoSymbol) {
             debug("compose not complete, ignoring.\n");
             return;
@@ -405,18 +476,40 @@ glfw_xkb_handle_key_event(_GLFWwindow *window, _GLFWXKBData *xkb, xkb_keycode_t
             // are active (for example if ISO_Shift_Level_* mods are active
             // they are not reported by GLFW so the key should be the shifted
             // key). See https://github.com/kovidgoyal/kitty/issues/171#issuecomment-377557053
-            xkb_mod_mask_t consumed_unknown_mods = xkb_state_key_get_consumed_mods(xkb->state, code_for_sym) & xkb->activeUnknownModifiers;
-            if (xkb->activeUnknownModifiers) debug("%s", format_xkb_mods(xkb, "active_unknown_mods", xkb->activeUnknownModifiers));
+            xkb_mod_mask_t consumed_unknown_mods = xkb_state_key_get_consumed_mods(sg->state, code_for_sym) & sg->activeUnknownModifiers;
+            if (sg->activeUnknownModifiers) debug("%s", format_xkb_mods(xkb, "active_unknown_mods", sg->activeUnknownModifiers));
             if (consumed_unknown_mods) { debug("%s", format_xkb_mods(xkb, "consumed_unknown_mods", consumed_unknown_mods)); }
             else glfw_sym = clean_syms[0];
             // xkb returns text even if alt and/or super are pressed
-            if ( ((GLFW_MOD_CONTROL | GLFW_MOD_ALT | GLFW_MOD_SUPER) & xkb->modifiers) == 0) xkb_state_key_get_utf8(xkb->state, code_for_sym, text, sizeof(text));
+            if ( ((GLFW_MOD_CONTROL | GLFW_MOD_ALT | GLFW_MOD_SUPER) & sg->modifiers) == 0) xkb_state_key_get_utf8(sg->state, code_for_sym, key_event.text, sizeof(key_event.text));
             text_type = "text";
         }
-        if ((1 <= text[0] && text[0] <= 31) || text[0] == 127) text[0] = 0;  // dont send text for ascii control codes
-        if (text[0]) { debug("%s: %s ", text_type, text); }
+        if ((1 <= key_event.text[0] && key_event.text[0] <= 31) || key_event.text[0] == 127) key_event.text[0] = 0;  // don't send text for ascii control codes
+        if (key_event.text[0]) { debug("%s: %s ", text_type, key_event.text); }
     }
     int glfw_keycode = glfw_key_for_sym(glfw_sym);
-    debug("%sglfw_key: %s\n", format_mods(xkb->modifiers), _glfwGetKeyName(glfw_keycode));
-    _glfwInputKeyboard(window, glfw_keycode, glfw_sym, action, xkb->modifiers, text, 0);
+    GLFWbool is_fallback = GLFW_FALSE;
+    if (glfw_keycode == GLFW_KEY_UNKNOWN && !key_event.text[0]) {
+        int num_default_syms = xkb_state_key_get_syms(sg->default_state, code_for_sym, &default_syms);
+        if (num_default_syms > 0) {
+            glfw_sym = default_syms[0];
+            glfw_keycode = glfw_key_for_sym(glfw_sym);
+            is_fallback = GLFW_TRUE;
+        }
+    }
+    debug(
+        "%s%s: %d (%s) xkb_key: %d (%s)\n",
+        format_mods(sg->modifiers),
+        is_fallback ? "glfw_fallback_key" : "glfw_key", glfw_keycode, _glfwGetKeyName(glfw_keycode),
+        glfw_sym, glfw_xkb_keysym_name(glfw_sym)
+    );
+    key_event.action = action; key_event.glfw_modifiers = sg->modifiers;
+    key_event.keycode = scancode; key_event.keysym = glfw_sym;
+    key_event.window_id = window->id; key_event.glfw_keycode = glfw_keycode;
+    key_event.ibus_sym = syms[0];
+    if (ibus_process_key(&key_event, &xkb->ibus)) {
+        debug("↳ to IBUS: keycode: 0x%x keysym: 0x%x (%s) %s\n", key_event.ibus_keycode, key_event.ibus_sym, glfw_xkb_keysym_name(key_event.ibus_sym), format_mods(key_event.glfw_modifiers));
+    } else {
+        _glfwInputKeyboard(window, glfw_keycode, glfw_sym, action, sg->modifiers, key_event.text, 0);
+    }
 }

glfw/xkb_glfw.h

@@ -32,12 +32,23 @@
 #include <xkbcommon/xkbcommon-x11.h>
 #endif
 
+#include "ibus_glfw.h"
+
+typedef struct {
+    struct xkb_state*       state;
+    struct xkb_state*       clean_state;
+    struct xkb_state*       default_state;
+    struct xkb_compose_state* composeState;
+    xkb_mod_mask_t          activeUnknownModifiers;
+    unsigned int            modifiers;
+} XKBStateGroup;
+
+
 typedef struct {
     struct xkb_context*     context;
     struct xkb_keymap*      keymap;
-    struct xkb_state*       state;
-    struct xkb_state*       clean_state;
-    struct xkb_compose_state* composeState;
+    struct xkb_keymap*      default_keymap;
+    XKBStateGroup           states;
 
     xkb_mod_index_t         controlIdx;
     xkb_mod_index_t         altIdx;
@@ -51,9 +62,8 @@ typedef struct {
     xkb_mod_mask_t          superMask;
     xkb_mod_mask_t          capsLockMask;
     xkb_mod_mask_t          numLockMask;
-    xkb_mod_mask_t          activeUnknownModifiers;
-    unsigned int            modifiers;
     xkb_mod_index_t         unknownModifiers[256];
+    _GLFWIBUSData           ibus;
 
 #ifdef _GLFW_X11
     int32_t                 keyboard_device_id;
@@ -81,3 +91,6 @@ GLFWbool glfw_xkb_should_repeat(_GLFWXKBData *xkb, xkb_keycode_t scancode);
 const char* glfw_xkb_keysym_name(xkb_keysym_t sym);
 xkb_keysym_t glfw_xkb_sym_for_key(int key);
 void glfw_xkb_handle_key_event(_GLFWwindow *window, _GLFWXKBData *xkb, xkb_keycode_t scancode, int action);
+int glfw_xkb_keysym_from_name(const char *name, GLFWbool case_sensitive);
+void glfw_xkb_update_ime_state(_GLFWwindow *w, _GLFWXKBData *xkb, int which, int a, int b, int c, int d);
+void glfw_xkb_key_from_ime(KeyEvent *ev, GLFWbool handled_by_ime, GLFWbool failed);

graphics-protocol.asciidoc

@@ -1,385 +1 @@
-= The terminal graphics protocol
-:toc:
-:toc-placement!:
-
-The goal of this specification is to create a flexible and performant protocol
-that allows the program running in the terminal, hereafter called the _client_,
-to render arbitrary pixel (raster) graphics to the screen of the terminal
-emulator. The major design goals are
-
- * Should not require terminal emulators to understand image formats.
- * Should allow specifying graphics to be drawn at individual pixel positions.
- * The graphics should integrate with the text, in particular it should be possible to draw graphics
-   below as well as above the text, with alpha blending. The graphics should also scroll with the text, automatically.
- * Should use optimizations when the client is running on the same computer as the terminal emulator.
-
-For some discussion regarding the design choices, see link:../../issues/33[#33].
-
-To see a quick demo, inside a kitty terminal run:
-
-```
-kitty icat path/to/some/image.png
-```
-
-You can also see a screenshot with more sophisticated features such as alpha-blending and text over graphics
-link:https://github.com/kovidgoyal/kitty/issues/33#issuecomment-334436100[here].
-
-Some third-party programs that use the kitty graphics protocol:
-
- * link:https://github.com/dsanson/termpdf[termdpf] - a terminal PDF/DJVU/CBR viewer
- * link:https://github.com/ranger/ranger[ranger] - a terminal file manager, with image previews, see this link:https://github.com/ranger/ranger/pull/1077[PR]
-
-toc::[]
-
-== Getting the window size
-
-In order to know what size of images to display and how to position them, the client must be able to get the
-window size in pixels and the number of cells per row and column. This can be done by using the `TIOCGWINSZ` ioctl.
-Some code to demonstrate its use
-
-In C:
-```C
-struct ttysize ts;
-ioctl(0, TIOCGWINSZ, &ts);
-printf("number of columns: %i, number of rows: %i, screen width: %i, screen height: %i\n", sz.ws_col, sz.ws_row, sz.ws_xpixel, sz.ws_ypixel);
-```
-
-In Python:
-```py
-import array, fcntl, termios
-buf = array.array('H', [0, 0, 0, 0])
-fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, buf)
-print('number of columns: {}, number of rows: {}, screen width: {}, screen height: {}'.format(*buf))
-```
-
-Note that some terminals return `0` for the width and height values. Such terminals should be modified to return the correct values.
-Examples of terminals that return correct values: `kitty, xterm`
-
-== The graphics escape code
-
-All graphics escape codes are of the form:
-
-```
-<ESC>_G<control data>;<payload><ESC>\
-```
-
-This is a so-called _Application Programming Command (APC)_. Most terminal
-emulators ignore APC codes, making it safe to use.
-
-The control data is a comma-separated list of `key=value` pairs.  The payload
-is arbitrary binary data, base64-encoded to prevent interoperation problems
-with legacy terminals that get confused by control codes within an APC code.
-The meaning of the payload is interpreted based on the control data.
-
-The first step is to transmit the actual image data.
-
-== Transferring pixel data
-
-The first consideration when transferring data between the client and the
-terminal emulator is the format in which to do so. Since there is a vast and
-growing number of image formats in existence, it does not make sense to have
-every terminal emulator implement support for them. Instead, the client should
-send simple pixel data to the terminal emulator. The obvious downside to this
-is performance, especially when the client is running on a remote machine.
-Techniques for remedying this limitation are discussed later. The terminal
-emulator must understand pixel data in three formats, 24-bit RGB, 32-bit RGBA and
-PNG. This is specified using the `f` key in the control data. `f=32` (which is the
-default) indicates 32-bit RGBA data and `f=24` indicates 24-bit RGB data and `f=100`
-indicates PNG data. The PNG format is supported for convenience and a compact way
-of transmitting paletted images.
-
-=== RGB and RGBA data
-
-In these formats the pixel data is stored directly as 3 or 4 bytes per pixel, respectively.
-When specifying images in this format, the image dimensions **must** be sent in the control data.
-For example:
-
-```
-<ESC>_Gf=24,s=10,v=20;<payload><ESC>\
-```
-
-Here the width and height are specified using the `s` and `v` keys respectively. Since
-`f=24` there are three bytes per pixel and therefore the pixel data must be `3 * 10 * 20 = 600`
-bytes.
-
-=== PNG data
-
-In this format any PNG image can be transmitted directly.  For example:
-
-```
-<ESC>_Gf=100;<payload><ESC>\
-
-```
-
-The PNG format is specified using the `f=100` key. The width and height of
-the image will be read from the PNG data itself. Note that if you use both PNG and
-compression, then you must provide the `S` key with the size of the PNG data.
-
-
-=== Compression
-
-The client can send compressed image data to the terminal emulator, by specifying the
-`o` key. Currently, only zlib based deflate compression is supported, which is specified using
-`o=z`. For example,
-
-```
-<ESC>_Gf=24,s=10,v=20,o=z;<payload><ESC>\
-```
-
-This is the same as the example from the RGB data section, except that the
-payload is now compressed using deflate. The terminal emulator will decompress
-it before rendering. You can specify compression for any format. The terminal
-emulator will decompress before interpreting the pixel data.
-
-
-=== The transmission medium
-
-The transmission medium is specified using the `t` key. The `t` key defaults to `d`
-and can take the values:
-
-|===
-| Value of `t` | Meaning
-
-| d | Direct (the data is transmitted within the escape code itself)
-| f | A simple file
-| t | A temporary file, the terminal emulator will delete the file after reading the pixel data
-| s | A http://man7.org/linux/man-pages/man7/shm_overview.7.html[POSIX shared memory object]. The terminal emulator will delete it after reading the pixel data
-|===
-
-==== Local client
-
-First let us consider the local client techniques (files and shared memory). Some examples:
-
-```
-<ESC>_Gf=100,t=f;<encoded /path/to/file.png><ESC>\
-```
-
-Here we tell the terminal emulator to read PNG data from the specified file of
-the specified size.
-
-```
-<ESC>_Gs=10,v=2,t=s,o=z;<encoded /some-shared-memory-name><ESC>\
-```
-
-Here we tell the terminal emulator to read compressed image data from
-the specified shared memory object.
-
-The client can also specify a size and offset to tell the terminal emulator
-to only read a part of the specified file. The is done using the `S` and `O`
-keys respectively. For example:
-
-```
-<ESC>_Gs=10,v=2,t=s,S=80,O=10;<encoded /some-shared-memory-name><ESC>\
-```
-
-This tells the terminal emulator to read `80` bytes starting from the offset `10`
-inside the specified shared memory buffer.
-
-
-==== Remote client
-
-Remote clients, those that are unable to use the filesystem/shared memory to
-transmit data, must send the pixel data directly using escape codes. Since
-escape codes are of limited maximum length, the data will need to be chunked up
-for transfer. This is done using the `m` key. The pixel data must first be
-base64 encoded then chunked up into chunks no larger than `4096` bytes. The client
-then sends the graphics escape code as usual, with the addition of an `m` key that
-must have the value `1` for all but the last chunk, where it must be `0`. For example,
-if the data is split into three chunks, the client would send the following
-sequence of escape codes to the terminal emulator:
-
-```
-<ESC>_Gs=100,v=30,m=1;<encoded pixel data first chunk><ESC>\
-<ESC>_Gm=1;<encoded pixel data second chunk><ESC>\
-<ESC>_Gm=0;<encoded pixel data last chunk><ESC>\
-```
-
-Note that only the first escape code needs to have the full set of control
-codes such as width, height, format etc. Subsequent chunks must have
-only the `m` key. The client **must** finish sending all chunks for a single image
-before sending any other graphics related escape codes.
-
-
-=== Detecting available transmission mediums
-
-Since a client has no a-priori knowledge of whether it shares a filesystem/shared memory
-with the terminal emulator, it can send an id with the control data, using the `i` key
-(which can be an arbitrary positive integer up to 4294967295, it must not be zero).
-If it does so, the terminal emulator will reply after trying to load the image, saying
-whether loading was successful or not. For example:
-
-```
-<ESC>_Gi=31,s=10,v=2,t=s;<encoded /some-shared-memory-name><ESC>\
-```
-
-to which the terminal emulator will reply (after trying to load the data):
-
-```
-<ESC>_Gi=31;error message or OK<ESC>\
-```
-
-Here the `i` value will be the same as was sent by the client in the original
-request.  The message data will be a ASCII encoded string containing only
-printable characters and spaces. The string will be `OK` if reading the pixel
-data succeeded or an error message.
-
-Sometimes, using an id is not appropriate, for example, if you do not want to
-replace a previously sent image with the same id, or if you are sending a dummy
-image and do not want it stored by the terminal emulator. In that case, you can
-use the *query action*, set `a=q`. Then the terminal emulator will try to load
-the image and respond with either OK or an error, as above, but it will not
-replace an existing image with the same id, nor will it store the image.
-
-
-== Display images on screen
-
-Every transmitted image can be displayed an arbitrary number of times on the
-screen, in different locations, using different parts of the source image, as
-needed. You can either simultaneously transmit and display an image using the
-action `a=T`, or first transmit the image with a id, such as `i=10` and then display
-it with `a=p,i=10` which will display the previously transmitted image at the current
-cursor position. When specifying an image id, the terminal emulator will reply with an
-acknowledgement code, which will be either:
-
-```
-<ESC>_Gi=<id>;OK<ESC>\
-```
-
-when the image referred to by id was found, or
-
-```
-<ESC>_Gi=<id>;ENOENT:<some detailed error msg><ESC>\
-```
-
-when the image with the specified id was not found. This is similar to the
-scheme described above for querying available transmission media, except that
-here we are querying if the image with the specified id is available or needs to
-be re-transmitted.
-
-=== Controlling displayed image layout
-
-The image is rendered at the current cursor position, from the upper left corner of
-the current cell. You can also specify extra `X=3` and `Y=4` pixel offsets to display from
-a different origin within the cell. Note that the offsets must be smaller that the size of the cell.
-
-By default, the entire image will be displayed (images wider than the available
-width will be truncated on the right edge). You can choose a source rectangle (in pixels)
-as the part of the image to display. This is done with the keys: `x, y, w, h` which specify
-the top-left corner, width and height of the source rectangle.
-
-You can also ask the terminal emulator to display the image in a specified rectangle
-(num of columns / num of lines), using the control codes `c,r`. `c` is the number of columns
-and `r` the number of rows. The image will be scaled (enlarged/shrunk) as needed to fit
-the specified area. Note that if you specify a start cell offset via the `X,Y` keys, it is not
-added to the number of rows/columns.
-
-Finally, you can specify the image *z-index*, i.e. the vertical stacking order. Images
-placed in the same location with different z-index values will be blended if
-they are semi-transparent. You can specify z-index values using the `z` key.
-Negative z-index values mean that the images will be drawn under the text. This
-allows rendering of text on top of images.
-
-== Deleting images
-
-Images can be deleted by using the delete action `a=d`. If specified without any
-other keys, it will delete all images visible on screen. To delete specific images,
-use the `d` key as described in the table below. Note that each value of d has
-both a lowercase and an uppercase variant. The lowercase variant only deletes the
-images without necessarily freeing up the stored image data, so that the images can be
-re-displayed without needing to resend the data. The uppercase variants will delete
-the image data as well, provided that the image is not referenced elsewhere, such as in the
-scrollback buffer. The values of the `x` and `y` keys are the same as cursor positions (i.e.
-x=1, y=1 is the top left cell).
-
-|===
-| Value of `d` | Meaning
-
-| `a` or `A` | Delete all images visible on screen
-| `i` or `I` | Delete all images with the specified id, specified using the `i` key.
-| `c` or `C` | Delete all images that intersect with the current cursor position.
-| `p` or `P` | Delete all images that intersect a specific cell, the cell is specified using the `x` and `y` keys
-| `q` or `Q` | Delete all images that intersect a specific cell having a specific z-index. The cell and z-index is specified using the `x`, `y` and `z` keys.
-| `x` or `X` | Delete all images that intersect the specified column, specified using the `x` key.
-| `y` or `Y` | Delete all images that intersect the specified row, specified using the `y` key.
-| `z` or `Z` | Delete all images that have the specified z-index, specified using the `z` key.
-
-|===
-
-
-Some examples:
-
-```
-<ESC>_Ga=d<ESC>\   # delete all visible images
-<ESC>_Ga=d,i=10<ESC>\   # delete the image with id=10
-<ESC>_Ga=Z,z=-1<ESC>\   # delete the images with z-index -1, also freeing up image data
-<ESC>_Ga=P,x=3,y=4<ESC>\   # delete all images that intersect the cell at (3, 4)
-```
-
-=== Image persistence and storage quotas
-
-In order to avoid *Denial-of-Service* attacks, terminal emulators should have a
-maximum storage quota for image data. It should allow at least a few full
-screen images.  For example the quota in kitty is 320MB per buffer. When adding
-a new image, if the total size exceeds the quota, the terminal emulator should
-delete older images to make space for the new one.
-
-
-== Control data reference
-
-The table below shows all the control data keys as well as what values they can
-take, and the default value they take when missing. All integers are 32-bit.
-
-[cols="^1,<3,^1,<6"]
-|===
-|Key | Value | Default | Description
-
-| `a` | Single character. `(t, T, q, p, d)` | `t` | The overall action this graphics command is performing.
-
-4+^.^h| Keys for image transmission
-
-| `f` | Positive integer. `(24, 32, 100)`. | `32` | The format in which the image data is sent.
-| `t` | Single character. `(d, f, t, s)`. | `d` | The transmission medium used.
-| `s` | Positive integer. | `0` | The width of the image being sent.
-| `v` | Positive integer. | `0` | The height of the image being sent.
-| `S` | Positive integer. | `0` | The size of data to read from a file.
-| `O` | Positive integer. | `0` | The offset from which to read data from a file.
-| `i` | Positive integer. `(0 - 4294967295)` | `0` | The image id
-| `o` | Single character. `only z` | `null` | The type of data compression.
-| `m` | zero or one | `0` | Whether there is more chunked data available.
-
-4+^.^h| Keys for image display
-
-| `x` | Positive integer | `0` | The left edge (in pixels) of the image area to display
-| `y` | Positive integer | `0` | The top edge (in pixels) of the image area to display
-| `w` | Positive integer | `0` | The width (in pixels) of the image area to display. By default, the entire width is used.
-| `h` | Positive integer | `0` | The height (in pixels) of the image area to display. By default, the entire height is used
-| `X` | Positive integer | `0` | The x-offset within the first cell at which to start displaying the image
-| `Y` | Positive integer | `0` | The y-offset within the first cell at which to start displaying the image
-| `c` | Positive integer | `0` | The number of columns to display the image over
-| `r` | Positive integer | `0` | The number of rows to display the image over
-| `z` | Integer          | `0` | The *z-index* vertical stacking order of the image
-
-4+^.^h| Keys for deleting images
-
-| `d` | Single character. `(a, A, c, C, p, P, q, Q, x, X, y, Y, z, Z)`. | `a` | What to delete.
-
-|===
-
-
-== Interaction with other terminal actions
-
-When resetting the terminal, all images that are visible on the screen must be
-cleared.  When switching from the main screen to the alternate screen buffer
-(1049 private mode) all images in the alternate screen must be cleared, just as
-all text is cleared. The clear screen escape code (usually `<ESC>[2J`) should also
-clear all images. This is so that the clear command works.
-
-The other commands to erase text must have no effect on graphics.
-The dedicated delete graphics commands must be used for those.
-
-When scrolling the screen (such as when using index cursor movement commands,
-or scrolling through the history buffer), images must be scrolled along with
-text. When page margins are defined and the index commands are used, only
-images that are entirely within the page area (between the margins) must be
-scrolled. When scrolling them would cause them to extend outside the page area,
-they must be clipped.
+See https://sw.kovidgoyal.net/kitty/graphics-protocol.html

key_encoding.asciidoc

@@ -1,129 +1 @@
-= Key encoding for extended keyboard protocol
-
-See link:protocol-extensions.asciidoc#keyboard-handling[Keyboard Handling protocol extension]
- for more information and link:key_encoding.json[for this table in JSON format].
-
-|===
-| Name | Encoded representation (base64)
-
-| 0               | `G`
-| 1               | `H`
-| 2               | `I`
-| 3               | `J`
-| 4               | `K`
-| 5               | `L`
-| 6               | `M`
-| 7               | `N`
-| 8               | `O`
-| 9               | `P`
-| A               | `S`
-| APOSTROPHE      | `B`
-| B               | `T`
-| BACKSLASH       | `t`
-| BACKSPACE       | `1`
-| C               | `U`
-| CAPS LOCK       | `:`
-| COMMA           | `C`
-| D               | `V`
-| DELETE          | `3`
-| DOWN            | `6`
-| E               | `W`
-| END             | `-`
-| ENTER           | `z`
-| EQUAL           | `R`
-| ESCAPE          | `y`
-| F               | `X`
-| F1              | `/`
-| F10             | `]`
-| F11             | `{`
-| F12             | `}`
-| F13             | `@`
-| F14             | `%`
-| F15             | `$`
-| F16             | `#`
-| F17             | `BA`
-| F18             | `BB`
-| F19             | `BC`
-| F2              | `*`
-| F20             | `BD`
-| F21             | `BE`
-| F22             | `BF`
-| F23             | `BG`
-| F24             | `BH`
-| F25             | `BI`
-| F3              | `?`
-| F4              | `&`
-| F5              | `<`
-| F6              | `>`
-| F7              | `(`
-| F8              | `)`
-| F9              | `[`
-| G               | `Y`
-| GRAVE ACCENT    | `v`
-| H               | `Z`
-| HOME            | `.`
-| I               | `a`
-| INSERT          | `2`
-| J               | `b`
-| K               | `c`
-| KP 0            | `BJ`
-| KP 1            | `BK`
-| KP 2            | `BL`
-| KP 3            | `BM`
-| KP 4            | `BN`
-| KP 5            | `BO`
-| KP 6            | `BP`
-| KP 7            | `BQ`
-| KP 8            | `BR`
-| KP 9            | `BS`
-| KP ADD          | `BX`
-| KP DECIMAL      | `BT`
-| KP DIVIDE       | `BU`
-| KP ENTER        | `BY`
-| KP EQUAL        | `BZ`
-| KP MULTIPLY     | `BV`
-| KP SUBTRACT     | `BW`
-| L               | `d`
-| LEFT            | `5`
-| LEFT ALT        | `Bc`
-| LEFT BRACKET    | `s`
-| LEFT CONTROL    | `Bb`
-| LEFT SHIFT      | `Ba`
-| LEFT SUPER      | `Bd`
-| M               | `e`
-| MINUS           | `D`
-| N               | `f`
-| NUM LOCK        | `=`
-| O               | `g`
-| P               | `h`
-| PAGE DOWN       | `9`
-| PAGE UP         | `8`
-| PAUSE           | `!`
-| PERIOD          | `E`
-| PRINT SCREEN    | `^`
-| Q               | `i`
-| R               | `j`
-| RIGHT           | `4`
-| RIGHT ALT       | `Bg`
-| RIGHT BRACKET   | `u`
-| RIGHT CONTROL   | `Bf`
-| RIGHT SHIFT     | `Be`
-| RIGHT SUPER     | `Bh`
-| S               | `k`
-| SCROLL LOCK     | `+`
-| SEMICOLON       | `Q`
-| SLASH           | `F`
-| SPACE           | `A`
-| T               | `l`
-| TAB             | `0`
-| U               | `m`
-| UP              | `7`
-| V               | `n`
-| W               | `o`
-| WORLD 1         | `w`
-| WORLD 2         | `x`
-| X               | `p`
-| Y               | `q`
-| Z               | `r`
-
-|===
+See https://sw.kovidgoyal.net/kitty/key-encoding.html

kittens/ask/main.py

@@ -50,10 +50,11 @@ class HistoryCompleter:
 
     def __enter__(self):
         if self.history_path:
-            if os.path.exists(self.history_path):
+            try:
                 readline.read_history_file(self.history_path)
+            except Exception:
+                pass
             readline.set_completer(self.complete)
-            readline.parse_and_bind('tab: complete')
         return self
 
     def __exit__(self, *a):
@@ -87,6 +88,7 @@ def main(args):
     global readline
     import readline as rl
     readline = rl
+    from kitty.shell import init_readline
     msg = 'Ask the user for input'
     try:
         args, items = parse_args(args[1:], option_text, '', msg, 'kitty ask')
@@ -96,7 +98,7 @@ def main(args):
             input('Press enter to quit...')
         raise SystemExit(e.code)
 
-    readline.read_init_file()
+    init_readline(readline)
     ans = {'items': items}
 
     with alternate_screen(), HistoryCompleter(args.name):

kittens/clipboard/main.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+import sys
+
+from kitty.cli import parse_args
+
+from ..tui.handler import Handler
+from ..tui.loop import Loop
+
+
+class Clipboard(Handler):
+
+    def __init__(self, data_to_send, args):
+        self.args = args
+        self.clipboard_contents = None
+        self.data_to_send = data_to_send
+
+    def initialize(self):
+        if self.data_to_send is not None:
+            self.cmd.write_to_clipboard(self.data_to_send, self.args.use_primary)
+        if not self.args.get_clipboard:
+            self.quit_loop(0)
+            return
+        self.cmd.request_from_clipboard(self.args.use_primary)
+
+    def on_clipboard_response(self, text, from_primary=False):
+        self.clipboard_contents = text
+        self.quit_loop(0)
+
+
+OPTIONS = r'''
+--get-clipboard
+default=False
+type=bool-set
+Output the current contents of the clipboard to stdout. Note that this
+will not work if you have not enabled the option to allow reading the clipboard
+in kitty.conf
+
+
+--use-primary
+default=False
+type=bool-set
+Use the primary selection rather than the clipboard on systems that support it,
+such as X11.
+'''.format
+help_text = '''\
+Read or write to the system clipboard.
+
+To set the clipboard text, pipe in the new text on stdin. Use the
+:option:`--get-clipboard` option to output the current clipboard contents to
+:file:`stdout`. Note that you must enable reading of clipboard in
+:file:`kitty.conf` first.
+'''
+usage = ''
+
+
+def main(args):
+    args, items = parse_args(args[1:], OPTIONS, usage, help_text, 'kitty +kitten clipboard')
+    if items:
+        raise SystemExit('Unrecognized extra command line arguments')
+    data = None
+    if not sys.stdin.isatty():
+        data = sys.stdin.buffer.read()
+        sys.stdin = open(os.ctermid(), 'r')
+    loop = Loop()
+    handler = Clipboard(data, args)
+    loop.loop(handler)
+    if loop.return_code == 0 and handler.clipboard_contents:
+        sys.stdout.write(handler.clipboard_contents)
+        sys.stdout.flush()
+    raise SystemExit(loop.return_code)
+
+
+if __name__ == '__main__':
+    main(sys.argv)
+elif __name__ == '__doc__':
+    sys.cli_docs['usage'] = usage
+    sys.cli_docs['options'] = OPTIONS
+    sys.cli_docs['help_text'] = help_text

kittens/diff/README.asciidoc

@@ -0,0 +1 @@
+See https://sw.kovidgoyal.net/kitty/kittens/diff.html

kittens/diff/collect.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+import re
+from functools import lru_cache
+from hashlib import md5
+from mimetypes import guess_type
+
+path_name_map = {}
+
+
+class Segment:
+
+    __slots__ = ('start', 'end', 'start_code', 'end_code')
+
+    def __init__(self, start, start_code):
+        self.start = start
+        self.start_code = start_code
+
+    def __repr__(self):
+        return 'Segment(start={!r}, start_code={!r}, end={!r}, end_code={!r})'.format(
+                self.start, self.start_code, getattr(self, 'end', None), getattr(self, 'end_code', None)
+        )
+
+
+class Collection:
+
+    def __init__(self):
+        self.changes = {}
+        self.renames = {}
+        self.adds = set()
+        self.removes = set()
+        self.all_paths = []
+        self.type_map = {}
+        self.added_count = self.removed_count = 0
+
+    def add_change(self, left_path, right_path):
+        self.changes[left_path] = right_path
+        self.all_paths.append(left_path)
+        self.type_map[left_path] = 'diff'
+
+    def add_rename(self, left_path, right_path):
+        self.renames[left_path] = right_path
+        self.all_paths.append(left_path)
+        self.type_map[left_path] = 'rename'
+
+    def add_add(self, right_path):
+        self.adds.add(right_path)
+        self.all_paths.append(right_path)
+        self.type_map[right_path] = 'add'
+        if isinstance(data_for_path(right_path), str):
+            self.added_count += len(lines_for_path(right_path))
+
+    def add_removal(self, left_path):
+        self.removes.add(left_path)
+        self.all_paths.append(left_path)
+        self.type_map[left_path] = 'removal'
+        if isinstance(data_for_path(left_path), str):
+            self.removed_count += len(lines_for_path(left_path))
+
+    def finalize(self):
+        self.all_paths.sort(key=path_name_map.get)
+
+    def __iter__(self):
+        for path in self.all_paths:
+            typ = self.type_map[path]
+            if typ == 'diff':
+                data = self.changes[path]
+            elif typ == 'rename':
+                data = self.renames[path]
+            else:
+                data = None
+            yield path, typ, data
+
+    def __len__(self):
+        return len(self.all_paths)
+
+
+def collect_files(collection, left, right):
+    left_names, right_names = set(), set()
+    left_path_map, right_path_map = {}, {}
+
+    def walk(base, names, pmap):
+        for dirpath, dirnames, filenames in os.walk(base):
+            for filename in filenames:
+                path = os.path.abspath(os.path.join(dirpath, filename))
+                path_name_map[path] = name = os.path.relpath(path, base)
+                names.add(name)
+                pmap[name] = path
+
+    walk(left, left_names, left_path_map), walk(right, right_names, right_path_map)
+    common_names = left_names & right_names
+    changed_names = {n for n in common_names if data_for_path(left_path_map[n]) != data_for_path(right_path_map[n])}
+    for n in changed_names:
+        collection.add_change(left_path_map[n], right_path_map[n])
+
+    removed = left_names - common_names
+    added = right_names - common_names
+    ahash = {a: hash_for_path(right_path_map[a]) for a in added}
+    rhash = {r: hash_for_path(left_path_map[r]) for r in removed}
+    for name, rh in rhash.items():
+        for n, ah in ahash.items():
+            if ah == rh and data_for_path(left_path_map[name]) == data_for_path(right_path_map[n]):
+                collection.add_rename(left_path_map[name], right_path_map[n])
+                added.discard(n)
+                break
+        else:
+            collection.add_removal(left_path_map[name])
+
+    for name in added:
+        collection.add_add(right_path_map[name])
+
+
+sanitize_pat = re.compile('[\x00-\x09\x0b-\x1f\x7f\x80-\x9f]')
+
+
+def sanitize(text):
+    ntext = text.replace('\r\n', '⏎\n')
+    return sanitize_pat.sub('░', ntext)
+
+
+@lru_cache(maxsize=1024)
+def mime_type_for_path(path):
+    return guess_type(path)[0] or 'application/octet-stream'
+
+
+@lru_cache(maxsize=1024)
+def raw_data_for_path(path):
+    with open(path, 'rb') as f:
+        return f.read()
+
+
+def is_image(path):
+    return mime_type_for_path(path).startswith('image/')
+
+
+@lru_cache(maxsize=1024)
+def data_for_path(path):
+    ans = raw_data_for_path(path)
+    if not is_image(path) and not os.path.samefile(path, os.devnull):
+        try:
+            ans = ans.decode('utf-8')
+        except UnicodeDecodeError:
+            pass
+    return ans
+
+
+@lru_cache(maxsize=1024)
+def lines_for_path(path):
+    data = data_for_path(path).replace('\t', lines_for_path.replace_tab_by)
+    return tuple(sanitize(data).splitlines())
+
+
+lines_for_path.replace_tab_by = ' ' * 4
+
+
+@lru_cache(maxsize=1024)
+def hash_for_path(path):
+    return md5(raw_data_for_path(path)).digest()
+
+
+def create_collection(left, right):
+    collection = Collection()
+    if os.path.isdir(left):
+        collect_files(collection, left, right)
+    else:
+        pl, pr = os.path.abspath(left), os.path.abspath(right)
+        path_name_map[pl] = left
+        path_name_map[pr] = right
+        collection.add_change(pl, pr)
+    collection.finalize()
+    return collection
+
+
+highlight_data = {}
+
+
+def set_highlight_data(data):
+    global highlight_data
+    highlight_data = data
+
+
+def highlights_for_path(path):
+    return highlight_data.get(path, [])

kittens/diff/config.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+
+from kitty.conf.utils import (
+    init_config as _init_config, key_func, load_config as _load_config, merge_dicts,
+    parse_config_base, parse_kittens_key, resolve_config
+)
+from kitty.conf.definition import config_lines
+from kitty.constants import config_dir
+from kitty.rgb import color_as_sgr
+
+from .config_data import type_map, all_options
+
+defaults = None
+
+formats = {
+    'title': '',
+    'margin': '',
+    'text': '',
+}
+
+
+def set_formats(opts):
+    formats['text'] = '48' + color_as_sgr(opts.background)
+    formats['title'] = '38' + color_as_sgr(opts.title_fg) + ';48' + color_as_sgr(opts.title_bg) + ';1'
+    formats['margin'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.margin_bg)
+    formats['added_margin'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.added_margin_bg)
+    formats['removed_margin'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.removed_margin_bg)
+    formats['added'] = '48' + color_as_sgr(opts.added_bg)
+    formats['removed'] = '48' + color_as_sgr(opts.removed_bg)
+    formats['filler'] = '48' + color_as_sgr(opts.filler_bg)
+    formats['hunk_margin'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.hunk_margin_bg)
+    formats['hunk'] = '38' + color_as_sgr(opts.margin_fg) + ';48' + color_as_sgr(opts.hunk_bg)
+    formats['removed_highlight'] = '48' + color_as_sgr(opts.highlight_removed_bg)
+    formats['added_highlight'] = '48' + color_as_sgr(opts.highlight_added_bg)
+
+
+func_with_args, args_funcs = key_func()
+
+
+@func_with_args('scroll_by')
+def parse_scroll_by(func, rest):
+    try:
+        return func, int(rest)
+    except Exception:
+        return func, 1
+
+
+@func_with_args('scroll_to')
+def parse_scroll_to(func, rest):
+    rest = rest.lower()
+    if rest not in {'start', 'end', 'next-change', 'prev-change', 'next-page', 'prev-page', 'next-match', 'prev-match'}:
+        rest = 'start'
+    return func, rest
+
+
+@func_with_args('change_context')
+def parse_change_context(func, rest):
+    rest = rest.lower()
+    if rest in {'all', 'default'}:
+        return func, rest
+    try:
+        amount = int(rest)
+    except Exception:
+        amount = 5
+    return func, amount
+
+
+@func_with_args('start_search')
+def parse_start_search(func, rest):
+    rest = rest.lower().split()
+    is_regex = rest and rest[0] == 'regex'
+    is_backward = len(rest) > 1 and rest[1] == 'backward'
+    return func, (is_regex, is_backward)
+
+
+def special_handling(key, val, ans):
+    if key == 'map':
+        action, *key_def = parse_kittens_key(val, args_funcs)
+        ans['key_definitions'][tuple(key_def)] = action
+        return True
+
+
+def parse_config(lines, check_keys=True):
+    ans = {'key_definitions': {}}
+    parse_config_base(
+        lines,
+        defaults,
+        type_map,
+        special_handling,
+        ans,
+        check_keys=check_keys
+    )
+    return ans
+
+
+def merge_configs(defaults, vals):
+    ans = {}
+    for k, v in defaults.items():
+        if isinstance(v, dict):
+            newvals = vals.get(k, {})
+            ans[k] = merge_dicts(v, newvals)
+        else:
+            ans[k] = vals.get(k, v)
+    return ans
+
+
+def parse_defaults(lines, check_keys=False):
+    return parse_config(lines, check_keys)
+
+
+Options, defaults = _init_config(config_lines(all_options), parse_defaults)
+
+
+def load_config(*paths, overrides=None):
+    return _load_config(Options, defaults, parse_config, merge_configs, *paths, overrides=overrides)
+
+
+SYSTEM_CONF = '/etc/xdg/kitty/diff.conf'
+defconf = os.path.join(config_dir, 'diff.conf')
+
+
+def init_config(args):
+    config = tuple(resolve_config(SYSTEM_CONF, defconf, args.config))
+    overrides = (a.replace('=', ' ', 1) for a in args.override or ())
+    opts = load_config(*config, overrides=overrides)
+    set_formats(opts)
+    return opts

kittens/diff/config_data.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+# Utils  {{{
+from gettext import gettext as _
+from functools import partial
+
+from kitty.conf.definition import option_func
+from kitty.conf.utils import (
+    positive_int, python_string, to_color
+)
+
+# }}}
+
+all_options = {}
+o, k, g, all_groups = option_func(all_options, {
+    'colors': [_('Colors')],
+    'diff': [_('Diffing'), ],
+    'shortcuts': [_('Keyboard shortcuts')],
+})
+
+
+g('diff')
+
+
+def syntax_aliases(raw):
+    ans = {}
+    for x in raw.split():
+        a, b = x.partition(':')[::2]
+        if a and b:
+            ans[a.lower()] = b
+    return ans
+
+
+o('syntax_aliases', 'pyj:py recipe:py', option_type=syntax_aliases, long_text=_('''
+File extension aliases for syntax highlight
+For example, to syntax highlight :file:`file.xyz` as
+:file:`file.abc` use a setting of :code:`xyz:abc`
+'''))
+
+o('num_context_lines', 3, option_type=positive_int, long_text=_('''
+The number of lines of context to show around each change.'''))
+
+o('diff_cmd', 'auto', long_text=_('''
+The diff command to use. Must contain the placeholder :code:`_CONTEXT_`
+which will be replaced by the number of lines of context. The default
+is to search the system for either git or diff and use that, if found.
+'''))
+
+o('replace_tab_by', r'\x20\x20\x20\x20', option_type=python_string, long_text=_('''
+The string to replace tabs with. Default is to use four spaces.'''))
+
+
+g('colors')
+
+o('pygments_style', 'default', long_text=_('''
+The pygments color scheme to use for syntax highlighting.
+See :link:`pygments colors schemes <https://help.farbox.com/pygments.html>` for a list of schemes.'''))
+
+
+c = partial(o, option_type=to_color)
+c('foreground', 'black', long_text=_('Basic colors'))
+c('background', 'white')
+
+c('title_fg', 'black', long_text=_('Title colors'))
+c('title_bg', 'white')
+
+c('margin_bg', '#fafbfc', long_text=_('Margin colors'))
+c('margin_fg', '#aaaaaa')
+
+c('removed_bg', '#ffeef0', long_text=_('Removed text backgrounds'))
+c('highlight_removed_bg', '#fdb8c0')
+c('removed_margin_bg', '#ffdce0')
+
+c('added_bg', '#e6ffed', long_text=_('Added text backgrounds'))
+c('highlight_added_bg', '#acf2bd')
+c('added_margin_bg', '#cdffd8')
+
+c('filler_bg', '#fafbfc', long_text=_('Filler (empty) line background'))
+
+c('hunk_margin_bg', '#dbedff', long_text=_('Hunk header colors'))
+c('hunk_bg', '#f1f8ff')
+
+c('search_bg', '#444', long_text=_('Highlighting'))
+c('search_fg', 'white')
+c('select_bg', '#B4D5FE')
+c('select_fg', 'black')
+
+g('shortcuts')
+k('quit', 'q', 'quit', _('Quit'))
+k('quit', 'esc', 'quit', _('Quit'))
+
+k('scroll_down', 'j', 'scroll_by 1', _('Scroll down'))
+k('scroll_down', 'down', 'scroll_by 1', _('Scroll down'))
+k('scroll_up', 'k', 'scroll_by -1', _('Scroll up'))
+k('scroll_up', 'up', 'scroll_by -1', _('Scroll up'))
+
+k('scroll_top', 'home', 'scroll_to start', _('Scroll to top'))
+k('scroll_bottom', 'end', 'scroll_to end', _('Scroll to bottom'))
+
+k('scroll_page_down', 'page_down', 'scroll_to next-page', _('Scroll to next page'))
+k('scroll_page_down', 'space', 'scroll_to next-page', _('Scroll to next page'))
+k('scroll_page_up', 'page_up', 'scroll_to prev-page', _('Scroll to previous page'))
+
+k('next_change', 'n', 'scroll_to next-change', _('Scroll to next change'))
+k('prev_change', 'p', 'scroll_to prev-change', _('Scroll to previous change'))
+
+k('all_context', 'a', 'change_context all', _('Show all context'))
+k('default_context', '=', 'change_context default', _('Show default context'))
+k('increase_context', '+', 'change_context 5', _('Increase context'))
+k('decrease_context', '-', 'change_context -5', _('Decrease context'))
+
+k('search_forward', '/', 'start_search regex forward', _('Search forward'))
+k('search_backward', '?', 'start_search regex backward', _('Search backward'))
+k('next_match', '.', 'scroll_to next-match', _('Scroll to next search match'))
+k('prev_match', ',', 'scroll_to prev-match', _('Scroll to previous search match'))
+k('next_match', '>', 'scroll_to next-match', _('Scroll to next search match'))
+k('prev_match', '<', 'scroll_to prev-match', _('Scroll to previous search match'))
+k('search_forward_simple', 'f', 'start_search substring forward', _('Search forward (no regex)'))
+k('search_backward_simple', 'b', 'start_search substring backward', _('Search backward (no regex)'))
+
+type_map = {o.name: o.option_type for o in all_options.values() if hasattr(o, 'option_type')}

kittens/diff/highlight.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import concurrent
+import os
+import re
+
+from pygments import highlight
+from pygments.formatter import Formatter
+from pygments.lexers import get_lexer_for_filename
+from pygments.util import ClassNotFound
+
+from kitty.rgb import color_as_sgr, parse_sharp
+
+from .collect import Segment, data_for_path, lines_for_path
+
+
+class StyleNotFound(Exception):
+    pass
+
+
+class DiffFormatter(Formatter):
+
+    def __init__(self, style='default'):
+        try:
+            Formatter.__init__(self, style=style)
+            initialized = True
+        except ClassNotFound:
+            initialized = False
+        if not initialized:
+            raise StyleNotFound('pygments style "{}" not found'.format(style))
+
+        self.styles = {}
+        for token, style in self.style:
+            start = []
+            end = []
+            # a style item is a tuple in the following form:
+            # colors are readily specified in hex: 'RRGGBB'
+            if style['color']:
+                start.append('38' + color_as_sgr(parse_sharp(style['color'])))
+                end.append('39')
+            if style['bold']:
+                start.append('1')
+                end.append('22')
+            if style['italic']:
+                start.append('3')
+                end.append('23')
+            if style['underline']:
+                start.append('4')
+                end.append('24')
+            if start:
+                start = '\033[{}m'.format(';'.join(start))
+                end = '\033[{}m'.format(';'.join(end))
+            self.styles[token] = start or '', end or ''
+
+    def format(self, tokensource, outfile):
+        for ttype, value in tokensource:
+            not_found = True
+            if value.rstrip('\n'):
+                while ttype and not_found:
+                    tok = self.styles.get(ttype)
+                    if tok is None:
+                        ttype = ttype[:-1]
+                    else:
+                        on, off = tok
+                        lines = value.split('\n')
+                        for line in lines:
+                            if line:
+                                outfile.write(on + line + off)
+                            if line is not lines[-1]:
+                                outfile.write('\n')
+                        not_found = False
+
+            if not_found:
+                outfile.write(value)
+
+
+formatter = None
+
+
+def initialize_highlighter(style='default'):
+    global formatter
+    formatter = DiffFormatter(style)
+
+
+def highlight_data(code, filename, aliases=None):
+    if aliases:
+        base, ext = os.path.splitext(filename)
+        alias = aliases.get(ext[1:])
+        if alias is not None:
+            filename = base + '.' + alias
+    try:
+        lexer = get_lexer_for_filename(filename, stripnl=False)
+    except ClassNotFound:
+        pass
+    else:
+        return highlight(code, lexer, formatter)
+
+
+split_pat = re.compile(r'(\033\[.*?m)')
+
+
+def highlight_line(line):
+    ans = []
+    current = None
+    pos = 0
+    for x in split_pat.split(line):
+        if x.startswith('\033'):
+            if current is None:
+                current = Segment(pos, x)
+            else:
+                current.end = pos
+                current.end_code = x
+                ans.append(current)
+                current = None
+        else:
+            pos += len(x)
+    return ans
+
+
+def highlight_for_diff(path, aliases):
+    ans = []
+    lines = lines_for_path(path)
+    hd = highlight_data('\n'.join(lines), path, aliases)
+    if hd is not None:
+        for line in hd.splitlines():
+            ans.append(highlight_line(line))
+    return ans
+
+
+def highlight_collection(collection, aliases=None):
+    jobs = {}
+    ans = {}
+    with concurrent.futures.ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:
+        highlight_collection.processes = executor._processes
+        for path, item_type, other_path in collection:
+            if item_type != 'rename':
+                for p in (path, other_path):
+                    if p:
+                        is_binary = isinstance(data_for_path(p), bytes)
+                        if not is_binary:
+                            jobs[executor.submit(highlight_for_diff, p, aliases)] = p
+        for future in concurrent.futures.as_completed(jobs):
+            path = jobs[future]
+            try:
+                highlights = future.result()
+            except Exception as e:
+                return 'Running syntax highlighting for {} generated an exception: {}'.format(path, e)
+            ans[path] = highlights
+    return ans
+
+
+def main():
+    from .config import defaults
+    # kitty +runpy "from kittens.diff.highlight import main; main()" file
+    import sys
+    initialize_highlighter()
+    with open(sys.argv[-1]) as f:
+        highlighted = highlight_data(f.read(), f.name, defaults.syntax_aliases)
+    if highlighted is None:
+        raise SystemExit('Unknown filetype: {}'.format(sys.argv[-1]))
+    print(highlighted)

kittens/diff/main.py

@@ -0,0 +1,560 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+import signal
+import sys
+import warnings
+from collections import defaultdict
+from functools import partial
+from gettext import gettext as _
+
+from kitty.cli import CONFIG_HELP, parse_args
+from kitty.constants import appname
+from kitty.fast_data_types import wcswidth
+from kitty.key_encoding import ESCAPE, RELEASE, enter_key
+
+from ..tui.handler import Handler
+from ..tui.images import ImageManager
+from ..tui.line_edit import LineEdit
+from ..tui.loop import Loop
+from ..tui.operations import styled
+from .collect import (
+    create_collection, data_for_path, lines_for_path, sanitize,
+    set_highlight_data
+)
+from .config import init_config
+from .patch import Differ, set_diff_command, worker_processes
+from .render import ImageSupportWarning, LineRef, render_diff
+from .search import BadRegex, Search
+
+try:
+    from .highlight import initialize_highlighter, highlight_collection
+except ImportError:
+    initialize_highlighter = None
+
+
+INITIALIZING, COLLECTED, DIFFED, COMMAND, MESSAGE = range(5)
+
+
+def generate_diff(collection, context):
+    d = Differ()
+
+    for path, item_type, changed_path in collection:
+        if item_type == 'diff':
+            is_binary = isinstance(data_for_path(path), bytes) or isinstance(data_for_path(changed_path), bytes)
+            if not is_binary:
+                d.add_diff(path, changed_path)
+
+    return d(context)
+
+
+class DiffHandler(Handler):
+
+    image_manager_class = ImageManager
+
+    def __init__(self, args, opts, left, right):
+        self.state = INITIALIZING
+        self.message = ''
+        self.current_search_is_regex = True
+        self.current_search = None
+        self.line_edit = LineEdit()
+        self.opts = opts
+        self.left, self.right = left, right
+        self.report_traceback_on_exit = None
+        self.args = args
+        self.scroll_pos = self.max_scroll_pos = 0
+        self.current_context_count = self.original_context_count = self.args.context
+        if self.current_context_count < 0:
+            self.current_context_count = self.original_context_count = self.opts.num_context_lines
+        self.highlighting_done = False
+        self.restore_position = None
+        for key_def, action in self.opts.key_definitions.items():
+            self.add_shortcut(action, *key_def)
+
+    def perform_action(self, action):
+        func, args = action
+        if func == 'quit':
+            self.quit_loop(0)
+            return
+        if self.state <= DIFFED:
+            if func == 'scroll_by':
+                return self.scroll_lines(*args)
+            if func == 'scroll_to':
+                where = args[0]
+                if 'change' in where:
+                    return self.scroll_to_next_change(backwards='prev' in where)
+                if 'match' in where:
+                    return self.scroll_to_next_match(backwards='prev' in where)
+                if 'page' in where:
+                    amt = self.num_lines * (1 if 'next' in where else -1)
+                else:
+                    amt = len(self.diff_lines) * (1 if 'end' in where else -1)
+                return self.scroll_lines(amt)
+            if func == 'change_context':
+                new_ctx = self.current_context_count
+                to = args[0]
+                if to == 'all':
+                    new_ctx = 100000
+                elif to == 'default':
+                    new_ctx = self.original_context_count
+                else:
+                    new_ctx += to
+                return self.change_context_count(new_ctx)
+            if func == 'start_search':
+                self.start_search(*args)
+                return
+
+    def create_collection(self):
+
+        def collect_done(collection):
+            self.collection = collection
+            self.state = COLLECTED
+            self.generate_diff()
+
+        def collect(left, right):
+            collection = create_collection(left, right)
+            self.asyncio_loop.call_soon_threadsafe(collect_done, collection)
+
+        self.asyncio_loop.run_in_executor(None, collect, self.left, self.right)
+
+    def generate_diff(self):
+
+        def diff_done(diff_map):
+            if isinstance(diff_map, str):
+                self.report_traceback_on_exit = diff_map
+                self.quit_loop(1)
+                return
+            self.state = DIFFED
+            self.diff_map = diff_map
+            self.calculate_statistics()
+            self.render_diff()
+            self.scroll_pos = 0
+            if self.restore_position is not None:
+                self.current_position = self.restore_position
+                self.restore_position = None
+            self.draw_screen()
+            if initialize_highlighter is not None and not self.highlighting_done:
+                from .highlight import StyleNotFound
+                self.highlighting_done = True
+                try:
+                    initialize_highlighter(self.opts.pygments_style)
+                except StyleNotFound as e:
+                    self.report_traceback_on_exit = str(e)
+                    self.quit_loop(1)
+                    return
+                self.syntax_highlight()
+
+        def diff(collection, current_context_count):
+            diff_map = generate_diff(collection, current_context_count)
+            self.asyncio_loop.call_soon_threadsafe(diff_done, diff_map)
+
+        self.asyncio_loop.run_in_executor(None, diff, self.collection, self.current_context_count)
+
+    def syntax_highlight(self):
+
+        def highlighting_done(hdata):
+            if isinstance(hdata, str):
+                self.report_traceback_on_exit = hdata
+                self.quit_loop(1)
+                return
+            set_highlight_data(hdata)
+            self.render_diff()
+            self.draw_screen()
+
+        def highlight(*a):
+            result = highlight_collection(*a)
+            self.asyncio_loop.call_soon_threadsafe(highlighting_done, result)
+
+        self.asyncio_loop.run_in_executor(None, highlight, self.collection, self.opts.syntax_aliases)
+
+    def calculate_statistics(self):
+        self.added_count = self.collection.added_count
+        self.removed_count = self.collection.removed_count
+        for patch in self.diff_map.values():
+            self.added_count += patch.added_count
+            self.removed_count += patch.removed_count
+
+    def render_diff(self):
+        self.diff_lines = tuple(render_diff(self.collection, self.diff_map, self.args, self.screen_size.cols, self.image_manager))
+        self.margin_size = render_diff.margin_size
+        self.ref_path_map = defaultdict(list)
+        for i, l in enumerate(self.diff_lines):
+            self.ref_path_map[l.ref.path].append((i, l.ref))
+        self.max_scroll_pos = len(self.diff_lines) - self.num_lines
+        if self.current_search is not None:
+            self.current_search(self.diff_lines, self.margin_size, self.screen_size.cols)
+
+    @property
+    def current_position(self):
+        return self.diff_lines[min(len(self.diff_lines) - 1, self.scroll_pos)].ref
+
+    @current_position.setter
+    def current_position(self, ref):
+        num = None
+        if isinstance(ref.extra, LineRef):
+            sln = ref.extra.src_line_number
+            for i, q in self.ref_path_map[ref.path]:
+                if isinstance(q.extra, LineRef):
+                    if q.extra.src_line_number >= sln:
+                        if q.extra.src_line_number == sln:
+                            num = i
+                        break
+                    num = i
+        if num is None:
+            for i, q in self.ref_path_map[ref.path]:
+                num = i
+                break
+
+        if num is not None:
+            self.scroll_pos = min(num, self.max_scroll_pos)
+
+    @property
+    def num_lines(self):
+        return self.screen_size.rows - 1
+
+    def scroll_to_next_change(self, backwards=False):
+        if backwards:
+            r = range(self.scroll_pos - 1, -1, -1)
+        else:
+            r = range(self.scroll_pos + 1, len(self.diff_lines))
+        for i in r:
+            line = self.diff_lines[i]
+            if line.is_change_start:
+                self.scroll_lines(i - self.scroll_pos)
+                return
+        self.cmd.bell()
+
+    def scroll_to_next_match(self, backwards=False, include_current=False):
+        if self.current_search is not None:
+            offset = 0 if include_current else 1
+            if backwards:
+                r = range(self.scroll_pos - offset, -1, -1)
+            else:
+                r = range(self.scroll_pos + offset, len(self.diff_lines))
+            for i in r:
+                if i in self.current_search:
+                    self.scroll_lines(i - self.scroll_pos)
+                    return
+        self.cmd.bell()
+
+    def set_scrolling_region(self):
+        self.cmd.set_scrolling_region(self.screen_size, 0, self.num_lines - 2)
+
+    def scroll_lines(self, amt=1):
+        new_pos = max(0, min(self.scroll_pos + amt, self.max_scroll_pos))
+        if new_pos == self.scroll_pos:
+            self.cmd.bell()
+            return
+        if abs(new_pos - self.scroll_pos) >= self.num_lines - 1:
+            self.scroll_pos = new_pos
+            self.draw_screen()
+            return
+        self.enforce_cursor_state()
+        self.cmd.scroll_screen(amt)
+        self.scroll_pos = new_pos
+        if amt < 0:
+            self.cmd.set_cursor_position(0, 0)
+            self.draw_lines(-amt)
+        else:
+            self.cmd.set_cursor_position(0, self.num_lines - amt)
+            self.draw_lines(amt, self.num_lines - amt)
+        self.draw_status_line()
+
+    def init_terminal_state(self):
+        self.cmd.set_line_wrapping(False)
+        self.cmd.set_window_title(main.title)
+        self.cmd.set_default_colors(
+            fg=self.opts.foreground, bg=self.opts.background,
+            cursor=self.opts.foreground, select_fg=self.opts.select_fg,
+            select_bg=self.opts.select_bg)
+        self.cmd.set_cursor_shape('bar')
+
+    def finalize(self):
+        self.cmd.set_default_colors()
+        self.cmd.set_cursor_visible(True)
+        self.cmd.set_scrolling_region()
+
+    def initialize(self):
+        self.init_terminal_state()
+        self.set_scrolling_region()
+        self.draw_screen()
+        self.create_collection()
+
+    def enforce_cursor_state(self):
+        self.cmd.set_cursor_visible(self.state == COMMAND)
+
+    def draw_lines(self, num, offset=0):
+        offset += self.scroll_pos
+        image_involved = False
+        limit = len(self.diff_lines)
+        for i in range(num):
+            lpos = offset + i
+            if lpos >= limit:
+                text = ''
+            else:
+                line = self.diff_lines[lpos]
+                text = line.text
+                if line.image_data is not None:
+                    image_involved = True
+            self.write('\r\x1b[K' + text + '\x1b[0m')
+            if self.current_search is not None:
+                self.current_search.highlight_line(self.write, lpos)
+            if i < num - 1:
+                self.write('\n')
+        if image_involved:
+            self.place_images()
+
+    def place_images(self):
+        self.cmd.clear_images_on_screen()
+        offset = self.scroll_pos
+        limit = len(self.diff_lines)
+        in_image = False
+        for row in range(self.num_lines):
+            lpos = offset + row
+            if lpos >= limit:
+                break
+            line = self.diff_lines[lpos]
+            if in_image:
+                if line.image_data is None:
+                    in_image = False
+                continue
+            if line.image_data is not None:
+                left_placement, right_placement = line.image_data
+                if left_placement is not None:
+                    self.place_image(row, left_placement, True)
+                    in_image = True
+                if right_placement is not None:
+                    self.place_image(row, right_placement, False)
+                    in_image = True
+
+    def place_image(self, row, placement, is_left):
+        xpos = (0 if is_left else (self.screen_size.cols // 2)) + placement.image.margin_size
+        image_height_in_rows = placement.image.rows
+        topmost_visible_row = placement.row
+        num_visible_rows = image_height_in_rows - topmost_visible_row
+        visible_frac = min(num_visible_rows / image_height_in_rows, 1)
+        if visible_frac > 0:
+            height = int(visible_frac * placement.image.height)
+            top = placement.image.height - height
+            self.image_manager.show_image(placement.image.image_id, xpos, row, src_rect=(
+                0, top, placement.image.width, height))
+
+    def draw_screen(self):
+        self.enforce_cursor_state()
+        if self.state < DIFFED:
+            self.cmd.clear_screen()
+            self.write(_('Calculating diff, please wait...'))
+            return
+        self.cmd.clear_images_on_screen()
+        self.cmd.set_cursor_position(0, 0)
+        self.draw_lines(self.num_lines)
+        self.draw_status_line()
+
+    def draw_status_line(self):
+        if self.state < DIFFED:
+            return
+        self.enforce_cursor_state()
+        self.cmd.set_cursor_position(0, self.num_lines)
+        self.cmd.clear_to_eol()
+        if self.state is COMMAND:
+            self.line_edit.write(self.write)
+        elif self.state is MESSAGE:
+            self.cmd.styled(self.message, reverse=True)
+        else:
+            sp = '{:.0%}'.format(self.scroll_pos/self.max_scroll_pos) if self.scroll_pos and self.max_scroll_pos else '0%'
+            scroll_frac = styled(sp, fg=self.opts.margin_fg)
+            if self.current_search is None:
+                counts = '{}{}{}'.format(
+                        styled(str(self.added_count), fg=self.opts.highlight_added_bg),
+                        styled(',', fg=self.opts.margin_fg),
+                        styled(str(self.removed_count), fg=self.opts.highlight_removed_bg)
+                )
+            else:
+                counts = styled('{} matches'.format(len(self.current_search)), fg=self.opts.margin_fg)
+            suffix = counts + '  ' + scroll_frac
+            prefix = styled(':', fg=self.opts.margin_fg)
+            filler = self.screen_size.cols - wcswidth(prefix) - wcswidth(suffix)
+            text = '{}{}{}'.format(prefix, ' ' * filler, suffix)
+            self.write(text)
+
+    def change_context_count(self, new_ctx):
+        new_ctx = max(0, new_ctx)
+        if new_ctx != self.current_context_count:
+            self.current_context_count = new_ctx
+            self.state = COLLECTED
+            self.generate_diff()
+            self.restore_position = self.current_position
+            self.draw_screen()
+
+    def start_search(self, is_regex, is_backward):
+        if self.state != DIFFED:
+            self.cmd.bell()
+            return
+        self.state = COMMAND
+        self.line_edit.clear()
+        self.line_edit.add_text('?' if is_backward else '/')
+        self.current_search_is_regex = is_regex
+        self.draw_status_line()
+
+    def do_search(self):
+        self.current_search = None
+        query = self.line_edit.current_input
+        if len(query) < 2:
+            return
+        try:
+            self.current_search = Search(self.opts, query[1:], self.current_search_is_regex, query[0] == '?')
+        except BadRegex:
+            self.state = MESSAGE
+            self.message = sanitize(_('Bad regex: {}').format(query[1:]))
+            self.cmd.bell()
+        else:
+            if self.current_search(self.diff_lines, self.margin_size, self.screen_size.cols):
+                self.scroll_to_next_match(include_current=True)
+            else:
+                self.state = MESSAGE
+                self.message = sanitize(_('No matches found'))
+                self.cmd.bell()
+
+    def on_text(self, text, in_bracketed_paste=False):
+        if self.state is COMMAND:
+            self.line_edit.on_text(text, in_bracketed_paste)
+            self.draw_status_line()
+            return
+        if self.state is MESSAGE:
+            self.state = DIFFED
+            self.draw_status_line()
+            return
+        action = self.shortcut_action(text)
+        if action is not None:
+            return self.perform_action(action)
+
+    def on_key(self, key_event):
+        if self.state is MESSAGE:
+            if key_event.type is not RELEASE:
+                self.state = DIFFED
+                self.draw_status_line()
+            return
+        if self.state is COMMAND:
+            if self.line_edit.on_key(key_event):
+                if not self.line_edit.current_input:
+                    self.state = DIFFED
+                self.draw_status_line()
+                return
+        if key_event.type is RELEASE:
+            return
+        if self.state is COMMAND:
+            if key_event.key is ESCAPE:
+                self.state = DIFFED
+                self.draw_status_line()
+                return
+            if key_event is enter_key:
+                self.state = DIFFED
+                self.do_search()
+                self.line_edit.clear()
+                self.draw_screen()
+                return
+        if self.state >= DIFFED and self.current_search is not None and key_event.key is ESCAPE:
+            self.current_search = None
+            self.draw_screen()
+            return
+        action = self.shortcut_action(key_event)
+        if action is not None:
+            return self.perform_action(action)
+
+    def on_resize(self, screen_size):
+        self.screen_size = screen_size
+        self.set_scrolling_region()
+        if self.state > COLLECTED:
+            self.image_manager.delete_all_sent_images()
+            self.render_diff()
+        self.draw_screen()
+
+    def on_interrupt(self):
+        self.quit_loop(1)
+
+    def on_eot(self):
+        self.quit_loop(1)
+
+
+OPTIONS = partial('''\
+--context
+type=int
+default=-1
+Number of lines of context to show between changes. Negative values
+use the number set in diff.conf
+
+
+--config
+type=list
+{config_help}
+
+
+--override -o
+type=list
+Override individual configuration options, can be specified multiple times.
+Syntax: :italic:`name=value`. For example: :italic:`-o background=gray`
+
+'''.format, config_help=CONFIG_HELP.format(conf_name='diff', appname=appname))
+
+
+def showwarning(message, category, filename, lineno, file=None, line=None):
+    if category is ImageSupportWarning:
+        showwarning.warnings.append(message)
+
+
+showwarning.warnings = []
+help_text = 'Show a side-by-side diff of the specified files/directories'
+usage = 'file_or_directory_left file_or_directory_right'
+
+
+def terminate_processes(processes):
+    for pid in processes:
+        try:
+            os.kill(pid, signal.SIGKILL)
+        except Exception:
+            pass
+
+
+def main(args):
+    warnings.showwarning = showwarning
+    args, items = parse_args(args[1:], OPTIONS, usage, help_text, 'kitty +kitten diff')
+    if len(items) != 2:
+        raise SystemExit('You must specify exactly two files/directories to compare')
+    left, right = items
+    main.title = _('{} vs. {}').format(left, right)
+    if os.path.isdir(left) != os.path.isdir(right):
+        raise SystemExit('The items to be diffed should both be either directories or files. Comparing a directory to a file is not valid.')
+    opts = init_config(args)
+    set_diff_command(opts.diff_cmd)
+    lines_for_path.replace_tab_by = opts.replace_tab_by
+    for f in left, right:
+        if not os.path.exists(f):
+            raise SystemExit('{} does not exist'.format(f))
+
+    loop = Loop()
+    handler = DiffHandler(args, opts, left, right)
+    loop.loop(handler)
+    for message in showwarning.warnings:
+        from kitty.utils import safe_print
+        safe_print(message, file=sys.stderr)
+    highlight_processes = getattr(highlight_collection, 'processes', ())
+    terminate_processes(tuple(highlight_processes))
+    terminate_processes(tuple(worker_processes))
+    if loop.return_code != 0:
+        if handler.report_traceback_on_exit:
+            print(handler.report_traceback_on_exit, file=sys.stderr)
+            input('Press Enter to quit.')
+        raise SystemExit(loop.return_code)
+
+
+if __name__ == '__main__':
+    main(sys.argv)
+elif __name__ == '__doc__':
+    sys.cli_docs['usage'] = usage
+    sys.cli_docs['options'] = OPTIONS
+    sys.cli_docs['help_text'] = help_text
+elif __name__ == '__conf__':
+    from .config import all_options
+    sys.all_options = all_options

kittens/diff/patch.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import concurrent.futures
+import os
+import shlex
+import shutil
+import subprocess
+
+from .collect import lines_for_path
+from .diff_speedup import changed_center
+
+left_lines = right_lines = None
+GIT_DIFF = 'git diff --no-color --no-ext-diff --exit-code -U_CONTEXT_ --no-index --'
+DIFF_DIFF = 'diff -p -U _CONTEXT_ --'
+worker_processes = []
+
+
+def find_differ():
+    if shutil.which('git'):
+        return GIT_DIFF
+    if shutil.which('diff'):
+        return DIFF_DIFF
+
+
+def set_diff_command(opt):
+    if opt == 'auto':
+        cmd = find_differ()
+        if cmd is None:
+            raise SystemExit('Failed to find either the git or diff programs on your system')
+    else:
+        cmd = opt
+    set_diff_command.cmd = cmd
+
+
+def run_diff(file1, file2, context=3):
+    # returns: ok, is_different, patch
+    cmd = shlex.split(set_diff_command.cmd.replace('_CONTEXT_', str(context)))
+    # we resolve symlinks because git diff does not follow symlinks, while diff
+    # does. We want consistent behavior, also for integration with git difftool
+    # we always want symlinks to be followed.
+    path1, path2 = map(os.path.realpath, (file1, file2))
+    p = subprocess.Popen(
+            cmd + [path1, path2],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.DEVNULL)
+    worker_processes.append(p.pid)
+    stdout, stderr = p.communicate()
+    returncode = p.wait()
+    worker_processes.remove(p.pid)
+    if returncode in (0, 1):
+        return True, returncode == 1, stdout.decode('utf-8')
+    return False, returncode, stderr.decode('utf-8')
+
+
+class Chunk:
+
+    __slots__ = ('is_context', 'left_start', 'right_start', 'left_count', 'right_count', 'centers')
+
+    def __init__(self, left_start, right_start, is_context=False):
+        self.is_context = is_context
+        self.left_start = left_start
+        self.right_start = right_start
+        self.left_count = self.right_count = 0
+        self.centers = None
+
+    def add_line(self):
+        self.right_count += 1
+
+    def remove_line(self):
+        self.left_count += 1
+
+    def context_line(self):
+        self.left_count += 1
+        self.right_count += 1
+
+    def finalize(self):
+        if not self.is_context and self.left_count == self.right_count:
+            self.centers = tuple(
+                changed_center(left_lines[self.left_start + i], right_lines[self.right_start + i])
+                for i in range(self.left_count)
+            )
+
+    def __repr__(self):
+        return 'Chunk(is_context={}, left_start={}, left_count={}, right_start={}, right_count={})'.format(
+                self.is_context, self.left_start, self.left_count, self.right_start, self.right_count)
+
+
+class Hunk:
+
+    def __init__(self, title, left, right):
+        self.left_start, self.left_count = left
+        self.right_start, self.right_count = right
+        self.left_start -= 1  # 0-index
+        self.right_start -= 1  # 0-index
+        self.title = title
+        self.added_count = self.removed_count = 0
+        self.chunks = []
+        self.current_chunk = None
+        self.largest_line_number = max(self.left_start + self.left_count, self.right_start + self.right_count)
+
+    def new_chunk(self, is_context=False):
+        if self.chunks:
+            c = self.chunks[-1]
+            left_start = c.left_start + c.left_count
+            right_start = c.right_start + c.right_count
+        else:
+            left_start = self.left_start
+            right_start = self.right_start
+        return Chunk(left_start, right_start, is_context)
+
+    def ensure_diff_chunk(self):
+        if self.current_chunk is None:
+            self.current_chunk = self.new_chunk(is_context=False)
+        elif self.current_chunk.is_context:
+            self.chunks.append(self.current_chunk)
+            self.current_chunk = self.new_chunk(is_context=False)
+
+    def ensure_context_chunk(self):
+        if self.current_chunk is None:
+            self.current_chunk = self.new_chunk(is_context=True)
+        elif not self.current_chunk.is_context:
+            self.chunks.append(self.current_chunk)
+            self.current_chunk = self.new_chunk(is_context=True)
+
+    def add_line(self):
+        self.ensure_diff_chunk()
+        self.current_chunk.add_line()
+        self.added_count += 1
+
+    def remove_line(self):
+        self.ensure_diff_chunk()
+        self.current_chunk.remove_line()
+        self.removed_count += 1
+
+    def context_line(self):
+        self.ensure_context_chunk()
+        self.current_chunk.context_line()
+
+    def finalize(self):
+        self.chunks.append(self.current_chunk)
+        del self.current_chunk
+        # Sanity check
+        c = self.chunks[-1]
+        if c.left_start + c.left_count != self.left_start + self.left_count:
+            raise ValueError('Left side line mismatch {} != {}'.format(c.left_start + c.left_count, self.left_start + self.left_count))
+        if c.right_start + c.right_count != self.right_start + self.right_count:
+            raise ValueError('Left side line mismatch {} != {}'.format(c.right_start + c.right_count, self.right_start + self.right_count))
+        for c in self.chunks:
+            c.finalize()
+
+
+def parse_range(x):
+    parts = x[1:].split(',', 1)
+    start = abs(int(parts[0]))
+    count = 1 if len(parts) < 2 else int(parts[1])
+    return start, count
+
+
+def parse_hunk_header(line):
+    parts = tuple(filter(None, line.split('@@', 2)))
+    linespec = parts[0].strip()
+    title = ''
+    if len(parts) == 2:
+        title = parts[1].strip()
+    left, right = map(parse_range, linespec.split())
+    return Hunk(title, left, right)
+
+
+class Patch:
+
+    def __init__(self, all_hunks):
+        self.all_hunks = all_hunks
+        self.largest_line_number = self.all_hunks[-1].largest_line_number if self.all_hunks else 0
+        self.added_count = sum(h.added_count for h in all_hunks)
+        self.removed_count = sum(h.removed_count for h in all_hunks)
+
+    def __iter__(self):
+        return iter(self.all_hunks)
+
+    def __len__(self):
+        return len(self.all_hunks)
+
+
+def parse_patch(raw):
+    all_hunks = []
+    current_hunk = None
+    for line in raw.splitlines():
+        if line.startswith('@@ '):
+            current_hunk = parse_hunk_header(line)
+            all_hunks.append(current_hunk)
+        else:
+            if current_hunk is None:
+                continue
+            q = line[0]
+            if q == '+':
+                current_hunk.add_line()
+            elif q == '-':
+                current_hunk.remove_line()
+            elif q == '\\':
+                continue
+            else:
+                current_hunk.context_line()
+    for h in all_hunks:
+        h.finalize()
+    return Patch(all_hunks)
+
+
+class Differ:
+
+    diff_executor = None
+
+    def __init__(self):
+        self.jmap = {}
+        self.jobs = []
+        if Differ.diff_executor is None:
+            Differ.diff_executor = self.diff_executor = concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count())
+
+    def add_diff(self, file1, file2):
+        self.jmap[file1] = file2
+        self.jobs.append(file1)
+
+    def __call__(self, context=3):
+        global left_lines, right_lines
+        ans = {}
+        executor = Differ.diff_executor
+        jobs = {executor.submit(run_diff, key, self.jmap[key], context): key for key in self.jobs}
+        for future in concurrent.futures.as_completed(jobs):
+            key = jobs[future]
+            left_path, right_path = key, self.jmap[key]
+            try:
+                ok, returncode, output = future.result()
+            except FileNotFoundError as err:
+                return 'Could not find the {} executable. Is it in your PATH?'.format(err.filename)
+            except Exception as e:
+                return 'Running git diff for {} vs. {} generated an exception: {}'.format(left_path, right_path, e)
+            if not ok:
+                return output + '\nRunning git diff for {} vs. {} failed'.format(left_path, right_path)
+            left_lines = lines_for_path(left_path)
+            right_lines = lines_for_path(right_path)
+            try:
+                patch = parse_patch(output)
+            except Exception:
+                import traceback
+                return traceback.format_exc() + '\nParsing diff for {} vs. {} failed'.format(left_path, right_path)
+            else:
+                ans[key] = patch
+        return ans

kittens/diff/render.py

@@ -0,0 +1,490 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import warnings
+from gettext import gettext as _
+from itertools import repeat, zip_longest
+from math import ceil
+
+from kitty.fast_data_types import truncate_point_for_length, wcswidth
+
+from ..tui.images import can_display_images
+from .collect import (
+    Segment, data_for_path, highlights_for_path, is_image, lines_for_path,
+    path_name_map, sanitize
+)
+from .config import formats
+from .diff_speedup import split_with_highlights as _split_with_highlights
+
+
+class ImageSupportWarning(Warning):
+    pass
+
+
+def images_supported():
+    ans = getattr(images_supported, 'ans', None)
+    if ans is None:
+        images_supported.ans = ans = can_display_images()
+        if not ans:
+            warnings.warn('ImageMagick not found images cannot be displayed', ImageSupportWarning)
+    return ans
+
+
+class Ref:
+
+    def __setattr__(self, name, value):
+        raise AttributeError("can't set attribute")
+
+    def __repr__(self):
+        return '{}({})'.format(self.__class__.__name__, ', '.join(
+            '{}={}'.format(n, getattr(self, n)) for n in self.__slots__ if n != '_hash'))
+
+
+class LineRef(Ref):
+
+    __slots__ = ('src_line_number', 'wrapped_line_idx')
+
+    def __init__(self, sln, wli=0):
+        object.__setattr__(self, 'src_line_number', sln)
+        object.__setattr__(self, 'wrapped_line_idx', wli)
+
+
+class Reference(Ref):
+
+    __slots__ = ('path', 'extra')
+
+    def __init__(self, path, extra=None):
+        object.__setattr__(self, 'path', path)
+        object.__setattr__(self, 'extra', extra)
+
+
+class Line:
+
+    __slots__ = ('text', 'ref', 'is_change_start', 'image_data')
+
+    def __init__(self, text, ref, change_start=False, image_data=None):
+        self.text = text
+        self.ref = ref
+        self.is_change_start = change_start
+        self.image_data = image_data
+
+
+def yield_lines_from(iterator, reference, is_change_start=True):
+    for text in iterator:
+        yield Line(text, reference, is_change_start)
+        is_change_start = False
+
+
+def human_readable(size, sep=' '):
+    """ Convert a size in bytes into a human readable form """
+    divisor, suffix = 1, "B"
+    for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
+        if size < (1 << ((i + 1) * 10)):
+            divisor, suffix = (1 << (i * 10)), candidate
+            break
+    size = str(float(size)/divisor)
+    if size.find(".") > -1:
+        size = size[:size.find(".")+2]
+    if size.endswith('.0'):
+        size = size[:-2]
+    return size + sep + suffix
+
+
+def fit_in(text, count):
+    p = truncate_point_for_length(text, count)
+    if p >= len(text):
+        return text
+    if count > 1:
+        p = truncate_point_for_length(text, count - 1)
+    return text[:p] + '…'
+
+
+def fill_in(text, sz):
+    w = wcswidth(text)
+    if w < sz:
+        text += ' ' * (sz - w)
+    return text
+
+
+def place_in(text, sz):
+    return fill_in(fit_in(text, sz), sz)
+
+
+def format_func(which):
+    def formatted(text):
+        fmt = formats[which]
+        return '\x1b[' + fmt + 'm' + text + '\x1b[0m'
+    formatted.__name__ = which + '_format'
+    return formatted
+
+
+text_format = format_func('text')
+title_format = format_func('title')
+margin_format = format_func('margin')
+added_format = format_func('added')
+removed_format = format_func('removed')
+removed_margin_format = format_func('removed_margin')
+added_margin_format = format_func('added_margin')
+filler_format = format_func('filler')
+hunk_margin_format = format_func('hunk_margin')
+hunk_format = format_func('hunk')
+highlight_map = {'remove': ('removed_highlight', 'removed'), 'add': ('added_highlight', 'added')}
+
+
+def highlight_boundaries(ltype):
+    s, e = highlight_map[ltype]
+    start = '\x1b[' + formats[s] + 'm'
+    stop = '\x1b[' + formats[e] + 'm'
+    return start, stop
+
+
+def title_lines(left_path, right_path, args, columns, margin_size):
+    m = ' ' * margin_size
+    left_name, right_name = map(path_name_map.get, (left_path, right_path))
+    if right_name and right_name != left_name:
+        n1 = fit_in(m + sanitize(left_name), columns // 2 - margin_size)
+        n1 = place_in(n1, columns // 2)
+        n2 = fit_in(m + sanitize(right_name), columns // 2 - margin_size)
+        n2 = place_in(n2, columns // 2)
+        name = n1 + n2
+    else:
+        name = place_in(m + sanitize(left_name), columns)
+    yield title_format(place_in(name, columns))
+    yield title_format('━' * columns)
+
+
+def binary_lines(path, other_path, columns, margin_size):
+    template = _('Binary file: {}')
+    available_cols = columns // 2 - margin_size
+
+    def fl(path, fmt):
+        text = template.format(human_readable(len(data_for_path(path))))
+        text = place_in(text, available_cols)
+        return margin_format(' ' * margin_size) + fmt(text)
+
+    if path is None:
+        filler = render_diff_line('', '', 'filler', margin_size, available_cols)
+        yield filler + fl(other_path, added_format)
+    elif other_path is None:
+        filler = render_diff_line('', '', 'filler', margin_size, available_cols)
+        yield fl(path, removed_format) + filler
+    else:
+        yield fl(path, removed_format) + fl(other_path, added_format)
+
+
+def split_to_size(line, width):
+    if not line:
+        yield line
+    while line:
+        p = truncate_point_for_length(line, width)
+        yield line[:p]
+        line = line[p:]
+
+
+def truncate_points(line, width):
+    pos = 0
+    sz = len(line)
+    while True:
+        pos = truncate_point_for_length(line, width, pos)
+        if pos < sz:
+            yield pos
+        else:
+            break
+
+
+def split_with_highlights(line, width, highlights, bg_highlight=None):
+    truncate_pts = list(truncate_points(line, width))
+    return _split_with_highlights(line, truncate_pts, highlights, bg_highlight)
+
+
+margin_bg_map = {'filler': filler_format, 'remove': removed_margin_format, 'add': added_margin_format, 'context': margin_format}
+text_bg_map = {'filler': filler_format, 'remove': removed_format, 'add': added_format, 'context': text_format}
+
+
+class DiffData:
+
+    def __init__(self, left_path, right_path, available_cols, margin_size):
+        self.left_path, self.right_path = left_path, right_path
+        self.available_cols = available_cols
+        self.margin_size = margin_size
+        self.left_lines, self.right_lines = map(lines_for_path, (left_path, right_path))
+        self.filler_line = render_diff_line('', '', 'filler', margin_size, available_cols)
+        self.left_filler_line = render_diff_line('', '', 'remove', margin_size, available_cols)
+        self.right_filler_line = render_diff_line('', '', 'add', margin_size, available_cols)
+        self.left_hdata = highlights_for_path(left_path)
+        self.right_hdata = highlights_for_path(right_path)
+
+    def left_highlights_for_line(self, line_num):
+        if line_num < len(self.left_hdata):
+            return self.left_hdata[line_num]
+        return []
+
+    def right_highlights_for_line(self, line_num):
+        if line_num < len(self.right_hdata):
+            return self.right_hdata[line_num]
+        return []
+
+
+def render_diff_line(number, text, ltype, margin_size, available_cols):
+    margin = margin_bg_map[ltype](place_in(number, margin_size))
+    content = text_bg_map[ltype](fill_in(text or '', available_cols))
+    return margin + content
+
+
+def render_diff_pair(left_line_number, left, left_is_change, right_line_number, right, right_is_change, is_first, margin_size, available_cols):
+    ltype = 'filler' if left_line_number is None else ('remove' if left_is_change else 'context')
+    rtype = 'filler' if right_line_number is None else ('add' if right_is_change else 'context')
+    return (
+            render_diff_line(left_line_number if is_first else None, left, ltype, margin_size, available_cols) +
+            render_diff_line(right_line_number if is_first else None, right, rtype, margin_size, available_cols)
+    )
+
+
+def hunk_title(hunk_num, hunk, margin_size, available_cols):
+    m = hunk_margin_format(' ' * margin_size)
+    t = '@@ -{},{} +{},{} @@ {}'.format(hunk.left_start + 1, hunk.left_count, hunk.right_start + 1, hunk.right_count, hunk.title)
+    return m + hunk_format(place_in(t, available_cols))
+
+
+def render_half_line(line_number, line, highlights, ltype, margin_size, available_cols, changed_center=None):
+    bg_highlight = None
+    if changed_center is not None and changed_center[0]:
+        prefix_count, suffix_count = changed_center
+        line_sz = len(line)
+        if prefix_count + suffix_count < line_sz:
+            start, stop = highlight_boundaries(ltype)
+            seg = Segment(prefix_count, start)
+            seg.end = line_sz - suffix_count
+            seg.end_code = stop
+            bg_highlight = seg
+    if highlights or bg_highlight:
+        lines = split_with_highlights(line, available_cols, highlights, bg_highlight)
+    else:
+        lines = split_to_size(line, available_cols)
+    line_number = str(line_number + 1)
+    for line in lines:
+        yield render_diff_line(line_number, line, ltype, margin_size, available_cols)
+        line_number = ''
+
+
+def lines_for_chunk(data, hunk_num, chunk, chunk_num):
+    if chunk.is_context:
+        for i in range(chunk.left_count):
+            left_line_number = line_ref = chunk.left_start + i
+            right_line_number = chunk.right_start + i
+            highlights = data.left_highlights_for_line(left_line_number)
+            if highlights:
+                lines = split_with_highlights(data.left_lines[left_line_number], data.available_cols, highlights)
+            else:
+                lines = split_to_size(data.left_lines[left_line_number], data.available_cols)
+            left_line_number = str(left_line_number + 1)
+            right_line_number = str(right_line_number + 1)
+            for wli, text in enumerate(lines):
+                line = render_diff_line(left_line_number, text, 'context', data.margin_size, data.available_cols)
+                if right_line_number == left_line_number:
+                    r = line
+                else:
+                    r = render_diff_line(right_line_number, text, 'context', data.margin_size, data.available_cols)
+                ref = Reference(data.left_path, LineRef(line_ref, wli))
+                yield Line(line + r, ref)
+                left_line_number = right_line_number = ''
+    else:
+        common = min(chunk.left_count, chunk.right_count)
+        for i in range(max(chunk.left_count, chunk.right_count)):
+            ll, rl = [], []
+            if i < chunk.left_count:
+                rln = ref_ln = chunk.left_start + i
+                ll.extend(render_half_line(
+                    rln, data.left_lines[rln], data.left_highlights_for_line(rln),
+                    'remove', data.margin_size, data.available_cols,
+                    None if chunk.centers is None else chunk.centers[i]))
+                ref_path = data.left_path
+            if i < chunk.right_count:
+                rln = ref_ln = chunk.right_start + i
+                rl.extend(render_half_line(
+                    rln, data.right_lines[rln], data.right_highlights_for_line(rln),
+                    'add', data.margin_size, data.available_cols,
+                    None if chunk.centers is None else chunk.centers[i]))
+                ref_path = data.right_path
+            if i < common:
+                extra = len(ll) - len(rl)
+                if extra != 0:
+                    if extra < 0:
+                        x, fl = ll, data.left_filler_line
+                        extra = -extra
+                    else:
+                        x, fl = rl, data.right_filler_line
+                    x.extend(repeat(fl, extra))
+            else:
+                if ll:
+                    x, count = rl, len(ll)
+                else:
+                    x, count = ll, len(rl)
+                x.extend(repeat(data.filler_line, count))
+            for wli, (left_line, right_line) in enumerate(zip(ll, rl)):
+                ref = Reference(ref_path, LineRef(ref_ln, wli))
+                yield Line(left_line + right_line, ref, i == 0 and wli == 0)
+
+
+def lines_for_diff(left_path, right_path, hunks, args, columns, margin_size):
+    available_cols = columns // 2 - margin_size
+    data = DiffData(left_path, right_path, available_cols, margin_size)
+
+    for hunk_num, hunk in enumerate(hunks):
+        yield Line(hunk_title(hunk_num, hunk, margin_size, columns - margin_size), Reference(left_path, LineRef(hunk.left_start)))
+        for cnum, chunk in enumerate(hunk.chunks):
+            yield from lines_for_chunk(data, hunk_num, chunk, cnum)
+
+
+def all_lines(path, args, columns, margin_size, is_add=True):
+    available_cols = columns // 2 - margin_size
+    ltype = 'add' if is_add else 'remove'
+    lines = lines_for_path(path)
+    filler = render_diff_line('', '', 'filler', margin_size, available_cols)
+    msg_written = False
+    hdata = highlights_for_path(path)
+
+    def highlights(num):
+        return hdata[num] if num < len(hdata) else []
+
+    for line_number, line in enumerate(lines):
+        h = render_half_line(line_number, line, highlights(line_number), ltype, margin_size, available_cols)
+        for i, hl in enumerate(h):
+            ref = Reference(path, LineRef(line_number, i))
+            empty = filler
+            if not msg_written:
+                msg_written = True
+                empty = render_diff_line(
+                        '', _('This file was added') if is_add else _('This file was removed'),
+                        'filler', margin_size, available_cols)
+            text = (empty + hl) if is_add else (hl + empty)
+            yield Line(text, ref, line_number == 0 and i == 0)
+
+
+def rename_lines(path, other_path, args, columns, margin_size):
+    m = ' ' * margin_size
+    for line in split_to_size(_('The file {0} was renamed to {1}').format(
+            sanitize(path_name_map[path]), sanitize(path_name_map[other_path])), columns - margin_size):
+        yield m + line
+
+
+class Image:
+
+    def __init__(self, image_id, width, height, margin_size, screen_size):
+        self.image_id = image_id
+        self.width, self.height = width, height
+        self.rows = int(ceil(self.height / screen_size.cell_height))
+        self.columns = int(ceil(self.width / screen_size.cell_width))
+        self.margin_size = margin_size
+
+
+class ImagePlacement:
+
+    def __init__(self, image, row):
+        self.image = image
+        self.row = row
+
+
+def render_image(path, is_left, available_cols, margin_size, image_manager):
+    lnum = 0
+    margin_fmt = removed_margin_format if is_left else added_margin_format
+    m = margin_fmt(' ' * margin_size)
+    fmt = removed_format if is_left else added_format
+
+    def yield_split(text):
+        nonlocal lnum
+        for i, line in enumerate(split_to_size(text, available_cols)):
+            yield m + fmt(place_in(line, available_cols)), Reference(path, LineRef(lnum, i)), None
+        lnum += 1
+
+    try:
+        image_id, width, height = image_manager.send_image(path, available_cols - margin_size, image_manager.screen_size.rows - 2)
+    except Exception as e:
+        yield from yield_split(_('Failed to render image, with error:'))
+        yield from yield_split(' '.join(str(e).splitlines()))
+        return
+    meta = _('Dimensions: {0}x{1} pixels Size: {2}').format(
+            width, height, human_readable(len(data_for_path(path))))
+    yield from yield_split(meta)
+    bg_line = m + fmt(' ' * available_cols)
+    img = Image(image_id, width, height, margin_size, image_manager.screen_size)
+    for r in range(img.rows):
+        yield bg_line, Reference(path, LineRef(lnum)), ImagePlacement(img, r)
+        lnum += 1
+
+
+def image_lines(left_path, right_path, columns, margin_size, image_manager):
+    available_cols = columns // 2 - margin_size
+    left_lines, right_lines = iter(()), iter(())
+    if left_path is not None:
+        left_lines = render_image(left_path, True, available_cols, margin_size, image_manager)
+    if right_path is not None:
+        right_lines = render_image(right_path, False, available_cols, margin_size, image_manager)
+    filler = ' ' * (available_cols + margin_size)
+    is_change_start = True
+    for left, right in zip_longest(left_lines, right_lines):
+        left_placement = right_placement = None
+        if left is None:
+            left = filler
+            right, ref, right_placement = right
+        elif right is None:
+            right = filler
+            left, ref, left_placement = left
+        else:
+            right, ref, right_placement = right
+            left, ref, left_placement = left
+        image_data = (left_placement, right_placement) if left_placement or right_placement else None
+        yield Line(left + right, ref, is_change_start, image_data)
+        is_change_start = False
+
+
+def render_diff(collection, diff_map, args, columns, image_manager):
+    largest_line_number = 0
+    for path, item_type, other_path in collection:
+        if item_type == 'diff':
+            patch = diff_map.get(path)
+            if patch is not None:
+                largest_line_number = max(largest_line_number, patch.largest_line_number)
+
+    margin_size = render_diff.margin_size = max(3, len(str(largest_line_number)) + 1)
+    last_item_num = len(collection) - 1
+
+    for i, (path, item_type, other_path) in enumerate(collection):
+        item_ref = Reference(path)
+        is_binary = isinstance(data_for_path(path), bytes)
+        if not is_binary and item_type == 'diff' and isinstance(data_for_path(other_path), bytes):
+            is_binary = True
+        is_img = is_binary and (is_image(path) or is_image(other_path)) and images_supported()
+        yield from yield_lines_from(title_lines(path, other_path, args, columns, margin_size), item_ref, False)
+        if item_type == 'diff':
+            if is_binary:
+                if is_img:
+                    ans = image_lines(path, other_path, columns, margin_size, image_manager)
+                else:
+                    ans = yield_lines_from(binary_lines(path, other_path, columns, margin_size), item_ref)
+            else:
+                ans = lines_for_diff(path, other_path, diff_map[path], args, columns, margin_size)
+        elif item_type == 'add':
+            if is_binary:
+                if is_img:
+                    ans = image_lines(None, path, columns, margin_size, image_manager)
+                else:
+                    ans = yield_lines_from(binary_lines(None, path, columns, margin_size), item_ref)
+            else:
+                ans = all_lines(path, args, columns, margin_size, is_add=True)
+        elif item_type == 'removal':
+            if is_binary:
+                if is_img:
+                    ans = image_lines(path, None, columns, margin_size, image_manager)
+                else:
+                    ans = yield_lines_from(binary_lines(path, None, columns, margin_size), item_ref)
+            else:
+                ans = all_lines(path, args, columns, margin_size, is_add=False)
+        elif item_type == 'rename':
+            ans = yield_lines_from(rename_lines(path, other_path, args, columns, margin_size), item_ref)
+        else:
+            raise ValueError('Unsupported item type: {}'.format(item_type))
+        yield from ans
+        if i < last_item_num:
+            yield Line('', item_ref)

kittens/diff/search.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import re
+
+from kitty.fast_data_types import wcswidth
+
+from ..tui.operations import styled
+
+
+class BadRegex(ValueError):
+    pass
+
+
+class Search:
+
+    def __init__(self, opts, query, is_regex, is_backward):
+        self.matches = {}
+        self.count = 0
+        self.style = styled('|', fg=opts.search_fg, bg=opts.search_bg).split('|', 1)[0]
+        if not is_regex:
+            query = re.escape(query)
+        try:
+            self.pat = re.compile(query, flags=re.UNICODE | re.IGNORECASE)
+        except Exception:
+            raise BadRegex('Not a valid regex: {}'.format(query))
+
+    def __call__(self, diff_lines, margin_size, cols):
+        self.matches = {}
+        self.count = 0
+        half_width = cols // 2
+        strip_pat = re.compile('\033[[].*?m')
+        right_offset = half_width + 1 + margin_size
+        find = self.pat.finditer
+        for i, line in enumerate(diff_lines):
+            text = strip_pat.sub('', line.text)
+            left, right = text[margin_size:half_width + 1], text[right_offset:]
+            matches = []
+
+            def add(which, offset):
+                for m in find(which):
+                    before = which[:m.start()]
+                    matches.append((wcswidth(before) + offset, m.group()))
+                    self.count += 1
+
+            add(left, margin_size)
+            add(right, right_offset)
+            if matches:
+                self.matches[i] = matches
+        return bool(self.matches)
+
+    def __contains__(self, i):
+        return i in self.matches
+
+    def __len__(self):
+        return self.count
+
+    def highlight_line(self, write, line_num):
+        highlights = self.matches.get(line_num)
+        if not highlights:
+            return False
+        write(self.style)
+        for start, text in highlights:
+            write('\r\x1b[{}C{}'.format(start, text))
+        write('\x1b[m')
+        return True

kittens/diff/speedup.c

@@ -0,0 +1,179 @@
+/*
+ * speedup.c
+ * Copyright (C) 2018 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "data-types.h"
+
+static PyObject*
+changed_center(PyObject *self UNUSED, PyObject *args) {
+    unsigned int prefix_count = 0, suffix_count = 0;
+    PyObject *lp, *rp;
+    if (!PyArg_ParseTuple(args, "UU", &lp, &rp)) return NULL;
+    const size_t left_len = PyUnicode_GET_LENGTH(lp), right_len = PyUnicode_GET_LENGTH(rp);
+
+#define R(which, index) PyUnicode_READ(PyUnicode_KIND(which), PyUnicode_DATA(which), index)
+    while(prefix_count < MIN(left_len, right_len)) {
+        if (R(lp, prefix_count) != R(rp, prefix_count)) break;
+        prefix_count++;
+    }
+    if (left_len && right_len && prefix_count < MIN(left_len, right_len)) {
+        while(suffix_count < MIN(left_len - prefix_count, right_len - prefix_count)) {
+            if(R(lp, left_len - 1 - suffix_count) != R(rp, right_len - 1 - suffix_count)) break;
+            suffix_count++;
+        }
+    }
+#undef R
+    return Py_BuildValue("II", prefix_count, suffix_count);
+}
+
+typedef struct {
+    unsigned int start_pos, end_pos, current_pos;
+    PyObject *start_code, *end_code;
+} Segment;
+
+typedef struct {
+    Segment sg;
+    unsigned int num, pos;
+} SegmentPointer;
+
+static const Segment EMPTY_SEGMENT = { .current_pos = UINT_MAX };
+
+static inline bool
+convert_segment(PyObject *highlight, Segment *dest) {
+    PyObject *val = NULL;
+#define I
+#define A(x, d, c) { \
+    val = PyObject_GetAttrString(highlight, #x); \
+    if (val == NULL) return false; \
+    dest->d = c(val); Py_DECREF(val); \
+}
+    A(start, start_pos, PyLong_AsUnsignedLong);
+    A(end, end_pos, PyLong_AsUnsignedLong);
+    dest->current_pos = dest->start_pos;
+    A(start_code, start_code, I);
+    A(end_code, end_code, I);
+    if (!PyUnicode_Check(dest->start_code)) { PyErr_SetString(PyExc_TypeError, "start_code is not a string"); return false; }
+    if (!PyUnicode_Check(dest->end_code)) { PyErr_SetString(PyExc_TypeError, "end_code is not a string"); return false; }
+#undef A
+#undef I
+    return true;
+}
+
+static inline bool
+next_segment(SegmentPointer *s, PyObject *highlights) {
+    if (s->pos < s->num) {
+        if (!convert_segment(PyList_GET_ITEM(highlights, s->pos), &s->sg)) return false;
+        s->pos++;
+    } else s->sg.current_pos = UINT_MAX;
+    return true;
+}
+
+static inline bool
+insert_code(PyObject *code, Py_UCS4 *buf, size_t bufsz, unsigned int *buf_pos) {
+    unsigned int csz = PyUnicode_GET_LENGTH(code);
+    if (*buf_pos + csz >= bufsz) return false;
+    for (unsigned int s = 0; s < csz; s++) buf[(*buf_pos)++] = PyUnicode_READ(PyUnicode_KIND(code), PyUnicode_DATA(code), s);
+    return true;
+}
+
+static inline bool
+add_line(Segment *bg_segment, Segment *fg_segment, Py_UCS4 *buf, size_t bufsz, unsigned int *buf_pos, PyObject *ans) {
+    bool bg_is_active = bg_segment->current_pos == bg_segment->end_pos, fg_is_active = fg_segment->current_pos == fg_segment->end_pos;
+    if (bg_is_active) { if(!insert_code(bg_segment->end_code, buf, bufsz, buf_pos)) return false; }
+    if (fg_is_active) { if(!insert_code(fg_segment->end_code, buf, bufsz, buf_pos)) return false; }
+    PyObject *wl = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, *buf_pos);
+    if (!wl) return false;
+    int ret = PyList_Append(ans, wl); Py_DECREF(wl); if (ret != 0) return false;
+    *buf_pos = 0;
+    if (bg_is_active) { if(!insert_code(bg_segment->start_code, buf, bufsz, buf_pos)) return false; }
+    if (fg_is_active) { if(!insert_code(fg_segment->start_code, buf, bufsz, buf_pos)) return false; }
+    return true;
+}
+
+static PyObject*
+split_with_highlights(PyObject *self UNUSED, PyObject *args) {
+    PyObject *line, *truncate_points_py, *fg_highlights, *bg_highlight;
+    if (!PyArg_ParseTuple(args, "UO!O!O", &line, &PyList_Type, &truncate_points_py, &PyList_Type, &fg_highlights, &bg_highlight)) return NULL;
+    PyObject *ans = PyList_New(0);
+    if (!ans) return NULL;
+    static unsigned int truncate_points[256];
+    unsigned int num_truncate_pts = PyList_GET_SIZE(truncate_points_py), truncate_pos = 0, truncate_point;
+    for (unsigned int i = 0; i < MIN(num_truncate_pts, arraysz(truncate_points)); i++) {
+        truncate_points[i] = PyLong_AsUnsignedLong(PyList_GET_ITEM(truncate_points_py, i));
+    }
+    SegmentPointer fg_segment = { .sg = EMPTY_SEGMENT, .num = PyList_GET_SIZE(fg_highlights)}, bg_segment = { .sg = EMPTY_SEGMENT };
+    if (bg_highlight != Py_None) { if (!convert_segment(bg_highlight, &bg_segment.sg)) { Py_CLEAR(ans); return NULL; }; bg_segment.num = 1; }
+#define CHECK_CALL(func, ...) if (!func(__VA_ARGS__)) { Py_CLEAR(ans); if (!PyErr_Occurred()) PyErr_SetString(PyExc_ValueError, "line too long"); return NULL; }
+    CHECK_CALL(next_segment, &fg_segment, fg_highlights);
+
+#define NEXT_TRUNCATE_POINT truncate_point = (truncate_pos < num_truncate_pts) ? truncate_points[truncate_pos++] : UINT_MAX
+    NEXT_TRUNCATE_POINT;
+
+#define INSERT_CODE(x) { CHECK_CALL(insert_code, x, buf, arraysz(buf), &buf_pos); }
+
+#define ADD_LINE CHECK_CALL(add_line, &bg_segment.sg, &fg_segment.sg, buf, arraysz(buf), &buf_pos, ans);
+
+#define ADD_CHAR(x) { \
+    buf[buf_pos++] = x; \
+    if (buf_pos >= arraysz(buf)) { Py_CLEAR(ans); PyErr_SetString(PyExc_ValueError, "line too long"); return NULL; } \
+}
+#define CHECK_SEGMENT(sgp, is_fg) { \
+    if (i == sgp.sg.current_pos) { \
+        INSERT_CODE(sgp.sg.current_pos == sgp.sg.start_pos ? sgp.sg.start_code : sgp.sg.end_code); \
+        if (sgp.sg.current_pos == sgp.sg.start_pos) sgp.sg.current_pos = sgp.sg.end_pos; \
+        else { \
+            if (is_fg) { \
+                CHECK_CALL(next_segment, &fg_segment, fg_highlights); \
+                if (sgp.sg.current_pos == i) { \
+                    INSERT_CODE(sgp.sg.start_code); \
+                    sgp.sg.current_pos = sgp.sg.end_pos; \
+                } \
+            } else sgp.sg.current_pos = UINT_MAX; \
+        } \
+    }\
+}
+
+    const unsigned int line_sz = PyUnicode_GET_LENGTH(line);
+    static Py_UCS4 buf[4096];
+    unsigned int i = 0, buf_pos = 0;
+    for (; i < line_sz; i++) {
+        if (i == truncate_point) { ADD_LINE; NEXT_TRUNCATE_POINT; }
+        CHECK_SEGMENT(bg_segment, false);
+        CHECK_SEGMENT(fg_segment, true)
+        ADD_CHAR(PyUnicode_READ(PyUnicode_KIND(line), PyUnicode_DATA(line), i));
+    }
+    if (buf_pos) ADD_LINE;
+    return ans;
+#undef INSERT_CODE
+#undef CHECK_SEGMENT
+#undef CHECK_CALL
+#undef ADD_CHAR
+#undef ADD_LINE
+#undef NEXT_TRUNCATE_POINT
+}
+
+static PyMethodDef module_methods[] = {
+    {"changed_center", (PyCFunction)changed_center, METH_VARARGS, ""},
+    {"split_with_highlights", (PyCFunction)split_with_highlights, METH_VARARGS, ""},
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+static struct PyModuleDef module = {
+   .m_base = PyModuleDef_HEAD_INIT,
+   .m_name = "diff_speedup",   /* name of module */
+   .m_doc = NULL,
+   .m_size = -1,
+   .m_methods = module_methods
+};
+
+EXPORTED PyMODINIT_FUNC
+PyInit_diff_speedup(void) {
+    PyObject *m;
+
+    m = PyModule_Create(&module);
+    if (m == NULL) return NULL;
+    return m;
+}

kittens/hints/main.py

@@ -6,24 +6,25 @@ import os
 import re
 import string
 import sys
-from functools import lru_cache, partial
+from functools import lru_cache
 from gettext import gettext as _
+from itertools import repeat
 
 from kitty.cli import parse_args
 from kitty.fast_data_types import set_clipboard_string
 from kitty.key_encoding import ESCAPE, backspace_key, enter_key
+from kitty.utils import screen_size_function
 
 from ..tui.handler import Handler
 from ..tui.loop import Loop
-from ..tui.operations import (
-    clear_screen, faint, set_cursor_visible, set_window_title, styled
-)
+from ..tui.operations import faint, styled
 
 URL_PREFIXES = 'http https file ftp'.split()
 HINT_ALPHABET = string.digits + string.ascii_lowercase
+screen_size = screen_size_function()
 
 
-class Mark(object):
+class Mark:
 
     __slots__ = ('index', 'start', 'end', 'text')
 
@@ -46,62 +47,56 @@ def decode_hint(x):
     return int(x, 36)
 
 
-def render(lines, current_input):
-    ans = []
+def highlight_mark(m, text, current_input):
+    hint = encode_hint(m.index)
+    if current_input and not hint.startswith(current_input):
+        return faint(text)
+    hint = hint[len(current_input):] or ' '
+    text = text[len(hint):]
+    return styled(
+        hint,
+        fg='black',
+        bg='green',
+        bold=True
+    ) + styled(
+        text, fg='gray', fg_intense=True, bold=True
+    )
 
-    def mark(m):
-        hint = encode_hint(m.index)
-        text = m.text
-        if current_input and not hint.startswith(current_input):
-            return faint(text)
-        hint = hint[len(current_input):] or ' '
-        text = text[len(hint):]
-        return styled(
-            hint,
-            fg='black',
-            bg='green',
-            bold=True
-        ) + styled(
-            text, fg='gray', fg_intense=True, bold=True
-        )
 
-    for line, marks in lines:
-        if not marks:
-            ans.append(faint(line))
+def render(text, current_input, all_marks, ignore_mark_indices):
+    for mark in reversed(all_marks):
+        if mark.index in ignore_mark_indices:
             continue
-        buf = []
+        mtext = highlight_mark(mark, text[mark.start:mark.end], current_input)
+        text = text[:mark.start] + mtext + text[mark.end:]
 
-        for i, m in enumerate(marks):
-            if i == 0 and m.start:
-                buf.append(faint(line[:m.start]))
-            buf.append(mark(m))
-            if m is not marks[-1]:
-                buf.append(faint(line[m.end:marks[i + 1].start]))
+    text = text.replace('\0', '')
 
-        rest = line[marks[-1].end:]
-        if rest:
-            buf.append(faint(rest))
-
-        ans.append(''.join(buf))
-    return '\r\n'.join(ans)
+    return text.replace('\n', '\r\n').rstrip()
 
 
 class Hints(Handler):
 
-    def __init__(self, lines, index_map, args):
-        self.lines, self.index_map = tuple(lines), index_map
-        self.current_input = ''
-        self.current_text = None
+    def __init__(self, text, all_marks, index_map, args):
+        self.text, self.index_map = text, index_map
+        self.all_marks = all_marks
+        self.ignore_mark_indices = set()
         self.args = args
         self.window_title = _('Choose URL') if args.type == 'url' else _('Choose text')
-        self.chosen = None
+        self.multiple = args.multiple
+        self.chosen = []
+        self.reset()
+
+    def reset(self):
+        self.current_input = ''
+        self.current_text = None
 
     def init_terminal_state(self):
-        self.write(set_cursor_visible(False))
-        self.write(set_window_title(self.window_title))
+        self.cmd.set_cursor_visible(False)
+        self.cmd.set_window_title(self.window_title)
+        self.cmd.set_line_wrapping(False)
 
-    def initialize(self, *args):
-        Handler.initialize(self, *args)
+    def initialize(self):
         self.init_terminal_state()
         self.draw_screen()
 
@@ -113,13 +108,17 @@ class Hints(Handler):
                 changed = True
         if changed:
             matches = [
-                m.text for idx, m in self.index_map.items()
+                m for idx, m in self.index_map.items()
                 if encode_hint(idx).startswith(self.current_input)
             ]
             if len(matches) == 1:
-                self.chosen = matches[0]
-                self.quit_loop(0)
-                return
+                self.chosen.append(matches[0].text)
+                if self.multiple:
+                    self.ignore_mark_indices.add(matches[0].index)
+                    self.reset()
+                else:
+                    self.quit_loop(0)
+                    return
             self.current_text = None
             self.draw_screen()
 
@@ -129,11 +128,22 @@ class Hints(Handler):
             self.current_text = None
             self.draw_screen()
         elif key_event is enter_key and self.current_input:
-            idx = decode_hint(self.current_input)
-            self.chosen = self.index_map[idx].text
-            self.quit_loop(0)
+            try:
+                idx = decode_hint(self.current_input)
+                self.chosen.append(self.index_map[idx].text)
+                self.ignore_mark_indices.add(idx)
+            except Exception:
+                self.current_input = ''
+                self.current_text = None
+                self.draw_screen()
+            else:
+                if self.multiple:
+                    self.reset()
+                    self.draw_screen()
+                else:
+                    self.quit_loop(0)
         elif key_event.key is ESCAPE:
-            self.quit_loop(1)
+            self.quit_loop(0 if self.multiple else 1)
 
     def on_interrupt(self):
         self.quit_loop(1)
@@ -142,49 +152,90 @@ class Hints(Handler):
         self.quit_loop(1)
 
     def on_resize(self, new_size):
-        Handler.on_resize(self, new_size)
         self.draw_screen()
 
     def draw_screen(self):
         if self.current_text is None:
-            self.current_text = render(self.lines, self.current_input)
-        self.write(clear_screen())
+            self.current_text = render(self.text, self.current_input, self.all_marks, self.ignore_mark_indices)
+        self.cmd.clear_screen()
         self.write(self.current_text)
 
 
-def regex_finditer(pat, minimum_match_length, line):
-    for m in pat.finditer(line):
+def regex_finditer(pat, minimum_match_length, text):
+    for m in pat.finditer(text):
         s, e = m.span(pat.groups)
+        while e > s + 1 and text[e-1] == '\0':
+            e -= 1
         if e - s >= minimum_match_length:
             yield s, e
 
 
-def find_urls(pat, line):
-    for m in pat.finditer(line):
-        s, e = m.span()
-        if s > 4 and line[s - 5:s] == 'link:':  # asciidoc URLs
-            url = line[s:e]
-            idx = url.rfind('[')
-            if idx > -1:
-                e -= len(url) - idx
-        while line[e - 1] in '.,?!' and e > 1:  # remove trailing punctuation
+closing_bracket_map = {'(': ')', '[': ']', '{': '}', '<': '>', '*': '*', '"': '"', "'": "'"}
+opening_brackets = ''.join(closing_bracket_map)
+postprocessor_map = {}
+
+
+def postprocessor(func):
+    postprocessor_map[func.__name__] = func
+    return func
+
+
+@postprocessor
+def url(text, s, e):
+    if s > 4 and text[s - 5:s] == 'link:':  # asciidoc URLs
+        url = text[s:e]
+        idx = url.rfind('[')
+        if idx > -1:
+            e -= len(url) - idx
+    while text[e - 1] in '.,?!' and e > 1:  # remove trailing punctuation
+        e -= 1
+    # truncate url at closing bracket/quote
+    if s > 0 and e <= len(text) and text[s-1] in opening_brackets:
+        q = closing_bracket_map[text[s-1]]
+        idx = text.find(q, s)
+        if idx > s:
+            e = idx
+    # Restructured Text URLs
+    if e > 3 and text[e-2:e] == '`_':
+        e -= 2
+
+    return s, e
+
+
+@postprocessor
+def brackets(text, s, e):
+    # Remove matching brackets
+    if e > s and e <= len(text):
+        before = text[s]
+        if before in '({[<' and text[e-1] == closing_bracket_map[before]:
+            s += 1
             e -= 1
-        yield s, e
+    return s, e
 
 
-def mark(finditer, line, all_marks):
-    marks = []
-    for s, e in finditer(line):
-        idx = len(all_marks)
-        text = line[s:e]
-        marks.append(Mark(idx, s, e, text))
-        all_marks.append(marks[-1])
-    return line, marks
+@postprocessor
+def quotes(text, s, e):
+    # Remove matching quotes
+    if e > s and e <= len(text):
+        before = text[s]
+        if before in '\'"' and text[e-1] == before:
+            s += 1
+            e -= 1
+    return s, e
 
 
-def run_loop(args, lines, index_map):
+def mark(pattern, post_processors, text, args):
+    pat = re.compile(pattern)
+    for idx, (s, e) in enumerate(regex_finditer(pat, args.minimum_match_length, text)):
+        for func in post_processors:
+            s, e = func(text, s, e)
+        mark_text = text[s:e].replace('\n', '').replace('\0', '')
+        yield Mark(idx, s, e, mark_text)
+
+
+def run_loop(args, text, all_marks, index_map):
     loop = Loop()
-    handler = Hints(lines, index_map, args)
+    handler = Hints(text, all_marks, index_map, args)
     loop.loop(handler)
     if handler.chosen and loop.return_code == 0:
         return {'match': handler.chosen, 'program': args.program}
@@ -195,64 +246,100 @@ def escape(chars):
     return chars.replace('\\', '\\\\').replace('-', r'\-').replace(']', r'\]')
 
 
-def run(args, text):
+def functions_for(args):
+    post_processors = []
     if args.type == 'url':
         from .url_regex import url_delimiters
-        url_pat = '(?:{})://[^{}]{{3,}}'.format(
+        pattern = '(?:{})://[^{}]{{3,}}'.format(
             '|'.join(args.url_prefixes.split(',')), url_delimiters
         )
-        finditer = partial(find_urls, re.compile(url_pat))
+        post_processors.append(url)
     elif args.type == 'path':
-        finditer = partial(regex_finditer, re.compile(r'(?:\S*/\S+)|(?:\S+[.][a-zA-Z0-9]{2,5})'), args.minimum_match_length)
+        pattern = r'(?:\S*/\S+)|(?:\S+[.][a-zA-Z0-9]{2,7})'
+        post_processors.extend((brackets, quotes))
     elif args.type == 'line':
-        finditer = partial(regex_finditer, re.compile(r'(?m)^\s*(.+)\s*$'), args.minimum_match_length)
+        pattern = '(?m)^\\s*(.+)[\\s\0]*$'
+    elif args.type == 'hash':
+        pattern = '[0-9a-f]{7,128}'
     elif args.type == 'word':
         chars = args.word_characters
         if chars is None:
             import json
             chars = json.loads(os.environ['KITTY_COMMON_OPTS'])['select_by_word_characters']
-        pat = re.compile('(?u)[{}\w]{{{},}}'.format(escape(chars), args.minimum_match_length))
-        finditer = partial(regex_finditer, pat, args.minimum_match_length)
+        pattern = r'(?u)[{}\w]{{{},}}'.format(escape(chars), args.minimum_match_length)
+        post_processors.extend((brackets, quotes))
     else:
-        finditer = partial(regex_finditer, re.compile(args.regex), args.minimum_match_length)
+        pattern = args.regex
+    return pattern, post_processors
+
+
+def convert_text(text, cols):
     lines = []
-    all_marks = []
-    for line in text.splitlines():
-        marked = mark(finditer, line, all_marks)
-        lines.append(marked)
-    if not all_marks:
-        input(_('No {} found, press Enter to abort.').format(
-            'URLs' if args.type == 'url' else 'matches'
-            ))
-        return
-
-    largest_index = all_marks[-1].index
-    for m in all_marks:
-        m.index = largest_index - m.index
-    index_map = {m.index: m for m in all_marks}
-
-    return run_loop(args, lines, index_map)
+    empty_line = '\0' * cols
+    for full_line in text.split('\n'):
+        if full_line:
+            if not full_line.rstrip('\r'):  # empty lines
+                lines.extend(repeat(empty_line, len(full_line)))
+                continue
+            for line in full_line.split('\r'):
+                if line:
+                    lines.append(line.ljust(cols, '\0'))
+    return '\n'.join(lines)
 
 
-OPTIONS = partial(r'''
+def parse_input(text):
+    try:
+        cols = int(os.environ['OVERLAID_WINDOW_COLS'])
+    except KeyError:
+        cols = screen_size().cols
+    return convert_text(text, cols)
+
+
+def run(args, text):
+    try:
+        pattern, post_processors = functions_for(args)
+        text = parse_input(text)
+        all_marks = tuple(mark(pattern, post_processors, text, args))
+        if not all_marks:
+            input(_('No {} found, press Enter to quit.').format(
+                'URLs' if args.type == 'url' else 'matches'
+                ))
+            return
+
+        largest_index = all_marks[-1].index
+        for m in all_marks:
+            m.index = largest_index - m.index
+        index_map = {m.index: m for m in all_marks}
+    except Exception:
+        import traceback
+        traceback.print_exc()
+        input('Press Enter to quit.')
+        raise SystemExit(1)
+
+    return run_loop(args, text, all_marks, index_map)
+
+
+# CLI {{{
+OPTIONS = r'''
 --program
 default=default
 What program to use to open matched text. Defaults to the default open program
-for the operating system.  Use a value of - to paste the match into the
-terminal window instead. A value of @ will copy the match to the clipboard.
+for the operating system.  Use a value of :file:`-` to paste the match into the
+terminal window instead. A value of :file:`@` will copy the match to the clipboard.
 
 
 --type
 default=url
-choices=url,regex,path,line,word
+choices=url,regex,path,line,hash,word
 The type of text to search for.
 
 
 --regex
 default=(?m)^\s*(.+)\s*$
-The regular expression to use when --type=regex.  If you specify a group in the
-regular expression only the group will be matched. This allow you to match text
-ignoring a prefix/suffix, as needed. The default expression matches lines.
+The regular expression to use when :option:`kitty +kitten hints --type`=regex.
+If you specify a group in the regular expression only the group
+will be matched. This allow you to match text ignoring a prefix/suffix, as
+needed. The default expression matches lines.
 
 
 --url-prefixes
@@ -261,7 +348,7 @@ Comma separated list of recognized URL prefixes.
 
 
 --word-characters
-Characters to consider as part of a word. In addition, all chacraters marked as
+Characters to consider as part of a word. In addition, all characters marked as
 alpha-numeric in the unicode database will be considered as word characters.
 Defaults to the select_by_word_characters setting from kitty.conf.
 
@@ -270,11 +357,22 @@ Defaults to the select_by_word_characters setting from kitty.conf.
 default=3
 type=int
 The minimum number of characters to consider a match.
-'''.format, ','.join(sorted(URL_PREFIXES)))
+
+
+--multiple
+type=bool-set
+Select multiple matches and perform the action on all of them together at the end.
+In this mode, press :kbd:`Esc` to finish selecting.
+'''.format(','.join(sorted(URL_PREFIXES))).format
+help_text = 'Select text from the screen using the keyboard. Defaults to searching for URLs.'
+usage = ''
+
+
+def parse_hints_args(args):
+    return parse_args(args, OPTIONS, usage, help_text, 'kitty +kitten hints')
 
 
 def main(args):
-    msg = 'Select text from the screen using the keyboard. Defaults to searching for URLs.'
     text = ''
     if sys.stdin.isatty():
         if '--help' not in args and '-h' not in args:
@@ -283,9 +381,9 @@ def main(args):
             return
     else:
         text = sys.stdin.buffer.read().decode('utf-8')
-        sys.stdin = open('/dev/tty')
+        sys.stdin = open(os.ctermid())
     try:
-        args, items = parse_args(args[1:], OPTIONS, '', msg, 'hints')
+        args, items = parse_hints_args(args[1:])
     except SystemExit as e:
         if e.code != 0:
             print(e.args[0], file=sys.stderr)
@@ -300,14 +398,25 @@ def main(args):
 
 def handle_result(args, data, target_window_id, boss):
     program = data['program']
+    matches = tuple(filter(None, data['match']))
     if program == '-':
         w = boss.window_id_map.get(target_window_id)
         if w is not None:
-            w.paste(data['match'])
+            for m in matches:
+                w.paste(m)
     elif program == '@':
-        set_clipboard_string(data['match'])
+        set_clipboard_string(matches[-1])
     else:
-        boss.open_url(data['match'], None if program == 'default' else program)
+        cwd = None
+        w = boss.window_id_map.get(target_window_id)
+        if w is not None:
+            cwd = w.cwd_of_child
+        program = None if program == 'default' else program
+        for m in matches:
+            boss.open_url(m, program, cwd=cwd)
+
+
+handle_result.type_of_input = 'screen'
 
 
 if __name__ == '__main__':
@@ -315,3 +424,8 @@ if __name__ == '__main__':
     ans = main(sys.argv)
     if ans:
         print(ans)
+elif __name__ == '__doc__':
+    sys.cli_docs['usage'] = usage
+    sys.cli_docs['options'] = OPTIONS
+    sys.cli_docs['help_text'] = help_text
+# }}}

kittens/hints/url_regex.py

@@ -1 +1 @@
-url_delimiters = '\x00-\x20\x7f-\xa0\xad\u0600-\u0605\u061c\u06dd\u070f\u08e2\u1680\u180e\u2000-\u200f\u2028-\u202f\u205f-\u2064\u2066-\u206f\u3000\ud800-\uf8ff\ufeff\ufff9-\ufffb\U000110bd\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f\U000f0000-\U000ffffd\U00100000-\U0010fffd'  # noqa
+url_delimiters = '\x00-\x09\x0b-\x20\x7f-\xa0\xad\u0600-\u0605\u061c\u06dd\u070f\u08e2\u1680\u180e\u2000-\u200f\u2028-\u202f\u205f-\u2064\u2066-\u206f\u3000\ud800-\uf8ff\ufeff\ufff9-\ufffb\U000110bd\U000110cd\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f\U000f0000-\U000ffffd\U00100000-\U0010fffd'  # noqa

kittens/icat/main.py

@@ -2,42 +2,27 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
 
-import array
-import codecs
-import fcntl
 import mimetypes
 import os
 import re
 import signal
-import subprocess
 import sys
-import termios
 import zlib
 from base64 import standard_b64encode
 from collections import namedtuple
-from math import ceil, floor
+from math import ceil
 from tempfile import NamedTemporaryFile
 
 from kitty.cli import parse_args
 from kitty.constants import appname
-from kitty.utils import read_with_timeout
-
-try:
-    fsenc = sys.getfilesystemencoding() or 'utf-8'
-    codecs.lookup(fsenc)
-except Exception:
-    fsenc = 'utf-8'
-
-
-class OpenFailed(ValueError):
-
-    def __init__(self, path, message):
-        ValueError.__init__(
-            self, 'Failed to open: {} with error: {}'.format(path, message)
-        )
-        self.path = path
+from kitty.utils import TTYIO, fit_image, screen_size_function
 
+from ..tui.images import (
+    ConvertFailed, NoImageMagick, OpenFailed, convert, fsenc, identify
+)
+from ..tui.operations import clear_images_on_screen, serialize_gr_command
 
+screen_size = None
 OPTIONS = '''\
 --align
 type=choices
@@ -49,14 +34,14 @@ Horizontal alignment for the displayed image.
 --place
 Choose where on the screen to display the image. The image will
 be scaled to fit into the specified rectangle. The syntax for
-specifying rectanges is <|_ width|>x<|_ height|>@<|_ left|>x<|_ top|>. All measurements
-are in cells (i.e. cursor positions) with the origin |_ (0, 0)| at
+specifying rectanges is <:italic:`width`>x<:italic:`height`>@<:italic:`left`>x<:italic:`top`>. All measurements
+are in cells (i.e. cursor positions) with the origin :italic:`(0, 0)` at
 the top-left corner of the screen.
 
 
 --scale-up
 type=bool-set
-When used in combination with |_ --place| it will cause images that
+When used in combination with :option:`--place` it will cause images that
 are smaller than the specified area to be scaled up to use as much
 of the specified area as possible.
 
@@ -71,8 +56,8 @@ type=choices
 choices=detect,file,stream
 default=detect
 Which mechanism to use to transfer images to the terminal. The default is to
-auto-detect. |_ file| means to use a temporary file and |_ stream| means to
-send the data via terminal escape codes. Note that if you use the |_ file|
+auto-detect. :italic:`file` means to use a temporary file and :italic:`stream` means to
+send the data via terminal escape codes. Note that if you use the :italic:`file`
 transfer mode and you are connecting over a remote session then image display
 will not work.
 
@@ -81,7 +66,7 @@ will not work.
 type=bool-set
 Detect support for image display in the terminal. If not supported, will exit
 with exit code 1, otherwise will exit with code 0 and print the supported
-transfer mode to stderr, which can be used with the |_ --transfer-mode| option.
+transfer mode to stderr, which can be used with the :option:`--transfer-mode` option.
 
 
 --detection-timeout
@@ -89,6 +74,13 @@ type=float
 default=10
 The amount of time (in seconds) to wait for a response form the terminal, when
 detecting image display support.
+
+
+--print-window-size
+type=bool-set
+Print out the window size as :italic:`widthxheight` (in pixels) and quit. This is a
+convenience method to query the window size if using kitty icat from a
+scripting language that cannot make termios calls.
 '''
 
 
@@ -100,46 +92,11 @@ def options_spec():
     return options_spec.ans
 
 
-Size = namedtuple('Size', 'rows cols width height')
-
-
-def screen_size():
-    if screen_size.changed:
-        buf = array.array('H', [0, 0, 0, 0])
-        fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, buf)
-        screen_size.ans = Size(*buf)
-        screen_size.changed = False
-    return screen_size.ans
-
-
-screen_size.changed = True
-
-
 def write_gr_cmd(cmd, payload=None):
-    cmd = ','.join('{}={}'.format(k, v) for k, v in cmd.items())
-    w = sys.stdout.buffer.write
-    w(b'\033_G'), w(cmd.encode('ascii'))
-    if payload:
-        w(b';')
-        w(payload)
-    w(b'\033\\')
+    sys.stdout.buffer.write(serialize_gr_command(cmd, payload))
     sys.stdout.flush()
 
 
-def fit_image(width, height, pwidth, pheight):
-    if height > pheight:
-        corrf = pheight / float(height)
-        width, height = floor(corrf * width), pheight
-    if width > pwidth:
-        corrf = pwidth / float(width)
-        width, height = pwidth, floor(corrf * height)
-    if height > pheight:
-        corrf = pheight / float(height)
-        width, height = floor(corrf * width), pheight
-
-    return int(width), int(height)
-
-
 def calculate_in_cell_x_offset(width, cell_width, align):
     if align == 'left':
         return 0
@@ -217,52 +174,6 @@ def show(outfile, width, height, fmt, transmit_mode='t', align='center', place=N
         write_chunked(cmd, data)
 
 
-ImageData = namedtuple('ImageData', 'fmt width height mode')
-
-
-def run_imagemagick(path, cmd, keep_stdout=True):
-    try:
-        p = subprocess.run(cmd, stdout=subprocess.PIPE if keep_stdout else subprocess.DEVNULL, stderr=subprocess.PIPE)
-    except FileNotFoundError:
-        raise SystemExit('ImageMagick is required to cat images')
-    if p.returncode != 0:
-        raise OpenFailed(path, p.stderr.decode('utf-8'))
-    return p
-
-
-def identify(path):
-    p = run_imagemagick(path, ['identify', '-format', '%m %w %h %A', path])
-    parts = tuple(filter(None, p.stdout.decode('utf-8').split()))
-    mode = 'rgb' if parts[3].lower() == 'false' else 'rgba'
-    return ImageData(parts[0].lower(), int(parts[1]), int(parts[2]), mode)
-
-
-def convert(path, m, available_width, available_height, scale_up):
-    width, height = m.width, m.height
-    cmd = ['convert', '-background', 'none', path]
-    if scale_up:
-        if width < available_width:
-            r = available_width / width
-            width, height = available_width, int(height * r)
-    if width > available_width or height > available_height:
-        width, height = fit_image(width, height, available_width, available_height)
-        cmd += ['-resize', '{}x{}'.format(width, height)]
-    with NamedTemporaryFile(prefix='icat-', suffix='.' + m.mode, delete=False) as outfile:
-        run_imagemagick(path, cmd + [outfile.name])
-    # ImageMagick sometimes generated rgba images smaller than the specified
-    # size. See https://github.com/kovidgoyal/kitty/issues/276 for examples
-    sz = os.path.getsize(outfile.name)
-    bytes_per_pixel = 3 if m.mode == 'rgb' else 4
-    expected_size = bytes_per_pixel * width * height
-    if sz < expected_size:
-        missing = expected_size - sz
-        if missing % (bytes_per_pixel * width) != 0:
-            raise SystemExit('ImageMagick failed to convert {} correctly, it generated {} < {} of data'.format(path, sz, expected_size))
-        height -= missing // (bytes_per_pixel * width)
-
-    return outfile.name, width, height
-
-
 def process(path, args):
     m = identify(path)
     ss = screen_size()
@@ -318,7 +229,8 @@ def detect_support(wait_for=10, silent=False):
             f.write(b'abcd'), f.flush()
             write_gr_cmd(dict(a='q', s=1, v=1, i=1), standard_b64encode(b'abcd'))
             write_gr_cmd(dict(a='q', s=1, v=1, i=2, t='f'), standard_b64encode(f.name.encode(fsenc)))
-            read_with_timeout(more_needed, timeout=wait_for)
+            with TTYIO() as io:
+                io.recv(more_needed, timeout=float(wait_for))
     finally:
         if not silent:
             sys.stdout.buffer.write(b'\033[J'), sys.stdout.flush()
@@ -334,18 +246,29 @@ def parse_place(raw):
         return namedtuple('Place', 'width height left top')(w, h, l, t)
 
 
-def main(args=sys.argv):
-    msg = (
+help_text = (
         'A cat like utility to display images in the terminal.'
         ' You can specify multiple image files and/or directories.'
-        ' Directories are scanned recursively for image files.')
-    args, items = parse_args(args[1:], options_spec, 'image-file ...', msg, '{} icat'.format(appname))
+        ' Directories are scanned recursively for image files.'
+)
+usage = 'image-file ...'
 
+
+def main(args=sys.argv):
+    global screen_size
+    args, items = parse_args(args[1:], options_spec, usage, help_text, '{} +kitten icat'.format(appname))
+
+    if args.print_window_size:
+        screen_size_function.ans = None
+        with open(os.ctermid()) as tty:
+            ss = screen_size_function(tty)()
+        print('{}x{}'.format(ss.width, ss.height), end='')
+        raise SystemExit(0)
+
+    if not sys.stdout.isatty():
+        sys.stdout = open(os.ctermid(), 'w')
+    screen_size = screen_size_function()
     signal.signal(signal.SIGWINCH, lambda signum, frame: setattr(screen_size, 'changed', True))
-    if not sys.stdout.isatty() or not sys.stdin.isatty():
-        raise SystemExit(
-            'Must be run in a terminal, stdout and/or stdin is currently not a terminal'
-        )
     if screen_size().width == 0:
         if args.detect_support:
             raise SystemExit(1)
@@ -369,7 +292,7 @@ def main(args=sys.argv):
         detect_support.has_files = args.transfer_mode == 'file'
     errors = []
     if args.clear:
-        write_gr_cmd({'a': 'd'})
+        sys.stdout.buffer.write(clear_images_on_screen(delete_data=True))
         if not items:
             return
     if not items:
@@ -385,6 +308,10 @@ def main(args=sys.argv):
                     process(item, args)
             else:
                 process(item, args)
+        except NoImageMagick as e:
+            raise SystemExit(str(e))
+        except ConvertFailed as e:
+            raise SystemExit(str(e))
         except OpenFailed as e:
             errors.append(e)
     if args.place:
@@ -398,3 +325,7 @@ def main(args=sys.argv):
 
 if __name__ == '__main__':
     main()
+elif __name__ == '__doc__':
+    sys.cli_docs['usage'] = usage
+    sys.cli_docs['options'] = options_spec
+    sys.cli_docs['help_text'] = help_text

kittens/panel/main.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+import shutil
+import subprocess
+import sys
+
+from kitty.cli import parse_args
+from kitty.constants import is_macos, is_wayland
+
+OPTIONS = r'''
+--lines
+type=int
+default=1
+The number of lines shown in the panel (the height of the panel). Applies to horizontal panels.
+
+
+--columns
+type=int
+default=20
+The number of columns shown in the panel (the width of the panel). Applies to vertical panels.
+
+
+--edge
+choices=top,bottom,left,right
+default=top
+Which edge of the screen to place the panel on. Note that some window managers
+(such as i3) do not support placing docked windows on the left and right edges.
+
+
+--config -c
+type=list
+Path to config file to use for kitty when drawing the panel.
+
+
+--override -o
+type=list
+Override individual kitty configuration options, can be specified multiple times.
+Syntax: :italic:`name=value`. For example: :option:`kitty +kitten panel -o` font_size=20
+'''.format
+
+
+args = None
+help_text = 'Use a command line program to draw a GPU accelerated panel on your X11 desktop'
+usage = 'program-to-run'
+
+
+def parse_panel_args(args):
+    return parse_args(args, OPTIONS, usage, help_text, 'kitty +kitten panel')
+
+
+def call_xprop(*cmd, silent=False):
+    cmd = ['xprop'] + list(cmd)
+    try:
+        cp = subprocess.run(cmd, stdout=subprocess.DEVNULL if silent else None)
+    except FileNotFoundError:
+        raise SystemExit('You must have the xprop program installed')
+    if cp.returncode != 0:
+        raise SystemExit(cp.returncode)
+
+
+def create_strut(
+    win_id,
+    left=0, right=0, top=0, bottom=0, left_start_y=0, left_end_y=0,
+    right_start_y=0, right_end_y=0, top_start_x=0, top_end_x=0,
+    bottom_start_x=0, bottom_end_x=0
+):
+    call_xprop(
+            '-id',
+            str(int(win_id)), '-format', '_NET_WM_STRUT_PARTIAL', '32cccccccccccc',
+            '-set', '_NET_WM_STRUT_PARTIAL',
+            '{left},{right},{top},{bottom},'
+            '{left_start_y},{left_end_y},{right_start_y},{right_end_y},'
+            '{top_start_x},{top_end_x},{bottom_start_x},{bottom_end_x}'.format(**locals())
+    )
+
+
+def create_top_strut(win_id, width, height):
+    create_strut(win_id, top=height, top_end_x=width)
+
+
+def create_bottom_strut(win_id, width, height):
+    create_strut(win_id, bottom=height, bottom_end_x=width)
+
+
+def create_left_strut(win_id, width, height):
+    create_strut(win_id, left=width, left_end_y=height)
+
+
+def create_right_strut(win_id, width, height):
+    create_strut(win_id, right=width, right_end_y=height)
+
+
+def setup_x11_window(win_id):
+    call_xprop(
+            '-id', str(win_id), '-format', '_NET_WM_WINDOW_TYPE', '32a',
+            '-set', '_NET_WM_WINDOW_TYPE', '_NET_WM_WINDOW_TYPE_DOCK'
+    )
+    func = globals()['create_{}_strut'.format(args.edge)]
+    func(win_id, initial_window_size_func.width, initial_window_size_func.height)
+
+
+def initial_window_size_func(opts, *a):
+    from kitty.fast_data_types import glfw_primary_monitor_size, set_smallest_allowed_resize
+
+    def initial_window_size(cell_width, cell_height, dpi_x, dpi_y):
+        monitor_width, monitor_height = glfw_primary_monitor_size()
+        if args.edge in {'top', 'bottom'}:
+            h = initial_window_size_func.height = cell_height * args.lines + 1
+            initial_window_size_func.width = monitor_width
+            set_smallest_allowed_resize(100, h)
+        else:
+            w = initial_window_size_func.width = cell_width * args.columns + 1
+            initial_window_size_func.height = monitor_height
+            set_smallest_allowed_resize(w, 100)
+        return initial_window_size_func.width, initial_window_size_func.height
+
+    return initial_window_size
+
+
+def main(sys_args):
+    global args
+    if is_macos or is_wayland:
+        raise SystemExit('Currently the panel kitten is supported only on X11 desktops')
+    if not shutil.which('xprop'):
+        raise SystemExit('The xprop program is required for the panel kitten')
+    args, items = parse_panel_args(sys_args[1:])
+    if not items:
+        raise SystemExit('You must specify the program to run')
+    sys.argv = ['kitty']
+    if args.config:
+        sys.argv.append('--config={}'.format(args.config))
+    for override in args.override:
+        sys.argv.append('--override={}'.format(override))
+    sys.argv.extend(items)
+    from kitty.main import run_app, main
+    run_app.cached_values_name = 'panel'
+    run_app.first_window_callback = setup_x11_window
+    run_app.initial_window_size_func = initial_window_size_func
+    main()
+
+
+if __name__ == '__main__':
+    main(sys.argv)
+elif __name__ == '__doc__':
+    sys.cli_docs['usage'] = usage
+    sys.cli_docs['options'] = OPTIONS
+    sys.cli_docs['help_text'] = help_text

kittens/resize_window/main.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+import sys
+
+from kitty.cli import parse_args
+from kitty.cmds import cmap, parse_subcommand_cli
+from kitty.constants import version
+from kitty.key_encoding import CTRL, ESCAPE, RELEASE, N, S, T, W
+from kitty.remote_control import encode_send, parse_rc_args
+
+from ..tui.handler import Handler
+from ..tui.loop import Loop
+from ..tui.operations import styled
+
+global_opts = None
+
+
+class Resize(Handler):
+
+    print_on_fail = None
+
+    def __init__(self, opts):
+        self.opts = opts
+
+    def initialize(self):
+        global global_opts
+        global_opts = parse_rc_args(['kitty', '@resize-window'])[0]
+        self.original_size = self.screen_size
+        self.cmd.set_cursor_visible(False)
+        self.cmd.set_line_wrapping(False)
+        self.draw_screen()
+
+    def do_window_resize(self, is_decrease=False, is_horizontal=True, reset=False, multiplier=1):
+        resize_window = cmap['resize-window']
+        increment = self.opts.horizontal_increment if is_horizontal else self.opts.vertical_increment
+        increment *= multiplier
+        if is_decrease:
+            increment *= -1
+        axis = 'reset' if reset else ('horizontal' if is_horizontal else 'vertical')
+        cmdline = [resize_window.name, '--self', '--increment={}'.format(increment), '--axis=' + axis]
+        opts, items = parse_subcommand_cli(resize_window, cmdline)
+        payload = resize_window(global_opts, opts, items)
+        send = {'cmd': resize_window.name, 'version': version, 'payload': payload}
+        self.write(encode_send(send))
+
+    def on_kitty_cmd_response(self, response):
+        if not response.get('ok'):
+            err = response['error']
+            if response.get('tb'):
+                err += '\n' + response['tb']
+            self.print_on_fail = err
+            self.quit_loop(1)
+            return
+        res = response.get('data')
+        if res:
+            self.cmd.bell()
+
+    def on_text(self, text, in_bracketed_paste=False):
+        text = text.upper()
+        if text in 'WNTSR':
+            self.do_window_resize(is_decrease=text in 'NS', is_horizontal=text in 'WN', reset=text == 'R')
+        elif text == 'Q':
+            self.quit_loop(0)
+
+    def on_key(self, key_event):
+        if key_event.type is RELEASE:
+            return
+        if key_event.key is ESCAPE:
+            self.quit_loop(0)
+        elif key_event.key in (W, N, T, S) and key_event.mods & CTRL:
+            self.do_window_resize(is_decrease=key_event.key in (N, S), is_horizontal=key_event.key in (W, N), multiplier=2)
+
+    def on_resize(self, new_size):
+        self.draw_screen()
+
+    def draw_screen(self):
+        self.cmd.clear_screen()
+        print = self.print
+        print(styled('Resize this window', bold=True, fg='gray', fg_intense=True))
+        print()
+        print('Press one of the following keys:')
+        print('  {}ider'.format(styled('W', fg='green')))
+        print('  {}arrower'.format(styled('N', fg='green')))
+        print('  {}aller'.format(styled('T', fg='green')))
+        print('  {}horter'.format(styled('S', fg='green')))
+        print('  {}eset'.format(styled('R', fg='red')))
+        print()
+        print('Press {} to quit resize mode'.format(styled('Esc', italic=True)))
+        print('Hold down {} to double step size'.format(styled('Ctrl', italic=True)))
+        print()
+        print(styled('Sizes', bold=True, fg='white', fg_intense=True))
+        print('Original: {} rows {} cols'.format(self.original_size.rows, self.original_size.cols))
+        print('Current:  {} rows {} cols'.format(
+            styled(self.screen_size.rows, fg='magenta'), styled(self.screen_size.cols, fg='magenta')))
+
+
+OPTIONS = r'''
+--horizontal-increment
+default=2
+type=int
+The base horizontal increment.
+
+
+--vertical-increment
+default=2
+type=int
+The base vertical increment.
+'''.format
+
+
+def main(args):
+    msg = 'Resize the current window'
+    try:
+        args, items = parse_args(args[1:], OPTIONS, '', msg, 'resize_window')
+    except SystemExit as e:
+        if e.code != 0:
+            print(e.args[0], file=sys.stderr)
+            input('Press Enter to quit')
+        return
+
+    loop = Loop()
+    handler = Resize(args)
+    loop.loop(handler)
+    if handler.print_on_fail:
+        print(handler.print_on_fail, file=sys.stderr)
+        input('Press Enter to quit')
+    raise SystemExit(loop.return_code)

kittens/runner.py

@@ -12,34 +12,47 @@ aliases = {'url_hints': 'hints'}
 
 
 def resolved_kitten(k):
-    return aliases.get(k, k)
+    return aliases.get(k, k).replace('-', '_')
 
 
 def import_kitten_main_module(config_dir, kitten):
     if kitten.endswith('.py'):
+        path_modified = False
         path = os.path.expanduser(kitten)
         if not os.path.isabs(path):
             path = os.path.join(config_dir, path)
         path = os.path.abspath(path)
         if os.path.dirname(path):
             sys.path.insert(0, os.path.dirname(path))
+            path_modified = True
         with open(path) as f:
             src = f.read()
         code = compile(src, path, 'exec')
         g = {'__name__': 'kitten'}
         exec(code, g)
-        return {'start': g['main'], 'end': g['handle_result']}
-    else:
-        kitten = resolved_kitten(kitten)
-        m = importlib.import_module('kittens.{}.main'.format(kitten))
-        return {'start': m.main, 'end': m.handle_result}
+        hr = g.get('handle_result', lambda *a, **kw: None)
+        if path_modified:
+            del sys.path[0]
+        return {'start': g['main'], 'end': hr}
+
+    kitten = resolved_kitten(kitten)
+    m = importlib.import_module('kittens.{}.main'.format(kitten))
+    return {'start': m.main, 'end': getattr(m, 'handle_result', lambda *a, **k: None)}
 
 
 def create_kitten_handler(kitten, orig_args):
     from kitty.constants import config_dir
     kitten = resolved_kitten(kitten)
     m = import_kitten_main_module(config_dir, kitten)
-    return partial(m['end'], [kitten] + orig_args)
+    ans = partial(m['end'], [kitten] + orig_args)
+    ans.type_of_input = getattr(m['end'], 'type_of_input', None)
+    return ans
+
+
+def set_debug(kitten):
+    from kittens.tui.loop import debug
+    import builtins
+    builtins.debug = debug
 
 
 def launch(args):
@@ -49,20 +62,78 @@ def launch(args):
     args = [kitten] + args
     os.environ['KITTY_CONFIG_DIRECTORY'] = config_dir
     from kittens.tui.operations import clear_screen, reset_mode
+    set_debug(kitten)
     m = import_kitten_main_module(config_dir, kitten)
-    result = m['start'](args)
+    try:
+        result = m['start'](args)
+    finally:
+        sys.stdin = sys.__stdin__
     print(reset_mode('ALTERNATE_SCREEN') + clear_screen(), end='')
     if result is not None:
         import json
-        print('OK:', json.dumps(result))
+        data = json.dumps(result)
+        print('OK:', len(data), data)
     sys.stderr.flush()
     sys.stdout.flush()
 
 
-def run_kitten(kitten):
+def deserialize(output):
+    import json
+    if output.startswith('OK: '):
+        try:
+            prefix, sz, rest = output.split(' ', 2)
+            return json.loads(rest[:int(sz)])
+        except Exception:
+            raise ValueError('Failed to parse kitten output: {!r}'.format(output))
+
+
+def run_kitten(kitten, run_name='__main__'):
     import runpy
     kitten = resolved_kitten(kitten)
-    runpy.run_module('kittens.{}.main'.format(kitten), run_name='__main__')
+    set_debug(kitten)
+    try:
+        runpy.run_module('kittens.{}.main'.format(kitten), run_name=run_name)
+    except ImportError:
+        raise SystemExit('No kitten named {}'.format(kitten))
+
+
+def all_kitten_names():
+    ans = getattr(all_kitten_names, 'ans', None)
+    if ans is None:
+        n = []
+        import glob
+        base = os.path.dirname(os.path.abspath(__file__))
+        for x in glob.glob(os.path.join(base, '*', '__init__.py')):
+            q = os.path.basename(os.path.dirname(x))
+            if q != 'tui':
+                n.append(q)
+        all_kitten_names.ans = ans = frozenset(n)
+    return ans
+
+
+def list_kittens():
+    print('You must specify the name of a kitten to run')
+    print('Choose from:')
+    print()
+    for kitten in all_kitten_names():
+        print(kitten)
+
+
+def get_kitten_cli_docs(kitten):
+    sys.cli_docs = {}
+    run_kitten(kitten, run_name='__doc__')
+    ans = sys.cli_docs
+    del sys.cli_docs
+    if 'help_text' in ans and 'usage' in ans and 'options' in ans:
+        return ans
+
+
+def get_kitten_conf_docs(kitten):
+    sys.all_options = None
+    run_kitten(kitten, run_name='__conf__')
+    ans = sys.all_options
+    del sys.all_options
+    return ans
 
 
 def main():