version 0.4.2

Running python code in signal handlers makes me nervous

.gitignore

@@ -1,10 +1,11 @@
-*.js
-*.pyj-cached
 *.so
-.build-cache
+*.pyc
+*.pyo
 tags
 build
-README.html
 linux-package
 logo/*.iconset
 test-launcher
+kitty-profile
+dev
+__pycache__

.travis.yml

@@ -18,6 +18,9 @@ matrix:
                 - libxrandr-dev
                 - libxinerama-dev
                 - libxcursor-dev
+                - libunistring-dev
+                - libpng-dev
+                - python3-pil
 
         - os: linux
           dist: trusty
@@ -36,6 +39,9 @@ matrix:
                 - libxrandr-dev
                 - libxinerama-dev
                 - libxcursor-dev
+                - libunistring-dev
+                - libpng-dev
+                - python3-pil
 
         - os: linux
           dist: trusty
@@ -54,6 +60,9 @@ matrix:
                 - libxrandr-dev
                 - libxinerama-dev
                 - libxcursor-dev
+                - libunistring-dev
+                - libpng-dev
+                - python3-pil
                 - clang
         - os: osx
           language: generic
@@ -78,6 +87,7 @@ install: |
             brew update;
             brew install python3;
             brew install glfw;
+            brew install libunistring;
         else
             mkdir -p $SW;
             curl https://download.calibre-ebook.com/travis/kitty/osx.tar.xz | tar xJ -C $SW;

CHANGELOG.rst

@@ -0,0 +1,53 @@
+Changelog
+==============
+
+kitty is a feature full, cross-platform, *fast*, OpenGL based terminal emulator.
+
+
+version 0.4.2 [2017-10-23]
+---------------------------
+
+- Fix a regression in 0.4.0 that broke custom key mappings
+
+- Fix a regression in 0.4.0 that broke support for non-QWERTY keyboard layouts 
+
+- Avoid using threads to reap zombie child processes. Also prevent kitty from
+  hanging if the open program hangs when clicking on a URL.
+
+
+version 0.4.0 [2017-10-22]
+---------------------------
+
+- Support for drawing arbitrary raster graphics (images) in the terminal via a
+  new graphics protocol. kitty can draw images with full 32-bit color using both
+  ssh connections and files/shared memory (when available) for better
+  performance. The drawing primitives support alpha blending and z-index.
+  Images can be drawn both above and below text. See
+  https://github.com/kovidgoyal/kitty/blob/master/graphics-protocol.asciidoc
+  for details. 
+
+- Refactor kitty's internals to make it even faster and more efficient. The CPU
+  usage of kitty + X server while doing intensive tasks such as scrolling a
+  file continuously in less has been reduced by 50%. There are now two
+  configuration options ``repaint_delay`` and ``input_delay`` you can use to
+  fine tune kitty's performance vs CPU usage profile. The CPU usage of kitty +
+  X when scrolling in less is now significantly better than most (all?) other
+  terminals. See https://github.com/kovidgoyal/kitty#performance
+
+- Hovering over URLs with the mouse now underlines them to indicate they
+  can be clicked. Hold down Ctrl+Shift while clicking to open the URL.
+
+- Selection using the mouse is now more intelligent. It does not add
+  blank cells (i.e. cells that have no content) after the end of text in a
+  line to the selection.
+
+- The block cursor in now fully opaque but renders the character under it in
+  the background color, for enhanced visibility.
+
+- Allow combining multiple independent actions into a single shortcut
+
+- Add a new shortcut to pass the current selection to an external program
+
+- Allow creating shortcuts to open new windows running arbitrary commands. You
+  can also pass the current selection to the command as an arguments and the
+  contents of the screen + scrollback buffer as stdin to the command.

Makefile

@@ -3,3 +3,6 @@ all:
 
 test:
 	python3 setup.py test
+
+clean:
+	python3 setup.py clean

README.asciidoc

@@ -48,23 +48,25 @@ image::https://travis-ci.org/kovidgoyal/kitty.svg?branch=master[Build status, li
 * Supports tiling multiple terminal windows side by side in different
 layouts without needing to use an extra program like tmux
 
-* Supports all modern terminal features: unicode, true-color, mouse
-protocol, focus tracking, bracketed paste and so on.
+* Supports all modern terminal features: graphics (images), unicode,
+true-color, mouse protocol, focus tracking, bracketed paste and so on.
 
-* Allows you to view the scrollback buffer in a separate window
-using your favorite pager program such as less
-
-* Support startup sessions which allow you to specify the window/tab layout,
+* Supports startup sessions which allow you to specify the window/tab layout,
 working directories and programs to run on startup.
 
+* Uses OpenGL for rendering, offloads rendering to the GPU for lower system
+load and buttery smooth scrolling. Uses threaded rendering to minimize input
+latency.
+
+* Cross-platform support: kitty currently works on Linux and macOS, but because
+it uses only OpenGL for rendering, it should be trivial to port to other
+platforms. 
+
+* Allows you to view the scrollback buffer in a separate window using your
+favorite pager program such as less
+
 * Easily hackable (UI layer written in python, inner loops in C for
-speed). Less than ten thousand lines of code.
-
-* Uses OpenGL for rendering, offloads rendering work to the GPU for lower system load.
-
-* Cross-platform support: kitty currently works on Linux and macOS, but because it
-uses only OpenGL for rendering, it should be trivial to port to
-other platforms. 
+speed). Less than fifteen thousand lines of code.
 
 image::screenshot.png?raw=true[Screenshot, showing three programs in the "Tall" layout]
 
@@ -80,9 +82,14 @@ the following dependencies are installed first.
 
 * python >= 3.5
 * glfw   >= 3.2
+* libunistring 
+* zlib
+* libpng
 * glew   >= 2.0 (not needed on macOS)
 * fontconfig (not needed on macOS)
-* xrdb and xsel (only on X11 based systems)
+* xrdb (only on X11 based systems)
+* xsel (only on X11 systems with glfw < 3.3)
+* ImageMagick (optional, needed to use the `kitty icat` tool to display images in the terminal)
 * gcc or clang (required only for building)
 * pkg-config (required only for building)
 
@@ -123,7 +130,7 @@ or a similar package manager)
 kitty is designed for power keyboard users. To that end all its controls
 work with the keyboard (although it fully supports mouse interactions as
 well). Its configuration is a simple, human editable, single file for
-easy reproducability (I like to store config files in source control).
+easy reproducibility (I like to store config files in source control).
 
 The code in kitty is designed to be simple, modular and hackable. It is
 written in a mix of C (for performance sensitive parts) and Python (for
@@ -292,6 +299,48 @@ come in handy for applications like powerline, without the need to use patched
 fonts. See the various font related configuration directives in the
 link:kitty/kitty.conf[config file].
 
+== Performance
+
+The main goals for kitty performance are user perceived latency while typing
+and "smoothness" while scrolling as well as CPU usage. kitty tries hard to find
+an optimum balance for these. To that end it keeps a cache of each rendered
+glyph in video RAM so that font rendering is not a bottleneck.  Interaction
+with child programs takes place in a separate thread from rendering, to improve
+smoothness.
+
+There are two parameters you can tune to adjust the performance. ``repaint_delay``
+and ``input_delay``. These control the artificial delays introduced into the
+render loop to reduce CPU usage. See the link:kitty/kitty.conf[config file] for details.
+
+You can generate detailed per-function performance data using
+link:https://github.com/gperftools/gperftools[gperftools]. Build kitty with the
+`--profile` flag which will create an executable called `kitty-profile`.  Run
+that and perform the task you want to analyse, for example, scrolling a large 
+file with `less`. After you quit, function call statistics will be printed to
+`stdout` and you can use tools like *kcachegrind* for more detailed analysis.
+
+Here are some CPU usage numbers for the task of scrolling a file continuously in less.
+The CPU usage is for the terminal process and X together and is measured using htop.
+The measurements are taken at the same font and window size for all terminals on a 
+`Intel(R) Core(TM) i7-4820K CPU @ 3.70GHz` CPU with a 
+`Advanced Micro Devices, Inc. [AMD/ATI] Cape Verde XT [Radeon HD 7770/8760 / R7 250X]` GPU.
+
+|===
+| Terminal | CPU usage (X + terminal)
+
+| kitty    | 6 - 8%
+| xterm    | 5 - 7% (but scrolling was extremely janky)
+| termite  | 10 - 13%
+| urxvt    | 12 - 14%
+| gnome-terminal | 15 - 17%
+| konsole  | 29 - 31%
+
+|===
+
+As you can see, kitty uses much less CPU than all terminals, except xterm, but
+its scrolling "smoothness" is much better than that of xterm (at least to my,
+admittedly biased, eyes).
+
 == Note for Linux/macOS packagers
 
 While kitty does use python, it is not a traditional python package, so please do not install it in site-packages.

__main__.py

@@ -2,6 +2,12 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
 
+import sys
+sys.debug_gl = '--debug-kitty-gl' in sys.argv
 
-from kitty.main import main
-main()
+if len(sys.argv) > 1 and sys.argv[1] == 'icat':
+    from kitty.icat import main
+    main(sys.argv[1:])
+else:
+    from kitty.main import main
+    main()

count-lines-of-code

@@ -1,2 +1,2 @@
 #!/bin/bash
-cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/unicode-data.h\nkitty/gl.h\nkitty/glfw.c\nkitty/glfw.h\nkitty/charsets.c\nkitty/key_encoding.py\nkitty/rgb.py') kitty
+cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/glfw.c\nkitty/keys.h\nkitty/charsets.c\nkitty/key_encoding.py\nkitty/rgb.py\nkitty/gl.h') kitty

generate-unicode-data.py

@@ -1,57 +0,0 @@
-#!/usr/bin/env python3
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
-
-import unicodedata
-import itertools
-import sys
-
-IGNORED_CATEGORIES = ('Cc', 'Cf', 'Cs')
-
-
-def ranges(i):
-    for a, b in itertools.groupby(enumerate(i), lambda r: r[1] - r[0]):
-        b = list(b)
-        yield b[0][1], b[-1][1]
-
-
-def generate_data(chars):
-    points, cranges = [], []
-    for l, r in ranges(chars):
-        if l == r:
-            points.append(l)
-        else:
-            cranges.append((l, r))
-    return points, cranges
-
-
-def generate_predicate(name, chars):
-    points, cranges = generate_data(chars)
-    cranges = ['(0x%x %s ch && ch <= 0x%x)' % (l, '<' if l == 0 else '<=', r) for l, r in cranges]
-    points = ['(ch == 0x%x)' % p for p in points]
-    return '''
-static inline bool %s(uint32_t ch) {
-    return %s || %s;
-}
-    ''' % (name, '||'.join(cranges), '||'.join(points))
-
-
-def main():
-    combining_chars = []
-    igchars = []
-    for c in map(chr, range(sys.maxunicode + 1)):
-        if unicodedata.category(c) in IGNORED_CATEGORIES:
-            igchars.append(ord(c))
-        if unicodedata.combining(c):
-            combining_chars.append(ord(c))
-
-    cc = generate_predicate('is_combining_char', combining_chars)
-    ig = generate_predicate('is_ignored_char', igchars)
-    with open('kitty/unicode-data.h', 'w') as f:
-        print('#pragma once', file=f)
-        print(cc, file=f)
-        print(ig, file=f)
-
-
-if __name__ == '__main__':
-    main()

graphics-protocol.asciidoc

@@ -0,0 +1,371 @@
+= The terminal graphics protocol
+:toc:
+:toc-placement!:
+
+The goal of this specification is to create a flexible and performant protocol
+that allows the program running in the terminal, hereafter called the _client_,
+to render arbitrary pixel (raster) graphics to the screen of the terminal
+emulator. The major design goals are
+
+ * Should not require terminal emulators to understand image formats.
+ * Should allow specifying graphics to be drawn at individual pixel positions. 
+ * The graphics should integrate with the text, in particular it should be possible to draw graphics
+   below as well as above the text, with alpha blending. The graphics should also scroll with the text, automatically.
+ * Should use optimizations when the client is running on the same computer as the terminal emulator.
+
+For some discussion regarding the design choices, see link:../../issues/33[#33].
+
+To see a quick demo, inside a kitty terminal run:
+
+```
+kitty icat path/to/some/image.png
+```
+
+You can also see a screenshot with more sophisticated features such as alpha-blending and text over graphics 
+link:https://github.com/kovidgoyal/kitty/issues/33#issuecomment-334436100[here].
+
+toc::[]
+
+== Getting the window size
+
+In order to know what size of images to display and how to position them, the client must be able to get the
+window size in pixels and the number of cells per row and column. This can be done by using the `TIOCGWINSZ` ioctl.
+Some C code to demonstrate its use
+
+```C
+struct ttysize ts; 
+ioctl(0, TIOCGWINSZ, &ts);
+printf("number of columns: %i, number of rows: %i, screen width: %i, screen height: %i\n", sz.ws_col, sz.ws_row, sz.ws_xpixel, sz.ws_ypixel);
+```
+
+Note that some terminals return `0` for the width and height values. Such terminals should be modified to return the correct values.
+Examples of terminals that return correct values: `kitty, xterm`
+
+== The graphics escape code
+
+All graphics escape codes are of the form:
+
+```
+<ESC>_G<control data>;<payload><ESC>\
+```
+
+This is a so-called _Application Programming Command (APC)_. Most terminal
+emulators ignore APC codes, making it safe to use.  
+
+The control data is a comma-separated list of `key=value` pairs.  The payload
+is arbitrary binary data, base64-encoded to prevent interoperation problems
+with legacy terminals that get confused by control codes within an APC code.
+The meaning of the payload is interpreted based on the control data.
+
+The first step is to transmit the actual image data.
+
+== Transferring pixel data
+
+The first consideration when transferring data between the client and the
+terminal emulator is the format in which to do so. Since there is a vast and
+growing number of image formats in existence, it does not make sense to have
+every terminal emulator implement support for them. Instead, the client should
+send simple pixel data to the terminal emulator. The obvious downside to this
+is performance, especially when the client is running on a remote machine.
+Techniques for remedying this limitation are discussed later. The terminal
+emulator must understand pixel data in three formats, 24-bit RGB, 32-bit RGBA and
+PNG. This is specified using the `f` key in the control data. `f=32` (which is the
+default) indicates 32-bit RGBA data and `f=24` indicates 24-bit RGB data and `f=100`
+indicates PNG data. The PNG format is supported for convenience and a compact way
+of transmitting paletted images.
+
+=== RGB and RGBA data
+
+In these formats the pixel data is stored directly as 3 or 4 bytes per pixel, respectively.
+When specifying images in this format, the image dimensions **must** be sent in the control data.
+For example:
+
+```
+<ESC>_Gf=24,s=10,v=20;<payload><ESC>\
+```
+
+Here the width and height are specified using the `s` and `v` keys respectively. Since
+`f=24` there are three bytes per pixel and therefore the pixel data must be `3 * 10 * 20 = 600`
+bytes.
+
+=== PNG data
+
+In this format any PNG image can be transmitted directly.  For example:
+
+```
+<ESC>_Gf=100;<payload><ESC>\
+
+```
+
+The PNG format is specified using the `f=100` key. The width and height of
+the image will be read from the PNG data itself. Note that if you use both PNG and
+compression, then you must provide the `S` key with the size of the PNG data.
+
+
+=== Compression
+
+The client can send compressed image data to the terminal emulator, by specifying the
+`o` key. Currently, only zlib based deflate compression is supported, which is specified using
+`o=z`. For example,
+
+```
+<ESC>_Gf=24,s=10,v=20,o=z;<payload><ESC>\
+```
+
+This is the same as the example from the RGB data section, except that the
+payload is now compressed using deflate. The terminal emulator will decompress
+it before rendering. You can specify compression for any format. The terminal
+emulator will decompress before interpreting the pixel data.
+
+
+=== The transmission medium
+
+The transmission medium is specified using the `t` key. The `t` key defaults to `d`
+and can take the values:
+
+|===
+| Value of `t` | Meaning 
+
+| d | Direct (the data is transmitted within the escape code itself)
+| f | A simple file
+| t | A temporary file, the terminal emulator will delete the file after reading the pixel data
+| s | A http://man7.org/linux/man-pages/man7/shm_overview.7.html[POSIX shared memory object]. The terminal emulator will delete it after reading the pixel data 
+|===
+
+==== Local client
+
+First let us consider the local client techniques (files and shared memory). Some examples:
+
+```
+<ESC>_Gf=100,t=f;<encoded /path/to/file.png><ESC>\
+```
+
+Here we tell the terminal emulator to read PNG data from the specified file of
+the specified size.
+
+```
+<ESC>_Gs=10,v=2,t=s,o=z;<encoded /some-shared-memory-name><ESC>\
+```
+
+Here we tell the terminal emulator to read compressed image data from
+the specified shared memory object.
+
+The client can also specify a size and offset to tell the terminal emulator
+to only read a part of the specified file. The is done using the `S` and `O`
+keys respectively. For example:
+
+```
+<ESC>_Gs=10,v=2,t=s,S=80,O=10;<encoded /some-shared-memory-name><ESC>\
+```
+
+This tells the terminal emulator to read `80` bytes starting from the offset `10`
+inside the specified shared memory buffer.
+
+
+==== Remote client
+
+Remote clients, those that are unable to use the filesystem/shared memory to
+transmit data, must send the pixel data directly using escape codes. Since
+escape codes are of limited maximum length, the data will need to be chunked up
+for transfer. This is done using the `m` key. The pixel data must first be
+base64 encoded then chunked up into chunks no larger than `4096` bytes. The client
+then sends the graphics escape code as usual, with the addition of an `m` key that
+must have the value `1` for all but the last chunk, where it must be `0`. For example,
+if the data is split into three chunks, the client would send the following
+sequence of escape codes to the terminal emulator:
+
+```
+<ESC>_Gs=100,v=30,m=1;<encoded pixel data first chunk><ESC>\                  
+<ESC>_Gm=1;<encoded pixel data second chunk><ESC>\                  
+<ESC>_Gm=0;<encoded pixel data last chunk><ESC>\                  
+```
+
+Note that only the first escape code needs to have the full set of control
+codes such as width, height, format etc. Subsequent chunks must have
+only the `m` key. The client **must** finish sending all chunks for a single image
+before sending any other graphics related escape codes.
+
+
+=== Detecting available transmission mediums
+
+Since a client has no a-priori knowledge of whether it shares a filesystem/shared memory
+with the terminal emulator, it can send an id with the control data, using the `i` key 
+(which can be an arbitrary positive integer up to 4294967295, it must not be zero).
+If it does so, the terminal emulator will reply after trying to load the image, saying
+whether loading was successful or not. For example:
+
+```
+<ESC>_Gi=31,s=10,v=2,t=s;<encoded /some-shared-memory-name><ESC>\
+```
+
+to which the terminal emulator will reply (after trying to load the data):
+
+```
+<ESC>_Gi=31;error message or OK<ESC>\
+```
+
+Here the `i` value will be the same as was sent by the client in the original
+request.  The message data will be a ASCII encoded string containing only
+printable characters and spaces. The string will be `OK` if reading the pixel
+data succeeded or an error message. 
+
+Sometimes, using an id is not appropriate, for example, if you do not want to
+replace a previously sent image with the same id, or if you are sending a dummy
+image and do not want it stored by the terminal emulator. In that case, you can
+use the *query action*, set `a=q`. Then the terminal emulator will try to load
+the image and respond with either OK or an error, as above, but it will not
+replace an existing image with the same id, nor will it store the image.
+
+
+== Display images on screen
+
+Every transmitted image can be displayed an arbitrary number of times on the
+screen, in different locations, using different parts of the source image, as
+needed. You can either simultaneously transmit and display an image using the
+action `a=T`, or first transmit the image with a id, such as `i=10` and then display
+it with `a=p,i=10` which will display the previously transmitted image at the current
+cursor position. When specifying an image id, the terminal emulator will reply with an
+acknowledgement code, which will be either:
+
+```
+<ESC>_Gi=<id>;OK<ESC>\
+```
+
+when the image referred to by id was found, or
+
+```
+<ESC>_Gi=<id>;ENOENT:<some detailed error msg><ESC>\
+```
+
+when the image with the specified id was not found. This is similar to the
+scheme described above for querying available transmission media, except that
+here we are querying if the image with the specified id is available or needs to
+be re-transmitted.
+
+=== Controlling displayed image layout
+
+The image is rendered at the current cursor position, from the upper left corner of
+the current cell. You can also specify extra `X=3` and `Y=4` pixel offsets to display from 
+a different origin within the cell. Note that the offsets must be smaller that the size of the cell.
+
+By default, the entire image will be displayed (images wider than the available
+width will be truncated on the right edge). You can choose a source rectangle (in pixels)
+as the part of the image to display. This is done with the keys: `x, y, w, h` which specify
+the top-left corner, width and height of the source rectangle.
+
+You can also ask the terminal emulator to display the image in a specified rectangle
+(num of columns / num of lines), using the control codes `c,r`. `c` is the number of columns
+and `r` the number of rows. The image will be scaled (enlarged/shrunk) as needed to fit 
+the specified area. Note that if you specify a start cell offset via the `X,Y` keys, it is not
+added to the number of rows/columns.
+
+Finally, you can specify the image *z-index*, i.e. the vertical stacking order. Images
+placed in the same location with different z-index values will be blended if
+they are semi-transparent. You can specify z-index values using the `z` key.
+Negative z-index values mean that the images will be drawn under the text. This
+allows rendering of text on top of images.
+
+== Deleting images
+
+Images can be deleted by using the delete action `a=d`. If specified without any
+other keys, it will delete all images visible on screen. To delete specific images,
+use the `d` key as described in the table below. Note that each value of d has 
+both a lowercase and an uppercase variant. The lowercase variant only deletes the
+images without necessarily freeing up the stored image data, so that the images can be
+re-displayed without needing to resend the data. The uppercase variants will delete
+the image data as well, provided that the image is not referenced elsewhere, such as in the
+scrollback buffer. The values of the `x` and `y` keys are the same as cursor positions (i.e.
+x=1, y=1 is the top left cell).
+
+|===
+| Value of `d` | Meaning 
+
+| `a` or `A` | Delete all images visible on screen
+| `i` or `I` | Delete all images with the specified id, specified using the `i` key.
+| `c` or `C` | Delete all images that intersect with the current cursor position.
+| `p` or `P` | Delete all images that intersect a specific cell, the cell is specified using the `x` and `y` keys
+| `q` or `Q` | Delete all images that intersect a specific cell having a specific z-index. The cell and z-index is specified using the `x`, `y` and `z` keys.
+| `x` or `X` | Delete all images that intersect the specified column, specified using the `x` key.
+| `y` or `Y` | Delete all images that intersect the specified row, specified using the `y` key.
+| `z` or `Z` | Delete all images that have the specified z-index, specified using the `z` key.
+
+|===
+
+
+Some examples:
+
+```
+<ESC>_Ga=d<ESC>\   # delete all visible images
+<ESC>_Ga=d,i=10<ESC>\   # delete the image with id=10
+<ESC>_Ga=Z,z=-1<ESC>\   # delete the images with z-index -1, also freeing up image data
+<ESC>_Ga=P,x=3,y=4<ESC>\   # delete all images that intersect the cell at (3, 4)
+```
+
+=== Image persistence and storage quotas
+
+In order to avoid *Denial-of-Service* attacks, terminal emulators should have a
+maximum storage quota for image data. It should allow at least a few full
+screen images.  For example the quota in kitty is 320MB per buffer. When adding
+a new image, if the total size exceeds the quota, the terminal emulator should
+delete older images to make space for the new one.
+
+
+== Control data reference
+
+The table below shows all the control data keys as well as what values they can
+take, and the default value they take when missing. All integers are 32-bit. 
+
+[cols="^1,<3,^1,<6"]
+|===
+|Key | Value | Default | Description
+
+| `a` | Single character. `(t, T, q, p, d)` | `t` | The overall action this graphics command is performing.
+
+4+^.^h| Keys for image transmission
+
+| `f` | Positive integer. `(24, 32, 100)`. | `32` | The format in which the image data is sent.
+| `t` | Single character. `(d, f, t, s)`. | `d` | The transmission medium used.
+| `s` | Positive integer. | `0` | The width of the image being sent.
+| `v` | Positive integer. | `0` | The height of the image being sent.
+| `S` | Positive integer. | `0` | The size of data to read from a file.
+| `O` | Positive integer. | `0` | The offset from which to read data from a file.
+| `i` | Positive integer. `(0 - 4294967295)` | `0` | The image id
+| `o` | Single character. `only z` | `null` | The type of data compression.
+| `m` | zero or one | `0` | Whether there is more chunked data available.
+
+4+^.^h| Keys for image display
+
+| `x` | Positive integer | `0` | The left edge (in pixels) of the image area to display
+| `y` | Positive integer | `0` | The top edge (in pixels) of the image area to display
+| `w` | Positive integer | `0` | The width (in pixels) of the image area to display. By default, the entire width is used.
+| `h` | Positive integer | `0` | The height (in pixels) of the image area to display. By default, the entire height is used
+| `X` | Positive integer | `0` | The x-offset within the first cell at which to start displaying the image
+| `Y` | Positive integer | `0` | The y-offset within the first cell at which to start displaying the image
+| `c` | Positive integer | `0` | The number of columns to display the image over
+| `r` | Positive integer | `0` | The number of rows to display the image over
+| `z` | Integer          | `0` | The *z-index* vertical stacking order of the image
+
+4+^.^h| Keys for deleting images
+
+| `d` | Single character. `(a, A, c, C, p, P, q, Q, x, X, y, Y, z, Z)`. | `a` | What to delete.
+
+|===
+
+
+== Interaction with other terminal actions
+
+When resetting the terminal, all images that are visible on the screen must be
+cleared.  When switching from the main screen to the alternate screen buffer
+(1049 private mode) all images in the alternate screen must be cleared, just as
+all text is cleared. The clear screen escape code (usually `<ESC>[2J`) should also
+clear all images. This is so that the clear command works.
+
+The other commands to erase text must have no effect on graphics.
+The dedicated delete graphics commands must be used for those.
+
+When scrolling the screen (such as when using index cursor movement commands,
+or scrolling through the history buffer), images must be scrolled along with
+text. When page margins are defined and the index commands are used, only
+images that are entirely within the page area (between the margins) must be
+scrolled. When scrolling them would cause them to extend outside the page area,
+they must be clipped.

kitty/border_fragment.glsl

@@ -0,0 +1,7 @@
+#version GLSL_VERSION
+in vec3 color;
+out vec4 final_color;
+
+void main() {
+    final_color = vec4(color, 1);
+}

kitty/border_vertex.glsl

@@ -0,0 +1,35 @@
+#version GLSL_VERSION
+uniform uvec2 viewport;
+in uvec4 rect;  // left, top, right, bottom
+in uint rect_color;
+out vec3 color;
+
+// indices into the rect vector
+const int LEFT = 0;
+const int TOP = 1;
+const int RIGHT = 2;
+const int BOTTOM = 3;
+
+const uvec2 pos_map[] = uvec2[4](
+    uvec2(RIGHT, TOP),  
+    uvec2(RIGHT, BOTTOM),  
+    uvec2(LEFT, BOTTOM),  
+    uvec2(LEFT, TOP)   
+);
+
+vec2 to_opengl(uint x, uint y) {
+    return vec2(
+        -1.0 + 2.0 * (float(x) / float(viewport.x)),
+        1.0 - 2.0 * (float(y) / float(viewport.y))
+    );
+}
+
+float to_color(uint c) {
+    return float(c & uint(0xff)) / 255.0;
+}
+
+void main() {
+    uvec2 pos = pos_map[gl_VertexID];
+    gl_Position = vec4(to_opengl(rect[pos.x], rect[pos.y]), 0, 1);
+    color = vec3(to_color(rect_color >> 16), to_color(rect_color >> 8), to_color(rect_color));
+}

kitty/borders.py

@@ -2,110 +2,76 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-from ctypes import addressof
+from functools import partial
 from itertools import chain
-from threading import Lock
 
-from .constants import GLfloat, GLint, GLuint, viewport_size
-from .fast_data_types import GL_TRIANGLE_FAN, glMultiDrawArrays, glUniform3fv
-from .shaders import ShaderProgram
-from .utils import get_dpi
+from .constants import viewport_size
+from .fast_data_types import (
+    BORDERS_PROGRAM, add_borders_rect, compile_program, init_borders_program,
+    send_borders_rects
+)
+from .utils import color_as_int, pt_to_px, load_shaders
 
 
-def as_color(c):
-    return c[0] / 255, c[1] / 255, c[2] / 255
+def vertical_edge(color, width, top, bottom, left):
+    add_borders_rect(left, top, left + width, bottom, color)
 
 
-def to_opengl(x, y):
-    return -1 + 2 * x / viewport_size.width, 1 - 2 * y / viewport_size.height
+def horizontal_edge(color, height, left, right, top):
+    add_borders_rect(left, top, right, top + height, color)
 
 
-def as_rect(left, top, right, bottom, color=0):
-    for (x, y) in ((right, top), (right, bottom), (left, bottom), (left, top)):
-        x, y = to_opengl(x, y)
-        yield x
-        yield y
-        yield color
+def edge(func, color, sz, a, b):
+    return partial(func, color, sz, a, b)
 
 
-class BordersProgram(ShaderProgram):
-
-    def __init__(self):
-        ShaderProgram.__init__(self, '''\
-uniform vec3 colors[3];
-in vec3 rect;
-out vec3 color;
-
-void main() {
-    gl_Position = vec4(rect[0], rect[1], 0, 1);
-    color = colors[uint(rect[2])];
-}
-''', '''\
-in vec3 color;
-out vec4 final_color;
-
-void main() {
-    final_color = vec4(color, 1);
-}
-        ''')
-        self.add_vertex_array('rect')
-
-    def send_data(self, data):
-        self.send_vertex_data('rect', data)
-
-    def set_colors(self, color_buf):
-        glUniform3fv(self.uniform_location('colors'), 3, addressof(color_buf))
+def border(color, sz, left, top, right, bottom):
+    horz = edge(horizontal_edge, color, sz, left, right)
+    horz(top), horz(bottom - sz)  # top, bottom edges
+    vert = edge(vertical_edge, color, sz, top, bottom)
+    vert(left), vert(right - sz)  # left, right edges
 
 
 class Borders:
 
     def __init__(self, opts):
-        self.is_dirty = False
-        self.lock = Lock()
-        self.can_render = False
-        dpix, dpiy = get_dpi()['logical']
-        dpi = (dpix + dpiy) / 2
-        self.border_width = round(opts.window_border_width * dpi / 72)
-        self.color_buf = (GLfloat * 9)(
-            *as_color(opts.background),
-            *as_color(opts.active_border_color),
-            *as_color(opts.inactive_border_color)
-        )
+        self.border_width = pt_to_px(opts.window_border_width)
+        self.padding_width = pt_to_px(opts.window_padding_width)
+        compile_program(BORDERS_PROGRAM, *load_shaders('border'))
+        init_borders_program()
+        self.background = color_as_int(opts.background)
+        self.active_border = color_as_int(opts.active_border_color)
+        self.inactive_border = color_as_int(opts.inactive_border_color)
 
-    def __call__(self, windows, active_window, current_layout, extra_blank_rects, draw_window_borders=True):
-        rects = []
+    def __call__(
+        self,
+        windows,
+        active_window,
+        current_layout,
+        extra_blank_rects,
+        draw_window_borders=True
+    ):
+        add_borders_rect(0, 0, 0, 0, 0)
         for br in chain(current_layout.blank_rects, extra_blank_rects):
-            rects.extend(as_rect(*br))
-        if draw_window_borders and self.border_width > 0:
-            bw = self.border_width
+            add_borders_rect(*br, self.background)
+        bw, pw = self.border_width, self.padding_width
+        fw = bw + pw
+
+        if fw > 0:
             for w in windows:
                 g = w.geometry
-                color = 1 if w is active_window else 2
-                rects.extend(as_rect(g.left - bw, g.top - bw, g.left, g.bottom + bw, color))
-                rects.extend(as_rect(g.left - bw, g.top - bw, g.right + bw, g.top, color))
-                rects.extend(as_rect(g.right, g.top - bw, g.right + bw, g.bottom + bw, color))
-                rects.extend(as_rect(g.left - bw, g.bottom, g.right + bw, g.bottom + bw, color))
-        with self.lock:
-            self.num_of_rects = len(rects) // 12
-            self.rects = (GLfloat * len(rects))()
-            self.starts = (GLint * self.num_of_rects)()
-            self.counts = (GLuint * self.num_of_rects)()
-            for i, x in enumerate(rects):
-                self.rects[i] = x
-                if i % 12 == 0:
-                    idx = i // 12
-                    self.starts[idx] = i // 3
-                    self.counts[idx] = 4
-            self.is_dirty = True
-            self.can_render = True
-
-    def render(self, program):
-        with self.lock:
-            if not self.can_render:
-                return
-            with program:
-                if self.is_dirty:
-                    program.send_data(self.rects)
-                    program.set_colors(self.color_buf)
-                    self.is_dirty = False
-                glMultiDrawArrays(GL_TRIANGLE_FAN, addressof(self.starts), addressof(self.counts), self.num_of_rects)
+                if bw > 0 and draw_window_borders:
+                    # Draw the border rectangles
+                    color = self.active_border if w is active_window else self.inactive_border
+                    border(
+                        color, bw, g.left - fw, g.top - fw, g.right + fw,
+                        g.bottom + fw
+                    )
+                if pw > 0:
+                    # Draw the background rectangles over the padding region
+                    color = self.background
+                    border(
+                        color, pw, g.left - pw, g.top - pw, g.right + pw,
+                        g.bottom + pw
+                    )
+        send_borders_rects(viewport_size.width, viewport_size.height)

kitty/boss.py

@@ -2,124 +2,81 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
-import io
-import select
-import signal
-import struct
-import inspect
-from functools import wraps
-from threading import Thread, current_thread
-from time import monotonic
-from queue import Queue, Empty
 from gettext import gettext as _
+from weakref import WeakValueDictionary
 
 from .config import MINIMUM_FONT_SIZE
-from .constants import (
-    viewport_size, set_boss, wakeup, cell_size, MODIFIER_KEYS,
-    main_thread, mouse_button_pressed, mouse_cursor_pos
-)
+from .constants import cell_size, set_boss, viewport_size, wakeup
 from .fast_data_types import (
-    glViewport, glBlendFunc, GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GLFW_PRESS,
-    GLFW_REPEAT, GLFW_MOUSE_BUTTON_1, glfw_post_empty_event,
-    GLFW_CURSOR_NORMAL, GLFW_CURSOR, GLFW_CURSOR_HIDDEN, drain_read
+    GLFW_KEY_DOWN, GLFW_KEY_UP, ChildMonitor, destroy_global_data,
+    destroy_sprite_map, glfw_post_empty_event, layout_sprite_map
 )
-from .fonts.render import set_font_family
-from .borders import BordersProgram
-from .char_grid import cursor_shader, cell_shader
-from .constants import is_key_pressed
-from .keys import interpret_text_event, interpret_key_event, get_shortcut
+from .fonts.render import render_cell_wrapper, set_font_family
+from .keys import get_key_map, get_sent_data, get_shortcut
 from .session import create_session
-from .shaders import Sprites, ShaderProgram
-from .tabs import TabManager, SpecialWindow
-from .timers import Timers
-from .utils import handle_unix_signals, safe_print, pipe2
+from .tabs import SpecialWindow, TabManager
+from .utils import (
+    get_primary_selection, open_url, safe_print, set_primary_selection
+)
+from .window import load_shader_programs
 
 
-def conditional_run(w, i):
-    if w is None or not w.destroyed:
-        next(i, None)
+def initialize_renderer():
+    load_shader_programs()
+    layout_sprite_map(cell_size.width, cell_size.height, render_cell_wrapper)
 
 
-def callback(func):
-    ''' Wrapper for function that executes first half (up to a yield statement)
-    in the UI thread and the rest in the child thread. If the function yields
-    something, the destroyed attribute of that something is checked before
-    running the second half. If the function returns before the yield, the
-    second half is not run. '''
+class DumpCommands:  # {{{
 
-    assert inspect.isgeneratorfunction(func)
+    def __init__(self, args):
+        self.draw_dump_buf = []
+        if args.dump_bytes:
+            self.dump_bytes_to = open(args.dump_bytes, 'wb')
 
-    @wraps(func)
-    def f(self, *a):
-        i = func(self, *a)
-        try:
-            w = next(i)
-        except StopIteration:
-            pass
-        else:
-            self.queue_action(conditional_run, w, i)
-    return f
+    def __call__(self, *a):
+        if a:
+            if a[0] == 'draw':
+                if a[1] is None:
+                    if self.draw_dump_buf:
+                        safe_print('draw', ''.join(self.draw_dump_buf))
+                        self.draw_dump_buf = []
+                else:
+                    self.draw_dump_buf.append(a[1])
+            elif a[0] == 'bytes':
+                self.dump_bytes_to.write(a[1])
+                self.dump_bytes_to.flush()
+            else:
+                if self.draw_dump_buf:
+                    safe_print('draw', ''.join(self.draw_dump_buf))
+                    self.draw_dump_buf = []
+                safe_print(*a)
+# }}}
 
 
-class Boss(Thread):
-
-    daemon = True
+class Boss:
 
     def __init__(self, glfw_window, opts, args):
-        Thread.__init__(self, name='ChildMonitor')
+        self.window_id_map = WeakValueDictionary()
         startup_session = create_session(opts, args)
-        self.cursor_blink_zero_time = monotonic()
         self.cursor_blinking = True
         self.window_is_focused = True
         self.glfw_window_title = None
-        self.action_queue = Queue()
-        self.pending_resize = False
-        self.resize_gl_viewport = False
         self.shutting_down = False
-        self.screen_update_delay = opts.repaint_delay / 1000.0
-        self.signal_fd = handle_unix_signals()
-        self.read_wakeup_fd, self.write_wakeup_fd = pipe2()
-        self.read_dispatch_map = {self.signal_fd: self.signal_received, self.read_wakeup_fd: self.on_wakeup}
-        self.all_writers = []
-        self.timers = Timers()
-        self.ui_timers = Timers()
-        self.pending_ui_thread_calls = Queue()
-        self.write_dispatch_map = {}
+        self.child_monitor = ChildMonitor(
+            glfw_window.window_id(),
+            self.on_child_death,
+            DumpCommands(args) if args.dump_commands or args.dump_bytes else None)
         set_boss(self)
         self.current_font_size = opts.font_size
         cell_size.width, cell_size.height = set_font_family(opts)
         self.opts, self.args = opts, args
         self.glfw_window = glfw_window
         glfw_window.framebuffer_size_callback = self.on_window_resize
-        glfw_window.char_mods_callback = self.on_text_input
-        glfw_window.key_callback = self.on_key
-        glfw_window.mouse_button_callback = self.on_mouse_button
-        glfw_window.scroll_callback = self.on_mouse_scroll
-        glfw_window.cursor_pos_callback = self.on_mouse_move
         glfw_window.window_focus_callback = self.on_focus
-        self.tab_manager = TabManager(opts, args, startup_session)
-        self.sprites = Sprites()
-        self.cell_program = ShaderProgram(*cell_shader)
-        self.cursor_program = ShaderProgram(*cursor_shader)
-        self.borders_program = BordersProgram()
-        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
-        self.sprites.do_layout(cell_size.width, cell_size.height)
-        self.glfw_window.set_click_cursor(False)
-        self.show_mouse_cursor()
-        self.start_cursor_blink()
-
-    def signal_received(self):
-        try:
-            data = os.read(self.signal_fd, io.DEFAULT_BUFFER_SIZE)
-        except BlockingIOError:
-            return
-        if data:
-            signals = struct.unpack('%uB' % len(data), data)
-            if signal.SIGINT in signals or signal.SIGTERM in signals:
-                if not self.shutting_down:
-                    self.glfw_window.set_should_close(True)
-                    glfw_post_empty_event()
+        initialize_renderer()
+        self.tab_manager = TabManager(opts, args)
+        self.tab_manager.init(startup_session)
+        self.activate_tab_at = self.tab_manager.activate_tab_at
 
     @property
     def current_tab_bar_height(self):
@@ -132,109 +89,46 @@ class Boss(Thread):
         for t in self:
             yield from t
 
-    def queue_action(self, func, *args):
-        self.action_queue.put((func, args))
-        wakeup()
+    def add_child(self, window):
+        self.child_monitor.add_child(window.id, window.child.pid, window.child.child_fd, window.screen)
+        self.window_id_map[window.id] = window
 
-    def on_wakeup(self):
-        if not self.shutting_down:
-            drain_read(self.read_wakeup_fd)
-            while True:
-                try:
-                    func, args = self.action_queue.get_nowait()
-                except Empty:
-                    break
-                try:
-                    func(*args)
-                except Exception:
-                    import traceback
-                    safe_print(traceback.format_exc())
-
-    def add_child_fd(self, child_fd, read_ready, write_ready):
-        self.read_dispatch_map[child_fd] = read_ready
-        self.write_dispatch_map[child_fd] = write_ready
-
-    def remove_child_fd(self, child_fd):
-        self.read_dispatch_map.pop(child_fd, None)
-        self.write_dispatch_map.pop(child_fd, None)
-        try:
-            self.all_writers.remove(child_fd)
-        except Exception:
-            pass
-
-    def queue_ui_action(self, func, *args):
-        self.pending_ui_thread_calls.put((func, args))
-        glfw_post_empty_event()
+    def on_child_death(self, window_id):
+        w = self.window_id_map.pop(window_id, None)
+        if w is not None:
+            w.on_child_death()
 
     def close_window(self, window=None):
-        ' Can be called in either thread, will first kill the child, then remove the window from the gui '
         if window is None:
             window = self.active_window
-        if current_thread() is main_thread:
-            self.queue_action(self.close_window, window)
-        else:
-            self.remove_child_fd(window.child_fd)
-            window.destroy()
-            self.queue_ui_action(self.gui_close_window, window)
+        self.child_monitor.mark_for_close(window.id)
 
     def close_tab(self, tab=None):
-        ' Can be called in either thread, will first kill all children, then remove the tab from the gui '
         if tab is None:
             tab = self.active_tab
-        if current_thread() is main_thread:
-            self.queue_action(self.close_tab, tab)
-        else:
-            for window in tab:
-                self.remove_child_fd(window.child_fd)
-                window.destroy()
-                self.queue_ui_action(self.gui_close_window, window)
+        for window in tab:
+            self.close_window(window)
 
-    def run(self):
-        if self.args.profile:
-            import cProfile
-            import pstats
-            pr = cProfile.Profile()
-            pr.enable()
-        self.loop()
-        if self.args.profile:
-            pr.disable()
-            pr.create_stats()
-            s = pstats.Stats(pr)
-            s.dump_stats(self.args.profile)
+    def start(self):
+        if not getattr(self, 'io_thread_started', False):
+            self.child_monitor.start()
+            self.io_thread_started = True
 
-    def loop(self):
-        while not self.shutting_down:
-            all_readers = list(self.read_dispatch_map)
-            all_writers = [w.child_fd for w in self.iterwindows() if w.write_buf]
-            readers, writers, _ = select.select(all_readers, all_writers, [], self.timers.timeout())
-            for r in readers:
-                self.read_dispatch_map[r]()
-            for w in writers:
-                self.write_dispatch_map[w]()
-            self.timers()
-            for w in self.iterwindows():
-                if w.screen.is_dirty():
-                    self.timers.add_if_missing(self.screen_update_delay, w.update_screen)
-
-    @callback
     def on_window_resize(self, window, w, h):
-        # debounce resize events
-        self.pending_resize = True
-        yield
-        self.timers.add(0.02, self.apply_pending_resize, w, h)
-
-    def apply_pending_resize(self, w, h):
         viewport_size.width, viewport_size.height = w, h
         self.tab_manager.resize()
-        self.resize_gl_viewport = True
-        self.pending_resize = False
-        glfw_post_empty_event()
 
     def increase_font_size(self):
-        self.change_font_size(min(self.opts.font_size * 5, self.current_font_size + self.opts.font_size_delta))
+        self.change_font_size(
+            min(
+                self.opts.font_size * 5, self.current_font_size +
+                self.opts.font_size_delta))
 
     def decrease_font_size(self):
-        self.change_font_size(max(MINIMUM_FONT_SIZE, self.current_font_size - self.opts.font_size_delta))
+        self.change_font_size(
+            max(
+                MINIMUM_FONT_SIZE, self.current_font_size -
+                self.opts.font_size_delta))
 
     def restore_font_size(self):
         self.change_font_size(self.opts.font_size)
@@ -243,10 +137,17 @@ class Boss(Thread):
         if new_size == self.current_font_size:
             return
         self.current_font_size = new_size
+        w, h = cell_size.width, cell_size.height
+        windows = tuple(filter(None, self.window_id_map.values()))
         cell_size.width, cell_size.height = set_font_family(
             self.opts, override_font_size=self.current_font_size)
-        self.sprites.do_layout(cell_size.width, cell_size.height)
-        self.queue_action(self.resize_windows_after_font_size_change)
+        layout_sprite_map(cell_size.width, cell_size.height, render_cell_wrapper)
+        for window in windows:
+            window.screen.rescale_images(w, h)
+        self.resize_windows_after_font_size_change()
+        for window in windows:
+            window.screen.refresh_sprite_positions()
+        self.tab_manager.refresh_sprite_positions()
 
     def resize_windows_after_font_size_change(self):
         self.tab_manager.resize()
@@ -269,185 +170,80 @@ class Boss(Thread):
         if t is not None:
             return t.active_window
 
-    @callback
-    def on_text_input(self, window, codepoint, mods):
-        w = self.active_window
-        if w is not None:
-            yield w
-            if w is not None:
-                data = interpret_text_event(codepoint, mods, w)
-                if data:
-                    w.write_to_child(data)
+    def dispatch_special_key(self, key, scancode, action, mods):
+        # Handles shortcuts, return True if the key was consumed
+        key_action = get_shortcut(self.opts.keymap, mods, key, scancode)
+        self.current_key_press_info = key, scancode, action, mods
+        return self.dispatch_action(key_action)
 
-    @callback
-    def on_key(self, window, key, scancode, action, mods):
-        is_key_pressed[key] = action == GLFW_PRESS
-        self.start_cursor_blink()
-        self.cursor_blink_zero_time = monotonic()
-        func = None
-        if action == GLFW_PRESS or action == GLFW_REPEAT:
-            func = get_shortcut(self.opts.keymap, mods, key, scancode)
-            if func is not None:
-                f = getattr(self, func, None)
-                if f is not None:
-                    passthrough = f()
-                    if not passthrough:
-                        return
+    def dispatch_action(self, key_action):
+        if key_action is not None:
+            f = getattr(self, key_action.func, None)
+            if f is not None:
+                passthrough = f(*key_action.args)
+                if passthrough is not True:
+                    return True
         tab = self.active_tab
         if tab is None:
-            return
+            return False
         window = self.active_window
         if window is None:
-            return
-        yield window
-        if func is not None:
-            f = getattr(tab, func, getattr(window, func, None))
+            return False
+        if key_action is not None:
+            f = getattr(tab, key_action.func, getattr(window, key_action.func, None))
             if f is not None:
-                passthrough = f()
-                if not passthrough:
-                    return
-        if window.char_grid.scrolled_by and key not in MODIFIER_KEYS and action == GLFW_PRESS:
-            window.scroll_end()
-        data = interpret_key_event(key, scancode, mods, window, action)
+                passthrough = f(*key_action.args)
+                if passthrough is not True:
+                    return True
+        key, scancode, action, mods = self.current_key_press_info
+        data = get_sent_data(
+            self.opts.send_text_map, key, scancode, mods, window, action
+        )
         if data:
             window.write_to_child(data)
+            return True
+        return False
+
+    def combine(self, *actions):
+        for key_action in actions:
+            self.dispatch_action(key_action)
 
-    @callback
     def on_focus(self, window, focused):
         self.window_is_focused = focused
         w = self.active_window
         if w is not None:
-            yield w
             w.focus_changed(focused)
 
     def display_scrollback(self, data):
         if self.opts.scrollback_in_new_tab:
-            self.queue_ui_action(self.display_scrollback_in_new_tab, data)
+            self.display_scrollback_in_new_tab(data)
         else:
             tab = self.active_tab
             if tab is not None:
-                tab.new_special_window(SpecialWindow(self.opts.scrollback_pager, data, _('History')))
+                tab.new_special_window(
+                    SpecialWindow(
+                        self.opts.scrollback_pager, data, _('History')))
 
-    def window_for_pos(self, x, y):
+    def switch_focus_to(self, window_idx):
         tab = self.active_tab
-        if tab is not None:
-            for w in tab:
-                if w.is_visible_in_layout and w.contains(x, y):
-                    return w
+        tab.set_active_window_idx(window_idx)
+        old_focus = tab.active_window
+        if not old_focus.destroyed:
+            old_focus.focus_changed(False)
+        tab.active_window.focus_changed(True)
 
-    def in_tab_bar(self, y):
-        th = self.current_tab_bar_height
-        return th > 0 and y >= viewport_size.height - th
-
-    @callback
-    def on_mouse_button(self, window, button, action, mods):
-        mouse_button_pressed[button] = action == GLFW_PRESS
-        self.show_mouse_cursor()
-        x, y = mouse_cursor_pos
-        w = self.window_for_pos(x, y)
-        if w is None:
-            if self.in_tab_bar(y):
-                if button == GLFW_MOUSE_BUTTON_1 and action == GLFW_PRESS:
-                    self.tab_manager.activate_tab_at(x)
-            return
-        focus_moved = False
-        old_focus = self.active_window
+    def send_fake_scroll(self, window_idx, amt, upwards):
         tab = self.active_tab
-        yield
-        if button == GLFW_MOUSE_BUTTON_1 and w is not old_focus:
-            tab.set_active_window(w)
-            focus_moved = True
-        if focus_moved:
-            if old_focus is not None and not old_focus.destroyed:
-                old_focus.focus_changed(False)
-            w.focus_changed(True)
-        w.on_mouse_button(button, action, mods)
+        w = tab.windows[window_idx]
+        k = get_key_map(w.screen)[GLFW_KEY_UP if upwards else GLFW_KEY_DOWN]
+        w.write_to_child(k * amt)
 
-    @callback
-    def on_mouse_move(self, window, xpos, ypos):
-        mouse_cursor_pos[:2] = xpos, ypos = int(xpos * viewport_size.x_ratio), int(ypos * viewport_size.y_ratio)
-        self.show_mouse_cursor()
-        w = self.window_for_pos(xpos, ypos)
-        if w is not None:
-            yield w
-            w.on_mouse_move(xpos, ypos)
-        else:
-            self.change_mouse_cursor(self.in_tab_bar(ypos))
-
-    @callback
-    def on_mouse_scroll(self, window, x, y):
-        self.show_mouse_cursor()
-        w = self.window_for_pos(*mouse_cursor_pos)
-        if w is not None:
-            yield w
-            w.on_mouse_scroll(x, y)
-
-    # GUI thread API {{{
-
-    def show_mouse_cursor(self):
-        self.glfw_window.set_input_mode(GLFW_CURSOR, GLFW_CURSOR_NORMAL)
-        if self.opts.mouse_hide_wait > 0:
-            self.ui_timers.add(self.opts.mouse_hide_wait, self.hide_mouse_cursor)
-
-    def hide_mouse_cursor(self):
-        self.glfw_window.set_input_mode(GLFW_CURSOR, GLFW_CURSOR_HIDDEN)
-
-    def change_mouse_cursor(self, click=False):
-        self.glfw_window.set_click_cursor(click)
-
-    def request_attention(self):
-        try:
-            self.glfw_window.request_window_attention()
-        except AttributeError:
-            pass   # needs glfw 3.3
-
-    def start_cursor_blink(self):
-        self.cursor_blinking = True
-        if self.opts.cursor_stop_blinking_after > 0:
-            self.ui_timers.add(self.opts.cursor_stop_blinking_after, self.stop_cursor_blinking)
-
-    def stop_cursor_blinking(self):
-        self.cursor_blinking = False
-
-    def render(self):
-        if self.pending_resize:
-            return
-        if self.resize_gl_viewport:
-            glViewport(0, 0, viewport_size.width, viewport_size.height)
-            self.resize_gl_viewport = False
-        tab = self.active_tab
-        if tab is not None:
-            if tab.title != self.glfw_window_title:
-                self.glfw_window_title = tab.title
-                self.glfw_window.set_title(self.glfw_window_title)
-            with self.sprites:
-                self.sprites.render_dirty_cells()
-                tab.render()
-                render_data = {window: window.char_grid.prepare_for_render(self.sprites) for window in tab.visible_windows() if not window.needs_layout}
-                with self.cell_program:
-                    self.tab_manager.render(self.cell_program, self.sprites)
-                    for window, rd in render_data.items():
-                        if rd is not None:
-                            window.render_cells(rd, self.cell_program, self.sprites)
-                active = self.active_window
-                rd = render_data.get(active)
-                if rd is not None:
-                    draw_cursor = True
-                    if self.cursor_blinking and self.opts.cursor_blink_interval > 0 and self.window_is_focused:
-                        now = monotonic() - self.cursor_blink_zero_time
-                        t = int(now * 1000)
-                        d = int(self.opts.cursor_blink_interval * 1000)
-                        n = t // d
-                        draw_cursor = n % 2 == 0
-                        self.ui_timers.add_if_missing(((n + 1) * d / 1000) - now, glfw_post_empty_event)
-                    if draw_cursor:
-                        with self.cursor_program:
-                            active.char_grid.render_cursor(rd, self.cursor_program, self.window_is_focused)
+    def open_url(self, url):
+        if url:
+            open_url(url, self.opts.open_url_with)
 
     def gui_close_window(self, window):
-        if window.char_grid.buffer_id is not None:
-            self.sprites.destroy_sprite_map(window.char_grid.buffer_id)
-            window.char_grid.buffer_id = None
+        window.destroy()
         for tab in self.tab_manager:
             if window in tab:
                 break
@@ -463,41 +259,95 @@ class Boss(Thread):
                     glfw_post_empty_event()
 
     def destroy(self):
-        # Must be called in the main thread as it manipulates signal handlers
-        signal.signal(signal.SIGINT, signal.SIG_DFL)
-        signal.signal(signal.SIGTERM, signal.SIG_DFL)
         self.shutting_down = True
+        self.child_monitor.shutdown()
         wakeup()
-        self.join()
+        self.child_monitor.join()
         for t in self.tab_manager:
             t.destroy()
         del self.tab_manager
-        self.sprites.destroy()
-        del self.sprites
+        destroy_sprite_map()
+        destroy_global_data()
         del self.glfw_window
 
-    def paste_from_clipboard(self):
-        text = self.glfw_window.get_clipboard_string()
+    def paste_to_active_window(self, text):
         if text:
             w = self.active_window
             if w is not None:
-                self.queue_action(w.paste, text)
+                w.paste(text)
+
+    def paste_from_clipboard(self):
+        text = self.glfw_window.get_clipboard_string()
+        self.paste_to_active_window(text)
+
+    def paste_from_selection(self):
+        text = get_primary_selection()
+        self.paste_to_active_window(text)
+
+    def set_primary_selection(self):
+        w = self.active_window
+        if w is not None and not w.destroyed:
+            text = w.text_for_selection()
+            if text:
+                set_primary_selection(text)
 
     def next_tab(self):
-        self.queue_action(self.tab_manager.next_tab)
+        self.tab_manager.next_tab()
 
     def previous_tab(self):
-        self.queue_action(self.tab_manager.next_tab, -1)
+        self.tab_manager.next_tab(-1)
 
-    def new_tab(self):
-        self.tab_manager.new_tab()
+    def args_to_special_window(self, args):
+        args = list(args)
+        stdin = None
+        w = self.active_window
+
+        def data_for_at(arg):
+            if arg == '@selection':
+                return w.text_for_selection()
+            if arg == '@ansi':
+                return w.buffer_as_ansi()
+            if arg == '@text':
+                return w.buffer_as_text()
+
+        if args[0].startswith('@'):
+            stdin = data_for_at(args[0]) or None
+            if stdin is not None:
+                stdin = stdin.encode('utf-8')
+            del args[0]
+
+        cmd = []
+        for arg in args:
+            if arg == '@selection':
+                arg = data_for_at(arg)
+                if not arg:
+                    continue
+            cmd.append(arg)
+        return SpecialWindow(cmd, stdin)
+
+    def new_tab(self, *args):
+        special_window = None
+        if args:
+            special_window = self.args_to_special_window(args)
+        self.tab_manager.new_tab(special_window=special_window)
+
+    def new_window(self, *args):
+        tab = self.active_tab
+        if tab is not None:
+            if args:
+                tab.new_special_window(self.args_to_special_window(args))
+            else:
+                tab.new_window()
 
     def move_tab_forward(self):
-        self.queue_action(self.tab_manager.move_tab, 1)
+        self.tab_manager.move_tab(1)
 
     def move_tab_backward(self):
-        self.queue_action(self.tab_manager.move_tab, -1)
+        self.tab_manager.move_tab(-1)
 
     def display_scrollback_in_new_tab(self, data):
-        self.tab_manager.new_tab(special_window=SpecialWindow(self.opts.scrollback_pager, data, _('History')))
+        self.tab_manager.new_tab(
+            special_window=SpecialWindow(
+                self.opts.scrollback_pager, data, _('History')))
+
     # }}}

kitty/cell_fragment.glsl

@@ -0,0 +1,49 @@
+#version GLSL_VERSION
+#define WHICH_PROGRAM
+#if defined(FOREGROUND) || defined(ALL)
+uniform sampler2DArray sprites;
+in vec3 sprite_pos;
+in vec3 underline_pos;
+in vec3 strike_pos;
+in vec3 foreground;
+in vec3 decoration_fg;
+#endif
+in vec3 background;
+#ifdef SPECIAL
+in vec4 special_bg;
+#endif
+
+out vec4 final_color;
+
+vec3 blend(float alpha, vec3 over, vec3 under) {
+    return over + (1 - alpha) * under;
+}
+
+void main() {
+#if defined(FOREGROUND) || defined(ALL)
+    float text_alpha = texture(sprites, sprite_pos).r;
+    float underline_alpha = texture(sprites, underline_pos).r;
+    float strike_alpha = texture(sprites, strike_pos).r;
+    vec3 underline = underline_alpha * decoration_fg;
+    vec3 strike = strike_alpha * foreground;
+    vec3 fg = text_alpha * foreground;
+    vec3 decoration = blend(underline_alpha, underline, strike);
+    vec3 combined_fg = blend(text_alpha, fg, decoration);
+    float combined_alpha = max(max(underline_alpha, strike_alpha), text_alpha);
+
+#ifdef ALL
+    final_color = vec4(blend(combined_alpha, combined_fg, background), 1);
+#else
+    final_color = vec4(combined_fg, combined_alpha);
+#endif
+
+#else
+
+#ifdef SPECIAL
+    final_color = special_bg;
+#else
+    final_color = vec4(background, 1);
+#endif
+
+#endif
+}

kitty/cell_vertex.glsl

@@ -0,0 +1,152 @@
+#version GLSL_VERSION
+#define WHICH_PROGRAM
+layout(std140) uniform CellRenderData {
+    float xstart, ystart, dx, dy, sprite_dx, sprite_dy;
+
+    uint default_fg, default_bg, highlight_fg, highlight_bg, cursor_color, url_color;
+
+    int color1, color2;
+
+    uint xnum, ynum, cursor_x, cursor_y, cursor_w, url_xl, url_yl, url_xr, url_yr;
+
+    uint color_table[256]; 
+};
+
+// Have to use fixed locations here as all variants of the cell program share the same VAO
+layout(location=0) in uvec3 colors;
+layout(location=1) in uvec4 sprite_coords;
+layout(location=2) in float is_selected;
+
+#if defined(FOREGROUND) || defined(ALL)
+
+out vec3 sprite_pos;
+out vec3 underline_pos;
+out vec3 strike_pos;
+out vec3 foreground;
+out vec3 decoration_fg;
+#endif
+out vec3 background;
+#ifdef SPECIAL
+out vec4 special_bg;
+#endif
+
+const uvec2 pos_map[] = uvec2[4](
+    uvec2(1, 0),  // right, top
+    uvec2(1, 1),  // right, bottom
+    uvec2(0, 1),  // left, bottom
+    uvec2(0, 0)   // left, top
+);
+
+const uint BYTE_MASK = uint(0xFF);
+const uint SHORT_MASK = uint(0xFFFF);
+const uint ZERO = uint(0);
+const uint ONE = uint(1);
+const uint TWO = uint(2);
+const uint THREE = uint(3);
+const uint DECORATION_MASK = uint(3);
+const uint STRIKE_MASK = uint(1);
+const uint REVERSE_MASK = uint(1);
+
+vec3 color_to_vec(uint c) {
+    uint r, g, b;
+    r = (c >> 16) & BYTE_MASK;
+    g = (c >> 8) & BYTE_MASK;
+    b = c & BYTE_MASK;
+    return vec3(float(r) / 255.0, float(g) / 255.0, float(b) / 255.0);
+}
+
+uint resolve_color(uint c, uint defval) {
+    // Convert a cell color to an actual color based on the color table
+    int t = int(c & BYTE_MASK);
+    uint r;
+    switch(t) {
+        case 1:
+            r = color_table[(c >> 8) & BYTE_MASK];
+            break;
+        case 2:
+            r = c >> 8;
+            break;
+        default:
+            r = defval;
+    }
+    return r;
+}
+
+vec3 to_color(uint c, uint defval) {
+    return color_to_vec(resolve_color(c, defval));
+}
+
+vec3 to_sprite_pos(uvec2 pos, uint x, uint y, uint z) {
+    vec2 s_xpos = vec2(x, float(x) + 1.0) * sprite_dx;
+    vec2 s_ypos = vec2(y, float(y) + 1.0) * sprite_dy;
+    return vec3(s_xpos[pos.x], s_ypos[pos.y], z);
+}
+
+vec3 choose_color(float q, vec3 a, vec3 b) {
+    return q * a + (1.0 - q) * b;
+}
+
+float in_range(uint x, uint y) {
+    if (url_yl == y && url_xl <= x && x <= url_xr) return 1.0;
+    return 0.0;
+}
+
+float is_cursor(uint x, uint y) {
+    if (y == cursor_y && (x == cursor_x || x == cursor_w)) return 1.0;
+    return 0.0;
+}
+
+void main() {
+    float cursor;
+    uint instance_id = uint(gl_InstanceID);
+    // The current cell being rendered
+    uint r = instance_id / xnum;
+    uint c = instance_id - r * xnum;
+
+    // The position of this vertex, at a corner of the cell
+    float left = xstart + c * dx;
+    float top = ystart - r * dy;
+    vec2 xpos = vec2(left, left + dx);
+    vec2 ypos = vec2(top, top - dy);
+    uvec2 pos = pos_map[gl_VertexID];
+    gl_Position = vec4(xpos[pos.x], ypos[pos.y], 0, 1);
+
+    uvec2 default_colors = uvec2(default_fg, default_bg);
+    ivec2 color_indices = ivec2(color1, color2);
+
+    uint text_attrs = sprite_coords[3];
+    int fg_index = color_indices[(text_attrs >> 6) & REVERSE_MASK];
+    int bg_index = color_indices[1 - fg_index];
+    background = to_color(colors[bg_index], default_colors[bg_index]);
+
+#if defined(FOREGROUND) || defined(ALL)
+    // The character sprite being rendered
+    sprite_pos = to_sprite_pos(pos, sprite_coords.x, sprite_coords.y, sprite_coords.z & SHORT_MASK);
+
+    // Foreground and background colors
+    uint resolved_fg = resolve_color(colors[fg_index], default_colors[fg_index]);
+    foreground = color_to_vec(resolved_fg);
+    // Selection
+    foreground = choose_color(is_selected, color_to_vec(highlight_fg), foreground);
+    // Underline and strike through (rendered via sprites)
+    float in_url = in_range(c, r);
+    decoration_fg = choose_color(in_url, color_to_vec(url_color), to_color(colors[2], resolved_fg));
+    underline_pos = choose_color(in_url, to_sprite_pos(pos, TWO, ZERO, ZERO), to_sprite_pos(pos, (text_attrs >> 2) & DECORATION_MASK, ZERO, ZERO));
+    strike_pos = to_sprite_pos(pos, ((text_attrs >> 7) & STRIKE_MASK) * THREE, ZERO, ZERO);
+    // Block cursor rendering
+    cursor = is_cursor(c, r);
+    foreground = choose_color(cursor, background, foreground);
+    decoration_fg = choose_color(cursor, background, decoration_fg);
+#endif
+
+#if defined(ALL)
+    // Selection and cursor
+    background = choose_color(is_selected, color_to_vec(highlight_bg), background);
+    background = choose_color(cursor, color_to_vec(cursor_color), background);
+#elif defined(SPECIAL)
+    cursor = is_cursor(c, r);
+    background = choose_color(is_selected, color_to_vec(highlight_bg), background);
+    background = choose_color(cursor, color_to_vec(cursor_color), background);
+    special_bg = vec4(background, max(cursor, is_selected));
+#endif
+}

kitty/char_grid.py

@@ -1,520 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
-
-import re
-import sys
-from enum import Enum
-from collections import namedtuple
-from ctypes import addressof, memmove, sizeof
-from threading import Lock
-
-from .config import build_ansi_color_table, defaults
-from .constants import (
-    GLuint, ScreenGeometry, cell_size, get_boss, viewport_size
-)
-from .fast_data_types import (
-    CURSOR_BEAM, CURSOR_BLOCK, CURSOR_UNDERLINE, DATA_CELL_SIZE, GL_BLEND,
-    GL_LINE_LOOP, GL_TRIANGLE_FAN, ColorProfile, glDisable, glDrawArrays,
-    glDrawArraysInstanced, glEnable, glUniform1i, glUniform2f, glUniform2ui,
-    glUniform4f
-)
-from .rgb import to_color
-from .utils import (
-    color_as_int, color_from_int, get_logical_dpi, open_url, safe_print,
-    set_primary_selection
-)
-
-Cursor = namedtuple('Cursor', 'x y shape blink')
-
-
-class DynamicColor(Enum):
-    default_fg, default_bg, cursor_color, highlight_fg, highlight_bg = range(1, 6)
-
-
-if DATA_CELL_SIZE % 3:
-    raise ValueError('Incorrect data cell size, must be a multiple of 3')
-
-
-# cell shader {{{
-
-cell_shader = (
-    '''\
-uniform uvec2 dimensions;  // xnum, ynum
-uniform vec4 steps;  // xstart, ystart, dx, dy
-uniform vec2 sprite_layout;  // dx, dy
-uniform usamplerBuffer sprite_map; // gl_InstanceID -> x, y, z
-uniform uvec2 color_indices;  // which color to use as fg and which as bg
-out vec3 sprite_pos;
-out vec3 underline_pos;
-out vec3 strike_pos;
-out vec3 foreground;
-out vec3 background;
-out vec3 decoration_fg;
-
-const uvec2 pos_map[] = uvec2[4](
-    uvec2(1, 0),  // right, top
-    uvec2(1, 1),  // right, bottom
-    uvec2(0, 1),  // left, bottom
-    uvec2(0, 0)   // left, top
-);
-
-const uint BYTE_MASK = uint(255);
-const uint ZERO = uint(0);
-const uint SMASK = uint(3);
-
-vec3 to_color(uint c) {
-    uint r, g, b;
-    r = (c >> 16) & BYTE_MASK;
-    g = (c >> 8) & BYTE_MASK;
-    b = c & BYTE_MASK;
-    return vec3(float(r) / 255.0, float(g) / 255.0, float(b) / 255.0);
-}
-
-vec3 to_sprite_pos(uvec2 pos, uint x, uint y, uint z) {
-    vec2 s_xpos = vec2(x, float(x) + 1.0) * sprite_layout[0];
-    vec2 s_ypos = vec2(y, float(y) + 1.0) * sprite_layout[1];
-    return vec3(s_xpos[pos[0]], s_ypos[pos[1]], z);
-}
-
-void main() {
-    uint instance_id = uint(gl_InstanceID);
-    uint r = instance_id / dimensions[0];
-    uint c = instance_id - r * dimensions[0];
-    float left = steps[0] + c * steps[2];
-    float top = steps[1] - r * steps[3];
-    vec2 xpos = vec2(left, left + steps[2]);
-    vec2 ypos = vec2(top, top - steps[3]);
-    uvec2 pos = pos_map[gl_VertexID];
-    gl_Position = vec4(xpos[pos[0]], ypos[pos[1]], 0, 1);
-
-    int sprite_id = gl_InstanceID * STRIDE;
-    uvec4 spos = texelFetch(sprite_map, sprite_id);
-    uvec4 colors = texelFetch(sprite_map, sprite_id + 1);
-    sprite_pos = to_sprite_pos(pos, spos[0], spos[1], spos[2]);
-    foreground = to_color(colors[color_indices[0]]);
-    background = to_color(colors[color_indices[1]]);
-    uint decoration = colors[2];
-    decoration_fg = to_color(decoration);
-    underline_pos = to_sprite_pos(pos, (decoration >> 24) & SMASK, ZERO, ZERO);
-    strike_pos = to_sprite_pos(pos, (decoration >> 26) & SMASK, ZERO, ZERO);
-}
-'''.replace('STRIDE', str(DATA_CELL_SIZE // 3)),
-
-    '''\
-uniform sampler2DArray sprites;
-in vec3 sprite_pos;
-in vec3 underline_pos;
-in vec3 strike_pos;
-in vec3 foreground;
-in vec3 background;
-in vec3 decoration_fg;
-out vec4 final_color;
-
-vec3 blend(float alpha, vec3 over, vec3 under) {
-    return over + (1 - alpha) * under;
-}
-
-void main() {
-    float text_alpha = texture(sprites, sprite_pos).r;
-    float underline_alpha = texture(sprites, underline_pos).r;
-    float strike_alpha = texture(sprites, strike_pos).r;
-    vec3 underline = underline_alpha * decoration_fg;
-    vec3 strike = strike_alpha * foreground;
-    vec3 fg = text_alpha * foreground;
-    vec3 decoration = blend(underline_alpha, underline, strike);
-    vec3 combined_fg = blend(text_alpha, fg, decoration);
-    float combined_alpha = max(max(underline_alpha, strike_alpha), text_alpha);
-    final_color = vec4(blend(combined_alpha, combined_fg, background), 1);
-}
-''')
-# }}}
-
-# cursor shader {{{
-
-cursor_shader = (
-    '''\
-uniform vec2 xpos;
-uniform vec2 ypos;
-
-const uvec2 pos_map[] = uvec2[4](
-    uvec2(1, 0),  // right, top
-    uvec2(1, 1),  // right, bottom
-    uvec2(0, 1),  // left, bottom
-    uvec2(0, 0)   // left, top
-);
-
-void main() {
-    uvec2 pos = pos_map[gl_VertexID];
-    gl_Position = vec4(xpos[pos[0]], ypos[pos[1]], 0, 1);
-}
-''',
-
-    '''\
-uniform vec4 color;
-out vec4 final_color;
-
-void main() {
-    final_color = color;
-}
-''')
-# }}}
-
-
-class Selection:  # {{{
-
-    __slots__ = tuple('in_progress start_x start_y start_scrolled_by end_x end_y end_scrolled_by'.split())
-
-    def __init__(self):
-        self.clear()
-
-    def clear(self):
-        self.in_progress = False
-        self.start_x = self.start_y = self.end_x = self.end_y = 0
-        self.start_scrolled_by = self.end_scrolled_by = 0
-
-    def limits(self, scrolled_by, lines, columns):
-
-        def coord(x, y, ydelta):
-            y = y - ydelta + scrolled_by
-            if y < 0:
-                x, y = 0, 0
-            elif y >= lines:
-                x, y = columns - 1, lines - 1
-            return x, y
-
-        a = coord(self.start_x, self.start_y, self.start_scrolled_by)
-        b = coord(self.end_x, self.end_y, self.end_scrolled_by)
-        return (a, b) if a[1] < b[1] or (a[1] == b[1] and a[0] <= b[0]) else (b, a)
-
-    def text(self, linebuf, historybuf):
-        sy = self.start_y - self.start_scrolled_by
-        ey = self.end_y - self.end_scrolled_by
-        if sy == ey and self.start_x == self.end_x:
-            return ''
-        a, b = (sy, self.start_x), (ey, self.end_x)
-        if a > b:
-            a, b = b, a
-
-        def line(y):
-            if y < 0:
-                return historybuf.line(-1 - y)
-            return linebuf.line(y)
-
-        lines = []
-        for y in range(a[0], b[0] + 1):
-            startx, endx = 0, linebuf.xnum - 1
-            if y == a[0]:
-                startx = max(0, min(a[1], endx))
-            if y == b[0]:
-                endx = max(0, min(b[1], endx))
-            l = line(y)
-            is_continued = l.is_continued()
-            if endx - startx >= linebuf.xnum - 1:
-                l = str(l).rstrip(' ')
-            else:
-                l = ''.join(l[x] for x in range(startx, endx + 1))
-            if not is_continued and startx == 0 and len(lines) > 0:
-                l = '\n' + l
-            lines.append(l)
-        return ''.join(lines)
-# }}}
-
-
-def calculate_gl_geometry(window_geometry):
-    dx, dy = 2 * cell_size.width / viewport_size.width, 2 * cell_size.height / viewport_size.height
-    xmargin = window_geometry.left / viewport_size.width
-    ymargin = window_geometry.top / viewport_size.height
-    xstart = -1 + 2 * xmargin
-    ystart = 1 - 2 * ymargin
-    return ScreenGeometry(xstart, ystart, window_geometry.xnum, window_geometry.ynum, dx, dy)
-
-
-def render_cells(buffer_id, sg, cell_program, sprites, invert_colors=False):
-    sprites.bind_sprite_map(buffer_id)
-    ul = cell_program.uniform_location
-    glUniform2ui(ul('dimensions'), sg.xnum, sg.ynum)
-    glUniform2ui(ul('color_indices'), 1 if invert_colors else 0, 0 if invert_colors else 1)
-    glUniform4f(ul('steps'), sg.xstart, sg.ystart, sg.dx, sg.dy)
-    glUniform1i(ul('sprites'), sprites.sampler_num)
-    glUniform1i(ul('sprite_map'), sprites.buffer_sampler_num)
-    glUniform2f(ul('sprite_layout'), *(sprites.layout))
-    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, sg.xnum * sg.ynum)
-
-
-class CharGrid:
-
-    url_pat = re.compile('(?:http|https|file|ftp)://\S+', re.IGNORECASE)
-
-    def __init__(self, screen, opts):
-        self.buffer_lock = Lock()
-        self.buffer_id = None
-        self.current_selection = Selection()
-        self.last_rendered_selection = self.current_selection.limits(0, screen.lines, screen.columns)
-        self.render_buf_is_dirty = True
-        self.render_data = None
-        self.scrolled_by = 0
-        self.color_profile = ColorProfile()
-        self.color_profile.update_ansi_color_table(build_ansi_color_table(opts))
-        self.screen = screen
-        self.opts = opts
-        self.default_bg = color_as_int(opts.background)
-        self.default_fg = color_as_int(opts.foreground)
-        self.dpix, self.dpiy = get_logical_dpi()
-        self.opts = opts
-        self.default_cursor = self.current_cursor = Cursor(0, 0, opts.cursor_shape, opts.cursor_blink_interval > 0)
-        self.opts = opts
-        self.highlight_fg, self.highlight_bg = map(color_as_int, (opts.selection_foreground, opts.selection_background))
-        self.sprite_map_type = self.main_sprite_map = self.scroll_sprite_map = self.render_buf = None
-
-        def escape(chars):
-            return ''.join(frozenset(chars)).replace('\\', r'\\').replace(']', r'\]').replace('-', r'\-')
-
-        try:
-            self.word_pat = re.compile(r'[\w{}]'.format(escape(self.opts.select_by_word_characters)), re.UNICODE)
-        except Exception:
-            safe_print('Invalid characters in select_by_word_characters, ignoring', file=sys.stderr)
-            self.word_pat = re.compile(r'[\w{}]'.format(escape(defaults.select_by_word_characters)), re.UNICODE)
-
-    def update_position(self, window_geometry):
-        self.screen_geometry = calculate_gl_geometry(window_geometry)
-
-    def resize(self, window_geometry):
-        self.update_position(window_geometry)
-        self.sprite_map_type = (GLuint * (self.screen_geometry.ynum * self.screen_geometry.xnum * DATA_CELL_SIZE))
-        self.main_sprite_map = self.sprite_map_type()
-        self.scroll_sprite_map = self.sprite_map_type()
-        with self.buffer_lock:
-            self.render_buf = self.sprite_map_type()
-            self.selection_buf = self.sprite_map_type()
-            self.render_buf_is_dirty = True
-            self.current_selection.clear()
-
-    def change_colors(self, changes):
-        dirtied = False
-
-        def item(raw):
-            if raw is None:
-                return 0
-            val = to_color(raw)
-            return None if val is None else (color_as_int(val) << 8) | 1
-
-        for which, val in changes.items():
-            val = item(val)
-            if val is None:
-                continue
-            dirtied = True
-            setattr(self.screen, which.name, val)
-        if dirtied:
-            self.screen.mark_as_dirty()
-
-    def scroll(self, amt, upwards=True):
-        if not isinstance(amt, int):
-            amt = {'line': 1, 'page': self.screen.lines - 1, 'full': self.screen.historybuf.count}[amt]
-        if not upwards:
-            amt *= -1
-        y = max(0, min(self.scrolled_by + amt, self.screen.historybuf.count))
-        if y != self.scrolled_by:
-            self.scrolled_by = y
-            self.update_cell_data()
-
-    def update_cell_data(self, force_full_refresh=False):
-        sprites = get_boss().sprites
-        is_dirty = self.screen.is_dirty()
-        with sprites.lock:
-            fg = self.screen.default_fg
-            fg = fg >> 8 if fg & 1 else self.default_fg
-            bg = self.screen.default_bg
-            bg = bg >> 8 if bg & 1 else self.default_bg
-            cursor_changed, history_line_added_count = self.screen.update_cell_data(
-                sprites.backend, self.color_profile, addressof(self.main_sprite_map), fg, bg, force_full_refresh)
-            if self.scrolled_by:
-                self.scrolled_by = min(self.scrolled_by + history_line_added_count, self.screen.historybuf.count)
-                self.screen.set_scroll_cell_data(
-                    sprites.backend, self.color_profile, addressof(self.main_sprite_map), fg, bg,
-                    self.scrolled_by, addressof(self.scroll_sprite_map))
-
-        data = self.scroll_sprite_map if self.scrolled_by else self.main_sprite_map
-        with self.buffer_lock:
-            if is_dirty:
-                self.current_selection.clear()
-            memmove(self.render_buf, data, sizeof(type(data)))
-            self.render_data = self.screen_geometry
-            self.render_buf_is_dirty = True
-        if cursor_changed:
-            c = self.screen.cursor
-            self.current_cursor = Cursor(c.x, c.y, c.shape, c.blink)
-
-    def cell_for_pos(self, x, y):
-        x, y = int(x // cell_size.width), int(y // cell_size.height)
-        if 0 <= x < self.screen.columns and 0 <= y < self.screen.lines:
-            return x, y
-        return None, None
-
-    def update_drag(self, is_press, mx, my):
-        x, y = self.cell_for_pos(mx, my)
-        if x is None:
-            x = 0 if mx <= cell_size.width else self.screen.columns - 1
-            y = 0 if my <= cell_size.height else self.screen.lines - 1
-        ps = None
-        with self.buffer_lock:
-            if is_press:
-                self.current_selection.start_x = self.current_selection.end_x = x
-                self.current_selection.start_y = self.current_selection.end_y = y
-                self.current_selection.start_scrolled_by = self.current_selection.end_scrolled_by = self.scrolled_by
-                self.current_selection.in_progress = True
-            elif self.current_selection.in_progress:
-                self.current_selection.end_x = x
-                self.current_selection.end_y = y
-                self.current_selection.end_scrolled_by = self.scrolled_by
-                if is_press is False:
-                    self.current_selection.in_progress = False
-                    ps = self.text_for_selection()
-        if ps and ps.strip():
-            set_primary_selection(ps)
-
-    def has_url_at(self, x, y):
-        x, y = self.cell_for_pos(x, y)
-        if x is not None:
-            l = self.screen_line(y)
-            if l is not None:
-                text = l.as_base_text()
-                for m in self.url_pat.finditer(text):
-                    if m.start() <= x < m.end():
-                        return True
-        return False
-
-    def click_url(self, x, y):
-        x, y = self.cell_for_pos(x, y)
-        if x is not None:
-            l = self.screen_line(y)
-            if l is not None:
-                text = l.as_base_text()
-                for m in self.url_pat.finditer(text):
-                    if m.start() <= x < m.end():
-                        url = ''.join(l[i] for i in range(*m.span())).rstrip('.')
-                        # Remove trailing "] and similar
-                        url = re.sub(r'''["'][)}\]]$''', '', url)
-                        # Remove closing trailing character if it is matched by it's
-                        # corresponding opening character before the url
-                        if m.start() > 0:
-                            before = l[m.start() - 1]
-                            closing = {'(': ')', '[': ']', '{': '}', '<': '>', '"': '"', "'": "'", '`': '`', '|': '|', ':': ':'}.get(before)
-                            if closing is not None and url.endswith(closing):
-                                url = url[:-1]
-                        if url:
-                            open_url(url, self.opts.open_url_with)
-
-    def screen_line(self, y):
-        ' Return the Line object corresponding to the yth line on the rendered screen '
-        if y >= 0 and y < self.screen.lines:
-            if self.scrolled_by:
-                if y < self.scrolled_by:
-                    return self.screen.historybuf.line(self.scrolled_by - 1 - y)
-                return self.screen.line(y - self.scrolled_by)
-            else:
-                return self.screen.line(y)
-
-    def multi_click(self, count, x, y):
-        x, y = self.cell_for_pos(x, y)
-        if x is not None:
-            line = self.screen_line(y)
-            if line is not None and count in (2, 3):
-                s = self.current_selection
-                s.start_scrolled_by = s.end_scrolled_by = self.scrolled_by
-                s.start_y = s.end_y = y
-                s.in_progress = False
-                if count == 3:
-                    for i in range(self.screen.columns):
-                        if line[i] != ' ':
-                            s.start_x = i
-                            break
-                    else:
-                        s.start_x = 0
-                    for i in range(self.screen.columns):
-                        c = self.screen.columns - 1 - i
-                        if line[c] != ' ':
-                            s.end_x = c
-                            break
-                    else:
-                        s.end_x = self.screen.columns - 1
-                elif count == 2:
-                    i = x
-                    while i >= 0 and self.word_pat.match(line[i]) is not None:
-                        i -= 1
-                    s.start_x = i if i == x else i + 1
-                    i = x
-                    while i < self.screen.columns and self.word_pat.match(line[i]) is not None:
-                        i += 1
-                    s.end_x = i if i == x else i - 1
-            ps = self.text_for_selection()
-            if ps:
-                set_primary_selection(ps)
-
-    def get_scrollback_as_ansi(self):
-        ans = []
-        self.screen.historybuf.as_ansi(ans.append)
-        self.screen.linebuf.as_ansi(ans.append)
-        return ''.join(ans).encode('utf-8')
-
-    def text_for_selection(self, sel=None):
-        s = sel or self.current_selection
-        return s.text(self.screen.linebuf, self.screen.historybuf)
-
-    def prepare_for_render(self, sprites):
-        with self.buffer_lock:
-            sg = self.render_data
-            if sg is None:
-                return
-            if self.buffer_id is None:
-                self.buffer_id = sprites.add_sprite_map()
-            buf = self.render_buf
-            start, end = sel = self.current_selection.limits(self.scrolled_by, self.screen.lines, self.screen.columns)
-            if start != end:
-                buf = self.selection_buf
-                if self.render_buf_is_dirty or sel != self.last_rendered_selection:
-                    memmove(buf, self.render_buf, sizeof(type(buf)))
-                    fg = self.screen.highlight_fg
-                    fg = fg >> 8 if fg & 1 else self.highlight_fg
-                    bg = self.screen.highlight_bg
-                    bg = bg >> 8 if bg & 1 else self.highlight_bg
-                    self.screen.apply_selection(addressof(buf), start[0], start[1], end[0], end[1], fg, bg)
-            if self.render_buf_is_dirty or self.last_rendered_selection != sel:
-                sprites.set_sprite_map(self.buffer_id, buf)
-                self.render_buf_is_dirty = False
-                self.last_rendered_selection = sel
-        return sg
-
-    def render_cells(self, sg, cell_program, sprites, invert_colors=False):
-        render_cells(self.buffer_id, sg, cell_program, sprites, invert_colors=invert_colors)
-
-    def render_cursor(self, sg, cursor_program, is_focused):
-        cursor = self.current_cursor
-        if not self.screen.cursor_visible or self.scrolled_by:
-            return
-
-        def width(w=2, vert=True):
-            dpi = self.dpix if vert else self.dpiy
-            w *= dpi / 72.0  # as pixels
-            factor = 2 / (viewport_size.width if vert else viewport_size.height)
-            return w * factor
-
-        ul = cursor_program.uniform_location
-        left = sg.xstart + cursor.x * sg.dx
-        top = sg.ystart - cursor.y * sg.dy
-        cc = self.screen.cursor_color
-        col = color_from_int(cc >> 8) if cc & 1 else self.opts.cursor
-        shape = cursor.shape or self.default_cursor.shape
-        alpha = self.opts.cursor_opacity
-        if alpha < 1.0 and shape == CURSOR_BLOCK:
-            glEnable(GL_BLEND)
-        mult = self.screen.current_char_width()
-        right = left + (width(1.5) if shape == CURSOR_BEAM else sg.dx * mult)
-        bottom = top - sg.dy
-        if shape == CURSOR_UNDERLINE:
-            top = bottom + width(vert=False)
-        glUniform4f(ul('color'), col[0] / 255.0, col[1] / 255.0, col[2] / 255.0, alpha)
-        glUniform2f(ul('xpos'), left, right)
-        glUniform2f(ul('ypos'), top, bottom)
-        glDrawArrays(GL_TRIANGLE_FAN if is_focused else GL_LINE_LOOP, 0, 4)
-        glDisable(GL_BLEND)

kitty/charsets.c

@@ -237,3 +237,60 @@ decode_utf8(uint32_t* state, uint32_t* codep, uint8_t byte) {
   *state = utf8_data[256 + *state*16 + type];
   return *state;
 }
+
+unsigned int
+encode_utf8(uint32_t ch, char* dest) {
+    if (ch < 0x80) {
+        dest[0] = (char)ch;
+        return 1;
+    }
+    if (ch < 0x800) {
+        dest[0] = (ch>>6) | 0xC0;
+        dest[1] = (ch & 0x3F) | 0x80;
+        return 2;
+    }
+    if (ch < 0x10000) {
+        dest[0] = (ch>>12) | 0xE0;
+        dest[1] = ((ch>>6) & 0x3F) | 0x80;
+        dest[2] = (ch & 0x3F) | 0x80;
+        return 3;
+    }
+    if (ch < 0x110000) {
+        dest[0] = (ch>>18) | 0xF0;
+        dest[1] = ((ch>>12) & 0x3F) | 0x80;
+        dest[2] = ((ch>>6) & 0x3F) | 0x80;
+        dest[3] = (ch & 0x3F) | 0x80;
+        return 4;
+    }
+    return 0;
+}
+
+
+// Base64
+// standard decoding using + and / with = being the padding character
+static uint8_t b64_decoding_table[256] = {
+0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+
+const char*
+base64_decode(const uint32_t *src, size_t src_sz, uint8_t *dest, size_t dest_capacity, size_t *dest_sz) {
+    if (!src_sz) { *dest_sz = 0; return NULL; }
+    if (src_sz % 4 != 0) return "base64 encoded data must have a length that is a multiple of four";
+    *dest_sz = (src_sz / 4) * 3;
+    if (src[src_sz - 1] == '=') (*dest_sz)--; 
+    if (src[src_sz - 2] == '=') (*dest_sz)--;
+    if (*dest_sz > dest_capacity) return "output buffer too small";
+    for (size_t i = 0, j = 0; i < src_sz;) {
+        uint32_t sextet_a = src[i] == '=' ? 0 & i++ : b64_decoding_table[src[i++] & 0xff];
+        uint32_t sextet_b = src[i] == '=' ? 0 & i++ : b64_decoding_table[src[i++] & 0xff];
+        uint32_t sextet_c = src[i] == '=' ? 0 & i++ : b64_decoding_table[src[i++] & 0xff];
+        uint32_t sextet_d = src[i] == '=' ? 0 & i++ : b64_decoding_table[src[i++] & 0xff];
+        uint32_t triple = (sextet_a << 3 * 6) + (sextet_b << 2 * 6) + (sextet_c << 1 * 6) + (sextet_d << 0 * 6);
+
+        if (j < *dest_sz) dest[j++] = (triple >> 2 * 8) & 0xFF;
+        if (j < *dest_sz) dest[j++] = (triple >> 1 * 8) & 0xFF;
+        if (j < *dest_sz) dest[j++] = (triple >> 0 * 8) & 0xFF;
+    }
+    return NULL;
+}

kitty/child-monitor.c

@@ -0,0 +1,886 @@
+/*
+ * child-monitor.c
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#ifdef __APPLE__
+#include <pthread.h>
+// I cant figure out how to get pthread.h to include this definition on macOS. MACOSX_DEPLOYMENT_TARGET does not work.
+extern int pthread_setname_np(const char *name);
+#else
+// Need _GNU_SOURCE for pthread_setname_np on linux
+#define _GNU_SOURCE
+#include <pthread.h>
+#undef _GNU_SOURCE
+#endif
+#include "state.h"
+#include "screen.h"
+#include <termios.h>
+#include <unistd.h>
+#include <float.h>
+#include <fcntl.h>
+#ifndef __APPLE__
+#include <stropts.h>
+#endif
+#include <sys/ioctl.h>
+#include <sys/wait.h>
+#include <signal.h>
+#include <GLFW/glfw3.h>
+
+#define EXTRA_FDS 2
+#define wakeup_main_loop glfwPostEmptyEvent
+
+static void (*parse_func)(Screen*, PyObject*);
+
+typedef struct {
+    Screen *screen;
+    bool needs_removal;
+    int fd;
+    unsigned long id;
+    pid_t pid;
+} Child;
+
+static const Child EMPTY_CHILD = {0};
+#define screen_mutex(op, which) \
+    pthread_mutex_##op(&screen->which##_buf_lock);
+#define children_mutex(op) \
+    pthread_mutex_##op(&children_lock);
+
+
+static Child children[MAX_CHILDREN] = {{0}};
+static Child scratch[MAX_CHILDREN] = {{0}};
+static Child add_queue[MAX_CHILDREN] = {{0}}, remove_queue[MAX_CHILDREN] = {{0}};
+static unsigned long remove_notify[MAX_CHILDREN] = {0};
+static size_t add_queue_count = 0, remove_queue_count = 0;
+static struct pollfd fds[MAX_CHILDREN + EXTRA_FDS] = {{0}};
+static pthread_mutex_t children_lock;
+static bool signal_received = false;
+static ChildMonitor *the_monitor = NULL;
+static uint8_t drain_buf[1024];
+static int signal_fds[2], wakeup_fds[2];
+static void *glfw_window_id = NULL;
+
+
+static inline void
+set_thread_name(const char *name) {
+    int ret = 0;
+#ifdef __APPLE__
+    ret = pthread_setname_np(name);
+#else
+    ret = pthread_setname_np(pthread_self(), name);
+#endif
+    if (ret != 0) perror("Failed to set thread name");
+}
+
+
+// Main thread functions {{{
+
+#define FREE_CHILD(x) \
+    Py_CLEAR((x).screen); x = EMPTY_CHILD;
+
+#define XREF_CHILD(x, OP) OP(x.screen); 
+#define INCREF_CHILD(x) XREF_CHILD(x, Py_INCREF)
+#define DECREF_CHILD(x) XREF_CHILD(x, Py_DECREF)
+
+// The max time (in secs) to wait for events from the window system
+// before ticking over the main loop. Negative values mean wait forever.
+static double maximum_wait = -1.0;
+
+static inline void
+set_maximum_wait(double val) {
+    if (val >= 0 && (val < maximum_wait || maximum_wait < 0)) maximum_wait = val;
+}
+
+static void
+handle_signal(int sig_num) {
+    int save_err = errno;
+    unsigned char byte = (unsigned char)sig_num;
+    while(true) {
+        ssize_t ret = write(signal_fds[1], &byte, 1);
+        if (ret < 0 && errno == EINTR) continue;
+        break;
+    }
+    errno = save_err;
+}
+
+static inline bool
+self_pipe(int fds[2]) {
+    int flags;
+    flags = pipe(fds);
+    if (flags != 0) return false;
+    flags = fcntl(fds[0], F_GETFD);
+    if (flags == -1) {  return false; }
+    if (fcntl(fds[0], F_SETFD, flags | FD_CLOEXEC) == -1) { return false; }
+    flags = fcntl(fds[0], F_GETFL);
+    if (flags == -1) { return false; }
+    if (fcntl(fds[0], F_SETFL, flags | O_NONBLOCK) == -1) { return false; }
+    return true;
+}
+
+static PyObject *
+new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
+    ChildMonitor *self;
+    PyObject *dump_callback, *death_notify, *wid; 
+    int ret;
+
+    if (the_monitor) { PyErr_SetString(PyExc_RuntimeError, "Can have only a single ChildMonitor instance"); return NULL; }
+    if (!PyArg_ParseTuple(args, "OOO", &wid, &death_notify, &dump_callback)) return NULL; 
+    glfw_window_id = PyLong_AsVoidPtr(wid);
+    if ((ret = pthread_mutex_init(&children_lock, NULL)) != 0) {
+        PyErr_Format(PyExc_RuntimeError, "Failed to create children_lock mutex: %s", strerror(ret));
+        return NULL;
+    }
+    if (!self_pipe(wakeup_fds)) return PyErr_SetFromErrno(PyExc_OSError);
+    if (!self_pipe(signal_fds)) return PyErr_SetFromErrno(PyExc_OSError);
+    if (signal(SIGINT, handle_signal) == SIG_ERR) return PyErr_SetFromErrno(PyExc_OSError);
+    if (signal(SIGTERM, handle_signal) == SIG_ERR) return PyErr_SetFromErrno(PyExc_OSError);
+    if (siginterrupt(SIGINT, false) != 0) return PyErr_SetFromErrno(PyExc_OSError);
+    if (siginterrupt(SIGTERM, false) != 0) return PyErr_SetFromErrno(PyExc_OSError);
+    self = (ChildMonitor *)type->tp_alloc(type, 0);
+    if (self == NULL) return PyErr_NoMemory();
+    self->death_notify = death_notify; Py_INCREF(death_notify);
+    if (dump_callback != Py_None) {
+        self->dump_callback = dump_callback; Py_INCREF(dump_callback);
+        parse_func = parse_worker_dump;
+    } else parse_func = parse_worker;
+    self->count = 0; 
+    fds[0].fd = wakeup_fds[0]; fds[1].fd = signal_fds[0];
+    fds[0].events = POLLIN; fds[1].events = POLLIN;
+    the_monitor = self;
+
+    return (PyObject*) self;
+}
+
+static void
+dealloc(ChildMonitor* self) {
+    pthread_mutex_destroy(&children_lock);
+    Py_CLEAR(self->dump_callback);
+    Py_CLEAR(self->death_notify);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+    while (remove_queue_count) {
+        remove_queue_count--;
+        FREE_CHILD(remove_queue[remove_queue_count]);
+    }
+    while (add_queue_count) {
+        add_queue_count--;
+        FREE_CHILD(add_queue[add_queue_count]);
+    }
+    close(wakeup_fds[0]);
+    close(wakeup_fds[1]);
+    close(signal_fds[0]); 
+    close(signal_fds[1]);
+}
+
+static void
+wakeup_io_loop() {
+    while(true) {
+        ssize_t ret = write(wakeup_fds[1], "w", 1);
+        if (ret < 0) {
+            if (errno == EINTR) continue;
+            perror("Failed to write to wakeup fd with error");
+        }
+        break;
+    }
+}
+
+static void* io_loop(void *data);
+
+static PyObject *
+start(ChildMonitor *self) {
+#define start_doc "start() -> Start the I/O thread"
+    int ret = pthread_create(&self->io_thread, NULL, io_loop, self);
+    if (ret != 0) return PyErr_SetFromErrno(PyExc_OSError);
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+join(ChildMonitor *self) {
+#define join_doc "join() -> Wait for the I/O thread to finish"
+    int ret = pthread_join(self->io_thread, NULL);
+    if (ret != 0) return PyErr_SetFromErrno(PyExc_OSError);
+    Py_RETURN_NONE;
+}
+
+
+static PyObject *
+wakeup(ChildMonitor UNUSED *self) {
+#define wakeup_doc "wakeup() -> wakeup the ChildMonitor I/O thread, forcing it to exit from poll() if it is waiting there."
+    wakeup_io_loop();
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+add_child(ChildMonitor *self, PyObject *args) {
+#define add_child_doc "add_child(id, pid, fd, screen) -> Add a child."
+    children_mutex(lock);
+    if (self->count + add_queue_count >= MAX_CHILDREN) { PyErr_SetString(PyExc_ValueError, "Too many children"); children_mutex(unlock); return NULL; }
+    add_queue[add_queue_count] = EMPTY_CHILD;
+#define A(attr) &add_queue[add_queue_count].attr
+    if (!PyArg_ParseTuple(args, "kiiO", A(id), A(pid), A(fd), A(screen))) {
+        children_mutex(unlock);
+        return NULL; 
+    }
+#undef A
+    INCREF_CHILD(add_queue[add_queue_count]);
+    add_queue_count++;
+    children_mutex(unlock);
+    wakeup_io_loop();
+    Py_RETURN_NONE;
+}
+
+bool
+schedule_write_to_child(unsigned long id, const char *data, size_t sz) {
+    ChildMonitor *self = the_monitor;
+    bool found = false;
+    children_mutex(lock);
+    for (size_t i = 0; i < self->count; i++) {
+        if (children[i].id == id) { 
+            found = true;
+            Screen *screen = children[i].screen;
+            screen_mutex(lock, write);
+            size_t space_left = screen->write_buf_sz - screen->write_buf_used;
+            if (space_left < sz) { 
+                if (screen->write_buf_used + sz > 100 * 1024 * 1024) {
+                    fprintf(stderr, "Too much data being sent to child with id: %lu, ignoring it\n", id);
+                    screen_mutex(unlock, write);
+                    break;
+                }
+                screen->write_buf_sz = screen->write_buf_used + sz;
+                screen->write_buf = PyMem_RawRealloc(screen->write_buf, screen->write_buf_sz);
+                if (screen->write_buf == NULL) { fatal("Out of memory."); }
+            }
+            memcpy(screen->write_buf + screen->write_buf_used, data, sz);
+            screen->write_buf_used += sz;
+            if (screen->write_buf_sz > BUFSIZ && screen->write_buf_used < BUFSIZ) {
+                screen->write_buf_sz = BUFSIZ;
+                screen->write_buf = PyMem_RawRealloc(screen->write_buf, screen->write_buf_sz);
+                if (screen->write_buf == NULL) { fatal("Out of memory."); }
+            }
+            if (screen->write_buf_used) wakeup_io_loop();
+            screen_mutex(unlock, write);
+            break;
+        }
+    }
+    children_mutex(unlock);
+    return found;
+}
+
+static PyObject *
+needs_write(ChildMonitor UNUSED *self, PyObject *args) {
+#define needs_write_doc "needs_write(id, data) -> Queue data to be written to child."
+    unsigned long id, sz;
+    const char *data;
+    if (!PyArg_ParseTuple(args, "ks#", &id, &data, &sz)) return NULL; 
+    if (schedule_write_to_child(id, data, sz)) { Py_RETURN_TRUE; }
+    Py_RETURN_FALSE;
+}
+
+static PyObject *
+shutdown(ChildMonitor *self) {
+#define shutdown_doc "shutdown() -> Shutdown the monitor loop."
+    signal(SIGINT, SIG_DFL);
+    signal(SIGTERM, SIG_DFL);
+    self->shutting_down = true;
+    Py_RETURN_NONE;
+}
+
+static inline void
+do_parse(ChildMonitor *self, Screen *screen, double now) {
+    screen_mutex(lock, read);
+    if (screen->read_buf_sz) {
+        double time_since_new_input = now - screen->new_input_at;
+        if (time_since_new_input >= OPT(input_delay)) {
+            parse_func(screen, self->dump_callback);
+            if (screen->read_buf_sz >= READ_BUF_SZ) wakeup_io_loop();  // Ensure the read fd has POLLIN set
+            screen->read_buf_sz = 0;
+            screen->new_input_at = 0;
+        } else set_maximum_wait(OPT(input_delay) - time_since_new_input);
+    }
+    screen_mutex(unlock, read);
+}
+
+static void
+parse_input(ChildMonitor *self) {
+    // Parse all available input that was read in the I/O thread.
+    size_t count = 0, remove_count = 0;
+    double now = monotonic();
+    children_mutex(lock);
+    while (remove_queue_count) {
+        remove_queue_count--; 
+        remove_notify[remove_count] = remove_queue[remove_queue_count].id;
+        remove_count++;
+        FREE_CHILD(remove_queue[remove_queue_count]);
+    }
+
+    if (UNLIKELY(signal_received)) {
+        glfwSetWindowShouldClose(glfw_window_id, true);
+    } else {
+        count = self->count;
+        for (size_t i = 0; i < count; i++) {
+            scratch[i] = children[i];
+            INCREF_CHILD(scratch[i]);
+        }
+    }
+    children_mutex(unlock);
+
+    while(remove_count) {
+        // must be done while no locks are held, since the locks are non-recursive and
+        // the python function could call into other functions in this module
+        remove_count--;
+        PyObject *t = PyObject_CallFunction(self->death_notify, "k", remove_notify[remove_count]);
+        if (t == NULL) PyErr_Print();
+        else Py_DECREF(t);
+    }
+
+    for (size_t i = 0; i < count; i++) {
+        if (!scratch[i].needs_removal) {
+            do_parse(self, scratch[i].screen, now);
+        }
+        DECREF_CHILD(scratch[i]);
+    }
+}
+
+static PyObject *
+mark_for_close(ChildMonitor *self, PyObject *args) {
+#define mark_for_close_doc "Mark a child to be removed from the child monitor"
+    unsigned long window_id;
+    if (!PyArg_ParseTuple(args, "k", &window_id)) return NULL;
+    children_mutex(lock);
+    for (size_t i = 0; i < self->count; i++) {
+        if (children[i].id == window_id) {
+            children[i].needs_removal = true;
+            break;
+        }
+    }
+    children_mutex(unlock);
+    wakeup_io_loop();
+    Py_RETURN_NONE;
+}
+
+static inline bool
+pty_resize(int fd, struct winsize *dim) {
+    while(true) {
+        if (ioctl(fd, TIOCSWINSZ, dim) == -1) {
+            if (errno == EINTR) continue;
+            if (errno != EBADF && errno != ENOTTY) {
+                fprintf(stderr, "Failed to resize tty associated with fd: %d with error: %s", fd, strerror(errno));
+                return false;
+            }
+        }
+        break;
+    }
+    return true;
+}
+
+static PyObject *
+resize_pty(ChildMonitor *self, PyObject *args) {
+#define resize_pty_doc "Resize the pty associated with the specified child"
+    unsigned long window_id;
+    struct winsize dim;
+    int fd = -1;
+    if (!PyArg_ParseTuple(args, "kHHHH", &window_id, &dim.ws_row, &dim.ws_col, &dim.ws_xpixel, &dim.ws_ypixel)) return NULL;
+    children_mutex(lock);
+#define FIND(queue, count) { \
+    for (size_t i = 0; i < count; i++) { \
+        if (queue[i].id == window_id) { \
+            fd = queue[i].fd; \
+            break; \
+        } \
+    }}
+    FIND(children, self->count);
+    if (fd == -1) FIND(add_queue, add_queue_count);
+    if (fd != -1) {
+        if (!pty_resize(fd, &dim)) PyErr_SetFromErrno(PyExc_OSError);
+    } else fprintf(stderr, "Failed to send resize signal to child with id: %lu (children count: %u) (add queue: %lu)\n", window_id, self->count, add_queue_count);
+    children_mutex(unlock);
+    if (PyErr_Occurred()) return NULL;
+    Py_RETURN_NONE;
+}
+
+bool
+set_iutf8(int UNUSED fd, bool UNUSED on) {
+#ifdef IUTF8
+    struct termios attrs;
+    if (tcgetattr(fd, &attrs) != 0) return false;
+    if (on) attrs.c_iflag |= IUTF8;
+    else attrs.c_iflag &= ~IUTF8;
+    if (tcsetattr(fd, TCSANOW, &attrs) != 0) return false;
+#endif
+    return true;
+}
+
+static PyObject*
+pyset_iutf8(ChildMonitor *self, PyObject *args) {
+    unsigned long window_id;
+    int on;
+    PyObject *found = Py_False;
+    if (!PyArg_ParseTuple(args, "kp", &window_id, &on)) return NULL;
+    children_mutex(lock);
+    for (size_t i = 0; i < self->count; i++) {
+        if (children[i].id == window_id) {
+            found = Py_True;
+            if (!set_iutf8(fds[EXTRA_FDS + i].fd, on & 1)) PyErr_SetFromErrno(PyExc_OSError);
+            break;
+        }
+    }
+    children_mutex(unlock);
+    if (PyErr_Occurred()) return NULL;
+    Py_INCREF(found);
+    return found;
+}
+
+#undef FREE_CHILD
+#undef INCREF_CHILD
+#undef DECREF_CHILD
+
+static double last_render_at = -DBL_MAX;
+draw_borders_func draw_borders = NULL;
+draw_cells_func draw_cells = NULL;
+draw_cursor_func draw_cursor = NULL;
+
+static inline double
+cursor_width(double w, bool vert) {
+    double dpi = vert ? global_state.logical_dpi_x : global_state.logical_dpi_y;
+    double ans = w * dpi / 72.0;  // as pixels
+    double factor = 2.0 / (vert ? global_state.viewport_width : global_state.viewport_height);
+    return ans * factor;
+}
+
+extern void cocoa_update_title(PyObject*);
+
+static inline void
+collect_cursor_info(CursorRenderInfo *ans, Window *w, double now) {
+    ScreenRenderData *rd = &w->render_data;
+    if (rd->screen->scrolled_by || !screen_is_cursor_visible(rd->screen)) {
+        ans->is_visible = false;
+        return;
+    }
+    double time_since_start_blink = now - global_state.cursor_blink_zero_time;
+    bool cursor_blinking = OPT(cursor_blink_interval) > 0 && global_state.application_focused && time_since_start_blink <= OPT(cursor_stop_blinking_after) ? true : false;
+    bool do_draw_cursor = true;
+    if (cursor_blinking) {
+        int t = (int)(time_since_start_blink * 1000);
+        int d = (int)(OPT(cursor_blink_interval) * 1000);
+        int n = t / d;
+        do_draw_cursor = n % 2 == 0 ? true : false;
+        double bucket = (n + 1) * d;
+        double delay = (bucket / 1000.0) - time_since_start_blink;
+        set_maximum_wait(delay);
+    }
+    if (!do_draw_cursor) { ans->is_visible = false; return; }
+    ans->is_visible = true;
+    ColorProfile *cp = rd->screen->color_profile;
+    Cursor *cursor = rd->screen->cursor;
+    ans->shape = cursor->shape ? cursor->shape : OPT(cursor_shape);
+    ans->color = colorprofile_to_color(cp, cp->overridden.cursor_color, cp->configured.cursor_color);
+    if (ans->shape == CURSOR_BLOCK) return;
+    double left = rd->xstart + cursor->x * rd->dx;
+    double top = rd->ystart - cursor->y * rd->dy;
+    unsigned long mult = MAX(1, screen_current_char_width(rd->screen));
+    double right = left + (ans->shape == CURSOR_BEAM ? cursor_width(1.5, true) : rd->dx * mult);
+    double bottom = top - rd->dy;
+    if (ans->shape == CURSOR_UNDERLINE) top = bottom + cursor_width(2.0, false);
+    ans->left = left; ans->right = right; ans->top = top; ans->bottom = bottom;
+}
+
+static inline void
+update_window_title(Window *w) {
+    if (w->title && w->title != global_state.application_title) {
+        global_state.application_title = w->title;
+        glfwSetWindowTitle(glfw_window_id, PyUnicode_AsUTF8(w->title));
+#ifdef __APPLE__
+        cocoa_update_title(w->title);
+#endif
+    }
+}
+
+static PyObject*
+simple_render_screen(PyObject UNUSED *self, PyObject *args) {
+#define simple_render_screen_doc "Render a Screen object, with no cursor"
+    Screen *screen; 
+    ssize_t vao_idx, gvao_idx;
+    float xstart, ystart, dx, dy;
+    if (!PyArg_ParseTuple(args, "O!nnffff", &Screen_Type, &screen, &vao_idx, &gvao_idx, &xstart, &ystart, &dx, &dy)) return NULL;
+    static CursorRenderInfo cursor_info = {0};
+    draw_cells(vao_idx, gvao_idx, xstart, ystart, dx, dy, screen, &cursor_info);
+    Py_RETURN_NONE;
+}
+
+static inline void
+render(double now) {
+    double time_since_last_render = now - last_render_at;
+    static CursorRenderInfo cursor_info;
+    if (time_since_last_render < OPT(repaint_delay)) { 
+        set_maximum_wait(OPT(repaint_delay) - time_since_last_render);
+        return;
+    }
+
+    draw_borders();
+    cursor_info.is_visible = false;
+#define TD global_state.tab_bar_render_data
+    if (TD.screen && global_state.num_tabs > 1) draw_cells(TD.vao_idx, 0, TD.xstart, TD.ystart, TD.dx, TD.dy, TD.screen, &cursor_info);
+#undef TD
+    if (global_state.num_tabs) {
+        Tab *tab = global_state.tabs + global_state.active_tab;
+        for (unsigned int i = 0; i < tab->num_windows; i++) {
+            Window *w = tab->windows + i;
+#define WD w->render_data
+            if (w->visible && WD.screen) {
+                if (w->last_drag_scroll_at > 0) {
+                    if (now - w->last_drag_scroll_at >= 0.02) { 
+                        if (drag_scroll(w)) {
+                            w->last_drag_scroll_at = now;
+                            set_maximum_wait(0.02);
+                        } else w->last_drag_scroll_at = 0;
+                    } else set_maximum_wait(now - w->last_drag_scroll_at);
+                }
+                bool is_active_window = i == tab->active_window;
+                if (is_active_window) {
+                    collect_cursor_info(&cursor_info, w, now);
+                    update_window_title(w);
+                } else cursor_info.is_visible = false;
+                draw_cells(WD.vao_idx, WD.gvao_idx, WD.xstart, WD.ystart, WD.dx, WD.dy, WD.screen, &cursor_info);
+                if (is_active_window && cursor_info.is_visible && cursor_info.shape != CURSOR_BLOCK) draw_cursor(&cursor_info);
+                if (WD.screen->start_visual_bell_at != 0) {
+                    double bell_left = global_state.opts.visual_bell_duration - (now - WD.screen->start_visual_bell_at);
+                    set_maximum_wait(bell_left);
+                }
+            }
+        }
+        
+#undef WD
+    }
+    glfwSwapBuffers(glfw_window_id);
+    last_render_at = now;
+}
+
+typedef struct { int fd; uint8_t *buf; size_t sz; } ThreadWriteData;
+
+static inline ThreadWriteData*
+alloc_twd(size_t sz) {
+    ThreadWriteData *data = malloc(sizeof(ThreadWriteData));
+    if (data != NULL) {
+        data->sz = sz;
+        data->buf = malloc(sz);
+        if (data->buf == NULL) { free(data); data = NULL; }
+    }
+    return data;
+}
+
+static inline void
+free_twd(ThreadWriteData *x) {
+    if (x != NULL) free(x->buf);
+    free(x);
+}
+
+static void*
+thread_write(void *x) {
+    ThreadWriteData *data = (ThreadWriteData*)x;
+    set_thread_name("KittyWriteStdin");
+    FILE *f = fdopen(data->fd, "w");
+    if (fwrite(data->buf, 1, data->sz, f) != data->sz) {
+        fprintf(stderr, "Failed to write all data\n");
+    }
+    fclose(f);
+    free_twd(data);
+    return 0;
+}
+
+PyObject*
+cm_thread_write(PyObject UNUSED *self, PyObject *args) {
+    static pthread_t thread;
+    int fd;
+    Py_ssize_t sz;
+    const char *buf;
+    if (!PyArg_ParseTuple(args, "is#", &fd, &buf, &sz)) return NULL;
+    ThreadWriteData *data = alloc_twd(sz);
+    if (data == NULL) return PyErr_NoMemory();
+    data->fd = fd; 
+    memcpy(data->buf, buf, data->sz);
+    int ret = pthread_create(&thread, NULL, thread_write, data);
+    if (ret != 0) { free_twd(data); return PyErr_SetFromErrno(PyExc_OSError); }
+    Py_RETURN_NONE;
+}
+
+static inline void
+hide_mouse(double now) {
+    if (glfwGetInputMode(glfw_window_id, GLFW_CURSOR) == GLFW_CURSOR_NORMAL && OPT(mouse_hide_wait) > 0 && now - global_state.last_mouse_activity_at > OPT(mouse_hide_wait)) {
+        glfwSetInputMode(glfw_window_id, GLFW_CURSOR, GLFW_CURSOR_HIDDEN);
+    }
+}
+
+
+static inline void
+wait_for_events() {
+    if (maximum_wait < 0) glfwWaitEvents();
+    else if (maximum_wait > 0) glfwWaitEventsTimeout(maximum_wait);
+    maximum_wait = -1;
+}
+
+
+static PyObject*
+main_loop(ChildMonitor *self) {
+#define main_loop_doc "The main thread loop"
+    while (!glfwWindowShouldClose(glfw_window_id)) {
+        double now = monotonic();
+        render(now);
+        hide_mouse(now);
+        wait_for_events();
+        parse_input(self);
+    }
+    if (PyErr_Occurred()) return NULL;
+    Py_RETURN_NONE;
+}
+
+// }}}
+
+// I/O thread functions {{{
+
+static inline void
+add_children(ChildMonitor *self) {
+    for (; add_queue_count > 0 && self->count < MAX_CHILDREN;) {
+        add_queue_count--;
+        children[self->count] = add_queue[add_queue_count];
+        add_queue[add_queue_count] = EMPTY_CHILD;
+        fds[EXTRA_FDS + self->count].fd = children[self->count].fd;
+        fds[EXTRA_FDS + self->count].events = POLLIN;
+        self->count++;
+    }
+}
+
+
+static inline void
+hangup(pid_t pid) {
+    errno = 0;
+    pid_t pgid = getpgid(pid);
+    if (errno == ESRCH) return;
+    if (errno != 0) { perror("Failed to get process group id for child"); return; }
+    if (killpg(pgid, SIGHUP) != 0) {
+        if (errno != ESRCH) perror("Failed to kill child"); 
+    }
+}
+
+
+static inline void
+cleanup_child(ssize_t i) {
+    close(children[i].fd);
+    hangup(children[i].pid);
+}
+
+
+static inline void
+remove_children(ChildMonitor *self) {
+    if (self->count > 0) {
+        size_t count = 0; 
+        for (ssize_t i = self->count - 1; i >= 0; i--) {
+            if (children[i].needs_removal) {
+                count++;
+                cleanup_child(i);
+                remove_queue[remove_queue_count] = children[i];
+                remove_queue_count++;
+                children[i] = EMPTY_CHILD;
+                fds[EXTRA_FDS + i].fd = -1;
+                size_t num_to_right = self->count - 1 - i;
+                if (num_to_right > 0) {
+                    memmove(children + i, children + i + 1, num_to_right * sizeof(Child));
+                    memmove(fds + EXTRA_FDS + i, fds + EXTRA_FDS + i + 1, num_to_right * sizeof(struct pollfd));
+                }
+            }
+        }
+        self->count -= count;
+    }
+}
+
+
+static bool
+read_bytes(int fd, Screen *screen) {
+    ssize_t len;
+    size_t available_buffer_space, orig_sz;
+
+    screen_mutex(lock, read);
+    orig_sz = screen->read_buf_sz;
+    if (orig_sz >= READ_BUF_SZ) { screen_mutex(unlock, read); return true; }  // screen read buffer is full
+    available_buffer_space = READ_BUF_SZ - orig_sz;
+    screen_mutex(unlock, read);
+
+    while(true) {
+        len = read(fd, screen->read_buf + orig_sz, available_buffer_space);
+        if (len < 0) {
+            if (errno == EINTR) continue;
+            if (errno != EIO) perror("Call to read() from child fd failed");
+            return false;
+        }
+        break;
+    }
+    if (UNLIKELY(len == 0)) return false;
+
+    screen_mutex(lock, read);
+    if (screen->new_input_at == 0) screen->new_input_at = monotonic();
+    if (orig_sz != screen->read_buf_sz) {
+        // The other thread consumed some of the screen read buffer
+        memmove(screen->read_buf + screen->read_buf_sz, screen->read_buf + orig_sz, len);
+    }
+    screen->read_buf_sz += len;
+    screen_mutex(unlock, read);
+    return true;
+}
+
+
+static inline void
+drain_fd(int fd) {
+    while(true) {
+        ssize_t len = read(fd, drain_buf, sizeof(drain_buf));
+        if (len < 0) {
+            if (errno == EINTR) continue;
+            if (errno != EIO) perror("Call to read() from drain fd failed");
+            break;
+        }
+        break;
+    }
+}
+
+static inline void
+write_to_child(int fd, Screen *screen) {
+    size_t written = 0;
+    ssize_t ret = 0;
+    screen_mutex(lock, write);
+    while (written < screen->write_buf_used) {
+        ret = write(fd, screen->write_buf + written, screen->write_buf_used - written);
+        if (ret > 0) { written += ret; }
+        else if (ret == 0) { 
+            // could mean anything, ignore
+            break;
+        } else {
+            if (errno == EINTR) continue;
+            if (errno == EWOULDBLOCK || errno == EAGAIN) break;
+            perror("Call to write() to child fd failed, discarding data.");
+            written = screen->write_buf_used;
+        }
+    }
+    screen->write_buf_used -= written;
+    screen_mutex(unlock, write);
+}
+
+static void*
+io_loop(void *data) {
+    // The I/O thread loop
+    size_t i;
+    int ret;
+    bool has_more, data_received; 
+    Screen *screen;
+    ChildMonitor *self = (ChildMonitor*)data;
+    set_thread_name("KittyChildMon");
+
+    while (LIKELY(!self->shutting_down)) {
+        children_mutex(lock);
+        remove_children(self);
+        add_children(self);
+        children_mutex(unlock);
+        data_received = false;
+        for (i = 0; i < self->count + EXTRA_FDS; i++) fds[i].revents = 0;
+        for (i = 0; i < self->count; i++) {
+            screen = children[i].screen;
+            /* printf("i:%lu id:%lu fd: %d read_buf_sz: %lu write_buf_used: %lu\n", i, children[i].id, children[i].fd, screen->read_buf_sz, screen->write_buf_used); */
+            screen_mutex(lock, read); screen_mutex(lock, write);
+            fds[EXTRA_FDS + i].events = (screen->read_buf_sz < READ_BUF_SZ ? POLLIN : 0) | (screen->write_buf_used ? POLLOUT  : 0);
+            screen_mutex(unlock, read); screen_mutex(unlock, write);
+        }
+        ret = poll(fds, self->count + EXTRA_FDS, -1);
+        if (ret > 0) {
+            if (fds[0].revents && POLLIN) drain_fd(fds[0].fd); // wakeup
+            if (fds[1].revents && POLLIN) { 
+                data_received = true;
+                drain_fd(fds[1].fd);
+                children_mutex(lock);
+                signal_received = true;
+                children_mutex(unlock);
+            }
+            for (i = 0; i < self->count; i++) {
+                if (fds[EXTRA_FDS + i].revents & (POLLIN | POLLHUP)) {
+                    data_received = true;
+                    has_more = read_bytes(fds[EXTRA_FDS + i].fd, children[i].screen);
+                    if (!has_more) { 
+                        // child is dead
+                        children_mutex(lock);
+                        children[i].needs_removal = true;
+                        children_mutex(unlock);
+                    }
+                }
+                if (fds[EXTRA_FDS + i].revents & POLLOUT) {
+                    write_to_child(children[i].fd, children[i].screen);
+                }
+                if (fds[EXTRA_FDS + i].revents & POLLNVAL) {
+                    // fd was closed
+                    children_mutex(lock);
+                    children[i].needs_removal = true;
+                    children_mutex(unlock);
+                    fprintf(stderr, "The child %lu had its fd unexpectedly closed\n", children[i].id);
+                }
+            }
+#ifdef DEBUG_POLL_EVENTS
+            for (i = 0; i < self->count + EXTRA_FDS; i++) {
+#define P(w) if (fds[i].revents & w) printf("i:%lu %s\n", i, #w);
+                P(POLLIN); P(POLLPRI); P(POLLOUT); P(POLLERR); P(POLLHUP); P(POLLNVAL);
+#undef P
+            }
+#endif
+        } else if (ret < 0) {
+            if (errno != EAGAIN && errno != EINTR) {
+                perror("Call to poll() failed");
+            }
+        }
+        if (data_received) wakeup_main_loop();
+    }
+    children_mutex(lock);
+    for (i = 0; i < self->count; i++) children[i].needs_removal = true;
+    remove_children(self);
+    children_mutex(unlock);
+    return 0;
+}
+// }}}
+
+// Boilerplate {{{
+static PyMethodDef methods[] = {
+    METHOD(add_child, METH_VARARGS)
+    METHOD(needs_write, METH_VARARGS)
+    METHOD(start, METH_NOARGS)
+    METHOD(join, METH_NOARGS)
+    METHOD(wakeup, METH_NOARGS)
+    METHOD(shutdown, METH_NOARGS)
+    METHOD(main_loop, METH_NOARGS)
+    METHOD(mark_for_close, METH_VARARGS)
+    METHOD(resize_pty, METH_VARARGS)
+    {"set_iutf8", (PyCFunction)pyset_iutf8, METH_VARARGS, ""},
+    {NULL}  /* Sentinel */
+};
+
+
+PyTypeObject ChildMonitor_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "fast_data_types.ChildMonitor",
+    .tp_basicsize = sizeof(ChildMonitor),
+    .tp_dealloc = (destructor)dealloc, 
+    .tp_flags = Py_TPFLAGS_DEFAULT,        
+    .tp_doc = "ChildMonitor",
+    .tp_methods = methods,
+    .tp_new = new,                
+};
+
+static PyMethodDef module_methods[] = {
+    METHOD(simple_render_screen, METH_VARARGS)
+    {NULL}  /* Sentinel */
+};
+
+bool
+init_child_monitor(PyObject *module) {
+    if (PyType_Ready(&ChildMonitor_Type) < 0) return false;
+    if (PyModule_AddObject(module, "ChildMonitor", (PyObject *)&ChildMonitor_Type) != 0) return false;
+    Py_INCREF(&ChildMonitor_Type); 
+    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
+    return true;
+}
+
+// }}}
+

kitty/child.py

@@ -2,15 +2,13 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
-import termios
-import struct
 import fcntl
-import signal
-from threading import Thread
+import os
+import sys
+
+import kitty.fast_data_types as fast_data_types
 
 from .constants import terminfo_dir
-import kitty.fast_data_types as fast_data_types
 
 
 def remove_cloexec(fd):
@@ -34,12 +32,11 @@ class Child:
         self.forked = True
         master, slave = os.openpty()  # Note that master and slave are in blocking mode
         remove_cloexec(slave)
-        self.set_iutf8(fd=master)
+        fast_data_types.set_iutf8(master, True)
         stdin, self.stdin = self.stdin, None
         if stdin is not None:
             stdin_read_fd, stdin_write_fd = os.pipe()
             remove_cloexec(stdin_read_fd)
-            stdin_file = os.fdopen(stdin_write_fd, 'wb')
         pid = os.fork()
         if pid == 0:  # child
             try:
@@ -64,50 +61,18 @@ class Child:
             try:
                 os.execvp(self.argv[0], self.argv)
             except Exception as err:
+                # Report he failure and exec a shell instead so that
+                # we are not left with a forked but not execed process
                 print('Could not launch:', self.argv[0])
                 print('\t', err)
-                input('\nPress Enter to exit:')
+                print('\nPress Enter to exit:', end=' ')
+                sys.stdout.flush()
+                os.execvp('/bin/sh', ['/bin/sh', '-c', 'read w'])
         else:  # master
             os.close(slave)
             self.pid = pid
             self.child_fd = master
             if stdin is not None:
-                t = Thread(name='WriteStdin', target=stdin_file.write, args=(stdin,))
-                t.daemon = True
-                t.start()
+                os.close(stdin_read_fd)
+                fast_data_types.thread_write(stdin_write_fd, stdin)
             return pid
-
-    def resize_pty(self, w, h, ww, wh):
-        if self.child_fd is not None:
-            fcntl.ioctl(self.child_fd, termios.TIOCSWINSZ, struct.pack('4H', h, w, ww, wh))
-
-    def set_iutf8(self, on=True, fd=None):
-        fd = fd or self.child_fd
-        if fd is not None and hasattr(fast_data_types, 'IUTF8'):
-            attrs = termios.tcgetattr(fd)
-            if on:
-                attrs[0] |= fast_data_types.IUTF8
-            else:
-                attrs[0] &= ~fast_data_types.IUTF8
-            termios.tcsetattr(fd, termios.TCSANOW, attrs)
-
-    def hangup(self):
-        if self.pid is not None:
-            pid, self.pid = self.pid, None
-            try:
-                pgrp = os.getpgid(pid)
-            except ProcessLookupError:
-                return
-            os.killpg(pgrp, signal.SIGHUP)
-            os.close(self.child_fd)
-            self.child_fd = None
-
-    def __del__(self):
-        self.hangup()
-
-    def get_child_status(self):
-        if self.pid is not None:
-            try:
-                return os.waitid(os.P_PID, self.pid, os.WEXITED | os.WNOHANG)
-            except ChildProcessError:
-                self.pid = None

kitty/client.py

@@ -125,14 +125,14 @@ def write_osc(code, string=''):
     write(OSC + str(code) + string + '\x07')
 
 
-set_dynamic_color = write_osc
+set_dynamic_color = set_color_table_color = write_osc
 
 
 def replay(raw):
     for line in raw.splitlines():
         if line.strip():
             cmd, rest = line.partition(' ')[::2]
-            if cmd in {'draw', 'set_title', 'set_icon'}:
+            if cmd in {'draw', 'set_title', 'set_icon', 'set_dynamic_color', 'set_color_table_color'}:
                 globals()[cmd](rest)
             else:
                 rest = map(int, rest.split()) if rest else ()

kitty/cocoa_window.m

@@ -9,19 +9,193 @@
 #include "data-types.h"
 #include <Cocoa/Cocoa.h>
 
-#ifndef NSWindowStyleMaskTitled
-#define NSWindowStyleMaskTitled NSTitledWindowMask 
+#include <AvailabilityMacros.h>
+// Needed for _NSGetProgname
+#include <crt_externs.h>
+
+#if (MAC_OS_X_VERSION_MAX_ALLOWED < 101200)
+#define NSWindowStyleMaskResizable NSResizableWindowMask
+#define NSEventModifierFlagOption NSAlternateKeyMask
+#define NSEventModifierFlagCommand NSCommandKeyMask
+#define NSEventModifierFlagControl NSControlKeyMask
 #endif
 
+@interface MenuDispatcher : NSObject
+@end
+
+@implementation MenuDispatcher
+@end
+
+static NSObject* menu_dispatcher = NULL;
+static NSMenuItem* title_menu = NULL;
+
+static NSString* 
+find_app_name(void) {
+    size_t i;
+    NSDictionary* infoDictionary = [[NSBundle mainBundle] infoDictionary];
+
+    // Keys to search for as potential application names
+    NSString* name_keys[] =
+    {
+        @"CFBundleDisplayName",
+        @"CFBundleName",
+        @"CFBundleExecutable",
+    };
+
+    for (i = 0;  i < sizeof(name_keys) / sizeof(name_keys[0]);  i++)
+    {
+        id name = [infoDictionary objectForKey:name_keys[i]];
+        if (name &&
+            [name isKindOfClass:[NSString class]] &&
+            ![name isEqualToString:@""])
+        {
+            return name;
+        }
+    }
+
+    char** progname = _NSGetProgname();
+    if (progname && *progname)
+        return [NSString stringWithUTF8String:*progname];
+
+    // Really shouldn't get here
+    return @"kitty";
+}
+
+
 PyObject*
-cocoa_hide_titlebar(PyObject UNUSED *self, PyObject *window_id) {
+cocoa_init(PyObject UNUSED *_self) {
+    // Press and Hold prevents some keys from emitting repeated characters
+    // See https://github.com/glfw/glfw/issues/1010
+    NSDictionary* defaults = [NSDictionary dictionaryWithObjectsAndKeys:[NSNumber numberWithBool:NO], @"ApplePressAndHoldEnabled", nil];
+    [[NSUserDefaults standardUserDefaults] registerDefaults:defaults];
+    Py_RETURN_NONE;
+}
+
+
+PyObject*
+cocoa_create_global_menu(PyObject UNUSED *_self) {
+    if (menu_dispatcher != NULL) { Py_RETURN_NONE; }
+    NSString* app_name = find_app_name();
+    menu_dispatcher = [[MenuDispatcher alloc] init];
+    NSMenu* bar = [[NSMenu alloc] init];
+    [NSApp setMainMenu:bar];
+
+    NSMenuItem* appMenuItem =
+        [bar addItemWithTitle:@"" action:NULL keyEquivalent:@""];
+    NSMenu* appMenu = [[NSMenu alloc] init];
+    [appMenuItem setSubmenu:appMenu];
+
+    [appMenu addItemWithTitle:[NSString stringWithFormat:@"About %@", app_name]
+                       action:@selector(orderFrontStandardAboutPanel:)
+                       keyEquivalent:@""];
+    [appMenu addItem:[NSMenuItem separatorItem]];
+    [appMenu addItemWithTitle:[NSString stringWithFormat:@"Hide %@", app_name]
+                       action:@selector(hide:)
+                keyEquivalent:@"h"];
+    [[appMenu addItemWithTitle:@"Hide Others"
+                       action:@selector(hideOtherApplications:)
+                keyEquivalent:@"h"]
+        setKeyEquivalentModifierMask:NSEventModifierFlagOption | NSEventModifierFlagCommand];
+    [appMenu addItemWithTitle:@"Show All"
+                       action:@selector(unhideAllApplications:)
+                keyEquivalent:@""];
+    [appMenu addItem:[NSMenuItem separatorItem]];
+
+    NSMenu* servicesMenu = [[NSMenu alloc] init];
+    [NSApp setServicesMenu:servicesMenu];
+    [[appMenu addItemWithTitle:@"Services"
+                       action:NULL
+                keyEquivalent:@""] setSubmenu:servicesMenu];
+    [servicesMenu release];
+
+    [appMenu addItem:[NSMenuItem separatorItem]];
+
+    [appMenu addItemWithTitle:[NSString stringWithFormat:@"Quit %@", app_name]
+                       action:@selector(terminate:)
+                       keyEquivalent:@"q"];
+    [appMenu release];
+
+    NSMenuItem* windowMenuItem =
+        [bar addItemWithTitle:@"" action:NULL keyEquivalent:@""];
+    NSMenu* windowMenu = [[NSMenu alloc] initWithTitle:@"Window"];
+    [windowMenuItem setSubmenu:windowMenu];
+
+    [windowMenu addItemWithTitle:@"Minimize"
+                          action:@selector(performMiniaturize:)
+                   keyEquivalent:@"m"];
+    [windowMenu addItemWithTitle:@"Zoom"
+                          action:@selector(performZoom:)
+                   keyEquivalent:@""];
+    [windowMenu addItem:[NSMenuItem separatorItem]];
+    [windowMenu addItemWithTitle:@"Bring All to Front"
+                          action:@selector(arrangeInFront:)
+                   keyEquivalent:@""];
+
+    [windowMenu addItem:[NSMenuItem separatorItem]];
+    [[windowMenu addItemWithTitle:@"Enter Full Screen"
+                           action:@selector(toggleFullScreen:)
+                    keyEquivalent:@"f"]
+     setKeyEquivalentModifierMask:NSEventModifierFlagControl | NSEventModifierFlagCommand];
+    [NSApp setWindowsMenu:windowMenu];
+    [windowMenu release];
+
+
+    [bar release];
+    Py_RETURN_NONE;
+}
+
+void
+cocoa_update_title(PyObject *pytitle) {
+    NSString *title = [[NSString alloc] initWithUTF8String:PyUnicode_AsUTF8(pytitle)];
+    NSMenu *bar = [NSApp mainMenu];
+    if (title_menu != NULL) {
+        [bar removeItem:title_menu];
+    }
+    title_menu = [bar addItemWithTitle:@"" action:NULL keyEquivalent:@""];
+    NSMenu *m = [[NSMenu alloc] initWithTitle:[NSString stringWithFormat:@" :: %@", title]];
+    [title_menu setSubmenu:m];
+    [m release];
+    [title release];
+}
+
+PyObject*
+cocoa_make_window_resizable(PyObject UNUSED *self, PyObject *window_id) {
     NSWindow *window = (NSWindow*)PyLong_AsVoidPtr(window_id);
     
     @try {
         [window setStyleMask:
-            [window styleMask] & ~NSWindowStyleMaskTitled];
+            [window styleMask] | NSWindowStyleMaskResizable];
     } @catch (NSException *e) {
         return PyErr_Format(PyExc_ValueError, "Failed to set style mask: %s: %s", [[e name] UTF8String], [[e reason] UTF8String]);
     }
     Py_RETURN_NONE;
 }
+
+ 
+PyObject*
+cocoa_get_lang(PyObject UNUSED *self) {
+    NSString* locale = nil;
+    NSString* lang_code = [[NSLocale currentLocale] objectForKey:NSLocaleLanguageCode];
+    NSString* country_code = [[NSLocale currentLocale] objectForKey:NSLocaleCountryCode];
+    if (lang_code && country_code) {
+        locale = [NSString stringWithFormat:@"%@_%@", lang_code, country_code];
+    } else {
+        locale = [[NSLocale currentLocale] localeIdentifier];
+    }
+    if (!locale) { Py_RETURN_NONE; }
+    return Py_BuildValue("s", [locale UTF8String]);
+}
+
+static PyMethodDef module_methods[] = {
+    {"cocoa_get_lang", (PyCFunction)cocoa_get_lang, METH_NOARGS, ""}, \
+    {"cocoa_make_window_resizable", (PyCFunction)cocoa_make_window_resizable, METH_O, ""}, \
+    {"cocoa_create_global_menu", (PyCFunction)cocoa_create_global_menu, METH_NOARGS, ""}, \
+    {"cocoa_init", (PyCFunction)cocoa_init, METH_NOARGS, ""}, \
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+bool
+init_cocoa(PyObject *module) {
+    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
+    return true;
+}

kitty/colors.c

@@ -6,6 +6,7 @@
  */
 
 #include "data-types.h"
+#include <structmember.h>
 
 static uint32_t FG_BG_256[256] = {
     0x000000,  // 0
@@ -59,6 +60,7 @@ new(PyTypeObject *type, PyObject UNUSED *args, PyObject UNUSED *kwds) {
         if (FG_BG_256[255] == 0) create_256_color_table();
         memcpy(self->color_table, FG_BG_256, sizeof(FG_BG_256));
         memcpy(self->orig_color_table, FG_BG_256, sizeof(FG_BG_256));
+        self->dirty = true; 
     }
     return (PyObject*) self;
 }
@@ -68,6 +70,11 @@ dealloc(ColorProfile* self) {
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
+ColorProfile*
+alloc_color_profile() {
+    return (ColorProfile*)new(&ColorProfile_Type, NULL, NULL);
+}
+
 
 static PyObject*
 update_ansi_color_table(ColorProfile *self, PyObject *val) {
@@ -80,9 +87,25 @@ update_ansi_color_table(ColorProfile *self, PyObject *val) {
         self->color_table[i] = PyLong_AsUnsignedLong(PyList_GET_ITEM(val, i));
         self->orig_color_table[i] = self->color_table[i];
     }
+    self->dirty = true;
     Py_RETURN_NONE;
 }
 
+color_type 
+colorprofile_to_color(ColorProfile *self, color_type entry, color_type defval) {
+    color_type t = entry & 0xFF, r;
+    switch(t) {
+        case 1:
+            r = (entry >> 8) & 0xff;
+            return self->color_table[r];
+        case 2:
+            return entry >> 8;
+        default:
+            return defval;
+    }
+}
+
+
 static PyObject*
 as_color(ColorProfile *self, PyObject *val) {
 #define as_color_doc "Convert the specified terminal color into an (r, g, b) tuple based on the current profile values"
@@ -111,6 +134,7 @@ static PyObject*
 reset_color_table(ColorProfile *self) {
 #define reset_color_table_doc "Reset all customized colors back to defaults"
     memcpy(self->color_table, self->orig_color_table, sizeof(FG_BG_256));
+    self->dirty = true;
     Py_RETURN_NONE;
 }
 
@@ -119,6 +143,7 @@ reset_color(ColorProfile *self, PyObject *val) {
 #define reset_color_doc "Reset the specified color"
     uint8_t i = PyLong_AsUnsignedLong(val) & 0xff;
     self->color_table[i] = self->orig_color_table[i];
+    self->dirty = true;
     Py_RETURN_NONE;
 }
 
@@ -129,33 +154,69 @@ set_color(ColorProfile *self, PyObject *args) {
     unsigned long val;
     if (!PyArg_ParseTuple(args, "Bk", &i, &val)) return NULL;
     self->color_table[i] = val;
+    self->dirty = true;
     Py_RETURN_NONE;
 }
 
-
-uint32_t 
-to_color(ColorProfile *self, uint32_t entry, uint32_t defval) {
-    unsigned int t = entry & 0xFF, r;
-    switch(t) {
-        case 1:
-            r = (entry >> 8) & 0xff;
-            return self->color_table[r];
-        case 2:
-            return entry >> 8;
-        default:
-            return defval;
-    }
+static PyObject*
+set_configured_colors(ColorProfile *self, PyObject *args) {
+#define set_configured_colors_doc "Set the configured colors"
+    if (!PyArg_ParseTuple(args, "II|III", &(self->configured.default_fg), &(self->configured.default_bg), &(self->configured.cursor_color), &(self->configured.highlight_fg), &(self->configured.highlight_bg))) return NULL;
+    self->dirty = true;
+    Py_RETURN_NONE;
 }
 
+void
+copy_color_table_to_buffer(ColorProfile *self, color_type *buf, int offset, size_t stride) {
+    size_t i;
+    stride = MAX(1, stride);
+    for (i = 0, buf = buf + offset; i < sizeof(self->color_table)/sizeof(self->color_table[0]); i++, buf += stride) {
+        *buf = self->color_table[i];
+    }
+    self->dirty = false;
+}
+ 
+static PyObject*
+color_table_address(ColorProfile *self) {
+#define color_table_address_doc "Pointer address to start of color table"
+    return PyLong_FromVoidPtr((void*)self->color_table);
+}
+ 
+
 // Boilerplate {{{
 
+#define CGETSET(name) \
+    static PyObject* name##_get(ColorProfile *self, void UNUSED *closure) { return PyLong_FromUnsignedLong(colorprofile_to_color(self, self->overridden.name, self->configured.name));  } \
+    static int name##_set(ColorProfile *self, PyObject *val, void UNUSED *closure) { if (val == NULL) { PyErr_SetString(PyExc_TypeError, "Cannot delete attribute"); return -1; } self->overridden.name = (color_type) PyLong_AsUnsignedLong(val); self->dirty = true; return 0; }
+
+CGETSET(default_fg)
+CGETSET(default_bg)
+CGETSET(cursor_color)
+CGETSET(highlight_fg)
+CGETSET(highlight_bg)
+
+static PyGetSetDef getsetters[] = {
+    GETSET(default_fg)
+    GETSET(default_bg)
+    GETSET(cursor_color)
+    GETSET(highlight_fg)
+    GETSET(highlight_bg)
+    {NULL}  /* Sentinel */
+};
+
+
+static PyMemberDef members[] = {
+    {NULL}
+};
 
 static PyMethodDef methods[] = {
     METHOD(update_ansi_color_table, METH_O)
     METHOD(reset_color_table, METH_NOARGS)
+    METHOD(color_table_address, METH_NOARGS)
     METHOD(as_color, METH_O)
     METHOD(reset_color, METH_O)
     METHOD(set_color, METH_VARARGS)
+    METHOD(set_configured_colors, METH_VARARGS)
     {NULL}  /* Sentinel */
 };
 
@@ -167,7 +228,9 @@ PyTypeObject ColorProfile_Type = {
     .tp_dealloc = (destructor)dealloc, 
     .tp_flags = Py_TPFLAGS_DEFAULT,        
     .tp_doc = "ColorProfile",
+    .tp_members = members,
     .tp_methods = methods,
+    .tp_getset = getsetters,
     .tp_new = new,                
 };
 

kitty/config.py

@@ -2,6 +2,7 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
+import ast
 import json
 import os
 import re
@@ -37,8 +38,9 @@ def to_cursor_shape(x):
         return cshapes[x.lower()]
     except KeyError:
         raise ValueError(
-            'Invalid cursor shape: {} allowed values are {}'.
-            format(x, ', '.join(cshapes))
+            'Invalid cursor shape: {} allowed values are {}'.format(
+                x, ', '.join(cshapes)
+            )
         )
 
 
@@ -46,10 +48,6 @@ def to_bool(x):
     return x.lower() in 'y yes true'.split()
 
 
-def to_opacity(x):
-    return max(0.3, min(float(x), 1))
-
-
 def parse_mods(parts):
 
     def map_mod(m):
@@ -61,8 +59,9 @@ def parse_mods(parts):
             mods |= getattr(defines, 'GLFW_MOD_' + map_mod(m.upper()))
         except AttributeError:
             safe_print(
-                'Shortcut: {} has an unknown modifier, ignoring'.
-                format(parts.join('+')),
+                'Shortcut: {} has an unknown modifier, ignoring'.format(
+                    parts.join('+')
+                ),
                 file=sys.stderr
             )
             return
@@ -84,23 +83,57 @@ named_keys = {
 }
 
 
-def parse_key(val, keymap):
-    sc, action = val.partition(' ')[::2]
-    action = action.strip()
-    sc = sc.strip()
-    if not sc or not action:
-        return
+def parse_shortcut(sc):
     parts = sc.split('+')
     mods = parse_mods(parts[:-1])
     key = parts[-1].upper()
     key = getattr(defines, 'GLFW_KEY_' + named_keys.get(key, key), None)
+    if key is not None:
+        return mods, key
+    return None, None
+
+
+KeyAction = namedtuple('KeyAction', 'func args')
+shlex_actions = {'pass_selection_to_program', 'new_window', 'new_tab'}
+
+
+def parse_key_action(action):
+    parts = action.split(' ', 1)
+    func = parts[0]
+    if len(parts) == 1:
+        return KeyAction(func, ())
+    rest = parts[1]
+    if func == 'combine':
+        sep, rest = rest.split(' ', 1)
+        parts = re.split(r'\s*' + re.escape(sep) + r'\s*', rest)
+        args = tuple(map(parse_key_action, filter(None, parts)))
+    elif func in shlex_actions:
+        args = shlex.split(rest)
+    return KeyAction(func, args)
+
+
+def parse_key(val, keymap):
+    sc, action = val.partition(' ')[::2]
+    sc, action = sc.strip(), action.strip()
+    if not sc or not action:
+        return
+    mods, key = parse_shortcut(sc)
     if key is None:
         safe_print(
             'Shortcut: {} has an unknown key, ignoring'.format(val),
             file=sys.stderr
         )
         return
-    keymap[(mods, key)] = action
+    try:
+        paction = parse_key_action(action)
+    except Exception:
+        safe_print(
+            'Invalid shortcut action: {}. Ignoring.'.format(action),
+            file=sys.stderr
+        )
+    else:
+        if paction is not None:
+            keymap[(mods, key)] = paction
 
 
 def parse_symbol_map(val):
@@ -137,6 +170,42 @@ def parse_symbol_map(val):
     return symbol_map
 
 
+def parse_send_text(val):
+    parts = val.split(' ')
+
+    def abort(msg):
+        safe_print(
+            'Send text: {} is invalid ({}), ignoring'.format(val, msg),
+            file=sys.stderr
+        )
+        return {}
+
+    if len(parts) < 3:
+        return abort('Incomplete')
+
+    text = ' '.join(parts[2:])
+    mode, sc = parts[:2]
+    mods, key = parse_shortcut(sc.strip())
+    if key is None:
+        return abort('Invalid shortcut')
+    text = ast.literal_eval("'''" + text + "'''").encode('utf-8')
+    if not text:
+        return abort('Empty text')
+
+    if mode in ('all', '*'):
+        modes = parse_send_text.all_modes
+    else:
+        modes = frozenset(mode.split(',')).intersection(
+            parse_send_text.all_modes
+        )
+        if not modes:
+            return abort('Invalid keyboard modes')
+    return {mode: {(mods, key): text} for mode in modes}
+
+
+parse_send_text.all_modes = frozenset({'normal', 'application', 'kitty'})
+
+
 def to_open_url_modifiers(val):
     return parse_mods(val.split('+'))
 
@@ -150,35 +219,52 @@ def to_layout_names(raw):
             raise ValueError('The window layout {} is unknown'.format(p))
 
 
+def positive_int(x):
+    return max(0, int(x))
+
+
+def positive_float(x):
+    return max(0, float(x))
+
+
+def adjust_line_height(x):
+    if x.endswith('%'):
+        return float(x[:-1].strip()) / 100.0
+    return int(x)
+
+
 type_map = {
-    'scrollback_lines': int,
+    'adjust_line_height': adjust_line_height,
+    'scrollback_lines': positive_int,
     'scrollback_pager': shlex.split,
     'scrollback_in_new_tab': to_bool,
     'font_size': to_font_size,
-    'font_size_delta': float,
+    'font_size_delta': positive_float,
     'cursor_shape': to_cursor_shape,
-    'cursor_opacity': to_opacity,
     'open_url_modifiers': to_open_url_modifiers,
-    'repaint_delay': int,
-    'window_border_width': float,
+    'repaint_delay': positive_int,
+    'input_delay': positive_int,
+    'window_border_width': positive_float,
+    'window_margin_width': positive_float,
+    'window_padding_width': positive_float,
     'wheel_scroll_multiplier': float,
-    'visual_bell_duration': float,
+    'visual_bell_duration': positive_float,
     'enable_audio_bell': to_bool,
-    'click_interval': float,
-    'mouse_hide_wait': float,
-    'cursor_blink_interval': float,
-    'cursor_stop_blinking_after': float,
+    'click_interval': positive_float,
+    'mouse_hide_wait': positive_float,
+    'cursor_blink_interval': positive_float,
+    'cursor_stop_blinking_after': positive_float,
     'enabled_layouts': to_layout_names,
     'remember_window_size': to_bool,
-    'initial_window_width': int,
-    'initial_window_height': int,
+    'initial_window_width': positive_int,
+    'initial_window_height': positive_int,
     'use_system_wcwidth': to_bool,
     'macos_hide_titlebar': to_bool,
 }
 
 for name in (
     'foreground background cursor active_border_color inactive_border_color'
-    ' selection_foreground selection_background'
+    ' selection_foreground selection_background url_color'
 ).split():
     type_map[name] = lambda x: to_color(x, validate=True)
 for i in range(16):
@@ -188,8 +274,18 @@ for a in ('active', 'inactive'):
         type_map['%s_tab_%s' % (a, b)] = lambda x: to_color(x, validate=True)
 
 
-def parse_config(lines):
-    ans = {'keymap': {}, 'symbol_map': {}}
+def parse_config(lines, check_keys=True):
+    ans = {
+        'keymap': {},
+        'symbol_map': {},
+        'send_text_map': {
+            'kitty': {},
+            'normal': {},
+            'application': {}
+        }
+    }
+    if check_keys:
+        all_keys = defaults._asdict()
     for line in lines:
         line = line.strip()
         if not line or line.startswith('#'):
@@ -203,6 +299,18 @@ def parse_config(lines):
             if key == 'symbol_map':
                 ans['symbol_map'].update(parse_symbol_map(val))
                 continue
+            if key == 'send_text':
+                stvals = parse_send_text(val)
+                for k, v in ans['send_text_map'].items():
+                    v.update(stvals.get(k, {}))
+                continue
+            if check_keys:
+                if key not in all_keys:
+                    safe_print(
+                        'Ignoring unknown config key: {}'.format(key),
+                        file=sys.stderr
+                    )
+                    continue
             tm = type_map.get(key)
             if tm is not None:
                 val = tm(val)
@@ -213,38 +321,47 @@ def parse_config(lines):
 with open(
     os.path.join(os.path.dirname(os.path.abspath(__file__)), 'kitty.conf')
 ) as f:
-    defaults = parse_config(f.readlines())
+    defaults = parse_config(f.readlines(), check_keys=False)
 Options = namedtuple('Defaults', ','.join(defaults.keys()))
 defaults = Options(**defaults)
-actions = frozenset(defaults.keymap.values())
+actions = frozenset(a.func for a in defaults.keymap.values()) | frozenset({'combine'})
+no_op_actions = frozenset({'noop', 'no-op', 'no_op'})
 
 
-def update_dict(a, b):
-    a.update(b)
-    return a
+def merge_keymaps(defaults, newvals):
+    ans = defaults.copy()
+    for k, v in newvals.items():
+        f = v.func
+        if f in no_op_actions:
+            ans.pop(k, None)
+            continue
+        if f in actions:
+            ans[k] = v
+    return ans
 
 
-def merge_dicts(vals, defaults):
-    return {
-        k: update_dict(v, vals.get(k, {}))
-        if isinstance(v, dict) else vals.get(k, v)
-        for k, v in defaults.items()
-    }
+def merge_dicts(defaults, newvals):
+    ans = defaults.copy()
+    ans.update(newvals)
+    return ans
 
 
-def merge_configs(ans, vals):
-    vals['keymap'] = {
-        k: v
-        for k, v in vals.get('keymap', {}).items() if v in actions
-    }
-    remove_keys = {
-        k
-        for k, v in vals.get('keymap', {}).items()
-        if v in ('noop', 'no-op', 'no_op')
-    }
-    ans = merge_dicts(vals, ans)
-    for k in remove_keys:
-        ans['keymap'].pop(k, None)
+def merge_configs(defaults, vals):
+    ans = {}
+    for k, v in defaults.items():
+        if isinstance(v, dict):
+            newvals = vals.get(k, {})
+            if k == 'keymap':
+                ans['keymap'] = merge_keymaps(v, newvals)
+            elif k == 'send_text_map':
+                ans[k] = {
+                    m: merge_dicts(mm, newvals.get(m, {}))
+                    for m, mm in v.items()
+                }
+            else:
+                ans[k] = merge_dicts(v, newvals)
+        else:
+            ans[k] = vals.get(k, v)
     return ans
 
 
@@ -266,7 +383,7 @@ def load_config(*paths, overrides=None) -> Options:
     return Options(**ans)
 
 
-def build_ansi_color_table(opts: Options=defaults):
+def build_ansi_color_table(opts: Options = defaults):
 
     def as_int(x):
         return (x[0] << 16) | (x[1] << 8) | x[2]

kitty/constants.py

@@ -3,19 +3,15 @@
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
 import os
-import threading
 import pwd
 import ctypes
 import sys
-from collections import namedtuple, defaultdict
+from collections import namedtuple
 
-from .fast_data_types import (
-    GLFW_KEY_LEFT_SHIFT, GLFW_KEY_RIGHT_SHIFT, GLFW_KEY_LEFT_ALT,
-    GLFW_KEY_RIGHT_ALT, GLFW_KEY_LEFT_CONTROL, GLFW_KEY_RIGHT_CONTROL,
-    GLFW_KEY_LEFT_SUPER, GLFW_KEY_RIGHT_SUPER)
+from .fast_data_types import set_boss as set_c_boss
 
 appname = 'kitty'
-version = (0, 2, 6)
+version = (0, 4, 2)
 str_version = '.'.join(map(str, version))
 _plat = sys.platform.lower()
 isosx = 'darwin' in _plat
@@ -32,15 +28,13 @@ def _get_config_dir():
 
     candidate = os.path.abspath(os.path.expanduser(os.environ.get('XDG_CONFIG_HOME') or ('~/Library/Preferences' if isosx else '~/.config')))
     ans = os.path.join(candidate, appname)
-    try:
-        os.makedirs(ans)
-    except FileExistsError:
-        pass
+    os.makedirs(ans, exist_ok=True)
     return ans
 
 
 config_dir = _get_config_dir()
 del _get_config_dir
+defconf = os.path.join(config_dir, 'kitty.conf')
 
 
 class ViewportSize:
@@ -61,25 +55,18 @@ def get_boss():
 
 def set_boss(m):
     get_boss.boss = m
+    set_c_boss(m)
 
 
 def wakeup():
-    os.write(get_boss.boss.write_wakeup_fd, b'1')
+    get_boss.boss.child_monitor.wakeup()
 
 
-def queue_action(func, *args):
-    get_boss.boss.queue_action(func, *args)
-
-
-is_key_pressed = defaultdict(lambda: False)
-mouse_button_pressed = defaultdict(lambda: False)
-mouse_cursor_pos = [0, 0]
 viewport_size = ViewportSize()
 cell_size = ViewportSize()
 base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 terminfo_dir = os.path.join(base_dir, 'terminfo')
 logo_data_file = os.path.join(base_dir, 'logo', 'kitty.rgba')
-main_thread = threading.current_thread()
 shell_path = pwd.getpwuid(os.geteuid()).pw_shell or '/bin/sh'
 
 GLint = ctypes.c_int if ctypes.sizeof(ctypes.c_int) == 4 else ctypes.c_long
@@ -90,7 +77,59 @@ if ctypes.sizeof(GLfloat) != 4:
 if ctypes.sizeof(GLint) != 4:
     raise RuntimeError('int size is not 4')
 
-MODIFIER_KEYS = (
-    GLFW_KEY_LEFT_SHIFT, GLFW_KEY_RIGHT_SHIFT, GLFW_KEY_LEFT_ALT,
-    GLFW_KEY_RIGHT_ALT, GLFW_KEY_LEFT_CONTROL, GLFW_KEY_RIGHT_CONTROL,
-    GLFW_KEY_LEFT_SUPER, GLFW_KEY_RIGHT_SUPER)
+
+def get_glfw_lib_name():
+    try:
+        for line in open('/proc/self/maps'):
+            lib = line.split()[-1]
+            if '/libglfw.so' in lib:
+                return lib
+    except Exception as err:
+        try:
+            print(str(err), file=sys.stderr)
+        except Exception:
+            pass
+    return 'libglfw.so.3'
+
+
+def glfw_lib():
+    ans = getattr(glfw_lib, 'ans', None)
+    if ans is None:
+        ans = glfw_lib.ans = ctypes.CDLL('libglfw.3.dylib' if isosx else get_glfw_lib_name())
+    return ans
+
+
+def selection_clipboard_funcs():
+    ans = getattr(selection_clipboard_funcs, 'ans', None)
+    if ans is None:
+        lib = glfw_lib()
+        if hasattr(lib, 'glfwGetX11SelectionString'):
+            g = lib.glfwGetX11SelectionString
+            g.restype = ctypes.c_char_p
+            g.argtypes = []
+            s = lib.glfwSetX11SelectionString
+            s.restype = None
+            s.argtypes = [ctypes.c_char_p]
+            ans = g, s
+        else:
+            ans = None, None
+        selection_clipboard_funcs.ans = ans
+    return ans
+
+
+def x11_window_id(window):
+    lib = glfw_lib()
+    lib.glfwGetX11Window.restype = ctypes.c_int32
+    lib.glfwGetX11Window.argtypes = [ctypes.c_void_p]
+    return lib.glfwGetX11Window(window.window_id())
+
+
+def x11_display():
+    lib = glfw_lib()
+    ans = lib.glfwGetX11Display
+    ans.restype = ctypes.c_void_p
+    ans.argtypes = []
+    return ans()
+
+
+iswayland = not isosx and hasattr(glfw_lib(), 'glfwGetWaylandDisplay')

kitty/core_text.m

@@ -11,6 +11,8 @@
 #include <math.h>
 #import <CoreGraphics/CGBitmapContext.h>
 #import <CoreText/CTFont.h>
+#include <Foundation/Foundation.h>
+#include <CoreText/CoreText.h>
 #import <Foundation/NSString.h>
 #import <Foundation/NSDictionary.h>
 
@@ -20,40 +22,113 @@ typedef struct {
     unsigned int units_per_em;
     float ascent, descent, leading, underline_position, underline_thickness, point_sz, scaled_point_sz;
     CTFontRef font;
-    PyObject *family_name, *full_name, *postscript_name;
+    PyObject *family_name, *full_name, *postscript_name, *path;
 } Face;
 
 
-static PyObject*
-convert_cfstring(CFStringRef src) {
+static inline PyObject*
+convert_cfstring(CFStringRef src, int free_src) {
 #define SZ 2048
     static char buf[SZ+2] = {0};
-    if(!CFStringGetCString(src, buf, SZ, kCFStringEncodingUTF8)) { PyErr_SetString(PyExc_ValueError, "Failed to convert CFString"); return NULL; }
-    return PyUnicode_FromString(buf);
+    PyObject *ans = NULL;
+    if(!CFStringGetCString(src, buf, SZ, kCFStringEncodingUTF8)) PyErr_SetString(PyExc_ValueError, "Failed to convert CFString");
+    else ans = PyUnicode_FromString(buf);
+    if (free_src) CFRelease(src);
+    return ans;
 #undef SZ
 }
 
+static PyObject*
+font_descriptor_to_python(CTFontDescriptorRef descriptor) {
+    NSURL *url = (NSURL *) CTFontDescriptorCopyAttribute(descriptor, kCTFontURLAttribute);
+    NSString *psName = (NSString *) CTFontDescriptorCopyAttribute(descriptor, kCTFontNameAttribute);  
+    NSString *family = (NSString *) CTFontDescriptorCopyAttribute(descriptor, kCTFontFamilyNameAttribute);
+    NSString *style = (NSString *) CTFontDescriptorCopyAttribute(descriptor, kCTFontStyleNameAttribute);
+    NSDictionary *traits = (NSDictionary *) CTFontDescriptorCopyAttribute(descriptor, kCTFontTraitsAttribute);
+    unsigned int straits = [traits[(id)kCTFontSymbolicTrait] unsignedIntValue];
+    NSNumber *weightVal = traits[(id)kCTFontWeightTrait];
+    NSNumber *widthVal = traits[(id)kCTFontWidthTrait];
+
+    PyObject *ans = Py_BuildValue("{ssssssss sOsOsO sfsfsI}", 
+            "path", [[url path] UTF8String], 
+            "postscript_name", [psName UTF8String],
+            "family", [family UTF8String],
+            "style", [style UTF8String],
+
+            "bold", (straits & kCTFontBoldTrait) != 0 ? Py_True : Py_False,
+            "italic", (straits & kCTFontItalicTrait) != 0 ? Py_True : Py_False,
+            "monospace", (straits & kCTFontMonoSpaceTrait) != 0 ? Py_True : Py_False,
+
+            "weight", [weightVal floatValue],
+            "width", [widthVal floatValue],
+            "traits", straits
+    );
+    [url release];
+    [psName release];
+    [family release];
+    [style release];
+    [traits release];
+    return ans;
+}
+
+static CTFontDescriptorRef
+font_descriptor_from_python(PyObject *src) {
+    CTFontSymbolicTraits symbolic_traits = 0;
+    NSMutableDictionary *attrs = [NSMutableDictionary dictionary];
+    PyObject *t = PyDict_GetItemString(src, "traits");
+    if (t == NULL) {
+        symbolic_traits = (
+            (PyDict_GetItemString(src, "bold") == Py_True) ? kCTFontBoldTrait : 0 |
+            (PyDict_GetItemString(src, "italic") == Py_True) ? kCTFontItalicTrait : 0 |
+            (PyDict_GetItemString(src, "monospace") == Py_True) ? kCTFontMonoSpaceTrait : 0);
+    } else {
+        symbolic_traits = PyLong_AsUnsignedLong(t);
+    }
+    NSDictionary *traits = @{(id)kCTFontSymbolicTrait:[NSNumber numberWithUnsignedInt:symbolic_traits]};
+    attrs[(id)kCTFontTraitsAttribute] = traits;
+
+#define SET(x, attr) \
+    t = PyDict_GetItemString(src, #x); \
+    if (t) attrs[(id)attr] = [NSString stringWithUTF8String:PyUnicode_AsUTF8(t)];
+
+    SET(family, kCTFontFamilyNameAttribute);
+    SET(style, kCTFontStyleNameAttribute);
+    SET(postscript_name, kCTFontNameAttribute);
+#undef SET
+
+    return CTFontDescriptorCreateWithAttributes((CFDictionaryRef) attrs);
+}
+
+PyObject*
+coretext_all_fonts(PyObject UNUSED *_self) {
+    static CTFontCollectionRef collection = NULL;
+    if (collection == NULL) collection = CTFontCollectionCreateFromAvailableFonts(NULL);
+    NSArray *matches = (NSArray *) CTFontCollectionCreateMatchingFontDescriptors(collection);  
+    PyObject *ans = PyTuple_New([matches count]), *temp;
+    if (ans == NULL) return PyErr_NoMemory();
+    for (unsigned int i = 0; i < [matches count]; i++) {
+        temp = font_descriptor_to_python((CTFontDescriptorRef) matches[i]);
+        if (temp == NULL) { Py_DECREF(ans); return NULL; }
+        PyTuple_SET_ITEM(ans, i, temp); temp = NULL;
+    }
+    return ans;
+}
+
 
 static PyObject*
 new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
     Face *self;
-    int bold, italic, monospace;
-    char *cfamily;
+    PyObject *descriptor;
     float point_sz, dpi;
-    if(!PyArg_ParseTuple(args, "spppff", &cfamily, &bold, &italic, &monospace, &point_sz, &dpi)) return NULL;
-    NSString *family = [[NSString alloc] initWithCString:cfamily encoding:NSUTF8StringEncoding];
-    if (family == NULL) return PyErr_NoMemory();
+    if(!PyArg_ParseTuple(args, "Off", &descriptor, &point_sz, &dpi)) return NULL;
     self = (Face *)type->tp_alloc(type, 0);
     if (self) {
-        CTFontSymbolicTraits symbolic_traits = (bold ? kCTFontBoldTrait : 0) | (italic ? kCTFontItalicTrait : 0) | (monospace ? kCTFontMonoSpaceTrait : 0);
-        NSDictionary *font_traits = [NSDictionary dictionaryWithObject:[NSNumber numberWithInt:symbolic_traits] forKey:(NSString *)kCTFontSymbolicTrait];
-        NSDictionary *font_attributes = [NSDictionary dictionaryWithObjectsAndKeys:family, kCTFontFamilyNameAttribute, font_traits, kCTFontTraitsAttribute, nil];
-        CTFontDescriptorRef descriptor = CTFontDescriptorCreateWithAttributes((CFDictionaryRef)font_attributes);
-        if (descriptor) {
+        CTFontDescriptorRef desc = font_descriptor_from_python(descriptor);
+        if (desc) {
             self->point_sz = point_sz;
             self->scaled_point_sz = (dpi / 72.0) * point_sz;
-            self->font = CTFontCreateWithFontDescriptor(descriptor, self->scaled_point_sz, NULL);
-            CFRelease(descriptor);
+            self->font = CTFontCreateWithFontDescriptor(desc, self->scaled_point_sz, NULL);
+            CFRelease(desc);
             if (!self->font) { Py_CLEAR(self); PyErr_SetString(PyExc_ValueError, "Failed to create CTFont object"); }
             else {
                 self->units_per_em = CTFontGetUnitsPerEm(self->font);
@@ -63,17 +138,19 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
                 self->underline_position = CTFontGetUnderlinePosition(self->font);
                 self->underline_thickness = CTFontGetUnderlineThickness(self->font);
                 self->scaled_point_sz = CTFontGetSize(self->font);
-                self->family_name = convert_cfstring(CTFontCopyFamilyName(self->font));
-                self->full_name = convert_cfstring(CTFontCopyFullName(self->font));
-                self->postscript_name = convert_cfstring(CTFontCopyPostScriptName(self->font));
-                if (self->family_name == NULL || self->full_name == NULL || self->postscript_name == NULL) { Py_CLEAR(self); }
+                self->family_name = convert_cfstring(CTFontCopyFamilyName(self->font), 1);
+                self->full_name = convert_cfstring(CTFontCopyFullName(self->font), 1);
+                self->postscript_name = convert_cfstring(CTFontCopyPostScriptName(self->font), 1);
+                NSURL *url = (NSURL*)CTFontCopyAttribute(self->font, kCTFontURLAttribute);
+                self->path = PyUnicode_FromString([[url path] UTF8String]);
+                [url release];
+                if (self->family_name == NULL || self->full_name == NULL || self->postscript_name == NULL || self->path == NULL) { Py_CLEAR(self); }
             }
         } else {
             Py_CLEAR(self);
             PyErr_NoMemory();
         }
     }
-    [ family release ];
     return (PyObject*)self;
 }
 
@@ -81,7 +158,7 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
 static void
 dealloc(Face* self) {
     if (self->font) CFRelease(self->font);
-    Py_CLEAR(self->family_name); Py_CLEAR(self->full_name); Py_CLEAR(self->postscript_name);
+    Py_CLEAR(self->family_name); Py_CLEAR(self->full_name); Py_CLEAR(self->postscript_name); Py_CLEAR(self->path);
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
@@ -123,14 +200,13 @@ cell_size(Face *self) {
 #define count (128 - 32)
     unichar chars[count+1] = {0};
     CGGlyph glyphs[count+1] = {0};
-    for (int i = 0; i < count; i++) chars[i] = 32 + i;
+    unsigned int width = 0, w, i;
+    for (i = 0; i < count; i++) chars[i] = 32 + i;
     CTFontGetGlyphsForCharacters(self->font, chars, glyphs, count);
-    CGSize advances[1] = {0};
-    unsigned int width = 0, w;
-    for (int i = 0; i < count; i++) {
+    for (i = 0; i < count; i++) {
         if (glyphs[i]) {
-            CTFontGetAdvancesForGlyphs(self->font, kCTFontOrientationHorizontal, glyphs+1, advances, 1);
-            w = (unsigned int)(ceilf(advances[0].width));
+            w = (unsigned int)(ceilf(
+                        CTFontGetAdvancesForGlyphs(self->font, kCTFontOrientationHorizontal, glyphs+i, NULL, 1)));
             if (w > width) width = w; 
         }
     }
@@ -177,9 +253,8 @@ render_char(Face *self, PyObject *args) {
         // TODO: Scale the glyph if its bbox is larger than the image by using a non-identity transform
         /* CGRect rect = CTFontGetBoundingRectsForGlyphs(font, kCTFontOrientationHorizontal, glyphs, 0, 1); */
         CGContextSetTextMatrix(ctx, transform);
-        CGFloat leading = self->leading > 0 ? self->leading : 1;  // Ensure at least one pixel of leading so that antialiasing works at the left edge
-        CGFloat pos_x = leading, pos_y = height - self->ascent;  
-        CGContextSetTextPosition(ctx, pos_x, pos_y);
+        CGFloat pos_y = height - self->ascent; 
+        CGContextSetTextPosition(ctx, 0, pos_y); 
         CTFontDrawGlyphs(font, &glyph, &CGPointZero, 1, ctx);
     }
 
@@ -198,8 +273,8 @@ repr(Face *self) {
     snprintf(buf, sizeof(buf)/sizeof(buf[0]), "ascent=%.1f, descent=%.1f, leading=%.1f, point_sz=%.1f, scaled_point_sz=%.1f, underline_position=%.1f underline_thickness=%.1f", 
         (self->ascent), (self->descent), (self->leading), (self->point_sz), (self->scaled_point_sz), (self->underline_position), (self->underline_thickness));
     return PyUnicode_FromFormat(
-        "Face(family=%U, full_name=%U, postscript_name=%U, units_per_em=%u, %s)",
-        self->family_name, self->full_name, self->postscript_name, self->units_per_em, buf
+        "Face(family=%U, full_name=%U, postscript_name=%U, path=%U, units_per_em=%u, %s)",
+        self->family_name, self->full_name, self->postscript_name, self->path, self->units_per_em, buf
     );
 }
 
@@ -217,6 +292,7 @@ static PyMemberDef members[] = {
     MEM(underline_position, T_FLOAT),
     MEM(underline_thickness, T_FLOAT),
     MEM(family_name, T_OBJECT),
+    MEM(path, T_OBJECT),
     MEM(full_name, T_OBJECT),
     MEM(postscript_name, T_OBJECT),
     {NULL}  /* Sentinel */
@@ -244,11 +320,16 @@ PyTypeObject Face_Type = {
     .tp_repr = (reprfunc)repr,
 };
 
+static PyMethodDef module_methods[] = {
+    {"coretext_all_fonts", (PyCFunction)coretext_all_fonts, METH_NOARGS, ""},
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
 
 int 
 init_CoreText(PyObject *module) {
     if (PyType_Ready(&Face_Type) < 0) return 0;
     if (PyModule_AddObject(module, "CTFace", (PyObject *)&Face_Type) != 0) return 0;
+    if (PyModule_AddFunctions(module, module_methods) != 0) return 0;
     return 1;
 }
 

kitty/cursor.c

@@ -27,12 +27,15 @@ static int __eq__(Cursor *a, Cursor *b) {
     return EQ(bold) && EQ(italic) && EQ(strikethrough) && EQ(reverse) && EQ(decoration) && EQ(fg) && EQ(bg) && EQ(decoration_fg) && EQ(x) && EQ(y) && EQ(shape) && EQ(blink);
 }
 
+static const char* cursor_names[NUM_OF_CURSOR_SHAPES] = { "NO_SHAPE", "BLOCK", "BEAM", "UNDERLINE" };
+
 #define BOOL(x) ((x) ? Py_True : Py_False)
 static PyObject *
 repr(Cursor *self) {
     return PyUnicode_FromFormat(
-        "Cursor(x=%u, y=%u, shape=%d, blink=%R, fg=#%08x, bg=#%08x, bold=%R, italic=%R, reverse=%R, strikethrough=%R, decoration=%d, decoration_fg=#%08x)",
-        self->x, self->y, self->shape, BOOL(self->blink), self->fg, self->bg, BOOL(self->bold), BOOL(self->italic), BOOL(self->reverse), BOOL(self->strikethrough), self->decoration, self->decoration_fg
+        "Cursor(x=%u, y=%u, shape=%s, blink=%R, fg=#%08x, bg=#%08x, bold=%R, italic=%R, reverse=%R, strikethrough=%R, decoration=%d, decoration_fg=#%08x)",
+        self->x, self->y, (self->shape < NUM_OF_CURSOR_SHAPES ? cursor_names[self->shape] : "INVALID"),
+        BOOL(self->blink), self->fg, self->bg, BOOL(self->bold), BOOL(self->italic), BOOL(self->reverse), BOOL(self->strikethrough), self->decoration, self->decoration_fg
     );
 }
 
@@ -51,7 +54,7 @@ reset_display_attrs(Cursor *self) {
 void cursor_reset(Cursor *self) {
     cursor_reset_display_attrs(self);
     self->x = 0; self->y = 0;
-    self->shape = 0; self->blink = false;
+    self->shape = NO_CURSOR_SHAPE; self->blink = false;
 }
 
 void cursor_copy_to(Cursor *src, Cursor *dest) {
@@ -75,7 +78,7 @@ BOOL_GETSET(Cursor, blink)
 static PyMemberDef members[] = {
     {"x", T_UINT, offsetof(Cursor, x), 0, "x"},
     {"y", T_UINT, offsetof(Cursor, y), 0, "y"},
-    {"shape", T_UBYTE, offsetof(Cursor, shape), 0, "shape"},
+    {"shape", T_INT, offsetof(Cursor, shape), 0, "shape"},
     {"decoration", T_UBYTE, offsetof(Cursor, decoration), 0, "decoration"},
     {"fg", T_ULONG, offsetof(Cursor, fg), 0, "fg"},
     {"bg", T_ULONG, offsetof(Cursor, bg), 0, "bg"},

kitty/cursor_fragment.glsl

@@ -0,0 +1,7 @@
+#version GLSL_VERSION
+uniform vec3 color;
+out vec4 final_color;
+
+void main() {
+    final_color = vec4(color, 1);
+}

kitty/cursor_vertex.glsl

@@ -0,0 +1,18 @@
+#version GLSL_VERSION
+uniform vec4 pos;
+
+const uvec2 pos_map[] = uvec2[4](
+    uvec2(1, 0),  // right, top
+    uvec2(1, 1),  // right, bottom
+    uvec2(0, 1),  // left, bottom
+    uvec2(0, 0)   // left, top
+);
+
+void main() {
+    vec2 xpos = vec2(pos[0], pos[2]);
+    vec2 ypos = vec2(pos[1], pos[3]);
+    uvec2 pos = pos_map[gl_VertexID];
+    gl_Position = vec4(xpos[pos[0]], ypos[pos[1]], 0, 1);
+}
+
+

kitty/data-types.c

@@ -6,19 +6,53 @@
  */
 
 #include "data-types.h"
-#include "glfw.h"
-#include "gl.h"
 #include "modes.h"
+#include <stddef.h>
+#include <termios.h>
+#include <signal.h>
+#include <sys/wait.h>
+#ifdef WITH_PROFILER
+#include <gperftools/profiler.h>
+#endif
 
-static char drain_buf[1024] = {0};
+/* To millisecond (10^-3) */
+#define SEC_TO_MS 1000
 
-static PyObject*
-drain_read(PyObject UNUSED *self, PyObject *fd) {
-    ALLOW_UNUSED_RESULT
-    read(PyLong_AsLong(fd), drain_buf, sizeof(drain_buf));
-    END_ALLOW_UNUSED_RESULT 
-    Py_RETURN_NONE;
+/* To microseconds (10^-6) */
+#define MS_TO_US 1000
+#define SEC_TO_US (SEC_TO_MS * MS_TO_US)
+
+/* To nanoseconds (10^-9) */
+#define US_TO_NS 1000
+#define MS_TO_NS (MS_TO_US * US_TO_NS)
+#define SEC_TO_NS (SEC_TO_MS * MS_TO_NS)
+
+/* Conversion from nanoseconds */
+#define NS_TO_MS (1000 * 1000)
+#define NS_TO_US (1000)
+
+#ifdef __APPLE__
+#include <mach/mach_time.h>
+static mach_timebase_info_data_t timebase = {0};
+static inline double monotonic_() {
+	return ((double)(mach_absolute_time() * timebase.numer) / timebase.denom)/SEC_TO_NS;
 }
+#else
+#include <time.h>
+static inline double monotonic_() {
+    struct timespec ts = {0};
+#ifdef CLOCK_HIGHRES
+	clock_gettime(CLOCK_HIGHRES, &ts);
+#elif CLOCK_MONOTONIC_RAW
+	clock_gettime(CLOCK_MONOTONIC_RAW, &ts);
+#else
+	clock_gettime(CLOCK_MONOTONIC, &ts);
+#endif
+	return (((double)ts.tv_nsec) / SEC_TO_NS) + (double)ts.tv_sec;
+}
+#endif
+
+double monotonic() { return monotonic_(); }
 
 static PyObject*
 wcwidth_wrap(PyObject UNUSED *self, PyObject *chr) {
@@ -31,19 +65,75 @@ change_wcwidth_wrap(PyObject UNUSED *self, PyObject *use9) {
     Py_RETURN_NONE;
 }
 
+static PyObject*
+redirect_std_streams(PyObject UNUSED *self, PyObject *args) {
+    char *devnull = NULL;
+    if (!PyArg_ParseTuple(args, "s", &devnull)) return NULL;
+    if (freopen(devnull, "r", stdin) == NULL) return PyErr_SetFromErrno(PyExc_EnvironmentError);
+    if (freopen(devnull, "w", stdout) == NULL) return PyErr_SetFromErrno(PyExc_EnvironmentError);
+    if (freopen(devnull, "w", stderr) == NULL)  return PyErr_SetFromErrno(PyExc_EnvironmentError);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+pyset_iutf8(PyObject UNUSED *self, PyObject *args) {
+    int fd, on;
+    if (!PyArg_ParseTuple(args, "ip", &fd, &on)) return NULL;
+    if (!set_iutf8(fd, on & 1)) return PyErr_SetFromErrno(PyExc_OSError);
+    Py_RETURN_NONE;
+}
+
+static void
+handle_sigchld(int UNUSED signum, siginfo_t *sinfo, void UNUSED *unused) {
+    if (sinfo->si_code != CLD_EXITED) return;
+    int sav_errno = errno, status;
+    while(true) {
+        if (waitpid(sinfo->si_pid, &status, WNOHANG) == -1) {
+            if (errno != EINTR) break;
+        } else break;
+    }
+    errno = sav_errno;
+}
+
+static PyObject*
+install_sigchld_handler(PyObject UNUSED *self) {
+    struct sigaction sa;
+    sa.sa_flags = SA_SIGINFO;
+    sa.sa_sigaction = handle_sigchld;
+    sigemptyset(&sa.sa_mask);
+    if (sigaction(SIGCHLD, &sa, NULL) == -1) return PyErr_SetFromErrno(PyExc_OSError);
+    Py_RETURN_NONE;
+}
+
+#ifdef WITH_PROFILER
+static PyObject*
+start_profiler(PyObject UNUSED *self, PyObject *args) {
+    char *path;
+    if (!PyArg_ParseTuple(args, "s", &path)) return NULL;
+    ProfilerStart(path);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+stop_profiler(PyObject UNUSED *self) {
+    ProfilerStop();
+    Py_RETURN_NONE;
+}
+#endif
+
 static PyMethodDef module_methods[] = {
-    GL_METHODS
-    {"drain_read", (PyCFunction)drain_read, METH_O, ""},
+    {"set_iutf8", (PyCFunction)pyset_iutf8, METH_VARARGS, ""},
+    {"thread_write", (PyCFunction)cm_thread_write, METH_VARARGS, ""},
     {"parse_bytes", (PyCFunction)parse_bytes, METH_VARARGS, ""},
     {"parse_bytes_dump", (PyCFunction)parse_bytes_dump, METH_VARARGS, ""},
-    {"read_bytes", (PyCFunction)read_bytes, METH_VARARGS, ""},
-    {"read_bytes_dump", (PyCFunction)read_bytes_dump, METH_VARARGS, ""},
+    {"redirect_std_streams", (PyCFunction)redirect_std_streams, METH_VARARGS, ""},
     {"wcwidth", (PyCFunction)wcwidth_wrap, METH_O, ""},
     {"change_wcwidth", (PyCFunction)change_wcwidth_wrap, METH_O, ""},
-#ifndef __APPLE__
-    {"get_fontconfig_font", (PyCFunction)get_fontconfig_font, METH_VARARGS, ""},
+    {"install_sigchld_handler", (PyCFunction)install_sigchld_handler, METH_NOARGS, ""},
+#ifdef WITH_PROFILER
+    {"start_profiler", (PyCFunction)start_profiler, METH_VARARGS, ""},
+    {"stop_profiler", (PyCFunction)stop_profiler, METH_NOARGS, ""},
 #endif
-    GLFW_FUNC_WRAPPERS
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
@@ -56,34 +146,71 @@ static struct PyModuleDef module = {
    .m_methods = module_methods
 };
 
-#include <termios.h>
 
-PyMODINIT_FUNC
+extern int init_LineBuf(PyObject *);
+extern int init_HistoryBuf(PyObject *);
+extern int init_Cursor(PyObject *);
+extern bool init_child_monitor(PyObject *);
+extern int init_Line(PyObject *);
+extern int init_ColorProfile(PyObject *);
+extern int init_Screen(PyObject *);
+extern int init_Face(PyObject *);
+extern bool init_freetype_library(PyObject*);
+extern bool init_fontconfig_library(PyObject*);
+extern bool init_glfw(PyObject *m);
+extern bool init_sprites(PyObject *module);
+extern bool init_state(PyObject *module);
+extern bool init_keys(PyObject *module);
+extern bool init_graphics(PyObject *module);
+extern bool init_shaders(PyObject *module);
+extern bool init_shaders_debug(PyObject *module);
+#ifdef __APPLE__
+extern int init_CoreText(PyObject *);
+extern bool init_cocoa(PyObject *module);
+#endif
+
+
+EXPORTED PyMODINIT_FUNC
 PyInit_fast_data_types(void) {
     PyObject *m;
 
     m = PyModule_Create(&module);
     if (m == NULL) return NULL;
+#ifdef __APPLE__
+    mach_timebase_info(&timebase);
+#endif
 
     if (m != NULL) {
         if (!init_LineBuf(m)) return NULL;
         if (!init_HistoryBuf(m)) return NULL;
         if (!init_Line(m)) return NULL;
         if (!init_Cursor(m)) return NULL;
+        if (!init_child_monitor(m)) return NULL;
         if (!init_ColorProfile(m)) return NULL;
-        if (!init_SpriteMap(m)) return NULL;
-        if (!init_ChangeTracker(m)) return NULL;
         if (!init_Screen(m)) return NULL;
-        if (!add_module_gl_constants(m)) return NULL;
         if (!init_glfw(m)) return NULL;
-        if (!init_Window(m)) return NULL;
+        if (!init_sprites(m)) return NULL;
+        if (!init_state(m)) return NULL;
+        if (!init_keys(m)) return NULL;
+        if (!init_graphics(m)) return NULL;
+        if (PySys_GetObject("debug_gl") == Py_True) {
+            if (!init_shaders_debug(m)) return NULL;
+        } else { 
+            if (!init_shaders(m)) return NULL;
+        }
 #ifdef __APPLE__
         if (!init_CoreText(m)) return NULL;
+        if (!init_cocoa(m)) return NULL;
 #else
         if (!init_Face(m)) return NULL;
         if (!init_freetype_library(m)) return NULL;
         if (!init_fontconfig_library(m)) return NULL;
 #endif
+
+#define OOF(n) #n, offsetof(Cell, n)
+        if (PyModule_AddObject(m, "CELL", Py_BuildValue("{sI sI sI sI sI sI sI sI sI}",
+                    OOF(ch), OOF(fg), OOF(bg), OOF(decoration_fg), OOF(cc), OOF(sprite_x), OOF(sprite_y), OOF(sprite_z), "size", sizeof(Cell))) != 0) return NULL;
+#undef OOF
         PyModule_AddIntConstant(m, "BOLD", BOLD_SHIFT);
         PyModule_AddIntConstant(m, "ITALIC", ITALIC_SHIFT);
         PyModule_AddIntConstant(m, "REVERSE", REVERSE_SHIFT);
@@ -99,17 +226,6 @@ PyInit_fast_data_types(void) {
         PyModule_AddIntMacro(m, DECCOLM);
         PyModule_AddIntMacro(m, DECOM);
         PyModule_AddIntMacro(m, IRM);
-        PyModule_AddIntMacro(m, DATA_CELL_SIZE);
-        PyModule_AddIntMacro(m, ANY_MODE);
-        PyModule_AddIntMacro(m, MOTION_MODE);
-        PyModule_AddIntMacro(m, BUTTON_MODE);
-        PyModule_AddIntMacro(m, SGR_PROTOCOL);
-        PyModule_AddIntMacro(m, NORMAL_PROTOCOL);
-        PyModule_AddIntMacro(m, URXVT_PROTOCOL);
-        PyModule_AddIntMacro(m, UTF8_PROTOCOL);
-#ifdef IUTF8
-        PyModule_AddIntMacro(m, IUTF8);
-#endif
     }
 
     return m;

kitty/data-types.h

@@ -10,36 +10,36 @@
 
 #include <stdint.h>
 #include <stdbool.h>
+#include <poll.h>
+#include <pthread.h>
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #define UNUSED __attribute__ ((unused))
+#define EXPORTED __attribute__ ((visibility ("default")))
+#define LIKELY(x)    __builtin_expect (!!(x), 1)
+#define UNLIKELY(x)  __builtin_expect (!!(x), 0)
 #define MAX(x, y) (((x) > (y)) ? (x) : (y))
 #define MIN(x, y) (((x) > (y)) ? (y) : (x))
 #define xstr(s) str(s)
 #define str(s) #s
+#define fatal(...) { fprintf(stderr, __VA_ARGS__); fprintf(stderr, "\n"); exit(EXIT_FAILURE); }
 
-typedef Py_UCS4 char_type;
+typedef uint32_t char_type;
 typedef uint32_t color_type;
 typedef uint32_t combining_type;
 typedef unsigned int index_type;
+typedef uint16_t sprite_index;
+typedef uint16_t attrs_type;
+typedef enum CursorShapes { NO_CURSOR_SHAPE, CURSOR_BLOCK, CURSOR_BEAM, CURSOR_UNDERLINE, NUM_OF_CURSOR_SHAPES } CursorShape;
 
 #define ERROR_PREFIX "[PARSE ERROR]"
-#define ANY_MODE 3
-#define MOTION_MODE 2
-#define BUTTON_MODE 1
-#define NORMAL_PROTOCOL 0
-#define UTF8_PROTOCOL 1
-#define SGR_PROTOCOL 2
-#define URXVT_PROTOCOL 3
+typedef enum MouseTrackingModes { NO_TRACKING, BUTTON_MODE, MOTION_MODE, ANY_MODE } MouseTrackingMode;
+typedef enum MouseTrackingProtocols { NORMAL_PROTOCOL, UTF8_PROTOCOL, SGR_PROTOCOL, URXVT_PROTOCOL} MouseTrackingProtocol;
+typedef enum MouseShapes { BEAM, HAND, ARROW } MouseShape;
 
-#define CELL_FIELD_COUNT 5
-#define CELL_SIZE (CELL_FIELD_COUNT * 4)
-// The data cell size must be a multiple of 3
-#define DATA_CELL_SIZE 2 * 3
-
-#define CHAR_MASK 0xFFFFFF
-#define ATTRS_SHIFT 24
-#define ATTRS_MASK_WITHOUT_WIDTH 0xFC000000
+#define MAX_CHILDREN 256
+#define BLANK_CHAR 0
+#define ATTRS_MASK_WITHOUT_WIDTH 0xFFC
 #define WIDTH_MASK  3
 #define DECORATION_SHIFT  2
 #define DECORATION_MASK 3
@@ -54,50 +54,24 @@ typedef unsigned int index_type;
 #define UTF8_REJECT 1
 #define UNDERCURL_CODE 6
 #define DECORATION_FG_CODE 58
+#define CHAR_IS_BLANK(ch) ((ch) == 32 || (ch) == 0)
 
-#define CURSOR_BLOCK 1
-#define CURSOR_BEAM 2
-#define CURSOR_UNDERLINE 3
 #define FG 1
 #define BG 2
 
 #define CURSOR_TO_ATTRS(c, w) \
     ((w) | (((c->decoration & 3) << DECORATION_SHIFT) | ((c->bold & 1) << BOLD_SHIFT) | \
-            ((c->italic & 1) << ITALIC_SHIFT) | ((c->reverse & 1) << REVERSE_SHIFT) | ((c->strikethrough & 1) << STRIKE_SHIFT))) << ATTRS_SHIFT
+            ((c->italic & 1) << ITALIC_SHIFT) | ((c->reverse & 1) << REVERSE_SHIFT) | ((c->strikethrough & 1) << STRIKE_SHIFT))) 
 
 #define ATTRS_TO_CURSOR(a, c) \
     c->decoration = (a >> DECORATION_SHIFT) & 3; c->bold = (a >> BOLD_SHIFT) & 1; c->italic = (a >> ITALIC_SHIFT) & 1; \
     c->reverse = (a >> REVERSE_SHIFT) & 1; c->strikethrough = (a >> STRIKE_SHIFT) & 1;
 
-#define SET_ATTRIBUTE(chars, shift, val) \
-    mask = shift == DECORATION_SHIFT ? 3 : 1; \
-    val = (val & mask) << (ATTRS_SHIFT + shift); \
-    mask = ~(mask << (ATTRS_SHIFT + shift)); \
-    for (index_type i = 0; i < self->xnum; i++)  (chars)[i] = ((chars)[i] & mask) | val;
-
 #define COPY_CELL(src, s, dest, d) \
-        (dest)->chars[d] = (src)->chars[s]; \
-        (dest)->fg_colors[d] = (src)->fg_colors[s]; \
-        (dest)->bg_colors[d] = (src)->bg_colors[s]; \
-        (dest)->decoration_fg[d] = (src)->decoration_fg[s]; \
-        (dest)->combining_chars[d] = (src)->combining_chars[s];
+    (dest)->cells[d] = (src)->cells[s];
 
 #define COPY_SELF_CELL(s, d) COPY_CELL(self, s, self, d)
 
-#define COPY_LINE(src, dest) \
-    memcpy((dest)->chars, (src)->chars, sizeof(char_type) * MIN((src)->xnum, (dest)->xnum)); \
-    memcpy((dest)->fg_colors, (src)->fg_colors, sizeof(color_type) * MIN((src)->xnum, (dest)->xnum)); \
-    memcpy((dest)->bg_colors, (src)->bg_colors, sizeof(color_type) * MIN((src)->xnum, (dest)->xnum)); \
-    memcpy((dest)->decoration_fg, (src)->decoration_fg, sizeof(color_type) * MIN((src)->xnum, (dest)->xnum)); \
-    memcpy((dest)->combining_chars, (src)->combining_chars, sizeof(combining_type) * MIN((src)->xnum, (dest)->xnum)); 
-
-#define CLEAR_LINE(l, num) \
-    for (index_type i = 0; i < (num); i++) (l)->chars[i] = (1 << ATTRS_SHIFT) | 32; \
-    memset((l)->fg_colors, 0, (num) * sizeof(color_type)); \
-    memset((l)->bg_colors, 0, (num) * sizeof(color_type)); \
-    memset((l)->decoration_fg, 0, (num) * sizeof(color_type)); \
-    memset((l)->combining_chars, 0, (num) * sizeof(combining_type));
-
 #define METHOD(name, arg_type) {#name, (PyCFunction)name, arg_type, name##_doc},
 
 #define BOOL_GETSET(type, x) \
@@ -107,11 +81,15 @@ typedef unsigned int index_type;
 #define GETSET(x) \
     {#x, (getter) x##_get, (setter) x##_set, #x, NULL},
 
+#ifndef EXTRA_INIT
+#define EXTRA_INIT
+#endif
 #define INIT_TYPE(type) \
     int init_##type(PyObject *module) {\
         if (PyType_Ready(&type##_Type) < 0) return 0; \
         if (PyModule_AddObject(module, #type, (PyObject *)&type##_Type) != 0) return 0; \
         Py_INCREF(&type##_Type); \
+        EXTRA_INIT; \
         return 1; \
     }
 
@@ -141,14 +119,18 @@ typedef unsigned int index_type;
 #define END_ALLOW_UNUSED_RESULT _Pragma("GCC diagnostic pop")
 #endif
 
+typedef struct {
+    char_type ch;
+    color_type fg, bg, decoration_fg;
+    combining_type cc;
+    sprite_index sprite_x, sprite_y, sprite_z;
+    attrs_type attrs;
+} Cell;
+
 typedef struct {
     PyObject_HEAD
 
-    char_type *chars;
-    color_type *fg_colors;
-    color_type *bg_colors;
-    color_type *decoration_fg;
-    combining_type *combining_chars;
+    Cell *cells;
     index_type xnum, ynum;
     bool continued;
     bool needs_free;
@@ -159,28 +141,22 @@ PyTypeObject Line_Type;
 typedef struct {
     PyObject_HEAD
 
-    uint8_t *buf;
+    Cell *buf;
     index_type xnum, ynum, *line_map, *scratch;
-    index_type block_size;
     bool *continued_map;
     Line *line;
-
-    // Pointers into buf
-    char_type *chars;
-    color_type *fg_colors;
-    color_type *bg_colors;
-    color_type *decoration_fg;
-    combining_type *combining_chars;
 } LineBuf;
 PyTypeObject LineBuf_Type;
 
+
 typedef struct {
     PyObject_HEAD
 
-    uint32_t *buf;
+    Cell *buf;
     index_type xnum, ynum;
     Line *line;
     index_type start_of_data, count;
+    bool *continued_map;
 } HistoryBuf;
 PyTypeObject HistoryBuf_Type;
 
@@ -189,7 +165,8 @@ typedef struct {
 
     bool bold, italic, reverse, strikethrough, blink;
     unsigned int x, y;
-    uint8_t decoration, shape;
+    uint8_t decoration;
+    CursorShape shape;
     unsigned long fg, bg, decoration_fg;
 
 } Cursor;
@@ -198,60 +175,20 @@ PyTypeObject Cursor_Type;
 PyTypeObject Face_Type;
 PyTypeObject Window_Type;
 
+typedef struct {
+    color_type default_fg, default_bg, cursor_color, highlight_fg, highlight_bg;
+} DynamicColor;
+
 typedef struct {
     PyObject_HEAD
 
+    bool dirty;
     uint32_t color_table[256];
     uint32_t orig_color_table[256];
-
+    DynamicColor configured, overridden;
 } ColorProfile;
 PyTypeObject ColorProfile_Type;
 
-typedef struct SpritePosition SpritePosition;
-struct SpritePosition {
-    SpritePosition *next;
-    unsigned int x, y, z;
-    char_type ch;
-    combining_type cc;
-    bool is_second;
-    bool filled;
-    bool rendered;
-};
-PyTypeObject SpritePosition_Type;
-
-typedef struct {
-    PyObject_HEAD
-
-    size_t max_array_len, max_texture_size, max_y;
-    unsigned int x, y, z, xnum, ynum;
-    SpritePosition cache[1024];
-    bool dirty;
-
-} SpriteMap;
-PyTypeObject SpriteMap_Type;
-
-typedef struct {
-    PyObject_HEAD
-
-    index_type xnum, ynum;
-    bool screen_changed;
-    bool cursor_changed;
-    bool dirty;
-    bool *changed_lines;
-    bool *lines_with_changed_cells;
-    bool *changed_cells;
-    unsigned int history_line_added_count;
-} ChangeTracker;
-PyTypeObject ChangeTracker_Type;
-
-
-typedef struct {
-    bool mLNM, mIRM, mDECTCEM, mDECSCNM, mDECOM, mDECAWM, mDECCOLM, mDECARM, mDECCKM,
-         mBRACKETED_PASTE, mFOCUS_TRACKING, mEXTENDED_KEYBOARD;
-    unsigned long mouse_tracking_mode, mouse_tracking_protocol;
-} ScreenModes;
-PyTypeObject ScreenModes_Type;
-
 #define SAVEPOINTS_SZ 256
 
 typedef struct {
@@ -272,154 +209,66 @@ typedef struct {
 #define PARSER_BUF_SZ (8 * 1024)
 #define READ_BUF_SZ (1024*1024)
 
+typedef struct {
+    double at;
+    PyObject *callback;
+    PyObject *args;
+} TimerEvent;
+
 typedef struct {
     PyObject_HEAD
 
-    unsigned int columns, lines, margin_top, margin_bottom, charset;
-    uint32_t utf8_state, utf8_codepoint, *g0_charset, *g1_charset, *g_charset;
-    bool use_latin1;
-    Cursor *cursor;
-    SavepointBuffer main_savepoints, alt_savepoints;
-    PyObject *callbacks;
-    LineBuf *linebuf, *main_linebuf, *alt_linebuf;
-    HistoryBuf *historybuf;
-    bool *tabstops, *main_tabstops, *alt_tabstops;
-    ChangeTracker *change_tracker;
-    ScreenModes modes;
+    PyObject *dump_callback, *update_screen, *death_notify;
+    unsigned int count;
+    bool shutting_down;
+    pthread_t io_thread;
+} ChildMonitor;
+PyTypeObject ChildMonitor_Type;
 
-    uint32_t parser_buf[PARSER_BUF_SZ];
-    unsigned int parser_state, parser_text_start, parser_buf_pos;
-    bool parser_has_pending_text;
-    uint8_t read_buf[READ_BUF_SZ];
-    uint32_t default_fg, default_bg, highlight_fg, highlight_bg, cursor_color;
-
-} Screen;
-PyTypeObject Screen_Type;
+#define clear_sprite_position(cell) (cell).sprite_x = 0; (cell).sprite_y = 0; (cell).sprite_z = 0; 
 
 #define left_shift_line(line, at, num) \
     for(index_type __i__ = (at); __i__ < (line)->xnum - (num); __i__++) { \
         COPY_CELL(line, __i__ + (num), line, __i__) \
     } \
-    if ((((line)->chars[(at)] >> ATTRS_SHIFT) & WIDTH_MASK) != 1) (line)->chars[(at)] = (1 << ATTRS_SHIFT) | 32;
-
+    if ((((line)->cells[(at)].attrs) & WIDTH_MASK) != 1) { \
+        (line)->cells[(at)].ch = BLANK_CHAR; \
+        (line)->cells[(at)].attrs = BLANK_CHAR ? 1 : 0; \
+        clear_sprite_position((line)->cells[(at)]); \
+    }
 
 // Global functions 
+const char* base64_decode(const uint32_t *src, size_t src_sz, uint8_t *dest, size_t dest_capacity, size_t *dest_sz);
 Line* alloc_line();
 Cursor* alloc_cursor();
 LineBuf* alloc_linebuf(unsigned int, unsigned int);
 HistoryBuf* alloc_historybuf(unsigned int, unsigned int);
-ChangeTracker* alloc_change_tracker(unsigned int, unsigned int);
-int init_LineBuf(PyObject *);
-int init_HistoryBuf(PyObject *);
-int init_Cursor(PyObject *);
-int init_Line(PyObject *);
-int init_ColorProfile(PyObject *);
-int init_SpriteMap(PyObject *);
-int init_ChangeTracker(PyObject *);
-int init_Screen(PyObject *);
-int init_Face(PyObject *);
-int init_Window(PyObject *);
-int init_CoreText(PyObject *);
+ColorProfile* alloc_color_profile();
 PyObject* create_256_color_table();
-PyObject* read_bytes_dump(PyObject UNUSED *, PyObject *);
-PyObject* read_bytes(PyObject UNUSED *, PyObject *);
 PyObject* parse_bytes_dump(PyObject UNUSED *, PyObject *);
 PyObject* parse_bytes(PyObject UNUSED *, PyObject *);
 uint32_t decode_utf8(uint32_t*, uint32_t*, uint8_t byte);
+unsigned int encode_utf8(uint32_t ch, char* dest);
 void cursor_reset(Cursor*);
 Cursor* cursor_copy(Cursor*);
 void cursor_copy_to(Cursor *src, Cursor *dest);
 void cursor_reset_display_attrs(Cursor*);
-bool update_cell_range_data(ScreenModes *modes, SpriteMap *, Line *, unsigned int, unsigned int, ColorProfile *, const uint32_t, const uint32_t, unsigned int *);
-uint32_t to_color(ColorProfile *, uint32_t, uint32_t);
+void set_sprite_position(Cell *cell, Cell *previous_cell);
 
-PyObject* line_text_at(char_type, combining_type);
-void line_clear_text(Line *self, unsigned int at, unsigned int num, int ch);
-void line_apply_cursor(Line *self, Cursor *cursor, unsigned int at, unsigned int num, bool clear_char);
-void line_set_char(Line *, unsigned int , uint32_t , unsigned int , Cursor *);
-void line_right_shift(Line *, unsigned int , unsigned int );
-void line_add_combining_char(Line *, uint32_t , unsigned int );
-index_type line_as_ansi(Line *self, Py_UCS4 *buf, index_type buflen);
-unsigned int line_length(Line *self);
+double monotonic();
+PyObject* cm_thread_write(PyObject *self, PyObject *args);
+bool schedule_write_to_child(unsigned long id, const char *data, size_t sz);
+bool set_iutf8(int, bool);
 
-void linebuf_init_line(LineBuf *, index_type);
-void linebuf_clear(LineBuf *, char_type ch);
-void linebuf_init_line(LineBuf *, index_type);
-void linebuf_index(LineBuf* self, index_type top, index_type bottom);
-void linebuf_reverse_index(LineBuf *self, index_type top, index_type bottom);
-void linebuf_clear_line(LineBuf *self, index_type y);
-void linebuf_insert_lines(LineBuf *self, unsigned int num, unsigned int y, unsigned int bottom);
-void linebuf_delete_lines(LineBuf *self, index_type num, index_type y, index_type bottom);
-void linebuf_set_attribute(LineBuf *, unsigned int , unsigned int );
-void linebuf_rewrap(LineBuf *self, LineBuf *other, int *cursor_y_out, HistoryBuf *);
-unsigned int linebuf_char_width_at(LineBuf *self, index_type x, index_type y);
-bool historybuf_resize(HistoryBuf *self, index_type lines);
-void historybuf_add_line(HistoryBuf *self, const Line *line);
-void historybuf_rewrap(HistoryBuf *self, HistoryBuf *other);
-void historybuf_init_line(HistoryBuf *self, index_type num, Line *l);
+color_type colorprofile_to_color(ColorProfile *self, color_type entry, color_type defval);
+void copy_color_table_to_buffer(ColorProfile *self, color_type *address, int offset, size_t stride);
 
 unsigned int safe_wcwidth(uint32_t ch);
 void change_wcwidth(bool use9);
-void screen_align(Screen*);
-void screen_restore_cursor(Screen *);
-void screen_save_cursor(Screen *);
-void screen_cursor_position(Screen*, unsigned int, unsigned int);
-void screen_cursor_back(Screen *self, unsigned int count/*=1*/, int move_direction/*=-1*/);
-void screen_erase_in_line(Screen *, unsigned int, bool);
-void screen_erase_in_display(Screen *, unsigned int, bool);
-void screen_draw(Screen *screen, uint32_t codepoint);
-void screen_ensure_bounds(Screen *self, bool use_margins);
-void screen_toggle_screen_buffer(Screen *self);
-void screen_normal_keypad_mode(Screen *self); 
-void screen_alternate_keypad_mode(Screen *self);  
-void screen_change_default_color(Screen *self, unsigned int which, uint32_t col);
-void screen_alignment_display(Screen *self);
-void screen_reverse_index(Screen *self);
-void screen_index(Screen *self);
-void screen_scroll(Screen *self, unsigned int count);
-void screen_reverse_scroll(Screen *self, unsigned int count);
-void screen_reset(Screen *self);
-void screen_set_tab_stop(Screen *self);
-void screen_tab(Screen *self);
-void screen_backtab(Screen *self, unsigned int);
-void screen_clear_tab_stop(Screen *self, unsigned int how);
-void screen_set_mode(Screen *self, unsigned int mode);
-void screen_reset_mode(Screen *self, unsigned int mode);
-void screen_insert_characters(Screen *self, unsigned int count);
-void screen_cursor_up(Screen *self, unsigned int count/*=1*/, bool do_carriage_return/*=false*/, int move_direction/*=-1*/);
-void screen_set_cursor(Screen *self, unsigned int mode, uint8_t secondary);
-void screen_cursor_to_column(Screen *self, unsigned int column);
-void screen_cursor_down(Screen *self, unsigned int count/*=1*/);
-void screen_cursor_forward(Screen *self, unsigned int count/*=1*/);
-void screen_cursor_down1(Screen *self, unsigned int count/*=1*/);
-void screen_cursor_up1(Screen *self, unsigned int count/*=1*/);
-void screen_cursor_to_line(Screen *screen, unsigned int line);
-void screen_insert_lines(Screen *self, unsigned int count/*=1*/);
-void screen_delete_lines(Screen *self, unsigned int count/*=1*/);
-void screen_delete_characters(Screen *self, unsigned int count);
-void screen_erase_characters(Screen *self, unsigned int count);
-void screen_set_margins(Screen *self, unsigned int top, unsigned int bottom);
-void screen_change_charset(Screen *, uint32_t to);
-void screen_designate_charset(Screen *, uint32_t which, uint32_t as);
-void screen_use_latin1(Screen *, bool);
-void set_title(Screen *self, PyObject*);
-void set_icon(Screen *self, PyObject*);
-void set_dynamic_color(Screen *self, unsigned int code, PyObject*);
-void set_color_table_color(Screen *self, unsigned int code, PyObject*);
-uint32_t* translation_table(uint32_t which);
-uint32_t *latin1_charset;
-void screen_request_capabilities(Screen *, PyObject *);
-void report_device_attributes(Screen *self, unsigned int UNUSED mode, char start_modifier);
-void select_graphic_rendition(Screen *self, unsigned int *params, unsigned int count);
-void report_device_status(Screen *self, unsigned int which, bool UNUSED);
-void report_mode_status(Screen *self, unsigned int which, bool);
-#define DECLARE_CH_SCREEN_HANDLER(name) void screen_##name(Screen *screen);
-DECLARE_CH_SCREEN_HANDLER(bell)
-DECLARE_CH_SCREEN_HANDLER(backspace)
-DECLARE_CH_SCREEN_HANDLER(tab)
-DECLARE_CH_SCREEN_HANDLER(linefeed)
-DECLARE_CH_SCREEN_HANDLER(carriage_return)
-
-bool init_freetype_library(PyObject*);
-bool init_fontconfig_library(PyObject*);
-PyObject *get_fontconfig_font(PyObject *self, PyObject *args);
+void set_mouse_cursor(MouseShape);
+void mouse_event(int, int);
+void scroll_event(double, double);
+void set_special_key_combo(int glfw_key, int mods);
+void on_text_input(unsigned int codepoint, int mods);
+void on_key_input(int key, int scancode, int action, int mods);
+void request_window_attention();

kitty/fontconfig.c

@@ -8,7 +8,7 @@
 #include "data-types.h"
 #include <fontconfig/fontconfig.h>
 
-PyObject*
+static PyObject*
 get_fontconfig_font(PyObject UNUSED *self, PyObject *args) {
     char *family;
     int bold, italic, allow_bitmapped_fonts, index = 0, hint_style=0, weight=0, slant=0;
@@ -74,11 +74,21 @@ end:
     return ans;
 }
 
+static PyMethodDef module_methods[] = {
+    {"get_fontconfig_font", (PyCFunction)get_fontconfig_font, METH_VARARGS, ""},
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
 bool 
-init_fontconfig_library(PyObject UNUSED *m) {
+init_fontconfig_library(PyObject *module) {
     if (!FcInit()) {
         PyErr_SetString(PyExc_RuntimeError, "Failed to initialize the fontconfig library");
         return false;
     }
+    if (Py_AtExit(FcFini) != 0) {
+        PyErr_SetString(PyExc_RuntimeError, "Failed to register the fontconfig library at exit handler");
+        return false;
+    }
+    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
     return true;
 }

kitty/fonts/core_text.py

@@ -3,18 +3,73 @@
 # License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
 
 import ctypes
-from kitty.fast_data_types import CTFace as Face
-from kitty.utils import get_logical_dpi, wcwidth, ceil_int
+import re
+import sys
+
+from kitty.fast_data_types import CTFace as Face, coretext_all_fonts
+from kitty.utils import ceil_int, get_logical_dpi, safe_print, wcwidth, adjust_line_height
 
 main_font = {}
 symbol_map = {}
 cell_width = cell_height = baseline = CellTexture = WideCellTexture = underline_thickness = underline_position = None
+attr_map = {(False, False): 'font_family', (True, False): 'bold_font', (False, True): 'italic_font', (True, True): 'bold_italic_font'}
 
 
-def install_symbol_map(val, font_size, dpi):
+def create_font_map(all_fonts):
+    ans = {'family_map': {}, 'ps_map': {}, 'full_map': {}}
+    for x in all_fonts:
+        f = (x['family'] or '').lower()
+        s = (x['style'] or '').lower()
+        ps = (x['postscript_name'] or '').lower()
+        ans['family_map'].setdefault(f, []).append(x)
+        ans['ps_map'].setdefault(ps, []).append(x)
+        ans['full_map'].setdefault(f + ' ' + s, []).append(x)
+    return ans
+
+
+def find_best_match(font_map, family, bold, italic):
+    q = re.sub(r'\s+', ' ', family.lower())
+
+    def score(candidate):
+        style_match = 1 if candidate['bold'] == bold and candidate['italic'] == italic else 0
+        monospace_match = 1 if candidate['monospace'] else 0
+        return style_match, monospace_match
+
+    # First look for an exact match
+    for selector in ('ps_map', 'full_map'):
+        candidates = font_map[selector].get(q)
+        if candidates:
+            candidates.sort(key=score)
+            return candidates[-1]
+
+    # Let CoreText choose the font if the family exists, otherwise
+    # fallback to Menlo
+    if q not in font_map['family_map']:
+        safe_print('The font {} was not found, falling back to Menlo'.format(family), file=sys.stderr)
+        family = 'Menlo'
+    return {
+        'monospace': True,
+        'bold': bold,
+        'italic': italic,
+        'family': family
+    }
+
+
+def get_face(font_map, family, main_family, font_size, dpi, bold=False, italic=False):
+    def resolve_family(f):
+        if (bold or italic) and f == 'auto':
+            f = main_family
+        if f.lower() == 'monospace':
+            f = 'Menlo'
+        return f
+    descriptor = find_best_match(font_map, resolve_family(family), bold, italic)
+    return Face(descriptor, font_size, dpi)
+
+
+def install_symbol_map(all_fonts, val, font_size, dpi):
     global symbol_map
     symbol_map = {}
-    family_map = {f: Face(f, False, False, False, font_size, dpi) for f in set(val.values())}
+    family_map = {f: get_face(all_fonts, f, 'Menlo', font_size, dpi) for f in set(val.values())}
     for ch, family in val.items():
         symbol_map[ch] = family_map[family]
 
@@ -28,23 +83,16 @@ def set_font_family(opts, override_font_size=None, ignore_dpi_failure=False):
             raise
         dpi = (72, 72)  # Happens when running via develop() in an ssh session
     dpi = sum(dpi) / 2.0
-    attr_map = {(False, False): 'font_family', (True, False): 'bold_font', (False, True): 'italic_font', (True, True): 'bold_italic_font'}
-
-    def get_family(bold, italic):
-        ans = getattr(opts, attr_map[(bold, italic)])
-        if ans.lower() == 'monospace':
-            ans = 'Menlo'
-        if ans == 'auto' and (bold or italic):
-            ans = get_family(False, False)
-        return ans
     font_size = override_font_size or opts.font_size
+    all_fonts = create_font_map(coretext_all_fonts())
 
-    for bold in (False, True):
-        for italic in (False, True):
-            main_font[(bold, italic)] = Face(get_family(bold, italic), bold, italic, True, font_size, dpi)
-    install_symbol_map(opts.symbol_map, font_size, dpi)
+    for (bold, italic), attr in attr_map.items():
+        main_font[(bold, italic)] = get_face(all_fonts, getattr(opts, attr), opts.font_family, font_size, dpi, bold, italic)
+
+    install_symbol_map(all_fonts, opts.symbol_map, font_size, dpi)
     mf = main_font[(False, False)]
     cell_width, cell_height = mf.cell_size()
+    cell_height = adjust_line_height(cell_height, opts.adjust_line_height)
     CellTexture = ctypes.c_ubyte * (cell_width * cell_height)
     WideCellTexture = ctypes.c_ubyte * (2 * cell_width * cell_height)
     baseline = int(round(mf.ascent))
@@ -53,6 +101,20 @@ def set_font_family(opts, override_font_size=None, ignore_dpi_failure=False):
     return cell_width, cell_height
 
 
+def test_font_matching(name='Menlo', bold=False, italic=False, dpi=72.0, font_size=11.0):
+    all_fonts = create_font_map(coretext_all_fonts())
+    face = get_face(all_fonts, name, 'Menlo', font_size, dpi, bold, italic)
+    return face
+
+
+def test_family_matching(name='Menlo', dpi=72.0, font_size=11.0):
+    all_fonts = create_font_map(coretext_all_fonts())
+    for bold in (False, True):
+        for italic in (False, True):
+            face = get_face(all_fonts, name, 'Menlo', font_size, dpi, bold, italic)
+            print(bold, italic, face)
+
+
 def current_cell():
     return CellTexture, cell_width, cell_height, baseline, underline_thickness, underline_position
 

kitty/fonts/fontconfig.py

@@ -4,7 +4,6 @@
 
 import os
 import re
-import subprocess
 from collections import namedtuple
 
 from kitty.fast_data_types import Face, get_fontconfig_font
@@ -35,51 +34,7 @@ def font_not_found(err, char):
     return FontNotFound(msg)
 
 
-def get_font_subprocess(
-    family='monospace',
-    bold=False,
-    italic=False,
-    allow_bitmaped_fonts=False,
-    size_in_pts=None,
-    character=None,
-    dpi=None
-):
-    query = escape_family_name(family)
-    if character is not None:
-        query += ':charset={:x}'.format(ord(character[0]))
-    if not allow_bitmaped_fonts:
-        query += ':scalable=true:outline=true'
-    if size_in_pts is not None:
-        query += ':size={:.1f}'.format(size_in_pts)
-    if dpi is not None:
-        query += ':dpi={:.1f}'.format(dpi)
-    if bold:
-        query += ':weight=200'
-    if italic:
-        query += ':slant=100'
-    try:
-        raw = subprocess.check_output([
-            'fc-match', query, '-f',
-            '%{file}\x1e%{hinting}\x1e%{hintstyle}\x1e%{scalable}\x1e%{outline}\x1e%{weight}\x1e%{slant}\x1e%{index}'
-        ]).decode('utf-8')
-    except subprocess.CalledProcessError as err:
-        raise font_not_found(err, character)
-    parts = raw.split('\x1e')
-    try:
-        path, hinting, hintstyle, scalable, outline, weight, slant, index = parts
-    except ValueError:
-        path = parts[0]
-        hintstyle, hinting, scalable, outline, weight, slant, index = 1, 'True', 'True', 'True', 100, 0, 0
-    return Font(
-        path,
-        to_bool(hinting),
-        int(hintstyle), bold, italic,
-        to_bool(scalable),
-        to_bool(outline), int(weight), int(slant), int(index)
-    )
-
-
-def get_font_lib(
+def get_font(
     family='monospace',
     bold=False,
     italic=False,
@@ -102,9 +57,6 @@ def get_font_lib(
     )
 
 
-get_font = get_font_lib
-
-
 def find_font_for_character(
     family,
     char,

kitty/fonts/freetype.py

@@ -4,15 +4,13 @@
 
 import ctypes
 import sys
-import unicodedata
 from collections import namedtuple
 from functools import lru_cache
 from itertools import chain
-from threading import Lock
 
-from kitty.fast_data_types import FT_PIXEL_MODE_GRAY, Face
+from kitty.fast_data_types import FT_PIXEL_MODE_GRAY, Face, FreeTypeError
 from kitty.fonts.box_drawing import render_missing_glyph
-from kitty.utils import ceil_int, get_logical_dpi, safe_print, wcwidth
+from kitty.utils import ceil_int, get_logical_dpi, safe_print, wcwidth, adjust_line_height
 
 from .fontconfig import (
     FontNotFound, find_font_for_character, font_for_family, get_font_files
@@ -93,6 +91,7 @@ def set_font_family(opts, override_font_size=None):
     cell_height = font_units_to_pixels(
         face.height, face.units_per_EM, size_in_pts, dpi[1]
     )
+    cell_height = adjust_line_height(cell_height, opts.adjust_line_height)
     baseline = font_units_to_pixels(
         face.ascender, face.units_per_EM, size_in_pts, dpi[1]
     )
@@ -113,7 +112,6 @@ def set_font_family(opts, override_font_size=None):
 CharBitmap = namedtuple(
     'CharBitmap', 'data bearingX bearingY advance rows columns'
 )
-freetype_lock = Lock()
 
 
 def render_to_bitmap(font, face, text):
@@ -161,26 +159,25 @@ def render_char(text, bold=False, italic=False, width=1):
     elif italic:
         key = 'italic'
     ch = text[0]
-    with freetype_lock:
-        font = symbol_map.get(ch)
-        if font is None or not font.face.get_char_index(ch):
-            font = current_font_family.get(key) or current_font_family['regular']
-            face = font.face
-            if not face.get_char_index(ch):
-                try:
-                    font = font_for_char(ch, bold, italic)
-                except FontNotFound:
-                    font = font_for_char(
-                        ch, bold, italic, allow_bitmaped_fonts=True
-                    )
-                face = alt_face_cache.get(font)
-                if face is None:
-                    face = alt_face_cache[font] = Face(font.face, font.index)
-                    if face.is_scalable:
-                        set_char_size(face, **cff_size)
-        else:
-            face = font.face
-        return render_using_face(font, face, text, width, italic, bold)
+    font = symbol_map.get(ch)
+    if font is None or not font.face.get_char_index(ch):
+        font = current_font_family.get(key) or current_font_family['regular']
+        face = font.face
+        if not face.get_char_index(ch):
+            try:
+                font = font_for_char(ch, bold, italic)
+            except FontNotFound:
+                font = font_for_char(
+                    ch, bold, italic, allow_bitmaped_fonts=True
+                )
+            face = alt_face_cache.get(font)
+            if face is None:
+                face = alt_face_cache[font] = Face(font.face, font.index)
+                if face.is_scalable:
+                    set_char_size(face, **cff_size)
+    else:
+        face = font.face
+    return render_using_face(font, face, text, width, italic, bold)
 
 
 def place_char_in_cell(bitmap_char):
@@ -254,13 +251,16 @@ def missing_glyph(width):
 def render_cell(text=' ', bold=False, italic=False):
     # TODO: Handle non-normalizable combining chars. Probably need to use
     # harfbuzz for that
-    text = unicodedata.normalize('NFC', text)[0]
+    text = text[0]
     width = wcwidth(text)
     try:
         bitmap_char = render_char(text, bold, italic, width)
     except FontNotFound as err:
         safe_print('ERROR:', err, file=sys.stderr)
         return missing_glyph(width)
+    except FreeTypeError as err:
+        safe_print('Failed to render text:', repr(text), 'with error:', err, file=sys.stderr)
+        return missing_glyph(width)
     second = None
     if width == 2:
         if bitmap_char.columns > cell_width:

kitty/fonts/render.py

@@ -3,6 +3,7 @@
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
 import ctypes
+from math import sin, pi, ceil, floor, sqrt
 
 from kitty.constants import isosx
 from .box_drawing import render_box_char, is_renderable_box_char
@@ -12,7 +13,7 @@ else:
     from .freetype import set_font_family, render_cell as rc, current_cell  # noqa
 
 
-def add_line(buf, cell_width, position, thickness):
+def add_line(buf, cell_width, position, thickness, cell_height):
     y = position - thickness // 2
     while thickness:
         thickness -= 1
@@ -22,15 +23,28 @@ def add_line(buf, cell_width, position, thickness):
         y += 1
 
 
-def add_curl(buf, cell_width, position, thickness):
-    for y in range(position - thickness, position):
-        for x in range(0, cell_width // 2):
-            offset = cell_width * y
-            buf[offset + x] = 255
-    for y in range(position, position + thickness):
-        for x in range(cell_width // 2, cell_width):
-            offset = cell_width * y
-            buf[offset + x] = 255
+def add_curl(buf, cell_width, position, thickness, cell_height):
+    xfactor = 2.0 * pi / cell_width
+    yfactor = thickness
+
+    def clamp_y(y):
+        return max(0, min(int(y), cell_height - 1))
+
+    def clamp_x(x):
+        return max(0, min(int(x), cell_width - 1))
+
+    def add_intensity(x, y, distance):
+        buf[cell_width * y + x] = min(255, buf[cell_width * y + x] + int(255 * (1 - distance)))
+
+    for x_exact in range(cell_width):
+        y_exact = yfactor * sin(x_exact * xfactor) + position
+        y_below = clamp_y(floor(y_exact))
+        y_above = clamp_y(ceil(y_exact))
+        x_before, x_after = map(clamp_x, (x_exact - 1, x_exact + 1))
+        for x in {x_before, x_exact, x_after}:
+            for y in {y_below, y_above}:
+                dist = sqrt((x - x_exact)**2 + (y - y_exact)**2) / 2
+                add_intensity(x, y, dist)
 
 
 def render_cell(text=' ', bold=False, italic=False, underline=0, strikethrough=False):
@@ -49,14 +63,30 @@ def render_cell(text=' ', bold=False, italic=False, underline=0, strikethrough=F
         t = underline_thickness
         if underline == 2:
             t = max(1, min(cell_height - underline_position - 1, t))
-        dl(add_curl if underline == 2 else add_line, underline_position, t)
+        dl(add_curl if underline == 2 else add_line, underline_position, t, cell_height)
     if strikethrough:
         pos = int(0.65 * baseline)
-        dl(add_line, pos, underline_thickness)
+        dl(add_line, pos, underline_thickness, cell_height)
 
     return first, second
 
 
+class Buf:
+
+    def __init__(self, buf):
+        self.buf = buf
+
+    def __call__(self):
+        return ctypes.addressof(self.buf)
+
+
+def render_cell_wrapper(text, bold, italic, underline, strikethrough, is_second):
+    first, second = render_cell(text, bold, italic, underline, strikethrough)
+    ans = second if is_second else first
+    ans = ans or render_cell()[0]
+    return Buf(ans)
+
+
 def join_cells(cell_width, cell_height, *cells):
     dstride = len(cells) * cell_width
     ans = (ctypes.c_ubyte * (cell_height * dstride))()

kitty/freetype.c

@@ -16,8 +16,10 @@ typedef struct {
     unsigned int units_per_EM;
     int ascender, descender, height, max_advance_width, max_advance_height, underline_position, underline_thickness;
     bool is_scalable;
+    PyObject *path;
 } Face;
 
+static PyObject* FreeType_Exception = NULL;
 
 void 
 set_freetype_error(const char* prefix, int err_code) {
@@ -41,12 +43,12 @@ set_freetype_error(const char* prefix, int err_code) {
 
     while(ft_errors[i].err_msg != NULL) {
         if (ft_errors[i].err_code == err_code) {
-            PyErr_Format(PyExc_ValueError, "%s %s", prefix, ft_errors[i].err_msg);
+            PyErr_Format(FreeType_Exception, "%s %s", prefix, ft_errors[i].err_msg);
             return;
         }
         i++;
     }
-    PyErr_Format(PyExc_ValueError, "%s (error code: %d)", prefix, err_code);
+    PyErr_Format(FreeType_Exception, "%s (error code: %d)", prefix, err_code);
 }
 
 static FT_Library  library;
@@ -63,9 +65,9 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
 
     self = (Face *)type->tp_alloc(type, 0);
     if (self != NULL) {
-        Py_BEGIN_ALLOW_THREADS;
+        self->path = PyTuple_GET_ITEM(args, 0);
+        Py_INCREF(self->path);
         error = FT_New_Face(library, path, index, &(self->face));
-        Py_END_ALLOW_THREADS;
         if(error) { set_freetype_error("Failed to load face, with error:", error); Py_CLEAR(self); return NULL; }
 #define CPY(n) self->n = self->face->n;
         CPY(units_per_EM); CPY(ascender); CPY(descender); CPY(height); CPY(max_advance_width); CPY(max_advance_height); CPY(underline_position); CPY(underline_thickness);
@@ -78,9 +80,20 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
 static void
 dealloc(Face* self) {
     FT_Done_Face(self->face);
+    Py_CLEAR(self->path);
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
+static PyObject *
+repr(Face *self) {
+    return PyUnicode_FromFormat(
+        "Face(path=%S, is_scalable=%S, units_per_EM=%u, ascender=%i, descender=%i, height=%i, max_advance_width=%i max_advance_height=%i, underline_position=%i, underline_thickness=%i)",
+        self->path, self->is_scalable ? Py_True : Py_False, 
+        self->ascender, self->descender, self->height, self->max_advance_width, self->max_advance_height, self->underline_position, self->underline_thickness
+    );
+}
+
+
 static PyObject*
 set_char_size(Face *self, PyObject *args) {
 #define set_char_size_doc "set_char_size(width, height, xdpi, ydpi) -> set the character size. width, height is in 1/64th of a pt. dpi is in pixels per inch"
@@ -88,9 +101,7 @@ set_char_size(Face *self, PyObject *args) {
     unsigned int xdpi, ydpi;
     int error;
     if (!PyArg_ParseTuple(args, "llII", &char_width, &char_height, &xdpi, &ydpi)) return NULL;
-    Py_BEGIN_ALLOW_THREADS;
     error = FT_Set_Char_Size(self->face, char_width, char_height, xdpi, ydpi);
-    Py_END_ALLOW_THREADS;
     if (error) { set_freetype_error("Failed to set char size, with error:", error); Py_CLEAR(self); return NULL; }
     Py_RETURN_NONE;
 }
@@ -107,9 +118,7 @@ load_char(Face *self, PyObject *args) {
         if (hintstyle >= 3) flags |= FT_LOAD_TARGET_NORMAL;
         else if (0 < hintstyle  && hintstyle < 3) flags |= FT_LOAD_TARGET_LIGHT;
     } else flags |= FT_LOAD_NO_HINTING;
-    Py_BEGIN_ALLOW_THREADS;
     error = FT_Load_Glyph(self->face, glyph_index, flags);
-    Py_END_ALLOW_THREADS;
     if (error) { set_freetype_error("Failed to load glyph, with error:", error); Py_CLEAR(self); return NULL; }
     Py_RETURN_NONE;
 }
@@ -120,9 +129,7 @@ get_char_index(Face *self, PyObject *args) {
     int code;
     unsigned int ans;
     if (!PyArg_ParseTuple(args, "C", &code)) return NULL;
-    Py_BEGIN_ALLOW_THREADS;
     ans = FT_Get_Char_Index(self->face, code);
-    Py_END_ALLOW_THREADS;
 
     return Py_BuildValue("I", ans);
 }
@@ -198,7 +205,7 @@ trim_to_width(Face UNUSED *self, PyObject *args) {
     rows = PyLong_AsUnsignedLong(PyStructSequence_GET_ITEM(bitmap, 0));
     width = PyLong_AsUnsignedLong(PyStructSequence_GET_ITEM(bitmap, 1));
     extra = width - cell_width;
-    if (extra >= cell_width) { PyErr_SetString(PyExc_ValueError, "Too large for trimming"); return NULL; }
+    if (extra >= cell_width) { PyErr_SetString(FreeType_Exception, "Too large for trimming"); return NULL; }
     PyObject *ans = PyStructSequence_New(&BitmapType);
     if (ans == NULL) return PyErr_NoMemory();
     src = (unsigned char*)PyByteArray_AS_STRING(PyStructSequence_GET_ITEM(bitmap, 3));
@@ -240,6 +247,7 @@ static PyMemberDef members[] = {
     MEM(underline_position, T_INT),
     MEM(underline_thickness, T_INT),
     MEM(is_scalable, T_BOOL),
+    MEM(path, T_OBJECT_EX),
     {NULL}  /* Sentinel */
 };
 
@@ -264,18 +272,30 @@ PyTypeObject Face_Type = {
     .tp_methods = methods,
     .tp_members = members,
     .tp_new = new,                
+    .tp_repr = (reprfunc)repr,
 };
 
 INIT_TYPE(Face)
 
+static void
+free_freetype() {
+    FT_Done_FreeType(library);
+}
 
 bool 
 init_freetype_library(PyObject *m) {
+    FreeType_Exception = PyErr_NewException("fast_data_types.FreeTypeError", NULL, NULL);
+    if (FreeType_Exception == NULL) return false;
+    if (PyModule_AddObject(m, "FreeTypeError", FreeType_Exception) != 0) return false;
     int error = FT_Init_FreeType(&library);
     if (error) {
         set_freetype_error("Failed to initialize FreeType library, with error:", error);
         return false;
     }
+    if (Py_AtExit(free_freetype) != 0) {
+        PyErr_SetString(FreeType_Exception, "Failed to register the freetype library at exit handler");
+        return false;
+    }
     if (PyStructSequence_InitType2(&GlpyhMetricsType, &gm_desc) != 0) return false;
     if (PyStructSequence_InitType2(&BitmapType, &bm_desc) != 0) return false;
     PyModule_AddObject(m, "GlyphMetrics", (PyObject*)&GlpyhMetricsType);

kitty/glfw.c

@@ -4,7 +4,7 @@
  * Distributed under terms of the GPL3 license.
  */
 
-#include "data-types.h"
+#include "state.h"
 #include <structmember.h>
 #include <GLFW/glfw3.h>
 #if defined(__APPLE__)
@@ -16,125 +16,166 @@
 #error "glfw >= 3.2 required"
 #endif
 
-#define MAX_WINDOWS 256
+#if GLFW_VERSION_MAJOR > 3 || (GLFW_VERSION_MAJOR == 3 && GLFW_VERSION_MINOR > 2)
+#define has_request_attention
+#define has_init_hint_string
+#endif
+
+#if GLFW_KEY_LAST >= MAX_KEY_COUNT
+#error "glfw has too many keys, you should increase MAX_KEY_COUNT"
+#endif
 
 #define CALLBACK(name, fmt, ...) \
     if ((name) != NULL) { \
-        PyGILState_STATE _pystate = PyGILState_Ensure(); \
         PyObject *_pyret = PyObject_CallFunction((name), fmt, __VA_ARGS__); \
-        if (_pyret == NULL && PyErr_Occurred() != NULL) PyErr_Print(); \
-        Py_CLEAR(_pyret); \
-        PyGILState_Release(_pystate); \
+        if (_pyret == NULL) PyErr_Print(); \
+        else Py_DECREF(_pyret); \
     } 
 
 #define WINDOW_CALLBACK(name, fmt, ...) \
-    Window *self = find_window(w); \
-    if (self) { CALLBACK(self->name, "O" fmt, self, __VA_ARGS__); }
+    CALLBACK(the_window->name, "O" fmt, the_window, __VA_ARGS__);
 
 typedef struct {
     PyObject_HEAD
 
     GLFWwindow *window;
-    PyObject *framebuffer_size_callback, *char_mods_callback, *key_callback, *mouse_button_callback, *scroll_callback, *cursor_pos_callback, *window_focus_callback;
-    GLFWcursor *standard_cursor, *click_cursor;
-} Window;
+    PyObject *framebuffer_size_callback, *window_focus_callback;
+    GLFWcursor *standard_cursor, *click_cursor, *arrow_cursor;
+} WindowWrapper;
+
+static void update_viewport(GLFWwindow *window) {
+    int w, h;
+    glfwGetFramebufferSize(window, &global_state.viewport_width, &global_state.viewport_height);
+    glfwGetWindowSize(window, &w, &h);
+    global_state.viewport_x_ratio = (double)global_state.viewport_width / (double)w;
+    global_state.viewport_y_ratio = (double)global_state.viewport_height / (double)h;
+}
 
 // callbacks {{{
-static Window* window_weakrefs[MAX_WINDOWS] = {0};
-
-static inline Window*
-find_window(GLFWwindow *w) {
-    for(int i = 0; i < MAX_WINDOWS; i++) {
-        if (window_weakrefs[i] == NULL) break; 
-        if (window_weakrefs[i]->window == w) return window_weakrefs[i];
-    }
-    return NULL;
-}
+static WindowWrapper* the_window = NULL;
+update_viewport_size_func update_viewport_size = NULL;
 
 static void 
 framebuffer_size_callback(GLFWwindow *w, int width, int height) {
-    WINDOW_CALLBACK(framebuffer_size_callback, "ii", width, height);
+    if (width > 100 && height > 100) {
+        update_viewport_size(width, height);
+        update_viewport(w);
+        WINDOW_CALLBACK(framebuffer_size_callback, "ii", width, height);
+        glfwPostEmptyEvent();
+    } else fprintf(stderr, "Ignoring resize request for tiny size: %dx%d\n", width, height);
 }
 
 static void 
-char_mods_callback(GLFWwindow *w, unsigned int codepoint, int mods) {
-    WINDOW_CALLBACK(char_mods_callback, "Ii", codepoint, mods);
+char_mods_callback(GLFWwindow UNUSED *w, unsigned int codepoint, int mods) {
+    global_state.cursor_blink_zero_time = monotonic();
+    on_text_input(codepoint, mods);
 }
 
 static void 
-key_callback(GLFWwindow *w, int key, int scancode, int action, int mods) {
-    WINDOW_CALLBACK(key_callback, "iiii", key, scancode, action, mods);
+key_callback(GLFWwindow UNUSED *w, int key, int scancode, int action, int mods) {
+    global_state.cursor_blink_zero_time = monotonic();
+    if (key >= 0 && key <= GLFW_KEY_LAST) {
+        global_state.is_key_pressed[key] = action == GLFW_RELEASE ? false : true;
+        on_key_input(key, scancode, action, mods);
+    }
 }
 
 static void 
 mouse_button_callback(GLFWwindow *w, int button, int action, int mods) {
-    WINDOW_CALLBACK(mouse_button_callback, "iii", button, action, mods);
-}
-
-static void 
-scroll_callback(GLFWwindow *w, double xoffset, double yoffset) {
-    WINDOW_CALLBACK(scroll_callback, "dd", xoffset, yoffset);
+    if (glfwGetInputMode(w, GLFW_CURSOR) != GLFW_CURSOR_NORMAL) { glfwSetInputMode(w, GLFW_CURSOR, GLFW_CURSOR_NORMAL); } 
+    double now = monotonic();
+    global_state.last_mouse_activity_at = now;
+    if (button >= 0 && (unsigned int)button < sizeof(global_state.mouse_button_pressed)/sizeof(global_state.mouse_button_pressed[0])) {
+        global_state.mouse_button_pressed[button] = action == GLFW_PRESS ? true : false;
+        mouse_event(button, mods);
+    }
 }
 
 static void 
 cursor_pos_callback(GLFWwindow *w, double x, double y) {
-    WINDOW_CALLBACK(cursor_pos_callback, "dd", x, y);
+    if (glfwGetInputMode(w, GLFW_CURSOR) != GLFW_CURSOR_NORMAL) { glfwSetInputMode(w, GLFW_CURSOR, GLFW_CURSOR_NORMAL); } 
+    double now = monotonic();
+    global_state.last_mouse_activity_at = now;
+    global_state.cursor_blink_zero_time = now;
+    global_state.mouse_x = x * global_state.viewport_x_ratio;
+    global_state.mouse_y = y * global_state.viewport_y_ratio;
+    mouse_event(-1, 0);
 }
 
 static void 
-window_focus_callback(GLFWwindow *w, int focused) {
+scroll_callback(GLFWwindow *w, double xoffset, double yoffset) {
+    if (glfwGetInputMode(w, GLFW_CURSOR) != GLFW_CURSOR_NORMAL) { glfwSetInputMode(w, GLFW_CURSOR, GLFW_CURSOR_NORMAL); } 
+    double now = monotonic();
+    global_state.last_mouse_activity_at = now;
+    scroll_event(xoffset, yoffset);
+}
+
+static void 
+window_focus_callback(GLFWwindow UNUSED *w, int focused) {
+    global_state.application_focused = focused ? true : false;
+    double now = monotonic();
+    global_state.last_mouse_activity_at = now;
+    global_state.cursor_blink_zero_time = now;
     WINDOW_CALLBACK(window_focus_callback, "O", focused ? Py_True : Py_False);
 }
 // }}}
 
+void
+set_mouse_cursor(MouseShape type) {
+    switch(type) {
+        case HAND:
+            glfwSetCursor(the_window->window, the_window->click_cursor);
+            break;
+        case ARROW:
+            glfwSetCursor(the_window->window, the_window->arrow_cursor);
+            break;
+        default:
+            glfwSetCursor(the_window->window, the_window->standard_cursor);
+            break;
+    }
+}
+
 static PyObject*
 new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
-    Window *self;
+    WindowWrapper *self;
     char *title;
-    int width, height, i;
+    int width, height;
     if (!PyArg_ParseTuple(args, "iis", &width, &height, &title)) return NULL;
+    if (the_window != NULL) { PyErr_SetString(PyExc_ValueError, "Only one glfw window is supported");  return NULL; }
 
-    self = (Window *)type->tp_alloc(type, 0);
+    self = (WindowWrapper *)type->tp_alloc(type, 0);
     if (self != NULL) {
+        the_window = self;
         self->window = glfwCreateWindow(width, height, title, NULL, NULL);
-        if (self->window == NULL) { Py_CLEAR(self); PyErr_SetString(PyExc_ValueError, "Failed to create GLFWwindow"); return NULL; }
-        for(i = 0; i < MAX_WINDOWS; i++) {
-            if (window_weakrefs[i] == NULL) { window_weakrefs[i] = self; break; }
-        }
-        if (i >= MAX_WINDOWS) { Py_CLEAR(self); PyErr_SetString(PyExc_ValueError, "Too many windows created"); return NULL; }
+        if (self->window == NULL) { Py_CLEAR(self); the_window = NULL; PyErr_SetString(PyExc_ValueError, "Failed to create GLFWwindow"); return NULL; }
+        update_viewport(self->window);
         self->standard_cursor = glfwCreateStandardCursor(GLFW_IBEAM_CURSOR);
         self->click_cursor = glfwCreateStandardCursor(GLFW_HAND_CURSOR);
-        if (self->standard_cursor == NULL || self->click_cursor == NULL) { Py_CLEAR(self); PyErr_SetString(PyExc_ValueError, "Failed to create standard mouse cursors"); return NULL; }
+        self->arrow_cursor = glfwCreateStandardCursor(GLFW_ARROW_CURSOR);
+        if (self->standard_cursor == NULL || self->click_cursor == NULL || self->arrow_cursor == NULL) {
+            Py_CLEAR(self); PyErr_SetString(PyExc_ValueError, "Failed to create standard mouse cursors"); return NULL; }
+        set_mouse_cursor(0);
         glfwSetFramebufferSizeCallback(self->window, framebuffer_size_callback);
         glfwSetCharModsCallback(self->window, char_mods_callback);
-        glfwSetKeyCallback(self->window, key_callback);
         glfwSetMouseButtonCallback(self->window, mouse_button_callback);
         glfwSetScrollCallback(self->window, scroll_callback);
         glfwSetCursorPosCallback(self->window, cursor_pos_callback);
+        glfwSetKeyCallback(self->window, key_callback);
         glfwSetWindowFocusCallback(self->window, window_focus_callback);
     }
     return (PyObject*)self;
 }
  
 // Global functions {{{
-static PyObject *error_callback = NULL;
-
 static void 
-cb_error_callback(int error, const char* description) {
-    CALLBACK(error_callback, "is", error, description) else fprintf(stderr, "[glfw error]: %s\n", description);
-}
-
-PyObject*
-glfw_set_error_callback(PyObject UNUSED *self, PyObject *callback) {
-    Py_CLEAR(error_callback);
-    error_callback = callback;
-    Py_INCREF(callback);
-    Py_RETURN_NONE;
+error_callback(int error, const char* description) {
+    fprintf(stderr, "[glfw error %d]: %s\n", error, description);
 }
 
 
 PyObject*
 glfw_init(PyObject UNUSED *self) {
+    glfwSetErrorCallback(error_callback);
     PyObject *ans = glfwInit() ? Py_True: Py_False;
     Py_INCREF(ans);
     return ans;
@@ -169,10 +210,8 @@ glfw_wait_events(PyObject UNUSED *self, PyObject *args) {
         time = PyFloat_AsDouble(PyTuple_GET_ITEM(args, 0));
         if (PyErr_Occurred()) PyErr_Clear();
     }
-    Py_BEGIN_ALLOW_THREADS;
     if (time < 0) glfwWaitEvents();
     else glfwWaitEventsTimeout(time);
-    Py_END_ALLOW_THREADS;
     Py_RETURN_NONE;
 }
 
@@ -208,53 +247,78 @@ glfw_get_key_name(PyObject UNUSED *self, PyObject *args) {
     return Py_BuildValue("s", glfwGetKeyName(key, scancode));
 }
 
+PyObject*
+glfw_init_hint_string(PyObject UNUSED *self, PyObject *args) {
+    int hint_id;
+    char *hint;
+    if (!PyArg_ParseTuple(args, "is", &hint_id, &hint)) return NULL;
+#ifdef has_init_hint_string
+    glfwInitHintString(hint_id, hint);
+#endif
+    Py_RETURN_NONE;
+}
+
+
 // }}}
 
 static void
-dealloc(Window* self) {
-    for(unsigned int i = 0; i < MAX_WINDOWS; i++) {
-        if (window_weakrefs[i] == self) window_weakrefs[i] = NULL;
-    }
-    Py_CLEAR(self->framebuffer_size_callback); Py_CLEAR(self->char_mods_callback); Py_CLEAR(self->key_callback); Py_CLEAR(self->mouse_button_callback); Py_CLEAR(self->scroll_callback); Py_CLEAR(self->cursor_pos_callback); Py_CLEAR(self->window_focus_callback);
+dealloc(WindowWrapper* self) {
+    the_window = NULL;
+    Py_CLEAR(self->framebuffer_size_callback); Py_CLEAR(self->window_focus_callback);
     if (self->window != NULL) glfwDestroyWindow(self->window);
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
 static PyObject*
-swap_buffers(Window *self) {
-    Py_BEGIN_ALLOW_THREADS;
+swap_buffers(WindowWrapper *self) {
     glfwSwapBuffers(self->window);
-    Py_END_ALLOW_THREADS;
     Py_RETURN_NONE;
 }
 
 static PyObject*
-make_context_current(Window *self) {
+make_context_current(WindowWrapper *self) {
     glfwMakeContextCurrent(self->window);
     Py_RETURN_NONE;
 }
 
 static PyObject*
-should_close(Window *self) {
+window_id(WindowWrapper *self) {
+    return PyLong_FromVoidPtr(self->window);
+}
+
+static PyObject*
+show(WindowWrapper *self) {
+    glfwShowWindow(self->window);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+hide(WindowWrapper *self) {
+    glfwHideWindow(self->window);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+should_close(WindowWrapper *self) {
     PyObject *ans = glfwWindowShouldClose(self->window) ? Py_True : Py_False;
     Py_INCREF(ans);
     return ans;
 }
 
 static PyObject*
-get_clipboard_string(Window *self) {
+get_clipboard_string(WindowWrapper *self) {
     return Py_BuildValue("s", glfwGetClipboardString(self->window));
 }
 
 static PyObject*
-get_cursor_pos(Window *self) {
+get_cursor_pos(WindowWrapper *self) {
     double x=0, y=0;
     glfwGetCursorPos(self->window, &x, &y);
     return Py_BuildValue("dd", x, y);
 }
 
 static PyObject*
-set_should_close(Window *self, PyObject *args) {
+set_should_close(WindowWrapper *self, PyObject *args) {
     int c;
     if (!PyArg_ParseTuple(args, "p", &c)) return NULL;
     glfwSetWindowShouldClose(self->window, c);
@@ -262,15 +326,15 @@ set_should_close(Window *self, PyObject *args) {
 }
 
 static PyObject*
-set_input_mode(Window *self, PyObject *args) {
-    int which, value;
-    if (!PyArg_ParseTuple(args, "ii", &which, &value)) return NULL;
-    glfwSetInputMode(self->window, which, value);
+set_pos(WindowWrapper *self, PyObject *args) {
+    int x, y;
+    if (!PyArg_ParseTuple(args, "ii", &x, &y)) return NULL;
+    glfwSetWindowPos(self->window, x, y);
     Py_RETURN_NONE;
 }
 
 static PyObject*
-is_key_pressed(Window *self, PyObject *args) {
+is_key_pressed(WindowWrapper *self, PyObject *args) {
     int c;
     if (!PyArg_ParseTuple(args, "i", &c)) return NULL;
     PyObject *ans = glfwGetKey(self->window, c) == GLFW_PRESS ? Py_True : Py_False;
@@ -279,15 +343,7 @@ is_key_pressed(Window *self, PyObject *args) {
 }
 
 static PyObject*
-set_click_cursor(Window *self, PyObject *args) {
-    int c;
-    if (!PyArg_ParseTuple(args, "p", &c)) return NULL;
-    glfwSetCursor(self->window, c ? self->click_cursor : self->standard_cursor);
-    Py_RETURN_NONE;
-}
-
-static PyObject*
-set_clipboard_string(Window *self, PyObject *args) {
+set_clipboard_string(WindowWrapper *self, PyObject *args) {
     char *title;
     if(!PyArg_ParseTuple(args, "s", &title)) return NULL;
     glfwSetClipboardString(self->window, title);
@@ -295,7 +351,7 @@ set_clipboard_string(Window *self, PyObject *args) {
 }
 
 static PyObject*
-set_window_icon(Window *self, PyObject *args) {
+set_window_icon(WindowWrapper *self, PyObject *args) {
     GLFWimage logo;
     Py_ssize_t sz;
     if(!PyArg_ParseTuple(args, "s#ii", &(logo.pixels), &sz, &(logo.width), &(logo.height))) return NULL;
@@ -305,23 +361,14 @@ set_window_icon(Window *self, PyObject *args) {
 
 
 static PyObject*
-_set_title(Window *self, PyObject *args) {
-    char *title;
-    if(!PyArg_ParseTuple(args, "s", &title)) return NULL;
-    glfwSetWindowTitle(self->window, title);
-    Py_RETURN_NONE;
-}
-
-
-static PyObject*
-get_framebuffer_size(Window *self) {
+get_framebuffer_size(WindowWrapper *self) {
     int w, h;
     glfwGetFramebufferSize(self->window, &w, &h);
     return Py_BuildValue("ii", w, h);
 }
 
 static PyObject*
-get_window_size(Window *self) {
+get_window_size(WindowWrapper *self) {
     int w, h;
     glfwGetWindowSize(self->window, &w, &h);
     return Py_BuildValue("ii", w, h);
@@ -363,29 +410,36 @@ current_monitor(GLFWwindow *window) {
 }
 
 static PyObject*
-current_monitor_dpi(Window *self) {
+current_monitor_dpi(WindowWrapper *self) {
     GLFWmonitor *m = current_monitor(self->window);
     if (m == NULL) return NULL;
     return get_physical_dpi(m);
 }
 
-#ifdef glfwRequestWindowAttention
-static PyObject*
-request_window_attention(Window *self) {
-    glfwRequestWindowAttention(self->window);
-    Py_RETURN_NONE;
-}
+void
+request_window_attention() {
+#ifdef has_request_attention
+    glfwRequestWindowAttention(the_window->window);
 #endif
+    glfwPostEmptyEvent();
+}
 
 #ifdef __APPLE__
 static PyObject*
-cocoa_window_id(Window *self) {
+cocoa_window_id(WindowWrapper *self) {
     void *wid = glfwGetCocoaWindow(self->window);
     if (wid == NULL) { PyErr_SetString(PyExc_ValueError, "Failed to get native window handle"); return NULL; }
     return PyLong_FromVoidPtr(wid);
 }
 #endif
 
+static PyObject*
+primary_monitor_size(PyObject UNUSED *self) {
+    GLFWmonitor* monitor = glfwGetPrimaryMonitor();
+    const GLFWvidmode* mode = glfwGetVideoMode(monitor);
+    return Py_BuildValue("ii", mode->width, mode->height);
+}
+
 // Boilerplate {{{
 #define MND(name, args) {#name, (PyCFunction)name, args, ""}
 
@@ -397,40 +451,33 @@ static PyMethodDef methods[] = {
     MND(get_framebuffer_size, METH_NOARGS),
     MND(get_window_size, METH_NOARGS),
     MND(current_monitor_dpi, METH_NOARGS),
-#ifdef glfwRequestWindowAttention
-    MND(request_window_attention, METH_NOARGS),
-#endif
 #ifdef __APPLE__
     MND(cocoa_window_id, METH_NOARGS),
 #endif
     MND(set_should_close, METH_VARARGS),
-    MND(set_input_mode, METH_VARARGS),
     MND(is_key_pressed, METH_VARARGS),
-    MND(set_click_cursor, METH_VARARGS),
+    MND(set_pos, METH_VARARGS),
     MND(set_clipboard_string, METH_VARARGS),
     MND(make_context_current, METH_NOARGS),
-    {"set_title", (PyCFunction)_set_title, METH_VARARGS, ""},
+    MND(window_id, METH_NOARGS),
+    MND(show, METH_NOARGS),
+    MND(hide, METH_NOARGS),
     {"set_icon", (PyCFunction)set_window_icon, METH_VARARGS, ""},
     {NULL}  /* Sentinel */
 };
 
 static PyMemberDef members[] = {
-#define CBE(name) {#name, T_OBJECT_EX, offsetof(Window, name), 0, #name}
+#define CBE(name) {#name, T_OBJECT_EX, offsetof(WindowWrapper, name), 0, #name}
     CBE(framebuffer_size_callback),
-    CBE(char_mods_callback),
-    CBE(key_callback),
-    CBE(mouse_button_callback),
-    CBE(scroll_callback),
-    CBE(cursor_pos_callback),
     CBE(window_focus_callback),
     {NULL}
 #undef CBE
 };
  
-PyTypeObject Window_Type = {
+PyTypeObject WindowWrapper_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
-    .tp_name = "fast_data_types.Window",
-    .tp_basicsize = sizeof(Window),
+    .tp_name = "fast_data_types.GLFWWindow",
+    .tp_basicsize = sizeof(WindowWrapper),
     .tp_dealloc = (destructor)dealloc, 
     .tp_flags = Py_TPFLAGS_DEFAULT,        
     .tp_doc = "A GLFW window",
@@ -439,15 +486,32 @@ PyTypeObject Window_Type = {
     .tp_new = new,                
 };
 
-INIT_TYPE(Window)
-
+static PyMethodDef module_methods[] = {
+    {"glfw_init", (PyCFunction)glfw_init, METH_NOARGS, ""}, \
+    {"glfw_terminate", (PyCFunction)glfw_terminate, METH_NOARGS, ""}, \
+    {"glfw_window_hint", (PyCFunction)glfw_window_hint, METH_VARARGS, ""}, \
+    {"glfw_swap_interval", (PyCFunction)glfw_swap_interval, METH_VARARGS, ""}, \
+    {"glfw_wait_events", (PyCFunction)glfw_wait_events, METH_VARARGS, ""}, \
+    {"glfw_post_empty_event", (PyCFunction)glfw_post_empty_event, METH_NOARGS, ""}, \
+    {"glfw_get_physical_dpi", (PyCFunction)glfw_get_physical_dpi, METH_NOARGS, ""}, \
+    {"glfw_get_key_name", (PyCFunction)glfw_get_key_name, METH_VARARGS, ""}, \
+    {"glfw_init_hint_string", (PyCFunction)glfw_init_hint_string, METH_VARARGS, ""}, \
+    {"glfw_primary_monitor_size", (PyCFunction)primary_monitor_size, METH_NOARGS, ""}, \
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
 
 // constants {{{
 bool
 init_glfw(PyObject *m) {
-    PyEval_InitThreads();
-    glfwSetErrorCallback(cb_error_callback);
+    if (PyModule_AddFunctions(m, module_methods) != 0) return false;
+    if (PyType_Ready(&WindowWrapper_Type) < 0) return false; 
+    if (PyModule_AddObject(m, "GLFWWindow", (PyObject *)&WindowWrapper_Type) != 0) return 0;
+    Py_INCREF(&WindowWrapper_Type); 
 #define ADDC(n) if(PyModule_AddIntConstant(m, #n, n) != 0) return false;
+#ifdef GLFW_X11_WM_CLASS_NAME
+    ADDC(GLFW_X11_WM_CLASS_NAME)
+    ADDC(GLFW_X11_WM_CLASS_CLASS)
+#endif
     ADDC(GLFW_RELEASE);
     ADDC(GLFW_PRESS);
     ADDC(GLFW_REPEAT);

kitty/glfw.h

@@ -1,39 +0,0 @@
-/*
- * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#pragma once
-
-bool init_glfw(PyObject *m);
-
-PyObject* glfw_set_error_callback(PyObject UNUSED *self, PyObject *callback);
-PyObject* glfw_init(PyObject UNUSED *self);
-PyObject* glfw_terminate(PyObject UNUSED *self);
-PyObject* glfw_window_hint(PyObject UNUSED *self, PyObject *args);
-PyObject* glfw_swap_interval(PyObject UNUSED *self, PyObject *args);
-PyObject* glfw_wait_events(PyObject UNUSED *self, PyObject*);
-PyObject* glfw_post_empty_event(PyObject UNUSED *self);
-PyObject* glfw_get_physical_dpi(PyObject UNUSED *self);
-PyObject* glfw_get_key_name(PyObject UNUSED *self, PyObject *args);
-
-#ifdef __APPLE__
-PyObject* cocoa_hide_titlebar(PyObject UNUSED *self, PyObject *window_id);
-#define COCOA_HIDE_TITLEBAR {"cocoa_hide_titlebar", (PyCFunction)cocoa_hide_titlebar, METH_O, ""}, 
-#else
-#define COCOA_HIDE_TITLEBAR
-#endif
-
-#define GLFW_FUNC_WRAPPERS \
-    {"glfw_set_error_callback", (PyCFunction)glfw_set_error_callback, METH_O, ""}, \
-    {"glfw_init", (PyCFunction)glfw_init, METH_NOARGS, ""}, \
-    {"glfw_terminate", (PyCFunction)glfw_terminate, METH_NOARGS, ""}, \
-    {"glfw_window_hint", (PyCFunction)glfw_window_hint, METH_VARARGS, ""}, \
-    {"glfw_swap_interval", (PyCFunction)glfw_swap_interval, METH_VARARGS, ""}, \
-    {"glfw_wait_events", (PyCFunction)glfw_wait_events, METH_VARARGS, ""}, \
-    {"glfw_post_empty_event", (PyCFunction)glfw_post_empty_event, METH_NOARGS, ""}, \
-    {"glfw_get_physical_dpi", (PyCFunction)glfw_get_physical_dpi, METH_NOARGS, ""}, \
-    {"glfw_get_key_name", (PyCFunction)glfw_get_key_name, METH_VARARGS, ""}, \
-    COCOA_HIDE_TITLEBAR
-

kitty/graphics.c

@@ -0,0 +1,997 @@
+/*
+ * graphics.c
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "graphics.h"
+#include "state.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <stdlib.h>
+
+#include <zlib.h>
+#include <png.h>
+#include <structmember.h>
+
+#define STORAGE_LIMIT (320 * (1024 * 1024))
+
+#define REPORT_ERROR(...) { fprintf(stderr, __VA_ARGS__); fprintf(stderr, "\n"); }
+
+static bool send_to_gpu = true;
+
+GraphicsManager*
+grman_alloc() {
+    GraphicsManager *self = (GraphicsManager *)GraphicsManager_Type.tp_alloc(&GraphicsManager_Type, 0);
+    self->images_capacity = 64;
+    self->images = calloc(self->images_capacity, sizeof(Image));
+    self->capacity = 64;
+    self->render_data = calloc(self->capacity, sizeof(ImageRenderData));
+    if (self->images == NULL || self->render_data == NULL) {
+        PyErr_NoMemory();
+        Py_CLEAR(self); return NULL;
+    }
+    return self;
+}
+
+static inline void
+free_refs_data(Image *img) {
+    free(img->refs); img->refs = NULL;
+    img->refcnt = 0; img->refcap = 0;
+}
+
+static inline void
+free_load_data(LoadData *ld) {
+    free(ld->buf); ld->buf_used = 0; ld->buf_capacity = 0;
+    ld->buf = NULL;
+
+    if (ld->mapped_file) munmap(ld->mapped_file, ld->mapped_file_sz);
+    ld->mapped_file = NULL; ld->mapped_file_sz = 0;
+}
+
+free_texture_func free_texture = NULL;
+send_image_to_gpu_func send_image_to_gpu = NULL;
+
+static inline void
+free_image(GraphicsManager *self, Image *img) {
+    if (img->texture_id) free_texture(&img->texture_id);
+    free_refs_data(img);
+    free_load_data(&(img->load_data));
+    self->used_storage -= img->used_storage;
+}
+
+
+static void
+dealloc(GraphicsManager* self) {
+    size_t i;
+    if (self->images) {
+        for (i = 0; i < self->image_count; i++) free_image(self, self->images + i);
+        free(self->images);
+    }
+    free(self->render_data);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+static size_t internal_id_counter = 1;
+
+static inline void*
+ensure_space(void *array, size_t *capacity, size_t count, size_t item_size, bool initialize) {
+    if (count < *capacity) return array;
+    void *ans = realloc(array, (*capacity) * item_size * 2);
+    if (ans == NULL) fatal("Out of memory re-allocating array.");
+    if (initialize) {
+        memset(((uint8_t*)array) + ((*capacity) * item_size), 0, ((*capacity) * item_size));
+    }
+    *capacity *= 2;
+    return ans;
+}
+
+static inline void
+remove_from_array(void *array, size_t item_size, size_t idx, size_t array_count) {
+    size_t num_to_right = array_count - 1 - idx;
+    uint8_t *p = (uint8_t*)array;
+    if (num_to_right > 0) memmove(p + (idx * item_size), p + ((idx + 1) * item_size), num_to_right * item_size);  
+    memset(p + (item_size * (array_count - 1)), 0, item_size);
+}
+
+static inline Image*
+img_by_internal_id(GraphicsManager *self, size_t id) {
+    for (size_t i = 0; i < self->image_count; i++) {
+        if (self->images[i].internal_id == id) return self->images + i;
+    }
+    return NULL;
+}
+
+static inline Image*
+img_by_client_id(GraphicsManager *self, uint32_t id) {
+    for (size_t i = 0; i < self->image_count; i++) {
+        if (self->images[i].client_id == id) return self->images + i;
+    }
+    return NULL;
+}
+
+static inline void
+remove_image(GraphicsManager *self, size_t idx) {
+    free_image(self, self->images + idx);
+    remove_from_array(self->images, sizeof(Image), idx, self->image_count--);
+    self->layers_dirty = true;
+}
+
+static inline void
+remove_images(GraphicsManager *self, bool(*predicate)(Image*), Image* skip_image) {
+    for (size_t i = self->image_count; i-- > 0;) {
+        Image *img = self->images + i;
+        if (img != skip_image && predicate(img)) {
+            remove_image(self, i);
+        }
+    }
+}
+
+
+// Loading image data {{{
+
+static bool
+trim_predicate(Image *img) {
+    return !img->data_loaded || !img->refcnt;
+}
+
+
+static int
+oldest_last(const void* a, const void *b) {
+    double ans = ((Image*)(b))->atime - ((Image*)(a))->atime;
+    return ans < 0 ? -1 : (ans == 0 ? 0 : 1); 
+}
+
+static inline void
+apply_storage_quota(GraphicsManager *self, size_t storage_limit, Image *currently_added_image) {
+    // First remove unreferenced images, even if they have an id
+    remove_images(self, trim_predicate, currently_added_image);
+    if (self->used_storage < storage_limit) return;
+
+    qsort(self->images, self->image_count, sizeof(Image), oldest_last);
+    while (self->used_storage > storage_limit && self->image_count > 0) {
+        remove_image(self, self->image_count - 1);
+    }
+    if (!self->image_count) self->used_storage = 0;  // sanity check
+}
+
+static char add_response[512] = {0};
+static bool has_add_respose = false;
+
+static inline void
+set_add_response(const char *code, const char *fmt, ...) {
+    va_list args;
+    va_start(args, fmt);
+    size_t sz = sizeof(add_response)/sizeof(add_response[0]);
+    int num = snprintf(add_response, sz, "%s:", code);
+    vsnprintf(add_response + num, sz - num, fmt, args);
+    va_end(args);
+    has_add_respose = true;
+}
+
+// Decode formats {{{
+#define ABRT(code, ...) { set_add_response(#code, __VA_ARGS__); goto err; }
+
+static inline bool
+mmap_img_file(GraphicsManager UNUSED *self, Image *img, int fd, size_t sz, off_t offset) {
+    if (!sz) {
+        struct stat s;
+        if (fstat(fd, &s) != 0) ABRT(EBADF, "Failed to fstat() the fd: %d file with error: [%d] %s", fd, errno, strerror(errno));
+        sz = s.st_size;
+    }
+    void *addr = mmap(0, sz, PROT_READ, MAP_SHARED, fd, offset);
+    if (addr == MAP_FAILED) ABRT(EBADF, "Failed to map image file fd: %d at offset: %zd with size: %zu with error: [%d] %s", fd, offset, sz, errno, strerror(errno)); 
+    img->load_data.mapped_file = addr;
+    img->load_data.mapped_file_sz = sz;
+    return true;
+err:
+    return false;
+}
+
+
+static inline const char*
+zlib_strerror(int ret) {
+#define Z(x) case x: return #x;
+    static char buf[128];
+    switch(ret) {
+        case Z_ERRNO:
+            return strerror(errno);
+        default:
+            snprintf(buf, sizeof(buf)/sizeof(buf[0]), "Unknown error: %d", ret);
+            return buf;
+        Z(Z_STREAM_ERROR);
+        Z(Z_DATA_ERROR);
+        Z(Z_MEM_ERROR);
+        Z(Z_BUF_ERROR);
+        Z(Z_VERSION_ERROR);
+    }
+#undef Z
+}
+
+static inline bool
+inflate_zlib(GraphicsManager UNUSED *self, Image *img, uint8_t *buf, size_t bufsz) {
+    bool ok = false;
+    z_stream z;
+    uint8_t *decompressed = malloc(img->load_data.data_sz);
+    if (decompressed == NULL) fatal("Out of memory allocating decompression buffer");
+    z.zalloc = Z_NULL;
+    z.zfree = Z_NULL;
+    z.opaque = Z_NULL;
+    z.avail_in = bufsz;
+    z.next_in = (Bytef*)buf;
+    z.avail_out = img->load_data.data_sz;
+    z.next_out = decompressed;
+    int ret;
+    if ((ret = inflateInit(&z)) != Z_OK) ABRT(ENOMEM, "Failed to initialize inflate with error: %s", zlib_strerror(ret));
+    if ((ret = inflate(&z, Z_FINISH)) != Z_STREAM_END) ABRT(EINVAL, "Failed to inflate image data with error: %s", zlib_strerror(ret));
+    if (z.avail_out) ABRT(EINVAL, "Image data size post inflation does not match expected size");
+    free_load_data(&img->load_data);
+    img->load_data.buf_capacity = img->load_data.data_sz;
+    img->load_data.buf = decompressed;
+    img->load_data.buf_used = img->load_data.data_sz;
+    ok = true;
+err:
+    inflateEnd(&z);
+    if (!ok) free(decompressed);
+    return ok;
+}
+
+struct fake_file { uint8_t *buf; size_t sz, cur; };
+
+static void
+read_png_from_buffer(png_structp png, png_bytep out, png_size_t length) {
+    struct fake_file *f = png_get_io_ptr(png);
+    if (f) {
+        size_t amt = MIN(length, f->sz - f->cur);
+        memcpy(out, f->buf + f->cur, amt);
+        f->cur += amt;
+    }
+}
+
+static void
+read_png_error_handler(png_structp png_ptr, png_const_charp msg) {
+    jmp_buf *jb;
+    set_add_response("EBADPNG", msg);
+    jb = png_get_error_ptr(png_ptr);
+    if (jb == NULL) fatal("read_png_error_handler: could not retrieve jmp_buf");
+    longjmp(*jb, 1);
+}
+
+static void
+read_png_warn_handler(png_structp UNUSED png_ptr, png_const_charp UNUSED msg) {
+    // ignore warnings
+}
+
+struct png_jmp_data { uint8_t *decompressed; bool ok; png_bytep *row_pointers; int width, height; size_t sz; };
+
+static void
+inflate_png_inner(struct png_jmp_data *d, uint8_t *buf, size_t bufsz) {
+    struct fake_file f = {.buf = buf, .sz = bufsz};
+    png_structp png = NULL;
+    png_infop info = NULL;
+    jmp_buf jb;
+    png = png_create_read_struct(PNG_LIBPNG_VER_STRING, &jb, read_png_error_handler, read_png_warn_handler);
+    if (!png) ABRT(ENOMEM, "Failed to create PNG read structure");
+    info = png_create_info_struct(png);
+    if (!info) ABRT(ENOMEM, "Failed to create PNG info structure");
+
+    if (setjmp(jb)) goto err;
+    
+    png_set_read_fn(png, &f, read_png_from_buffer);
+    png_read_info(png, info);
+    png_byte color_type, bit_depth;
+    d->width      = png_get_image_width(png, info);
+    d->height     = png_get_image_height(png, info);
+    color_type = png_get_color_type(png, info);
+    bit_depth  = png_get_bit_depth(png, info);
+
+    // Ensure we get RGBA data out of libpng
+    if (bit_depth == 16) png_set_strip_16(png);
+    if (color_type == PNG_COLOR_TYPE_PALETTE) png_set_palette_to_rgb(png);
+    // PNG_COLOR_TYPE_GRAY_ALPHA is always 8 or 16bit depth.
+    if (color_type == PNG_COLOR_TYPE_GRAY && bit_depth < 8) png_set_expand_gray_1_2_4_to_8(png);
+
+    if (png_get_valid(png, info, PNG_INFO_tRNS)) png_set_tRNS_to_alpha(png);
+
+    // These color_type don't have an alpha channel then fill it with 0xff.
+    if (color_type == PNG_COLOR_TYPE_RGB || color_type == PNG_COLOR_TYPE_GRAY || color_type == PNG_COLOR_TYPE_PALETTE) png_set_filler(png, 0xFF, PNG_FILLER_AFTER);
+
+    if (color_type == PNG_COLOR_TYPE_GRAY || color_type == PNG_COLOR_TYPE_GRAY_ALPHA) png_set_gray_to_rgb(png);
+    png_read_update_info(png, info);
+
+    int rowbytes = png_get_rowbytes(png, info);
+    d->sz = rowbytes * d->height * sizeof(png_byte);
+    d->decompressed = malloc(d->sz + 16);
+    if (d->decompressed == NULL) ABRT(ENOMEM, "Out of memory allocating decompression buffer for PNG");
+    d->row_pointers = malloc(d->height * sizeof(png_bytep));
+    if (d->row_pointers == NULL) ABRT(ENOMEM, "Out of memory allocating row_pointers buffer for PNG");
+    for (int i = 0; i < d->height; i++) d->row_pointers[i] = d->decompressed + i * rowbytes;
+    png_read_image(png, d->row_pointers);
+
+    d->ok = true;
+err:
+    if (png) png_destroy_read_struct(&png, info ? &info : NULL, NULL);
+    return;
+}
+
+static inline bool
+inflate_png(GraphicsManager UNUSED *self, Image *img, uint8_t *buf, size_t bufsz) {
+    struct png_jmp_data d = {0};
+    inflate_png_inner(&d, buf, bufsz);
+    if (d.ok) {
+        free_load_data(&img->load_data);
+        img->load_data.buf = d.decompressed;
+        img->load_data.buf_capacity = d.sz; 
+        img->load_data.buf_used = d.sz;
+        img->load_data.data_sz = d.sz;
+        img->width = d.width; img->height = d.height;
+    }
+    else free(d.decompressed);
+    free(d.row_pointers);
+    return d.ok;
+}
+#undef ABRT
+// }}}
+
+static bool
+add_trim_predicate(Image *img) {
+    return !img->data_loaded || (!img->client_id && !img->refcnt);
+}
+
+static inline Image*
+find_or_create_image(GraphicsManager *self, uint32_t id, bool *existing) {
+    if (id) {
+        for (size_t i = 0; i < self->image_count; i++) {
+            if (self->images[i].client_id == id) {
+                *existing = true;
+                return self->images + i;
+            }
+        }
+    }
+    *existing = false;
+    self->images = ensure_space(self->images, &self->images_capacity, self->image_count, sizeof(Image), true);
+    return self->images + self->image_count++;
+}
+
+
+static Image*
+handle_add_command(GraphicsManager *self, const GraphicsCommand *g, const uint8_t *payload, bool *is_dirty, uint32_t iid) {
+#define ABRT(code, ...) { set_add_response(#code, __VA_ARGS__); self->loading_image = 0; if (img) img->data_loaded = false; return NULL; }
+#define MAX_DATA_SZ (4 * 100000000)
+    has_add_respose = false;
+    bool existing, init_img = true;
+    Image *img = NULL;
+    unsigned char tt = g->transmission_type ? g->transmission_type : 'd';
+    enum FORMATS { RGB=24, RGBA=32, PNG=100 };
+    uint32_t fmt = g->format ? g->format : RGBA;
+    if (tt == 'd' && self->loading_image) init_img = false;
+    if (init_img) {
+        self->last_init_graphics_command = *g;
+        self->last_init_graphics_command.id = iid;
+        self->loading_image = 0;
+        if (g->data_width > 10000 || g->data_height > 10000) ABRT(EINVAL, "Image too large");
+        remove_images(self, add_trim_predicate, NULL);
+        img = find_or_create_image(self, iid, &existing);
+        if (existing) {
+            free_load_data(&img->load_data);
+            img->data_loaded = false;
+            free_refs_data(img);
+            *is_dirty = true;
+            self->layers_dirty = true;
+        } else {
+            img->internal_id = internal_id_counter++;
+            img->client_id = iid;
+        }
+        img->atime = monotonic(); img->used_storage = 0;
+        img->width = g->data_width; img->height = g->data_height;
+        switch(fmt) {
+            case PNG:  
+                if (g->data_sz > MAX_DATA_SZ) ABRT(EINVAL, "PNG data size too large");
+                img->load_data.is_4byte_aligned = true;
+                img->load_data.is_opaque = false;
+                img->load_data.data_sz = g->data_sz ? g->data_sz : 1024 * 100;
+                break;
+            case RGB:
+            case RGBA:
+                img->load_data.data_sz = g->data_width * g->data_height * (fmt / 8);
+                if (!img->load_data.data_sz) ABRT(EINVAL, "Zero width/height not allowed");
+                img->load_data.is_4byte_aligned = fmt == RGBA || (img->width % 4 == 0);
+                img->load_data.is_opaque = fmt == RGB;
+                break;
+            default:
+                ABRT(EINVAL, "Unknown image format: %u", fmt);
+        }
+        if (tt == 'd') {
+            if (g->more) self->loading_image = img->internal_id;
+            img->load_data.buf_capacity = img->load_data.data_sz + (g->compressed ? 1024 : 10);  // compression header
+            img->load_data.buf = malloc(img->load_data.buf_capacity);
+            img->load_data.buf_used = 0;
+            if (img->load_data.buf == NULL) {
+                ABRT(ENOMEM, "Out of memory");
+                img->load_data.buf_capacity = 0; img->load_data.buf_used = 0;
+            }
+        }
+    } else {
+        self->last_init_graphics_command.more = g->more;
+        self->last_init_graphics_command.payload_sz = g->payload_sz;
+        g = &self->last_init_graphics_command;
+        tt = g->transmission_type ? g->transmission_type : 'd';
+        fmt = g->format ? g->format : RGBA;
+        img = img_by_internal_id(self, self->loading_image);
+        if (img == NULL) {
+            self->loading_image = 0;
+            ABRT(EILSEQ, "More payload loading refers to non-existent image");
+        }
+    }
+    int fd;
+    static char fname[2056] = {0};
+    switch(tt) {
+        case 'd':  // direct
+            if (img->load_data.buf_capacity - img->load_data.buf_used < g->payload_sz) {
+                if (img->load_data.buf_used + g->payload_sz > MAX_DATA_SZ || fmt != PNG) ABRT(EFBIG, "Too much data");
+                img->load_data.buf_capacity = MIN(2 * img->load_data.buf_capacity, MAX_DATA_SZ);
+                img->load_data.buf = realloc(img->load_data.buf, img->load_data.buf_capacity);
+                if (img->load_data.buf == NULL) {
+                    ABRT(ENOMEM, "Out of memory");
+                    img->load_data.buf_capacity = 0; img->load_data.buf_used = 0;
+                }
+            }
+            memcpy(img->load_data.buf + img->load_data.buf_used, payload, g->payload_sz);
+            img->load_data.buf_used += g->payload_sz;
+            if (!g->more) { img->data_loaded = true; self->loading_image = 0; }
+            break;
+        case 'f': // file
+        case 't': // temporary file
+        case 's': // POSIX shared memory
+            if (g->payload_sz > 2048) ABRT(EINVAL, "Filename too long");
+            snprintf(fname, sizeof(fname)/sizeof(fname[0]), "%.*s", (int)g->payload_sz, payload);
+            if (tt == 's') fd = shm_open(fname, O_RDONLY, 0);
+            else fd = open(fname, O_CLOEXEC | O_RDONLY);
+            if (fd == -1) ABRT(EBADF, "Failed to open file %s for graphics transmission with error: [%d] %s", fname, errno, strerror(errno));
+            img->data_loaded = mmap_img_file(self, img, fd, g->data_sz, g->data_offset);
+            close(fd);
+            if (tt == 't') unlink(fname);
+            else if (tt == 's') shm_unlink(fname);
+            break;
+        default:
+            ABRT(EINVAL, "Unknown transmission type: %c", g->transmission_type);
+    }
+    if (!img->data_loaded) return NULL;
+    self->loading_image = 0;
+    bool needs_processing = g->compressed || fmt == PNG;
+    if (needs_processing) {
+        uint8_t *buf; size_t bufsz;
+#define IB { if (img->load_data.buf) { buf = img->load_data.buf; bufsz = img->load_data.buf_used; } else { buf = img->load_data.mapped_file; bufsz = img->load_data.mapped_file_sz; } }
+        switch(g->compressed) {
+            case 'z':
+                IB;
+                if (!inflate_zlib(self, img, buf, bufsz)) {
+                    img->data_loaded = false; return NULL;
+                }
+                break;
+            case 0:
+                break;
+            default:
+                ABRT(EINVAL, "Unknown image compression: %c", g->compressed);
+        }
+        switch(fmt) {
+            case PNG:
+                IB;
+                if (!inflate_png(self, img, buf, bufsz)) {
+                    img->data_loaded = false; return NULL;
+                }
+                break;
+            default: break;
+        }
+#undef IB
+        img->load_data.data = img->load_data.buf;
+        if (img->load_data.buf_used < img->load_data.data_sz) {
+            ABRT(ENODATA, "Insufficient image data: %zu < %zu", img->load_data.buf_used, img->load_data.data_sz);
+        }
+        if (img->load_data.mapped_file) {
+            munmap(img->load_data.mapped_file, img->load_data.mapped_file_sz);
+            img->load_data.mapped_file = NULL; img->load_data.mapped_file_sz = 0;
+        }
+    } else {
+        if (tt == 'd') {
+            if (img->load_data.buf_used < img->load_data.data_sz) {
+                ABRT(ENODATA, "Insufficient image data: %zu < %zu",  img->load_data.buf_used, img->load_data.data_sz);
+            } else img->load_data.data = img->load_data.buf;
+        } else {
+            if (img->load_data.mapped_file_sz < img->load_data.data_sz) {
+                ABRT(ENODATA, "Insufficient image data: %zu < %zu",  img->load_data.mapped_file_sz, img->load_data.data_sz);
+            } else img->load_data.data = img->load_data.mapped_file;
+        }
+    }
+    size_t required_sz = (img->load_data.is_opaque ? 3 : 4) * img->width * img->height;
+    if (img->load_data.data_sz != required_sz) ABRT(EINVAL, "Image dimensions: %ux%u do not match data size: %zu, expected size: %zu", img->width, img->height, img->load_data.data_sz, required_sz);
+    if (LIKELY(img->data_loaded && send_to_gpu)) {
+        send_image_to_gpu(&img->texture_id, img->load_data.data, img->width, img->height, img->load_data.is_opaque, img->load_data.is_4byte_aligned);
+        free_load_data(&img->load_data);
+        self->used_storage += required_sz;
+        img->used_storage = required_sz;
+    }
+    return img;
+#undef MAX_DATA_SZ
+#undef ABRT
+}
+
+static inline const char*
+create_add_response(GraphicsManager UNUSED *self, bool data_loaded, uint32_t iid) {
+    static char rbuf[sizeof(add_response)/sizeof(add_response[0])];
+    if (iid) {
+        if (!has_add_respose) {
+            if (!data_loaded) return NULL;
+            snprintf(add_response, 10, "OK"); 
+        }
+        snprintf(rbuf, sizeof(rbuf)/sizeof(rbuf[0]) - 1, "\033_Gi=%u;%s\033\\", iid, add_response);
+        return rbuf;
+    }
+    return NULL;
+}
+
+// }}}
+
+// Displaying images {{{
+
+#define ensure_space_for(base, array, type, num, capacity, initial_cap) \
+    if (base->capacity < num) { \
+        base->capacity = MAX(initial_cap, MAX(2 * base->capacity, num)); \
+        base->array = realloc(base->array, sizeof(type) * base->capacity); \
+        if (base->array == NULL) fatal("Out of memory while ensuring space in array"); \
+    }
+
+static inline void
+update_src_rect(ImageRef *ref, Image *img) {
+    // The src rect in OpenGL co-ords [0, 1] with origin at top-left corner of image
+    ref->src_rect.left = (float)ref->src_x / (float)img->width;
+    ref->src_rect.right = (float)(ref->src_x + ref->src_width) / (float)img->width;
+    ref->src_rect.top = (float)ref->src_y / (float)img->height;
+    ref->src_rect.bottom = (float)(ref->src_y + ref->src_height) / (float)img->height;
+}
+
+static inline void
+update_dest_rect(ImageRef *ref, uint32_t num_cols, uint32_t num_rows) {
+    uint32_t t;
+    if (num_cols == 0) {
+        t = ref->src_width + ref->cell_x_offset;
+        num_cols = t / global_state.cell_width;
+        if (t > num_cols * global_state.cell_width) num_cols += 1;
+    } 
+    if (num_rows == 0) {
+        t = ref->src_height + ref->cell_y_offset;
+        num_rows = t / global_state.cell_height;
+        if (t > num_rows * global_state.cell_height) num_rows += 1;
+    } 
+    ref->effective_num_rows = num_rows;
+    ref->effective_num_cols = num_cols;
+}
+
+
+static void 
+handle_put_command(GraphicsManager *self, const GraphicsCommand *g, Cursor *c, bool *is_dirty, Image *img) {
+    has_add_respose = false;
+    if (img == NULL) img = img_by_client_id(self, g->id);
+    if (img == NULL) { set_add_response("ENOENT", "Put command refers to non-existent image with id: %u", g->id); return; }
+    if (!img->data_loaded) { set_add_response("ENOENT", "Put command refers to image with id: %u that could not load its data", g->id); return; }
+    ensure_space_for(img, refs, ImageRef, img->refcnt + 1, refcap, 10);
+    *is_dirty = true;
+    self->layers_dirty = true;
+    ImageRef *ref = NULL;
+    for (size_t i=0; i < img->refcnt; i++) {
+        if ((unsigned)img->refs[i].start_row == c->x && (unsigned)img->refs[i].start_column == c->y) {
+            ref = img->refs + i;
+            break;
+        }
+    }
+    if (ref == NULL) ref = img->refs + img->refcnt++;
+    img->atime = monotonic();
+    ref->src_x = g->x_offset; ref->src_y = g->y_offset; ref->src_width = g->width ? g->width : img->width; ref->src_height = g->height ? g->height : img->height;
+    ref->src_width = MIN(ref->src_width, img->width - (img->width > ref->src_x ? ref->src_x : img->width));
+    ref->src_height = MIN(ref->src_height, img->height - (img->height > ref->src_y ? ref->src_y : img->height));
+    ref->z_index = g->z_index;
+    ref->start_row = c->y; ref->start_column = c->x;
+    ref->cell_x_offset = MIN(g->cell_x_offset, global_state.cell_width - 1);
+    ref->cell_y_offset = MIN(g->cell_y_offset, global_state.cell_height - 1);
+    ref->num_cols = g->num_cells; ref->num_rows = g->num_lines;
+    update_src_rect(ref, img);
+    update_dest_rect(ref, g->num_cells, g->num_lines);
+    // Move the cursor, the screen will take care of ensuring it is in bounds
+    c->x += ref->effective_num_cols; c->y += ref->effective_num_rows - 1;
+}
+
+static int 
+cmp_by_zindex_and_image(const void *a_, const void *b_) { 
+    const ImageRenderData *a = (const ImageRenderData*)a_, *b = (const ImageRenderData*)b_;
+    int ans = a->z_index - b->z_index;
+    if (ans == 0) ans = a->image_id - b->image_id;
+    return ans; 
+}
+
+bool
+grman_update_layers(GraphicsManager *self, unsigned int scrolled_by, float screen_left, float screen_top, float dx, float dy, unsigned int num_cols, unsigned int num_rows) {
+    if (self->last_scrolled_by != scrolled_by) self->layers_dirty = true;
+    self->last_scrolled_by = scrolled_by;
+    if (!self->layers_dirty) return false;
+    self->layers_dirty = false;
+    size_t i, j;
+    self->num_of_negative_refs = 0; self->num_of_positive_refs = 0;
+    Image *img; ImageRef *ref;
+    ImageRect r;
+    float screen_width = dx * num_cols, screen_height = dy * num_rows;
+    float screen_bottom = screen_top - screen_height;
+    float screen_width_px = num_cols * global_state.cell_width;
+    float screen_height_px = num_rows * global_state.cell_height;
+    float y0 = screen_top - dy * scrolled_by;
+
+    // Iterate over all visible refs and create render data
+    self->count = 0;
+    for (i = 0; i < self->image_count; i++) { img = self->images + i; for (j = 0; j < img->refcnt; j++) { ref = img->refs + j;
+        r.top = y0 - ref->start_row * dy - dy * (float)ref->cell_y_offset / (float)global_state.cell_height;
+        if (ref->num_rows > 0) r.bottom = y0 - (ref->start_row + (int32_t)ref->num_rows) * dy;
+        else r.bottom = r.top - screen_height * (float)ref->src_height / screen_height_px;
+        if (r.top <= screen_bottom || r.bottom >= screen_top) continue;  // not visible
+
+        r.left = screen_left + ref->start_column * dx + dx * (float)ref->cell_x_offset / (float) global_state.cell_width;
+        if (ref->num_cols > 0) r.right = screen_left + (ref->start_column + (int32_t)ref->num_cols) * dx;
+        else r.right = r.left + screen_width * (float)ref->src_width / screen_width_px;
+
+        if (ref->z_index < 0) self->num_of_negative_refs++; else self->num_of_positive_refs++;
+        ensure_space_for(self, render_data, ImageRenderData, self->count + 1, capacity, 100);
+        ImageRenderData *rd = self->render_data + self->count;
+#define R(n, a, b) rd->vertices[n*4] = ref->src_rect.a; rd->vertices[n*4 + 1] = ref->src_rect.b; rd->vertices[n*4 + 2] = r.a; rd->vertices[n*4 + 3] = r.b;
+        R(0, right, top); R(1, right, bottom); R(2, left, bottom); R(3, left, top);
+#undef R
+        self->count++;
+        rd->z_index = ref->z_index; rd->image_id = img->internal_id;
+        rd->texture_id = img->texture_id;
+    }}
+    if (!self->count) return false;
+    // Sort visible refs in draw order (z-index, img)
+    qsort(self->render_data, self->count, sizeof(ImageRenderData), cmp_by_zindex_and_image);
+    // Calculate the group counts
+    i = 0;
+    while (i < self->count) {
+        size_t image_id = self->render_data[i].image_id, start = i;
+        if (start == self->count - 1) i = self->count;
+        else {
+            while (i < self->count - 1 && self->render_data[++i].image_id == image_id) {}
+        }
+        self->render_data[start].group_count = i - start;
+    }
+    return true;
+}
+
+// }}}
+
+// Image lifetime/scrolling {{{
+
+static inline void
+filter_refs(GraphicsManager *self, const void* data, bool free_images, bool (*filter_func)(ImageRef*, Image*, const void*)) {
+    Image *img; ImageRef *ref;
+    size_t i, j;
+
+    if (self->image_count) {
+        for (i = self->image_count; i-- > 0;) { 
+            img = self->images + i;
+            for (j = img->refcnt; j-- > 0;) { 
+                ref = img->refs + j;
+                if (filter_func(ref, img, data)) {
+                    remove_from_array(img->refs, sizeof(ImageRef), j, img->refcnt--);
+                }
+            }
+            if (img->refcnt == 0 && (free_images || img->client_id == 0)) remove_image(self, i);
+        }
+        self->layers_dirty = true;
+    }
+}
+
+static inline bool
+scroll_filter_func(ImageRef *ref, Image UNUSED *img, const void *data) {
+    ScrollData *d = (ScrollData*)data;
+    ref->start_row += d->amt;
+    return ref->start_row + (int32_t)ref->effective_num_rows <= d->limit;
+}
+
+static inline bool
+ref_within_region(ImageRef *ref, index_type margin_top, index_type margin_bottom) {
+    return ref->start_row >= (int32_t)margin_top && ref->start_row + ref->effective_num_rows <= margin_bottom;
+}
+
+static inline bool
+ref_outside_region(ImageRef *ref, index_type margin_top, index_type margin_bottom) {
+    return ref->start_row + ref->effective_num_rows <= margin_top || ref->start_row > (int32_t)margin_bottom;
+}
+
+static inline bool
+scroll_filter_margins_func(ImageRef* ref, Image* img, const void* data) {
+    ScrollData *d = (ScrollData*)data;
+    if (ref_within_region(ref, d->margin_top, d->margin_bottom)) {
+        ref->start_row += d->amt;
+        if (ref_outside_region(ref, d->margin_top, d->margin_bottom)) return true;
+        // Clip the image if scrolling has resulted in part of it being outside the page area
+        uint32_t clip_amt, clipped_rows;
+        if (ref->start_row < (int32_t)d->margin_top) {
+            // image moved up
+            clipped_rows = d->margin_top - ref->start_row;
+            clip_amt = global_state.cell_height * clipped_rows;
+            if (ref->src_height <= clip_amt) return true;
+            ref->src_y += clip_amt; ref->src_height -= clip_amt;
+            ref->effective_num_rows -= clipped_rows;
+            update_src_rect(ref, img);
+            ref->start_row += clipped_rows;
+        } else if (ref->start_row + ref->effective_num_rows > d->margin_bottom) {
+            // image moved down
+            clipped_rows = ref->start_row + ref->effective_num_rows - d->margin_bottom;
+            clip_amt = global_state.cell_height * clipped_rows;
+            if (ref->src_height <= clip_amt) return true;
+            ref->src_height -= clip_amt;
+            ref->effective_num_rows -= clipped_rows;
+            update_src_rect(ref, img);
+        }
+        return ref_outside_region(ref, d->margin_top, d->margin_bottom);
+    }
+    return false;
+}
+
+void 
+grman_scroll_images(GraphicsManager *self, const ScrollData *data) {
+    filter_refs(self, data, true, data->has_margins ? scroll_filter_margins_func : scroll_filter_func);
+}
+
+static inline bool
+clear_filter_func(ImageRef *ref, Image UNUSED *img, const void UNUSED *data) {
+    return ref->start_row + (int32_t)ref->effective_num_rows > 0;
+}
+
+void
+grman_clear(GraphicsManager *self) {
+    filter_refs(self, NULL, true, clear_filter_func);
+}
+
+static inline bool
+id_filter_func(ImageRef UNUSED *ref, Image *img, const void *data) {
+    uint32_t iid = *(uint32_t*)data;
+    return img->client_id == iid;
+}
+
+static inline bool
+x_filter_func(ImageRef *ref, Image UNUSED *img, const void *data) {
+    const GraphicsCommand *g = data;
+    return ref->start_column <= (int32_t)g->x_offset - 1 && ((int32_t)g->x_offset - 1) < ((int32_t)(ref->start_column + ref->effective_num_cols));
+}
+
+static inline bool
+y_filter_func(ImageRef *ref, Image UNUSED *img, const void *data) {
+    const GraphicsCommand *g = data;
+    return ref->start_row <= (int32_t)g->y_offset - 1 && ((int32_t)(g->y_offset - 1 < ref->start_row + ref->effective_num_rows));
+}
+
+static inline bool
+z_filter_func(ImageRef *ref, Image UNUSED *img, const void *data) {
+    const GraphicsCommand *g = data;
+    return ref->z_index == g->z_index;
+}
+
+
+static inline bool
+point_filter_func(ImageRef *ref, Image *img, const void *data) {
+    return x_filter_func(ref, img, data) && y_filter_func(ref, img, data);
+}
+
+static inline bool
+point3d_filter_func(ImageRef *ref, Image *img, const void *data) {
+    return z_filter_func(ref, img, data) && point_filter_func(ref, img, data);
+}
+
+
+static void
+handle_delete_command(GraphicsManager *self, const GraphicsCommand *g, Cursor *c, bool *is_dirty) {
+    static GraphicsCommand d;
+    switch (g->delete_action) {
+#define I(u, data, func) filter_refs(self, data, g->delete_action == u, func); *is_dirty = true; break
+#define D(l, u, data, func) case l: case u: I(u, data, func)
+#define G(l, u, func) D(l, u, g, func)
+        case 0:
+        D('a', 'A', NULL, clear_filter_func);
+        D('i', 'I', &g->id, id_filter_func);
+        G('p', 'P', point_filter_func);
+        G('q', 'Q', point3d_filter_func);
+        G('x', 'X', x_filter_func);
+        G('y', 'Y', y_filter_func);
+        G('z', 'Z', z_filter_func);
+        case 'c':
+        case 'C':
+            d.x_offset = c->x + 1; d.y_offset = c->y + 1;
+            I('C', &d, point_filter_func);
+        default:
+            REPORT_ERROR("Unknown graphics command delete action: %c", g->delete_action);
+            break;
+#undef G
+#undef D
+#undef I
+    }
+}
+
+// }}}
+
+void 
+grman_resize(GraphicsManager *self, index_type UNUSED old_lines, index_type UNUSED lines, index_type UNUSED old_columns, index_type UNUSED columns) {
+    self->layers_dirty = true;
+}
+
+void 
+grman_rescale(GraphicsManager *self, unsigned int UNUSED old_cell_width, unsigned int UNUSED old_cell_height) {
+    ImageRef *ref; Image *img;
+    self->layers_dirty = true;
+    for (size_t i = self->image_count; i-- > 0;) { 
+        img = self->images + i;
+        for (size_t j = img->refcnt; j-- > 0;) { 
+            ref = img->refs + j;
+            ref->cell_x_offset = MIN(ref->cell_x_offset, global_state.cell_width - 1);
+            ref->cell_y_offset = MIN(ref->cell_y_offset, global_state.cell_height - 1);
+            update_dest_rect(ref, ref->num_cols, ref->num_rows);
+        }
+    }
+}
+
+const char*
+grman_handle_command(GraphicsManager *self, const GraphicsCommand *g, const uint8_t *payload, Cursor *c, bool *is_dirty) {
+    Image *image;
+    const char *ret = NULL;
+    uint32_t iid, q_iid;
+
+    switch(g->action) {
+        case 0:
+        case 't':
+        case 'T':
+        case 'q':
+            iid = g->id;
+            if (g->action == 'q') { q_iid = iid; iid = 0; if (!q_iid) { REPORT_ERROR("Query graphics command without image id"); break; } }
+            image = handle_add_command(self, g, payload, is_dirty, iid);
+            ret = create_add_response(self, image != NULL, g->action == 'q' ? q_iid: self->last_init_graphics_command.id);
+            if (self->last_init_graphics_command.action == 'T' && image && image->data_loaded) handle_put_command(self, &self->last_init_graphics_command, c, is_dirty, image);
+            if (g->action == 'q') remove_images(self, add_trim_predicate, NULL);
+            if (self->used_storage > STORAGE_LIMIT) apply_storage_quota(self, STORAGE_LIMIT, image);
+            break;
+        case 'p':
+            if (!g->id) {
+                REPORT_ERROR("Put graphics command without image id");
+                break;
+            }
+            handle_put_command(self, g, c, is_dirty, NULL);
+            ret = create_add_response(self, true, g->id);
+            break;
+        case 'd':
+            handle_delete_command(self, g, c, is_dirty);
+            break;
+        default:
+            REPORT_ERROR("Unknown graphics command action: %c", g->action);
+            break;
+    }
+    return ret;
+}
+
+
+// Boilerplate {{{
+static PyObject *
+new(PyTypeObject UNUSED *type, PyObject UNUSED *args, PyObject UNUSED *kwds) {
+    PyObject *ans = (PyObject*)grman_alloc();
+    if (ans == NULL) PyErr_NoMemory();
+    return ans;
+}
+
+static inline PyObject*
+image_as_dict(Image *img) {
+#define U(x) #x, img->x
+    return Py_BuildValue("{sI sI sI sI sI sI sH sH sN}",
+        U(texture_id), U(client_id), U(width), U(height), U(internal_id), U(refcnt), U(data_loaded),
+        "is_4byte_aligned", img->load_data.is_4byte_aligned,
+        "data", Py_BuildValue("y#", img->load_data.data, img->load_data.data_sz)
+    );
+#undef U
+
+}
+
+#define W(x) static PyObject* py##x(GraphicsManager UNUSED *self, PyObject *args)
+#define PA(fmt, ...) if(!PyArg_ParseTuple(args, fmt, __VA_ARGS__)) return NULL;
+
+W(image_for_client_id) {
+    unsigned long id = PyLong_AsUnsignedLong(args);
+    bool existing = false;
+    Image *img = find_or_create_image(self, id, &existing);
+    if (!existing) { Py_RETURN_NONE; }
+    return image_as_dict(img);
+}
+
+W(shm_write) {
+    const char *name, *data;
+    Py_ssize_t sz;
+    PA("ss#", &name, &data, &sz);
+    int fd = shm_open(name, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
+    if (fd == -1) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, name); return NULL; }
+    int ret = ftruncate(fd, sz);
+    if (ret != 0) { close(fd); PyErr_SetFromErrnoWithFilename(PyExc_OSError, name); return NULL; }
+    void *addr = mmap(0, sz, PROT_WRITE, MAP_SHARED, fd, 0);
+    if (addr == MAP_FAILED) { close(fd); PyErr_SetFromErrnoWithFilename(PyExc_OSError, name); return NULL; }
+    memcpy(addr, data, sz);
+    if (munmap(addr, sz) != 0) { close(fd); PyErr_SetFromErrnoWithFilename(PyExc_OSError, name); return NULL; }
+    close(fd);
+    Py_RETURN_NONE;
+}
+
+W(shm_unlink) {
+    char *name;
+    PA("s", &name);
+    int ret = shm_unlink(name);
+    if (ret == -1) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, name); return NULL; }
+    Py_RETURN_NONE;
+}
+
+W(set_send_to_gpu) {
+    send_to_gpu = PyObject_IsTrue(args) ? true : false;
+    Py_RETURN_NONE;
+}
+
+W(update_layers) {
+    unsigned int scrolled_by, sx, sy; float xstart, ystart, dx, dy;
+    PA("IffffII", &scrolled_by, &xstart, &ystart, &dx, &dy, &sx, &sy);
+    grman_update_layers(self, scrolled_by, xstart, ystart, dx, dy, sx, sy);
+    PyObject *ans = PyTuple_New(self->count);
+    for (size_t i = 0; i < self->count; i++) {
+        ImageRenderData *r = self->render_data + i;
+#define R(offset) Py_BuildValue("{sf sf sf sf}", "left", r->vertices[offset + 8], "top", r->vertices[offset + 1], "right", r->vertices[offset], "bottom", r->vertices[offset + 5])
+        PyTuple_SET_ITEM(ans, i, 
+            Py_BuildValue("{sN sN sI si sI}", "src_rect", R(0), "dest_rect", R(2), "group_count", r->group_count, "z_index", r->z_index, "image_id", r->image_id)
+        );
+#undef R
+    }
+    return ans;
+}
+
+#define M(x, va) {#x, (PyCFunction)py##x, va, ""}
+
+static PyMethodDef methods[] = {
+    M(image_for_client_id, METH_O),
+    M(update_layers, METH_VARARGS),
+    {NULL}  /* Sentinel */
+};
+
+static PyMemberDef members[] = {
+    {"image_count", T_UINT, offsetof(GraphicsManager, image_count), 0, "image_count"},
+    {NULL},
+};
+
+PyTypeObject GraphicsManager_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "fast_data_types.GraphicsManager",
+    .tp_basicsize = sizeof(GraphicsManager),
+    .tp_dealloc = (destructor)dealloc, 
+    .tp_flags = Py_TPFLAGS_DEFAULT,        
+    .tp_doc = "GraphicsManager",
+    .tp_new = new,                
+    .tp_methods = methods,
+    .tp_members = members,
+};
+
+static PyMethodDef module_methods[] = {
+    M(shm_write, METH_VARARGS),
+    M(shm_unlink, METH_VARARGS),
+    M(set_send_to_gpu, METH_O),
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+
+bool
+init_graphics(PyObject *module) {
+    if (PyType_Ready(&GraphicsManager_Type) < 0) return false;
+    if (PyModule_AddObject(module, "GraphicsManager", (PyObject *)&GraphicsManager_Type) != 0) return false; 
+    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
+    Py_INCREF(&GraphicsManager_Type);
+    return true;
+}
+// }}}

kitty/graphics.h

@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#pragma once
+#include "data-types.h"
+
+typedef struct {
+    unsigned char action, transmission_type, compressed, delete_action;
+    uint32_t format, more, id, data_sz, data_offset;
+    uint32_t width, height, x_offset, y_offset, data_height, data_width, num_cells, num_lines, cell_x_offset, cell_y_offset;
+    int32_t z_index;
+    size_t payload_sz;
+} GraphicsCommand;
+
+typedef struct {
+    uint8_t *buf;
+    size_t buf_capacity, buf_used;
+
+    uint8_t *mapped_file;
+    size_t mapped_file_sz;
+
+    size_t data_sz;
+    uint8_t *data;
+    bool is_4byte_aligned;
+    bool is_opaque;
+} LoadData;
+
+typedef struct {
+    float left, top, right, bottom;
+} ImageRect;
+
+typedef struct {
+    uint32_t src_width, src_height, src_x, src_y;
+    uint32_t cell_x_offset, cell_y_offset, num_cols, num_rows, effective_num_rows, effective_num_cols;
+    int32_t z_index;
+    int32_t start_row, start_column;
+    ImageRect src_rect;
+} ImageRef;
+
+
+typedef struct {
+    uint32_t texture_id, client_id, width, height;
+    size_t internal_id;
+
+    bool data_loaded;
+    LoadData load_data;
+
+    ImageRef *refs;
+    size_t refcnt, refcap;
+    double atime;
+    size_t used_storage;
+} Image;
+
+typedef struct {
+    float vertices[16];
+    uint32_t texture_id, group_count;
+    int z_index;
+    size_t image_id;
+} ImageRenderData;
+
+typedef struct {
+    PyObject_HEAD
+
+    size_t image_count, images_capacity, loading_image;
+    GraphicsCommand last_init_graphics_command;
+    Image *images;
+    size_t count, capacity;
+    ImageRenderData *render_data;
+    bool layers_dirty;
+    size_t num_of_negative_refs, num_of_positive_refs;
+    unsigned int last_scrolled_by;
+    size_t used_storage;
+} GraphicsManager;
+PyTypeObject GraphicsManager_Type;
+
+
+typedef struct {
+    int32_t amt, limit;
+    index_type margin_top, margin_bottom;
+    bool has_margins;
+} ScrollData;
+
+GraphicsManager* grman_alloc();
+void grman_clear(GraphicsManager*);
+const char* grman_handle_command(GraphicsManager *self, const GraphicsCommand *g, const uint8_t *payload, Cursor *c, bool *is_dirty);
+bool grman_update_layers(GraphicsManager *self, unsigned int scrolled_by, float screen_left, float screen_top, float dx, float dy, unsigned int num_cols, unsigned int num_rows);
+void grman_scroll_images(GraphicsManager *self, const ScrollData*);
+void grman_resize(GraphicsManager*, index_type, index_type, index_type, index_type);
+void grman_rescale(GraphicsManager *self, unsigned int old_cell_width, unsigned int old_cell_height);

kitty/graphics_fragment.glsl

@@ -0,0 +1,10 @@
+#version GLSL_VERSION
+
+uniform sampler2D image;
+
+in vec2 texcoord;
+out vec4 color;
+
+void main() {
+    color = texture(image, texcoord);
+}

kitty/graphics_vertex.glsl

@@ -0,0 +1,9 @@
+#version GLSL_VERSION
+
+in vec4 src;
+out vec2 texcoord;
+
+void main() {
+    texcoord = vec2(src[0], src[1]);
+    gl_Position = vec4(src[2], src[3], 0, 1);
+}

kitty/history.c

@@ -6,15 +6,12 @@
  */
 
 #include "data-types.h"
+#include "lineops.h"
 #include <structmember.h>
 
-#define CELL_FIELD_COUNT_H (CELL_FIELD_COUNT + 1)
-#define CELL_SIZE_H (CELL_FIELD_COUNT_H * 4)
-
-static inline uint32_t* 
-start_of(HistoryBuf *self, index_type num) {
-    // Pointer to the start of the line with index (buffer position) num
-    return self->buf + CELL_FIELD_COUNT_H * num * self->xnum;
+static inline Cell*
+lineptr(HistoryBuf *linebuf, index_type y) {
+    return linebuf->buf + y * linebuf->xnum;
 }
 
 static PyObject *
@@ -33,17 +30,17 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
     if (self != NULL) {
         self->xnum = xnum;
         self->ynum = ynum;
-        self->buf = (uint32_t*)PyMem_Calloc(xnum * ynum, CELL_SIZE_H);
+        self->buf = PyMem_Calloc(xnum * ynum, sizeof(Cell));
+        self->continued_map = PyMem_Calloc(ynum, sizeof(bool));
         self->line = alloc_line();
-        if (self->buf == NULL || self->line == NULL) {
+        if (self->buf == NULL || self->line == NULL || self->continued_map == NULL) {
             PyErr_NoMemory();
-            PyMem_Free(self->buf); Py_CLEAR(self->line);
+            PyMem_Free(self->buf); Py_CLEAR(self->line); PyMem_Free(self->continued_map);
             Py_CLEAR(self);
         } else {
             self->line->xnum = xnum;
             for(index_type y = 0; y < self->ynum; y++) {
-                self->line->chars = start_of(self, y) + 1;
-                for (index_type i = 0; i < self->xnum; i++) self->line->chars[i] = (1 << ATTRS_SHIFT) | 32;
+                clear_chars_in_line(lineptr(self, y), self->xnum, BLANK_CHAR);
             }
         }
     }
@@ -55,9 +52,17 @@ static void
 dealloc(HistoryBuf* self) {
     Py_CLEAR(self->line);
     PyMem_Free(self->buf);
+    PyMem_Free(self->continued_map);
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
+void 
+historybuf_refresh_sprite_positions(HistoryBuf *self) {
+    for (index_type i = 0; i < self->ynum; i++) {
+        update_sprites_in_line(lineptr(self, i), self->xnum);
+    }
+}
+
 static inline index_type 
 index_of(HistoryBuf *self, index_type lnum) {
     // The index (buffer position) of the line with line number lnum
@@ -70,13 +75,8 @@ index_of(HistoryBuf *self, index_type lnum) {
 static inline void 
 init_line(HistoryBuf *self, index_type num, Line *l) {
     // Initialize the line l, setting its pointer to the offsets for the line at index (buffer position) num
-    uint32_t *start_ptr = start_of(self, num);
-    l->continued = (bool)(*start_ptr);
-    l->chars = (char_type*)(start_ptr + 1);
-    l->fg_colors = (color_type*)(l->chars + self->xnum);
-    l->bg_colors = (color_type*)(l->fg_colors + self->xnum);
-    l->decoration_fg = (color_type*)(l->bg_colors + self->xnum);
-    l->combining_chars = (combining_type*)(l->decoration_fg + self->xnum);
+    l->cells = lineptr(self, num);
+    l->continued = self->continued_map[num];
 }
 
 void 
@@ -84,8 +84,8 @@ historybuf_init_line(HistoryBuf *self, index_type lnum, Line *l) {
     init_line(self, index_of(self, lnum), l);
 }
 
-static inline 
-index_type historybuf_push(HistoryBuf *self) {
+static inline index_type 
+historybuf_push(HistoryBuf *self) {
     index_type idx = (self->start_of_data + self->count) % self->ynum;
     init_line(self, idx, self->line);
     if (self->count == self->ynum) self->start_of_data = (self->start_of_data + 1) % self->ynum;
@@ -99,20 +99,21 @@ historybuf_resize(HistoryBuf *self, index_type lines) {
     t.xnum=self->xnum;
     t.ynum=lines;
     if (t.ynum > 0 && t.ynum != self->ynum) {
-        t.buf = (uint32_t*)PyMem_Calloc(t.xnum * t.ynum, CELL_SIZE_H);
+        t.buf = PyMem_Calloc(t.xnum * t.ynum, sizeof(Cell));
         if (t.buf == NULL) { PyErr_NoMemory(); return false; }
+        t.continued_map = PyMem_Calloc(t.ynum, sizeof(bool));
+        if (t.continued_map == NULL) { PyMem_Free(t.buf); PyErr_NoMemory(); return false; }
         t.count = MIN(self->count, t.ynum);
-        if (t.count > 0) {
-            for (index_type s=0; s < t.count; s++) {
-                uint32_t *src = start_of(self, index_of(self, s)), *dest = start_of(&t, index_of(&t, s));
-                memcpy(dest, src, CELL_SIZE_H * t.xnum);
-            }
+        for (index_type s=0; s < t.count; s++) {
+            index_type si = index_of(self, s), ti = index_of(&t, s);
+            copy_cells(lineptr(self, si), lineptr(&t, ti), t.xnum);
+            t.continued_map[ti] = self->continued_map[si];
         }
         self->count = t.count;
         self->start_of_data = t.start_of_data;
         self->ynum = t.ynum;
-        PyMem_Free(self->buf);
-        self->buf = t.buf;
+        PyMem_Free(self->buf); PyMem_Free(self->continued_map);
+        self->buf = t.buf; self->continued_map = t.continued_map;
     }
     return true;
 }
@@ -120,8 +121,8 @@ historybuf_resize(HistoryBuf *self, index_type lines) {
 void 
 historybuf_add_line(HistoryBuf *self, const Line *line) {
     index_type idx = historybuf_push(self);
-    COPY_LINE(line, self->line);
-    *(start_of(self, idx)) = line->continued;
+    copy_line(line, self->line);
+    self->continued_map[idx] = line->continued;
 }
 
 static PyObject*
@@ -142,6 +143,22 @@ line(HistoryBuf *self, PyObject *val) {
     return (PyObject*)self->line;
 }
 
+static PyObject*
+__str__(HistoryBuf *self) {
+    PyObject *lines = PyTuple_New(self->ynum);
+    if (lines == NULL) return PyErr_NoMemory();
+    for (index_type i = 0; i < self->count; i++) {
+        init_line(self, index_of(self, i), self->line);
+        PyObject *t = PyObject_Str((PyObject*)self->line);
+        if (t == NULL) { Py_CLEAR(lines); return NULL; }
+        PyTuple_SET_ITEM(lines, i, t);
+    }
+    PyObject *sep = PyUnicode_FromString("\n");
+    PyObject *ans = PyUnicode_Join(sep, lines);
+    Py_CLEAR(lines); Py_CLEAR(sep);
+    return ans;
+}
+
 static PyObject*
 push(HistoryBuf *self, PyObject *args) {
 #define push_doc "Push a line into this buffer, removing the oldest line, if necessary"
@@ -159,7 +176,7 @@ as_ansi(HistoryBuf *self, PyObject *callback) {
     for(unsigned int i = 0; i < self->count; i++) {
         init_line(self, i, &l);
         if (i < self->count - 1) {
-            l.continued = *start_of(self, index_of(self, i + 1));
+            l.continued = self->continued_map[index_of(self, i + 1)];
         } else l.continued = false;
         index_type num = line_as_ansi(&l, t, 5120);
         if (!(l.continued) && num < 5119) t[num++] = 10; // 10 = \n
@@ -202,6 +219,7 @@ PyTypeObject HistoryBuf_Type = {
     .tp_doc = "History buffers",
     .tp_methods = methods,
     .tp_members = members,            
+    .tp_str = (reprfunc)__str__,
     .tp_new = new
 };
 
@@ -218,9 +236,9 @@ HistoryBuf *alloc_historybuf(unsigned int lines, unsigned int columns) {
 
 #define init_src_line(src_y) init_line(src, map_src_index(src_y), src->line);
 
-#define is_src_line_continued(src_y) (map_src_index(src_y) < src->ynum - 1 ? (bool)(*(start_of(src, map_src_index(src_y + 1)))) : false)
+#define is_src_line_continued(src_y) (map_src_index(src_y) < src->ynum - 1 ? src->continued_map[map_src_index(src_y + 1)] : false)
 
-#define next_dest_line(cont) *start_of(dest, historybuf_push(dest)) = cont; dest->line->continued = cont; 
+#define next_dest_line(cont) dest->continued_map[historybuf_push(dest)] = cont; dest->line->continued = cont; 
 
 #define first_dest_line next_dest_line(false); 
 
@@ -229,10 +247,9 @@ HistoryBuf *alloc_historybuf(unsigned int lines, unsigned int columns) {
 void historybuf_rewrap(HistoryBuf *self, HistoryBuf *other) {
     // Fast path
     if (other->xnum == self->xnum && other->ynum == self->ynum) {
-        Py_BEGIN_ALLOW_THREADS;
-        memcpy(other->buf, self->buf, CELL_SIZE_H * self->xnum * self->ynum);
+        memcpy(other->buf, self->buf, sizeof(Cell) * self->xnum * self->ynum);
+        memcpy(other->continued_map, self->continued_map, sizeof(bool) * self->ynum);
         other->count = self->count; other->start_of_data = self->start_of_data;
-        Py_END_ALLOW_THREADS;
         return;
     }
     other->count = 0; other->start_of_data = 0;

kitty/icat.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+import argparse
+import codecs
+import fcntl
+import mimetypes
+import os
+import re
+import selectors
+import signal
+import struct
+import subprocess
+import sys
+import termios
+import tty
+import zlib
+from base64 import standard_b64encode
+from collections import namedtuple
+from gettext import gettext as _
+from math import ceil, floor
+from tempfile import NamedTemporaryFile
+from time import monotonic
+
+try:
+    from kitty.constants import appname
+except ImportError:
+    appname = ''
+
+try:
+    fsenc = sys.getfilesystemencoding() or 'utf-8'
+    codecs.lookup(fsenc)
+except Exception:
+    fsenc = 'utf-8'
+
+
+class OpenFailed(ValueError):
+
+    def __init__(self, path, message):
+        ValueError.__init__(
+            self, 'Failed to open: {} with error: {}'.format(path, message)
+        )
+        self.path = path
+
+
+def option_parser():
+    parser = argparse.ArgumentParser(
+        prog=appname + '-icat' if appname else 'icat',
+        description=_('Display images in the terminal')
+    )
+    a = parser.add_argument
+    a(
+        'items',
+        nargs='+',
+        help=_(
+            'Image files or directories. Directories are scanned recursively.'
+        )
+    )
+    return parser
+
+
+Size = namedtuple('Size', 'rows cols width height')
+
+
+def screen_size(refresh=False):
+    if refresh or getattr(screen_size, 'ans', None) is None:
+        s = struct.pack('HHHH', 0, 0, 0, 0)
+        x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
+        screen_size.ans = Size(*struct.unpack('HHHH', x))
+    return screen_size.ans
+
+
+def write_gr_cmd(cmd, payload):
+    cmd = ','.join('{}={}'.format(k, v) for k, v in cmd.items())
+    w = sys.stdout.buffer.write
+    w(b'\033_G'), w(cmd.encode('ascii')), w(b';'), w(payload), w(b'\033\\')
+    sys.stdout.flush()
+
+
+def fit_image(width, height, pwidth, pheight):
+    if height > pheight:
+        corrf = pheight / float(height)
+        width, height = floor(corrf * width), pheight
+    if width > pwidth:
+        corrf = pwidth / float(width)
+        width, height = pwidth, floor(corrf * height)
+    if height > pheight:
+        corrf = pheight / float(height)
+        width, height = floor(corrf * width), pheight
+
+    return int(width), int(height)
+
+
+def set_cursor(cmd, width, height):
+    ss = screen_size()
+    cw = int(ss.width / ss.cols)
+    num_of_cells_needed = int(ceil(width / cw))
+    if num_of_cells_needed > ss.cols:
+        w, h = fit_image(width, height, ss.width, height)
+        ch = int(ss.height / ss.rows)
+        num_of_rows_needed = int(ceil(height / ch))
+        y_off = height % ch
+        cmd['c'], cmd['r'] = ss.cols, num_of_rows_needed
+        cmd['Y'] = y_off
+    else:
+        x_off = width % cw
+        cmd['X'] = x_off
+        extra_cells = (ss.cols - num_of_cells_needed) // 2
+        if extra_cells:
+            sys.stdout.buffer.write(b' ' * extra_cells)
+
+
+def write_chunked(cmd, data):
+    if cmd['f'] != 100:
+        data = zlib.compress(data)
+        cmd['o'] = 'z'
+    data = standard_b64encode(data)
+    while data:
+        chunk, data = data[:4096], data[4096:]
+        m = 1 if data else 0
+        cmd['m'] = m
+        write_gr_cmd(cmd, chunk)
+        cmd.clear()
+
+
+def show(outfile, width, height, fmt, transmit_mode):
+    cmd = {'a': 'T', 'f': fmt, 's': width, 'v': height}
+    set_cursor(cmd, width, height)
+    if detect_support.has_files:
+        cmd['t'] = transmit_mode
+        write_gr_cmd(cmd, standard_b64encode(os.path.abspath(outfile).encode(fsenc)))
+    else:
+        with open(outfile, 'rb') as f:
+            data = f.read()
+        if fmt == 100:
+            cmd['S'] = len(data)
+        write_chunked(cmd, data)
+
+
+ImageData = namedtuple('ImageData', 'fmt width height mode')
+
+
+def run_imagemagick(path, cmd, keep_stdout=True):
+    try:
+        p = subprocess.run(cmd, stdout=subprocess.PIPE if keep_stdout else subprocess.DEVNULL, stderr=subprocess.PIPE)
+    except FileNotFoundError:
+        raise SystemExit('ImageMagick is required to cat images')
+    if p.returncode != 0:
+        raise OpenFailed(path, p.stderr.decode('utf-8'))
+    return p
+
+
+def identify(path):
+    p = run_imagemagick(path, ['identify', '-format', '%m %w %h %A', path])
+    parts = tuple(filter(None, p.stdout.decode('utf-8').split()))
+    mode = 'rgb' if parts[3].lower() == 'false' else 'rgba'
+    return ImageData(parts[0].lower(), int(parts[1]), int(parts[2]), mode)
+
+
+def convert(path, m, ss):
+    width, height = m.width, m.height
+    cmd = ['convert', '-background', 'none', path]
+    if width > ss.width:
+        width, height = fit_image(width, height, ss.width, height)
+        cmd += ['-resize', '{}x{}'.format(width, height)]
+    with NamedTemporaryFile(prefix='icat-', suffix='.' + m.mode, delete=False) as outfile:
+        run_imagemagick(path, cmd + [outfile.name])
+    return outfile.name, width, height
+
+
+def process(path):
+    m = identify(path)
+    ss = screen_size()
+    needs_scaling = m.width > ss.width
+    if m.fmt == 'png' and not needs_scaling:
+        outfile = path
+        transmit_mode = 'f'
+        fmt = 100
+        width, height = m.width, m.height
+    else:
+        fmt = 24 if m.mode == 'rgb' else 32
+        transmit_mode = 't'
+        outfile, width, height = convert(path, m, ss)
+    show(outfile, width, height, fmt, transmit_mode)
+    print()  # ensure cursor is on a new line
+
+
+def scan(d):
+    for dirpath, dirnames, filenames in os.walk(d):
+        for f in filenames:
+            mt = mimetypes.guess_type(f)[0]
+            if mt and mt.startswith('image/'):
+                yield os.path.join(dirpath, f), mt
+
+
+def detect_support(wait_for=10):
+    fd = sys.stdin.fileno()
+    old = termios.tcgetattr(fd)
+    oldfl = fcntl.fcntl(fd, fcntl.F_GETFL)
+    fcntl.fcntl(fd, fcntl.F_SETFL, oldfl | os.O_NONBLOCK)
+    print('Checking for graphics ({}s max. wait)...'.format(wait_for), end='\r')
+    sys.stdout.flush()
+    tty.setraw(fd)
+    try:
+        received = b''
+        start_time = monotonic()
+        responses = {}
+
+        def parse_responses():
+            for m in re.finditer(b'\033_Gi=([1|2]);(.+?)\033\\\\', received):
+                iid = m.group(1)
+                if iid in (b'1', b'2'):
+                    iid = int(iid.decode('ascii'))
+                    if iid not in responses:
+                        responses[iid] = m.group(2) == b'OK'
+
+        def read():
+            nonlocal received
+            d = sys.stdin.buffer.read()
+            if not d:  # EOF
+                responses[1] = responses[2] = False
+                return
+            received += d
+            parse_responses()
+
+        with NamedTemporaryFile() as f:
+            f.write(b'abcd'), f.flush()
+            write_gr_cmd(dict(a='q', s=1, v=1, i=1), standard_b64encode(b'abcd'))
+            write_gr_cmd(dict(a='q', s=1, v=1, i=2, t='f'), standard_b64encode(f.name.encode(fsenc)))
+            sel = selectors.DefaultSelector()
+            sel.register(sys.stdin, selectors.EVENT_READ, read)
+            while monotonic() - start_time < wait_for and 1 not in responses and 2 not in responses:
+                for key, mask in sel.select(0.1):
+                    read()
+    finally:
+        sys.stdout.buffer.write(b'\033[J'), sys.stdout.flush()
+        termios.tcsetattr(fd, termios.TCSADRAIN, old)
+        fcntl.fcntl(fd, fcntl.F_SETFL, oldfl)
+    detect_support.has_files = bool(responses.get(2))
+    return responses.get(1, False)
+
+
+def main(args=sys.argv):
+    signal.signal(signal.SIGWINCH, lambda: screen_size(refresh=True))
+    if not sys.stdout.isatty() or not sys.stdin.isatty():
+        raise SystemExit(
+            'Must be run in a terminal, stdout is currently not a terminal'
+        )
+    if screen_size().width == 0:
+        raise SystemExit(
+            'Terminal does not support reporting screen sizes via the TIOCGWINSZ ioctl'
+        )
+    args = option_parser().parse_args(args[1:])
+    if not args.items:
+        raise SystemExit('You must specify at least one file to cat')
+    if not detect_support():
+        raise SystemExit('This terminal emulator does not support the graphics protocol, use a terminal emulator such as kitty that does support it')
+    errors = []
+    for item in args.items:
+        try:
+            if os.path.isdir(item):
+                for x in scan(item):
+                    process(item)
+            else:
+                process(item)
+        except OpenFailed as e:
+            errors.append(e)
+    if not errors:
+        return
+    for err in errors:
+        print(err, file=sys.stderr)
+    raise SystemExit(1)
+
+
+if __name__ == '__main__':
+    main()

kitty/keys.c

@@ -0,0 +1,203 @@
+/*
+ * keys.c
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "keys.h"
+#include "state.h"
+#include "screen.h"
+#include <GLFW/glfw3.h>
+
+const uint8_t*
+key_to_bytes(int glfw_key, bool smkx, bool extended, int mods, int action) {
+    if ((action & 3) == 3) return NULL;
+    if ((unsigned)glfw_key >= sizeof(key_map)/sizeof(key_map[0]) || glfw_key < 0) return NULL;
+    uint16_t key = key_map[glfw_key];
+    if (key == UINT8_MAX) return NULL;
+    mods &= 0xF;
+    key |= (mods & 0xF) << 7;
+    key |= (action & 3) << 11;
+    key |= (smkx & 1) << 13;
+    key |= (extended & 1) << 14;
+    if (key >= SIZE_OF_KEY_BYTES_MAP) return NULL;
+    return key_bytes[key];
+}
+
+#define SPECIAL_INDEX(key) ((key & 0x7f) | ( (mods & 0xF) << 7))
+
+void
+set_special_key_combo(int glfw_key, int mods) {
+    uint16_t key = key_map[glfw_key];
+    if (key != UINT8_MAX) {
+        key = SPECIAL_INDEX(key);
+        needs_special_handling[key] = true;
+    }
+}
+
+static inline Window*
+active_window() {
+    Tab *t = global_state.tabs + global_state.active_tab;
+    Window *w = t->windows + t->active_window;
+    if (w->render_data.screen) return w;
+    return NULL;
+}
+
+void
+on_text_input(unsigned int codepoint, int mods) {
+    Window *w = active_window();
+    static char buf[10];
+    unsigned int sz = 0;
+
+    if (w != NULL) {
+        Screen *screen = w->render_data.screen;
+        bool in_alt_mods = !screen->modes.mEXTENDED_KEYBOARD && (mods == GLFW_MOD_ALT || mods == (GLFW_MOD_ALT | GLFW_MOD_SHIFT));
+        bool is_text = mods <= GLFW_MOD_SHIFT;
+        if (in_alt_mods) {
+            sz = encode_utf8(codepoint, buf + 1);
+            if (sz) {
+                buf[0] = 033;
+                sz++;
+            }
+        } else if (is_text) sz = encode_utf8(codepoint, buf);
+        if (sz) schedule_write_to_child(w->id, buf, sz);
+    }
+}
+
+static inline bool
+is_modifier_key(int key) {
+    switch(key) {
+        case GLFW_KEY_LEFT_SHIFT:
+        case GLFW_KEY_RIGHT_SHIFT: 
+        case GLFW_KEY_LEFT_ALT:
+        case GLFW_KEY_RIGHT_ALT:
+        case GLFW_KEY_LEFT_CONTROL:
+        case GLFW_KEY_RIGHT_CONTROL:
+        case GLFW_KEY_LEFT_SUPER:
+        case GLFW_KEY_RIGHT_SUPER:
+            return true;
+        default:
+            return false;
+    }
+}
+
+static inline int
+get_localized_key(int key, int scancode) {
+    const char *name = glfwGetKeyName(key, scancode);
+    if (name == NULL || name[1] != 0) return key;
+    switch(name[0]) {
+#define K(ch, name) case ch: return GLFW_KEY_##name
+        // key names {{{
+        K('A', A); K('a', A);
+        K('B', B); K('b', B);
+        K('C', C); K('c', C); 
+        K('D', D); K('d', D);
+        K('E', E); K('e', E);
+        K('F', F); K('f', F);
+        K('G', G); K('g', G);
+        K('H', H); K('h', H);
+        K('I', I); K('i', I);
+        K('J', J); K('j', J);
+        K('K', K); K('k', K);
+        K('L', L); K('l', L);
+        K('M', M); K('m', M);
+        K('N', N); K('n', N);
+        K('O', O); K('o', O);
+        K('P', P); K('p', P);
+        K('Q', Q); K('q', Q);
+        K('R', R); K('r', R);
+        K('S', S); K('s', S);
+        K('T', T); K('t', T);
+        K('U', U); K('u', U);
+        K('V', V); K('v', V);
+        K('W', W); K('w', W);
+        K('X', X); K('x', X);
+        K('Y', Y); K('y', Y);
+        K('Z', Z); K('z', Z);
+        K('0', 0);
+        K('1', 1);
+        K('2', 2);
+        K('3', 3);
+        K('5', 5);
+        K('6', 6);
+        K('7', 7);
+        K('8', 8);
+        K('9', 9);
+        K('\'', APOSTROPHE);
+        K(',', COMMA);
+        K('.', PERIOD);
+        K('/', SLASH);
+        K('-', MINUS);
+        K(';', SEMICOLON);
+        K('=', EQUAL);
+        K('[', LEFT_BRACKET);
+        K(']', RIGHT_BRACKET);
+        K('`', GRAVE_ACCENT);
+        K('\\', BACKSLASH);
+        // }}}
+#undef K
+        default:
+            return key;
+    }
+}
+
+void
+on_key_input(int key, int scancode, int action, int mods) {
+    Window *w = active_window();
+    if (!w) return;
+    Screen *screen = w->render_data.screen;
+    int lkey = get_localized_key(key, scancode);
+    if (action == GLFW_PRESS || action == GLFW_REPEAT) {
+        uint16_t qkey = key_map[lkey];
+        bool special = false;
+        if (qkey != UINT8_MAX) {
+            qkey = SPECIAL_INDEX(qkey);
+            special = needs_special_handling[qkey];
+        }
+        /* printf("key: %s mods: %d special: %d\n", key_name(lkey), mods, special); */
+        if (special) {
+            PyObject *ret = PyObject_CallMethod(global_state.boss, "dispatch_special_key", "iiii", lkey, scancode, action, mods);
+            if (ret == NULL) { PyErr_Print(); }
+            else {
+                bool consumed = ret == Py_True;
+                Py_DECREF(ret);
+                if (consumed) return;
+            }
+        }
+    }
+    if (screen->scrolled_by && action == GLFW_PRESS && !is_modifier_key(key)) {
+        screen_history_scroll(screen, SCROLL_FULL, false);  // scroll back to bottom
+    }
+    if (
+            action == GLFW_PRESS ||
+            (action == GLFW_REPEAT && screen->modes.mDECARM) ||
+            screen->modes.mEXTENDED_KEYBOARD
+       ) {
+        const uint8_t *data = key_to_bytes(lkey, screen->modes.mDECCKM, screen->modes.mEXTENDED_KEYBOARD, mods, action);
+        if (data) schedule_write_to_child(w->id, (char*)(data + 1), *data);
+    }
+}
+
+#define PYWRAP1(name) static PyObject* py##name(PyObject UNUSED *self, PyObject *args)
+#define PA(fmt, ...) if(!PyArg_ParseTuple(args, fmt, __VA_ARGS__)) return NULL;
+#define M(name, arg_type) {#name, (PyCFunction)py##name, arg_type, NULL}
+
+PYWRAP1(key_to_bytes) {
+    int glfw_key, smkx, extended, mods, action;
+    PA("ippii", &glfw_key, &smkx, &extended, &mods, &action);
+    const uint8_t *ans = key_to_bytes(glfw_key, smkx & 1, extended & 1, mods, action);
+    if (ans == NULL) return Py_BuildValue("y#", "", 0);
+    return Py_BuildValue("y#", ans + 1, *ans);
+}
+
+static PyMethodDef module_methods[] = {
+    M(key_to_bytes, METH_VARARGS), 
+    {0}
+};
+
+bool
+init_keys(PyObject *module) {
+    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
+    return true;
+}

kitty/keys.py

@@ -85,6 +85,7 @@ control_codes.update({
 })
 control_codes[defines.GLFW_KEY_6] = (30,)
 control_codes[defines.GLFW_KEY_SLASH] = (31,)
+control_codes[defines.GLFW_KEY_SPACE] = (0,)
 
 
 rmkx_key_map = smkx_key_map.copy()
@@ -104,42 +105,22 @@ def get_key_map(screen):
     return cursor_key_mode_map[screen.cursor_key_mode]
 
 
-valid_localized_key_names = {
-    k: getattr(defines, 'GLFW_KEY_' + k)
-    for k in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
-}
-
-for name, ch in {
-    'APOSTROPHE': "'",
-    'COMMA': ',',
-    'PERIOD': '.',
-    'SLASH': '/',
-    'MINUS': '-',
-    'SEMICOLON': ';',
-    'EQUAL': '=',
-    'LEFT_BRACKET': '[',
-    'RIGHT_BRACKET': ']',
-    'GRAVE_ACCENT': '`',
-    'BACKSLASH': '\\'
-}.items():
-    valid_localized_key_names[ch] = getattr(defines, 'GLFW_KEY_' + name)
-
-
-def get_localized_key(key, scancode):
-    name = defines.glfw_get_key_name(key, scancode)
-    return valid_localized_key_names.get((name or '').upper(), key)
+def keyboard_mode_name(screen):
+    if screen.extended_keyboard:
+        return 'kitty'
+    return 'application' if screen.cursor_key_mode else 'normal'
 
 
 action_map = {
-    defines.GLFW_PRESS: b'p',
-    defines.GLFW_RELEASE: b'r',
-    defines.GLFW_REPEAT: b't'
+    defines.GLFW_PRESS: 'p',
+    defines.GLFW_RELEASE: 'r',
+    defines.GLFW_REPEAT: 't'
 }
 
 
-def extended_key_event(key, scancode, mods, action):
+def extended_key_event(key, mods, action):
     if key >= defines.GLFW_KEY_LAST or key == defines.GLFW_KEY_UNKNOWN or (
-        # Shifted printable key should be handled by interpret_text_event()
+        # Shifted printable key should be handled by on_text_input()
         mods == defines.GLFW_MOD_SHIFT and 32 <= key <= 126
     ):
         return b''
@@ -155,42 +136,128 @@ def extended_key_event(key, scancode, mods, action):
     ).encode('ascii')
 
 
-def interpret_key_event(key, scancode, mods, window, action, get_localized_key=get_localized_key):
-    screen = window.screen
-    key = get_localized_key(key, scancode)
-    if screen.extended_keyboard:
-        return extended_key_event(key, scancode, mods, action)
+def key_to_bytes(key, smkx, extended, mods, action):
+    if extended:
+        return extended_key_event(key, mods, action)
     data = bytearray()
-    if action == defines.GLFW_PRESS or (
-        action == defines.GLFW_REPEAT and screen.auto_repeat_enabled
-    ):
-        if mods == defines.GLFW_MOD_CONTROL and key in control_codes:
-            # Map Ctrl-key to ascii control code
-            data.extend(control_codes[key])
-        elif mods in alt_mods and key in alt_codes:
-            # Printable keys handled by interpret_text_event()
-            data.extend((alt_codes if mods == defines.GLFW_MOD_ALT else shift_alt_codes)[key])
-        else:
-            key_map = get_key_map(screen)
-            x = key_map.get(key)
-            if x is not None:
-                if mods == defines.GLFW_MOD_SHIFT:
-                    x = SHIFTED_KEYS.get(key, x)
-                data.extend(x)
+    if mods == defines.GLFW_MOD_CONTROL and key in control_codes:
+        # Map Ctrl-key to ascii control code
+        data.extend(control_codes[key])
+    elif mods in alt_mods and key in alt_codes:
+        # Printable keys handled by on_text_input()
+        data.extend((alt_codes if mods == defines.GLFW_MOD_ALT else shift_alt_codes)[key])
+    else:
+        key_map = cursor_key_mode_map[smkx]
+        x = key_map.get(key)
+        if x is not None:
+            if mods == defines.GLFW_MOD_SHIFT:
+                x = SHIFTED_KEYS.get(key, x)
+            data.extend(x)
     return bytes(data)
 
 
-def interpret_text_event(codepoint, mods, window):
+def interpret_key_event(key, scancode, mods, window, action):
     screen = window.screen
-    if mods > defines.GLFW_MOD_SHIFT:
-        if mods in alt_mods and not screen.extended_keyboard:
-            data = chr(codepoint).encode('utf-8')
-            return b'\x1b' + data
-        return b''  # Handled by interpret_key_event above
-    data = chr(codepoint).encode('utf-8')
-    return data
+    if (
+            action == defines.GLFW_PRESS or
+            (action == defines.GLFW_REPEAT and screen.auto_repeat_enabled) or
+            screen.extended_keyboard
+    ):
+        return defines.key_to_bytes(key, screen.cursor_key_mode, screen.extended_keyboard, mods, action)
+    return b''
 
 
 def get_shortcut(keymap, mods, key, scancode):
-    key = get_localized_key(key, scancode)
     return keymap.get((mods & 0b1111, key))
+
+
+def get_sent_data(send_text_map, key, scancode, mods, window, action):
+    if action in (defines.GLFW_PRESS, defines.GLFW_REPEAT):
+        m = keyboard_mode_name(window.screen)
+        keymap = send_text_map[m]
+        return keymap.get((mods & 0b1111, key))
+
+
+def generate_key_table():
+    # To run this, use: python3 . -c "from kitty.keys import *; generate_key_table()"
+    import os
+    from functools import partial
+    f = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'keys.h'), 'w')
+    w = partial(print, file=f)
+    w('// auto-generated from keys.py, do not edit!')
+    w('#pragma once')
+    w('#include <stddef.h>')
+    w('#include <stdint.h>')
+    w('#include <stdbool.h>')
+    w('#include <limits.h>')
+    w('static bool needs_special_handling[%d] = {0};' % (128 * 16))
+    number_of_keys = defines.GLFW_KEY_LAST + 1
+    w('// map glfw key numbers to 7-bit numbers for compact data storage')
+    w('static const uint8_t key_map[%d] = {' % number_of_keys)
+    key_count = 0
+
+    def key_name(k):
+        return k[len('GLFW_KEY_'):]
+
+    keys = {v: k for k, v in vars(defines).items() if k.startswith('GLFW_KEY_') and k not in {'GLFW_KEY_LAST', 'GLFW_KEY_UNKNOWN'}}
+    key_rmap = []
+    for i in range(number_of_keys):
+        k = keys.get(i)
+        if k is None:
+            w('UINT8_MAX,')
+        else:
+            w('%d, /* %s */' % (key_count, key_name(k)))
+            key_rmap.append(i)
+            key_count += 1
+            if key_count > 128:
+                raise OverflowError('Too many keys')
+    w('};\n')
+    w('static inline const char* key_name(int key) { switch(key) {')
+    for i in range(number_of_keys):
+        k = keys.get(i)
+        if k is not None:
+            w('case %d: return "%s";' % (i, key_name(k)))
+    w('default: return NULL; }}\n')
+    number_entries = 128 * 256
+    inits = []
+    longest = 0
+    for i in range(number_entries):
+        key = i & 0x7f  # lowest seven bits
+        if key < key_count:
+            glfw_key = key_rmap[key]
+            k = keys.get(glfw_key)
+        else:
+            k = None
+        if k is None:
+            inits.append(None)
+        else:
+            mods = (i >> 7) & 0b1111
+            rest = i >> 11
+            action = rest & 0b11
+            if action == 0b11:  # no such action
+                inits.append(None)
+            else:
+                smkx = bool(rest & 0b100)
+                extended = bool(rest & 0b1000)
+                data = key_to_bytes(glfw_key, smkx, extended, mods, action)
+                if data:
+                    longest = max(len(data), longest)
+                    inits.append((data, k, mods, smkx, extended))
+                else:
+                    inits.append(None)
+    longest += 1
+    w('#define SIZE_OF_KEY_BYTES_MAP %d\n' % number_entries)
+    w('static const uint8_t key_bytes[%d][%d] = {' % (number_entries, longest))
+    # empty = '{' + ('0, ' * longest) + '},'
+    empty = '{0},'
+    all_mods = {k.rpartition('_')[2]: v for k, v in vars(defines).items() if k.startswith('GLFW_MOD_')}
+    all_mods = {k: v for k, v in sorted(all_mods.items(), key=lambda x: x[0])}
+    for b in inits:
+        if b is None:
+            w(empty)
+        else:
+            b, k, mods, smkx, extended = b
+            b = bytearray(b)
+            name = '+'.join([k for k, v in all_mods.items() if v & mods] + [key_name(k)])
+            w('{%d, ' % len(b) + ', '.join(map(hex, b)) + '}, //', name, 'smkx:', smkx, 'extended:', extended)
+    w('};')

kitty/kitty.conf

@@ -7,6 +7,12 @@
 # font_family Operator Mono Book
 # bold_font Operator Mono Thick
 # bold_italic_font Operator Mono Medium
+# or
+# font_family SF Mono Medium
+# bold_font        SF Mono Semibold
+# bold_italic_font SF Mono Semibold 
+# Note that you should use the full family name but do not add Bold or Italic qualifiers
+# to the name.
 font_family      monospace
 italic_font      auto
 bold_font        auto
@@ -19,6 +25,14 @@ font_size        11.0
 # the font size in a running terminal.
 font_size_delta 2
 
+
+# Adjust the line height. 
+# You can use either numbers, which are interpreted as pixels or percentages
+# (number followed by %), which are interpreted as percentages of the
+# unmodified line height. You can use negative pixels or percentages less than
+# 100% to reduce line height (but this might cause rendering artifacts).
+adjust_line_height 0
+
 # The foreground color
 foreground       #dddddd
 
@@ -31,21 +45,22 @@ selection_foreground #000000
 # The background for selections
 selection_background #FFFACD
 
-# The cursor color
-cursor           #ffffff
+# The color for highlighting URLs on mouse-over
+url_color #0087BD
 
-# The cursor opacity
-cursor_opacity   0.7
+# The cursor color
+cursor           #cccccc
 
 # The cursor shape can be one of (block, beam, underline)
 cursor_shape     block
 
 # The interval (in seconds) at which to blink the cursor. Set to zero to
-# disable blinking.
+# disable blinking. Note that numbers smaller than repaint_delay will be 
+# limited to repaint_delay.
 cursor_blink_interval     0.5
 
 # Stop blinking cursor after the specified number of seconds of keyboard inactivity. Set to
-# zero or a negative number to never stop blinking.
+# zero to never stop blinking.
 cursor_stop_blinking_after 15.0
 
 # Number of lines of history to keep in memory for scrolling back
@@ -59,7 +74,8 @@ scrollback_pager less +G -R
 # When viewing scrollback in a new window, put it in a new tab as well
 scrollback_in_new_tab no
 
-# Wheel scroll multiplier (modify the amount scrolled by the mouse wheel)
+# Wheel scroll multiplier (modify the amount scrolled by the mouse wheel). Use negative
+# numbers to change scroll direction.
 wheel_scroll_multiplier 5.0
 
 # The interval between successive clicks to detect double/triple clicks (in seconds)
@@ -71,7 +87,7 @@ click_interval 0.5
 select_by_word_characters :@-./_~?&=%+#
 
 # Hide mouse cursor after the specified number of seconds of the mouse not being used. Set to
-# zero or a negative number to disable mouse cursor hiding.
+# zero to disable mouse cursor hiding.
 mouse_hide_wait 3.0
 
 # The enabled window layouts. A comma separated list of layout names. The special value * means
@@ -86,11 +102,18 @@ remember_window_size   yes
 initial_window_width   640
 initial_window_height  400
 
-# Delay (in milliseconds) between screen updates. Decreasing it, increases fps
-# at the cost of more CPU usage. The default value yields ~100fps which is more
-# that sufficient for most uses.
+# Delay (in milliseconds) between screen updates. Decreasing it, increases
+# frames-per-second (FPS) at the cost of more CPU usage. The default value
+# yields ~100 FPS which is more than sufficient for most uses.
 repaint_delay    10
 
+# Delay (in milliseconds) before input from the program running in the terminal
+# is processed. Note that decreasing it will increase responsiveness, but also
+# increase CPU usage and might cause flicker in full screen programs that
+# redraw the entire screen on each loop, because kitty is so fast that partial
+# screen updates will be drawn.
+input_delay 3
+
 # Visual bell duration. Flash the screen when a bell occurs for the specified number of
 # seconds. Set to zero to disable.
 visual_bell_duration 0.0
@@ -118,8 +141,15 @@ use_system_wcwidth yes
 term xterm-kitty
 
 # The width (in pts) of window borders. Will be rounded to the nearest number of pixels based on screen resolution.
+# Note that borders are displayed only when more than one window is visible. They are meant to separate multiple windows.
 window_border_width 1
 
+# The window margin (in pts) (blank area outside the border)
+window_margin_width 0
+
+# The window padding (in pts) (blank area between the text and the window border)
+window_padding_width 0
+
 # The color for the border of the active window
 active_border_color #00ff00
 
@@ -168,17 +198,31 @@ color14  #14ffff
 color7   #dddddd
 color15  #ffffff
 
+
 # Key mapping
 # For a list of key names, see: http://www.glfw.org/docs/latest/group__keys.html
 # For a list of modifier names, see: http://www.glfw.org/docs/latest/group__mods.html
+# 
 # You can use the special action no_op to unmap a keyboard shortcut that is
 # assigned in the default configuration.
+# 
+# You can combine multiple actions to be triggered by a single shortcut, using the
+# syntax below:
+# map key combine <separator> action1 <separator> action2 <separator> action3 ...
+# For example:
+# map ctrl+shift+e combine : new_window : next_layout
+# this will create a new window and switch to the next available layout
 
 # Clipboard
 map ctrl+shift+v        paste_from_clipboard
 map ctrl+shift+s        paste_from_selection
 map ctrl+shift+c        copy_to_clipboard
 map shift+insert        paste_from_selection
+# You can also pass the contents of the current selection to any program using
+# pass_selection_to_program. By default, the system's open program is used, but
+# you can specify your own, for example:
+# map ctrl+shift+o      pass_selection_to_program firefox
+map ctrl+shift+o        pass_selection_to_program 
 
 # Scrolling
 map ctrl+shift+up        scroll_line_up
@@ -192,7 +236,7 @@ map ctrl+shift+end       scroll_end
 map ctrl+shift+h         show_scrollback
 
 # Window management
-map ctrl+shift+enter    new_window
+map ctrl+shift+enter    new_window 
 map ctrl+shift+w        close_window
 map ctrl+shift+]        next_window
 map ctrl+shift+[        previous_window
@@ -209,6 +253,15 @@ map ctrl+shift+7        seventh_window
 map ctrl+shift+8        eighth_window
 map ctrl+shift+9        ninth_window
 map ctrl+shift+0        tenth_window
+# You can also open a new window running an arbitrary program, for example:
+# map ctrl+shift+y      new_window mutt
+# You can also pass the current selection to the new program by using the @selection placeholder
+# map ctrl+shift+y      new_window less @selection
+# Finally, you can even send the contents of the current screen + history buffer as stdin using
+# the placeholders @text (which is the plain text) and @ansi (which includes text styling escape codes)
+# For example, the following command opens the scrollback buffer in less in a new window.
+# map ctrl+shift+y      new_window @ansi less +G -R
+
 
 # Tab management
 map ctrl+shift+right    next_tab
@@ -218,12 +271,32 @@ map ctrl+shift+q        close_tab
 map ctrl+shift+l        next_layout
 map ctrl+shift+.        move_tab_forward
 map ctrl+shift+,        move_tab_backward
+# Just as with new_window above, you can also pass the name of arbitrary
+# commands to run when using new_tab.
+
 
 # Miscellaneous
 map ctrl+shift+equal    increase_font_size
 map ctrl+shift+minus    decrease_font_size
 map ctrl+shift+backspace restore_font_size
 
+# Sending arbitrary text on shortcut key presses
+# You can tell kitty to send arbitrary (UTF-8) encoded text to
+# the client program when pressing specified shortcut keys. For example:
+# send_text all ctrl+alt+a Special text
+# This will send "Special text" when you press the Ctrl+Alt+a key combination.
+# The text to be sent is a python string literal so you can use escapes like
+# \x1b to send control codes or \u21fb to send unicode characters (or you can
+# just input the unicode characters directly as UTF-8 text). The first argument
+# to send_text is the keyboard modes in which to activate the shortcut. The possible
+# values are normal or application or kitty or a comma separated combination of them.
+# The special keyword all means all modes. The modes normal and application refer to
+# the DECCKM cursor key mode for terminals, and kitty refers to the special kitty
+# extended keyboard protocol. Another example, that outputs a word and then moves the cursor
+# to the start of the line (same as pressing the Home key):
+# send_text normal ctrl+alt+a Word\x1b[H
+# send_text application ctrl+alt+a Word\x1bOH
+
 # Symbol mapping (special font for specified unicode code points). Map the
 # specified unicode codepoints to a particular font. Useful if you need special
 # rendering for some symbols, such as for Powerline. Avoids the need for

kitty/layout.py

@@ -6,25 +6,29 @@ from collections import namedtuple
 from itertools import islice
 
 from .constants import WindowGeometry, viewport_size, cell_size, get_boss
+from .utils import pt_to_px
 
 
 def available_height():
     return viewport_size.height - get_boss().current_tab_bar_height
 
 
-def layout_dimension(length, cell_length, number_of_windows=1, border_length=0, left_align=False):
+def layout_dimension(length, cell_length, number_of_windows=1, border_length=0, margin_length=0, padding_length=0, left_align=False):
     number_of_cells = length // cell_length
+    border_length += padding_length
     space_needed_for_border = number_of_windows * 2 * border_length
+    space_needed_for_padding = number_of_windows * 2 * margin_length
+    space_needed = space_needed_for_padding + space_needed_for_border
     extra = length - number_of_cells * cell_length
-    while extra < space_needed_for_border:
+    while extra < space_needed:
         number_of_cells -= 1
         extra = length - number_of_cells * cell_length
     cells_per_window = number_of_cells // number_of_windows
-    extra -= space_needed_for_border
+    extra -= space_needed
     pos = 0 if left_align else (extra // 2)
-    pos += border_length
+    pos += border_length + margin_length
     inner_length = cells_per_window * cell_length
-    window_length = 2 * border_length + inner_length
+    window_length = 2 * (border_length + margin_length) + inner_length
     extra = number_of_cells - (cells_per_window * number_of_windows)
     while number_of_windows > 0:
         number_of_windows -= 1
@@ -43,6 +47,8 @@ class Layout:
     def __init__(self, opts, border_width, windows):
         self.opts = opts
         self.border_width = border_width
+        self.margin_width = pt_to_px(opts.window_margin_width)
+        self.padding_width = pt_to_px(opts.window_padding_width)
         # A set of rectangles corresponding to the blank spaces at the edges of
         # this layout, i.e. spaces that are not covered by any window
         self.blank_rects = ()
@@ -76,9 +82,9 @@ def window_geometry(xstart, xnum, ystart, ynum):
     return WindowGeometry(left=xstart, top=ystart, xnum=xnum, ynum=ynum, right=xstart + cell_size.width * xnum, bottom=ystart + cell_size.height * ynum)
 
 
-def layout_single_window():
-    xstart, xnum = next(layout_dimension(viewport_size.width, cell_size.width))
-    ystart, ynum = next(layout_dimension(available_height(), cell_size.height))
+def layout_single_window(margin_length, padding_length):
+    xstart, xnum = next(layout_dimension(viewport_size.width, cell_size.width, margin_length=margin_length, padding_length=padding_length))
+    ystart, ynum = next(layout_dimension(available_height(), cell_size.height, margin_length=margin_length, padding_length=padding_length))
     return window_geometry(xstart, xnum, ystart, ynum)
 
 
@@ -116,14 +122,14 @@ class Stack(Layout):
 
     def set_active_window(self, windows, active_window_idx):
         for i, w in enumerate(windows):
-            w.is_visible_in_layout = i == active_window_idx
+            w.set_visible_in_layout(i, i == active_window_idx)
 
     def __call__(self, windows, active_window_idx):
         self.blank_rects = []
-        wg = layout_single_window()
+        wg = layout_single_window(self.margin_width, self.padding_width)
         for i, w in enumerate(windows):
-            w.is_visible_in_layout = i == active_window_idx
-            w.set_geometry(wg)
+            w.set_visible_in_layout(i, i == active_window_idx)
+            w.set_geometry(i, wg)
             if w.is_visible_in_layout:
                 self.blank_rects = blank_rects_for_window(w)
 
@@ -135,19 +141,25 @@ class Tall(Layout):
     def __call__(self, windows, active_window_idx):
         self.blank_rects = br = []
         if len(windows) == 1:
-            wg = layout_single_window()
-            windows[0].set_geometry(wg)
+            wg = layout_single_window(self.margin_width, self.padding_width)
+            windows[0].set_geometry(0, wg)
             self.blank_rects = blank_rects_for_window(windows[0])
             return
-        xlayout = layout_dimension(viewport_size.width, cell_size.width, 2, self.border_width)
+        xlayout = layout_dimension(
+            viewport_size.width, cell_size.width, 2, self.border_width,
+            margin_length=self.margin_width, padding_length=self.padding_width)
         xstart, xnum = next(xlayout)
-        ystart, ynum = next(layout_dimension(available_height(), cell_size.height, 1, self.border_width, left_align=True))
-        windows[0].set_geometry(window_geometry(xstart, xnum, ystart, ynum))
+        ystart, ynum = next(layout_dimension(
+            available_height(), cell_size.height, 1, self.border_width, left_align=True,
+            margin_length=self.margin_width, padding_length=self.padding_width))
+        windows[0].set_geometry(0, window_geometry(xstart, xnum, ystart, ynum))
         vh = available_height()
         xstart, xnum = next(xlayout)
-        ylayout = layout_dimension(available_height(), cell_size.height, len(windows) - 1, self.border_width, left_align=True)
-        for w, (ystart, ynum) in zip(islice(windows, 1, None), ylayout):
-            w.set_geometry(window_geometry(xstart, xnum, ystart, ynum))
+        ylayout = layout_dimension(
+            available_height(), cell_size.height, len(windows) - 1, self.border_width, left_align=True,
+            margin_length=self.margin_width, padding_length=self.padding_width)
+        for i, (w, (ystart, ynum)) in enumerate(zip(islice(windows, 1, None), ylayout)):
+            w.set_geometry(i + 1, window_geometry(xstart, xnum, ystart, ynum))
         left_blank_rect(windows[0], br, vh), top_blank_rect(windows[0], br, vh), right_blank_rect(windows[-1], br, vh)
         br.append(Rect(windows[0].geometry.right, 0, windows[1].geometry.left, vh))
         br.append(Rect(0, windows[0].geometry.bottom, windows[0].geometry.right, vh))

kitty/line-buf.c

@@ -6,27 +6,33 @@
  */
 
 #include "data-types.h"
+#include "lineops.h"
 #include <structmember.h>
 
+static inline Cell*
+lineptr(LineBuf *linebuf, index_type y) {
+    return linebuf->buf + y * linebuf->xnum;
+}
+
 static inline void
 clear_chars_to(LineBuf* linebuf, index_type y, char_type ch) {
-    char_type *chars = linebuf->chars + linebuf->xnum * y;
-    for (index_type i = 0; i < linebuf->xnum; i++) chars[i] = (1 << ATTRS_SHIFT) | ch;
+    clear_chars_in_line(lineptr(linebuf, y), linebuf->xnum, ch);
 }
 
-void linebuf_clear(LineBuf *self, char_type ch) {
-    memset(self->buf, 0, self->block_size * CELL_SIZE);
+void 
+linebuf_clear(LineBuf *self, char_type ch) {
+    memset(self->buf, 0, self->xnum * self->ynum * sizeof(Cell));
     memset(self->continued_map, 0, self->ynum * sizeof(bool));
-    for (index_type i = 0; i < self->ynum; i++) {
-        clear_chars_to(self, i, ch);
-        self->line_map[i] = i;
+    for (index_type i = 0; i < self->ynum; i++) self->line_map[i] = i;
+    if (ch != 0) {
+        for (index_type i = 0; i < self->ynum; i++) clear_chars_to(self, i, ch);
     }
 }
 
 static PyObject*
 clear(LineBuf *self) {
 #define clear_doc "Clear all lines in this LineBuf"
-    linebuf_clear(self, ' ');
+    linebuf_clear(self, BLANK_CHAR);
     Py_RETURN_NONE;
 }
 
@@ -51,8 +57,7 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
     if (self != NULL) {
         self->xnum = xnum;
         self->ynum = ynum;
-        self->block_size = xnum * ynum;
-        self->buf = PyMem_Calloc(xnum * ynum, CELL_SIZE);
+        self->buf = PyMem_Calloc(xnum * ynum, sizeof(Cell));
         self->line_map = PyMem_Calloc(ynum, sizeof(index_type));
         self->scratch = PyMem_Calloc(ynum, sizeof(index_type));
         self->continued_map = PyMem_Calloc(ynum, sizeof(bool));
@@ -62,15 +67,10 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
             PyMem_Free(self->buf); PyMem_Free(self->line_map); PyMem_Free(self->continued_map); Py_CLEAR(self->line);
             Py_CLEAR(self);
         } else {
-            self->chars = (char_type*)self->buf;
-            self->fg_colors = (color_type*)(self->chars + self->block_size);
-            self->bg_colors = (color_type*)(self->fg_colors + self->block_size);
-            self->decoration_fg = (color_type*)(self->bg_colors + self->block_size);
-            self->combining_chars = (combining_type*)(self->decoration_fg + self->block_size);
             self->line->xnum = xnum;
             for(index_type i = 0; i < ynum; i++) {
                 self->line_map[i] = i;
-                clear_chars_to(self, i, ' ');
+                if (BLANK_CHAR != 0) clear_chars_to(self, i, BLANK_CHAR);
             }
         }
     }
@@ -88,18 +88,17 @@ dealloc(LineBuf* self) {
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
-#define INIT_LINE(lb, l, ynum) \
-    (l)->chars           = (lb)->chars + (ynum) * (lb)->xnum; \
-    (l)->fg_colors          = (lb)->fg_colors + (ynum) * (lb)->xnum; \
-    (l)->bg_colors          = (lb)->bg_colors + (ynum) * (lb)->xnum; \
-    (l)->decoration_fg   = (lb)->decoration_fg + (ynum) * (lb)->xnum; \
-    (l)->combining_chars = (lb)->combining_chars + (ynum) * (lb)->xnum;
+static inline void
+init_line(LineBuf *lb, Line *l, index_type ynum) {
+    l->cells = lineptr(lb, ynum);
+}
 
-void linebuf_init_line(LineBuf *self, index_type idx) {
+void 
+linebuf_init_line(LineBuf *self, index_type idx) {
     self->line->ynum = idx;
     self->line->xnum = self->xnum;
     self->line->continued = self->continued_map[idx];
-    INIT_LINE(self, self->line, self->line_map[idx]);
+    init_line(self, self->line, self->line_map[idx]);
 }
 
 static PyObject*
@@ -115,15 +114,15 @@ line(LineBuf *self, PyObject *y) {
     return (PyObject*)self->line;
 }
 
-unsigned int linebuf_char_width_at(LineBuf *self, index_type x, index_type y) {
-    char_type *chars = self->chars + self->line_map[y] * self->xnum;
-    return (chars[x] >> ATTRS_SHIFT) & WIDTH_MASK;
+unsigned int 
+linebuf_char_width_at(LineBuf *self, index_type x, index_type y) {
+    return (lineptr(self, self->line_map[y])[x].attrs) & WIDTH_MASK;
 }
 
-void linebuf_set_attribute(LineBuf *self, unsigned int shift, unsigned int val) {
-    char_type mask;
+void 
+linebuf_set_attribute(LineBuf *self, unsigned int shift, unsigned int val) {
     for (index_type y = 0; y < self->ynum; y++) {
-        SET_ATTRIBUTE(self->chars + y * self->xnum, shift, val);
+        set_attribute_on_line(lineptr(self, y), shift, val, self->xnum);
     }
 }
 
@@ -148,36 +147,22 @@ set_continued(LineBuf *self, PyObject *args) {
     Py_RETURN_NONE;
 }
 
-static inline int
+static inline bool
 allocate_line_storage(Line *line, bool initialize) {
     if (initialize) {
-        line->chars = PyMem_Calloc(line->xnum, sizeof(char_type));
-        line->fg_colors = PyMem_Calloc(line->xnum, sizeof(color_type));
-        line->bg_colors = PyMem_Calloc(line->xnum, sizeof(color_type));
-        line->decoration_fg = PyMem_Calloc(line->xnum, sizeof(color_type));
-        line->combining_chars = PyMem_Calloc(line->xnum, sizeof(combining_type));
-        for (index_type i = 0; i < line->xnum; i++) line->chars[i] = (1 << ATTRS_SHIFT) | 32;
+        line->cells = PyMem_Calloc(line->xnum, sizeof(Cell));
+        if (line->cells == NULL) { PyErr_NoMemory(); return false; }
+        if (BLANK_CHAR != 0) clear_chars_in_line(line->cells, line->xnum, BLANK_CHAR);
     } else {
-        line->chars = PyMem_Malloc(line->xnum * sizeof(char_type));
-        line->fg_colors = PyMem_Malloc(line->xnum * sizeof(color_type));
-        line->bg_colors = PyMem_Malloc(line->xnum * sizeof(color_type));
-        line->decoration_fg = PyMem_Malloc(line->xnum * sizeof(color_type));
-        line->combining_chars = PyMem_Malloc(line->xnum * sizeof(combining_type));
-    }
-    if (line->chars == NULL || line->fg_colors == NULL || line->bg_colors == NULL || line->decoration_fg == NULL || line->combining_chars == NULL) {
-        PyMem_Free(line->chars); line->chars = NULL;
-        PyMem_Free(line->fg_colors); line->fg_colors = NULL;
-        PyMem_Free(line->bg_colors); line->bg_colors = NULL;
-        PyMem_Free(line->decoration_fg); line->decoration_fg = NULL;
-        PyMem_Free(line->combining_chars); line->combining_chars = NULL;
-        PyErr_NoMemory();
-        return 0;
+        line->cells = PyMem_Malloc(line->xnum * sizeof(Cell));
+        if (line->cells == NULL) { PyErr_NoMemory(); return false; }
     }
     line->needs_free = 1;
-    return 1;
+    return true;
 }
 
-static inline PyObject* create_line_copy_inner(LineBuf* self, index_type y) {
+static inline PyObject* 
+create_line_copy_inner(LineBuf* self, index_type y) {
     Line src, *line;
     line = alloc_line();
     if (line == NULL) return PyErr_NoMemory();
@@ -185,8 +170,8 @@ static inline PyObject* create_line_copy_inner(LineBuf* self, index_type y) {
     if (!allocate_line_storage(line, 0)) { Py_CLEAR(line); return PyErr_NoMemory(); }
     line->ynum = y;
     line->continued = self->continued_map[y];
-    INIT_LINE(self, &src, self->line_map[y]);
-    COPY_LINE(&src, line);
+    init_line(self, &src, self->line_map[y]);
+    copy_line(&src, line);
     return (PyObject*)line;
 }
 
@@ -207,15 +192,21 @@ copy_line_to(LineBuf *self, PyObject *args) {
     src.xnum = self->xnum; dest->xnum = self->xnum;
     dest->ynum = y;
     dest->continued = self->continued_map[y];
-    INIT_LINE(self, &src, self->line_map[y]);
-    COPY_LINE(&src, dest);
+    init_line(self, &src, self->line_map[y]);
+    copy_line(&src, dest);
     Py_RETURN_NONE;
 }
 
+static inline void
+clear_line_(Line *l, index_type xnum) {
+    memset(l->cells, 0, xnum * sizeof(Cell));
+    if (BLANK_CHAR != 0) clear_chars_in_line(l->cells, xnum, BLANK_CHAR);
+}
+
 void linebuf_clear_line(LineBuf *self, index_type y) {
     Line l;
-    INIT_LINE(self, &l, self->line_map[y]);
-    CLEAR_LINE(&l, self->xnum);
+    init_line(self, &l, self->line_map[y]);
+    clear_line_(&l, self->xnum);
     self->continued_map[y] = 0;
 }
 
@@ -299,8 +290,8 @@ void linebuf_insert_lines(LineBuf *self, unsigned int num, unsigned int y, unsig
         }
         Line l;
         for (i = y; i < y + num; i++) {
-            INIT_LINE(self, &l, self->line_map[i]);
-            CLEAR_LINE(&l, self->xnum);
+            init_line(self, &l, self->line_map[i]);
+            clear_line_(&l, self->xnum);
             self->continued_map[i] = 0;
         }
     }
@@ -334,8 +325,8 @@ linebuf_delete_lines(LineBuf *self, index_type num, index_type y, index_type bot
     }
     Line l;
     for (i = ylimit - num; i < ylimit; i++) {
-        INIT_LINE(self, &l, self->line_map[i]);
-        CLEAR_LINE(&l, self->xnum);
+        init_line(self, &l, self->line_map[i]);
+        clear_line_(&l, self->xnum);
         self->continued_map[i] = 0;
     }
 }
@@ -356,7 +347,7 @@ as_ansi(LineBuf *self, PyObject *callback) {
     Line l = {.xnum=self->xnum};
     for(index_type i = 0; i < self->ynum; i++) {
         l.continued = (i < self->ynum - 1) ? self->continued_map[i+1] : self->continued_map[i];
-        INIT_LINE(self, (&l), self->line_map[i]);
+        init_line(self, (&l), self->line_map[i]);
         index_type num = line_as_ansi(&l, t, 5120);
         if (!(l.continued) && num < 5119) t[num++] = 10; // 10 = \n
         PyObject *ans = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, t, num);
@@ -369,6 +360,29 @@ as_ansi(LineBuf *self, PyObject *callback) {
     Py_RETURN_NONE;
 }
 
+void 
+linebuf_refresh_sprite_positions(LineBuf *self) {
+    for (index_type i = 0; i < self->ynum; i++) {
+        update_sprites_in_line(lineptr(self, i), self->xnum);
+    }
+}
+
+static PyObject*
+__str__(LineBuf *self) {
+    PyObject *lines = PyTuple_New(self->ynum);
+    if (lines == NULL) return PyErr_NoMemory();
+    for (index_type i = 0; i < self->ynum; i++) {
+        init_line(self, self->line, self->line_map[i]);
+        PyObject *t = PyObject_Str((PyObject*)self->line);
+        if (t == NULL) { Py_CLEAR(lines); return NULL; }
+        PyTuple_SET_ITEM(lines, i, t);
+    }
+    PyObject *sep = PyUnicode_FromString("\n");
+    PyObject *ans = PyUnicode_Join(sep, lines);
+    Py_CLEAR(lines); Py_CLEAR(sep);
+    return ans;
+}
+
 // Boilerplate {{{
 static PyObject*
 copy_old(LineBuf *self, PyObject *y);
@@ -412,6 +426,7 @@ PyTypeObject LineBuf_Type = {
     .tp_doc = "Line buffers",
     .tp_methods = methods,
     .tp_members = members,            
+    .tp_str = (reprfunc)__str__,
     .tp_new = new
 };
 
@@ -430,8 +445,8 @@ copy_old(LineBuf *self, PyObject *y) {
         index_type s = self->ynum - 1 - i, o = other->ynum - 1 - i;
         self->continued_map[s] = other->continued_map[o];
         s = self->line_map[s]; o = other->line_map[o];
-        INIT_LINE(self, &sl, s); INIT_LINE(other, &ol, o);
-        COPY_LINE(&ol, &sl);
+        init_line(self, &sl, s); init_line(other, &ol, o);
+        copy_line(&ol, &sl);
     }
     Py_RETURN_NONE;
 }
@@ -439,47 +454,50 @@ copy_old(LineBuf *self, PyObject *y) {
 #include "rewrap.h"
 
 void 
-linebuf_rewrap(LineBuf *self, LineBuf *other, int *cursor_y_out, HistoryBuf *historybuf) {
+linebuf_rewrap(LineBuf *self, LineBuf *other, index_type *num_content_lines_before, index_type *num_content_lines_after, HistoryBuf *historybuf) {
     index_type first, i;
     bool is_empty = true;
 
     // Fast path
     if (other->xnum == self->xnum && other->ynum == self->ynum) {
-        Py_BEGIN_ALLOW_THREADS;
         memcpy(other->line_map, self->line_map, sizeof(index_type) * self->ynum);
         memcpy(other->continued_map, self->continued_map, sizeof(bool) * self->ynum);
-        memcpy(other->buf, self->buf, self->xnum * self->ynum * CELL_SIZE);
-        Py_END_ALLOW_THREADS;
+        memcpy(other->buf, self->buf, self->xnum * self->ynum * sizeof(Cell));
+        *num_content_lines_before = self->ynum; *num_content_lines_after = self->ynum;
         return;
     }
 
     // Find the first line that contains some content
-    Py_BEGIN_ALLOW_THREADS;
-    for (first = self->ynum - 1; true; first--) {
-        char_type *chars = self->chars + self->xnum * first;
+    first = self->ynum;
+    do {
+        first--;
+        Cell *cells = lineptr(self, self->line_map[first]);
         for(i = 0; i < self->xnum; i++) {
-            if ((chars[i] & CHAR_MASK) != 32) { is_empty = false; break; }
+            if ((cells[i].ch) != BLANK_CHAR) { is_empty = false; break; }
         }
-        if (!is_empty || !first) break;
-    }
-    Py_END_ALLOW_THREADS;
+    } while(is_empty && first > 0);
 
-    if (first == 0) { *cursor_y_out = 0; return; }  // All lines are empty
+    if (is_empty) {  // All lines are empty
+        *num_content_lines_after = 0; 
+        *num_content_lines_before = 0;
+        return; 
+    }
 
     rewrap_inner(self, other, first + 1, historybuf);
-    *cursor_y_out = other->line->ynum;
+    *num_content_lines_after = other->line->ynum + 1;
+    *num_content_lines_before = first + 1;
 }
 
 static PyObject*
 rewrap(LineBuf *self, PyObject *args) {
     LineBuf* other;
     HistoryBuf *historybuf;
-    int cursor_y = -1;
+    unsigned int nclb, ncla;
 
     if (!PyArg_ParseTuple(args, "O!O!", &LineBuf_Type, &other, &HistoryBuf_Type, &historybuf)) return NULL;
-    linebuf_rewrap(self, other, &cursor_y, historybuf);
+    linebuf_rewrap(self, other, &nclb, &ncla, historybuf);
 
-    return Py_BuildValue("i", cursor_y);
+    return Py_BuildValue("II", nclb, ncla);
 }
 
 LineBuf *alloc_linebuf(unsigned int lines, unsigned int columns) {

kitty/line.c

@@ -6,6 +6,8 @@
  */
 
 #include "data-types.h"
+#include "unicode-data.h"
+#include "lineops.h"
 
 static PyObject *
 new(PyTypeObject UNUSED *type, PyObject UNUSED *args, PyObject UNUSED *kwds) {
@@ -16,19 +18,16 @@ new(PyTypeObject UNUSED *type, PyObject UNUSED *args, PyObject UNUSED *kwds) {
 static void
 dealloc(Line* self) {
     if (self->needs_free) {
-        PyMem_Free(self->chars);
-        PyMem_Free(self->fg_colors);
-        PyMem_Free(self->bg_colors);
-        PyMem_Free(self->decoration_fg);
-        PyMem_Free(self->combining_chars);
+        PyMem_Free(self->cells);
     }
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
 unsigned int
 line_length(Line *self) {
-    for (int i = self->xnum - 1; i >= 0; i++) {
-        if ((self->chars[i] & CHAR_MASK) != 32) return i + 1;
+    index_type last = self->xnum - 1;
+    for (index_type i = 0; i < self->xnum; i++) {
+        if ((self->cells[last - i].ch) != BLANK_CHAR) return self->xnum - i;
     }
     return 0;
 }
@@ -42,6 +41,8 @@ line_text_at(char_type ch, combining_type cc) {
         PyUnicode_WriteChar(ans, 0, ch);
     } else {
         Py_UCS4 cc1 = cc & CC_MASK, cc2 = cc >> 16;
+        Py_UCS4 normalized = normalize(ch, cc1, cc2);
+        if (normalized) { return line_text_at(normalized, 0); }
         Py_UCS4 maxc = (ch > cc1) ? MAX(ch, cc2) : MAX(cc1, cc2);
         ans = PyUnicode_New(cc2 ? 3 : 2, maxc);
         if (ans == NULL) return PyErr_NoMemory();
@@ -53,54 +54,152 @@ line_text_at(char_type ch, combining_type cc) {
     return ans;
 }
 
+// URL detection {{{
+
+static const char* url_prefixes[4] = {"https", "http", "file", "ftp"};
+static size_t url_prefix_lengths[sizeof(url_prefixes)/sizeof(url_prefixes[0])] = {0};
+typedef enum URL_PARSER_STATES {ANY, FIRST_SLASH, SECOND_SLASH} URL_PARSER_STATE;
+
+static inline index_type
+find_colon_slash(Line *self, index_type x, index_type limit) {
+    // Find :// at or before x
+    index_type pos = x;
+    URL_PARSER_STATE state = ANY;
+    limit = MAX(2, limit);
+    if (pos < limit) return 0;
+    do {
+        char_type ch = self->cells[pos].ch;
+        if (!is_url_char(ch)) return false;
+        switch(state) {
+            case ANY:
+                if (ch == '/') state = FIRST_SLASH; 
+                break;
+            case FIRST_SLASH:
+                state = ch == '/' ? SECOND_SLASH : ANY;
+                break;
+            case SECOND_SLASH:
+                if (ch == ':') return pos;
+                state = ANY;
+                break;
+        }
+        pos--;
+    } while(pos >= limit);
+    return 0;
+}
+
+static inline bool
+prefix_matches(Line *self, index_type at, const char* prefix, index_type prefix_len) {
+    if (prefix_len > at) return false;
+    index_type p, i;
+    for (p = at - prefix_len, i = 0; i < prefix_len && p < self->xnum; i++, p++) {
+        if ((self->cells[p].ch) != (unsigned char)prefix[i]) return false;
+    }
+    return i == prefix_len;
+}
+
+static inline bool
+has_url_prefix_at(Line *self, index_type at, index_type min_prefix_len, index_type *ans) {
+    if (UNLIKELY(!url_prefix_lengths[0])) {
+        for (index_type i = 0; i < sizeof(url_prefixes)/sizeof(url_prefixes[0]); i++) url_prefix_lengths[i] = strlen(url_prefixes[i]);
+    }
+    for (index_type i = 0; i < sizeof(url_prefixes)/sizeof(url_prefixes[0]); i++) {
+        index_type prefix_len = url_prefix_lengths[i];
+        if (at < prefix_len || prefix_len < min_prefix_len) continue;
+        if (prefix_matches(self, at, url_prefixes[i], prefix_len)) { *ans = at - prefix_len; return true; }
+    }
+    return false;
+}
+
+#define MAX_URL_SCHEME_LEN 5
+#define MIN_URL_LEN 5
+
+static inline bool
+has_url_beyond(Line *self, index_type x) {
+    if (self->xnum <= x + MIN_URL_LEN + 3) return false;
+    for (index_type i = x; i < MIN(x + MIN_URL_LEN + 3, self->xnum); i++) {
+        if (!is_url_char(self->cells[i].ch)) return false;
+    }
+    return true;
+}
+
+index_type
+line_url_start_at(Line *self, index_type x) {
+    // Find the starting cell for a URL that contains the position x. A URL is defined as
+    // known-prefix://url-chars. If no URL is found self->xnum is returned.
+    if (x >= self->xnum || self->xnum <= MIN_URL_LEN + 3) return self->xnum;
+    index_type ds_pos = 0, t;
+    // First look for :// ahead of x
+    if (self->xnum - x > MAX_URL_SCHEME_LEN + 3) ds_pos = find_colon_slash(self, x + MAX_URL_SCHEME_LEN + 3, x < 2 ? 0 : x - 2);
+    if (ds_pos != 0 && has_url_beyond(self, ds_pos)) {
+        if (has_url_prefix_at(self, ds_pos, ds_pos > x ? ds_pos - x: 0, &t)) return t;
+    }
+    ds_pos = find_colon_slash(self, x, 0);
+    if (ds_pos == 0 || self->xnum < ds_pos + MIN_URL_LEN + 3 || !has_url_beyond(self, ds_pos)) return self->xnum;
+    if (has_url_prefix_at(self, ds_pos, 0, &t)) return t;
+    return self->xnum;
+}
+
+index_type
+line_url_end_at(Line *self, index_type x) {
+    index_type ans = x;
+    if (x >= self->xnum || self->xnum <= MIN_URL_LEN + 3) return 0;
+    while (ans < self->xnum && is_url_char(self->cells[ans].ch)) ans++;
+    ans--;
+    while (ans > x && can_strip_from_end_of_url(self->cells[ans].ch)) ans--;
+    return ans;
+}
+
+static PyObject*
+url_start_at(Line *self, PyObject *x) {
+#define url_start_at_doc "url_start_at(x) -> Return the start cell number for a URL containing x or self->xnum if not found"
+    return PyLong_FromUnsignedLong((unsigned long)line_url_start_at(self, PyLong_AsUnsignedLong(x)));
+}
+
+static PyObject*
+url_end_at(Line *self, PyObject *x) {
+#define url_end_at_doc "url_end_at(x) -> Return the end cell number for a URL containing x or 0 if not found"
+    return PyLong_FromUnsignedLong((unsigned long)line_url_end_at(self, PyLong_AsUnsignedLong(x)));
+}
+
+// }}}
+
 static PyObject*
 text_at(Line* self, Py_ssize_t xval) {
 #define text_at_doc "[x] -> Return the text in the specified cell"
-    char_type ch;
-    combining_type cc;
-
     if (xval >= self->xnum) { PyErr_SetString(PyExc_IndexError, "Column number out of bounds"); return NULL; }
-    ch = self->chars[xval] & CHAR_MASK;
-    cc = self->combining_chars[xval];
-    return line_text_at(ch, cc);
+    return line_text_at(self->cells[xval].ch, self->cells[xval].cc);
+}
+
+PyObject*
+unicode_in_range(Line *self, index_type start, index_type limit, bool include_cc, char leading_char) {
+    size_t n = 0;
+    static Py_UCS4 buf[4096];
+    if (leading_char) buf[n++] = leading_char; 
+    char_type previous_width = 0;
+    for(index_type i = start; i < limit && n < sizeof(buf)/sizeof(buf[0]) - 4; i++) {
+        char_type ch = self->cells[i].ch;
+        if (ch == 0) {
+            if (previous_width == 2) { previous_width = 0; continue; };
+            ch = ' ';
+        }
+        buf[n++] = ch;
+        if (include_cc) {
+            char_type cc = self->cells[i].cc;
+            Py_UCS4 cc1 = cc & CC_MASK, cc2;
+            if (cc1) {
+                buf[n++] = cc1;
+                cc2 = cc >> 16;
+                if (cc2) buf[n++] = cc2;
+            }
+        }
+        previous_width = self->cells[i].attrs & WIDTH_MASK;
+    }
+    return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, n);
 }
 
 static PyObject *
 as_unicode(Line* self) {
-    Py_ssize_t n = 0;
-    Py_UCS4 *buf = PyMem_Malloc(3 * self->xnum * sizeof(Py_UCS4));
-    if (buf == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-    for(index_type i = 0; i < self->xnum; i++) {
-        char_type attrs = self->chars[i] >> ATTRS_SHIFT;
-        if ((attrs & WIDTH_MASK) < 1) continue;
-        buf[n++] = self->chars[i] & CHAR_MASK;
-        char_type cc = self->combining_chars[i];
-        Py_UCS4 cc1 = cc & CC_MASK, cc2;
-        if (cc1) {
-            buf[n++] = cc1;
-            cc2 = cc >> 16;
-            if (cc2) buf[n++] = cc2;
-        }
-    }
-    PyObject *ans = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, n);
-    PyMem_Free(buf);
-    return ans;
-}
-
-static PyObject *
-as_base_text(Line* self) {
-#define as_base_text_doc "Return the line as a string, discarding all combining characters"
-    PyObject *ans = PyUnicode_New(self->xnum, 1114111);
-    if (ans == NULL) return PyErr_NoMemory();
-    for(index_type i = 0; i < self->xnum; i++) {
-        char_type attrs = self->chars[i] >> ATTRS_SHIFT;
-        if ((attrs & WIDTH_MASK) < 1) continue;
-        PyUnicode_WRITE(PyUnicode_4BYTE_KIND, PyUnicode_DATA(ans), i, self->chars[i] & CHAR_MASK);
-    }
-    return ans;
+    return unicode_in_range(self, 0, xlimit_for_line(self), true, 0);
 }
 
 static inline bool
@@ -141,21 +240,18 @@ line_as_ansi(Line *self, Py_UCS4 *buf, index_type buflen) {
 #define CHECK_COLOR(name, val, off_code) if (name != (val)) { name = (val); WRITE_COLOR(name, off_code); }
 #define WRITE_CH(val) if (i > buflen - 1) return i; buf[i++] = val; 
 
-    index_type limit = self->xnum, i=0;
-    int r;
-    if (!self->continued) {  // Trim trailing spaces
-        for(r = self->xnum - 1; r >= 0; r--) {
-            if ((self->chars[r] & CHAR_MASK) != 32) break;
-        }
-        limit = r + 1;
-    }
+    index_type limit = xlimit_for_line(self), i=0;
     bool bold = false, italic = false, reverse = false, strike = false;
     uint32_t fg = 0, bg = 0, decoration_fg = 0, decoration = 0;
+    char_type previous_width = 0;
 
     WRITE_SGR(0);
     for (index_type pos=0; pos < limit; pos++) {
-        char_type attrs = self->chars[pos] >> ATTRS_SHIFT, ch = self->chars[pos] & CHAR_MASK;
-        if (ch == 0 || (attrs & WIDTH_MASK) < 1) continue;
+        char_type attrs = self->cells[pos].attrs, ch = self->cells[pos].ch;
+        if (ch == 0) {
+            if (previous_width == 2) { previous_width = 0; continue; }
+            ch = ' ';
+        }
         CHECK_BOOL(bold, BOLD_SHIFT, 1, 22);
         CHECK_BOOL(italic, ITALIC_SHIFT, 3, 23);
         CHECK_BOOL(reverse, REVERSE_SHIFT, 7, 27);
@@ -171,17 +267,18 @@ line_as_ansi(Line *self, Py_UCS4 *buf, index_type buflen) {
                     WRITE_SGR(0); break;
             }
         }
-        CHECK_COLOR(fg, self->fg_colors[pos], 38);
-        CHECK_COLOR(bg, self->bg_colors[pos], 48);
-        CHECK_COLOR(decoration_fg, self->decoration_fg[pos], DECORATION_FG_CODE);
+        CHECK_COLOR(fg, self->cells[pos].fg, 38);
+        CHECK_COLOR(bg, self->cells[pos].bg, 48);
+        CHECK_COLOR(decoration_fg, self->cells[pos].decoration_fg, DECORATION_FG_CODE);
         WRITE_CH(ch);
-        char_type cc = self->combining_chars[pos];
+        char_type cc = self->cells[pos].cc;
         Py_UCS4 cc1 = cc & CC_MASK;
         if (cc1) {
             WRITE_CH(cc1);
             cc1 = cc >> 16;
             if (cc1) { WRITE_CH(cc1); }
         }
+        previous_width = attrs & WIDTH_MASK;
     }
     return i;
 #undef CHECK_BOOL
@@ -222,14 +319,18 @@ width(Line *self, PyObject *val) {
 #define width_doc "width(x) -> the width of the character at x"
     unsigned long x = PyLong_AsUnsignedLong(val);
     if (x >= self->xnum) { PyErr_SetString(PyExc_ValueError, "Out of bounds"); return NULL; }
-    char_type attrs = self->chars[x] >> ATTRS_SHIFT;
+    char_type attrs = self->cells[x].attrs;
     return PyLong_FromUnsignedLong((unsigned long) (attrs & WIDTH_MASK));
 }
 
-void line_add_combining_char(Line *self, uint32_t ch, unsigned int x) {
-    combining_type c = self->combining_chars[x];
-    if (c & CC_MASK) self->combining_chars[x] = (c & CC_MASK) | ( (ch & CC_MASK) << CC_SHIFT );
-    else self->combining_chars[x] = ch & CC_MASK;
+#define set_sprite_position_at(x) set_sprite_position(self->cells + x, x == 0 ? NULL : self->cells + x - 1);
+
+void 
+line_add_combining_char(Line *self, uint32_t ch, unsigned int x) {
+    combining_type c = self->cells[x].cc;
+    if (c & CC_MASK) self->cells[x].cc = (c & CC_MASK) | ( (ch & CC_MASK) << CC_SHIFT );
+    else self->cells[x].cc = ch & CC_MASK;
+    set_sprite_position_at(x);
 }
 
 static PyObject*
@@ -274,11 +375,13 @@ set_text(Line* self, PyObject *args) {
     color_type dfg = cursor->decoration_fg & COL_MASK;
 
     for (index_type i = cursor->x; offset < limit && i < self->xnum; i++, offset++) {
-        self->chars[i] = (PyUnicode_READ(kind, buf, offset) & CHAR_MASK) | attrs;
-        self->fg_colors[i] = fg;
-        self->bg_colors[i] = bg;
-        self->decoration_fg[i] = dfg;
-        self->combining_chars[i] = 0;
+        self->cells[i].ch = (PyUnicode_READ(kind, buf, offset));
+        self->cells[i].attrs = attrs;
+        self->cells[i].fg = fg;
+        self->cells[i].bg = bg;
+        self->cells[i].decoration_fg = dfg;
+        self->cells[i].cc = 0;
+        set_sprite_position_at(i);
     }
 
     Py_RETURN_NONE;
@@ -297,28 +400,35 @@ cursor_from(Line* self, PyObject *args) {
     ans = alloc_cursor();
     if (ans == NULL) { PyErr_NoMemory(); return NULL; }
     ans->x = x; ans->y = y;
-    char_type attrs = self->chars[x] >> ATTRS_SHIFT;
+    char_type attrs = self->cells[x].attrs;
     ATTRS_TO_CURSOR(attrs, ans);
-    ans->fg = self->fg_colors[x]; ans->bg = self->bg_colors[x];
-    ans->decoration_fg = self->decoration_fg[x] & COL_MASK;
+    ans->fg = self->cells[x].fg; ans->bg = self->cells[x].bg;
+    ans->decoration_fg = self->cells[x].decoration_fg & COL_MASK;
 
     return (PyObject*)ans;
 }
 
 void 
-line_clear_text(Line *self, unsigned int at, unsigned int num, int ch) {
-    const char_type repl = ((char_type)ch & CHAR_MASK) | (1 << ATTRS_SHIFT);
-    for (index_type i = at; i < MIN(self->xnum, at + num); i++) {
-        self->chars[i] = (self->chars[i] & ATTRS_MASK_WITHOUT_WIDTH) | repl;
+line_clear_text(Line *self, unsigned int at, unsigned int num, char_type ch) {
+    attrs_type width = ch ? 1 : 0;
+#define PREFIX \
+    for (index_type i = at; i < MIN(self->xnum, at + num); i++) { \
+        self->cells[i].ch = ch; self->cells[i].cc = 0; \
+        self->cells[i].attrs = (self->cells[i].attrs & ATTRS_MASK_WITHOUT_WIDTH) | width;
+    if (CHAR_IS_BLANK(ch)) {
+        PREFIX
+        clear_sprite_position(self->cells[i]); }
+    } else {
+        PREFIX
+        set_sprite_position_at(i)}
     }
-    memset(self->combining_chars + at, 0, MIN(num, self->xnum - at) * sizeof(combining_type));
 }
 
 static PyObject*
 clear_text(Line* self, PyObject *args) {
-#define clear_text_doc "clear_text(at, num, ch=' ') -> Clear characters in the specified range, preserving formatting."
+#define clear_text_doc "clear_text(at, num, ch=BLANK_CHAR) -> Clear characters in the specified range, preserving formatting."
     unsigned int at, num;
-    int ch = 32;
+    int ch = BLANK_CHAR;
     if (!PyArg_ParseTuple(args, "II|C", &at, &num, &ch)) return NULL;
     line_clear_text(self, at, num, ch);
     Py_RETURN_NONE;
@@ -329,18 +439,21 @@ line_apply_cursor(Line *self, Cursor *cursor, unsigned int at, unsigned int num,
     char_type attrs = CURSOR_TO_ATTRS(cursor, 1);
     color_type fg = (cursor->fg & COL_MASK), bg = (cursor->bg & COL_MASK);
     color_type dfg = cursor->decoration_fg & COL_MASK;
-    if (!clear_char) attrs = ((attrs >> ATTRS_SHIFT) & ~WIDTH_MASK) << ATTRS_SHIFT;
+    if (!clear_char) attrs = attrs & ATTRS_MASK_WITHOUT_WIDTH;
     
     for (index_type i = at; i < self->xnum && i < at + num; i++) {
         if (clear_char) {
-            self->chars[i] = 32 | attrs;
-            self->combining_chars[i] = 0;
+            self->cells[i].ch = BLANK_CHAR;
+            self->cells[i].cc = 0;
+            self->cells[i].attrs = attrs;
+            clear_sprite_position(self->cells[i]);
         } else {
-            char_type w = ((self->chars[i] >> ATTRS_SHIFT) & WIDTH_MASK) << ATTRS_SHIFT;
-            self->chars[i] = (self->chars[i] & CHAR_MASK) | attrs | w;
+            attrs_type w = self->cells[i].attrs & WIDTH_MASK;
+            self->cells[i].attrs = attrs | w;
+            set_sprite_position_at(i);
         }
-        self->fg_colors[i] = fg; self->bg_colors[i] = bg;
-        self->decoration_fg[i] = dfg;
+        self->cells[i].fg = fg; self->cells[i].bg = bg;
+        self->cells[i].decoration_fg = dfg;
     }
 }
 
@@ -360,8 +473,12 @@ void line_right_shift(Line *self, unsigned int at, unsigned int num) {
         COPY_SELF_CELL(i - num, i)
     }
     // Check if a wide character was split at the right edge
-    char_type w = (self->chars[self->xnum - 1] >> ATTRS_SHIFT) & 3;
-    if (w != 1) self->chars[self->xnum - 1] = (1 << ATTRS_SHIFT) | 32;
+    char_type w = (self->cells[self->xnum - 1].attrs) & WIDTH_MASK;
+    if (w != 1) {
+        self->cells[self->xnum - 1].ch = BLANK_CHAR;
+        self->cells[self->xnum - 1].attrs = BLANK_CHAR ? 1 : 0;
+        clear_sprite_position(self->cells[self->xnum - 1]);
+    }
 }
 
 static PyObject*
@@ -393,18 +510,19 @@ left_shift(Line *self, PyObject *args) {
 }
  
 void 
-line_set_char(Line *self, unsigned int at, uint32_t ch, unsigned int width, Cursor *cursor) {
-    char_type attrs;
+line_set_char(Line *self, unsigned int at, uint32_t ch, unsigned int width, Cursor *cursor, bool is_second) {
     if (cursor == NULL) {
-        attrs = (((self->chars[at] >> ATTRS_SHIFT) & ~3) | (width & 3)) << ATTRS_SHIFT;
+        self->cells[at].attrs = (self->cells[at].attrs & ATTRS_MASK_WITHOUT_WIDTH) | width;
     } else {
-        attrs = CURSOR_TO_ATTRS(cursor, width & 3);
-        self->fg_colors[at] = (cursor->fg & COL_MASK);
-        self->bg_colors[at] = (cursor->bg & COL_MASK);
-        self->decoration_fg[at] = cursor->decoration_fg & COL_MASK;
+        self->cells[at].attrs = CURSOR_TO_ATTRS(cursor, width & WIDTH_MASK);
+        self->cells[at].fg = (cursor->fg & COL_MASK);
+        self->cells[at].bg = (cursor->bg & COL_MASK);
+        self->cells[at].decoration_fg = cursor->decoration_fg & COL_MASK;
     }
-    self->chars[at] = (ch & CHAR_MASK) | attrs;
-    self->combining_chars[at] = 0;
+    self->cells[at].ch = ch;
+    self->cells[at].cc = 0;
+    if (!is_second && CHAR_IS_BLANK(ch)) { clear_sprite_position(self->cells[at]); }
+    else set_sprite_position_at(at);
 }
 
 static PyObject*
@@ -419,7 +537,7 @@ set_char(Line *self, PyObject *args) {
         PyErr_SetString(PyExc_ValueError, "Out of bounds");
         return NULL;
     }
-    line_set_char(self, at, ch, width, cursor);
+    line_set_char(self, at, ch, width, cursor, false);
     Py_RETURN_NONE;
 }
 
@@ -427,25 +545,19 @@ static PyObject*
 set_attribute(Line *self, PyObject *args) {
 #define set_attribute_doc "set_attribute(which, val) -> Set the attribute on all cells in the line."
     unsigned int shift, val;
-    char_type mask;
     if (!PyArg_ParseTuple(args, "II", &shift, &val)) return NULL;
     if (shift < DECORATION_SHIFT || shift > STRIKE_SHIFT) { PyErr_SetString(PyExc_ValueError, "Unknown attribute"); return NULL; }
-    SET_ATTRIBUTE(self->chars, shift, val);
+    set_attribute_on_line(self->cells, shift, val, self->xnum);
     Py_RETURN_NONE;
 }
 
 static Py_ssize_t
 __len__(PyObject *self) {
-    return (Py_ssize_t)(((Line*)self)->ynum);
+    return (Py_ssize_t)(((Line*)self)->xnum);
 }
 
 static int __eq__(Line *a, Line *b) {
-    return a->xnum == b->xnum && \
-                    memcmp(a->chars, b->chars, sizeof(char_type) * a->xnum) == 0 && \
-                    memcmp(a->fg_colors, b->fg_colors, sizeof(color_type) * a->xnum) == 0 && \
-                    memcmp(a->bg_colors, b->bg_colors, sizeof(color_type) * a->xnum) == 0 && \
-                    memcmp(a->decoration_fg, b->decoration_fg, sizeof(color_type) * a->xnum) == 0 && \
-                    memcmp(a->combining_chars, b->combining_chars, sizeof(combining_type) * a->xnum) == 0;
+    return a->xnum == b->xnum && memcmp(a->cells, b->cells, sizeof(Cell) * a->xnum) == 0;
 }
 
 // Boilerplate {{{
@@ -473,10 +585,11 @@ static PyMethodDef methods[] = {
     METHOD(left_shift, METH_VARARGS)
     METHOD(set_char, METH_VARARGS)
     METHOD(set_attribute, METH_VARARGS)
-    METHOD(as_base_text, METH_NOARGS)
     METHOD(as_ansi, METH_NOARGS)
     METHOD(is_continued, METH_NOARGS)
     METHOD(width, METH_O)
+    METHOD(url_start_at, METH_O)
+    METHOD(url_end_at, METH_O)
         
     {NULL}  /* Sentinel */
 };
@@ -503,7 +616,8 @@ Line *alloc_line() {
 }
 
 RICHCMP(Line)
-// }}
+INIT_TYPE(Line)
+// }}}
  
 static PyObject*
 copy_char(Line* self, PyObject *args) {
@@ -514,12 +628,7 @@ copy_char(Line* self, PyObject *args) {
         PyErr_SetString(PyExc_ValueError, "Out of bounds");
         return NULL;
     }
-    to->chars[dest] = self->chars[src];
-    to->fg_colors[dest] = self->fg_colors[src];
-    to->bg_colors[dest] = self->bg_colors[src];
-    to->decoration_fg[dest] = self->decoration_fg[src];
-    to->combining_chars[dest] = self->combining_chars[src];
+    COPY_CELL(self, src, to, dest);
     Py_RETURN_NONE;
 }
 
-INIT_TYPE(Line)

kitty/lineops.h

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#pragma once
+
+#include "data-types.h"
+
+static inline void
+update_sprites_in_line(Cell *cells, index_type xnum) {
+    if (LIKELY(xnum > 0)) {
+        if (CHAR_IS_BLANK(cells->ch)) { clear_sprite_position(cells[0]); }
+        else set_sprite_position(cells, NULL);
+        for (index_type i = 1; i < xnum; i++) {
+            if (CHAR_IS_BLANK(cells[i].ch)) { clear_sprite_position(cells[i]); }
+            else set_sprite_position(cells + i, cells + i - 1);
+        }
+    }
+}
+
+static inline void
+set_attribute_on_line(Cell *cells, uint32_t shift, uint32_t val, index_type xnum) {
+    // Set a single attribute on all cells in the line
+    attrs_type mask = shift == DECORATION_SHIFT ? 3 : 1;
+    attrs_type aval = (val & mask) << shift; 
+    mask = ~(mask << shift);
+    for (index_type i = 0; i < xnum; i++) cells[i].attrs = (cells[i].attrs & mask) | aval;
+    if (shift == BOLD_SHIFT || shift == ITALIC_SHIFT) update_sprites_in_line(cells, xnum);
+}
+
+static inline void
+copy_cells(const Cell *src, Cell *dest, index_type xnum) {
+    memcpy(dest, src, sizeof(Cell) * xnum);
+}
+
+static inline void
+copy_line(const Line *src, Line *dest) {
+    copy_cells(src->cells, dest->cells, MIN(src->xnum, dest->xnum));
+}
+
+static inline void
+clear_chars_in_line(Cell *cells, index_type xnum, char_type ch) {
+    // Clear only the char part of each cell, the rest must have been cleared by a memset or similar
+    if (ch) {
+        for (index_type i = 0; i < xnum; i++) { cells[i].ch = ch; cells[i].attrs = 1; }
+    }
+}
+
+static inline index_type
+xlimit_for_line(Line *line) {
+    index_type xlimit = line->xnum;
+    if (BLANK_CHAR == 0) {
+        while (xlimit > 0 && (line->cells[xlimit - 1].ch) == BLANK_CHAR) xlimit--;
+    }
+    return xlimit;
+}
+
+PyObject* line_text_at(char_type, combining_type);
+void line_clear_text(Line *self, unsigned int at, unsigned int num, char_type ch);
+void line_apply_cursor(Line *self, Cursor *cursor, unsigned int at, unsigned int num, bool clear_char);
+void line_set_char(Line *, unsigned int , uint32_t , unsigned int , Cursor *, bool);
+void line_right_shift(Line *, unsigned int , unsigned int );
+void line_add_combining_char(Line *, uint32_t , unsigned int );
+index_type line_url_start_at(Line *self, index_type x);
+index_type line_url_end_at(Line *self, index_type x);
+index_type line_as_ansi(Line *self, Py_UCS4 *buf, index_type buflen);
+unsigned int line_length(Line *self);
+PyObject* unicode_in_range(Line *self, index_type start, index_type limit, bool include_cc, char leading_char);
+
+void linebuf_init_line(LineBuf *, index_type);
+void linebuf_clear(LineBuf *, char_type ch);
+void linebuf_init_line(LineBuf *, index_type);
+void linebuf_index(LineBuf* self, index_type top, index_type bottom);
+void linebuf_reverse_index(LineBuf *self, index_type top, index_type bottom);
+void linebuf_clear_line(LineBuf *self, index_type y);
+void linebuf_insert_lines(LineBuf *self, unsigned int num, unsigned int y, unsigned int bottom);
+void linebuf_delete_lines(LineBuf *self, index_type num, index_type y, index_type bottom);
+void linebuf_set_attribute(LineBuf *, unsigned int , unsigned int );
+void linebuf_rewrap(LineBuf *self, LineBuf *other, index_type *, index_type *, HistoryBuf *);
+unsigned int linebuf_char_width_at(LineBuf *self, index_type x, index_type y);
+void linebuf_refresh_sprite_positions(LineBuf *self);
+bool historybuf_resize(HistoryBuf *self, index_type lines);
+void historybuf_add_line(HistoryBuf *self, const Line *line);
+void historybuf_rewrap(HistoryBuf *self, HistoryBuf *other);
+void historybuf_init_line(HistoryBuf *self, index_type num, Line *l);
+void historybuf_refresh_sprite_positions(HistoryBuf *self);

kitty/main.py

@@ -5,34 +5,35 @@
 import argparse
 import locale
 import os
+import signal
 import sys
-import tempfile
+from contextlib import contextmanager
 from gettext import gettext as _
 
-from queue import Empty
-
 from .boss import Boss
 from .config import (
     cached_values, load_cached_values, load_config, save_cached_values
 )
 from .constants import (
-    appname, config_dir, isosx, logo_data_file, str_version, viewport_size
+    appname, defconf, isosx, iswayland, logo_data_file, str_version,
+    viewport_size
 )
 from .fast_data_types import (
-    GL_COLOR_BUFFER_BIT, GLFW_CONTEXT_VERSION_MAJOR,
-    GLFW_CONTEXT_VERSION_MINOR, GLFW_OPENGL_CORE_PROFILE,
+    GL_VERSION_REQUIRED, GLFW_CONTEXT_VERSION_MAJOR,
+    GLFW_CONTEXT_VERSION_MINOR, GLFW_DECORATED, GLFW_OPENGL_CORE_PROFILE,
     GLFW_OPENGL_FORWARD_COMPAT, GLFW_OPENGL_PROFILE, GLFW_SAMPLES,
-    GLFW_STENCIL_BITS, Window, change_wcwidth,
-    enable_automatic_opengl_error_checking, glClear, glClearColor, glewInit,
-    glfw_init, glfw_set_error_callback, glfw_swap_interval, glfw_terminate,
-    glfw_wait_events, glfw_window_hint
+    GLFW_STENCIL_BITS, GLFWWindow, change_wcwidth, check_for_extensions,
+    clear_buffers, glewInit, glfw_init, glfw_init_hint_string,
+    glfw_swap_interval, glfw_terminate, glfw_window_hint,
+    install_sigchld_handler, set_logical_dpi, set_options
 )
 from .layout import all_layouts
-from .shaders import GL_VERSION
-from .utils import safe_print
+from .utils import color_as_int, detach, get_logical_dpi, safe_print
 
-
-defconf = os.path.join(config_dir, 'kitty.conf')
+try:
+    from .fast_data_types import GLFW_X11_WM_CLASS_NAME, GLFW_X11_WM_CLASS_CLASS
+except ImportError:
+    GLFW_X11_WM_CLASS_NAME = GLFW_X11_WM_CLASS_CLASS = None
 
 
 def option_parser():
@@ -83,18 +84,19 @@ def option_parser():
         action='version',
         version='{} {} by Kovid Goyal'.format(appname, str_version)
     )
-    a(
-        '--profile',
-        action='store_true',
-        default=False,
-        help=_('Show profiling data after exit')
-    )
     a(
         '--dump-commands',
         action='store_true',
         default=False,
         help=_('Output commands received from child process to stdout')
     )
+    if not isosx:
+        a(
+            '--detach',
+            action='store_true',
+            default=False,
+            help=_('Detach from the controlling terminal, if any')
+        )
     a(
         '--replay-commands',
         default=None,
@@ -106,7 +108,7 @@ def option_parser():
                ' child process. Useful for debugging.')
     )
     a(
-        '--debug-gl',
+        '--debug-kitty-gl',
         action='store_true',
         default=False,
         help=_('Debug OpenGL commands. This will cause all OpenGL calls'
@@ -137,9 +139,11 @@ def option_parser():
     return parser
 
 
-def setup_opengl():
-    glfw_window_hint(GLFW_CONTEXT_VERSION_MAJOR, GL_VERSION[0])
-    glfw_window_hint(GLFW_CONTEXT_VERSION_MINOR, GL_VERSION[1])
+def setup_opengl(opts):
+    if opts.macos_hide_titlebar:
+        glfw_window_hint(GLFW_DECORATED, False)
+    glfw_window_hint(GLFW_CONTEXT_VERSION_MAJOR, GL_VERSION_REQUIRED[0])
+    glfw_window_hint(GLFW_CONTEXT_VERSION_MINOR, GL_VERSION_REQUIRED[1])
     glfw_window_hint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE)
     glfw_window_hint(GLFW_OPENGL_FORWARD_COMPAT, True)
     glfw_window_hint(GLFW_SAMPLES, 0)
@@ -148,34 +152,23 @@ def setup_opengl():
         glfw_window_hint(GLFW_STENCIL_BITS, 8)
 
 
-def clear_buffers(window, opts):
-    bg = opts.background
-    glClearColor(bg.red / 255, bg.green / 255, bg.blue / 255, 1)
+def initialize_window(window, opts):
+    set_logical_dpi(*get_logical_dpi())
+    viewport_size.width, viewport_size.height = window.get_framebuffer_size()
+    w, h = window.get_window_size()
+    viewport_size.x_ratio = viewport_size.width / float(w)
+    viewport_size.y_ratio = viewport_size.height / float(h)
+    glewInit()
     glfw_swap_interval(0)
-    glClear(GL_COLOR_BUFFER_BIT)
-    window.swap_buffers()
-    glClear(GL_COLOR_BUFFER_BIT)
+    clear_buffers(window.swap_buffers, color_as_int(opts.background))
     # We dont turn this on as it causes rendering performance to be much worse,
     # for example, dragging the mouse to select is laggy
     # glfw_swap_interval(1)
 
 
-def dispatch_pending_calls(boss):
-    while True:
-        try:
-            func, args = boss.pending_ui_thread_calls.get_nowait()
-        except Empty:
-            break
-        try:
-            func(*args)
-        except Exception:
-            import traceback
-            safe_print(traceback.format_exc())
-    boss.ui_timers()
-
-
 def run_app(opts, args):
-    setup_opengl()
+    set_options(opts)
+    setup_opengl(opts)
     load_cached_values()
     if 'window-size' in cached_values and opts.remember_window_size:
         ws = cached_values['window-size']
@@ -188,30 +181,29 @@ def run_app(opts, args):
     else:
         viewport_size.width = opts.initial_window_width
         viewport_size.height = opts.initial_window_height
-    window = Window(viewport_size.width, viewport_size.height, args.cls)
-    window.set_title(appname)
+    try:
+        window = GLFWWindow(viewport_size.width, viewport_size.height, args.cls)
+    except ValueError:
+        safe_print('Failed to create GLFW window with initial size:', viewport_size)
+        viewport_size.width = 640
+        viewport_size.height = 400
+        window = GLFWWindow(viewport_size.width, viewport_size.height, args.cls)
     window.make_context_current()
     if isosx:
+        from .fast_data_types import cocoa_make_window_resizable, cocoa_create_global_menu, cocoa_init
+        check_for_extensions()
+        cocoa_init()
+        cocoa_create_global_menu()
         if opts.macos_hide_titlebar:
-            from .fast_data_types import cocoa_hide_titlebar
-            cocoa_hide_titlebar(window.cocoa_window_id())
-    else:
+            cocoa_make_window_resizable(window.cocoa_window_id())
+    elif not iswayland:  # no window icons on wayland
         with open(logo_data_file, 'rb') as f:
             window.set_icon(f.read(), 256, 256)
-    viewport_size.width, viewport_size.height = window.get_framebuffer_size()
-    w, h = window.get_window_size()
-    viewport_size.x_ratio = viewport_size.width / float(w)
-    viewport_size.y_ratio = viewport_size.height / float(h)
-    glewInit()
+    initialize_window(window, opts)
     boss = Boss(window, opts, args)
     boss.start()
-    clear_buffers(window, opts)
     try:
-        while not window.should_close():
-            boss.render()
-            window.swap_buffers()
-            glfw_wait_events(boss.ui_timers.timeout())
-            dispatch_pending_calls(boss)
+        boss.child_monitor.main_loop()
     finally:
         boss.destroy()
     del window
@@ -219,21 +211,65 @@ def run_app(opts, args):
     save_cached_values()
 
 
-def on_glfw_error(code, msg):
-    if isinstance(msg, bytes):
+def ensure_osx_locale():
+    # Ensure the LANG env var is set. See
+    # https://github.com/kovidgoyal/kitty/issues/90
+    from .fast_data_types import cocoa_get_lang
+    if 'LANG' not in os.environ:
+        lang = cocoa_get_lang()
+        if lang is not None:
+            os.environ['LANG'] = lang + '.UTF-8'
+
+
+@contextmanager
+def setup_profiling(args):
+    try:
+        from .fast_data_types import start_profiler, stop_profiler
+    except ImportError:
+        start_profiler = stop_profiler = None
+    if start_profiler is not None:
+        start_profiler('/tmp/kitty-profile.log')
+    yield
+    if stop_profiler is not None:
+        import subprocess
+        stop_profiler()
+        exe = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'kitty-profile')
+        cg = '/tmp/kitty-profile.callgrind'
+        print('Post processing profile data for', exe, '...')
+        subprocess.call(['pprof', '--callgrind', exe, '/tmp/kitty-profile.log'], stdout=open(cg, 'wb'))
         try:
-            msg = msg.decode('utf-8')
-        except Exception:
-            msg = repr(msg)
-    safe_print('[glfw error] ', msg, file=sys.stderr)
+            subprocess.Popen(['kcachegrind', cg])
+        except FileNotFoundError:
+            subprocess.call(['pprof', '--text', exe, '/tmp/kitty-profile.log'])
+            print('To view the graphical call data, use: kcachegrind', cg)
 
 
 def main():
-    locale.setlocale(locale.LC_ALL, '')
+    try:
+        sys.setswitchinterval(1000.0)  # we have only a single python thread
+    except AttributeError:
+        pass  # python compiled without threading
+    if isosx:
+        ensure_osx_locale()
+    try:
+        locale.setlocale(locale.LC_ALL, '')
+    except Exception:
+        if not isosx:
+            raise
+        print('Failed to set locale with LANG:', os.environ.get('LANG'), file=sys.stderr)
+        os.environ.pop('LANG')
+        try:
+            locale.setlocale(locale.LC_ALL, '')
+        except Exception:
+            print('Failed to set locale with no LANG, ignoring', file=sys.stderr)
     if os.environ.pop('KITTY_LAUNCHED_BY_LAUNCH_SERVICES',
                       None) == '1' and getattr(sys, 'frozen', True):
         os.chdir(os.path.expanduser('~'))
+    if not os.path.isdir(os.getcwd()):
+        os.chdir(os.path.expanduser('~'))
     args = option_parser().parse_args()
+    if getattr(args, 'detach', False):
+        detach()
     if args.cmd:
         exec(args.cmd)
         return
@@ -245,28 +281,15 @@ def main():
     overrides = (a.replace('=', ' ', 1) for a in args.override or ())
     opts = load_config(*config, overrides=overrides)
     change_wcwidth(not opts.use_system_wcwidth)
-    glfw_set_error_callback(on_glfw_error)
-    enable_automatic_opengl_error_checking(args.debug_gl)
+    if GLFW_X11_WM_CLASS_CLASS is not None:
+        glfw_init_hint_string(GLFW_X11_WM_CLASS_CLASS, args.cls)
     if not glfw_init():
         raise SystemExit('GLFW initialization failed')
     try:
-        if args.profile:
-            tf = tempfile.NamedTemporaryFile(prefix='kitty-profiling-stats-')
-            args.profile = tf.name
-            import cProfile
-            import pstats
-            pr = cProfile.Profile()
-            pr.enable()
-            run_app(opts, args)
-            pr.disable()
-            pr.create_stats()
-            s = pstats.Stats(pr)
-            s.add(args.profile)
-            tf.close()
-            s.strip_dirs()
-            s.sort_stats('time', 'name')
-            s.print_stats(30)
-        else:
+        with setup_profiling(args):
+            # Avoid needing to launch threads to reap zombies
+            install_sigchld_handler()
             run_app(opts, args)
+            signal.signal(signal.SIGCHLD, signal.SIG_DFL)
     finally:
         glfw_terminate()

kitty/mouse.c

@@ -0,0 +1,358 @@
+/*
+ * mouse.c
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "state.h"
+#include "screen.h"
+#include "lineops.h"
+#include <limits.h>
+#include <math.h>
+#include <GLFW/glfw3.h>
+
+static MouseShape mouse_cursor_shape = BEAM;
+typedef enum MouseActions { PRESS, RELEASE, DRAG, MOVE } MouseAction;
+
+#define SHIFT_INDICATOR  (1 << 2)
+#define ALT_INDICATOR (1 << 3)
+#define CONTROL_INDICATOR (1 << 4)
+#define MOTION_INDICATOR  (1 << 5)
+#define EXTRA_BUTTON_INDICATOR (1 << 6)
+
+static inline unsigned int
+button_map(int button) {
+    switch(button) {
+        case GLFW_MOUSE_BUTTON_1:
+            return 0;
+        case GLFW_MOUSE_BUTTON_2:
+            return 1;
+        case GLFW_MOUSE_BUTTON_3:
+            return 2;
+        case GLFW_MOUSE_BUTTON_4:
+            return EXTRA_BUTTON_INDICATOR;
+        case GLFW_MOUSE_BUTTON_5:
+            return EXTRA_BUTTON_INDICATOR | 1;
+        default:
+            return UINT_MAX;
+    }
+}
+
+static char mouse_event_buf[64];
+
+size_t
+encode_mouse_event(Window *w, int button, MouseAction action, int mods) {
+    unsigned int x = w->mouse_cell_x + 1, y = w->mouse_cell_y + 1; // 1 based indexing
+    unsigned int cb = 0;
+    Screen *screen = w->render_data.screen;
+
+    if (action == MOVE) {
+        if (screen->modes.mouse_tracking_protocol != SGR_PROTOCOL) cb = 3;
+    } else {
+        cb = button_map(button);
+        if (cb == UINT_MAX) return 0;
+    }
+    if (action == DRAG || action == MOVE) cb |= MOTION_INDICATOR;
+    else if (action == RELEASE && screen->modes.mouse_tracking_protocol != SGR_PROTOCOL) cb = 3;
+    if (mods & GLFW_MOD_SHIFT) cb |= SHIFT_INDICATOR;
+    if (mods & GLFW_MOD_ALT) cb |= ALT_INDICATOR;
+    if (mods & GLFW_MOD_CONTROL) cb |= CONTROL_INDICATOR;
+    switch(screen->modes.mouse_tracking_protocol) {
+        case SGR_PROTOCOL:
+            return snprintf(mouse_event_buf, sizeof(mouse_event_buf), "\033[<%d;%d;%d%s", cb, x, y, action == RELEASE ? "m" : "M");
+            break;
+        case URXVT_PROTOCOL:
+            return snprintf(mouse_event_buf, sizeof(mouse_event_buf), "\033[%d;%d;%dM", cb, x, y);
+            break;
+        case UTF8_PROTOCOL:
+            mouse_event_buf[0] = 033; mouse_event_buf[1] = '['; mouse_event_buf[2] = cb + 32; 
+            unsigned int sz = 3;
+            sz += encode_utf8(x + 32, mouse_event_buf + sz);
+            sz += encode_utf8(y + 32, mouse_event_buf + sz);
+            return sz;
+            break;
+        default:
+            if (x > 223 || y > 223) return 0;
+            else {
+                mouse_event_buf[0] = 033; mouse_event_buf[1] = '['; mouse_event_buf[2] = 'M'; mouse_event_buf[3] = cb + 32; mouse_event_buf[4] = x + 32; mouse_event_buf[5] = y + 32;
+                return 6;
+            }
+            break;
+    }
+    return 0;
+}
+
+static inline bool
+contains_mouse(Window *w) {
+    WindowGeometry *g = &w->geometry;
+    double x = global_state.mouse_x, y = global_state.mouse_y;
+    return (w->visible && g->left <= x && x <= g->right && g->top <= y && y <= g->bottom);
+}
+
+static inline bool
+cell_for_pos(Window *w, unsigned int *x, unsigned int *y) {
+    WindowGeometry *g = &w->geometry;
+    unsigned int qx = (unsigned int)((double)(global_state.mouse_x - g->left) / global_state.cell_width);
+    unsigned int qy = (unsigned int)((double)(global_state.mouse_y - g->top) / global_state.cell_height);
+    bool ret = false;
+    Screen *screen = w->render_data.screen;
+    if (screen && qx <= screen->columns && qy <= screen->lines) {
+        *x = qx; *y = qy; ret = true;
+    }
+    return ret;
+}
+
+#define HANDLER(name) static inline void name(Window UNUSED *w, int UNUSED button, int UNUSED modifiers, unsigned int UNUSED window_idx)
+
+static inline void
+update_drag(bool from_button, Window *w, bool is_release) {
+    Screen *screen = w->render_data.screen;
+    if (from_button) {
+        if (is_release) screen_update_selection(screen, w->mouse_cell_x, w->mouse_cell_y, true);
+        else screen_start_selection(screen, w->mouse_cell_x, w->mouse_cell_y);
+    } else if (screen->selection.in_progress) {
+        screen_update_selection(screen, w->mouse_cell_x, w->mouse_cell_y, false);
+        call_boss(set_primary_selection, NULL);
+    }
+}
+
+
+bool
+drag_scroll(Window *w) {
+    unsigned int margin = global_state.cell_height / 2;
+    double x = global_state.mouse_x, y = global_state.mouse_y;
+    if (y < w->geometry.top || y > w->geometry.bottom) return false;
+    if (x < w->geometry.left || x > w->geometry.right) return false;
+    bool upwards = y <= (w->geometry.top + margin);
+    if (upwards || y >= w->geometry.bottom - margin) {
+        Screen *screen = w->render_data.screen;
+        if (screen->linebuf == screen->main_linebuf) {
+            screen_history_scroll(screen, SCROLL_LINE, upwards);
+            update_drag(false, w, false);
+            global_state.last_mouse_activity_at = monotonic();
+            if (mouse_cursor_shape != ARROW) {
+                mouse_cursor_shape = ARROW;
+                set_mouse_cursor(mouse_cursor_shape);
+            }
+            return true;
+        }
+    }
+    return false;
+}
+
+
+static inline void
+detect_url(Window *w, Screen *screen, unsigned int x, unsigned int y) {
+    bool has_url = false;
+    index_type url_start, url_end = 0;
+    Line *line = screen_visual_line(screen, y);
+    if (line) {
+        url_start = line_url_start_at(line, x);
+        if (url_start < line->xnum) url_end = line_url_end_at(line, x);
+        has_url = url_end > url_start;
+    }
+    if (has_url) {
+        mouse_cursor_shape = HAND;
+        screen_mark_url(screen, url_start, w->mouse_cell_y, url_end, w->mouse_cell_y);
+    } else {
+        mouse_cursor_shape = BEAM;
+        screen_mark_url(screen, 0, 0, 0, 0);
+    }
+}
+
+
+HANDLER(handle_move_event) {
+    unsigned int x, y;
+    if (!cell_for_pos(w, &x, &y)) return;
+    Screen *screen = w->render_data.screen;
+    detect_url(w, screen, x, y);
+    bool mouse_cell_changed = x != w->mouse_cell_x || y != w->mouse_cell_y;
+    w->mouse_cell_x = x; w->mouse_cell_y = y;
+    bool handle_in_kitty = (
+            (screen->modes.mouse_tracking_mode == ANY_MODE ||
+            (screen->modes.mouse_tracking_mode == MOTION_MODE && button >= 0)) &&
+            !(global_state.is_key_pressed[GLFW_KEY_LEFT_SHIFT] || global_state.is_key_pressed[GLFW_KEY_RIGHT_SHIFT])
+    ) ? false : true;
+    if (handle_in_kitty) {
+        if (screen->selection.in_progress && button == GLFW_MOUSE_BUTTON_LEFT) {
+            double now = monotonic();
+            if ((now - w->last_drag_scroll_at) >= 0.02 || mouse_cell_changed) {
+                update_drag(false, w, false);
+                w->last_drag_scroll_at = monotonic();
+            }
+        }
+    } else {
+        if (!mouse_cell_changed) return;
+        size_t sz = encode_mouse_event(w, MAX(0, button), button >=0 ? DRAG : MOVE, 0);
+        if (sz) schedule_write_to_child(w->id, mouse_event_buf, sz);
+    }
+}
+
+static inline void
+multi_click(Window *w, unsigned int count) {
+    Screen *screen = w->render_data.screen;
+    index_type start, end;
+    bool found_selection = false;
+    switch(count) {
+        case 2:
+            found_selection = screen_selection_range_for_word(screen, w->mouse_cell_x, w->mouse_cell_y, &start, &end);
+            break;
+        case 3:
+            found_selection = screen_selection_range_for_line(screen, w->mouse_cell_y, &start, &end);
+            break;
+        default:
+            break;
+    }
+    if (found_selection) {
+        screen_start_selection(screen, start, w->mouse_cell_y);
+        screen_update_selection(screen, end, w->mouse_cell_y, true);
+        call_boss(set_primary_selection, NULL);
+    }
+}
+
+HANDLER(add_click) {
+    ClickQueue *q = &w->click_queue;
+    if (q->length == CLICK_QUEUE_SZ) { memmove(q->clicks, q->clicks + 1, sizeof(Click) * (CLICK_QUEUE_SZ - 1)); q->length--; }
+    double now = monotonic();
+#define N(n) (q->clicks[q->length - n])
+    N(0).at = now; N(0).button = button; N(0).modifiers = modifiers;
+    q->length++;
+    // Now dispatch the multi-click if any
+    if (q->length > 2 && N(1).at - N(3).at <= 2 * OPT(click_interval)) {
+        multi_click(w, 3);
+        q->length = 0;
+    } else if (q->length > 1 && N(1).at - N(2).at <= OPT(click_interval)) {
+        multi_click(w, 2);
+    }
+#undef N
+}
+
+static inline void
+open_url(Window *w) {
+    Line *line = screen_visual_line(w->render_data.screen, w->mouse_cell_y);
+    if (line) {
+        index_type start = line_url_start_at(line, w->mouse_cell_x); 
+        if (start < line->xnum) {
+            index_type end = line_url_end_at(line, w->mouse_cell_x);
+            if (end > start) {
+                call_boss(open_url, "N", unicode_in_range(line, start, end + 1, true, 0));
+            }
+        }
+    }
+}
+
+HANDLER(handle_button_event) {
+    Tab *t = global_state.tabs + global_state.active_tab;
+    bool is_release = !global_state.mouse_button_pressed[button];
+    if (window_idx != t->active_window) {
+        call_boss(switch_focus_to, "I", window_idx);
+    }
+    Screen *screen = w->render_data.screen;
+    if (!screen) return;
+    bool handle_in_kitty = (
+            modifiers == GLFW_MOD_SHIFT || 
+            screen->modes.mouse_tracking_mode == 0 ||
+            button == GLFW_MOUSE_BUTTON_MIDDLE ||
+            (modifiers == (int)OPT(open_url_modifiers) && button == GLFW_MOUSE_BUTTON_LEFT)
+        );
+    if (handle_in_kitty) {
+        switch(button) {
+            case GLFW_MOUSE_BUTTON_LEFT:
+                update_drag(true, w, is_release);
+                if (is_release) {
+                    if (modifiers == (int)OPT(open_url_modifiers)) {
+                        open_url(w);
+                    } else {
+                        if (is_release) add_click(w, button, modifiers, window_idx);
+                    }
+                }
+                break;
+            case GLFW_MOUSE_BUTTON_MIDDLE:
+                if (is_release && !modifiers) { call_boss(paste_from_selection, NULL); return; }
+                break;
+        }
+    } else {
+        size_t sz = encode_mouse_event(w, button, is_release ? RELEASE : PRESS, modifiers);
+        if (sz) schedule_write_to_child(w->id, mouse_event_buf, sz);
+    }
+}
+
+HANDLER(handle_event) {
+    switch(button) {
+        case -1:
+            for (int i = 0; i < GLFW_MOUSE_BUTTON_5; i++) { if (global_state.mouse_button_pressed[i]) { button = i; break; } }
+            handle_move_event(w, button, modifiers, window_idx);
+            break;
+        case GLFW_MOUSE_BUTTON_LEFT:  
+        case GLFW_MOUSE_BUTTON_RIGHT:
+        case GLFW_MOUSE_BUTTON_MIDDLE:
+        case GLFW_MOUSE_BUTTON_4:
+        case GLFW_MOUSE_BUTTON_5:
+            handle_button_event(w, button, modifiers, window_idx);
+            break;
+        default:
+            break;
+    }
+}
+
+static inline void 
+handle_tab_bar_mouse(int button, int UNUSED modifiers) {
+    if (button != GLFW_MOUSE_BUTTON_LEFT || !global_state.mouse_button_pressed[button]) return;
+    call_boss(activate_tab_at, "d", global_state.mouse_x);
+}
+
+static inline Window*
+window_for_event(unsigned int *window_idx, bool *in_tab_bar) {
+    *in_tab_bar = global_state.num_tabs > 1 && global_state.mouse_y >= global_state.viewport_height - global_state.cell_height;
+    if (!*in_tab_bar) {
+        Tab *t = global_state.tabs + global_state.active_tab;
+        for (unsigned int i = 0; i < t->num_windows; i++) {
+            if (contains_mouse(t->windows + i) && t->windows[i].render_data.screen) {
+                *window_idx = i; return t->windows + i;
+            }
+        }
+    }
+    return NULL;
+
+}
+
+void
+mouse_event(int button, int modifiers) {
+    MouseShape old_cursor = mouse_cursor_shape;
+    bool in_tab_bar;
+    unsigned int window_idx;
+    Window *w = window_for_event(&window_idx, &in_tab_bar);
+    if (in_tab_bar) { 
+        mouse_cursor_shape = HAND;
+        handle_tab_bar_mouse(button, modifiers); 
+    } else if(w) {
+        handle_event(w, button, modifiers, window_idx);
+    }
+    if (mouse_cursor_shape != old_cursor) {
+        set_mouse_cursor(mouse_cursor_shape);
+    }
+}
+
+void
+scroll_event(double UNUSED xoffset, double yoffset) {
+    int s = (int) round(yoffset * OPT(wheel_scroll_multiplier));
+    if (s == 0) return;
+    bool upwards = s > 0;
+    bool in_tab_bar;
+    unsigned int window_idx;
+    Window *w = window_for_event(&window_idx, &in_tab_bar);
+    if (w) {
+        Screen *screen = w->render_data.screen;
+        if (screen->linebuf == screen->main_linebuf) {
+            screen_history_scroll(screen, abs(s), upwards);
+        } else {
+            if (screen->modes.mouse_tracking_mode) {
+                size_t sz = encode_mouse_event(w, upwards ? GLFW_MOUSE_BUTTON_4 : GLFW_MOUSE_BUTTON_5, PRESS, 0);
+                if (sz) schedule_write_to_child(w->id, mouse_event_buf, sz);
+            } else {
+                call_boss(send_fake_scroll, "IiO", window_idx, abs(s), upwards ? Py_True : Py_False);
+            }
+        }
+    }
+}

kitty/mouse.py

@@ -1,62 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
-
-from .fast_data_types import (
-    GLFW_MOUSE_BUTTON_2, GLFW_MOUSE_BUTTON_3, GLFW_MOD_ALT, GLFW_MOD_CONTROL,
-    GLFW_MOD_SHIFT, GLFW_MOUSE_BUTTON_4, GLFW_MOUSE_BUTTON_5, SGR_PROTOCOL,
-    GLFW_MOUSE_BUTTON_1, URXVT_PROTOCOL, UTF8_PROTOCOL
-)
-
-PRESS, RELEASE, DRAG, MOVE = range(4)
-SHIFT_INDICATOR = 1 << 2
-ALT_INDICATOR = 1 << 3
-CONTROL_INDICATOR = 1 << 4
-MOTION_INDICATOR = 1 << 5
-EXTRA_BUTTON_INDICATOR = 1 << 6
-
-cb_map = {
-    GLFW_MOUSE_BUTTON_1: 0,
-    GLFW_MOUSE_BUTTON_2: 0b1,
-    GLFW_MOUSE_BUTTON_3: 0b10,
-    GLFW_MOUSE_BUTTON_4: EXTRA_BUTTON_INDICATOR,
-    GLFW_MOUSE_BUTTON_5: EXTRA_BUTTON_INDICATOR | 0b1
-}
-
-
-def encode_mouse_event(tracking_mode, tracking_protocol, button, action, mods, x, y):
-    x, y = x + 1, y + 1  # One based indexing
-    cb = 0
-    if action is MOVE:
-        if tracking_protocol != SGR_PROTOCOL:
-            cb = 0b11
-    else:
-        cb = cb_map.get(button)
-        if cb is None:
-            return
-    if action in (DRAG, MOVE):
-        cb |= MOTION_INDICATOR
-    elif action is RELEASE:
-        if tracking_protocol != SGR_PROTOCOL:
-            cb = 0b11
-    if mods & GLFW_MOD_SHIFT:
-        cb |= SHIFT_INDICATOR
-    if mods & GLFW_MOD_ALT:
-        cb |= ALT_INDICATOR
-    if mods & GLFW_MOD_CONTROL:
-        cb |= CONTROL_INDICATOR
-    ans = None
-    if tracking_protocol == SGR_PROTOCOL:
-        ans = '\033[<%d;%d;%d%s' % (cb, x, y, 'm' if action is RELEASE else 'M')
-        ans = ans.encode('ascii')
-    elif tracking_protocol == URXVT_PROTOCOL:
-        ans = '\033[%d;%d;%dM' % (cb + 32, x, y)
-        ans = ans.encode('ascii')
-    elif tracking_protocol == UTF8_PROTOCOL:
-        ans = bytearray([0o33, ord('['), cb + 32])
-        ans.extend(chr(x + 32).encode('utf-8') + chr(y + 32).encode('utf-8'))
-        ans = bytes(ans)
-    else:
-        if x <= 223 and y <= 223:
-            ans = bytearray([0o33, ord('['), cb + 32, x + 32, y + 32])
-    return ans

kitty/parser.c

@@ -6,22 +6,25 @@
 
 #include "data-types.h"
 #include "control-codes.h"
+#include "screen.h"
+#include "graphics.h"
+#include <time.h>
 
 // utils {{{
-static unsigned int pow10[10] = {
-    1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000
+static unsigned long pow10[] = {
+    1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000
 };
 
-static inline unsigned int 
+static inline unsigned long
 utoi(uint32_t *buf, unsigned int sz) {
-    unsigned int ans = 0;
+    unsigned long ans = 0;
     uint32_t *p = buf;
     // Ignore leading zeros
     while(sz > 0) {
         if (*p == '0') { p++; sz--; }
         else break;
     }
-    if (sz < sizeof(pow10)/sizeof(pow10[10])) {
+    if (sz < sizeof(pow10)/sizeof(pow10[0])) {
         for (int i = sz-1, j=0; i >=0; i--, j++) {
             ans += (p[i] - '0') * pow10[j];
         }
@@ -84,6 +87,7 @@ _report_params(PyObject *dump_callback, const char *name, unsigned int *params,
 
 #define GET_MACRO(_1,_2,_3,NAME,...) NAME
 #define REPORT_COMMAND(...) GET_MACRO(__VA_ARGS__, REPORT_COMMAND3, REPORT_COMMAND2, REPORT_COMMAND1, SENTINEL)(__VA_ARGS__)
+#define REPORT_VA_COMMAND(...) Py_XDECREF(PyObject_CallFunction(dump_callback, __VA_ARGS__)); PyErr_Clear();
 
 #define REPORT_DRAW(ch) \
     Py_XDECREF(PyObject_CallFunction(dump_callback, "sC", "draw", ch)); PyErr_Clear();
@@ -106,6 +110,7 @@ _report_params(PyObject *dump_callback, const char *name, unsigned int *params,
 #define REPORT_ERROR(...) fprintf(stderr, "%s ", ERROR_PREFIX); fprintf(stderr, __VA_ARGS__); fprintf(stderr, "\n");
 
 #define REPORT_COMMAND(...)
+#define REPORT_VA_COMMAND(...) 
 #define REPORT_DRAW(ch)
 #define REPORT_PARAMS(...)
 #define FLUSH_DRAW
@@ -526,6 +531,193 @@ dispatch_dcs(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
 }
 // }}}
 
+// APC mode {{{
+
+static inline void
+parse_graphics_code(Screen *screen, PyObject UNUSED *dump_callback) {
+    unsigned int pos = 1;
+    enum GR_STATES { KEY, EQUAL, UINT, INT, FLAG, AFTER_VALUE, PAYLOAD };
+    enum GR_STATES state = KEY, value_state = FLAG;
+    enum KEYS { 
+        action='a', 
+        delete_action='d',
+        transmission_type='t',
+        compressed='o',
+        format = 'f',
+        more = 'm',
+        id = 'i',
+        width = 'w', 
+        height = 'h',
+        x_offset = 'x',
+        y_offset = 'y',
+        data_height = 'v', 
+        data_width = 's',
+        data_sz = 'S',
+        data_offset = 'O',
+        num_cells = 'c',
+        num_lines = 'r',
+        cell_x_offset = 'X',
+        cell_y_offset = 'Y',
+        z_index = 'z'  
+    };
+    enum KEYS key = 'a';
+    static GraphicsCommand g;
+    unsigned int i, code;
+    unsigned long lcode;
+    bool is_negative;
+    memset(&g, 0, sizeof(g));
+    static uint8_t payload[4096];
+    size_t sz;
+    const char *err;
+
+    while (pos < screen->parser_buf_pos) {
+        switch(state) {
+
+            case KEY:
+                key = screen->parser_buf[pos++];
+                switch(key) {
+#define KS(n, vs) case n: state = EQUAL; value_state = vs; break
+#define U(x) KS(x, UINT)
+                    KS(action, FLAG); KS(delete_action, FLAG); KS(transmission_type, FLAG); KS(compressed, FLAG); KS(z_index, INT);
+                    U(format); U(more); U(id); U(data_sz); U(data_offset); U(width); U(height); U(x_offset); U(y_offset); U(data_height); U(data_width); U(num_cells); U(num_lines); U(cell_x_offset); U(cell_y_offset);
+#undef U
+#undef KS
+                    default:
+                        REPORT_ERROR("Malformed graphics control block, invalid key character: 0x%x", key);
+                        return;
+                }
+                break;
+
+            case EQUAL:
+                if (screen->parser_buf[pos++] != '=') {
+                    REPORT_ERROR("Malformed graphics control block, no = after key, found: 0x%x instead", screen->parser_buf[pos-1]);
+                    return;
+                }
+                state = value_state;
+                break;
+
+            case FLAG:
+                switch(key) {
+#define F(a) case a: g.a = screen->parser_buf[pos++] & 0xff; break
+                    F(action); F(delete_action); F(transmission_type); F(compressed);
+                    default:
+                        break;
+                }
+                state = AFTER_VALUE;
+                break;
+#undef F
+
+            case INT:
+#define READ_UINT \
+                for (i = pos; i < MIN(screen->parser_buf_pos, pos + 10); i++) { \
+                    if (screen->parser_buf[i] < '0' || screen->parser_buf[i] > '9') break; \
+                } \
+                if (i == pos) { REPORT_ERROR("Malformed graphics control block, expecting an integer value for key: %c", key & 0xFF); return; } \
+                lcode = utoi(screen->parser_buf + pos, i - pos); pos = i; \
+                if (lcode > UINT32_MAX) { REPORT_ERROR("id is too large"); return; } \
+                code = lcode;
+
+                is_negative = false;
+                if(screen->parser_buf[pos] == '-') { is_negative = true; pos++; }
+#define U(x) case x: g.x = is_negative ? 0 - (int32_t)code : (int32_t)code; break
+                READ_UINT; 
+                switch(key) { 
+                    U(z_index);
+                    default: break;
+                }
+                state = AFTER_VALUE;
+                break;
+#undef U
+            case UINT:
+                READ_UINT;
+#define U(x) case x: g.x = code; break
+                switch(key) { 
+                    U(format); U(more); U(id); U(data_sz); U(data_offset); U(width); U(height); U(x_offset); U(y_offset); U(data_height); U(data_width); U(num_cells); U(num_lines); U(cell_x_offset); U(cell_y_offset);
+                    default: break; 
+                }
+                state = AFTER_VALUE;
+                break;
+#undef U
+#undef SET_ATTR
+#undef READ_UINT
+            case AFTER_VALUE:
+                switch (screen->parser_buf[pos++]) {
+                    case ',':
+                        state = KEY;
+                        break;
+                    case ';':
+                        state = PAYLOAD;
+                        break;
+                    default:
+                        REPORT_ERROR("Malformed graphics control block, expecting a comma or semi-colon after a value, found: 0x%x", screen->parser_buf[pos - 1]); 
+                        return;
+                }
+                break;
+
+            case PAYLOAD:
+                sz = screen->parser_buf_pos - pos;
+                err = base64_decode(screen->parser_buf + pos, sz, payload, sizeof(payload), &g.payload_sz);
+                if (err != NULL) { REPORT_ERROR("Failed to parse graphics command payload with error: %s", err); return; }
+                pos = screen->parser_buf_pos;
+                break;  
+        }
+    }
+    switch(state) {
+        case EQUAL:
+            REPORT_ERROR("Malformed graphics control block, no = after key"); return;
+        case INT:
+        case UINT:
+            REPORT_ERROR("Malformed graphics control block, expecting an integer value"); return; 
+        case FLAG:
+            REPORT_ERROR("Malformed graphics control block, expecting a flag value"); return; 
+        default:
+            break;
+    }
+#define A(x) #x, g.x
+#define U(x) #x, (unsigned int)(g.x)
+#define I(x) #x, (int)(g.x)
+    REPORT_VA_COMMAND("s {sc sc sc sc sI sI sI sI sI  sI sI sI sI sI sI sI sI sI sI  sI si} y#", "graphics_command", 
+            A(action), A(delete_action), A(transmission_type), A(compressed),
+            U(format), U(more), U(id), U(data_sz), U(data_offset),
+            U(width), U(height), U(x_offset), U(y_offset), U(data_height), U(data_width), U(num_cells), U(num_lines), U(cell_x_offset), U(cell_y_offset),
+            U(payload_sz), I(z_index),
+            payload, g.payload_sz
+    );
+#undef U
+#undef A
+#undef I
+    screen_handle_graphics_command(screen, &g, payload);
+}
+
+static inline void
+dispatch_apc(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
+    if (screen->parser_buf_pos < 2) return;
+    switch(screen->parser_buf[0]) {
+        case 'G':
+            parse_graphics_code(screen, dump_callback);
+            break;
+        default:
+            REPORT_ERROR("Unrecognized APC code: 0x%x", screen->parser_buf[0]);
+            break;
+    }
+}
+
+// }}}
+
+// PM mode {{{
+static inline void
+dispatch_pm(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
+    if (screen->parser_buf_pos < 2) return;
+    switch(screen->parser_buf[0]) {
+        default:
+            REPORT_ERROR("Unrecognized PM code: 0x%x", screen->parser_buf[0]);
+            break;
+    }
+}
+
+
+// }}}
+
 // Parse loop {{{
 
 static inline bool
@@ -687,10 +879,10 @@ dispatch_unicode_char(Screen *screen, uint32_t codepoint, PyObject DUMP_UNUSED *
             if (accumulate_osc(screen, codepoint, dump_callback)) { dispatch_osc(screen, dump_callback); SET_STATE(0); }
             break;
         case APC:
-            if (accumulate_oth(screen, codepoint, dump_callback)) { SET_STATE(0); }
+            if (accumulate_oth(screen, codepoint, dump_callback)) { dispatch_apc(screen, dump_callback); SET_STATE(0); }
             break;
         case PM:
-            if (accumulate_oth(screen, codepoint, dump_callback)) { SET_STATE(0); }
+            if (accumulate_oth(screen, codepoint, dump_callback)) { dispatch_pm(screen, dump_callback); SET_STATE(0); }
             break;
         case DCS:
             if (accumulate_dcs(screen, codepoint, dump_callback)) { dispatch_dcs(screen, dump_callback); SET_STATE(0); }
@@ -724,7 +916,7 @@ FLUSH_DRAW;
 }
 // }}}
 
-// Boilerplate {{{
+// Entry points {{{
 #ifdef DUMP_COMMANDS
 #define FNAME(x) x##_dump
 #else
@@ -745,38 +937,13 @@ FNAME(parse_bytes)(PyObject UNUSED *self, PyObject *args) {
     Py_RETURN_NONE;
 }
 
-PyObject*
-FNAME(read_bytes)(PyObject UNUSED *self, PyObject *args) {
-    PyObject *dump_callback = NULL;
-    Py_ssize_t len;
-    Screen *screen;
-    int fd;
-#ifdef DUMP_COMMANDS
-    if (!PyArg_ParseTuple(args, "OOi", &dump_callback, &screen, &fd)) return NULL;
-#else
-    if (!PyArg_ParseTuple(args, "Oi", &screen, &fd)) return NULL;
-#endif
 
-    while(true) {
-        Py_BEGIN_ALLOW_THREADS;
-        len = read(fd, screen->read_buf, READ_BUF_SZ);
-        Py_END_ALLOW_THREADS;
-        if (len == -1) {
-            if (errno == EINTR) continue;
-            if (errno == EIO) { Py_RETURN_FALSE; }
-            return PyErr_SetFromErrno(PyExc_OSError);
-        }
-        /* PyObject_Print(Py_BuildValue("y#", screen->read_buf, len), stderr, 0); */
-        break;
-    }
+void
+FNAME(parse_worker)(Screen *screen, PyObject *dump_callback) {
 #ifdef DUMP_COMMANDS
-    if (len > 0) {
-        Py_XDECREF(PyObject_CallFunction(dump_callback, "sy#", "bytes", screen->read_buf, len)); PyErr_Clear();
-    }
+    Py_XDECREF(PyObject_CallFunction(dump_callback, "sy#", "bytes", screen->read_buf, screen->read_buf_sz)); PyErr_Clear();
 #endif
-    _parse_bytes(screen, screen->read_buf, len, dump_callback);
-    if(len > 0) { Py_RETURN_TRUE; }
-    Py_RETURN_FALSE;
-}
+    _parse_bytes(screen, screen->read_buf, screen->read_buf_sz, dump_callback);
 #undef FNAME
+}
 // }}}

kitty/rewrap.h

@@ -12,11 +12,11 @@
 #endif
 
 #ifndef init_src_line
-#define init_src_line(src_y) INIT_LINE(src, src->line, src->line_map[src_y]);
+#define init_src_line(src_y) init_line(src, src->line, src->line_map[src_y]);
 #endif
 
 #ifndef init_dest_line
-#define init_dest_line(dest_y) INIT_LINE(dest, dest->line, dest->line_map[dest_y]);
+#define init_dest_line(dest_y) init_line(dest, dest->line, dest->line_map[dest_y]);
 #endif
 
 #ifndef first_dest_line
@@ -28,7 +28,7 @@
     if (dest_y >= dest->ynum - 1) { \
         linebuf_index(dest, 0, dest->ynum - 1); \
         if (historybuf != NULL) { \
-            linebuf_init_line(dest, dest->ynum - 1); \
+            init_dest_line(dest->ynum - 1); \
             historybuf_add_line(historybuf, dest->line); \
         }\
         linebuf_clear_line(dest, dest->ynum - 1); \
@@ -41,19 +41,16 @@
 #define is_src_line_continued(src_y) (src_y < src->ynum - 1 ? src->continued_map[src_y + 1] : false)
 #endif
 
-static inline void copy_range(Line *src, index_type src_at, Line* dest, index_type dest_at, index_type num) {
-    memcpy(dest->chars + dest_at, src->chars + src_at, num * sizeof(char_type));
-    memcpy(dest->fg_colors + dest_at, src->fg_colors + src_at, num * sizeof(color_type));
-    memcpy(dest->bg_colors + dest_at, src->bg_colors + src_at, num * sizeof(color_type));
-    memcpy(dest->decoration_fg + dest_at, src->decoration_fg + src_at, num * sizeof(color_type));
-    memcpy(dest->combining_chars + dest_at, src->combining_chars + src_at, num * sizeof(combining_type));
+static inline void 
+copy_range(Line *src, index_type src_at, Line* dest, index_type dest_at, index_type num) {
+    memcpy(dest->cells + dest_at, src->cells + src_at, num * sizeof(Cell));
 }
 
 
-static void rewrap_inner(BufType *src, BufType *dest, const index_type src_limit, HistoryBuf UNUSED *historybuf) {
+static void 
+rewrap_inner(BufType *src, BufType *dest, const index_type src_limit, HistoryBuf UNUSED *historybuf) {
     bool src_line_is_continued = false;
     index_type src_y = 0, src_x = 0, dest_x = 0, dest_y = 0, num = 0, src_x_limit = 0;
-    Py_BEGIN_ALLOW_THREADS;
 
     first_dest_line;
     do {
@@ -61,8 +58,8 @@ static void rewrap_inner(BufType *src, BufType *dest, const index_type src_limit
         src_line_is_continued = is_src_line_continued(src_y);
         src_x_limit = src->xnum;
         if (!src_line_is_continued) {
-            // Trim trailing white-space since there is a hard line break at the end of this line
-            while(src_x_limit && (src->line->chars[src_x_limit - 1] & CHAR_MASK) == 32) src_x_limit--;
+            // Trim trailing blanks since there is a hard line break at the end of this line
+            while(src_x_limit && (src->line->cells[src_x_limit - 1].ch) == BLANK_CHAR) src_x_limit--;
             
         }
         while (src_x < src_x_limit) {
@@ -75,5 +72,4 @@ static void rewrap_inner(BufType *src, BufType *dest, const index_type src_limit
         if (!src_line_is_continued && src_y < src_limit) { next_dest_line(false); dest_x = 0; }
     } while (src_y < src_limit);
     dest->line->ynum = dest_y;
-    Py_END_ALLOW_THREADS;
 }

kitty/screen.h

@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#pragma once
+
+#include "graphics.h"
+
+typedef enum ScrollTypes { SCROLL_LINE = -999999, SCROLL_PAGE, SCROLL_FULL } ScrollType;
+
+typedef struct {
+    bool mLNM, mIRM, mDECTCEM, mDECSCNM, mDECOM, mDECAWM, mDECCOLM, mDECARM, mDECCKM,
+         mBRACKETED_PASTE, mFOCUS_TRACKING, mEXTENDED_KEYBOARD;
+    MouseTrackingMode mouse_tracking_mode;
+    MouseTrackingProtocol mouse_tracking_protocol;
+} ScreenModes;
+
+typedef struct {
+    unsigned int x, y;
+} SelectionBoundary;
+
+typedef struct {
+    unsigned int start_x, start_y, start_scrolled_by, end_x, end_y, end_scrolled_by;
+    bool in_progress;
+} Selection;
+    
+typedef struct {
+    PyObject_HEAD
+
+    unsigned int columns, lines, margin_top, margin_bottom, charset, scrolled_by, last_selection_scrolled_by, window_id;
+    uint32_t utf8_state, utf8_codepoint, *g0_charset, *g1_charset, *g_charset;
+    Selection selection;
+    SelectionBoundary last_rendered_selection_start, last_rendered_selection_end;
+    Selection url_range;
+    bool use_latin1, selection_updated_once, is_dirty, scroll_changed;
+    Cursor *cursor;
+    SavepointBuffer main_savepoints, alt_savepoints;
+    PyObject *callbacks, *test_child;
+    LineBuf *linebuf, *main_linebuf, *alt_linebuf;
+    GraphicsManager *grman, *main_grman, *alt_grman;
+    HistoryBuf *historybuf;
+    unsigned int history_line_added_count;
+    bool *tabstops, *main_tabstops, *alt_tabstops;
+    ScreenModes modes;
+    ColorProfile *color_profile;
+    double start_visual_bell_at;
+
+    uint32_t parser_buf[PARSER_BUF_SZ];
+    unsigned int parser_state, parser_text_start, parser_buf_pos;
+    bool parser_has_pending_text;
+    uint8_t read_buf[READ_BUF_SZ], *write_buf;
+    double new_input_at;
+    size_t read_buf_sz, write_buf_sz, write_buf_used;
+    pthread_mutex_t read_buf_lock, write_buf_lock;
+
+} Screen;
+PyTypeObject Screen_Type;
+
+
+void parse_worker(Screen *screen, PyObject *dump_callback);
+void parse_worker_dump(Screen *screen, PyObject *dump_callback);
+void screen_align(Screen*);
+void screen_restore_cursor(Screen *);
+void screen_save_cursor(Screen *);
+void screen_cursor_position(Screen*, unsigned int, unsigned int);
+void screen_cursor_back(Screen *self, unsigned int count/*=1*/, int move_direction/*=-1*/);
+void screen_erase_in_line(Screen *, unsigned int, bool);
+void screen_erase_in_display(Screen *, unsigned int, bool);
+void screen_draw(Screen *screen, uint32_t codepoint);
+void screen_ensure_bounds(Screen *self, bool use_margins);
+void screen_toggle_screen_buffer(Screen *self);
+void screen_normal_keypad_mode(Screen *self); 
+void screen_alternate_keypad_mode(Screen *self);  
+void screen_change_default_color(Screen *self, unsigned int which, uint32_t col);
+void screen_alignment_display(Screen *self);
+void screen_reverse_index(Screen *self);
+void screen_index(Screen *self);
+void screen_scroll(Screen *self, unsigned int count);
+void screen_reverse_scroll(Screen *self, unsigned int count);
+void screen_reset(Screen *self);
+void screen_set_tab_stop(Screen *self);
+void screen_tab(Screen *self);
+void screen_backtab(Screen *self, unsigned int);
+void screen_clear_tab_stop(Screen *self, unsigned int how);
+void screen_set_mode(Screen *self, unsigned int mode);
+void screen_reset_mode(Screen *self, unsigned int mode);
+void screen_insert_characters(Screen *self, unsigned int count);
+void screen_cursor_up(Screen *self, unsigned int count/*=1*/, bool do_carriage_return/*=false*/, int move_direction/*=-1*/);
+void screen_set_cursor(Screen *self, unsigned int mode, uint8_t secondary);
+void screen_cursor_to_column(Screen *self, unsigned int column);
+void screen_cursor_down(Screen *self, unsigned int count/*=1*/);
+void screen_cursor_forward(Screen *self, unsigned int count/*=1*/);
+void screen_cursor_down1(Screen *self, unsigned int count/*=1*/);
+void screen_cursor_up1(Screen *self, unsigned int count/*=1*/);
+void screen_cursor_to_line(Screen *screen, unsigned int line);
+void screen_insert_lines(Screen *self, unsigned int count/*=1*/);
+void screen_delete_lines(Screen *self, unsigned int count/*=1*/);
+void screen_delete_characters(Screen *self, unsigned int count);
+void screen_erase_characters(Screen *self, unsigned int count);
+void screen_set_margins(Screen *self, unsigned int top, unsigned int bottom);
+void screen_change_charset(Screen *, uint32_t to);
+void screen_designate_charset(Screen *, uint32_t which, uint32_t as);
+void screen_use_latin1(Screen *, bool);
+void set_title(Screen *self, PyObject*);
+void set_icon(Screen *self, PyObject*);
+void set_dynamic_color(Screen *self, unsigned int code, PyObject*);
+void set_color_table_color(Screen *self, unsigned int code, PyObject*);
+uint32_t* translation_table(uint32_t which);
+uint32_t *latin1_charset;
+void screen_request_capabilities(Screen *, PyObject *);
+void report_device_attributes(Screen *self, unsigned int UNUSED mode, char start_modifier);
+void select_graphic_rendition(Screen *self, unsigned int *params, unsigned int count);
+void report_device_status(Screen *self, unsigned int which, bool UNUSED);
+void report_mode_status(Screen *self, unsigned int which, bool);
+void screen_apply_selection(Screen *self, void *address, size_t size);
+bool screen_is_selection_dirty(Screen *self);
+bool screen_invert_colors(Screen *self);
+void screen_update_cell_data(Screen *self, void *address, size_t sz);
+bool screen_is_cursor_visible(Screen *self);
+bool screen_selection_range_for_line(Screen *self, index_type y, index_type *start, index_type *end);
+bool screen_selection_range_for_word(Screen *self, index_type x, index_type y, index_type *start, index_type *end);
+void screen_start_selection(Screen *self, index_type x, index_type y);
+void screen_update_selection(Screen *self, index_type x, index_type y, bool ended);
+bool screen_history_scroll(Screen *self, int amt, bool upwards);
+Line* screen_visual_line(Screen *self, index_type y);
+unsigned long screen_current_char_width(Screen *self);
+void screen_url_range(Screen *self, uint32_t *);
+void screen_mark_url(Screen *self, index_type start_x, index_type start_y, index_type end_x, index_type end_y);
+void screen_handle_graphics_command(Screen *self, const GraphicsCommand *cmd, const uint8_t *payload);
+#define DECLARE_CH_SCREEN_HANDLER(name) void screen_##name(Screen *screen);
+DECLARE_CH_SCREEN_HANDLER(bell)
+DECLARE_CH_SCREEN_HANDLER(backspace)
+DECLARE_CH_SCREEN_HANDLER(tab)
+DECLARE_CH_SCREEN_HANDLER(linefeed)
+DECLARE_CH_SCREEN_HANDLER(carriage_return)

kitty/shaders.c

@@ -0,0 +1,669 @@
+/*
+ * shaders.c
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "gl.h"
+#include <sys/sysctl.h>
+
+enum { CELL_PROGRAM, CELL_BACKGROUND_PROGRAM, CELL_SPECIAL_PROGRAM, CELL_FOREGROUND_PROGRAM, CURSOR_PROGRAM, BORDERS_PROGRAM, GRAPHICS_PROGRAM, NUM_PROGRAMS };
+enum { SPRITE_MAP_UNIT, GRAPHICS_UNIT };
+
+// Sprites {{{
+typedef struct {
+    int xnum, ynum, x, y, z, last_num_of_layers, last_ynum;
+    unsigned int cell_width, cell_height;
+    GLuint texture_id;
+    GLenum texture_unit;
+    GLint max_texture_size, max_array_texture_layers;
+    PyObject *render_cell;
+} SpriteMap;
+
+static SpriteMap sprite_map = { .xnum = 1, .ynum = 1, .last_num_of_layers = 1, .last_ynum = -1, .texture_unit = GL_TEXTURE0 };
+
+#ifdef __APPLE__ 
+#define glCopyImageSubData(...)
+#define GLEW_ARB_copy_image false
+#endif
+
+static bool copy_image_warned = false;
+
+static void
+copy_image_sub_data(GLuint src_texture_id, GLuint dest_texture_id, unsigned int width, unsigned int height, unsigned int num_levels) {
+    if (!GLEW_ARB_copy_image) {
+        // ARB_copy_image not available, do a slow roundtrip copy
+        if (!copy_image_warned) {
+            copy_image_warned = true;
+            fprintf(stderr, "WARNING: Your system's OpenGL implementation does not have glCopyImageSubData, falling back to a slower implementation.\n");
+        }
+        uint8_t *src = malloc(5 * width * height * num_levels);
+        if (src == NULL) { fatal("Out of memory."); }
+        uint8_t *dest = src + (4 * width * height * num_levels);
+        glBindTexture(GL_TEXTURE_2D_ARRAY, src_texture_id); check_gl();
+        glGetTexImage(GL_TEXTURE_2D_ARRAY, 0, GL_RGBA, GL_UNSIGNED_BYTE, src); check_gl();
+        glBindTexture(GL_TEXTURE_2D_ARRAY, dest_texture_id); check_gl();
+        glPixelStorei(GL_UNPACK_ALIGNMENT, 1); check_gl();
+        for(size_t i = 0; i < width * height * num_levels; i++) dest[i] = src[4*i];
+        glTexSubImage3D(GL_TEXTURE_2D_ARRAY, 0, 0, 0, 0, width, height, num_levels, GL_RED, GL_UNSIGNED_BYTE, dest); check_gl();
+        free(src);
+    } else {
+        glCopyImageSubData(src_texture_id, GL_TEXTURE_2D_ARRAY, 0, 0, 0, 0, dest_texture_id, GL_TEXTURE_2D_ARRAY, 0, 0, 0, 0, width, height, num_levels); check_gl();
+    }
+}
+
+
+static void
+realloc_sprite_texture() {
+    GLuint tex;
+    glGenTextures(1, &tex); check_gl();
+    glBindTexture(GL_TEXTURE_2D_ARRAY, tex); check_gl();
+    // We use GL_NEAREST otherwise glyphs that touch the edge of the cell
+    // often show a border between cells
+    glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+    glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+    glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); check_gl();
+    unsigned int xnum, ynum, z, znum, width, height, src_ynum;
+    sprite_map_current_layout(&xnum, &ynum, &z);
+    znum = z + 1;
+    width = xnum * sprite_map.cell_width; height = ynum * sprite_map.cell_height;
+    glTexStorage3D(GL_TEXTURE_2D_ARRAY, 1, GL_R8, width, height, znum); check_gl();
+    if (sprite_map.texture_id) {
+        // need to re-alloc
+        src_ynum = MAX(1, sprite_map.last_ynum);
+        copy_image_sub_data(sprite_map.texture_id, tex, width, src_ynum * sprite_map.cell_height, sprite_map.last_num_of_layers);
+        glDeleteTextures(1, &sprite_map.texture_id); check_gl();
+    }
+    glBindTexture(GL_TEXTURE_2D_ARRAY, 0);
+    sprite_map.last_num_of_layers = znum;
+    sprite_map.last_ynum = ynum;
+    sprite_map.texture_id = tex;
+}
+
+static inline PyObject*
+render_cell(PyObject *text, bool bold, bool italic, unsigned int underline, bool strikethrough, bool is_second) {
+#define B(x) (x ? Py_True : Py_False)
+    PyObject *ret = PyObject_CallFunction(sprite_map.render_cell, "OOOIOO", text, B(bold), B(italic), underline, B(strikethrough), B(is_second));
+    if (ret == NULL) { PyErr_Print(); fatal("Rendering of a cell failed, aborting"); }
+    return ret;
+#undef B
+}
+
+static inline void
+ensure_sprite_map() {
+    static GLuint bound_texture_id = 0;
+    if (!sprite_map.texture_id) realloc_sprite_texture();
+    if (bound_texture_id != sprite_map.texture_id) {
+        glActiveTexture(GL_TEXTURE0 + SPRITE_MAP_UNIT); check_gl();
+        glBindTexture(GL_TEXTURE_2D_ARRAY, sprite_map.texture_id); check_gl();
+        bound_texture_id = sprite_map.texture_id;
+    }
+}
+
+static void 
+sprite_send_to_gpu(unsigned int x, unsigned int y, unsigned int z, PyObject *buf) {
+    unsigned int xnum, ynum, znum;
+    sprite_map_current_layout(&xnum, &ynum, &znum);
+    if ((int)znum >= sprite_map.last_num_of_layers || (znum == 0 && (int)ynum > sprite_map.last_ynum)) realloc_sprite_texture();
+    glBindTexture(GL_TEXTURE_2D_ARRAY, sprite_map.texture_id); check_gl();
+    glPixelStorei(GL_UNPACK_ALIGNMENT, 1); check_gl();
+    x *= sprite_map.cell_width; y *= sprite_map.cell_height;
+    PyObject *ret = PyObject_CallObject(buf, NULL);
+    if (ret == NULL) { PyErr_Print(); fatal("Failed to get address of rendered cell buffer"); }
+    void *address = PyLong_AsVoidPtr(ret);
+    Py_DECREF(ret);
+    glTexSubImage3D(GL_TEXTURE_2D_ARRAY, 0, x, y, z, sprite_map.cell_width, sprite_map.cell_height, 1, GL_RED, GL_UNSIGNED_BYTE, address); check_gl();
+    Py_DECREF(buf);
+}
+
+static void 
+render_and_send_dirty_sprites(PyObject *text, bool bold, bool italic, bool is_second, sprite_index x, sprite_index y, sprite_index z) {
+    if (text == NULL) { fatal("The text for a sprite was NULL, probably out of memory."); }
+    PyObject *buf = render_cell(text, bold, italic, false, false, is_second);
+    sprite_send_to_gpu(x, y, z, buf);
+}
+
+static inline sprite_index
+send_prerendered(unsigned int underline, bool strikethrough) {
+    sprite_index x, y, z;
+    PyObject *blank = PyUnicode_FromString(" ");
+    if (blank == NULL) { fatal("Out of memory"); }
+    PyObject *buf = render_cell(blank, false, false, underline, strikethrough, false);
+    Py_CLEAR(blank);
+    if (sprite_map_increment(&x, &y, &z) != 0) { fatal("Failed to increment sprite map for prerendering"); }
+    sprite_send_to_gpu(x, y, z, buf);
+    return x;
+}
+
+static void 
+layout_sprite_map(unsigned int cell_width, unsigned int cell_height, PyObject *render_cell) {
+    sprite_map.cell_width = MAX(1, cell_width);
+    sprite_map.cell_height = MAX(1, cell_height);
+    global_state.cell_width = sprite_map.cell_width;
+    global_state.cell_height = sprite_map.cell_height;
+    if (sprite_map.max_texture_size == 0) {
+        glGetIntegerv(GL_MAX_TEXTURE_SIZE, &(sprite_map.max_texture_size)); check_gl();
+        glGetIntegerv(GL_MAX_ARRAY_TEXTURE_LAYERS, &(sprite_map.max_array_texture_layers)); check_gl();
+        sprite_map_set_limits(sprite_map.max_texture_size, sprite_map.max_array_texture_layers);
+    }
+    sprite_map_set_layout(sprite_map.cell_width, sprite_map.cell_height);
+    Py_CLEAR(sprite_map.render_cell);
+    sprite_map.render_cell = render_cell; Py_INCREF(sprite_map.render_cell);
+    if (sprite_map.texture_id) { glDeleteTextures(1, &(sprite_map.texture_id)); sprite_map.texture_id = 0; }
+    realloc_sprite_texture();
+    // Pre-render the basic cells to ensure they have known sprite numbers
+    send_prerendered(0, false);
+    send_prerendered(1, false);
+    send_prerendered(2, false);
+    if (send_prerendered(0, true) != 3) { fatal("Available OpenGL texture size is too small"); }
+}
+
+static void
+destroy_sprite_map() {
+    sprite_map_free();
+    Py_CLEAR(sprite_map.render_cell);
+    if (sprite_map.texture_id) {
+        glDeleteTextures(1, &(sprite_map.texture_id));
+        check_gl();
+        sprite_map.texture_id = 0;
+    }
+}
+
+// }}}
+
+// Cell {{{
+
+typedef struct {
+    UniformBlock render_data;
+    ArrayInformation color_table;
+} CellProgramLayout;
+
+static CellProgramLayout cell_program_layouts[NUM_PROGRAMS];
+
+static void
+init_cell_program() {
+    for (int i = CELL_PROGRAM; i <= CELL_FOREGROUND_PROGRAM; i++) {
+        cell_program_layouts[i].render_data.index = block_index(i, "CellRenderData");
+        cell_program_layouts[i].render_data.size = block_size(i, cell_program_layouts[i].render_data.index);
+        cell_program_layouts[i].color_table.size = get_uniform_information(i, "color_table[0]", GL_UNIFORM_SIZE);
+        cell_program_layouts[i].color_table.offset = get_uniform_information(i, "color_table[0]", GL_UNIFORM_OFFSET);
+        cell_program_layouts[i].color_table.stride = get_uniform_information(i, "color_table[0]", GL_UNIFORM_ARRAY_STRIDE);
+    }
+    // Sanity check to ensure the attribute location binding worked
+#define C(p, name, expected) { int aloc = attrib_location(p, #name); if (aloc != expected && aloc != -1) fatal("The attribute location for %s is %d != %d in program: %d", #name, aloc, expected, p); }
+    for (int p = CELL_PROGRAM; p <= CELL_FOREGROUND_PROGRAM; p++) {
+        C(p, colors, 0); C(p, sprite_coords, 1); C(p, is_selected, 2);
+    }
+#undef C
+
+}
+
+#define CELL_BUFFERS enum { cell_data_buffer, selection_buffer, uniform_buffer };
+
+static ssize_t
+create_cell_vao() {
+    ssize_t vao_idx = create_vao();
+#define A(name, size, dtype, offset, stride) \
+    add_attribute_to_vao(CELL_PROGRAM, vao_idx, #name, \
+            /*size=*/size, /*dtype=*/dtype, /*stride=*/stride, /*offset=*/offset, /*divisor=*/1);
+#define A1(name, size, dtype, offset) A(name, size, dtype, (void*)(offsetof(Cell, offset)), sizeof(Cell))
+
+    add_buffer_to_vao(vao_idx, GL_ARRAY_BUFFER);
+    A1(sprite_coords, 4, GL_UNSIGNED_SHORT, sprite_x);
+    A1(colors, 3, GL_UNSIGNED_INT, fg);
+
+    add_buffer_to_vao(vao_idx, GL_ARRAY_BUFFER);
+    A(is_selected, 1, GL_FLOAT, NULL, 0);
+
+    size_t bufnum = add_buffer_to_vao(vao_idx, GL_UNIFORM_BUFFER);
+    alloc_vao_buffer(vao_idx, cell_program_layouts[CELL_PROGRAM].render_data.size, bufnum, GL_STREAM_DRAW);
+
+    return vao_idx;
+#undef A
+#undef A1
+}
+
+static ssize_t
+create_graphics_vao() {
+    ssize_t vao_idx = create_vao();
+    add_buffer_to_vao(vao_idx, GL_ARRAY_BUFFER);
+    add_attribute_to_vao(GRAPHICS_PROGRAM, vao_idx, "src", 4, GL_FLOAT, 0, NULL, 0);
+    return vao_idx;
+}
+
+static inline void
+cell_update_uniform_block(ssize_t vao_idx, Screen *screen, int uniform_buffer, GLfloat xstart, GLfloat ystart, GLfloat dx, GLfloat dy, CursorRenderInfo *cursor) {
+    struct CellRenderData {
+        GLfloat xstart, ystart, dx, dy, sprite_dx, sprite_dy;
+
+        GLuint default_fg, default_bg, highlight_fg, highlight_bg, cursor_color, url_color;
+
+        GLint color1, color2;
+
+        GLuint xnum, ynum, cursor_x, cursor_y, cursor_w, url_xl, url_yl, url_xr, url_yr;
+    };
+    static struct CellRenderData *rd;
+
+    bool inverted = screen_invert_colors(screen);
+    // Send the uniform data
+    rd = (struct CellRenderData*)map_vao_buffer(vao_idx, uniform_buffer, GL_WRITE_ONLY);
+    if (UNLIKELY(screen->color_profile->dirty)) {
+        copy_color_table_to_buffer(screen->color_profile, (GLuint*)rd, cell_program_layouts[CELL_PROGRAM].color_table.offset / sizeof(GLuint), cell_program_layouts[CELL_PROGRAM].color_table.stride / sizeof(GLuint));
+    }
+    // Cursor position
+    if (cursor->is_visible && cursor->shape == CURSOR_BLOCK) { 
+        rd->cursor_x = screen->cursor->x, rd->cursor_y = screen->cursor->y; 
+    } else {
+        rd->cursor_x = screen->columns, rd->cursor_y = screen->lines; 
+    }
+    rd->cursor_w = rd->cursor_x + MAX(1, screen_current_char_width(screen)) - 1;
+
+    rd->xnum = screen->columns; rd->ynum = screen->lines;
+    screen_url_range(screen, &rd->url_xl);
+    
+    rd->xstart = xstart; rd->ystart = ystart; rd->dx = dx; rd->dy = dy;
+    unsigned int x, y, z;
+    sprite_map_current_layout(&x, &y, &z);
+    rd->sprite_dx = 1.0f / (float)x; rd->sprite_dy = 1.0f / (float)y;
+    rd->color1 = inverted & 1; rd->color2 = 1 - (inverted & 1);
+
+#define COLOR(name) colorprofile_to_color(screen->color_profile, screen->color_profile->overridden.name, screen->color_profile->configured.name)
+    rd->default_fg = COLOR(default_fg); rd->default_bg = COLOR(default_bg); rd->highlight_fg = COLOR(highlight_fg); rd->highlight_bg = COLOR(highlight_bg);
+#undef COLOR
+    rd->cursor_color = cursor->color; rd->url_color = OPT(url_color);
+
+    unmap_vao_buffer(vao_idx, uniform_buffer); rd = NULL;
+}
+
+static inline void
+cell_prepare_to_render(ssize_t vao_idx, ssize_t gvao_idx, Screen *screen, GLfloat xstart, GLfloat ystart, GLfloat dx, GLfloat dy, CursorRenderInfo *cursor) {
+    size_t sz;
+    CELL_BUFFERS;
+    void *address;
+    if (screen->scroll_changed || screen->is_dirty) {
+        sz = sizeof(Cell) * screen->lines * screen->columns;
+        address = alloc_and_map_vao_buffer(vao_idx, sz, cell_data_buffer, GL_STREAM_DRAW, GL_WRITE_ONLY);
+        screen_update_cell_data(screen, address, sz);
+        unmap_vao_buffer(vao_idx, cell_data_buffer); address = NULL;
+    }
+
+    if (screen_is_selection_dirty(screen)) {
+        sz = sizeof(GLfloat) * screen->lines * screen->columns;
+        address = alloc_and_map_vao_buffer(vao_idx, sz, selection_buffer, GL_STREAM_DRAW, GL_WRITE_ONLY);
+        screen_apply_selection(screen, address, sz);
+        unmap_vao_buffer(vao_idx, selection_buffer); address = NULL;
+    }
+
+    if (gvao_idx && grman_update_layers(screen->grman, screen->scrolled_by, xstart, ystart, dx, dy, screen->columns, screen->lines)) {
+        sz = sizeof(GLfloat) * 16 * screen->grman->count;
+        GLfloat *a = alloc_and_map_vao_buffer(gvao_idx, sz, 0, GL_STREAM_DRAW, GL_WRITE_ONLY);
+        for (size_t i = 0; i < screen->grman->count; i++, a += 16) memcpy(a, screen->grman->render_data[i].vertices, sizeof(screen->grman->render_data[0].vertices));
+        unmap_vao_buffer(gvao_idx, 0); a = NULL;
+    }
+
+    cell_update_uniform_block(vao_idx, screen, uniform_buffer, xstart, ystart, dx, dy, cursor);
+
+    ensure_sprite_map();
+    render_dirty_sprites(render_and_send_dirty_sprites);
+
+    bind_vao_uniform_buffer(vao_idx, uniform_buffer, cell_program_layouts[CELL_PROGRAM].render_data.index);
+    bind_vertex_array(vao_idx);
+}
+
+static void
+draw_graphics(ssize_t vao_idx, ssize_t gvao_idx, ImageRenderData *data, GLuint start, GLuint count) {
+    bind_vertex_array(gvao_idx);
+    bind_program(GRAPHICS_PROGRAM);
+    glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+    static bool graphics_constants_set = false;
+    if (!graphics_constants_set) { 
+        glUniform1i(glGetUniformLocation(program_id(GRAPHICS_PROGRAM), "image"), GRAPHICS_UNIT); check_gl(); 
+        graphics_constants_set = true; 
+    }
+    glActiveTexture(GL_TEXTURE0 + GRAPHICS_UNIT); check_gl();
+
+    GLuint base = 4 * start;
+    glEnable(GL_SCISSOR_TEST);
+    for (GLuint i=0; i < count;) {
+        ImageRenderData *rd = data + start + i;
+        glBindTexture(GL_TEXTURE_2D, rd->texture_id); check_gl();
+        // You could reduce the number of draw calls by using
+        // glDrawArraysInstancedBaseInstance but Apple chose to abandon OpenGL
+        // before implementing it.
+        for (GLuint k=0; k < rd->group_count; k++, base += 4, i++) glDrawArrays(GL_TRIANGLE_FAN, base, 4);
+    }
+    glDisable(GL_SCISSOR_TEST);
+    bind_vertex_array(vao_idx);
+}
+
+static void
+draw_all_cells(ssize_t vao_idx, ssize_t gvao_idx, Screen *screen) {
+    bind_program(CELL_PROGRAM); 
+    static bool cell_constants_set = false;
+    if (!cell_constants_set) { 
+        glUniform1i(glGetUniformLocation(program_id(CELL_PROGRAM), "sprites"), SPRITE_MAP_UNIT); check_gl(); 
+        cell_constants_set = true; 
+    }
+    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); check_gl();
+    if (screen->grman->count) draw_graphics(vao_idx, gvao_idx, screen->grman->render_data, 0, screen->grman->count);
+}
+
+static void
+draw_cells_interleaved(ssize_t vao_idx, ssize_t gvao_idx, Screen *screen) {
+    bind_program(CELL_BACKGROUND_PROGRAM); 
+    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); check_gl();
+
+    if (screen->grman->num_of_negative_refs) draw_graphics(vao_idx, gvao_idx, screen->grman->render_data, 0, screen->grman->num_of_negative_refs);
+
+    bind_program(CELL_SPECIAL_PROGRAM); 
+    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); check_gl();
+
+    bind_program(CELL_FOREGROUND_PROGRAM); 
+    glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
+    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); check_gl();
+
+    if (screen->grman->num_of_positive_refs) draw_graphics(vao_idx, gvao_idx, screen->grman->render_data, screen->grman->num_of_negative_refs, screen->grman->num_of_positive_refs);
+}
+
+static void 
+draw_cells_impl(ssize_t vao_idx, ssize_t gvao_idx, GLfloat xstart, GLfloat ystart, GLfloat dx, GLfloat dy, Screen *screen, CursorRenderInfo *cursor) {
+    GLfloat h = (GLfloat)screen->lines * dy;
+#define SCALE(w, x) ((GLfloat)(global_state.viewport_##w) * (GLfloat)(x))
+    glScissor(
+            (GLint)(SCALE(width, (xstart + 1.0f) / 2.0f)), 
+            (GLint)(SCALE(height, ((ystart - h) + 1.0f) / 2.0f)),
+            (GLsizei)(ceilf(SCALE(width, (float)screen->columns * dx / 2.0f))),
+            (GLsizei)(ceilf(SCALE(height, h / 2.0f)))
+    );
+#undef SCALE
+    cell_prepare_to_render(vao_idx, gvao_idx, screen, xstart, ystart, dx, dy, cursor);
+    if (screen->grman->num_of_negative_refs) draw_cells_interleaved(vao_idx, gvao_idx, screen);
+    else draw_all_cells(vao_idx, gvao_idx, screen);
+}
+// }}}
+
+// Cursor {{{
+enum CursorUniforms { CURSOR_color, CURSOR_pos, NUM_CURSOR_UNIFORMS };
+static GLint cursor_uniform_locations[NUM_CURSOR_UNIFORMS] = {0};
+static ssize_t cursor_vertex_array;
+
+static void
+init_cursor_program() {
+    Program *p = programs + CURSOR_PROGRAM;
+    int left = NUM_CURSOR_UNIFORMS;
+    cursor_vertex_array = create_vao();
+    for (int i = 0; i < p->num_of_uniforms; i++, left--) {
+#define SET_LOC(which) if (strcmp(p->uniforms[i].name, #which) == 0) cursor_uniform_locations[CURSOR_##which] = p->uniforms[i].location
+        SET_LOC(color);
+        else SET_LOC(pos);
+        else { fatal("Unknown uniform in cursor program"); }
+    }
+    if (left) { fatal("Left over uniforms in cursor program"); }
+#undef SET_LOC
+}
+
+static void 
+draw_cursor_impl(CursorRenderInfo *cursor) {
+    bind_program(CURSOR_PROGRAM); bind_vertex_array(cursor_vertex_array); check_gl();
+    glUniform3f(cursor_uniform_locations[CURSOR_color], ((cursor->color >> 16) & 0xff) / 255.0, ((cursor->color >> 8) & 0xff) / 255.0, (cursor->color & 0xff) / 255.0); check_gl();
+    glUniform4f(cursor_uniform_locations[CURSOR_pos], cursor->left, cursor->top, cursor->right, cursor->bottom); check_gl();
+    glDrawArrays(global_state.application_focused ? GL_TRIANGLE_FAN : GL_LINE_LOOP, 0, 4); check_gl();
+    unbind_vertex_array(); unbind_program();
+}
+// }}}
+
+// Borders {{{
+enum BorderUniforms { BORDER_viewport, NUM_BORDER_UNIFORMS };
+static GLint border_uniform_locations[NUM_BORDER_UNIFORMS] = {0};
+static ssize_t border_vertex_array;
+static GLsizei num_border_rects = 0;
+static GLuint rect_buf[5 * 1024];
+static GLuint *rect_pos = NULL;
+
+static void
+init_borders_program() {
+    Program *p = programs + BORDERS_PROGRAM;
+    int left = NUM_BORDER_UNIFORMS;
+    border_vertex_array = create_vao();
+    for (int i = 0; i < p->num_of_uniforms; i++, left--) {
+#define SET_LOC(which) if (strcmp(p->uniforms[i].name, #which) == 0) border_uniform_locations[BORDER_##which] = p->uniforms[i].location
+        SET_LOC(viewport);
+        else { fatal("Unknown uniform in borders program"); return; }
+    }
+    if (left) { fatal("Left over uniforms in borders program"); return; }
+#undef SET_LOC
+    add_buffer_to_vao(border_vertex_array, GL_ARRAY_BUFFER);
+    add_attribute_to_vao(BORDERS_PROGRAM, border_vertex_array, "rect",
+            /*size=*/4, /*dtype=*/GL_UNSIGNED_INT, /*stride=*/sizeof(GLuint)*5, /*offset=*/0, /*divisor=*/1);
+    add_attribute_to_vao(BORDERS_PROGRAM, border_vertex_array, "rect_color",
+            /*size=*/1, /*dtype=*/GL_UNSIGNED_INT, /*stride=*/sizeof(GLuint)*5, /*offset=*/(void*)(sizeof(GLuint)*4), /*divisor=*/1);
+}
+
+static void
+draw_borders_impl() {
+    if (num_border_rects) {
+        bind_program(BORDERS_PROGRAM);
+        bind_vertex_array(border_vertex_array);
+        glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, num_border_rects);
+        check_gl();
+        unbind_vertex_array();
+        unbind_program();
+    }
+}
+
+static void
+add_borders_rect(GLuint left, GLuint top, GLuint right, GLuint bottom, GLuint color) {
+    if (!left && !top && !right && !bottom) { num_border_rects = 0;  rect_pos = rect_buf; return; }
+    num_border_rects++;
+    *(rect_pos++) = left;
+    *(rect_pos++) = top;
+    *(rect_pos++) = right;
+    *(rect_pos++) = bottom;
+    *(rect_pos++) = color;
+}
+
+static void
+send_borders_rects(GLuint vw, GLuint vh) {
+    if (num_border_rects) {
+        size_t sz = sizeof(GLuint) * 5 * num_border_rects;
+        void *borders_buf_address = alloc_and_map_vao_buffer(border_vertex_array, sz, 0, GL_STATIC_DRAW, GL_WRITE_ONLY);
+        if (borders_buf_address) memcpy(borders_buf_address, rect_buf, sz);
+        unmap_vao_buffer(border_vertex_array, 0);
+    }
+    bind_program(BORDERS_PROGRAM);
+    glUniform2ui(border_uniform_locations[BORDER_viewport], vw, vh);
+    check_gl();
+    unbind_program();
+}
+// }}}
+
+// Python API {{{
+static PyObject*
+compile_program(PyObject UNUSED *self, PyObject *args) {
+    const char *vertex_shader, *fragment_shader;
+    int which;
+    GLuint vertex_shader_id = 0, fragment_shader_id = 0;
+    if (!PyArg_ParseTuple(args, "iss", &which, &vertex_shader, &fragment_shader)) return NULL;
+    if (which < 0 || which >= NUM_PROGRAMS) { PyErr_Format(PyExc_ValueError, "Unknown program: %d", which); return NULL; }
+    if (programs[which].id != 0) { PyErr_SetString(PyExc_ValueError, "program already compiled"); return NULL; }
+    programs[which].id = glCreateProgram();
+    check_gl();
+    vertex_shader_id = compile_shader(GL_VERTEX_SHADER, vertex_shader); check_gl();
+    fragment_shader_id = compile_shader(GL_FRAGMENT_SHADER, fragment_shader); check_gl();
+    glAttachShader(programs[which].id, vertex_shader_id); check_gl();
+    glAttachShader(programs[which].id, fragment_shader_id); check_gl();
+    glLinkProgram(programs[which].id); check_gl();
+    GLint ret = GL_FALSE;
+    glGetProgramiv(programs[which].id, GL_LINK_STATUS, &ret);
+    if (ret != GL_TRUE) {
+        GLsizei len;
+        glGetProgramInfoLog(programs[which].id, sizeof(glbuf), &len, glbuf);
+        fprintf(stderr, "Failed to compile GLSL shader!\n%s", glbuf);
+        PyErr_SetString(PyExc_ValueError, "Failed to compile shader");
+        goto end;
+    }
+    init_uniforms(which);
+
+end:
+    if (vertex_shader_id != 0) glDeleteShader(vertex_shader_id);
+    if (fragment_shader_id != 0) glDeleteShader(fragment_shader_id);
+    check_gl();
+    if (PyErr_Occurred()) { glDeleteProgram(programs[which].id); programs[which].id = 0; return NULL;}
+    return Py_BuildValue("I", programs[which].id);
+    Py_RETURN_NONE;
+}
+
+#define PYWRAP0(name) static PyObject* py##name(PyObject UNUSED *self)
+#define PYWRAP1(name) static PyObject* py##name(PyObject UNUSED *self, PyObject *args)
+#define PYWRAP2(name) static PyObject* py##name(PyObject UNUSED *self, PyObject *args, PyObject *kw)
+#define PA(fmt, ...) if(!PyArg_ParseTuple(args, fmt, __VA_ARGS__)) return NULL;
+#define ONE_INT(name) PYWRAP1(name) { name(PyLong_AsSsize_t(args)); Py_RETURN_NONE; } 
+#define TWO_INT(name) PYWRAP1(name) { int a, b; PA("ii", &a, &b); name(a, b); Py_RETURN_NONE; } 
+#define NO_ARG(name) PYWRAP0(name) { name(); Py_RETURN_NONE; }
+#define NO_ARG_INT(name) PYWRAP0(name) { return PyLong_FromSsize_t(name()); }
+
+ONE_INT(bind_program)
+NO_ARG(unbind_program)
+
+PYWRAP0(create_vao) {
+    int ans = create_vao();
+    if (ans < 0) return NULL;
+    return Py_BuildValue("i", ans);
+}
+
+ONE_INT(remove_vao)
+
+ONE_INT(bind_vertex_array)
+NO_ARG(unbind_vertex_array)
+TWO_INT(unmap_vao_buffer)
+
+NO_ARG(init_cursor_program)
+
+NO_ARG(init_borders_program)
+PYWRAP1(add_borders_rect) { unsigned int a, b, c, d, e; PA("IIIII", &a, &b, &c, &d, &e); add_borders_rect(a, b, c, d, e); Py_RETURN_NONE; }
+TWO_INT(send_borders_rects)
+
+NO_ARG(init_cell_program)
+NO_ARG_INT(create_cell_vao)
+NO_ARG_INT(create_graphics_vao)
+NO_ARG(destroy_sprite_map)
+PYWRAP1(layout_sprite_map) {
+    unsigned int cell_width, cell_height;
+    PyObject *render_cell;
+    PA("IIO", &cell_width, &cell_height, &render_cell);
+    layout_sprite_map(cell_width, cell_height, render_cell);
+    Py_RETURN_NONE;
+}
+
+PYWRAP1(clear_buffers) {
+    PyObject *swap_buffers;
+    unsigned int bg;
+    PA("OI", &swap_buffers, &bg);
+#define C(shift) ((float)((bg >> shift) & 0xff)) / 255.0
+    glClearColor(C(16), C(8), C(0), 1);
+#undef C
+    glClear(GL_COLOR_BUFFER_BIT);
+    PyObject *ret = PyObject_CallFunctionObjArgs(swap_buffers, NULL);
+    if (ret == NULL) return NULL;
+    Py_DECREF(ret);
+    glClear(GL_COLOR_BUFFER_BIT);
+    Py_RETURN_NONE;
+}
+
+PYWRAP0(check_for_extensions) {
+    GLint n = 0, i, left = 2;
+    glGetIntegerv(GL_NUM_EXTENSIONS, &n);
+    bool texture_storage = false;
+#define CHECK(name) if (!name) { \
+    if (strstr((const char*)ext, "GL_ARB_" #name) == (const char *)ext) { left--; name = true; } \
+}
+    for (i = 0; i < n; i++) {
+        const GLubyte *ext = glGetStringi(GL_EXTENSIONS, i);
+        CHECK(texture_storage); 
+        if (left < 1) break;
+    }
+#undef CHECK
+    if (left > 0) {
+#define CHECK(name) if (!name) { PyErr_Format(PyExc_RuntimeError, "The OpenGL driver on this system is missing the required extension: GL_ARB_%s", #name); return NULL; }
+        CHECK(texture_storage); 
+#undef CHECK
+    }
+    Py_RETURN_NONE;
+}
+
+#define M(name, arg_type) {#name, (PyCFunction)name, arg_type, NULL}
+#define MW(name, arg_type) {#name, (PyCFunction)py##name, arg_type, NULL}
+static PyMethodDef module_methods[] = {
+    {"glewInit", (PyCFunction)glew_init, METH_NOARGS, NULL}, 
+    M(compile_program, METH_VARARGS),
+    MW(check_for_extensions, METH_NOARGS),
+    MW(create_vao, METH_NOARGS),
+    MW(remove_vao, METH_O),
+    MW(bind_vertex_array, METH_O),
+    MW(unbind_vertex_array, METH_NOARGS),
+    MW(unmap_vao_buffer, METH_VARARGS),
+    MW(bind_program, METH_O),
+    MW(unbind_program, METH_NOARGS),
+    MW(init_cursor_program, METH_NOARGS),
+    MW(init_borders_program, METH_NOARGS),
+    MW(add_borders_rect, METH_VARARGS),
+    MW(send_borders_rects, METH_VARARGS),
+    MW(init_cell_program, METH_NOARGS),
+    MW(create_cell_vao, METH_NOARGS),
+    MW(create_graphics_vao, METH_NOARGS),
+    MW(layout_sprite_map, METH_VARARGS),
+    MW(destroy_sprite_map, METH_NOARGS),
+    MW(clear_buffers, METH_VARARGS),
+
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+bool
+#ifdef ENABLE_DEBUG_GL
+init_shaders_debug(PyObject *module) {
+#else
+init_shaders(PyObject *module) {
+#endif
+#define C(x) if (PyModule_AddIntConstant(module, #x, x) != 0) { PyErr_NoMemory(); return false; }
+    C(CELL_PROGRAM); C(CELL_BACKGROUND_PROGRAM); C(CELL_SPECIAL_PROGRAM); C(CELL_FOREGROUND_PROGRAM); C(CURSOR_PROGRAM); C(BORDERS_PROGRAM); C(GRAPHICS_PROGRAM);
+    C(GLSL_VERSION);
+    C(GL_VERSION);
+    C(GL_VENDOR);
+    C(GL_SHADING_LANGUAGE_VERSION);
+    C(GL_RENDERER);
+    C(GL_TRIANGLE_FAN); C(GL_TRIANGLE_STRIP); C(GL_TRIANGLES); C(GL_LINE_LOOP);
+    C(GL_COLOR_BUFFER_BIT);
+    C(GL_VERTEX_SHADER);
+    C(GL_FRAGMENT_SHADER);
+    C(GL_TRUE);
+    C(GL_FALSE);
+    C(GL_COMPILE_STATUS);
+    C(GL_LINK_STATUS);
+    C(GL_TEXTURE0); C(GL_TEXTURE1); C(GL_TEXTURE2); C(GL_TEXTURE3); C(GL_TEXTURE4); C(GL_TEXTURE5); C(GL_TEXTURE6); C(GL_TEXTURE7); C(GL_TEXTURE8);
+    C(GL_MAX_ARRAY_TEXTURE_LAYERS); C(GL_TEXTURE_BINDING_BUFFER); C(GL_MAX_TEXTURE_BUFFER_SIZE);
+    C(GL_MAX_TEXTURE_SIZE);
+    C(GL_TEXTURE_2D_ARRAY);
+    C(GL_LINEAR); C(GL_CLAMP_TO_EDGE); C(GL_NEAREST);
+    C(GL_TEXTURE_MIN_FILTER); C(GL_TEXTURE_MAG_FILTER);
+    C(GL_TEXTURE_WRAP_S); C(GL_TEXTURE_WRAP_T);
+    C(GL_UNPACK_ALIGNMENT);
+    C(GL_R8); C(GL_RED); C(GL_UNSIGNED_BYTE); C(GL_UNSIGNED_SHORT); C(GL_R32UI); C(GL_RGB32UI); C(GL_RGBA);
+    C(GL_TEXTURE_BUFFER); C(GL_STATIC_DRAW); C(GL_STREAM_DRAW); C(GL_DYNAMIC_DRAW);
+    C(GL_SRC_ALPHA); C(GL_ONE_MINUS_SRC_ALPHA); 
+    C(GL_WRITE_ONLY); C(GL_READ_ONLY); C(GL_READ_WRITE);
+    C(GL_BLEND); C(GL_FLOAT); C(GL_UNSIGNED_INT); C(GL_ARRAY_BUFFER); C(GL_UNIFORM_BUFFER);
+
+#undef C
+    PyModule_AddObject(module, "GL_VERSION_REQUIRED", Py_BuildValue("II", REQUIRED_VERSION_MAJOR, REQUIRED_VERSION_MINOR));
+    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
+    update_viewport_size = &update_viewport_size_impl;
+    draw_borders = &draw_borders_impl;
+    draw_cells = &draw_cells_impl;
+    draw_cursor = &draw_cursor_impl;
+    free_texture = &free_texture_impl;
+    send_image_to_gpu = &send_image_to_gpu_impl;
+    return true;
+}
+// }}}

kitty/shaders.py

@@ -1,267 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
-
-import sys
-from ctypes import addressof, sizeof
-from functools import lru_cache
-from threading import Lock
-
-from .fonts.render import render_cell
-from .fast_data_types import (
-    glCreateProgram, glAttachShader, GL_FRAGMENT_SHADER, GL_VERTEX_SHADER,
-    glLinkProgram, GL_TRUE, GL_LINK_STATUS, glGetProgramiv,
-    glGetProgramInfoLog, glDeleteShader, glDeleteProgram, glGenVertexArrays,
-    glCreateShader, glShaderSource, glCompileShader, glGetShaderiv,
-    GL_COMPILE_STATUS, glGetShaderInfoLog, glGetUniformLocation,
-    glGetAttribLocation, glUseProgram, glBindVertexArray, GL_TEXTURE0,
-    GL_TEXTURE1, glGetIntegerv, GL_MAX_ARRAY_TEXTURE_LAYERS, glNamedBufferData,
-    GL_MAX_TEXTURE_SIZE, glDeleteTexture, GL_TEXTURE_2D_ARRAY, glGenTextures,
-    glBindTexture, glTexParameteri, GL_CLAMP_TO_EDGE, glDeleteBuffer,
-    GL_TEXTURE_MAG_FILTER, GL_TEXTURE_MIN_FILTER, GL_TEXTURE_WRAP_S,
-    GL_NEAREST, GL_TEXTURE_WRAP_T, glGenBuffers, GL_R8, GL_RED,
-    GL_UNPACK_ALIGNMENT, GL_UNSIGNED_BYTE, GL_STATIC_DRAW, GL_STREAM_DRAW,
-    GL_TEXTURE_BUFFER, GL_RGB32UI, GL_FLOAT, GL_ARRAY_BUFFER, glBindBuffer,
-    glPixelStorei, glTexBuffer, glActiveTexture, glTexStorage3D,
-    glCopyImageSubData, glTexSubImage3D, ITALIC, BOLD, SpriteMap,
-    glEnableVertexAttribArray, glVertexAttribPointer, copy_image_sub_data
-)
-from .utils import safe_print
-
-GL_VERSION = (3, 3)
-VERSION = GL_VERSION[0] * 100 + GL_VERSION[1] * 10
-ITALIC_MASK = 1 << ITALIC
-BOLD_MASK = 1 << BOLD
-
-
-class Sprites:
-    ''' Maintain sprite sheets of all rendered characters on the GPU as a texture
-    array with each texture being a sprite sheet. '''
-
-    # TODO: Rewrite this class using the ARB_shader_image_load_store and ARB_shader_storage_buffer_object
-    # extensions one they become available.
-
-    def __init__(self):
-        self.xnum = self.ynum = 1
-        self.sampler_num = 0
-        self.buffer_sampler_num = 1
-        self.first_cell_cache = {}
-        self.second_cell_cache = {}
-        self.x = self.y = self.z = 0
-        self.texture_id = self.buffer_texture_id = None
-        self.last_num_of_layers = 1
-        self.last_ynum = -1
-        self.texture_unit = GL_TEXTURE0
-        self.backend = SpriteMap(glGetIntegerv(GL_MAX_TEXTURE_SIZE), glGetIntegerv(GL_MAX_ARRAY_TEXTURE_LAYERS))
-        self.lock = Lock()
-
-    def do_layout(self, cell_width=1, cell_height=1):
-        self.cell_width, self.cell_height = cell_width, cell_height
-        self.backend.layout(cell_width or 1, cell_height or 1)
-        if self.texture_id is not None:
-            glDeleteTexture(self.texture_id)
-            self.texture_id = None
-        self.ensure_state()
-        self.pre_render()
-
-    def pre_render(self):
-        # Pre-render the basic cells to ensure they have known sprite numbers
-
-        def send(*a, **kw):
-            buf = render_cell(*a, **kw)[0]
-            x, y, z = self.backend.increment()
-            self.send_to_gpu(x, y, z, buf)
-            return x
-
-        send()  # blank
-        send(underline=1)
-        send(underline=2)
-        if send(strikethrough=True) != 3:
-            raise RuntimeError('Available OpenGL texture size is too small')
-
-    @property
-    def layout(self):
-        return 1 / self.backend.xnum, 1 / self.backend.ynum
-
-    def render_cell(self, text, bold, italic, is_second):
-        first, second = render_cell(text, bold, italic)
-        ans = second if is_second else first
-        return ans or render_cell()[0]
-
-    def render_dirty_cells(self):
-        with self.lock:
-            self.backend.render_dirty_cells(self.render_cell, self.send_to_gpu)
-
-    def send_to_gpu(self, x, y, z, buf):
-        if self.backend.z >= self.last_num_of_layers:
-            self.realloc_texture()
-        else:
-            if self.backend.z == 0 and self.backend.ynum > self.last_ynum:
-                self.realloc_texture()
-        tgt = GL_TEXTURE_2D_ARRAY
-        glBindTexture(tgt, self.texture_id)
-        glPixelStorei(GL_UNPACK_ALIGNMENT, 1)
-        x, y = x * self.cell_width, y * self.cell_height
-        glTexSubImage3D(tgt, 0, x, y, z, self.cell_width, self.cell_height, 1, GL_RED, GL_UNSIGNED_BYTE, addressof(buf))
-        glBindTexture(tgt, 0)
-
-    def realloc_texture(self):
-        tgt = GL_TEXTURE_2D_ARRAY
-        tex = glGenTextures(1)
-        glBindTexture(tgt, tex)
-        # We use GL_NEAREST otherwise glyphs that touch the edge of the cell
-        # often show a border between cells
-        glTexParameteri(tgt, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
-        glTexParameteri(tgt, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
-        glTexParameteri(tgt, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
-        glTexParameteri(tgt, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
-        znum = self.backend.z + 1
-        width, height = self.backend.xnum * self.cell_width, self.backend.ynum * self.cell_height
-        glTexStorage3D(tgt, 1, GL_R8, width, height, znum)
-        if self.texture_id is not None:
-            ynum = self.backend.ynum
-            if self.backend.z == 0:
-                ynum -= 1  # Only copy the previous rows
-            try:
-                glCopyImageSubData(self.texture_id, tgt, 0, 0, 0, 0, tex, tgt, 0, 0, 0, 0,
-                                   width, ynum * self.cell_height, self.last_num_of_layers)
-            except RuntimeError:
-                # OpenGL does not have ARB_copy_image
-                if not hasattr(self, 'realloc_warned'):
-                    safe_print(
-                        'WARNING: Your system\'s OpenGL implementation does not have glCopyImageSubData, falling back to a slower implementation',
-                        file=sys.stderr)
-                    self.realloc_warned = True
-                copy_image_sub_data(self.texture_id, tex, width, ynum * self.cell_height, self.last_num_of_layers)
-                glBindTexture(tgt, tex)
-            glDeleteTexture(self.texture_id)
-        self.last_num_of_layers = znum
-        self.last_ynum = self.backend.ynum
-        self.texture_id = tex
-        glBindTexture(tgt, 0)
-
-    def destroy(self):
-        if self.texture_id is not None:
-            glDeleteTexture(self.texture_id)
-            self.texture_id = None
-        if self.buffer_texture_id is not None:
-            glDeleteTexture(self.buffer_texture_id)
-
-    def ensure_state(self):
-        if self.texture_id is None:
-            self.realloc_texture()
-            self.buffer_texture_id = glGenTextures(1)
-            self.buffer_texture_unit = GL_TEXTURE1
-
-    def add_sprite_map(self):
-        return glGenBuffers(1)
-
-    def set_sprite_map(self, buf_id, data, usage=GL_STREAM_DRAW):
-        self.bind_sprite_map(buf_id)
-        glNamedBufferData(buf_id, sizeof(data), addressof(data), usage)
-
-    def bind_sprite_map(self, buf_id):
-        glBindBuffer(GL_TEXTURE_BUFFER, buf_id)
-        glTexBuffer(GL_TEXTURE_BUFFER, GL_RGB32UI, buf_id)
-
-    def destroy_sprite_map(self, buf_id):
-        glDeleteBuffer(buf_id)
-
-    def __enter__(self):
-        self.ensure_state()
-        glActiveTexture(self.texture_unit)
-        glBindTexture(GL_TEXTURE_2D_ARRAY, self.texture_id)
-
-        glActiveTexture(self.buffer_texture_unit)
-        glBindTexture(GL_TEXTURE_BUFFER, self.buffer_texture_id)
-
-    def __exit__(self, *a):
-        glBindTexture(GL_TEXTURE_2D_ARRAY, 0)
-        glBindTexture(GL_TEXTURE_BUFFER, 0)
-        glBindBuffer(GL_TEXTURE_BUFFER, 0)
-        glTexBuffer(GL_TEXTURE_BUFFER, GL_RGB32UI, 0)
-
-
-class ShaderProgram:
-    """ Helper class for using GLSL shader programs """
-
-    def __init__(self, vertex, fragment):
-        """
-        Create a shader program.
-
-        """
-        self.program_id = glCreateProgram()
-        vs_id = self.add_shader(vertex, GL_VERTEX_SHADER)
-        glAttachShader(self.program_id, vs_id)
-
-        frag_id = self.add_shader(fragment, GL_FRAGMENT_SHADER)
-        glAttachShader(self.program_id, frag_id)
-
-        glLinkProgram(self.program_id)
-        if glGetProgramiv(self.program_id, GL_LINK_STATUS) != GL_TRUE:
-            info = glGetProgramInfoLog(self.program_id)
-            glDeleteProgram(self.program_id)
-            glDeleteShader(vs_id)
-            glDeleteShader(frag_id)
-            raise ValueError('Error linking shader program: \n%s' % info.decode('utf-8'))
-        glDeleteShader(vs_id)
-        glDeleteShader(frag_id)
-        self.vao_id = glGenVertexArrays(1)
-        self.buffers = {}
-
-    def add_vertex_array(self, name, size=3, dtype=GL_FLOAT, normalized=False, stride=0, offset=0):
-        glBindVertexArray(self.vao_id)
-        if name not in self.buffers:
-            self.buffers[name] = glGenBuffers(1)
-        glBindBuffer(GL_ARRAY_BUFFER, self.buffers[name])
-        aid = self.attribute_location(name)
-        glEnableVertexAttribArray(aid)
-        glVertexAttribPointer(aid, size, dtype, normalized, stride, offset)
-        glBindBuffer(GL_ARRAY_BUFFER, 0)
-        glBindVertexArray(0)
-
-    def send_vertex_data(self, name, data, usage=GL_STATIC_DRAW):
-        bufid = self.buffers[name]
-        glNamedBufferData(bufid, sizeof(data), addressof(data), usage)
-
-    def __hash__(self) -> int:
-        return self.program_id
-
-    def __eq__(self, other) -> bool:
-        return isinstance(other, ShaderProgram) and other.program_id == self.program_id
-
-    def __ne__(self, other) -> bool:
-        return not self.__eq__(other)
-
-    def add_shader(self, source: str, shader_type: int) -> int:
-        ' Compile a shader and return its id, or raise an exception if compilation fails '
-        shader_id = glCreateShader(shader_type)
-        source = '#version {}\n{}'.format(VERSION, source)
-        try:
-            glShaderSource(shader_id, source)
-            glCompileShader(shader_id)
-            if glGetShaderiv(shader_id, GL_COMPILE_STATUS) != GL_TRUE:
-                info = glGetShaderInfoLog(shader_id)
-                raise ValueError('GLSL {} compilation failed: \n{}'.format(shader_type, info.decode('utf-8')))
-            return shader_id
-        except Exception:
-            glDeleteShader(shader_id)
-            raise
-
-    @lru_cache(maxsize=2**6)
-    def uniform_location(self, name: str) -> int:
-        ' Return the id for the uniform variable `name` or -1 if not found. '
-        return glGetUniformLocation(self.program_id, name)
-
-    @lru_cache(maxsize=2**6)
-    def attribute_location(self, name: str) -> int:
-        ' Return the id for the attribute variable `name` or -1 if not found. '
-        return glGetAttribLocation(self.program_id, name)
-
-    def __enter__(self):
-        glUseProgram(self.program_id)
-        glBindVertexArray(self.vao_id)
-
-    def __exit__(self, *args):
-        glUseProgram(0)
-        glBindVertexArray(0)

kitty/sprites.c

@@ -6,107 +6,37 @@
  */
 
 #include "data-types.h"
+#include "lineops.h"
 #include <structmember.h>
 
-static PyObject*
-new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
-    SpriteMap *self;
-    unsigned long mlen, msz;
-    if (!PyArg_ParseTuple(args, "kk", &msz, &mlen)) return NULL;
+typedef struct SpritePosition SpritePosition;
+struct SpritePosition {
+    SpritePosition *next;
+    sprite_index x, y, z;
+    char_type ch;
+    combining_type cc;
+    bool bold, italic;
+    bool is_second;
+    bool filled;
+    bool rendered;
+};
 
-    self = (SpriteMap *)type->tp_alloc(type, 0);
-    if (self != NULL) {
-        self->max_array_len = mlen;
-        self->max_texture_size = msz;
-        self->dirty = true;
-    }
-    return (PyObject*) self;
-}
+typedef struct {
+    size_t max_array_len, max_texture_size, max_y;
+    unsigned int x, y, z, xnum, ynum;
+    SpritePosition cache[1024];
+    bool dirty;
+} SpriteMap;
 
-static void
-dealloc(SpriteMap* self) {
-    SpritePosition *s, *t;
-    for (size_t i = 0; i < sizeof(self->cache)/sizeof(self->cache[0]); i++) {
-        s = &(self->cache[i]);
-        s = s->next;
-        while (s) {
-            t = s;
-            s = s->next;
-            PyMem_Free(t);
-        }
-    }
-    Py_TYPE(self)->tp_free((PyObject*)self);
-}
+static SpriteMap sprite_map = {
+    .max_array_len = 1000,
+    .max_texture_size = 1000,
+    .max_y = 100,
+    .dirty = true
+};
 
-static PyObject*
-layout(SpriteMap *self, PyObject *args) {
-#define layout_doc "layout(cell_width, cell_height) -> Invalidate the cache and prepare it for new cell size"
-    unsigned long cell_width, cell_height;
-    if (!PyArg_ParseTuple(args, "kk", &cell_width, &cell_height)) return NULL;
-    self->xnum = MAX(1, self->max_texture_size / cell_width);
-    self->max_y = MAX(1, self->max_texture_size / cell_height);
-    self->ynum = 1;
-    self->x = 0; self->y = 0; self->z = 0;
-
-    for (size_t i = 0; i < sizeof(self->cache)/sizeof(self->cache[0]); i++) {
-        SpritePosition *s = &(self->cache[i]);
-        do {
-            s->filled = false;
-            s->is_second = false;
-            s->rendered = false;
-            s->ch = 0; s->cc = 0;
-            s->x = 0; s->y = 0; s->z = 0;
-            s = s->next;
-        } while (s != NULL);
-    }
-    Py_RETURN_NONE;
-}
-
-static void
-do_increment(SpriteMap *self, int *error) {
-    self->x++;
-    if (self->x >= self->xnum) {
-        self->x = 0; self->y++;
-        self->ynum = MIN(MAX(self->ynum, self->y + 1), self->max_y);
-        if (self->y >= self->max_y) {
-            self->y = 0; self->z++;
-            if (self->z >= self->max_array_len) *error = 2;
-        }
-    }
-}
-
-static SpritePosition*
-sprite_position_for(SpriteMap *self, char_type ch, combining_type cc, bool is_second, int *error) {
-    char_type attrs = ch >> ATTRS_SHIFT, pos_char;
-    uint8_t bold = (attrs >> BOLD_SHIFT) & 1, italic = (attrs >> ITALIC_SHIFT) & 1;
-    size_t idx = (ch & 0xff) | (bold << 8) | (italic << 9);
-    attrs = bold << BOLD_SHIFT | italic << ITALIC_SHIFT;
-    pos_char = (ch & CHAR_MASK) | (attrs << ATTRS_SHIFT);
-    SpritePosition *s = &(self->cache[idx]);
-    while(true) {
-        if (s->filled) {
-            if (s->ch == pos_char && s->cc == cc && s->is_second == is_second) return s;  // Cache hit
-        } else {
-            break;
-        }
-        if (!s->next) {
-            s->next = PyMem_Calloc(1, sizeof(SpritePosition));
-            if (s->next == NULL) { *error = 1; return NULL; }
-        }
-        s = s->next;
-    }
-    s->ch = pos_char;
-    s->cc = cc;
-    s->is_second = is_second;
-    s->filled = true;
-    s->rendered = false;
-    s->x = self->x; s->y = self->y; s->z = self->z;
-    do_increment(self, error);
-    self->dirty = true;
-    return s;
-}
-
-static void set_sprite_error(int error) {
+static inline void 
+sprite_map_set_error(int error) {
     switch(error) {
         case 1:
             PyErr_NoMemory(); break;
@@ -117,125 +47,182 @@ static void set_sprite_error(int error) {
     }
 }
 
-static PyObject*
-increment(SpriteMap *self) {
-#define increment_doc "Increment the current position and return the old (x, y, z) values"
-    unsigned int x = self->x, y = self->y, z = self->z;
-    int error = 0;
-    do_increment(self, &error);
-    if (error) { set_sprite_error(error); return NULL; }
-    return Py_BuildValue("III", x, y, z);
+void
+sprite_map_set_limits(size_t max_texture_size, size_t max_array_len) {
+    sprite_map.max_texture_size = max_texture_size;
+    sprite_map.max_array_len = max_array_len;
 }
 
 static PyObject*
-position_for(SpriteMap *self, PyObject *args) {
-#define position_for_doc "position_for(ch, cc, is_second) -> x, y, z the sprite position for the specified text"
-    unsigned long ch = 0;
-    unsigned long long cc = 0;
-    int is_second = 0, error = 0;
-    if (!PyArg_ParseTuple(args, "|kKp", &ch, &cc, &is_second)) return NULL;
-    SpritePosition *pos = sprite_position_for(self, ch, cc, is_second, &error);
-    if (pos == NULL) {set_sprite_error(error); return NULL; }
-    return Py_BuildValue("III", pos->x, pos->y, pos->z);
-}
-
-bool
-update_cell_range_data(ScreenModes *modes, SpriteMap *self, Line *line, unsigned int xstart, unsigned int xmax, ColorProfile *color_profile, const uint32_t default_bg, const uint32_t default_fg, unsigned int *data) {
-    SpritePosition *sp;
-    char_type previous_ch=0, ch;
-    uint8_t previous_width = 0;
-    int err = 0;
-    const bool screen_reversed = modes->mDECSCNM;
-
-    size_t base = line->ynum * line->xnum * DATA_CELL_SIZE;
-    for (size_t i = xstart, offset = base + xstart * DATA_CELL_SIZE; i <= xmax; i++, offset += DATA_CELL_SIZE) {
-        ch = line->chars[i];
-        if (previous_width == 2) sp = sprite_position_for(self, previous_ch, 0, true, &err);
-        else sp = sprite_position_for(self, ch, line->combining_chars[i], false, &err);
-        if (sp == NULL) { set_sprite_error(err); return false; }
-        char_type attrs = ch >> ATTRS_SHIFT;
-        unsigned int decoration = (attrs >> DECORATION_SHIFT) & DECORATION_MASK;
-        unsigned int strikethrough = ((attrs >> STRIKE_SHIFT) & 1) ? 3 : 0;
-        bool reverse = ((attrs >> REVERSE_SHIFT) & 1) ^ screen_reversed;
-        data[offset] = sp->x;
-        data[offset+1] = sp->y;
-        data[offset+2] = sp->z;
-        data[offset+(reverse ? 4 : 3)] = to_color(color_profile, line->fg_colors[i] & COL_MASK, default_fg);
-        data[offset+(reverse ? 3 : 4)] = to_color(color_profile, line->bg_colors[i] & COL_MASK, default_bg);
-        unsigned int decoration_fg = to_color(color_profile, line->decoration_fg[i] & COL_MASK, data[offset+3]);
-        data[offset+5] = (decoration_fg & COL_MASK) | (decoration << 24) | (strikethrough << 26);
-        previous_ch = ch; previous_width = (attrs) & WIDTH_MASK;
-    }
-    return true;
-}
-
-static PyObject*
-render_dirty_cells(SpriteMap *self, PyObject *args) {
-#define render_dirty_cells_doc "Render all cells that are marked as dirty"
-    PyObject *render_cell, *send_to_gpu;
-
-    if (!PyArg_ParseTuple(args, "OO", &render_cell, &send_to_gpu)) return NULL;
-
-    if (!self->dirty) { Py_RETURN_NONE; }
-
-    for (size_t i = 0; i < sizeof(self->cache)/sizeof(self->cache[0]); i++) {
-        SpritePosition *sp = &(self->cache[i]);
-        while (sp) {
-            if (sp->filled && !sp->rendered) {
-                PyObject *text = line_text_at(sp->ch & CHAR_MASK, sp->cc);
-                if (text == NULL) return NULL;
-                char_type attrs = sp->ch >> ATTRS_SHIFT;
-                bool bold = (attrs >> BOLD_SHIFT) & 1, italic = (attrs >> ITALIC_SHIFT) & 1;
-                PyObject *rcell = PyObject_CallFunctionObjArgs(render_cell, text, bold ? Py_True : Py_False, italic ? Py_True : Py_False, sp->is_second ? Py_True : Py_False, NULL);
-                Py_CLEAR(text);
-                if (rcell == NULL) return NULL;
-                PyObject *ret = PyObject_CallFunction(send_to_gpu, "IIIO", sp->x, sp->y, sp->z, rcell);
-                Py_CLEAR(rcell);
-                if (ret == NULL) return NULL;
-                Py_CLEAR(ret); 
-                sp->rendered = true; 
-            }
-            sp = sp->next;
-        }
-    }
-    self->dirty = false;
+sprite_map_set_limits_py(PyObject UNUSED *self, PyObject *args) {
+    unsigned int w, h;
+    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
+    sprite_map_set_limits(w, h);
     Py_RETURN_NONE;
 }
 
-// Boilerplate {{{
+void
+sprite_map_free() {
+    SpritePosition *s, *t;
+    for (size_t i = 0; i < sizeof(sprite_map.cache)/sizeof(sprite_map.cache[0]); i++) {
+        s = &(sprite_map.cache[i]);
+        s = s->next;
+        while (s) {
+            t = s;
+            s = s->next;
+            PyMem_Free(t);
+        }
+    }
+}
 
-static PyMemberDef members[] = {
-    {"xnum", T_UINT, offsetof(SpriteMap, xnum), 0, "xnum"},
-    {"ynum", T_UINT, offsetof(SpriteMap, ynum), 0, "ynum"},
-    {"x", T_UINT, offsetof(SpriteMap, x), 0, "x"},
-    {"y", T_UINT, offsetof(SpriteMap, y), 0, "y"},
-    {"z", T_UINT, offsetof(SpriteMap, z), 0, "z"},
-    {NULL}  /* Sentinel */
+static inline void
+do_increment(int *error) {
+    sprite_map.x++;
+    if (sprite_map.x >= sprite_map.xnum) {
+        sprite_map.x = 0; sprite_map.y++;
+        sprite_map.ynum = MIN(MAX(sprite_map.ynum, sprite_map.y + 1), sprite_map.max_y);
+        if (sprite_map.y >= sprite_map.max_y) {
+            sprite_map.y = 0; sprite_map.z++;
+            if (sprite_map.z >= MIN(UINT16_MAX, sprite_map.max_array_len)) *error = 2;
+        }
+    }
+}
+
+SpritePosition*
+sprite_map_position_for(char_type ch, attrs_type attrs, combining_type cc, bool is_second, int *error) {
+    uint8_t bold_italic = (attrs >> BOLD_SHIFT) & 3;
+    unsigned int idx = (ch & 0xFF) | (bold_italic << 8); // Only bold italic bits and lowest byte of char
+    SpritePosition *s = &(sprite_map.cache[idx]);
+    // Optimize for the common case of an ASCII char already in the cache
+    if (LIKELY(s->ch == ch && s->filled && s->cc == cc && s->is_second == is_second)) return s;  // Cache hit
+    while(true) {
+        if (s->filled) {
+            if (s->ch == ch && s->cc == cc && s->is_second == is_second) return s;  // Cache hit
+        } else {
+            break;
+        }
+        if (!s->next) {
+            s->next = PyMem_Calloc(1, sizeof(SpritePosition));
+            if (s->next == NULL) { *error = 1; return NULL; }
+        }
+        s = s->next;
+    }
+    s->ch = ch;
+    s->cc = cc;
+    s->is_second = is_second;
+    s->filled = true;
+    s->rendered = false;
+    s->bold = bold_italic & 1;
+    s->italic = bold_italic >> 1;
+    s->x = sprite_map.x; s->y = sprite_map.y; s->z = sprite_map.z;
+    do_increment(error);
+    sprite_map.dirty = true;
+    return s;
+}
+
+
+void
+set_sprite_position(Cell *cell, Cell *previous_cell) {
+    SpritePosition *sp;
+    static int error;
+    if (UNLIKELY(previous_cell != NULL && ((previous_cell->attrs) & WIDTH_MASK) == 2)) {
+        sp = sprite_map_position_for(previous_cell->ch, previous_cell->attrs, 0, true, &error);
+    } else {
+        sp = sprite_map_position_for(cell->ch, cell->attrs, cell->cc, false, &error);
+    }
+    cell->sprite_x = sp->x;
+    cell->sprite_y = sp->y;
+    cell->sprite_z = sp->z;
+}
+
+int 
+sprite_map_increment(sprite_index *x, sprite_index *y, sprite_index *z) {
+    int error = 0;
+    *x = sprite_map.x; *y = sprite_map.y; *z = sprite_map.z;
+    do_increment(&error);
+    return error;
+}
+
+void
+sprite_map_set_layout(unsigned int cell_width, unsigned int cell_height) {
+    // Invalidate cache since cell size has changed.
+    SpritePosition *s;
+    sprite_map.xnum = MIN(MAX(1, sprite_map.max_texture_size / cell_width), UINT16_MAX);
+    sprite_map.max_y = MIN(MAX(1, sprite_map.max_texture_size / cell_height), UINT16_MAX);
+    sprite_map.ynum = 1;
+    sprite_map.x = 0; sprite_map.y = 0; sprite_map.z = 0;
+
+    for (size_t i = 0; i < sizeof(sprite_map.cache)/sizeof(sprite_map.cache[0]); i++) {
+        s = &(sprite_map.cache[i]);
+        do {
+            s->filled = false;
+            s->is_second = false;
+            s->rendered = false;
+            s->ch = 0; s->cc = 0;
+            s->x = 0; s->y = 0; s->z = 0;
+            s = s->next;
+        } while (s != NULL);
+    }
+    sprite_map.dirty = true;
+}
+
+static PyObject*
+sprite_map_set_layout_py(PyObject UNUSED *self, PyObject *args) {
+    unsigned int w, h;
+    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
+    sprite_map_set_layout(w, h);
+    Py_RETURN_NONE;
+}
+
+void
+sprite_map_current_layout(unsigned int *x, unsigned int *y, unsigned int *z) {
+    *x = sprite_map.xnum; *y = sprite_map.ynum; *z = sprite_map.z;
+}
+
+PyObject*
+sprite_position_for(PyObject UNUSED *self, PyObject *args) {
+#define sprite_position_for_doc "sprite_position_for(ch, cc, is_second, attrs) -> x, y, z the sprite position for the specified text"
+    unsigned long ch = 0;
+    unsigned long long cc = 0;
+    unsigned int attrs = 1;
+    int is_second = 0, error = 0;
+    if (!PyArg_ParseTuple(args, "|kKpI", &ch, &cc, &is_second, &attrs)) return NULL;
+    SpritePosition *pos = sprite_map_position_for(ch, attrs, cc, is_second, &error);
+    if (pos == NULL) { sprite_map_set_error(error); return NULL; }
+    return Py_BuildValue("III", pos->x, pos->y, pos->z);
+}
+
+void
+render_dirty_sprites(void (*render)(PyObject*, bool, bool, bool, sprite_index, sprite_index, sprite_index)) {
+#define render_dirty_cells_doc "Render all cells that are marked as dirty"
+    if (!sprite_map.dirty) return;
+
+    for (size_t i = 0; i < sizeof(sprite_map.cache)/sizeof(sprite_map.cache[0]); i++) {
+        SpritePosition *sp = &(sprite_map.cache[i]);
+        do {
+            if (sp->filled && !sp->rendered) {
+                PyObject *text = line_text_at(sp->ch, sp->cc);
+                render(text, sp->bold, sp->italic, sp->is_second, sp->x, sp->y, sp->z);
+                Py_CLEAR(text);
+                sp->rendered = true; 
+            }
+            sp = sp->next;
+        } while(sp);
+    }
+    sprite_map.dirty = false;
+}
+
+
+static PyMethodDef module_methods[] = {
+    {"sprite_position_for", (PyCFunction)sprite_position_for, METH_VARARGS, ""},
+    {"sprite_map_set_layout", (PyCFunction)sprite_map_set_layout_py, METH_VARARGS, ""},
+    {"sprite_map_set_limits", (PyCFunction)sprite_map_set_limits_py, METH_VARARGS, ""},
+
+    {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
-
-static PyMethodDef methods[] = {
-    METHOD(layout, METH_VARARGS)
-    METHOD(position_for, METH_VARARGS)
-    METHOD(render_dirty_cells, METH_VARARGS)
-    METHOD(increment, METH_NOARGS)
-    {NULL}  /* Sentinel */
-};
-
-
-PyTypeObject SpriteMap_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    .tp_name = "fast_data_types.SpriteMap",
-    .tp_basicsize = sizeof(SpriteMap),
-    .tp_dealloc = (destructor)dealloc, 
-    .tp_flags = Py_TPFLAGS_DEFAULT,        
-    .tp_doc = "SpriteMap",
-    .tp_methods = methods,
-    .tp_members = members,
-    .tp_new = new,                
-};
-
-INIT_TYPE(SpriteMap)
-// }}}
-
-
+bool 
+init_sprites(PyObject *module) {
+    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
+    return true;
+}

kitty/sprites.h

@@ -0,0 +1,15 @@
+/*
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#pragma once
+
+
+void sprite_map_current_layout(unsigned int *x, unsigned int *y, unsigned int*);
+void sprite_map_set_layout(unsigned int cell_width, unsigned int cell_height);
+void sprite_map_set_limits(size_t max_texture_size, size_t max_array_len);
+void sprite_map_free();
+int sprite_map_increment(sprite_index *x, sprite_index *y, sprite_index *z);
+void render_dirty_sprites(void (*render)(PyObject*, bool, bool, bool, sprite_index, sprite_index, sprite_index));

kitty/state.c

@@ -0,0 +1,300 @@
+/*
+ * state.c
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "state.h"
+
+GlobalState global_state = {{0}};
+static const Tab EMPTY_TAB = {0};
+static const Window EMPTY_WINDOW = {0};
+
+#define ensure_can_add(array, count, msg) if (count >= sizeof(array)/sizeof(array[0]) - 1) fatal(msg);
+
+#define noop(...)
+#define REMOVER(array, qid, count, empty, structure, destroy) { \
+    size_t capacity = sizeof(array)/sizeof(array[0]); \
+    for (size_t i = 0; i < count; i++) { \
+        if (array[i].id == qid) { \
+            destroy(array[i]); \
+            array[i] = empty; \
+            size_t num_to_right = capacity - count - 1; \
+            if (num_to_right) memmove(array + i, array + i + 1, num_to_right * sizeof(structure)); \
+            (count)--; \
+        } \
+    }} 
+#define WITH_TAB(tab_id) \
+    for (size_t t = 0; t < global_state.num_tabs; t++) { \
+        if (global_state.tabs[t].id == tab_id) { \
+            Tab *tab = global_state.tabs + t;
+#define END_WITH_TAB break; }}
+
+static inline void
+add_tab(unsigned int id) {
+    ensure_can_add(global_state.tabs, global_state.num_tabs, "Too many children (add_tab)");
+    global_state.tabs[global_state.num_tabs] = EMPTY_TAB;
+    global_state.tabs[global_state.num_tabs].id = id;
+    global_state.num_tabs++;
+}
+
+static inline void
+add_window(unsigned int tab_id, unsigned int id, PyObject *title) {
+    WITH_TAB(tab_id);
+    ensure_can_add(tab->windows, tab->num_windows, "Too many children (add_window)");
+    tab->windows[tab->num_windows] = EMPTY_WINDOW;
+    tab->windows[tab->num_windows].id = id;
+    tab->windows[tab->num_windows].visible = true;
+    tab->windows[tab->num_windows].title = title;
+    Py_INCREF(tab->windows[tab->num_windows].title);
+    tab->num_windows++;
+    END_WITH_TAB;
+}
+
+static inline void
+update_window_title(unsigned int tab_id, unsigned int window_id, PyObject *title) {
+    WITH_TAB(tab_id);
+    for (size_t i = 0; i < tab->num_windows; i++) {
+        if (tab->windows[i].id == window_id) {
+            Py_CLEAR(tab->windows[i].title);
+            tab->windows[i].title = title;
+            Py_INCREF(tab->windows[i].title);
+            break;
+        }
+    }
+    END_WITH_TAB;
+}
+
+static inline void
+remove_tab(unsigned int id) {
+    REMOVER(global_state.tabs, id, global_state.num_tabs, EMPTY_TAB, Tab, noop);
+}
+
+static inline void
+remove_window(unsigned int tab_id, unsigned int id) {
+    WITH_TAB(tab_id);
+#define destroy_window(w) Py_CLEAR(w.render_data.screen); Py_CLEAR(w.title);
+    REMOVER(tab->windows, id, tab->num_windows, EMPTY_WINDOW, Window, destroy_window);
+#undef destroy_window
+    END_WITH_TAB;
+}
+
+
+static inline void
+set_active_tab(unsigned int idx) {
+    global_state.active_tab = idx;
+}
+
+static inline void
+set_active_window(unsigned int tab_id, unsigned int idx) {
+    WITH_TAB(tab_id);
+    tab->active_window = idx;
+    END_WITH_TAB;
+}
+
+static inline void
+swap_tabs(unsigned int a, unsigned int b) {
+    Tab t = global_state.tabs[b];
+    global_state.tabs[b] = global_state.tabs[a];
+    global_state.tabs[a] = t;
+}
+
+static inline void
+swap_windows(unsigned int tab_id, unsigned int a, unsigned int b) {
+    WITH_TAB(tab_id);
+    Window w = tab->windows[b];
+    tab->windows[b] = tab->windows[a];
+    tab->windows[a] = w;
+    END_WITH_TAB;
+}
+
+// Python API {{{
+#define PYWRAP0(name) static PyObject* py##name(PyObject UNUSED *self)
+#define PYWRAP1(name) static PyObject* py##name(PyObject UNUSED *self, PyObject *args)
+#define PYWRAP2(name) static PyObject* py##name(PyObject UNUSED *self, PyObject *args, PyObject *kw)
+#define PA(fmt, ...) if(!PyArg_ParseTuple(args, fmt, __VA_ARGS__)) return NULL;
+#define ONE_UINT(name) PYWRAP1(name) { name((unsigned int)PyLong_AsUnsignedLong(args)); Py_RETURN_NONE; }
+#define TWO_UINT(name) PYWRAP1(name) { unsigned int a, b; PA("II", &a, &b); name(a, b); Py_RETURN_NONE; }
+#define THREE_UINT(name) PYWRAP1(name) { unsigned int a, b, c; PA("III", &a, &b, &c); name(a, b, c); Py_RETURN_NONE; }
+
+static inline color_type
+color_as_int(PyObject *color) {
+    if (!PyTuple_Check(color)) { PyErr_SetString(PyExc_TypeError, "Not a color tuple"); return 0; }
+#define I(n, s) ((PyLong_AsUnsignedLong(PyTuple_GET_ITEM(color, n)) & 0xff) << s)
+    return (I(0, 16) | I(1, 8) | I(2, 0)) & 0xffffff;
+#undef I
+}
+
+static inline double
+repaint_delay(PyObject *val) {
+    return (double)(PyLong_AsUnsignedLong(val)) / 1000.0;
+}
+
+#define dict_iter(d) { \
+    PyObject *key, *value; Py_ssize_t pos = 0; \
+    while (PyDict_Next(d, &pos, &key, &value))
+
+static inline void
+set_special_keys(PyObject *dict) {
+    dict_iter(dict) {
+        if (!PyTuple_Check(key)) { PyErr_SetString(PyExc_TypeError, "dict keys for special keys must be tuples"); return; }
+        int mods = PyLong_AsLong(PyTuple_GET_ITEM(key, 0));
+        int glfw_key = PyLong_AsLong(PyTuple_GET_ITEM(key, 1));
+        set_special_key_combo(glfw_key, mods);
+    }}
+}
+
+PYWRAP1(set_options) {
+    PyObject *ret;
+#define GA(name) ret = PyObject_GetAttrString(args, #name); if (ret == NULL) return NULL;
+#define S(name, convert) { GA(name); global_state.opts.name = convert(ret); Py_DECREF(ret); if (PyErr_Occurred()) return NULL; }
+    S(visual_bell_duration, PyFloat_AsDouble);
+    S(enable_audio_bell, PyObject_IsTrue);
+    S(cursor_blink_interval, PyFloat_AsDouble);
+    S(cursor_stop_blinking_after, PyFloat_AsDouble);
+    S(cursor_shape, PyLong_AsLong);
+    S(mouse_hide_wait, PyFloat_AsDouble);
+    S(wheel_scroll_multiplier, PyFloat_AsDouble);
+    S(open_url_modifiers, PyLong_AsUnsignedLong);
+    S(click_interval, PyFloat_AsDouble);
+    S(url_color, color_as_int);
+    S(repaint_delay, repaint_delay);
+    S(input_delay, repaint_delay);
+
+    PyObject *chars = PyObject_GetAttrString(args, "select_by_word_characters");
+    if (chars == NULL) return NULL;
+    for (size_t i = 0; i < MIN((size_t)PyUnicode_GET_LENGTH(chars), sizeof(OPT(select_by_word_characters))/sizeof(OPT(select_by_word_characters[0]))); i++) {
+        OPT(select_by_word_characters)[i] = PyUnicode_READ(PyUnicode_KIND(chars), PyUnicode_DATA(chars), i);
+    }
+    OPT(select_by_word_characters_count) = PyUnicode_GET_LENGTH(chars);
+
+    GA(keymap); set_special_keys(ret);
+    Py_DECREF(ret); if (PyErr_Occurred()) return NULL;
+    GA(send_text_map);
+    dict_iter(ret) {
+        set_special_keys(value);
+    }}
+    Py_DECREF(ret); if (PyErr_Occurred()) return NULL;
+
+    Py_DECREF(chars);
+#undef S
+    Py_RETURN_NONE;
+}
+
+PYWRAP1(set_tab_bar_render_data) {
+#define A(name) &(global_state.tab_bar_render_data.name)
+    Py_CLEAR(global_state.tab_bar_render_data.screen);
+    PA("iffffO", A(vao_idx), A(xstart), A(ystart), A(dx), A(dy), A(screen));
+    Py_INCREF(global_state.tab_bar_render_data.screen);
+    Py_RETURN_NONE;
+#undef A
+}
+
+PYWRAP1(set_window_render_data) {
+#define A(name) &(d.name)
+#define B(name) &(g.name)
+    unsigned int window_idx, tab_id;
+    static ScreenRenderData d = {0};
+    static WindowGeometry g = {0};
+    PA("IIiiffffOIIII", &tab_id, &window_idx, A(vao_idx), A(gvao_idx), A(xstart), A(ystart), A(dx), A(dy), A(screen), B(left), B(top), B(right), B(bottom));
+
+    WITH_TAB(tab_id);
+    Py_CLEAR(tab->windows[window_idx].render_data.screen);
+    tab->windows[window_idx].render_data = d;
+    tab->windows[window_idx].geometry = g;
+    Py_INCREF(tab->windows[window_idx].render_data.screen);
+    END_WITH_TAB;
+    Py_RETURN_NONE;
+#undef A
+#undef B
+}
+
+PYWRAP1(update_window_visibility) {
+    unsigned int window_idx, tab_id;
+    int visible;
+    PA("IIp", &tab_id, &window_idx, &visible);
+    WITH_TAB(tab_id);
+    tab->windows[window_idx].visible = visible & 1;
+    END_WITH_TAB;
+    Py_RETURN_NONE;
+}
+
+PYWRAP1(set_logical_dpi) {
+    PA("dd", &global_state.logical_dpi_x, &global_state.logical_dpi_y);
+    Py_RETURN_NONE;
+}
+
+PYWRAP1(set_boss) {
+    Py_CLEAR(global_state.boss);
+    global_state.boss = args;
+    Py_INCREF(global_state.boss);
+    Py_RETURN_NONE;
+}
+
+PYWRAP0(destroy_global_data) {
+    Py_CLEAR(global_state.tab_bar_render_data.screen);
+    Py_CLEAR(global_state.boss);
+    Py_RETURN_NONE;
+}
+
+PYWRAP1(set_display_state) {
+    PA("iiII", &global_state.viewport_width, &global_state.viewport_height, &global_state.cell_width, &global_state.cell_height);
+    Py_RETURN_NONE;
+}
+
+#define WF(name) PYWRAP1(name) { \
+    unsigned int tab_id, window_id; \
+    PyObject *title; \
+    PA("IIO", &tab_id, &window_id, &title); \
+    name(tab_id, window_id, title); \
+    Py_RETURN_NONE; \
+}
+WF(add_window)
+WF(update_window_title)
+
+ONE_UINT(add_tab)
+ONE_UINT(remove_tab)
+TWO_UINT(remove_window)
+ONE_UINT(set_active_tab)
+TWO_UINT(set_active_window)
+TWO_UINT(swap_tabs)
+THREE_UINT(swap_windows)
+
+#define M(name, arg_type) {#name, (PyCFunction)name, arg_type, NULL}
+#define MW(name, arg_type) {#name, (PyCFunction)py##name, arg_type, NULL}
+
+static PyMethodDef module_methods[] = {
+    MW(set_options, METH_O),
+    MW(set_logical_dpi, METH_VARARGS),
+    MW(add_tab, METH_O),
+    MW(add_window, METH_VARARGS),
+    MW(update_window_title, METH_VARARGS),
+    MW(remove_tab, METH_O),
+    MW(remove_window, METH_VARARGS),
+    MW(set_active_tab, METH_O),
+    MW(set_active_window, METH_VARARGS),
+    MW(swap_tabs, METH_VARARGS),
+    MW(swap_windows, METH_VARARGS),
+    MW(set_tab_bar_render_data, METH_VARARGS),
+    MW(set_window_render_data, METH_VARARGS),
+    MW(update_window_visibility, METH_VARARGS),
+    MW(set_boss, METH_O),
+    MW(set_display_state, METH_VARARGS),
+    MW(destroy_global_data, METH_NOARGS),
+
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+
+bool 
+init_state(PyObject *module) {
+    double now = monotonic();
+    global_state.application_focused = true;
+    global_state.cursor_blink_zero_time = now;
+    global_state.last_mouse_activity_at = now;
+    global_state.cell_width = 1; global_state.cell_height = 1;
+    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
+    return true;
+}
+// }}}

kitty/state.h

@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#pragma once
+#include "data-types.h"
+#include "screen.h"
+
+#define OPT(name) global_state.opts.name
+
+typedef struct {
+    double visual_bell_duration, cursor_blink_interval, cursor_stop_blinking_after, mouse_hide_wait, click_interval, wheel_scroll_multiplier;
+    bool enable_audio_bell;
+    CursorShape cursor_shape;
+    unsigned int open_url_modifiers;
+    char_type select_by_word_characters[256]; size_t select_by_word_characters_count;
+    color_type url_color;
+    double repaint_delay, input_delay;
+} Options;
+
+typedef struct {
+    ssize_t vao_idx, gvao_idx;
+    float xstart, ystart, dx, dy;
+    Screen *screen;
+} ScreenRenderData;
+
+typedef struct {
+    unsigned int left, top, right, bottom;
+} WindowGeometry;
+
+typedef struct {
+    double at;
+    int button, modifiers;
+} Click;
+
+#define CLICK_QUEUE_SZ 3
+typedef struct {
+    Click clicks[CLICK_QUEUE_SZ];
+    unsigned int length;
+} ClickQueue;
+
+typedef struct {
+    unsigned int id;
+    bool visible;
+    PyObject *title;
+    ScreenRenderData render_data;
+    unsigned int mouse_cell_x, mouse_cell_y;
+    WindowGeometry geometry;
+    ClickQueue click_queue;
+    double last_drag_scroll_at;
+} Window;
+
+typedef struct {
+    unsigned int id, active_window, num_windows;
+    Window windows[MAX_CHILDREN];
+} Tab;
+
+#define MAX_KEY_COUNT 512
+
+typedef struct {
+    Options opts;
+
+    Tab tabs[MAX_CHILDREN];
+    unsigned int active_tab, num_tabs;
+    ScreenRenderData tab_bar_render_data;
+    bool application_focused;
+    double cursor_blink_zero_time, last_mouse_activity_at;
+    double logical_dpi_x, logical_dpi_y;
+    double mouse_x, mouse_y;
+    bool mouse_button_pressed[20];
+    int viewport_width, viewport_height;
+    double viewport_x_ratio, viewport_y_ratio;
+    unsigned int cell_width, cell_height;
+    PyObject *application_title;
+    PyObject *boss;
+    bool is_key_pressed[MAX_KEY_COUNT];
+} GlobalState;
+
+extern GlobalState global_state;
+
+typedef struct {
+    bool is_visible;
+    CursorShape shape;
+    double left, right, top, bottom;
+    color_type color;
+} CursorRenderInfo;
+
+bool drag_scroll(Window *);
+
+#define call_boss(name, ...) { \
+    PyObject *cret_ = PyObject_CallMethod(global_state.boss, #name, __VA_ARGS__); \
+    if (cret_ == NULL) { PyErr_Print(); } \
+    else Py_DECREF(cret_); \
+}
+
+#define EXTERNAL_FUNC(name, ret, ...) typedef ret (*name##_func)(__VA_ARGS__); extern name##_func name
+#define EXTERNAL_FUNC0(name, ret) typedef ret (*name##_func)(); extern name##_func name
+EXTERNAL_FUNC0(draw_borders, void);
+EXTERNAL_FUNC(draw_cells, void, ssize_t, ssize_t, float, float, float, float, Screen *, CursorRenderInfo *);
+EXTERNAL_FUNC(draw_cursor, void, CursorRenderInfo *);
+EXTERNAL_FUNC(update_viewport_size, void, int, int);
+EXTERNAL_FUNC(free_texture, void, uint32_t*);
+EXTERNAL_FUNC(send_image_to_gpu, void, uint32_t*, const void*, int32_t, int32_t, bool, bool);

kitty/tabs.py

@@ -2,50 +2,64 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-from collections import deque
+from collections import deque, namedtuple
 from functools import partial
-from threading import Lock
-from ctypes import addressof
+from itertools import count
 
+from .borders import Borders
 from .child import Child
 from .config import build_ansi_color_table
-from .constants import get_boss, appname, shell_path, cell_size, queue_action, viewport_size, WindowGeometry, GLuint
-from .fast_data_types import glfw_post_empty_event, Screen, DECAWM, DATA_CELL_SIZE, ColorProfile
-from .char_grid import calculate_gl_geometry, render_cells
-from .layout import all_layouts, Rect
+from .constants import (
+    WindowGeometry, appname, cell_size, get_boss, shell_path, viewport_size
+)
+from .fast_data_types import (
+    DECAWM, Screen, add_tab, add_window, create_cell_vao,
+    glfw_post_empty_event, remove_tab, remove_window, set_active_tab,
+    set_active_window, set_tab_bar_render_data, swap_tabs, swap_windows
+)
+from .layout import Rect, all_layouts
 from .utils import color_as_int
-from .borders import Borders
-from .window import Window
+from .window import Window, calculate_gl_geometry
+
+TabbarData = namedtuple('TabbarData', 'title is_active is_last')
+borders = None
+tab_counter = count()
+next(tab_counter)
 
 
 def SpecialWindow(cmd, stdin=None, override_title=None):
     return (cmd, stdin, override_title)
 
 
-class Tab:
+class Tab:  # {{{
 
     def __init__(self, opts, args, on_title_change, session_tab=None, special_window=None):
+        global borders
+        self.id = next(tab_counter)
+        add_tab(self.id)
         self.opts, self.args = opts, args
         self.name = getattr(session_tab, 'name', '')
         self.on_title_change = on_title_change
-        self.enabled_layouts = list((session_tab or opts).enabled_layouts)
-        self.borders = Borders(opts)
+        self.enabled_layouts = list(getattr(session_tab, 'enabled_layouts', None) or opts.enabled_layouts)
+        if borders is None:
+            borders = Borders(opts)
         self.windows = deque()
         self.active_window_idx = 0
+        for i, which in enumerate('first second third fourth fifth sixth seventh eighth ninth tenth'.split()):
+            setattr(self, which + '_window', partial(self.nth_window, num=i))
         if session_tab is None:
             self.cwd = args.directory
             l = self.enabled_layouts[0]
+            self.current_layout = all_layouts[l](opts, borders.border_width, self.windows)
             if special_window is None:
-                queue_action(self.new_window)
+                self.new_window()
             else:
-                queue_action(self.new_special_window, special_window)
+                self.new_special_window(special_window)
         else:
             self.cwd = session_tab.cwd or args.directory
             l = session_tab.layout
-            queue_action(self.startup, session_tab)
-        self.current_layout = all_layouts[l](opts, self.borders.border_width, self.windows)
-        for i, which in enumerate('first second third fourth fifth sixth seventh eighth ninth tenth'.split()):
-            setattr(self, which + '_window', partial(self.nth_window, num=i))
+            self.current_layout = all_layouts[l](opts, borders.border_width, self.windows)
+            self.startup(session_tab)
 
     def startup(self, session_tab):
         for cmd in session_tab.windows:
@@ -80,8 +94,8 @@ class Tab:
 
     def relayout_borders(self):
         tm = get_boss().tab_manager
-        self.borders(self.windows, self.active_window, self.current_layout, tm.blank_rects,
-                     self.current_layout.needs_window_borders and len(self.windows) > 1)
+        borders(self.windows, self.active_window, self.current_layout, tm.blank_rects,
+                self.current_layout.needs_window_borders and len(self.windows) > 1)
 
     def next_layout(self):
         if len(self.opts.enabled_layouts) > 1:
@@ -90,9 +104,9 @@ class Tab:
             except Exception:
                 idx = -1
             nl = self.opts.enabled_layouts[(idx + 1) % len(self.opts.enabled_layouts)]
-            self.current_layout = all_layouts[nl](self.opts, self.borders.border_width, self.windows)
-            for w in self.windows:
-                w.is_visible_in_layout = True
+            self.current_layout = all_layouts[nl](self.opts, borders.border_width, self.windows)
+            for i, w in enumerate(self.windows):
+                w.set_visible_in_layout(i, True)
             self.relayout()
 
     def launch_child(self, use_shell=False, cmd=None, stdin=None):
@@ -110,8 +124,11 @@ class Tab:
         window = Window(self, child, self.opts, self.args)
         if override_title is not None:
             window.title = window.override_title = override_title
-        get_boss().add_child_fd(child.child_fd, window.read_ready, window.write_ready)
+        # Must add child before laying out so that resize_pty succeeds
+        get_boss().add_child(window)
+        add_window(self.id, window.id, window.override_title or window.title or appname)
         self.active_window_idx = self.current_layout.add_window(self.windows, window, self.active_window_idx)
+        set_active_window(self.id, self.active_window_idx)
         self.relayout_borders()
         glfw_post_empty_event()
         return window
@@ -124,7 +141,9 @@ class Tab:
             self.remove_window(self.windows[self.active_window_idx])
 
     def remove_window(self, window):
+        remove_window(self.id, window.id)
         self.active_window_idx = self.current_layout.remove_window(self.windows, window, self.active_window_idx)
+        set_active_window(self.id, self.active_window_idx)
         self.relayout_borders()
         glfw_post_empty_event()
 
@@ -132,6 +151,7 @@ class Tab:
         if idx != self.active_window_idx:
             self.current_layout.set_active_window(self.windows, idx)
             self.active_window_idx = idx
+            set_active_window(self.id, self.active_window_idx)
             self.relayout_borders()
             glfw_post_empty_event()
 
@@ -149,6 +169,7 @@ class Tab:
     def _next_window(self, delta=1):
         if len(self.windows) > 1:
             self.active_window_idx = self.current_layout.next_window(self.windows, self.active_window_idx, delta)
+            set_active_window(self.id, self.active_window_idx)
             self.relayout_borders()
             glfw_post_empty_event()
 
@@ -163,7 +184,9 @@ class Tab:
             idx = self.active_window_idx
             nidx = (idx + len(self.windows) + delta) % len(self.windows)
             self.windows[nidx], self.windows[idx] = self.windows[idx], self.windows[nidx]
+            swap_windows(self.id, nidx, idx)
             self.active_window_idx = nidx
+            set_active_window(self.id, self.active_window_idx)
             self.relayout()
 
     def move_window_to_top(self):
@@ -185,75 +208,135 @@ class Tab:
         return window in self.windows
 
     def destroy(self):
-        if hasattr(self, 'windows'):
-            for w in self.windows:
-                w.destroy()
-            del self.windows
-
-    def render(self):
-        self.borders.render(get_boss().borders_program)
+        self.windows = deque()
 
     def __repr__(self):
         return 'Tab(title={}, id={})'.format(self.name or self.title, hex(id(self)))
+# }}}
 
 
-class TabManager:
+class TabBar:  # {{{
 
-    def __init__(self, opts, args, startup_session):
-        self.opts, self.args = opts, args
-        self.buffer_id = None
-        self.tabbar_lock = Lock()
-        self.tabs = [Tab(opts, args, self.title_changed, t) for t in startup_session.tabs]
-        self.cell_ranges = []
-        self.active_tab_idx = startup_session.active_tab_idx
-        self.tabbar_dirty = True
-        self.color_profile = ColorProfile()
-        self.color_profile.update_ansi_color_table(build_ansi_color_table(opts))
-        self.default_fg = color_as_int(opts.inactive_tab_foreground)
-        self.default_bg = color_as_int(opts.inactive_tab_background)
-        self.tab_bar_blank_rects = ()
+    def __init__(self, opts):
+        self.num_tabs = 1
+        self.cell_width = 1
+        self.data_buffer_size = 0
+        self.vao_id = create_cell_vao()
+        self.layout_changed = None
+        self.dirty = True
+        self.screen = s = Screen(None, 1, 10)
+        s.color_profile.update_ansi_color_table(build_ansi_color_table(opts))
+        s.color_profile.set_configured_colors(
+            color_as_int(opts.inactive_tab_foreground),
+            color_as_int(opts.inactive_tab_background)
+        )
+        self.blank_rects = ()
 
         def as_rgb(x):
             return (x << 8) | 2
 
         self.active_bg = as_rgb(color_as_int(opts.active_tab_background))
         self.active_fg = as_rgb(color_as_int(opts.active_tab_foreground))
-        self.can_render = False
+
+    def layout(self, viewport_width, viewport_height, cell_width, cell_height):
+        self.cell_width = cell_width
+        s = self.screen
+        ncells = viewport_width // cell_width
+        s.resize(1, ncells)
+        s.reset_mode(DECAWM)
+        self.layout_changed = True
+        margin = (viewport_width - ncells * cell_width) // 2
+        self.window_geometry = g = WindowGeometry(
+            margin, viewport_height - cell_height, viewport_width - margin, viewport_height, s.columns, s.lines)
+        if margin > 0:
+            self.tab_bar_blank_rects = (Rect(0, g.top, g.left, g.bottom), Rect(g.right - 1, g.top, viewport_width, g.bottom))
+        else:
+            self.tab_bar_blank_rects = ()
+        self.screen_geometry = sg = calculate_gl_geometry(g, viewport_width, viewport_height, cell_width, cell_height)
+        set_tab_bar_render_data(self.vao_id, sg.xstart, sg.ystart, sg.dx, sg.dy, self.screen)
+
+    def update(self, data):
+        if self.layout_changed is None:
+            return
+        s = self.screen
+        s.cursor.x = 0
+        s.erase_in_line(2, False)
+        max_title_length = (self.screen_geometry.xnum // max(1, len(data))) - 1
+        cr = []
+
+        for t in data:
+            s.cursor.bg = self.active_bg if t.is_active else 0
+            s.cursor.fg = self.active_fg if t.is_active else 0
+            s.cursor.bold = s.cursor.italic = t.is_active
+            before = s.cursor.x
+            s.draw(t.title)
+            extra = s.cursor.x - before - max_title_length
+            if extra > 0:
+                s.cursor.x -= extra + 1
+                s.draw('…')
+            cr.append((before, s.cursor.x))
+            s.cursor.bold = s.cursor.italic = False
+            s.cursor.fg = s.cursor.bg = 0
+            s.draw('┇')
+            if s.cursor.x > s.columns - max_title_length and not t.is_last:
+                s.draw('…')
+                break
+        s.erase_in_line(0, False)  # Ensure no long titles bleed after the last tab
+        self.cell_ranges = cr
+        glfw_post_empty_event()
+
+    def tab_at(self, x):
+        x = (x - self.window_geometry.left) // self.cell_width
+        for i, (a, b) in enumerate(self.cell_ranges):
+            if a <= x <= b:
+                return i
+# }}}
+
+
+class TabManager:  # {{{
+
+    def __init__(self, opts, args):
+        self.opts, self.args = opts, args
+        self.tabs = []
+        self.tab_bar = TabBar(opts)
+        self.refresh_sprite_positions = self.tab_bar.screen.refresh_sprite_positions
+        self.tab_bar.layout(*self.tab_bar_layout_data)
+        self.active_tab_idx = 0
+
+    def _add_tab(self, tab):
+        self.tabs.append(tab)
+
+    def _remove_tab(self, tab):
+        remove_tab(tab.id)
+        self.tabs.remove(tab)
+
+    def _set_active_tab(self, idx):
+        self.active_tab_idx = idx
+        set_active_tab(idx)
+
+    def init(self, startup_session):
+        for t in startup_session.tabs:
+            self._add_tab(Tab(self.opts, self.args, self.title_changed, t))
+        self._set_active_tab(max(0, min(startup_session.active_tab_idx, len(self.tabs) - 1)))
         if len(self.tabs) > 1:
-            self.layout_tab_bar()
+            get_boss().tabbar_visibility_changed()
+        self.update_tab_bar()
+
+    def update_tab_bar(self):
+        if len(self.tabs) > 1:
+            self.tab_bar.update(self.tab_bar_data)
 
     def resize(self, only_tabs=False):
         if not only_tabs:
-            self.layout_tab_bar()
+            self.tab_bar.layout(*self.tab_bar_layout_data)
+            self.update_tab_bar()
         for tab in self.tabs:
             tab.relayout()
 
-    def layout_tab_bar(self):
-        self.can_render = False
-        ncells = viewport_size.width // cell_size.width
-        s = Screen(None, 1, ncells)
-        s.reset_mode(DECAWM)
-        self.sprite_map_type = (GLuint * (s.lines * s.columns * DATA_CELL_SIZE))
-        with self.tabbar_lock:
-            self.sprite_map = self.sprite_map_type()
-            self.tab_bar_screen = s
-            self.tabbar_dirty = True
-        margin = (viewport_size.width - ncells * cell_size.width) // 2
-        self.window_geometry = g = WindowGeometry(
-            margin, viewport_size.height - cell_size.height, viewport_size.width - margin, viewport_size.height, s.columns, s.lines)
-        if margin > 0:
-            self.tab_bar_blank_rects = (Rect(0, g.top, g.left, g.bottom), Rect(g.right - 1, g.top, viewport_size.width, g.bottom))
-        else:
-            self.tab_bar_blank_rects = ()
-        self.screen_geometry = calculate_gl_geometry(g)
-        self.screen = s
-        self.can_render = True
-
     def set_active_tab(self, idx):
-        self.active_tab_idx = idx
-        self.tabbar_dirty = True
+        self._set_active_tab(idx)
         self.active_tab.relayout_borders()
-        glfw_post_empty_event()
+        self.update_tab_bar()
 
     def next_tab(self, delta=1):
         if len(self.tabs) > 1:
@@ -278,83 +361,55 @@ class TabManager:
             idx = self.active_tab_idx
             nidx = (idx + len(self.tabs) + delta) % len(self.tabs)
             self.tabs[idx], self.tabs[nidx] = self.tabs[nidx], self.tabs[idx]
-            self.active_tab_idx = nidx
-            glfw_post_empty_event()
+            swap_tabs(idx, nidx)
+            self._set_active_tab(nidx)
+            self.update_tab_bar()
 
     def title_changed(self, new_title):
-        with self.tabbar_lock:
-            self.tabbar_dirty = True
+        self.update_tab_bar()
 
     def new_tab(self, special_window=None):
-        ' Must be called in the GUI thread '
         needs_resize = len(self.tabs) == 1
-        self.active_tab_idx = len(self.tabs)
-        self.tabs.append(Tab(self.opts, self.args, self.title_changed, special_window=special_window))
+        idx = len(self.tabs)
+        self._add_tab(Tab(self.opts, self.args, self.title_changed, special_window=special_window))
+        self._set_active_tab(idx)
+        self.update_tab_bar()
         if needs_resize:
-            if not self.can_render:
-                queue_action(self.layout_tab_bar)
-            queue_action(get_boss().tabbar_visibility_changed)
+            get_boss().tabbar_visibility_changed()
 
     def remove(self, tab):
-        ' Must be called in the GUI thread '
         needs_resize = len(self.tabs) == 2
-        self.tabs.remove(tab)
-        self.active_tab_idx = max(0, min(self.active_tab_idx, len(self.tabs) - 1))
-        self.tabbar_dirty = True
+        self._remove_tab(tab)
+        self._set_active_tab(max(0, min(self.active_tab_idx, len(self.tabs) - 1)))
+        self.update_tab_bar()
         tab.destroy()
         if needs_resize:
-            queue_action(get_boss().tabbar_visibility_changed)
+            get_boss().tabbar_visibility_changed()
 
-    def update_tab_bar_data(self, sprites):
-        s = self.tab_bar_screen
-        s.cursor.x = 0
-        s.erase_in_line(2, False)
+    @property
+    def tab_bar_layout_data(self):
+        return viewport_size.width, viewport_size.height, cell_size.width, cell_size.height
+
+    @property
+    def tab_bar_data(self):
         at = self.active_tab
-        max_title_length = (self.screen_geometry.xnum // len(self.tabs)) - 1
-        self.cell_ranges = []
-
+        ans = []
         for t in self.tabs:
             title = (t.name or t.title or appname) + ' '
-            s.cursor.bg = self.active_bg if t is at else 0
-            s.cursor.fg = self.active_fg if t is at else 0
-            s.cursor.bold = s.cursor.italic = t is at
-            before = s.cursor.x
-            s.draw(title)
-            extra = s.cursor.x - before - max_title_length
-            if extra > 0:
-                s.cursor.x -= extra + 1
-                s.draw('…')
-            self.cell_ranges.append((before, s.cursor.x))
-            s.cursor.bold = s.cursor.italic = False
-            s.cursor.fg = s.cursor.bg = 0
-            s.draw('┇')
-            if s.cursor.x > s.columns - max_title_length:
-                s.draw('…')
-                break
-        s.update_cell_data(
-            sprites.backend, self.color_profile, addressof(self.sprite_map), self.default_fg, self.default_bg, True)
-        sprites.render_dirty_cells()
-        if self.buffer_id is None:
-            self.buffer_id = sprites.add_sprite_map()
-        sprites.set_sprite_map(self.buffer_id, self.sprite_map)
+            ans.append(TabbarData(title, t is at, t is self.tabs[-1]))
+        return ans
 
     def activate_tab_at(self, x):
-        x = (x - self.window_geometry.left) // cell_size.width
-        for i, (a, b) in enumerate(self.cell_ranges):
-            if a <= x <= b:
-                queue_action(self.set_active_tab, i)
-                return
+        i = self.tab_bar.tab_at(x)
+        if i is not None:
+            self.set_active_tab(i)
 
     @property
     def blank_rects(self):
-        if len(self.tabs) < 2:
-            return ()
-        return self.tab_bar_blank_rects
+        return self.tab_bar.blank_rects if len(self.tabs) > 1 else ()
 
-    def render(self, cell_program, sprites):
-        if not self.can_render or len(self.tabs) < 2:
+    def render(self):
+        if len(self.tabs) < 2:
             return
-        with self.tabbar_lock:
-            if self.tabbar_dirty:
-                self.update_tab_bar_data(sprites)
-        render_cells(self.buffer_id, self.screen_geometry, cell_program, sprites)
+        self.tab_bar.render()
+# }}}

kitty/timers.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
-
-from collections import namedtuple
-from operator import itemgetter
-from time import monotonic
-
-from .utils import safe_print
-
-Event = namedtuple('Event', 'at callback args')
-get_at = itemgetter(0)
-
-
-class Timers:
-
-    def __init__(self):
-        self.timers = []
-
-    def _add(self, delay, callback, args):
-        self.timers.append(Event(monotonic() + delay, callback, args))
-        self.timers.sort(key=get_at)
-
-    def add(self, delay, callback, *args):
-        self.remove(callback)
-        self._add(delay, callback, args)
-
-    def add_if_missing(self, delay, callback, *args):
-        for ev in self.timers:
-            if ev.callback == callback:
-                return
-        self._add(delay, callback, args)
-
-    def remove(self, callback):
-        for i, ev in enumerate(self.timers):
-            if ev.callback == callback:
-                break
-        else:
-            return
-        del self.timers[i]
-
-    def timeout(self):
-        if self.timers:
-            return max(0, self.timers[0][0] - monotonic())
-
-    def __call__(self):
-        if self.timers:
-            now = monotonic()
-            expired_timers, waiting_timers = [], []
-            for ev in self.timers:
-                (expired_timers if ev[0] <= now else waiting_timers).append(ev)
-            self.timers = waiting_timers
-            for ev in expired_timers:
-                    try:
-                        ev.callback(*ev.args)
-                    except Exception:
-                        import traceback
-                        safe_print(traceback.format_exc())

kitty/tracker.c

@@ -1,267 +0,0 @@
-/*
- * tracker.c
- * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "data-types.h"
-#include "tracker.h"
-
-
-bool tracker_resize(ChangeTracker *self, unsigned int ynum, unsigned int xnum) {
-#define ALLOC_VAR(name, sz) \
-    bool *name = PyMem_Calloc(sz, sizeof(bool)); \
-    if (name == NULL) { PyErr_NoMemory(); return false; } \
-    PyMem_Free(self->name); self->name = name;
-
-    self->ynum = ynum; self->xnum = xnum;
-    ALLOC_VAR(changed_lines, self->ynum);
-    ALLOC_VAR(changed_cells, self->xnum * self->ynum);
-    ALLOC_VAR(lines_with_changed_cells, self->ynum);
-    RESET_STATE_VARS(self);
-    return true;
-}
-
-static PyObject*
-resize(ChangeTracker *self, PyObject *args) {
-#define resize_doc "Resize this change tracker must be called when the screen it is tracking for is resized"
-    unsigned int ynum=1, xnum=1;
-    if (!PyArg_ParseTuple(args, "|II", &ynum, &xnum)) return NULL;
-    if (!tracker_resize(self, ynum, xnum)) return NULL;
-    Py_RETURN_NONE;
-}
-
-static PyObject*
-new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
-    ChangeTracker *self;
-    self = (ChangeTracker *)type->tp_alloc(type, 0);
-    if (self != NULL) {
-        PyObject *ret = resize(self, args);
-        if (ret == NULL) { Py_CLEAR(self); return NULL; }
-        Py_CLEAR(ret);
-    }
-    return (PyObject*) self;
-}
-
-static void
-dealloc(ChangeTracker* self) {
-    PyMem_Free(self->changed_lines); PyMem_Free(self->changed_cells); PyMem_Free(self->lines_with_changed_cells);
-    Py_TYPE(self)->tp_free((PyObject*)self);
-}
-
-static PyObject*
-reset(ChangeTracker *self) {
-#define reset_doc "Reset all changes"
-    tracker_reset(self);
-
-    Py_RETURN_NONE;
-}
-
-static PyObject*
-cursor_changed(ChangeTracker *self) {
-#define cursor_changed_doc ""
-    tracker_cursor_changed(self);
-    Py_RETURN_NONE;
-}
-
-static PyObject*
-line_added_to_history(ChangeTracker *self) {
-#define line_added_to_history_doc ""
-    tracker_line_added_to_history(self);
-    Py_RETURN_NONE;
-}
-
-static PyObject*
-update_screen(ChangeTracker *self) {
-#define update_screen_doc ""
-    tracker_update_screen(self);
-    Py_RETURN_NONE;
-}
-
-static PyObject*
-update_line_range(ChangeTracker *self, PyObject *args) {
-#define update_line_range_doc ""
-    unsigned int f, l;
-    if (!PyArg_ParseTuple(args, "II", &f, &l)) return NULL;
-    tracker_update_line_range(self, f, l);
-    Py_RETURN_NONE;
-}
-
-static PyObject*
-update_cell_range(ChangeTracker *self, PyObject *args) {
-#define update_cell_range_doc ""
-    unsigned int line, f, l;
-    if (!PyArg_ParseTuple(args, "III", &line, &f, &l)) return NULL;
-    tracker_update_cell_range(self, line, f, l);
-    Py_RETURN_NONE;
-}
-
-bool tracker_update_cell_data(ScreenModes *modes, ChangeTracker *self, LineBuf *lb, SpriteMap *spm, ColorProfile *color_profile, unsigned int *data, unsigned long default_fg, unsigned long default_bg, bool force_screen_refresh) {
-    unsigned int y;
-    Py_ssize_t start;
-    default_fg &= COL_MASK;
-    default_bg &= COL_MASK;
-
-#define UPDATE_RANGE(xstart, xmax) \
-    linebuf_init_line(lb, y); \
-    if (!update_cell_range_data(modes, spm, lb->line, (xstart), (xmax), color_profile, default_bg, default_fg, data)) return false;
-
-    if (self->screen_changed || force_screen_refresh) {
-        for (y = 0; y < self->ynum; y++) {
-            UPDATE_RANGE(0, self->xnum - 1);
-        }
-    } else {
-        for (y = 0; y < self->ynum; y++) {
-            if (self->changed_lines[y]) {
-                UPDATE_RANGE(0, self->xnum - 1);
-            } else if (self->lines_with_changed_cells[y]) {
-                start = -1;
-                bool *line = self->changed_cells + y * self->xnum;
-                for (unsigned int i = 0; i < self->xnum; i++) {
-                    if (line[i]) {
-                        if (start == -1) {
-                            start = i;
-                        }
-                    } else {
-                        if (start != -1) {
-                            UPDATE_RANGE(start, i - 1);
-                            start = -1;
-                        }
-                    }
-                }
-                if (start != -1) {
-                    UPDATE_RANGE(start, self->xnum - 1);
-                }
-            }
-        }
-    }
-    tracker_reset(self);
-    return true;
-}
-
-static inline PyObject*
-get_ranges(bool *line, unsigned int xnum) {
-    PyObject *ans = PyList_New(0), *t;
-    Py_ssize_t start = -1;
-    if (ans == NULL) return PyErr_NoMemory();
-
-#define APPEND_RANGE(x) \
-    t = Py_BuildValue("nI", start, x); \
-    if (t == NULL) { Py_CLEAR(ans); return NULL; } \
-    if (PyList_Append(ans, t) != 0) { Py_CLEAR(ans); Py_CLEAR(t); return NULL; } \
-    Py_CLEAR(t);
-
-    for (unsigned int i = 0; i < xnum; i++) {
-        if (line[i]) {
-            if (start == -1) {
-                start = i;
-            }
-        } else {
-            if (start != -1) {
-                APPEND_RANGE(i - 1);
-                start = -1;
-            }
-        }
-    }
-    if (start != -1) {
-        APPEND_RANGE(xnum - 1);
-    }
-
-    return ans;
-}
-
-PyObject*
-tracker_consolidate_changes(ChangeTracker *self) {
-    PyObject *ans = PyDict_New();
-    if (ans == NULL) return PyErr_NoMemory();
-    if (PyDict_SetItemString(ans, "screen", self->screen_changed ? Py_True : Py_False) != 0) { Py_CLEAR(ans); return NULL; }
-    if (PyDict_SetItemString(ans, "cursor", self->cursor_changed ? Py_True : Py_False) != 0) { Py_CLEAR(ans); return NULL; }
-    PyObject *t = PyLong_FromUnsignedLong((unsigned long)self->history_line_added_count);
-    if (t == NULL) { Py_CLEAR(ans); return NULL; }
-    if (PyDict_SetItemString(ans, "history_line_added_count", t) != 0) { Py_CLEAR(t); Py_CLEAR(ans); return NULL; }
-    Py_CLEAR(t);
-
-    // Changed lines
-    Py_ssize_t num = 0;
-    if (!self->screen_changed) {
-        for (unsigned int i = 0; i < self->ynum; i++) { num += self->changed_lines[i]; }
-    }
-    t = PyTuple_New(num);
-    if (t == NULL) { Py_CLEAR(ans); return NULL; }
-    if (num > 0) {
-        for (unsigned int i = 0, j=0; i < self->ynum; i++) { 
-            if (self->changed_lines[i]) {
-                PyObject *n = PyLong_FromUnsignedLong(i);
-                if (n == NULL) { Py_CLEAR(t); Py_CLEAR(ans); return NULL; }
-                PyTuple_SET_ITEM(t, j++, n);
-            }
-        }
-    }
-    if (PyDict_SetItemString(ans, "lines", t) != 0) { Py_CLEAR(t); Py_CLEAR(ans); return NULL; }
-    Py_CLEAR(t);
-
-    // Changed cells
-    t = PyDict_New();
-    if (t == NULL) { Py_CLEAR(ans); return PyErr_NoMemory(); }
-    if (!self->screen_changed) {
-        for (unsigned int i = 0; i < self->ynum; i++) { 
-            if (self->lines_with_changed_cells[i] && !self->changed_lines[i]) {
-                PyObject *ranges = get_ranges(self->changed_cells + i * self->xnum, self->xnum);
-                if (ranges == NULL) { Py_CLEAR(t); Py_CLEAR(ans); return NULL; }
-                PyObject *key = PyLong_FromUnsignedLong(i);
-                if (key == NULL) { Py_CLEAR(t); Py_CLEAR(ans); Py_CLEAR(ranges); return NULL; }
-                if (PyDict_SetItem(t, key, ranges) != 0) { Py_CLEAR(key); Py_CLEAR(t); Py_CLEAR(ans); Py_CLEAR(ranges); return NULL; }
-                Py_CLEAR(key); Py_CLEAR(ranges);
-            }
-        }
-    }
-
-    if (PyDict_SetItemString(ans, "cells", t) != 0) { Py_CLEAR(t); Py_CLEAR(ans); return NULL; }
-    Py_CLEAR(t);
-
-    tracker_reset(self);
-    return ans;
-}
-
-// Boilerplate {{{
-
-BOOL_GETSET(ChangeTracker, dirty)
-static PyGetSetDef getseters[] = {
-    GETSET(dirty)
-    {NULL}  /* Sentinel */
-};
-
-static PyMethodDef methods[] = {
-    METHOD(resize, METH_VARARGS)
-    METHOD(reset, METH_NOARGS)
-    METHOD(cursor_changed, METH_NOARGS)
-    {"consolidate_changes", (PyCFunction)tracker_consolidate_changes, METH_NOARGS, ""},
-    METHOD(line_added_to_history, METH_NOARGS)
-    METHOD(update_screen, METH_NOARGS)
-    METHOD(update_line_range, METH_VARARGS)
-    METHOD(update_cell_range, METH_VARARGS)
-    {NULL}  /* Sentinel */
-};
-
-
-PyTypeObject ChangeTracker_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    .tp_name = "fast_data_types.ChangeTracker",
-    .tp_basicsize = sizeof(ChangeTracker),
-    .tp_dealloc = (destructor)dealloc, 
-    .tp_flags = Py_TPFLAGS_DEFAULT,        
-    .tp_doc = "ChangeTracker",
-    .tp_methods = methods,
-    .tp_getset = getseters,
-    .tp_new = new,                
-};
-
-INIT_TYPE(ChangeTracker)
-// }}}
-
-ChangeTracker* alloc_change_tracker(unsigned int ynum, unsigned int xnum) {
-    ChangeTracker *self = (ChangeTracker *)(&ChangeTracker_Type)->tp_alloc((&ChangeTracker_Type), 0);
-    if (!tracker_resize(self, ynum, xnum)) { Py_CLEAR(self); return NULL; }
-    return self;
-}

kitty/tracker.h

@@ -1,55 +0,0 @@
-/*
- * tracker.h
- * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#pragma once
-
-static inline void tracker_cursor_changed(ChangeTracker *self) {
-    self->cursor_changed = true;
-    self->dirty = true;
-}
-
-static inline void tracker_line_added_to_history(ChangeTracker *self) {
-    self->history_line_added_count++;
-    self->dirty = true;
-}
-
-static inline void tracker_update_screen(ChangeTracker *self) {
-    self->screen_changed = true;
-    self->dirty = true;
-}
-
-static inline void tracker_update_line_range(ChangeTracker *self, unsigned int first_line, unsigned int last_line) {
-    if (!self->screen_changed) {
-        for (unsigned int i = first_line; i <= MIN(self->ynum - 1, last_line); i++) self->changed_lines[i] = true;
-        self->dirty = true;
-    }
-}
-
-static inline void tracker_update_cell_range(ChangeTracker *self, unsigned int line, unsigned int first_cell, unsigned int last_cell) {
-    if (!self->screen_changed && line < self->ynum && !self->changed_lines[line]) {
-        self->lines_with_changed_cells[line] = true;
-        unsigned int base = line * self->xnum;
-        for (unsigned int i = first_cell; i <= MIN(self->xnum - 1, last_cell); i++) self->changed_cells[base + i] = true;
-        self->dirty = true;
-    }
-}
-
-#define RESET_STATE_VARS(self) \
-    self->screen_changed = false; self->cursor_changed = false; self->dirty = false; self->history_line_added_count = 0; 
-
-static inline void tracker_reset(ChangeTracker *self) {
-    self->screen_changed = false; self->cursor_changed = false; self->dirty = false;
-    self->history_line_added_count = 0;
-    memset(self->changed_lines, 0, self->ynum * sizeof(bool));
-    memset(self->changed_cells, 0, self->ynum * self->xnum * sizeof(bool));
-    memset(self->lines_with_changed_cells, 0, self->ynum * sizeof(bool));
-    RESET_STATE_VARS(self);
-}
-
-PyObject* tracker_consolidate_changes(ChangeTracker *self);
-bool tracker_resize(ChangeTracker *self, unsigned int ynum, unsigned int xnum);
-bool tracker_update_cell_data(ScreenModes*, ChangeTracker *, LineBuf *, SpriteMap *, ColorProfile *, unsigned int *, unsigned long, unsigned long, bool);

kitty/unicode-data.h

@@ -1,11 +1,41 @@
 #pragma once
 
-static inline bool is_combining_char(uint32_t ch) {
-    return (0x300 <= ch && ch <= 0x34e)||(0x350 <= ch && ch <= 0x36f)||(0x483 <= ch && ch <= 0x487)||(0x591 <= ch && ch <= 0x5bd)||(0x5c1 <= ch && ch <= 0x5c2)||(0x5c4 <= ch && ch <= 0x5c5)||(0x610 <= ch && ch <= 0x61a)||(0x64b <= ch && ch <= 0x65f)||(0x6d6 <= ch && ch <= 0x6dc)||(0x6df <= ch && ch <= 0x6e4)||(0x6e7 <= ch && ch <= 0x6e8)||(0x6ea <= ch && ch <= 0x6ed)||(0x730 <= ch && ch <= 0x74a)||(0x7eb <= ch && ch <= 0x7f3)||(0x816 <= ch && ch <= 0x819)||(0x81b <= ch && ch <= 0x823)||(0x825 <= ch && ch <= 0x827)||(0x829 <= ch && ch <= 0x82d)||(0x859 <= ch && ch <= 0x85b)||(0x8e3 <= ch && ch <= 0x8ff)||(0x951 <= ch && ch <= 0x954)||(0xc55 <= ch && ch <= 0xc56)||(0xe38 <= ch && ch <= 0xe3a)||(0xe48 <= ch && ch <= 0xe4b)||(0xeb8 <= ch && ch <= 0xeb9)||(0xec8 <= ch && ch <= 0xecb)||(0xf18 <= ch && ch <= 0xf19)||(0xf71 <= ch && ch <= 0xf72)||(0xf7a <= ch && ch <= 0xf7d)||(0xf82 <= ch && ch <= 0xf84)||(0xf86 <= ch && ch <= 0xf87)||(0x1039 <= ch && ch <= 0x103a)||(0x135d <= ch && ch <= 0x135f)||(0x1939 <= ch && ch <= 0x193b)||(0x1a17 <= ch && ch <= 0x1a18)||(0x1a75 <= ch && ch <= 0x1a7c)||(0x1ab0 <= ch && ch <= 0x1abd)||(0x1b6b <= ch && ch <= 0x1b73)||(0x1baa <= ch && ch <= 0x1bab)||(0x1bf2 <= ch && ch <= 0x1bf3)||(0x1cd0 <= ch && ch <= 0x1cd2)||(0x1cd4 <= ch && ch <= 0x1ce0)||(0x1ce2 <= ch && ch <= 0x1ce8)||(0x1cf8 <= ch && ch <= 0x1cf9)||(0x1dc0 <= ch && ch <= 0x1df5)||(0x1dfc <= ch && ch <= 0x1dff)||(0x20d0 <= ch && ch <= 0x20dc)||(0x20e5 <= ch && ch <= 0x20f0)||(0x2cef <= ch && ch <= 0x2cf1)||(0x2de0 <= ch && ch <= 0x2dff)||(0x302a <= ch && ch <= 0x302f)||(0x3099 <= ch && ch <= 0x309a)||(0xa674 <= ch && ch <= 0xa67d)||(0xa69e <= ch && ch <= 0xa69f)||(0xa6f0 <= ch && ch <= 0xa6f1)||(0xa8e0 <= ch && ch <= 0xa8f1)||(0xa92b <= ch && ch <= 0xa92d)||(0xaab2 <= ch && ch <= 0xaab4)||(0xaab7 <= ch && ch <= 0xaab8)||(0xaabe <= ch && ch <= 0xaabf)||(0xfe20 <= ch && ch <= 0xfe2f)||(0x10376 <= ch && ch <= 0x1037a)||(0x10a38 <= ch && ch <= 0x10a3a)||(0x10ae5 <= ch && ch <= 0x10ae6)||(0x110b9 <= ch && ch <= 0x110ba)||(0x11100 <= ch && ch <= 0x11102)||(0x11133 <= ch && ch <= 0x11134)||(0x11235 <= ch && ch <= 0x11236)||(0x112e9 <= ch && ch <= 0x112ea)||(0x11366 <= ch && ch <= 0x1136c)||(0x11370 <= ch && ch <= 0x11374)||(0x114c2 <= ch && ch <= 0x114c3)||(0x115bf <= ch && ch <= 0x115c0)||(0x116b6 <= ch && ch <= 0x116b7)||(0x16af0 <= ch && ch <= 0x16af4)||(0x16b30 <= ch && ch <= 0x16b36)||(0x1d165 <= ch && ch <= 0x1d169)||(0x1d16d <= ch && ch <= 0x1d172)||(0x1d17b <= ch && ch <= 0x1d182)||(0x1d185 <= ch && ch <= 0x1d18b)||(0x1d1aa <= ch && ch <= 0x1d1ad)||(0x1d242 <= ch && ch <= 0x1d244)||(0x1e8d0 <= ch && ch <= 0x1e8d6) || (ch == 0x5bf)||(ch == 0x5c7)||(ch == 0x670)||(ch == 0x711)||(ch == 0x93c)||(ch == 0x94d)||(ch == 0x9bc)||(ch == 0x9cd)||(ch == 0xa3c)||(ch == 0xa4d)||(ch == 0xabc)||(ch == 0xacd)||(ch == 0xb3c)||(ch == 0xb4d)||(ch == 0xbcd)||(ch == 0xc4d)||(ch == 0xcbc)||(ch == 0xccd)||(ch == 0xd4d)||(ch == 0xdca)||(ch == 0xf35)||(ch == 0xf37)||(ch == 0xf39)||(ch == 0xf74)||(ch == 0xf80)||(ch == 0xfc6)||(ch == 0x1037)||(ch == 0x108d)||(ch == 0x1714)||(ch == 0x1734)||(ch == 0x17d2)||(ch == 0x17dd)||(ch == 0x18a9)||(ch == 0x1a60)||(ch == 0x1a7f)||(ch == 0x1b34)||(ch == 0x1b44)||(ch == 0x1be6)||(ch == 0x1c37)||(ch == 0x1ced)||(ch == 0x1cf4)||(ch == 0x20e1)||(ch == 0x2d7f)||(ch == 0xa66f)||(ch == 0xa806)||(ch == 0xa8c4)||(ch == 0xa953)||(ch == 0xa9b3)||(ch == 0xa9c0)||(ch == 0xaab0)||(ch == 0xaac1)||(ch == 0xaaf6)||(ch == 0xabed)||(ch == 0xfb1e)||(ch == 0x101fd)||(ch == 0x102e0)||(ch == 0x10a0d)||(ch == 0x10a0f)||(ch == 0x10a3f)||(ch == 0x11046)||(ch == 0x1107f)||(ch == 0x11173)||(ch == 0x111c0)||(ch == 0x111ca)||(ch == 0x1133c)||(ch == 0x1134d)||(ch == 0x1163f)||(ch == 0x1172b)||(ch == 0x1bc9e);
+#include <unictype.h>
+#include <uninorm.h>
+
+static inline bool 
+is_combining_char(uint32_t ch) {
+    return uc_combining_class(ch) != UC_CCC_NR;
 }
     
 
-static inline bool is_ignored_char(uint32_t ch) {
-    return (0x0 < ch && ch <= 0x1f)||(0x7f <= ch && ch <= 0x9f)||(0x600 <= ch && ch <= 0x605)||(0x200b <= ch && ch <= 0x200f)||(0x202a <= ch && ch <= 0x202e)||(0x2060 <= ch && ch <= 0x2064)||(0x2066 <= ch && ch <= 0x206f)||(0xd800 <= ch && ch <= 0xdfff)||(0xfff9 <= ch && ch <= 0xfffb)||(0x1bca0 <= ch && ch <= 0x1bca3)||(0x1d173 <= ch && ch <= 0x1d17a)||(0xe0020 <= ch && ch <= 0xe007f) || (ch == 0xad)||(ch == 0x61c)||(ch == 0x6dd)||(ch == 0x70f)||(ch == 0x180e)||(ch == 0xfeff)||(ch == 0x110bd)||(ch == 0xe0001);
+static inline bool 
+is_ignored_char(uint32_t ch) {
+    return uc_is_general_category_withtable(ch, UC_CATEGORY_MASK_Cc | UC_CATEGORY_MASK_Cf | UC_CATEGORY_MASK_Cs);
+}
+
+static inline bool 
+is_word_char(uint32_t ch) {
+    return uc_is_general_category_withtable(ch, UC_CATEGORY_MASK_L | UC_CATEGORY_MASK_N);
+}
+
+static inline bool
+is_url_char(uint32_t ch) {
+    return ch && !uc_is_general_category_withtable(ch, UC_CATEGORY_MASK_C | UC_CATEGORY_MASK_Z);
+}
+
+static inline uint32_t
+normalize(uint32_t ch, uint32_t cc1, uint32_t cc2) {
+    uint32_t ans = uc_composition(ch, cc1);
+    if (ans && cc2) ans = uc_composition(ans, cc2);
+    return ans;
+}
+
+static inline bool
+can_strip_from_end_of_url(uint32_t ch) {
+    // remove trailing punctuation
+    return (
+        (uc_is_general_category_withtable(ch, UC_CATEGORY_MASK_P) && ch != '/') ||
+        ch == '>'
+    ) ? true : false;
 }
-    

kitty/utils.py

@@ -6,17 +6,27 @@ import math
 import os
 import re
 import shlex
-import signal
 import string
 import subprocess
 from contextlib import contextmanager
 from functools import lru_cache
 from time import monotonic
 
-from .constants import isosx
-from .fast_data_types import glfw_get_physical_dpi, wcwidth as wcwidth_impl
+from .constants import isosx, iswayland, selection_clipboard_funcs, x11_display
+from .fast_data_types import (
+    GLSL_VERSION, glfw_get_physical_dpi, redirect_std_streams,
+    wcwidth as wcwidth_impl
+)
 from .rgb import Color, to_color
 
+BASE = os.path.dirname(os.path.abspath(__file__))
+
+
+def load_shaders(name):
+    vert = open(os.path.join(BASE, '{}_vertex.glsl'.format(name))).read().replace('GLSL_VERSION', str(GLSL_VERSION), 1)
+    frag = open(os.path.join(BASE, '{}_fragment.glsl'.format(name))).read().replace('GLSL_VERSION', str(GLSL_VERSION), 1)
+    return vert, frag
+
 
 def safe_print(*a, **k):
     try:
@@ -37,6 +47,13 @@ def wcwidth(c: str) -> int:
         return wcwidth_impl(ord(c[0]))
 
 
+@lru_cache()
+def pt_to_px(pts):
+    dpix, dpiy = get_dpi()['logical']
+    dpi = (dpix + dpiy) / 2
+    return round(pts * dpi / 72)
+
+
 @contextmanager
 def timeit(name, do_timing=False):
     if do_timing:
@@ -55,7 +72,6 @@ def load_libx11():
     import ctypes
     from ctypes.util import find_library
     libx11 = ctypes.CDLL(find_library('X11'))
-    ans = []
 
     def cdef(name, restype, *argtypes):
         f = getattr(libx11, name)
@@ -63,49 +79,33 @@ def load_libx11():
             f.restype = restype
         if argtypes:
             f.argtypes = argtypes
-        ans.append(f)
+        return f
 
-    cdef('XOpenDisplay', ctypes.c_void_p, ctypes.c_char_p)
-    cdef('XCloseDisplay', ctypes.c_int, ctypes.c_void_p)
-    cdef('XResourceManagerString', ctypes.c_char_p, ctypes.c_void_p)
-    return ans
+    return cdef('XResourceManagerString', ctypes.c_char_p, ctypes.c_void_p)
 
 
 def parse_xrdb(raw):
     q = 'Xft.dpi:\t'
-    for line in raw.decode('utf-8').splitlines():
+    for line in raw.decode('utf-8', 'replace').splitlines():
         if line.startswith(q):
             return float(line[len(q):])
 
 
-def x11_dpi_native():
-    XOpenDisplay, XCloseDisplay, XResourceManagerString = load_libx11()
-    display = XOpenDisplay(None)
-    if display is None:
-        raise RuntimeError('Could not connect to the X server')
-    try:
-        raw = XResourceManagerString(display)
-        return parse_xrdb(raw)
-    finally:
-        XCloseDisplay(display)
-
-
 def x11_dpi():
-    try:
-        return x11_dpi_native()
-    except Exception:
-        pass
-    try:
-        raw = subprocess.check_output(['xrdb', '-query'])
-        return parse_xrdb(raw)
-    except Exception:
-        pass
+    XResourceManagerString = load_libx11()
+    display = x11_display()
+    if display:
+        try:
+            raw = XResourceManagerString(display)
+            return parse_xrdb(raw)
+        except Exception:
+            pass
 
 
 def get_logical_dpi():
     if not hasattr(get_logical_dpi, 'ans'):
-        if isosx:
-            # TODO: Investigate if this needs a different implementation on OS X
+        if isosx or iswayland:
+            # TODO: Investigate if this needs a different implementation on OS X or Wayland
             get_logical_dpi.ans = glfw_get_physical_dpi()
         else:
             # See https://github.com/glfw/glfw/issues/1019 for why we cant use
@@ -146,35 +146,33 @@ def parse_color_set(raw):
             continue
 
 
-def pipe2():
-    try:
-        read_fd, write_fd = os.pipe2(os.O_NONBLOCK | os.O_CLOEXEC)
-    except AttributeError:
-        import fcntl
-        read_fd, write_fd = os.pipe()
-        for fd in (read_fd, write_fd):
-            flag = fcntl.fcntl(fd, fcntl.F_GETFD)
-            fcntl.fcntl(fd, fcntl.F_SETFD, flag | fcntl.FD_CLOEXEC)
-            flag = fcntl.fcntl(fd, fcntl.F_GETFL)
-            fcntl.fcntl(fd, fcntl.F_SETFL, flag | os.O_NONBLOCK)
-    return read_fd, write_fd
-
-
-def handle_unix_signals():
-    read_fd, write_fd = pipe2()
-    for sig in (signal.SIGINT, signal.SIGTERM):
-        signal.signal(sig, lambda x, y: None)
-        signal.siginterrupt(sig, False)
-    signal.set_wakeup_fd(write_fd)
-    return read_fd
+def set_primary_selection(text):
+    if isosx:
+        return  # There is no primary selection on OS X
+    if isinstance(text, str):
+        text = text.encode('utf-8')
+    s = selection_clipboard_funcs()[1]
+    if s is None:
+        p = subprocess.Popen(['xsel', '-i', '-p'], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        p.stdin.write(text), p.stdin.close()
+        p.wait()
+    else:
+        s(text)
 
 
 def get_primary_selection():
     if isosx:
         return ''  # There is no primary selection on OS X
-    # glfw has no way to get the primary selection
-    # https://github.com/glfw/glfw/issues/894
-    return subprocess.check_output(['xsel', '-p']).decode('utf-8')
+    g = selection_clipboard_funcs()[0]
+    if g is None:
+        # We cannot use check_output as we set a SIGCHLD handler to reap zombies
+        ans = subprocess.Popen(['xsel', '-p'], stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE).stdout.read().decode('utf-8')
+        if ans:
+            # Without this for some reason repeated pastes dont work
+            set_primary_selection(ans)
+    else:
+        ans = (g() or b'').decode('utf-8', 'replace')
+    return ans
 
 
 def base64_encode(
@@ -191,14 +189,11 @@ def base64_encode(
     return ans
 
 
-def set_primary_selection(text):
-    if isosx:
-        return  # There is no primary selection on OS X
-    if isinstance(text, str):
-        text = text.encode('utf-8')
-    p = subprocess.Popen(['xsel', '-i', '-p'], stdin=subprocess.PIPE)
-    p.stdin.write(text), p.stdin.close()
-    p.wait()
+def open_cmd(cmd, arg=None):
+    if arg is not None:
+        cmd = list(cmd)
+        cmd.append(arg)
+    return subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
 
 def open_url(url, program='default'):
@@ -206,5 +201,21 @@ def open_url(url, program='default'):
         cmd = ['open'] if isosx else ['xdg-open']
     else:
         cmd = shlex.split(program)
-    cmd.append(url)
-    subprocess.Popen(cmd).wait()
+    return open_cmd(cmd, url)
+
+
+def detach(fork=True, setsid=True, redirect=True):
+    if fork:
+        # Detach from the controlling process.
+        if os.fork() != 0:
+            raise SystemExit(0)
+    if setsid:
+        os.setsid()
+    if redirect:
+        redirect_std_streams(os.devnull)
+
+
+def adjust_line_height(cell_height, val):
+    if isinstance(val, int):
+        return cell_height + val
+    return int(cell_height * val)

kitty/window.py

@@ -2,25 +2,33 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
+import sys
 import weakref
 from collections import deque
-from functools import partial
-from time import monotonic
+from enum import Enum
+from itertools import count
 
-from .char_grid import CharGrid, DynamicColor
-from .constants import wakeup, get_boss, appname, WindowGeometry, is_key_pressed, mouse_button_pressed, cell_size
-from .fast_data_types import (
-    BRACKETED_PASTE_START, BRACKETED_PASTE_END, Screen, read_bytes_dump,
-    read_bytes, GLFW_MOD_SHIFT, GLFW_MOUSE_BUTTON_1, GLFW_PRESS,
-    GLFW_MOUSE_BUTTON_MIDDLE, GLFW_RELEASE, glfw_post_empty_event,
-    GLFW_MOUSE_BUTTON_5, ANY_MODE, MOTION_MODE, GLFW_KEY_LEFT_SHIFT,
-    GLFW_KEY_RIGHT_SHIFT, GLFW_KEY_UP, GLFW_KEY_DOWN, GLFW_MOUSE_BUTTON_4
+from .config import build_ansi_color_table
+from .constants import (
+    ScreenGeometry, WindowGeometry, appname, cell_size, get_boss,
+    viewport_size, wakeup
 )
-from .keys import get_key_map
-from .mouse import encode_mouse_event, PRESS, RELEASE, MOVE, DRAG
+from .fast_data_types import (
+    BRACKETED_PASTE_END, BRACKETED_PASTE_START, CELL_BACKGROUND_PROGRAM,
+    CELL_FOREGROUND_PROGRAM, CELL_PROGRAM, CELL_SPECIAL_PROGRAM,
+    CURSOR_PROGRAM, GRAPHICS_PROGRAM, SCROLL_FULL, SCROLL_LINE, SCROLL_PAGE,
+    Screen, compile_program, create_cell_vao, create_graphics_vao,
+    glfw_post_empty_event, init_cell_program, init_cursor_program, remove_vao,
+    set_window_render_data, update_window_title, update_window_visibility
+)
+from .rgb import to_color
 from .terminfo import get_capabilities
-from .utils import sanitize_title, get_primary_selection, parse_color_set, safe_print
+from .utils import color_as_int, load_shaders, parse_color_set, sanitize_title, open_url, open_cmd
+
+
+class DynamicColor(Enum):
+    default_fg, default_bg, cursor_color, highlight_fg, highlight_bg = range(1, 6)
+
 
 DYNAMIC_COLOR_CODES = {
     10: DynamicColor.default_fg,
@@ -30,13 +38,46 @@ DYNAMIC_COLOR_CODES = {
     19: DynamicColor.highlight_fg,
 }
 DYNAMIC_COLOR_CODES.update({k+100: v for k, v in DYNAMIC_COLOR_CODES.items()})
-dump_bytes_opened = False
+window_counter = count()
+next(window_counter)
+
+
+def calculate_gl_geometry(window_geometry, viewport_width, viewport_height, cell_width, cell_height):
+    dx, dy = 2 * cell_width / viewport_width, 2 * cell_height / viewport_height
+    xmargin = window_geometry.left / viewport_width
+    ymargin = window_geometry.top / viewport_height
+    xstart = -1 + 2 * xmargin
+    ystart = 1 - 2 * ymargin
+    return ScreenGeometry(xstart, ystart, window_geometry.xnum, window_geometry.ynum, dx, dy)
+
+
+def load_shader_programs():
+    v, f = load_shaders('cell')
+    compile_program(GRAPHICS_PROGRAM, *load_shaders('graphics'))
+    for which, p in {
+            'ALL': CELL_PROGRAM, 'BACKGROUND': CELL_BACKGROUND_PROGRAM, 'SPECIAL': CELL_SPECIAL_PROGRAM,
+            'FOREGROUND': CELL_FOREGROUND_PROGRAM
+    }.items():
+        vv, ff = v.replace('WHICH_PROGRAM', which), f.replace('WHICH_PROGRAM', which)
+        compile_program(p, vv, ff)
+    init_cell_program()
+    compile_program(CURSOR_PROGRAM, *load_shaders('cursor'))
+    init_cursor_program()
+
+
+def setup_colors(screen, opts):
+    screen.color_profile.update_ansi_color_table(build_ansi_color_table(opts))
+    screen.color_profile.set_configured_colors(*map(color_as_int, (
+        opts.foreground, opts.background, opts.cursor, opts.selection_foreground, opts.selection_background)))
 
 
 class Window:
 
     def __init__(self, tab, child, opts, args):
-        global dump_bytes_opened
+        self.id = next(window_counter)
+        self.vao_id = create_cell_vao()
+        self.gvao_id = create_graphics_vao()
+        self.tab_id = tab.id
         self.tabref = weakref.ref(tab)
         self.override_title = None
         self.last_mouse_cursor_pos = 0, 0
@@ -45,32 +86,19 @@ class Window:
         self.geometry = WindowGeometry(0, 0, 0, 0, 0, 0)
         self.needs_layout = True
         self.title = appname
-        self._is_visible_in_layout = True
+        self.is_visible_in_layout = True
         self.child, self.opts = child, opts
-        self.child_fd = child.child_fd
-        self.start_visual_bell_at = None
-        self.screen = Screen(self, 24, 80, opts.scrollback_lines)
-        self.read_bytes = partial(read_bytes_dump, self.dump_commands) if args.dump_commands or args.dump_bytes else read_bytes
-        if args.dump_bytes:
-            mode = 'ab' if dump_bytes_opened else 'wb'
-            self.dump_bytes_to = open(args.dump_bytes, mode)
-            dump_bytes_opened = True
-        self.draw_dump_buf = []
-        self.write_buf = memoryview(b'')
-        self.char_grid = CharGrid(self.screen, opts)
+        self.screen = Screen(self, 24, 80, opts.scrollback_lines, self.id)
+        setup_colors(self.screen, opts)
 
     def __repr__(self):
-        return 'Window(title={}, id={})'.format(self.title, hex(id(self)))
+        return 'Window(title={}, id={})'.format(self.title, self.id)
 
-    @property
-    def is_visible_in_layout(self):
-        return self._is_visible_in_layout
-
-    @is_visible_in_layout.setter
-    def is_visible_in_layout(self, val):
+    def set_visible_in_layout(self, window_idx, val):
         val = bool(val)
-        if val != self._is_visible_in_layout:
-            self._is_visible_in_layout = val
+        if val is not self.is_visible_in_layout:
+            self.is_visible_in_layout = val
+            update_window_visibility(self.tab_id, window_idx, val)
             if val:
                 self.refresh()
 
@@ -78,16 +106,26 @@ class Window:
         self.screen.mark_as_dirty()
         wakeup()
 
-    def set_geometry(self, new_geometry):
+    def update_position(self, window_geometry):
+        self.screen_geometry = sg = calculate_gl_geometry(window_geometry, viewport_size.width, viewport_size.height, cell_size.width, cell_size.height)
+        return sg
+
+    def set_geometry(self, window_idx, new_geometry):
+        if self.destroyed:
+            return
         if self.needs_layout or new_geometry.xnum != self.screen.columns or new_geometry.ynum != self.screen.lines:
+            boss = get_boss()
             self.screen.resize(new_geometry.ynum, new_geometry.xnum)
-            self.child.resize_pty(self.screen.columns, self.screen.lines,
-                                  max(0, new_geometry.right - new_geometry.left), max(0, new_geometry.bottom - new_geometry.top))
-            self.char_grid.resize(new_geometry)
+            current_pty_size = (
+                self.screen.lines, self.screen.columns,
+                max(0, new_geometry.right - new_geometry.left), max(0, new_geometry.bottom - new_geometry.top))
+            sg = self.update_position(new_geometry)
             self.needs_layout = False
+            boss.child_monitor.resize_pty(self.id, *current_pty_size)
         else:
-            self.char_grid.update_position(new_geometry)
-        self.geometry = new_geometry
+            sg = self.update_position(new_geometry)
+        self.geometry = g = new_geometry
+        set_window_render_data(self.tab_id, window_idx, self.vao_id, self.gvao_id, sg.xstart, sg.ystart, sg.dx, sg.dy, self.screen, *g[:4])
 
     def contains(self, x, y):
         g = self.geometry
@@ -96,51 +134,24 @@ class Window:
     def close(self):
         get_boss().close_window(self)
 
-    def destroy(self):
+    def on_child_death(self):
+        if self.destroyed:
+            return
         self.destroyed = True
-        self.child.hangup()
-        self.child.get_child_status()  # Ensure child does not become zombie
-        # At this point this window can still render to screen using its
-        # existing buffers in char_grid. The rest of the cleanup must be
-        # performed in the GUI thread.
-
-    def read_ready(self):
-        if self.read_bytes(self.screen, self.child_fd) is False:
-            self.close()  # EOF
-
-    def write_ready(self):
-        while self.write_buf:
-            try:
-                n = os.write(self.child_fd, self.write_buf)
-            except BlockingIOError:
-                n = 0
-            if not n:
-                return
-            self.write_buf = self.write_buf[n:]
+        # Remove cycles so that screen is de-allocated immediately
+        boss = get_boss()
+        self.screen.reset_callbacks()
+        boss.gui_close_window(self)
+        self.screen = None
 
     def write_to_child(self, data):
-        self.write_buf = memoryview(self.write_buf.tobytes() + data)
-        wakeup()
-
-    def bell(self):
-        if self.opts.enable_audio_bell:
-            try:
-                with open('/dev/tty', 'wb') as f:
-                    f.write(b'\007')
-            except EnvironmentError:
-                pass  # failure to beep is not critical
-        if self.opts.visual_bell_duration > 0:
-            self.start_visual_bell_at = monotonic()
-        tm = get_boss()
-        tm.queue_ui_action(tm.request_attention)
-        glfw_post_empty_event()
+        if data:
+            if get_boss().child_monitor.needs_write(self.id, data) is not True:
+                print('Failed to write to child %d as it does not exist' % self.id, file=sys.stderr)
 
+    # screen callbacks {{{
     def use_utf8(self, on):
-        self.child.set_iutf8(on)
-
-    def update_screen(self):
-        self.char_grid.update_cell_data()
-        glfw_post_empty_event()
+        get_boss().child_monitor.set_iutf8(self.window_id, on)
 
     def focus_changed(self, focused):
         if focused:
@@ -153,6 +164,7 @@ class Window:
     def title_changed(self, new_title):
         if self.override_title is None:
             self.title = sanitize_title(new_title or appname)
+            update_window_title(self.tab_id, self.id, self.title)
             t = self.tabref()
             if t is not None:
                 t.title_changed(self)
@@ -161,6 +173,24 @@ class Window:
     def icon_changed(self, new_icon):
         pass  # TODO: Implement this
 
+    def change_colors(self, changes):
+        dirtied = False
+
+        def item(raw):
+            if raw is None:
+                return 0
+            val = to_color(raw)
+            return None if val is None else (color_as_int(val) << 8) | 2
+
+        for which, val in changes.items():
+            val = item(val)
+            if val is None:
+                continue
+            dirtied = True
+            setattr(self.screen.color_profile, which.name, val)
+        if dirtied:
+            self.screen.mark_as_dirty()
+
     def set_dynamic_color(self, code, value):
         if isinstance(value, bytes):
             value = value.decode('utf-8')
@@ -172,17 +202,18 @@ class Window:
                     val = None
                 color_changes[w] = val
             code += 1
-        self.char_grid.change_colors(color_changes)
+        self.change_colors(color_changes)
         glfw_post_empty_event()
 
     def set_color_table_color(self, code, value):
+        cp = self.screen.color_profile
         if code == 4:
             for c, val in parse_color_set(value):
-                self.char_grid.color_profile.set_color(c, val)
+                cp.set_color(c, val)
             self.refresh()
         elif code == 104:
             if not value.strip():
-                self.char_grid.color_profile.reset_color_table()
+                cp.reset_color_table()
             else:
                 for c in value.split(';'):
                     try:
@@ -190,120 +221,36 @@ class Window:
                     except Exception:
                         continue
                     if 0 <= c <= 255:
-                        self.char_grid.color_profile.reset_color(c)
+                        cp.reset_color(c)
             self.refresh()
 
     def request_capabilities(self, q):
         self.write_to_child(get_capabilities(q))
 
-    def dispatch_multi_click(self, x, y):
-        if len(self.click_queue) > 2 and self.click_queue[-1] - self.click_queue[-3] <= 2 * self.opts.click_interval:
-            self.char_grid.multi_click(3, x, y)
-            glfw_post_empty_event()
-        elif len(self.click_queue) > 1 and self.click_queue[-1] - self.click_queue[-2] <= self.opts.click_interval:
-            self.char_grid.multi_click(2, x, y)
-            glfw_post_empty_event()
+    # }}}
 
-    def on_mouse_button(self, button, action, mods):
-        mode = self.screen.mouse_tracking_mode()
-        handle_event = mods == GLFW_MOD_SHIFT or mode == 0 or button == GLFW_MOUSE_BUTTON_MIDDLE or (
-            mods == self.opts.open_url_modifiers and button == GLFW_MOUSE_BUTTON_1)
-        x, y = self.last_mouse_cursor_pos
-        if handle_event:
-            if button == GLFW_MOUSE_BUTTON_1:
-                self.char_grid.update_drag(action == GLFW_PRESS, x, y)
-                if action == GLFW_RELEASE:
-                    if mods == self.char_grid.opts.open_url_modifiers:
-                        self.char_grid.click_url(x, y)
-                    self.click_queue.append(monotonic())
-                    self.dispatch_multi_click(x, y)
-            elif button == GLFW_MOUSE_BUTTON_MIDDLE:
-                if action == GLFW_RELEASE:
-                    self.paste_from_selection()
-        else:
-            x, y = self.char_grid.cell_for_pos(x, y)
-            if x is not None:
-                ev = encode_mouse_event(mode, self.screen.mouse_tracking_protocol(),
-                                        button, PRESS if action == GLFW_PRESS else RELEASE, mods, x, y)
-                if ev:
-                    self.write_to_child(ev)
+    def text_for_selection(self):
+        return ''.join(self.screen.text_for_selection())
 
-    def on_mouse_move(self, x, y):
-        button = None
-        for b in range(0, GLFW_MOUSE_BUTTON_5 + 1):
-            if mouse_button_pressed[b]:
-                button = b
-                break
-        action = MOVE if button is None else DRAG
-        mode = self.screen.mouse_tracking_mode()
-        send_event = (mode == ANY_MODE or (mode == MOTION_MODE and button is not None)) and not (
-            is_key_pressed[GLFW_KEY_LEFT_SHIFT] or is_key_pressed[GLFW_KEY_RIGHT_SHIFT])
-        x, y = max(0, x - self.geometry.left), max(0, y - self.geometry.top)
-        self.last_mouse_cursor_pos = x, y
-        tm = get_boss()
-        tm.queue_ui_action(tm.change_mouse_cursor, self.char_grid.has_url_at(x, y))
-        if send_event:
-            x, y = self.char_grid.cell_for_pos(x, y)
-            if x is not None:
-                ev = encode_mouse_event(mode, self.screen.mouse_tracking_protocol(),
-                                        button, action, 0, x, y)
-                if ev:
-                    self.write_to_child(ev)
-        else:
-            if self.char_grid.current_selection.in_progress:
-                self.char_grid.update_drag(None, x, y)
-                margin = cell_size.height // 2
-                if y <= margin or y >= self.geometry.bottom - margin:
-                    get_boss().timers.add(0.02, self.drag_scroll)
+    def destroy(self):
+        if self.vao_id is not None:
+            remove_vao(self.vao_id)
+            remove_vao(self.gvao_id)
+            self.vao_id = self.gvao_id = None
 
-    def drag_scroll(self):
-        x, y = self.last_mouse_cursor_pos
-        tm = get_boss()
-        margin = cell_size.height // 2
-        if y <= margin or y >= self.geometry.bottom - margin:
-            self.scroll_line_up() if y < 50 else self.scroll_line_down()
-            self.char_grid.update_drag(None, x, y)
-            tm.timers.add(0.02, self.drag_scroll)
+    def buffer_as_ansi(self):
+        data = []
+        self.screen.historybuf.as_ansi(data.append)
+        self.screen.linebuf.as_ansi(data.append)
+        return ''.join(data)
 
-    def on_mouse_scroll(self, x, y):
-        s = int(round(y * self.opts.wheel_scroll_multiplier))
-        if abs(s) < 0:
-            return
-        upwards = s > 0
-        if self.screen.is_main_linebuf():
-            self.char_grid.scroll(abs(s), upwards)
-            glfw_post_empty_event()
-        else:
-            mode = self.screen.mouse_tracking_mode()
-            send_event = mode > 0
-            if send_event:
-                x, y = self.last_mouse_cursor_pos
-                x, y = self.char_grid.cell_for_pos(x, y)
-                if x is not None:
-                    ev = encode_mouse_event(mode, self.screen.mouse_tracking_protocol(),
-                                            GLFW_MOUSE_BUTTON_4 if upwards else GLFW_MOUSE_BUTTON_5, PRESS, 0, x, y)
-                    if ev:
-                        self.write_to_child(ev)
-            else:
-                k = get_key_map(self.screen)[GLFW_KEY_UP if upwards else GLFW_KEY_DOWN]
-                self.write_to_child(k * abs(s))
-
-    def buf_toggled(self, is_main_linebuf):
-        self.char_grid.scroll('full', False)
-
-    def render_cells(self, render_data, program, sprites):
-        invert_colors = False
-        if self.start_visual_bell_at is not None:
-            invert_colors = monotonic() - self.start_visual_bell_at <= self.opts.visual_bell_duration
-            if not invert_colors:
-                self.start_visual_bell_at = None
-        self.char_grid.render_cells(render_data, program, sprites, invert_colors)
+    def buffer_as_text(self):
+        return str(self.screen.historybuf) + '\n' + str(self.screen.linebuf)
 
     # actions {{{
 
     def show_scrollback(self):
-        data = self.char_grid.get_scrollback_as_ansi()
-        get_boss().display_scrollback(data)
+        get_boss().display_scrollback(self.buffer_as_ansi().encode('utf-8'))
 
     def paste(self, text):
         if text and not self.destroyed:
@@ -313,65 +260,40 @@ class Window:
                 text = BRACKETED_PASTE_START.encode('ascii') + text + BRACKETED_PASTE_END.encode('ascii')
             self.write_to_child(text)
 
-    def paste_from_selection(self):
-        text = get_primary_selection()
-        if text:
-            if isinstance(text, bytes):
-                text = text.decode('utf-8')
-            self.paste(text)
-
     def copy_to_clipboard(self):
-        text = self.char_grid.text_for_selection()
+        text = self.text_for_selection()
         if text:
-            tm = get_boss()
-            tm.queue_ui_action(tm.glfw_window.set_clipboard_string, text)
+            get_boss().glfw_window.set_clipboard_string(text)
+
+    def pass_selection_to_program(self, *args):
+        text = self.text_for_selection()
+        if text:
+            if args:
+                open_cmd(args, text)
+            else:
+                open_url(text)
 
     def scroll_line_up(self):
         if self.screen.is_main_linebuf():
-            self.char_grid.scroll('line', True)
-            glfw_post_empty_event()
+            self.screen.scroll(SCROLL_LINE, True)
 
     def scroll_line_down(self):
         if self.screen.is_main_linebuf():
-            self.char_grid.scroll('line', False)
-            glfw_post_empty_event()
+            self.screen.scroll(SCROLL_LINE, False)
 
     def scroll_page_up(self):
         if self.screen.is_main_linebuf():
-            self.char_grid.scroll('page', True)
-            glfw_post_empty_event()
+            self.screen.scroll(SCROLL_PAGE, True)
 
     def scroll_page_down(self):
         if self.screen.is_main_linebuf():
-            self.char_grid.scroll('page', False)
-            glfw_post_empty_event()
+            self.screen.scroll(SCROLL_PAGE, False)
 
     def scroll_home(self):
         if self.screen.is_main_linebuf():
-            self.char_grid.scroll('full', True)
-            glfw_post_empty_event()
+            self.screen.scroll(SCROLL_FULL, True)
 
     def scroll_end(self):
         if self.screen.is_main_linebuf():
-            self.char_grid.scroll('full', False)
-            glfw_post_empty_event()
-    # }}}
-
-    def dump_commands(self, *a):  # {{{
-        if a:
-            if a[0] == 'draw':
-                if a[1] is None:
-                    if self.draw_dump_buf:
-                        safe_print('draw', ''.join(self.draw_dump_buf))
-                        self.draw_dump_buf = []
-                else:
-                    self.draw_dump_buf.append(a[1])
-            elif a[0] == 'bytes':
-                self.dump_bytes_to.write(a[1])
-                self.dump_bytes_to.flush()
-            else:
-                if self.draw_dump_buf:
-                    safe_print('draw', ''.join(self.draw_dump_buf))
-                    self.draw_dump_buf = []
-                safe_print(*a)
+            self.screen.scroll(SCROLL_FULL, False)
     # }}}

kitty_tests/__init__.py

@@ -12,7 +12,7 @@ class Callbacks:
     def __init__(self):
         self.clear()
 
-    def write_to_child(self, data):
+    def write(self, data):
         self.wtcbuf += data
 
     def title_changed(self, data):
@@ -30,16 +30,12 @@ class Callbacks:
     def request_capabilities(self, q):
         self.qbuf += q
 
-    def buf_toggled(self, is_alt):
-        self.is_alt = is_alt
-
     def use_utf8(self, on):
         self.iutf8 = on
 
     def clear(self):
         self.wtcbuf = b''
         self.iconbuf = self.titlebuf = self.colorbuf = self.qbuf = self.ctbuf = ''
-        self.is_alt = False
         self.iutf8 = True
 
 
@@ -72,9 +68,11 @@ def filled_history_buf(ynum=5, xnum=5, cursor=Cursor()):
 class BaseTest(TestCase):
 
     ae = TestCase.assertEqual
+    maxDiff = 2000
 
     def create_screen(self, cols=5, lines=5, scrollback=5):
-        return Screen(Callbacks(), lines, cols, scrollback)
+        c = Callbacks()
+        return Screen(c, lines, cols, scrollback, 0, c)
 
     def assertEqualAttributes(self, c1, c2):
         x1, y1, c1.x, c1.y = c1.x, c1.y, 0, 0
@@ -83,17 +81,3 @@ class BaseTest(TestCase):
             self.assertEqual(c1, c2)
         finally:
             c1.x, c1.y, c2.x, c2.y = x1, y1, x2, y2
-
-    def assertChanges(self, s, ignore='', **expected_changes):
-        actual_changes = s.consolidate_changes()
-        ignore = frozenset(ignore.split())
-        for k, v in actual_changes.items():
-            if k not in ignore:
-                if isinstance(v, dict):
-                    v = {ky: tuple(vy) for ky, vy in v.items()}
-                if k == 'lines':
-                    v = set(v)
-                if k in expected_changes:
-                    self.ae(expected_changes[k], v)
-                else:
-                    self.assertFalse(v, 'The property {} was expected to be empty but is: {}'.format(k, v))

kitty_tests/datatypes.py

@@ -5,10 +5,14 @@
 import os
 from unittest import skipIf
 
-from . import BaseTest, filled_line_buf, filled_cursor, filled_history_buf
 from kitty.config import build_ansi_color_table, defaults
-from kitty.utils import wcwidth, sanitize_title
-from kitty.fast_data_types import LineBuf, Cursor as C, REVERSE, ColorProfile, SpriteMap, HistoryBuf, Cursor
+from kitty.fast_data_types import (
+    REVERSE, ColorProfile, Cursor as C, HistoryBuf, LineBuf,
+    sprite_map_set_layout, sprite_map_set_limits, sprite_position_for
+)
+from kitty.utils import sanitize_title, wcwidth
+
+from . import BaseTest, filled_cursor, filled_history_buf, filled_line_buf
 
 
 def create_lbuf(*lines):
@@ -30,7 +34,7 @@ class TestDataTypes(BaseTest):
         new.copy_old(old)
         self.ae(new.line(0), old.line(1))
         new.clear()
-        self.ae(str(new.line(0)), ' ' * new.xnum)
+        self.ae(str(new.line(0)), '')
         old.set_attribute(REVERSE, False)
         for y in range(old.ynum):
             for x in range(old.xnum):
@@ -136,22 +140,23 @@ class TestDataTypes(BaseTest):
         lb = LineBuf(2, 3)
         for y in range(lb.ynum):
             line = lb.line(y)
-            self.ae(str(line), ' ' * lb.xnum)
+            self.ae(str(line), '')
             for x in range(lb.xnum):
-                self.ae(line[x], ' ')
+                self.ae(line[x], '\0')
         with self.assertRaises(IndexError):
             lb.line(lb.ynum)
         with self.assertRaises(IndexError):
             lb.line(0)[lb.xnum]
         l = lb.line(0)
+        l.set_text(' ', 0, len(' '), C())
         l.add_combining_char(0, '1')
         self.ae(l[0], ' 1')
         l.add_combining_char(0, '2')
         self.ae(l[0], ' 12')
         l.add_combining_char(0, '3')
         self.ae(l[0], ' 13')
-        self.ae(l[1], ' ')
-        self.ae(str(l), ' 13  ')
+        self.ae(l[1], '\0')
+        self.ae(str(l), ' 13')
         t = 'Testing with simple text'
         lb = LineBuf(2, len(t))
         l = lb.line(0)
@@ -210,10 +215,49 @@ class TestDataTypes(BaseTest):
         l.set_char(0, 'x', 1, q)
         self.assertEqualAttributes(l.cursor_from(0), q)
 
+    def test_url_at(self):
+        def create(t):
+            lb = create.lb = LineBuf(1, len(t))
+            l = lb.line(0)
+            l.set_text(t, 0, len(t), C())
+            return l
+
+        for trail in '.,]>)\\':
+            l = create("http://xyz.com" + trail)
+            self.ae(l.url_end_at(0), len(l) - 2)
+        l = create("ftp://abc/")
+        self.ae(l.url_end_at(0), len(l) - 1)
+        l = create("http://-abcd] ")
+        self.ae(l.url_end_at(0), len(l) - 3)
+
+        def lspace_test(n, scheme='http'):
+            l = create(' ' * n + scheme + '://acme.com')
+            for i in range(0, n):
+                self.ae(l.url_start_at(i), len(l))
+            for i in range(n, len(l)):
+                self.ae(l.url_start_at(i), n)
+        for i in range(7):
+            for scheme in 'http https ftp file'.split():
+                lspace_test(i)
+        l = create('b https://testing.me a')
+        for s in (0, 1, len(l) - 1, len(l) - 2):
+            self.ae(l.url_start_at(s), len(l), 'failed with start at: %d' % s)
+        for s in range(2, len(l) - 2):
+            self.ae(l.url_start_at(s), 2, 'failed with start at: %d (%s)' % (s, str(l)[s:]))
+
+        def no_url(t):
+            l = create(t)
+            for s in range(len(l)):
+                self.ae(l.url_start_at(s), len(l))
+        no_url('https:// testing.me a')
+        no_url('h ttp://acme.com')
+        no_url('http: //acme.com')
+        no_url('http:/ /acme.com')
+
     def rewrap(self, lb, lb2):
         hb = HistoryBuf(lb2.ynum, lb2.xnum)
         cy = lb.rewrap(lb2, hb)
-        return hb, cy
+        return hb, cy[1]
 
     def test_rewrap_simple(self):
         ' Same width buffers '
@@ -224,7 +268,7 @@ class TestDataTypes(BaseTest):
             self.ae(lb2.line(i), lb.line(i))
         lb2 = LineBuf(8, 5)
         cy = self.rewrap(lb, lb2)[1]
-        self.ae(cy, 4)
+        self.ae(cy, 5)
         for i in range(lb.ynum):
             self.ae(lb2.line(i), lb.line(i))
         empty = LineBuf(1, lb2.xnum)
@@ -232,7 +276,7 @@ class TestDataTypes(BaseTest):
             self.ae(str(lb2.line(i)), str(empty.line(0)))
         lb2 = LineBuf(3, 5)
         cy = self.rewrap(lb, lb2)[1]
-        self.ae(cy, 2)
+        self.ae(cy, 3)
         for i in range(lb2.ynum):
             self.ae(lb2.line(i), lb.line(i + 2))
 
@@ -253,18 +297,21 @@ class TestDataTypes(BaseTest):
     def test_rewrap_wider(self):
         ' New buffer wider '
         lb = create_lbuf('0123 ', '56789')
-        lb2 = self.line_comparison_rewrap(lb, '0123 5', '6789  ', ' ' * 6)
+        lb2 = self.line_comparison_rewrap(lb, '0123 5', '6789', '')
         self.assertContinued(lb2, False, True)
 
         lb = create_lbuf('12', 'abc')
-        lb2 = self.line_comparison_rewrap(lb, '12  ', 'abc ')
+        lb2 = self.line_comparison_rewrap(lb, '12', 'abc')
         self.assertContinued(lb2, False, False)
 
     def test_rewrap_narrower(self):
         ' New buffer narrower '
-        lb = create_lbuf('123 ', 'abcde')
-        lb2 = self.line_comparison_rewrap(lb, '123', 'abc', 'de ')
+        lb = create_lbuf('123', 'abcde')
+        lb2 = self.line_comparison_rewrap(lb, '123', 'abc', 'de')
         self.assertContinued(lb2, False, False, True)
+        lb = create_lbuf('123  ', 'abcde')
+        lb2 = self.line_comparison_rewrap(lb, '123', '  a', 'bcd', 'e')
+        self.assertContinued(lb2, False, True, True, True)
 
     @skipIf('ANCIENT_WCWIDTH' in os.environ, 'wcwidth() is too old')
     def test_utils(self):
@@ -280,17 +327,18 @@ class TestDataTypes(BaseTest):
         self.ae(c.as_color(255 << 8 | 1), (0xee, 0xee, 0xee))
 
     def test_sprite_map(self):
-        s = SpriteMap(10, 2)
-        s.layout(5, 5)
-        self.ae(s.position_for(0), (0, 0, 0))
-        self.ae(s.position_for(1), (1, 0, 0))
-        self.ae(s.position_for(2), (0, 1, 0))
-        self.ae(s.position_for(3), (1, 1, 0))
-        self.ae(s.position_for(4), (0, 0, 1))
-        self.ae(s.position_for(5), (1, 0, 1))
-        self.ae(s.position_for(0, 1), (0, 1, 1))
-        self.ae(s.position_for(0, 2), (1, 1, 1))
-        self.ae(s.position_for(0, 2), (1, 1, 1))
+        sprite_map_set_limits(10, 2)
+        sprite_map_set_layout(5, 5)
+        self.ae(sprite_position_for(0), (0, 0, 0))
+        self.ae(sprite_position_for(1), (1, 0, 0))
+        self.ae(sprite_position_for(2), (0, 1, 0))
+        self.ae(sprite_position_for(3), (1, 1, 0))
+        self.ae(sprite_position_for(4), (0, 0, 1))
+        self.ae(sprite_position_for(5), (1, 0, 1))
+        self.ae(sprite_position_for(0, 1), (0, 1, 1))
+        self.ae(sprite_position_for(0, 2), (1, 1, 1))
+        self.ae(sprite_position_for(0, 2), (1, 1, 1))
+        sprite_map_set_limits(1000, 1000)
 
     def test_historybuf(self):
         lb = filled_line_buf()
@@ -340,7 +388,7 @@ class TestDataTypes(BaseTest):
         lb.as_ansi(a.append)
         self.ae(a, ['\x1b[0m' + str(lb.line(i)) + '\n' for i in range(lb.ynum)])
         l = lb.line(0)
-        c = Cursor()
+        c = C()
         c.bold = c.italic = c.reverse = c.strikethrough = True
         c.fg = (4 << 8) | 1
         c.bg = (1 << 24) | (2 << 16) | (3 << 8) | 2

kitty_tests/gr.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+import sys
+import zlib
+from base64 import standard_b64encode
+
+write = getattr(sys.stdout, 'buffer', sys.stdout).write
+
+
+def clear_screen():
+    write(b'\033[2J')
+
+
+def move_cursor(x, y):
+    write('\033[{};{}H'.format(y, x).encode('ascii'))
+
+
+def write_gr_cmd(cmd, payload):
+    cmd = ','.join('{}={}'.format(k, v) for k, v in cmd.items())
+    w = write
+    w(b'\033_G'), w(cmd.encode('ascii')), w(b';'), w(payload), w(b'\033\\')
+    sys.stdout.flush()
+
+
+def display(data, width, height, x, y, z, ncols=0, nrows=0):
+    move_cursor(x, y)
+    cmd = {'a': 'T', 's': width, 'v': height, 'c': ncols, 'r': nrows, 'S': len(data), 'z': z}
+    data = zlib.compress(data)
+    cmd['o'] = 'z'
+    data = standard_b64encode(data)
+    while data:
+        chunk, data = data[:4096], data[4096:]
+        m = 1 if data else 0
+        cmd['m'] = m
+        write_gr_cmd(cmd, chunk)
+        cmd.clear()
+
+
+def display_png_file(path):
+    cmd = {'a': 'T', 't': 'f', 'f': '100'}
+    path = os.path.abspath(path)
+    if not isinstance(path, bytes):
+        path = path.encode(sys.getfilesystemencoding() or 'utf-8')
+    data = standard_b64encode(path)
+    write_gr_cmd(cmd, data)
+
+
+def main():
+    clear_screen()
+    display(b'\xdd\xdd\xdd\xff', 1, 1, 0, 0, -10, 40, 20)
+    display(open('logo/kitty.rgba', 'rb').read(), 256, 256, 0, 5, -9)
+    display(b'\0\0\0\xaa', 1, 1, 0, 7, -8, 40, 3)
+    move_cursor(5, 8)
+    print('kitty is \033[3m\033[32mawesome\033[m!')
+    move_cursor(0, 21)
+    print('Lenna...')
+    display_png_file('kitty_tests/Lenna.png')
+    try:
+        try:
+            raw_input()
+        except NameError:
+            input()
+    except (EOFError, KeyboardInterrupt):
+        pass
+
+
+if __name__ == '__main__':
+    os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
+    main()

kitty_tests/graphics.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+import tempfile
+import unittest
+import zlib
+from base64 import standard_b64encode
+from io import BytesIO
+
+from kitty.fast_data_types import (
+    parse_bytes, set_display_state, set_send_to_gpu, shm_unlink, shm_write
+)
+
+from . import BaseTest
+
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+
+set_send_to_gpu(False)
+
+
+def relpath(name):
+    base = os.path.dirname(os.path.abspath(__file__))
+    return os.path.join(base, name)
+
+
+def send_command(screen, cmd, payload=b''):
+    cmd = '\033_G' + cmd
+    if payload:
+        if isinstance(payload, str):
+            payload = payload.encode('utf-8')
+        payload = standard_b64encode(payload).decode('ascii')
+        cmd += ';' + payload
+    cmd += '\033\\'
+    c = screen.callbacks
+    c.clear()
+    parse_bytes(screen, cmd.encode('ascii'))
+    return c.wtcbuf
+
+
+def parse_response(res):
+    if not res:
+        return
+    return res.decode('ascii').partition(';')[2].partition('\033')[0]
+
+
+all_bytes = bytes(bytearray(range(256)))
+
+
+def byte_block(sz):
+    d, m = divmod(sz, len(all_bytes))
+    return (all_bytes * d) + all_bytes[:m]
+
+
+def load_helpers(self):
+    s = self.create_screen()
+    g = s.grman
+
+    def l(payload, **kw):
+        kw.setdefault('i', 1)
+        cmd = ','.join('%s=%s' % (k, v) for k, v in kw.items())
+        res = send_command(s, cmd, payload)
+        return parse_response(res)
+
+    def sl(payload, **kw):
+        if isinstance(payload, str):
+            payload = payload.encode('utf-8')
+        data = kw.pop('expecting_data', payload)
+        cid = kw.setdefault('i', 1)
+        self.ae('OK', l(payload, **kw))
+        img = g.image_for_client_id(cid)
+        self.ae(img['client_id'], cid)
+        self.ae(img['data'], data)
+        if 's' in kw:
+            self.ae((kw['s'], kw['v']), (img['width'], img['height']))
+        self.ae(img['is_4byte_aligned'], kw.get('f') != 24)
+        return img
+
+    return s, g, l, sl
+
+
+def put_helpers(self, cw, ch):
+    iid = 0
+
+    def create_screen():
+        s = self.create_screen(10, 5)
+        set_display_state(s.columns * cw, s.lines * ch, cw, ch)
+        return s, 2 / s.columns, 2 / s.lines
+
+    def put_cmd(z=0, num_cols=0, num_lines=0, x_off=0, y_off=0, width=0, height=0, cell_x_off=0, cell_y_off=0):
+        return 'z=%d,c=%d,r=%d,x=%d,y=%d,w=%d,h=%d,X=%d,Y=%d' % (z, num_cols, num_lines, x_off, y_off, width, height, cell_x_off, cell_y_off)
+
+    def put_image(screen, w, h, **kw):
+        nonlocal iid
+        iid += 1
+        cmd = 'a=T,f=24,i=%d,s=%d,v=%d,%s' % (iid, w, h, put_cmd(**kw))
+        data = b'x' * w * h * 3
+        res = send_command(screen, cmd, data)
+        return iid, parse_response(res)
+
+    def put_ref(screen, **kw):
+        cmd = 'a=p,i=%d,%s' % (iid, put_cmd(**kw))
+        send_command(screen, cmd)
+
+    def layers(screen, scrolled_by=0, xstart=-1, ystart=1):
+        return screen.grman.update_layers(scrolled_by, xstart, ystart, dx, dy, screen.columns, screen.lines)
+
+    def rect_eq(r, left, top, right, bottom):
+        for side in 'left top right bottom'.split():
+            a, b = r[side], locals()[side]
+            if abs(a - b) > 0.0001:
+                self.ae(a, b, 'the %s side is not equal' % side)
+
+    s, dx, dy = create_screen()
+    return s, dx, dy, put_image, put_ref, layers, rect_eq
+
+
+class TestGraphics(BaseTest):
+
+    def test_load_images(self):
+        s, g, l, sl = load_helpers(self)
+
+        # Test load query
+        self.ae(l('abcd', s=1, v=1, a='q'), 'OK')
+        self.ae(g.image_count, 0)
+
+        # Test simple load
+        for f in 32, 24:
+            p = 'abc' + ('d' if f == 32 else '')
+            img = sl(p, s=1, v=1, f=f)
+            self.ae(bool(img['is_4byte_aligned']), f == 32)
+
+        # Test chuunked load
+        self.assertIsNone(l('abcd', s=2, v=2, m=1))
+        self.assertIsNone(l('efgh', m=1))
+        self.assertIsNone(l('ijkl', m=1))
+        self.ae(l('mnop', m=0), 'OK')
+        img = g.image_for_client_id(1)
+        self.ae(img['data'], b'abcdefghijklmnop')
+
+        # Test compression
+        random_data = byte_block(3 * 1024)
+        compressed_random_data = zlib.compress(random_data)
+        sl(
+            compressed_random_data,
+            s=24,
+            v=32,
+            o='z',
+            expecting_data=random_data
+        )
+
+        # Test chunked + compressed
+        b = len(compressed_random_data) // 2
+        self.assertIsNone(l(compressed_random_data[:b], s=24, v=32, o='z', m=1))
+        self.ae(l(compressed_random_data[b:], m=0), 'OK')
+        img = g.image_for_client_id(1)
+        self.ae(img['data'], random_data)
+
+        # Test loading from file
+        f = tempfile.NamedTemporaryFile()
+        f.write(random_data), f.flush()
+        sl(f.name, s=24, v=32, t='f', expecting_data=random_data)
+        self.assertTrue(os.path.exists(f.name))
+        f.seek(0), f.truncate(), f.write(compressed_random_data), f.flush()
+        sl(f.name, s=24, v=32, t='t', o='z', expecting_data=random_data)
+        self.assertRaises(
+            FileNotFoundError, f.close
+        )  # check that file was deleted
+
+        # Test loading from POSIX SHM
+        name = '/kitty-test-shm'
+        shm_write(name, random_data)
+        sl(name, s=24, v=32, t='s', expecting_data=random_data)
+        self.assertRaises(
+            FileNotFoundError, shm_unlink, name
+        )  # check that file was deleted
+
+    @unittest.skipIf(Image is None, 'PIL not available, skipping PNG tests')
+    def test_load_png(self):
+        s, g, l, sl = load_helpers(self)
+        w, h = 5, 3
+        rgba_data = byte_block(w * h * 4)
+        img = Image.frombytes('RGBA', (w, h), rgba_data)
+        rgb_data = img.convert('RGB').convert('RGBA').tobytes()
+
+        def png(mode='RGBA'):
+            buf = BytesIO()
+            i = img
+            if mode != i.mode:
+                i = img.convert(mode)
+            i.save(buf, 'PNG')
+            return buf.getvalue()
+
+        for mode in 'RGBA RGB'.split():
+            data = png(mode)
+            sl(data, f=100, expecting_data=rgb_data if mode == 'RGB' else rgba_data)
+
+        for m in 'LP':
+            img = img.convert(m)
+            rgba_data = img.convert('RGBA').tobytes()
+            data = png(m)
+        sl(data, f=100, expecting_data=rgba_data)
+
+        self.ae(l(b'a' * 20, f=100, S=20).partition(':')[0], 'EBADPNG')
+
+    def test_image_put(self):
+        cw, ch = 10, 20
+        s, dx, dy, put_image, put_ref, layers, rect_eq = put_helpers(self, cw, ch)
+        self.ae(put_image(s, 10, 20)[1], 'OK')
+        l = layers(s)
+        self.ae(len(l), 1)
+        rect_eq(l[0]['src_rect'], 0, 0, 1, 1)
+        rect_eq(l[0]['dest_rect'], -1, 1, -1 + dx, 1 - dy)
+        self.ae(l[0]['group_count'], 1)
+        self.ae(s.cursor.x, 1), self.ae(s.cursor.y, 0)
+        put_ref(s, num_cols=s.columns, x_off=2, y_off=1, width=3, height=5, cell_x_off=3, cell_y_off=1, z=-1)
+        l = layers(s)
+        self.ae(len(l), 2)
+        rect_eq(l[0]['src_rect'], 2 / 10, 1 / 20, (2 + 3) / 10, (1 + 5)/20)
+        left, top = -1 + dx + 3 * dx / cw, 1 - 1 * dy / ch
+        rect_eq(l[0]['dest_rect'], left, top, -1 + (1 + s.columns) * dx, top - dy * 5 / ch)
+        rect_eq(l[1]['src_rect'], 0, 0, 1, 1)
+        rect_eq(l[1]['dest_rect'], -1, 1, -1 + dx, 1 - dy)
+        self.ae(l[0]['group_count'], 1), self.ae(l[1]['group_count'], 1)
+        self.ae(s.cursor.x, 0), self.ae(s.cursor.y, 1)
+
+    def test_gr_scroll(self):
+        cw, ch = 10, 20
+        s, dx, dy, put_image, put_ref, layers, rect_eq = put_helpers(self, cw, ch)
+        put_image(s, 10, 20)  # a one cell image at (0, 0)
+        self.ae(len(layers(s)), 1)
+        for i in range(s.lines):
+            s.index()
+        self.ae(len(layers(s)), 0), self.ae(s.grman.image_count, 1)
+        for i in range(s.historybuf.ynum - 1):
+            s.index()
+            self.ae(len(layers(s)), 0), self.ae(s.grman.image_count, 1)
+        s.index()
+        self.ae(s.grman.image_count, 0)
+
+        # Now test with margins
+        s.reset()
+        # Test images outside page area untouched
+        put_image(s, cw, ch)  # a one cell image at (0, 0)
+        for i in range(s.lines - 1):
+            s.index()
+        put_image(s, cw, ch)  # a one cell image at (0, bottom)
+        s.set_margins(2, 4)  # 1-based indexing
+        self.ae(s.grman.image_count, 2)
+        for i in range(s.lines + s.historybuf.ynum):
+            s.index()
+            self.ae(s.grman.image_count, 2)
+        for i in range(s.lines):  # ensure cursor is at top margin
+            s.reverse_index()
+        # Test clipped scrolling during index
+        put_image(s, cw, 2*ch, z=-1)  # 1x2 cell image
+        self.ae(s.grman.image_count, 3)
+        self.ae(layers(s)[0]['src_rect'], {'left': 0.0, 'top': 0.0, 'right': 1.0, 'bottom': 1.0})
+        s.index(), s.index()
+        l = layers(s)
+        self.ae(len(l), 3)
+        self.ae(layers(s)[0]['src_rect'],  {'left': 0.0, 'top': 0.5, 'right': 1.0, 'bottom': 1.0})
+        s.index()
+        self.ae(s.grman.image_count, 2)
+        # Test clipped scrolling during reverse_index
+        for i in range(s.lines):
+            s.reverse_index()
+        put_image(s, cw, 2*ch, z=-1)  # 1x2 cell image
+        self.ae(s.grman.image_count, 3)
+        self.ae(layers(s)[0]['src_rect'], {'left': 0.0, 'top': 0.0, 'right': 1.0, 'bottom': 1.0})
+        while s.cursor.y != 1:
+            s.reverse_index()
+        s.reverse_index()
+        self.ae(layers(s)[0]['src_rect'], {'left': 0.0, 'top': 0.0, 'right': 1.0, 'bottom': 0.5})
+        s.reverse_index()
+        self.ae(s.grman.image_count, 2)
+
+    def test_gr_reset(self):
+        cw, ch = 10, 20
+        s, dx, dy, put_image, put_ref, layers, rect_eq = put_helpers(self, cw, ch)
+        put_image(s, cw, ch)  # a one cell image at (0, 0)
+        self.ae(len(layers(s)), 1)
+        s.reset()
+        self.ae(s.grman.image_count, 0)
+        put_image(s, cw, ch)  # a one cell image at (0, 0)
+        self.ae(s.grman.image_count, 1)
+        for i in range(s.lines):
+            s.index()
+        s.reset()
+        self.ae(s.grman.image_count, 1)
+
+    def test_gr_delete(self):
+        cw, ch = 10, 20
+        s, dx, dy, put_image, put_ref, layers, rect_eq = put_helpers(self, cw, ch)
+
+        def delete(ac=None, **kw):
+            cmd = 'a=d'
+            if ac:
+                cmd += ',d={}'.format(ac)
+            if kw:
+                cmd += ',' + ','.join('{}={}'.format(k, v) for k, v in kw.items())
+            send_command(s, cmd)
+
+        put_image(s, cw, ch)
+        delete()
+        self.ae(len(layers(s)), 0), self.ae(s.grman.image_count, 1)
+        delete('A')
+        self.ae(s.grman.image_count, 0)
+        iid = put_image(s, cw, ch)[0]
+        delete('I', i=iid)
+        self.ae(s.grman.image_count, 0)
+        s.reset()
+        put_image(s, cw, ch)
+        put_image(s, cw, ch)
+        delete('C')
+        self.ae(s.grman.image_count, 2)
+        s.cursor_position(1, 1)
+        delete('C')
+        self.ae(s.grman.image_count, 1)
+        delete('P', x=2, y=1)
+        self.ae(s.grman.image_count, 0)
+        put_image(s, cw, ch, z=9)
+        delete('Z', z=9)
+        self.ae(s.grman.image_count, 0)

kitty_tests/keys.py

@@ -39,7 +39,6 @@ class TestParser(BaseTest):
                 mods,
                 w,
                 defines.GLFW_PRESS,
-                get_localized_key=lambda k, s: k
             )
             self.ae(b'\033' + expected.encode('ascii'), actual)
 

kitty_tests/parser.py

@@ -45,15 +45,15 @@ class TestParser(BaseTest):
         pb = partial(self.parse_bytes_dump, s)
 
         pb('12', '12')
-        self.ae(str(s.line(0)), '12   ')
+        self.ae(str(s.line(0)), '12')
         self.ae(s.cursor.x, 2)
         pb('3456', '3456')
         self.ae(str(s.line(0)), '12345')
-        self.ae(str(s.line(1)), '6    ')
+        self.ae(str(s.line(1)), '6')
         pb(b'\n123\n\r45', ('screen_linefeed',), '123', ('screen_linefeed',), ('screen_carriage_return',), '45')
-        self.ae(str(s.line(1)), '6    ')
-        self.ae(str(s.line(2)), ' 123 ')
-        self.ae(str(s.line(3)), '45   ')
+        self.ae(str(s.line(1)), '6')
+        self.ae(str(s.line(2)), ' 123')
+        self.ae(str(s.line(3)), '45')
         parse_bytes(s, b'\rabcde')
         self.ae(str(s.line(3)), 'abcde')
         pb('\rßxyz1', ('screen_carriage_return',), 'ßxyz1')
@@ -65,11 +65,11 @@ class TestParser(BaseTest):
         s = self.create_screen()
         pb = partial(self.parse_bytes_dump, s)
         pb('12\033Da', '12', ('screen_index',), 'a')
-        self.ae(str(s.line(0)), '12   ')
-        self.ae(str(s.line(1)), '  a  ')
+        self.ae(str(s.line(0)), '12')
+        self.ae(str(s.line(1)), '  a')
         pb('\033x', ('Unknown char after ESC: 0x%x' % ord('x'),))
         pb('\033c123', ('screen_reset', ), '123')
-        self.ae(str(s.line(0)), '123  ')
+        self.ae(str(s.line(0)), '123')
 
     def test_charsets(self):
         s = self.create_screen()
@@ -79,14 +79,14 @@ class TestParser(BaseTest):
         s = self.create_screen()
         pb = partial(self.parse_bytes_dump, s)
         pb('\033)0\x0e/_', ('screen_designate_charset', 1, ord('0')), ('screen_change_charset', 1), '/_')
-        self.ae(str(s.line(0)), '/\xa0   ')
+        self.ae(str(s.line(0)), '/\xa0')
         self.assertTrue(s.callbacks.iutf8)
         pb('\033%@_', ('screen_use_latin1', 1), '_')
         self.assertFalse(s.callbacks.iutf8)
         s = self.create_screen()
         pb = partial(self.parse_bytes_dump, s)
         pb('\033(0/_', ('screen_designate_charset', 0, ord('0')), '/_')
-        self.ae(str(s.line(0)), '/\xa0   ')
+        self.ae(str(s.line(0)), '/\xa0')
 
     def test_csi_codes(self):
         s = self.create_screen()
@@ -185,6 +185,52 @@ class TestParser(BaseTest):
     def test_oth_codes(self):
         s = self.create_screen()
         pb = partial(self.parse_bytes_dump, s)
-        for prefix in '\033_', '\033^', '\u009e', '\u009f':
+        for prefix in '\033_', '\u009f':
             for suffix in '\u009c', '\033\\':
-                pb('a{}+\\++{}bcde'.format(prefix, suffix), 'abcde')
+                pb('a{}+\\++{}bcde'.format(prefix, suffix), ('draw', 'a'), ('Unrecognized APC code: 0x2b',), ('draw', 'bcde'))
+        for prefix in '\033^', '\u009e':
+            for suffix in '\u009c', '\033\\':
+                pb('a{}+\\++{}bcde'.format(prefix, suffix), ('draw', 'a'), ('Unrecognized PM code: 0x2b',), ('draw', 'bcde'))
+
+    def test_graphics_command(self):
+        from base64 import standard_b64encode
+
+        def enc(x):
+            return standard_b64encode(x.encode('utf-8') if isinstance(x, str) else x).decode('ascii')
+
+        def c(**k):
+            for p, v in tuple(k.items()):
+                if isinstance(v, str) and p != 'payload':
+                    k[p] = v.encode('ascii')
+            for f in 'action delete_action transmission_type compressed'.split():
+                k.setdefault(f, b'\0')
+            for f in ('format more id data_sz data_offset width height x_offset y_offset data_height data_width'
+                      ' num_cells num_lines cell_x_offset cell_y_offset z_index').split():
+                k.setdefault(f, 0)
+            p = k.pop('payload', '').encode('utf-8')
+            k['payload_sz'] = len(p)
+            return ('graphics_command', k, p)
+
+        def t(cmd, **kw):
+            pb('\033_G{};{}\033\\'.format(cmd, enc(kw.get('payload', ''))), c(**kw))
+
+        def e(cmd, err):
+            pb('\033_G{}\033\\'.format(cmd), (err,))
+
+        s = self.create_screen()
+        pb = partial(self.parse_bytes_dump, s)
+        uint32_max = 2**32 - 1
+        t('i=%d' % uint32_max, id=uint32_max)
+        e('i=%d' % (uint32_max + 1), 'id is too large')
+        pb('\033_Gi=12\033\\', c(id=12))
+        t('a=t,t=d,s=100,z=-9', payload='X', action='t', transmission_type='d', data_width=100, z_index=-9, payload_sz=1)
+        t('a=t,t=d,s=100,z=9', payload='payload', action='t', transmission_type='d', data_width=100, z_index=9, payload_sz=7)
+        t('a=t,t=d,s=100,z=9', action='t', transmission_type='d', data_width=100, z_index=9)
+        e(',s=1', 'Malformed graphics control block, invalid key character: 0x2c')
+        e('W=1', 'Malformed graphics control block, invalid key character: 0x57')
+        e('1=1', 'Malformed graphics control block, invalid key character: 0x31')
+        e('a=t,,w=2', 'Malformed graphics control block, invalid key character: 0x2c')
+        e('s', 'Malformed graphics control block, no = after key')
+        e('s=', 'Malformed graphics control block, expecting an integer value')
+        e('s==', 'Malformed graphics control block, expecting an integer value for key: s')
+        e('s=1=', 'Malformed graphics control block, expecting a comma or semi-colon after a value, found: 0x3d')

kitty_tests/screen.py

@@ -18,15 +18,13 @@ class TestScreen(BaseTest):
         s.draw('a' * 5)
         self.ae(str(s.line(0)), 'a' * 5)
         self.ae(s.cursor.x, 5), self.ae(s.cursor.y, 0)
-        self.assertChanges(s, ignore='cursor', cells={0: ((0, 4),)})
         s.draw('b' * 7)
         self.assertTrue(s.linebuf.is_continued(1))
         self.assertTrue(s.linebuf.is_continued(2))
         self.ae(str(s.line(0)), 'a' * 5)
         self.ae(str(s.line(1)), 'b' * 5)
-        self.ae(str(s.line(2)), 'b' * 2 + ' ' * 3)
+        self.ae(str(s.line(2)), 'b' * 2)
         self.ae(s.cursor.x, 2), self.ae(s.cursor.y, 2)
-        self.assertChanges(s, ignore='cursor', cells={1: ((0, 4),), 2: ((0, 1),)})
         s.draw('c' * 15)
         self.ae(str(s.line(0)), 'b' * 5)
         self.ae(str(s.line(1)), 'bbccc')
@@ -37,11 +35,9 @@ class TestScreen(BaseTest):
         s.draw('0123456789')
         self.ae(str(s.line(0)), '01239')
         self.ae(s.cursor.x, 5), self.ae(s.cursor.y, 0)
-        self.assertChanges(s, ignore='cursor', cells={0: ((0, 4),)})
         s.draw('ab')
         self.ae(str(s.line(0)), '0123b')
         self.ae(s.cursor.x, 5), self.ae(s.cursor.y, 0)
-        self.assertChanges(s, ignore='cursor', cells={0: ((4, 4),)})
 
         # Now test in insert mode
         s.reset(), s.reset_dirty()
@@ -53,7 +49,6 @@ class TestScreen(BaseTest):
         s.draw('ab')
         self.ae(str(s.line(4)), 'ab123')
         self.ae((s.cursor.x, s.cursor.y), (2, 4))
-        self.assertChanges(s, ignore='cursor', cells={4: ((0, 4),)})
 
     @skipIf('ANCIENT_WCWIDTH' in os.environ, 'wcwidth() is too old')
     def test_draw_char(self):
@@ -63,23 +58,19 @@ class TestScreen(BaseTest):
         self.ae(str(s.line(0)), 'ココx')
         self.ae(tuple(map(s.line(0).width, range(5))), (2, 0, 2, 0, 1))
         self.ae(s.cursor.x, 5), self.ae(s.cursor.y, 0)
-        self.assertChanges(s, ignore='cursor', cells={0: ((0, 4),)})
         s.draw('ニチハ')
         self.ae(str(s.line(0)), 'ココx')
-        self.ae(str(s.line(1)), 'ニチ ')
-        self.ae(str(s.line(2)), 'ハ   ')
-        self.assertChanges(s, ignore='cursor', cells={1: ((0, 3),), 2: ((0, 1),)})
+        self.ae(str(s.line(1)), 'ニチ')
+        self.ae(str(s.line(2)), 'ハ')
         self.ae(s.cursor.x, 2), self.ae(s.cursor.y, 2)
         s.draw('Ƶ̧\u0308')
-        self.ae(str(s.line(2)), 'ハƵ̧\u0308  ')
+        self.ae(str(s.line(2)), 'ハƵ̧\u0308')
         self.ae(s.cursor.x, 3), self.ae(s.cursor.y, 2)
-        self.assertChanges(s, ignore='cursor', cells={2: ((2, 2),)})
         s.draw('xy'), s.draw('\u0306')
         self.ae(str(s.line(2)), 'ハƵ̧\u0308xy\u0306')
         self.ae(s.cursor.x, 5), self.ae(s.cursor.y, 2)
-        self.assertChanges(s, ignore='cursor', cells={2: ((3, 4),)})
         s.draw('c' * 15)
-        self.ae(str(s.line(0)), 'ニチ ')
+        self.ae(str(s.line(0)), 'ニチ')
 
         # Now test without line-wrap
         s.reset(), s.reset_dirty()
@@ -87,11 +78,9 @@ class TestScreen(BaseTest):
         s.draw('0\u030612345\u03066789\u0306')
         self.ae(str(s.line(0)), '0\u03061239\u0306')
         self.ae(s.cursor.x, 5), self.ae(s.cursor.y, 0)
-        self.assertChanges(s, ignore='cursor', cells={0: ((0, 4),)})
         s.draw('ab\u0306')
         self.ae(str(s.line(0)), '0\u0306123b\u0306')
         self.ae(s.cursor.x, 5), self.ae(s.cursor.y, 0)
-        self.assertChanges(s, ignore='cursor', cells={0: ((4, 4),)})
 
         # Now test in insert mode
         s.reset(), s.reset_dirty()
@@ -103,7 +92,6 @@ class TestScreen(BaseTest):
         s.draw('a\u0306b')
         self.ae(str(s.line(4)), 'a\u0306b1\u030623')
         self.ae((s.cursor.x, s.cursor.y), (2, 4))
-        self.assertChanges(s, ignore='cursor', cells={4: ((0, 4),)})
 
     @skipIf('ANCIENT_WCWIDTH' in os.environ, 'wcwidth() is too old')
     def test_char_manipulation(self):
@@ -121,17 +109,14 @@ class TestScreen(BaseTest):
         s.insert_characters(2)
         self.ae(str(s.line(0)), 'a  bc')
         self.assertTrue(s.line(0).cursor_from(1).bold)
-        self.assertChanges(s, ignore='cursor', cells={0: ((1, 4),)})
         s.cursor_back(1)
         s.insert_characters(20)
-        self.ae(str(s.line(0)), '     ')
-        self.assertChanges(s, ignore='cursor', cells={0: ((0, 4),)})
+        self.ae(str(s.line(0)), '')
         s.draw('xココ')
         s.cursor_back(5)
         s.reset_dirty()
         s.insert_characters(1)
-        self.ae(str(s.line(0)), ' xコ ')
-        self.assertChanges(s, ignore='cursor', cells={0: ((0, 4),)})
+        self.ae(str(s.line(0)), ' xコ')
         c = Cursor()
         c.italic = True
         s.line(0).apply_cursor(c, 0, 5)
@@ -141,34 +126,29 @@ class TestScreen(BaseTest):
 
         init()
         s.delete_characters(2)
-        self.ae(str(s.line(0)), 'ade  ')
+        self.ae(str(s.line(0)), 'ade')
         self.assertTrue(s.line(0).cursor_from(4).bold)
         self.assertFalse(s.line(0).cursor_from(2).bold)
-        self.assertChanges(s, ignore='cursor', cells={0: ((1, 4),)})
 
         init()
         s.erase_characters(2)
         self.ae(str(s.line(0)), 'a  de')
         self.assertTrue(s.line(0).cursor_from(1).bold)
         self.assertFalse(s.line(0).cursor_from(4).bold)
-        self.assertChanges(s, cells={0: ((1, 2),)})
         s.erase_characters(20)
-        self.ae(str(s.line(0)), 'a    ')
+        self.ae(str(s.line(0)), 'a')
 
         init()
         s.erase_in_line()
-        self.ae(str(s.line(0)), 'a    ')
+        self.ae(str(s.line(0)), 'a')
         self.assertTrue(s.line(0).cursor_from(1).bold)
         self.assertFalse(s.line(0).cursor_from(0).bold)
-        self.assertChanges(s, cells={0: ((1, 4),)})
         init()
         s.erase_in_line(1)
         self.ae(str(s.line(0)), '  cde')
-        self.assertChanges(s, cells={0: ((0, 1),)})
         init()
         s.erase_in_line(2)
-        self.ae(str(s.line(0)), '     ')
-        self.assertChanges(s, cells={0: ((0, 4),)})
+        self.ae(str(s.line(0)), '')
         init()
         s.erase_in_line(2, True)
         self.ae((False, False, False, False, False), tuple(map(lambda i: s.line(0).cursor_from(i).bold, range(5))))
@@ -188,23 +168,19 @@ class TestScreen(BaseTest):
 
         init()
         s.erase_in_display()
-        self.ae(all_lines(s), ('12345', '12   ', '     ', '     ', '     '))
-        self.assertChanges(s, lines={2, 3, 4}, cells={1: ((2, 4),)})
+        self.ae(all_lines(s), ('12345', '12', '', '', ''))
 
         init()
         s.erase_in_display(1)
-        self.ae(all_lines(s), ('     ', '   45', '12345', '12345', '12345'))
-        self.assertChanges(s, lines={0}, cells={1: ((0, 2),)})
+        self.ae(all_lines(s), ('', '   45', '12345', '12345', '12345'))
 
         init()
         s.erase_in_display(2)
-        self.ae(all_lines(s), ('     ', '     ', '     ', '     ', '     '))
-        self.assertChanges(s, lines=set(range(5)))
+        self.ae(all_lines(s), ('', '', '', '', ''))
         self.assertTrue(s.line(0).cursor_from(1).bold)
         init()
         s.erase_in_display(2, True)
-        self.ae(all_lines(s), ('     ', '     ', '     ', '     ', '     '))
-        self.assertChanges(s, lines=set(range(5)))
+        self.ae(all_lines(s), ('', '', '', '', ''))
         self.assertFalse(s.line(0).cursor_from(1).bold)
 
     def test_cursor_movement(self):
@@ -227,13 +203,13 @@ class TestScreen(BaseTest):
         s = self.create_screen()
         s.draw('12345' * 5)
         s.index()
-        self.ae(str(s.line(4)), ' ' * 5)
+        self.ae(str(s.line(4)), '')
         for i in range(4):
             self.ae(str(s.line(i)), '12345')
         s.draw('12345' * 5)
         s.cursor_up(5)
         s.reverse_index()
-        self.ae(str(s.line(0)), ' ' * 5)
+        self.ae(str(s.line(0)), '')
         for i in range(1, 5):
             self.ae(str(s.line(i)), '12345')
 
@@ -241,20 +217,25 @@ class TestScreen(BaseTest):
         s = self.create_screen(scrollback=6)
         s.draw(''.join([str(i) * s.columns for i in range(s.lines)]))
         s.resize(3, 10)
-        self.ae(str(s.line(0)), '0'*5 + '1'*5)
-        self.ae(str(s.line(1)), '2'*5 + '3'*5)
-        self.ae(str(s.line(2)), '4'*5 + ' '*5)
+        self.ae(str(s.line(0)), '2'*5 + '3'*5)
+        self.ae(str(s.line(1)), '4'*5)
+        self.ae(str(s.line(2)), ''*5)
         s.resize(5, 1)
         self.ae(str(s.line(0)), '4')
         hb = s.historybuf
-        for i in range(hb.ynum):
-            self.ae(str(hb.line(i)), '4' if i == 0 else '3')
-        s = self.create_screen(scrollback=6)
-        s.draw(''.join([str(i) * s.columns for i in range(s.lines*2)]))
-        self.ae(str(s.line(4)), '9'*5)
+        self.ae(str(hb), '4\n3\n3\n3\n3\n3')
+        s = self.create_screen(scrollback=20)
+        s.draw(''.join(str(i) * s.columns for i in range(s.lines*2)))
+        self.ae(str(s.linebuf), '55555\n66666\n77777\n88888\n99999')
         s.resize(5, 2)
-        self.ae(str(s.line(3)), '9 ')
-        self.ae(str(s.line(4)), '  ')
+        self.ae(str(s.linebuf), '88\n99\n99\n9\n')
+
+    def test_cursor_after_resize(self):
+        s = self.create_screen()
+        s.draw('123'), s.linefeed(), s.carriage_return(), s.draw('123'), s.linefeed(), s.carriage_return()
+        y_before = s.cursor.y
+        s.resize(s.lines, s.columns-1)
+        self.ae(y_before, s.cursor.y)
 
     def test_tab_stops(self):
         # Taken from vttest/main.c

linux-launcher.c

@@ -84,9 +84,13 @@ int main(int argc, char *argv[]) {
     char lib[PATH_MAX+1] = {0};
     char *final_argv[MAX_ARGC + 1] = {0};
     wchar_t *argvw[MAX_ARGC + 1] = {0};
+#ifdef WITH_PROFILER
+    num = snprintf(lib, PATH_MAX, "%s%s", exe_dir, "/");
+#else
     num = snprintf(lib, PATH_MAX, "%s%s", exe_dir, "/../lib/kitty");
+#endif
 
-    if (num < 0 || num >= PATH_MAX) { fprintf(stderr, "Failed to create path to /../lib/kitty\n"); return 1; }
+    if (num < 0 || num >= PATH_MAX) { fprintf(stderr, "Failed to create path to kitty lib\n"); return 1; }
     final_argv[0] = exe;
     final_argv[1] = lib;
     for (i = 1, num_args=2; i < argc && i + 1 <= MAX_ARGC; i++) {

protocol-extensions.asciidoc

@@ -70,260 +70,8 @@ link:http://vt100.net/docs/vt510-rm/DECRPM[DECRPM]
 
 == Graphics rendering
 
-The goal of this specification is to create a flexible and performant protocol
-that allows the program running in the terminal, hereafter called the _client_,
-to render arbitrary pixel (raster) graphics to the screen of the terminal
-emulator. The major design goals are
-
- * Should not require terminal emulators to understand image formats.
- * Should allow specifying graphics to be drawn per individual character cell. This allows graphics to mix with text using
-   the existing cursor based protocols.
- * Should use optimizations when the client is running on the same computer as the terminal emulator.
-
-For some discussion regarding the design choices, see link:../../issues/33[#33].
-
-=== Getting the window size
-
-In order to know what size of images to display and how to position them, the client must be able to get the
-window size in pixels and the number of cells per row and column. This can be done by using the `TIOCGWINSZ` ioctl.
-Some C code to demonstrate its use
-
-```C
-struct ttysize ts; 
-ioctl(0, TIOCGWINSZ, &ts);
-printf("number of columns: %i, number of rows: %i, screen width: %i, screen height: %i\n", sz.ws_col, sz.ws_row, sz.ws_xpixel, sz.ws_ypixel);
-```
-
-Note that some terminals return `0` for the width and height values. Such terminals should be modified to return the correct values.
-Examples of terminals that return correct values: `kitty, xterm`
-
-=== Transferring pixel data
-
-```
-<ESC>_G<control data>;<payload><ESC>\
-```
-
-Before describing this escape code in detail, lets see some quick examples to get a flavor of it in action.
-
-```
-# Draw 10x20 pixels starting at the top-left corner of the current cell.
-<ESC>_Gw=10,h=20,s=100;<pixel data><ESC>\                  
-
-# Ditto, getting the pixel data from /tmp/pixel_data
-<ESC>_Gw=10,h=20,t=f,s=100;<encoded /tmp/pixel_data><ESC>\      
-
-# Ditto, getting the pixel data from /dev/shm/pixel_data, deleting the file after reading data
-<ESC>_Gw=10,h=20,t=t,s=100;<encoded /dev/shm/pixel_data><ESC>\  
-
-# Draw 10x20 pixels starting at the top-left corner of the current cell, ignoring the first 4 rows and 3 columns of the pixel data
-<ESC>_Gw=10,h=20,x=3,y=4,s=100;<pixel data><ESC>\     
-```
-
-This control code is an _Application-Programming Command (APC)_, indicated by
-the leading `<ESC>_`. No modern terminals that I know of use APC codes, and
-well-behaved terminals are supposed to ignore APC codes they do not understand.
-
-The next character `G` indicates this APC code is for graphics data. In the future, we might
-have different first letters for different needs. 
-
-The control data is a comma-separated list of key-value pairs with the restriction that
-keys and values must contain only the characters `0-9a-zA-Z_-+/*`. The payload is arbitrary binary
-data interpreted based on the control codes. The binary data must be base-64 encoded so as to minimize
-the probability of problems with legacy systems that might interpret control
-codes in the binary data incorrectly.
-
-The key to the operation of this escape code is understanding the way the control data works.
-The control data's keys are split up into categories for easier reference.
-
-==== Controlling drawing
-
-|===
-| Key | Default     | Meaning 
-
-| w   | full width  | width -- number of columns of the pixel data to draw 
-| h   | full height | height -- number of rows of the pixel data to draw 
-| x   | zero        | x-offset -- the column in the pixel data to start from (0-based)
-| y   | zero        | y-offset -- the row in the pixel data to start from (0-based)
-|===
-
-The origin for `(x, y)` is the top left corner of the pixel data, with `x`
-increasing from left-to-right and `y` increasing downwards. The terminal
-emulator will draw the specified region starting at the top-left corner of the
-current cell. If the width is greater than a single cell, the cursor will be
-moved one cell to the right and drawing will continue.  If the cursor reaches
-the end of the line, it moves to the next line and starts drawing the next row
-of data.  This means that the displayed image will be truncated at the right
-edge of the screen. If the cursor needs to move past the bottom of the screen,
-the screen is scrolled. After the entire region is drawn, the cursor will be
-positioned at the first cell after the image.
-
-Setting the width and/or height to zero means that no drawing is done and the
-cursor position remains unchanged.
-
-
-==== Transmitting data
-
-The first consideration when transferring data between the client and the
-terminal emulator is the format in which to do so. Since there is a vast and
-growing number of image formats in existence, it does not make sense to have
-every terminal emulator implement support for them. Instead, the client should
-send simple pixel data to the terminal emulator. The obvious downside to this
-is performance, especially when the client is running on a remote machine.
-Techniques for remedying this limitation are discussed later. The terminal
-emulator must understand pixel data in two formats, 24-bit RGB and 32-bit RGBA.
-This is specified using the `f` key in the control data. `f=32` (which is the
-default) indicates 32-bit RGBA data and `f=24` indicates 24-bit RGB data.
-
-One additional parameter is needed to describe the pixel data, the _stride_,
-that is the number of pixels per row. This is encoded using the `s` key, which
-is **required**. For example, `s=100` means there are one hundred pixels per
-row in the pixel data.
-
-Now let us turn to considering how the data is actually transmitted. 
-
-
-===== Local client
-
-When the client and the terminal emulator are on the same computer and share a
-filesystem or shared memory, transfer can happen efficiently using files or
-shared memory objects to pass the data around. The type of transfer is
-controlled by the `t` key. When sending data via files/shared memory, `t` can
-take three values, described below:
-
-|===
-| Value of `t` | Meaning 
-
-| f | A simple file
-| t | A temporary file, the terminal emulator will delete the file after reading the pixel data
-| s | A http://man7.org/linux/man-pages/man7/shm_overview.7.html[POSIX shared memory object]. The terminal emulator will delete it after reading the pixel data 
-|===
-
-In all these cases, the payload data must be the base-64 encoded absolute file path.
-
-[[query]]An important consideration is how the client can tell if the terminal emulator
-and it share a filesystem. This can be done by using the _response mode_, specifying
-the `q` key, with some unique id as the value. For example,
-
-```
-<ESC>_Gt=t,s=100,q=33;<encoded /tmp/pixel_data><ESC>\      
-```
-
-When the terminal emulator receives this escape code, it will read and display
-the pixel data as normal, and also send an escape code back to the client
-indicating whether the reading of the data was successful or not. The returned
-escape code will look like:
-
-```
-<ESC>_Gq=33;<encoded error message or OK><ESC>\
-```
-
-Here the `q` value will be the same as was sent by the client in the original
-request.  The payload data will be a base-64 encoded UTF-8 string. The string
-will be `OK` if reading the pixel data succeeded or an error message. Clients 
-can set the width and height to zero to avoid actually drawing anything on
-screen during the test.
-
-
-===== Remote client
-
-Remote clients, those that are unable to use the filesystem/shared memory to
-transmit data, must send the pixel data directly using escape codes. Since
-escape codes are of limited maximum length, the data will need to be chunked up
-for transfer. This is done using the `m` key. The pixel data must first be
-base64 encoded then chunked up into chunks no larger than `4096` bytes. The client
-then sends the graphics escape code as usual, with the addition of an `m` key that
-must have the value `1` for all but the last chunk, where it must be `0`. For example,
-if the data is split into three chunks, the client would send the following
-sequence of escape codes to the terminal emulator:
-
-```
-<ESC>_Gw=100,h=30,s=100,m=1;<base-64 pixel data first chunk><ESC>\                  
-<ESC>_Gm=1;<base-64 pixel data second chunk><ESC>\                  
-<ESC>_Gm=0;<base-64 pixel data last chunk><ESC>\                  
-```
-
-Note that only the first escape code needs to have the full set of control
-codes such as stride, width, height, format etc. Subsequent chunks must have
-only the `m` key. The client must finish sending all chunks for a single image
-before sending any other graphics related escape codes.
-
-
-=== Image persistence
-
-Full screen applications may need to render the same image multiple times or
-even render different parts of an image, in different locations, for example,
-if the image is sprite map. Resending the image data each time this happens is
-wasteful. Instead this protocol allows the client to have the terminal emulator
-manage a persistent store of images. 
-
-Persistence is implemented by simply assigning an id to transmitted pixel data using the 
-key `i`. So for example,
-
-```
-<ESC>_Gt=t,s=100,i=some-id;<encoded /tmp/pixel_data><ESC>\
-```
-
-Now, if the client wants to redraw that image in the future, all it has to do is send
-a code with the keys `t=i,i=some-id`, and no payload, like this:
-
-```
-<ESC>_Gt=i,i=some-id;<ESC>\
-```
-
-The client can use the `w, h, x, y` keys to draw different parts of the image
-and draw it at different locations by positioning the cursor before sending the
-code.
-
-Saved images can be deleted, to free up resources, by using the code:
-
-```
-<ESC>_Gt=d,i=some-id;<ESC>\
-```
-
-The special value of `i=*` will cause the terminal emulator to delete all
-stored images.  Well behaved clients should send this code before terminating.
-
-Terminal emulators may limit the maximum amount of saved data to avoid denial-of-service
-attacks.  Terminal emulators should make the limit fairly generous, at least a
-few hundred, full screen, RGBA images worth of data should be allowed. 
-
-Client applications can check if an image is still stored by sending the `q`
-key, as described <<query,above>>. For example,
-
-```
-<ESC>_Gt=i,i=some-id,q=some-id;<ESC>\
-```
-
-The terminal emulator will respond with:
-
-```
-<ESC>_Gq=some-id;<encoded OK or error message><ESC>\
-```
-
-If `OK` is sent the image was successfully loaded from the persistent storage, if not,
-then it must be resent.
-
-Note that when using the local filesystem to send data (`t=f`) mode, there is
-no need to use this persistence mechanism, as the client can directly refer to
-the file repeatedly with no overhead.
-
-=== A summary of the control keys used
-
-|===
-|Key | Description
-
-| f  | The _format_ of the transmitted pixel data
-| h  | _height_ -- number of rows of the pixel data to draw 
-| i  | _id_ to save transmitted data in persistent storage
-| m  | indicates whether there is _more_ data to come during a chunked transfer
-| q  | _query_ the terminal emulator to see if transmission succeeded
-| s  | The _stride_ of the transmitted pixel data
-| t  | The _type_ of transmission medium used
-| w  | _width_ -- number of columns of the pixel data to draw 
-| x  | _x-offset_ -- the column in the pixel data to start from (0-based)
-| y  | _y-offset_ -- the row in the pixel data to start from (0-based)
-
-|===
+See link:graphics-protocol.asciidoc[Graphics Protocol] for a description
+of this protocol to enable drawing of arbitrary raster images in the terminal.
 
 
 == Keyboard handling

setup.py

@@ -99,7 +99,7 @@ def get_sanitize_args(cc, ccver):
     return sanitize_args
 
 
-def init_env(debug=False, sanitize=False, native_optimizations=True):
+def init_env(debug=False, sanitize=False, native_optimizations=True, profile=False):
     global cflags, ldflags, cc, ldpaths
     native_optimizations = native_optimizations and not sanitize and not debug
     cc, ccver = cc_version()
@@ -115,10 +115,10 @@ def init_env(debug=False, sanitize=False, native_optimizations=True):
     cflags = os.environ.get(
         'OVERRIDE_CFLAGS', (
             '-Wextra -Wno-missing-field-initializers -Wall -std=c99 -D_XOPEN_SOURCE=700'
-            ' -pedantic-errors -Werror {} {} -D{}DEBUG -fwrapv {} {} -pipe {}'
+            ' -pedantic-errors -Werror {} {} -D{}DEBUG -fwrapv {} {} -pipe {} -fvisibility=hidden'
         ).format(
-            optimize, ' '.join(sanitize_args), ('' if debug else 'N'), stack_protector, missing_braces, '-march=native'
-            if native_optimizations else ''
+            optimize, ' '.join(sanitize_args), ('' if debug else 'N'), stack_protector, missing_braces,
+            '-march=native' if native_optimizations else '',
         )
     )
     cflags = shlex.split(cflags
@@ -129,7 +129,13 @@ def init_env(debug=False, sanitize=False, native_optimizations=True):
     ldflags = shlex.split(ldflags)
     cflags += shlex.split(os.environ.get('CFLAGS', ''))
     ldflags += shlex.split(os.environ.get('LDFLAGS', ''))
+    if not debug:
+        cflags.append('-flto'), ldflags.append('-flto')
 
+    if profile:
+        cflags.append('-DWITH_PROFILER')
+        cflags.append('-g')
+        ldflags.append('-lprofiler')
     cflags.append('-pthread')
     # We add 4000 to the primary version because vim turns on SGR mouse mode
     # automatically if this version is high enough
@@ -149,6 +155,7 @@ def init_env(debug=False, sanitize=False, native_optimizations=True):
             raise SystemExit(
                 'glew >= 2.0.0 is required, found version: ' + ver
             )
+    cflags.extend(pkg_config('libpng', '--cflags-only-I'))
     if not isosx:
         cflags.extend(pkg_config('glew', '--cflags-only-I'))
     if isosx:
@@ -166,7 +173,12 @@ def init_env(debug=False, sanitize=False, native_optimizations=True):
     else:
         glfw_ldflags = pkg_config('glfw3', '--libs')
         glew_libs = pkg_config('glew', '--libs')
-    ldpaths = pylib + glew_libs + font_libs + glfw_ldflags
+    libpng = pkg_config('libpng', '--libs')
+    ldpaths = pylib + glew_libs + font_libs + glfw_ldflags + libpng + ['-lunistring']
+    if not isosx:
+        ldpaths += ['-lrt']
+    if '-lz' not in ldpaths:
+        ldpaths.append('-lz')
 
     try:
         os.mkdir(build_dir)
@@ -190,6 +202,7 @@ def run_tool(cmd):
 
 SPECIAL_SOURCES = {
     'kitty/parser_dump.c': ('kitty/parser.c', ['DUMP_COMMANDS']),
+    'kitty/shaders_debug.c': ('kitty/shaders.c', ['ENABLE_DEBUG_GL']),
 }
 
 
@@ -204,6 +217,26 @@ def newer(dest, *sources):
     return False
 
 
+def dependecies_for(src, obj, all_headers):
+    dep_file = obj.rpartition('.')[0] + '.d'
+    try:
+        deps = open(dep_file).read()
+    except FileNotFoundError:
+        yield src
+        yield from iter(all_headers)
+    else:
+        RE_INC = re.compile(r'^(?P<target>.+?):\s+(?P<deps>.+?)$', re.MULTILINE)
+        SPACE_TOK = '\x1B'
+
+        text = deps.replace('\\\n', ' ').replace('\\ ', SPACE_TOK)
+        for match in RE_INC.finditer(text):
+            files = (f.replace(SPACE_TOK, ' ') for f in match.group('deps').split())
+            for path in files:
+                path = os.path.abspath(path)
+                if path.startswith(base):
+                    yield path
+
+
 def compile_c_extension(module, incremental, sources, headers):
     prefix = os.path.basename(module)
     objects = [
@@ -218,8 +251,8 @@ def compile_c_extension(module, incremental, sources, headers):
             cflgs.extend(map(define, defines))
 
         src = os.path.join(base, src)
-        if not incremental or newer(dest, src, *headers):
-            run_tool([cc] + cflgs + ['-c', src] + ['-o', dest])
+        if not incremental or newer(dest, *dependecies_for(src, dest, headers)):
+            run_tool([cc, '-MMD'] + cflgs + ['-c', src] + ['-o', dest])
     dest = os.path.join(base, module + '.so')
     if not incremental or newer(dest, *objects):
         run_tool([cc] + ldflags + objects + ldpaths + ['-o', dest])
@@ -231,7 +264,7 @@ def option_parser():
         'action',
         nargs='?',
         default='build',
-        choices='build test linux-package osx-bundle'.split(),
+        choices='build test linux-package osx-bundle clean'.split(),
         help='Action to perform (default is build)'
     )
     p.add_argument(
@@ -260,6 +293,12 @@ def option_parser():
         action='store_true',
         help='Only build changed files'
     )
+    p.add_argument(
+        '--profile',
+        default=False,
+        action='store_true',
+        help='Use the -pg compile flag to add profiling information'
+    )
     return p
 
 
@@ -277,21 +316,19 @@ def find_c_files():
         key=lambda x: os.path.getmtime(os.path.join(base, x)), reverse=True
     )
     ans.append('kitty/parser_dump.c')
+    ans.append('kitty/shaders_debug.c')
     return tuple(ans), tuple(headers)
 
 
 def build(args, native_optimizations=True):
-    init_env(args.debug, args.sanitize, native_optimizations)
+    init_env(args.debug, args.sanitize, native_optimizations, args.profile)
     compile_c_extension(
         'kitty/fast_data_types', args.incremental, *find_c_files()
     )
 
 
 def safe_makedirs(path):
-    try:
-        os.makedirs(path)
-    except FileExistsError:
-        pass
+    os.makedirs(path, exist_ok=True)
 
 
 def build_test_launcher(args):
@@ -306,6 +343,25 @@ def build_test_launcher(args):
     run_tool(cmd)
 
 
+def build_linux_launcher(args, launcher_dir='.', for_bundle=False):
+    cflags = '-Wall -Werror -fpie'.split()
+    libs = []
+    if args.profile:
+        cflags.append('-DWITH_PROFILER'), cflags.append('-g')
+        libs.append('-lprofiler')
+    else:
+        cflags.append('-O3')
+    if for_bundle:
+        cflags.append('-DFOR_BUNDLE')
+        cflags.append('-DPYVER="{}"'.format(sysconfig.get_python_version()))
+    pylib = get_python_flags(cflags)
+    exe = 'kitty-profile' if args.profile else 'kitty'
+    cmd = [cc] + cflags + [
+        'linux-launcher.c', '-o', os.path.join(launcher_dir, exe)
+    ] + libs + pylib
+    run_tool(cmd)
+
+
 def package(args, for_bundle=False):  # {{{
     ddir = args.prefix
     libdir = os.path.join(ddir, 'lib', 'kitty')
@@ -322,7 +378,7 @@ def package(args, for_bundle=False):  # {{{
     def src_ignore(parent, entries):
         return [
             x for x in entries
-            if '.' in x and x.rpartition('.')[2] not in ('py', 'so', 'conf')
+            if '.' in x and x.rpartition('.')[2] not in ('py', 'so', 'conf', 'glsl')
         ]
 
     shutil.copytree('kitty', os.path.join(libdir, 'kitty'), ignore=src_ignore)
@@ -334,15 +390,7 @@ def package(args, for_bundle=False):  # {{{
             os.chmod(path, 0o755 if f.endswith('.so') else 0o644)
     launcher_dir = os.path.join(ddir, 'bin')
     safe_makedirs(launcher_dir)
-    cflags = '-O3 -Wall -Werror -fpie'.split()
-    if for_bundle:
-        cflags.append('-DFOR_BUNDLE')
-        cflags.append('-DPYVER="{}"'.format(sysconfig.get_python_version()))
-    pylib = get_python_flags(cflags)
-    cmd = [cc] + cflags + [
-        'linux-launcher.c', '-o', os.path.join(launcher_dir, 'kitty')
-    ] + pylib
-    run_tool(cmd)
+    build_linux_launcher(args, launcher_dir, for_bundle)
     if not isosx:  # {{{ linux desktop gunk
         icdir = os.path.join(ddir, 'share', 'icons', 'hicolor', '256x256')
         safe_makedirs(icdir)
@@ -377,6 +425,15 @@ Categories=System;
     # }}}
 
 
+def clean():
+    for f in subprocess.check_output('git ls-files --others --ignored --exclude-from=.gitignore'.split()).decode('utf-8').splitlines():
+        if f.startswith('logo/kitty.iconset') or f.startswith('dev/'):
+            continue
+        os.unlink(f)
+        if os.sep in f and not os.listdir(os.path.dirname(f)):
+            os.rmdir(os.path.dirname(f))
+
+
 def main():
     if sys.version_info < (3, 5):
         raise SystemExit('python >= 3.5 required')
@@ -386,6 +443,9 @@ def main():
     if args.action == 'build':
         build(args)
         build_test_launcher(args)
+        if args.profile:
+            build_linux_launcher(args)
+            print('kitty profile executable is', 'kitty-profile')
     elif args.action == 'test':
         os.execlp(
             sys.executable, sys.executable, os.path.join(base, 'test.py')
@@ -396,6 +456,8 @@ def main():
     elif args.action == 'osx-bundle':
         build(args, native_optimizations=False)
         package(args, for_bundle=True)
+    elif args.action == 'clean':
+        clean()
 
 
 if __name__ == '__main__':