version 0.5.1

Fixes #194

.gitattributes

@@ -0,0 +1,7 @@
+kitty/wcwidth9.h linguist-generated=true
+kitty/keys.h linguist-generated=true
+kitty/charsets.c linguist-generated=true
+kitty/key_encoding.py linguist-generated=true
+kitty/rgb.py linguist-generated=true
+kitty/gl-wrapper.* linguist-generated=true
+kitty/khrplatform.h linguist-generated=true

.gitignore

@@ -5,7 +5,9 @@ tags
 build
 linux-package
 logo/*.iconset
-test-launcher
+compile_commands.json
+glad/out
+asan-launcher
 kitty-profile
 dev
 __pycache__

.travis.yml

@@ -13,7 +13,7 @@ matrix:
             apt:
               packages:
                 - libfontconfig1-dev
-                - libglew-dev
+                - libharfbuzz-dev
                 - libxi-dev
                 - libxrandr-dev
                 - libxinerama-dev
@@ -34,7 +34,7 @@ matrix:
             apt:
               packages:
                 - libfontconfig1-dev
-                - libglew-dev
+                - libharfbuzz-dev
                 - libxi-dev
                 - libxrandr-dev
                 - libxinerama-dev
@@ -55,7 +55,7 @@ matrix:
             apt:
               packages:
                 - libfontconfig1-dev
-                - libglew-dev
+                - libharfbuzz-dev
                 - libxi-dev
                 - libxrandr-dev
                 - libxinerama-dev
@@ -75,8 +75,8 @@ matrix:
 env:
     global:
         - PYTHON=python3
-        - PKG_CONFIG_PATH=$HOME/glfw/lib/pkgconfig:$PKG_CONFIG_PATH
-        - LD_LIBRARY_PATH=$HOME/glfw/lib:$LD_LIBRARY_PATH
+        - PKG_CONFIG_PATH=$HOME/glfw/lib/pkgconfig:$HOME/harfbuzz/lib/pkgconfig:$PKG_CONFIG_PATH
+        - LD_LIBRARY_PATH=$HOME/glfw/lib:$HOME/harfbuzz/lib:$LD_LIBRARY_PATH
         - ASAN_OPTIONS=leak_check_at_exit=0
 
 install: |
@@ -88,20 +88,32 @@ install: |
             brew install python3;
             brew install glfw;
             brew install libunistring;
+            brew install freetype;
+            brew install harfbuzz --without-glib --without-gobject-introspection --without-graphite2 --without-icu4c;
         else
             mkdir -p $SW;
             curl https://download.calibre-ebook.com/travis/kitty/osx.tar.xz | tar xJ -C $SW;
         fi
     else
+        pushd /tmp
         wget -O glfw-3.2.1.zip https://github.com/glfw/glfw/archive/3.2.1.zip
         unzip -q glfw-3.2.1.zip
         cd glfw-3.2.1
         cmake -DBUILD_SHARED_LIBS=ON -DGLFW_BUILD_EXAMPLES=OFF -DGLFW_BUILD_TESTS=OFF -DGLFW_BUILD_DOCS=OFF -DCMAKE_INSTALL_PREFIX=$HOME/glfw
         make
         make install
+
+        wget https://github.com/behdad/harfbuzz/releases/download/1.6.3/harfbuzz-1.6.3.tar.bz2
+        tar xf harfbuzz-1.6.3.tar.bz2
+        cd harfbuzz-1.6.3
+        ./configure --prefix=$HOME/harfbuzz --disable-dependency-tracking --disable-static --with-glib=no --with-gobject=no --with-cairo=no --with-fontconfig=no --with-icu=no --with-directwrite=no --with-freetype=yes
+        make -j2
+        make install
         cd ..
+        popd
     fi
     pkg-config --cflags glfw3
+    pkg-config --cflags harfbuzz
     if [[ "$TRAVIS_OS_NAME" != 'osx' ]]; then
         PLIB=$(ldd `which python` | grep libpython | cut -d ' ' -f 3)
         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`dirname $PLIB`
@@ -111,9 +123,8 @@ install: |
 before_script:
     - echo $LD_LIBRARY_PATH
     - $PYTHON setup.py build --debug $SANITIZE_ARG;
-    - if [[ "$TRAVIS_OS_NAME" != 'osx' ]]; then ldd ./test-launcher `which $PYTHON`; fi
 
 script:
-    - ./test-launcher
+    - if [[ -z $SANITIZE_ARG ]]; then $PYTHON test.py; else ./asan-launcher test.py; fi
     - if [[ "$RUN_FLAKE" == "1" ]]; then flake8 --count .; fi
     - if [[ "$BUILD_PKG" == "1" ]]; then $PYTHON setup.py linux-package; fi

CHANGELOG.rst

@@ -1,14 +1,66 @@
 Changelog
 ==============
 
-kitty is a feature full, cross-platform, *fast*, OpenGL based terminal emulator.
+kitty is a feature full, cross-platform, *fast*, GPU based terminal emulator.
+
+version 0.5.1 [2017-12-01]
+---------------------------
+
+- Add an option to control the thickness of lines in box drawing characters
+
+- Increase max. allowed ligature length to nine characters
+
+- Fix text not vertically centered when adjusting line height
+
+- Fix unicode block characters not being rendered properly
+
+- Fix shift+up/down not generating correct escape codes
+
+- Image display: Fix displaying images taller than two screen heights not
+  scrolling properly
 
 
-version 0.4.1 [2017-10-23]
+version 0.5.0 [2017-11-19]
+---------------------------
+
+- Add support for ligature fonts such as Fira Code, Hasklig, etc. kitty now
+  uses harfbuzz for text shaping which allow it to support advanced OpenType
+  features such as contextual alternates/ligatures/combining glyphs/etc.
+
+- Make it easy to select fonts by allowing listing of monospace fonts using:
+  kitty list-fonts
+
+- Add an option to have window focus follow mouse
+
+- Add a keyboard shortcut (ctrl+shift+f11) to toggle fullscreen mode
+
+- macOS: Fix handling of option key. It now behaves just like the alt key on
+  Linux. There is an option to make it type unicode characters instead.
+
+- Linux: Add support for startup notification on X11 desktops. kitty will
+  now inform the window manager when its startup is complete.
+
+- Fix shell prompt being duplicated when window is resized
+
+- Fix crash when displaying more than 64 images in the same session
+
+- Add support for colons in SGR color codes. These are generated by some
+  applications such as neovim when they mistakenly identify kitty as a libvte
+  based terminal.
+
+- Fix mouse interaction not working in apps using obsolete mouse interaction
+  protocols
+
+- Linux: no longer require glew as a dependency
+
+
+version 0.4.2 [2017-10-23]
 ---------------------------
 
 - Fix a regression in 0.4.0 that broke custom key mappings
 
+- Fix a regression in 0.4.0 that broke support for non-QWERTY keyboard layouts 
+
 - Avoid using threads to reap zombie child processes. Also prevent kitty from
   hanging if the open program hangs when clicking on a URL.
 

Makefile

@@ -6,3 +6,11 @@ test:
 
 clean:
 	python3 setup.py clean
+
+# A debug build
+debug:
+	python3 setup.py build --debug
+
+# Build with the ASAN and UBSAN sanitizers
+asan:
+	python3 setup.py build --debug --sanitize

README.asciidoc

@@ -5,10 +5,12 @@
 :sc_close_tab: pass:quotes[`ctrl+shift+q`]
 :sc_close_window: pass:quotes[`ctrl+shift+w`]
 :sc_copy_to_clipboard: pass:quotes[`ctrl+shift+c`]
+:sc_decrease_font_size: pass:quotes[`ctrl+shift+minus`]
 :sc_eighth_window: pass:quotes[`ctrl+shift+8`]
 :sc_fifth_window: pass:quotes[`ctrl+shift+5`]
 :sc_first_window: pass:quotes[`ctrl+shift+1`]
 :sc_fourth_window: pass:quotes[`ctrl+shift+4`]
+:sc_increase_font_size: pass:quotes[`ctrl+shift+equal`]
 :sc_move_tab_backward: pass:quotes[`ctrl+shift+,`]
 :sc_move_tab_forward: pass:quotes[`ctrl+shift+.`]
 :sc_move_window_backward: pass:quotes[`ctrl+shift+b`]
@@ -20,10 +22,12 @@
 :sc_next_tab: pass:quotes[`ctrl+shift+right`]
 :sc_next_window: pass:quotes[`ctrl+shift+]`]
 :sc_ninth_window: pass:quotes[`ctrl+shift+9`]
+:sc_pass_selection_to_program: pass:quotes[`ctrl+shift+o`]
 :sc_paste_from_clipboard: pass:quotes[`ctrl+shift+v`]
 :sc_paste_from_selection: pass:quotes[`ctrl+shift+s` or `shift+insert`]
 :sc_previous_tab: pass:quotes[`ctrl+shift+left`]
 :sc_previous_window: pass:quotes[`ctrl+shift+[`]
+:sc_restore_font_size: pass:quotes[`ctrl+shift+backspace`]
 :sc_scroll_end: pass:quotes[`ctrl+shift+end`]
 :sc_scroll_home: pass:quotes[`ctrl+shift+home`]
 :sc_scroll_line_down: pass:quotes[`ctrl+shift+down` or `ctrl+shift+j`]
@@ -36,6 +40,7 @@
 :sc_sixth_window: pass:quotes[`ctrl+shift+6`]
 :sc_tenth_window: pass:quotes[`ctrl+shift+0`]
 :sc_third_window: pass:quotes[`ctrl+shift+3`]
+:sc_toggle_fullscreen: pass:quotes[`ctrl+shift+f11`]
 // END_SHORTCUT_BLOCK
 
 ++++
@@ -49,7 +54,7 @@ image::https://travis-ci.org/kovidgoyal/kitty.svg?branch=master[Build status, li
 layouts without needing to use an extra program like tmux
 
 * Supports all modern terminal features: graphics (images), unicode,
-true-color, mouse protocol, focus tracking, bracketed paste and so on.
+true-color, OpenType ligatures, mouse protocol, focus tracking, bracketed paste and so on.
 
 * Supports startup sessions which allow you to specify the window/tab layout,
 working directories and programs to run on startup.
@@ -62,8 +67,10 @@ latency.
 it uses only OpenGL for rendering, it should be trivial to port to other
 platforms. 
 
-* Allows you to view the scrollback buffer in a separate window using your
-favorite pager program such as less
+* Allows you to open the scrollback buffer and window contents in a separate
+window using arbitrary programs of your choice. This is useful for browsing
+the history comfortably in a pager or editor and also for doing things like
+opening URLs using only the keyboard.
 
 * Easily hackable (UI layer written in python, inner loops in C for
 speed). Less than fifteen thousand lines of code.
@@ -85,9 +92,9 @@ the following dependencies are installed first.
 * libunistring 
 * zlib
 * libpng
-* glew   >= 2.0 (not needed on macOS)
+* freetype
 * fontconfig (not needed on macOS)
-* xrdb (only on X11 based systems)
+* harfbuzz >= 1.5.0 
 * xsel (only on X11 systems with glfw < 3.3)
 * ImageMagick (optional, needed to use the `kitty icat` tool to display images in the terminal)
 * gcc or clang (required only for building)
@@ -115,6 +122,8 @@ python3 /path/to/kitty/folder
 
 * Arch linux: AUR git package https://aur.archlinux.org/packages/kitty-git/
 
+* NixOS / nixpkgs: https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/kitty/default.nix
+
 
 === macOS packages
 
@@ -207,6 +216,19 @@ windows are:
 
 |===
 
+=== Miscellaneous
+
+|===
+|Action |Shortcut
+
+|Increase font size   | {sc_increase_font_size}
+|Decrease font size   | {sc_decrease_font_size}
+|Restore font size    | {sc_restore_font_size} 
+|Toggle fullscreen    | {sc_toggle_fullscreen}
+|Pass current selection to program | {sc_pass_selection_to_program}
+|===
+
+
 
 == The scrollback buffer
 

__main__.py

@@ -3,11 +3,13 @@
 # License: GPL v3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
 
 import sys
-sys.debug_gl = '--debug-kitty-gl' in sys.argv
 
 if len(sys.argv) > 1 and sys.argv[1] == 'icat':
     from kitty.icat import main
     main(sys.argv[1:])
+elif len(sys.argv) > 1 and sys.argv[1] == 'list-fonts':
+    from kitty.fonts.list import main
+    main(sys.argv[1:])
 else:
     from kitty.main import main
     main()

asan-launcher.c

@@ -0,0 +1,19 @@
+/*
+ * linux-launcher.c
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include <Python.h>
+
+#define MAX_ARGC 1024
+
+int main(int argc, char *argv[]) {
+    wchar_t *argvw[MAX_ARGC + 1] = {0};
+    argvw[0] = L"kitty";
+    for (int i = 1; i < argc; i++) argvw[i] = Py_DecodeLocale(argv[i], NULL);
+    int ret = Py_Main(argc, argvw);
+    for (int i = 1; i < argc; i++) PyMem_RawFree(argvw[i]);
+    return ret;
+}

count-lines-of-code

@@ -1,2 +1,2 @@
 #!/bin/bash
-cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/glfw.c\nkitty/keys.h\nkitty/charsets.c\nkitty/key_encoding.py\nkitty/rgb.py\nkitty/gl.h') kitty
+cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/glfw.c\nkitty/keys.h\nkitty/charsets.c\nkitty/key_encoding.py\nkitty/rgb.py\nkitty/gl.h\nkitty/gl-wrapper.h\nkitty/gl-wrapper.c\nkitty/khrplatform.h') kitty

glad/generate.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+import re
+import shlex
+import shutil
+import subprocess
+
+cmdline = (
+    'glad --profile core --out-path {dest} --api gl=3.3 --generator {generator} --spec gl'
+    ' --extensions GL_ARB_texture_storage,GL_ARB_copy_image,GL_ARB_multisample,GL_ARB_robustness,GL_KHR_debug'
+)
+
+
+def clean(x):
+    if os.path.exists(x):
+        shutil.rmtree(x)
+
+
+def regenerate():
+    clean('out')
+
+    subprocess.check_call(
+        shlex.split(cmdline.format(dest='out', generator='c-debug'))
+    )
+
+
+def export():
+    c = open('out/src/glad.c', 'rb').read().decode('utf-8')
+    functions = []
+
+    def sub(m):
+        functions.append(m.group(2))
+        return m.group()
+
+    c = re.sub(r'^([A-Z0-9]+) glad_debug_([a-zA-Z0-9]+) = glad_debug_impl_\2;$', sub, c, flags=re.M)
+    switch = ['glad_debug_{0} = glad_{0}'.format(f) for f in functions]
+    c = c.replace('<glad/glad.h>', '"gl-wrapper.h"', 1)
+    c = '#pragma GCC diagnostic ignored "-Wpedantic"\n' + c
+    c += '''
+int
+init_glad(GLADloadproc load, int debug) {
+    int ret = gladLoadGLLoader(load);
+    if (ret && !debug) {
+        SUB;
+    }
+    return ret;
+}'''.replace('SUB', ';\n        '.join(switch), 1)
+    open('../kitty/gl-wrapper.c', 'w').write(c)
+    raw = open('out/include/glad/glad.h').read()
+    raw = raw.replace('<KHR/khrplatform.h>', '"khrplatform.h"')
+    raw += '\nint init_glad(GLADloadproc, int);\n'
+    open('../kitty/gl-wrapper.h', 'w').write(raw)
+    shutil.copy2('out/include/KHR/khrplatform.h', '../kitty')
+
+
+if __name__ == '__main__':
+    os.chdir(os.path.dirname(os.path.abspath(__file__)))
+    regenerate()
+    export()

kitty/boss.py

@@ -11,8 +11,8 @@ from .fast_data_types import (
     GLFW_KEY_DOWN, GLFW_KEY_UP, ChildMonitor, destroy_global_data,
     destroy_sprite_map, glfw_post_empty_event, layout_sprite_map
 )
-from .fonts.render import render_cell_wrapper, set_font_family
-from .keys import get_key_map, get_sent_data, get_shortcut
+from .fonts.render import prerender, resize_fonts, set_font_family
+from .keys import get_key_map, get_shortcut
 from .session import create_session
 from .tabs import SpecialWindow, TabManager
 from .utils import (
@@ -23,7 +23,8 @@ from .window import load_shader_programs
 
 def initialize_renderer():
     load_shader_programs()
-    layout_sprite_map(cell_size.width, cell_size.height, render_cell_wrapper)
+    layout_sprite_map(cell_size.width, cell_size.height)
+    prerender()
 
 
 class DumpCommands:  # {{{
@@ -109,6 +110,9 @@ class Boss:
         for window in tab:
             self.close_window(window)
 
+    def toggle_fullscreen(self):
+        self.glfw_window.toggle_fullscreen()
+
     def start(self):
         if not getattr(self, 'io_thread_started', False):
             self.child_monitor.start()
@@ -139,9 +143,9 @@ class Boss:
         self.current_font_size = new_size
         w, h = cell_size.width, cell_size.height
         windows = tuple(filter(None, self.window_id_map.values()))
-        cell_size.width, cell_size.height = set_font_family(
-            self.opts, override_font_size=self.current_font_size)
-        layout_sprite_map(cell_size.width, cell_size.height, render_cell_wrapper)
+        cell_size.width, cell_size.height = resize_fonts(self.current_font_size)
+        layout_sprite_map(cell_size.width, cell_size.height)
+        prerender()
         for window in windows:
             window.screen.rescale_images(w, h)
         self.resize_windows_after_font_size_change()
@@ -195,13 +199,6 @@ class Boss:
                 passthrough = f(*key_action.args)
                 if passthrough is not True:
                     return True
-        key, scancode, action, mods = self.current_key_press_info
-        data = get_sent_data(
-            self.opts.send_text_map, key, scancode, mods, window, action
-        )
-        if data:
-            window.write_to_child(data)
-            return True
         return False
 
     def combine(self, *actions):
@@ -263,11 +260,10 @@ class Boss:
         self.child_monitor.shutdown()
         wakeup()
         self.child_monitor.join()
-        for t in self.tab_manager:
-            t.destroy()
-        del self.tab_manager
+        self.tab_manager.destroy()
         destroy_sprite_map()
         destroy_global_data()
+        del self.tab_manager
         del self.glfw_window
 
     def paste_to_active_window(self, text):

kitty/child-monitor.c

@@ -21,13 +21,11 @@ extern int pthread_setname_np(const char *name);
 #include <unistd.h>
 #include <float.h>
 #include <fcntl.h>
-#ifndef __APPLE__
-#include <stropts.h>
-#endif
 #include <sys/ioctl.h>
 #include <sys/wait.h>
 #include <signal.h>
 #include <GLFW/glfw3.h>
+extern PyTypeObject Screen_Type;
 
 #define EXTRA_FDS 2
 #define wakeup_main_loop glfwPostEmptyEvent
@@ -393,7 +391,7 @@ resize_pty(ChildMonitor *self, PyObject *args) {
     if (fd == -1) FIND(add_queue, add_queue_count);
     if (fd != -1) {
         if (!pty_resize(fd, &dim)) PyErr_SetFromErrno(PyExc_OSError);
-    } else fprintf(stderr, "Failed to send resize signal to child with id: %lu (children count: %u) (add queue: %lu)\n", window_id, self->count, add_queue_count);
+    } else fprintf(stderr, "Failed to send resize signal to child with id: %lu (children count: %u) (add queue: %zu)\n", window_id, self->count, add_queue_count);
     children_mutex(unlock);
     if (PyErr_Occurred()) return NULL;
     Py_RETURN_NONE;
@@ -436,9 +434,6 @@ pyset_iutf8(ChildMonitor *self, PyObject *args) {
 #undef DECREF_CHILD
 
 static double last_render_at = -DBL_MAX;
-draw_borders_func draw_borders = NULL;
-draw_cells_func draw_cells = NULL;
-draw_cursor_func draw_cursor = NULL;
 
 static inline double
 cursor_width(double w, bool vert) {

kitty/colors.c

@@ -8,6 +8,8 @@
 #include "data-types.h"
 #include <structmember.h>
 
+PyTypeObject ColorProfile_Type;
+
 static uint32_t FG_BG_256[256] = {
     0x000000,  // 0
     0xcd0000,  // 1

kitty/config.py

@@ -19,7 +19,7 @@ from .rgb import to_color
 from .utils import safe_print
 
 key_pat = re.compile(r'([a-zA-Z][a-zA-Z0-9_-]*)\s+(.+)$')
-MINIMUM_FONT_SIZE = 6
+MINIMUM_FONT_SIZE = 4
 
 
 def to_font_size(x):
@@ -107,6 +107,8 @@ def parse_key_action(action):
         sep, rest = rest.split(' ', 1)
         parts = re.split(r'\s*' + re.escape(sep) + r'\s*', rest)
         args = tuple(map(parse_key_action, filter(None, parts)))
+    elif func == 'send_text':
+        args = rest.split(' ', 1)
     elif func in shlex_actions:
         args = shlex.split(rest)
     return KeyAction(func, args)
@@ -165,12 +167,15 @@ def parse_symbol_map(val):
             return abort()
         if b < a or max(a, b) > sys.maxunicode or min(a, b) < 1:
             return abort()
-        for y in range(a, b + 1):
-            symbol_map[chr(y)] = family
+        symbol_map[(a, b)] = family
     return symbol_map
 
 
-def parse_send_text(val):
+def parse_send_text_bytes(text):
+    return ast.literal_eval("'''" + text + "'''").encode('utf-8')
+
+
+def parse_send_text(val, keymap):
     parts = val.split(' ')
 
     def abort(msg):
@@ -182,28 +187,10 @@ def parse_send_text(val):
 
     if len(parts) < 3:
         return abort('Incomplete')
-
-    text = ' '.join(parts[2:])
     mode, sc = parts[:2]
-    mods, key = parse_shortcut(sc.strip())
-    if key is None:
-        return abort('Invalid shortcut')
-    text = ast.literal_eval("'''" + text + "'''").encode('utf-8')
-    if not text:
-        return abort('Empty text')
-
-    if mode in ('all', '*'):
-        modes = parse_send_text.all_modes
-    else:
-        modes = frozenset(mode.split(',')).intersection(
-            parse_send_text.all_modes
-        )
-        if not modes:
-            return abort('Invalid keyboard modes')
-    return {mode: {(mods, key): text} for mode in modes}
-
-
-parse_send_text.all_modes = frozenset({'normal', 'application', 'kitty'})
+    text = ' '.join(parts[2:])
+    key_str = '{} send_text {} {}'.format(sc, mode, text)
+    return parse_key(key_str, keymap)
 
 
 def to_open_url_modifiers(val):
@@ -233,6 +220,13 @@ def adjust_line_height(x):
     return int(x)
 
 
+def box_drawing_scale(x):
+    ans = tuple(float(x.strip()) for x in x.split(','))
+    if len(ans) != 4:
+        raise ValueError('Invalid box_drawing scale, must have four entries')
+    return ans
+
+
 type_map = {
     'adjust_line_height': adjust_line_height,
     'scrollback_lines': positive_int,
@@ -240,6 +234,7 @@ type_map = {
     'scrollback_in_new_tab': to_bool,
     'font_size': to_font_size,
     'font_size_delta': positive_float,
+    'focus_follows_mouse': to_bool,
     'cursor_shape': to_cursor_shape,
     'open_url_modifiers': to_open_url_modifiers,
     'repaint_delay': positive_int,
@@ -260,6 +255,8 @@ type_map = {
     'initial_window_height': positive_int,
     'use_system_wcwidth': to_bool,
     'macos_hide_titlebar': to_bool,
+    'macos_option_as_alt': to_bool,
+    'box_drawing_scale': box_drawing_scale,
 }
 
 for name in (
@@ -278,11 +275,6 @@ def parse_config(lines, check_keys=True):
     ans = {
         'keymap': {},
         'symbol_map': {},
-        'send_text_map': {
-            'kitty': {},
-            'normal': {},
-            'application': {}
-        }
     }
     if check_keys:
         all_keys = defaults._asdict()
@@ -300,9 +292,8 @@ def parse_config(lines, check_keys=True):
                 ans['symbol_map'].update(parse_symbol_map(val))
                 continue
             if key == 'send_text':
-                stvals = parse_send_text(val)
-                for k, v in ans['send_text_map'].items():
-                    v.update(stvals.get(k, {}))
+                # For legacy compatibility
+                parse_send_text(val, ans['keymap'])
                 continue
             if check_keys:
                 if key not in all_keys:
@@ -324,7 +315,7 @@ with open(
     defaults = parse_config(f.readlines(), check_keys=False)
 Options = namedtuple('Defaults', ','.join(defaults.keys()))
 defaults = Options(**defaults)
-actions = frozenset(a.func for a in defaults.keymap.values()) | frozenset({'combine'})
+actions = frozenset(a.func for a in defaults.keymap.values()) | frozenset({'combine', 'send_text'})
 no_op_actions = frozenset({'noop', 'no-op', 'no_op'})
 
 
@@ -353,11 +344,6 @@ def merge_configs(defaults, vals):
             newvals = vals.get(k, {})
             if k == 'keymap':
                 ans['keymap'] = merge_keymaps(v, newvals)
-            elif k == 'send_text_map':
-                ans[k] = {
-                    m: merge_dicts(mm, newvals.get(m, {}))
-                    for m, mm in v.items()
-                }
             else:
                 ans[k] = merge_dicts(v, newvals)
         else:

kitty/constants.py

@@ -11,7 +11,7 @@ from collections import namedtuple
 from .fast_data_types import set_boss as set_c_boss
 
 appname = 'kitty'
-version = (0, 4, 1)
+version = (0, 5, 1)
 str_version = '.'.join(map(str, version))
 _plat = sys.platform.lower()
 isosx = 'darwin' in _plat
@@ -67,7 +67,11 @@ cell_size = ViewportSize()
 base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 terminfo_dir = os.path.join(base_dir, 'terminfo')
 logo_data_file = os.path.join(base_dir, 'logo', 'kitty.rgba')
-shell_path = pwd.getpwuid(os.geteuid()).pw_shell or '/bin/sh'
+try:
+    shell_path = pwd.getpwuid(os.geteuid()).pw_shell or '/bin/sh'
+except KeyError:
+    print('Failed to read login shell from /etc/passwd for current user, falling back to /bin/sh', file=sys.stderr)
+    shell_path = '/bin/sh'
 
 GLint = ctypes.c_int if ctypes.sizeof(ctypes.c_int) == 4 else ctypes.c_long
 GLuint = ctypes.c_uint if ctypes.sizeof(ctypes.c_uint) == 4 else ctypes.c_ulong

kitty/core_text.m

@@ -5,7 +5,8 @@
  * Distributed under terms of the GPL3 license.
  */
 
-#include "data-types.h"
+#include "state.h"
+#include "fonts.h"
 #include <structmember.h>
 #include <stdint.h>
 #include <math.h>
@@ -16,25 +17,16 @@
 #import <Foundation/NSString.h>
 #import <Foundation/NSDictionary.h>
 
-typedef struct {
-    PyObject_HEAD
 
-    unsigned int units_per_em;
-    float ascent, descent, leading, underline_position, underline_thickness, point_sz, scaled_point_sz;
-    CTFontRef font;
-    PyObject *family_name, *full_name, *postscript_name, *path;
-} Face;
-
-
-static inline PyObject*
+static inline char*
 convert_cfstring(CFStringRef src, int free_src) {
 #define SZ 2048
     static char buf[SZ+2] = {0};
-    PyObject *ans = NULL;
+    bool ok = false;
     if(!CFStringGetCString(src, buf, SZ, kCFStringEncodingUTF8)) PyErr_SetString(PyExc_ValueError, "Failed to convert CFString");
-    else ans = PyUnicode_FromString(buf);
+    else ok = true;
     if (free_src) CFRelease(src);
-    return ans;
+    return ok ? buf : NULL;
 #undef SZ
 }
 
@@ -78,9 +70,9 @@ font_descriptor_from_python(PyObject *src) {
     PyObject *t = PyDict_GetItemString(src, "traits");
     if (t == NULL) {
         symbolic_traits = (
-            (PyDict_GetItemString(src, "bold") == Py_True) ? kCTFontBoldTrait : 0 |
-            (PyDict_GetItemString(src, "italic") == Py_True) ? kCTFontItalicTrait : 0 |
-            (PyDict_GetItemString(src, "monospace") == Py_True) ? kCTFontMonoSpaceTrait : 0);
+            (PyDict_GetItemString(src, "bold") == Py_True ? kCTFontBoldTrait : 0) |
+            (PyDict_GetItemString(src, "italic") == Py_True ? kCTFontItalicTrait : 0) |
+            (PyDict_GetItemString(src, "monospace") == Py_True ? kCTFontMonoSpaceTrait : 0));
     } else {
         symbolic_traits = PyLong_AsUnsignedLong(t);
     }
@@ -114,221 +106,84 @@ coretext_all_fonts(PyObject UNUSED *_self) {
     return ans;
 }
 
-
-static PyObject*
-new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
-    Face *self;
-    PyObject *descriptor;
-    float point_sz, dpi;
-    if(!PyArg_ParseTuple(args, "Off", &descriptor, &point_sz, &dpi)) return NULL;
-    self = (Face *)type->tp_alloc(type, 0);
-    if (self) {
-        CTFontDescriptorRef desc = font_descriptor_from_python(descriptor);
-        if (desc) {
-            self->point_sz = point_sz;
-            self->scaled_point_sz = (dpi / 72.0) * point_sz;
-            self->font = CTFontCreateWithFontDescriptor(desc, self->scaled_point_sz, NULL);
-            CFRelease(desc);
-            if (!self->font) { Py_CLEAR(self); PyErr_SetString(PyExc_ValueError, "Failed to create CTFont object"); }
-            else {
-                self->units_per_em = CTFontGetUnitsPerEm(self->font);
-                self->ascent = CTFontGetAscent(self->font);
-                self->descent = CTFontGetDescent(self->font);
-                self->leading = CTFontGetLeading(self->font);
-                self->underline_position = CTFontGetUnderlinePosition(self->font);
-                self->underline_thickness = CTFontGetUnderlineThickness(self->font);
-                self->scaled_point_sz = CTFontGetSize(self->font);
-                self->family_name = convert_cfstring(CTFontCopyFamilyName(self->font), 1);
-                self->full_name = convert_cfstring(CTFontCopyFullName(self->font), 1);
-                self->postscript_name = convert_cfstring(CTFontCopyPostScriptName(self->font), 1);
-                NSURL *url = (NSURL*)CTFontCopyAttribute(self->font, kCTFontURLAttribute);
-                self->path = PyUnicode_FromString([[url path] UTF8String]);
-                [url release];
-                if (self->family_name == NULL || self->full_name == NULL || self->postscript_name == NULL || self->path == NULL) { Py_CLEAR(self); }
-            }
-        } else {
-            Py_CLEAR(self);
-            PyErr_NoMemory();
-        }
-    }
-    return (PyObject*)self;
-}
-
-
 static void
-dealloc(Face* self) {
-    if (self->font) CFRelease(self->font);
-    Py_CLEAR(self->family_name); Py_CLEAR(self->full_name); Py_CLEAR(self->postscript_name); Py_CLEAR(self->path);
-    Py_TYPE(self)->tp_free((PyObject*)self);
+free_font(void *f) {
+    CFRelease((CTFontRef)f);
 }
 
-static void
-encode_utf16_pair(uint32_t character, unichar *units) {
-    unsigned int code;
-    assert(0x10000 <= character && character <= 0x10FFFF);
-    code = (character - 0x10000);
-    units[0] = 0xD800 | (code >> 10);
-    units[1] = 0xDC00 | (code & 0x3FF);
+static inline PyObject*
+ft_face(CTFontRef font) {
+    const char *psname = convert_cfstring(CTFontCopyPostScriptName(font), 1);
+    NSURL *url = (NSURL*)CTFontCopyAttribute(font, kCTFontURLAttribute);
+    PyObject *path = PyUnicode_FromString([[url path] UTF8String]);
+    [url release];
+    if (path == NULL) { CFRelease(font); return NULL; }
+    PyObject *ans =  ft_face_from_path_and_psname(path, psname, (void*)font, free_font, true, 3, CTFontGetLeading(font));
+    Py_DECREF(path);
+    if (ans == NULL) { CFRelease(font); }
+    return ans;
 }
 
-static PyObject*
-has_char(Face *self, PyObject *args) {
-#define has_char_doc "True iff this font has glyphs for the specified character"
-    int ch, count = 1;
-    unichar chars[2] = {0};
-    CGGlyph glyphs[2] = {0};
-    if (!PyArg_ParseTuple(args, "C", &ch)) return NULL;
-    if (ch <= 0xffff) chars[0] = (unichar)ch;
-    else { encode_utf16_pair(ch, chars); count = 2; }
-    PyObject *ret = (CTFontGetGlyphsForCharacters(self->font, chars, glyphs, count)) ? Py_True : Py_False;
-    Py_INCREF(ret);
-    return ret;
-}
-
-static PyObject*
-font_units_to_pixels(Face *self, PyObject *args) {
-#define font_units_to_pixels_doc "Convert the specified value from font units to pixels at the current font size"
-    double x;
-    if (!PyArg_ParseTuple(args, "d", &x)) return NULL;
-    x *= self->scaled_point_sz / self->units_per_em;
-    return Py_BuildValue("i", (int)ceil(x));
-}
-
-static PyObject*
-cell_size(Face *self) {
-#define cell_size_doc "Return the best cell size for this font based on the advances for the ASCII chars from 32 to 127"
-#define count (128 - 32)
-    unichar chars[count+1] = {0};
-    CGGlyph glyphs[count+1] = {0};
-    unsigned int width = 0, w, i;
-    for (i = 0; i < count; i++) chars[i] = 32 + i;
-    CTFontGetGlyphsForCharacters(self->font, chars, glyphs, count);
-    for (i = 0; i < count; i++) {
-        if (glyphs[i]) {
-            w = (unsigned int)(ceilf(
-                        CTFontGetAdvancesForGlyphs(self->font, kCTFontOrientationHorizontal, glyphs+i, NULL, 1)));
-            if (w > width) width = w; 
-        }
+static inline CTFontRef
+find_substitute_face(CFStringRef str, CTFontRef old_font) {
+    // CTFontCreateForString returns the original font when there are combining
+    // diacritics in the font and the base character is in the original font,
+    // so we have to check each character individually
+    CFIndex len = CFStringGetLength(str), start = 0, amt = len;
+    while (start < len) {
+        CTFontRef new_font = CTFontCreateForString(old_font, str, CFRangeMake(start, amt));
+        if (amt == len && len != 1) amt = 1;
+        else start++;
+        if (new_font == NULL) { PyErr_SetString(PyExc_ValueError, "Failed to find fallback CTFont"); return NULL; }
+        if (new_font == old_font) { CFRelease(new_font); continue; }
+        return new_font;
     }
-    // See https://stackoverflow.com/questions/5511830/how-does-line-spacing-work-in-core-text-and-why-is-it-different-from-nslayoutm
-    CGFloat leading = MAX(0, self->leading);
-    leading = floor(leading + 0.5);
-    CGFloat line_height = floor(self->ascent + 0.5) + floor(self->descent + 0.5) + leading;
-    CGFloat ascender_delta = (leading > 0) ? 0 : floor(0.2 * line_height + 0.5);
-    return Py_BuildValue("II", width, (unsigned int)(line_height + ascender_delta));  
-#undef count
+    PyErr_SetString(PyExc_ValueError, "CoreText returned the same font as a fallback font"); 
+    return NULL;
 }
 
-static PyObject*
-render_char(Face *self, PyObject *args) {
-#define render_char_doc "Render the specified character into the specified buffer. Combining unicode chars should be handled automatically (I hope)"
-    char *s;
-    unsigned int width, height;
-    PyObject *pbuf;
-    CGColorSpaceRef color_space = NULL;
-    CGContextRef ctx = NULL;
-    CTFontRef font = NULL;
-    if (!PyArg_ParseTuple(args, "esIIO!", "UTF-8", &s, &width, &height, &PyLong_Type, &pbuf)) return NULL;
-    uint8_t *buf = (uint8_t*)PyLong_AsVoidPtr(pbuf);
-    CFStringRef str = CFStringCreateWithCString(NULL, s, kCFStringEncodingUTF8);
-    if (!str) return PyErr_NoMemory();
-    CGGlyph glyphs[10] = {0};
-    unichar chars[10] = {0};
-    CFRange range = CFRangeMake(0, CFStringGetLength(str));
-    CFStringGetCharacters(str, range, chars);
-    font = CTFontCreateForString(self->font, str, range);
-    if (font == NULL) { PyErr_SetString(PyExc_ValueError, "Failed to find fallback font"); goto end; }
-    CTFontGetGlyphsForCharacters(font, chars, glyphs, range.length);
-    color_space = CGColorSpaceCreateDeviceGray();
-    if (color_space == NULL) { PyErr_NoMemory(); goto end; }
-    ctx = CGBitmapContextCreate(buf, width, height, 8, width, color_space, (kCGBitmapAlphaInfoMask & kCGImageAlphaNone));
-    if (ctx == NULL) { PyErr_SetString(PyExc_ValueError, "Failed to create bitmap context"); goto end; }
-    CGContextSetShouldAntialias(ctx, true);
-    CGContextSetShouldSmoothFonts(ctx, true);  // sub-pixel antialias
-    CGContextSetRGBFillColor(ctx, 1, 1, 1, 1); // white glyphs
-    CGAffineTransform transform = CGAffineTransformIdentity;
-    CGContextSetTextDrawingMode(ctx, kCGTextFill);
-    CGGlyph glyph = glyphs[0];
-    if (glyph) {
-        // TODO: Scale the glyph if its bbox is larger than the image by using a non-identity transform
-        /* CGRect rect = CTFontGetBoundingRectsForGlyphs(font, kCTFontOrientationHorizontal, glyphs, 0, 1); */
-        CGContextSetTextMatrix(ctx, transform);
-        CGFloat pos_y = height - self->ascent; 
-        CGContextSetTextPosition(ctx, 0, pos_y); 
-        CTFontDrawGlyphs(font, &glyph, &CGPointZero, 1, ctx);
-    }
-
-end:
+PyObject*
+create_fallback_face(PyObject *base_face, Cell* cell, bool UNUSED bold, bool UNUSED italic) {
+    PyObject *lp = PyObject_CallMethod(base_face, "extra_data", NULL);
+    if (lp == NULL) return NULL;
+    CTFontRef font = PyLong_AsVoidPtr(lp);
+    Py_CLEAR(lp);
+    static char text[128];
+    cell_as_utf8(cell, true, text, ' ');
+    CFStringRef str = CFStringCreateWithCString(NULL, text, kCFStringEncodingUTF8);
+    if (str == NULL) return PyErr_NoMemory();
+    CTFontRef new_font = find_substitute_face(str, font);
     CFRelease(str);
-    if (ctx) CGContextRelease(ctx);
-    if (color_space) CGColorSpaceRelease(color_space);
-    if (font && font != self->font) CFRelease(font);
-    if (PyErr_Occurred()) return NULL;
-    Py_RETURN_NONE;
+    if (new_font == NULL) return NULL;
+    return ft_face(new_font);
 }
 
-static PyObject *
-repr(Face *self) {
-    char buf[400] = {0};
-    snprintf(buf, sizeof(buf)/sizeof(buf[0]), "ascent=%.1f, descent=%.1f, leading=%.1f, point_sz=%.1f, scaled_point_sz=%.1f, underline_position=%.1f underline_thickness=%.1f", 
-        (self->ascent), (self->descent), (self->leading), (self->point_sz), (self->scaled_point_sz), (self->underline_position), (self->underline_thickness));
-    return PyUnicode_FromFormat(
-        "Face(family=%U, full_name=%U, postscript_name=%U, path=%U, units_per_em=%u, %s)",
-        self->family_name, self->full_name, self->postscript_name, self->path, self->units_per_em, buf
-    );
+PyObject*
+face_from_descriptor(PyObject *descriptor) {
+    CTFontDescriptorRef desc = font_descriptor_from_python(descriptor);
+    if (!desc) return NULL;
+    float scaled_point_sz = ((global_state.logical_dpi_x + global_state.logical_dpi_y) / 144.0) * global_state.font_sz_in_pts;
+    CTFontRef font = CTFontCreateWithFontDescriptor(desc, scaled_point_sz, NULL);
+    CFRelease(desc); desc = NULL;
+    if (!font) { PyErr_SetString(PyExc_ValueError, "Failed to create CTFont object"); return NULL; }
+    return ft_face(font);
 }
 
+PyObject*
+specialize_font_descriptor(PyObject *base_descriptor) {
+    Py_INCREF(base_descriptor);
+    return base_descriptor;
+}
 
 // Boilerplate {{{
 
-static PyMemberDef members[] = {
-#define MEM(name, type) {#name, type, offsetof(Face, name), READONLY, #name}
-    MEM(units_per_em, T_UINT),
-    MEM(point_sz, T_FLOAT),
-    MEM(scaled_point_sz, T_FLOAT),
-    MEM(ascent, T_FLOAT),
-    MEM(descent, T_FLOAT),
-    MEM(leading, T_FLOAT),
-    MEM(underline_position, T_FLOAT),
-    MEM(underline_thickness, T_FLOAT),
-    MEM(family_name, T_OBJECT),
-    MEM(path, T_OBJECT),
-    MEM(full_name, T_OBJECT),
-    MEM(postscript_name, T_OBJECT),
-    {NULL}  /* Sentinel */
-};
-
-static PyMethodDef methods[] = {
-    METHOD(has_char, METH_VARARGS)
-    METHOD(cell_size, METH_NOARGS)
-    METHOD(font_units_to_pixels, METH_VARARGS)
-    METHOD(render_char, METH_VARARGS)
-    {NULL}  /* Sentinel */
-};
-
-
-PyTypeObject Face_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0)
-    .tp_name = "fast_data_types.CTFace",
-    .tp_basicsize = sizeof(Face),
-    .tp_dealloc = (destructor)dealloc, 
-    .tp_flags = Py_TPFLAGS_DEFAULT,        
-    .tp_doc = "CoreText Font face",
-    .tp_methods = methods,
-    .tp_members = members,
-    .tp_new = new,                
-    .tp_repr = (reprfunc)repr,
-};
-
 static PyMethodDef module_methods[] = {
-    {"coretext_all_fonts", (PyCFunction)coretext_all_fonts, METH_NOARGS, ""},
+    METHODB(coretext_all_fonts, METH_NOARGS),
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
 int 
 init_CoreText(PyObject *module) {
-    if (PyType_Ready(&Face_Type) < 0) return 0;
-    if (PyModule_AddObject(module, "CTFace", (PyObject *)&Face_Type) != 0) return 0;
     if (PyModule_AddFunctions(module, module_methods) != 0) return 0;
     return 1;
 }

kitty/data-types.c

@@ -9,6 +9,8 @@
 #include "modes.h"
 #include <stddef.h>
 #include <termios.h>
+#include <signal.h>
+#include <sys/wait.h>
 #ifdef WITH_PROFILER
 #include <gperftools/profiler.h>
 #endif
@@ -81,6 +83,28 @@ pyset_iutf8(PyObject UNUSED *self, PyObject *args) {
     Py_RETURN_NONE;
 }
 
+static void
+handle_sigchld(int UNUSED signum, siginfo_t *sinfo, void UNUSED *unused) {
+    if (sinfo->si_code != CLD_EXITED) return;
+    int sav_errno = errno, status;
+    while(true) {
+        if (waitpid(sinfo->si_pid, &status, WNOHANG) == -1) {
+            if (errno != EINTR) break;
+        } else break;
+    }
+    errno = sav_errno;
+}
+
+static PyObject*
+install_sigchld_handler(PyObject UNUSED *self) {
+    struct sigaction sa;
+    sa.sa_flags = SA_SIGINFO;
+    sa.sa_sigaction = handle_sigchld;
+    sigemptyset(&sa.sa_mask);
+    if (sigaction(SIGCHLD, &sa, NULL) == -1) return PyErr_SetFromErrno(PyExc_OSError);
+    Py_RETURN_NONE;
+}
+
 #ifdef WITH_PROFILER
 static PyObject*
 start_profiler(PyObject UNUSED *self, PyObject *args) {
@@ -105,6 +129,7 @@ static PyMethodDef module_methods[] = {
     {"redirect_std_streams", (PyCFunction)redirect_std_streams, METH_VARARGS, ""},
     {"wcwidth", (PyCFunction)wcwidth_wrap, METH_O, ""},
     {"change_wcwidth", (PyCFunction)change_wcwidth_wrap, METH_O, ""},
+    {"install_sigchld_handler", (PyCFunction)install_sigchld_handler, METH_NOARGS, ""},
 #ifdef WITH_PROFILER
     {"start_profiler", (PyCFunction)start_profiler, METH_VARARGS, ""},
     {"stop_profiler", (PyCFunction)stop_profiler, METH_NOARGS, ""},
@@ -129,16 +154,14 @@ extern bool init_child_monitor(PyObject *);
 extern int init_Line(PyObject *);
 extern int init_ColorProfile(PyObject *);
 extern int init_Screen(PyObject *);
-extern int init_Face(PyObject *);
 extern bool init_freetype_library(PyObject*);
 extern bool init_fontconfig_library(PyObject*);
+extern bool init_fonts(PyObject*);
 extern bool init_glfw(PyObject *m);
-extern bool init_sprites(PyObject *module);
 extern bool init_state(PyObject *module);
 extern bool init_keys(PyObject *module);
 extern bool init_graphics(PyObject *module);
 extern bool init_shaders(PyObject *module);
-extern bool init_shaders_debug(PyObject *module);
 #ifdef __APPLE__
 extern int init_CoreText(PyObject *);
 extern bool init_cocoa(PyObject *module);
@@ -164,23 +187,18 @@ PyInit_fast_data_types(void) {
         if (!init_ColorProfile(m)) return NULL;
         if (!init_Screen(m)) return NULL;
         if (!init_glfw(m)) return NULL;
-        if (!init_sprites(m)) return NULL;
         if (!init_state(m)) return NULL;
         if (!init_keys(m)) return NULL;
         if (!init_graphics(m)) return NULL;
-        if (PySys_GetObject("debug_gl") == Py_True) {
-            if (!init_shaders_debug(m)) return NULL;
-        } else { 
-            if (!init_shaders(m)) return NULL;
-        }
+        if (!init_shaders(m)) return NULL;
+        if (!init_freetype_library(m)) return NULL;
 #ifdef __APPLE__
         if (!init_CoreText(m)) return NULL;
         if (!init_cocoa(m)) return NULL;
 #else
-        if (!init_Face(m)) return NULL;
-        if (!init_freetype_library(m)) return NULL;
         if (!init_fontconfig_library(m)) return NULL;
 #endif
+        if (!init_fonts(m)) return NULL;
 
 #define OOF(n) #n, offsetof(Cell, n)
         if (PyModule_AddObject(m, "CELL", Py_BuildValue("{sI sI sI sI sI sI sI sI sI}",

kitty/data-types.h

@@ -30,6 +30,7 @@ typedef uint32_t combining_type;
 typedef unsigned int index_type;
 typedef uint16_t sprite_index;
 typedef uint16_t attrs_type;
+typedef uint8_t line_attrs_type;
 typedef enum CursorShapes { NO_CURSOR_SHAPE, CURSOR_BLOCK, CURSOR_BEAM, CURSOR_UNDERLINE, NUM_OF_CURSOR_SHAPES } CursorShape;
 
 #define ERROR_PREFIX "[PARSE ERROR]"
@@ -45,6 +46,7 @@ typedef enum MouseShapes { BEAM, HAND, ARROW } MouseShape;
 #define DECORATION_MASK 3
 #define BOLD_SHIFT 4
 #define ITALIC_SHIFT 5
+#define BI_VAL(attrs) ((attrs >> 4) & 3)
 #define REVERSE_SHIFT 6
 #define STRIKE_SHIFT 7
 #define COL_MASK 0xFFFFFFFF
@@ -55,6 +57,8 @@ typedef enum MouseShapes { BEAM, HAND, ARROW } MouseShape;
 #define UNDERCURL_CODE 6
 #define DECORATION_FG_CODE 58
 #define CHAR_IS_BLANK(ch) ((ch) == 32 || (ch) == 0)
+#define CONTINUED_MASK 1
+#define TEXT_DIRTY_MASK 2
 
 #define FG 1
 #define BG 2
@@ -73,6 +77,7 @@ typedef enum MouseShapes { BEAM, HAND, ARROW } MouseShape;
 #define COPY_SELF_CELL(s, d) COPY_CELL(self, s, self, d)
 
 #define METHOD(name, arg_type) {#name, (PyCFunction)name, arg_type, name##_doc},
+#define METHODB(name, arg_type) {#name, (PyCFunction)name, arg_type, ""}
 
 #define BOOL_GETSET(type, x) \
     static PyObject* x##_get(type *self, void UNUSED *closure) { PyObject *ans = self->x ? Py_True : Py_False; Py_INCREF(ans); return ans; } \
@@ -132,10 +137,8 @@ typedef struct {
 
     Cell *cells;
     index_type xnum, ynum;
-    bool continued;
-    bool needs_free;
+    bool continued, needs_free, has_dirty_text;
 } Line;
-PyTypeObject Line_Type;
 
 
 typedef struct {
@@ -143,10 +146,9 @@ typedef struct {
 
     Cell *buf;
     index_type xnum, ynum, *line_map, *scratch;
-    bool *continued_map;
+    line_attrs_type *line_attrs;
     Line *line;
 } LineBuf;
-PyTypeObject LineBuf_Type;
 
 
 typedef struct {
@@ -156,9 +158,8 @@ typedef struct {
     index_type xnum, ynum;
     Line *line;
     index_type start_of_data, count;
-    bool *continued_map;
+    line_attrs_type *line_attrs;
 } HistoryBuf;
-PyTypeObject HistoryBuf_Type;
 
 typedef struct {
     PyObject_HEAD
@@ -170,10 +171,6 @@ typedef struct {
     unsigned long fg, bg, decoration_fg;
 
 } Cursor;
-PyTypeObject Cursor_Type;
-
-PyTypeObject Face_Type;
-PyTypeObject Window_Type;
 
 typedef struct {
     color_type default_fg, default_bg, cursor_color, highlight_fg, highlight_bg;
@@ -187,7 +184,6 @@ typedef struct {
     uint32_t orig_color_table[256];
     DynamicColor configured, overridden;
 } ColorProfile;
-PyTypeObject ColorProfile_Type;
 
 #define SAVEPOINTS_SZ 256
 
@@ -223,7 +219,6 @@ typedef struct {
     bool shutting_down;
     pthread_t io_thread;
 } ChildMonitor;
-PyTypeObject ChildMonitor_Type;
 
 #define clear_sprite_position(cell) (cell).sprite_x = 0; (cell).sprite_y = 0; (cell).sprite_z = 0; 
 
@@ -237,6 +232,16 @@ PyTypeObject ChildMonitor_Type;
         clear_sprite_position((line)->cells[(at)]); \
     }
 
+#define ensure_space_for(base, array, type, num, capacity, initial_cap, zero_mem) \
+    if ((base)->capacity < num) { \
+        size_t _newcap = MAX(initial_cap, MAX(2 * (base)->capacity, num)); \
+        (base)->array = realloc((base)->array, sizeof(type) * _newcap); \
+        if ((base)->array == NULL) fatal("Out of memory while ensuring space in array"); \
+        if (zero_mem) memset((base)->array + (base)->capacity, 0, sizeof(type) * (_newcap - (base)->capacity)); \
+        (base)->capacity = _newcap; \
+    }
+
+
 // Global functions 
 const char* base64_decode(const uint32_t *src, size_t src_sz, uint8_t *dest, size_t dest_capacity, size_t *dest_sz);
 Line* alloc_line();
@@ -253,7 +258,6 @@ void cursor_reset(Cursor*);
 Cursor* cursor_copy(Cursor*);
 void cursor_copy_to(Cursor *src, Cursor *dest);
 void cursor_reset_display_attrs(Cursor*);
-void set_sprite_position(Cell *cell, Cell *previous_cell);
 
 double monotonic();
 PyObject* cm_thread_write(PyObject *self, PyObject *args);

kitty/fontconfig.c

@@ -5,28 +5,153 @@
  * Distributed under terms of the GPL3 license.
  */
 
-#include "data-types.h"
+#include "state.h"
+#include "lineops.h"
+#include "fonts.h"
 #include <fontconfig/fontconfig.h>
+#ifndef FC_COLOR
+#define FC_COLOR "color"
+#endif
+
+static inline PyObject*
+pybool(FcBool x) { PyObject *ans = x ? Py_True: Py_False; Py_INCREF(ans); return ans; }
+
+static inline PyObject*
+pyspacing(int val) {
+#define S(x) case FC_##x: return PyUnicode_FromString(#x)
+    switch(val) { S(PROPORTIONAL); S(DUAL); S(MONO); S(CHARCELL); default: return PyUnicode_FromString("UNKNOWN"); }
+#undef S
+}
+
+static inline PyObject*
+pattern_as_dict(FcPattern *pat) {
+    PyObject *ans = PyDict_New();
+    if (ans == NULL) return NULL;
+#define PS(x) PyUnicode_FromString((char*)x)
+#define G(type, get, which, conv, name) { \
+    type out; PyObject *p; \
+    if (get(pat, which, 0, &out) == FcResultMatch) { \
+        p = conv(out); if (p == NULL) { Py_CLEAR(ans); return NULL; } \
+        if (PyDict_SetItemString(ans, #name, p) != 0) { Py_CLEAR(p); Py_CLEAR(ans); return NULL; } \
+        Py_CLEAR(p); \
+    }}
+#define S(which, key) G(FcChar8*, FcPatternGetString, which, PS, key)
+#define I(which, key) G(int, FcPatternGetInteger, which, PyLong_FromLong, key)
+#define B(which, key) G(int, FcPatternGetBool, which, pybool, key)
+#define E(which, key, conv) G(int, FcPatternGetInteger, which, conv, key)
+    S(FC_FILE, path);
+    S(FC_FAMILY, family);
+    S(FC_STYLE, style);
+    S(FC_FULLNAME, full_name);
+    S(FC_POSTSCRIPT_NAME, postscript_name);
+    I(FC_WEIGHT, weight);
+    I(FC_SLANT, slant);
+    I(FC_HINT_STYLE, hint_style);
+    I(FC_INDEX, index);
+    B(FC_HINTING, hinting);
+    B(FC_SCALABLE, scalable);
+    B(FC_OUTLINE, outline);
+    B(FC_COLOR, color);
+    E(FC_SPACING, spacing, pyspacing);
+
+    return ans;
+#undef PS
+#undef S
+#undef I
+#undef B
+#undef E
+#undef G
+}
+
+static inline PyObject*
+font_set(FcFontSet *fs) {
+    PyObject *ans = PyTuple_New(fs->nfont);
+    if (ans == NULL) return NULL;
+    for (int i = 0; i < fs->nfont; i++) {
+        PyObject *d = pattern_as_dict(fs->fonts[i]);
+        if (d == NULL) { Py_CLEAR(ans); break; }
+        PyTuple_SET_ITEM(ans, i, d);
+    }
+    return ans;
+}
+
+#define AP(func, which, in, desc) if (!func(pat, which, in)) { PyErr_Format(PyExc_ValueError, "Failed to add %s to fontconfig pattern", desc, NULL); goto end; }
 
 static PyObject*
-get_fontconfig_font(PyObject UNUSED *self, PyObject *args) {
-    char *family;
-    int bold, italic, allow_bitmapped_fonts, index = 0, hint_style=0, weight=0, slant=0;
-    double size_in_pts, dpi;
-    unsigned int character;
-    FcBool hinting, scalable, outline;
-    FcChar8 *path = NULL;
-    FcPattern *pat = NULL, *match = NULL;
+fc_list(PyObject UNUSED *self, PyObject *args) {
+    int allow_bitmapped_fonts = 0, only_monospaced_fonts = 1;
+    PyObject *ans = NULL;
+    FcObjectSet *os = NULL;
+    FcPattern *pat = NULL;
+    FcFontSet *fs = NULL;
+    if (!PyArg_ParseTuple(args, "|pp", &only_monospaced_fonts, &allow_bitmapped_fonts)) return NULL;
+    pat = FcPatternCreate();
+    if (pat == NULL) return PyErr_NoMemory();
+    if (!allow_bitmapped_fonts) {
+        AP(FcPatternAddBool, FC_OUTLINE, true, "outline");
+        AP(FcPatternAddBool, FC_SCALABLE, true, "scalable");
+    }
+    if (only_monospaced_fonts) AP(FcPatternAddInteger, FC_SPACING, FC_MONO, "spacing");
+    os = FcObjectSetBuild(FC_FILE, FC_POSTSCRIPT_NAME, FC_FAMILY, FC_STYLE, FC_FULLNAME, FC_WEIGHT, FC_WIDTH, FC_SLANT, FC_HINT_STYLE, FC_INDEX, FC_HINTING, FC_SCALABLE, FC_OUTLINE, FC_COLOR, FC_SPACING, NULL);
+    if (!os) { PyErr_SetString(PyExc_ValueError, "Failed to create fontconfig object set"); goto end; }
+    fs = FcFontList(NULL, pat, os);
+    if (!fs) { PyErr_SetString(PyExc_ValueError, "Failed to create fontconfig font set"); goto end; }
+    ans = font_set(fs);
+end:
+    if (pat != NULL) FcPatternDestroy(pat);
+    if (os != NULL) FcObjectSetDestroy(os);
+    if (fs != NULL) FcFontSetDestroy(fs);
+    return ans;
+}
+
+static inline PyObject*
+_fc_match(FcPattern *pat) {
+    FcPattern *match = NULL;
+    PyObject *ans = NULL;
     FcResult result;
+    FcConfigSubstitute(NULL, pat, FcMatchPattern);
+    FcDefaultSubstitute(pat);
+    match = FcFontMatch(NULL, pat, &result);
+    if (match == NULL) { PyErr_SetString(PyExc_KeyError, "FcFontMatch() failed"); goto end; }
+    ans = pattern_as_dict(match);
+end:
+    if (match) FcPatternDestroy(match);
+    return ans;
+}
+
+static Py_UCS4 char_buf[1024];
+
+static inline void
+add_charset(FcPattern *pat, size_t num) {
     FcCharSet *charset = NULL;
+    if (num) {
+        charset = FcCharSetCreate();
+        if (charset == NULL) { PyErr_NoMemory(); goto end; }
+        for (size_t i = 0; i < num; i++) {
+            if (!FcCharSetAddChar(charset, char_buf[i])) { 
+                PyErr_SetString(PyExc_RuntimeError, "Failed to add character to fontconfig charset"); 
+                goto end; 
+            }
+        }
+        AP(FcPatternAddCharSet, FC_CHARSET, charset, "charset");
+    }
+end:
+    if (charset != NULL) FcCharSetDestroy(charset);
+}
+
+static PyObject*
+fc_match(PyObject UNUSED *self, PyObject *args) {
+    char *family = NULL;
+    int bold = 0, italic = 0, allow_bitmapped_fonts = 0;
+    double size_in_pts = 0, dpi = 0;
+    FcPattern *pat = NULL;
     PyObject *ans = NULL;
 
-    if (!PyArg_ParseTuple(args, "spppdId", &family, &bold, &italic, &allow_bitmapped_fonts, &size_in_pts, &character, &dpi)) return NULL;
+    if (!PyArg_ParseTuple(args, "|zpppdd", &family, &bold, &italic, &allow_bitmapped_fonts, &size_in_pts, &dpi)) return NULL;
     pat = FcPatternCreate();
     if (pat == NULL) return PyErr_NoMemory();
 
-#define AP(func, which, in, desc) if (!func(pat, which, in)) { PyErr_Format(PyExc_RuntimeError, "Failed to add %s to fontconfig patter", desc, NULL); goto end; }
-    AP(FcPatternAddString, FC_FAMILY, (const FcChar8*)family, "family");
+    if (family && strlen(family) > 0) AP(FcPatternAddString, FC_FAMILY, (const FcChar8*)family, "family");
     if (!allow_bitmapped_fonts) {
         AP(FcPatternAddBool, FC_OUTLINE, true, "outline");
         AP(FcPatternAddBool, FC_SCALABLE, true, "scalable");
@@ -35,47 +160,52 @@ get_fontconfig_font(PyObject UNUSED *self, PyObject *args) {
     if (dpi > 0) { AP(FcPatternAddDouble, FC_DPI, dpi, "dpi"); }
     if (bold) { AP(FcPatternAddInteger, FC_WEIGHT, FC_WEIGHT_BOLD, "weight"); }
     if (italic) { AP(FcPatternAddInteger, FC_SLANT, FC_SLANT_ITALIC, "slant"); }
-    if (character > 0) {
-        charset = FcCharSetCreate();
-        if (charset == NULL) { PyErr_NoMemory(); goto end; }
-        if (!FcCharSetAddChar(charset, character)) { PyErr_SetString(PyExc_RuntimeError, "Failed to add character to fontconfig charset"); goto end; }
-        AP(FcPatternAddCharSet, FC_CHARSET, charset, "charset");
-    }
-#undef AP
-    FcConfigSubstitute(NULL, pat, FcMatchPattern);
-    FcDefaultSubstitute(pat);
-    match = FcFontMatch(NULL, pat, &result);
-    if (match == NULL) { PyErr_SetString(PyExc_KeyError, "FcFontMatch() failed"); goto end; }
-
-#define GI(func, which, out, desc) \
-    if (func(match, which, 0, & out) != FcResultMatch) { \
-        PyErr_Format(PyExc_RuntimeError, "Failed to get %s from match object", desc, NULL); goto end; \
-    }
-
-    GI(FcPatternGetString, FC_FILE, path, "file path");
-    GI(FcPatternGetInteger, FC_INDEX, index, "face index");
-    GI(FcPatternGetInteger, FC_WEIGHT, weight, "weight");
-    GI(FcPatternGetInteger, FC_SLANT, slant, "slant");
-    GI(FcPatternGetInteger, FC_HINT_STYLE, hint_style, "hint style");
-    GI(FcPatternGetBool, FC_HINTING, hinting, "hinting");
-    GI(FcPatternGetBool, FC_SCALABLE, scalable, "scalable");
-    GI(FcPatternGetBool, FC_OUTLINE, outline, "outline");
-#undef GI
-
-#define BP(x) (x ? Py_True : Py_False)
-    ans = Py_BuildValue("siiOOOii", path, index, hint_style, BP(hinting), BP(scalable), BP(outline), weight, slant);
-#undef BP
+    ans = _fc_match(pat);
 
 end:
     if (pat != NULL) FcPatternDestroy(pat);
-    if (match != NULL) FcPatternDestroy(match);
-    if (charset != NULL) FcCharSetDestroy(charset);
-    if (PyErr_Occurred()) return NULL;
     return ans;
 }
 
+PyObject*
+specialize_font_descriptor(PyObject *base_descriptor) {
+    PyObject *p = PyDict_GetItemString(base_descriptor, "path"), *ans = NULL;
+    PyObject *idx = PyDict_GetItemString(base_descriptor, "index");
+    if (p == NULL) { PyErr_SetString(PyExc_ValueError, "Base descriptor has no path"); return NULL; }
+    if (idx == NULL) { PyErr_SetString(PyExc_ValueError, "Base descriptor has no index"); return NULL; }
+    FcPattern *pat = FcPatternCreate();
+    if (pat == NULL) return PyErr_NoMemory();
+    AP(FcPatternAddString, FC_FILE, (const FcChar8*)PyUnicode_AsUTF8(p), "path");
+    AP(FcPatternAddInteger, FC_INDEX, PyLong_AsLong(idx), "index");
+    AP(FcPatternAddDouble, FC_SIZE, global_state.font_sz_in_pts, "size");
+    AP(FcPatternAddDouble, FC_DPI, (global_state.logical_dpi_x + global_state.logical_dpi_y) / 2.0, "dpi");
+    ans = _fc_match(pat);
+end:
+    if (pat != NULL) FcPatternDestroy(pat);
+    return ans;
+}
+
+PyObject*
+create_fallback_face(PyObject UNUSED *base_face, Cell* cell, bool bold, bool italic) {
+    PyObject *ans = NULL;
+    FcPattern *pat = FcPatternCreate();
+    if (pat == NULL) return PyErr_NoMemory();
+    AP(FcPatternAddString, FC_FAMILY, (const FcChar8*)"monospace", "family");
+    if (bold) { AP(FcPatternAddInteger, FC_WEIGHT, FC_WEIGHT_BOLD, "weight"); }
+    if (italic) { AP(FcPatternAddInteger, FC_SLANT, FC_SLANT_ITALIC, "slant"); }
+    size_t num = cell_as_unicode(cell, true, char_buf, ' ');
+    add_charset(pat, num);
+    PyObject *d = _fc_match(pat);
+    if (d) { ans = face_from_descriptor(d); Py_CLEAR(d); }
+end:
+    if (pat != NULL) FcPatternDestroy(pat);
+    return ans;
+}
+
+#undef AP
 static PyMethodDef module_methods[] = {
-    {"get_fontconfig_font", (PyCFunction)get_fontconfig_font, METH_VARARGS, ""},
+    METHODB(fc_list, METH_VARARGS),
+    METHODB(fc_match, METH_VARARGS),
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
@@ -90,5 +220,11 @@ init_fontconfig_library(PyObject *module) {
         return false;
     }
     if (PyModule_AddFunctions(module, module_methods) != 0) return false;
+    PyModule_AddIntMacro(module, FC_WEIGHT_REGULAR);
+    PyModule_AddIntMacro(module, FC_WEIGHT_MEDIUM);
+    PyModule_AddIntMacro(module, FC_WEIGHT_SEMIBOLD);
+    PyModule_AddIntMacro(module, FC_WEIGHT_BOLD);
+    PyModule_AddIntMacro(module, FC_SLANT_ITALIC);
+    PyModule_AddIntMacro(module, FC_SLANT_ROMAN);
     return true;
 }

kitty/fonts.c

@@ -0,0 +1,932 @@
+/*
+ * fonts.c
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "fonts.h"
+#include "state.h"
+
+#define MISSING_GLYPH 4
+#define MAX_NUM_EXTRA_GLYPHS 8
+
+typedef uint16_t glyph_index;
+typedef void (*send_sprite_to_gpu_func)(unsigned int, unsigned int, unsigned int, uint8_t*);
+send_sprite_to_gpu_func current_send_sprite_to_gpu = NULL;
+static PyObject *python_send_to_gpu_impl = NULL;
+extern PyTypeObject Line_Type;
+
+typedef struct SpritePosition SpritePosition;
+typedef struct SpecialGlyphCache SpecialGlyphCache;
+enum {NO_FONT=-3, MISSING_FONT=-2, BLANK_FONT=-1, BOX_FONT=0};
+
+
+typedef struct {
+    glyph_index data[MAX_NUM_EXTRA_GLYPHS];
+} ExtraGlyphs;
+
+struct SpritePosition {
+    SpritePosition *next;
+    bool filled, rendered;
+    sprite_index x, y, z;
+    uint8_t ligature_index;
+    glyph_index glyph;
+    ExtraGlyphs extra_glyphs;
+};
+
+struct SpecialGlyphCache {
+    SpecialGlyphCache *next;
+    glyph_index glyph;
+    bool is_special, filled;
+};
+
+typedef struct {
+    size_t max_array_len, max_texture_size, max_y;
+    unsigned int x, y, z, xnum, ynum;
+} GPUSpriteTracker;
+
+
+static GPUSpriteTracker sprite_tracker = {0};
+static hb_buffer_t *harfbuzz_buffer = NULL;
+static char_type shape_buffer[2048] = {0};
+
+
+typedef struct {
+    PyObject *face;
+    hb_font_t *hb_font;
+    // Map glyphs to sprite map co-ords
+    SpritePosition sprite_map[1024]; 
+    SpecialGlyphCache special_glyph_cache[1024];
+    bool bold, italic;
+} Font;
+
+typedef struct {
+    char_type left, right;
+    size_t font_idx;
+} SymbolMap;
+
+typedef struct {
+    Font *fonts;
+    SymbolMap* symbol_maps;
+    size_t fonts_capacity, fonts_count, symbol_maps_capacity, symbol_maps_count, symbol_map_fonts_count, fallback_fonts_count;
+    ssize_t box_font_idx, medium_font_idx, bold_font_idx, italic_font_idx, bi_font_idx, first_symbol_font_idx, first_fallback_font_idx;
+} Fonts;
+
+static Fonts fonts = {0};
+
+
+// Sprites {{{
+
+static inline void 
+sprite_map_set_error(int error) {
+    switch(error) {
+        case 1:
+            PyErr_NoMemory(); break;
+        case 2:
+            PyErr_SetString(PyExc_RuntimeError, "Out of texture space for sprites"); break;
+        default:
+            PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while allocating sprites"); break;
+    }
+}
+
+void
+sprite_tracker_set_limits(size_t max_texture_size, size_t max_array_len) {
+    sprite_tracker.max_texture_size = max_texture_size;
+    sprite_tracker.max_array_len = max_array_len;
+}
+
+static inline void
+do_increment(int *error) {
+    sprite_tracker.x++;
+    if (sprite_tracker.x >= sprite_tracker.xnum) {
+        sprite_tracker.x = 0; sprite_tracker.y++;
+        sprite_tracker.ynum = MIN(MAX(sprite_tracker.ynum, sprite_tracker.y + 1), sprite_tracker.max_y);
+        if (sprite_tracker.y >= sprite_tracker.max_y) {
+            sprite_tracker.y = 0; sprite_tracker.z++;
+            if (sprite_tracker.z >= MIN(UINT16_MAX, sprite_tracker.max_array_len)) *error = 2;
+        }
+    }
+}
+
+
+static inline bool
+extra_glyphs_equal(ExtraGlyphs *a, ExtraGlyphs *b) {
+    for (size_t i = 0; i < MAX_NUM_EXTRA_GLYPHS; i++) {
+        if (a->data[i] != b->data[i]) return false;
+        if (a->data[i] == 0) return true;
+    }
+    return true;
+}
+
+
+static SpritePosition*
+sprite_position_for(Font *font, glyph_index glyph, ExtraGlyphs *extra_glyphs, uint8_t ligature_index, int *error) {
+    glyph_index idx = glyph & 0x3ff;
+    SpritePosition *s = font->sprite_map + idx;
+    // Optimize for the common case of glyph under 1024 already in the cache
+    if (LIKELY(s->glyph == glyph && s->filled && extra_glyphs_equal(&s->extra_glyphs, extra_glyphs) && s->ligature_index == ligature_index)) return s;  // Cache hit
+    while(true) {
+        if (s->filled) {
+            if (s->glyph == glyph && extra_glyphs_equal(&s->extra_glyphs, extra_glyphs) && s->ligature_index == ligature_index) return s;  // Cache hit
+        } else {
+            break;
+        }
+        if (!s->next) {
+            s->next = calloc(1, sizeof(SpritePosition));
+            if (s->next == NULL) { *error = 1; return NULL; }
+        }
+        s = s->next;
+    }
+    s->glyph = glyph;
+    memcpy(&s->extra_glyphs, extra_glyphs, sizeof(ExtraGlyphs));
+    s->ligature_index = ligature_index;
+    s->filled = true;
+    s->rendered = false;
+    s->x = sprite_tracker.x; s->y = sprite_tracker.y; s->z = sprite_tracker.z;
+    do_increment(error);
+    return s;
+}
+
+static SpecialGlyphCache*
+special_glyph_cache_for(Font *font, glyph_index glyph) {
+    SpecialGlyphCache *s = font->special_glyph_cache + (glyph & 0x3ff);
+    // Optimize for the common case of glyph under 1024 already in the cache
+    if (LIKELY(s->glyph == glyph && s->filled)) return s;  // Cache hit
+    while(true) {
+        if (s->filled) {
+            if (s->glyph == glyph) return s;  // Cache hit
+        } else {
+            break;
+        }
+        if (!s->next) {
+            s->next = calloc(1, sizeof(SpecialGlyphCache));
+            if (s->next == NULL) return NULL; 
+        }
+        s = s->next;
+    }
+    s->glyph = glyph;
+    return s;
+}
+
+void
+sprite_tracker_current_layout(unsigned int *x, unsigned int *y, unsigned int *z) {
+    *x = sprite_tracker.xnum; *y = sprite_tracker.ynum; *z = sprite_tracker.z;
+}
+
+void
+free_maps(Font *font) {
+#define free_a_map(type, attr) {\
+    type *s, *t; \
+    for (size_t i = 0; i < sizeof(font->attr)/sizeof(font->attr[0]); i++) { \
+        s = font->attr[i].next; \
+        while (s) { \
+            t = s; \
+            s = s->next; \
+            free(t); \
+        } \
+    }\
+    memset(font->attr, 0, sizeof(font->attr)); \
+}
+    free_a_map(SpritePosition, sprite_map);
+    free_a_map(SpecialGlyphCache, special_glyph_cache);
+#undef free_a_map
+}
+
+void
+clear_sprite_map(Font *font) {
+#define CLEAR(s) s->filled = false; s->rendered = false; s->glyph = 0; memset(&s->extra_glyphs, 0, sizeof(ExtraGlyphs)); s->x = 0; s->y = 0; s->z = 0; s->ligature_index = 0;
+    SpritePosition *s;
+    for (size_t i = 0; i < sizeof(font->sprite_map)/sizeof(font->sprite_map[0]); i++) {
+        s = font->sprite_map + i;
+        CLEAR(s);
+        while ((s = s->next)) {
+            CLEAR(s);
+        }
+    }
+#undef CLEAR
+}
+
+void
+clear_special_glyph_cache(Font *font) {
+#define CLEAR(s) s->filled = false; s->glyph = 0; 
+    SpecialGlyphCache *s;
+    for (size_t i = 0; i < sizeof(font->special_glyph_cache)/sizeof(font->special_glyph_cache[0]); i++) {
+        s = font->special_glyph_cache + i;
+        CLEAR(s);
+        while ((s = s->next)) {
+            CLEAR(s);
+        }
+    }
+#undef CLEAR
+}
+
+void
+sprite_tracker_set_layout(unsigned int cell_width, unsigned int cell_height) {
+    sprite_tracker.xnum = MIN(MAX(1, sprite_tracker.max_texture_size / cell_width), UINT16_MAX);
+    sprite_tracker.max_y = MIN(MAX(1, sprite_tracker.max_texture_size / cell_height), UINT16_MAX);
+    sprite_tracker.ynum = 1;
+    sprite_tracker.x = 0; sprite_tracker.y = 0; sprite_tracker.z = 0;
+}
+// }}}
+
+static inline PyObject*
+desc_to_face(PyObject *desc) {
+    PyObject *d = specialize_font_descriptor(desc);
+    if (d == NULL) return NULL;
+    PyObject *ans = face_from_descriptor(d);
+    Py_DECREF(d);
+    return ans;
+}
+
+
+static inline bool
+init_font(Font *f, PyObject *descriptor, bool bold, bool italic, bool is_face) {
+    PyObject *face;
+    if (is_face) { face = descriptor; Py_INCREF(face); }
+    else { face = desc_to_face(descriptor); if (face == NULL) return false; }
+    f->face = face; 
+    f->hb_font = harfbuzz_font_for_face(face);
+    if (f->hb_font == NULL) { PyErr_NoMemory(); return false; }
+    f->bold = bold; f->italic = italic;
+    return true;
+}
+
+static inline void
+del_font(Font *f) { 
+    f->hb_font = NULL;
+    Py_CLEAR(f->face); 
+    free_maps(f);
+    f->bold = false; f->italic = false;
+}
+
+static unsigned int cell_width = 0, cell_height = 0, baseline = 0, underline_position = 0, underline_thickness = 0;
+static uint8_t *canvas = NULL;
+#define CELLS_IN_CANVAS ((MAX_NUM_EXTRA_GLYPHS + 1) * 3)
+static inline void 
+clear_canvas(void) { memset(canvas, 0, CELLS_IN_CANVAS * cell_width * cell_height); }
+
+static void
+python_send_to_gpu(unsigned int x, unsigned int y, unsigned int z, uint8_t* buf) {
+    if (python_send_to_gpu_impl != NULL && python_send_to_gpu_impl != Py_None) {
+        PyObject *ret = PyObject_CallFunction(python_send_to_gpu_impl, "IIIN", x, y, z, PyBytes_FromStringAndSize((const char*)buf, cell_width * cell_height));
+        if (ret == NULL) PyErr_Print();
+        else Py_DECREF(ret);
+    }
+}
+
+
+static inline PyObject*
+update_cell_metrics() {
+#define CALL(idx, desired_height, force) { if (idx >= 0) { Font *f = fonts.fonts + idx; if ((f)->face) { if(!set_size_for_face((f)->face, desired_height, force)) return NULL; (f)->hb_font = harfbuzz_font_for_face((f)->face); } clear_sprite_map((f)); }}
+    CALL(BOX_FONT, 0, false); CALL(fonts.medium_font_idx, 0, false);
+    CALL(fonts.bold_font_idx, 0, false); CALL(fonts.italic_font_idx, 0, false); CALL(fonts.bi_font_idx, 0, false);
+    cell_metrics(fonts.fonts[fonts.medium_font_idx].face, &cell_width, &cell_height, &baseline, &underline_position, &underline_thickness);
+    if (!cell_width) { PyErr_SetString(PyExc_ValueError, "Failed to calculate cell width for the specified font."); return NULL; }
+    unsigned int before_cell_height = cell_height;
+    if (OPT(adjust_line_height_px) != 0) cell_height += OPT(adjust_line_height_px);
+    if (OPT(adjust_line_height_frac) != 0.f) cell_height *= OPT(adjust_line_height_frac);
+    int line_height_adjustment = cell_height - before_cell_height;
+    if (cell_height < 4) { PyErr_SetString(PyExc_ValueError, "line height too small after adjustment"); return NULL; }
+    if (cell_height > 1000) { PyErr_SetString(PyExc_ValueError, "line height too large after adjustment"); return NULL; }
+    underline_position = MIN(cell_height - 1, underline_position);
+    if (line_height_adjustment > 1) {
+        baseline += MIN(cell_height - 1, (unsigned)line_height_adjustment / 2);
+        underline_position += MIN(cell_height - 1, (unsigned)line_height_adjustment / 2);
+    }
+    sprite_tracker_set_layout(cell_width, cell_height);
+    global_state.cell_width = cell_width; global_state.cell_height = cell_height;
+    free(canvas); canvas = malloc(CELLS_IN_CANVAS * cell_width * cell_height);
+    if (canvas == NULL) return PyErr_NoMemory();
+    for (ssize_t i = 0, j = fonts.first_symbol_font_idx; i < (ssize_t)fonts.symbol_map_fonts_count; i++, j++)  {
+        CALL(j, cell_height, true);
+    }
+    for (ssize_t i = 0, j = fonts.first_fallback_font_idx; i < (ssize_t)fonts.fallback_fonts_count; i++, j++)  {
+        CALL(j, cell_height, true);
+    }
+    return Py_BuildValue("IIIII", cell_width, cell_height, baseline, underline_position, underline_thickness);
+#undef CALL
+}
+
+static PyObject*
+set_font_size(PyObject UNUSED *m, PyObject *args) {
+    if (!PyArg_ParseTuple(args, "f", &global_state.font_sz_in_pts)) return NULL;
+    return update_cell_metrics();
+}
+
+static inline bool
+face_has_codepoint(PyObject* face, char_type cp) {
+    return glyph_id_for_codepoint(face, cp) > 0;
+}
+
+static inline bool 
+has_cell_text(Font *self, Cell *cell) {
+    if (!face_has_codepoint(self->face, cell->ch)) return false;
+    if (cell->cc) {
+        if (!face_has_codepoint(self->face, cell->cc & CC_MASK)) return false;
+        char_type cc = cell->cc >> 16;
+        if (cc && !face_has_codepoint(self->face, cc)) return false;
+    }
+    return true;
+}
+
+static inline ssize_t
+fallback_font(Cell *cell) {
+    bool bold = (cell->attrs >> BOLD_SHIFT) & 1;
+    bool italic = (cell->attrs >> ITALIC_SHIFT) & 1;
+    ssize_t f;
+
+    for (size_t i = 0, j = fonts.first_fallback_font_idx; i < fonts.fallback_fonts_count; i++, j++)  {
+        Font *ff = fonts.fonts +j;
+        if (ff->bold == bold && ff->italic == italic && has_cell_text(ff, cell)) {
+            return j;
+        }
+    }
+
+    if (fonts.fallback_fonts_count > 100) { fprintf(stderr, "Too many fallback fonts\n"); return MISSING_FONT; }
+
+    if (bold) f = fonts.italic_font_idx > 0 ? fonts.bi_font_idx : fonts.bold_font_idx;
+    else f = italic ? fonts.italic_font_idx : fonts.medium_font_idx;
+    if (f < 0) f = fonts.medium_font_idx;
+
+    PyObject *face = create_fallback_face(fonts.fonts[f].face, cell, bold, italic);
+    if (face == NULL) { PyErr_Print(); return MISSING_FONT; }
+    if (face == Py_None) { Py_DECREF(face); return MISSING_FONT; }
+    set_size_for_face(face, cell_height, true);
+
+    ensure_space_for(&fonts, fonts, Font, fonts.fonts_count + 1, fonts_capacity, 5, true);
+    ssize_t ans = fonts.first_fallback_font_idx + fonts.fallback_fonts_count;
+    Font *af = &fonts.fonts[ans];
+    if (!init_font(af, face, bold, italic, true)) fatal("Out of memory");
+    Py_DECREF(face);
+    fonts.fallback_fonts_count++;
+    fonts.fonts_count++;
+    return ans;
+}
+
+static inline ssize_t
+in_symbol_maps(char_type ch) {
+    for (size_t i = 0; i < fonts.symbol_maps_count; i++) {
+        if (fonts.symbol_maps[i].left <= ch && ch <= fonts.symbol_maps[i].right) return fonts.first_symbol_font_idx + fonts.symbol_maps[i].font_idx;
+    }
+    return NO_FONT;
+}
+
+
+static ssize_t
+font_for_cell(Cell *cell) {
+START_ALLOW_CASE_RANGE
+    ssize_t ans;
+    switch(cell->ch) {
+        case 0:
+        case ' ':
+            return BLANK_FONT;
+        case 0x2500 ... 0x2570:
+        case 0x2574 ... 0x259f:
+        case 0xe0b0:
+        case 0xe0b2:
+            return BOX_FONT;
+        default:
+            ans = in_symbol_maps(cell->ch);
+            if (ans > -1) return ans;
+            switch(BI_VAL(cell->attrs)) {
+                case 0:
+                    ans = fonts.medium_font_idx; break;
+                case 1:
+                    ans = fonts.bold_font_idx ; break;
+                case 2:
+                    ans = fonts.italic_font_idx; break;
+                case 3:
+                    ans = fonts.bi_font_idx; break;
+            }
+            if (ans < 0) ans = fonts.medium_font_idx;
+            if (has_cell_text(fonts.fonts + ans, cell)) return ans;
+            return fallback_font(cell);
+    }
+END_ALLOW_CASE_RANGE
+}
+
+static inline void
+set_sprite(Cell *cell, sprite_index x, sprite_index y, sprite_index z) {
+    cell->sprite_x = x; cell->sprite_y = y; cell->sprite_z = z;
+}
+
+static inline glyph_index
+box_glyph_id(char_type ch) {
+START_ALLOW_CASE_RANGE
+    switch(ch) {
+        case 0x2500 ... 0x259f:
+            return ch - 0x2500;
+        case 0xe0b0:
+            return 0xfa;
+        case 0xe0b2:
+            return 0xfb;
+        default:
+            return 0xff;
+    }
+END_ALLOW_CASE_RANGE
+}
+
+static PyObject* box_drawing_function = NULL;
+
+static void 
+render_box_cell(Cell *cell) {
+    int error = 0;
+    glyph_index glyph = box_glyph_id(cell->ch);
+    static ExtraGlyphs extra_glyphs = {{0}};
+    SpritePosition *sp = sprite_position_for(&fonts.fonts[BOX_FONT], glyph, &extra_glyphs, false, &error);
+    if (sp == NULL) {
+        sprite_map_set_error(error); PyErr_Print();
+        set_sprite(cell, 0, 0, 0);
+        return;
+    }
+    set_sprite(cell, sp->x, sp->y, sp->z);
+    if (sp->rendered) return;
+    sp->rendered = true;
+    PyObject *ret = PyObject_CallFunction(box_drawing_function, "I", cell->ch);
+    if (ret == NULL) { PyErr_Print(); return; }
+    current_send_sprite_to_gpu(sp->x, sp->y, sp->z, PyLong_AsVoidPtr(PyTuple_GET_ITEM(ret, 0)));
+    Py_DECREF(ret);
+}
+
+static inline void
+load_hb_buffer(Cell *first_cell, index_type num_cells) {  
+    index_type num;
+    hb_buffer_clear_contents(harfbuzz_buffer);
+    while (num_cells) {
+        attrs_type prev_width = 0;
+        for (num = 0; num_cells && num < sizeof(shape_buffer)/sizeof(shape_buffer[0]) - 20; first_cell++, num_cells--) {
+            if (prev_width == 2) { prev_width = 0; continue; }
+            shape_buffer[num++] = first_cell->ch;
+            prev_width = first_cell->attrs & WIDTH_MASK;
+            if (first_cell->cc) {
+                shape_buffer[num++] = first_cell->cc & CC_MASK;
+                combining_type cc2 = first_cell->cc >> 16;
+                if (cc2) shape_buffer[num++] = cc2 & CC_MASK;
+            }
+        }
+        hb_buffer_add_utf32(harfbuzz_buffer, shape_buffer, num, 0, num);
+    }
+    hb_buffer_guess_segment_properties(harfbuzz_buffer);
+}
+
+
+static inline void
+set_cell_sprite(Cell *cell, SpritePosition *sp) {
+    cell->sprite_x = sp->x; cell->sprite_y = sp->y; cell->sprite_z = sp->z;
+}
+
+static inline uint8_t*
+extract_cell_from_canvas(unsigned int i, unsigned int num_cells) {
+    uint8_t *ans = canvas + (cell_width * cell_height * (CELLS_IN_CANVAS - 1)), *dest = ans, *src = canvas + (i * cell_width);
+    unsigned int stride = cell_width * num_cells;
+    for (unsigned int r = 0; r < cell_height; r++, dest += cell_width, src += stride) memcpy(dest, src, cell_width);
+    return ans;
+}
+
+static inline void
+render_group(unsigned int num_cells, unsigned int num_glyphs, Cell *cells, hb_glyph_info_t *info, hb_glyph_position_t *positions, Font *font, glyph_index glyph, ExtraGlyphs *extra_glyphs) {
+    static SpritePosition* sprite_position[16];
+    int error = 0;
+    num_cells = MIN(sizeof(sprite_position)/sizeof(sprite_position[0]), num_cells);
+    for (unsigned int i = 0; i < num_cells; i++) {
+        sprite_position[i] = sprite_position_for(font, glyph, extra_glyphs, (uint8_t)i, &error);
+        if (error != 0) { sprite_map_set_error(error); PyErr_Print(); return; }
+    }
+    if (sprite_position[0]->rendered) {
+        for (unsigned int i = 0; i < num_cells; i++) { set_cell_sprite(cells + i, sprite_position[i]); }
+        return;
+    }
+
+    clear_canvas();
+    render_glyphs_in_cells(font->face, font->bold, font->italic, info, positions, num_glyphs, canvas, cell_width, cell_height, num_cells, baseline);
+
+    for (unsigned int i = 0; i < num_cells; i++) { 
+        sprite_position[i]->rendered = true;
+        set_cell_sprite(cells + i, sprite_position[i]); 
+        uint8_t *buf = num_cells == 1 ? canvas : extract_cell_from_canvas(i, num_cells);
+        current_send_sprite_to_gpu(sprite_position[i]->x, sprite_position[i]->y, sprite_position[i]->z, buf);
+    }
+
+}
+
+typedef struct {
+    Cell *cell;
+    unsigned int num_codepoints;
+    unsigned int codepoints_consumed;
+    char_type current_codepoint;
+} CellData;
+
+
+static inline bool
+is_special_glyph(glyph_index glyph_id, Font *font, CellData* cell_data) {
+    // A glyph is special if the codepoint it corresponds to matches a
+    // different glyph in the font
+    SpecialGlyphCache *s = special_glyph_cache_for(font, glyph_id);
+    if (s == NULL) return false;
+    if (!s->filled) {
+        s->is_special = cell_data->current_codepoint ? (
+            glyph_id != glyph_id_for_codepoint(font->face, cell_data->current_codepoint) ? true : false) 
+            :
+            false;
+        s->filled = true;
+    } 
+    return s->is_special;
+}
+
+static inline unsigned int
+num_codepoints_in_cell(Cell *cell) {
+    unsigned int ans = 1;
+    if (cell->cc) ans += ((cell->cc >> CC_SHIFT) & CC_MASK) ? 2 : 1;
+    return ans;
+}
+
+static inline unsigned int
+check_cell_consumed(CellData *cell_data, Cell *last_cell) {
+    cell_data->codepoints_consumed++;
+    if (cell_data->codepoints_consumed >= cell_data->num_codepoints) {
+        attrs_type width = cell_data->cell->attrs & WIDTH_MASK;
+        cell_data->cell += MAX(1, width);
+        cell_data->codepoints_consumed = 0;
+        if (cell_data->cell <= last_cell) {
+            cell_data->num_codepoints = num_codepoints_in_cell(cell_data->cell);
+            cell_data->current_codepoint = cell_data->cell->ch;
+        } else cell_data->current_codepoint = 0;
+        return width;
+    } else {
+        switch(cell_data->codepoints_consumed) {
+            case 0:
+                cell_data->current_codepoint = cell_data->cell->ch;
+                break;
+            case 1:
+                cell_data->current_codepoint = cell_data->cell->cc & CC_MASK;
+                break;
+            case 2:
+                cell_data->current_codepoint = (cell_data->cell->cc >> CC_SHIFT) & CC_MASK;
+                break;
+            default:
+                cell_data->current_codepoint = 0;
+                break;
+        }
+    }
+    return 0;
+}
+
+static inline glyph_index
+next_group(Font *font, unsigned int *num_group_cells, unsigned int *num_group_glyphs, Cell *cells, hb_glyph_info_t *info, unsigned int max_num_glyphs, unsigned int max_num_cells, ExtraGlyphs *extra_glyphs) {
+    // See https://github.com/behdad/harfbuzz/issues/615 for a discussion of
+    // how to break text into cells. In addition, we have to deal with
+    // monospace ligature fonts that use dummy glyphs of zero size to implement
+    // their ligatures.
+    
+    CellData cell_data;
+    cell_data.cell = cells; cell_data.num_codepoints = num_codepoints_in_cell(cells); cell_data.codepoints_consumed = 0; cell_data.current_codepoint = cells->ch;
+#define LIMIT (MAX_NUM_EXTRA_GLYPHS + 1)
+    glyph_index glyphs_in_group[LIMIT];
+    unsigned int ncells = 0, nglyphs = 0, n;
+    uint32_t previous_cluster = UINT32_MAX, cluster;
+    Cell *last_cell = cells + max_num_cells;
+    unsigned int cell_limit = MIN(max_num_cells, LIMIT + 1), glyph_limit = MIN(LIMIT, max_num_glyphs);
+    bool is_special, prev_was_special = false;
+
+    while(nglyphs < glyph_limit && ncells < cell_limit) {
+        glyph_index glyph_id = info[nglyphs].codepoint;
+        cluster = info[nglyphs].cluster;
+        is_special = is_special_glyph(glyph_id, font, &cell_data);
+        if (prev_was_special && !is_special) break; 
+        glyphs_in_group[nglyphs++] = glyph_id;
+        // Soak up a number of codepoints indicated by the difference in cluster numbers.
+        if (cluster > previous_cluster || nglyphs == 1) {
+            n = nglyphs == 1 ? 1 : cluster - previous_cluster;
+            unsigned int before = ncells;
+            while(n-- && ncells < max_num_cells) ncells += check_cell_consumed(&cell_data, last_cell);
+            if (ncells > before && !is_special) break;
+        }
+        previous_cluster = cluster;
+        prev_was_special = is_special;
+    }
+    *num_group_cells = MAX(1, MIN(ncells, cell_limit));
+    *num_group_glyphs = MAX(1, MIN(nglyphs, glyph_limit));
+    memset(extra_glyphs, 0, sizeof(ExtraGlyphs));
+
+    switch(nglyphs) {
+        case 0:
+        case 1:
+            break;
+        default:
+            memcpy(&extra_glyphs->data, glyphs_in_group + 1, sizeof(glyph_index) * MIN(MAX_NUM_EXTRA_GLYPHS, nglyphs - 1));
+            break;
+    }
+#undef LIMIT
+    return glyphs_in_group[0];
+}
+
+static inline unsigned int
+shape(Cell *first_cell, index_type num_cells, hb_font_t *font, hb_glyph_info_t **info, hb_glyph_position_t **positions) {
+    load_hb_buffer(first_cell, num_cells);
+    hb_shape(font, harfbuzz_buffer, NULL, 0);
+    unsigned int info_length, positions_length;
+    *info = hb_buffer_get_glyph_infos(harfbuzz_buffer, &info_length);
+    *positions = hb_buffer_get_glyph_positions(harfbuzz_buffer, &positions_length);
+    if (!info || !positions) return 0;
+    return MIN(info_length, positions_length);
+}
+
+
+static inline void
+shape_run(Cell *first_cell, index_type num_cells, Font *font) {
+    hb_glyph_info_t *info;
+    hb_glyph_position_t *positions;
+    unsigned int num_glyphs = shape(first_cell, num_cells, font->hb_font, &info, &positions);
+#if 0
+        // You can also generate this easily using hb-shape --show-flags --show-extents --cluster-level=1 --shapers=ot /path/to/font/file text
+        hb_buffer_serialize_glyphs(harfbuzz_buffer, 0, num_glyphs, (char*)canvas, CELLS_IN_CANVAS * cell_width * cell_height, NULL, font->hb_font, HB_BUFFER_SERIALIZE_FORMAT_TEXT, HB_BUFFER_SERIALIZE_FLAG_DEFAULT | HB_BUFFER_SERIALIZE_FLAG_GLYPH_EXTENTS | HB_BUFFER_SERIALIZE_FLAG_GLYPH_FLAGS);
+        printf("\n%s\n", canvas);
+        clear_canvas();
+#endif
+    unsigned int run_pos = 0, cell_pos = 0, num_group_glyphs, num_group_cells;
+    ExtraGlyphs extra_glyphs;
+    glyph_index first_glyph;
+    while(run_pos < num_glyphs && cell_pos < num_cells) {
+        first_glyph = next_group(font, &num_group_cells, &num_group_glyphs, first_cell + cell_pos, info + run_pos, num_glyphs - run_pos, num_cells - cell_pos, &extra_glyphs);
+        /* printf("Group: num_group_cells: %u num_group_glyphs: %u\n", num_group_cells, num_group_glyphs); */
+        render_group(num_group_cells, num_group_glyphs, first_cell + cell_pos, info + run_pos, positions + run_pos, font, first_glyph, &extra_glyphs);
+        run_pos += num_group_glyphs; cell_pos += num_group_cells;
+    }
+}
+
+static PyObject*
+test_shape(PyObject UNUSED *self, PyObject *args) {
+    Line *line;
+    char *path = NULL;
+    int index = 0;
+    if(!PyArg_ParseTuple(args, "O!|zi", &Line_Type, &line, &path, &index)) return NULL;
+    index_type num = 0;
+    while(num < line->xnum && line->cells[num].ch) num += line->cells[num].attrs & WIDTH_MASK;
+    PyObject *face = NULL;
+    Font *font = fonts.fonts + fonts.medium_font_idx;
+    if (path) {
+        face = face_from_path(path, index);
+        if (face == NULL) return NULL;
+        font = calloc(1, sizeof(Font));
+        font->hb_font = harfbuzz_font_for_face(face); 
+        font->face = face;
+    } 
+    hb_glyph_info_t *info;
+    hb_glyph_position_t *positions;
+    unsigned int num_glyphs = shape(line->cells, num, font->hb_font, &info, &positions);
+
+    PyObject *ans = PyList_New(0);
+    unsigned int run_pos = 0, cell_pos = 0, num_group_glyphs, num_group_cells;
+    ExtraGlyphs extra_glyphs; 
+    glyph_index first_glyph;
+    while(run_pos < num_glyphs && cell_pos < num) {
+        first_glyph = next_group(font, &num_group_cells, &num_group_glyphs, line->cells + cell_pos, info + run_pos, num_glyphs - run_pos, num - cell_pos, &extra_glyphs);
+        PyObject *eg = PyTuple_New(MAX_NUM_EXTRA_GLYPHS);
+        for (size_t g = 0; g < MAX_NUM_EXTRA_GLYPHS; g++) PyTuple_SET_ITEM(eg, g, Py_BuildValue("H", extra_glyphs.data[g]));
+        PyList_Append(ans, Py_BuildValue("IIIN", num_group_cells, num_group_glyphs, first_glyph, eg));
+        run_pos += num_group_glyphs; cell_pos += num_group_cells;
+    }
+    if (face) { Py_CLEAR(face); free(font); }
+    return ans;
+}
+
+static inline void 
+render_run(Cell *first_cell, index_type num_cells, ssize_t font_idx) {
+    switch(font_idx) {
+        default:
+            shape_run(first_cell, num_cells, &fonts.fonts[font_idx]);
+            break;
+        case BLANK_FONT:
+            while(num_cells--) set_sprite(first_cell++, 0, 0, 0);
+            break;
+        case BOX_FONT:
+            while(num_cells--) render_box_cell(first_cell++);
+            break;
+        case MISSING_FONT:
+            while(num_cells--) set_sprite(first_cell++, MISSING_GLYPH, 0, 0);
+            break;
+    }
+}
+
+void
+render_line(Line *line) {
+#define RENDER if (run_font_idx != NO_FONT && i > first_cell_in_run) render_run(line->cells + first_cell_in_run, i - first_cell_in_run, run_font_idx);
+    ssize_t run_font_idx = NO_FONT;
+    index_type first_cell_in_run, i;
+    attrs_type prev_width = 0;
+    for (i=0, first_cell_in_run=0; i < line->xnum; i++) {
+        if (prev_width == 2) { prev_width = 0; continue; }
+        Cell *cell = line->cells + i;
+        ssize_t cell_font_idx = font_for_cell(cell);
+        prev_width = cell->attrs & WIDTH_MASK;
+        if (run_font_idx == NO_FONT) run_font_idx = cell_font_idx;
+        if (run_font_idx == cell_font_idx) continue;
+        RENDER;
+        run_font_idx = cell_font_idx;
+        first_cell_in_run = i;
+    }
+    RENDER;
+#undef RENDER
+}
+
+static PyObject*
+set_font(PyObject UNUSED *m, PyObject *args) {
+    PyObject *sm, *smf, *medium, *bold = NULL, *italic = NULL, *bi = NULL;
+    Py_CLEAR(box_drawing_function);
+    if (!PyArg_ParseTuple(args, "OO!O!fO|OOO", &box_drawing_function, &PyTuple_Type, &sm, &PyTuple_Type, &smf, &global_state.font_sz_in_pts, &medium, &bold, &italic, &bi)) return NULL;
+    Py_INCREF(box_drawing_function);
+    fonts.symbol_map_fonts_count = PyTuple_GET_SIZE(smf);
+    size_t num_fonts = 5 + fonts.symbol_map_fonts_count;
+    for (size_t i = 0; i < fonts.fonts_count; i++) del_font(fonts.fonts + i);
+    ensure_space_for(&fonts, fonts, Font, num_fonts, fonts_capacity, 5, true);
+    fonts.fonts_count = 1;
+#define A(attr, bold, italic) { if(attr) { if (!init_font(&fonts.fonts[fonts.fonts_count], attr, bold, italic, false)) return NULL; fonts.attr##_font_idx = fonts.fonts_count++; } else fonts.attr##_font_idx = -1; } 
+    A(medium, false, false);
+    A(bold, true, false); A(italic, false, true); A(bi, true, true);
+#undef A
+
+    fonts.first_symbol_font_idx = fonts.fonts_count;
+    fonts.symbol_maps_count = PyTuple_GET_SIZE(sm);
+    ensure_space_for(&fonts, symbol_maps, SymbolMap, fonts.symbol_maps_count, symbol_maps_capacity, 5, true);
+    for (size_t i = 0; i < fonts.symbol_map_fonts_count; i++) {
+        PyObject *face;
+        int bold, italic;
+        if (!PyArg_ParseTuple(PyTuple_GET_ITEM(smf, i), "Opp", &face, &bold, &italic)) return NULL;
+        if (!init_font(fonts.fonts + fonts.fonts_count++, face, bold != 0, italic != 0, false)) return NULL;
+    }
+    for (size_t i = 0; i < fonts.symbol_maps_count; i++) {
+        unsigned int left, right, font_idx;
+        if (!PyArg_ParseTuple(PyTuple_GET_ITEM(sm, i), "III", &left, &right, &font_idx)) return NULL;
+        fonts.symbol_maps[i].left = left; fonts.symbol_maps[i].right = right; fonts.symbol_maps[i].font_idx = font_idx;
+    }
+    fonts.first_fallback_font_idx = fonts.fonts_count;
+    fonts.fallback_fonts_count = 0;
+    return update_cell_metrics();
+}
+
+static void
+finalize(void) {
+    Py_CLEAR(python_send_to_gpu_impl);
+    free(canvas);
+    Py_CLEAR(box_drawing_function);
+    for (size_t i = 0; i < fonts.fonts_count; i++) del_font(fonts.fonts + i);
+    free(fonts.symbol_maps); free(fonts.fonts);
+    if (harfbuzz_buffer) hb_buffer_destroy(harfbuzz_buffer);
+}
+
+static PyObject*
+sprite_map_set_limits(PyObject UNUSED *self, PyObject *args) {
+    unsigned int w, h;
+    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
+    sprite_tracker_set_limits(w, h);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+sprite_map_set_layout(PyObject UNUSED *self, PyObject *args) {
+    unsigned int w, h;
+    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
+    sprite_tracker_set_layout(w, h);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+test_sprite_position_for(PyObject UNUSED *self, PyObject *args) {
+    glyph_index glyph;
+    ExtraGlyphs extra_glyphs = {{0}};
+    if (!PyArg_ParseTuple(args, "H|I", &glyph, &extra_glyphs.data)) return NULL;
+    int error;
+    SpritePosition *pos = sprite_position_for(&fonts.fonts[fonts.medium_font_idx], glyph, &extra_glyphs, 0, &error);
+    if (pos == NULL) { sprite_map_set_error(error); return NULL; }
+    return Py_BuildValue("HHH", pos->x, pos->y, pos->z);
+}
+
+static PyObject*
+send_prerendered_sprites(PyObject UNUSED *s, PyObject *args) {
+    int error = 0;
+    sprite_index x = 0, y = 0, z = 0;
+    // blank cell
+    clear_canvas();
+    current_send_sprite_to_gpu(x, y, z, canvas);
+    do_increment(&error);
+    if (error != 0) { sprite_map_set_error(error); return NULL; }
+    for (ssize_t i = 0; i < PyTuple_GET_SIZE(args); i++) {
+        x = sprite_tracker.x; y = sprite_tracker.y; z = sprite_tracker.z;
+        do_increment(&error);
+        if (error != 0) { sprite_map_set_error(error); return NULL; }
+        current_send_sprite_to_gpu(x, y, z, PyLong_AsVoidPtr(PyTuple_GET_ITEM(args, i)));
+    }
+    return Py_BuildValue("H", x);
+}
+
+static PyObject*
+set_send_sprite_to_gpu(PyObject UNUSED *self, PyObject *func) {
+    Py_CLEAR(python_send_to_gpu_impl);
+    python_send_to_gpu_impl = func;
+    Py_INCREF(func);
+    current_send_sprite_to_gpu = func == Py_None ? send_sprite_to_gpu : python_send_to_gpu;
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+test_render_line(PyObject UNUSED *self, PyObject *args) {
+    PyObject *line;
+    if (!PyArg_ParseTuple(args, "O!", &Line_Type, &line)) return NULL;
+    render_line((Line*)line);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+concat_cells(PyObject UNUSED *self, PyObject *args) {
+    unsigned int cell_width, cell_height;
+    PyObject *cells;
+    if (!PyArg_ParseTuple(args, "IIO!", &cell_width, &cell_height, &PyTuple_Type, &cells)) return NULL;
+    size_t num_cells = PyTuple_GET_SIZE(cells), r, c, i;
+    PyObject *ans = PyBytes_FromStringAndSize(NULL, 3 * cell_width * cell_height * num_cells);
+    if (ans == NULL) return PyErr_NoMemory();
+    uint8_t *dest = (uint8_t*)PyBytes_AS_STRING(ans), *src;
+    for (r = 0; r < cell_height; r++) {
+        for (c = 0; c < num_cells; c++) {
+            src = ((uint8_t*)PyBytes_AS_STRING(PyTuple_GET_ITEM(cells, c))) + cell_width * r;
+            for (i = 0; i < cell_width; i++, dest += 3) {
+                dest[0] = src[i]; dest[1] = src[i]; dest[2] = src[i];
+            }
+
+        }
+    }
+    return ans;
+}
+
+static PyObject*
+current_fonts(PyObject UNUSED *self) {
+    PyObject *ans = PyDict_New();
+    if (!ans) return NULL;
+#define SET(key, val) {if (PyDict_SetItemString(ans, #key, fonts.fonts[val].face) != 0) { goto error; }}
+    SET(medium, fonts.medium_font_idx);
+    if (fonts.bold_font_idx) SET(bold, fonts.bold_font_idx);
+    if (fonts.italic_font_idx) SET(italic, fonts.italic_font_idx);
+    if (fonts.bi_font_idx) SET(bi, fonts.bi_font_idx);
+    PyObject *ff = PyTuple_New(fonts.fallback_fonts_count);
+    if (!ff) goto error;
+    for (size_t i = 0; i < fonts.fallback_fonts_count; i++) {
+        Py_INCREF(fonts.fonts[fonts.first_fallback_font_idx + i].face);
+        PyTuple_SET_ITEM(ff, i, fonts.fonts[fonts.first_fallback_font_idx + i].face);
+    }
+    PyDict_SetItemString(ans, "fallback", ff);
+    Py_CLEAR(ff);
+    return ans;
+error:
+    Py_CLEAR(ans); return NULL;
+#undef SET
+}
+
+static PyObject*
+get_fallback_font(PyObject UNUSED *self, PyObject *args) {
+    PyObject *text; 
+    int bold, italic;
+    if (!PyArg_ParseTuple(args, "Upp", &text, &bold, &italic)) return NULL;
+    static Py_UCS4 char_buf[16];
+    if (!PyUnicode_AsUCS4(text, char_buf, sizeof(char_buf)/sizeof(char_buf[0]), 1)) return NULL;
+    Cell cell = {0};
+    cell.ch = char_buf[0];
+    if (PyUnicode_GetLength(text) > 1) cell.cc |= char_buf[1] & CC_MASK;
+    if (PyUnicode_GetLength(text) > 2) cell.cc |= (char_buf[2] & CC_MASK) << 16;
+    if (bold) cell.attrs |= 1 << BOLD_SHIFT;
+    if (italic) cell.attrs |= 1 << ITALIC_SHIFT;
+    ssize_t ans = fallback_font(&cell);
+    if (ans < 0) { PyErr_SetString(PyExc_ValueError, "Too many fallback fonts"); return NULL; }
+    return fonts.fonts[ans].face;
+}
+
+
+static PyMethodDef module_methods[] = {
+    METHODB(set_font_size, METH_VARARGS),
+    METHODB(set_font, METH_VARARGS),
+    METHODB(sprite_map_set_limits, METH_VARARGS),
+    METHODB(sprite_map_set_layout, METH_VARARGS),
+    METHODB(send_prerendered_sprites, METH_VARARGS),
+    METHODB(test_sprite_position_for, METH_VARARGS),
+    METHODB(concat_cells, METH_VARARGS),
+    METHODB(set_send_sprite_to_gpu, METH_O),
+    METHODB(test_shape, METH_VARARGS),
+    METHODB(current_fonts, METH_NOARGS),
+    METHODB(test_render_line, METH_VARARGS),
+    METHODB(get_fallback_font, METH_VARARGS),
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+bool 
+init_fonts(PyObject *module) {
+    if (Py_AtExit(finalize) != 0) {
+        PyErr_SetString(PyExc_RuntimeError, "Failed to register the fonts module at exit handler");
+        return false;
+    }
+    harfbuzz_buffer = hb_buffer_create();
+    if (harfbuzz_buffer == NULL || !hb_buffer_allocation_successful(harfbuzz_buffer) || !hb_buffer_pre_allocate(harfbuzz_buffer, 2048)) { PyErr_NoMemory(); return false; }
+    hb_buffer_set_cluster_level(harfbuzz_buffer, HB_BUFFER_CLUSTER_LEVEL_MONOTONE_CHARACTERS);
+    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
+    current_send_sprite_to_gpu = send_sprite_to_gpu;
+    sprite_tracker_set_limits(2000, 2000);
+    return true;
+}

kitty/fonts.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#pragma once
+
+#include "lineops.h"
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
+#include <hb.h>
+#pragma GCC diagnostic pop
+
+
+
+unsigned int glyph_id_for_codepoint(PyObject *, char_type);
+hb_font_t* harfbuzz_font_for_face(PyObject*);
+bool set_size_for_face(PyObject*, unsigned int, bool);
+void cell_metrics(PyObject*, unsigned int*, unsigned int*, unsigned int*, unsigned int*, unsigned int*); 
+void sprite_tracker_current_layout(unsigned int *x, unsigned int *y, unsigned int *z);
+bool render_glyphs_in_cells(PyObject *f, bool bold, bool italic, hb_glyph_info_t *info, hb_glyph_position_t *positions, unsigned int num_glyphs, uint8_t *canvas, unsigned int cell_width, unsigned int cell_height, unsigned int num_cells, unsigned int baseline);
+void render_line(Line *line);
+void sprite_tracker_set_limits(size_t max_texture_size, size_t max_array_len);
+void sprite_tracker_set_layout(unsigned int cell_width, unsigned int cell_height);
+typedef void (*free_extra_data_func)(void*);
+PyObject* ft_face_from_data(const uint8_t* data, size_t sz, void *extra_data, free_extra_data_func fed, PyObject *path, int hinting, int hintstyle, float);
+PyObject* ft_face_from_path_and_psname(PyObject* path, const char* psname, void *extra_data, free_extra_data_func fed, int hinting, int hintstyle, float);
+PyObject* specialize_font_descriptor(PyObject *base_descriptor);
+PyObject* create_fallback_face(PyObject *base_face, Cell* cell, bool bold, bool italic);
+PyObject* face_from_descriptor(PyObject*);
+PyObject* face_from_path(const char *path, int index);

kitty/fonts/box_drawing.py

@@ -3,15 +3,23 @@
 # License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
 
 import math
+import os
 from functools import partial as p
 from itertools import repeat
 
 from kitty.utils import get_logical_dpi
 
+scale = (0.001, 1, 1.5, 2)
+
+
+def set_scale(new_scale):
+    global scale
+    scale = tuple(new_scale)
+
 
 def thickness(level=1, horizontal=True):
     dpi = get_logical_dpi()[0 if horizontal else 1]
-    pts = (0.001, 1, 1.5, 2)[level]
+    pts = scale[level]
     return int(math.ceil(pts * dpi / 72.0))
 
 
@@ -264,6 +272,49 @@ def inner_corner(buf, width, height, which='tl', level=1):
     draw_vline(buf, width, y1, y2, width // 2 + (xd * hgap), level)
 
 
+def vblock(buf, width, height, frac=1, gravity='top'):
+    num_rows = min(height, round(frac * height))
+    start = 0 if gravity == 'top' else height - num_rows
+    for r in range(start, start + num_rows):
+        off = r * width
+        for c in range(off, off + width):
+            buf[c] = 255
+
+
+def hblock(buf, width, height, frac=1, gravity='left'):
+    num_cols = min(width, round(frac * width))
+    start = 0 if gravity == 'left' else width - num_cols
+    for r in range(height):
+        off = r * width + start
+        for c in range(off, off + num_cols):
+            buf[c] = 255
+
+
+def shade(buf, width, height, frac=1/4):
+    rand = bytearray(os.urandom(width * height))
+    cutoff = int(frac * 255)
+
+    for r in range(height):
+        off = width * r
+        for c in range(width):
+            q = off + c
+            if rand[q] < cutoff:
+                buf[q] = 255
+
+
+def quad(buf, width, height, x=0, y=0):
+    num_cols = width // 2
+    left = x * num_cols
+    right = width if x else num_cols
+    num_rows = height // 2
+    top = y * num_rows
+    bottom = height if y else num_rows
+    for r in range(top, bottom):
+        off = r * width
+        for c in range(left, right):
+            buf[off + c] = 255
+
+
 box_chars = {
     '─': [hline],
     '━': [p(hline, level=3)],
@@ -308,6 +359,38 @@ box_chars = {
     '╣': [p(inner_corner, which='tl'), p(inner_corner, which='bl'), p(dvline, only='right')],
     '╦': [p(inner_corner, which='bl'), p(inner_corner, which='br'), p(dhline, only='top')],
     '╩': [p(inner_corner, which='tl'), p(inner_corner, which='tr'), p(dhline, only='bottom')],
+    '▀': [p(vblock, frac=1/2)],
+    '▁': [p(vblock, frac=1/8, gravity='bottom')],
+    '▂': [p(vblock, frac=1/4, gravity='bottom')],
+    '▃': [p(vblock, frac=3/8, gravity='bottom')],
+    '▄': [p(vblock, frac=1/2, gravity='bottom')],
+    '▅': [p(vblock, frac=5/8, gravity='bottom')],
+    '▆': [p(vblock, frac=3/4, gravity='bottom')],
+    '▇': [p(vblock, frac=7/8, gravity='bottom')],
+    '█': [p(vblock, frac=1, gravity='bottom')],
+    '▉': [p(hblock, frac=7/8)],
+    '▊': [p(hblock, frac=3/4)],
+    '▋': [p(hblock, frac=5/8)],
+    '▌': [p(hblock, frac=1/2)],
+    '▍': [p(hblock, frac=3/8)],
+    '▎': [p(hblock, frac=1/4)],
+    '▏': [p(hblock, frac=1/8)],
+    '▐': [p(hblock, frac=1/2, gravity='right')],
+    '░': [p(shade, frac=1/4)],
+    '▒': [p(shade, frac=2/4)],
+    '▓': [p(shade, frac=3/4)],
+    '▔': [p(vblock, frac=1/8)],
+    '▕': [p(hblock, frac=1/8, gravity='right')],
+    '▖': [p(quad, y=1)],
+    '▗': [p(quad, x=1, y=1)],
+    '▘': [quad],
+    '▙': [quad, p(quad, y=1), p(quad, x=1, y=1)],
+    '▚': [quad, p(quad, x=1, y=1)],
+    '▛': [quad, p(quad, x=1), p(quad, y=1)],
+    '▜': [quad, p(quad, x=1, y=1), p(quad, x=1)],
+    '▝': [p(quad, x=1)],
+    '▞': [p(quad, x=1), p(quad, y=1)],
+    '▟': [p(quad, x=1), p(quad, y=1), p(quad, x=1, y=1)],
 }
 
 t, f = 1, 3
@@ -336,8 +419,6 @@ for chars, func in (('╒╕╘╛', dvcorner), ('╓╖╙╜', dhcorner), ('
     for ch in chars:
         box_chars[ch] = [p(func, which=ch)]
 
-is_renderable_box_char = box_chars.__contains__
-
 
 def render_box_char(ch, buf, width, height):
     for func in box_chars[ch]:
@@ -354,34 +435,41 @@ def render_missing_glyph(buf, width, height):
     draw_vline(buf, width, vgap, height - vgap + 1, width - hgap, 0)
 
 
-def join_rows(width, height, rows):
-    import ctypes
-    ans = (ctypes.c_ubyte * (width * height * len(rows)))()
-    pos = ctypes.addressof(ans)
-    row_sz = width * height
-    for row in rows:
-        ctypes.memmove(pos, ctypes.addressof(row), row_sz)
-        pos += row_sz
-    return ans
-
-
 def test_drawing(sz=32, family='monospace'):
-    from .render import join_cells, display_bitmap, render_cell, set_font_family
-    from kitty.config import defaults
-    opts = defaults._replace(font_family=family, font_size=sz)
-    width, height = set_font_family(opts)
+    from .render import display_bitmap, setup_for_testing
+    from kitty.fast_data_types import concat_cells, set_send_sprite_to_gpu
+
+    width, height = setup_for_testing(family, sz)[1:]
+    space = bytearray(width * height)
+
+    def join_cells(cells):
+        cells = tuple(bytes(x) for x in cells)
+        return concat_cells(width, height, cells)
+
+    def render_chr(ch):
+        if ch in box_chars:
+            cell = bytearray(len(space))
+            render_box_char(ch, cell, width, height)
+            return cell
+        return space
+
     pos = 0x2500
     rows = []
-    space = render_cell()[0]
-    space_row = join_cells(width, height, *repeat(space, 32))
+    space_row = join_cells(repeat(space, 32))
 
-    for r in range(8):
-        row = []
-        for i in range(16):
-            row.append(render_cell(chr(pos))[0]), row.append(space)
-            pos += 1
-        rows.append(join_cells(width, height, *row))
-        rows.append(space_row)
-    width *= 32
-    im = join_rows(width, height, rows)
-    display_bitmap(im, width, height * len(rows))
+    try:
+        for r in range(10):
+            row = []
+            for i in range(16):
+                row.append(render_chr(chr(pos)))
+                row.append(space)
+                pos += 1
+            rows.append(join_cells(row))
+            rows.append(space_row)
+        rgb_data = b''.join(rows)
+        width *= 32
+        height *= len(rows)
+        assert len(rgb_data) == width * height * 3, '{} != {}'.format(len(rgb_data), width * height * 3)
+        display_bitmap(rgb_data, width, height)
+    finally:
+        set_send_sprite_to_gpu(None)

kitty/fonts/core_text.py

@@ -2,17 +2,16 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
 
-import ctypes
 import re
 import sys
 
-from kitty.fast_data_types import CTFace as Face, coretext_all_fonts
-from kitty.utils import ceil_int, get_logical_dpi, safe_print, wcwidth, adjust_line_height
+from kitty.fast_data_types import coretext_all_fonts
+from kitty.utils import safe_print
 
-main_font = {}
-symbol_map = {}
-cell_width = cell_height = baseline = CellTexture = WideCellTexture = underline_thickness = underline_position = None
-attr_map = {(False, False): 'font_family', (True, False): 'bold_font', (False, True): 'italic_font', (True, True): 'bold_italic_font'}
+attr_map = {(False, False): 'font_family',
+            (True, False): 'bold_font',
+            (False, True): 'italic_font',
+            (True, True): 'bold_italic_font'}
 
 
 def create_font_map(all_fonts):
@@ -27,11 +26,30 @@ def create_font_map(all_fonts):
     return ans
 
 
-def find_best_match(font_map, family, bold, italic):
+def all_fonts_map():
+    ans = getattr(all_fonts_map, 'ans', None)
+    if ans is None:
+        ans = all_fonts_map.ans = create_font_map(coretext_all_fonts())
+    return ans
+
+
+def list_fonts():
+    for fd in coretext_all_fonts():
+        f = fd['family']
+        if f:
+            fn = (f + ' ' + (fd['style'] or '')).strip()
+            is_mono = bool(fd['monospace'])
+            yield {'family': f, 'full_name': fn, 'is_monospace': is_mono}
+
+
+def find_best_match(family, bold=False, italic=False):
     q = re.sub(r'\s+', ' ', family.lower())
+    font_map = all_fonts_map()
 
     def score(candidate):
-        style_match = 1 if candidate['bold'] == bold and candidate['italic'] == italic else 0
+        style_match = 1 if candidate['bold'] == bold and candidate[
+            'italic'
+        ] == italic else 0
         monospace_match = 1 if candidate['monospace'] else 0
         return style_match, monospace_match
 
@@ -45,7 +63,10 @@ def find_best_match(font_map, family, bold, italic):
     # Let CoreText choose the font if the family exists, otherwise
     # fallback to Menlo
     if q not in font_map['family_map']:
-        safe_print('The font {} was not found, falling back to Menlo'.format(family), file=sys.stderr)
+        safe_print(
+            'The font {} was not found, falling back to Menlo'.format(family),
+            file=sys.stderr
+        )
         family = 'Menlo'
     return {
         'monospace': True,
@@ -55,111 +76,28 @@ def find_best_match(font_map, family, bold, italic):
     }
 
 
-def get_face(font_map, family, main_family, font_size, dpi, bold=False, italic=False):
-    def resolve_family(f):
-        if (bold or italic) and f == 'auto':
-            f = main_family
-        if f.lower() == 'monospace':
-            f = 'Menlo'
-        return f
-    descriptor = find_best_match(font_map, resolve_family(family), bold, italic)
-    return Face(descriptor, font_size, dpi)
+def resolve_family(f, main_family, bold=False, italic=False):
+    if (bold or italic) and f == 'auto':
+        f = main_family
+    if f.lower() == 'monospace':
+        f = 'Menlo'
+    return f
 
 
-def install_symbol_map(all_fonts, val, font_size, dpi):
-    global symbol_map
-    symbol_map = {}
-    family_map = {f: get_face(all_fonts, f, 'Menlo', font_size, dpi) for f in set(val.values())}
-    for ch, family in val.items():
-        symbol_map[ch] = family_map[family]
-
-
-def set_font_family(opts, override_font_size=None, ignore_dpi_failure=False):
-    global cell_width, cell_height, baseline, CellTexture, WideCellTexture, underline_thickness, underline_position
-    try:
-        dpi = get_logical_dpi()
-    except Exception:
-        if not ignore_dpi_failure:
-            raise
-        dpi = (72, 72)  # Happens when running via develop() in an ssh session
-    dpi = sum(dpi) / 2.0
-    font_size = override_font_size or opts.font_size
-    all_fonts = create_font_map(coretext_all_fonts())
-
+def get_font_files(opts):
+    ans = {}
     for (bold, italic), attr in attr_map.items():
-        main_font[(bold, italic)] = get_face(all_fonts, getattr(opts, attr), opts.font_family, font_size, dpi, bold, italic)
-
-    install_symbol_map(all_fonts, opts.symbol_map, font_size, dpi)
-    mf = main_font[(False, False)]
-    cell_width, cell_height = mf.cell_size()
-    cell_height = adjust_line_height(cell_height, opts.adjust_line_height)
-    CellTexture = ctypes.c_ubyte * (cell_width * cell_height)
-    WideCellTexture = ctypes.c_ubyte * (2 * cell_width * cell_height)
-    baseline = int(round(mf.ascent))
-    underline_position = int(round(baseline - mf.underline_position))
-    underline_thickness = ceil_int(mf.underline_thickness)
-    return cell_width, cell_height
+        face = find_best_match(resolve_family(getattr(opts, attr), opts.font_family, bold, italic), bold, italic)
+        key = {(False, False): 'medium',
+               (True, False): 'bold',
+               (False, True): 'italic',
+               (True, True): 'bi'}[(bold, italic)]
+        ans[key] = face
+        if key == 'medium':
+            get_font_files.medium_family = face['family']
+    return ans
 
 
-def test_font_matching(name='Menlo', bold=False, italic=False, dpi=72.0, font_size=11.0):
-    all_fonts = create_font_map(coretext_all_fonts())
-    face = get_face(all_fonts, name, 'Menlo', font_size, dpi, bold, italic)
-    return face
-
-
-def test_family_matching(name='Menlo', dpi=72.0, font_size=11.0):
-    all_fonts = create_font_map(coretext_all_fonts())
-    for bold in (False, True):
-        for italic in (False, True):
-            face = get_face(all_fonts, name, 'Menlo', font_size, dpi, bold, italic)
-            print(bold, italic, face)
-
-
-def current_cell():
-    return CellTexture, cell_width, cell_height, baseline, underline_thickness, underline_position
-
-
-def split(buf, cell_width, cell_height):
-    first, second = CellTexture(), CellTexture()
-    for y in range(cell_height):
-        offset, woffset = y * cell_width, y * cell_width * 2
-        for x in range(cell_width):
-            first[offset + x] = buf[woffset + x]
-            second[offset + x] = buf[woffset + cell_width + x]
-    return first, second
-
-
-def render_cell(text=' ', bold=False, italic=False):
-    ch = text[0]
-    width = wcwidth(ch)
-    face = symbol_map.get(ch) or main_font[(bold, italic)]
-    if width == 2:
-        buf, width = WideCellTexture(), cell_width * 2
-    else:
-        buf, width = CellTexture(), cell_width
-    face.render_char(text, width, cell_height, ctypes.addressof(buf))
-    if width == 2:
-        first, second = split(buf, cell_width, cell_height)
-    else:
-        first, second = buf, None
-    return first, second
-
-
-def develop(family='monospace', sz=288):
-    import pickle
-    from .render import render_string
-    from kitty.fast_data_types import glfw_init
-    from kitty.config import defaults
-    import os
-    glfw_init()
-    try:
-        os.remove('/tmp/cell.data')
-    except EnvironmentError:
-        pass
-    opts = defaults._replace(font_family=family, font_size=sz)
-    set_font_family(opts, ignore_dpi_failure=True)
-    for (bold, italic), face in main_font.items():
-        print('bold: {} italic: {} {}'.format(bold, italic, face))
-    print('cell_width: {}, cell_height: {}, baseline: {}'.format(cell_width, cell_height, baseline))
-    buf, w, h = render_string()
-    open('/tmp/cell.data', 'wb').write(pickle.dumps((bytearray(buf), w, h)))
+def font_for_family(family):
+    ans = find_best_match(resolve_family(family, get_font_files.medium_family))
+    return ans, ans['bold'], ans['italic']

kitty/fonts/fontconfig.py

@@ -2,120 +2,89 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-import os
 import re
-from collections import namedtuple
+from functools import lru_cache
 
-from kitty.fast_data_types import Face, get_fontconfig_font
-
-
-def escape_family_name(name):
-    return re.sub(r'([-:,\\])', lambda m: '\\' + m.group(1), name)
-
-
-Font = namedtuple(
-    'Font',
-    'face hinting hintstyle bold italic scalable outline weight slant index'
+from kitty.fast_data_types import (
+    FC_SLANT_ITALIC, FC_SLANT_ROMAN, FC_WEIGHT_BOLD, FC_WEIGHT_REGULAR,
+    fc_list, fc_match,
 )
 
-
-class FontNotFound(ValueError):
-    pass
+attr_map = {(False, False): 'font_family',
+            (True, False): 'bold_font',
+            (False, True): 'italic_font',
+            (True, True): 'bold_italic_font'}
 
 
-def to_bool(x):
-    return x.lower() == 'true'
-
-
-def font_not_found(err, char):
-    msg = 'Failed to find font'
-    if char is not None:
-        msg = 'Failed to find font for character U+{:X}, error from fontconfig: {}'.  format(ord(char[0]), err)
-    return FontNotFound(msg)
-
-
-def get_font(
-    family='monospace',
-    bold=False,
-    italic=False,
-    allow_bitmaped_fonts=False,
-    size_in_pts=None,
-    character=None,
-    dpi=None
-):
-    try:
-        path, index, hintstyle, hinting, scalable, outline, weight, slant = get_fontconfig_font(
-            family, bold, italic, allow_bitmaped_fonts, size_in_pts or 0,
-            0 if character is None else ord(character[0]), dpi or 0
-        )
-    except KeyError as err:
-        raise font_not_found(err, character)
-
-    return Font(
-        path, hinting, hintstyle, bold, italic, scalable, outline, weight,
-        slant, index
-    )
-
-
-def find_font_for_character(
-    family,
-    char,
-    bold=False,
-    italic=False,
-    allow_bitmaped_fonts=False,
-    size_in_pts=None,
-    dpi=None
-):
-    ans = get_font(
-        family,
-        bold,
-        italic,
-        character=char,
-        allow_bitmaped_fonts=allow_bitmaped_fonts,
-        size_in_pts=size_in_pts,
-        dpi=dpi
-    )
-    if not ans.face or not os.path.exists(ans.face):
-        raise FontNotFound(
-            'Failed to find font for character U+{:X}'.format(ord(char[0]))
-        )
+def create_font_map(all_fonts):
+    ans = {'family_map': {}, 'ps_map': {}, 'full_map': {}}
+    for x in all_fonts:
+        if 'path' not in x:
+            continue
+        f = (x.get('family') or '').lower()
+        full = (x.get('full_name') or '').lower()
+        ps = (x.get('postscript_name') or '').lower()
+        ans['family_map'].setdefault(f, []).append(x)
+        ans['ps_map'].setdefault(ps, []).append(x)
+        ans['full_map'].setdefault(full, []).append(x)
     return ans
 
 
-def get_font_information(family, bold=False, italic=False):
-    return get_font(family, bold, italic)
+@lru_cache()
+def all_fonts_map(monospaced=True):
+    return create_font_map(fc_list(monospaced))
+
+
+def list_fonts():
+    for fd in fc_list(False):
+        f = fd.get('family')
+        if f:
+            fn = fd.get('full_name') or (f + ' ' + fd.get('style', '')).strip()
+            is_mono = fd.get('spacing') == 'MONO'
+            yield {'family': f, 'full_name': fn, 'is_monospace': is_mono}
+
+
+def find_best_match(family, bold=False, italic=False, monospaced=True):
+    q = re.sub(r'\s+', ' ', family.lower())
+    font_map = all_fonts_map(monospaced)
+
+    def score(candidate):
+        bold_score = abs((FC_WEIGHT_BOLD if bold else FC_WEIGHT_REGULAR) - candidate.get('weight', 0))
+        italic_score = abs((FC_SLANT_ITALIC if italic else FC_SLANT_ROMAN) - candidate.get('slant', 0))
+        monospace_match = 0 if candidate.get('spacing') == 'MONO' else 1
+        return bold_score + italic_score, monospace_match
+
+    # First look for an exact match
+    for selector in ('ps_map', 'full_map', 'family_map'):
+        candidates = font_map[selector].get(q)
+        if candidates:
+            candidates.sort(key=score)
+            return candidates[0]
+
+    # Use fc-match with a generic family
+    family = 'monospace' if monospaced else 'sans-serif'
+    return fc_match(family, bold, italic)
+
+
+def resolve_family(f, main_family, bold, italic):
+    if (bold or italic) and f == 'auto':
+        f = main_family
+    return f
 
 
 def get_font_files(opts):
     ans = {}
-    attr_map = {
-        'bold': 'bold_font',
-        'italic': 'italic_font',
-        'bi': 'bold_italic_font'
-    }
-
-    def get_family(key=None):
-        ans = getattr(opts, attr_map.get(key, 'font_family'))
-        if ans == 'auto' and key:
-            ans = get_family()
-        return ans
-
-    n = get_font_information(get_family())
-    ans['regular'] = n._replace(face=Face(n.face, n.index))
-
-    def do(key):
-        b = get_font_information(
-            get_family(key),
-            bold=key in ('bold', 'bi'),
-            italic=key in ('italic', 'bi')
-        )
-        if b.face != n.face:
-            ans[key] = b._replace(face=Face(b.face, b.index))
-
-    do('bold'), do('italic'), do('bi')
+    for (bold, italic), attr in attr_map.items():
+        rf = resolve_family(getattr(opts, attr), opts.font_family, bold, italic)
+        font = find_best_match(rf, bold, italic)
+        key = {(False, False): 'medium',
+               (True, False): 'bold',
+               (False, True): 'italic',
+               (True, True): 'bi'}[(bold, italic)]
+        ans[key] = font
     return ans
 
 
 def font_for_family(family):
-    ans = get_font_information(family)
-    return ans._replace(face=Face(ans.face, ans.index))
+    ans = find_best_match(family, monospaced=False)
+    return ans, ans.get('weight', 0) >= FC_WEIGHT_BOLD, ans.get('slant', FC_SLANT_ROMAN) != FC_SLANT_ROMAN

kitty/fonts/freetype.py

@@ -1,289 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=utf-8
-# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
-
-import ctypes
-import sys
-from collections import namedtuple
-from functools import lru_cache
-from itertools import chain
-
-from kitty.fast_data_types import FT_PIXEL_MODE_GRAY, Face, FreeTypeError
-from kitty.fonts.box_drawing import render_missing_glyph
-from kitty.utils import ceil_int, get_logical_dpi, safe_print, wcwidth, adjust_line_height
-
-from .fontconfig import (
-    FontNotFound, find_font_for_character, font_for_family, get_font_files
-)
-
-current_font_family = current_font_family_name = cff_size = cell_width = cell_height = baseline = None
-CharTexture = underline_position = underline_thickness = None
-alt_face_cache = {}
-symbol_map = {}
-
-
-def set_char_size(face, width=0, height=0, hres=0, vres=0):
-    face.set_char_size(width, height, hres, vres)
-
-
-def load_char(font, face, text):
-    face.load_char(text, font.hinting, font.hintstyle)
-
-
-def calc_cell_width(font, face):
-    ans = 0
-    for i in range(32, 128):
-        ch = chr(i)
-        load_char(font, face, ch)
-        m = face.glyph_metrics()
-        ans = max(ans, ceil_int(m.horiAdvance / 64))
-    return ans
-
-
-@lru_cache(maxsize=2**10)
-def font_for_char(char, bold=False, italic=False, allow_bitmaped_fonts=False):
-    if allow_bitmaped_fonts:
-        return find_font_for_character(
-            current_font_family_name,
-            char,
-            bold,
-            italic,
-            allow_bitmaped_fonts=True,
-            size_in_pts=cff_size['width'] / 64,
-            dpi=(cff_size['hres'] + cff_size['vres']) / 2
-        )
-    return find_font_for_character(
-        current_font_family_name, char, bold, italic
-    )
-
-
-def install_symbol_map(val):
-    global symbol_map
-    symbol_map = {}
-    family_map = {f: font_for_family(f) for f in set(val.values())}
-    for ch, family in val.items():
-        symbol_map[ch] = family_map[family]
-
-
-def font_units_to_pixels(x, units_per_em, size_in_pts, dpi):
-    return ceil_int(x * ((size_in_pts * dpi) / (72 * units_per_em)))
-
-
-def set_font_family(opts, override_font_size=None):
-    global current_font_family, current_font_family_name, cff_size, cell_width, cell_height, CharTexture, baseline
-    global underline_position, underline_thickness
-    size_in_pts = override_font_size or opts.font_size
-    current_font_family = get_font_files(opts)
-    current_font_family_name = opts.font_family
-    dpi = get_logical_dpi()
-    cff_size = ceil_int(64 * size_in_pts)
-    cff_size = {
-        'width': cff_size,
-        'height': cff_size,
-        'hres': int(dpi[0]),
-        'vres': int(dpi[1])
-    }
-    install_symbol_map(opts.symbol_map)
-    for fobj in chain(current_font_family.values(), symbol_map.values()):
-        set_char_size(fobj.face, **cff_size)
-    face = current_font_family['regular'].face
-    cell_width = calc_cell_width(current_font_family['regular'], face)
-    cell_height = font_units_to_pixels(
-        face.height, face.units_per_EM, size_in_pts, dpi[1]
-    )
-    cell_height = adjust_line_height(cell_height, opts.adjust_line_height)
-    baseline = font_units_to_pixels(
-        face.ascender, face.units_per_EM, size_in_pts, dpi[1]
-    )
-    underline_position = min(
-        baseline - font_units_to_pixels(
-            face.underline_position, face.units_per_EM, size_in_pts, dpi[1]
-        ), cell_height - 1
-    )
-    underline_thickness = font_units_to_pixels(
-        face.underline_thickness, face.units_per_EM, size_in_pts, dpi[1]
-    )
-    CharTexture = ctypes.c_ubyte * (cell_width * cell_height)
-    font_for_char.cache_clear()
-    alt_face_cache.clear()
-    return cell_width, cell_height
-
-
-CharBitmap = namedtuple(
-    'CharBitmap', 'data bearingX bearingY advance rows columns'
-)
-
-
-def render_to_bitmap(font, face, text):
-    load_char(font, face, text)
-    bitmap = face.bitmap()
-    if bitmap.pixel_mode != FT_PIXEL_MODE_GRAY:
-        raise ValueError(
-            'FreeType rendered the glyph for {!r} with an unsupported pixel mode: {}'.
-            format(text, bitmap.pixel_mode)
-        )
-    return bitmap
-
-
-def render_using_face(font, face, text, width, italic, bold):
-    bitmap = render_to_bitmap(font, face, text)
-    if width == 1 and bitmap.width > cell_width:
-        extra = bitmap.width - cell_width
-        if italic and extra < cell_width // 2:
-            bitmap = face.trim_to_width(bitmap, cell_width)
-        elif extra > max(2, 0.3 * cell_width) and face.is_scalable:
-            # rescale the font size so that the glyph is visible in a single
-            # cell and hope somebody updates libc's wcwidth
-            sz = cff_size.copy()
-            sz['width'] = int(sz['width'] * cell_width / bitmap.width)
-            # Preserve aspect ratio
-            sz['height'] = int(sz['height'] * cell_width / bitmap.width)
-            try:
-                set_char_size(face, **sz)
-                bitmap = render_to_bitmap(font, face, text)
-            finally:
-                set_char_size(face, **cff_size)
-    m = face.glyph_metrics()
-    return CharBitmap(
-        bitmap.buffer,
-        ceil_int(abs(m.horiBearingX) / 64),
-        ceil_int(abs(m.horiBearingY) / 64),
-        ceil_int(m.horiAdvance / 64), bitmap.rows, bitmap.width
-    )
-
-
-def render_char(text, bold=False, italic=False, width=1):
-    key = 'regular'
-    if bold:
-        key = 'bi' if italic else 'bold'
-    elif italic:
-        key = 'italic'
-    ch = text[0]
-    font = symbol_map.get(ch)
-    if font is None or not font.face.get_char_index(ch):
-        font = current_font_family.get(key) or current_font_family['regular']
-        face = font.face
-        if not face.get_char_index(ch):
-            try:
-                font = font_for_char(ch, bold, italic)
-            except FontNotFound:
-                font = font_for_char(
-                    ch, bold, italic, allow_bitmaped_fonts=True
-                )
-            face = alt_face_cache.get(font)
-            if face is None:
-                face = alt_face_cache[font] = Face(font.face, font.index)
-                if face.is_scalable:
-                    set_char_size(face, **cff_size)
-    else:
-        face = font.face
-    return render_using_face(font, face, text, width, italic, bold)
-
-
-def place_char_in_cell(bitmap_char):
-    # We want the glyph to be positioned inside the cell based on the bearingX
-    # and bearingY values, making sure that it does not overflow the cell.
-
-    # Calculate column bounds
-    if bitmap_char.columns > cell_width:
-        src_start_column, dest_start_column = 0, 0
-    else:
-        src_start_column, dest_start_column = 0, bitmap_char.bearingX
-        extra = dest_start_column + bitmap_char.columns - cell_width
-        if extra > 0:
-            dest_start_column -= extra
-    column_count = min(
-        bitmap_char.columns - src_start_column, cell_width - dest_start_column
-    )
-
-    # Calculate row bounds, making sure the baseline is aligned with the cell
-    # baseline
-    if bitmap_char.bearingY > baseline:
-        src_start_row, dest_start_row = bitmap_char.bearingY - baseline, 0
-    else:
-        src_start_row, dest_start_row = 0, baseline - bitmap_char.bearingY
-    row_count = min(
-        bitmap_char.rows - src_start_row, cell_height - dest_start_row
-    )
-    return create_cell_buffer(
-        bitmap_char, src_start_row, dest_start_row, row_count,
-        src_start_column, dest_start_column, column_count
-    )
-
-
-def split_char_bitmap(bitmap_char):
-    stride = bitmap_char.columns
-    extra = stride - cell_width
-    rows = bitmap_char.rows
-    first_buf = (ctypes.c_ubyte * (cell_width * rows))()
-    second_buf = (ctypes.c_ubyte * (extra * rows))()
-    src = bitmap_char.data
-    for r in range(rows):
-        soff, off = r * stride, r * cell_width
-        first_buf[off:off + cell_width] = src[soff:soff + cell_width]
-        off = r * extra
-        soff += cell_width
-        second_buf[off:off + extra] = src[soff:soff + extra]
-    first = bitmap_char._replace(data=first_buf, columns=cell_width)
-    second = bitmap_char._replace(data=second_buf, bearingX=0, columns=extra)
-    return first, second
-
-
-def current_cell():
-    return CharTexture, cell_width, cell_height, baseline, underline_thickness, underline_position
-
-
-@lru_cache(maxsize=8)
-def missing_glyph(width):
-    w = cell_width * width
-    ans = bytearray(w * cell_height)
-    render_missing_glyph(ans, w, cell_height)
-    first, second = CharBitmap(ans, 0, 0, 0, cell_height, w), None
-    if width == 2:
-        first, second = split_char_bitmap(first)
-        second = create_cell_buffer(
-            second, 0, 0, second.rows, 0, 0, second.columns
-        )
-    first = create_cell_buffer(first, 0, 0, first.rows, 0, 0, first.columns)
-    return first, second
-
-
-def render_cell(text=' ', bold=False, italic=False):
-    # TODO: Handle non-normalizable combining chars. Probably need to use
-    # harfbuzz for that
-    text = text[0]
-    width = wcwidth(text)
-    try:
-        bitmap_char = render_char(text, bold, italic, width)
-    except FontNotFound as err:
-        safe_print('ERROR:', err, file=sys.stderr)
-        return missing_glyph(width)
-    except FreeTypeError as err:
-        safe_print('Failed to render text:', repr(text), 'with error:', err, file=sys.stderr)
-        return missing_glyph(width)
-    second = None
-    if width == 2:
-        if bitmap_char.columns > cell_width:
-            bitmap_char, second = split_char_bitmap(bitmap_char)
-            second = place_char_in_cell(second)
-        else:
-            second = render_cell()[0]
-
-    first = place_char_in_cell(bitmap_char)
-
-    return first, second
-
-
-def create_cell_buffer(
-    bitmap_char, src_start_row, dest_start_row, row_count, src_start_column,
-    dest_start_column, column_count
-):
-    src = bitmap_char.data
-    src_stride = bitmap_char.columns
-    dest = CharTexture()
-    for r in range(row_count):
-        sr, dr = src_start_column + (
-            src_start_row + r
-        ) * src_stride, dest_start_column + (dest_start_row + r) * cell_width
-        dest[dr:dr + column_count] = src[sr:sr + column_count]
-    return dest

kitty/fonts/list.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+import sys
+from kitty.constants import isosx
+
+if isosx:
+    from .core_text import list_fonts
+else:
+    from .fontconfig import list_fonts
+
+
+def create_family_groups(monospaced=True):
+    g = {}
+    for f in list_fonts():
+        if not monospaced or f['is_monospace']:
+            g.setdefault(f['family'], []).append(f)
+    return g
+
+
+def main(argv):
+    isatty = sys.stdout.isatty()
+    groups = create_family_groups()
+    for k in sorted(groups, key=lambda x: x.lower()):
+        if isatty:
+            print('\033[1;32m' + k + '\033[m')
+        else:
+            print(k)
+        for f in sorted(groups[k], key=lambda x: x['full_name'].lower()):
+            p = f['full_name']
+            if isatty:
+                p = '\033[3m' + p + '\033[m'
+            print('   ', p)
+        print()

kitty/fonts/render.py

@@ -3,14 +3,71 @@
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
 import ctypes
-from math import sin, pi, ceil, floor, sqrt
+import sys
+from collections import namedtuple
+from math import ceil, floor, pi, sin, sqrt
 
+from kitty.config import defaults
 from kitty.constants import isosx
-from .box_drawing import render_box_char, is_renderable_box_char
+from kitty.fast_data_types import (
+    Screen, change_wcwidth, get_fallback_font, send_prerendered_sprites,
+    set_font, set_font_size, set_logical_dpi, set_send_sprite_to_gpu,
+    sprite_map_set_limits, test_render_line, test_shape
+)
+from kitty.fonts.box_drawing import render_box_char, render_missing_glyph
+
 if isosx:
-    from .core_text import set_font_family, render_cell as rc, current_cell  # noqa
+    from .core_text import get_font_files, font_for_family
 else:
-    from .freetype import set_font_family, render_cell as rc, current_cell  # noqa
+    from .fontconfig import get_font_files, font_for_family
+
+
+def create_symbol_map(opts):
+    val = opts.symbol_map
+    family_map = {}
+    faces = []
+    for family in val.values():
+        if family not in family_map:
+            font, bold, italic = font_for_family(family)
+            family_map[family] = len(faces)
+            faces.append((font, bold, italic))
+    sm = tuple((a, b, family_map[f]) for (a, b), f in val.items())
+    return sm, tuple(faces)
+
+
+FontState = namedtuple(
+    'FontState',
+    'family pt_sz cell_width cell_height baseline underline_position underline_thickness'
+)
+
+
+def set_font_family(opts=None, override_font_size=None):
+    opts = opts or defaults
+    sz = override_font_size or opts.font_size
+    font_map = get_font_files(opts)
+    faces = [font_map['medium']]
+    for k in 'bold italic bi'.split():
+        if k in font_map:
+            faces.append(font_map[k])
+    sm, sfonts = create_symbol_map(opts)
+    cell_width, cell_height, baseline, underline_position, underline_thickness = set_font(
+        render_box_drawing, sm, sfonts, sz, *faces
+    )
+    set_font_family.state = FontState(
+        opts.font_family, sz, cell_width, cell_height, baseline,
+        underline_position, underline_thickness
+    )
+    return cell_width, cell_height
+
+
+def resize_fonts(new_sz):
+    s = set_font_family.state
+    cell_width, cell_height, baseline, underline_position, underline_thickness = set_font_size(new_sz)
+    set_font_family.state = FontState(
+        s.family, new_sz, cell_width, cell_height, baseline,
+        underline_position, underline_thickness
+    )
+    return cell_width, cell_height
 
 
 def add_line(buf, cell_width, position, thickness, cell_height):
@@ -34,7 +91,9 @@ def add_curl(buf, cell_width, position, thickness, cell_height):
         return max(0, min(int(x), cell_width - 1))
 
     def add_intensity(x, y, distance):
-        buf[cell_width * y + x] = min(255, buf[cell_width * y + x] + int(255 * (1 - distance)))
+        buf[cell_width * y + x] = min(
+            255, buf[cell_width * y + x] + int(255 * (1 - distance))
+        )
 
     for x_exact in range(cell_width):
         y_exact = yfactor * sin(x_exact * xfactor) + position
@@ -47,79 +106,145 @@ def add_curl(buf, cell_width, position, thickness, cell_height):
                 add_intensity(x, y, dist)
 
 
-def render_cell(text=' ', bold=False, italic=False, underline=0, strikethrough=False):
-    CharTexture, cell_width, cell_height, baseline, underline_thickness, underline_position = current_cell()
-    if is_renderable_box_char(text):
-        first, second = render_box_char(text, CharTexture(), cell_width, cell_height), None
-    else:
-        first, second = rc(text, bold, italic)
+def render_special(underline=0, strikethrough=False, missing=False):
+    s = set_font_family.state
+    cell_width, cell_height, baseline = s.cell_width, s.cell_height, s.baseline
+    underline_position, underline_thickness = s.underline_position, s.underline_thickness
+    underline_position = min(underline_position, cell_height - underline_thickness)
+    CharTexture = ctypes.c_ubyte * (cell_width * cell_height)
+    ans = CharTexture if missing else CharTexture()
 
     def dl(f, *a):
-        f(first, cell_width, *a)
-        if second is not None:
-            f(second, cell_width, *a)
+        f(ans, cell_width, *a)
 
     if underline:
         t = underline_thickness
         if underline == 2:
             t = max(1, min(cell_height - underline_position - 1, t))
-        dl(add_curl if underline == 2 else add_line, underline_position, t, cell_height)
+        dl(
+            add_curl
+            if underline == 2 else add_line, underline_position, t, cell_height
+        )
     if strikethrough:
         pos = int(0.65 * baseline)
         dl(add_line, pos, underline_thickness, cell_height)
 
-    return first, second
-
-
-class Buf:
-
-    def __init__(self, buf):
-        self.buf = buf
-
-    def __call__(self):
-        return ctypes.addressof(self.buf)
-
-
-def render_cell_wrapper(text, bold, italic, underline, strikethrough, is_second):
-    first, second = render_cell(text, bold, italic, underline, strikethrough)
-    ans = second if is_second else first
-    ans = ans or render_cell()[0]
-    return Buf(ans)
-
-
-def join_cells(cell_width, cell_height, *cells):
-    dstride = len(cells) * cell_width
-    ans = (ctypes.c_ubyte * (cell_height * dstride))()
-    for r in range(cell_height):
-        soff = r * cell_width
-        doff = r * dstride
-        for cnum, cell in enumerate(cells):
-            doff2 = doff + (cnum * cell_width)
-            ans[doff2:doff2 + cell_width] = cell[soff:soff + cell_width]
+    if missing:
+        buf = bytearray(cell_width * cell_height)
+        render_missing_glyph(buf, cell_width, cell_height)
+        ans = CharTexture.from_buffer(buf)
     return ans
 
 
-def display_bitmap(data, w, h):
-    from PIL import Image
-    img = Image.new('L', (w, h))
-    img.putdata(data)
-    img.show()
+def prerender():
+    # Pre-render the special blank, underline and strikethrough cells
+    cells = render_special(1), render_special(2), render_special(0, True), render_special(missing=True)
+    if send_prerendered_sprites(*map(ctypes.addressof, cells)) != len(cells):
+        raise RuntimeError('Your GPU has too small a max texture size')
 
 
-def render_string(text='\'Qing👁a⧽'):
+def render_box_drawing(codepoint):
+    s = set_font_family.state
+    cell_width, cell_height = s.cell_width, s.cell_height
+    CharTexture = ctypes.c_ubyte * (cell_width * cell_height)
+    buf = render_box_char(
+        chr(codepoint), CharTexture(), cell_width, cell_height
+    )
+    return ctypes.addressof(buf), buf
+
+
+def setup_for_testing(family='monospace', size=11.0, dpi=96.0):
+    from kitty.utils import get_logical_dpi
+    opts = defaults._replace(font_family=family)
+    sprites = {}
+
+    def send_to_gpu(x, y, z, data):
+        sprites[(x, y, z)] = data
+
+    sprite_map_set_limits(100000, 100)
+    set_send_sprite_to_gpu(send_to_gpu)
+    set_logical_dpi(dpi, dpi)
+    get_logical_dpi((dpi, dpi))
+    cell_width, cell_height = set_font_family(opts, override_font_size=size)
+    prerender()
+    return sprites, cell_width, cell_height
+
+
+def render_string(text, family='monospace', size=11.0, dpi=96.0):
+    try:
+        sprites, cell_width, cell_height = setup_for_testing(family, size, dpi)
+        s = Screen(None, 1, len(text)*2)
+        line = s.line(0)
+        s.draw(text)
+        test_render_line(line)
+    finally:
+        set_send_sprite_to_gpu(None)
     cells = []
-    for c in text:
-        f, s = render_cell(c, underline=2, strikethrough=True)
-        cells.append(f)
-        if s is not None:
-            cells.append(s)
-    cell_width, cell_height = current_cell()[1:3]
-    char_data = join_cells(cell_width, cell_height, *cells)
-    return char_data, cell_width * len(cells), cell_height
+    found_content = False
+    for i in reversed(range(s.columns)):
+        sp = line.sprite_at(i)
+        if sp == (0, 0, 0) and not found_content:
+            continue
+        found_content = True
+        cells.append(sprites[sp])
+    return cell_width, cell_height, list(reversed(cells))
 
 
-def test_rendering(text='\'Ping👁a⧽', sz=144, family='monospace'):
-    from kitty.config import defaults
-    opts = defaults._replace(font_family=family, font_size=sz)
-    set_font_family(opts)
-    display_bitmap(*render_string(text))
+def shape_string(text="abcd", family='monospace', size=11.0, dpi=96.0, path=None):
+    try:
+        sprites, cell_width, cell_height = setup_for_testing(family, size, dpi)
+        s = Screen(None, 1, len(text)*2)
+        line = s.line(0)
+        s.draw(text)
+        return test_shape(line, path)
+    finally:
+        set_send_sprite_to_gpu(None)
+
+
+def display_bitmap(rgb_data, width, height):
+    from tempfile import NamedTemporaryFile
+    from kitty.icat import detect_support, show
+    if not hasattr(display_bitmap, 'detected') and not detect_support():
+        raise SystemExit('Your terminal does not support the graphics protocol')
+    display_bitmap.detected = True
+    with NamedTemporaryFile(delete=False) as f:
+        f.write(rgb_data)
+    show(f.name, width, height, 24)
+
+
+def test_render_string(text='Hello, world!', family='monospace', size=144.0, dpi=96.0):
+    from kitty.fast_data_types import concat_cells, current_fonts
+
+    cell_width, cell_height, cells = render_string(text, family, size, dpi)
+    rgb_data = concat_cells(cell_width, cell_height, tuple(cells))
+    cf = current_fonts()
+    fonts = [cf['medium'].display_name()]
+    fonts.extend(f.display_name() for f in cf['fallback'])
+    msg = 'Rendered string {} below, with fonts: {}\n'.format(text, ', '.join(fonts))
+    try:
+        print(msg)
+    except UnicodeEncodeError:
+        sys.stdout.buffer.write(msg.encode('utf-8') + b'\n')
+    display_bitmap(rgb_data, cell_width * len(cells), cell_height)
+    print('\n')
+
+
+def test_fallback_font(qtext=None, bold=False, italic=False):
+    set_logical_dpi(96.0, 96.0)
+    set_font_family()
+    trials = (qtext,) if qtext else ('你', 'He\u0347\u0305', '\U0001F929')
+    for text in trials:
+        f = get_fallback_font(text, bold, italic)
+        try:
+            print(text, f)
+        except UnicodeEncodeError:
+            sys.stdout.buffer.write((text + ' %s\n' % f).encode('utf-8'))
+
+
+def showcase():
+    change_wcwidth(True)
+    f = 'monospace' if isosx else 'Liberation Mono'
+    test_render_string('He\u0347\u0305llo\u0337, w\u0302or\u0306l\u0354d!', family=f)
+    test_render_string('你好,世界', family=f)
+    test_render_string('|\U0001F601|\U0001F64f|\U0001F63a|', family=f)
+    test_render_string('A=>>B!=C', family='Fira Code')

kitty/freetype.c

@@ -5,9 +5,17 @@
  * Distributed under terms of the GPL3 license.
  */
 
-#include "data-types.h"
+#include "fonts.h"
+#include "state.h"
+#include <math.h>
 #include <structmember.h>
 #include <ft2build.h>
+#include <hb-ft.h>
+
+#if HB_VERSION_MAJOR > 1 || (HB_VERSION_MAJOR == 1 && (HB_VERSION_MINOR > 6 || (HB_VERSION_MINOR == 6 && HB_VERSION_MICRO >= 3)))
+#define HARFBUZZ_HAS_CHANGE_FONT
+#endif
+
 #include FT_FREETYPE_H
 typedef struct {
     PyObject_HEAD
@@ -15,9 +23,18 @@ typedef struct {
     FT_Face face;
     unsigned int units_per_EM;
     int ascender, descender, height, max_advance_width, max_advance_height, underline_position, underline_thickness;
+    int hinting, hintstyle, index;
     bool is_scalable;
+    float size_in_pts;
+    FT_F26Dot6 char_width, char_height;
+    FT_UInt xdpi, ydpi;
     PyObject *path;
+    hb_font_t *harfbuzz_font;
+    void *extra_data;
+    free_extra_data_func free_extra_data;
+    float apple_leading;
 } Face;
+PyTypeObject Face_Type;
 
 static PyObject* FreeType_Exception = NULL;
 
@@ -53,33 +70,186 @@ set_freetype_error(const char* prefix, int err_code) {
 
 static FT_Library  library;
 
+#define CALC_CELL_HEIGHT(self) font_units_to_pixels(self, self->height)
 
-static PyObject*
-new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
-    Face *self;
-    char *path;
-    int error;
-    long index;
-    /* unsigned int columns=80, lines=24, scrollback=0; */
-    if (!PyArg_ParseTuple(args, "sl", &path, &index)) return NULL;
+static inline int
+font_units_to_pixels(Face *self, int x) {
+    return ceil((double)FT_MulFix(x, self->face->size->metrics.y_scale) / 64.0);
+}
 
-    self = (Face *)type->tp_alloc(type, 0);
-    if (self != NULL) {
-        self->path = PyTuple_GET_ITEM(args, 0);
-        Py_INCREF(self->path);
-        error = FT_New_Face(library, path, index, &(self->face));
-        if(error) { set_freetype_error("Failed to load face, with error:", error); Py_CLEAR(self); return NULL; }
+static inline bool
+set_font_size(Face *self, FT_F26Dot6 char_width, FT_F26Dot6 char_height, FT_UInt xdpi, FT_UInt ydpi, unsigned int desired_height) {
+    int error = FT_Set_Char_Size(self->face, 0, char_height, xdpi, ydpi);
+    if (!error) {
+        unsigned int ch = CALC_CELL_HEIGHT(self);
+        if (desired_height && ch != desired_height) {
+            FT_F26Dot6 h = floor((double)char_height * (double)desired_height / (double) ch);
+            return set_font_size(self, 0, h, xdpi, ydpi, 0);
+        }
+        self->char_width = char_width; self->char_height = char_height; self->xdpi = xdpi; self->ydpi = ydpi;
+        if (self->harfbuzz_font != NULL) {
+#ifdef HARFBUZZ_HAS_CHANGE_FONT
+            hb_ft_font_changed(self->harfbuzz_font);
+#else
+            hb_font_set_scale(
+                self->harfbuzz_font,
+                (int) (((uint64_t) self->face->size->metrics.x_scale * (uint64_t) self->face->units_per_EM + (1u<<15)) >> 16),
+                (int) (((uint64_t) self->face->size->metrics.y_scale * (uint64_t) self->face->units_per_EM + (1u<<15)) >> 16)
+            );
+#endif
+        }
+    } else {
+        if (!self->is_scalable && self->face->num_fixed_sizes > 0) {
+            int32_t min_diff = INT32_MAX;
+            if (desired_height == 0) desired_height = global_state.cell_height;
+            if (desired_height == 0) {
+                desired_height = ceil(((double)char_height / 64.) * (double)ydpi / 72.);
+                desired_height += ceil(0.2 * desired_height);
+            }
+            FT_Int strike_index = -1;
+            for (FT_Int i = 0; i < self->face->num_fixed_sizes; i++) {
+                int h = self->face->available_sizes[i].height;
+                int32_t diff = h < (int32_t)desired_height ? (int32_t)desired_height - h : h - (int32_t)desired_height;
+                if (diff < min_diff) {
+                    min_diff = diff;
+                    strike_index = i;
+                }
+            }
+            if (strike_index > -1) {
+                error = FT_Select_Size(self->face, strike_index);
+                if (error) { set_freetype_error("Failed to set char size for non-scaleable font, with error:", error); return false; }
+                return true;
+            }
+        }
+        set_freetype_error("Failed to set char size, with error:", error); 
+        return false; 
+    }
+    return !error;
+}
+
+bool
+set_size_for_face(PyObject *s, unsigned int desired_height, bool force) {
+    Face *self = (Face*)s;
+    FT_F26Dot6 w = (FT_F26Dot6)(ceil(global_state.font_sz_in_pts * 64.0));
+    FT_UInt xdpi = (FT_UInt)global_state.logical_dpi_x, ydpi = (FT_UInt)global_state.logical_dpi_y;
+    if (!force && (self->char_width == w && self->char_height == w && self->xdpi == xdpi && self->ydpi == ydpi)) return true;
+    ((Face*)self)->size_in_pts = global_state.font_sz_in_pts;
+    return set_font_size(self, w, w, xdpi, ydpi, desired_height);
+}
+
+static inline int
+get_load_flags(int hinting, int hintstyle, int base) {
+    int flags = base;
+    if (hinting) {
+        if (hintstyle >= 3) flags |= FT_LOAD_TARGET_NORMAL;
+        else if (0 < hintstyle  && hintstyle < 3) flags |= FT_LOAD_TARGET_LIGHT;
+    } else flags |= FT_LOAD_NO_HINTING;
+    return flags;
+}
+
+
+static inline bool
+init_ft_face(Face *self, PyObject *path, int hinting, int hintstyle) {
 #define CPY(n) self->n = self->face->n;
-        CPY(units_per_EM); CPY(ascender); CPY(descender); CPY(height); CPY(max_advance_width); CPY(max_advance_height); CPY(underline_position); CPY(underline_thickness);
+    CPY(units_per_EM); CPY(ascender); CPY(descender); CPY(height); CPY(max_advance_width); CPY(max_advance_height); CPY(underline_position); CPY(underline_thickness);
 #undef CPY
-        self->is_scalable = FT_IS_SCALABLE(self->face);
+    self->is_scalable = FT_IS_SCALABLE(self->face);
+    self->hinting = hinting; self->hintstyle = hintstyle;
+    if (!set_size_for_face((PyObject*)self, 0, false)) return false;
+    self->harfbuzz_font = hb_ft_font_create(self->face, NULL);
+    if (self->harfbuzz_font == NULL) { PyErr_NoMemory(); return false; }
+    hb_ft_font_set_load_flags(self->harfbuzz_font, get_load_flags(self->hinting, self->hintstyle, FT_LOAD_DEFAULT));
+
+    self->path = path;
+    Py_INCREF(self->path);
+    self->index = self->face->face_index & 0xFFFF;
+    return true;
+}
+
+PyObject*
+ft_face_from_data(const uint8_t* data, size_t sz, void *extra_data, free_extra_data_func fed, PyObject *path, int hinting, int hintstyle, float apple_leading) {
+    Face *ans = (Face*)Face_Type.tp_alloc(&Face_Type, 0);
+    if (ans == NULL) return NULL; 
+    int error = FT_New_Memory_Face(library, data, sz, 0, &ans->face);
+    if(error) { set_freetype_error("Failed to load memory face, with error:", error); Py_CLEAR(ans); return NULL; }
+    if (!init_ft_face(ans, path, hinting, hintstyle)) { Py_CLEAR(ans); return NULL; }
+    ans->extra_data = extra_data;
+    ans->free_extra_data = fed;
+    ans->apple_leading = apple_leading;
+    return (PyObject*)ans;
+}
+
+static inline bool
+load_from_path_and_psname(const char *path, const char* psname, Face *ans) {
+    int error, num_faces, index = 0;
+    error = FT_New_Face(library, path, index, &ans->face);
+    if (error) { set_freetype_error("Failed to load face, with error:", error); ans->face = NULL; return false; }
+    num_faces = ans->face->num_faces;
+    if (num_faces < 2) return true;
+    do {
+        if (ans->face) {
+            if (strcmp(FT_Get_Postscript_Name(ans->face), psname) == 0) return true;
+            FT_Done_Face(ans->face); ans->face = NULL;
+        }
+        error = FT_New_Face(library, path, ++index, &ans->face);
+        if (error) ans->face = NULL;
+    } while(index < num_faces);
+    PyErr_Format(PyExc_ValueError, "No face matching the postscript name: %s found in: %s", psname, path);
+    return false;
+}
+
+PyObject*
+ft_face_from_path_and_psname(PyObject* path, const char* psname, void *extra_data, free_extra_data_func fed, int hinting, int hintstyle, float apple_leading) {
+    if (PyUnicode_READY(path) != 0) return NULL;
+    Face *ans = (Face*)Face_Type.tp_alloc(&Face_Type, 0);
+    if (!ans) return NULL;
+    if (!load_from_path_and_psname(PyUnicode_AsUTF8(path), psname, ans)) { Py_CLEAR(ans); return NULL; }
+    if (!init_ft_face(ans, path, hinting, hintstyle)) { Py_CLEAR(ans); return NULL; }
+    ans->extra_data = extra_data;
+    ans->free_extra_data = fed;
+    ans->apple_leading = apple_leading;
+    return (PyObject*)ans;
+}
+
+#ifndef __APPLE__
+PyObject*
+face_from_descriptor(PyObject *descriptor) {
+#define D(key, conv) { PyObject *t = PyDict_GetItemString(descriptor, #key); if (t == NULL) return NULL; key = conv(t); t = NULL; }
+    char *path;
+    long index;
+    bool hinting;
+    long hint_style;
+    D(path, PyUnicode_AsUTF8);
+    D(index, PyLong_AsLong);
+    D(hinting, PyObject_IsTrue);
+    D(hint_style, PyLong_AsLong);
+#undef D
+    Face *self = (Face *)Face_Type.tp_alloc(&Face_Type, 0);
+    if (self != NULL) {
+        int error = FT_New_Face(library, path, index, &(self->face));
+        if(error) { set_freetype_error("Failed to load face, with error:", error); Py_CLEAR(self); return NULL; }
+        if (!init_ft_face(self, PyDict_GetItemString(descriptor, "path"), hinting, hint_style)) { Py_CLEAR(self); return NULL; }
     }
     return (PyObject*)self;
 }
+#endif
  
+PyObject*
+face_from_path(const char *path, int index) {
+    Face *ans = (Face*)Face_Type.tp_alloc(&Face_Type, 0);
+    if (ans == NULL) return NULL;
+    int error;
+    error = FT_New_Face(library, path, index, &ans->face);
+    if (error) { set_freetype_error("Failed to load face, with error:", error); ans->face = NULL; return NULL; }
+    if (!init_ft_face(ans, Py_None, true, 3)) { Py_CLEAR(ans); return NULL; }
+    return (PyObject*)ans;
+}
+
 static void
 dealloc(Face* self) {
-    FT_Done_Face(self->face);
+    if (self->harfbuzz_font) hb_font_destroy(self->harfbuzz_font);
+    if (self->face) FT_Done_Face(self->face);
+    if (self->extra_data && self->free_extra_data) self->free_extra_data(self->extra_data);
     Py_CLEAR(self->path);
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
@@ -87,151 +257,192 @@ dealloc(Face* self) {
 static PyObject *
 repr(Face *self) {
     return PyUnicode_FromFormat(
-        "Face(path=%S, is_scalable=%S, units_per_EM=%u, ascender=%i, descender=%i, height=%i, max_advance_width=%i max_advance_height=%i, underline_position=%i, underline_thickness=%i)",
-        self->path, self->is_scalable ? Py_True : Py_False, 
+        "Face(family=%s, style=%s, ps_name=%s, path=%S, index=%d, is_scalable=%S, units_per_EM=%u, ascender=%i, descender=%i, height=%i, underline_position=%i, underline_thickness=%i)",
+        self->face->family_name ? self->face->family_name : "", self->face->style_name ? self->face->style_name : "",
+        FT_Get_Postscript_Name(self->face),
+        self->path, self->index, self->is_scalable ? Py_True : Py_False, 
         self->ascender, self->descender, self->height, self->max_advance_width, self->max_advance_height, self->underline_position, self->underline_thickness
     );
 }
 
 
-static PyObject*
-set_char_size(Face *self, PyObject *args) {
-#define set_char_size_doc "set_char_size(width, height, xdpi, ydpi) -> set the character size. width, height is in 1/64th of a pt. dpi is in pixels per inch"
-    long char_width, char_height;
-    unsigned int xdpi, ydpi;
-    int error;
-    if (!PyArg_ParseTuple(args, "llII", &char_width, &char_height, &xdpi, &ydpi)) return NULL;
-    error = FT_Set_Char_Size(self->face, char_width, char_height, xdpi, ydpi);
-    if (error) { set_freetype_error("Failed to set char size, with error:", error); Py_CLEAR(self); return NULL; }
-    Py_RETURN_NONE;
+static inline bool 
+load_glyph(Face *self, int glyph_index, int load_type) {
+    int flags = get_load_flags(self->hinting, self->hintstyle, load_type);
+    int error = FT_Load_Glyph(self->face, glyph_index, flags);
+    if (error) { set_freetype_error("Failed to load glyph, with error:", error); return false; }
+    return true;
 }
 
-static PyObject*
-load_char(Face *self, PyObject *args) {
-#define load_char_doc "load_char(char, hinting, hintstyle)"
-    int char_code, hinting, hintstyle, error;
-    if (!PyArg_ParseTuple(args, "Cpp", &char_code, &hinting, &hintstyle)) return NULL;
-
-    int glyph_index = FT_Get_Char_Index(self->face, char_code);
-    int flags = FT_LOAD_RENDER;
-    if (hinting) {
-        if (hintstyle >= 3) flags |= FT_LOAD_TARGET_NORMAL;
-        else if (0 < hintstyle  && hintstyle < 3) flags |= FT_LOAD_TARGET_LIGHT;
-    } else flags |= FT_LOAD_NO_HINTING;
-    error = FT_Load_Glyph(self->face, glyph_index, flags);
-    if (error) { set_freetype_error("Failed to load glyph, with error:", error); Py_CLEAR(self); return NULL; }
-    Py_RETURN_NONE;
-}
-
-static PyObject*
-get_char_index(Face *self, PyObject *args) {
-#define get_char_index_doc ""
-    int code;
-    unsigned int ans;
-    if (!PyArg_ParseTuple(args, "C", &code)) return NULL;
-    ans = FT_Get_Char_Index(self->face, code);
-
-    return Py_BuildValue("I", ans);
-}
- 
-static PyStructSequence_Field gm_fields[] = {
-    {"width", NULL},
-    {"height", NULL},
-    {"horiBearingX", NULL},
-    {"horiBearingY", NULL},
-    {"horiAdvance", NULL},
-    {"vertBearingX", NULL},
-    {"vertBearingY", NULL},
-    {"vertAdvance", NULL},
-    {NULL}
-};
-
-static PyStructSequence_Desc gm_desc = {"GlpyhMetrics", NULL, gm_fields, 8};
-static PyTypeObject GlpyhMetricsType = {{{0}}};
-
-static PyObject*
-glyph_metrics(Face *self) {
-#define glyph_metrics_doc ""
-    PyObject *ans = PyStructSequence_New(&GlpyhMetricsType);
-    if (ans != NULL) {
-#define SI(num, attr) PyStructSequence_SET_ITEM(ans, num, PyLong_FromLong(self->face->glyph->metrics.attr)); if (PyStructSequence_GET_ITEM(ans, num) == NULL) { Py_CLEAR(ans); return PyErr_NoMemory(); }
-        SI(0, width); SI(1, height);
-        SI(2, horiBearingX); SI(3, horiBearingY); SI(4, horiAdvance);
-        SI(5, vertBearingX); SI(6, vertBearingY); SI(7, vertAdvance);
-#undef SI
-    } else return PyErr_NoMemory();
-    return ans;
-}
-
-static PyStructSequence_Field bm_fields[] = {
-    {"rows", NULL},
-    {"width", NULL},
-    {"pitch", NULL},
-    {"buffer", NULL},
-    {"num_grays", NULL},
-    {"pixel_mode", NULL},
-    {"palette_mode", NULL},
-    {NULL}
-};
-static PyStructSequence_Desc bm_desc = {"Bitmap", NULL, bm_fields, 7};
-static PyTypeObject BitmapType = {{{0}}};
-
-static PyObject*
-bitmap(Face *self) {
-#define bitmap_doc ""
-    PyObject *ans = PyStructSequence_New(&BitmapType);
-    if (ans != NULL) {
-#define SI(num, attr, func, conv) PyStructSequence_SET_ITEM(ans, num, func((conv)self->face->glyph->bitmap.attr)); if (PyStructSequence_GET_ITEM(ans, num) == NULL) { Py_CLEAR(ans); return PyErr_NoMemory(); }
-        SI(0, rows, PyLong_FromUnsignedLong, unsigned long); SI(1, width, PyLong_FromUnsignedLong, unsigned long);
-        SI(2, pitch, PyLong_FromLong, long); 
-        PyObject *t = PyByteArray_FromStringAndSize((const char*)self->face->glyph->bitmap.buffer, self->face->glyph->bitmap.rows * self->face->glyph->bitmap.pitch);
-        if (t == NULL) { Py_CLEAR(ans); return PyErr_NoMemory(); }
-        PyStructSequence_SET_ITEM(ans, 3, t);
-        SI(4, num_grays, PyLong_FromUnsignedLong, unsigned long);
-        SI(5, pixel_mode, PyLong_FromUnsignedLong, unsigned long); SI(6, palette_mode, PyLong_FromUnsignedLong, unsigned long);
-#undef SI
-    } else return PyErr_NoMemory();
-    return ans;
-}
- 
-static PyObject*
-trim_to_width(Face UNUSED *self, PyObject *args) {
-#define trim_to_width_doc "Trim edges from the supplied bitmap to make it fit in the specified cell-width"
-    PyObject *bitmap, *t;
-    unsigned long cell_width, rows, width, rtrim = 0, extra, ltrim;
-    unsigned char *src, *dest;
-    bool column_has_text = false;
-    if (!PyArg_ParseTuple(args, "O!k", &BitmapType, &bitmap, &cell_width)) return NULL;
-    rows = PyLong_AsUnsignedLong(PyStructSequence_GET_ITEM(bitmap, 0));
-    width = PyLong_AsUnsignedLong(PyStructSequence_GET_ITEM(bitmap, 1));
-    extra = width - cell_width;
-    if (extra >= cell_width) { PyErr_SetString(FreeType_Exception, "Too large for trimming"); return NULL; }
-    PyObject *ans = PyStructSequence_New(&BitmapType);
-    if (ans == NULL) return PyErr_NoMemory();
-    src = (unsigned char*)PyByteArray_AS_STRING(PyStructSequence_GET_ITEM(bitmap, 3));
-    PyObject *abuf = PyByteArray_FromStringAndSize(NULL, cell_width * rows);
-    if (abuf == NULL) { Py_CLEAR(ans); return PyErr_NoMemory(); }
-    dest = (unsigned char*)PyByteArray_AS_STRING(abuf);
-    PyStructSequence_SET_ITEM(ans, 1, PyLong_FromUnsignedLong(cell_width));
-    PyStructSequence_SET_ITEM(ans, 2, PyLong_FromUnsignedLong(cell_width));
-    PyStructSequence_SET_ITEM(ans, 3, abuf);
-#define COPY(which) t = PyStructSequence_GET_ITEM(bitmap, which); Py_INCREF(t); PyStructSequence_SET_ITEM(ans, which, t);
-    COPY(0); COPY(4); COPY(5); COPY(6);
-#undef COPY
-
-    for (long x = width - 1; !column_has_text && x > -1 && rtrim < extra; x--) {
-        for (unsigned long y = 0; y < rows * width; y += width) {
-            if (src[x + y] > 200) { column_has_text = true; break; }
+static inline unsigned int
+calc_cell_width(Face *self) {
+    unsigned int ans = 0;
+    for (char_type i = 32; i < 128; i++) {
+        int glyph_index = FT_Get_Char_Index(self->face, i);
+        if (load_glyph(self, glyph_index, FT_LOAD_DEFAULT)) {
+            ans = MAX(ans, (unsigned long)ceilf((float)self->face->glyph->metrics.horiAdvance / 64.f));
         }
-        if (!column_has_text) rtrim++;
     }
-    rtrim = MIN(extra, rtrim);
-    ltrim = extra - rtrim;
-    for (unsigned long y = 0; y < rows; y++) {
-        memcpy(dest + y*cell_width, src + ltrim + y*width, cell_width);
+    return ans;
+}
+
+void 
+cell_metrics(PyObject *s, unsigned int* cell_width, unsigned int* cell_height, unsigned int* baseline, unsigned int* underline_position, unsigned int* underline_thickness) {
+    Face *self = (Face*)s;
+    *cell_width = calc_cell_width(self);
+    *cell_height = CALC_CELL_HEIGHT(self);
+#ifdef __APPLE__
+    // See https://stackoverflow.com/questions/5511830/how-does-line-spacing-work-in-core-text-and-why-is-it-different-from-nslayoutm
+    if (self->apple_leading <= 0) {
+        *cell_height += floor(0.2 * (double)(*cell_height) + 0.5);
+    }
+#endif
+    *baseline = font_units_to_pixels(self, self->ascender);
+    *underline_position = MIN(*cell_height - 1, (unsigned int)font_units_to_pixels(self, MAX(0, self->ascender - self->underline_position)));
+    *underline_thickness = MAX(1, font_units_to_pixels(self, self->underline_thickness));
+}
+
+unsigned int 
+glyph_id_for_codepoint(PyObject *s, char_type cp) {
+    return FT_Get_Char_Index(((Face*)s)->face, cp);
+}
+
+hb_font_t*
+harfbuzz_font_for_face(PyObject *self) { return ((Face*)self)->harfbuzz_font; }
+
+
+typedef struct {
+    unsigned char* buf;
+    size_t start_x, width, stride;
+    size_t rows;
+} ProcessedBitmap;
+
+
+static inline void
+trim_borders(ProcessedBitmap *ans, size_t extra) {
+    bool column_has_text = false;
+
+    // Trim empty columns from the right side of the bitmap
+    for (ssize_t x = ans->width - 1; !column_has_text && x > -1 && extra > 0; x--) {
+        for (size_t y = 0; y < ans->rows && !column_has_text; y++) {
+            if (ans->buf[x + y * ans->stride] > 200) column_has_text = true;
+        }
+        if (!column_has_text) { ans->width--; extra--; }
     }
 
-    return ans;
+    ans->start_x = extra;
+    ans->width -= extra;
+}
+
+
+
+static inline bool
+render_bitmap(Face *self, int glyph_id, ProcessedBitmap *ans, unsigned int cell_width, unsigned int num_cells, bool bold, bool italic, bool rescale) {
+    if (!load_glyph(self, glyph_id, FT_LOAD_RENDER)) return false;
+    unsigned int max_width = cell_width * num_cells;
+    FT_Bitmap *bitmap = &self->face->glyph->bitmap;
+    ans->buf = bitmap->buffer;
+    ans->start_x = 0; ans->width = bitmap->width;
+    ans->stride = bitmap->pitch < 0 ? -bitmap->pitch : bitmap->pitch;
+    ans->rows = bitmap->rows;
+    if (ans->width > max_width) {
+        size_t extra = bitmap->width - max_width;
+        if (italic && extra < cell_width / 2) {
+            trim_borders(ans, extra);
+        } else if (rescale && self->is_scalable && extra > MAX(2, cell_width / 3)) {
+            FT_F26Dot6 char_width = self->char_width, char_height = self->char_height;
+            float ar = (float)max_width / (float)bitmap->width;
+            if (set_font_size(self, (FT_F26Dot6)((float)self->char_width * ar), (FT_F26Dot6)((float)self->char_height * ar), self->xdpi, self->ydpi, 0)) {
+                if (!render_bitmap(self, glyph_id, ans, cell_width, num_cells, bold, italic, false)) return false;
+                if (!set_font_size(self, char_width, char_height, self->xdpi, self->ydpi, 0)) return false;
+            } else return false;
+        }
+    }
+    return true;
+}
+
+static inline void
+place_bitmap_in_canvas(unsigned char *cell, ProcessedBitmap *bm, size_t cell_width, size_t cell_height, float x_offset, float y_offset, FT_Glyph_Metrics *metrics, size_t baseline) {
+    // We want the glyph to be positioned inside the cell based on the bearingX
+    // and bearingY values, making sure that it does not overflow the cell.
+
+    // Calculate column bounds
+    float bearing_x = (float)metrics->horiBearingX / 64.f;
+    ssize_t xoff = (ssize_t)(x_offset + bearing_x);
+    size_t src_start_column = bm->start_x, dest_start_column = 0, extra;
+    if (xoff < 0) src_start_column += -xoff;
+    else dest_start_column = xoff;
+    // Move the dest start column back if the width overflows because of it
+    if (dest_start_column > 0 && dest_start_column + bm->width > cell_width) {
+        extra = dest_start_column + bm->width - cell_width;
+        dest_start_column = extra > dest_start_column ? 0 : dest_start_column - extra;
+    }
+
+    // Calculate row bounds
+    float bearing_y = (float)metrics->horiBearingY / 64.f;
+    ssize_t yoff = (ssize_t)(y_offset + bearing_y);
+    size_t src_start_row, dest_start_row;
+    if (yoff > 0 && (size_t)yoff > baseline) {
+        src_start_row = 0;
+        dest_start_row = 0;
+    } else {
+        src_start_row = 0;
+        dest_start_row = baseline - yoff;
+    }
+
+    /* printf("x_offset: %f bearing_x: %f y_offset: %f bearing_y: %f src_start_row: %zu src_start_column: %zu dest_start_row: %zu dest_start_column: %zu bm_width: %lu bitmap_rows: %lu\n", x_offset, bearing_x, y_offset, bearing_y, src_start_row, src_start_column, dest_start_row, dest_start_column, bm->width, bm->rows); */
+
+    for (size_t sr = src_start_row, dr = dest_start_row; sr < bm->rows && dr < cell_height; sr++, dr++) {
+        for(size_t sc = src_start_column, dc = dest_start_column; sc < bm->width && dc < cell_width; sc++, dc++) {
+            uint16_t val = cell[dr * cell_width + dc];
+            val = MIN(255, ((uint16_t)val + (uint16_t)bm->buf[sr * bm->stride + sc]));
+            cell[dr * cell_width + dc] = val;
+        }
+    }
+}
+
+static inline void
+right_shift_canvas(uint8_t *canvas, size_t width, size_t height, size_t amt) {
+    uint8_t *src;
+    size_t r;
+    for (r = 0, src = canvas; r < height; r++, src += width) {
+        memmove(src + amt, src, width - amt);
+        memset(src, 0, amt);
+    }
+}
+
+bool
+render_glyphs_in_cells(PyObject *f, bool bold, bool italic, hb_glyph_info_t *info, hb_glyph_position_t *positions, unsigned int num_glyphs, uint8_t *canvas, unsigned int cell_width, unsigned int cell_height, unsigned int num_cells, unsigned int baseline) {
+    Face *self = (Face*)f;
+    float x = 0.f, y = 0.f, x_offset = 0.f;
+    ProcessedBitmap bm;
+    unsigned int canvas_width = cell_width * num_cells;
+    for (unsigned int i = 0; i < num_glyphs; i++) {
+        if (!render_bitmap(self, info[i].codepoint, &bm, cell_width, num_cells, bold, italic, true)) return false;
+        x_offset = x + (float)positions[i].x_offset / 64.0f;
+        y = (float)positions[i].y_offset / 64.0f;
+        if (self->face->glyph->metrics.width > 0 && bm.width > 0) place_bitmap_in_canvas(canvas, &bm, canvas_width, cell_height, x_offset, y, &self->face->glyph->metrics, baseline);
+        x += (float)positions[i].x_advance / 64.0f;
+    }
+    unsigned int right_edge = (unsigned int)x, delta;
+    if (num_cells > 1 && right_edge < canvas_width && (delta = (canvas_width - right_edge) / 2) && delta > 1) {
+        // center the glyphs in the canvas
+        right_shift_canvas(canvas, canvas_width, cell_height, delta);
+    }
+    return true;
+}
+
+static PyObject*
+display_name(Face *self) {
+    const char *psname = FT_Get_Postscript_Name(self->face);
+    if (psname) return Py_BuildValue("s", psname);
+    Py_INCREF(self->path);
+    return self->path;
+}
+
+static PyObject*
+extra_data(Face *self) {
+    return PyLong_FromVoidPtr(self->extra_data);
 }
 
 // Boilerplate {{{
@@ -252,12 +463,8 @@ static PyMemberDef members[] = {
 };
 
 static PyMethodDef methods[] = {
-    METHOD(set_char_size, METH_VARARGS)
-    METHOD(load_char, METH_VARARGS)
-    METHOD(get_char_index, METH_VARARGS)
-    METHOD(glyph_metrics, METH_NOARGS)
-    METHOD(bitmap, METH_NOARGS)
-    METHOD(trim_to_width, METH_VARARGS)
+    METHODB(display_name, METH_NOARGS),
+    METHODB(extra_data, METH_NOARGS),
     {NULL}  /* Sentinel */
 };
 
@@ -271,12 +478,9 @@ PyTypeObject Face_Type = {
     .tp_doc = "FreeType Font face",
     .tp_methods = methods,
     .tp_members = members,
-    .tp_new = new,                
     .tp_repr = (reprfunc)repr,
 };
 
-INIT_TYPE(Face)
-
 static void
 free_freetype() {
     FT_Done_FreeType(library);
@@ -284,6 +488,9 @@ free_freetype() {
 
 bool 
 init_freetype_library(PyObject *m) {
+    if (PyType_Ready(&Face_Type) < 0) return 0;
+    if (PyModule_AddObject(m, "Face", (PyObject *)&Face_Type) != 0) return 0; 
+    Py_INCREF(&Face_Type); 
     FreeType_Exception = PyErr_NewException("fast_data_types.FreeTypeError", NULL, NULL);
     if (FreeType_Exception == NULL) return false;
     if (PyModule_AddObject(m, "FreeTypeError", FreeType_Exception) != 0) return false;
@@ -296,15 +503,6 @@ init_freetype_library(PyObject *m) {
         PyErr_SetString(FreeType_Exception, "Failed to register the freetype library at exit handler");
         return false;
     }
-    if (PyStructSequence_InitType2(&GlpyhMetricsType, &gm_desc) != 0) return false;
-    if (PyStructSequence_InitType2(&BitmapType, &bm_desc) != 0) return false;
-    PyModule_AddObject(m, "GlyphMetrics", (PyObject*)&GlpyhMetricsType);
-    PyModule_AddObject(m, "Bitmap", (PyObject*)&BitmapType);
-    PyModule_AddIntMacro(m, FT_LOAD_RENDER);
-    PyModule_AddIntMacro(m, FT_LOAD_TARGET_NORMAL);
-    PyModule_AddIntMacro(m, FT_LOAD_TARGET_LIGHT);
-    PyModule_AddIntMacro(m, FT_LOAD_NO_HINTING);
-    PyModule_AddIntMacro(m, FT_PIXEL_MODE_GRAY);
     return true;
 }
 

kitty/gl.h

@@ -6,17 +6,12 @@
 
 #pragma once
 
+#include "gl-wrapper.h"
 #include "state.h"
 #include "screen.h"
-#include "sprites.h"
-#ifdef __APPLE__
-#include <OpenGL/gl3.h>
-#include <OpenGL/gl3ext.h>
-#else
-#include <GL/glew.h>
-#endif
 #include <string.h>
 #include <stddef.h>
+#include <GLFW/glfw3.h>
 
 static char glbuf[4096];
 
@@ -26,19 +21,10 @@ static char glbuf[4096];
 #define REQUIRED_VERSION_MINOR 3
 #define GLSL_VERSION (REQUIRED_VERSION_MAJOR * 100 + REQUIRED_VERSION_MINOR * 10)
 
-#ifndef GL_STACK_UNDERFLOW
-#define GL_STACK_UNDERFLOW 0x0504
-#endif
-
-#ifndef GL_STACK_OVERFLOW
-#define GL_STACK_OVERFLOW 0x0503
-#endif
-
-#ifdef ENABLE_DEBUG_GL
 static void
-check_for_gl_error(int line) {
-#define f(msg) fatal("%s (at line: %d)", msg, line); break;
-    int code = glGetError();
+check_for_gl_error(const char *name, void UNUSED *funcptr, int UNUSED len_args, ...) {
+#define f(msg) fatal("OpenGL error: %s (calling function: %s)", msg, name); break;
+    GLenum code = glad_glGetError();
     switch(code) { 
         case GL_NO_ERROR: break;
         case GL_INVALID_ENUM: 
@@ -56,59 +42,40 @@ check_for_gl_error(int line) {
         case GL_STACK_OVERFLOW: 
             f("An attempt has been made to perform an operation that would cause an internal stack to underflow. (GL_STACK_OVERFLOW)"); 
         default: 
-            fatal("An unknown OpenGL error occurred with code: %d (at line: %d)", code, line); 
+            fatal("An unknown OpenGL error occurred with code: %d (calling function: %s)", code, name); 
             break;
     }
 }
 
-#define check_gl() { check_for_gl_error(__LINE__); }
-#else
-#define check_gl() {}
-#endif
-
 static PyObject* 
-glew_init(PyObject UNUSED *self) {
-#ifndef __APPLE__
-    GLenum err = glewInit();
-    if (err != GLEW_OK) {
-        PyErr_Format(PyExc_RuntimeError, "GLEW init failed: [%d] %s", err, glewGetErrorString(err));
-        return NULL;
+gl_init(PyObject UNUSED *self, PyObject *args) {
+    int is_wayland, debug;
+    if (!PyArg_ParseTuple(args, "pp", &is_wayland, &debug)) return NULL;
+    if (!init_glad((GLADloadproc) glfwGetProcAddress, debug)) {
+        fatal("Loading the OpenGL library failed");
     }
+    glad_set_post_callback(check_for_gl_error);
 #define ARB_TEST(name) \
-    if (!GLEW_ARB_##name) { \
-        PyErr_Format(PyExc_RuntimeError, "The OpenGL driver on this system is missing the required extension: ARB_%s", #name); \
-        return NULL; \
+    if (!GLAD_GL_ARB_##name) { \
+        fatal("The OpenGL driver on this system is missing the required extension: ARB_%s", #name); \
     }
     ARB_TEST(texture_storage);
 #undef ARB_TEST
-#endif
     glEnable(GL_BLEND);
     Py_RETURN_NONE;
 }
 
-static void
-update_viewport_size_impl(int w, int h) {
-    glViewport(0, 0, w, h); check_gl();
+void
+update_viewport_size(int w, int h) {
+    glViewport(0, 0, w, h); 
 }
 
-static void
-free_texture_impl(GLuint *tex_id) {
-    glDeleteTextures(1, tex_id); check_gl();
+void
+free_texture(GLuint *tex_id) {
+    glDeleteTextures(1, tex_id); 
     *tex_id = 0;
 }
 
-static void
-send_image_to_gpu_impl(GLuint *tex_id, const void* data, GLsizei width, GLsizei height, bool is_opaque, bool is_4byte_aligned) {
-    if (!(*tex_id)) { glGenTextures(1, tex_id); check_gl(); }
-    glBindTexture(GL_TEXTURE_2D, *tex_id); check_gl();
-    glPixelStorei(GL_UNPACK_ALIGNMENT, is_4byte_aligned ? 4 : 1); check_gl();
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); check_gl();
-    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, is_opaque ? GL_RGB : GL_RGBA, GL_UNSIGNED_BYTE, data); check_gl(); 
-}
-
 
 // }}}
 
@@ -139,11 +106,8 @@ static Program programs[64] = {{0}};
 static inline GLuint
 compile_shader(GLenum shader_type, const char *source) {
     GLuint shader_id = glCreateShader(shader_type);
-    check_gl();
     glShaderSource(shader_id, 1, (const GLchar **)&source, NULL);
-    check_gl();
     glCompileShader(shader_id);
-    check_gl();
     GLint ret = GL_FALSE;
     glGetShaderiv(shader_id, GL_COMPILE_STATUS, &ret);
     if (ret != GL_TRUE) {
@@ -164,11 +128,9 @@ static inline void
 init_uniforms(int program) {
     Program *p = programs + program;
     glGetProgramiv(p->id, GL_ACTIVE_UNIFORMS, &(p->num_of_uniforms));
-    check_gl();
     for (GLint i = 0; i < p->num_of_uniforms; i++) {
         Uniform *u = p->uniforms + i;
         glGetActiveUniform(p->id, (GLuint)i, sizeof(u->name)/sizeof(u->name[0]), NULL, &(u->size), &(u->type), u->name);
-        check_gl();
         u->location = glGetUniformLocation(p->id, u->name);
         u->idx = i;
     }
@@ -188,14 +150,12 @@ get_uniform_information(int program, const char *name, GLenum information_type)
 static inline GLint
 attrib_location(int program, const char *name) {
     GLint ans = glGetAttribLocation(programs[program].id, name);
-    check_gl();
     return ans;
 }
 
 static inline GLuint
 block_index(int program, const char *name) {
     GLuint ans = glGetUniformBlockIndex(programs[program].id, name);
-    check_gl();
     if (ans == GL_INVALID_INDEX) { fatal("Could not find block index"); }
     return ans;
 }
@@ -205,20 +165,17 @@ static inline GLint
 block_size(int program, GLuint block_index) {
     GLint ans;
     glGetActiveUniformBlockiv(programs[program].id, block_index, GL_UNIFORM_BLOCK_DATA_SIZE, &ans);
-    check_gl();
     return ans;
 }
 
 static inline void
 bind_program(int program) {
     glUseProgram(programs[program].id);
-    check_gl();
 }
 
 static inline void
 unbind_program() {
     glUseProgram(0);
-    check_gl();
 }
 // }}}
 
@@ -237,7 +194,6 @@ static ssize_t
 create_buffer(GLenum usage) {
     GLuint buffer_id;
     glGenBuffers(1, &buffer_id);
-    check_gl();
     for (size_t i = 0; i < sizeof(buffers)/sizeof(buffers[0]); i++) {
         if (buffers[i].id == 0) {
             buffers[i].id = buffer_id;
@@ -254,7 +210,6 @@ create_buffer(GLenum usage) {
 static void
 delete_buffer(ssize_t buf_idx) {
     glDeleteBuffers(1, &(buffers[buf_idx].id));
-    check_gl();
     buffers[buf_idx].id = 0;
     buffers[buf_idx].size = 0;
 }
@@ -262,14 +217,12 @@ delete_buffer(ssize_t buf_idx) {
 static GLuint
 bind_buffer(ssize_t buf_idx) {
     glBindBuffer(buffers[buf_idx].usage, buffers[buf_idx].id);
-    check_gl();
     return buffers[buf_idx].id;
 }
 
 static void
 unbind_buffer(ssize_t buf_idx) {
     glBindBuffer(buffers[buf_idx].usage, 0);
-    check_gl();
 }
 
 static inline void
@@ -278,20 +231,17 @@ alloc_buffer(ssize_t idx, GLsizeiptr size, GLenum usage) {
     if (b->size == size) return;
     b->size = size;
     glBufferData(b->usage, size, NULL, usage);
-    check_gl();
 }
 
 static inline void*
 map_buffer(ssize_t idx, GLenum access) {
     void *ans = glMapBuffer(buffers[idx].usage, access);
-    check_gl();
     return ans;
 }
 
 static inline void
 unmap_buffer(ssize_t idx) {
     glUnmapBuffer(buffers[idx].usage);
-    check_gl();
 }
 
 // }}}
@@ -310,13 +260,11 @@ static ssize_t
 create_vao() {
     GLuint vao_id;
     glGenVertexArrays(1, &vao_id);
-    check_gl();
     for (size_t i = 0; i < sizeof(vaos)/sizeof(vaos[0]); i++) {
         if (!vaos[i].id) {
             vaos[i].id = vao_id;
             vaos[i].num_buffers = 0;
             glBindVertexArray(vao_id);
-            check_gl();
             return i;
         }
     }
@@ -343,7 +291,6 @@ add_located_attribute_to_vao(ssize_t vao_idx, GLint aloc, GLint size, GLenum dat
     ssize_t buf = vao->buffers[vao->num_buffers - 1];
     bind_buffer(buf);
     glEnableVertexAttribArray(aloc);
-    check_gl();
     switch(data_type) {
         case GL_BYTE:
         case GL_UNSIGNED_BYTE:
@@ -357,10 +304,8 @@ add_located_attribute_to_vao(ssize_t vao_idx, GLint aloc, GLint size, GLenum dat
             glVertexAttribPointer(aloc, size, data_type, GL_FALSE, stride, offset);
             break;
     }
-    check_gl();
     if (divisor) {
         glVertexAttribDivisor(aloc, divisor);
-        check_gl();
     }
     unbind_buffer(buf);
 }
@@ -381,20 +326,17 @@ remove_vao(ssize_t vao_idx) {
         delete_buffer(vao->buffers[vao->num_buffers]);
     }
     glDeleteVertexArrays(1, &(vao->id));
-    check_gl();
     vaos[vao_idx].id = 0;
 }
 
 static void
 bind_vertex_array(ssize_t vao_idx) {
     glBindVertexArray(vaos[vao_idx].id);
-    check_gl();
 }
 
 static void
 unbind_vertex_array() {
     glBindVertexArray(0);
-    check_gl();
 }
 
 static ssize_t
@@ -422,7 +364,6 @@ static void
 bind_vao_uniform_buffer(ssize_t vao_idx, size_t bufnum, GLuint block_index) {
     ssize_t buf_idx = vaos[vao_idx].buffers[bufnum];
     glBindBufferBase(GL_UNIFORM_BUFFER, block_index, buffers[buf_idx].id);
-    check_gl();
 }
 
 static void

kitty/glfw.c

@@ -19,6 +19,7 @@
 #if GLFW_VERSION_MAJOR > 3 || (GLFW_VERSION_MAJOR == 3 && GLFW_VERSION_MINOR > 2)
 #define has_request_attention
 #define has_init_hint_string
+#define has_content_scale_query
 #endif
 
 #if GLFW_KEY_LAST >= MAX_KEY_COUNT
@@ -35,12 +36,19 @@
 #define WINDOW_CALLBACK(name, fmt, ...) \
     CALLBACK(the_window->name, "O" fmt, the_window, __VA_ARGS__);
 
+typedef struct {
+    int x, y, w, h;
+    bool is_set;
+} GLFWWindowGeometry;
+
+static GLFWcursor *standard_cursor = NULL, *click_cursor = NULL, *arrow_cursor = NULL;
+
 typedef struct {
     PyObject_HEAD
 
     GLFWwindow *window;
     PyObject *framebuffer_size_callback, *window_focus_callback;
-    GLFWcursor *standard_cursor, *click_cursor, *arrow_cursor;
+    GLFWWindowGeometry before_fullscreen;
 } WindowWrapper;
 
 static void update_viewport(GLFWwindow *window) {
@@ -53,7 +61,6 @@ static void update_viewport(GLFWwindow *window) {
 
 // callbacks {{{
 static WindowWrapper* the_window = NULL;
-update_viewport_size_func update_viewport_size = NULL;
 
 static void 
 framebuffer_size_callback(GLFWwindow *w, int width, int height) {
@@ -124,13 +131,13 @@ void
 set_mouse_cursor(MouseShape type) {
     switch(type) {
         case HAND:
-            glfwSetCursor(the_window->window, the_window->click_cursor);
+            glfwSetCursor(the_window->window, click_cursor);
             break;
         case ARROW:
-            glfwSetCursor(the_window->window, the_window->arrow_cursor);
+            glfwSetCursor(the_window->window, arrow_cursor);
             break;
         default:
-            glfwSetCursor(the_window->window, the_window->standard_cursor);
+            glfwSetCursor(the_window->window, standard_cursor);
             break;
     }
 }
@@ -149,11 +156,13 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
         self->window = glfwCreateWindow(width, height, title, NULL, NULL);
         if (self->window == NULL) { Py_CLEAR(self); the_window = NULL; PyErr_SetString(PyExc_ValueError, "Failed to create GLFWwindow"); return NULL; }
         update_viewport(self->window);
-        self->standard_cursor = glfwCreateStandardCursor(GLFW_IBEAM_CURSOR);
-        self->click_cursor = glfwCreateStandardCursor(GLFW_HAND_CURSOR);
-        self->arrow_cursor = glfwCreateStandardCursor(GLFW_ARROW_CURSOR);
-        if (self->standard_cursor == NULL || self->click_cursor == NULL || self->arrow_cursor == NULL) {
-            Py_CLEAR(self); PyErr_SetString(PyExc_ValueError, "Failed to create standard mouse cursors"); return NULL; }
+        if (standard_cursor == NULL) {
+            standard_cursor = glfwCreateStandardCursor(GLFW_IBEAM_CURSOR);
+            click_cursor = glfwCreateStandardCursor(GLFW_HAND_CURSOR);
+            arrow_cursor = glfwCreateStandardCursor(GLFW_ARROW_CURSOR);
+            if (standard_cursor == NULL || click_cursor == NULL || arrow_cursor == NULL) {
+                Py_CLEAR(self); PyErr_SetString(PyExc_ValueError, "Failed to create standard mouse cursors"); return NULL; }
+        }
         set_mouse_cursor(0);
         glfwSetFramebufferSizeCallback(self->window, framebuffer_size_callback);
         glfwSetCharModsCallback(self->window, char_mods_callback);
@@ -317,6 +326,18 @@ get_cursor_pos(WindowWrapper *self) {
     return Py_BuildValue("dd", x, y);
 }
 
+static PyObject*
+get_content_scale(WindowWrapper UNUSED *self) {
+#ifdef has_content_scale_query
+    float xscale, yscale;
+    glfwGetWindowContentScale(self->window, &xscale, &yscale);
+    return Py_BuildValue("ff", xscale, yscale);
+#else
+    PyErr_SetString(PyExc_NotImplementedError, "glfw version is too old");
+    return NULL;
+#endif
+}
+
 static PyObject*
 set_should_close(WindowWrapper *self, PyObject *args) {
     int c;
@@ -416,6 +437,28 @@ current_monitor_dpi(WindowWrapper *self) {
     return get_physical_dpi(m);
 }
 
+static PyObject*
+toggle_fullscreen(WindowWrapper *self) {
+    GLFWmonitor *monitor;
+    if ((monitor = glfwGetWindowMonitor(self->window)) == NULL) {
+        // make fullscreen
+        monitor = current_monitor(self->window);
+        if (monitor == NULL) return NULL;
+        const GLFWvidmode* mode = glfwGetVideoMode(monitor);
+        self->before_fullscreen.is_set = true;
+        glfwGetWindowSize(self->window, &self->before_fullscreen.w, &self->before_fullscreen.h);
+        glfwGetWindowPos(self->window, &self->before_fullscreen.x, &self->before_fullscreen.y);
+        glfwSetWindowMonitor(self->window, monitor, 0, 0, mode->width, mode->height, mode->refreshRate);
+        Py_RETURN_TRUE;
+    } else {
+        // make windowed
+        const GLFWvidmode* mode = glfwGetVideoMode(monitor);
+        if (self->before_fullscreen.is_set) glfwSetWindowMonitor(self->window, NULL, self->before_fullscreen.x, self->before_fullscreen.y, self->before_fullscreen.w, self->before_fullscreen.h, mode->refreshRate);
+        else glfwSetWindowMonitor(self->window, NULL, 0, 0, 600, 400, mode->refreshRate);
+        Py_RETURN_FALSE;
+    }
+}
+
 void
 request_window_attention() {
 #ifdef has_request_attention
@@ -440,6 +483,19 @@ primary_monitor_size(PyObject UNUSED *self) {
     return Py_BuildValue("ii", mode->width, mode->height);
 }
 
+static PyObject*
+primary_monitor_content_scale(PyObject UNUSED *self) {
+#ifdef has_content_scale_query
+    GLFWmonitor* monitor = glfwGetPrimaryMonitor();
+    float xscale, yscale;
+    glfwGetMonitorContentScale(monitor, &xscale, &yscale);
+    return Py_BuildValue("ff", xscale, yscale);
+#else
+    PyErr_SetString(PyExc_NotImplementedError, "glfw version is too old");
+    return NULL;
+#endif
+}
+
 // Boilerplate {{{
 #define MND(name, args) {#name, (PyCFunction)name, args, ""}
 
@@ -447,9 +503,11 @@ static PyMethodDef methods[] = {
     MND(swap_buffers, METH_NOARGS),
     MND(get_clipboard_string, METH_NOARGS),
     MND(get_cursor_pos, METH_NOARGS),
+    MND(get_content_scale, METH_NOARGS),
     MND(should_close, METH_NOARGS),
     MND(get_framebuffer_size, METH_NOARGS),
     MND(get_window_size, METH_NOARGS),
+    MND(toggle_fullscreen, METH_NOARGS),
     MND(current_monitor_dpi, METH_NOARGS),
 #ifdef __APPLE__
     MND(cocoa_window_id, METH_NOARGS),
@@ -497,6 +555,7 @@ static PyMethodDef module_methods[] = {
     {"glfw_get_key_name", (PyCFunction)glfw_get_key_name, METH_VARARGS, ""}, \
     {"glfw_init_hint_string", (PyCFunction)glfw_init_hint_string, METH_VARARGS, ""}, \
     {"glfw_primary_monitor_size", (PyCFunction)primary_monitor_size, METH_NOARGS, ""}, \
+    {"glfw_primary_monitor_content_scale", (PyCFunction)primary_monitor_content_scale, METH_NOARGS, ""}, \
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 

kitty/graphics.c

@@ -17,11 +17,13 @@
 #include <zlib.h>
 #include <png.h>
 #include <structmember.h>
+PyTypeObject GraphicsManager_Type;
 
 #define STORAGE_LIMIT (320 * (1024 * 1024))
 
 #define REPORT_ERROR(...) { fprintf(stderr, __VA_ARGS__); fprintf(stderr, "\n"); }
 
+
 static bool send_to_gpu = true;
 
 GraphicsManager*
@@ -53,9 +55,6 @@ free_load_data(LoadData *ld) {
     ld->mapped_file = NULL; ld->mapped_file_sz = 0;
 }
 
-free_texture_func free_texture = NULL;
-send_image_to_gpu_func send_image_to_gpu = NULL;
-
 static inline void
 free_image(GraphicsManager *self, Image *img) {
     if (img->texture_id) free_texture(&img->texture_id);
@@ -78,18 +77,6 @@ dealloc(GraphicsManager* self) {
 
 static size_t internal_id_counter = 1;
 
-static inline void*
-ensure_space(void *array, size_t *capacity, size_t count, size_t item_size, bool initialize) {
-    if (count < *capacity) return array;
-    void *ans = realloc(array, (*capacity) * item_size * 2);
-    if (ans == NULL) fatal("Out of memory re-allocating array.");
-    if (initialize) {
-        memset(((uint8_t*)array) + ((*capacity) * item_size), 0, ((*capacity) * item_size));
-    }
-    *capacity *= 2;
-    return ans;
-}
-
 static inline void
 remove_from_array(void *array, size_t item_size, size_t idx, size_t array_count) {
     size_t num_to_right = array_count - 1 - idx;
@@ -353,7 +340,7 @@ find_or_create_image(GraphicsManager *self, uint32_t id, bool *existing) {
         }
     }
     *existing = false;
-    self->images = ensure_space(self->images, &self->images_capacity, self->image_count, sizeof(Image), true);
+    ensure_space_for(self, images, Image, self->image_count + 1, images_capacity, 64, true);
     return self->images + self->image_count++;
 }
 
@@ -538,13 +525,6 @@ create_add_response(GraphicsManager UNUSED *self, bool data_loaded, uint32_t iid
 
 // Displaying images {{{
 
-#define ensure_space_for(base, array, type, num, capacity, initial_cap) \
-    if (base->capacity < num) { \
-        base->capacity = MAX(initial_cap, MAX(2 * base->capacity, num)); \
-        base->array = realloc(base->array, sizeof(type) * base->capacity); \
-        if (base->array == NULL) fatal("Out of memory while ensuring space in array"); \
-    }
-
 static inline void
 update_src_rect(ImageRef *ref, Image *img) {
     // The src rect in OpenGL co-ords [0, 1] with origin at top-left corner of image
@@ -578,7 +558,7 @@ handle_put_command(GraphicsManager *self, const GraphicsCommand *g, Cursor *c, b
     if (img == NULL) img = img_by_client_id(self, g->id);
     if (img == NULL) { set_add_response("ENOENT", "Put command refers to non-existent image with id: %u", g->id); return; }
     if (!img->data_loaded) { set_add_response("ENOENT", "Put command refers to image with id: %u that could not load its data", g->id); return; }
-    ensure_space_for(img, refs, ImageRef, img->refcnt + 1, refcap, 10);
+    ensure_space_for(img, refs, ImageRef, img->refcnt + 1, refcap, 16, true);
     *is_dirty = true;
     self->layers_dirty = true;
     ImageRef *ref = NULL;
@@ -641,7 +621,7 @@ grman_update_layers(GraphicsManager *self, unsigned int scrolled_by, float scree
         else r.right = r.left + screen_width * (float)ref->src_width / screen_width_px;
 
         if (ref->z_index < 0) self->num_of_negative_refs++; else self->num_of_positive_refs++;
-        ensure_space_for(self, render_data, ImageRenderData, self->count + 1, capacity, 100);
+        ensure_space_for(self, render_data, ImageRenderData, self->count + 1, capacity, 64, true);
         ImageRenderData *rd = self->render_data + self->count;
 #define R(n, a, b) rd->vertices[n*4] = ref->src_rect.a; rd->vertices[n*4 + 1] = ref->src_rect.b; rd->vertices[n*4 + 2] = r.a; rd->vertices[n*4 + 3] = r.b;
         R(0, right, top); R(1, right, bottom); R(2, left, bottom); R(3, left, top);
@@ -850,8 +830,8 @@ grman_handle_command(GraphicsManager *self, const GraphicsCommand *g, const uint
         case 't':
         case 'T':
         case 'q':
-            iid = g->id;
-            if (g->action == 'q') { q_iid = iid; iid = 0; if (!q_iid) { REPORT_ERROR("Query graphics command without image id"); break; } }
+            iid = g->id; q_iid = iid;
+            if (g->action == 'q') { iid = 0; if (!q_iid) { REPORT_ERROR("Query graphics command without image id"); break; } }
             image = handle_add_command(self, g, payload, is_dirty, iid);
             ret = create_add_response(self, image != NULL, g->action == 'q' ? q_iid: self->last_init_graphics_command.id);
             if (self->last_init_graphics_command.action == 'T' && image && image->data_loaded) handle_put_command(self, &self->last_init_graphics_command, c, is_dirty, image);

kitty/graphics.h

@@ -74,7 +74,6 @@ typedef struct {
     unsigned int last_scrolled_by;
     size_t used_storage;
 } GraphicsManager;
-PyTypeObject GraphicsManager_Type;
 
 
 typedef struct {

kitty/history.c

@@ -9,6 +9,8 @@
 #include "lineops.h"
 #include <structmember.h>
 
+extern PyTypeObject Line_Type;
+
 static inline Cell*
 lineptr(HistoryBuf *linebuf, index_type y) {
     return linebuf->buf + y * linebuf->xnum;
@@ -31,11 +33,11 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
         self->xnum = xnum;
         self->ynum = ynum;
         self->buf = PyMem_Calloc(xnum * ynum, sizeof(Cell));
-        self->continued_map = PyMem_Calloc(ynum, sizeof(bool));
+        self->line_attrs = PyMem_Calloc(ynum, sizeof(line_attrs_type));
         self->line = alloc_line();
-        if (self->buf == NULL || self->line == NULL || self->continued_map == NULL) {
+        if (self->buf == NULL || self->line == NULL || self->line_attrs == NULL) {
             PyErr_NoMemory();
-            PyMem_Free(self->buf); Py_CLEAR(self->line); PyMem_Free(self->continued_map);
+            PyMem_Free(self->buf); Py_CLEAR(self->line); PyMem_Free(self->line_attrs);
             Py_CLEAR(self);
         } else {
             self->line->xnum = xnum;
@@ -52,17 +54,10 @@ static void
 dealloc(HistoryBuf* self) {
     Py_CLEAR(self->line);
     PyMem_Free(self->buf);
-    PyMem_Free(self->continued_map);
+    PyMem_Free(self->line_attrs);
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
-void 
-historybuf_refresh_sprite_positions(HistoryBuf *self) {
-    for (index_type i = 0; i < self->ynum; i++) {
-        update_sprites_in_line(lineptr(self, i), self->xnum);
-    }
-}
-
 static inline index_type 
 index_of(HistoryBuf *self, index_type lnum) {
     // The index (buffer position) of the line with line number lnum
@@ -76,7 +71,8 @@ static inline void
 init_line(HistoryBuf *self, index_type num, Line *l) {
     // Initialize the line l, setting its pointer to the offsets for the line at index (buffer position) num
     l->cells = lineptr(self, num);
-    l->continued = self->continued_map[num];
+    l->continued = self->line_attrs[num] & CONTINUED_MASK;
+    l->has_dirty_text = self->line_attrs[num] & TEXT_DIRTY_MASK ? true : false;
 }
 
 void 
@@ -84,6 +80,16 @@ historybuf_init_line(HistoryBuf *self, index_type lnum, Line *l) {
     init_line(self, index_of(self, lnum), l);
 }
 
+void 
+historybuf_mark_line_clean(HistoryBuf *self, index_type y) {
+    self->line_attrs[index_of(self, y)] &= ~TEXT_DIRTY_MASK;
+}
+
+void 
+historybuf_mark_line_dirty(HistoryBuf *self, index_type y) {
+    self->line_attrs[index_of(self, y)] |= TEXT_DIRTY_MASK;
+}
+
 static inline index_type 
 historybuf_push(HistoryBuf *self) {
     index_type idx = (self->start_of_data + self->count) % self->ynum;
@@ -101,19 +107,19 @@ historybuf_resize(HistoryBuf *self, index_type lines) {
     if (t.ynum > 0 && t.ynum != self->ynum) {
         t.buf = PyMem_Calloc(t.xnum * t.ynum, sizeof(Cell));
         if (t.buf == NULL) { PyErr_NoMemory(); return false; }
-        t.continued_map = PyMem_Calloc(t.ynum, sizeof(bool));
-        if (t.continued_map == NULL) { PyMem_Free(t.buf); PyErr_NoMemory(); return false; }
+        t.line_attrs = PyMem_Calloc(t.ynum, sizeof(line_attrs_type));
+        if (t.line_attrs == NULL) { PyMem_Free(t.buf); PyErr_NoMemory(); return false; }
         t.count = MIN(self->count, t.ynum);
         for (index_type s=0; s < t.count; s++) {
             index_type si = index_of(self, s), ti = index_of(&t, s);
             copy_cells(lineptr(self, si), lineptr(&t, ti), t.xnum);
-            t.continued_map[ti] = self->continued_map[si];
+            t.line_attrs[ti] = self->line_attrs[si];
         }
         self->count = t.count;
         self->start_of_data = t.start_of_data;
         self->ynum = t.ynum;
-        PyMem_Free(self->buf); PyMem_Free(self->continued_map);
-        self->buf = t.buf; self->continued_map = t.continued_map;
+        PyMem_Free(self->buf); PyMem_Free(self->line_attrs);
+        self->buf = t.buf; self->line_attrs = t.line_attrs;
     }
     return true;
 }
@@ -122,7 +128,7 @@ void
 historybuf_add_line(HistoryBuf *self, const Line *line) {
     index_type idx = historybuf_push(self);
     copy_line(line, self->line);
-    self->continued_map[idx] = line->continued;
+    self->line_attrs[idx] = (line->continued & CONTINUED_MASK) | (line->has_dirty_text ? TEXT_DIRTY_MASK : 0);
 }
 
 static PyObject*
@@ -176,7 +182,7 @@ as_ansi(HistoryBuf *self, PyObject *callback) {
     for(unsigned int i = 0; i < self->count; i++) {
         init_line(self, i, &l);
         if (i < self->count - 1) {
-            l.continued = self->continued_map[index_of(self, i + 1)];
+            l.continued = self->line_attrs[index_of(self, i + 1)] & CONTINUED_MASK;
         } else l.continued = false;
         index_type num = line_as_ansi(&l, t, 5120);
         if (!(l.continued) && num < 5119) t[num++] = 10; // 10 = \n
@@ -190,6 +196,19 @@ as_ansi(HistoryBuf *self, PyObject *callback) {
     Py_RETURN_NONE;
 }
 
+static PyObject*
+dirty_lines(HistoryBuf *self) {
+#define dirty_lines_doc "dirty_lines() -> Line numbers of all lines that have dirty text."
+    PyObject *ans = PyList_New(0);
+    for (index_type i = 0; i < self->ynum; i++) {
+        if (self->line_attrs[i] & TEXT_DIRTY_MASK) {
+            PyList_Append(ans, PyLong_FromUnsignedLong(i));
+        }
+    }
+    return ans;
+}
+
+
 // Boilerplate {{{
 static PyObject* rewrap(HistoryBuf *self, PyObject *args);
 #define rewrap_doc ""
@@ -198,6 +217,7 @@ static PyMethodDef methods[] = {
     METHOD(change_num_of_lines, METH_O)
     METHOD(line, METH_O)
     METHOD(as_ansi, METH_O)
+    METHOD(dirty_lines, METH_NOARGS)
     METHOD(push, METH_VARARGS)
     METHOD(rewrap, METH_VARARGS)
     {NULL, NULL, 0, NULL}  /* Sentinel */
@@ -236,9 +256,9 @@ HistoryBuf *alloc_historybuf(unsigned int lines, unsigned int columns) {
 
 #define init_src_line(src_y) init_line(src, map_src_index(src_y), src->line);
 
-#define is_src_line_continued(src_y) (map_src_index(src_y) < src->ynum - 1 ? src->continued_map[map_src_index(src_y + 1)] : false)
+#define is_src_line_continued(src_y) (map_src_index(src_y) < src->ynum - 1 ? (src->line_attrs[map_src_index(src_y + 1)] & CONTINUED_MASK) : false)
 
-#define next_dest_line(cont) dest->continued_map[historybuf_push(dest)] = cont; dest->line->continued = cont; 
+#define next_dest_line(cont) dest->line_attrs[historybuf_push(dest)] = cont & CONTINUED_MASK; dest->line->continued = cont; 
 
 #define first_dest_line next_dest_line(false); 
 
@@ -248,12 +268,15 @@ void historybuf_rewrap(HistoryBuf *self, HistoryBuf *other) {
     // Fast path
     if (other->xnum == self->xnum && other->ynum == self->ynum) {
         memcpy(other->buf, self->buf, sizeof(Cell) * self->xnum * self->ynum);
-        memcpy(other->continued_map, self->continued_map, sizeof(bool) * self->ynum);
+        memcpy(other->line_attrs, self->line_attrs, sizeof(line_attrs_type) * self->ynum);
         other->count = self->count; other->start_of_data = self->start_of_data;
         return;
     }
     other->count = 0; other->start_of_data = 0;
-    if (self->count > 0) rewrap_inner(self, other, self->count, NULL);
+    if (self->count > 0) {
+        rewrap_inner(self, other, self->count, NULL);
+        for (index_type i = 0; i < other->count; i++) other->line_attrs[(other->start_of_data + i) % other->ynum] |= TEXT_DIRTY_MASK;
+    }
 }
 
 static PyObject*

kitty/icat.py

@@ -124,7 +124,7 @@ def write_chunked(cmd, data):
         cmd.clear()
 
 
-def show(outfile, width, height, fmt, transmit_mode):
+def show(outfile, width, height, fmt, transmit_mode='t'):
     cmd = {'a': 'T', 'f': fmt, 's': width, 'v': height}
     set_cursor(cmd, width, height)
     if detect_support.has_files:

kitty/keys.c

@@ -10,22 +10,18 @@
 #include "screen.h"
 #include <GLFW/glfw3.h>
 
-const uint8_t*
+const char*
 key_to_bytes(int glfw_key, bool smkx, bool extended, int mods, int action) {
     if ((action & 3) == 3) return NULL;
     if ((unsigned)glfw_key >= sizeof(key_map)/sizeof(key_map[0]) || glfw_key < 0) return NULL;
     uint16_t key = key_map[glfw_key];
     if (key == UINT8_MAX) return NULL;
-    mods &= 0xF;
-    key |= (mods & 0xF) << 7;
-    key |= (action & 3) << 11;
-    key |= (smkx & 1) << 13;
-    key |= (extended & 1) << 14;
-    if (key >= SIZE_OF_KEY_BYTES_MAP) return NULL;
-    return key_bytes[key];
+    KeyboardMode mode = extended ? EXTENDED : (smkx ? APPLICATION : NORMAL);
+    return key_lookup(key, mode, mods, action);
 }
 
 #define SPECIAL_INDEX(key) ((key & 0x7f) | ( (mods & 0xF) << 7))
+#define IS_ALT_MODS(mods) (mods == GLFW_MOD_ALT || mods == (GLFW_MOD_ALT | GLFW_MOD_SHIFT))
 
 void
 set_special_key_combo(int glfw_key, int mods) {
@@ -47,20 +43,15 @@ active_window() {
 void
 on_text_input(unsigned int codepoint, int mods) {
     Window *w = active_window();
-    static char buf[10];
+    static char buf[16];
     unsigned int sz = 0;
 
     if (w != NULL) {
-        Screen *screen = w->render_data.screen;
-        bool in_alt_mods = !screen->modes.mEXTENDED_KEYBOARD && (mods == GLFW_MOD_ALT || mods == (GLFW_MOD_ALT | GLFW_MOD_SHIFT));
         bool is_text = mods <= GLFW_MOD_SHIFT;
-        if (in_alt_mods) {
-            sz = encode_utf8(codepoint, buf + 1);
-            if (sz) {
-                buf[0] = 033;
-                sz++;
-            }
-        } else if (is_text) sz = encode_utf8(codepoint, buf);
+        if (is_text) sz = encode_utf8(codepoint, buf);
+#ifdef __APPLE__
+        if (!OPT(macos_option_as_alt) && IS_ALT_MODS(mods)) sz = encode_utf8(codepoint, buf);
+#endif
         if (sz) schedule_write_to_child(w->id, buf, sz);
     }
 }
@@ -89,31 +80,32 @@ get_localized_key(int key, int scancode) {
     switch(name[0]) {
 #define K(ch, name) case ch: return GLFW_KEY_##name
         // key names {{{
-        K('A', A);
-        K('B', B);
-        K('C', C);
-        K('D', D);
-        K('E', E);
-        K('F', F);
-        K('G', G);
-        K('H', H);
-        K('I', I);
-        K('J', J);
-        K('K', K);
-        K('L', L);
-        K('M', M);
-        K('N', N);
-        K('O', O);
-        K('P', P);
-        K('Q', Q);
-        K('S', S);
-        K('T', T);
-        K('U', U);
-        K('V', V);
-        K('W', W);
-        K('X', X);
-        K('Y', Y);
-        K('Z', Z);
+        K('A', A); K('a', A);
+        K('B', B); K('b', B);
+        K('C', C); K('c', C); 
+        K('D', D); K('d', D);
+        K('E', E); K('e', E);
+        K('F', F); K('f', F);
+        K('G', G); K('g', G);
+        K('H', H); K('h', H);
+        K('I', I); K('i', I);
+        K('J', J); K('j', J);
+        K('K', K); K('k', K);
+        K('L', L); K('l', L);
+        K('M', M); K('m', M);
+        K('N', N); K('n', N);
+        K('O', O); K('o', O);
+        K('P', P); K('p', P);
+        K('Q', Q); K('q', Q);
+        K('R', R); K('r', R);
+        K('S', S); K('s', S);
+        K('T', T); K('t', T);
+        K('U', U); K('u', U);
+        K('V', V); K('v', V);
+        K('W', W); K('w', W);
+        K('X', X); K('x', X);
+        K('Y', Y); K('y', Y);
+        K('Z', Z); K('z', Z);
         K('0', 0);
         K('1', 1);
         K('2', 2);
@@ -168,13 +160,16 @@ on_key_input(int key, int scancode, int action, int mods) {
     if (screen->scrolled_by && action == GLFW_PRESS && !is_modifier_key(key)) {
         screen_history_scroll(screen, SCROLL_FULL, false);  // scroll back to bottom
     }
+#ifdef __APPLE__
+    if (!OPT(macos_option_as_alt) && IS_ALT_MODS(mods)) return;
+#endif
     if (
             action == GLFW_PRESS ||
             (action == GLFW_REPEAT && screen->modes.mDECARM) ||
             screen->modes.mEXTENDED_KEYBOARD
        ) {
-        const uint8_t *data = key_to_bytes(lkey, screen->modes.mDECCKM, screen->modes.mEXTENDED_KEYBOARD, mods, action);
-        if (data) schedule_write_to_child(w->id, (char*)(data + 1), *data);
+        const char *data = key_to_bytes(lkey, screen->modes.mDECCKM, screen->modes.mEXTENDED_KEYBOARD, mods, action);
+        if (data) schedule_write_to_child(w->id, (data + 1), *data);
     }
 }
 
@@ -185,7 +180,7 @@ on_key_input(int key, int scancode, int action, int mods) {
 PYWRAP1(key_to_bytes) {
     int glfw_key, smkx, extended, mods, action;
     PA("ippii", &glfw_key, &smkx, &extended, &mods, &action);
-    const uint8_t *ans = key_to_bytes(glfw_key, smkx & 1, extended & 1, mods, action);
+    const char *ans = key_to_bytes(glfw_key, smkx & 1, extended & 1, mods, action);
     if (ans == NULL) return Py_BuildValue("y#", "", 0);
     return Py_BuildValue("y#", ans + 1, *ans);
 }

kitty/keys.py

@@ -2,10 +2,12 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
+import string
+
 from . import fast_data_types as defines
+from .key_encoding import KEY_MAP
 from .terminfo import key_as_bytes
 from .utils import base64_encode
-from .key_encoding import KEY_MAP
 
 
 def modify_key_bytes(keybytes, amt):
@@ -24,8 +26,9 @@ def modify_complex_key(name, amt):
 
 control_codes = {}
 smkx_key_map = {}
-alt_codes = {defines.GLFW_KEY_TAB: b'\033\t'}
-shift_alt_codes = {defines.GLFW_KEY_TAB: key_as_bytes('kcbt')}
+alt_codes = {defines.GLFW_KEY_TAB: b'\033\t', defines.GLFW_KEY_ENTER: b'\033\r', defines.GLFW_KEY_ESCAPE: b'\033\033', defines.GLFW_KEY_BACKSPACE: b'\033\177'}
+shift_alt_codes = alt_codes.copy()
+shift_alt_codes[defines.GLFW_KEY_TAB] = key_as_bytes('kcbt')
 alt_mods = (defines.GLFW_MOD_ALT, defines.GLFW_MOD_SHIFT | defines.GLFW_MOD_ALT)
 
 for kf, kn in {
@@ -76,6 +79,8 @@ SHIFTED_KEYS = {
     defines.GLFW_KEY_END: key_as_bytes('kEND'),
     defines.GLFW_KEY_LEFT: key_as_bytes('kLFT'),
     defines.GLFW_KEY_RIGHT: key_as_bytes('kRIT'),
+    defines.GLFW_KEY_UP: key_as_bytes('kri'),
+    defines.GLFW_KEY_DOWN: key_as_bytes('kind'),
 }
 
 control_codes.update({
@@ -87,7 +92,6 @@ control_codes[defines.GLFW_KEY_6] = (30,)
 control_codes[defines.GLFW_KEY_SLASH] = (31,)
 control_codes[defines.GLFW_KEY_SPACE] = (0,)
 
-
 rmkx_key_map = smkx_key_map.copy()
 rmkx_key_map.update({
     defines.GLFW_KEY_UP: b'\033[A',
@@ -131,11 +135,59 @@ def extended_key_event(key, mods, action):
     name = KEY_MAP.get(key)
     if name is None:
         return b''
+    m = 0
+    if mods & defines.GLFW_MOD_SHIFT:
+        m |= 0x1
+    if mods & defines.GLFW_MOD_ALT:
+        m |= 0x2
+    if mods & defines.GLFW_MOD_CONTROL:
+        m |= 0x4
+    if mods & defines.GLFW_MOD_SUPER:
+        m |= 0x8
     return '\033_K{}{}{}\033\\'.format(
-        action_map[action], base64_encode(mods), name
+        action_map[action], base64_encode(m), name
     ).encode('ascii')
 
 
+def pmap(names, r):
+    names = names.split()
+    r = [x.encode('ascii') for x in r]
+    if len(names) != len(r):
+        raise ValueError('Incorrect mapping for {}'.format(names))
+    names = [getattr(defines, 'GLFW_KEY_' + n) for n in names]
+    return dict(zip(names, r))
+
+
+UN_SHIFTED_PRINTABLE = {
+    getattr(defines, 'GLFW_KEY_' + x): x.lower().encode('ascii')
+    for x in string.digits + string.ascii_uppercase
+}
+UN_SHIFTED_PRINTABLE.update(pmap(
+    'SPACE APOSTROPHE COMMA MINUS PERIOD SLASH SEMICOLON EQUAL',
+    " ',-./;="
+))
+UN_SHIFTED_PRINTABLE.update(pmap(
+    'LEFT_BRACKET BACKSLASH RIGHT_BRACKET GRAVE_ACCENT',
+    "[\\]`"
+))
+SHIFTED_PRINTABLE = UN_SHIFTED_PRINTABLE.copy()
+SHIFTED_PRINTABLE.update({
+    getattr(defines, 'GLFW_KEY_' + x): x.encode('ascii') for x in string.ascii_uppercase
+})
+SHIFTED_PRINTABLE.update(pmap(
+    '1 2 3 4 5 6 7 8 9 0',
+    '!@#$%^&*()'
+))
+SHIFTED_PRINTABLE.update(pmap(
+    'APOSTROPHE COMMA MINUS PERIOD SLASH SEMICOLON EQUAL',
+    '"<_>?:+'
+))
+SHIFTED_PRINTABLE.update(pmap(
+    'LEFT_BRACKET BACKSLASH RIGHT_BRACKET GRAVE_ACCENT',
+    "{|}~"
+))
+
+
 def key_to_bytes(key, smkx, extended, mods, action):
     if extended:
         return extended_key_event(key, mods, action)
@@ -143,9 +195,13 @@ def key_to_bytes(key, smkx, extended, mods, action):
     if mods == defines.GLFW_MOD_CONTROL and key in control_codes:
         # Map Ctrl-key to ascii control code
         data.extend(control_codes[key])
-    elif mods in alt_mods and key in alt_codes:
-        # Printable keys handled by on_text_input()
-        data.extend((alt_codes if mods == defines.GLFW_MOD_ALT else shift_alt_codes)[key])
+    elif mods in alt_mods:
+        if key in alt_codes:
+            data.extend((alt_codes if mods == defines.GLFW_MOD_ALT else shift_alt_codes)[key])
+        elif key in UN_SHIFTED_PRINTABLE:
+            m = UN_SHIFTED_PRINTABLE if mods == defines.GLFW_MOD_ALT else SHIFTED_PRINTABLE
+            data.append(0o33)
+            data.extend(m[key])
     else:
         key_map = cursor_key_mode_map[smkx]
         x = key_map.get(key)
@@ -171,13 +227,6 @@ def get_shortcut(keymap, mods, key, scancode):
     return keymap.get((mods & 0b1111, key))
 
 
-def get_sent_data(send_text_map, key, scancode, mods, window, action):
-    if action in (defines.GLFW_PRESS, defines.GLFW_REPEAT):
-        m = keyboard_mode_name(window.screen)
-        keymap = send_text_map[m]
-        return keymap.get((mods & 0b1111, key))
-
-
 def generate_key_table():
     # To run this, use: python3 . -c "from kitty.keys import *; generate_key_table()"
     import os
@@ -218,46 +267,62 @@ def generate_key_table():
         if k is not None:
             w('case %d: return "%s";' % (i, key_name(k)))
     w('default: return NULL; }}\n')
-    number_entries = 128 * 256
-    inits = []
-    longest = 0
-    for i in range(number_entries):
-        key = i & 0x7f  # lowest seven bits
-        if key < key_count:
-            glfw_key = key_rmap[key]
-            k = keys.get(glfw_key)
-        else:
-            k = None
-        if k is None:
-            inits.append(None)
-        else:
-            mods = (i >> 7) & 0b1111
-            rest = i >> 11
-            action = rest & 0b11
-            if action == 0b11:  # no such action
-                inits.append(None)
+    w('typedef enum { NORMAL, APPLICATION, EXTENDED } KeyboardMode;\n')
+    w('static inline const char*\nkey_lookup(uint8_t key, KeyboardMode mode, uint8_t mods, uint8_t action) {')
+    i = 1
+
+    def ind(*a):
+        w(('  ' * i)[:-1], *a)
+    ind('switch(mode) {')
+    mmap = [(False, False), (True, False), (False, True)]
+    for (smkx, extended), mode in zip(mmap, 'NORMAL APPLICATION EXTENDED'.split()):
+        i += 1
+        ind('case {}:'.format(mode))
+        i += 1
+        ind('switch(action & 3) { case 3: return NULL; ')
+        for action in (defines.GLFW_RELEASE, defines.GLFW_PRESS, defines.GLFW_REPEAT):
+            i += 1
+            ind('case {}: // {}'.format(action, 'RELEASE PRESS REPEAT'.split()[action]))
+            i += 1
+            if action != defines.GLFW_RELEASE or mode == 'EXTENDED':
+                ind('switch (mods & 0xf) { ')
+                i += 1
+                for mods in range(16):
+                    key_bytes = {}
+                    for key in range(key_count):
+                        glfw_key = key_rmap[key]
+                        data = key_to_bytes(glfw_key, smkx, extended, mods, action)
+                        if data:
+                            key_bytes[key] = data, glfw_key
+                    i += 1
+                    ind('case 0x{:x}:'.format(mods))
+                    i += 1
+                    if key_bytes:
+                        ind('switch(key & 0x7f) { default: return NULL;')
+                        i += 1
+                        for key, (data, glfw_key) in key_bytes.items():
+                            ind('case {}: // {}'.format(key, key_name(keys[glfw_key])))
+                            i += 1
+                            items = bytearray(data)
+                            items.insert(0, len(items))
+                            ind('return "{}";'.format(''.join('\\x{:02x}'.format(x) for x in items)))
+                            i -= 1
+                        i -= 1
+                        ind('} // end switch(key)')
+                    else:
+                        ind('return NULL;')
+                    i -= 2
+                i -= 1
+                ind('}  // end switch(mods)\n')
+                i -= 1
             else:
-                smkx = bool(rest & 0b100)
-                extended = bool(rest & 0b1000)
-                data = key_to_bytes(glfw_key, smkx, extended, mods, action)
-                if data:
-                    longest = max(len(data), longest)
-                    inits.append((data, k, mods, smkx, extended))
-                else:
-                    inits.append(None)
-    longest += 1
-    w('#define SIZE_OF_KEY_BYTES_MAP %d\n' % number_entries)
-    w('static const uint8_t key_bytes[%d][%d] = {' % (number_entries, longest))
-    # empty = '{' + ('0, ' * longest) + '},'
-    empty = '{0},'
-    all_mods = {k.rpartition('_')[2]: v for k, v in vars(defines).items() if k.startswith('GLFW_MOD_')}
-    all_mods = {k: v for k, v in sorted(all_mods.items(), key=lambda x: x[0])}
-    for b in inits:
-        if b is None:
-            w(empty)
-        else:
-            b, k, mods, smkx, extended = b
-            b = bytearray(b)
-            name = '+'.join([k for k, v in all_mods.items() if v & mods] + [key_name(k)])
-            w('{%d, ' % len(b) + ', '.join(map(hex, b)) + '}, //', name, 'smkx:', smkx, 'extended:', extended)
-    w('};')
+                ind('return NULL;\n')
+                i -= 1
+            i -= 1
+        ind('}}  // end switch(action) in mode {}\n\n'.format(mode))
+        i -= 1
+    i -= 1
+    ind('}')
+    ind('return NULL;')
+    i -= 1
+    w('}')

kitty/khrplatform.h

@@ -0,0 +1,284 @@
+#ifndef __khrplatform_h_
+#define __khrplatform_h_
+
+/*
+** Copyright (c) 2008-2009 The Khronos Group Inc.
+**
+** Permission is hereby granted, free of charge, to any person obtaining a
+** copy of this software and/or associated documentation files (the
+** "Materials"), to deal in the Materials without restriction, including
+** without limitation the rights to use, copy, modify, merge, publish,
+** distribute, sublicense, and/or sell copies of the Materials, and to
+** permit persons to whom the Materials are furnished to do so, subject to
+** the following conditions:
+**
+** The above copyright notice and this permission notice shall be included
+** in all copies or substantial portions of the Materials.
+**
+** THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+** MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
+*/
+
+/* Khronos platform-specific types and definitions.
+ *
+ * $Revision: 32517 $ on $Date: 2016-03-11 02:41:19 -0800 (Fri, 11 Mar 2016) $
+ *
+ * Adopters may modify this file to suit their platform. Adopters are
+ * encouraged to submit platform specific modifications to the Khronos
+ * group so that they can be included in future versions of this file.
+ * Please submit changes by sending them to the public Khronos Bugzilla
+ * (http://khronos.org/bugzilla) by filing a bug against product
+ * "Khronos (general)" component "Registry".
+ *
+ * A predefined template which fills in some of the bug fields can be
+ * reached using http://tinyurl.com/khrplatform-h-bugreport, but you
+ * must create a Bugzilla login first.
+ *
+ *
+ * See the Implementer's Guidelines for information about where this file
+ * should be located on your system and for more details of its use:
+ *    http://www.khronos.org/registry/implementers_guide.pdf
+ *
+ * This file should be included as
+ *        #include <KHR/khrplatform.h>
+ * by Khronos client API header files that use its types and defines.
+ *
+ * The types in khrplatform.h should only be used to define API-specific types.
+ *
+ * Types defined in khrplatform.h:
+ *    khronos_int8_t              signed   8  bit
+ *    khronos_uint8_t             unsigned 8  bit
+ *    khronos_int16_t             signed   16 bit
+ *    khronos_uint16_t            unsigned 16 bit
+ *    khronos_int32_t             signed   32 bit
+ *    khronos_uint32_t            unsigned 32 bit
+ *    khronos_int64_t             signed   64 bit
+ *    khronos_uint64_t            unsigned 64 bit
+ *    khronos_intptr_t            signed   same number of bits as a pointer
+ *    khronos_uintptr_t           unsigned same number of bits as a pointer
+ *    khronos_ssize_t             signed   size
+ *    khronos_usize_t             unsigned size
+ *    khronos_float_t             signed   32 bit floating point
+ *    khronos_time_ns_t           unsigned 64 bit time in nanoseconds
+ *    khronos_utime_nanoseconds_t unsigned time interval or absolute time in
+ *                                         nanoseconds
+ *    khronos_stime_nanoseconds_t signed time interval in nanoseconds
+ *    khronos_boolean_enum_t      enumerated boolean type. This should
+ *      only be used as a base type when a client API's boolean type is
+ *      an enum. Client APIs which use an integer or other type for
+ *      booleans cannot use this as the base type for their boolean.
+ *
+ * Tokens defined in khrplatform.h:
+ *
+ *    KHRONOS_FALSE, KHRONOS_TRUE Enumerated boolean false/true values.
+ *
+ *    KHRONOS_SUPPORT_INT64 is 1 if 64 bit integers are supported; otherwise 0.
+ *    KHRONOS_SUPPORT_FLOAT is 1 if floats are supported; otherwise 0.
+ *
+ * Calling convention macros defined in this file:
+ *    KHRONOS_APICALL
+ *    KHRONOS_APIENTRY
+ *    KHRONOS_APIATTRIBUTES
+ *
+ * These may be used in function prototypes as:
+ *
+ *      KHRONOS_APICALL void KHRONOS_APIENTRY funcname(
+ *                                  int arg1,
+ *                                  int arg2) KHRONOS_APIATTRIBUTES;
+ */
+
+/*-------------------------------------------------------------------------
+ * Definition of KHRONOS_APICALL
+ *-------------------------------------------------------------------------
+ * This precedes the return type of the function in the function prototype.
+ */
+#if defined(_WIN32) && !defined(__SCITECH_SNAP__)
+#   define KHRONOS_APICALL __declspec(dllimport)
+#elif defined (__SYMBIAN32__)
+#   define KHRONOS_APICALL IMPORT_C
+#elif defined(__ANDROID__)
+#   define KHRONOS_APICALL __attribute__((visibility("default")))
+#else
+#   define KHRONOS_APICALL
+#endif
+
+/*-------------------------------------------------------------------------
+ * Definition of KHRONOS_APIENTRY
+ *-------------------------------------------------------------------------
+ * This follows the return type of the function  and precedes the function
+ * name in the function prototype.
+ */
+#if defined(_WIN32) && !defined(_WIN32_WCE) && !defined(__SCITECH_SNAP__)
+    /* Win32 but not WinCE */
+#   define KHRONOS_APIENTRY __stdcall
+#else
+#   define KHRONOS_APIENTRY
+#endif
+
+/*-------------------------------------------------------------------------
+ * Definition of KHRONOS_APIATTRIBUTES
+ *-------------------------------------------------------------------------
+ * This follows the closing parenthesis of the function prototype arguments.
+ */
+#if defined (__ARMCC_2__)
+#define KHRONOS_APIATTRIBUTES __softfp
+#else
+#define KHRONOS_APIATTRIBUTES
+#endif
+
+/*-------------------------------------------------------------------------
+ * basic type definitions
+ *-----------------------------------------------------------------------*/
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || defined(__GNUC__) || defined(__SCO__) || defined(__USLC__)
+
+
+/*
+ * Using <stdint.h>
+ */
+#include <stdint.h>
+typedef int32_t                 khronos_int32_t;
+typedef uint32_t                khronos_uint32_t;
+typedef int64_t                 khronos_int64_t;
+typedef uint64_t                khronos_uint64_t;
+#define KHRONOS_SUPPORT_INT64   1
+#define KHRONOS_SUPPORT_FLOAT   1
+
+#elif defined(__VMS ) || defined(__sgi)
+
+/*
+ * Using <inttypes.h>
+ */
+#include <inttypes.h>
+typedef int32_t                 khronos_int32_t;
+typedef uint32_t                khronos_uint32_t;
+typedef int64_t                 khronos_int64_t;
+typedef uint64_t                khronos_uint64_t;
+#define KHRONOS_SUPPORT_INT64   1
+#define KHRONOS_SUPPORT_FLOAT   1
+
+#elif defined(_WIN32) && !defined(__SCITECH_SNAP__)
+
+/*
+ * Win32
+ */
+typedef __int32                 khronos_int32_t;
+typedef unsigned __int32        khronos_uint32_t;
+typedef __int64                 khronos_int64_t;
+typedef unsigned __int64        khronos_uint64_t;
+#define KHRONOS_SUPPORT_INT64   1
+#define KHRONOS_SUPPORT_FLOAT   1
+
+#elif defined(__sun__) || defined(__digital__)
+
+/*
+ * Sun or Digital
+ */
+typedef int                     khronos_int32_t;
+typedef unsigned int            khronos_uint32_t;
+#if defined(__arch64__) || defined(_LP64)
+typedef long int                khronos_int64_t;
+typedef unsigned long int       khronos_uint64_t;
+#else
+typedef long long int           khronos_int64_t;
+typedef unsigned long long int  khronos_uint64_t;
+#endif /* __arch64__ */
+#define KHRONOS_SUPPORT_INT64   1
+#define KHRONOS_SUPPORT_FLOAT   1
+
+#elif 0
+
+/*
+ * Hypothetical platform with no float or int64 support
+ */
+typedef int                     khronos_int32_t;
+typedef unsigned int            khronos_uint32_t;
+#define KHRONOS_SUPPORT_INT64   0
+#define KHRONOS_SUPPORT_FLOAT   0
+
+#else
+
+/*
+ * Generic fallback
+ */
+#include <stdint.h>
+typedef int32_t                 khronos_int32_t;
+typedef uint32_t                khronos_uint32_t;
+typedef int64_t                 khronos_int64_t;
+typedef uint64_t                khronos_uint64_t;
+#define KHRONOS_SUPPORT_INT64   1
+#define KHRONOS_SUPPORT_FLOAT   1
+
+#endif
+
+
+/*
+ * Types that are (so far) the same on all platforms
+ */
+typedef signed   char          khronos_int8_t;
+typedef unsigned char          khronos_uint8_t;
+typedef signed   short int     khronos_int16_t;
+typedef unsigned short int     khronos_uint16_t;
+
+/*
+ * Types that differ between LLP64 and LP64 architectures - in LLP64,
+ * pointers are 64 bits, but 'long' is still 32 bits. Win64 appears
+ * to be the only LLP64 architecture in current use.
+ */
+#ifdef _WIN64
+typedef signed   long long int khronos_intptr_t;
+typedef unsigned long long int khronos_uintptr_t;
+typedef signed   long long int khronos_ssize_t;
+typedef unsigned long long int khronos_usize_t;
+#else
+typedef signed   long  int     khronos_intptr_t;
+typedef unsigned long  int     khronos_uintptr_t;
+typedef signed   long  int     khronos_ssize_t;
+typedef unsigned long  int     khronos_usize_t;
+#endif
+
+#if KHRONOS_SUPPORT_FLOAT
+/*
+ * Float type
+ */
+typedef          float         khronos_float_t;
+#endif
+
+#if KHRONOS_SUPPORT_INT64
+/* Time types
+ *
+ * These types can be used to represent a time interval in nanoseconds or
+ * an absolute Unadjusted System Time.  Unadjusted System Time is the number
+ * of nanoseconds since some arbitrary system event (e.g. since the last
+ * time the system booted).  The Unadjusted System Time is an unsigned
+ * 64 bit value that wraps back to 0 every 584 years.  Time intervals
+ * may be either signed or unsigned.
+ */
+typedef khronos_uint64_t       khronos_utime_nanoseconds_t;
+typedef khronos_int64_t        khronos_stime_nanoseconds_t;
+#endif
+
+/*
+ * Dummy value used to pad enum types to 32 bits.
+ */
+#ifndef KHRONOS_MAX_ENUM
+#define KHRONOS_MAX_ENUM 0x7FFFFFFF
+#endif
+
+/*
+ * Enumerated boolean type
+ *
+ * Values other than zero should be considered to be true.  Therefore
+ * comparisons should not be made against KHRONOS_TRUE.
+ */
+typedef enum {
+    KHRONOS_FALSE = 0,
+    KHRONOS_TRUE  = 1,
+    KHRONOS_BOOLEAN_ENUM_FORCE_SIZE = KHRONOS_MAX_ENUM
+} khronos_boolean_enum_t;
+
+#endif /* __khrplatform_h_ */

kitty/kitty.conf

@@ -5,14 +5,12 @@
 # by the OSes font system. Setting them manually is useful for font families
 # that have many weight variants like Book, Medium, Thick, etc. For example:
 # font_family Operator Mono Book
-# bold_font Operator Mono Thick
-# bold_italic_font Operator Mono Medium
-# or
-# font_family SF Mono Medium
-# bold_font        SF Mono Semibold
-# bold_italic_font SF Mono Semibold 
-# Note that you should use the full family name but do not add Bold or Italic qualifiers
-# to the name.
+# bold_font Operator Mono Medium
+# italic_font Operator Mono Book Italic
+# bold_italic_font Operator Mono Medium Italic
+# 
+# You can get a list of full family names available on your computer by running
+# kitty list-fonts
 font_family      monospace
 italic_font      auto
 bold_font        auto
@@ -33,6 +31,12 @@ font_size_delta 2
 # 100% to reduce line height (but this might cause rendering artifacts).
 adjust_line_height 0
 
+# Change the sizes of the lines used for the box drawing unicode characters
+# These values are in pts. They will be scaled by the monitor DPI to arrive at 
+# a pixel value. There must be four values corresponding to thin, normal, thick,
+# and very thick lines;
+box_drawing_scale 0.001, 1, 1.5, 2
+
 # The foreground color
 foreground       #dddddd
 
@@ -90,6 +94,9 @@ select_by_word_characters :@-./_~?&=%+#
 # zero to disable mouse cursor hiding.
 mouse_hide_wait 3.0
 
+# Set the active window to the window under the mouse when moving the mouse around
+focus_follows_mouse no
+
 # The enabled window layouts. A comma separated list of layout names. The special value * means
 # all layouts. The first listed layout will be used as the startup layout.
 # For a list of available layouts, see the file layouts.py
@@ -279,11 +286,12 @@ map ctrl+shift+,        move_tab_backward
 map ctrl+shift+equal    increase_font_size
 map ctrl+shift+minus    decrease_font_size
 map ctrl+shift+backspace restore_font_size
+map ctrl+shift+f11      toggle_fullscreen
 
 # Sending arbitrary text on shortcut key presses
 # You can tell kitty to send arbitrary (UTF-8) encoded text to
 # the client program when pressing specified shortcut keys. For example:
-# send_text all ctrl+alt+a Special text
+# map ctrl+alt+a send_text all Special text
 # This will send "Special text" when you press the Ctrl+Alt+a key combination.
 # The text to be sent is a python string literal so you can use escapes like
 # \x1b to send control codes or \u21fb to send unicode characters (or you can
@@ -294,8 +302,8 @@ map ctrl+shift+backspace restore_font_size
 # the DECCKM cursor key mode for terminals, and kitty refers to the special kitty
 # extended keyboard protocol. Another example, that outputs a word and then moves the cursor
 # to the start of the line (same as pressing the Home key):
-# send_text normal ctrl+alt+a Word\x1b[H
-# send_text application ctrl+alt+a Word\x1bOH
+# map ctrl+alt+a send_text normal Word\x1b[H
+# map ctrl+alt+a send_text application Word\x1bOH
 
 # Symbol mapping (special font for specified unicode code points). Map the
 # specified unicode codepoints to a particular font. Useful if you need special
@@ -316,3 +324,9 @@ map ctrl+shift+backspace restore_font_size
 
 # Hide the kitty window's title bar on macOS.
 macos_hide_titlebar no
+
+# Use the option key as an alt key. With this set to no, kitty will use 
+# the macOS native Option+Key = unicode character behavior. This will
+# break any Alt+key keyboard shortcuts in your terminal programs, but you
+# can use the macOS unicode input technique.
+macos_option_as_alt yes

kitty/line-buf.c

@@ -9,6 +9,9 @@
 #include "lineops.h"
 #include <structmember.h>
 
+extern PyTypeObject Line_Type;
+extern PyTypeObject HistoryBuf_Type;
+
 static inline Cell*
 lineptr(LineBuf *linebuf, index_type y) {
     return linebuf->buf + y * linebuf->xnum;
@@ -22,13 +25,26 @@ clear_chars_to(LineBuf* linebuf, index_type y, char_type ch) {
 void 
 linebuf_clear(LineBuf *self, char_type ch) {
     memset(self->buf, 0, self->xnum * self->ynum * sizeof(Cell));
-    memset(self->continued_map, 0, self->ynum * sizeof(bool));
+    memset(self->line_attrs, 0, self->ynum * sizeof(line_attrs_type));
     for (index_type i = 0; i < self->ynum; i++) self->line_map[i] = i;
     if (ch != 0) {
-        for (index_type i = 0; i < self->ynum; i++) clear_chars_to(self, i, ch);
+        for (index_type i = 0; i < self->ynum; i++) {
+            clear_chars_to(self, i, ch);
+            self->line_attrs[i] = TEXT_DIRTY_MASK;
+        }
     }
 }
 
+void
+linebuf_mark_line_dirty(LineBuf *self, index_type y) {
+    self->line_attrs[y] |= TEXT_DIRTY_MASK;
+}
+
+void
+linebuf_mark_line_clean(LineBuf *self, index_type y) {
+    self->line_attrs[y] &= ~TEXT_DIRTY_MASK;
+}
+
 static PyObject*
 clear(LineBuf *self) {
 #define clear_doc "Clear all lines in this LineBuf"
@@ -60,11 +76,11 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
         self->buf = PyMem_Calloc(xnum * ynum, sizeof(Cell));
         self->line_map = PyMem_Calloc(ynum, sizeof(index_type));
         self->scratch = PyMem_Calloc(ynum, sizeof(index_type));
-        self->continued_map = PyMem_Calloc(ynum, sizeof(bool));
+        self->line_attrs = PyMem_Calloc(ynum, sizeof(line_attrs_type));
         self->line = alloc_line();
-        if (self->buf == NULL || self->line_map == NULL || self->scratch == NULL || self->continued_map == NULL || self->line == NULL) {
+        if (self->buf == NULL || self->line_map == NULL || self->scratch == NULL || self->line_attrs == NULL || self->line == NULL) {
             PyErr_NoMemory();
-            PyMem_Free(self->buf); PyMem_Free(self->line_map); PyMem_Free(self->continued_map); Py_CLEAR(self->line);
+            PyMem_Free(self->buf); PyMem_Free(self->line_map); PyMem_Free(self->line_attrs); Py_CLEAR(self->line);
             Py_CLEAR(self);
         } else {
             self->line->xnum = xnum;
@@ -82,7 +98,7 @@ static void
 dealloc(LineBuf* self) {
     PyMem_Free(self->buf);
     PyMem_Free(self->line_map); 
-    PyMem_Free(self->continued_map); 
+    PyMem_Free(self->line_attrs); 
     PyMem_Free(self->scratch);
     Py_CLEAR(self->line);
     Py_TYPE(self)->tp_free((PyObject*)self);
@@ -97,7 +113,8 @@ void
 linebuf_init_line(LineBuf *self, index_type idx) {
     self->line->ynum = idx;
     self->line->xnum = self->xnum;
-    self->line->continued = self->continued_map[idx];
+    self->line->continued = self->line_attrs[idx] & CONTINUED_MASK ? true : false;
+    self->line->has_dirty_text = self->line_attrs[idx] & TEXT_DIRTY_MASK ? true : false;
     init_line(self, self->line, self->line_map[idx]);
 }
 
@@ -123,6 +140,7 @@ void
 linebuf_set_attribute(LineBuf *self, unsigned int shift, unsigned int val) {
     for (index_type y = 0; y < self->ynum; y++) {
         set_attribute_on_line(lineptr(self, y), shift, val, self->xnum);
+        self->line_attrs[y] |= TEXT_DIRTY_MASK;
     }
 }
 
@@ -143,10 +161,23 @@ set_continued(LineBuf *self, PyObject *args) {
     int val;
     if (!PyArg_ParseTuple(args, "Ip", &y, &val)) return NULL;
     if (y >= self->ynum) { PyErr_SetString(PyExc_ValueError, "Out of bounds."); return NULL; }
-    self->continued_map[y] = val & 1;
+    if (val) self->line_attrs[y] |= CONTINUED_MASK;
+    else self->line_attrs[y] &= ~CONTINUED_MASK;
     Py_RETURN_NONE;
 }
 
+static PyObject*
+dirty_lines(LineBuf *self) {
+#define dirty_lines_doc "dirty_lines() -> Line numbers of all lines that have dirty text."
+    PyObject *ans = PyList_New(0);
+    for (index_type i = 0; i < self->ynum; i++) {
+        if (self->line_attrs[i] & TEXT_DIRTY_MASK) {
+            PyList_Append(ans, PyLong_FromUnsignedLong(i));
+        }
+    }
+    return ans;
+}
+
 static inline bool
 allocate_line_storage(Line *line, bool initialize) {
     if (initialize) {
@@ -169,7 +200,8 @@ create_line_copy_inner(LineBuf* self, index_type y) {
     src.xnum = self->xnum; line->xnum = self->xnum;
     if (!allocate_line_storage(line, 0)) { Py_CLEAR(line); return PyErr_NoMemory(); }
     line->ynum = y;
-    line->continued = self->continued_map[y];
+    line->continued = self->line_attrs[y] & CONTINUED_MASK ? true : false;
+    line->has_dirty_text = self->line_attrs[y] & TEXT_DIRTY_MASK ? true : false;
     init_line(self, &src, self->line_map[y]);
     copy_line(&src, line);
     return (PyObject*)line;
@@ -191,7 +223,8 @@ copy_line_to(LineBuf *self, PyObject *args) {
     if (!PyArg_ParseTuple(args, "IO!", &y, &Line_Type, &dest)) return NULL;
     src.xnum = self->xnum; dest->xnum = self->xnum;
     dest->ynum = y;
-    dest->continued = self->continued_map[y];
+    dest->continued = self->line_attrs[y] & CONTINUED_MASK;
+    dest->has_dirty_text = self->line_attrs[y] & TEXT_DIRTY_MASK;
     init_line(self, &src, self->line_map[y]);
     copy_line(&src, dest);
     Py_RETURN_NONE;
@@ -201,13 +234,15 @@ static inline void
 clear_line_(Line *l, index_type xnum) {
     memset(l->cells, 0, xnum * sizeof(Cell));
     if (BLANK_CHAR != 0) clear_chars_in_line(l->cells, xnum, BLANK_CHAR);
+    l->has_dirty_text = false;
 }
 
-void linebuf_clear_line(LineBuf *self, index_type y) {
+void 
+linebuf_clear_line(LineBuf *self, index_type y) {
     Line l;
     init_line(self, &l, self->line_map[y]);
     clear_line_(&l, self->xnum);
-    self->continued_map[y] = 0;
+    self->line_attrs[y] = 0;
 }
 
 static PyObject*
@@ -219,20 +254,21 @@ clear_line(LineBuf *self, PyObject *val) {
     Py_RETURN_NONE;
 }
 
-void linebuf_index(LineBuf* self, index_type top, index_type bottom) {
+void 
+linebuf_index(LineBuf* self, index_type top, index_type bottom) {
     if (top >= self->ynum - 1 || bottom >= self->ynum || bottom <= top) return;
     index_type old_top = self->line_map[top];
-    bool old_cont = self->continued_map[top];
+    line_attrs_type old_attrs = self->line_attrs[top];
     for (index_type i = top; i < bottom; i++) {
         self->line_map[i] = self->line_map[i + 1];
-        self->continued_map[i] = self->continued_map[i + 1];
+        self->line_attrs[i] = self->line_attrs[i + 1];
     }
     self->line_map[bottom] = old_top;
-    self->continued_map[bottom] = old_cont;
+    self->line_attrs[bottom] = old_attrs;
 }
 
 static PyObject*
-index(LineBuf *self, PyObject *args) {
+pyw_index(LineBuf *self, PyObject *args) {
 #define index_doc "index(top, bottom) -> Scroll all lines in the range [top, bottom] by one upwards. After scrolling, bottom will be top."
     unsigned int top, bottom;
     if (!PyArg_ParseTuple(args, "II", &top, &bottom)) return NULL;
@@ -240,16 +276,17 @@ index(LineBuf *self, PyObject *args) {
     Py_RETURN_NONE;
 }
 
-void linebuf_reverse_index(LineBuf *self, index_type top, index_type bottom) {
+void 
+linebuf_reverse_index(LineBuf *self, index_type top, index_type bottom) {
     if (top >= self->ynum - 1 || bottom >= self->ynum || bottom <= top) return;
     index_type old_bottom = self->line_map[bottom];
-    bool old_cont = self->continued_map[bottom];
+    line_attrs_type old_attrs = self->line_attrs[bottom];
     for (index_type i = bottom; i > top; i--) {
         self->line_map[i] = self->line_map[i - 1];
-        self->continued_map[i] = self->continued_map[i - 1];
+        self->line_attrs[i] = self->line_attrs[i - 1];
     }
     self->line_map[top] = old_bottom;
-    self->continued_map[top] = old_cont;
+    self->line_attrs[top] = old_attrs;
 }
 
 static PyObject*
@@ -267,11 +304,12 @@ is_continued(LineBuf *self, PyObject *val) {
 #define is_continued_doc "is_continued(y) -> Whether the line y is continued or not"
     unsigned long y = PyLong_AsUnsignedLong(val);
     if (y >= self->ynum) { PyErr_SetString(PyExc_ValueError, "Out of bounds."); return NULL; }
-    if (self->continued_map[y]) { Py_RETURN_TRUE; }
+    if (self->line_attrs[y] & CONTINUED_MASK) { Py_RETURN_TRUE; }
     Py_RETURN_FALSE;
 }
 
-void linebuf_insert_lines(LineBuf *self, unsigned int num, unsigned int y, unsigned int bottom) {
+void 
+linebuf_insert_lines(LineBuf *self, unsigned int num, unsigned int y, unsigned int bottom) {
     index_type i;
     if (y >= self->ynum || y > bottom || bottom >= self->ynum) return;
     index_type ylimit = bottom + 1;
@@ -282,9 +320,9 @@ void linebuf_insert_lines(LineBuf *self, unsigned int num, unsigned int y, unsig
         }
         for (i = ylimit - 1; i >= y + num; i--) {
             self->line_map[i] = self->line_map[i - num];
-            self->continued_map[i] = self->continued_map[i - num];
+            self->line_attrs[i] = self->line_attrs[i - num];
         }
-        if (y + num < self->ynum) self->continued_map[y + num] = 0;
+        if (y + num < self->ynum) self->line_attrs[y + num] &= ~CONTINUED_MASK;
         for (i = 0; i < num; i++) {
             self->line_map[y + i] = self->scratch[ylimit - num + i];
         }
@@ -292,7 +330,7 @@ void linebuf_insert_lines(LineBuf *self, unsigned int num, unsigned int y, unsig
         for (i = y; i < y + num; i++) {
             init_line(self, &l, self->line_map[i]);
             clear_line_(&l, self->xnum);
-            self->continued_map[i] = 0;
+            self->line_attrs[i] = 0;
         }
     }
 }
@@ -317,9 +355,9 @@ linebuf_delete_lines(LineBuf *self, index_type num, index_type y, index_type bot
     }
     for (i = y; i < ylimit && i + num < self->ynum; i++) {
         self->line_map[i] = self->line_map[i + num];
-        self->continued_map[i] = self->continued_map[i + num];
+        self->line_attrs[i] = self->line_attrs[i + num];
     }
-    self->continued_map[y] = 0;
+    self->line_attrs[y] &= ~CONTINUED_MASK;
     for (i = 0; i < num; i++) {
         self->line_map[ylimit - num + i] = self->scratch[y + i];
     }
@@ -327,13 +365,13 @@ linebuf_delete_lines(LineBuf *self, index_type num, index_type y, index_type bot
     for (i = ylimit - num; i < ylimit; i++) {
         init_line(self, &l, self->line_map[i]);
         clear_line_(&l, self->xnum);
-        self->continued_map[i] = 0;
+        self->line_attrs[i] = 0;
     }
 }
  
 static PyObject*
 delete_lines(LineBuf *self, PyObject *args) {
-#define delete_lines_doc "delete_lines(num, y, bottom) -> Delete num blank lines at y, only changing lines in the range [y, bottom]."
+#define delete_lines_doc "delete_lines(num, y, bottom) -> Delete num lines at y, only changing lines in the range [y, bottom]."
     unsigned int y, num, bottom;
     if (!PyArg_ParseTuple(args, "III", &num, &y, &bottom)) return NULL;
     linebuf_delete_lines(self, num, y, bottom);
@@ -346,7 +384,7 @@ as_ansi(LineBuf *self, PyObject *callback) {
     static Py_UCS4 t[5120];
     Line l = {.xnum=self->xnum};
     for(index_type i = 0; i < self->ynum; i++) {
-        l.continued = (i < self->ynum - 1) ? self->continued_map[i+1] : self->continued_map[i];
+        l.continued = ((i < self->ynum - 1) ? self->line_attrs[i+1] : self->line_attrs[i]) & CONTINUED_MASK;
         init_line(self, (&l), self->line_map[i]);
         index_type num = line_as_ansi(&l, t, 5120);
         if (!(l.continued) && num < 5119) t[num++] = 10; // 10 = \n
@@ -360,13 +398,6 @@ as_ansi(LineBuf *self, PyObject *callback) {
     Py_RETURN_NONE;
 }
 
-void 
-linebuf_refresh_sprite_positions(LineBuf *self) {
-    for (index_type i = 0; i < self->ynum; i++) {
-        update_sprites_in_line(lineptr(self, i), self->xnum);
-    }
-}
-
 static PyObject*
 __str__(LineBuf *self) {
     PyObject *lines = PyTuple_New(self->ynum);
@@ -403,7 +434,8 @@ static PyMethodDef methods[] = {
     METHOD(as_ansi, METH_O)
     METHOD(set_attribute, METH_VARARGS)
     METHOD(set_continued, METH_VARARGS)
-    METHOD(index, METH_VARARGS)
+    METHOD(dirty_lines, METH_NOARGS)
+    {"index", (PyCFunction)pyw_index, METH_VARARGS, NULL},
     METHOD(reverse_index, METH_VARARGS)
     METHOD(insert_lines, METH_VARARGS)
     METHOD(delete_lines, METH_VARARGS)
@@ -443,7 +475,7 @@ copy_old(LineBuf *self, PyObject *y) {
 
     for (index_type i = 0; i < MIN(self->ynum, other->ynum); i++) {
         index_type s = self->ynum - 1 - i, o = other->ynum - 1 - i;
-        self->continued_map[s] = other->continued_map[o];
+        self->line_attrs[s] = other->line_attrs[o];
         s = self->line_map[s]; o = other->line_map[o];
         init_line(self, &sl, s); init_line(other, &ol, o);
         copy_line(&ol, &sl);
@@ -461,7 +493,7 @@ linebuf_rewrap(LineBuf *self, LineBuf *other, index_type *num_content_lines_befo
     // Fast path
     if (other->xnum == self->xnum && other->ynum == self->ynum) {
         memcpy(other->line_map, self->line_map, sizeof(index_type) * self->ynum);
-        memcpy(other->continued_map, self->continued_map, sizeof(bool) * self->ynum);
+        memcpy(other->line_attrs, self->line_attrs, sizeof(bool) * self->ynum);
         memcpy(other->buf, self->buf, self->xnum * self->ynum * sizeof(Cell));
         *num_content_lines_before = self->ynum; *num_content_lines_after = self->ynum;
         return;
@@ -485,6 +517,7 @@ linebuf_rewrap(LineBuf *self, LineBuf *other, index_type *num_content_lines_befo
 
     rewrap_inner(self, other, first + 1, historybuf);
     *num_content_lines_after = other->line->ynum + 1;
+    for (i = 0; i < *num_content_lines_after; i++) other->line_attrs[i] |= TEXT_DIRTY_MASK;
     *num_content_lines_before = first + 1;
 }
 

kitty/line.c

@@ -9,6 +9,8 @@
 #include "unicode-data.h"
 #include "lineops.h"
 
+extern PyTypeObject Cursor_Type;
+
 static PyObject *
 new(PyTypeObject UNUSED *type, PyObject UNUSED *args, PyObject UNUSED *kwds) {
     PyErr_SetString(PyExc_TypeError, "Line objects cannot be instantiated directly, create them using LineBuf.line()");
@@ -35,22 +37,15 @@ line_length(Line *self) {
 PyObject* 
 line_text_at(char_type ch, combining_type cc) {
     PyObject *ans;
-    if (cc == 0) {
-        ans = PyUnicode_New(1, ch);
-        if (ans == NULL) return PyErr_NoMemory();
-        PyUnicode_WriteChar(ans, 0, ch);
+    if (LIKELY(cc == 0)) {
+        ans = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, &ch, 1);
     } else {
-        Py_UCS4 cc1 = cc & CC_MASK, cc2 = cc >> 16;
-        Py_UCS4 normalized = normalize(ch, cc1, cc2);
-        if (normalized) { return line_text_at(normalized, 0); }
-        Py_UCS4 maxc = (ch > cc1) ? MAX(ch, cc2) : MAX(cc1, cc2);
-        ans = PyUnicode_New(cc2 ? 3 : 2, maxc);
-        if (ans == NULL) return PyErr_NoMemory();
-        PyUnicode_WriteChar(ans, 0, ch);
-        PyUnicode_WriteChar(ans, 1, cc1);
-        if (cc2) PyUnicode_WriteChar(ans, 2, cc2);
+        Py_UCS4 buf[3];
+        buf[0] = ch; buf[1] = cc & CC_MASK; buf[2] = cc >> 16;
+        Py_UCS4 normalized = normalize(ch, buf[1], buf[2]);
+        if (normalized) ans = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, &normalized, 1);
+        else ans = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, buf[2] ? 3 : 2);
     }
-
     return ans;
 }
 
@@ -166,10 +161,43 @@ url_end_at(Line *self, PyObject *x) {
 static PyObject*
 text_at(Line* self, Py_ssize_t xval) {
 #define text_at_doc "[x] -> Return the text in the specified cell"
-    if (xval >= self->xnum) { PyErr_SetString(PyExc_IndexError, "Column number out of bounds"); return NULL; }
+    if ((unsigned)xval >= self->xnum) { PyErr_SetString(PyExc_IndexError, "Column number out of bounds"); return NULL; }
     return line_text_at(self->cells[xval].ch, self->cells[xval].cc);
 }
 
+size_t
+cell_as_unicode(Cell *cell, bool include_cc, Py_UCS4 *buf, char_type zero_char) {
+    size_t n = 1;
+    buf[0] = cell->ch ? cell->ch : zero_char;
+    if (include_cc) {
+        char_type cc = cell->cc;
+        Py_UCS4 cc1 = cc & CC_MASK, cc2;
+        if (cc1) {
+            buf[1] = cc1; n++;
+            cc2 = cc >> 16;
+            if (cc2) { buf[2] = cc2; n++; }
+        }
+    }
+    return n;
+}
+
+size_t
+cell_as_utf8(Cell *cell, bool include_cc, char *buf, char_type zero_char) {
+    size_t n = encode_utf8(cell->ch ? cell->ch : zero_char, buf);
+    if (include_cc) {
+        char_type cc = cell->cc;
+        Py_UCS4 cc1 = cc & CC_MASK, cc2;
+        if (cc1) {
+            n += encode_utf8(cc1, buf + n);
+            cc2 = cc >> 16;
+            if (cc2) { n += encode_utf8(cc2, buf + n); }
+        }
+    }
+    buf[n] = 0;
+    return n;
+}
+
+
 PyObject*
 unicode_in_range(Line *self, index_type start, index_type limit, bool include_cc, char leading_char) {
     size_t n = 0;
@@ -180,18 +208,8 @@ unicode_in_range(Line *self, index_type start, index_type limit, bool include_cc
         char_type ch = self->cells[i].ch;
         if (ch == 0) {
             if (previous_width == 2) { previous_width = 0; continue; };
-            ch = ' ';
-        }
-        buf[n++] = ch;
-        if (include_cc) {
-            char_type cc = self->cells[i].cc;
-            Py_UCS4 cc1 = cc & CC_MASK, cc2;
-            if (cc1) {
-                buf[n++] = cc1;
-                cc2 = cc >> 16;
-                if (cc2) buf[n++] = cc2;
-            }
         }
+        n += cell_as_unicode(self->cells + i, include_cc, buf + n, ' ');
         previous_width = self->cells[i].attrs & WIDTH_MASK;
     }
     return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, n);
@@ -202,6 +220,16 @@ as_unicode(Line* self) {
     return unicode_in_range(self, 0, xlimit_for_line(self), true, 0);
 }
 
+static PyObject*
+sprite_at(Line* self, PyObject *x) {
+#define sprite_at_doc "[x] -> Return the sprite in the specified cell"
+    unsigned long xval = PyLong_AsUnsignedLong(x);
+    if (xval >= self->xnum) { PyErr_SetString(PyExc_IndexError, "Column number out of bounds"); return NULL; }
+    Cell *c = self->cells + xval;
+    return Py_BuildValue("HHH", c->sprite_x, c->sprite_y, c->sprite_z);
+}
+
+
 static inline bool
 write_sgr(unsigned int val, Py_UCS4 *buf, index_type buflen, index_type *i) {
     static char s[20] = {0};
@@ -323,14 +351,12 @@ width(Line *self, PyObject *val) {
     return PyLong_FromUnsignedLong((unsigned long) (attrs & WIDTH_MASK));
 }
 
-#define set_sprite_position_at(x) set_sprite_position(self->cells + x, x == 0 ? NULL : self->cells + x - 1);
-
 void 
 line_add_combining_char(Line *self, uint32_t ch, unsigned int x) {
+    if (!self->cells[x].ch) return;  // dont allow adding combining chars to a null cell
     combining_type c = self->cells[x].cc;
     if (c & CC_MASK) self->cells[x].cc = (c & CC_MASK) | ( (ch & CC_MASK) << CC_SHIFT );
     else self->cells[x].cc = ch & CC_MASK;
-    set_sprite_position_at(x);
 }
 
 static PyObject*
@@ -381,7 +407,6 @@ set_text(Line* self, PyObject *args) {
         self->cells[i].bg = bg;
         self->cells[i].decoration_fg = dfg;
         self->cells[i].cc = 0;
-        set_sprite_position_at(i);
     }
 
     Py_RETURN_NONE;
@@ -414,13 +439,12 @@ line_clear_text(Line *self, unsigned int at, unsigned int num, char_type ch) {
 #define PREFIX \
     for (index_type i = at; i < MIN(self->xnum, at + num); i++) { \
         self->cells[i].ch = ch; self->cells[i].cc = 0; \
-        self->cells[i].attrs = (self->cells[i].attrs & ATTRS_MASK_WITHOUT_WIDTH) | width;
+        self->cells[i].attrs = (self->cells[i].attrs & ATTRS_MASK_WITHOUT_WIDTH) | width; \
+    }
     if (CHAR_IS_BLANK(ch)) {
         PREFIX
-        clear_sprite_position(self->cells[i]); }
     } else {
         PREFIX
-        set_sprite_position_at(i)}
     }
 }
 
@@ -450,7 +474,6 @@ line_apply_cursor(Line *self, Cursor *cursor, unsigned int at, unsigned int num,
         } else {
             attrs_type w = self->cells[i].attrs & WIDTH_MASK;
             self->cells[i].attrs = attrs | w;
-            set_sprite_position_at(i);
         }
         self->cells[i].fg = fg; self->cells[i].bg = bg;
         self->cells[i].decoration_fg = dfg;
@@ -510,7 +533,7 @@ left_shift(Line *self, PyObject *args) {
 }
  
 void 
-line_set_char(Line *self, unsigned int at, uint32_t ch, unsigned int width, Cursor *cursor, bool is_second) {
+line_set_char(Line *self, unsigned int at, uint32_t ch, unsigned int width, Cursor *cursor, bool UNUSED is_second) {
     if (cursor == NULL) {
         self->cells[at].attrs = (self->cells[at].attrs & ATTRS_MASK_WITHOUT_WIDTH) | width;
     } else {
@@ -521,8 +544,6 @@ line_set_char(Line *self, unsigned int at, uint32_t ch, unsigned int width, Curs
     }
     self->cells[at].ch = ch;
     self->cells[at].cc = 0;
-    if (!is_second && CHAR_IS_BLANK(ch)) { clear_sprite_position(self->cells[at]); }
-    else set_sprite_position_at(at);
 }
 
 static PyObject*
@@ -590,6 +611,7 @@ static PyMethodDef methods[] = {
     METHOD(width, METH_O)
     METHOD(url_start_at, METH_O)
     METHOD(url_end_at, METH_O)
+    METHOD(sprite_at, METH_O)
         
     {NULL}  /* Sentinel */
 };

kitty/lineops.h

@@ -8,18 +8,6 @@
 
 #include "data-types.h"
 
-static inline void
-update_sprites_in_line(Cell *cells, index_type xnum) {
-    if (LIKELY(xnum > 0)) {
-        if (CHAR_IS_BLANK(cells->ch)) { clear_sprite_position(cells[0]); }
-        else set_sprite_position(cells, NULL);
-        for (index_type i = 1; i < xnum; i++) {
-            if (CHAR_IS_BLANK(cells[i].ch)) { clear_sprite_position(cells[i]); }
-            else set_sprite_position(cells + i, cells + i - 1);
-        }
-    }
-}
-
 static inline void
 set_attribute_on_line(Cell *cells, uint32_t shift, uint32_t val, index_type xnum) {
     // Set a single attribute on all cells in the line
@@ -27,7 +15,6 @@ set_attribute_on_line(Cell *cells, uint32_t shift, uint32_t val, index_type xnum
     attrs_type aval = (val & mask) << shift; 
     mask = ~(mask << shift);
     for (index_type i = 0; i < xnum; i++) cells[i].attrs = (cells[i].attrs & mask) | aval;
-    if (shift == BOLD_SHIFT || shift == ITALIC_SHIFT) update_sprites_in_line(cells, xnum);
 }
 
 static inline void
@@ -67,6 +54,8 @@ index_type line_url_start_at(Line *self, index_type x);
 index_type line_url_end_at(Line *self, index_type x);
 index_type line_as_ansi(Line *self, Py_UCS4 *buf, index_type buflen);
 unsigned int line_length(Line *self);
+size_t cell_as_unicode(Cell *cell, bool include_cc, Py_UCS4 *buf, char_type);
+size_t cell_as_utf8(Cell *cell, bool include_cc, char *buf, char_type);
 PyObject* unicode_in_range(Line *self, index_type start, index_type limit, bool include_cc, char leading_char);
 
 void linebuf_init_line(LineBuf *, index_type);
@@ -79,10 +68,14 @@ void linebuf_insert_lines(LineBuf *self, unsigned int num, unsigned int y, unsig
 void linebuf_delete_lines(LineBuf *self, index_type num, index_type y, index_type bottom);
 void linebuf_set_attribute(LineBuf *, unsigned int , unsigned int );
 void linebuf_rewrap(LineBuf *self, LineBuf *other, index_type *, index_type *, HistoryBuf *);
+void linebuf_mark_line_dirty(LineBuf *self, index_type y);
+void linebuf_mark_line_clean(LineBuf *self, index_type y);
 unsigned int linebuf_char_width_at(LineBuf *self, index_type x, index_type y);
 void linebuf_refresh_sprite_positions(LineBuf *self);
 bool historybuf_resize(HistoryBuf *self, index_type lines);
 void historybuf_add_line(HistoryBuf *self, const Line *line);
 void historybuf_rewrap(HistoryBuf *self, HistoryBuf *other);
 void historybuf_init_line(HistoryBuf *self, index_type num, Line *l);
+void historybuf_mark_line_clean(HistoryBuf *self, index_type y);
+void historybuf_mark_line_dirty(HistoryBuf *self, index_type y);
 void historybuf_refresh_sprite_positions(HistoryBuf *self);

kitty/main.py

@@ -3,7 +3,6 @@
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
 import argparse
-import errno
 import locale
 import os
 import signal
@@ -23,13 +22,16 @@ from .fast_data_types import (
     GL_VERSION_REQUIRED, GLFW_CONTEXT_VERSION_MAJOR,
     GLFW_CONTEXT_VERSION_MINOR, GLFW_DECORATED, GLFW_OPENGL_CORE_PROFILE,
     GLFW_OPENGL_FORWARD_COMPAT, GLFW_OPENGL_PROFILE, GLFW_SAMPLES,
-    GLFW_STENCIL_BITS, GLFWWindow, change_wcwidth, check_for_extensions,
-    clear_buffers, glewInit, glfw_init, glfw_init_hint_string,
-    glfw_swap_interval, glfw_terminate, glfw_window_hint, set_logical_dpi,
-    set_options
+    GLFW_STENCIL_BITS, GLFWWindow, change_wcwidth, clear_buffers, gl_init,
+    glfw_init, glfw_init_hint_string, glfw_swap_interval, glfw_terminate,
+    glfw_window_hint, install_sigchld_handler, set_logical_dpi, set_options
 )
+from .fonts.box_drawing import set_scale
 from .layout import all_layouts
-from .utils import color_as_int, detach, get_logical_dpi, safe_print
+from .utils import (
+    color_as_int, detach, end_startup_notification, get_logical_dpi,
+    init_startup_notification, safe_print
+)
 
 try:
     from .fast_data_types import GLFW_X11_WM_CLASS_NAME, GLFW_X11_WM_CLASS_CLASS
@@ -109,7 +111,7 @@ def option_parser():
                ' child process. Useful for debugging.')
     )
     a(
-        '--debug-kitty-gl',
+        '--debug-gl',
         action='store_true',
         default=False,
         help=_('Debug OpenGL commands. This will cause all OpenGL calls'
@@ -153,13 +155,12 @@ def setup_opengl(opts):
         glfw_window_hint(GLFW_STENCIL_BITS, 8)
 
 
-def initialize_window(window, opts):
-    set_logical_dpi(*get_logical_dpi())
+def initialize_window(window, opts, debug_gl=False):
     viewport_size.width, viewport_size.height = window.get_framebuffer_size()
     w, h = window.get_window_size()
     viewport_size.x_ratio = viewport_size.width / float(w)
     viewport_size.y_ratio = viewport_size.height / float(h)
-    glewInit()
+    gl_init(iswayland, debug_gl)
     glfw_swap_interval(0)
     clear_buffers(window.swap_buffers, color_as_int(opts.background))
     # We dont turn this on as it causes rendering performance to be much worse,
@@ -170,6 +171,7 @@ def initialize_window(window, opts):
 def run_app(opts, args):
     set_options(opts)
     setup_opengl(opts)
+    set_scale(opts.box_drawing_scale)
     load_cached_values()
     if 'window-size' in cached_values and opts.remember_window_size:
         ws = cached_values['window-size']
@@ -189,10 +191,10 @@ def run_app(opts, args):
         viewport_size.width = 640
         viewport_size.height = 400
         window = GLFWWindow(viewport_size.width, viewport_size.height, args.cls)
+    startup_ctx = init_startup_notification(window)
     window.make_context_current()
     if isosx:
         from .fast_data_types import cocoa_make_window_resizable, cocoa_create_global_menu, cocoa_init
-        check_for_extensions()
         cocoa_init()
         cocoa_create_global_menu()
         if opts.macos_hide_titlebar:
@@ -200,9 +202,11 @@ def run_app(opts, args):
     elif not iswayland:  # no window icons on wayland
         with open(logo_data_file, 'rb') as f:
             window.set_icon(f.read(), 256, 256)
-    initialize_window(window, opts)
+    set_logical_dpi(*get_logical_dpi())
+    initialize_window(window, opts, args.debug_gl)
     boss = Boss(window, opts, args)
     boss.start()
+    end_startup_notification(startup_ctx)
     try:
         boss.child_monitor.main_loop()
     finally:
@@ -245,19 +249,6 @@ def setup_profiling(args):
             print('To view the graphical call data, use: kcachegrind', cg)
 
 
-def reap_zombies(*a):
-    while True:
-        try:
-            pid, status = os.waitpid(-1, os.WNOHANG)
-            if pid == 0:
-                break
-        except OSError as err:
-            if err.errno != errno.EINTR:
-                break
-        except Exception:
-            break
-
-
 def main():
     try:
         sys.setswitchinterval(1000.0)  # we have only a single python thread
@@ -302,7 +293,7 @@ def main():
     try:
         with setup_profiling(args):
             # Avoid needing to launch threads to reap zombies
-            signal.signal(signal.SIGCHLD, reap_zombies)
+            install_sigchld_handler()
             run_app(opts, args)
             signal.signal(signal.SIGCHLD, signal.SIG_DFL)
     finally:

kitty/mouse.c

@@ -63,11 +63,11 @@ encode_mouse_event(Window *w, int button, MouseAction action, int mods) {
             return snprintf(mouse_event_buf, sizeof(mouse_event_buf), "\033[<%d;%d;%d%s", cb, x, y, action == RELEASE ? "m" : "M");
             break;
         case URXVT_PROTOCOL:
-            return snprintf(mouse_event_buf, sizeof(mouse_event_buf), "\033[%d;%d;%dM", cb, x, y);
+            return snprintf(mouse_event_buf, sizeof(mouse_event_buf), "\033[%d;%d;%dM", cb + 32, x, y);
             break;
         case UTF8_PROTOCOL:
-            mouse_event_buf[0] = 033; mouse_event_buf[1] = '['; mouse_event_buf[2] = cb + 32; 
-            unsigned int sz = 3;
+            mouse_event_buf[0] = 033; mouse_event_buf[1] = '['; mouse_event_buf[2] = 'M'; mouse_event_buf[3] = cb + 32; 
+            unsigned int sz = 4;
             sz += encode_utf8(x + 32, mouse_event_buf + sz);
             sz += encode_utf8(y + 32, mouse_event_buf + sz);
             return sz;
@@ -163,7 +163,13 @@ detect_url(Window *w, Screen *screen, unsigned int x, unsigned int y) {
 
 
 HANDLER(handle_move_event) {
-    unsigned int x, y;
+    unsigned int x = 0, y = 0;
+    if (OPT(focus_follows_mouse)) {
+        Tab *t = global_state.tabs + global_state.active_tab;
+        if (window_idx != t->active_window) {
+            call_boss(switch_focus_to, "I", window_idx);
+        }
+    }
     if (!cell_for_pos(w, &x, &y)) return;
     Screen *screen = w->render_data.screen;
     detect_url(w, screen, x, y);
@@ -321,7 +327,7 @@ void
 mouse_event(int button, int modifiers) {
     MouseShape old_cursor = mouse_cursor_shape;
     bool in_tab_bar;
-    unsigned int window_idx;
+    unsigned int window_idx = 0;
     Window *w = window_for_event(&window_idx, &in_tab_bar);
     if (in_tab_bar) { 
         mouse_cursor_shape = HAND;
@@ -340,7 +346,7 @@ scroll_event(double UNUSED xoffset, double yoffset) {
     if (s == 0) return;
     bool upwards = s > 0;
     bool in_tab_bar;
-    unsigned int window_idx;
+    unsigned int window_idx = 0;
     Window *w = window_for_event(&window_idx, &in_tab_bar);
     if (w) {
         Screen *screen = w->render_data.screen;

kitty/parser.c

@@ -10,14 +10,16 @@
 #include "graphics.h"
 #include <time.h>
 
+extern PyTypeObject Screen_Type;
+
 // utils {{{
-static unsigned long pow10[] = {
+static uint64_t pow10[] = {
     1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000
 };
 
-static inline unsigned long
+static inline uint64_t
 utoi(uint32_t *buf, unsigned int sz) {
-    unsigned long ans = 0;
+    uint64_t ans = 0;
     uint32_t *p = buf;
     // Ignore leading zeros
     while(sz > 0) {
@@ -25,12 +27,23 @@ utoi(uint32_t *buf, unsigned int sz) {
         else break;
     }
     if (sz < sizeof(pow10)/sizeof(pow10[0])) {
-        for (int i = sz-1, j=0; i >=0; i--, j++) {
+        for (int i = sz-1, j=0; i >= 0; i--, j++) {
             ans += (p[i] - '0') * pow10[j];
         }
     }
     return ans;
 }
+
+
+static inline const char*
+utf8(char_type codepoint) {
+    if (!codepoint) return "";
+    static char buf[8];
+    int n = encode_utf8(codepoint, buf);
+    buf[n] = 0;
+    return buf;
+}
+
 // }}}
 
 // Macros {{{
@@ -66,7 +79,9 @@ _report_params(PyObject *dump_callback, const char *name, unsigned int *params,
     static char buf[MAX_PARAMS*3] = {0};
     unsigned int i, p;
     for(i = 0, p=0; i < count && p < MAX_PARAMS*3-20; i++) {
-        p += snprintf(buf + p, MAX_PARAMS*3 - p, "%u ", params[i]);
+        int n = snprintf(buf + p, MAX_PARAMS*3 - p, "%u ", params[i]);
+        if (n < 0) break;
+        p += n;
     }
     buf[p] = 0;
     Py_XDECREF(PyObject_CallFunction(dump_callback, "ss", name, buf)); PyErr_Clear();
@@ -328,6 +343,7 @@ dispatch_osc(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
 // CSI mode {{{
 #define CSI_SECONDARY \
         case ';': \
+        case ':': \
         case '"': \
         case '*': \
         case '\'': \
@@ -352,6 +368,148 @@ restore_cursor(Screen *s, unsigned int UNUSED param, bool private) {
     else screen_restore_cursor(s);
 }
 
+static inline const char*
+repr_csi_params(unsigned int *params, unsigned int num_params) {
+    if (!num_params) return "";
+    static char buf[256];
+    unsigned int pos = 0;
+    while (pos < 200 && num_params && sizeof(buf) > pos + 1) {
+        const char *fmt = num_params > 1 ? "%u " : "%u";
+        int ret = snprintf(buf + pos, sizeof(buf) - pos - 1, fmt, params[num_params--]);
+        if (ret < 0) return "An error occurred formatting the params array";
+        pos += ret;
+    }
+    buf[pos] = 0;
+    return buf;
+}
+
+static inline void
+parse_sgr(Screen *screen, uint32_t *buf, unsigned int num, unsigned int *params, PyObject DUMP_UNUSED *dump_callback) {
+    enum State { START, NORMAL, MULTIPLE, COLOR, COLOR1, COLOR3 };
+    enum State state = START;
+    unsigned int num_params, num_start, i;
+
+#define READ_PARAM { params[num_params++] = utoi(buf + num_start, i - num_start); }
+#define SEND_SGR { REPORT_PARAMS(select_graphic_rendition, params, num_params); select_graphic_rendition(screen, params, num_params); state = START; num_params = 0; }
+
+    for (i=0, num_start=0, num_params=0; i < num && num_params < MAX_PARAMS; i++) {
+        switch(buf[i]) {
+            IS_DIGIT
+                switch(state) {
+                    case START:
+                        num_start = i;
+                        state = NORMAL;
+                        num_params = 0;
+                        break;
+                    default:
+                        break;
+                }
+                break;
+            case ';':
+                switch(state) {
+                    case START:
+                        params[num_params++] = 0;
+                        SEND_SGR;
+                        break;
+                    case NORMAL:
+                        READ_PARAM;
+                        switch(params[0]) {
+                            case 38:
+                            case 48:
+                            case 58:
+                                state = COLOR;
+                                num_start = i + 1;
+                                break;
+                            default:
+                                SEND_SGR;
+                                break;
+                        }
+                        break;
+                    case MULTIPLE:
+                        READ_PARAM;
+                        SEND_SGR;
+                        break;
+                    case COLOR:
+                        READ_PARAM;
+                        switch(params[1]) {
+                            case 2:
+                                state = COLOR3;
+                                break;
+                            case 5:
+                                state = COLOR1;
+                                break;
+                            default:
+                                REPORT_ERROR("Invalid SGR color code with unknown color type: %u", params[1]);
+                                return;
+                        }
+                        num_start = i + 1;
+                        break;
+                    case COLOR1:
+                        READ_PARAM;
+                        SEND_SGR;
+                        break;
+                    case COLOR3:
+                        READ_PARAM;
+                        if (num_params == 5) { SEND_SGR; }
+                        else num_start = i + 1;
+                        break;
+                }
+                break;
+            case ':':
+                switch(state) {
+                    case START:
+                        REPORT_ERROR("Invalid SGR code containing ':' at an invalid location: %u", i);
+                        return;
+                    case NORMAL:
+                        READ_PARAM;
+                        state = MULTIPLE;
+                        num_start = i + 1;
+                        break;
+                    case MULTIPLE:
+                        READ_PARAM;
+                        num_start = i + 1;
+                        break;
+                    case COLOR: 
+                    case COLOR1: 
+                    case COLOR3:
+                        REPORT_ERROR("Invalid SGR code containing disallowed character: %s", utf8(buf[i]));
+                        return;
+                }
+                break;
+            default:
+                REPORT_ERROR("Invalid SGR code containing disallowed character: %s", utf8(buf[i]));
+                return;
+        }
+    }
+    switch(state) {
+        case START:
+            if (num_params < MAX_PARAMS) params[num_params++] = 0;
+            SEND_SGR;
+            break;
+        case COLOR1:
+        case NORMAL:
+        case MULTIPLE:
+            if (i > num_start && num_params < MAX_PARAMS) { READ_PARAM; }
+            if (num_params) { SEND_SGR; }
+            else { REPORT_ERROR("Incomplete SGR code"); }
+            break;
+        case COLOR:
+            REPORT_ERROR("Invalid SGR code containing incomplete semi-colon separated color sequence");
+            break;
+        case COLOR3:
+            if (i > num_start && num_params < MAX_PARAMS) READ_PARAM;
+            if (num_params != 5) { 
+                REPORT_ERROR("Invalid SGR code containing incomplete semi-colon separated color sequence");
+                break;
+            }
+            if (num_params) { SEND_SGR; }
+            else { REPORT_ERROR("Incomplete SGR code"); }
+            break;
+    }
+#undef READ_PARAM
+#undef SEND_SGR
+}
+
 static inline void
 dispatch_csi(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
 #define CALL_CSI_HANDLER1(name, defval) \
@@ -394,12 +552,6 @@ dispatch_csi(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
     } \
     break;
 
-#define CSI_HANDLER_MULTIPLE(name) \
-    REPORT_PARAMS(name, params, num_params); \
-    name(screen, params, num_params); \
-    break;
-
-
     char start_modifier = 0, end_modifier = 0;
     uint32_t *buf = screen->parser_buf, code = screen->parser_buf[screen->parser_buf_pos];
     unsigned int num = screen->parser_buf_pos, start, i, num_params=0, p1, p2;
@@ -409,6 +561,10 @@ dispatch_csi(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
         start_modifier = (char)screen->parser_buf[0];
         buf++; num--;
     }
+    if (code == SGR && !start_modifier) {
+        parse_sgr(screen, buf, num, params, dump_callback);
+        return;
+    }
     if (num > 0) {
         switch(buf[num-1]) {
             CSI_SECONDARY
@@ -476,8 +632,6 @@ dispatch_csi(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
             SET_MODE(screen_set_mode); 
         case RM: 
             SET_MODE(screen_reset_mode); 
-        case SGR: 
-            CSI_HANDLER_MULTIPLE(select_graphic_rendition); 
         case DSR: 
             CALL_CSI_HANDLER1P(report_device_status, 0, '?'); 
         case SC: 
@@ -497,11 +651,11 @@ dispatch_csi(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
                 // DECRQM
                 CALL_CSI_HANDLER1P(report_mode_status, 0, '?'); 
             } else {
-                REPORT_ERROR("Unknown DECSTR CSI sequence with start and end modifiers: 0x%x 0x%x", start_modifier, end_modifier);
+                REPORT_ERROR("Unknown DECSTR CSI sequence with start and end modifiers: '%c' '%c'", start_modifier, end_modifier);
             }
             break;
         default:
-            REPORT_ERROR("Unknown CSI code: 0x%x", code);
+            REPORT_ERROR("Unknown CSI code: '%s' with start_modifier: '%c' and end_modifier: '%c' and parameters: '%s'", utf8(code), start_modifier, end_modifier, repr_csi_params(params, num_params));
     }
 }
 // }}}
@@ -894,10 +1048,12 @@ dispatch_unicode_char(Screen *screen, uint32_t codepoint, PyObject DUMP_UNUSED *
 #undef HANDLE
 }
 
+extern uint32_t *latin1_charset;
+
 static inline void 
 _parse_bytes(Screen *screen, uint8_t *buf, Py_ssize_t len, PyObject DUMP_UNUSED *dump_callback) {
     uint32_t prev = screen->utf8_state;
-    for (unsigned int i = 0; i < len; i++) {
+    for (unsigned int i = 0; i < (unsigned int)len; i++) {
         if (screen->use_latin1) dispatch_unicode_char(screen, latin1_charset[buf[i]], dump_callback);
         else {
             switch (decode_utf8(&screen->utf8_state, &screen->utf8_codepoint, buf[i])) {

kitty/rewrap.h

@@ -34,11 +34,11 @@
         linebuf_clear_line(dest, dest->ynum - 1); \
     } else dest_y++; \
     init_dest_line(dest_y); \
-    dest->continued_map[dest_y] = continued;
+    dest->line_attrs[dest_y] = continued ? CONTINUED_MASK : 0;
 #endif
 
 #ifndef is_src_line_continued
-#define is_src_line_continued(src_y) (src_y < src->ynum - 1 ? src->continued_map[src_y + 1] : false)
+#define is_src_line_continued(src_y) (src_y < src->ynum - 1 ? (src->line_attrs[src_y + 1] & CONTINUED_MASK) : false)
 #endif
 
 static inline void 

kitty/screen.c

@@ -8,6 +8,7 @@
 #define EXTRA_INIT PyModule_AddIntMacro(module, SCROLL_LINE); PyModule_AddIntMacro(module, SCROLL_PAGE); PyModule_AddIntMacro(module, SCROLL_FULL);
 
 #include "state.h"
+#include "fonts.h"
 #include "lineops.h"
 #include "screen.h"
 #include <structmember.h>
@@ -81,7 +82,8 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
         self->main_linebuf = alloc_linebuf(lines, columns); self->alt_linebuf = alloc_linebuf(lines, columns);
         self->linebuf = self->main_linebuf;
         self->historybuf = alloc_historybuf(MAX(scrollback, lines), columns);
-        self->main_grman = grman_alloc(); self->alt_grman = grman_alloc();
+        self->main_grman = grman_alloc(); 
+        self->alt_grman = grman_alloc();
         self->grman = self->main_grman;
         self->main_tabstops = PyMem_Calloc(2 * self->columns, sizeof(bool));
         if (self->cursor == NULL || self->main_linebuf == NULL || self->alt_linebuf == NULL || self->main_tabstops == NULL || self->historybuf == NULL || self->main_grman == NULL || self->alt_grman == NULL || self->color_profile == NULL) {
@@ -139,7 +141,8 @@ screen_resize(Screen *self, unsigned int lines, unsigned int columns) {
 
     bool is_main = self->linebuf == self->main_linebuf;
     index_type num_content_lines_before, num_content_lines_after;
-    index_type num_content_lines;
+    index_type num_content_lines = 0, old_columns = self->columns;
+    bool cursor_on_last_content_line = false;
 
     // Resize main linebuf
     HistoryBuf *nh = realloc_hb(self->historybuf, self->historybuf->ynum, columns);
@@ -148,7 +151,10 @@ screen_resize(Screen *self, unsigned int lines, unsigned int columns) {
     LineBuf *n = realloc_lb(self->main_linebuf, lines, columns, &num_content_lines_before, &num_content_lines_after, self->historybuf);
     if (n == NULL) return false;
     Py_CLEAR(self->main_linebuf); self->main_linebuf = n;
-    if (is_main) num_content_lines = num_content_lines_after;
+    if (is_main) {
+        num_content_lines = num_content_lines_after;
+        cursor_on_last_content_line = num_content_lines_before == self->cursor->y + 1 || !num_content_lines_before;
+    }
     grman_resize(self->main_grman, self->lines, lines, self->columns, columns);
 
     // Resize alt linebuf
@@ -173,9 +179,15 @@ screen_resize(Screen *self, unsigned int lines, unsigned int columns) {
     self->selection = EMPTY_SELECTION;
     self->url_range = EMPTY_SELECTION;
 
-    // Ensure cursor is on a blank line
+    // Ensure cursor is on the correct line
     self->cursor->x = 0;
-    self->cursor->y = MIN(self->lines - 1, num_content_lines); 
+    if (cursor_on_last_content_line) {
+        index_type delta;
+        if (self->columns > old_columns) delta = 1;
+        else delta = (old_columns / self->columns) + 1;
+        self->cursor->y = num_content_lines > delta ? num_content_lines - delta : 0;
+    } else self->cursor->y = num_content_lines;
+    self->cursor->y = MIN(self->cursor->y, self->lines - 1);
     if (num_content_lines >= self->lines) screen_index(self);
 
     return true;
@@ -187,13 +199,6 @@ screen_rescale_images(Screen *self, unsigned int old_cell_width, unsigned int ol
     grman_rescale(self->alt_grman, old_cell_width, old_cell_height);
 }
 
-static void
-screen_refresh_sprite_positions(Screen *self) {
-    linebuf_refresh_sprite_positions(self->main_linebuf);
-    linebuf_refresh_sprite_positions(self->alt_linebuf);
-    historybuf_refresh_sprite_positions(self->historybuf);
-}
-
 
 static bool 
 screen_change_scrollback_size(Screen *self, unsigned int size) {
@@ -213,7 +218,8 @@ static void
 dealloc(Screen* self) {
     pthread_mutex_destroy(&self->read_buf_lock);
     pthread_mutex_destroy(&self->write_buf_lock);
-    Py_CLEAR(self->grman); Py_CLEAR(self->alt_grman);
+    Py_CLEAR(self->main_grman); 
+    Py_CLEAR(self->alt_grman);
     PyMem_RawFree(self->write_buf);
     Py_CLEAR(self->callbacks);
     Py_CLEAR(self->test_child);
@@ -275,12 +281,13 @@ void
 screen_draw(Screen *self, uint32_t och) {
     if (is_ignored_char(och)) return;
     uint32_t ch = och < 256 ? self->g_charset[och] : och;
-    unsigned int char_width = safe_wcwidth(ch);
+    bool is_cc = is_combining_char(ch);
+    unsigned int char_width = is_cc ? 0 : safe_wcwidth(ch);
     if (self->columns - self->cursor->x < char_width) {
         if (self->modes.mDECAWM) {
             screen_carriage_return(self);
             screen_linefeed(self);
-            self->linebuf->continued_map[self->cursor->y] = true;
+            self->linebuf->line_attrs[self->cursor->y] |= CONTINUED_MASK;
         } else {
             self->cursor->x = self->columns - char_width;
         }
@@ -297,15 +304,18 @@ screen_draw(Screen *self, uint32_t och) {
             self->cursor->x++;
         }
         self->is_dirty = true;
+        linebuf_mark_line_dirty(self->linebuf, self->cursor->y);
     } else if (is_combining_char(ch)) {
         if (self->cursor->x > 0) {
             linebuf_init_line(self->linebuf, self->cursor->y);
             line_add_combining_char(self->linebuf->line, ch, self->cursor->x - 1);
             self->is_dirty = true;
+            linebuf_mark_line_dirty(self->linebuf, self->cursor->y);
         } else if (self->cursor->y > 0) {
             linebuf_init_line(self->linebuf, self->cursor->y - 1);
             line_add_combining_char(self->linebuf->line, ch, self->columns - 1);
             self->is_dirty = true;
+            linebuf_mark_line_dirty(self->linebuf, self->cursor->y);
         }
     }
 }
@@ -321,17 +331,20 @@ screen_align(Screen *self) {
 
 // Graphics {{{
 
-void screen_alignment_display(Screen *self) {
+void 
+screen_alignment_display(Screen *self) {
     // http://www.vt100.net/docs/vt510-rm/DECALN.html 
     screen_cursor_position(self, 1, 1);
     self->margin_top = 0; self->margin_bottom = self->lines - 1;
     for (unsigned int y = 0; y < self->linebuf->ynum; y++) {
         linebuf_init_line(self->linebuf, y);
         line_clear_text(self->linebuf->line, 0, self->linebuf->xnum, 'E');
+        linebuf_mark_line_dirty(self->linebuf, y);
     }
 }
 
-void select_graphic_rendition(Screen *self, unsigned int *params, unsigned int count) {
+void 
+select_graphic_rendition(Screen *self, unsigned int *params, unsigned int count) {
 #define SET_COLOR(which) \
     if (i < count) { \
         attr = params[i++];\
@@ -423,7 +436,8 @@ screen_handle_graphics_command(Screen *self, const GraphicsCommand *cmd, const u
     if (response != NULL) write_to_child(self, response, strlen(response));
     if (x != self->cursor->x || y != self->cursor->y) {
         if (self->cursor->x >= self->columns) { self->cursor->x = 0; self->cursor->y++; }
-        if (self->cursor->y > self->margin_bottom) { screen_scroll(self, self->cursor->y - self->margin_bottom); }
+        if (self->cursor->y > self->margin_bottom) screen_scroll(self, self->cursor->y - self->margin_bottom); 
+        screen_ensure_bounds(self, false);
     }
 }
 // }}}
@@ -688,7 +702,6 @@ screen_index(Screen *self) {
 void 
 screen_scroll(Screen *self, unsigned int count) {
     // Scroll the screen up by count lines, not moving the cursor
-    count = MIN(self->lines, count);
     unsigned int top = self->margin_top, bottom = self->margin_bottom;
     while (count > 0) {
         count--;
@@ -823,7 +836,8 @@ screen_cursor_to_line(Screen *self, unsigned int line) {
 
 // Editing {{{
 
-void screen_erase_in_line(Screen *self, unsigned int how, bool private) {
+void 
+screen_erase_in_line(Screen *self, unsigned int how, bool private) {
     /*Erases a line in a specific way.
 
         :param int how: defines the way the line should be erased in:
@@ -859,10 +873,12 @@ void screen_erase_in_line(Screen *self, unsigned int how, bool private) {
             line_apply_cursor(self->linebuf->line, self->cursor, s, n, true);
         }
         self->is_dirty = true;
+        linebuf_mark_line_dirty(self->linebuf, self->cursor->y);
     }
 }
 
-void screen_erase_in_display(Screen *self, unsigned int how, bool private) {
+void 
+screen_erase_in_display(Screen *self, unsigned int how, bool private) {
     /* Erases display in a specific way.
 
         :param int how: defines the way the line should be erased in:
@@ -895,6 +911,7 @@ void screen_erase_in_display(Screen *self, unsigned int how, bool private) {
             } else {
                 line_apply_cursor(self->linebuf->line, self->cursor, 0, self->columns, true);
             }
+            linebuf_mark_line_dirty(self->linebuf, i);
         }
         self->is_dirty = true;
     }
@@ -903,7 +920,8 @@ void screen_erase_in_display(Screen *self, unsigned int how, bool private) {
     }
 }
 
-void screen_insert_lines(Screen *self, unsigned int count) {
+void 
+screen_insert_lines(Screen *self, unsigned int count) {
     unsigned int top = self->margin_top, bottom = self->margin_bottom;
     if (count == 0) count = 1;
     if (top <= self->cursor->y && self->cursor->y <= bottom) {
@@ -913,7 +931,8 @@ void screen_insert_lines(Screen *self, unsigned int count) {
     }
 }
 
-void screen_delete_lines(Screen *self, unsigned int count) {
+void 
+screen_delete_lines(Screen *self, unsigned int count) {
     unsigned int top = self->margin_top, bottom = self->margin_bottom;
     if (count == 0) count = 1;
     if (top <= self->cursor->y && self->cursor->y <= bottom) {
@@ -923,7 +942,8 @@ void screen_delete_lines(Screen *self, unsigned int count) {
     }
 }
 
-void screen_insert_characters(Screen *self, unsigned int count) {
+void 
+screen_insert_characters(Screen *self, unsigned int count) {
     unsigned int top = self->margin_top, bottom = self->margin_bottom;
     if (count == 0) count = 1;
     if (top <= self->cursor->y && self->cursor->y <= bottom) {
@@ -932,11 +952,13 @@ void screen_insert_characters(Screen *self, unsigned int count) {
         linebuf_init_line(self->linebuf, self->cursor->y);
         line_right_shift(self->linebuf->line, x, num);
         line_apply_cursor(self->linebuf->line, self->cursor, x, num, true);
+        linebuf_mark_line_dirty(self->linebuf, self->cursor->y);
         self->is_dirty = true;
     }
 }
 
-void screen_delete_characters(Screen *self, unsigned int count) {
+void 
+screen_delete_characters(Screen *self, unsigned int count) {
     // Delete characters, later characters are moved left
     unsigned int top = self->margin_top, bottom = self->margin_bottom;
     if (count == 0) count = 1;
@@ -946,17 +968,20 @@ void screen_delete_characters(Screen *self, unsigned int count) {
         linebuf_init_line(self->linebuf, self->cursor->y);
         left_shift_line(self->linebuf->line, x, num);
         line_apply_cursor(self->linebuf->line, self->cursor, self->columns - num, num, true);
+        linebuf_mark_line_dirty(self->linebuf, self->cursor->y);
         self->is_dirty = true;
     }
 }
 
-void screen_erase_characters(Screen *self, unsigned int count) {
+void 
+screen_erase_characters(Screen *self, unsigned int count) {
     // Delete characters replacing them by spaces
     if (count == 0) count = 1;
     unsigned int x = self->cursor->x;
     unsigned int num = MIN(self->columns - x, count);
     linebuf_init_line(self->linebuf, self->cursor->y);
     line_apply_cursor(self->linebuf->line, self->cursor, x, num, true);
+    linebuf_mark_line_dirty(self->linebuf, self->cursor->y);
     self->is_dirty = true;
 }
 
@@ -1149,16 +1174,27 @@ screen_reset_dirty(Screen *self) {
 void
 screen_update_cell_data(Screen *self, void *address, size_t UNUSED sz) {
     unsigned int history_line_added_count = self->history_line_added_count;
+    index_type lnum;
     bool selection_must_be_cleared = self->is_dirty ? true : false;
     if (self->scrolled_by) self->scrolled_by = MIN(self->scrolled_by + history_line_added_count, self->historybuf->count);
     screen_reset_dirty(self);
     self->scroll_changed = false;
     for (index_type y = 0; y < MIN(self->lines, self->scrolled_by); y++) {
-        historybuf_init_line(self->historybuf, self->scrolled_by - 1 - y, self->historybuf->line);
+        lnum = self->scrolled_by - 1 - y;
+        historybuf_init_line(self->historybuf, lnum, self->historybuf->line);
+        if (self->historybuf->line->has_dirty_text) {
+            render_line(self->historybuf->line);
+            historybuf_mark_line_clean(self->historybuf, lnum);
+        }
         update_line_data(self->historybuf->line, y, address);
     }
     for (index_type y = self->scrolled_by; y < self->lines; y++) {
-        linebuf_init_line(self->linebuf, y - self->scrolled_by);
+        lnum = y - self->scrolled_by;
+        linebuf_init_line(self->linebuf, lnum);
+        if (self->linebuf->line->has_dirty_text) {
+            render_line(self->linebuf->line);
+            linebuf_mark_line_clean(self->linebuf, lnum);
+        }
         update_line_data(self->linebuf->line, y, address);
     }
     if (selection_must_be_cleared) {
@@ -1246,6 +1282,17 @@ screen_url_range(Screen *self, uint32_t *data) {
 #define WRAP1E(name, defval, ...) static PyObject* name(Screen *self, PyObject *args) { unsigned int v=defval; if(!PyArg_ParseTuple(args, "|I", &v)) return NULL; screen_##name(self, v, __VA_ARGS__); Py_RETURN_NONE; }
 #define WRAP2(name, defval1, defval2) static PyObject* name(Screen *self, PyObject *args) { unsigned int a=defval1, b=defval2; if(!PyArg_ParseTuple(args, "|II", &a, &b)) return NULL; screen_##name(self, a, b); Py_RETURN_NONE; }
 
+static PyObject*
+refresh_sprite_positions(Screen *self) {
+    self->is_dirty = true;
+    for (index_type i = 0; i < self->lines; i++) {
+        linebuf_mark_line_dirty(self->main_linebuf, i);
+        linebuf_mark_line_dirty(self->alt_linebuf, i);
+    }
+    for (index_type i = 0; i < self->historybuf->count; i++) historybuf_mark_line_dirty(self->historybuf, i);
+    Py_RETURN_NONE;
+}
+
 static PyObject*
 screen_wcswidth(Screen UNUSED *self, PyObject *str) {
     if (PyUnicode_READY(str) != 0) return NULL;
@@ -1253,7 +1300,11 @@ screen_wcswidth(Screen UNUSED *self, PyObject *str) {
     void *data = PyUnicode_DATA(str);
     Py_ssize_t len = PyUnicode_GET_LENGTH(str), i;
     unsigned long ans = 0;
-    for (i = 0; i < len; i++) ans += safe_wcwidth(PyUnicode_READ(kind, data, i));
+    for (i = 0; i < len; i++) {
+        char_type ch = PyUnicode_READ(kind, data, i);
+        bool is_cc = is_combining_char(ch);
+        ans += is_cc ? 0 : safe_wcwidth(ch);
+    }
     return PyLong_FromUnsignedLong(ans);
 }
 
@@ -1352,7 +1403,6 @@ cursor_up(Screen *self, PyObject *args) {
 
 WRAP0x(index)
 WRAP0(reverse_index)
-WRAP0(refresh_sprite_positions)
 WRAP0(reset)
 WRAP0(set_tab_stop)
 WRAP1(clear_tab_stop, 0)
@@ -1564,6 +1614,7 @@ static PyMethodDef methods[] = {
     MND(cursor_forward, METH_VARARGS)
     {"wcswidth", (PyCFunction)screen_wcswidth, METH_O, ""},
     {"index", (PyCFunction)xxx_index, METH_VARARGS, ""},
+    MND(refresh_sprite_positions, METH_NOARGS)
     MND(tab, METH_NOARGS)
     MND(backspace, METH_NOARGS)
     MND(linefeed, METH_NOARGS)
@@ -1571,7 +1622,6 @@ static PyMethodDef methods[] = {
     MND(set_tab_stop, METH_NOARGS)
     MND(clear_tab_stop, METH_VARARGS)
     MND(reverse_index, METH_NOARGS)
-    MND(refresh_sprite_positions, METH_NOARGS)
     MND(mark_as_dirty, METH_NOARGS)
     MND(resize, METH_VARARGS)
     MND(set_margins, METH_VARARGS)

kitty/screen.h

@@ -56,7 +56,6 @@ typedef struct {
     pthread_mutex_t read_buf_lock, write_buf_lock;
 
 } Screen;
-PyTypeObject Screen_Type;
 
 
 void parse_worker(Screen *screen, PyObject *dump_callback);
@@ -108,7 +107,6 @@ void set_icon(Screen *self, PyObject*);
 void set_dynamic_color(Screen *self, unsigned int code, PyObject*);
 void set_color_table_color(Screen *self, unsigned int code, PyObject*);
 uint32_t* translation_table(uint32_t which);
-uint32_t *latin1_charset;
 void screen_request_capabilities(Screen *, PyObject *);
 void report_device_attributes(Screen *self, unsigned int UNUSED mode, char start_modifier);
 void select_graphic_rendition(Screen *self, unsigned int *params, unsigned int count);

kitty/shaders.c

@@ -6,6 +6,7 @@
  */
 
 #include "gl.h"
+#include "fonts.h"
 #include <sys/sysctl.h>
 
 enum { CELL_PROGRAM, CELL_BACKGROUND_PROGRAM, CELL_SPECIAL_PROGRAM, CELL_FOREGROUND_PROGRAM, CURSOR_PROGRAM, BORDERS_PROGRAM, GRAPHICS_PROGRAM, NUM_PROGRAMS };
@@ -18,21 +19,15 @@ typedef struct {
     GLuint texture_id;
     GLenum texture_unit;
     GLint max_texture_size, max_array_texture_layers;
-    PyObject *render_cell;
 } SpriteMap;
 
 static SpriteMap sprite_map = { .xnum = 1, .ynum = 1, .last_num_of_layers = 1, .last_ynum = -1, .texture_unit = GL_TEXTURE0 };
 
-#ifdef __APPLE__ 
-#define glCopyImageSubData(...)
-#define GLEW_ARB_copy_image false
-#endif
-
 static bool copy_image_warned = false;
 
 static void
 copy_image_sub_data(GLuint src_texture_id, GLuint dest_texture_id, unsigned int width, unsigned int height, unsigned int num_levels) {
-    if (!GLEW_ARB_copy_image) {
+    if (!GLAD_GL_ARB_copy_image) {
         // ARB_copy_image not available, do a slow roundtrip copy
         if (!copy_image_warned) {
             copy_image_warned = true;
@@ -41,15 +36,15 @@ copy_image_sub_data(GLuint src_texture_id, GLuint dest_texture_id, unsigned int
         uint8_t *src = malloc(5 * width * height * num_levels);
         if (src == NULL) { fatal("Out of memory."); }
         uint8_t *dest = src + (4 * width * height * num_levels);
-        glBindTexture(GL_TEXTURE_2D_ARRAY, src_texture_id); check_gl();
-        glGetTexImage(GL_TEXTURE_2D_ARRAY, 0, GL_RGBA, GL_UNSIGNED_BYTE, src); check_gl();
-        glBindTexture(GL_TEXTURE_2D_ARRAY, dest_texture_id); check_gl();
-        glPixelStorei(GL_UNPACK_ALIGNMENT, 1); check_gl();
+        glBindTexture(GL_TEXTURE_2D_ARRAY, src_texture_id); 
+        glGetTexImage(GL_TEXTURE_2D_ARRAY, 0, GL_RGBA, GL_UNSIGNED_BYTE, src); 
+        glBindTexture(GL_TEXTURE_2D_ARRAY, dest_texture_id); 
+        glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 
         for(size_t i = 0; i < width * height * num_levels; i++) dest[i] = src[4*i];
-        glTexSubImage3D(GL_TEXTURE_2D_ARRAY, 0, 0, 0, 0, width, height, num_levels, GL_RED, GL_UNSIGNED_BYTE, dest); check_gl();
+        glTexSubImage3D(GL_TEXTURE_2D_ARRAY, 0, 0, 0, 0, width, height, num_levels, GL_RED, GL_UNSIGNED_BYTE, dest); 
         free(src);
     } else {
-        glCopyImageSubData(src_texture_id, GL_TEXTURE_2D_ARRAY, 0, 0, 0, 0, dest_texture_id, GL_TEXTURE_2D_ARRAY, 0, 0, 0, 0, width, height, num_levels); check_gl();
+        glCopyImageSubData(src_texture_id, GL_TEXTURE_2D_ARRAY, 0, 0, 0, 0, dest_texture_id, GL_TEXTURE_2D_ARRAY, 0, 0, 0, 0, width, height, num_levels); 
     }
 }
 
@@ -57,24 +52,24 @@ copy_image_sub_data(GLuint src_texture_id, GLuint dest_texture_id, unsigned int
 static void
 realloc_sprite_texture() {
     GLuint tex;
-    glGenTextures(1, &tex); check_gl();
-    glBindTexture(GL_TEXTURE_2D_ARRAY, tex); check_gl();
+    glGenTextures(1, &tex); 
+    glBindTexture(GL_TEXTURE_2D_ARRAY, tex); 
     // We use GL_NEAREST otherwise glyphs that touch the edge of the cell
     // often show a border between cells
     glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
     glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
     glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); check_gl();
+    glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 
     unsigned int xnum, ynum, z, znum, width, height, src_ynum;
-    sprite_map_current_layout(&xnum, &ynum, &z);
+    sprite_tracker_current_layout(&xnum, &ynum, &z);
     znum = z + 1;
     width = xnum * sprite_map.cell_width; height = ynum * sprite_map.cell_height;
-    glTexStorage3D(GL_TEXTURE_2D_ARRAY, 1, GL_R8, width, height, znum); check_gl();
+    glTexStorage3D(GL_TEXTURE_2D_ARRAY, 1, GL_R8, width, height, znum); 
     if (sprite_map.texture_id) {
         // need to re-alloc
         src_ynum = MAX(1, sprite_map.last_ynum);
         copy_image_sub_data(sprite_map.texture_id, tex, width, src_ynum * sprite_map.cell_height, sprite_map.last_num_of_layers);
-        glDeleteTextures(1, &sprite_map.texture_id); check_gl();
+        glDeleteTextures(1, &sprite_map.texture_id); 
     }
     glBindTexture(GL_TEXTURE_2D_ARRAY, 0);
     sprite_map.last_num_of_layers = znum;
@@ -82,91 +77,65 @@ realloc_sprite_texture() {
     sprite_map.texture_id = tex;
 }
 
-static inline PyObject*
-render_cell(PyObject *text, bool bold, bool italic, unsigned int underline, bool strikethrough, bool is_second) {
-#define B(x) (x ? Py_True : Py_False)
-    PyObject *ret = PyObject_CallFunction(sprite_map.render_cell, "OOOIOO", text, B(bold), B(italic), underline, B(strikethrough), B(is_second));
-    if (ret == NULL) { PyErr_Print(); fatal("Rendering of a cell failed, aborting"); }
-    return ret;
-#undef B
-}
-
 static inline void
 ensure_sprite_map() {
     static GLuint bound_texture_id = 0;
     if (!sprite_map.texture_id) realloc_sprite_texture();
     if (bound_texture_id != sprite_map.texture_id) {
-        glActiveTexture(GL_TEXTURE0 + SPRITE_MAP_UNIT); check_gl();
-        glBindTexture(GL_TEXTURE_2D_ARRAY, sprite_map.texture_id); check_gl();
+        glActiveTexture(GL_TEXTURE0 + SPRITE_MAP_UNIT); 
+        glBindTexture(GL_TEXTURE_2D_ARRAY, sprite_map.texture_id); 
         bound_texture_id = sprite_map.texture_id;
     }
 }
 
-static void 
-sprite_send_to_gpu(unsigned int x, unsigned int y, unsigned int z, PyObject *buf) {
+void 
+send_sprite_to_gpu(unsigned int x, unsigned int y, unsigned int z, uint8_t *buf) {
     unsigned int xnum, ynum, znum;
-    sprite_map_current_layout(&xnum, &ynum, &znum);
+    sprite_tracker_current_layout(&xnum, &ynum, &znum);
     if ((int)znum >= sprite_map.last_num_of_layers || (znum == 0 && (int)ynum > sprite_map.last_ynum)) realloc_sprite_texture();
-    glBindTexture(GL_TEXTURE_2D_ARRAY, sprite_map.texture_id); check_gl();
-    glPixelStorei(GL_UNPACK_ALIGNMENT, 1); check_gl();
+    glBindTexture(GL_TEXTURE_2D_ARRAY, sprite_map.texture_id); 
+    glPixelStorei(GL_UNPACK_ALIGNMENT, 1); 
     x *= sprite_map.cell_width; y *= sprite_map.cell_height;
-    PyObject *ret = PyObject_CallObject(buf, NULL);
-    if (ret == NULL) { PyErr_Print(); fatal("Failed to get address of rendered cell buffer"); }
-    void *address = PyLong_AsVoidPtr(ret);
-    Py_DECREF(ret);
-    glTexSubImage3D(GL_TEXTURE_2D_ARRAY, 0, x, y, z, sprite_map.cell_width, sprite_map.cell_height, 1, GL_RED, GL_UNSIGNED_BYTE, address); check_gl();
+    glTexSubImage3D(GL_TEXTURE_2D_ARRAY, 0, x, y, z, sprite_map.cell_width, sprite_map.cell_height, 1, GL_RED, GL_UNSIGNED_BYTE, buf); 
     Py_DECREF(buf);
 }
 
-static void 
-render_and_send_dirty_sprites(PyObject *text, bool bold, bool italic, bool is_second, sprite_index x, sprite_index y, sprite_index z) {
-    if (text == NULL) { fatal("The text for a sprite was NULL, probably out of memory."); }
-    PyObject *buf = render_cell(text, bold, italic, false, false, is_second);
-    sprite_send_to_gpu(x, y, z, buf);
+void
+send_image_to_gpu(GLuint *tex_id, const void* data, GLsizei width, GLsizei height, bool is_opaque, bool is_4byte_aligned) {
+    if (!(*tex_id)) { glGenTextures(1, tex_id);  }
+    glBindTexture(GL_TEXTURE_2D, *tex_id); 
+    glPixelStorei(GL_UNPACK_ALIGNMENT, is_4byte_aligned ? 4 : 1); 
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); 
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, is_opaque ? GL_RGB : GL_RGBA, GL_UNSIGNED_BYTE, data);  
 }
 
-static inline sprite_index
-send_prerendered(unsigned int underline, bool strikethrough) {
-    sprite_index x, y, z;
-    PyObject *blank = PyUnicode_FromString(" ");
-    if (blank == NULL) { fatal("Out of memory"); }
-    PyObject *buf = render_cell(blank, false, false, underline, strikethrough, false);
-    Py_CLEAR(blank);
-    if (sprite_map_increment(&x, &y, &z) != 0) { fatal("Failed to increment sprite map for prerendering"); }
-    sprite_send_to_gpu(x, y, z, buf);
-    return x;
-}
+static bool limits_updated = false;
 
 static void 
-layout_sprite_map(unsigned int cell_width, unsigned int cell_height, PyObject *render_cell) {
+layout_sprite_map(unsigned int cell_width, unsigned int cell_height) {
     sprite_map.cell_width = MAX(1, cell_width);
     sprite_map.cell_height = MAX(1, cell_height);
     global_state.cell_width = sprite_map.cell_width;
     global_state.cell_height = sprite_map.cell_height;
-    if (sprite_map.max_texture_size == 0) {
-        glGetIntegerv(GL_MAX_TEXTURE_SIZE, &(sprite_map.max_texture_size)); check_gl();
-        glGetIntegerv(GL_MAX_ARRAY_TEXTURE_LAYERS, &(sprite_map.max_array_texture_layers)); check_gl();
-        sprite_map_set_limits(sprite_map.max_texture_size, sprite_map.max_array_texture_layers);
+    if (!limits_updated) {
+        glGetIntegerv(GL_MAX_TEXTURE_SIZE, &(sprite_map.max_texture_size)); 
+        glGetIntegerv(GL_MAX_ARRAY_TEXTURE_LAYERS, &(sprite_map.max_array_texture_layers)); 
+        sprite_tracker_set_limits(sprite_map.max_texture_size, sprite_map.max_array_texture_layers);
+        limits_updated = true;
     }
-    sprite_map_set_layout(sprite_map.cell_width, sprite_map.cell_height);
-    Py_CLEAR(sprite_map.render_cell);
-    sprite_map.render_cell = render_cell; Py_INCREF(sprite_map.render_cell);
+    sprite_tracker_set_layout(sprite_map.cell_width, sprite_map.cell_height);
     if (sprite_map.texture_id) { glDeleteTextures(1, &(sprite_map.texture_id)); sprite_map.texture_id = 0; }
     realloc_sprite_texture();
-    // Pre-render the basic cells to ensure they have known sprite numbers
-    send_prerendered(0, false);
-    send_prerendered(1, false);
-    send_prerendered(2, false);
-    if (send_prerendered(0, true) != 3) { fatal("Available OpenGL texture size is too small"); }
 }
 
 static void
 destroy_sprite_map() {
-    sprite_map_free();
-    Py_CLEAR(sprite_map.render_cell);
+    /* sprite_map_free(); */
     if (sprite_map.texture_id) {
         glDeleteTextures(1, &(sprite_map.texture_id));
-        check_gl();
         sprite_map.texture_id = 0;
     }
 }
@@ -265,7 +234,7 @@ cell_update_uniform_block(ssize_t vao_idx, Screen *screen, int uniform_buffer, G
     
     rd->xstart = xstart; rd->ystart = ystart; rd->dx = dx; rd->dy = dy;
     unsigned int x, y, z;
-    sprite_map_current_layout(&x, &y, &z);
+    sprite_tracker_current_layout(&x, &y, &z);
     rd->sprite_dx = 1.0f / (float)x; rd->sprite_dy = 1.0f / (float)y;
     rd->color1 = inverted & 1; rd->color2 = 1 - (inverted & 1);
 
@@ -282,6 +251,9 @@ cell_prepare_to_render(ssize_t vao_idx, ssize_t gvao_idx, Screen *screen, GLfloa
     size_t sz;
     CELL_BUFFERS;
     void *address;
+
+    ensure_sprite_map();
+
     if (screen->scroll_changed || screen->is_dirty) {
         sz = sizeof(Cell) * screen->lines * screen->columns;
         address = alloc_and_map_vao_buffer(vao_idx, sz, cell_data_buffer, GL_STREAM_DRAW, GL_WRITE_ONLY);
@@ -305,9 +277,6 @@ cell_prepare_to_render(ssize_t vao_idx, ssize_t gvao_idx, Screen *screen, GLfloa
 
     cell_update_uniform_block(vao_idx, screen, uniform_buffer, xstart, ystart, dx, dy, cursor);
 
-    ensure_sprite_map();
-    render_dirty_sprites(render_and_send_dirty_sprites);
-
     bind_vao_uniform_buffer(vao_idx, uniform_buffer, cell_program_layouts[CELL_PROGRAM].render_data.index);
     bind_vertex_array(vao_idx);
 }
@@ -319,16 +288,16 @@ draw_graphics(ssize_t vao_idx, ssize_t gvao_idx, ImageRenderData *data, GLuint s
     glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
     static bool graphics_constants_set = false;
     if (!graphics_constants_set) { 
-        glUniform1i(glGetUniformLocation(program_id(GRAPHICS_PROGRAM), "image"), GRAPHICS_UNIT); check_gl(); 
+        glUniform1i(glGetUniformLocation(program_id(GRAPHICS_PROGRAM), "image"), GRAPHICS_UNIT);  
         graphics_constants_set = true; 
     }
-    glActiveTexture(GL_TEXTURE0 + GRAPHICS_UNIT); check_gl();
+    glActiveTexture(GL_TEXTURE0 + GRAPHICS_UNIT); 
 
     GLuint base = 4 * start;
     glEnable(GL_SCISSOR_TEST);
     for (GLuint i=0; i < count;) {
         ImageRenderData *rd = data + start + i;
-        glBindTexture(GL_TEXTURE_2D, rd->texture_id); check_gl();
+        glBindTexture(GL_TEXTURE_2D, rd->texture_id); 
         // You could reduce the number of draw calls by using
         // glDrawArraysInstancedBaseInstance but Apple chose to abandon OpenGL
         // before implementing it.
@@ -343,32 +312,32 @@ draw_all_cells(ssize_t vao_idx, ssize_t gvao_idx, Screen *screen) {
     bind_program(CELL_PROGRAM); 
     static bool cell_constants_set = false;
     if (!cell_constants_set) { 
-        glUniform1i(glGetUniformLocation(program_id(CELL_PROGRAM), "sprites"), SPRITE_MAP_UNIT); check_gl(); 
+        glUniform1i(glGetUniformLocation(program_id(CELL_PROGRAM), "sprites"), SPRITE_MAP_UNIT);  
         cell_constants_set = true; 
     }
-    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); check_gl();
+    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); 
     if (screen->grman->count) draw_graphics(vao_idx, gvao_idx, screen->grman->render_data, 0, screen->grman->count);
 }
 
 static void
 draw_cells_interleaved(ssize_t vao_idx, ssize_t gvao_idx, Screen *screen) {
     bind_program(CELL_BACKGROUND_PROGRAM); 
-    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); check_gl();
+    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); 
 
     if (screen->grman->num_of_negative_refs) draw_graphics(vao_idx, gvao_idx, screen->grman->render_data, 0, screen->grman->num_of_negative_refs);
 
     bind_program(CELL_SPECIAL_PROGRAM); 
-    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); check_gl();
+    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); 
 
     bind_program(CELL_FOREGROUND_PROGRAM); 
     glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
-    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); check_gl();
+    glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, screen->lines * screen->columns); 
 
     if (screen->grman->num_of_positive_refs) draw_graphics(vao_idx, gvao_idx, screen->grman->render_data, screen->grman->num_of_negative_refs, screen->grman->num_of_positive_refs);
 }
 
-static void 
-draw_cells_impl(ssize_t vao_idx, ssize_t gvao_idx, GLfloat xstart, GLfloat ystart, GLfloat dx, GLfloat dy, Screen *screen, CursorRenderInfo *cursor) {
+void 
+draw_cells(ssize_t vao_idx, ssize_t gvao_idx, GLfloat xstart, GLfloat ystart, GLfloat dx, GLfloat dy, Screen *screen, CursorRenderInfo *cursor) {
     GLfloat h = (GLfloat)screen->lines * dy;
 #define SCALE(w, x) ((GLfloat)(global_state.viewport_##w) * (GLfloat)(x))
     glScissor(
@@ -404,12 +373,12 @@ init_cursor_program() {
 #undef SET_LOC
 }
 
-static void 
-draw_cursor_impl(CursorRenderInfo *cursor) {
-    bind_program(CURSOR_PROGRAM); bind_vertex_array(cursor_vertex_array); check_gl();
-    glUniform3f(cursor_uniform_locations[CURSOR_color], ((cursor->color >> 16) & 0xff) / 255.0, ((cursor->color >> 8) & 0xff) / 255.0, (cursor->color & 0xff) / 255.0); check_gl();
-    glUniform4f(cursor_uniform_locations[CURSOR_pos], cursor->left, cursor->top, cursor->right, cursor->bottom); check_gl();
-    glDrawArrays(global_state.application_focused ? GL_TRIANGLE_FAN : GL_LINE_LOOP, 0, 4); check_gl();
+void 
+draw_cursor(CursorRenderInfo *cursor) {
+    bind_program(CURSOR_PROGRAM); bind_vertex_array(cursor_vertex_array); 
+    glUniform3f(cursor_uniform_locations[CURSOR_color], ((cursor->color >> 16) & 0xff) / 255.0, ((cursor->color >> 8) & 0xff) / 255.0, (cursor->color & 0xff) / 255.0); 
+    glUniform4f(cursor_uniform_locations[CURSOR_pos], cursor->left, cursor->top, cursor->right, cursor->bottom); 
+    glDrawArrays(global_state.application_focused ? GL_TRIANGLE_FAN : GL_LINE_LOOP, 0, 4); 
     unbind_vertex_array(); unbind_program();
 }
 // }}}
@@ -441,13 +410,12 @@ init_borders_program() {
             /*size=*/1, /*dtype=*/GL_UNSIGNED_INT, /*stride=*/sizeof(GLuint)*5, /*offset=*/(void*)(sizeof(GLuint)*4), /*divisor=*/1);
 }
 
-static void
-draw_borders_impl() {
+void
+draw_borders() {
     if (num_border_rects) {
         bind_program(BORDERS_PROGRAM);
         bind_vertex_array(border_vertex_array);
         glDrawArraysInstanced(GL_TRIANGLE_FAN, 0, 4, num_border_rects);
-        check_gl();
         unbind_vertex_array();
         unbind_program();
     }
@@ -474,7 +442,6 @@ send_borders_rects(GLuint vw, GLuint vh) {
     }
     bind_program(BORDERS_PROGRAM);
     glUniform2ui(border_uniform_locations[BORDER_viewport], vw, vh);
-    check_gl();
     unbind_program();
 }
 // }}}
@@ -489,12 +456,11 @@ compile_program(PyObject UNUSED *self, PyObject *args) {
     if (which < 0 || which >= NUM_PROGRAMS) { PyErr_Format(PyExc_ValueError, "Unknown program: %d", which); return NULL; }
     if (programs[which].id != 0) { PyErr_SetString(PyExc_ValueError, "program already compiled"); return NULL; }
     programs[which].id = glCreateProgram();
-    check_gl();
-    vertex_shader_id = compile_shader(GL_VERTEX_SHADER, vertex_shader); check_gl();
-    fragment_shader_id = compile_shader(GL_FRAGMENT_SHADER, fragment_shader); check_gl();
-    glAttachShader(programs[which].id, vertex_shader_id); check_gl();
-    glAttachShader(programs[which].id, fragment_shader_id); check_gl();
-    glLinkProgram(programs[which].id); check_gl();
+    vertex_shader_id = compile_shader(GL_VERTEX_SHADER, vertex_shader); 
+    fragment_shader_id = compile_shader(GL_FRAGMENT_SHADER, fragment_shader); 
+    glAttachShader(programs[which].id, vertex_shader_id); 
+    glAttachShader(programs[which].id, fragment_shader_id); 
+    glLinkProgram(programs[which].id); 
     GLint ret = GL_FALSE;
     glGetProgramiv(programs[which].id, GL_LINK_STATUS, &ret);
     if (ret != GL_TRUE) {
@@ -509,7 +475,6 @@ compile_program(PyObject UNUSED *self, PyObject *args) {
 end:
     if (vertex_shader_id != 0) glDeleteShader(vertex_shader_id);
     if (fragment_shader_id != 0) glDeleteShader(fragment_shader_id);
-    check_gl();
     if (PyErr_Occurred()) { glDeleteProgram(programs[which].id); programs[which].id = 0; return NULL;}
     return Py_BuildValue("I", programs[which].id);
     Py_RETURN_NONE;
@@ -551,9 +516,8 @@ NO_ARG_INT(create_graphics_vao)
 NO_ARG(destroy_sprite_map)
 PYWRAP1(layout_sprite_map) {
     unsigned int cell_width, cell_height;
-    PyObject *render_cell;
-    PA("IIO", &cell_width, &cell_height, &render_cell);
-    layout_sprite_map(cell_width, cell_height, render_cell);
+    PA("II", &cell_width, &cell_height);
+    layout_sprite_map(cell_width, cell_height);
     Py_RETURN_NONE;
 }
 
@@ -572,33 +536,11 @@ PYWRAP1(clear_buffers) {
     Py_RETURN_NONE;
 }
 
-PYWRAP0(check_for_extensions) {
-    GLint n = 0, i, left = 2;
-    glGetIntegerv(GL_NUM_EXTENSIONS, &n);
-    bool texture_storage = false;
-#define CHECK(name) if (!name) { \
-    if (strstr((const char*)ext, "GL_ARB_" #name) == (const char *)ext) { left--; name = true; } \
-}
-    for (i = 0; i < n; i++) {
-        const GLubyte *ext = glGetStringi(GL_EXTENSIONS, i);
-        CHECK(texture_storage); 
-        if (left < 1) break;
-    }
-#undef CHECK
-    if (left > 0) {
-#define CHECK(name) if (!name) { PyErr_Format(PyExc_RuntimeError, "The OpenGL driver on this system is missing the required extension: GL_ARB_%s", #name); return NULL; }
-        CHECK(texture_storage); 
-#undef CHECK
-    }
-    Py_RETURN_NONE;
-}
-
 #define M(name, arg_type) {#name, (PyCFunction)name, arg_type, NULL}
 #define MW(name, arg_type) {#name, (PyCFunction)py##name, arg_type, NULL}
 static PyMethodDef module_methods[] = {
-    {"glewInit", (PyCFunction)glew_init, METH_NOARGS, NULL}, 
+    M(gl_init, METH_VARARGS),
     M(compile_program, METH_VARARGS),
-    MW(check_for_extensions, METH_NOARGS),
     MW(create_vao, METH_NOARGS),
     MW(remove_vao, METH_O),
     MW(bind_vertex_array, METH_O),
@@ -621,11 +563,7 @@ static PyMethodDef module_methods[] = {
 };
 
 bool
-#ifdef ENABLE_DEBUG_GL
-init_shaders_debug(PyObject *module) {
-#else
 init_shaders(PyObject *module) {
-#endif
 #define C(x) if (PyModule_AddIntConstant(module, #x, x) != 0) { PyErr_NoMemory(); return false; }
     C(CELL_PROGRAM); C(CELL_BACKGROUND_PROGRAM); C(CELL_SPECIAL_PROGRAM); C(CELL_FOREGROUND_PROGRAM); C(CURSOR_PROGRAM); C(BORDERS_PROGRAM); C(GRAPHICS_PROGRAM);
     C(GLSL_VERSION);
@@ -658,12 +596,6 @@ init_shaders(PyObject *module) {
 #undef C
     PyModule_AddObject(module, "GL_VERSION_REQUIRED", Py_BuildValue("II", REQUIRED_VERSION_MAJOR, REQUIRED_VERSION_MINOR));
     if (PyModule_AddFunctions(module, module_methods) != 0) return false;
-    update_viewport_size = &update_viewport_size_impl;
-    draw_borders = &draw_borders_impl;
-    draw_cells = &draw_cells_impl;
-    draw_cursor = &draw_cursor_impl;
-    free_texture = &free_texture_impl;
-    send_image_to_gpu = &send_image_to_gpu_impl;
     return true;
 }
 // }}}

kitty/sprites.c

@@ -1,228 +0,0 @@
-/*
- * sprites.c
- * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "data-types.h"
-#include "lineops.h"
-#include <structmember.h>
-
-typedef struct SpritePosition SpritePosition;
-struct SpritePosition {
-    SpritePosition *next;
-    sprite_index x, y, z;
-    char_type ch;
-    combining_type cc;
-    bool bold, italic;
-    bool is_second;
-    bool filled;
-    bool rendered;
-};
-
-typedef struct {
-    size_t max_array_len, max_texture_size, max_y;
-    unsigned int x, y, z, xnum, ynum;
-    SpritePosition cache[1024];
-    bool dirty;
-} SpriteMap;
-
-static SpriteMap sprite_map = {
-    .max_array_len = 1000,
-    .max_texture_size = 1000,
-    .max_y = 100,
-    .dirty = true
-};
-
-static inline void 
-sprite_map_set_error(int error) {
-    switch(error) {
-        case 1:
-            PyErr_NoMemory(); break;
-        case 2:
-            PyErr_SetString(PyExc_RuntimeError, "Out of texture space for sprites"); break;
-        default:
-            PyErr_SetString(PyExc_RuntimeError, "Unknown error occurred while allocating sprites"); break;
-    }
-}
-
-void
-sprite_map_set_limits(size_t max_texture_size, size_t max_array_len) {
-    sprite_map.max_texture_size = max_texture_size;
-    sprite_map.max_array_len = max_array_len;
-}
-
-static PyObject*
-sprite_map_set_limits_py(PyObject UNUSED *self, PyObject *args) {
-    unsigned int w, h;
-    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
-    sprite_map_set_limits(w, h);
-    Py_RETURN_NONE;
-}
-
-void
-sprite_map_free() {
-    SpritePosition *s, *t;
-    for (size_t i = 0; i < sizeof(sprite_map.cache)/sizeof(sprite_map.cache[0]); i++) {
-        s = &(sprite_map.cache[i]);
-        s = s->next;
-        while (s) {
-            t = s;
-            s = s->next;
-            PyMem_Free(t);
-        }
-    }
-}
-
-static inline void
-do_increment(int *error) {
-    sprite_map.x++;
-    if (sprite_map.x >= sprite_map.xnum) {
-        sprite_map.x = 0; sprite_map.y++;
-        sprite_map.ynum = MIN(MAX(sprite_map.ynum, sprite_map.y + 1), sprite_map.max_y);
-        if (sprite_map.y >= sprite_map.max_y) {
-            sprite_map.y = 0; sprite_map.z++;
-            if (sprite_map.z >= MIN(UINT16_MAX, sprite_map.max_array_len)) *error = 2;
-        }
-    }
-}
-
-SpritePosition*
-sprite_map_position_for(char_type ch, attrs_type attrs, combining_type cc, bool is_second, int *error) {
-    uint8_t bold_italic = (attrs >> BOLD_SHIFT) & 3;
-    unsigned int idx = (ch & 0xFF) | (bold_italic << 8); // Only bold italic bits and lowest byte of char
-    SpritePosition *s = &(sprite_map.cache[idx]);
-    // Optimize for the common case of an ASCII char already in the cache
-    if (LIKELY(s->ch == ch && s->filled && s->cc == cc && s->is_second == is_second)) return s;  // Cache hit
-    while(true) {
-        if (s->filled) {
-            if (s->ch == ch && s->cc == cc && s->is_second == is_second) return s;  // Cache hit
-        } else {
-            break;
-        }
-        if (!s->next) {
-            s->next = PyMem_Calloc(1, sizeof(SpritePosition));
-            if (s->next == NULL) { *error = 1; return NULL; }
-        }
-        s = s->next;
-    }
-    s->ch = ch;
-    s->cc = cc;
-    s->is_second = is_second;
-    s->filled = true;
-    s->rendered = false;
-    s->bold = bold_italic & 1;
-    s->italic = bold_italic >> 1;
-    s->x = sprite_map.x; s->y = sprite_map.y; s->z = sprite_map.z;
-    do_increment(error);
-    sprite_map.dirty = true;
-    return s;
-}
-
-
-void
-set_sprite_position(Cell *cell, Cell *previous_cell) {
-    SpritePosition *sp;
-    static int error;
-    if (UNLIKELY(previous_cell != NULL && ((previous_cell->attrs) & WIDTH_MASK) == 2)) {
-        sp = sprite_map_position_for(previous_cell->ch, previous_cell->attrs, 0, true, &error);
-    } else {
-        sp = sprite_map_position_for(cell->ch, cell->attrs, cell->cc, false, &error);
-    }
-    cell->sprite_x = sp->x;
-    cell->sprite_y = sp->y;
-    cell->sprite_z = sp->z;
-}
-
-int 
-sprite_map_increment(sprite_index *x, sprite_index *y, sprite_index *z) {
-    int error = 0;
-    *x = sprite_map.x; *y = sprite_map.y; *z = sprite_map.z;
-    do_increment(&error);
-    return error;
-}
-
-void
-sprite_map_set_layout(unsigned int cell_width, unsigned int cell_height) {
-    // Invalidate cache since cell size has changed.
-    SpritePosition *s;
-    sprite_map.xnum = MIN(MAX(1, sprite_map.max_texture_size / cell_width), UINT16_MAX);
-    sprite_map.max_y = MIN(MAX(1, sprite_map.max_texture_size / cell_height), UINT16_MAX);
-    sprite_map.ynum = 1;
-    sprite_map.x = 0; sprite_map.y = 0; sprite_map.z = 0;
-
-    for (size_t i = 0; i < sizeof(sprite_map.cache)/sizeof(sprite_map.cache[0]); i++) {
-        s = &(sprite_map.cache[i]);
-        do {
-            s->filled = false;
-            s->is_second = false;
-            s->rendered = false;
-            s->ch = 0; s->cc = 0;
-            s->x = 0; s->y = 0; s->z = 0;
-            s = s->next;
-        } while (s != NULL);
-    }
-    sprite_map.dirty = true;
-}
-
-static PyObject*
-sprite_map_set_layout_py(PyObject UNUSED *self, PyObject *args) {
-    unsigned int w, h;
-    if(!PyArg_ParseTuple(args, "II", &w, &h)) return NULL;
-    sprite_map_set_layout(w, h);
-    Py_RETURN_NONE;
-}
-
-void
-sprite_map_current_layout(unsigned int *x, unsigned int *y, unsigned int *z) {
-    *x = sprite_map.xnum; *y = sprite_map.ynum; *z = sprite_map.z;
-}
-
-PyObject*
-sprite_position_for(PyObject UNUSED *self, PyObject *args) {
-#define sprite_position_for_doc "sprite_position_for(ch, cc, is_second, attrs) -> x, y, z the sprite position for the specified text"
-    unsigned long ch = 0;
-    unsigned long long cc = 0;
-    unsigned int attrs = 1;
-    int is_second = 0, error = 0;
-    if (!PyArg_ParseTuple(args, "|kKpI", &ch, &cc, &is_second, &attrs)) return NULL;
-    SpritePosition *pos = sprite_map_position_for(ch, attrs, cc, is_second, &error);
-    if (pos == NULL) { sprite_map_set_error(error); return NULL; }
-    return Py_BuildValue("III", pos->x, pos->y, pos->z);
-}
-
-void
-render_dirty_sprites(void (*render)(PyObject*, bool, bool, bool, sprite_index, sprite_index, sprite_index)) {
-#define render_dirty_cells_doc "Render all cells that are marked as dirty"
-    if (!sprite_map.dirty) return;
-
-    for (size_t i = 0; i < sizeof(sprite_map.cache)/sizeof(sprite_map.cache[0]); i++) {
-        SpritePosition *sp = &(sprite_map.cache[i]);
-        do {
-            if (sp->filled && !sp->rendered) {
-                PyObject *text = line_text_at(sp->ch, sp->cc);
-                render(text, sp->bold, sp->italic, sp->is_second, sp->x, sp->y, sp->z);
-                Py_CLEAR(text);
-                sp->rendered = true; 
-            }
-            sp = sp->next;
-        } while(sp);
-    }
-    sprite_map.dirty = false;
-}
-
-
-static PyMethodDef module_methods[] = {
-    {"sprite_position_for", (PyCFunction)sprite_position_for, METH_VARARGS, ""},
-    {"sprite_map_set_layout", (PyCFunction)sprite_map_set_layout_py, METH_VARARGS, ""},
-    {"sprite_map_set_limits", (PyCFunction)sprite_map_set_limits_py, METH_VARARGS, ""},
-
-    {NULL, NULL, 0, NULL}        /* Sentinel */
-};
-
-bool 
-init_sprites(PyObject *module) {
-    if (PyModule_AddFunctions(module, module_methods) != 0) return false;
-    return true;
-}

kitty/sprites.h

@@ -1,15 +0,0 @@
-/*
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#pragma once
-
-
-void sprite_map_current_layout(unsigned int *x, unsigned int *y, unsigned int*);
-void sprite_map_set_layout(unsigned int cell_width, unsigned int cell_height);
-void sprite_map_set_limits(size_t max_texture_size, size_t max_array_len);
-void sprite_map_free();
-int sprite_map_increment(sprite_index *x, sprite_index *y, sprite_index *z);
-void render_dirty_sprites(void (*render)(PyObject*, bool, bool, bool, sprite_index, sprite_index, sprite_index));

kitty/state.c

@@ -151,6 +151,7 @@ PYWRAP1(set_options) {
 #define S(name, convert) { GA(name); global_state.opts.name = convert(ret); Py_DECREF(ret); if (PyErr_Occurred()) return NULL; }
     S(visual_bell_duration, PyFloat_AsDouble);
     S(enable_audio_bell, PyObject_IsTrue);
+    S(focus_follows_mouse, PyObject_IsTrue);
     S(cursor_blink_interval, PyFloat_AsDouble);
     S(cursor_stop_blinking_after, PyFloat_AsDouble);
     S(cursor_shape, PyLong_AsLong);
@@ -161,6 +162,7 @@ PYWRAP1(set_options) {
     S(url_color, color_as_int);
     S(repaint_delay, repaint_delay);
     S(input_delay, repaint_delay);
+    S(macos_option_as_alt, PyObject_IsTrue);
 
     PyObject *chars = PyObject_GetAttrString(args, "select_by_word_characters");
     if (chars == NULL) return NULL;
@@ -168,16 +170,20 @@ PYWRAP1(set_options) {
         OPT(select_by_word_characters)[i] = PyUnicode_READ(PyUnicode_KIND(chars), PyUnicode_DATA(chars), i);
     }
     OPT(select_by_word_characters_count) = PyUnicode_GET_LENGTH(chars);
+    Py_DECREF(chars);
 
     GA(keymap); set_special_keys(ret);
     Py_DECREF(ret); if (PyErr_Occurred()) return NULL;
-    GA(send_text_map);
-    dict_iter(ret) {
-        set_special_keys(value);
-    }}
-    Py_DECREF(ret); if (PyErr_Occurred()) return NULL;
 
-    Py_DECREF(chars);
+    PyObject *al = PyObject_GetAttrString(args, "adjust_line_height");
+    if (PyFloat_Check(al)) { 
+        OPT(adjust_line_height_frac) = (float)PyFloat_AsDouble(al);
+        OPT(adjust_line_height_px) = 0;
+    } else {
+        OPT(adjust_line_height_frac) = 0;
+        OPT(adjust_line_height_px) = (int)PyLong_AsLong(al);
+    }
+    Py_DECREF(al);
 #undef S
     Py_RETURN_NONE;
 }

kitty/state.h

@@ -18,6 +18,10 @@ typedef struct {
     char_type select_by_word_characters[256]; size_t select_by_word_characters_count;
     color_type url_color;
     double repaint_delay, input_delay;
+    bool focus_follows_mouse;
+    bool macos_option_as_alt;
+    int adjust_line_height_px;
+    float adjust_line_height_frac;
 } Options;
 
 typedef struct {
@@ -68,6 +72,7 @@ typedef struct {
     bool application_focused;
     double cursor_blink_zero_time, last_mouse_activity_at;
     double logical_dpi_x, logical_dpi_y;
+    float font_sz_in_pts;
     double mouse_x, mouse_y;
     bool mouse_button_pressed[20];
     int viewport_width, viewport_height;
@@ -87,19 +92,17 @@ typedef struct {
     color_type color;
 } CursorRenderInfo;
 
-bool drag_scroll(Window *);
-
 #define call_boss(name, ...) { \
     PyObject *cret_ = PyObject_CallMethod(global_state.boss, #name, __VA_ARGS__); \
     if (cret_ == NULL) { PyErr_Print(); } \
     else Py_DECREF(cret_); \
 }
 
-#define EXTERNAL_FUNC(name, ret, ...) typedef ret (*name##_func)(__VA_ARGS__); extern name##_func name
-#define EXTERNAL_FUNC0(name, ret) typedef ret (*name##_func)(); extern name##_func name
-EXTERNAL_FUNC0(draw_borders, void);
-EXTERNAL_FUNC(draw_cells, void, ssize_t, ssize_t, float, float, float, float, Screen *, CursorRenderInfo *);
-EXTERNAL_FUNC(draw_cursor, void, CursorRenderInfo *);
-EXTERNAL_FUNC(update_viewport_size, void, int, int);
-EXTERNAL_FUNC(free_texture, void, uint32_t*);
-EXTERNAL_FUNC(send_image_to_gpu, void, uint32_t*, const void*, int32_t, int32_t, bool, bool);
+bool drag_scroll(Window *);
+void draw_borders();
+void draw_cells(ssize_t, ssize_t, float, float, float, float, Screen *, CursorRenderInfo *);
+void draw_cursor(CursorRenderInfo *);
+void update_viewport_size(int, int);
+void free_texture(uint32_t*);
+void send_image_to_gpu(uint32_t*, const void*, int32_t, int32_t, bool, bool);
+void send_sprite_to_gpu(unsigned int, unsigned int, unsigned int, uint8_t*);

kitty/tabs.py

@@ -49,16 +49,16 @@ class Tab:  # {{{
             setattr(self, which + '_window', partial(self.nth_window, num=i))
         if session_tab is None:
             self.cwd = args.directory
-            l = self.enabled_layouts[0]
-            self.current_layout = all_layouts[l](opts, borders.border_width, self.windows)
+            sl = self.enabled_layouts[0]
+            self.current_layout = all_layouts[sl](opts, borders.border_width, self.windows)
             if special_window is None:
                 self.new_window()
             else:
                 self.new_special_window(special_window)
         else:
             self.cwd = session_tab.cwd or args.directory
-            l = session_tab.layout
-            self.current_layout = all_layouts[l](opts, borders.border_width, self.windows)
+            l0 = session_tab.layout
+            self.current_layout = all_layouts[l0](opts, borders.border_width, self.windows)
             self.startup(session_tab)
 
     def startup(self, session_tab):
@@ -285,6 +285,10 @@ class TabBar:  # {{{
         self.cell_ranges = cr
         glfw_post_empty_event()
 
+    def destroy(self):
+        self.screen.reset_callbacks()
+        del self.screen
+
     def tab_at(self, x):
         x = (x - self.window_geometry.left) // self.cell_width
         for i, (a, b) in enumerate(self.cell_ranges):
@@ -412,4 +416,11 @@ class TabManager:  # {{{
         if len(self.tabs) < 2:
             return
         self.tab_bar.render()
+
+    def destroy(self):
+        for t in self:
+            t.destroy()
+        self.tab_bar.destroy()
+        del self.tab_bar
+        del self.refresh_sprite_positions
 # }}}

kitty/terminfo.py

@@ -190,6 +190,8 @@ string_capabilities = {
     'kpp': r'\E[5~',
     # Scroll backwards (reverse index)
     'kri': r'\E[1;2A',
+    # scroll forwards (index)
+    'kind': r'\E[1;2B',
     # Restore cursor
     'rc': r'\E8',
     # Reverse video
@@ -276,8 +278,6 @@ string_capabilities = {
     # 'is2': r'\E[!p\E[?3;4l\E[4l\E>',
     # # Enter/send key
     # 'kent': r'\EOM',
-    # # scroll forwards
-    # 'kind': r'\E[1;2B',
     # # reset2
     # 'rs2': r'\E[!p\E[?3;4l\E[4l\E>',
 }
@@ -350,6 +350,7 @@ termcap_aliases.update({
     'kN': 'knp',
     'kP': 'kpp',
     'kR': 'kri',
+    'kF': 'kind',
     'rc': 'rc',
     'mr': 'rev',
     'sr': 'ri',
@@ -400,7 +401,6 @@ termcap_aliases.update({
     # 'mk': 'invis',
     # 'is': 'is2',
     # '@8': 'kent',
-    # 'kF': 'kind',
     # 'r2': 'rs2',
 })
 

kitty/utils.py

@@ -2,20 +2,23 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
+import ctypes
 import math
 import os
 import re
 import shlex
 import string
 import subprocess
+import sys
 from contextlib import contextmanager
+from ctypes.util import find_library
 from functools import lru_cache
 from time import monotonic
 
-from .constants import isosx, iswayland, selection_clipboard_funcs, x11_display
+from .constants import isosx, iswayland, selection_clipboard_funcs, x11_display, x11_window_id
 from .fast_data_types import (
-    GLSL_VERSION, glfw_get_physical_dpi, redirect_std_streams,
-    wcwidth as wcwidth_impl
+    GLSL_VERSION, glfw_get_physical_dpi, glfw_primary_monitor_content_scale,
+    redirect_std_streams, wcwidth as wcwidth_impl
 )
 from .rgb import Color, to_color
 
@@ -92,6 +95,8 @@ def parse_xrdb(raw):
 
 
 def x11_dpi():
+    if iswayland:
+        return
     XResourceManagerString = load_libx11()
     display = x11_display()
     if display:
@@ -102,19 +107,25 @@ def x11_dpi():
             pass
 
 
-def get_logical_dpi():
+def get_logical_dpi(override_dpi=None):
+    # See https://github.com/glfw/glfw/issues/1019 for why we cant use
+    # glfw_get_physical_dpi()
+    if override_dpi is not None:
+        get_logical_dpi.ans = override_dpi
     if not hasattr(get_logical_dpi, 'ans'):
-        if isosx or iswayland:
-            # TODO: Investigate if this needs a different implementation on OS X or Wayland
+        if isosx:
+            # TODO: Investigate if this needs a different implementation on OS X
             get_logical_dpi.ans = glfw_get_physical_dpi()
         else:
-            # See https://github.com/glfw/glfw/issues/1019 for why we cant use
-            # glfw_get_physical_dpi()
-            dpi = x11_dpi()
-            if dpi is None:
-                get_logical_dpi.ans = glfw_get_physical_dpi()
+            try:
+                xscale, yscale = glfw_primary_monitor_content_scale()
+            except NotImplementedError:  # glfw < 3.3
+                xdpi = ydpi = x11_dpi()
+                if xdpi is None:
+                    xdpi, ydpi = glfw_get_physical_dpi()
             else:
-                get_logical_dpi.ans = dpi, dpi
+                xdpi, ydpi = xscale * 96.0, yscale * 96.0
+            get_logical_dpi.ans = xdpi, ydpi
     return get_logical_dpi.ans
 
 
@@ -219,3 +230,68 @@ def adjust_line_height(cell_height, val):
     if isinstance(val, int):
         return cell_height + val
     return int(cell_height * val)
+
+
+def init_startup_notification_x11(window_id):
+    # https://specifications.freedesktop.org/startup-notification-spec/startup-notification-latest.txt
+    display = x11_display()
+    if not display:
+        return
+    lib = find_library('startup-notification-1')
+    if not lib:
+        safe_print('libstartup-notification-1.so not found, disabling startup notification', file=sys.stderr)
+        return
+    lib = init_startup_notification_x11.lib = ctypes.CDLL(lib)
+    f = lib.sn_display_new
+    f.restype = ctypes.c_void_p
+    f.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
+    display = f(x11_display(), None, None)
+    f = lib.sn_launchee_context_new_from_environment
+    f.restype = ctypes.c_void_p
+    f.argtypes = [ctypes.c_void_p, ctypes.c_int]
+    ctx = f(display, 0)
+    f = lib.sn_display_unref
+    f.argtypes = [ctypes.c_void_p]
+    f.restype = None
+    f(display)
+    os.environ.pop('DESKTOP_STARTUP_ID', None)  # ensure child processes dont get this env var
+    if ctx:
+        f = lib.sn_launchee_context_setup_window
+        f.argtypes = [ctypes.c_void_p, ctypes.c_int32]
+        f(ctx, x11_window_id(window_id))
+    return ctx
+
+
+def end_startup_notification_x11(ctx):
+    lib = init_startup_notification_x11.lib
+    del init_startup_notification_x11.lib
+    f = lib.sn_launchee_context_complete
+    f.restype = None
+    f.argtypes = [ctypes.c_void_p]
+    f(ctx)
+    f = lib.sn_launchee_context_unref
+    f.restype = None
+    f.argtypes = [ctypes.c_void_p]
+    f(ctx)
+
+
+def init_startup_notification(window):
+    if isosx or iswayland:
+        return
+    try:
+        return init_startup_notification_x11(window)
+    except Exception:
+        import traceback
+        traceback.print_exc()
+
+
+def end_startup_notification(ctx):
+    if not ctx:
+        return
+    if isosx or iswayland:
+        return
+    try:
+        end_startup_notification_x11(ctx)
+    except Exception:
+        import traceback
+        traceback.print_exc()

kitty/window.py

@@ -8,7 +8,7 @@ from collections import deque
 from enum import Enum
 from itertools import count
 
-from .config import build_ansi_color_table
+from .config import build_ansi_color_table, parse_send_text_bytes
 from .constants import (
     ScreenGeometry, WindowGeometry, appname, cell_size, get_boss,
     viewport_size, wakeup
@@ -21,6 +21,7 @@ from .fast_data_types import (
     glfw_post_empty_event, init_cell_program, init_cursor_program, remove_vao,
     set_window_render_data, update_window_title, update_window_visibility
 )
+from .keys import keyboard_mode_name
 from .rgb import to_color
 from .terminfo import get_capabilities
 from .utils import color_as_int, load_shaders, parse_color_set, sanitize_title, open_url, open_cmd
@@ -80,7 +81,6 @@ class Window:
         self.tab_id = tab.id
         self.tabref = weakref.ref(tab)
         self.override_title = None
-        self.last_mouse_cursor_pos = 0, 0
         self.destroyed = False
         self.click_queue = deque(maxlen=3)
         self.geometry = WindowGeometry(0, 0, 0, 0, 0, 0)
@@ -144,6 +144,17 @@ class Window:
         boss.gui_close_window(self)
         self.screen = None
 
+    def send_text(self, *args):
+        mode = keyboard_mode_name(self.screen)
+        required_mode, text = args[-2:]
+        required_mode = frozenset(required_mode.split(','))
+        if not required_mode & {mode, 'all'}:
+            return True
+        data = parse_send_text_bytes(text)
+        if not data:
+            return True
+        self.write_to_child(data)
+
     def write_to_child(self, data):
         if data:
             if get_boss().child_monitor.needs_write(self.id, data) is not True:
@@ -257,7 +268,8 @@ class Window:
             if isinstance(text, str):
                 text = text.encode('utf-8')
             if self.screen.in_bracketed_paste_mode:
-                text = BRACKETED_PASTE_START.encode('ascii') + text + BRACKETED_PASTE_END.encode('ascii')
+                bpe = BRACKETED_PASTE_END.encode('ascii')
+                text = BRACKETED_PASTE_START.encode('ascii') + text.replace(bpe, b'') + bpe
             self.write_to_child(text)
 
     def copy_to_clipboard(self):

kitty_tests/datatypes.py

@@ -7,8 +7,7 @@ from unittest import skipIf
 
 from kitty.config import build_ansi_color_table, defaults
 from kitty.fast_data_types import (
-    REVERSE, ColorProfile, Cursor as C, HistoryBuf, LineBuf,
-    sprite_map_set_layout, sprite_map_set_limits, sprite_position_for
+    REVERSE, ColorProfile, Cursor as C, HistoryBuf, LineBuf
 )
 from kitty.utils import sanitize_title, wcwidth
 
@@ -19,10 +18,10 @@ def create_lbuf(*lines):
     maxw = max(map(len, lines))
     ans = LineBuf(len(lines), maxw)
     prev_full_length = False
-    for i, l in enumerate(lines):
-        ans.line(i).set_text(l, 0, len(l), C())
+    for i, l0 in enumerate(lines):
+        ans.line(i).set_text(l0, 0, len(l0), C())
         ans.set_continued(i, prev_full_length)
-        prev_full_length = len(l) == maxw
+        prev_full_length = len(l0) == maxw
     return ans
 
 
@@ -38,8 +37,8 @@ class TestDataTypes(BaseTest):
         old.set_attribute(REVERSE, False)
         for y in range(old.ynum):
             for x in range(old.xnum):
-                l = old.line(y)
-                c = l.cursor_from(x)
+                l0 = old.line(y)
+                c = l0.cursor_from(x)
                 self.assertFalse(c.reverse)
                 self.assertTrue(c.bold)
         self.assertFalse(old.is_continued(0))
@@ -62,9 +61,9 @@ class TestDataTypes(BaseTest):
         self.ae(lb.line(3), lb2.line(1))
         self.ae(lb.line(4), lb2.line(4))
         self.ae(lb.create_line_copy(1), lb2.line(2))
-        l = lb.create_line_copy(2)
-        lb.copy_line_to(1, l)
-        self.ae(l, lb2.line(2))
+        l2 = lb.create_line_copy(2)
+        lb.copy_line_to(1, l2)
+        self.ae(l2, lb2.line(2))
         lb.clear_line(0)
         self.ae(lb.line(0), LineBuf(1, lb.xnum).create_line_copy(0))
         lb = filled_line_buf(5, 5, filled_cursor())
@@ -124,11 +123,11 @@ class TestDataTypes(BaseTest):
         self.ae(lb.line(4), clb.line(4))
 
         lb = filled_line_buf(5, 5, filled_cursor())
-        l = lb.line(0)
-        l.add_combining_char(1, 'a')
-        l.clear_text(1, 2)
-        self.ae(str(l), '0  00')
-        self.assertEqualAttributes(l.cursor_from(1), l.cursor_from(0))
+        l0 = lb.line(0)
+        l0.add_combining_char(1, 'a')
+        l0.clear_text(1, 2)
+        self.ae(str(l0), '0  00')
+        self.assertEqualAttributes(l0.cursor_from(1), l0.cursor_from(0))
 
         lb = filled_line_buf(10, 10, filled_cursor())
         lb.clear()
@@ -147,23 +146,23 @@ class TestDataTypes(BaseTest):
             lb.line(lb.ynum)
         with self.assertRaises(IndexError):
             lb.line(0)[lb.xnum]
-        l = lb.line(0)
-        l.set_text(' ', 0, len(' '), C())
-        l.add_combining_char(0, '1')
-        self.ae(l[0], ' 1')
-        l.add_combining_char(0, '2')
-        self.ae(l[0], ' 12')
-        l.add_combining_char(0, '3')
-        self.ae(l[0], ' 13')
-        self.ae(l[1], '\0')
-        self.ae(str(l), ' 13')
+        l0 = lb.line(0)
+        l0.set_text(' ', 0, len(' '), C())
+        l0.add_combining_char(0, '1')
+        self.ae(l0[0], ' 1')
+        l0.add_combining_char(0, '2')
+        self.ae(l0[0], ' 12')
+        l0.add_combining_char(0, '3')
+        self.ae(l0[0], ' 13')
+        self.ae(l0[1], '\0')
+        self.ae(str(l0), ' 13')
         t = 'Testing with simple text'
         lb = LineBuf(2, len(t))
-        l = lb.line(0)
-        l.set_text(t, 0, len(t), C())
-        self.ae(str(l), t)
-        l.set_text('a', 0, 1, C())
-        self.assertEqual(str(l), 'a' + t[1:])
+        l0 = lb.line(0)
+        l0.set_text(t, 0, len(t), C())
+        self.ae(str(l0), t)
+        l0.set_text('a', 0, 1, C())
+        self.assertEqual(str(l0), 'a' + t[1:])
 
         c = C(3, 5)
         c.bold = c.italic = c.reverse = c.strikethrough = True
@@ -174,81 +173,81 @@ class TestDataTypes(BaseTest):
         self.ae(c, c2)
         c2.bold = False
         self.assertNotEqual(c, c2)
-        l.set_text(t, 0, len(t), C())
-        l.apply_cursor(c2, 3)
-        self.assertEqualAttributes(c2, l.cursor_from(3))
-        l.apply_cursor(c2, 0, len(l))
-        for i in range(len(l)):
-            self.assertEqualAttributes(c2, l.cursor_from(i))
-        l.apply_cursor(c3, 0)
-        self.assertEqualAttributes(c3, l.cursor_from(0))
-        l.copy_char(0, l, 1)
-        self.assertEqualAttributes(c3, l.cursor_from(1))
+        l0.set_text(t, 0, len(t), C())
+        l0.apply_cursor(c2, 3)
+        self.assertEqualAttributes(c2, l0.cursor_from(3))
+        l0.apply_cursor(c2, 0, len(l0))
+        for i in range(len(l0)):
+            self.assertEqualAttributes(c2, l0.cursor_from(i))
+        l0.apply_cursor(c3, 0)
+        self.assertEqualAttributes(c3, l0.cursor_from(0))
+        l0.copy_char(0, l0, 1)
+        self.assertEqualAttributes(c3, l0.cursor_from(1))
 
         t = '0123456789'
         lb = LineBuf(1, len(t))
-        l = lb.line(0)
-        l.set_text(t, 0, len(t), C())
-        self.ae(t, str(l))
-        l.right_shift(4, 2)
-        self.ae('0123454567', str(l))
-        l.set_text(t, 0, len(t), C())
-        l.right_shift(0, 0)
-        self.ae(t, str(l))
-        l.right_shift(0, 1)
-        self.ae(str(l), '0' + t[:-1])
-        l.set_text(t, 0, len(t), C())
-        l.left_shift(0, 2)
-        self.ae(str(l), t[2:] + '89')
-        l.set_text(t, 0, len(t), C())
-        l.left_shift(7, 3)
-        self.ae(str(l), t)
+        l3 = lb.line(0)
+        l3.set_text(t, 0, len(t), C())
+        self.ae(t, str(l3))
+        l3.right_shift(4, 2)
+        self.ae('0123454567', str(l3))
+        l3.set_text(t, 0, len(t), C())
+        l3.right_shift(0, 0)
+        self.ae(t, str(l3))
+        l3.right_shift(0, 1)
+        self.ae(str(l3), '0' + t[:-1])
+        l3.set_text(t, 0, len(t), C())
+        l3.left_shift(0, 2)
+        self.ae(str(l3), t[2:] + '89')
+        l3.set_text(t, 0, len(t), C())
+        l3.left_shift(7, 3)
+        self.ae(str(l3), t)
 
-        l.set_text(t, 0, len(t), C())
+        l3.set_text(t, 0, len(t), C())
         q = C()
         q.bold = q.italic = q.reverse = q.strikethrough = True
         q.decoration = 2
         c = C()
         c.x = 3
-        l.set_text('axyb', 1, 2, c)
-        self.ae(str(l), '012xy56789')
-        l.set_char(0, 'x', 1, q)
-        self.assertEqualAttributes(l.cursor_from(0), q)
+        l3.set_text('axyb', 1, 2, c)
+        self.ae(str(l3), '012xy56789')
+        l3.set_char(0, 'x', 1, q)
+        self.assertEqualAttributes(l3.cursor_from(0), q)
 
     def test_url_at(self):
         def create(t):
             lb = create.lb = LineBuf(1, len(t))
-            l = lb.line(0)
-            l.set_text(t, 0, len(t), C())
-            return l
+            lf = lb.line(0)
+            lf.set_text(t, 0, len(t), C())
+            return lf
 
         for trail in '.,]>)\\':
-            l = create("http://xyz.com" + trail)
-            self.ae(l.url_end_at(0), len(l) - 2)
-        l = create("ftp://abc/")
-        self.ae(l.url_end_at(0), len(l) - 1)
-        l = create("http://-abcd] ")
-        self.ae(l.url_end_at(0), len(l) - 3)
+            lx = create("http://xyz.com" + trail)
+            self.ae(lx.url_end_at(0), len(lx) - 2)
+        l0 = create("ftp://abc/")
+        self.ae(l0.url_end_at(0), len(l0) - 1)
+        l2 = create("http://-abcd] ")
+        self.ae(l2.url_end_at(0), len(l2) - 3)
 
         def lspace_test(n, scheme='http'):
-            l = create(' ' * n + scheme + '://acme.com')
+            lf = create(' ' * n + scheme + '://acme.com')
             for i in range(0, n):
-                self.ae(l.url_start_at(i), len(l))
-            for i in range(n, len(l)):
-                self.ae(l.url_start_at(i), n)
+                self.ae(lf.url_start_at(i), len(lf))
+            for i in range(n, len(lf)):
+                self.ae(lf.url_start_at(i), n)
         for i in range(7):
             for scheme in 'http https ftp file'.split():
                 lspace_test(i)
-        l = create('b https://testing.me a')
-        for s in (0, 1, len(l) - 1, len(l) - 2):
-            self.ae(l.url_start_at(s), len(l), 'failed with start at: %d' % s)
-        for s in range(2, len(l) - 2):
-            self.ae(l.url_start_at(s), 2, 'failed with start at: %d (%s)' % (s, str(l)[s:]))
+        l3 = create('b https://testing.me a')
+        for s in (0, 1, len(l3) - 1, len(l3) - 2):
+            self.ae(l3.url_start_at(s), len(l3), 'failed with start at: %d' % s)
+        for s in range(2, len(l3) - 2):
+            self.ae(l3.url_start_at(s), 2, 'failed with start at: %d (%s)' % (s, str(l3)[s:]))
 
         def no_url(t):
-            l = create(t)
-            for s in range(len(l)):
-                self.ae(l.url_start_at(s), len(l))
+            lf = create(t)
+            for s in range(len(lf)):
+                self.ae(lf.url_start_at(s), len(lf))
         no_url('https:// testing.me a')
         no_url('h ttp://acme.com')
         no_url('http: //acme.com')
@@ -279,11 +278,13 @@ class TestDataTypes(BaseTest):
         self.ae(cy, 3)
         for i in range(lb2.ynum):
             self.ae(lb2.line(i), lb.line(i + 2))
+        self.assertFalse(lb.dirty_lines())
+        self.ae(lb2.dirty_lines(), list(range(lb2.ynum)))
 
     def line_comparison(self, buf, *lines):
-        for i, l in enumerate(lines):
+        for i, l0 in enumerate(lines):
             l2 = buf.line(i)
-            self.ae(l, str(l2))
+            self.ae(l0, str(l2))
 
     def line_comparison_rewrap(self, lb, *lines):
         lb2 = LineBuf(len(lines), max(map(len, lines)))
@@ -299,6 +300,7 @@ class TestDataTypes(BaseTest):
         lb = create_lbuf('0123 ', '56789')
         lb2 = self.line_comparison_rewrap(lb, '0123 5', '6789', '')
         self.assertContinued(lb2, False, True)
+        self.ae(lb2.dirty_lines(), [0, 1])
 
         lb = create_lbuf('12', 'abc')
         lb2 = self.line_comparison_rewrap(lb, '12', 'abc')
@@ -326,20 +328,6 @@ class TestDataTypes(BaseTest):
             self.assertEqual(c.as_color(i << 8 | 1), (col[0], col[1], col[2]))
         self.ae(c.as_color(255 << 8 | 1), (0xee, 0xee, 0xee))
 
-    def test_sprite_map(self):
-        sprite_map_set_limits(10, 2)
-        sprite_map_set_layout(5, 5)
-        self.ae(sprite_position_for(0), (0, 0, 0))
-        self.ae(sprite_position_for(1), (1, 0, 0))
-        self.ae(sprite_position_for(2), (0, 1, 0))
-        self.ae(sprite_position_for(3), (1, 1, 0))
-        self.ae(sprite_position_for(4), (0, 0, 1))
-        self.ae(sprite_position_for(5), (1, 0, 1))
-        self.ae(sprite_position_for(0, 1), (0, 1, 1))
-        self.ae(sprite_position_for(0, 2), (1, 1, 1))
-        self.ae(sprite_position_for(0, 2), (1, 1, 1))
-        sprite_map_set_limits(1000, 1000)
-
     def test_historybuf(self):
         lb = filled_line_buf()
         hb = HistoryBuf(5, 5)
@@ -372,6 +360,7 @@ class TestDataTypes(BaseTest):
         hb.rewrap(hb2)
         for i in range(hb2.ynum):
             self.ae(hb2.line(i), hb.line(i))
+        self.ae(hb2.dirty_lines(), list(range(hb2.ynum)))
         hb = filled_history_buf(5, 5)
         hb2 = HistoryBuf(hb.ynum, hb.xnum * 2)
         hb.rewrap(hb2)
@@ -382,19 +371,19 @@ class TestDataTypes(BaseTest):
 
     def test_ansi_repr(self):
         lb = filled_line_buf()
-        l = lb.line(0)
-        self.ae(l.as_ansi(), '\x1b[0m00000')
+        l0 = lb.line(0)
+        self.ae(l0.as_ansi(), '\x1b[0m00000')
         a = []
         lb.as_ansi(a.append)
         self.ae(a, ['\x1b[0m' + str(lb.line(i)) + '\n' for i in range(lb.ynum)])
-        l = lb.line(0)
+        l2 = lb.line(0)
         c = C()
         c.bold = c.italic = c.reverse = c.strikethrough = True
         c.fg = (4 << 8) | 1
         c.bg = (1 << 24) | (2 << 16) | (3 << 8) | 2
         c.decoration_fg = (5 << 8) | 1
-        l.set_text('1', 0, 1, c)
-        self.ae(l.as_ansi(), '\x1b[0m\x1b[1m\x1b[3m\x1b[7m\x1b[9m\x1b[38;5;4m\x1b[48;2;1;2;3m\x1b[58;5;5m' '1'
+        l2.set_text('1', 0, 1, c)
+        self.ae(l2.as_ansi(), '\x1b[0m\x1b[1m\x1b[3m\x1b[7m\x1b[9m\x1b[38;5;4m\x1b[48;2;1;2;3m\x1b[58;5;5m' '1'
                 '\x1b[22m\x1b[23m\x1b[27m\x1b[29m\x1b[39m\x1b[49m\x1b[59m' '0000')
         lb = filled_line_buf()
         for i in range(lb.ynum):

kitty_tests/fonts.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+from collections import OrderedDict
+
+from kitty.constants import isosx
+from kitty.fast_data_types import (
+    change_wcwidth, set_logical_dpi, set_send_sprite_to_gpu,
+    sprite_map_set_layout, sprite_map_set_limits, test_render_line,
+    test_sprite_position_for, wcwidth
+)
+from kitty.fonts.box_drawing import box_chars
+from kitty.fonts.render import (
+    prerender, render_string, set_font_family, shape_string
+)
+from kitty.utils import get_logical_dpi
+
+from . import BaseTest
+
+
+class Rendering(BaseTest):
+
+    def setUp(self):
+        sprite_map_set_limits(100000, 100)
+        self.sprites = OrderedDict()
+
+        def send_to_gpu(x, y, z, data):
+            self.sprites[(x, y, z)] = data
+
+        set_send_sprite_to_gpu(send_to_gpu)
+        set_logical_dpi(96.0, 96.0)
+        get_logical_dpi((96.0, 96.0))
+        self.cell_width, self.cell_height = set_font_family()
+        prerender()
+        self.assertEqual([k[0] for k in self.sprites], [0, 1, 2, 3, 4])
+
+    def tearDown(self):
+        set_send_sprite_to_gpu(None)
+        del self.sprites
+
+    def test_sprite_map(self):
+        sprite_map_set_limits(10, 2)
+        sprite_map_set_layout(5, 5)
+        self.ae(test_sprite_position_for(0), (0, 0, 0))
+        self.ae(test_sprite_position_for(1), (1, 0, 0))
+        self.ae(test_sprite_position_for(2), (0, 1, 0))
+        self.ae(test_sprite_position_for(3), (1, 1, 0))
+        self.ae(test_sprite_position_for(4), (0, 0, 1))
+        self.ae(test_sprite_position_for(5), (1, 0, 1))
+        self.ae(test_sprite_position_for(0, 1), (0, 1, 1))
+        self.ae(test_sprite_position_for(0, 2), (1, 1, 1))
+        self.ae(test_sprite_position_for(0, 2), (1, 1, 1))
+
+    def test_box_drawing(self):
+        prerendered = len(self.sprites)
+        s = self.create_screen(cols=len(box_chars), lines=1, scrollback=0)
+        s.draw(''.join(box_chars))
+        line = s.line(0)
+        test_render_line(line)
+        self.assertEqual(len(self.sprites), prerendered + len(box_chars))
+
+    def test_font_rendering(self):
+        render_string('ab\u0347\u0305你好|\U0001F601|\U0001F64f|\U0001F63a|')
+        text = 'He\u0347\u0305llo\u0341, w\u0302or\u0306l\u0354d!'
+        # macOS has no fonts capable of rendering combining chars
+        if isosx:
+            text = text.encode('ascii', 'ignore').decode('ascii')
+        cells = render_string(text)[-1]
+        self.ae(len(cells), len(text.encode('ascii', 'ignore')))
+        text = '你好,世界'
+        sz = sum(map(lambda x: wcwidth(ord(x)), text))
+        cells = render_string(text)[-1]
+        self.ae(len(cells), sz)
+
+    def test_shaping(self):
+        change_wcwidth(True)
+        try:
+
+            def groups(text, path=None):
+                return [x[:2] for x in shape_string(text, path=path)]
+
+            self.ae(groups('abcd'), [(1, 1) for i in range(4)])
+            self.ae(groups('A=>>B!=C', path='kitty_tests/FiraCode-Medium.otf'), [(1, 1), (3, 3), (1, 1), (2, 2), (1, 1)])
+            colon_glyph = shape_string('9:30', path='kitty_tests/FiraCode-Medium.otf')[1][2]
+            self.assertNotEqual(colon_glyph, shape_string(':', path='kitty_tests/FiraCode-Medium.otf')[0][2])
+            self.ae(colon_glyph, 998)
+            self.ae(groups('9:30', path='kitty_tests/FiraCode-Medium.otf'), [(1, 1), (1, 1), (1, 1), (1, 1)])
+            self.ae(groups('|\U0001F601|\U0001F64f|\U0001F63a|'), [(1, 1), (2, 1), (1, 1), (2, 1), (1, 1), (2, 1), (1, 1)])
+            self.ae(groups('He\u0347\u0305llo\u0337,', path='kitty_tests/LiberationMono-Regular.ttf'),
+                    [(1, 1), (1, 3), (1, 1), (1, 1), (1, 2), (1, 1)])
+        finally:
+            change_wcwidth(False)

kitty_tests/graphics.py

@@ -60,7 +60,7 @@ def load_helpers(self):
     s = self.create_screen()
     g = s.grman
 
-    def l(payload, **kw):
+    def pl(payload, **kw):
         kw.setdefault('i', 1)
         cmd = ','.join('%s=%s' % (k, v) for k, v in kw.items())
         res = send_command(s, cmd, payload)
@@ -71,7 +71,7 @@ def load_helpers(self):
             payload = payload.encode('utf-8')
         data = kw.pop('expecting_data', payload)
         cid = kw.setdefault('i', 1)
-        self.ae('OK', l(payload, **kw))
+        self.ae('OK', pl(payload, **kw))
         img = g.image_for_client_id(cid)
         self.ae(img['client_id'], cid)
         self.ae(img['data'], data)
@@ -80,7 +80,7 @@ def load_helpers(self):
         self.ae(img['is_4byte_aligned'], kw.get('f') != 24)
         return img
 
-    return s, g, l, sl
+    return s, g, pl, sl
 
 
 def put_helpers(self, cw, ch):
@@ -211,21 +211,21 @@ class TestGraphics(BaseTest):
         cw, ch = 10, 20
         s, dx, dy, put_image, put_ref, layers, rect_eq = put_helpers(self, cw, ch)
         self.ae(put_image(s, 10, 20)[1], 'OK')
-        l = layers(s)
-        self.ae(len(l), 1)
-        rect_eq(l[0]['src_rect'], 0, 0, 1, 1)
-        rect_eq(l[0]['dest_rect'], -1, 1, -1 + dx, 1 - dy)
-        self.ae(l[0]['group_count'], 1)
+        l0 = layers(s)
+        self.ae(len(l0), 1)
+        rect_eq(l0[0]['src_rect'], 0, 0, 1, 1)
+        rect_eq(l0[0]['dest_rect'], -1, 1, -1 + dx, 1 - dy)
+        self.ae(l0[0]['group_count'], 1)
         self.ae(s.cursor.x, 1), self.ae(s.cursor.y, 0)
         put_ref(s, num_cols=s.columns, x_off=2, y_off=1, width=3, height=5, cell_x_off=3, cell_y_off=1, z=-1)
-        l = layers(s)
-        self.ae(len(l), 2)
-        rect_eq(l[0]['src_rect'], 2 / 10, 1 / 20, (2 + 3) / 10, (1 + 5)/20)
+        l2 = layers(s)
+        self.ae(len(l2), 2)
+        rect_eq(l2[0]['src_rect'], 2 / 10, 1 / 20, (2 + 3) / 10, (1 + 5)/20)
         left, top = -1 + dx + 3 * dx / cw, 1 - 1 * dy / ch
-        rect_eq(l[0]['dest_rect'], left, top, -1 + (1 + s.columns) * dx, top - dy * 5 / ch)
-        rect_eq(l[1]['src_rect'], 0, 0, 1, 1)
-        rect_eq(l[1]['dest_rect'], -1, 1, -1 + dx, 1 - dy)
-        self.ae(l[0]['group_count'], 1), self.ae(l[1]['group_count'], 1)
+        rect_eq(l2[0]['dest_rect'], left, top, -1 + (1 + s.columns) * dx, top - dy * 5 / ch)
+        rect_eq(l2[1]['src_rect'], 0, 0, 1, 1)
+        rect_eq(l2[1]['dest_rect'], -1, 1, -1 + dx, 1 - dy)
+        self.ae(l2[0]['group_count'], 1), self.ae(l2[1]['group_count'], 1)
         self.ae(s.cursor.x, 0), self.ae(s.cursor.y, 1)
 
     def test_gr_scroll(self):
@@ -261,8 +261,8 @@ class TestGraphics(BaseTest):
         self.ae(s.grman.image_count, 3)
         self.ae(layers(s)[0]['src_rect'], {'left': 0.0, 'top': 0.0, 'right': 1.0, 'bottom': 1.0})
         s.index(), s.index()
-        l = layers(s)
-        self.ae(len(l), 3)
+        l0 = layers(s)
+        self.ae(len(l0), 3)
         self.ae(layers(s)[0]['src_rect'],  {'left': 0.0, 'top': 0.5, 'right': 1.0, 'bottom': 1.0})
         s.index()
         self.ae(s.grman.image_count, 2)

kitty_tests/parser.py

@@ -115,19 +115,32 @@ class TestParser(BaseTest):
         pb('\033[?1000;1004h', ('screen_set_mode', 1000, 1), ('screen_set_mode', 1004, 1))
         pb('\033[20;4;20l', ('screen_reset_mode', 20, 0), ('screen_reset_mode', 4, 0), ('screen_reset_mode', 20, 0))
         s.reset()
-        pb('\033[1;3;4;7;9;34;44m', ('select_graphic_rendition', '1 3 4 7 9 34 44 '))
+
+        def sgr(params):
+            return (('select_graphic_rendition', '{} '.format(x)) for x in params.split())
+
+        pb('\033[1;3;4;7;9;34;44m', *sgr('1 3 4 7 9 34 44'))
         for attr in 'bold italic reverse strikethrough'.split():
             self.assertTrue(getattr(s.cursor, attr))
         self.ae(s.cursor.decoration, 1)
         self.ae(s.cursor.fg, 4 << 8 | 1)
         self.ae(s.cursor.bg, 4 << 8 | 1)
-        pb('\033[38;5;1;48;5;7m', ('select_graphic_rendition', '38 5 1 48 5 7 '))
+        pb('\033[38;5;1;48;5;7m', ('select_graphic_rendition', '38 5 1 '), ('select_graphic_rendition', '48 5 7 '))
         self.ae(s.cursor.fg, 1 << 8 | 1)
         self.ae(s.cursor.bg, 7 << 8 | 1)
-        pb('\033[38;2;1;2;3;48;2;7;8;9m', ('select_graphic_rendition', '38 2 1 2 3 48 2 7 8 9 '))
+        pb('\033[38;2;1;2;3;48;2;7;8;9m', ('select_graphic_rendition', '38 2 1 2 3 '), ('select_graphic_rendition', '48 2 7 8 9 '))
         self.ae(s.cursor.fg, 1 << 24 | 2 << 16 | 3 << 8 | 2)
         self.ae(s.cursor.bg, 7 << 24 | 8 << 16 | 9 << 8 | 2)
-        pb('\033[;2m', ('select_graphic_rendition', '0 2 '))
+        pb('\033[0;2m', *sgr('0 2'))
+        pb('\033[;2m', *sgr('0 2'))
+        pb('\033[m', *sgr('0 '))
+        pb('\033[1;;2m', *sgr('1 0 2'))
+        pb('\033[38;5;1m', ('select_graphic_rendition', '38 5 1 '))
+        pb('\033[58;2;1;2;3m', ('select_graphic_rendition', '58 2 1 2 3 '))
+        pb('\033[38;2;1;2;3m', ('select_graphic_rendition', '38 2 1 2 3 '))
+        pb('\033[1001:2:1:2:3m', ('select_graphic_rendition', '1001 2 1 2 3 '))
+        pb('\033[38:2:1:2:3;48:5:9;58;5;7m', (
+            'select_graphic_rendition', '38 2 1 2 3 '), ('select_graphic_rendition', '48 5 9 '), ('select_graphic_rendition', '58 5 7 '))
         c = s.callbacks
         pb('\033[5n', ('report_device_status', 5, 0))
         self.ae(c.wtcbuf, b'\033[0n')

kitty_tests/screen.py

@@ -228,7 +228,7 @@ class TestScreen(BaseTest):
         s.draw(''.join(str(i) * s.columns for i in range(s.lines*2)))
         self.ae(str(s.linebuf), '55555\n66666\n77777\n88888\n99999')
         s.resize(5, 2)
-        self.ae(str(s.linebuf), '88\n99\n99\n9\n')
+        self.ae(str(s.linebuf), '88\n88\n99\n99\n9')
 
     def test_cursor_after_resize(self):
         s = self.create_screen()
@@ -324,3 +324,41 @@ class TestScreen(BaseTest):
         s.cursor_visible = False
         s.toggle_alt_screen()
         self.assertFalse(s.cursor_visible)
+
+    def test_dirty_lines(self):
+        s = self.create_screen()
+        self.assertFalse(s.linebuf.dirty_lines())
+        s.draw('a' * (s.columns * 2))
+        self.ae(s.linebuf.dirty_lines(), [0, 1])
+        self.assertFalse(s.historybuf.dirty_lines())
+        while not s.historybuf.count:
+            s.draw('a' * (s.columns * 2))
+        self.ae(s.historybuf.dirty_lines(), list(range(s.historybuf.count)))
+        self.ae(s.linebuf.dirty_lines(), list(range(s.lines)))
+        s.cursor.x, s.cursor.y = 0, 1
+        s.insert_lines(2)
+        self.ae(s.linebuf.dirty_lines(), [0, 3, 4])
+        s.draw('a' * (s.columns * s.lines))
+        self.ae(s.linebuf.dirty_lines(), list(range(s.lines)))
+        s.cursor.x, s.cursor.y = 0, 1
+        s.delete_lines(2)
+        self.ae(s.linebuf.dirty_lines(), [0, 1, 2])
+
+        s = self.create_screen()
+        self.assertFalse(s.linebuf.dirty_lines())
+        s.erase_in_line(0, False)
+        self.ae(s.linebuf.dirty_lines(), [0])
+        s.index(), s.index()
+        s.erase_in_display(0, False)
+        self.ae(s.linebuf.dirty_lines(), [0, 2, 3, 4])
+
+        s = self.create_screen()
+        self.assertFalse(s.linebuf.dirty_lines())
+        s.insert_characters(2)
+        self.ae(s.linebuf.dirty_lines(), [0])
+        s.cursor.y = 1
+        s.delete_characters(2)
+        self.ae(s.linebuf.dirty_lines(), [0, 1])
+        s.cursor.y = 2
+        s.erase_characters(2)
+        self.ae(s.linebuf.dirty_lines(), [0, 1, 2])

protocol-extensions.asciidoc

@@ -139,7 +139,7 @@ The escape sequence encodes the following properties:
 
 Where `<type>` is one of `p` -- press, `r` -- release and `t` -- repeat.
 Modifiers is a bitmask represented as a single base64 digit.  Shift -- `0x1`,
-Control -- `0x2`, Alt -- `0x4` and Super -- `0x8`.  `<key>` is a number
+Alt -- `0x2`, Control -- `0x4` and Super -- `0x8`.  `<key>` is a number
 (encoded in base85) corresponding to the key pressed. The key name to number
 mapping is defined in link:key_encoding.asciidoc[this table].
 

setup.py

@@ -2,14 +2,15 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
+import argparse
+import json
 import os
 import re
-import sys
-import sysconfig
 import shlex
 import shutil
 import subprocess
-import argparse
+import sys
+import sysconfig
 
 base = os.path.dirname(os.path.abspath(__file__))
 build_dir = os.path.join(base, 'build')
@@ -45,8 +46,27 @@ def pkg_config(pkg, *args):
     )
 
 
+def at_least_version(package, major, minor=0):
+    q = '{}.{}'.format(major, minor)
+    if subprocess.run([PKGCONFIG, package, '--atleast-version=' + q]
+                      ).returncode != 0:
+        try:
+            ver = subprocess.check_output([PKGCONFIG, package, '--modversion']
+                                          ).decode('utf-8').strip()
+            qmajor, qminor = map(int, re.match(r'(\d+).(\d+)', ver).groups())
+        except Exception:
+            ver = 'not found'
+            qmajor = qminor = 0
+        if qmajor < major or (qmajor == major and qminor < minor):
+            raise SystemExit(
+                '{} >= {}.{} is required, found version: {}'.format(
+                    package, major, minor, ver
+                )
+            )
+
+
 def cc_version():
-    cc = os.environ.get('CC', 'gcc')
+    cc = os.environ.get('CC', 'clang' if isosx else 'gcc')
     raw = subprocess.check_output([cc, '-dumpversion']).decode('utf-8')
     ver = raw.split('.')[:2]
     try:
@@ -89,17 +109,18 @@ def get_python_flags(cflags):
 
 
 def get_sanitize_args(cc, ccver):
-    sanitize_args = set()
-    sanitize_args.add('-fno-omit-frame-pointer')
-    sanitize_args.add('-fsanitize=address')
-    if (cc == 'gcc' and ccver >= (5, 0)) or cc == 'clang':
-        sanitize_args.add('-fsanitize=undefined')
+    sanitize_args = ['-fsanitize=address']
+    if ccver >= (5, 0):
+        sanitize_args.append('-fsanitize=undefined')
         # if cc == 'gcc' or (cc == 'clang' and ccver >= (4, 2)):
-        #     sanitize_args.add('-fno-sanitize-recover=all')
+        #     sanitize_args.append('-fno-sanitize-recover=all')
+    sanitize_args.append('-fno-omit-frame-pointer')
     return sanitize_args
 
 
-def init_env(debug=False, sanitize=False, native_optimizations=True, profile=False):
+def init_env(
+    debug=False, sanitize=False, native_optimizations=True, profile=False
+):
     global cflags, ldflags, cc, ldpaths
     native_optimizations = native_optimizations and not sanitize and not debug
     cc, ccver = cc_version()
@@ -110,71 +131,70 @@ def init_env(debug=False, sanitize=False, native_optimizations=True, profile=Fal
     missing_braces = ''
     if ccver < (5, 2) and cc == 'gcc':
         missing_braces = '-Wno-missing-braces'
-    optimize = '-ggdb' if debug or sanitize else '-O3'
+    df = '-g3'
+    if ccver >= (5, 0):
+        df += ' -Og'
+    optimize = df if debug or sanitize else '-O3'
     sanitize_args = get_sanitize_args(cc, ccver) if sanitize else set()
     cflags = os.environ.get(
         'OVERRIDE_CFLAGS', (
             '-Wextra -Wno-missing-field-initializers -Wall -std=c99 -D_XOPEN_SOURCE=700'
             ' -pedantic-errors -Werror {} {} -D{}DEBUG -fwrapv {} {} -pipe {} -fvisibility=hidden'
         ).format(
-            optimize, ' '.join(sanitize_args), ('' if debug else 'N'), stack_protector, missing_braces,
+            optimize,
+            ' '.join(sanitize_args),
+            ('' if debug else 'N'),
+            stack_protector,
+            missing_braces,
             '-march=native' if native_optimizations else '',
         )
     )
-    cflags = shlex.split(cflags
-                         ) + shlex.split(sysconfig.get_config_var('CCSHARED'))
+    cflags = shlex.split(cflags) + shlex.split(
+        sysconfig.get_config_var('CCSHARED')
+    )
     ldflags = os.environ.get(
-        'OVERRIDE_LDFLAGS', '-Wall ' + ' '.join(sanitize_args) + ('' if debug else ' -O3')
+        'OVERRIDE_LDFLAGS',
+        '-Wall ' + ' '.join(sanitize_args) + ('' if debug else ' -O3')
     )
     ldflags = shlex.split(ldflags)
     cflags += shlex.split(os.environ.get('CFLAGS', ''))
     ldflags += shlex.split(os.environ.get('LDFLAGS', ''))
-    if not debug:
+    if not debug and not sanitize:
+        # See https://github.com/google/sanitizers/issues/647
         cflags.append('-flto'), ldflags.append('-flto')
 
     if profile:
         cflags.append('-DWITH_PROFILER')
-        cflags.append('-g')
+        cflags.append('-g3')
         ldflags.append('-lprofiler')
     cflags.append('-pthread')
     # We add 4000 to the primary version because vim turns on SGR mouse mode
     # automatically if this version is high enough
     cflags.append('-DPRIMARY_VERSION={}'.format(version[0] + 4000))
     cflags.append('-DSECONDARY_VERSION={}'.format(version[1]))
-    if not is_travis and not isosx and subprocess.Popen(
-        [PKGCONFIG, 'glew', '--atleast-version=2']
-    ).wait() != 0:
-        try:
-            ver = subprocess.check_output([PKGCONFIG, 'glew', '--modversion']
-                                          ).decode('utf-8').strip()
-            major = int(re.match(r'\d+', ver).group())
-        except Exception:
-            ver = 'not found'
-            major = 0
-        if major < 2:
-            raise SystemExit(
-                'glew >= 2.0.0 is required, found version: ' + ver
-            )
+    at_least_version('harfbuzz', 1, 5)
     cflags.extend(pkg_config('libpng', '--cflags-only-I'))
-    if not isosx:
-        cflags.extend(pkg_config('glew', '--cflags-only-I'))
     if isosx:
         font_libs = ['-framework', 'CoreText', '-framework', 'CoreGraphics']
+        cflags.extend(pkg_config('freetype2', '--cflags-only-I'))
+        font_libs += pkg_config('freetype2', '--libs')
     else:
         cflags.extend(pkg_config('fontconfig', '--cflags-only-I'))
         font_libs = pkg_config('fontconfig', '--libs')
+    cflags.extend(pkg_config('harfbuzz', '--cflags-only-I'))
+    font_libs.extend(pkg_config('harfbuzz', '--libs'))
     cflags.extend(pkg_config('glfw3', '--cflags-only-I'))
     ldflags.append('-shared')
     pylib = get_python_flags(cflags)
     if isosx:
         glfw_ldflags = pkg_config('--libs', '--static', 'glfw3'
                                   ) + ['-framework', 'OpenGL']
-        glew_libs = []
     else:
         glfw_ldflags = pkg_config('glfw3', '--libs')
-        glew_libs = pkg_config('glew', '--libs')
     libpng = pkg_config('libpng', '--libs')
-    ldpaths = pylib + glew_libs + font_libs + glfw_ldflags + libpng + ['-lunistring']
+    ldpaths = pylib + font_libs + glfw_ldflags + libpng + [
+        '-lunistring'
+    ]
     if not isosx:
         ldpaths += ['-lrt']
     if '-lz' not in ldpaths:
@@ -202,7 +222,6 @@ def run_tool(cmd):
 
 SPECIAL_SOURCES = {
     'kitty/parser_dump.c': ('kitty/parser.c', ['DUMP_COMMANDS']),
-    'kitty/shaders_debug.c': ('kitty/shaders.c', ['ENABLE_DEBUG_GL']),
 }
 
 
@@ -225,34 +244,47 @@ def dependecies_for(src, obj, all_headers):
         yield src
         yield from iter(all_headers)
     else:
-        RE_INC = re.compile(r'^(?P<target>.+?):\s+(?P<deps>.+?)$', re.MULTILINE)
+        RE_INC = re.compile(
+            r'^(?P<target>.+?):\s+(?P<deps>.+?)$', re.MULTILINE
+        )
         SPACE_TOK = '\x1B'
 
         text = deps.replace('\\\n', ' ').replace('\\ ', SPACE_TOK)
         for match in RE_INC.finditer(text):
-            files = (f.replace(SPACE_TOK, ' ') for f in match.group('deps').split())
+            files = (
+                f.replace(SPACE_TOK, ' ') for f in match.group('deps').split()
+            )
             for path in files:
                 path = os.path.abspath(path)
                 if path.startswith(base):
                     yield path
 
 
-def compile_c_extension(module, incremental, sources, headers):
+def compile_c_extension(module, incremental, compilation_database, sources, headers):
     prefix = os.path.basename(module)
     objects = [
         os.path.join(build_dir, prefix + '-' + os.path.basename(src) + '.o')
         for src in sources
     ]
 
-    for src, dest in zip(sources, objects):
+    for original_src, dest in zip(sources, objects):
+        src = original_src
         cflgs = cflags[:]
-        if src in SPECIAL_SOURCES:
+        is_special = src in SPECIAL_SOURCES
+        if is_special:
             src, defines = SPECIAL_SOURCES[src]
             cflgs.extend(map(define, defines))
 
+        cmd = [cc, '-MMD'] + cflgs
+        cmd_changed = compilation_database.get(original_src, [])[:-4] != cmd
+        must_compile = not incremental or cmd_changed
         src = os.path.join(base, src)
-        if not incremental or newer(dest, *dependecies_for(src, dest, headers)):
-            run_tool([cc, '-MMD'] + cflgs + ['-c', src] + ['-o', dest])
+        if must_compile or newer(
+            dest, *dependecies_for(src, dest, headers)
+        ):
+            cmd += ['-c', src] + ['-o', dest]
+            compilation_database[original_src] = cmd
+            run_tool(cmd)
     dest = os.path.join(base, module + '.so')
     if not incremental or newer(dest, *objects):
         run_tool([cc] + ldflags + objects + ldpaths + ['-o', dest])
@@ -288,10 +320,11 @@ def option_parser():
         help='Where to create the linux package'
     )
     p.add_argument(
-        '--incremental',
-        default=False,
-        action='store_true',
-        help='Only build changed files'
+        '--full',
+        dest='incremental',
+        default=True,
+        action='store_false',
+        help='Do a full build, even for unchanged files'
     )
     p.add_argument(
         '--profile',
@@ -305,7 +338,7 @@ def option_parser():
 def find_c_files():
     ans, headers = [], []
     d = os.path.join(base, 'kitty')
-    exclude = {'freetype.c', 'fontconfig.c'} if isosx else {'core_text.m', 'cocoa_window.m'}
+    exclude = {'fontconfig.c'} if isosx else {'core_text.m', 'cocoa_window.m'}
     for x in os.listdir(d):
         ext = os.path.splitext(x)[1]
         if ext in ('.c', '.m') and os.path.basename(x) not in exclude:
@@ -316,30 +349,44 @@ def find_c_files():
         key=lambda x: os.path.getmtime(os.path.join(base, x)), reverse=True
     )
     ans.append('kitty/parser_dump.c')
-    ans.append('kitty/shaders_debug.c')
     return tuple(ans), tuple(headers)
 
 
 def build(args, native_optimizations=True):
+    try:
+        with open('compile_commands.json') as f:
+            compilation_database = json.load(f)
+    except FileNotFoundError:
+        compilation_database = []
+    compilation_database = {
+        k['file']: k['arguments'] for k in compilation_database
+    }
     init_env(args.debug, args.sanitize, native_optimizations, args.profile)
     compile_c_extension(
-        'kitty/fast_data_types', args.incremental, *find_c_files()
+        'kitty/fast_data_types', args.incremental, compilation_database, *find_c_files()
     )
+    compilation_database = [
+        {'file': k, 'arguments': v, 'directory': base} for k, v in compilation_database.items()
+    ]
+    with open('compile_commands.json', 'w') as f:
+        json.dump(compilation_database, f, indent=2, sort_keys=True)
 
 
 def safe_makedirs(path):
     os.makedirs(path, exist_ok=True)
 
 
-def build_test_launcher(args):
+def build_asan_launcher(args):
+    dest = 'asan-launcher'
+    src = 'asan-launcher.c'
+    if args.incremental and not newer(dest, src):
+        return
     cc, ccver = cc_version()
-    cflags = '-g -Wall -Werror -fpie'.split()
+    cflags = '-g3 -Wall -Werror -fpie -std=c99'.split()
     pylib = get_python_flags(cflags)
-    sanitize_lib = (['-lasan'] if cc == 'gcc' else []) if args.sanitize else []
-    cflags.extend(get_sanitize_args(cc, ccver) if args.sanitize else [])
-    cmd = [cc] + cflags + [
-        'test-launcher.c', '-o', 'test-launcher',
-    ] + sanitize_lib + pylib
+    sanitize_lib = ['-lasan'] if cc == 'gcc' and not isosx else []
+    cflags.extend(get_sanitize_args(cc, ccver))
+    cmd = [cc] + cflags + [src, '-o', dest] + sanitize_lib + pylib
     run_tool(cmd)
 
 
@@ -357,7 +404,8 @@ def build_linux_launcher(args, launcher_dir='.', for_bundle=False):
     pylib = get_python_flags(cflags)
     exe = 'kitty-profile' if args.profile else 'kitty'
     cmd = [cc] + cflags + [
-        'linux-launcher.c', '-o', os.path.join(launcher_dir, exe)
+        'linux-launcher.c', '-o',
+        os.path.join(launcher_dir, exe)
     ] + libs + pylib
     run_tool(cmd)
 
@@ -378,7 +426,8 @@ def package(args, for_bundle=False):  # {{{
     def src_ignore(parent, entries):
         return [
             x for x in entries
-            if '.' in x and x.rpartition('.')[2] not in ('py', 'so', 'conf', 'glsl')
+            if '.' in x and x.rpartition('.')[2] not in
+            ('py', 'so', 'conf', 'glsl')
         ]
 
     shutil.copytree('kitty', os.path.join(libdir, 'kitty'), ignore=src_ignore)
@@ -392,7 +441,7 @@ def package(args, for_bundle=False):  # {{{
     safe_makedirs(launcher_dir)
     build_linux_launcher(args, launcher_dir, for_bundle)
     if not isosx:  # {{{ linux desktop gunk
-        icdir = os.path.join(ddir, 'share', 'icons', 'hicolor', '256x256')
+        icdir = os.path.join(ddir, 'share', 'icons', 'hicolor', '256x256', 'apps')
         safe_makedirs(icdir)
         shutil.copy2('logo/kitty.png', icdir)
         deskdir = os.path.join(ddir, 'share', 'applications')
@@ -426,7 +475,9 @@ Categories=System;
 
 
 def clean():
-    for f in subprocess.check_output('git ls-files --others --ignored --exclude-from=.gitignore'.split()).decode('utf-8').splitlines():
+    for f in subprocess.check_output(
+        'git ls-files --others --ignored --exclude-from=.gitignore'.split()
+    ).decode('utf-8').splitlines():
         if f.startswith('logo/kitty.iconset') or f.startswith('dev/'):
             continue
         os.unlink(f)
@@ -442,7 +493,7 @@ def main():
     os.chdir(os.path.dirname(os.path.abspath(__file__)))
     if args.action == 'build':
         build(args)
-        build_test_launcher(args)
+        build_asan_launcher(args)
         if args.profile:
             build_linux_launcher(args)
             print('kitty profile executable is', 'kitty-profile')

terminfo/kitty.terminfo

@@ -85,6 +85,7 @@ xterm-kitty|KovIdTTY,
 	khlp=,
 	khome=\EOH,
 	kich1=\E[2~,
+	kind=\E[1;2B,
 	kmous=\E[M,
 	knp=\E[6~,
 	kpp=\E[5~,

test-launcher.c

@@ -1,13 +0,0 @@
-/*
- * linux-launcher.c
- * Copyright (C) 2017 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include <Python.h>
-
-int main(int argc, char *argv[]) {
-    wchar_t *wargv[2] = {L"kitty-test", L"test.py"};
-    return Py_Main(2, wargv);
-}

update-on-ox

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+import os
+import shlex
+import subprocess
+import sys
+
+base = os.path.dirname(os.path.abspath(__file__))
+if True:
+    sys.path.insert(0, base)
+    from kitty.constants import str_version
+
+dmg = f'kitty-{str_version}.dmg'
+
+
+def run(what):
+    ret = subprocess.run(shlex.split(what))
+    if ret.returncode != 0:
+        raise SystemExit(ret.returncode)
+
+
+os.chdir(os.path.expanduser('~/work/build-kitty'))
+run('./osx kitty --debug-build')
+run(f'scp build/osx/dist/{dmg} ox:/tmp')
+run(f'ssh ox /Users/kovid/install-dmg /tmp/{dmg}')