Ensure the extended keyboard protocol key encoding is stable

Also use base85 instead of base64 for keyname encoding to reduce average
length

count-lines-of-code

@@ -1,2 +1,2 @@
 #!/bin/bash
-cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/unicode-data.h\nkitty/gl.h\nkitty/glfw.c\nkitty/glfw.h\nkitty/charsets.c') kitty
+cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/unicode-data.h\nkitty/gl.h\nkitty/glfw.c\nkitty/glfw.h\nkitty/charsets.c\nkitty/key_encoding.py') kitty

key_encoding.asciidoc

@@ -21,95 +21,95 @@ See link:protocol-extensions.asciidoc#keyboard-handling[Keyboard Handling protoc
 | BACKSLASH       | `t`
 | BACKSPACE       | `1`
 | C               | `U`
-| CAPS LOCK       | `BA`
+| CAPS LOCK       | `:`
 | COMMA           | `C`
 | D               | `V`
 | DELETE          | `3`
 | DOWN            | `6`
 | E               | `W`
-| END             | `/`
+| END             | `-`
 | ENTER           | `z`
 | EQUAL           | `R`
 | ESCAPE          | `y`
 | F               | `X`
-| F1              | `BF`
+| F1              | `/`
-| F10             | `BO`
+| F10             | `]`
-| F11             | `BP`
+| F11             | `{`
-| F12             | `BQ`
+| F12             | `}`
-| F13             | `BR`
+| F13             | `@`
-| F14             | `BS`
+| F14             | `%`
-| F15             | `BT`
+| F15             | `$`
-| F16             | `BU`
+| F16             | `#`
-| F17             | `BV`
+| F17             | `BA`
-| F18             | `BW`
+| F18             | `BB`
-| F19             | `BX`
+| F19             | `BC`
-| F2              | `BG`
+| F2              | `*`
-| F20             | `BY`
+| F20             | `BD`
-| F21             | `BZ`
+| F21             | `BE`
-| F22             | `Ba`
+| F22             | `BF`
-| F23             | `Bb`
+| F23             | `BG`
-| F24             | `Bc`
+| F24             | `BH`
-| F25             | `Bd`
+| F25             | `BI`
-| F3              | `BH`
+| F3              | `?`
-| F4              | `BI`
+| F4              | `&`
-| F5              | `BJ`
+| F5              | `<`
-| F6              | `BK`
+| F6              | `>`
-| F7              | `BL`
+| F7              | `(`
-| F8              | `BM`
+| F8              | `)`
-| F9              | `BN`
+| F9              | `[`
 | G               | `Y`
 | GRAVE ACCENT    | `v`
 | H               | `Z`
-| HOME            | `+`
+| HOME            | `.`
 | I               | `a`
 | INSERT          | `2`
 | J               | `b`
 | K               | `c`
-| KP 0            | `Be`
+| KP 0            | `BJ`
-| KP 1            | `Bf`
+| KP 1            | `BK`
-| KP 2            | `Bg`
+| KP 2            | `BL`
-| KP 3            | `Bh`
+| KP 3            | `BM`
-| KP 4            | `Bi`
+| KP 4            | `BN`
-| KP 5            | `Bj`
+| KP 5            | `BO`
-| KP 6            | `Bk`
+| KP 6            | `BP`
-| KP 7            | `Bl`
+| KP 7            | `BQ`
-| KP 8            | `Bm`
+| KP 8            | `BR`
-| KP 9            | `Bn`
+| KP 9            | `BS`
-| KP ADD          | `Bs`
+| KP ADD          | `BX`
-| KP DECIMAL      | `Bo`
+| KP DECIMAL      | `BT`
-| KP DIVIDE       | `Bp`
+| KP DIVIDE       | `BU`
-| KP ENTER        | `Bt`
+| KP ENTER        | `BY`
-| KP EQUAL        | `Bu`
+| KP EQUAL        | `BZ`
-| KP MULTIPLY     | `Bq`
+| KP MULTIPLY     | `BV`
-| KP SUBTRACT     | `Br`
+| KP SUBTRACT     | `BW`
 | L               | `d`
 | LEFT            | `5`
-| LEFT ALT        | `Bx`
+| LEFT ALT        | `Bc`
 | LEFT BRACKET    | `s`
-| LEFT CONTROL    | `Bw`
+| LEFT CONTROL    | `Bb`
-| LEFT SHIFT      | `Bv`
+| LEFT SHIFT      | `Ba`
-| LEFT SUPER      | `By`
+| LEFT SUPER      | `Bd`
 | M               | `e`
 | MINUS           | `D`
 | N               | `f`
-| NUM LOCK        | `BC`
+| NUM LOCK        | `=`
 | O               | `g`
 | P               | `h`
 | PAGE DOWN       | `9`
 | PAGE UP         | `8`
-| PAUSE           | `BE`
+| PAUSE           | `!`
 | PERIOD          | `E`
-| PRINT SCREEN    | `BD`
+| PRINT SCREEN    | `^`
 | Q               | `i`
 | R               | `j`
 | RIGHT           | `4`
-| RIGHT ALT       | `B1`
+| RIGHT ALT       | `Bg`
 | RIGHT BRACKET   | `u`
-| RIGHT CONTROL   | `B0`
+| RIGHT CONTROL   | `Bf`
-| RIGHT SHIFT     | `Bz`
+| RIGHT SHIFT     | `Be`
-| RIGHT SUPER     | `B2`
+| RIGHT SUPER     | `Bh`
 | S               | `k`
-| SCROLL LOCK     | `BB`
+| SCROLL LOCK     | `+`
 | SEMICOLON       | `Q`
 | SLASH           | `F`
 | SPACE           | `A`

kitty/key_encoding.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+import string
+
+from . import fast_data_types as defines
+
+# ENCODING {{{
+ENCODING = {
+    '0': 'G',
+    '1': 'H',
+    '2': 'I',
+    '3': 'J',
+    '4': 'K',
+    '5': 'L',
+    '6': 'M',
+    '7': 'N',
+    '8': 'O',
+    '9': 'P',
+    'A': 'S',
+    'APOSTROPHE': 'B',
+    'B': 'T',
+    'BACKSLASH': 't',
+    'BACKSPACE': '1',
+    'C': 'U',
+    'CAPS LOCK': ':',
+    'COMMA': 'C',
+    'D': 'V',
+    'DELETE': '3',
+    'DOWN': '6',
+    'E': 'W',
+    'END': '-',
+    'ENTER': 'z',
+    'EQUAL': 'R',
+    'ESCAPE': 'y',
+    'F': 'X',
+    'F1': '/',
+    'F10': ']',
+    'F11': '{',
+    'F12': '}',
+    'F13': '@',
+    'F14': '%',
+    'F15': '$',
+    'F16': '#',
+    'F17': 'BA',
+    'F18': 'BB',
+    'F19': 'BC',
+    'F2': '*',
+    'F20': 'BD',
+    'F21': 'BE',
+    'F22': 'BF',
+    'F23': 'BG',
+    'F24': 'BH',
+    'F25': 'BI',
+    'F3': '?',
+    'F4': '&',
+    'F5': '<',
+    'F6': '>',
+    'F7': '(',
+    'F8': ')',
+    'F9': '[',
+    'G': 'Y',
+    'GRAVE ACCENT': 'v',
+    'H': 'Z',
+    'HOME': '.',
+    'I': 'a',
+    'INSERT': '2',
+    'J': 'b',
+    'K': 'c',
+    'KP 0': 'BJ',
+    'KP 1': 'BK',
+    'KP 2': 'BL',
+    'KP 3': 'BM',
+    'KP 4': 'BN',
+    'KP 5': 'BO',
+    'KP 6': 'BP',
+    'KP 7': 'BQ',
+    'KP 8': 'BR',
+    'KP 9': 'BS',
+    'KP ADD': 'BX',
+    'KP DECIMAL': 'BT',
+    'KP DIVIDE': 'BU',
+    'KP ENTER': 'BY',
+    'KP EQUAL': 'BZ',
+    'KP MULTIPLY': 'BV',
+    'KP SUBTRACT': 'BW',
+    'L': 'd',
+    'LEFT': '5',
+    'LEFT ALT': 'Bc',
+    'LEFT BRACKET': 's',
+    'LEFT CONTROL': 'Bb',
+    'LEFT SHIFT': 'Ba',
+    'LEFT SUPER': 'Bd',
+    'M': 'e',
+    'MINUS': 'D',
+    'N': 'f',
+    'NUM LOCK': '=',
+    'O': 'g',
+    'P': 'h',
+    'PAGE DOWN': '9',
+    'PAGE UP': '8',
+    'PAUSE': '!',
+    'PERIOD': 'E',
+    'PRINT SCREEN': '^',
+    'Q': 'i',
+    'R': 'j',
+    'RIGHT': '4',
+    'RIGHT ALT': 'Bg',
+    'RIGHT BRACKET': 'u',
+    'RIGHT CONTROL': 'Bf',
+    'RIGHT SHIFT': 'Be',
+    'RIGHT SUPER': 'Bh',
+    'S': 'k',
+    'SCROLL LOCK': '+',
+    'SEMICOLON': 'Q',
+    'SLASH': 'F',
+    'SPACE': 'A',
+    'T': 'l',
+    'TAB': '0',
+    'U': 'm',
+    'UP': '7',
+    'V': 'n',
+    'W': 'o',
+    'WORLD 1': 'w',
+    'WORLD 2': 'x',
+    'X': 'p',
+    'Y': 'q',
+    'Z': 'r'
+}
+
+# END_ENCODING }}}
+
+
+def encode(
+    integer,
+    chars=string.ascii_uppercase + string.ascii_lowercase + string.digits +
+    '.-:+=^!/*?&<>()[]{}@%$#'
+):
+    ans = ''
+    d = len(chars)
+    while True:
+        integer, remainder = divmod(integer, d)
+        ans = chars[remainder] + ans
+        if integer == 0:
+            break
+    return ans
+
+
+def symbolic_name(glfw_name):
+    return glfw_name[9:].replace('_', ' ')
+
+
+def generate_extended_key_map(symbolic=False):
+    keys = (a for a in dir(defines) if a.startswith('GLFW_KEY_'))
+    ans = {}
+    for k in keys:
+        name = symbolic_name(k)
+        enc = ENCODING.get(name)
+        if name is not None:
+            ans[getattr(defines, k)] = enc
+    return ans
+
+
+def update_encoding():
+    import re
+    import subprocess
+    from pprint import pformat
+    keys = {a for a in dir(defines) if a.startswith('GLFW_KEY_')}
+    ans = ENCODING
+    i = len(ans)
+    for k in sorted(keys, key=lambda k: getattr(defines, k)):
+        val = getattr(defines, k)
+        name = symbolic_name(k)
+        if val < defines.GLFW_KEY_LAST and val != defines.GLFW_KEY_UNKNOWN and name not in ans:
+            ans[name] = encode(i)
+            i += 1
+    with open(__file__, 'r+') as f:
+        raw = f.read()
+        nraw = re.sub(
+            r'^ENCODING = {.+^# END_ENCODING',
+            'ENCODING = {}\n# END_ENCODING'.format(pformat(ans, indent=4)),
+            raw,
+            flags=re.MULTILINE | re.DOTALL
+        )
+        if raw == nraw:
+            raise SystemExit('Failed to replace ENCODING dict')
+        f.seek(0), f.truncate()
+        f.write(nraw)
+    subprocess.check_call(['yapf', '-i', __file__])

kitty/keys.py

@@ -2,10 +2,10 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-import string
-
 from . import fast_data_types as defines
 from .terminfo import key_as_bytes
+from .utils import base64_encode
+from .key_encoding import generate_extended_key_map
 
 smkx_key_map = {
     defines.GLFW_KEY_UP: 'kcuu1',
@@ -114,36 +114,7 @@ action_map = {
     defines.GLFW_REPEAT: b't'
 }
 
-
+extended_key_map = generate_extended_key_map()
-def base64_encode(
-    integer,
-    chars=string.ascii_uppercase + string.ascii_lowercase + string.digits +
-    '+/'
-):
-    ans = ''
-    while True:
-        integer, remainder = divmod(integer, 64)
-        ans = chars[remainder] + ans
-        if integer == 0:
-            break
-    return ans
-
-
-def generate_key_extended_map(symbolic=False):
-    ans = {}
-    i = 0
-    keys = {a for a in dir(defines) if a.startswith('GLFW_KEY_')}
-
-    for k in sorted(keys, key=lambda k: getattr(defines, k)):
-        val = getattr(defines, k)
-        if val < defines.GLFW_KEY_LAST and val != defines.GLFW_KEY_UNKNOWN:
-            key = k[9:] if symbolic else val
-            ans[key] = base64_encode(i)
-            i += 1
-    return ans
-
-
-key_extended_map = generate_key_extended_map()
 
 
 def extended_key_event(key, scancode, mods, action):
@@ -156,8 +127,11 @@ def extended_key_event(key, scancode, mods, action):
         defines.GLFW_KEY_BACKSPACE, defines.GLFW_KEY_ENTER
     ):
         return smkx_key_map[key]
+    name = extended_key_map.get(key)
+    if name is None:
+        return b''
     return '\033_K{}{}{}\033\\'.format(
-        action_map[action], base64_encode(mods), key_extended_map[key]
+        action_map[action], base64_encode(mods), name
     ).encode('ascii')
 
 

kitty/utils.py

@@ -2,12 +2,13 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
-import re
-import os
-import signal
-import shlex
-import subprocess
 import math
+import os
+import re
+import shlex
+import signal
+import string
+import subprocess
 from collections import namedtuple
 from contextlib import contextmanager
 from functools import lru_cache
@@ -56,7 +57,9 @@ def get_logical_dpi():
             get_logical_dpi.ans = glfw_get_physical_dpi()
         else:
             raw = subprocess.check_output(['xdpyinfo']).decode('utf-8')
-            m = re.search(r'^\s*resolution:\s*(\d+)+x(\d+)', raw, flags=re.MULTILINE)
+            m = re.search(
+                r'^\s*resolution:\s*(\d+)+x(\d+)', raw, flags=re.MULTILINE
+            )
             get_logical_dpi.ans = int(m.group(1)), int(m.group(2))
     return get_logical_dpi.ans
 
@@ -67,11 +70,13 @@ def get_dpi():
         get_dpi.ans = {'physical': pdpi, 'logical': get_logical_dpi()}
     return get_dpi.ans
 
+
 # Color names  {{{
 
-
 color_pat = re.compile(r'^#([a-fA-F0-9]{3}|[a-fA-F0-9]{6})$')
-color_pat2 = re.compile(r'rgb:([a-f0-9]{2})/([a-f0-9]{2})/([a-f0-9]{2})$', re.IGNORECASE)
+color_pat2 = re.compile(
+    r'rgb:([a-f0-9]{2})/([a-f0-9]{2})/([a-f0-9]{2})$', re.IGNORECASE
+)
 
 color_names = {
     'aliceblue': 'f0f8ff',
@@ -223,6 +228,7 @@ color_names = {
     'yellowgreen': '9acd32',
 }
 Color = namedtuple('Color', 'red green blue')
+
 # }}}
 
 
@@ -295,6 +301,20 @@ def get_primary_selection():
     return subprocess.check_output(['xsel', '-p']).decode('utf-8')
 
 
+def base64_encode(
+    integer,
+    chars=string.ascii_uppercase + string.ascii_lowercase + string.digits +
+    '+/'
+):
+    ans = ''
+    while True:
+        integer, remainder = divmod(integer, 64)
+        ans = chars[remainder] + ans
+        if integer == 0:
+            break
+    return ans
+
+
 def set_primary_selection(text):
     if isosx:
         return  # There is no primary selection on OS X

preprocess-readme.py

@@ -37,7 +37,7 @@ if raw != nraw:
 
 raw = subprocess.check_output([
     'kitty', '-c',
-    'from kitty.keys import *; import json; print(json.dumps(generate_key_extended_map(True)))'
+    'from kitty.key_encoding import *; import json; print(json.dumps(ENCODING))'
 ]).decode('utf-8')
 key_map = json.loads(raw)
 lines = [
@@ -45,7 +45,7 @@ lines = [
     '', '|===', '| Name | Encoded representation (base64)', ''
 ]
 for k in sorted(key_map):
-    lines.append('| {:15s} | `{}`'.format(k.replace('_', ' '), key_map[k]))
+    lines.append('| {:15s} | `{}`'.format(k.replace('_', ' '), key_map[k].replace('`', '\\`')))
 lines += ['', '|===']
 with open('key_encoding.asciidoc', 'w') as f:
     print('= Key encoding for extended keyboard protocol\n', file=f)

protocol-extensions.asciidoc

@@ -385,7 +385,7 @@ The escape sequence encodes the following properties:
 Where `<type>` is one of `p` -- press, `r` -- release and `t` -- repeat.
 Modifiers is a bitmask represented as a single base64 digit.  Shift -- `0x1`,
 Control -- `0x2`, Alt -- `0x4` and Super -- `0x8`.  `<key>` is a number
-(encoded in base64) corresponding to the key pressed. The key name to number
+(encoded in base85) corresponding to the key pressed. The key name to number
 mapping is defined in link:key_encoding.asciidoc[this table].
 
 For example: