Implement protocol for atomic screen updates

See https://gitlab.com/gnachman/iterm2/wikis/synchronized-updates-spec

kitty/child-monitor.c

@@ -25,7 +25,7 @@ extern PyTypeObject Screen_Type;
 #define MSG_NOSIGNAL 0
 #endif
 
-static void (*parse_func)(Screen*, PyObject*);
+static void (*parse_func)(Screen*, PyObject*, double);
 
 typedef struct {
     char *data;
@@ -314,13 +314,17 @@ shutdown_monitor(ChildMonitor *self, PyObject *a UNUSED) {
 static inline void
 do_parse(ChildMonitor *self, Screen *screen, double now) {
     screen_mutex(lock, read);
-    if (screen->read_buf_sz) {
+    if (screen->read_buf_sz || screen->pending_mode.used) {
         double time_since_new_input = now - screen->new_input_at;
         if (time_since_new_input >= OPT(input_delay)) {
-            parse_func(screen, self->dump_callback);
+            bool read_buf_full = screen->read_buf_sz >= READ_BUF_SZ;
-            if (screen->read_buf_sz >= READ_BUF_SZ) wakeup_io_loop(false);  // Ensure the read fd has POLLIN set
+            parse_func(screen, self->dump_callback, now);
-            screen->read_buf_sz = 0;
+            if (read_buf_full) wakeup_io_loop(false);  // Ensure the read fd has POLLIN set
             screen->new_input_at = 0;
+            if (screen->pending_mode.activated_at) {
+                double time_since_pending = MAX(0, now - screen->pending_mode.activated_at);
+                set_maximum_wait(screen->pending_mode.wait_time - time_since_pending);
+            }
         } else set_maximum_wait(OPT(input_delay) - time_since_new_input);
     }
     screen_mutex(unlock, read);

kitty/parser.c

@@ -775,6 +775,8 @@ startswith(const uint32_t *string, size_t sz, const char *prefix) {
     return true;
 }
 
+#define PENDING_MODE_CHAR '='
+
 static inline void
 dispatch_dcs(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
     if (screen->parser_buf_pos < 2) return;
@@ -792,6 +794,14 @@ dispatch_dcs(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
                 REPORT_ERROR("Unrecognized DCS %c code: 0x%x", (char)screen->parser_buf[0], screen->parser_buf[1]);
             }
             break;
+        case PENDING_MODE_CHAR:
+            if (screen->parser_buf[1] == 's') {
+                screen->pending_mode.activated_at = monotonic();
+                REPORT_COMMAND(screen_start_pending_mode);
+            } else {
+                REPORT_ERROR("Unrecognized DCS %c code: 0x%x", (char)screen->parser_buf[0], screen->parser_buf[1]);
+            }
+            break;
         case '@':
 #define CMD_PREFIX "kitty-cmd{"
             if (startswith(screen->parser_buf + 1, screen->parser_buf_pos - 2, CMD_PREFIX)) {
@@ -1003,45 +1013,45 @@ END_ALLOW_CASE_RANGE
 #undef ENSURE_SPACE
 }
 
-static inline void
+#define dispatch_unicode_char(codepoint, watch_for_pending) { \
-dispatch_unicode_char(Screen *screen, uint32_t codepoint, PyObject DUMP_UNUSED *dump_callback) {
+    switch(screen->parser_state) { \
-#define HANDLE(name) handle_##name(screen, codepoint, dump_callback); break
+        case ESC: \
-    switch(screen->parser_state) {
+            handle_esc_mode_char(screen, codepoint, dump_callback); \
-        case ESC:
+            break; \
-            HANDLE(esc_mode_char);
+        case CSI: \
-        case CSI:
+            if (accumulate_csi(screen, codepoint, dump_callback)) { dispatch_csi(screen, dump_callback); SET_STATE(0); } \
-            if (accumulate_csi(screen, codepoint, dump_callback)) { dispatch_csi(screen, dump_callback); SET_STATE(0); }
+            break; \
-            break;
+        case OSC: \
-        case OSC:
+            if (accumulate_osc(screen, codepoint, dump_callback)) { dispatch_osc(screen, dump_callback); SET_STATE(0); } \
-            if (accumulate_osc(screen, codepoint, dump_callback)) { dispatch_osc(screen, dump_callback); SET_STATE(0); }
+            break; \
-            break;
+        case APC: \
-        case APC:
+            if (accumulate_oth(screen, codepoint, dump_callback)) { dispatch_apc(screen, dump_callback); SET_STATE(0); } \
-            if (accumulate_oth(screen, codepoint, dump_callback)) { dispatch_apc(screen, dump_callback); SET_STATE(0); }
+            break; \
-            break;
+        case PM: \
-        case PM:
+            if (accumulate_oth(screen, codepoint, dump_callback)) { dispatch_pm(screen, dump_callback); SET_STATE(0); } \
-            if (accumulate_oth(screen, codepoint, dump_callback)) { dispatch_pm(screen, dump_callback); SET_STATE(0); }
+            break; \
-            break;
+        case DCS: \
-        case DCS:
+            if (accumulate_dcs(screen, codepoint, dump_callback)) { dispatch_dcs(screen, dump_callback); SET_STATE(0); watch_for_pending; } \
-            if (accumulate_dcs(screen, codepoint, dump_callback)) { dispatch_dcs(screen, dump_callback); SET_STATE(0); }
+            if (screen->parser_state == ESC) { handle_esc_mode_char(screen, codepoint, dump_callback); break; } \
-            if (screen->parser_state == ESC) { HANDLE(esc_mode_char); }
+            break; \
-            break;
+        default: \
-        default:
+            handle_normal_mode_char(screen, codepoint, dump_callback); \
-            HANDLE(normal_mode_char);
+            break; \
-    }
+    } \
-#undef HANDLE
+} \
-}
 
 extern uint32_t *latin1_charset;
 
 static inline void
-_parse_bytes(Screen *screen, uint8_t *buf, Py_ssize_t len, PyObject DUMP_UNUSED *dump_callback) {
+_parse_bytes(Screen *screen, const uint8_t *buf, Py_ssize_t len, PyObject DUMP_UNUSED *dump_callback) {
     uint32_t prev = screen->utf8_state;
     for (unsigned int i = 0; i < (unsigned int)len; i++) {
-        if (screen->use_latin1) dispatch_unicode_char(screen, latin1_charset[buf[i]], dump_callback);
+        if (screen->use_latin1) {
-        else {
+            dispatch_unicode_char(latin1_charset[buf[i]], ;);
+        } else {
             switch (decode_utf8(&screen->utf8_state, &screen->utf8_codepoint, buf[i])) {
                 case UTF8_ACCEPT:
-                    dispatch_unicode_char(screen, screen->utf8_codepoint, dump_callback);
+                    dispatch_unicode_char(screen->utf8_codepoint, ;);
                     break;
                 case UTF8_REJECT:
                     screen->utf8_state = UTF8_ACCEPT;
@@ -1053,6 +1063,162 @@ _parse_bytes(Screen *screen, uint8_t *buf, Py_ssize_t len, PyObject DUMP_UNUSED
     }
 FLUSH_DRAW;
 }
+
+static inline size_t
+_parse_bytes_watching_for_pending(Screen *screen, const uint8_t *buf, Py_ssize_t len, PyObject DUMP_UNUSED *dump_callback) {
+    uint32_t prev = screen->utf8_state;
+    size_t i = 0;
+    while(i < (size_t)len) {
+        uint8_t ch = buf[i++];
+        if (screen->use_latin1) {
+            dispatch_unicode_char(latin1_charset[ch], if (screen->pending_mode.activated_at) goto end);
+        } else {
+            switch (decode_utf8(&screen->utf8_state, &screen->utf8_codepoint, ch)) {
+                case UTF8_ACCEPT:
+                    dispatch_unicode_char(screen->utf8_codepoint, if (screen->pending_mode.activated_at) goto end);
+                    break;
+                case UTF8_REJECT:
+                    screen->utf8_state = UTF8_ACCEPT;
+                    if (prev != UTF8_ACCEPT && i > 0) i--;
+                    break;
+            }
+            prev = screen->utf8_state;
+        }
+    }
+end:
+FLUSH_DRAW;
+    return i;
+}
+
+
+static inline size_t
+_queue_pending_bytes(Screen *screen, const uint8_t *buf, size_t len, PyObject *dump_callback DUMP_UNUSED) {
+    size_t pos = 0;
+    enum STATE { NORMAL, MAYBE_DCS, IN_DCS, EXPECTING_START_OR_ESC, EXPECTING_ESC, EXPECTING_SLASH };
+    enum STATE state = screen->pending_mode.state;
+    bool is_end_code = true;
+#define COPY(what) screen->pending_mode.buf[screen->pending_mode.used++] = what
+    while (pos < len) {
+        uint8_t ch = buf[pos++];
+        switch(state) {
+            case NORMAL:
+                if (ch == ESC) state = MAYBE_DCS;
+                else COPY(ch);
+                break;
+            case MAYBE_DCS:
+                if (ch == 'P') state = IN_DCS;
+                else {
+                    state = NORMAL;
+                    COPY(0x1b); COPY(ch);
+                }
+                break;
+            case IN_DCS:
+                if (ch == PENDING_MODE_CHAR) { state = EXPECTING_START_OR_ESC; is_end_code = true; }
+                else {
+                    state = NORMAL;
+                    COPY(0x1b); COPY('P'); COPY(ch);
+                }
+                break;
+            case EXPECTING_START_OR_ESC:
+                if (ch == 0x1b) state = EXPECTING_SLASH;
+                else if (ch == 's') {
+                    state = EXPECTING_ESC;
+                    is_end_code = false;
+                } else {
+                    state = NORMAL;
+                    COPY(0x1b); COPY('P'); COPY('='); COPY(ch);
+                }
+                break;
+            case EXPECTING_ESC:
+                if (ch == 0x1b) state = EXPECTING_SLASH;
+                else {
+                    state = NORMAL;
+                    COPY(0x1b); COPY('P'); COPY('=');
+                    if (!is_end_code) COPY('s');
+                    COPY(ch);
+                }
+                break;
+            case EXPECTING_SLASH:
+                if (ch == '\\') {
+                    // We found a pending mode sequence
+                    if (is_end_code) {
+                        REPORT_COMMAND(screen_stop_pending_mode);
+                        screen->pending_mode.activated_at = 0;
+                        goto end;
+                    } else {
+                        REPORT_COMMAND(screen_start_pending_mode);
+                        screen->pending_mode.activated_at = monotonic();
+                    }
+                } else {
+                    state = NORMAL;
+                    COPY(0x1b); COPY('P'); COPY('=');
+                    if (!is_end_code) { COPY('s'); }
+                    COPY(0x1b); COPY(ch);
+                }
+                break;
+        }
+    }
+end:
+    screen->pending_mode.state = state;
+    return pos;
+#undef COPY
+}
+
+static inline void
+do_parse_bytes(Screen *screen, const uint8_t *read_buf, const size_t read_buf_sz, double now, PyObject *dump_callback DUMP_UNUSED) {
+    enum STATE {START, PARSE_PENDING, PARSE_READ_BUF, QUEUE_PENDING};
+    enum STATE state = START;
+    size_t read_buf_pos = 0;
+
+    do {
+        switch(state) {
+            case START:
+                if (screen->pending_mode.activated_at) {
+                    if (screen->pending_mode.activated_at + screen->pending_mode.wait_time < now) {
+                        screen->pending_mode.activated_at = 0;
+                        state = screen->pending_mode.used ? PARSE_PENDING : PARSE_READ_BUF;
+                    } else state = QUEUE_PENDING;
+                } else {
+                    if (screen->pending_mode.used) state = PARSE_PENDING;
+                    else {
+                        state = PARSE_READ_BUF;
+                    }
+                }
+                break;
+
+            case PARSE_PENDING:
+                _parse_bytes(screen, screen->pending_mode.buf, screen->pending_mode.used, dump_callback);
+                screen->pending_mode.used = 0; screen->pending_mode.state = 0;
+                screen->pending_mode.activated_at = 0;  // ignore any pending starts in the pending bytes
+                state = START;
+                break;
+
+            case PARSE_READ_BUF:
+                screen->pending_mode.activated_at = 0; screen->pending_mode.state = 0;
+                read_buf_pos += _parse_bytes_watching_for_pending(screen, read_buf + read_buf_pos, read_buf_sz - read_buf_pos, dump_callback);
+                state = START;
+                break;
+
+            case QUEUE_PENDING: {
+                if (screen->pending_mode.capacity - screen->pending_mode.used < read_buf_sz) {
+                    if (screen->pending_mode.capacity >= READ_BUF_SZ) {
+                        // Too much pending data, drain it
+                        screen->pending_mode.activated_at = 0;
+                        state = START;
+                        break;
+                    }
+                    screen->pending_mode.capacity = MAX(screen->pending_mode.capacity * 2, screen->pending_mode.used + read_buf_sz);
+                    screen->pending_mode.buf = realloc(screen->pending_mode.buf, screen->pending_mode.capacity);
+                    if (!screen->pending_mode.buf) fatal("Out of memory");
+                }
+                read_buf_pos += _queue_pending_bytes(screen, read_buf + read_buf_pos, read_buf_sz - read_buf_pos, dump_callback);
+                state = START;
+            }   break;
+        }
+    } while(read_buf_pos < read_buf_sz || (!screen->pending_mode.activated_at && screen->pending_mode.used));
+
+}
+
 // }}}
 
 // Entry points {{{
@@ -1072,17 +1238,20 @@ FNAME(parse_bytes)(PyObject UNUSED *self, PyObject *args) {
 #else
     if (!PyArg_ParseTuple(args, "O!y*", &Screen_Type, &screen, &pybuf)) return NULL;
 #endif
-    _parse_bytes(screen, pybuf.buf, pybuf.len, dump_callback);
+    do_parse_bytes(screen, pybuf.buf, pybuf.len, monotonic(), dump_callback);
     Py_RETURN_NONE;
 }
 
 
 void
-FNAME(parse_worker)(Screen *screen, PyObject *dump_callback) {
+FNAME(parse_worker)(Screen *screen, PyObject *dump_callback, double now) {
 #ifdef DUMP_COMMANDS
-    Py_XDECREF(PyObject_CallFunction(dump_callback, "sy#", "bytes", screen->read_buf, screen->read_buf_sz)); PyErr_Clear();
+    if (screen->read_buf_sz) {
+        Py_XDECREF(PyObject_CallFunction(dump_callback, "sy#", "bytes", screen->read_buf, screen->read_buf_sz)); PyErr_Clear();
+    }
 #endif
-    _parse_bytes(screen, screen->read_buf, screen->read_buf_sz, dump_callback);
+    do_parse_bytes(screen, screen->read_buf, screen->read_buf_sz, now, dump_callback);
-#undef FNAME
+    screen->read_buf_sz = 0;
 }
+#undef FNAME
 // }}}

kitty/screen.c

@@ -111,6 +111,7 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
         self->main_grman = grman_alloc();
         self->alt_grman = grman_alloc();
         self->grman = self->main_grman;
+        self->pending_mode.wait_time = 2.0;
         self->main_tabstops = PyMem_Calloc(2 * self->columns, sizeof(bool));
         if (self->cursor == NULL || self->main_linebuf == NULL || self->alt_linebuf == NULL || self->main_tabstops == NULL || self->historybuf == NULL || self->main_grman == NULL || self->alt_grman == NULL || self->color_profile == NULL) {
             Py_CLEAR(self); return NULL;
@@ -272,6 +273,7 @@ dealloc(Screen* self) {
     PyMem_Free(self->overlay_line.cpu_cells);
     PyMem_Free(self->overlay_line.gpu_cells);
     PyMem_Free(self->main_tabstops);
+    free(self->pending_mode.buf);
     Py_TYPE(self)->tp_free((PyObject*)self);
 } // }}}
 
@@ -659,6 +661,7 @@ void
 screen_set_8bit_controls(Screen *self, bool yes) {
     self->modes.eight_bit_controls = yes;
 }
+
 // }}}
 
 // Cursor {{{
@@ -1603,6 +1606,14 @@ deactivate_overlay_line(Screen *self) {
 #define WRAP2(name, defval1, defval2) static PyObject* name(Screen *self, PyObject *args) { unsigned int a=defval1, b=defval2; if(!PyArg_ParseTuple(args, "|II", &a, &b)) return NULL; screen_##name(self, a, b); Py_RETURN_NONE; }
 #define WRAP2B(name) static PyObject* name(Screen *self, PyObject *args) { unsigned int a, b; int p; if(!PyArg_ParseTuple(args, "IIp", &a, &b, &p)) return NULL; screen_##name(self, a, b, (bool)p); Py_RETURN_NONE; }
 
+static PyObject*
+set_pending_timeout(Screen *self, PyObject *val) {
+    if (!PyFloat_Check(val)) { PyErr_SetString(PyExc_TypeError, "timeout must be a float"); return NULL; }
+    PyObject *ans = PyFloat_FromDouble(self->pending_mode.wait_time);
+    self->pending_mode.wait_time = PyFloat_AS_DOUBLE(val);
+    return ans;
+}
+
 static PyObject*
 as_text(Screen *self, PyObject *args) {
     as_text_generic(args, self, visual_line_, self->lines, self->columns);
@@ -2095,6 +2106,7 @@ static PyMethodDef methods[] = {
     MND(cursor_down1, METH_VARARGS)
     MND(cursor_forward, METH_VARARGS)
     {"index", (PyCFunction)xxx_index, METH_VARARGS, ""},
+    MND(set_pending_timeout, METH_O)
     MND(as_text, METH_VARARGS)
     MND(as_text_non_visual, METH_VARARGS)
     MND(tab, METH_NOARGS)

kitty/screen.h

@@ -97,11 +97,18 @@ typedef struct {
 
     CursorRenderInfo cursor_render_info;
 
+    struct {
+        size_t capacity, used;
+        uint8_t *buf;
+        double activated_at, wait_time;
+        int state;
+    } pending_mode;
+
 } Screen;
 
 
-void parse_worker(Screen *screen, PyObject *dump_callback);
+void parse_worker(Screen *screen, PyObject *dump_callback, double now);
-void parse_worker_dump(Screen *screen, PyObject *dump_callback);
+void parse_worker_dump(Screen *screen, PyObject *dump_callback, double now);
 void screen_align(Screen*);
 void screen_restore_cursor(Screen *);
 void screen_save_cursor(Screen *);

kitty_tests/parser.py

@@ -2,6 +2,7 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
+import time
 from binascii import hexlify
 from functools import partial
 
@@ -226,6 +227,29 @@ class TestParser(BaseTest):
         pb('\033[0c', ('report_device_attributes', 0, 0))
         self.ae(c.wtcbuf, b'\x9b?62;c')
 
+    def test_pending(self):
+        s = self.create_screen()
+        timeout = 0.1
+        s.set_pending_timeout(timeout)
+        pb = partial(self.parse_bytes_dump, s)
+        pb('\033P=s\033\\', ('screen_start_pending_mode',))
+        pb('a')
+        self.ae(str(s.line(0)), '')
+        pb('\033P=\033\\', ('screen_stop_pending_mode',), ('draw', 'a'))
+        self.ae(str(s.line(0)), 'a')
+        pb('\033P=s\033\\', ('screen_start_pending_mode',))
+        pb('b')
+        self.ae(str(s.line(0)), 'a')
+        time.sleep(timeout)
+        pb('c', ('draw', 'bc'))
+        self.ae(str(s.line(0)), 'abc')
+        pb('\033P=s\033\\d', ('screen_start_pending_mode',))
+        pb('\033P=\033\\', ('screen_stop_pending_mode',), ('draw', 'd'))
+        pb('\033P=s\033\\e', ('screen_start_pending_mode',))
+        pb('\033P'), pb('=')
+        pb('\033\\', ('screen_stop_pending_mode',), ('draw', 'e'))
+        pb('\033P=s\033\\f\033P=s\033\\', ('screen_start_pending_mode',), ('screen_start_pending_mode',))
+
     def test_oth_codes(self):
         s = self.create_screen()
         pb = partial(self.parse_bytes_dump, s)