Get rid of kitty's special OSC 52 protocol

A better solution from an ecosystem perspective is to just work with the
original protocol. I have modified kitty's escape parser to special case
OSC 52 handling without changing its max escape code size.

Basically, it works by splitting up OSC 52 escape codes longer than the
max size into a series of partial OSC 52 escape codes. These get
dispatched to the UI layer where it accumulates them upto the 8MB limit
and then sends to clipboard when the partial sequence ends.

See https://github.com/ranger/ranger/issues/1861

docs/changelog.rst

@@ -16,6 +16,12 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
 - When opening hyperlinks, allow defining open actions for directories
   (:pull:`3836`)
 
+- When using the OSC 52 escape code to copy to clipboard allow large
+  copies (up to 8MB) without needing a kitty specific chunking protocol.
+  Note that if you used the chunking protocol in the past, it will no longer
+  work and you should switch to using the unmodified protocol which has the
+  advantage of working with all terminal emulators.
+
 - Fix a bug in the implementation of the synchronized updates escape code that
   could cause incorrect parsing if either the pending buffer capacity or the
   pending timeout were exceeded (:iss:`3779`)

docs/clipboard.rst

@@ -1,29 +0,0 @@
-Pasting to clipboard
-=======================
-
-|kitty| implements the OSC 52 escape code protocol to get/set the clipboard
-contents (controlled via the :opt:`clipboard_control` setting). There is one
-difference in kitty's implementation compared to some other terminal emulators.
-|kitty| allows sending arbitrary amounts of text to the clipboard. It does so
-by modifying the protocol slightly. Successive OSC 52 escape codes to set the
-clipboard will concatenate, so::
-
-    <ESC>]52;c;<payload1><ESC>\
-    <ESC>]52;c;<payload2><ESC>\
-
-will result in the clipboard having the contents ``payload1 + payload2``. To
-send a new string to the clipboard send an OSC 52 sequence with an invalid payload
-first, for example::
-
-    <ESC>]52;c;!<ESC>\
-
-Here ``!`` is not valid base64 encoded text, so it clears the clipboard.
-Further, since it is invalid, it should be ignored by terminal emulators
-that do not support this extension, thereby making it safe to use, simply
-always send it before starting a new OSC 52 paste, even if you aren't chunking
-up large pastes, that way kitty won't concatenate your paste, and it will have
-no ill-effects in other terminal emulators.
-
-In case you're using software that can't be easily adapted to this
-protocol extension, it can be disabled by specifying ``no-append`` to the
-:opt:`clipboard_control` setting.

docs/protocol-extensions.rst

@@ -28,6 +28,5 @@ please do so by opening issues in the `GitHub
    keyboard-protocol
    desktop-notifications
    unscroll
-   clipboard
    color-stack
    deccara

kittens/tui/operations.py

@@ -352,24 +352,17 @@ def set_default_colors(
 
 @cmd
 def write_to_clipboard(data: Union[str, bytes], use_primary: bool = False) -> str:
-    if isinstance(data, str):
-        data = data.encode('utf-8')
     from base64 import standard_b64encode
     fmt = 'p' if use_primary else 'c'
-
+    if isinstance(data, str):
-    def esc(chunk: str) -> str:
+        data = data.encode('utf-8')
-        return '\x1b]52;{};{}\x07'.format(fmt, chunk)
+    payload = standard_b64encode(data).decode('ascii')
-
+    return f'\x1b]52;{fmt};{payload}\a'
-    ans = esc('!')  # clear clipboard buffer
-    for chunk in (data[i:i+512] for i in range(0, len(data), 512)):
-        s = standard_b64encode(chunk).decode('ascii')
-        ans += esc(s)
-    return ans
 
 
 @cmd
 def request_from_clipboard(use_primary: bool = False) -> str:
-    return '\x1b]52;{};?\x07'.format('p' if use_primary else 'c')
+    return '\x1b]52;{};?\a'.format('p' if use_primary else 'c')
 
 
 # Boilerplate to make operations available via Handler.cmd  {{{

kitty/client.py

@@ -208,10 +208,27 @@ def write_osc(code: int, string: str = '') -> None:
 
 
 set_dynamic_color = set_color_table_color = process_cwd_notification = write_osc
+clipboard_control_pending: str = ''
+
+
+def clipboard_control(payload: str) -> None:
+    global clipboard_control_pending
+    code, data = payload.split(';', 1)
+    if code == '-52':
+        if clipboard_control_pending:
+            clipboard_control_pending += data.lstrip(';')
+        else:
+            clipboard_control_pending = payload
+        return
+    if clipboard_control_pending:
+        clipboard_control_pending += data.lstrip(';')
+        payload = clipboard_control_pending
+        clipboard_control_pending = ''
+    write(OSC + payload + '\x07')
 
 
 def replay(raw: str) -> None:
-    specials = {'draw', 'set_title', 'set_icon', 'set_dynamic_color', 'set_color_table_color', 'process_cwd_notification'}
+    specials = {'draw', 'set_title', 'set_icon', 'set_dynamic_color', 'set_color_table_color', 'process_cwd_notification', 'clipboard_control'}
     for line in raw.splitlines():
         if line.strip() and not line.startswith('#'):
             cmd, rest = line.partition(' ')[::2]

kitty/parser.c

@@ -16,6 +16,7 @@
 #include <time.h>
 
 extern PyTypeObject Screen_Type;
+#define EXTENDED_OSC_SENTINEL 0x1bu
 
 // utils {{{
 static const uint64_t pow_10_array[] = {
@@ -125,7 +126,7 @@ _report_params(PyObject *dump_callback, const char *name, int *params, unsigned
     Py_XDECREF(PyObject_CallFunction(dump_callback, "sO", #name, string)); PyErr_Clear();
 
 #define REPORT_OSC2(name, code, string) \
-    Py_XDECREF(PyObject_CallFunction(dump_callback, "sIO", #name, code, string)); PyErr_Clear();
+    Py_XDECREF(PyObject_CallFunction(dump_callback, "siO", #name, code, string)); PyErr_Clear();
 
 #define REPORT_HYPERLINK(id, url) \
     Py_XDECREF(PyObject_CallFunction(dump_callback, "szz", "set_active_hyperlink", id, url)); PyErr_Clear();
@@ -354,7 +355,18 @@ dispatch_hyperlink(Screen *screen, size_t pos, size_t size, PyObject DUMP_UNUSED
     free(data);
 }
 
-static inline void
+static void
+continue_osc_52(Screen *screen) {
+    screen->parser_buf[0] = '5'; screen->parser_buf[1] = '2'; screen->parser_buf[2] = ';';
+    screen->parser_buf[3] = ';'; screen->parser_buf_pos = 4;
+}
+
+static bool
+is_extended_osc(const Screen *screen) {
+    return screen->parser_buf_pos > 2 && screen->parser_buf[0] == EXTENDED_OSC_SENTINEL && screen->parser_buf[1] == 1 && screen->parser_buf[2] == ';';
+}
+
+static void
 dispatch_osc(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
 #define DISPATCH_OSC_WITH_CODE(name) REPORT_OSC2(name, code, string); name(screen, code, string);
 #define DISPATCH_OSC(name) REPORT_OSC(name, string); name(screen, string);
@@ -372,6 +384,12 @@ dispatch_osc(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
     if (i > 0) {
         code = utoi(screen->parser_buf, i);
         if (i < limit && screen->parser_buf[i] == ';') i++;
+    } else {
+        if (is_extended_osc(screen)) {
+            // partial OSC 52
+            i = 3;
+            code = -52;
+        }
     }
     switch(code) {
         case 0:
@@ -418,8 +436,10 @@ dispatch_osc(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
             DISPATCH_OSC_WITH_CODE(set_dynamic_color);
             END_DISPATCH
         case 52:
+        case -52:
             START_DISPATCH
-            DISPATCH_OSC(clipboard_control);
+            DISPATCH_OSC_WITH_CODE(clipboard_control);
+            if (code == -52) continue_osc_52(screen);
             END_DISPATCH
         case 30001:
             REPORT_COMMAND(screen_push_dynamic_colors);
@@ -1088,8 +1108,16 @@ dispatch_pm(Screen *screen, PyObject DUMP_UNUSED *dump_callback) {
 
 // Parse loop {{{
 
-static inline bool
+static bool
-accumulate_osc(Screen *screen, uint32_t ch, PyObject DUMP_UNUSED *dump_callback) {
+handle_extended_osc_code(Screen *screen) {
+    // Handle extra long OSC 52 codes
+    if (screen->parser_buf[0] != '5' || screen->parser_buf[1] != '2' || screen->parser_buf[2] != ';') return false;
+    screen->parser_buf[0] = EXTENDED_OSC_SENTINEL; screen->parser_buf[1] = 1;
+    return true;
+}
+
+static bool
+accumulate_osc(Screen *screen, uint32_t ch, PyObject DUMP_UNUSED *dump_callback, bool *extended_osc_code) {
     switch(ch) {
         case ST:
             return true;
@@ -1106,7 +1134,8 @@ accumulate_osc(Screen *screen, uint32_t ch, PyObject DUMP_UNUSED *dump_callback)
             /* fallthrough */
         default:
             if (screen->parser_buf_pos >= PARSER_BUF_SZ - 1) {
-                REPORT_ERROR("OSC sequence too long, truncating.");
+                if (handle_extended_osc_code(screen)) *extended_osc_code = true;
+                else REPORT_ERROR("OSC sequence too long, truncating.");
                 return true;
             }
             screen->parser_buf[screen->parser_buf_pos++] = ch;
@@ -1250,7 +1279,15 @@ END_ALLOW_CASE_RANGE
             if (accumulate_csi(screen, codepoint, dump_callback)) { dispatch##_csi(screen, dump_callback); SET_STATE(0); watch_for_pending; } \
             break; \
         case OSC: \
-            if (accumulate_osc(screen, codepoint, dump_callback)) { dispatch##_osc(screen, dump_callback); SET_STATE(0); } \
+            { \
+                bool extended_osc_code = false; \
+                if (accumulate_osc(screen, codepoint, dump_callback, &extended_osc_code)) {  \
+                    dispatch##_osc(screen, dump_callback); \
+                    if (extended_osc_code) { \
+                        if (accumulate_osc(screen, codepoint, dump_callback, &extended_osc_code)) { dispatch##_osc(screen, dump_callback); SET_STATE(0); } \
+                    } else { SET_STATE(0); } \
+                } \
+            } \
             break; \
         case APC: \
             if (accumulate_oth(screen, codepoint, dump_callback)) { dispatch##_apc(screen, dump_callback); SET_STATE(0); } \
@@ -1371,7 +1408,13 @@ pending_escape_code(Screen *screen, char_type start_ch, char_type end_ch) {
 
 static void pending_pm(Screen *screen, PyObject *dump_callback UNUSED) { pending_escape_code(screen, PM, ST); }
 static void pending_apc(Screen *screen, PyObject *dump_callback UNUSED) { pending_escape_code(screen, APC, ST); }
-static void pending_osc(Screen *screen, PyObject *dump_callback UNUSED) { pending_escape_code(screen, OSC, ST); }
+
+static void
+pending_osc(Screen *screen, PyObject *dump_callback UNUSED) {
+    const bool extended = is_extended_osc(screen);
+    pending_escape_code(screen, OSC, ST);
+    if (extended) continue_osc_52(screen);
+}
 
 
 static void

kitty/screen.c

@@ -1708,8 +1708,8 @@ set_dynamic_color(Screen *self, unsigned int code, PyObject *color) {
 }
 
 void
-clipboard_control(Screen *self, PyObject *data) {
+clipboard_control(Screen *self, int code, PyObject *data) {
-    CALLBACK("clipboard_control", "O", data);
+    CALLBACK("clipboard_control", "OO", data, code == -52 ? Py_True: Py_False);
 }
 
 void

kitty/screen.h

@@ -193,7 +193,7 @@ void set_title(Screen *self, PyObject*);
 void desktop_notify(Screen *self, unsigned int, PyObject*);
 void set_icon(Screen *self, PyObject*);
 void set_dynamic_color(Screen *self, unsigned int code, PyObject*);
-void clipboard_control(Screen *self, PyObject*);
+void clipboard_control(Screen *self, int code, PyObject*);
 void set_color_table_color(Screen *self, unsigned int code, PyObject*);
 void process_cwd_notification(Screen *self, unsigned int code, PyObject*);
 uint32_t* translation_table(uint32_t which);

kitty/window.py

@@ -12,8 +12,8 @@ from functools import partial
 from gettext import gettext as _
 from itertools import chain
 from typing import (
-    Any, Callable, Deque, Dict, Iterable, List, Optional, Pattern, Sequence,
+    Any, Callable, Deque, Dict, Iterable, List, NamedTuple, Optional, Pattern,
-    Tuple, Union
+    Sequence, Tuple, Union
 )
 
 from .child import ProcessDesc
@@ -72,6 +72,11 @@ class PipeData(TypedDict):
     text: str
 
 
+class ClipboardPending(NamedTuple):
+    where: str
+    data: str
+
+
 class DynamicColor(IntEnum):
     default_fg, default_bg, cursor_color, highlight_fg, highlight_bg = range(1, 6)
 
@@ -337,7 +342,7 @@ class Window:
         self.tab_id = tab.id
         self.os_window_id = tab.os_window_id
         self.tabref: Callable[[], Optional[TabType]] = weakref.ref(tab)
-        self.clipboard_control_buffers = {'p': '', 'c': ''}
+        self.clipboard_pending: Optional[ClipboardPending] = None
         self.destroyed = False
         self.geometry: WindowGeometry = WindowGeometry(0, 0, 0, 0, 0, 0)
         self.needs_layout = True
@@ -777,9 +782,24 @@ class Window:
     def send_cmd_response(self, response: Any) -> None:
         self.screen.send_escape_code_to_child(DCS, '@kitty-cmd' + json.dumps(response))
 
-    def clipboard_control(self, data: str) -> None:
+    def clipboard_control(self, data: str, is_partial: bool = False) -> None:
         where, text = data.partition(';')[::2]
+        if is_partial:
+            if self.clipboard_pending is None:
+                self.clipboard_pending = ClipboardPending(where, text)
+            else:
+                self.clipboard_pending = self.clipboard_pending._replace(data=self.clipboard_pending[1] + text)
+                if len(self.clipboard_pending.data) > 8 * 1024 * 1024:
+                    log_error('Discarding part of too large OSC 52 paste request')
+                    self.clipboard_pending = self.clipboard_pending._replace(data='')
+            return
+
         if not where:
+            if self.clipboard_pending is not None:
+                text = self.clipboard_pending.data + text
+                where = self.clipboard_pending.where
+                self.clipboard_pending = None
+            else:
                 where = 's0'
         cc = get_options().clipboard_control
         if text == '?':
@@ -802,22 +822,13 @@ class Window:
             except Exception:
                 text = ''
 
-            def write(key: str, func: Callable[[str], None]) -> None:
-                if text:
-                    if ('no-append' in cc or
-                            len(self.clipboard_control_buffers[key]) > 1024*1024):
-                        self.clipboard_control_buffers[key] = ''
-                    self.clipboard_control_buffers[key] += text
-                else:
-                    self.clipboard_control_buffers[key] = ''
-                func(self.clipboard_control_buffers[key])
-
             if 's' in where or 'c' in where:
                 if 'write-clipboard' in cc:
-                    write('c', set_clipboard_string)
+                    set_clipboard_string(text)
             if 'p' in where:
                 if 'write-primary' in cc:
-                    write('p', set_primary_selection)
+                    set_primary_selection(text)
+        self.clipboard_pending = None
 
     def manipulate_title_stack(self, pop: bool, title: str, icon: Any) -> None:
         if title:

kitty_tests/__init__.py

@@ -50,12 +50,20 @@ class Callbacks:
     def open_url(self, url: str, hyperlink_id: int) -> None:
         self.open_urls.append((url, hyperlink_id))
 
+    def clipboard_control(self, data: str, is_partial: bool = False) -> None:
+        self.cc_buf.append((data, is_partial))
+
     def clear(self) -> None:
         self.wtcbuf = b''
         self.iconbuf = self.titlebuf = self.colorbuf = self.ctbuf = ''
         self.iutf8 = True
         self.notifications = []
         self.open_urls = []
+        self.cc_buf = []
+        self.bell_count = 0
+
+    def on_bell(self) -> None:
+        self.bell_count += 1
 
     def on_activity_since_last_focus(self) -> None:
         pass

kitty_tests/parser.py

@@ -239,6 +239,11 @@ class TestParser(BaseTest):
         pb('\033]8;moo\x07', ('Ignoring malformed OSC 8 code',))
         pb('\033]8;id=xyz;\x07', ('set_active_hyperlink', 'xyz', None))
         pb('\033]8;moo:x=z:id=xyz:id=abc;http://yay;.com\x07', ('set_active_hyperlink', 'xyz', 'http://yay;.com'))
+        c.clear()
+        payload = '1' * 1024
+        pb(f'\033]52;p;{payload}\x07', ('clipboard_control', 52, f'p;{payload}'))
+        c.clear()
+        pb('\033]52;p;xyz\x07', ('clipboard_control', 52, 'p;xyz'))
 
     def test_desktop_notify(self):
         reset_registry()

kitty_tests/screen.py

@@ -794,6 +794,29 @@ class TestScreen(BaseTest):
         self.ae(str(s.linebuf), '0\n5\n6\n7\n\n')
         self.ae(str(s.historybuf), '')
 
+    def test_osc_52(self):
+        s = self.create_screen()
+        c = s.callbacks
+
+        def send(what: str):
+            return parse_bytes(s, f'\033]52;p;{what}\a'.encode('ascii'))
+
+        def t(q, use_pending_mode, *expected):
+            c.clear()
+            if use_pending_mode:
+                parse_bytes(s, b'\033[?2026h')
+            send(q)
+            if use_pending_mode:
+                self.ae(c.cc_buf, [])
+                parse_bytes(s, b'\033[?2026l')
+            self.ae(c.cc_buf, list(expected))
+
+        for use_pending_mode in (False, True):
+            t('XYZ', use_pending_mode, ('p;XYZ', False))
+            t('a' * 8192, use_pending_mode, ('p;' + 'a' * (8192 - 6), True), (';' + 'a' * 6, False))
+            t('', use_pending_mode, ('p;', False))
+            t('!', use_pending_mode, ('p;!', False))
+
     def test_key_encoding_flags_stack(self):
         s = self.create_screen()
         c = s.callbacks