Graphics protocol: Support for frame composition

Fixes #3809

docs/changelog.rst

@@ -20,6 +20,9 @@ To update |kitty|, :doc:`follow the instructions <binary>`.
   could cause incorrect parsing if either the pending buffer capacity or the
   pending timeout were exceeded (:iss:`3779`)
 
+- Graphics protocol: Add support for composing rectangles from one animation
+  frame onto another (:iss:`3809`)
+
 - diff kitten: Remove limit on max line length of 4096 characters (:iss:`3806`)
 
 - Fix turning off cursor blink via escape codes not working (:iss:`3808`)

docs/graphics-protocol.rst

@@ -632,6 +632,45 @@ static background.
 In particular, the first frame or *root frame* is created with the base image
 data and has no gap, so its gap must be set using this control code.
 
+Composing animation frames
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 0.21.3
+   Support for frame composition
+
+Clients can *compose* animation frames, this means that they can compose pixels
+in rectangular regions from one frame onto another frame. This allows for fast
+and low band-width modification of frames.
+
+To achieve this use the ``a=c`` key. The source frame is specified with
+``r=frame number`` and the destination frame as ``c=frame number``. The size of
+the rectangle is specified as ``w=width,h=height`` pixels. If unspecified, the
+full image width and height are used. The offset of the rectangle from the
+top-left corner for the source frame is specified by the ``x,y`` keys and the
+destination frame by the ``X,Y`` keys. The composition operation is specified
+by the ``C`` key with the default being to alpha blend the source rectangle
+onto the destination rectangle. With ``C=1`` it will be a simple replacement
+of pixels. For example::
+
+    <ESC>_Gi=1,r=7,c=9,w=23,h=27,X=4,Y=8,x=1,y=3<ESC>\
+
+Will compose a ``23x27`` rectangle located at ``(4, 8)`` in the ``7th frame``
+onto the rectangle located at ``(1, 3)`` in the ``9th frame``. These will be
+in the image with ``id=1``.
+
+If the frames or the image are not found the terminal emulator must
+respond with `ENOENT`. If the rectangles go out of bounds of the image
+the terminal must respond with `EINVAL`. If the source and destination frames are
+the same and the rectangles overlap, the terminal must respond with `EINVAL`.
+
+
+.. note::
+   In kitty, doing a composition will cause a frame to be *fully rendered*
+   potentially increasing its storage requirements, when the frame was previously
+   stored as a set of operations on other frames. If this happens and there
+   is not enough storage space, kitty will respond with ENOSPC.
+
+
 Image persistence and storage quotas
 -----------------------------------------
 
@@ -654,10 +693,10 @@ take, and the default value they take when missing. All integers are 32-bit.
 Key      Value                 Default    Description
 =======  ====================  =========  =================
 ``a``    Single character.     ``t``      The overall action this graphics command is performing.
-         ``(a, d, f,``                    ``t`` - transmit data, ``T`` - transmit data and display image,
+         ``(a, c, d, f, `                 ``t`` - transmit data, ``T`` - transmit data and display image,
          ``p, q, t, T)``                  ``q`` - query terminal, ``p`` - put (display) previous transmitted image,
                                           ``d`` - delete image, ``f`` - transmit data for animation frames,
-                                          ``a`` - control animation
+                                          ``a`` - control animation, ``c`` - compose animation frames
 
 ``q``    ``0, 1, 2``           ``0``      Suppress responses from the terminal to this graphics command.
 
@@ -711,6 +750,20 @@ Key      Value                 Default    Description
 ``Y``    Positive integer      ``0``      The background color for pixels not
                                           specified in the frame data. Must be in 32-bit RGBA format
 
+**Keys for animation frame composition**
+-----------------------------------------------------------
+
+``c``    Positive integer      ``0``      The 1-based frame number of the frame whose image data serves as the base data
+``r``    Positive integer      ``0``      The 1-based frame number of the frame that is being edited.
+``x``    Positive integer      ``0``      The left edge (in pixels) of the destination rectangle
+``y``    Positive integer      ``0``      The top edge (in pixels) of the destination rectangle
+``w``    Positive integer      ``0``      The width (in pixels) of the source and destination rectangles. By default, the entire width is used
+``h``    Positive integer      ``0``      The height (in pixels) of the source and destination rectangles. By default, the entire height is used
+``X``    Positive integer      ``0``      The left edge (in pixels) of the source rectangle
+``Y``    Positive integer      ``0``      The top edge (in pixels) of the source rectangle
+``C``    Positive integer      ``0``      The composition mode for blending
+                                          pixels. Default is full alpha blending. ``1`` means a simple overwrite.
+
 
 **Keys for animation control**
 -----------------------------------------------------------

gen-apc-parsers.py

@@ -250,7 +250,7 @@ def write_header(text: str, path: str) -> None:
 def graphics_parser() -> None:
     flag = frozenset
     keymap: KeymapType = {
-        'a': ('action', flag('tTqpdfa')),
+        'a': ('action', flag('tTqpdfac')),
         'd': ('delete_action', flag('aAiIcCfFnNpPqQxXyYzZ')),
         't': ('transmission_type', flag('dfts')),
         'o': ('compressed', flag('z')),

kitty/graphics.c

@@ -24,6 +24,7 @@ PyTypeObject GraphicsManager_Type;
 
 #define DEFAULT_STORAGE_LIMIT 320u * (1024u * 1024u)
 #define REPORT_ERROR(...) { log_error(__VA_ARGS__); }
+#define FREE_CFD_AFTER_FUNCTION __attribute__((cleanup(cfd_free)))
 
 // caching {{{
 #define CACHE_KEY_BUFFER_SIZE 32
@@ -825,7 +826,7 @@ frame_for_id(Image *img, const uint32_t frame_id) {
     return NULL;
 }
 
-static inline Frame*
+static Frame*
 frame_for_number(Image *img, const uint32_t frame_number) {
     switch(frame_number) {
         case 1:
@@ -872,10 +873,53 @@ alpha_blend(uint8_t *dest_px, const uint8_t *src_px) {
 typedef struct {
     bool needs_blending;
     uint32_t over_px_sz, under_px_sz;
-    uint32_t over_width, over_height, under_width, under_height, over_offset_x, over_offset_y;
+    uint32_t over_width, over_height, under_width, under_height, over_offset_x, over_offset_y, under_offset_x, under_offset_y;
+    uint32_t stride;
 } ComposeData;
 
-static inline void
+#define COPY_RGB under_px[0] = over_px[0]; under_px[1] = over_px[1]; under_px[2] = over_px[2];
+#define COPY_PIXELS \
+    if (d.needs_blending) { \
+        if (d.under_px_sz == 3) { \
+            ROW_ITER PIX_ITER blend_on_opaque(under_px, over_px); }} \
+        } else { \
+            ROW_ITER PIX_ITER alpha_blend(under_px, over_px); }} \
+        } \
+    } else { \
+        if (d.under_px_sz == 4) { \
+            if (d.over_px_sz == 4) { \
+                ROW_ITER PIX_ITER COPY_RGB under_px[3] = over_px[3]; }} \
+            } else { \
+                ROW_ITER PIX_ITER COPY_RGB under_px[3] = 255; }} \
+            } \
+        } else { \
+            ROW_ITER PIX_ITER COPY_RGB }} \
+        } \
+    } \
+
+
+static void
+compose_rectangles(const ComposeData d, uint8_t *under_data, const uint8_t *over_data) {
+    // compose two equal sized, non-overlapping rectangles at different offsets
+    // does not do bounds checking on the data arrays
+    const bool can_copy_rows = !d.needs_blending && d.over_px_sz == d.under_px_sz;
+    const unsigned min_width = MIN(d.under_width, d.over_width);
+#define ROW_ITER for (unsigned y = 0; y < d.under_height && y < d.over_height; y++) { \
+        uint8_t *under_row = under_data + (y + d.under_offset_y) * d.under_px_sz * d.stride + (d.under_offset_x * d.under_px_sz); \
+        const uint8_t *over_row = over_data + (y + d.over_offset_y) * d.over_px_sz * d.stride + (d.over_offset_x * d.over_px_sz);
+    if (can_copy_rows) {
+        ROW_ITER memcpy(under_row, over_row, (size_t)d.over_px_sz * min_width);}
+        return;
+    }
+#define PIX_ITER for (unsigned x = 0; x < min_width; x++) { \
+        uint8_t *under_px = under_row + (d.under_px_sz * x); \
+        const uint8_t *over_px = over_row + (d.over_px_sz * x);
+    COPY_PIXELS
+#undef PIX_ITER
+#undef ROW_ITER
+}
+
+static void
 compose(const ComposeData d, uint8_t *under_data, const uint8_t *over_data) {
     const bool can_copy_rows = !d.needs_blending && d.over_px_sz == d.under_px_sz;
     unsigned min_row_sz = d.over_offset_x < d.under_width ? d.under_width - d.over_offset_x : 0;
@@ -890,24 +934,7 @@ compose(const ComposeData d, uint8_t *under_data, const uint8_t *over_data) {
 #define PIX_ITER for (unsigned x = 0; x < min_row_sz; x++) { \
         uint8_t *under_px = under_row + (d.under_px_sz * x); \
         const uint8_t *over_px = over_row + (d.over_px_sz * x);
-#define COPY_RGB under_px[0] = over_px[0]; under_px[1] = over_px[1]; under_px[2] = over_px[2];
+    COPY_PIXELS
-    if (d.needs_blending) {
-        if (d.under_px_sz == 3) {
-            ROW_ITER PIX_ITER blend_on_opaque(under_px, over_px); }}
-        } else {
-            ROW_ITER PIX_ITER alpha_blend(under_px, over_px); }}
-        }
-    } else {
-        if (d.under_px_sz == 4) {
-            if (d.over_px_sz == 4) {
-                ROW_ITER PIX_ITER COPY_RGB under_px[3] = over_px[3]; }}
-            } else {
-                ROW_ITER PIX_ITER COPY_RGB under_px[3] = 255; }}
-            }
-        } else {
-            ROW_ITER PIX_ITER COPY_RGB }}
-        }
-    }
 #undef COPY_RGB
 #undef PIX_ITER
 #undef ROW_ITER
@@ -982,7 +1009,7 @@ get_coalesced_frame_data_impl(GraphicsManager *self, Image *img, const Frame *f,
     return base_data;
 }
 
-static inline CoalescedFrameData
+static CoalescedFrameData
 get_coalesced_frame_data(GraphicsManager *self, Image *img, const Frame *f) {
     return get_coalesced_frame_data_impl(self, img, f, 0);
 }
@@ -1278,6 +1305,73 @@ scan_active_animations(GraphicsManager *self, const monotonic_t now, monotonic_t
 }
 // }}}
 
+// {{{ composition a=c
+static void
+cfd_free(void *p) { free(((CoalescedFrameData*)p)->buf); }
+
+static void
+handle_compose_command(GraphicsManager *self, bool *is_dirty, const GraphicsCommand *g, Image *img) {
+    Frame *src_frame = frame_for_number(img, g->_frame_number);
+    if (!src_frame) {
+        set_command_failed_response("ENOENT", "No source frame number %u exists in image id: %u\n", g->_frame_number, img->client_id);
+        return;
+    }
+    Frame *dest_frame = frame_for_number(img, g->_other_frame_number);
+    if (!dest_frame) {
+        set_command_failed_response("ENOENT", "No destination frame number %u exists in image id: %u\n", g->_other_frame_number, img->client_id);
+        return;
+    }
+    const unsigned int width = g->width ? g->width : img->width;
+    const unsigned int height = g->height ? g->height : img->height;
+    const unsigned int dest_x = g->x_offset, dest_y = g->y_offset, src_x = g->cell_x_offset, src_y = g->cell_y_offset;
+    if (dest_x + width > img->width || dest_y + height > img->height) {
+        set_command_failed_response("EINVAL", "The destination rectangle is out of bounds");
+        return;
+    }
+    if (src_x + width > img->width || src_y + height > img->height) {
+        set_command_failed_response("EINVAL", "The source rectangle is out of bounds");
+        return;
+    }
+    if (src_frame == dest_frame) {
+        bool x_overlaps = MAX(src_x, dest_x) < (MIN(src_x, dest_x) + width);
+        bool y_overlaps = MAX(src_y, dest_y) < (MIN(src_y, dest_y) + height);
+        if (x_overlaps && y_overlaps) {
+            set_command_failed_response("EINVAL", "The source and destination rectangles overlap and the src and destination frames are the same");
+            return;
+        }
+    }
+
+    FREE_CFD_AFTER_FUNCTION CoalescedFrameData src_data = get_coalesced_frame_data(self, img, src_frame);
+    if (!src_data.buf) {
+        set_command_failed_response("EINVAL", "Failed to get data for src frame: %u", g->_frame_number - 1);
+        return;
+    }
+    FREE_CFD_AFTER_FUNCTION CoalescedFrameData dest_data = get_coalesced_frame_data(self, img, dest_frame);
+    if (!dest_data.buf) {
+        set_command_failed_response("EINVAL", "Failed to get data for destination frame: %u", g->_other_frame_number - 1);
+        return;
+    }
+    ComposeData d = {
+        .over_px_sz = src_data.is_opaque ? 3 : 4, .under_px_sz = dest_data.is_opaque ? 3: 4,
+        .needs_blending = !g->cursor_movement && !src_data.is_opaque,
+        .over_offset_x = src_x, .over_offset_y = src_y,
+        .under_offset_x = dest_x, .under_offset_y = dest_y,
+        .over_width = width, .over_height = height, .under_width = width, .under_height = height,
+        .stride = img->width
+    };
+    compose_rectangles(d, dest_data.buf, src_data.buf);
+    const ImageAndFrame key = { .image_id = img->internal_id, .frame_id = dest_frame->id };
+    if (!add_to_cache(self, key, dest_data.buf, (dest_data.is_opaque ? 3 : 4) * img->width * img->height)) {
+        if (PyErr_Occurred()) PyErr_Print();
+        set_command_failed_response("ENOSPC", "Failed to store image data in disk cache");
+    }
+    // frame is now a fully coalesced frame
+    dest_frame->x = 0; dest_frame->y = 0; dest_frame->width = img->width; dest_frame->height = img->height;
+    dest_frame->base_frame_id = 0; dest_frame->bgcolor = 0;
+    *is_dirty = (g->_other_frame_number - 1) == img->current_frame_index;
+}
+// }}}
+
 // Image lifetime/scrolling {{{
 
 static inline void
@@ -1564,6 +1658,20 @@ grman_handle_command(GraphicsManager *self, const GraphicsCommand *g, const uint
         case 'd':
             handle_delete_command(self, g, c, is_dirty, cell);
             break;
+        case 'c':
+            if (!g->id && !g->image_number) {
+                REPORT_ERROR("Compose frame data command without image id or number");
+                break;
+            }
+            Image *img = g->id ? img_by_client_id(self, g->id) : img_by_client_number(self, g->image_number);
+            if (!img) {
+                set_command_failed_response("ENOENT", "Animation command refers to non-existent image with id: %u and number: %u", g->id, g->image_number);
+                ret = finish_command_response(g, false);
+            } else {
+                handle_compose_command(self, is_dirty, g, img);
+                ret = finish_command_response(g, true);
+            }
+            break;
         default:
             REPORT_ERROR("Unknown graphics command action: %c", g->action);
             break;

kitty/parse-graphics-command.h

@@ -148,9 +148,9 @@ static inline void parse_graphics_code(Screen *screen,
 
       case action: {
         g.action = screen->parser_buf[pos++] & 0xff;
-        if (g.action != 't' && g.action != 'a' && g.action != 'T' &&
+        if (g.action != 'q' && g.action != 'p' && g.action != 't' &&
-            g.action != 'f' && g.action != 'd' && g.action != 'p' &&
+            g.action != 'd' && g.action != 'c' && g.action != 'a' &&
-            g.action != 'q') {
+            g.action != 'T' && g.action != 'f') {
           REPORT_ERROR("Malformed GraphicsCommand control block, unknown flag "
                        "value for action: 0x%x",
                        g.action);
@@ -160,16 +160,16 @@ static inline void parse_graphics_code(Screen *screen,
 
       case delete_action: {
         g.delete_action = screen->parser_buf[pos++] & 0xff;
-        if (g.delete_action != 'Q' && g.delete_action != 'N' &&
+        if (g.delete_action != 'q' && g.delete_action != 'Q' &&
-            g.delete_action != 'z' && g.delete_action != 'p' &&
+            g.delete_action != 'c' && g.delete_action != 'C' &&
-            g.delete_action != 'X' && g.delete_action != 'a' &&
+            g.delete_action != 'N' && g.delete_action != 'i' &&
-            g.delete_action != 'y' && g.delete_action != 'f' &&
+            g.delete_action != 'A' && g.delete_action != 'y' &&
-            g.delete_action != 'Z' && g.delete_action != 'F' &&
+            g.delete_action != 'a' && g.delete_action != 'I' &&
-            g.delete_action != 'Y' && g.delete_action != 'n' &&
+            g.delete_action != 'F' && g.delete_action != 'p' &&
-            g.delete_action != 'C' && g.delete_action != 'q' &&
+            g.delete_action != 'z' && g.delete_action != 'x' &&
-            g.delete_action != 'c' && g.delete_action != 'i' &&
+            g.delete_action != 'n' && g.delete_action != 'X' &&
-            g.delete_action != 'A' && g.delete_action != 'P' &&
+            g.delete_action != 'Y' && g.delete_action != 'P' &&
-            g.delete_action != 'I' && g.delete_action != 'x') {
+            g.delete_action != 'Z' && g.delete_action != 'f') {
           REPORT_ERROR("Malformed GraphicsCommand control block, unknown flag "
                        "value for delete_action: 0x%x",
                        g.delete_action);

kitty_tests/graphics.py

@@ -758,6 +758,30 @@ class TestGraphics(BaseTest):
         self.ae(g.image_count, 0)
         self.assertEqual(g.disk_cache.total_size, 0)
 
+        # test frame composition
+        self.assertEqual(li(a='t').code, 'OK')
+        self.assertEqual(g.disk_cache.total_size, 36)
+        t(payload='2' * 36)
+        t(payload='3' * 36, frame_number=3)
+        img = g.image_for_client_id(1)
+        self.assertEqual(img['extra_frames'], (
+            {'gap': 40, 'id': 2, 'data': b'2' * 36},
+            {'gap': 40, 'id': 3, 'data': b'3' * 36},
+        ))
+        self.assertEqual(li(a='c', i=11).code, 'ENOENT')
+        self.assertEqual(li(a='c', i=1, r=1, c=2).code, 'OK')
+        img = g.image_for_client_id(1)
+        self.assertEqual(img['extra_frames'], (
+            {'gap': 40, 'id': 2, 'data': b'abcdefghijkl'*3},
+            {'gap': 40, 'id': 3, 'data': b'3' * 36},
+        ))
+        self.assertEqual(li(a='c', i=1, r=2, c=3, w=1, h=2, x=1, y=1).code, 'OK')
+        img = g.image_for_client_id(1)
+        self.assertEqual(img['extra_frames'], (
+            {'gap': 40, 'id': 2, 'data': b'abcdefghijkl'*3},
+            {'gap': 40, 'id': 3, 'data': b'3' * 12 + (b'333abc' + b'3' * 6) * 2},
+        ))
+
     def test_graphics_quota_enforcement(self):
         s = self.create_screen()
         g = s.grman