Implement the index operations

Scrolling of line buffer one line at a time, optionally saving the
scrolled of line in a history buffer.

kitty/data-types.h

@@ -99,6 +99,8 @@ typedef struct {
     decoration_type *decoration_fg;
     combining_type *combining_chars;
     index_type xnum, ynum;
+    uint8_t continued;
+    uint8_t needs_free;
 } Line;
 
 

kitty/line-buf.c

@@ -7,6 +7,7 @@
 
 #include "data-types.h"
 #include <structmember.h>
+extern PyTypeObject Line_Type;
 
 static inline void
 clear_chars_to_space(LineBuf* linebuf, index_type y) {
@@ -54,9 +55,8 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
         self->line = alloc_line();
         if (self->buf == NULL || self->line_map == NULL || self->continued_map == NULL || self->line == NULL) {
             PyErr_NoMemory();
-            PyMem_Free(self->buf); PyMem_Free(self->line_map); PyMem_Free(self->continued_map); Py_XDECREF(self->line);
+            PyMem_Free(self->buf); PyMem_Free(self->line_map); PyMem_Free(self->continued_map); Py_CLEAR(self->line);
-            Py_DECREF(self);
+            Py_CLEAR(self);
-            self = NULL;
         } else {
             self->chars = (char_type*)self->buf;
             self->colors = (color_type*)(self->chars + self->block_size);
@@ -76,7 +76,7 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
 static void
 dealloc(LineBuf* self) {
     PyMem_Free(self->buf); PyMem_Free(self->line_map); PyMem_Free(self->continued_map);
-    Py_XDECREF(self->line);
+    Py_CLEAR(self->line);
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
@@ -94,9 +94,9 @@ line(LineBuf *self, PyObject *y) {
         PyErr_SetString(PyExc_ValueError, "Line number too large");
         return NULL;
     }
-    self->line->ynum = self->line_map[idx];
+    self->line->ynum = idx;
     self->line->xnum = self->xnum;
-    INIT_LINE(self, self->line, self->line->ynum);
+    INIT_LINE(self, self->line, self->line_map[idx]);
     Py_INCREF(self->line);
     return (PyObject*)self->line;
 }
@@ -125,6 +125,113 @@ set_continued(LineBuf *self, PyObject *args) {
     Py_RETURN_NONE;
 }
 
+static inline int
+allocate_line_storage(Line *line, uint8_t initialize) {
+    if (initialize) {
+        line->chars = PyMem_Calloc(line->xnum, sizeof(char_type));
+        line->colors = PyMem_Calloc(line->xnum, sizeof(color_type));
+        line->decoration_fg = PyMem_Calloc(line->xnum, sizeof(decoration_type));
+        line->combining_chars = PyMem_Calloc(line->xnum, sizeof(combining_type));
+        for (index_type i = 0; i < line->xnum; i++) line->chars[i] = (1 << ATTRS_SHIFT) | 32;
+    } else {
+        line->chars = PyMem_Malloc(line->xnum * sizeof(char_type));
+        line->colors = PyMem_Malloc(line->xnum * sizeof(color_type));
+        line->decoration_fg = PyMem_Malloc(line->xnum * sizeof(decoration_type));
+        line->combining_chars = PyMem_Malloc(line->xnum * sizeof(combining_type));
+    }
+    if (line->chars == NULL || line->colors == NULL || line->decoration_fg == NULL || line->combining_chars == NULL) {
+        PyMem_Free(line->chars); line->chars = NULL;
+        PyMem_Free(line->colors); line->colors = NULL;
+        PyMem_Free(line->decoration_fg); line->decoration_fg = NULL;
+        PyMem_Free(line->combining_chars); line->combining_chars = NULL;
+        PyErr_NoMemory();
+        return 0;
+    }
+    line->needs_free = 1;
+    return 1;
+}
+
+static PyObject*
+create_line_copy(LineBuf *self, PyObject *ynum) {
+#define create_line_copy_doc "Create a new Line object that is a copy of the line at ynum. Note that this line has its own copy of the data and does not refer to the data in the LineBuf."
+    Line src, *line;
+    index_type y = (index_type)PyLong_AsUnsignedLong(ynum);
+    if (y >= self->ynum) { PyErr_SetString(PyExc_ValueError, "Out of bounds"); return NULL; }
+    line = alloc_line();
+    if (line == NULL) return PyErr_NoMemory();
+    src.xnum = self->xnum; line->xnum = self->xnum;
+    if (!allocate_line_storage(line, 0)) { Py_DECREF(line); return NULL; }
+    line->ynum = y;
+    line->continued = self->continued_map[y];
+    INIT_LINE(self, &src, self->line_map[y]);
+    COPY_LINE(&src, line);
+    return (PyObject*)line;
+}
+
+static PyObject*
+copy_line_to(LineBuf *self, PyObject *args) {
+#define copy_line_to_doc "Copy the line at ynum to the provided line object."
+    unsigned int y;
+    Line src, *dest;
+    if (!PyArg_ParseTuple(args, "IO!", &y, &Line_Type, &dest)) return NULL;
+    src.xnum = self->xnum; dest->xnum = self->xnum;
+    dest->ynum = y;
+    dest->continued = self->continued_map[y];
+    INIT_LINE(self, &src, self->line_map[y]);
+    COPY_LINE(&src, dest);
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+clear_line(LineBuf *self, PyObject *val) {
+#define clear_line_doc "clear_line(y) -> Clear the specified line"
+    index_type y = (index_type)PyLong_AsUnsignedLong(val);
+    Line l;
+    if (y >= self->ynum) { PyErr_SetString(PyExc_ValueError, "Out of bounds"); return NULL; }
+    INIT_LINE(self, &l, self->line_map[y]);
+    for (index_type i = 0; i < self->xnum; i++) l.chars[i] = (1 << ATTRS_SHIFT) | 32;
+    memset(l.colors, 0, self->xnum * sizeof(color_type));
+    memset(l.decoration_fg, 0, self->xnum * sizeof(decoration_type));
+    memset(l.combining_chars, 0, self->xnum * sizeof(combining_type));
+    self->continued_map[y] = 0;
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+index(LineBuf *self, PyObject *args) {
+#define index_doc "index(top, bottom) -> Scroll all lines in the range [top, bottom] by one upwards. After scrolling, bottom will be top."
+    unsigned int top, bottom;
+    if (!PyArg_ParseTuple(args, "II", &top, &bottom)) return NULL;
+    if (top >= self->ynum - 1 || bottom >= self->ynum || bottom <= top) { PyErr_SetString(PyExc_ValueError, "Out of bounds"); return NULL; }
+    index_type old_top = self->line_map[top];
+    uint8_t old_cont = self->continued_map[top];
+    for (index_type i = top; i < bottom; i++) {
+        self->line_map[i] = self->line_map[i + 1];
+        self->continued_map[i] = self->continued_map[i + 1];
+    }
+    self->line_map[bottom] = old_top;
+    self->continued_map[bottom] = old_cont;
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+reverse_index(LineBuf *self, PyObject *args) {
+#define reverse_index_doc "reverse_index(top, bottom) -> Scroll all lines in the range [top, bottom] by one down. After scrolling, top will be bottom."
+    unsigned int top, bottom;
+    if (!PyArg_ParseTuple(args, "II", &top, &bottom)) return NULL;
+    if (top >= self->ynum - 1 || bottom >= self->ynum || bottom <= top) { PyErr_SetString(PyExc_ValueError, "Out of bounds"); return NULL; }
+    index_type old_bottom = self->line_map[bottom];
+    uint8_t old_cont = self->continued_map[bottom];
+    for (index_type i = bottom; i > top; i--) {
+        self->line_map[i] = self->line_map[i - 1];
+        self->continued_map[i] = self->continued_map[i - 1];
+    }
+    self->line_map[top] = old_bottom;
+    self->continued_map[top] = old_cont;
+    Py_RETURN_NONE;
+}
+
+
 static PyObject*
 is_continued(LineBuf *self, PyObject *val) {
 #define is_continued_doc "is_continued(y) -> Whether the line y is continued or not"
@@ -142,10 +249,15 @@ copy_old(LineBuf *self, PyObject *y);
 
 static PyMethodDef methods[] = {
     METHOD(line, METH_O)
+    METHOD(clear_line, METH_O)
     METHOD(copy_old, METH_O)
+    METHOD(copy_line_to, METH_VARARGS)
+    METHOD(create_line_copy, METH_O)
     METHOD(clear, METH_NOARGS)
     METHOD(set_attribute, METH_VARARGS)
     METHOD(set_continued, METH_VARARGS)
+    METHOD(index, METH_VARARGS)
+    METHOD(reverse_index, METH_VARARGS)
     METHOD(is_continued, METH_O)
     {NULL, NULL, 0, NULL}  /* Sentinel */
 };

kitty/line.c

@@ -15,7 +15,13 @@ new(PyTypeObject UNUSED *type, PyObject UNUSED *args, PyObject UNUSED *kwds) {
 }
 
 static void
-dealloc(LineBuf* self) {
+dealloc(Line* self) {
+    if (self->needs_free) {
+        PyMem_Free(self->chars);
+        PyMem_Free(self->colors);
+        PyMem_Free(self->decoration_fg);
+        PyMem_Free(self->combining_chars);
+    }
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
@@ -56,9 +62,8 @@ as_unicode(Line* self) {
         return NULL;
     }
     for(index_type i = 0; i < self->xnum; i++) {
-        char_type ch = self->chars[i] & CHAR_MASK;
+        buf[n++] = self->chars[i] & CHAR_MASK;
         char_type cc = self->combining_chars[i];
-        buf[n++] = ch & CHAR_MASK;
         Py_UCS4 cc1 = cc & CC_MASK, cc2;
         if (cc1) {
             buf[n++] = cc1;
@@ -137,14 +142,14 @@ cursor_from(Line* self, PyObject *args) {
         PyErr_SetString(PyExc_ValueError, "Out of bounds x");
         return NULL;
     }
-    ans = PyObject_New(Cursor, &Cursor_Type);
+    ans = alloc_cursor();
     if (ans == NULL) { PyErr_NoMemory(); return NULL; }
     xo = PyLong_FromUnsignedLong(x); yo = PyLong_FromUnsignedLong(y);
     if (xo == NULL || yo == NULL) {
-        Py_DECREF(ans); Py_XDECREF(xo); Py_XDECREF(yo);
+        Py_CLEAR(ans); Py_CLEAR(xo); Py_CLEAR(yo);
         PyErr_NoMemory(); return NULL;
     }
-    Py_XDECREF(ans->x); Py_XDECREF(ans->y);
+    Py_CLEAR(ans->x); Py_CLEAR(ans->y);
     ans->x = xo; ans->y = yo;
     char_type attrs = self->chars[x] >> ATTRS_SHIFT;
     ATTRS_TO_CURSOR(attrs, ans);
@@ -297,7 +302,7 @@ static PyMethodDef methods[] = {
     {NULL}  /* Sentinel */
 };
 
-static PyTypeObject Line_Type = {
+PyTypeObject Line_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     .tp_name = "fast_data_types.Line",
     .tp_basicsize = sizeof(Line),
@@ -312,7 +317,9 @@ static PyTypeObject Line_Type = {
 };
 
 Line *alloc_line() {
-    return (Line*)PyType_GenericAlloc(&Line_Type, 0);
+    Line *ans = (Line*)PyType_GenericAlloc(&Line_Type, 0);
+    ans->needs_free = 0;
+    return ans;
 }
 
 RICHCMP(Line)

kitty/screen.py

@@ -66,7 +66,7 @@ class Screen:
         self.columns = columns
         self.lines = lines
         sz = max(1000, opts.scrollback_lines)
-        self.tophistorybuf = LineBuf(sz, self.columns)
+        self.tophistorybuf = deque(maxlen=sz)
         self.main_linebuf, self.alt_linebuf = LineBuf(self.lines, self.columns), LineBuf(self.lines, self.columns)
         self.linebuf = self.main_linebuf
         self.reset(notify=False)
@@ -75,8 +75,8 @@ class Screen:
         sz = max(1000, opts.scrollback_lines)
         if sz != self.tophistorybuf.maxlen:
             previous = self.tophistorybuf
-            self.tophistorybuf = LineBuf(opts.scrollback_lines, self.columns)
+            self.tophistorybuf = deque(maxlen=sz)
-            self.tophistorybuf.copy_old(previous)
+            self.tophistorybuf.extend(previous)
 
     def line(self, i):
         return self.linebuf.line(i)
@@ -381,14 +381,14 @@ class Screen:
             if mo.DECAWM in self.mode:
                 self.carriage_return()
                 self.linefeed()
-                self.linebuf[self.cursor.y].continued = True
+                self.linebuf.set_continued(self.cursor.y, True)
             else:
                 self.cursor.x = self.columns - char_width
 
         do_insert = mo.IRM in self.mode
 
         cx = self.cursor.x
-        line = self.linebuf[self.cursor.y]
+        line = self.linebuf.line(self.cursor.y)
         if char_width > 0:
             if do_insert:
                 line.right_shift(self.cursor.x, char_width)
@@ -406,7 +406,7 @@ class Screen:
                 line.add_combining_char(cx - 1, char)
                 self.update_cell_range(self.cursor.y, cx - 1, cx - 1)
             elif self.cursor.y > 0:
-                lline = self.linebuf[self.cursor.y - 1]
+                lline = self.linebuf.line(self.cursor.y - 1)
                 lline.add_combining_char(self.columns - 1, char)
                 self.update_cell_range(self.cursor.y - 1, self.columns - 1, self.columns - 1)
 
@@ -458,11 +458,15 @@ class Screen:
         top, bottom = self.margins
 
         if self.cursor.y == bottom:
-            l = self.linebuf.pop(top)
+            self.linebuf.index(top, bottom)
             if self.linebuf is self.main_linebuf:
-                self.tophistorybuf.append(l)
+                if len(self.tophistorybuf) >= self.tophistorybuf.maxlen:
+                    l = self.tophistorybuf.popleft()
+                    self.linebuf.copy_line_to(bottom, l)
+                else:
+                    self.tophistorybuf.append(self.linebuf.create_line_copy(bottom))
+                self.linebuf.clear_line(bottom)
                 self.line_added_to_history()
-            self.linebuf.insert(bottom, Line(self.columns))
             if bottom - top >= self.lines - 1:
                 self.update_screen()
             else:
@@ -477,12 +481,12 @@ class Screen:
         top, bottom = self.margins
 
         if self.cursor.y == top:
-            self.linebuf.pop(bottom)
+            self.linebuf.reverse_index(top, bottom)
-            self.linebuf.insert(top, Line(self.columns))
             if bottom - top >= self.lines - 1:
                 self.update_screen()
             else:
                 self.update_line_range(top, bottom)
+            self.linebuf.clear_line(top)
         else:
             self.cursor_up()
 

kitty_tests/datatypes.py

@@ -22,7 +22,8 @@ class TestDataTypes(BaseTest):
         old.set_attribute(REVERSE, False)
         for y in range(old.ynum):
             for x in range(old.xnum):
-                c = old.line(y).cursor_from(x)
+                l = old.line(y)
+                c = l.cursor_from(x)
                 self.assertFalse(c.reverse)
                 self.assertTrue(c.bold)
         self.assertFalse(old.is_continued(0))
@@ -30,17 +31,43 @@ class TestDataTypes(BaseTest):
         self.assertTrue(old.is_continued(0))
         self.assertFalse(old.is_continued(1))
 
+        lb = filled_line_buf(5, 5, filled_cursor())
+        lb2 = LineBuf(5, 5)
+        lb2.copy_old(lb)
+        lb.index(0, 4)
+        for i in range(0, 4):
+            self.ae(lb.line(i), lb2.line(i+1))
+        self.ae(lb.line(4), lb2.line(0))
+        lb = filled_line_buf(5, 5, filled_cursor())
+        lb.index(1, 3)
+        self.ae(lb.line(0), lb2.line(0))
+        self.ae(lb.line(1), lb2.line(2))
+        self.ae(lb.line(2), lb2.line(3))
+        self.ae(lb.line(3), lb2.line(1))
+        self.ae(lb.line(4), lb2.line(4))
+        self.ae(lb.create_line_copy(1), lb2.line(2))
+        l = lb.create_line_copy(2)
+        lb.copy_line_to(1, l)
+        self.ae(l, lb2.line(2))
+        lb.clear_line(0)
+        self.ae(lb.line(0), LineBuf(1, lb.xnum).create_line_copy(0))
+        lb = filled_line_buf(5, 5, filled_cursor())
+        lb.reverse_index(0, 4)
+        self.ae(lb.line(0), lb2.line(4))
+        for i in range(1, 5):
+            self.ae(lb.line(i), lb2.line(i-1))
+
     def test_line(self):
         lb = LineBuf(2, 3)
-        for y in range(2):
+        for y in range(lb.ynum):
             line = lb.line(y)
-            self.ae(str(line), ' ' * 3)
+            self.ae(str(line), ' ' * lb.xnum)
-            for x in range(3):
+            for x in range(lb.xnum):
                 self.ae(line[x], ' ')
         with self.assertRaises(ValueError):
-            lb.line(5)
+            lb.line(lb.ynum)
         with self.assertRaises(ValueError):
-            lb.line(0)[5]
+            lb.line(0)[lb.xnum]
         l = lb.line(0)
         l.add_combining_char(0, '1')
         self.ae(l[0], ' 1')