Merge branch 'pagerhist' of https://github.com/martinetd/kitty

docs/changelog.rst

@@ -3,6 +3,14 @@ Changelog
 
 |kitty| is a feature full, cross-platform, *fast*, GPU based terminal emulator.
 
+0.13.0 [future]
+------------------------------
+
+- Add an option :opt:`scrollback_pager_history_size` to tell kitty to store
+  extended scrollback to use when viewing the scrollback buffer in a pager
+  (:iss:`970`)
+
+
 0.12.3 [2018-09-29]
 ------------------------------
 

docs/index.rst

@@ -370,6 +370,9 @@ arbitrary, command running in a new window, tab or overlay, for example::
 Would open the scrollback buffer in a new window when you press the :kbd:`F1`
 key. See :sc:`show_scrollback` for details.
 
+If you wish to store very large amounts of scrollback to view using the piping or
+:sc:`show_scrollback` features, you can use the :opt:`scrollback_pager_history_size`
+option.
 
 Frequently Asked Questions
 ---------------------------------

kitty/config_data.py

@@ -323,6 +323,11 @@ def scrollback_lines(x):
     return x
 
 
+def scrollback_pager_history_size(x):
+    ans = int(max(0, float(x)) * 1024 * 1024)
+    return min(ans, 4096 * 1024 * 1024 - 1)
+
+
 o('scrollback_lines', 2000, option_type=scrollback_lines, long_text=_('''
 Number of lines of history to keep in memory for scrolling back. Memory is allocated
 on demand. Negative numbers are (effectively) infinite scrollback. Note that using
@@ -336,6 +341,14 @@ use can handle ANSI escape sequences for colors and text formatting.
 INPUT_LINE_NUMBER in the command line above will be replaced by an integer
 representing which line should be at the top of the screen.'''))
 
+o('scrollback_pager_history_size', 0, option_type=scrollback_pager_history_size, long_text=_('''
+Separate scrollback history size, used only for browsing the scrollback buffer (in MB).
+This separate buffer is not available for interactive scrolling but will be
+piped to the pager program when viewing scrollback buffer in a separate window.
+The current implementation stores one character in 4 bytes, so approximatively
+2500 lines per megabyte at 100 chars per line. A value of zero or less disables
+this feature. The maximum allowed size is 4GB.'''))
+
 o('wheel_scroll_multiplier', 5.0, long_text=_('''
 Modify the amount scrolled by the mouse wheel. Note this is only used for low
 precision scrolling devices, not for high precision scrolling on platforms such

kitty/cursor.c

@@ -230,31 +230,45 @@ color_as_sgr(char *buf, size_t sz, unsigned long val, unsigned simple_code, unsi
 static inline const char*
 decoration_as_sgr(uint8_t decoration) {
     switch(decoration) {
-        case 1: return "4";
+        case 1: return "4;";
-        case 2: return "4:2";
+        case 2: return "4:2;";
-        case 3: return "4:3";
+        case 3: return "4:3;";
-        default: return "24";
+        default: return "24;";
     }
 }
 
 const char*
-cursor_as_sgr(Cursor *self, Cursor *prev) {
+cell_as_sgr(GPUCell *cell, GPUCell *prev) {
     static char buf[128];
 #define SZ sizeof(buf) - (p - buf) - 2
-#define P(fmt, ...) { p += snprintf(p, SZ, fmt ";", __VA_ARGS__); }
+#define P(s) { size_t len = strlen(s); if (SZ > len) { memcpy(p, s, len); p += len; } }
     char *p = buf;
-    bool intensity_differs = self->bold != prev->bold || self->dim != prev->dim;
+#define CMP(attr) (attr(cell) != attr(prev))
+#define BOLD(cell) (cell->attrs & (1 << BOLD_SHIFT))
+#define DIM(cell) (cell->attrs & (1 << DIM_SHIFT))
+#define ITALIC(cell) (cell->attrs & (1 << ITALIC_SHIFT))
+#define REVERSE(cell) (cell->attrs & (1 << REVERSE_SHIFT))
+#define STRIKETHROUGH(cell) (cell->attrs & (1 << STRIKE_SHIFT))
+#define DECORATION(cell) (cell->attrs & (DECORATION_MASK << DECORATION_SHIFT))
+    bool intensity_differs = CMP(BOLD) || CMP(DIM);
     if (intensity_differs) {
-        if (!self->bold && !self->dim) { P("%d", 22); }
+        if (!BOLD(cell) && !DIM(cell)) { P("22;"); }
-        else { if (self->bold) P("%d", 1); if (self->dim) P("%d", 2); }
+        else { if (BOLD(cell)) P("1;"); if (DIM(cell)) P("2;"); }
     }
-    if (self->italic != prev->italic) P("%d", self->italic ? 3 : 23);
+    if (CMP(ITALIC)) P(ITALIC(cell) ? "3;" : "23;");
-    if (self->reverse != prev->reverse) P("%d", self->reverse ? 7 : 27);
+    if (CMP(REVERSE)) P(REVERSE(cell) ? "7;" : "27;");
-    if (self->strikethrough != prev->strikethrough) P("%d", self->strikethrough ? 9 : 29);
+    if (CMP(STRIKETHROUGH)) P(STRIKETHROUGH(cell) ? "9;" : "29;");
-    if (self->decoration != prev->decoration) P("%s", decoration_as_sgr(self->decoration));
+    if (cell->fg != prev->fg) p += color_as_sgr(p, SZ, cell->fg, 30, 90, 38);
-    if (self->fg != prev->fg) p += color_as_sgr(p, SZ, self->fg, 30, 90, 38);
+    if (cell->bg != prev->bg) p += color_as_sgr(p, SZ, cell->bg, 40, 100, 48);
-    if (self->bg != prev->bg) p += color_as_sgr(p, SZ, self->bg, 40, 100, 48);
+    if (cell->decoration_fg != prev->decoration_fg) p += color_as_sgr(p, SZ, cell->decoration_fg, 0, 0, DECORATION_FG_CODE);
-    if (self->decoration_fg != prev->decoration_fg) p += color_as_sgr(p, SZ, self->decoration_fg, 0, 0, DECORATION_FG_CODE);
+    if (CMP(DECORATION)) P(decoration_as_sgr((cell->attrs >> DECORATION_SHIFT) & DECORATION_MASK));
+#undef CMP
+#undef BOLD
+#undef DIM
+#undef ITALIC
+#undef REVERSE
+#undef STRIKETHROUGH
+#undef DECORATION
 #undef P
 #undef SZ
     if (p > buf) *(p - 1) = 0;  // remove trailing semi-colon

kitty/data-types.h

@@ -173,11 +173,20 @@ typedef struct {
     line_attrs_type *line_attrs;
 } HistoryBufSegment;
 
+typedef struct {
+    index_type bufsize, maxsz;
+    Py_UCS4 *buffer;
+    index_type start, end;
+    index_type bufend;
+    bool rewrap_needed;
+} PagerHistoryBuf;
+
 typedef struct {
     PyObject_HEAD
 
     index_type xnum, ynum, num_segments;
     HistoryBufSegment *segments;
+    PagerHistoryBuf *pagerhist;
     Line *line;
     index_type start_of_data, count;
 } HistoryBuf;
@@ -254,7 +263,7 @@ const char* base64_decode(const uint32_t *src, size_t src_sz, uint8_t *dest, siz
 Line* alloc_line();
 Cursor* alloc_cursor();
 LineBuf* alloc_linebuf(unsigned int, unsigned int);
-HistoryBuf* alloc_historybuf(unsigned int, unsigned int);
+HistoryBuf* alloc_historybuf(unsigned int, unsigned int, unsigned int);
 ColorProfile* alloc_color_profile();
 PyObject* create_256_color_table();
 PyObject* parse_bytes_dump(PyObject UNUSED *, PyObject *);
@@ -267,7 +276,7 @@ void cursor_copy_to(Cursor *src, Cursor *dest);
 void cursor_reset_display_attrs(Cursor*);
 void cursor_from_sgr(Cursor *self, unsigned int *params, unsigned int count);
 void apply_sgr_to_cells(GPUCell *first_cell, unsigned int cell_count, unsigned int *params, unsigned int count);
-const char* cursor_as_sgr(Cursor*, Cursor*);
+const char* cell_as_sgr(GPUCell *, GPUCell *);
 
 double monotonic();
 PyObject* cm_thread_write(PyObject *self, PyObject *args);

kitty/history.c

@@ -54,12 +54,34 @@ attrptr(HistoryBuf *self, index_type y) {
     seg_ptr(line_attrs, 1);
 }
 
+static inline PagerHistoryBuf*
+alloc_pagerhist(unsigned int pagerhist_sz) {
+    PagerHistoryBuf *ph;
+    if (!pagerhist_sz) return NULL;
+    ph = PyMem_Calloc(1, sizeof(PagerHistoryBuf));
+    ph->maxsz = pagerhist_sz / sizeof(Py_UCS4);
+    ph->bufsize = 1024*1024 / sizeof(Py_UCS4);
+    ph->buffer = PyMem_RawMalloc(1024*1024);
+    if (!ph->buffer) { PyMem_Free(ph); return NULL; }
+    return ph;
+}
+
+static inline bool
+pagerhist_extend(PagerHistoryBuf *ph) {
+    if (ph->bufsize >= ph->maxsz) return false;
+    void *newbuf = PyMem_Realloc(ph->buffer, ph->bufsize * sizeof(Py_UCS4) + 1024*1024);
+    if (!newbuf) return false;
+    ph->buffer = newbuf;
+    ph->bufsize += 1024*1024 / sizeof(Py_UCS4);
+    return true;
+}
+
 static PyObject *
 new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
     HistoryBuf *self;
-    unsigned int xnum = 1, ynum = 1;
+    unsigned int xnum = 1, ynum = 1, pagerhist_sz = 0;
 
-    if (!PyArg_ParseTuple(args, "II", &ynum, &xnum)) return NULL;
+    if (!PyArg_ParseTuple(args, "II|I", &ynum, &xnum, &pagerhist_sz)) return NULL;
 
     if (xnum == 0 || ynum == 0) {
         PyErr_SetString(PyExc_ValueError, "Cannot create an empty history buffer");
@@ -74,6 +96,7 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
         add_segment(self);
         self->line = alloc_line();
         self->line->xnum = xnum;
+        self->pagerhist = alloc_pagerhist(pagerhist_sz);
     }
 
     return (PyObject*)self;
@@ -88,6 +111,8 @@ dealloc(HistoryBuf* self) {
         PyMem_Free(self->segments[i].line_attrs);
     }
     PyMem_Free(self->segments);
+    if (self->pagerhist) PyMem_Free(self->pagerhist->buffer);
+    PyMem_Free(self->pagerhist);
     Py_TYPE(self)->tp_free((PyObject*)self);
 }
 
@@ -132,12 +157,32 @@ historybuf_clear(HistoryBuf *self) {
     self->start_of_data = 0;
 }
 
+static inline void
+pagerhist_push(HistoryBuf *self) {
+    PagerHistoryBuf *ph = self->pagerhist;
+    if (!ph) return;
+    Line l = {.xnum=self->xnum};
+    init_line(self, self->start_of_data, &l);
+    if (ph->start != ph->end && !l.continued) {
+        ph->buffer[ph->end++] = '\n';
+    }
+    if (ph->bufsize - ph->end < 1024 && !pagerhist_extend(ph)) {
+        ph->bufend = ph->end; ph->end = 0;
+    }
+    ph->end += line_as_ansi(&l, ph->buffer + ph->end, 1023);
+    ph->buffer[ph->end++] = '\r';
+    if (ph->bufend)
+        ph->start = ph->end + 1 < ph->bufend ? ph->end + 1 : 0;
+}
+
 static inline index_type
 historybuf_push(HistoryBuf *self) {
     index_type idx = (self->start_of_data + self->count) % self->ynum;
     init_line(self, idx, self->line);
-    if (self->count == self->ynum) self->start_of_data = (self->start_of_data + 1) % self->ynum;
+    if (self->count == self->ynum) {
-    else self->count++;
+        pagerhist_push(self);
+        self->start_of_data = (self->start_of_data + 1) % self->ynum;
+    } else self->count++;
     return idx;
 }
 
@@ -209,6 +254,94 @@ as_ansi(HistoryBuf *self, PyObject *callback) {
 static inline Line*
 get_line(HistoryBuf *self, index_type y, Line *l) { init_line(self, index_of(self, self->count - y - 1), l); return l; }
 
+static void
+pagerhist_rewrap(PagerHistoryBuf *ph, index_type xnum) {
+    Py_UCS4 *buf = PyMem_RawMalloc(ph->bufsize * sizeof(Py_UCS4));
+    if (!buf) return;
+    index_type s = ph->start, i = s, dest = 0, dest_bufend = 0, x = 0;
+    index_type end = ph->bufend ? ph->bufend : ph->end;
+    index_type lastmod_s = 0, lastmod_len = 0;
+#define CPY(_s, _l) { if (dest + (_l) >= ph->bufsize - 1) { dest_bufend = dest; dest = 0; } \
+              memcpy(buf + dest, ph->buffer + (_s), (_l) * sizeof(Py_UCS4)); dest += (_l); }
+    while (i < end) {
+        switch (ph->buffer[i]) {
+        case '\n':
+            CPY(s, i - s + 1);
+            x = 0; s = i + 1; lastmod_len = 0;
+            break;
+        case '\r':
+            CPY(s, i - s);
+            if (!memcmp(ph->buffer + lastmod_s, ph->buffer + i + 1, lastmod_len * sizeof(Py_UCS4)))
+                i += lastmod_len;
+            s = i + 1;
+            break;
+        case '\x1b':
+            if (ph->buffer[i+1] != '[') break;
+            lastmod_s = i;
+            while (ph->buffer[++i] != 'm');
+            lastmod_len = i - lastmod_s + 1;
+            break;
+        default:
+            x++; break;
+        }
+        i++;
+        if (ph->bufend && i == ph->bufend) {
+            if (s != i) CPY(s, i - s);
+            end = ph->end; i = s = 0;
+        }
+        if (x == xnum) {
+            CPY(s, i - s); buf[dest++] = '\r'; s = i; x = 0;
+            if (!(ph->buffer[i] == '\x1b' && ph->buffer[i+1] == '[') && lastmod_len)
+                CPY(lastmod_s, lastmod_len);
+        }
+    }
+#undef CPY
+    PyMem_Free(ph->buffer);
+    ph->buffer = buf;
+    ph->end = dest; ph->bufend = dest_bufend;
+    ph->start = dest_bufend ? dest + 1 : 0;
+    ph->rewrap_needed = false;
+}
+
+static PyObject *
+pagerhist_as_text(HistoryBuf *self, PyObject *callback) {
+    PagerHistoryBuf *ph = self->pagerhist;
+    PyObject *ret = NULL, *t = NULL;
+    Py_UCS4 *buf = NULL;
+    index_type num;
+    if (!ph) Py_RETURN_NONE;
+
+    if (ph->rewrap_needed) pagerhist_rewrap(ph, self->xnum);
+
+    for (int i = 0; i < 3; i++) {
+        switch(i) {
+        case 0:
+            num = (ph->bufend ? ph->bufend : ph->end) - ph->start;
+            buf = ph->buffer + ph->start;
+            t = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, num);
+            break;
+        case 1:
+            if (!ph->bufend) continue;
+            num = ph->end; buf = ph->buffer;
+            t = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, num);
+            break;
+        case 2:
+            { Line l = {.xnum=self->xnum}; get_line(self, 0, &l); if (l.continued) continue; }
+            t = PyUnicode_FromString("\n");
+            break;
+        }
+        if (t == NULL) goto end;
+        ret = PyObject_CallFunctionObjArgs(callback, t, NULL);
+        Py_DECREF(t);
+        if (ret == NULL) goto end;
+        Py_DECREF(ret);
+    }
+
+end:
+    if (PyErr_Occurred()) return NULL;
+    Py_RETURN_NONE;
+}
+
 static PyObject*
 as_text(HistoryBuf *self, PyObject *args) {
     Line l = {.xnum=self->xnum};
@@ -238,6 +371,7 @@ static PyObject* rewrap(HistoryBuf *self, PyObject *args);
 static PyMethodDef methods[] = {
     METHOD(line, METH_O)
     METHOD(as_ansi, METH_O)
+    METHODB(pagerhist_as_text, METH_O),
     METHODB(as_text, METH_VARARGS),
     METHOD(dirty_lines, METH_NOARGS)
     METHOD(push, METH_VARARGS)
@@ -267,8 +401,8 @@ PyTypeObject HistoryBuf_Type = {
 
 INIT_TYPE(HistoryBuf)
 
-HistoryBuf *alloc_historybuf(unsigned int lines, unsigned int columns) {
+HistoryBuf *alloc_historybuf(unsigned int lines, unsigned int columns, unsigned int pagerhist_sz) {
-    return (HistoryBuf*)new(&HistoryBuf_Type, Py_BuildValue("II", lines, columns), NULL);
+    return (HistoryBuf*)new(&HistoryBuf_Type, Py_BuildValue("III", lines, columns, pagerhist_sz), NULL);
 }
 // }}}
 
@@ -298,6 +432,8 @@ void historybuf_rewrap(HistoryBuf *self, HistoryBuf *other) {
         other->count = self->count; other->start_of_data = self->start_of_data;
         return;
     }
+    if (other->pagerhist && other->xnum != self->xnum && other->pagerhist->end != other->pagerhist->start)
+        other->pagerhist->rewrap_needed = true;
     other->count = 0; other->start_of_data = 0;
     index_type x = 0, y = 0;
     if (self->count > 0) {

kitty/line.c

@@ -235,35 +235,36 @@ line_as_ansi(Line *self, Py_UCS4 *buf, index_type buflen) {
     if (limit == 0) return 0;
     char_type previous_width = 0;
 
-    WRITE_SGR("0");
+    GPUCell blank_cell = { 0 };
-    Cursor c1 = {{0}}, c2 = {{0}};
+    GPUCell *cell, *prev_cell = &blank_cell;
-    Cursor *cursor = &c1, *prev_cursor = &c2, *t;
 
     for (index_type pos=0; pos < limit; pos++) {
-        char_type attrs = self->gpu_cells[pos].attrs, ch = self->cpu_cells[pos].ch;
+        char_type ch = self->cpu_cells[pos].ch;
         if (ch == 0) {
             if (previous_width == 2) { previous_width = 0; continue; }
             ch = ' ';
         }
-        ATTRS_TO_CURSOR(attrs, cursor);
-        cursor->fg = self->gpu_cells[pos].fg; cursor->bg = self->gpu_cells[pos].bg;
-        cursor->decoration_fg = self->gpu_cells[pos].decoration_fg & COL_MASK;
 
-        const char *sgr = cursor_as_sgr(cursor, prev_cursor);
+        cell = &self->gpu_cells[pos];
-        t = prev_cursor; prev_cursor = cursor; cursor = t;
+
+#define CMP_ATTRS (cell->attrs & ATTRS_MASK_WITHOUT_WIDTH) != (prev_cell->attrs & ATTRS_MASK_WITHOUT_WIDTH)
+#define CMP(x) cell->x != prev_cell->x
+        if (CMP_ATTRS || CMP(fg) || CMP(bg) || CMP(decoration_fg)) {
+            const char *sgr = cell_as_sgr(cell, prev_cell);
             if (*sgr) WRITE_SGR(sgr);
+        }
+        prev_cell = cell;
         WRITE_CH(ch);
         for(unsigned c = 0; c < arraysz(self->cpu_cells[pos].cc_idx) && self->cpu_cells[pos].cc_idx[c]; c++) {
             WRITE_CH(codepoint_for_mark(self->cpu_cells[pos].cc_idx[c]));
         }
-        previous_width = attrs & WIDTH_MASK;
+        previous_width = cell->attrs & WIDTH_MASK;
     }
     return i;
-#undef CHECK_BOOL
+#undef CMP_ATTRS
-#undef CHECK_COLOR
+#undef CMP
 #undef WRITE_SGR
 #undef WRITE_CH
-#undef WRITE_COLOR
 }
 
 static PyObject*

kitty/screen.c

@@ -107,7 +107,7 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
         self->color_profile = alloc_color_profile();
         self->main_linebuf = alloc_linebuf(lines, columns); self->alt_linebuf = alloc_linebuf(lines, columns);
         self->linebuf = self->main_linebuf;
-        self->historybuf = alloc_historybuf(MAX(scrollback, lines), columns);
+        self->historybuf = alloc_historybuf(MAX(scrollback, lines), columns, OPT(scrollback_pager_history_size));
         self->main_grman = grman_alloc();
         self->alt_grman = grman_alloc();
         self->grman = self->main_grman;
@@ -165,8 +165,9 @@ screen_dirty_sprite_positions(Screen *self) {
 
 static inline HistoryBuf*
 realloc_hb(HistoryBuf *old, unsigned int lines, unsigned int columns) {
-    HistoryBuf *ans = alloc_historybuf(lines, columns);
+    HistoryBuf *ans = alloc_historybuf(lines, columns, 0);
     if (ans == NULL) { PyErr_NoMemory(); return NULL; }
+    ans->pagerhist = old->pagerhist; old->pagerhist = NULL;
     historybuf_rewrap(old, ans);
     return ans;
 }
@@ -1405,7 +1406,6 @@ screen_request_capabilities(Screen *self, char c, PyObject *q) {
     static char buf[128];
     int shape = 0;
     const char *query;
-    Cursor blank_cursor = {{0}};
     switch(c) {
         case '+':
             CALLBACK("request_capabilities", "O", q);
@@ -1429,7 +1429,13 @@ screen_request_capabilities(Screen *self, char c, PyObject *q) {
                 shape = snprintf(buf, sizeof(buf), "1$r%d q", shape);
             } else if (strcmp("m", query) == 0) {
                 // SGR
-                shape = snprintf(buf, sizeof(buf), "1$r%sm", cursor_as_sgr(self->cursor, &blank_cursor));
+                GPUCell blank_cell = { 0 }, cursor_cell = {
+                    .attrs = CURSOR_TO_ATTRS(self->cursor, 1),
+                    .fg = self->cursor->fg & COL_MASK,
+                    .bg = self->cursor->bg & COL_MASK,
+                    .decoration_fg = self->cursor->decoration_fg & COL_MASK,
+                };
+                shape = snprintf(buf, sizeof(buf), "1$r%sm", cell_as_sgr(&cursor_cell, &blank_cell));
             } else if (strcmp("r", query) == 0) {
                 shape = snprintf(buf, sizeof(buf), "1$r%u;%ur", self->margin_top + 1, self->margin_bottom + 1);
             } else {

kitty/state.c

@@ -367,6 +367,7 @@ PYWRAP1(set_options) {
     S(dynamic_background_opacity, PyObject_IsTrue);
     S(inactive_text_alpha, PyFloat_AsDouble);
     S(window_padding_width, PyFloat_AsDouble);
+    S(scrollback_pager_history_size, PyLong_AsUnsignedLong);
     S(cursor_shape, PyLong_AsLong);
     S(url_style, PyLong_AsUnsignedLong);
     S(tab_bar_edge, PyLong_AsLong);

kitty/state.h

@@ -19,6 +19,7 @@ typedef struct {
     unsigned int open_url_modifiers;
     unsigned int rectangle_select_modifiers;
     unsigned int url_style;
+    unsigned int scrollback_pager_history_size;
     char_type select_by_word_characters[256]; size_t select_by_word_characters_count;
     color_type url_color, background, active_border_color, inactive_border_color, bell_border_color;
     double repaint_delay, input_delay;

kitty/window.py

@@ -436,7 +436,7 @@ class Window:
             self.screen.reset_callbacks()
         self.screen = None
 
-    def as_text(self, as_ansi=False, add_history=False, add_wrap_markers=False, alternate_screen=False):
+    def as_text(self, as_ansi=False, add_history=False, add_pager_history=False, add_wrap_markers=False, alternate_screen=False):
         lines = []
         add_history = add_history and not self.screen.is_using_alternate_linebuf() and not alternate_screen
         if alternate_screen:
@@ -446,6 +446,9 @@ class Window:
         f(lines.append, as_ansi, add_wrap_markers)
         if add_history:
             h = []
+            if add_pager_history:
+                # assert as_ansi and add_wrap_markers?
+                self.screen.historybuf.pagerhist_as_text(h.append)
             self.screen.historybuf.as_text(h.append, as_ansi, add_wrap_markers)
             lines = h + lines
         return ''.join(lines)
@@ -461,7 +464,7 @@ class Window:
     # actions {{{
 
     def show_scrollback(self):
-        data = self.as_text(as_ansi=True, add_history=True, add_wrap_markers=True)
+        data = self.as_text(as_ansi=True, add_history=True, add_pager_history=True, add_wrap_markers=True)
         data = data.replace('\r\n', '\n').replace('\r', '\n')
         lines = data.count('\n')
         input_line_number = (lines - (self.screen.lines - 1) - self.screen.scrolled_by)

kitty_tests/__init__.py

@@ -4,6 +4,8 @@
 
 from unittest import TestCase
 
+from kitty.config import Options, defaults, merge_configs
+from kitty.fast_data_types import set_options
 from kitty.fast_data_types import LineBuf, Cursor, Screen, HistoryBuf
 
 
@@ -72,7 +74,9 @@ class BaseTest(TestCase):
     ae = TestCase.assertEqual
     maxDiff = 2000
 
-    def create_screen(self, cols=5, lines=5, scrollback=5, cell_width=10, cell_height=20):
+    def create_screen(self, cols=5, lines=5, scrollback=5, cell_width=10, cell_height=20, options={}):
+        options = Options(merge_configs(defaults._asdict(), options))
+        set_options(options)
         c = Callbacks()
         return Screen(c, lines, cols, scrollback, cell_width, cell_height, 0, c)
 

kitty_tests/bench_scrollback.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+
+from time import clock_gettime, CLOCK_MONOTONIC, sleep
+from argparse import ArgumentParser
+from random import Random
+from string import printable
+import sys
+
+
+def main():
+    parser = ArgumentParser(description='Generate text')
+    parser.add_argument('--freq', default=10000, type=int, help='Number of lines to try to write per second. Will warn if not attained.')
+    parser.add_argument('--color', action='store_true', help='Add color to the output')
+    parser.add_argument('--unicode', action='store_true', help='Mix in some unicode characters')
+    parser.add_argument('--length', default=50, type=int, help='Average line length')
+    parser.add_argument('--lengthvar', default=0.3, type=float, help='Variation for line length, in ratio of line length')
+    parser.add_argument('--emptylines', default=0.1, type=float, help='ratio of empty lines')
+    parser.add_argument('--linesperwrite', default=1, type=int, help='number of lines to repeat/write at a time')
+    parser.add_argument('--patterns', default=1000, type=int, help='number of different pattern to alternate')
+    parser.add_argument('--seed', default=sys.argv[0], type=str, help='seed to get different output')
+    args = parser.parse_args()
+
+    rng = Random()
+    rng.seed(args.seed)
+
+    characters = [c for c in printable if c not in '\r\n\x0b\x0c']
+    if args.color:
+        characters += ['\x1b[91m', '\x1b[0m', '\x1b[1;32m', '\x1b[22m', '\x1b[35m']
+
+    if args.unicode:
+        characters += [u'日', u'本', u'💜', u'☃', u'🎩', u'🍀', u'、']
+
+    patterns = []
+    for _ in range(0, args.patterns):
+        s = ""
+        for _ in range(0, args.linesperwrite):
+            cnt = int(rng.gauss(args.length, args.length * args.lengthvar))
+            if cnt < 0 or rng.random() < args.emptylines:
+                cnt = 0
+            s += "".join(rng.choices(characters, k=cnt)) + '\n'
+        patterns += [s]
+
+    time_per_print = args.linesperwrite / args.freq
+    t1 = clock_gettime(CLOCK_MONOTONIC)
+    cnt = 0
+    while True:
+        sys.stdout.write(patterns[rng.randrange(0, args.patterns)])
+        sys.stdout.flush()
+        cnt += 1
+        t2 = clock_gettime(CLOCK_MONOTONIC)
+        if t2 - t1 < cnt * time_per_print:
+            sleep(cnt * time_per_print - (t2 - t1))
+            t1 = t2
+            cnt = 0
+        elif cnt >= 100:
+            print("Cannot print fast enough, printed %d lines in %f seconds instead of %f seconds target" %
+                  (cnt * args.linesperwrite, t2 - t1, cnt * time_per_print))
+            break
+        else:
+            cnt += 1
+
+
+if __name__ == '__main__':
+    main()

kitty_tests/datatypes.py

@@ -447,10 +447,10 @@ class TestDataTypes(BaseTest):
     def test_ansi_repr(self):
         lb = filled_line_buf()
         l0 = lb.line(0)
-        self.ae(l0.as_ansi(), '\x1b[0m00000')
+        self.ae(l0.as_ansi(), '00000')
         a = []
         lb.as_ansi(a.append)
-        self.ae(a, ['\x1b[0m' + str(lb.line(i)) + '\n' for i in range(lb.ynum)])
+        self.ae(a, [str(lb.line(i)) + '\n' for i in range(lb.ynum)])
         l2 = lb.line(0)
         c = C()
         c.bold = c.italic = c.reverse = c.strikethrough = c.dim = True
@@ -458,15 +458,15 @@ class TestDataTypes(BaseTest):
         c.bg = (1 << 24) | (2 << 16) | (3 << 8) | 2
         c.decoration_fg = (5 << 8) | 1
         l2.set_text('1', 0, 1, c)
-        self.ae(l2.as_ansi(), '\x1b[0m\x1b[1;2;3;7;9;34;48:2:1:2:3;58:5:5m' '1'
+        self.ae(l2.as_ansi(), '\x1b[1;2;3;7;9;34;48:2:1:2:3;58:5:5m' '1'
                 '\x1b[22;23;27;29;39;49;59m' '0000')
         lb = filled_line_buf()
         for i in range(lb.ynum):
             lb.set_continued(i, True)
         a = []
         lb.as_ansi(a.append)
-        self.ae(a, ['\x1b[0m' + str(lb.line(i)) for i in range(lb.ynum)])
+        self.ae(a, [str(lb.line(i)) for i in range(lb.ynum)])
         hb = filled_history_buf(5, 5)
         a = []
         hb.as_ansi(a.append)
-        self.ae(a, ['\x1b[0m' + str(hb.line(i)) + '\n' for i in range(hb.count - 1, -1, -1)])
+        self.ae(a, [str(hb.line(i)) + '\n' for i in range(hb.count - 1, -1, -1)])

kitty_tests/screen.py

@@ -441,4 +441,4 @@ class TestScreen(BaseTest):
             return ''.join(d)
 
         self.ae(as_text(), 'ababababab\nc\n\n')
-        self.ae(as_text(True), '\x1b[0mababa\x1b[0mbabab\n\x1b[0mc\n\n')
+        self.ae(as_text(True), 'ababababab\nc\n\n')