Start work on C implementation of the core data types

kitty/data-types.c

@@ -5,7 +5,9 @@
  * Distributed under terms of the GPL3 license.
  */
 
-#include <Python.h>
+#include "data-types.h"
+
+extern PyTypeObject LineBuf_Type;
 
 static PyMethodDef module_methods[] = {
     {NULL, NULL, 0, NULL}        /* Sentinel */
@@ -14,9 +16,8 @@ static PyMethodDef module_methods[] = {
 static struct PyModuleDef module = {
    PyModuleDef_HEAD_INIT,
    "fast_data_types",   /* name of module */
-   NULL, /* module documentation, may be NULL */
+   NULL, 
-   -1,       /* size of per-interpreter state of the module,
+   -1,       
-                or -1 if the module keeps state in global variables. */
    module_methods
 };
 
@@ -24,9 +25,15 @@ PyMODINIT_FUNC
 PyInit_fast_data_types(void) {
     PyObject *m;
 
+
+    if (PyType_Ready(&LineBuf_Type) < 0) return NULL;
     m = PyModule_Create(&module);
     if (m == NULL) return NULL;
 
+    if (m != NULL) {
+        Py_INCREF(&LineBuf_Type);
+        PyModule_AddObject(m, "LineBuf", (PyObject *)&LineBuf_Type);
+    }
+
     return m;
 }
-

kitty/data-types.h

@@ -0,0 +1,40 @@
+/*
+ * data-types.h
+ * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#pragma once
+
+
+#include <stdint.h>
+#include <Python.h>
+#define UNUSED __attribute__ ((unused))
+#define MAX(x, y) (((x) > (y)) ? (x) : (y))
+
+typedef Py_UCS4 char_type;
+typedef uint64_t color_type;
+typedef uint32_t decoration_type;
+typedef uint32_t combining_type;
+typedef unsigned int index_type;
+#define CELL_SIZE (sizeof(char_type) + sizeof(color_type) + sizeof(decoration_type) + sizeof(combining_type))
+#define CHAR_MASK 0xFFFFFF
+
+typedef struct {
+    PyObject_HEAD
+
+    uint8_t *buf;
+    index_type xnum, ynum, *line_map;
+    index_type block_size;
+    uint8_t *continued_map;
+
+    // Pointers into buf
+    char_type *chars;
+    color_type *colors;
+    decoration_type *decoration_fg;
+    combining_type *combining_chars;
+} LineBuf;
+
+
+

kitty/line-buf.c

@@ -0,0 +1,147 @@
+/*
+ * line-buf.c
+ * Copyright (C) 2016 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "data-types.h"
+
+static inline void
+clear_chars_to_space(LineBuf* linebuf, index_type y) {
+    char_type *chars = linebuf->chars + linebuf->xnum * y;
+    for (index_type i = 0; i < linebuf->xnum; i++) chars[i] = 32;
+}
+
+static PyObject *
+LineBuf_new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) {
+    LineBuf *self;
+    index_type xnum, ynum;
+
+    if (!PyArg_ParseTuple(args, "II", &ynum, &xnum)) return NULL;
+
+    if (xnum > 5000 || ynum > 50000) {
+        PyErr_SetString(PyExc_ValueError, "Number of rows or columns is too large.");
+        return NULL;
+    }
+
+    if (xnum * ynum == 0) {
+        PyErr_SetString(PyExc_ValueError, "Cannot create an empty LineBuf");
+        return NULL;
+    }
+
+    self = (LineBuf *)type->tp_alloc(type, 0);
+    if (self != NULL) {
+        self->xnum = xnum;
+        self->ynum = ynum;
+        self->block_size = xnum * ynum;
+        self->buf = PyMem_Calloc(xnum * ynum, CELL_SIZE);
+        self->line_map = PyMem_Calloc(ynum, sizeof(index_type));
+        self->continued_map = PyMem_Calloc(ynum, sizeof(uint8_t));
+        if (self->buf == NULL || self->line_map == NULL || self->continued_map == NULL) {
+            PyErr_NoMemory();
+            PyMem_Free(self->buf); PyMem_Free(self->line_map); PyMem_Free(self->continued_map);
+            Py_DECREF(self);
+            self = NULL;
+        } else {
+            self->chars = (char_type*)self->buf;
+            self->colors = (color_type*)(self->chars + self->block_size);
+            self->decoration_fg = (decoration_type*)(self->colors + self->block_size);
+            self->combining_chars = (combining_type*)(self->decoration_fg + self->block_size);
+            for(index_type i = 0; i < ynum; i++) {
+                self->line_map[i] = i;
+                clear_chars_to_space(self, i);
+            }
+        }
+    }
+
+    return (PyObject*)self;
+}
+
+static void
+LineBuf_dealloc(LineBuf* self) {
+    PyMem_Free(self->buf); PyMem_Free(self->line_map); PyMem_Free(self->continued_map);
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+static PyObject *
+text_at(LineBuf* self, PyObject *args) {
+    index_type ynum, xnum, idx;
+    char_type ch;
+    combining_type cc;
+    PyObject * ans;
+
+    if (!PyArg_ParseTuple(args, "II", &ynum, &xnum)) return NULL;
+    if (ynum >= self->ynum) { PyErr_SetString(PyExc_ValueError, "Line number out of bounds"); return NULL; }
+    if (xnum >= self->xnum) { PyErr_SetString(PyExc_ValueError, "Column number out of bounds"); return NULL; }
+
+    idx = xnum + ynum * self->xnum;
+    ch = self->chars[idx] & CHAR_MASK;
+    cc = self->combining_chars[idx];
+    if (cc == 0) {
+        ans = PyUnicode_New(1, ch);
+        if (ans == NULL) return PyErr_NoMemory();
+        PyUnicode_WriteChar(ans, 0, ch);
+    } else {
+        Py_UCS4 cc1 = cc & 0xFFFF, cc2 = cc >> 16;
+        Py_UCS4 maxc = (ch > cc1) ? MAX(ch, cc2) : MAX(cc1, cc2);
+        ans = PyUnicode_New(cc2 ? 3 : 2, maxc);
+        if (ans == NULL) return PyErr_NoMemory();
+        PyUnicode_WriteChar(ans, 0, ch);
+        PyUnicode_WriteChar(ans, 1, cc1);
+        if (cc2) PyUnicode_WriteChar(ans, 2, cc2);
+    }
+
+    return ans;
+}
+
+// Boilerplate {{{
+static PyMethodDef LineBuf_methods[] = {
+    {"text_at", (PyCFunction)text_at, METH_VARARGS,
+     "Return the text in the specified cell"
+    },
+    {NULL}  /* Sentinel */
+};
+
+PyTypeObject LineBuf_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "fast_data_types.LineBuf",
+    sizeof(LineBuf),
+    0,                         /* tp_itemsize */
+    (destructor)LineBuf_dealloc, /* tp_dealloc */
+    0,                         /* tp_print */
+    0,                         /* tp_getattr */
+    0,                         /* tp_setattr */
+    0,                         /* tp_reserved */
+    0,                         /* tp_repr */
+    0,                         /* tp_as_number */
+    0,                         /* tp_as_sequence */
+    0,                         /* tp_as_mapping */
+    0,                         /* tp_hash  */
+    0,                         /* tp_call */
+    0,                         /* tp_str */
+    0,                         /* tp_getattro */
+    0,                         /* tp_setattro */
+    0,                         /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,        /* tp_flags */
+    "Line buffers",            /* tp_doc */
+    0,                         /* tp_traverse */
+    0,                         /* tp_clear */
+    0,                         /* tp_richcompare */
+    0,                         /* tp_weaklistoffset */
+    0,                         /* tp_iter */
+    0,                         /* tp_iternext */
+    LineBuf_methods,           /* tp_methods */
+    0,                         /* tp_members */
+    0,                         /* tp_getset */
+    0,                         /* tp_base */
+    0,                         /* tp_dict */
+    0,                         /* tp_descr_get */
+    0,                         /* tp_descr_set */
+    0,                         /* tp_dictoffset */
+    0,                         /* tp_init */
+    0,                         /* tp_alloc */
+    LineBuf_new,               /* tp_new */
+};
+// }}
+

kitty_tests/datatypes.py

@@ -8,10 +8,17 @@ from . import BaseTest
 
 from kitty.data_types import Line, Cursor
 from kitty.utils import is_simple_string, wcwidth, sanitize_title
+from kitty.fast_data_types import LineBuf
 
 
 class TestDataTypes(BaseTest):
 
+    def text_line_buf(self):
+        lb = LineBuf(2, 3)
+        for y in range(2):
+            for x in range(3):
+                self.ae(lb.text_at(y, x), ' ')
+
     def test_line_ops(self):
         t = 'Testing with simple text'
         l = Line(len(t))

setup.py

@@ -25,7 +25,8 @@ def init_env():
     global cflags, ldflags, cc, ldpaths
     cc = os.environ.get('CC', 'gcc')
     cflags = os.environ.get('OVERRIDE_CFLAGS',
-                            '-Wall -Werror -O3 -DNDEBUG -fwrapv -fstack-protector-strong -pipe')
+                            '-Wextra -Wno-missing-field-initializers -Wall -std=c99 -D_XOPEN_SOURCE=700'
+                            ' -pedantic-errors -Werror -O3 -DNDEBUG -fwrapv -fstack-protector-strong -pipe')
     cflags = shlex.split(cflags) + shlex.split(sysconfig.get_config_var('CCSHARED'))
     ldflags = os.environ.get('OVERRIDE_LDFLAGS', '-Wall -O3')
     ldflags = shlex.split(ldflags)
@@ -67,4 +68,4 @@ if __name__ == '__main__':
     if sys.version_info < (3, 5):
         raise SystemExit('python >= 3.5 required')
     init_env()
-    compile_c_extension('kitty/fast_data_types', 'kitty/data-types.c')
+    compile_c_extension('kitty/fast_data_types', 'kitty/data-types.c', 'kitty/line-buf.c')