From c346cbc252d3909a096a7205c3978b49faa86b1a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 31 Dec 2020 13:54:33 +0530 Subject: [PATCH] Use a single file for the disk cache This has the advantage that on crash it is automatically cleaned --- kitty/constants.py | 7 ---- kitty/disk-cache.c | 87 ++++++++++++++++++++-------------------------- 2 files changed, 38 insertions(+), 56 deletions(-) diff --git a/kitty/constants.py b/kitty/constants.py index 77ba219b0..8ead0384a 100644 --- a/kitty/constants.py +++ b/kitty/constants.py @@ -142,13 +142,6 @@ def cache_dir() -> str: return candidate -def dir_for_disk_cache() -> str: - from tempfile import mkdtemp - ans = os.path.join(cache_dir(), 'disk-cache') - os.makedirs(ans, exist_ok=True) - return mkdtemp(dir=ans) - - def wakeup() -> None: from .fast_data_types import get_boss b = get_boss() diff --git a/kitty/disk-cache.c b/kitty/disk-cache.c index 215139171..032e18fc1 100644 --- a/kitty/disk-cache.c +++ b/kitty/disk-cache.c @@ -22,32 +22,27 @@ typedef struct { size_t hash_keylen, data_sz; bool written_to_disk; uint8_t encryption_key[64]; - char filename[8]; + off_t pos_in_cache_file; UT_hash_handle hh; } CacheEntry; typedef struct { PyObject_HEAD - char *path; - int path_fd; + char *cache_dir; + int cache_file_fd; pthread_mutex_t lock; pthread_t write_thread; bool thread_started, lock_inited, loop_data_inited, shutting_down, fully_initialized; LoopData loop_data; - PyObject *rmtree; CacheEntry *entries, currently_writing; } DiskCache; void -free_cache_entry(const DiskCache *self, CacheEntry *e) { +free_cache_entry(CacheEntry *e) { if (e->hash_key) { free(e->hash_key); e->hash_key = NULL; } if (e->data) { free(e->data); e->data = NULL; } - if (self->path_fd > -1 && e->filename[0]) { - unlinkat(self->path_fd, e->filename, 0); - e->filename[0] = 0; - } free(e); } @@ -58,16 +53,27 @@ new(PyTypeObject *type, PyObject UNUSED *args, PyObject UNUSED *kwds) { DiskCache *self; self = (DiskCache*)type->tp_alloc(type, 0); if (self) { - self->path_fd = -1; - PyObject *shutil = PyImport_ImportModule("shutil"); - if (!shutil) { Py_CLEAR(self); return NULL; } - self->rmtree = PyObject_GetAttrString(shutil, "rmtree"); - Py_CLEAR(shutil); - if (!self->rmtree) { Py_CLEAR(self); return NULL; } + self->cache_file_fd = -1; } return (PyObject*) self; } +static int +open_cache_file(const char *cache_path) { + size_t sz = strlen(cache_path) + 16; + char *buf = calloc(1, sz); + if (!buf) { errno = ENOMEM; return -1; } + snprintf(buf, sz - 1, "%s/XXXXXXXXXXXX", cache_path); + int fd = -1; + while (fd < 0) { + fd = mkostemp(buf, O_CLOEXEC); + if (fd > -1 || errno != EINTR) break; + } + if (fd > -1) unlink(buf); + free(buf); + return fd; +} + static void* write_loop(void *data) { DiskCache *self = (DiskCache*)data; @@ -101,27 +107,24 @@ ensure_state(DiskCache *self) { self->thread_started = true; } - if (!self->path) { + if (!self->cache_dir) { PyObject *kc = NULL, *cache_dir = NULL; kc = PyImport_ImportModule("kitty.constants"); if (kc) { cache_dir = PyObject_CallMethod(kc, "dir_for_disk_cache", NULL); if (cache_dir) { - self->path = strdup(PyUnicode_AsUTF8(cache_dir)); - if (!self->path) PyErr_NoMemory(); + self->cache_dir = strdup(PyUnicode_AsUTF8(cache_dir)); + if (!self->cache_dir) PyErr_NoMemory(); } } Py_CLEAR(kc); Py_CLEAR(cache_dir); if (PyErr_Occurred()) return false; } - if (self->path_fd < 0) { - while (self->path_fd < 0) { - self->path_fd = open(self->path, O_DIRECTORY | O_RDWR | O_CLOEXEC); - if (self->path_fd > -1 || errno != EINTR) break; - } - if (self->path_fd < 0) { - PyErr_SetFromErrnoWithFilename(PyExc_OSError, self->path); + if (self->cache_file_fd < 0) { + self->cache_file_fd = open_cache_file(self->cache_dir); + if (self->cache_file_fd < 0) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, self->cache_dir); return false; } } @@ -155,21 +158,16 @@ dealloc(DiskCache* self) { CacheEntry *tmp, *s; HASH_ITER(hh, self->entries, s, tmp) { HASH_DEL(self->entries, s); - free_cache_entry(self, s); s = NULL; + free_cache_entry(s); s = NULL; } self->entries = NULL; } - if (self->path_fd > -1) { - safe_close(self->path_fd, __FILE__, __LINE__); - self->path_fd = -1; - } - if (self->path) { - PyObject_CallFunction(self->rmtree, "sO", self->path, Py_True); - free(self->path); self->path = NULL; + if (self->cache_file_fd > -1) { + safe_close(self->cache_file_fd, __FILE__, __LINE__); + self->cache_file_fd = -1; } if (self->currently_writing.hash_key) free(self->currently_writing.hash_key); if (self->currently_writing.data) free(self->currently_writing.data); - Py_CLEAR(self->rmtree); Py_TYPE(self)->tp_free((PyObject*)self); } @@ -219,36 +217,27 @@ xor_data(const uint8_t* restrict key, const size_t key_sz, uint8_t* restrict dat static void read_from_cache_entry(const DiskCache *self, const CacheEntry *s, uint8_t *dest) { - int fd = -1; - while (fd < 0) { - fd = openat(self->path_fd, s->filename, O_CLOEXEC | O_RDONLY); - if (fd > 0 || errno != EINTR) break; - } - if (fd < 0) { - PyErr_SetFromErrnoWithFilename(PyExc_OSError, s->filename); - return; - } uint8_t *p = dest; size_t sz = s->data_sz; + off_t pos = s->pos_in_cache_file; while (sz) { - ssize_t n = read(fd, p, sz); + ssize_t n = pread(self->cache_file_fd, p, sz, pos); if (n > 0) { sz -= n; p += n; + pos += n; continue; } if (n < 0) { if (errno == EINTR || errno == EAGAIN) continue; - PyErr_SetFromErrnoWithFilename(PyExc_OSError, s->filename); - goto end; + PyErr_SetFromErrnoWithFilename(PyExc_OSError, self->cache_dir); + break; } if (n == 0) { PyErr_SetString(PyExc_OSError, "Disk cache file truncated"); - goto end; + break; } } -end: - safe_close(fd, __FILE__, __LINE__); } bool