From 34cbf5ceac1ecf9f40c8318b953014351411f59f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 10 Mar 2023 13:22:10 +0530 Subject: [PATCH] Get rid of prewarming Don't need it anymore since all major UI kittens are ported to Go and so don't have startup latency. --- kitty/boss.py | 6 +- kitty/child-monitor.c | 40 +-- kitty/child.py | 92 ++---- kitty/fast_data_types.pyi | 1 - kitty/main.py | 14 +- kitty/prewarm.py | 621 -------------------------------------- kitty_tests/prewarm.py | 145 --------- 7 files changed, 31 insertions(+), 888 deletions(-) delete mode 100644 kitty/prewarm.py delete mode 100644 kitty_tests/prewarm.py diff --git a/kitty/boss.py b/kitty/boss.py index 80466f8c8..89d8e0b12 100644 --- a/kitty/boss.py +++ b/kitty/boss.py @@ -120,7 +120,6 @@ from .notify import notification_activated from .options.types import Options from .options.utils import MINIMUM_FONT_SIZE, KeyMap, SubSequenceMap from .os_window_size import initial_window_size_func -from .prewarm import PrewarmProcess from .rgb import color_from_int from .session import Session, create_sessions, get_os_window_sizing_data from .tabs import SpecialWindow, SpecialWindowInstance, Tab, TabDict, TabManager @@ -320,7 +319,6 @@ class Boss: args: CLIOptions, cached_values: Dict[str, Any], global_shortcuts: Dict[str, SingleKey], - prewarm: PrewarmProcess, ): set_layout_options(opts) self.clipboard = Clipboard() @@ -357,11 +355,10 @@ class Boss: if args.listen_on and self.allow_remote_control in ('y', 'socket', 'socket-only', 'password'): listen_fd = listen_on(args.listen_on) self.listening_on = args.listen_on - self.prewarm = prewarm self.child_monitor = ChildMonitor( self.on_child_death, DumpCommands(args) if args.dump_commands or args.dump_bytes else None, - talk_fd, listen_fd, self.prewarm.take_from_worker_fd() + talk_fd, listen_fd, ) set_boss(self) self.args = args @@ -2402,7 +2399,6 @@ class Boss: for w in self.all_windows: self.default_bg_changed_for(w.id) w.refresh(reload_all_gpu_data=True) - self.prewarm.reload_kitty_config() @ac('misc', ''' Reload the config file diff --git a/kitty/child-monitor.c b/kitty/child-monitor.c index 3414a0785..e3ee2c3c3 100644 --- a/kitty/child-monitor.c +++ b/kitty/child-monitor.c @@ -57,7 +57,7 @@ typedef struct { bool shutting_down; pthread_t io_thread, talk_thread; - int talk_fd, listen_fd, prewarm_fd; + int talk_fd, listen_fd; Message *messages; size_t messages_capacity, messages_count; LoopData io_loop_data; @@ -158,11 +158,11 @@ static PyObject * new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) { ChildMonitor *self; PyObject *dump_callback, *death_notify; - int talk_fd = -1, listen_fd = -1, prewarm_fd = -1; + int talk_fd = -1, listen_fd = -1; int ret; if (the_monitor) { PyErr_SetString(PyExc_RuntimeError, "Can have only a single ChildMonitor instance"); return NULL; } - if (!PyArg_ParseTuple(args, "OO|iii", &death_notify, &dump_callback, &talk_fd, &listen_fd, &prewarm_fd)) return NULL; + if (!PyArg_ParseTuple(args, "OO|ii", &death_notify, &dump_callback, &talk_fd, &listen_fd)) return NULL; if ((ret = pthread_mutex_init(&children_lock, NULL)) != 0) { PyErr_Format(PyExc_RuntimeError, "Failed to create children_lock mutex: %s", strerror(ret)); return NULL; @@ -175,7 +175,6 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) { if (!init_loop_data(&self->io_loop_data, KITTY_HANDLED_SIGNALS)) return PyErr_SetFromErrno(PyExc_OSError); self->talk_fd = talk_fd; self->listen_fd = listen_fd; - self->prewarm_fd = prewarm_fd; if (self == NULL) return PyErr_NoMemory(); self->death_notify = death_notify; Py_INCREF(death_notify); if (dump_callback != Py_None) { @@ -184,7 +183,6 @@ new(PyTypeObject *type, PyObject *args, PyObject UNUSED *kwds) { } else parse_func = parse_worker; self->count = 0; children_fds[0].fd = self->io_loop_data.wakeup_read_fd; children_fds[1].fd = self->io_loop_data.signal_read_fd; - children_fds[2].fd = self->prewarm_fd; children_fds[0].events = POLLIN; children_fds[1].events = POLLIN; children_fds[2].events = POLLIN; the_monitor = self; @@ -211,7 +209,6 @@ dealloc(ChildMonitor* self) { FREE_CHILD(add_queue[add_queue_count]); } free_loop_data(&self->io_loop_data); - safe_close(self->prewarm_fd, __FILE__, __LINE__); self->prewarm_fd = -1; Py_TYPE(self)->tp_free((PyObject*)self); } @@ -1365,34 +1362,6 @@ mark_monitored_pids(pid_t pid, int status) { children_mutex(unlock); } -static void -reap_prewarmed_children(ChildMonitor *self, int fd, bool enable_close_on_child_death) { - static char buf[256]; - static size_t buf_pos = 0; - while(true) { - ssize_t len = read(fd, buf + buf_pos, sizeof(buf) - buf_pos); - if (len < 0) { - if (errno == EINTR) continue; - if (errno != EIO && errno != EAGAIN) log_error("Call to read() from reap_prewarmed_children() failed with error: %s", strerror(errno)); - break; - } - buf_pos += len; - char *nl; - while (buf_pos > 1 && (nl = memchr(buf, '\n', buf_pos)) != NULL) { - size_t sz = nl - buf + 1; - if (enable_close_on_child_death) { - *nl = 0; - int pid = atoi(buf); - if (pid) mark_child_for_removal(self, pid); - } - memmove(buf, buf + sz, sz); - buf_pos -= sz; - } - if (len == 0) break; - } - -} - static void reap_children(ChildMonitor *self, bool enable_close_on_child_death) { int status; @@ -1508,9 +1477,6 @@ io_loop(void *data) { } if (ss.child_died) reap_children(self, OPT(close_on_child_death)); } - if (children_fds[2].revents && POLLIN) { - reap_prewarmed_children(self, children_fds[2].fd, OPT(close_on_child_death)); - } for (i = 0; i < self->count; i++) { if (children_fds[EXTRA_FDS + i].revents & (POLLIN | POLLHUP)) { data_received = true; diff --git a/kitty/child.py b/kitty/child.py index fcb0c97e3..a6761d38a 100644 --- a/kitty/child.py +++ b/kitty/child.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, DefaultDict, Dict, Generator, List, Optional, import kitty.fast_data_types as fast_data_types -from .constants import handled_signals, is_freebsd, is_macos, kitten_exe, kitty_base_dir, shell_path, terminfo_dir, wrapped_kitten_names +from .constants import handled_signals, is_freebsd, is_macos, kitten_exe, kitty_base_dir, shell_path, terminfo_dir from .types import run_once from .utils import log_error, which @@ -185,48 +185,11 @@ class ProcessDesc(TypedDict): cmdline: Optional[Sequence[str]] -def is_prewarmable(argv: List[str]) -> Tuple[bool, List[str]]: - if len(argv) < 3 or os.path.basename(argv[0]) != 'kitty': - return False, argv - if argv[1][:1] != '+': - return False, argv - sw = '' - if argv[1] == '+': - which = argv[2] - if len(argv) > 3: - sw = argv[3] - else: - which = argv[1][1:] - if len(argv) > 2: - sw = argv[2] - if which == 'open': - return False, argv - if which == 'kitten' and sw in wrapped_kitten_names(): - argv = list(argv) - argv[0] = kitten_exe() - if argv[1] == '+': - del argv[1:3] - else: - del argv[1] - return False, argv - return True, argv - - -@run_once -def cmdline_of_prewarmer() -> List[str]: - # we need this check in case the prewarmed process has done an exec and - # changed its cmdline - with suppress(Exception): - return cmdline_of_pid(fast_data_types.get_boss().prewarm.worker_pid) - return [''] - - class Child: child_fd: Optional[int] = None pid: Optional[int] = None forked = False - is_prewarmed = False def __init__( self, @@ -296,18 +259,16 @@ class Child: self.forked = True master, slave = openpty() stdin, self.stdin = self.stdin, None - self.is_prewarmed, self.argv = is_prewarmable(self.argv) - if not self.is_prewarmed: - ready_read_fd, ready_write_fd = os.pipe() - os.set_inheritable(ready_write_fd, False) - os.set_inheritable(ready_read_fd, True) - if stdin is not None: - stdin_read_fd, stdin_write_fd = os.pipe() - os.set_inheritable(stdin_write_fd, False) - os.set_inheritable(stdin_read_fd, True) - else: - stdin_read_fd = stdin_write_fd = -1 - env = tuple(f'{k}={v}' for k, v in self.final_env().items()) + ready_read_fd, ready_write_fd = os.pipe() + os.set_inheritable(ready_write_fd, False) + os.set_inheritable(ready_read_fd, True) + if stdin is not None: + stdin_read_fd, stdin_write_fd = os.pipe() + os.set_inheritable(stdin_write_fd, False) + os.set_inheritable(stdin_read_fd, True) + else: + stdin_read_fd = stdin_write_fd = -1 + env = tuple(f'{k}={v}' for k, v in self.final_env().items()) argv = list(self.argv) exe = argv[0] if is_macos and exe == shell_path: @@ -328,23 +289,17 @@ class Child: argv[0] = (f'-{exe.split("/")[-1]}') self.final_exe = which(exe) or exe self.final_argv0 = argv[0] - if self.is_prewarmed: - fe = self.final_env() - self.prewarmed_child = fast_data_types.get_boss().prewarm(slave, self.argv, self.cwd, fe, stdin) - pid = self.prewarmed_child.child_process_pid - else: - pid = fast_data_types.spawn( - self.final_exe, self.cwd, tuple(argv), env, master, slave, stdin_read_fd, stdin_write_fd, - ready_read_fd, ready_write_fd, tuple(handled_signals), kitten_exe()) + pid = fast_data_types.spawn( + self.final_exe, self.cwd, tuple(argv), env, master, slave, stdin_read_fd, stdin_write_fd, + ready_read_fd, ready_write_fd, tuple(handled_signals), kitten_exe()) os.close(slave) self.pid = pid self.child_fd = master - if not self.is_prewarmed: - if stdin is not None: - os.close(stdin_read_fd) - fast_data_types.thread_write(stdin_write_fd, stdin) - os.close(ready_read_fd) - self.terminal_ready_fd = ready_write_fd + if stdin is not None: + os.close(stdin_read_fd) + fast_data_types.thread_write(stdin_write_fd, stdin) + os.close(ready_read_fd) + self.terminal_ready_fd = ready_write_fd if self.child_fd is not None: os.set_blocking(self.child_fd, False) return pid @@ -356,18 +311,15 @@ class Child: self.terminal_ready_fd = -1 def mark_terminal_ready(self) -> None: - if self.is_prewarmed: - fast_data_types.get_boss().prewarm.mark_child_as_ready(self.prewarmed_child.child_id) - else: - os.close(self.terminal_ready_fd) - self.terminal_ready_fd = -1 + os.close(self.terminal_ready_fd) + self.terminal_ready_fd = -1 def cmdline_of_pid(self, pid: int) -> List[str]: try: ans = cmdline_of_pid(pid) except Exception: ans = [] - if pid == self.pid and (not ans or (self.is_prewarmed and ans == cmdline_of_prewarmer())): + if pid == self.pid and (not ans): ans = list(self.argv) return ans diff --git a/kitty/fast_data_types.pyi b/kitty/fast_data_types.pyi index a32675dd8..acf0e1293 100644 --- a/kitty/fast_data_types.pyi +++ b/kitty/fast_data_types.pyi @@ -1232,7 +1232,6 @@ class ChildMonitor: dump_callback: Optional[Callable[[bytes], None]], talk_fd: int = -1, listen_fd: int = -1, - prewarm_fd: int = -1, ): pass diff --git a/kitty/main.py b/kitty/main.py index f09c5965a..7b5a150f4 100644 --- a/kitty/main.py +++ b/kitty/main.py @@ -49,7 +49,6 @@ from .fonts.render import set_font_family from .options.types import Options from .options.utils import DELETE_ENV_VAR from .os_window_size import initial_window_size_func -from .prewarm import PrewarmProcess, fork_prewarm_process from .session import create_sessions, get_os_window_sizing_data from .types import SingleInstanceData from .utils import ( @@ -202,7 +201,7 @@ def set_x11_window_icon() -> None: set_default_window_icon(f'{path}-128{ext}') -def _run_app(opts: Options, args: CLIOptions, prewarm: PrewarmProcess, bad_lines: Sequence[BadLine] = ()) -> None: +def _run_app(opts: Options, args: CLIOptions, bad_lines: Sequence[BadLine] = ()) -> None: global_shortcuts: Dict[str, SingleKey] = {} if is_macos: from collections import defaultdict @@ -249,7 +248,7 @@ def _run_app(opts: Options, args: CLIOptions, prewarm: PrewarmProcess, bad_lines pre_show_callback, args.title or appname, args.name or args.cls or appname, wincls, wstate, load_all_shaders, disallow_override_title=bool(args.title)) - boss = Boss(opts, args, cached_values, global_shortcuts, prewarm) + boss = Boss(opts, args, cached_values, global_shortcuts) boss.start(window_id, startup_sessions) if bad_lines: boss.show_bad_config_lines(bad_lines) @@ -266,12 +265,12 @@ class AppRunner: self.first_window_callback = lambda window_handle: None self.initial_window_size_func = initial_window_size_func - def __call__(self, opts: Options, args: CLIOptions, prewarm: PrewarmProcess, bad_lines: Sequence[BadLine] = ()) -> None: + def __call__(self, opts: Options, args: CLIOptions, bad_lines: Sequence[BadLine] = ()) -> None: set_scale(opts.box_drawing_scale) set_options(opts, is_wayland(), args.debug_rendering, args.debug_font_fallback) try: set_font_family(opts, debug_font_matching=args.debug_font_fallback) - _run_app(opts, args, prewarm, bad_lines) + _run_app(opts, args, bad_lines) finally: set_options(None) free_font_data() # must free font data before glfw/freetype/fontconfig/opengl etc are finalized @@ -497,9 +496,6 @@ def _main() -> None: bad_lines: List[BadLine] = [] opts = create_opts(cli_opts, accumulate_bad_lines=bad_lines) setup_environment(opts, cli_opts) - prewarm = fork_prewarm_process(opts) - if prewarm is None: - raise SystemExit(1) # set_locale on macOS uses cocoa APIs when LANG is not set, so we have to # call it after the fork @@ -522,7 +518,7 @@ def _main() -> None: try: with setup_profiling(): # Avoid needing to launch threads to reap zombies - run_app(opts, cli_opts, prewarm, bad_lines) + run_app(opts, cli_opts, bad_lines) finally: glfw_terminate() cleanup_ssh_control_masters() diff --git a/kitty/prewarm.py b/kitty/prewarm.py deleted file mode 100644 index 4868b1d0f..000000000 --- a/kitty/prewarm.py +++ /dev/null @@ -1,621 +0,0 @@ -#!/usr/bin/env python -# License: GPLv3 Copyright: 2022, Kovid Goyal - -import fcntl -import io -import json -import os -import select -import signal -import sys -import termios -import time -import traceback -import warnings -from contextlib import suppress -from dataclasses import dataclass -from importlib import import_module -from itertools import count -from typing import IO, TYPE_CHECKING, Any, Callable, Dict, Iterator, List, NoReturn, Optional, Tuple, TypeVar, Union, cast - -from kitty.constants import kitty_exe, running_in_kitty -from kitty.entry_points import main as main_entry_point -from kitty.fast_data_types import ( - CLD_EXITED, - CLD_KILLED, - CLD_STOPPED, - clearenv, - get_options, - install_signal_handlers, - read_signals, - remove_signal_handlers, - safe_pipe, - set_options, - set_use_os_log, -) -from kitty.options.types import Options -from kitty.shm import SharedMemory -from kitty.types import SignalInfo -from kitty.utils import log_error, safer_fork - -if TYPE_CHECKING: - from _typeshed import ReadableBuffer, WriteableBuffer - - -error_events = select.POLLERR | select.POLLNVAL | select.POLLHUP -TIMEOUT = 5.0 - - -def restore_python_signal_handlers() -> None: - remove_signal_handlers() - signal.signal(signal.SIGINT, signal.default_int_handler) - signal.signal(signal.SIGPIPE, signal.SIG_IGN) - signal.signal(signal.SIGUSR1, signal.SIG_DFL) - signal.signal(signal.SIGCHLD, signal.SIG_DFL) - - -def print_error(*a: Any) -> None: - log_error('Prewarm zygote:', *a) - - -class PrewarmProcessFailed(Exception): - pass - - -@dataclass -class Child: - child_id: int - child_process_pid: int - - -def wait_for_child_death(child_pid: int, timeout: float = 1, options: int = 0) -> Optional[int]: - st = time.monotonic() - while not timeout or time.monotonic() - st < timeout: - try: - pid, status = os.waitpid(child_pid, options | os.WNOHANG) - except ChildProcessError: - return 0 - else: - if pid == child_pid: - return status - if not timeout: - break - time.sleep(0.01) - return None - - -class PrewarmProcess: - - def __init__( - self, - prewarm_process_pid: int, - to_prewarm_stdin: int, - from_prewarm_stdout: int, - from_prewarm_death_notify: int, - ) -> None: - self.children: Dict[int, Child] = {} - self.worker_pid = prewarm_process_pid - self.from_prewarm_death_notify = from_prewarm_death_notify - self.write_to_process_fd = to_prewarm_stdin - self.read_from_process_fd = from_prewarm_stdout - self.poll = select.poll() - self.poll.register(self.read_from_process_fd, select.POLLIN) - - def take_from_worker_fd(self, create_file: bool = False) -> int: - if create_file: - os.set_blocking(self.from_prewarm_death_notify, True) - self.from_worker = open(self.from_prewarm_death_notify, mode='r', closefd=True) - self.from_prewarm_death_notify = -1 - return -1 - ans, self.from_prewarm_death_notify = self.from_prewarm_death_notify, -1 - return ans - - def __del__(self) -> None: - if self.write_to_process_fd > -1: - safe_close(self.write_to_process_fd) - self.write_to_process_fd = -1 - if self.from_prewarm_death_notify > -1: - safe_close(self.from_prewarm_death_notify) - self.from_prewarm_death_notify = -1 - if self.read_from_process_fd > -1: - safe_close(self.read_from_process_fd) - self.read_from_process_fd = -1 - - if hasattr(self, 'from_worker'): - self.from_worker.close() - del self.from_worker - if self.worker_pid > 0: - if wait_for_child_death(self.worker_pid) is None: - log_error('Prewarm process failed to quit gracefully, killing it') - os.kill(self.worker_pid, signal.SIGKILL) - os.waitpid(self.worker_pid, 0) - - def poll_to_send(self, yes: bool = True) -> None: - if yes: - self.poll.register(self.write_to_process_fd, select.POLLOUT) - else: - self.poll.unregister(self.write_to_process_fd) - - def reload_kitty_config(self, opts: Optional[Options] = None) -> None: - if opts is None: - opts = get_options() - data = json.dumps({'paths': opts.config_paths, 'overrides': opts.config_overrides}) - if self.write_to_process_fd > -1: - self.send_to_prewarm_process(f'reload_kitty_config:{data}\n') - - def __call__( - self, - tty_fd: int, - argv: List[str], - cwd: str = '', - env: Optional[Dict[str, str]] = None, - stdin_data: Optional[Union[str, bytes]] = None, - timeout: float = TIMEOUT, - ) -> Child: - tty_name = os.ttyname(tty_fd) - if isinstance(stdin_data, str): - stdin_data = stdin_data.encode() - if env is None: - env = dict(os.environ) - cmd: Dict[str, Union[int, List[str], str, Dict[str, str]]] = { - 'tty_name': tty_name, 'cwd': cwd or os.getcwd(), 'argv': argv, 'env': env, - } - total_size = 0 - if stdin_data is not None: - cmd['stdin_size'] = len(stdin_data) - total_size += len(stdin_data) - data = json.dumps(cmd).encode() - total_size += len(data) + SharedMemory.num_bytes_for_size - with SharedMemory(size=total_size, unlink_on_exit=True) as shm: - shm.write_data_with_size(data) - if stdin_data: - shm.write(stdin_data) - shm.flush() - self.send_to_prewarm_process(f'fork:{shm.name}\n') - input_buf = b'' - st = time.monotonic() - while time.monotonic() - st < timeout: - for (fd, event) in self.poll.poll(2): - if event & error_events: - raise PrewarmProcessFailed('Failed doing I/O with prewarm process') - if fd == self.read_from_process_fd and event & select.POLLIN: - d = os.read(self.read_from_process_fd, io.DEFAULT_BUFFER_SIZE) - input_buf += d - while (idx := input_buf.find(b'\n')) > -1: - line = input_buf[:idx].decode() - input_buf = input_buf[idx+1:] - if line.startswith('CHILD:'): - _, cid, pid = line.split(':') - child = self.add_child(int(cid), int(pid)) - shm.unlink_on_exit = False - return child - if line.startswith('ERR:'): - raise PrewarmProcessFailed(line.split(':', 1)[-1]) - raise PrewarmProcessFailed('Timed out waiting for I/O with prewarm process') - - def add_child(self, child_id: int, pid: int) -> Child: - self.children[child_id] = c = Child(child_id, pid) - return c - - def send_to_prewarm_process(self, output_buf: Union[str, bytes] = b'', timeout: float = TIMEOUT) -> None: - if isinstance(output_buf, str): - output_buf = output_buf.encode() - st = time.monotonic() - while time.monotonic() - st < timeout and output_buf: - self.poll_to_send(bool(output_buf)) - for (fd, event) in self.poll.poll(2): - if event & error_events: - raise PrewarmProcessFailed(f'Failed doing I/O with prewarm process: {event}') - if fd == self.write_to_process_fd and event & select.POLLOUT: - n = os.write(self.write_to_process_fd, output_buf) - output_buf = output_buf[n:] - self.poll_to_send(False) - if output_buf: - raise PrewarmProcessFailed('Timed out waiting to write to prewarm process') - - def mark_child_as_ready(self, child_id: int) -> bool: - c = self.children.pop(child_id, None) - if c is None: - return False - self.send_to_prewarm_process(f'ready:{child_id}\n') - return True - - -def reload_kitty_config(payload: str) -> None: - d = json.loads(payload) - from kittens.tui.utils import set_kitty_opts - set_kitty_opts(paths=d['paths'], overrides=d['overrides']) - - -def prewarm() -> None: - from kittens.runner import all_kitten_names - for kitten in all_kitten_names(): - with suppress(Exception): - import_module(f'kittens.{kitten}.main') - - -class MemoryViewReadWrapperBytes(io.BufferedIOBase): - - def __init__(self, mw: memoryview): - self.mw = mw - self.pos = 0 - - def detach(self) -> io.RawIOBase: - raise io.UnsupportedOperation('detach() not supported') - - def read(self, size: Optional[int] = -1) -> bytes: - if size is None or size < 0: - size = max(0, len(self.mw) - self.pos) - oldpos = self.pos - self.pos = min(len(self.mw), self.pos + size) - if self.pos <= oldpos: - return b'' - return bytes(self.mw[oldpos:self.pos]) - - def readinto(self, b: 'WriteableBuffer') -> int: - if not isinstance(b, memoryview): - b = memoryview(b) - b = b.cast('B') - data = self.read(len(b)) - n = len(data) - b[:n] = data - return n - readinto1 = readinto - - def readall(self) -> bytes: - return self.read() - - def write(self, b: 'ReadableBuffer') -> int: - raise io.UnsupportedOperation('readonly stream') - - def readable(self) -> bool: - return True - - -class MemoryViewReadWrapper(io.TextIOWrapper): - - def __init__(self, mw: memoryview): - super().__init__(cast(IO[bytes], MemoryViewReadWrapperBytes(mw)), encoding='utf-8', errors='replace') - - -parent_tty_name = '' -is_zygote = True - - -def debug(*a: Any) -> None: - if parent_tty_name: - with open(parent_tty_name, 'w') as f: - print(*a, file=f) - - -def child_main(cmd: Dict[str, Any], ready_fd: int = -1, prewarm_type: str = 'direct') -> NoReturn: - getattr(sys, 'kitty_run_data')['prewarmed'] = prewarm_type - cwd = cmd.get('cwd') - if cwd: - with suppress(OSError): - os.chdir(cwd) - env = cmd.get('env') - if env is not None: - os.environ.clear() - # os.environ.clear() does not delete all existing env vars from the - # libc environ pointer in some circumstances, I havent figured out - # which exactly. Presumably there is something that alters the - # libc environ pointer?? The environ pointer is used by os.exec and - # therefore by subprocess and friends, so we need to ensure it is - # cleared. - clearenv() - os.environ.update(env) - argv = cmd.get('argv') - if argv: - sys.argv = list(argv) - if ready_fd > -1: - poll = select.poll() - poll.register(ready_fd, select.POLLIN) - tuple(poll.poll()) - safe_close(ready_fd) - main_entry_point() - raise SystemExit(0) - - -def fork(shm_address: str, free_non_child_resources: Callable[[], None]) -> Tuple[int, int]: - global is_zygote - sz = pos = 0 - with SharedMemory(name=shm_address, unlink_on_exit=True) as shm: - data = shm.read_data_with_size() - cmd = json.loads(data) - sz = cmd.get('stdin_size', 0) - if sz: - pos = shm.tell() - shm.unlink_on_exit = False - - r, w = safe_pipe() - ready_fd_read, ready_fd_write = safe_pipe() - try: - child_pid = safer_fork() - except OSError: - safe_close(r) - safe_close(w) - safe_close(ready_fd_read) - safe_close(ready_fd_write) - if sz: - with SharedMemory(shm_address, unlink_on_exit=True): - pass - raise - if child_pid: - # master process - safe_close(w) - safe_close(ready_fd_read) - poll = select.poll() - poll.register(r, select.POLLIN) - tuple(poll.poll()) - safe_close(r) - return child_pid, ready_fd_write - # child process - is_zygote = False - restore_python_signal_handlers() - safe_close(r) - safe_close(ready_fd_write) - free_non_child_resources() - os.setsid() - tty_name = cmd.get('tty_name') - if tty_name: - sys.__stdout__.flush() - sys.__stderr__.flush() - establish_controlling_tty(tty_name, sys.__stdin__.fileno(), sys.__stdout__.fileno(), sys.__stderr__.fileno()) - safe_close(w) - if shm.unlink_on_exit: - child_main(cmd, ready_fd_read) - else: - with SharedMemory(shm_address, unlink_on_exit=True) as shm: - stdin_data = memoryview(shm.mmap)[pos:pos + sz] - if stdin_data: - sys.stdin = MemoryViewReadWrapper(stdin_data) - try: - child_main(cmd, ready_fd_read) - finally: - stdin_data.release() - sys.stdin = sys.__stdin__ - return 0, -1 # type: ignore - - -Funtion = TypeVar('Funtion', bound=Callable[..., Any]) - - -def eintr_retry(func: Funtion) -> Funtion: - def ret(*a: Any, **kw: Any) -> Any: - while True: - with suppress(InterruptedError): - return func(*a, **kw) - return cast(Funtion, ret) - - -safe_close = eintr_retry(os.close) -safe_open = eintr_retry(os.open) -safe_ioctl = eintr_retry(fcntl.ioctl) -safe_dup2 = eintr_retry(os.dup2) - - -def establish_controlling_tty(fd_or_tty_name: Union[str, int], *dups: int, closefd: bool = True) -> int: - tty_name = os.ttyname(fd_or_tty_name) if isinstance(fd_or_tty_name, int) else fd_or_tty_name - with open(safe_open(tty_name, os.O_RDWR | os.O_CLOEXEC), 'w', closefd=closefd) as f: - tty_fd = f.fileno() - safe_ioctl(tty_fd, termios.TIOCSCTTY, 0) - for fd in dups: - safe_dup2(tty_fd, fd) - return -1 if closefd else tty_fd - - -interactive_and_job_control_signals = ( - signal.SIGINT, signal.SIGQUIT, signal.SIGTSTP, signal.SIGTTIN, signal.SIGTTOU -) - - -def main(stdin_fd: int, stdout_fd: int, notify_child_death_fd: int) -> None: - global parent_tty_name - with suppress(OSError): - parent_tty_name = os.ttyname(sys.stdout.fileno()) - os.set_blocking(notify_child_death_fd, False) - os.set_blocking(stdin_fd, False) - os.set_blocking(stdout_fd, False) - signal_read_fd = install_signal_handlers(signal.SIGCHLD, signal.SIGUSR1)[0] - poll = select.poll() - poll.register(stdin_fd, select.POLLIN) - poll.register(signal_read_fd, select.POLLIN) - input_buf = output_buf = child_death_buf = b'' - child_ready_fds: Dict[int, int] = {} - child_pid_map: Dict[int, int] = {} - child_id_counter = count() - # runpy issues a warning when running modules that have already been - # imported. Ignore it. - warnings.filterwarnings('ignore', category=RuntimeWarning, module='runpy') - prewarm() - - def get_all_non_child_fds() -> Iterator[int]: - yield notify_child_death_fd - yield stdin_fd - yield stdout_fd - # the signal fds are closed by remove_signal_handlers() - yield from child_ready_fds.values() - - def free_non_child_resources() -> None: - for fd in get_all_non_child_fds(): - if fd > -1: - safe_close(fd) - - def check_event(event: int, err_msg: str) -> None: - if event & select.POLLHUP: - raise SystemExit(0) - if event & error_events: - print_error(err_msg) - raise SystemExit(1) - - def handle_input(event: int) -> None: - nonlocal input_buf, output_buf - check_event(event, 'Polling of input pipe failed') - if not (event & select.POLLIN): - return - d = os.read(stdin_fd, io.DEFAULT_BUFFER_SIZE) - if not d: - raise SystemExit(0) - input_buf += d - while (idx := input_buf.find(b'\n')) > -1: - line = input_buf[:idx].decode() - input_buf = input_buf[idx+1:] - cmd, _, payload = line.partition(':') - if cmd == 'reload_kitty_config': - reload_kitty_config(payload) - elif cmd == 'ready': - child_id = int(payload) - cfd = child_ready_fds.pop(child_id, None) - if cfd is not None: - safe_close(cfd) - elif cmd == 'quit': - raise SystemExit(0) - elif cmd == 'fork': - try: - child_pid, ready_fd_write = fork(payload, free_non_child_resources) - except Exception as e: - es = str(e).replace('\n', ' ') - output_buf += f'ERR:{es}\n'.encode() - else: - if is_zygote: - child_id = next(child_id_counter) - child_pid_map[child_pid] = child_id - child_ready_fds[child_id] = ready_fd_write - output_buf += f'CHILD:{child_id}:{child_pid}\n'.encode() - elif cmd == 'echo': - output_buf += f'{payload}\n'.encode() - - def handle_output(event: int) -> None: - nonlocal output_buf - check_event(event, 'Polling of output pipe failed') - if not (event & select.POLLOUT): - return - if output_buf: - n = os.write(stdout_fd, output_buf) - if not n: - raise SystemExit(0) - output_buf = output_buf[n:] - if not output_buf: - poll.unregister(stdout_fd) - - def handle_notify_child_death(event: int) -> None: - nonlocal child_death_buf - check_event(event, 'Polling of notify child death pipe failed') - if not (event & select.POLLOUT): - return - if child_death_buf: - n = os.write(notify_child_death_fd, child_death_buf) - if not n: - raise SystemExit(0) - child_death_buf = child_death_buf[n:] - if not child_death_buf: - poll.unregister(notify_child_death_fd) - - def handle_child_death(dead_child_id: int, dead_child_pid: int) -> None: - nonlocal child_death_buf - xfd = child_ready_fds.pop(dead_child_id, None) - if xfd is not None: - safe_close(xfd) - child_death_buf += f'{dead_child_pid}\n'.encode() - - def handle_signals(event: int) -> None: - check_event(event, 'Polling of signal pipe failed') - if not event & select.POLLIN: - return - - def handle_signal(siginfo: SignalInfo) -> None: - if siginfo.si_signo != signal.SIGCHLD or siginfo.si_code not in (CLD_KILLED, CLD_EXITED, CLD_STOPPED): - return - while True: - try: - pid, status = os.waitpid(-1, os.WNOHANG | os.WUNTRACED) - except ChildProcessError: - pid = 0 - if not pid: - break - child_id = child_pid_map.pop(pid, None) - if child_id is not None: - handle_child_death(child_id, pid) - - read_signals(signal_read_fd, handle_signal) - - keep_type_checker_happy = True - try: - while is_zygote and keep_type_checker_happy: - if output_buf: - poll.register(stdout_fd, select.POLLOUT) - if child_death_buf: - poll.register(notify_child_death_fd, select.POLLOUT) - for (q, event) in poll.poll(): - if q == stdin_fd: - handle_input(event) - elif q == stdout_fd: - handle_output(event) - elif q == signal_read_fd: - handle_signals(event) - elif q == notify_child_death_fd: - handle_notify_child_death(event) - except (KeyboardInterrupt, EOFError, BrokenPipeError): - if is_zygote: - raise SystemExit(1) - raise - except Exception: - if is_zygote: - traceback.print_exc() - raise - finally: - if is_zygote: - restore_python_signal_handlers() - for fmd in child_ready_fds.values(): - with suppress(OSError): - safe_close(fmd) - - -def exec_main(stdin_read: int, stdout_write: int, death_notify_write: int) -> None: - os.setsid() - os.set_inheritable(stdin_read, False) - os.set_inheritable(stdout_write, False) - os.set_inheritable(death_notify_write, False) - running_in_kitty(False) - for x in (sys.stdout, sys.stdin, sys.stderr): - if not x.line_buffering: # happens if the parent kitty instance has stdout not pointing to a terminal - x.reconfigure(line_buffering=True) # type: ignore - try: - main(stdin_read, stdout_write, death_notify_write) - finally: - set_options(None) - - -def fork_prewarm_process(opts: Options, use_exec: bool = False) -> Optional[PrewarmProcess]: - stdin_read, stdin_write = safe_pipe() - stdout_read, stdout_write = safe_pipe() - death_notify_read, death_notify_write = safe_pipe() - if use_exec: - import subprocess - tp = subprocess.Popen( - [kitty_exe(), '+runpy', f'from kitty.prewarm import exec_main; exec_main({stdin_read}, {stdout_write}, {death_notify_write})'], - pass_fds=(stdin_read, stdout_write, death_notify_write)) - child_pid = tp.pid - tp.returncode = 0 # prevent a warning when the popen object is deleted with the process still running - os.set_blocking(stdout_read, True) - os.set_blocking(stdout_read, False) - else: - child_pid = safer_fork() - if child_pid: - # master - safe_close(stdin_read) - safe_close(stdout_write) - safe_close(death_notify_write) - p = PrewarmProcess(child_pid, stdin_write, stdout_read, death_notify_read) - if use_exec: - p.reload_kitty_config() - return p - # child - set_use_os_log(False) - safe_close(stdin_write) - safe_close(stdout_read) - safe_close(death_notify_read) - set_options(opts) - exec_main(stdin_read, stdout_write, death_notify_write) - raise SystemExit(0) diff --git a/kitty_tests/prewarm.py b/kitty_tests/prewarm.py deleted file mode 100644 index a6b5e404b..000000000 --- a/kitty_tests/prewarm.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python -# License: GPLv3 Copyright: 2022, Kovid Goyal - - -import json -import os -import select -import signal -import subprocess -import tempfile -import time - -from kitty.constants import kitty_exe -from kitty.fast_data_types import CLD_EXITED, CLD_KILLED, CLD_STOPPED, get_options, has_sigqueue, install_signal_handlers, read_signals, sigqueue - -from . import BaseTest - - -class Prewarm(BaseTest): - - maxDiff = None - - def test_prewarming(self): - from kitty.prewarm import fork_prewarm_process - - cwd = tempfile.gettempdir() - env = {'TEST_ENV_PASS': 'xyz'} - cols = 317 - stdin_data = 'from_stdin' - pty = self.create_pty(cols=cols) - ttyname = os.ttyname(pty.slave_fd) - opts = get_options() - opts.config_overrides = 'font_family prewarm', - os.environ['SHOULD_NOT_BE_PRESENT'] = '1' - p = fork_prewarm_process(opts, use_exec=True) - del os.environ['SHOULD_NOT_BE_PRESENT'] - if p is None: - return - p.take_from_worker_fd(create_file=True) - child = p(pty.slave_fd, [kitty_exe(), '+runpy', """\ -import os, json; from kitty.utils import *; from kitty.fast_data_types import get_options; print(json.dumps({ - 'cterm': os.ctermid(), - 'ttyname': os.ttyname(sys.stdout.fileno()), - 'cols': read_screen_size().cols, - 'cwd': os.getcwd(), - 'env': os.environ.copy(), - 'pid': os.getpid(), - 'font_family': get_options().font_family, - 'stdin': sys.stdin.read(), - }, indent=2), "ALL_OUTPUT_PRESENT", sep="")"""], cwd=cwd, env=env, stdin_data=stdin_data, timeout=15.0) - self.assertFalse(pty.screen_contents().strip()) - p.mark_child_as_ready(child.child_id) - pty.wait_till(lambda: 'ALL_OUTPUT_PRESENT' in pty.screen_contents()) - data = json.JSONDecoder().raw_decode(pty.screen_contents())[0] - self.ae(data['cols'], cols) - self.assertTrue(data['cterm']) - self.ae(data['ttyname'], ttyname) - self.ae(os.path.realpath(data['cwd']), os.path.realpath(cwd)) - self.ae(data['env']['TEST_ENV_PASS'], env['TEST_ENV_PASS']) - self.assertNotIn('SHOULD_NOT_BE_PRESENT', data['env']) - self.ae(data['font_family'], 'prewarm') - self.ae(int(p.from_worker.readline()), data['pid']) - - def test_signal_handling(self): - from kitty.prewarm import restore_python_signal_handlers, wait_for_child_death - expecting_code = 0 - expecting_signal = signal.SIGCHLD - expecting_value = 0 - found_signal = False - - def handle_signals(signals): - nonlocal found_signal - for siginfo in signals: - if siginfo.si_signo != expecting_signal.value: - continue - if expecting_code is not None: - self.ae(siginfo.si_code, expecting_code) - self.ae(siginfo.sival_int, expecting_value) - if expecting_code in (CLD_EXITED, CLD_KILLED): - p.wait(1) - p.stdin.close() - found_signal = True - - def assert_signal(): - nonlocal found_signal - found_signal = False - st = time.monotonic() - while time.monotonic() - st < 30: - for (fd, event) in poll.poll(10): - if fd == signal_read_fd: - signals = [] - read_signals(signal_read_fd, signals.append) - handle_signals(signals) - if found_signal: - break - self.assertTrue(found_signal, f'Failed to get signal: {expecting_signal!r}') - - def t(signal, q, expecting_sig=signal.SIGCHLD): - nonlocal expecting_code, found_signal, expecting_signal - expecting_code = q - expecting_signal = expecting_sig - if signal is not None: - p.send_signal(signal) - assert_signal() - - poll = select.poll() - - def run(): - return subprocess.Popen([kitty_exe(), '+runpy', 'import sys; sys.stdin.read()'], stderr=subprocess.DEVNULL, stdin=subprocess.PIPE) - p = run() - orig_mask = signal.pthread_sigmask(signal.SIG_BLOCK, ()) - signal_read_fd = install_signal_handlers(signal.SIGCHLD, signal.SIGUSR1)[0] - try: - poll.register(signal_read_fd, select.POLLIN) - t(signal.SIGINT, CLD_KILLED) - p = run() - p.stdin.close() - t(None, CLD_EXITED) - expecting_code = None - expecting_signal = signal.SIGUSR1 - os.kill(os.getpid(), signal.SIGUSR1) - assert_signal() - expecting_value = 17 if has_sigqueue else 0 - sigqueue(os.getpid(), signal.SIGUSR1.value, expecting_value) - assert_signal() - - expecting_code = None - expecting_value = 0 - p = run() - p.send_signal(signal.SIGSTOP) - s = wait_for_child_death(p.pid, options=os.WUNTRACED, timeout=5) - self.assertTrue(os.WIFSTOPPED(s)) - t(None, CLD_STOPPED) - p.send_signal(signal.SIGCONT) - s = wait_for_child_death(p.pid, options=os.WCONTINUED, timeout=5) - self.assertTrue(os.WIFCONTINUED(s)) - # macOS does not send SIGCHLD when child is continued - # https://stackoverflow.com/questions/48487935/sigchld-is-sent-on-sigcont-on-linux-but-not-on-macos - p.stdin.close() - p.wait(3) - for fd, event in poll.poll(0): - read_signals(signal_read_fd, lambda si: None) - finally: - restore_python_signal_handlers() - signal.pthread_sigmask(signal.SIG_SETMASK, orig_mask)