From f0fab80f5b9dec2c8eafc995fb9bdd7818bcb2ce Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 17 Sep 2021 13:41:45 +0530 Subject: [PATCH] Start work on rsync support for file transfers --- .github/workflows/ci.py | 2 +- Brewfile | 1 + bypy/linux/__main__.py | 2 +- bypy/macos/__main__.py | 3 ++- bypy/sources.json | 28 ++++++++++++++------- docs/build.rst | 1 + kittens/transfer/rsync.c | 35 ++++++++++++++++++++++++++ kitty_tests/check_build.py | 3 ++- setup.py | 50 +++++++++++++++++++++++++++++++++----- 9 files changed, 106 insertions(+), 19 deletions(-) create mode 100644 kittens/transfer/rsync.c diff --git a/.github/workflows/ci.py b/.github/workflows/ci.py index f8bc10a3a..d9587b006 100644 --- a/.github/workflows/ci.py +++ b/.github/workflows/ci.py @@ -36,7 +36,7 @@ def install_deps(): run('sudo apt-get update') run('sudo apt-get install -y libgl1-mesa-dev libxi-dev libxrandr-dev libxinerama-dev ca-certificates' ' libxcursor-dev libxcb-xkb-dev libdbus-1-dev libxkbcommon-dev libharfbuzz-dev libx11-xcb-dev' - ' libpng-dev liblcms2-dev libfontconfig-dev libxkbcommon-x11-dev libcanberra-dev uuid-dev') + ' libpng-dev liblcms2-dev libfontconfig-dev libxkbcommon-x11-dev libcanberra-dev librsync-dev uuid-dev') if is_bundle: install_bundle() else: diff --git a/Brewfile b/Brewfile index 11f1e8e21..c43be8725 100644 --- a/Brewfile +++ b/Brewfile @@ -1,5 +1,6 @@ pkg-config zlib +librsync python imagemagick harfbuzz diff --git a/bypy/linux/__main__.py b/bypy/linux/__main__.py index 96b75746c..5fe0cba5a 100644 --- a/bypy/linux/__main__.py +++ b/bypy/linux/__main__.py @@ -29,7 +29,7 @@ kitty_constants = iv['kitty_constants'] def binary_includes(): return tuple(map(get_dll_path, ( 'expat', 'sqlite3', 'ffi', 'z', 'lzma', 'png16', 'lcms2', 'crypt', - 'iconv', 'pcre', 'graphite2', 'glib-2.0', 'freetype', + 'iconv', 'pcre', 'graphite2', 'glib-2.0', 'freetype', 'rsync', 'harfbuzz', 'xkbcommon', 'xkbcommon-x11', 'ncursesw', 'readline', 'brotlicommon', 'brotlienc', 'brotlidec' ))) + ( diff --git a/bypy/macos/__main__.py b/bypy/macos/__main__.py index 4e4ec7131..15e5fbe29 100644 --- a/bypy/macos/__main__.py +++ b/bypy/macos/__main__.py @@ -222,7 +222,7 @@ class Freeze(object): @flush def get_local_dependencies(self, path_to_lib): for x, is_id in self.get_dependencies(path_to_lib): - for y in (PREFIX + '/lib/', PREFIX + '/python/Python.framework/'): + for y in (PREFIX + '/lib/', PREFIX + '/python/Python.framework/', '@rpath/'): if x.startswith(y): if y == PREFIX + '/python/Python.framework/': y = PREFIX + '/python/' @@ -288,6 +288,7 @@ class Freeze(object): 'lcms2.2', 'crypto.1.1', 'ssl.1.1', + 'rsync.2', ): print('\nAdding', x) x = 'lib%s.dylib' % x diff --git a/bypy/sources.json b/bypy/sources.json index 19fe284f1..5a94f16a0 100644 --- a/bypy/sources.json +++ b/bypy/sources.json @@ -28,6 +28,15 @@ } }, + { + "name": "openssl", + "unix": { + "filename": "openssl-1.1.1i.tar.gz", + "hash": "sha256:e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242", + "urls": ["https://www.openssl.org/source/{filename}"] + } + }, + { "name": "cmake", "os": "macos", @@ -90,15 +99,6 @@ } }, - { - "name": "openssl", - "unix": { - "filename": "openssl-1.1.1i.tar.gz", - "hash": "sha256:e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242", - "urls": ["https://www.openssl.org/source/{filename}"] - } - }, - { "name": "ncurses", "os": "linux", @@ -128,6 +128,16 @@ } }, + { + "name": "librsync", + "unix": { + "filename": "librsync-2.3.2.tar.gz", + "hash": "sha256:ef8ce23df38d5076d25510baa2cabedffbe0af460d887d86c2413a1c2b0c676f", + "urls": ["https://github.com/librsync/librsync/releases/download/v2.3.2/{filename}"] + } + }, + + { "name": "xcrypt", "os": "linux", diff --git a/docs/build.rst b/docs/build.rst index a54e191e6..3c9bfbbf8 100644 --- a/docs/build.rst +++ b/docs/build.rst @@ -34,6 +34,7 @@ Run-time dependencies: * ``zlib`` * ``libpng`` * ``liblcms2`` +* ``librsync`` * ``freetype`` (not needed on macOS) * ``fontconfig`` (not needed on macOS) * ``libcanberra`` (not needed on macOS) diff --git a/kittens/transfer/rsync.c b/kittens/transfer/rsync.c new file mode 100644 index 000000000..cbf538725 --- /dev/null +++ b/kittens/transfer/rsync.c @@ -0,0 +1,35 @@ +/* + * rsync.c + * Copyright (C) 2021 Kovid Goyal + * + * Distributed under terms of the GPL3 license. + */ + +#include "data-types.h" +#include + +static PyMethodDef module_methods[] = { + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static int +exec_module(PyObject *m UNUSED) { + return 0; +} + +IGNORE_PEDANTIC_WARNINGS +static PyModuleDef_Slot slots[] = { {Py_mod_exec, (void*)exec_module}, {0, NULL} }; +END_IGNORE_PEDANTIC_WARNINGS + +static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "rsync", /* name of module */ + .m_doc = NULL, + .m_slots = slots, + .m_methods = module_methods +}; + +EXPORTED PyMODINIT_FUNC +PyInit_rsync(void) { + return PyModuleDef_Init(&module); +} diff --git a/kitty_tests/check_build.py b/kitty_tests/check_build.py index 798ef5631..2cdf5d13f 100644 --- a/kitty_tests/check_build.py +++ b/kitty_tests/check_build.py @@ -23,7 +23,8 @@ class TestBuild(BaseTest): from kittens.choose import subseq_matcher from kittens.diff import diff_speedup from kittens.unicode_input import unicode_names - del fdt, unicode_names, subseq_matcher, diff_speedup + from kittens.transfer import rsync + del fdt, unicode_names, subseq_matcher, diff_speedup, rsync def test_loading_shaders(self) -> None: from kitty.utils import load_shaders diff --git a/setup.py b/setup.py index b737c3c71..fcce729fc 100755 --- a/setup.py +++ b/setup.py @@ -746,23 +746,28 @@ def compile_kittens(compilation_database: CompilationDatabase) -> None: output: str, extra_headers: Sequence[str] = (), extra_sources: Sequence[str] = (), - filter_sources: Optional[Callable[[str], bool]] = None - ) -> Tuple[List[str], List[str], str]: + filter_sources: Optional[Callable[[str], bool]] = None, + cflags: Sequence[str] = (), ldflags: Sequence[str] = (), + ) -> Tuple[str, List[str], List[str], str, Sequence[str], Sequence[str]]: sources = list(filter(filter_sources, list(extra_sources) + list_files(os.path.join('kittens', kitten, '*.c')))) headers = list_files(os.path.join('kittens', kitten, '*.h')) + list(extra_headers) - return (sources, headers, 'kittens/{}/{}'.format(kitten, output)) + return kitten, sources, headers, 'kittens/{}/{}'.format(kitten, output), cflags, ldflags - for sources, all_headers, dest in ( + for kitten, sources, all_headers, dest, cflags, ldflags in ( files('unicode_input', 'unicode_names'), files('diff', 'diff_speedup'), + files('transfer', 'rsync', ldflags=('-lrsync',)), files( 'choose', 'subseq_matcher', extra_headers=('kitty/charsets.h',), extra_sources=('kitty/charsets.c',), filter_sources=lambda x: 'windows_compat.c' not in x), ): + final_env = kenv.copy() + final_env.cflags.extend(cflags) + final_env.ldflags.extend(ldflags) compile_c_extension( - kenv, dest, compilation_database, sources, all_headers + ['kitty/data-types.h']) + final_env, dest, compilation_database, sources, all_headers + ['kitty/data-types.h']) def init_env_from_args(args: Options, native_optimizations: bool = False) -> None: @@ -1186,7 +1191,7 @@ def option_parser() -> argparse.ArgumentParser: # {{{ 'action', nargs='?', default=Options.action, - choices='build test linux-package kitty.app linux-freeze macos-freeze build-launcher build-frozen-launcher clean export-ci-bundles'.split(), + choices='build test linux-package kitty.app linux-freeze macos-freeze build-launcher build-frozen-launcher clean export-ci-bundles build-dep'.split(), help='Action to perform (default is build)' ) p.add_argument( @@ -1300,8 +1305,41 @@ def option_parser() -> argparse.ArgumentParser: # {{{ # }}} +def build_dep() -> None: + class Options(argparse.Namespace): + platform: str + deps: List[str] + + p = argparse.ArgumentParser(prog=f'{sys.argv[0]} build-dep', description='Build dependencies for the kitty binary packages') + p.add_argument( + '--platform', + default='all', + choices='all macos linux linux-x86'.split(), + help='Platforms to build the dep for' + ) + p.add_argument( + 'deps', + nargs='*', + default=[], + help='Names of the dependencies, if none provided, build all' + ) + args = p.parse_args(sys.argv[2:], namespace=Options) + if args.platform == 'all': + platforms = ['linux', 'linux 32', 'macos'] + elif args.platform == 'linux-x86': + platforms = ['linux 32'] + else: + platforms = [args.platform] + base = [sys.executable, '../bypy'] + for pf in platforms: + cmd = base + pf.split() + args.deps + run_tool(cmd) + + def main() -> None: global verbose + if len(sys.argv) > 1 and sys.argv[1] == 'build-dep': + return build_dep() args = option_parser().parse_args(namespace=Options()) if not is_macos: args.build_universal_binary = False