Start work on rsync support for file transfers

.github/workflows/ci.py

@@ -36,7 +36,7 @@ def install_deps():
         run('sudo apt-get update')
         run('sudo apt-get install -y libgl1-mesa-dev libxi-dev libxrandr-dev libxinerama-dev ca-certificates'
             ' libxcursor-dev libxcb-xkb-dev libdbus-1-dev libxkbcommon-dev libharfbuzz-dev libx11-xcb-dev'
-            ' libpng-dev liblcms2-dev libfontconfig-dev libxkbcommon-x11-dev libcanberra-dev uuid-dev')
+            ' libpng-dev liblcms2-dev libfontconfig-dev libxkbcommon-x11-dev libcanberra-dev librsync-dev uuid-dev')
     if is_bundle:
         install_bundle()
     else:

Brewfile

@@ -1,5 +1,6 @@
 pkg-config
 zlib
+librsync
 python
 imagemagick
 harfbuzz

bypy/linux/__main__.py

@@ -29,7 +29,7 @@ kitty_constants = iv['kitty_constants']
 def binary_includes():
     return tuple(map(get_dll_path, (
             'expat', 'sqlite3', 'ffi', 'z', 'lzma', 'png16', 'lcms2', 'crypt',
-            'iconv', 'pcre', 'graphite2', 'glib-2.0', 'freetype',
+            'iconv', 'pcre', 'graphite2', 'glib-2.0', 'freetype', 'rsync',
             'harfbuzz', 'xkbcommon', 'xkbcommon-x11',
             'ncursesw', 'readline', 'brotlicommon', 'brotlienc', 'brotlidec'
         ))) + (

bypy/macos/__main__.py

@@ -222,7 +222,7 @@ class Freeze(object):
     @flush
     def get_local_dependencies(self, path_to_lib):
         for x, is_id in self.get_dependencies(path_to_lib):
-            for y in (PREFIX + '/lib/', PREFIX + '/python/Python.framework/'):
+            for y in (PREFIX + '/lib/', PREFIX + '/python/Python.framework/', '@rpath/'):
                 if x.startswith(y):
                     if y == PREFIX + '/python/Python.framework/':
                         y = PREFIX + '/python/'
@@ -288,6 +288,7 @@ class Freeze(object):
                 'lcms2.2',
                 'crypto.1.1',
                 'ssl.1.1',
+                'rsync.2',
         ):
             print('\nAdding', x)
             x = 'lib%s.dylib' % x

bypy/sources.json

@@ -28,6 +28,15 @@
         }
     },
 
+    {
+        "name": "openssl",
+        "unix": {
+            "filename": "openssl-1.1.1i.tar.gz",
+            "hash": "sha256:e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242",
+            "urls": ["https://www.openssl.org/source/{filename}"]
+        }
+    },
+
     {
         "name": "cmake",
         "os": "macos",
@@ -90,15 +99,6 @@
         }
     },
 
-    {
-        "name": "openssl",
-        "unix": {
-            "filename": "openssl-1.1.1i.tar.gz",
-            "hash": "sha256:e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242",
-            "urls": ["https://www.openssl.org/source/{filename}"]
-        }
-    },
-
     {
         "name": "ncurses",
         "os": "linux",
@@ -128,6 +128,16 @@
         }
     },
 
+    {
+        "name": "librsync",
+        "unix": {
+            "filename": "librsync-2.3.2.tar.gz",
+            "hash": "sha256:ef8ce23df38d5076d25510baa2cabedffbe0af460d887d86c2413a1c2b0c676f",
+            "urls": ["https://github.com/librsync/librsync/releases/download/v2.3.2/{filename}"]
+        }
+    },
+
+
     {
         "name": "xcrypt",
         "os": "linux",

docs/build.rst

@@ -34,6 +34,7 @@ Run-time dependencies:
 * ``zlib``
 * ``libpng``
 * ``liblcms2``
+* ``librsync``
 * ``freetype`` (not needed on macOS)
 * ``fontconfig`` (not needed on macOS)
 * ``libcanberra`` (not needed on macOS)

kittens/transfer/rsync.c

@@ -0,0 +1,35 @@
+/*
+ * rsync.c
+ * Copyright (C) 2021 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "data-types.h"
+#include <librsync.h>
+
+static PyMethodDef module_methods[] = {
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+static int
+exec_module(PyObject *m UNUSED) {
+    return 0;
+}
+
+IGNORE_PEDANTIC_WARNINGS
+static PyModuleDef_Slot slots[] = { {Py_mod_exec, (void*)exec_module}, {0, NULL} };
+END_IGNORE_PEDANTIC_WARNINGS
+
+static struct PyModuleDef module = {
+   .m_base = PyModuleDef_HEAD_INIT,
+   .m_name = "rsync",   /* name of module */
+   .m_doc = NULL,
+   .m_slots = slots,
+   .m_methods = module_methods
+};
+
+EXPORTED PyMODINIT_FUNC
+PyInit_rsync(void) {
+	return PyModuleDef_Init(&module);
+}

kitty_tests/check_build.py

@@ -23,7 +23,8 @@ class TestBuild(BaseTest):
         from kittens.choose import subseq_matcher
         from kittens.diff import diff_speedup
         from kittens.unicode_input import unicode_names
-        del fdt, unicode_names, subseq_matcher, diff_speedup
+        from kittens.transfer import rsync
+        del fdt, unicode_names, subseq_matcher, diff_speedup, rsync
 
     def test_loading_shaders(self) -> None:
         from kitty.utils import load_shaders

setup.py

@@ -746,23 +746,28 @@ def compile_kittens(compilation_database: CompilationDatabase) -> None:
             output: str,
             extra_headers: Sequence[str] = (),
             extra_sources: Sequence[str] = (),
-            filter_sources: Optional[Callable[[str], bool]] = None
+            filter_sources: Optional[Callable[[str], bool]] = None,
-    ) -> Tuple[List[str], List[str], str]:
+            cflags: Sequence[str] = (), ldflags: Sequence[str] = (),
+    ) -> Tuple[str, List[str], List[str], str, Sequence[str], Sequence[str]]:
         sources = list(filter(filter_sources, list(extra_sources) + list_files(os.path.join('kittens', kitten, '*.c'))))
         headers = list_files(os.path.join('kittens', kitten, '*.h')) + list(extra_headers)
-        return (sources, headers, 'kittens/{}/{}'.format(kitten, output))
+        return kitten, sources, headers, 'kittens/{}/{}'.format(kitten, output), cflags, ldflags
 
-    for sources, all_headers, dest in (
+    for kitten, sources, all_headers, dest, cflags, ldflags in (
         files('unicode_input', 'unicode_names'),
         files('diff', 'diff_speedup'),
+        files('transfer', 'rsync', ldflags=('-lrsync',)),
         files(
             'choose', 'subseq_matcher',
             extra_headers=('kitty/charsets.h',),
             extra_sources=('kitty/charsets.c',),
             filter_sources=lambda x: 'windows_compat.c' not in x),
     ):
+        final_env = kenv.copy()
+        final_env.cflags.extend(cflags)
+        final_env.ldflags.extend(ldflags)
         compile_c_extension(
-            kenv, dest, compilation_database, sources, all_headers + ['kitty/data-types.h'])
+            final_env, dest, compilation_database, sources, all_headers + ['kitty/data-types.h'])
 
 
 def init_env_from_args(args: Options, native_optimizations: bool = False) -> None:
@@ -1186,7 +1191,7 @@ def option_parser() -> argparse.ArgumentParser:  # {{{
         'action',
         nargs='?',
         default=Options.action,
-        choices='build test linux-package kitty.app linux-freeze macos-freeze build-launcher build-frozen-launcher clean export-ci-bundles'.split(),
+        choices='build test linux-package kitty.app linux-freeze macos-freeze build-launcher build-frozen-launcher clean export-ci-bundles build-dep'.split(),
         help='Action to perform (default is build)'
     )
     p.add_argument(
@@ -1300,8 +1305,41 @@ def option_parser() -> argparse.ArgumentParser:  # {{{
 # }}}
 
 
+def build_dep() -> None:
+    class Options(argparse.Namespace):
+        platform: str
+        deps: List[str]
+
+    p = argparse.ArgumentParser(prog=f'{sys.argv[0]} build-dep', description='Build dependencies for the kitty binary packages')
+    p.add_argument(
+        '--platform',
+        default='all',
+        choices='all macos linux linux-x86'.split(),
+        help='Platforms to build the dep for'
+    )
+    p.add_argument(
+        'deps',
+        nargs='*',
+        default=[],
+        help='Names of the dependencies, if none provided, build all'
+    )
+    args = p.parse_args(sys.argv[2:], namespace=Options)
+    if args.platform == 'all':
+        platforms = ['linux', 'linux 32', 'macos']
+    elif args.platform == 'linux-x86':
+        platforms = ['linux 32']
+    else:
+        platforms = [args.platform]
+    base = [sys.executable, '../bypy']
+    for pf in platforms:
+        cmd = base + pf.split() + args.deps
+        run_tool(cmd)
+
+
 def main() -> None:
     global verbose
+    if len(sys.argv) > 1 and sys.argv[1] == 'build-dep':
+        return build_dep()
     args = option_parser().parse_args(namespace=Options())
     if not is_macos:
         args.build_universal_binary = False