From fbf1ec43c76e7cf7f4cf6b20c122928a2d15ad3b Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sun, 5 Jun 2022 00:02:13 +0200 Subject: [PATCH 1/4] diff kitten: add options to ignore paths when comparing directories Tested locally & over SSH: $ kitty +kitten diff /local/path /local/path2 $ kitty +kitten diff /local/path ssh:remote:/path --- kittens/diff/collect.py | 8 ++++++++ kittens/diff/main.py | 2 ++ kittens/diff/options/definition.py | 11 +++++++++++ kittens/diff/options/parse.py | 8 +++++++- kittens/diff/options/types.py | 4 ++++ kittens/diff/options/utils.py | 6 +++++- 6 files changed, 37 insertions(+), 2 deletions(-) mode change 100644 => 100755 kittens/diff/collect.py mode change 100644 => 100755 kittens/diff/main.py mode change 100644 => 100755 kittens/diff/options/definition.py mode change 100644 => 100755 kittens/diff/options/utils.py diff --git a/kittens/diff/collect.py b/kittens/diff/collect.py old mode 100644 new mode 100755 index 104f8ee11..0140a2c5d --- a/kittens/diff/collect.py +++ b/kittens/diff/collect.py @@ -4,6 +4,7 @@ import os import re from contextlib import suppress +from fnmatch import fnmatch from functools import lru_cache from hashlib import md5 from kitty.guess_mime_type import guess_type @@ -37,6 +38,9 @@ class Segment: class Collection: + file_ignores = [''] + ignore_paths = [''] + def __init__(self) -> None: self.changes: Dict[str, str] = {} self.renames: Dict[str, str] = {} @@ -113,7 +117,11 @@ def collect_files(collection: Collection, left: str, right: str) -> None: def walk(base: str, names: Set[str], pmap: Dict[str, str]) -> None: for dirpath, dirnames, filenames in os.walk(base): + if any(fnmatch(dirpath, f"*/{pat}") for pat in collection.ignore_paths): + continue for filename in filenames: + if any(fnmatch(filename, f"{pat}") for pat in collection.file_ignores): + continue path = os.path.abspath(os.path.join(dirpath, filename)) path_name_map[path] = name = os.path.relpath(path, base) names.add(name) diff --git a/kittens/diff/main.py b/kittens/diff/main.py old mode 100644 new mode 100755 index 6d3097960..1ff51b3b2 --- a/kittens/diff/main.py +++ b/kittens/diff/main.py @@ -655,6 +655,8 @@ def main(args: List[str]) -> None: opts = init_config(cli_opts) set_diff_command(opts.diff_cmd) lines_for_path.replace_tab_by = opts.replace_tab_by + Collection.file_ignores = opts.file_ignores + Collection.ignore_paths = opts.ignore_paths left, right = map(get_remote_file, (left, right)) if os.path.isdir(left) != os.path.isdir(right): raise SystemExit('The items to be diffed should both be either directories or files. Comparing a directory to a file is not valid.') diff --git a/kittens/diff/options/definition.py b/kittens/diff/options/definition.py old mode 100644 new mode 100755 index 3b8529fc6..041b63024 --- a/kittens/diff/options/definition.py +++ b/kittens/diff/options/definition.py @@ -47,6 +47,17 @@ opt('replace_tab_by', '\\x20\\x20\\x20\\x20', option_type='python_string', long_text='The string to replace tabs with. Default is to use four spaces.' ) + +opt('file_ignores', '', + option_type='pattern_list', + long_text='List of file patterns that are ignored when directories are compared' + ) + +opt('ignore_paths', '', + option_type='pattern_list', + long_text='List of directory patterns that are ignored when directories are compared' + ) + egr() # }}} # colors {{{ diff --git a/kittens/diff/options/parse.py b/kittens/diff/options/parse.py index 6ba056c03..3ea9f1a23 100644 --- a/kittens/diff/options/parse.py +++ b/kittens/diff/options/parse.py @@ -1,7 +1,7 @@ # generated by gen-config.py DO NOT edit import typing -from kittens.diff.options.utils import parse_map, syntax_aliases +from kittens.diff.options.utils import parse_map, pattern_list, syntax_aliases from kitty.conf.utils import merge_dicts, positive_int, python_string, to_color, to_color_or_none @@ -86,6 +86,12 @@ class Parser: for k in parse_map(val): ans['map'].append(k) + def file_ignores(self, val: str, ans: typing.Dict[str, typing.List[str]]): + ans['file_ignores'] = pattern_list(val) + + def ignore_paths(self, val: str, ans: typing.Dict[str, typing.List[str]]): + ans['ignore_paths'] = pattern_list(val) + def create_result_dict() -> typing.Dict[str, typing.Any]: return { diff --git a/kittens/diff/options/types.py b/kittens/diff/options/types.py index 9a7bccec5..e3abc68ad 100644 --- a/kittens/diff/options/types.py +++ b/kittens/diff/options/types.py @@ -14,12 +14,14 @@ option_names = ( # {{{ 'added_margin_bg', 'background', 'diff_cmd', + 'file_ignores', 'filler_bg', 'foreground', 'highlight_added_bg', 'highlight_removed_bg', 'hunk_bg', 'hunk_margin_bg', + 'ignore_paths', 'map', 'margin_bg', 'margin_fg', @@ -43,12 +45,14 @@ class Options: added_margin_bg: Color = Color(205, 255, 216) background: Color = Color(255, 255, 255) diff_cmd: str = 'auto' + file_ignores: typing.List[str] = [''] filler_bg: Color = Color(250, 251, 252) foreground: Color = Color(0, 0, 0) highlight_added_bg: Color = Color(172, 242, 189) highlight_removed_bg: Color = Color(253, 184, 192) hunk_bg: Color = Color(241, 248, 255) hunk_margin_bg: Color = Color(219, 237, 255) + ignore_paths: typing.List[str] = [''] margin_bg: Color = Color(250, 251, 252) margin_fg: Color = Color(170, 170, 170) margin_filler_bg: typing.Optional[kitty.fast_data_types.Color] = None diff --git a/kittens/diff/options/utils.py b/kittens/diff/options/utils.py old mode 100644 new mode 100755 index 0e88b3bc4..000d1edd6 --- a/kittens/diff/options/utils.py +++ b/kittens/diff/options/utils.py @@ -3,7 +3,7 @@ # License: GPLv3 Copyright: 2021, Kovid Goyal -from typing import Any, Dict, Iterable, Tuple, Union +from typing import Any, Dict, Iterable, List, Tuple, Union from kitty.conf.utils import ( KeyFuncWrapper, KittensKeyDefinition, parse_kittens_key @@ -58,6 +58,10 @@ def syntax_aliases(raw: str) -> Dict[str, str]: return ans +def pattern_list(raw: str) -> List[str]: + return [pat for pat in raw.split(' ') if pat] + + def parse_map(val: str) -> Iterable[KittensKeyDefinition]: x = parse_kittens_key(val, func_with_args.args_funcs) if x is not None: From eea652f1d0693da5af2581b8df150fcf46f234d4 Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Sun, 5 Jun 2022 10:10:17 +0200 Subject: [PATCH 2/4] kittens/diff: move empty pattern check to dir tree walk --- kittens/diff/collect.py | 4 ++-- kittens/diff/options/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kittens/diff/collect.py b/kittens/diff/collect.py index 0140a2c5d..0a62b1127 100755 --- a/kittens/diff/collect.py +++ b/kittens/diff/collect.py @@ -117,10 +117,10 @@ def collect_files(collection: Collection, left: str, right: str) -> None: def walk(base: str, names: Set[str], pmap: Dict[str, str]) -> None: for dirpath, dirnames, filenames in os.walk(base): - if any(fnmatch(dirpath, f"*/{pat}") for pat in collection.ignore_paths): + if any(fnmatch(dirpath, f"*/{pat}") for pat in collection.ignore_paths if pat): continue for filename in filenames: - if any(fnmatch(filename, f"{pat}") for pat in collection.file_ignores): + if any(fnmatch(filename, f"{pat}") for pat in collection.file_ignores if pat): continue path = os.path.abspath(os.path.join(dirpath, filename)) path_name_map[path] = name = os.path.relpath(path, base) diff --git a/kittens/diff/options/utils.py b/kittens/diff/options/utils.py index 000d1edd6..e52e5efed 100755 --- a/kittens/diff/options/utils.py +++ b/kittens/diff/options/utils.py @@ -59,7 +59,7 @@ def syntax_aliases(raw: str) -> Dict[str, str]: def pattern_list(raw: str) -> List[str]: - return [pat for pat in raw.split(' ') if pat] + return raw.split(' ') def parse_map(val: str) -> Iterable[KittensKeyDefinition]: From 20b6a97159d81fb63922b373d08bee49d1fa7b7f Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Mon, 6 Jun 2022 09:58:18 +0200 Subject: [PATCH 3/4] Update in response to feedback - one configuration option: ignore_paths - use shlex to parse option to support whitespace and in-line comments - change option type to Tuple[str, ...] - remove ignored directories from dirnames to prevent scanning --- kittens/diff/collect.py | 33 ++++++++++++++++-------------- kittens/diff/main.py | 1 - kittens/diff/options/definition.py | 9 +++----- kittens/diff/options/parse.py | 5 +---- kittens/diff/options/types.py | 4 +--- kittens/diff/options/utils.py | 5 +++-- 6 files changed, 26 insertions(+), 31 deletions(-) diff --git a/kittens/diff/collect.py b/kittens/diff/collect.py index 0a62b1127..bda96cc1d 100755 --- a/kittens/diff/collect.py +++ b/kittens/diff/collect.py @@ -7,6 +7,7 @@ from contextlib import suppress from fnmatch import fnmatch from functools import lru_cache from hashlib import md5 +from itertools import product from kitty.guess_mime_type import guess_type from typing import TYPE_CHECKING, Dict, List, Set, Optional, Iterator, Tuple, Union @@ -38,8 +39,7 @@ class Segment: class Collection: - file_ignores = [''] - ignore_paths = [''] + ignore_paths: Tuple[str, ...] = () def __init__(self) -> None: self.changes: Dict[str, str] = {} @@ -109,26 +109,29 @@ def resolve_remote_name(path: str, default: str) -> str: return default +def walk(base: str, names: Set[str], pmap: Dict[str, str], ignore_paths: Tuple[str, ...]) -> None: + for dirpath, dirnames, filenames in os.walk(base): + ignored = [_dir for _dir, pat in product(dirnames, ignore_paths) if fnmatch(_dir, pat)] + for _dir in ignored: + dirnames.pop(dirnames.index(_dir)) + for filename in filenames: + if any(fnmatch(filename, pat) for pat in ignore_paths if pat): + continue + path = os.path.abspath(os.path.join(dirpath, filename)) + path_name_map[path] = name = os.path.relpath(path, base) + names.add(name) + pmap[name] = path + + def collect_files(collection: Collection, left: str, right: str) -> None: left_names: Set[str] = set() right_names: Set[str] = set() left_path_map: Dict[str, str] = {} right_path_map: Dict[str, str] = {} - def walk(base: str, names: Set[str], pmap: Dict[str, str]) -> None: - for dirpath, dirnames, filenames in os.walk(base): - if any(fnmatch(dirpath, f"*/{pat}") for pat in collection.ignore_paths if pat): - continue - for filename in filenames: - if any(fnmatch(filename, f"{pat}") for pat in collection.file_ignores if pat): - continue - path = os.path.abspath(os.path.join(dirpath, filename)) - path_name_map[path] = name = os.path.relpath(path, base) - names.add(name) - pmap[name] = path + walk(left, left_names, left_path_map, collection.ignore_paths) + walk(right, right_names, right_path_map, collection.ignore_paths) - walk(left, left_names, left_path_map) - walk(right, right_names, right_path_map) common_names = left_names & right_names changed_names = {n for n in common_names if data_for_path(left_path_map[n]) != data_for_path(right_path_map[n])} for n in changed_names: diff --git a/kittens/diff/main.py b/kittens/diff/main.py index 1ff51b3b2..f0e4676ba 100755 --- a/kittens/diff/main.py +++ b/kittens/diff/main.py @@ -655,7 +655,6 @@ def main(args: List[str]) -> None: opts = init_config(cli_opts) set_diff_command(opts.diff_cmd) lines_for_path.replace_tab_by = opts.replace_tab_by - Collection.file_ignores = opts.file_ignores Collection.ignore_paths = opts.ignore_paths left, right = map(get_remote_file, (left, right)) if os.path.isdir(left) != os.path.isdir(right): diff --git a/kittens/diff/options/definition.py b/kittens/diff/options/definition.py index 041b63024..9a7d2a669 100755 --- a/kittens/diff/options/definition.py +++ b/kittens/diff/options/definition.py @@ -48,14 +48,11 @@ opt('replace_tab_by', '\\x20\\x20\\x20\\x20', long_text='The string to replace tabs with. Default is to use four spaces.' ) -opt('file_ignores', '', - option_type='pattern_list', - long_text='List of file patterns that are ignored when directories are compared' - ) - opt('ignore_paths', '', option_type='pattern_list', - long_text='List of directory patterns that are ignored when directories are compared' + long_text='''List of patterns that are matched against directory and file +names and ignored when directories are compared. +''', ) egr() # }}} diff --git a/kittens/diff/options/parse.py b/kittens/diff/options/parse.py index 3ea9f1a23..e4ffa26b1 100644 --- a/kittens/diff/options/parse.py +++ b/kittens/diff/options/parse.py @@ -86,10 +86,7 @@ class Parser: for k in parse_map(val): ans['map'].append(k) - def file_ignores(self, val: str, ans: typing.Dict[str, typing.List[str]]): - ans['file_ignores'] = pattern_list(val) - - def ignore_paths(self, val: str, ans: typing.Dict[str, typing.List[str]]): + def ignore_paths(self, val: str, ans: typing.Dict[str, typing.Any]): ans['ignore_paths'] = pattern_list(val) diff --git a/kittens/diff/options/types.py b/kittens/diff/options/types.py index e3abc68ad..f265c8a95 100644 --- a/kittens/diff/options/types.py +++ b/kittens/diff/options/types.py @@ -14,7 +14,6 @@ option_names = ( # {{{ 'added_margin_bg', 'background', 'diff_cmd', - 'file_ignores', 'filler_bg', 'foreground', 'highlight_added_bg', @@ -45,14 +44,13 @@ class Options: added_margin_bg: Color = Color(205, 255, 216) background: Color = Color(255, 255, 255) diff_cmd: str = 'auto' - file_ignores: typing.List[str] = [''] filler_bg: Color = Color(250, 251, 252) foreground: Color = Color(0, 0, 0) highlight_added_bg: Color = Color(172, 242, 189) highlight_removed_bg: Color = Color(253, 184, 192) hunk_bg: Color = Color(241, 248, 255) hunk_margin_bg: Color = Color(219, 237, 255) - ignore_paths: typing.List[str] = [''] + ignore_paths: typing.Tuple[str, ...] = () margin_bg: Color = Color(250, 251, 252) margin_fg: Color = Color(170, 170, 170) margin_filler_bg: typing.Optional[kitty.fast_data_types.Color] = None diff --git a/kittens/diff/options/utils.py b/kittens/diff/options/utils.py index e52e5efed..4777306bf 100755 --- a/kittens/diff/options/utils.py +++ b/kittens/diff/options/utils.py @@ -3,6 +3,7 @@ # License: GPLv3 Copyright: 2021, Kovid Goyal +import shlex from typing import Any, Dict, Iterable, List, Tuple, Union from kitty.conf.utils import ( @@ -58,8 +59,8 @@ def syntax_aliases(raw: str) -> Dict[str, str]: return ans -def pattern_list(raw: str) -> List[str]: - return raw.split(' ') +def pattern_list(raw: str) -> Tuple[str, ...]: + return tuple(shlex.split(raw, comments=True)) def parse_map(val: str) -> Iterable[KittensKeyDefinition]: From 38cb18fe922717b21d218af05996a48d5101d7cc Mon Sep 17 00:00:00 2001 From: Suvayu Ali Date: Mon, 6 Jun 2022 11:01:45 +0200 Subject: [PATCH 4/4] diff kitten: tests for directory walking --- kitty_tests/diff.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/kitty_tests/diff.py b/kitty_tests/diff.py index 37bee2e00..7578854e2 100644 --- a/kitty_tests/diff.py +++ b/kitty_tests/diff.py @@ -43,3 +43,43 @@ class TestDiff(BaseTest): highlights = [h(0, 1, 1), h(1, 3, 2)] self.ae(['S1SaE1ES2SbcE2Ed'], split_with_highlights('abcd', 10, highlights)) + + def test_walk(self): + from pathlib import Path + import tempfile + from kittens.diff.collect import walk + + with tempfile.TemporaryDirectory() as tmpdir: + # /tmp/test/ + # ├── a + # │ └── b + # │ └── c + # ├── d + # ├── #d# + # ├── e + # ├── e~ + # └── f + # │ └── g + # └── h space + Path(tmpdir, "a/b").mkdir(parents=True) + Path(tmpdir, "a/b/c").touch() + Path(tmpdir, "b").touch() + Path(tmpdir, "d").touch() + Path(tmpdir, "#d#").touch() + Path(tmpdir, "e").touch() + Path(tmpdir, "e~").touch() + Path(tmpdir, "f").mkdir() + Path(tmpdir, "f/g").touch() + Path(tmpdir, "h space").touch() + expected_names = {"d", "e", "f/g", "h space"} + expected_pmap = { + "d": f"{tmpdir}/d", + "e": f"{tmpdir}/e", + "f/g": f"{tmpdir}/f/g", + "h space": f"{tmpdir}/h space" + } + names = set() + pmap = {} + walk(tmpdir, names, pmap, ("*~", "#*#", "b")) + self.ae(expected_names, names) + self.ae(expected_pmap, pmap)