From 0e5ed29d830035a74e9ae7df573a2485c26240b4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 9 Mar 2023 18:59:09 +0530 Subject: [PATCH] Fix generation of url regex for Go --- gen-wcwidth.py | 25 ++++++++++++++----------- kittens/hints/url_regex.py | 3 --- kitty_tests/main.py | 2 +- tools/cmd/hints/marks.go | 2 +- tools/cmd/hints/marks_test.go | 19 +++++++++++++++++++ tools/cmd/hints/url_regex.go | 2 +- tools/utils/misc.go | 4 ++-- 7 files changed, 38 insertions(+), 19 deletions(-) delete mode 100644 kittens/hints/url_regex.py diff --git a/gen-wcwidth.py b/gen-wcwidth.py index 84f7b449d..ad95f15a2 100755 --- a/gen-wcwidth.py +++ b/gen-wcwidth.py @@ -369,19 +369,25 @@ def codepoint_to_mark_map(p: Callable[..., None], mark_map: List[int]) -> Dict[i return rmap -def classes_to_regex(classes: Iterable[str], exclude: str = '') -> Iterable[str]: +def classes_to_regex(classes: Iterable[str], exclude: str = '', for_go: bool = True) -> Iterable[str]: chars: Set[int] = set() for c in classes: chars |= class_maps[c] for x in map(ord, exclude): chars.discard(x) - def as_string(codepoint: int) -> str: - if codepoint < 256: - return fr'\x{codepoint:02x}' - if codepoint <= 0xffff: - return fr'\u{codepoint:04x}' - return fr'\U{codepoint:08x}' + if for_go: + def as_string(codepoint: int) -> str: + if codepoint < 256: + return fr'\x{codepoint:02x}' + return fr'\x{{{codepoint:x}}}' + else: + def as_string(codepoint: int) -> str: + if codepoint < 256: + return fr'\x{codepoint:02x}' + if codepoint <= 0xffff: + return fr'\u{codepoint:04x}' + return fr'\U{codepoint:08x}' for spec in get_ranges(list(chars)): if isinstance(spec, tuple): @@ -439,13 +445,10 @@ def gen_ucd() -> None: f.write(raw) chars = ''.join(classes_to_regex(cz, exclude='\n\r')) - with open('kittens/hints/url_regex.py', 'w') as f: - f.write('# generated by gen-wcwidth.py, do not edit\n\n') - f.write(f"url_delimiters = '{chars}' # noqa") with open('tools/cmd/hints/url_regex.go', 'w') as f: f.write('// generated by gen-wcwidth.py, do not edit\n\n') f.write('package hints\n\n') - f.write(f"const URL_DELIMITERS = `{chars}`") + f.write(f'const URL_DELIMITERS = `{chars}`') def gen_names() -> None: diff --git a/kittens/hints/url_regex.py b/kittens/hints/url_regex.py deleted file mode 100644 index 28086da1b..000000000 --- a/kittens/hints/url_regex.py +++ /dev/null @@ -1,3 +0,0 @@ -# generated by gen-wcwidth.py, do not edit - -url_delimiters = '\x00-\x09\x0b-\x0c\x0e-\x20\x7f-\xa0\xad\u0600-\u0605\u061c\u06dd\u070f\u0890-\u0891\u08e2\u1680\u180e\u2000-\u200f\u2028-\u202f\u205f-\u2064\u2066-\u206f\u3000\ud800-\uf8ff\ufeff\ufff9-\ufffb\U000110bd\U000110cd\U00013430-\U0001343f\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f\U000f0000-\U000ffffd\U00100000-\U0010fffd' # noqa \ No newline at end of file diff --git a/kitty_tests/main.py b/kitty_tests/main.py index 9806580f5..9af9873dd 100644 --- a/kitty_tests/main.py +++ b/kitty_tests/main.py @@ -169,7 +169,7 @@ def run_python_tests(args: Any, go_proc: 'Optional[subprocess.Popen[bytes]]' = N def print_go() -> None: try: - print(go_proc.stdout.read().decode(), end='', flush=True) + print(go_proc.stdout.read().decode('utf-8', 'replace'), end='', flush=True) except KeyboardInterrupt: go_proc.terminate() if go_proc.wait(0.1) is None: diff --git a/tools/cmd/hints/marks.go b/tools/cmd/hints/marks.go index 5c41b2bfa..2a167d134 100644 --- a/tools/cmd/hints/marks.go +++ b/tools/cmd/hints/marks.go @@ -53,7 +53,7 @@ func process_escape_codes(text string) (ans string, hyperlinks []Mark) { idx++ } - ans = utils.ReplaceAll(utils.MustCompile("\x1b(?:\\[[0-9;:]*?m|\\].*?\x1b\\)"), text, func(raw string, groupdict map[string]utils.SubMatch) string { + ans = utils.ReplaceAll(utils.MustCompile("\x1b(?:\\[[0-9;:]*?m|\\].*?\x1b\\\\)"), text, func(raw string, groupdict map[string]utils.SubMatch) string { if !strings.HasPrefix(raw, "\x1b]8") { removed_size += len(raw) return "" diff --git a/tools/cmd/hints/marks_test.go b/tools/cmd/hints/marks_test.go index 67b3dcd2d..93b6b720b 100644 --- a/tools/cmd/hints/marks_test.go +++ b/tools/cmd/hints/marks_test.go @@ -4,10 +4,29 @@ package hints import ( "fmt" + "kitty/tools/utils" "testing" + + "github.com/google/go-cmp/cmp" ) var _ = fmt.Print func TestHintMarking(t *testing.T) { + + opts := &Options{Type: "url"} + r := func(text string, url ...string) { + ptext := convert_text(text, 20) + marks, _, err := find_marks(ptext, opts) + if err != nil { + t.Fatalf("%#v failed with error: %s", text, err) + } + actual := utils.Map(func(m Mark) string { return m.Text }, marks) + if diff := cmp.Diff(url, actual); diff != "" { + t.Fatalf("%#v failed:\n%s", text, diff) + } + } + + u := `http://test.me/` + r(u, u) } diff --git a/tools/cmd/hints/url_regex.go b/tools/cmd/hints/url_regex.go index 527fa5bf3..d637e3234 100644 --- a/tools/cmd/hints/url_regex.go +++ b/tools/cmd/hints/url_regex.go @@ -2,4 +2,4 @@ package hints -const URL_DELIMITERS = `\x00-\x09\x0b-\x0c\x0e-\x20\x7f-\xa0\xad\u0600-\u0605\u061c\u06dd\u070f\u0890-\u0891\u08e2\u1680\u180e\u2000-\u200f\u2028-\u202f\u205f-\u2064\u2066-\u206f\u3000\ud800-\uf8ff\ufeff\ufff9-\ufffb\U000110bd\U000110cd\U00013430-\U0001343f\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f\U000f0000-\U000ffffd\U00100000-\U0010fffd` +const URL_DELIMITERS = `\x00-\x09\x0b-\x0c\x0e-\x20\x7f-\xa0\xad\x{600}-\x{605}\x{61c}\x{6dd}\x{70f}\x{890}-\x{891}\x{8e2}\x{1680}\x{180e}\x{2000}-\x{200f}\x{2028}-\x{202f}\x{205f}-\x{2064}\x{2066}-\x{206f}\x{3000}\x{d800}-\x{f8ff}\x{feff}\x{fff9}-\x{fffb}\x{110bd}\x{110cd}\x{13430}-\x{1343f}\x{1bca0}-\x{1bca3}\x{1d173}-\x{1d17a}\x{e0001}\x{e0020}-\x{e007f}\x{f0000}-\x{ffffd}\x{100000}-\x{10fffd}` \ No newline at end of file diff --git a/tools/utils/misc.go b/tools/utils/misc.go index 39a0d56d4..ad94a40ae 100644 --- a/tools/utils/misc.go +++ b/tools/utils/misc.go @@ -57,8 +57,8 @@ func Filter[T any](s []T, f func(x T) bool) []T { return ans } -func Map[T any](f func(x T) T, s []T) []T { - ans := make([]T, 0, len(s)) +func Map[T any, O any](f func(x T) O, s []T) []O { + ans := make([]O, 0, len(s)) for _, x := range s { ans = append(ans, f(x)) }