Port custom processor for hints

docs/kittens/hints.rst

@@ -72,7 +72,8 @@ the :ref:`kitty config directory <confloc>` with the following contents:
             start, end = m.span()
             mark_text = text[start:end].replace('\n', '').replace('\0', '')
             # The empty dictionary below will be available as groupdicts
-            # in handle_result() and can contain arbitrary data.
+            # in handle_result() and can contain string keys and arbitrary JSON
+            # serializable values.
             yield Mark(idx, start, end, mark_text, {})
 
 

kittens/hints/main.py

@@ -5,6 +5,7 @@ import sys
 from functools import lru_cache
 from typing import Any, Dict, List, Optional, Sequence, Tuple
 
+from kitty.cli_stub import HintsCLIOptions
 from kitty.clipboard import set_clipboard_string, set_primary_selection
 from kitty.constants import website_url
 from kitty.fast_data_types import get_options
@@ -26,6 +27,48 @@ def load_custom_processor(customize_processing: str) -> Any:
     import runpy
     return runpy.run_path(custom_path, run_name='__main__')
 
+class Mark:
+
+    __slots__ = ('index', 'start', 'end', 'text', 'is_hyperlink', 'group_id', 'groupdict')
+
+    def __init__(
+            self,
+            index: int, start: int, end: int,
+            text: str,
+            groupdict: Any,
+            is_hyperlink: bool = False,
+            group_id: Optional[str] = None
+    ):
+        self.index, self.start, self.end = index, start, end
+        self.text = text
+        self.groupdict = groupdict
+        self.is_hyperlink = is_hyperlink
+        self.group_id = group_id
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            'index': self.index, 'start': self.start, 'end': self.end,
+            'text': self.text, 'groupdict': {str(k):v for k, v in (self.groupdict or {}).items()},
+            'group_id': self.group_id or '', 'is_hyperlink': self.is_hyperlink
+        }
+
+
+def parse_hints_args(args: List[str]) -> Tuple[HintsCLIOptions, List[str]]:
+    from kitty.cli import parse_args
+    return parse_args(args, OPTIONS, usage, help_text, 'kitty +kitten hints', result_class=HintsCLIOptions)
+
+
+def custom_marking() -> None:
+    import json
+    text = sys.stdin.read()
+    sys.stdin.close()
+    opts, extra_cli_args = parse_hints_args(sys.argv[1:])
+    m = load_custom_processor(opts.customize_processing or '::impossible::')
+    if 'mark' not in m:
+        raise SystemExit(2)
+    all_marks = tuple(x.as_dict() for x in m['mark'](text, opts, Mark, extra_cli_args))
+    sys.stdout.write(json.dumps(all_marks))
+    raise SystemExit(0)
 
 
 # CLI {{{

kitty_tests/main.py

@@ -146,8 +146,11 @@ def run_go(packages: Set[str], names: str) -> 'subprocess.Popen[bytes]':
     for name in names:
         cmd.extend(('-run', name))
     cmd += go_pkg_args
+    env = os.environ.copy()
+    from kitty.constants import kitty_exe
+    env['KITTY_PATH_TO_KITTY_EXE'] = kitty_exe()
     print(shlex.join(cmd), flush=True)
-    return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
 
 
 def reduce_go_pkgs(module: str, names: Sequence[str]) -> Set[str]:

tools/cmd/hints/main.go

@@ -81,7 +81,7 @@ type Result struct {
 	Multiple_joiner      string           `json:"multiple_joiner"`
 	Customize_processing string           `json:"customize_processing"`
 	Type                 string           `json:"type"`
-	Groupdicts           []map[string]string `json:"groupdicts"`
+	Groupdicts           []map[string]any `json:"groupdicts"`
 	Extra_cli_args       []string         `json:"extra_cli_args"`
 	Linenum_action       string           `json:"linenum_action"`
 	Cwd                  string           `json:"cwd"`
@@ -125,7 +125,7 @@ func main(_ *cli.Command, o *Options, args []string) (rc int, err error) {
 		return 1, nil
 	}
 	input_text := parse_input(utils.UnsafeBytesToString(stdin))
-	text, all_marks, index_map, err := find_marks(input_text, o)
+	text, all_marks, index_map, err := find_marks(input_text, o, os.Args[2:]...)
 	if err != nil {
 		tui.ReportError(err)
 		return 1, nil
@@ -313,7 +313,7 @@ func main(_ *cli.Command, o *Options, args []string) (rc int, err error) {
 		return lp.ExitCode(), nil
 	}
 	result.Match = make([]string, len(chosen))
-	result.Groupdicts = make([]map[string]string, len(chosen))
+	result.Groupdicts = make([]map[string]any, len(chosen))
 	for i, m := range chosen {
 		result.Match[i] = m.Text + match_suffix
 		result.Groupdicts[i] = m.Groupdict

tools/cmd/hints/marks.go

@@ -3,10 +3,14 @@
 package hints
 
 import (
+	"bytes"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"kitty"
 	"kitty/tools/config"
 	"kitty/tools/utils"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -32,10 +36,13 @@ func default_linenum_regex() string {
 }
 
 type Mark struct {
-	Index, Start, End int
+	Index        int            `json:"index"`
-	Text, Group_id    string
+	Start        int            `json:"start"`
-	Is_hyperlink      bool
+	End          int            `json:"end"`
-	Groupdict         map[string]string
+	Text         string         `json:"text"`
+	Group_id     string         `json:"group_id"`
+	Is_hyperlink bool           `json:"is_hyperlink"`
+	Groupdict    map[string]any `json:"groupdict"`
 }
 
 func process_escape_codes(text string) (ans string, hyperlinks []Mark) {
@@ -302,8 +309,12 @@ func mark(r *regexp.Regexp, post_processors []PostProcessorFunc, group_processor
 		for _, f := range group_processors {
 			f(gd)
 		}
+		gd2 := make(map[string]any, len(gd))
+		for k, v := range gd {
+			gd2[k] = v
+		}
 		ans = append(ans, Mark{
-			Index: i, Start: match_start, End: match_end, Text: full_match, Groupdict: gd,
+			Index: i, Start: match_start, End: match_end, Text: full_match, Groupdict: gd2,
 		})
 	}
 	return
@@ -322,18 +333,51 @@ func (self *ErrNoMatches) Error() string {
 	return fmt.Sprintf("No %s found", none_of)
 }
 
-func find_marks(text string, opts *Options) (sanitized_text string, ans []Mark, index_map map[int]*Mark, err error) {
+func find_marks(text string, opts *Options, cli_args ...string) (sanitized_text string, ans []Mark, index_map map[int]*Mark, err error) {
 	sanitized_text, hyperlinks := process_escape_codes(text)
+
+	run_basic_matching := func() error {
 		pattern, post_processors, group_processors := functions_for(opts)
-	if opts.Type == "hyperlink" {
-		ans = hyperlinks
-	} else {
 		r, err := regexp.Compile(pattern)
 		if err != nil {
-			return "", nil, nil, fmt.Errorf("Failed to compile the regex pattern: %#v with error: %w", pattern, err)
+			return fmt.Errorf("Failed to compile the regex pattern: %#v with error: %w", pattern, err)
 		}
 		ans = mark(r, post_processors, group_processors, sanitized_text, opts)
+		return nil
 	}
+
+	if opts.CustomizeProcessing != "" {
+		cmd := exec.Command(utils.KittyExe(), append([]string{"+runpy", "from kittens.hints.main import custom_marking; custom_marking()"}, cli_args...)...)
+		cmd.Stdin = strings.NewReader(sanitized_text)
+		stdout, stderr := bytes.Buffer{}, bytes.Buffer{}
+		cmd.Stdout, cmd.Stderr = &stdout, &stderr
+		err = cmd.Run()
+		if err != nil {
+			var e *exec.ExitError
+			if errors.As(err, &e) && e.ExitCode() == 2 {
+				err = run_basic_matching()
+				if err != nil {
+					return
+				}
+				goto process_answer
+			} else {
+				return "", nil, nil, fmt.Errorf("Failed to run custom processor %#v with error: %w\n%s", opts.CustomizeProcessing, err, stderr.String())
+			}
+		}
+		ans = make([]Mark, 0, 32)
+		err = json.Unmarshal(stdout.Bytes(), &ans)
+		if err != nil {
+			return "", nil, nil, fmt.Errorf("Failed to load output from custom processor %#v with error: %w", opts.CustomizeProcessing, err)
+		}
+	} else if opts.Type == "hyperlink" {
+		ans = hyperlinks
+	} else {
+		err = run_basic_matching()
+		if err != nil {
+			return
+		}
+	}
+process_answer:
 	if len(ans) == 0 {
 		return "", nil, nil, &ErrNoMatches{Type: opts.Type}
 	}

tools/cmd/hints/marks_test.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"kitty"
 	"kitty/tools/utils"
+	"os"
+	"path/filepath"
 	"strconv"
 	"testing"
 
@@ -17,11 +19,19 @@ var _ = fmt.Print
 
 func TestHintMarking(t *testing.T) {
 
-	opts := &Options{Type: "url", UrlPrefixes: "default", Regex: kitty.HintsDefaultRegex}
+	var opts *Options
 	cols := 20
-	r := func(text string, url ...string) {
+	cli_args := []string{}
+
+	reset := func() {
+		opts = &Options{Type: "url", UrlPrefixes: "default", Regex: kitty.HintsDefaultRegex}
+		cols = 20
+		cli_args = []string{}
+	}
+
+	r := func(text string, url ...string) (marks []Mark) {
 		ptext := convert_text(text, cols)
-		_, marks, _, err := find_marks(ptext, opts)
+		_, marks, _, err := find_marks(ptext, opts, cli_args...)
 		if err != nil {
 			var e *ErrNoMatches
 			if len(url) != 0 || !errors.As(err, &e) {
@@ -33,8 +43,10 @@ func TestHintMarking(t *testing.T) {
 		if diff := cmp.Diff(url, actual); diff != "" {
 			t.Fatalf("%#v failed:\n%s", text, diff)
 		}
+		return
 	}
 
+	reset()
 	u := `http://test.me/`
 	r(u, u)
 	r(`"`+u+`"`, u)
@@ -51,11 +63,11 @@ func TestHintMarking(t *testing.T) {
 	opts.Type = "linenum"
 	m := func(text, path string, line int) {
 		ptext := convert_text(text, cols)
-		_, marks, _, err := find_marks(ptext, opts)
+		_, marks, _, err := find_marks(ptext, opts, cli_args...)
 		if err != nil {
 			t.Fatalf("%#v failed with error: %s", text, err)
 		}
-		gd := map[string]string{"path": path, "line": strconv.Itoa(line)}
+		gd := map[string]any{"path": path, "line": strconv.Itoa(line)}
 		if diff := cmp.Diff(marks[0].Groupdict, gd); diff != "" {
 			t.Fatalf("%#v failed:\n%s", text, diff)
 		}
@@ -67,6 +79,7 @@ func TestHintMarking(t *testing.T) {
 	m("a/file.c:23:32", "a/file.c", 23)
 	m("~/file.c:23:32", utils.Expanduser("~/file.c"), 23)
 
+	reset()
 	opts.Type = "path"
 	r("file.c", "file.c")
 	r("file.c.", "file.c")
@@ -74,6 +87,7 @@ func TestHintMarking(t *testing.T) {
 	r("(file.epub)", "file.epub")
 	r("some/path", "some/path")
 
+	reset()
 	cols = 60
 	opts.Type = "ip"
 	r(`100.64.0.0`, `100.64.0.0`)
@@ -86,4 +100,25 @@ func TestHintMarking(t *testing.T) {
 	r(`255.255.255.256`)
 	r(`:1`)
 
+	reset()
+	tdir := t.TempDir()
+	simple := filepath.Join(tdir, "simple.py")
+	cli_args = []string{"--customize-processing", simple, "extra1"}
+	os.WriteFile(simple, []byte(`
+def mark(text, args, Mark, extra_cli_args, *a):
+    import re
+    for idx, m in enumerate(re.finditer(r'\w+', text)):
+        start, end = m.span()
+        mark_text = text[start:end].replace('\n', '').replace('\0', '')
+        yield Mark(idx, start, end, mark_text, {"idx": idx, "args": extra_cli_args})
+`), 0o600)
+	opts.Type = "regex"
+	opts.CustomizeProcessing = simple
+	marks := r("a b", `a`, `b`)
+	if diff := cmp.Diff(marks[0].Groupdict, map[string]any{"idx": float64(0), "args": []any{"extra1"}}); diff != "" {
+		t.Fatalf("Did not get expected groupdict from custom processor:\n%s", diff)
+	}
+	opts.Regex = "b"
+	os.WriteFile(simple, []byte(""), 0o600)
+	r("a b", `b`)
 }

tools/utils/paths.go

@@ -65,9 +65,12 @@ func Abspath(path string) string {
 var KittyExe = (&Once[string]{Run: func() string {
 	exe, err := os.Executable()
 	if err == nil {
-		return filepath.Join(filepath.Dir(exe), "kitty")
+		ans := filepath.Join(filepath.Dir(exe), "kitty")
+		if s, err := os.Stat(ans); err == nil && !s.IsDir() {
+			return ans
 		}
-	return ""
+	}
+	return os.Getenv("KITTY_PATH_TO_KITTY_EXE")
 }}).Get
 
 var ConfigDir = (&Once[string]{Run: func() (config_dir string) {