Start work on porting hints kitten to Go

gen-go-code.py

@@ -465,6 +465,7 @@ def generate_constants() -> str:
         assert m is not None
         placeholder_char = int(m.group(1), 16)
     dp = ", ".join(map(lambda x: f'"{serialize_as_go_string(x)}"', kc.default_pager_for_help))
+    url_prefixes = ','.join(f'"{x}"' for x in Options.url_prefixes)
     return f'''\
 package kitty
 
@@ -489,9 +490,11 @@ var RefMap = map[string]string{serialize_go_dict(ref_map['ref'])}
 var DocTitleMap = map[string]string{serialize_go_dict(ref_map['doc'])}
 var AllowedShellIntegrationValues = []string{{ {str(sorted(allowed_shell_integration_values))[1:-1].replace("'", '"')} }}
 var KittyConfigDefaults = struct {{
-Term, Shell_integration string
+Term, Shell_integration, Select_by_word_characters string
+Url_prefixes []string
 }}{{
-Term: "{Options.term}", Shell_integration: "{' '.join(Options.shell_integration)}",
+Term: "{Options.term}", Shell_integration: "{' '.join(Options.shell_integration)}", Url_prefixes: []string{{ {url_prefixes} }},
+Select_by_word_characters: `{Options.select_by_word_characters}`,
 }}
 '''  # }}}
 

gen-wcwidth.py

@@ -438,9 +438,14 @@ def gen_ucd() -> None:
             f.truncate()
             f.write(raw)
 
+    chars = ''.join(classes_to_regex(cz, exclude='\n\r'))
     with open('kittens/hints/url_regex.py', 'w') as f:
         f.write('# generated by gen-wcwidth.py, do not edit\n\n')
-        f.write("url_delimiters = '{}'  # noqa".format(''.join(classes_to_regex(cz, exclude='\n\r'))))
+        f.write(f"url_delimiters = '{chars}'  # noqa")
+    with open('tools/cmd/hints/url_regex.go', 'w') as f:
+        f.write('// generated by gen-wcwidth.py, do not edit\n\n')
+        f.write('package hints\n\n')
+        f.write(f"const URL_DELIMITERS = `{chars}`")
 
 
 def gen_names() -> None:

kittens/hints/main.py

@@ -864,6 +864,7 @@ if __name__ == '__main__':
 elif __name__ == '__doc__':
     cd = sys.cli_docs  # type: ignore
     cd['usage'] = usage
+    cd['short_desc'] = 'Select text from screen with keyboard'
     cd['options'] = OPTIONS
     cd['help_text'] = help_text
 # }}}

shell-integration/ssh/kitty

@@ -24,7 +24,7 @@ exec_kitty() {
 
 
 is_wrapped_kitten() {
-    wrapped_kittens="clipboard icat hyperlinked_grep ask unicode_input ssh"
+    wrapped_kittens="clipboard icat hyperlinked_grep ask hints unicode_input ssh"
     [ -n "$1" ] && {
         case " $wrapped_kittens " in
             *" $1 "*) printf "%s" "$1" ;;

tools/cli/markup/prettify.go

@@ -80,7 +80,7 @@ func replace_all_rst_roles(str string, repl func(rst_format_match) string) strin
 		m.role = groupdict["role"].Text
 		return repl(m)
 	}
-	return utils.ReplaceAll(":(?P<role>[a-z]+):(?:(?:`(?P<payload>[^`]+)`)|(?:'(?P<payload>[^']+)'))", str, rf)
+	return utils.ReplaceAll(utils.MustCompile(":(?P<role>[a-z]+):(?:(?:`(?P<payload>[^`]+)`)|(?:'(?P<payload>[^']+)'))"), str, rf)
 }
 
 func (self *Context) hyperlink_for_url(url string, text string) string {

tools/cmd/hints/main.go

@@ -0,0 +1,152 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package hints
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	"kitty/tools/cli"
+	"kitty/tools/tty"
+	"kitty/tools/tui"
+	"kitty/tools/tui/loop"
+	"kitty/tools/utils"
+	"kitty/tools/wcswidth"
+)
+
+var _ = fmt.Print
+
+func convert_text(text string, cols int) string {
+	lines := make([]string, 0, 64)
+	empty_line := strings.Repeat("\x00", cols) + "\n"
+	s1 := utils.NewLineScanner(text)
+	for s1.Scan() {
+		full_line := s1.Text()
+		if full_line == "" {
+			continue
+		}
+		if strings.TrimRight(full_line, "\r") == "" {
+			for i := 0; i < len(full_line); i++ {
+				lines = append(lines, empty_line)
+			}
+			continue
+		}
+		appended := false
+		s2 := utils.NewSeparatorScanner(full_line, "\r")
+		for s2.Scan() {
+			line := s2.Text()
+			if line != "" {
+				line_sz := wcswidth.Stringwidth(line)
+				extra := cols - line_sz
+				if extra > 0 {
+					line += strings.Repeat("\x00", extra)
+				}
+				lines = append(lines, line)
+				lines = append(lines, "\r")
+				appended = true
+			}
+		}
+		if appended {
+			lines[len(lines)-1] = "\n"
+		}
+	}
+	ans := strings.Join(lines, "")
+	return strings.TrimRight(ans, "\r\n")
+}
+
+func parse_input(text string) string {
+	cols, err := strconv.Atoi(os.Getenv("OVERLAID_WINDOW_COLS"))
+	if err == nil {
+		return convert_text(text, cols)
+	}
+	term, err := tty.OpenControllingTerm()
+	if err == nil {
+		sz, err := term.GetSize()
+		term.Close()
+		if err == nil {
+			return convert_text(text, int(sz.Col))
+		}
+	}
+	return convert_text(text, 80)
+}
+
+type Result struct {
+	Match                []string            `json:"match"`
+	Programs             []string            `json:"programs"`
+	Multiple_joiner      string              `json:"multiple_joiner"`
+	Customize_processing string              `json:"customize_processing"`
+	Type                 string              `json:"type"`
+	Groupdicts           []map[string]string `json:"groupdicts"`
+	Extra_cli_args       []string            `json:"extra_cli_args"`
+	Linenum_action       string              `json:"linenum_action"`
+	Cwd                  string              `json:"cwd"`
+}
+
+func main(_ *cli.Command, o *Options, args []string) (rc int, err error) {
+	output := tui.KittenOutputSerializer()
+	if tty.IsTerminal(os.Stdin.Fd()) {
+		tui.ReportError(fmt.Errorf("You must pass the text to be hinted on STDIN"))
+		return 1, nil
+	}
+	stdin, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		tui.ReportError(fmt.Errorf("Failed to read from STDIN with error: %w", err))
+		return 1, nil
+	}
+	if len(args) > 0 && o.CustomizeProcessing == "" && o.Type != "linenum" {
+		tui.ReportError(fmt.Errorf("Extra command line arguments present: %s", strings.Join(args, " ")))
+		return 1, nil
+	}
+	text := parse_input(utils.UnsafeBytesToString(stdin))
+	all_marks, index_map, err := find_marks(text, o)
+	if err != nil {
+		tui.ReportError(err)
+		return 1, nil
+	}
+
+	result := Result{
+		Programs: o.Program, Multiple_joiner: o.MultipleJoiner, Customize_processing: o.CustomizeProcessing, Type: o.Type,
+		Extra_cli_args: args, Linenum_action: o.LinenumAction,
+	}
+	result.Cwd, _ = os.Getwd()
+	alphabet := o.Alphabet
+	if alphabet == "" {
+		alphabet = DEFAULT_HINT_ALPHABET
+	}
+	ignore_mark_indices := utils.NewSet[int](8)
+	_, _, _ = all_marks, index_map, ignore_mark_indices
+	window_title := o.WindowTitle
+	if window_title == "" {
+		switch o.Type {
+		case "url":
+			window_title = "Choose URL"
+		default:
+			window_title = "Choose text"
+		}
+	}
+	lp, err := loop.New(loop.NoAlternateScreen) // no alternate screen reduces flicker on exit
+	if err != nil {
+		return
+	}
+	lp.OnInitialize = func() (string, error) {
+		lp.SendOverlayReady()
+		lp.SetCursorVisible(false)
+		lp.SetWindowTitle(window_title)
+		lp.AllowLineWrapping(false)
+		return "", nil
+	}
+	lp.OnFinalize = func() string {
+		lp.SetCursorVisible(true)
+		return ""
+	}
+
+	output(result)
+	return
+}
+
+func EntryPoint(parent *cli.Command) {
+	create_cmd(parent, main)
+}

tools/cmd/hints/marks.go

@@ -0,0 +1,332 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package hints
+
+import (
+	"fmt"
+	"kitty"
+	"kitty/tools/config"
+	"kitty/tools/utils"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/seancfoley/ipaddress-go/ipaddr"
+	"golang.org/x/exp/slices"
+)
+
+var _ = fmt.Print
+
+const (
+	DEFAULT_HINT_ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyz"
+	DEFAULT_REGEX         = `(?m)^\s*(.+)\s*$`
+	FILE_EXTENSION        = `\.(?:[a-zA-Z0-9]{2,7}|[ahcmo])(?!\.)`
+)
+
+func path_regex() string {
+	return fmt.Sprintf(`(?:\S*?/[\r\S]+)|(?:\S[\r\S]*{%s})\b`, FILE_EXTENSION)
+}
+
+func default_linenum_regex() string {
+	return fmt.Sprintf(`(?P<path>%s):(?P<line>\d+)`, path_regex())
+}
+
+type Mark struct {
+	Index, Start, End int
+	Text, Group_id    string
+	Is_hyperlink      bool
+	Groupdict         map[string]string
+}
+
+func process_escape_codes(text string) (ans string, hyperlinks []Mark) {
+	removed_size, idx := 0, 0
+	active_hyperlink_url := ""
+	active_hyperlink_id := ""
+	active_hyperlink_start_offset := 0
+
+	add_hyperlink := func(end int) {
+		hyperlinks = append(hyperlinks, Mark{
+			Index: idx, Start: active_hyperlink_start_offset, End: end, Text: active_hyperlink_url, Is_hyperlink: true, Group_id: active_hyperlink_id})
+		active_hyperlink_url, active_hyperlink_id = "", ""
+		active_hyperlink_start_offset = 0
+		idx++
+	}
+
+	ans = utils.ReplaceAll(utils.MustCompile("\x1b(?:\\[[0-9;:]*?m|\\].*?\x1b\\)"), text, func(raw string, groupdict map[string]utils.SubMatch) string {
+		if !strings.HasPrefix(raw, "\x1b]8") {
+			removed_size += len(raw)
+			return ""
+		}
+		start := groupdict[""].Start - removed_size
+		removed_size += len(raw)
+		if active_hyperlink_url != "" {
+			add_hyperlink(start)
+		}
+		raw = raw[4 : len(raw)-2]
+		if metadata, url, found := strings.Cut(raw, ";"); found && url != "" {
+			active_hyperlink_url = url
+			active_hyperlink_start_offset = start
+			if metadata != "" {
+				for _, entry := range strings.Split(metadata, ":") {
+					if strings.HasPrefix(entry, "id=") && len(entry) > 3 {
+						active_hyperlink_id = entry[3:]
+					}
+				}
+			}
+		}
+		return ""
+	})
+	if active_hyperlink_url != "" {
+		add_hyperlink(len(ans))
+	}
+	return
+}
+
+type PostProcessorFunc = func(string, int, int) (int, int)
+
+func is_punctuation(b string) bool {
+	switch b {
+	case ",", ".", "?", "!":
+		return true
+	}
+	return false
+}
+
+func closing_bracket_for(ch string) string {
+	switch ch {
+	case "(":
+		return ")"
+	case "[":
+		return "]"
+	case "{":
+		return "}"
+	case "<":
+		return ">"
+	case "*":
+		return "*"
+	case `"`:
+		return `"`
+	case "'":
+		return "'"
+	case "“":
+		return "”"
+	case "‘":
+		return "’"
+	}
+	return ""
+}
+
+func char_at(s string, i int) string {
+	ans, _ := utf8.DecodeRuneInString(s[i:])
+	if ans == utf8.RuneError {
+		return ""
+	}
+	return string(ans)
+}
+
+func matching_remover(openers ...string) PostProcessorFunc {
+	return func(text string, s, e int) (int, int) {
+		if s < e && e <= len(text) {
+			before := char_at(text, s)
+			if slices.Index(openers, before) > -1 {
+				q := closing_bracket_for(before)
+				if e > 0 && char_at(text, e-1) == q {
+					s++
+					e--
+				} else if char_at(text, e) == q {
+					s++
+				}
+			}
+		}
+		return s, e
+	}
+}
+
+var PostProcessorMap = (&utils.Once[map[string]PostProcessorFunc]{Run: func() map[string]PostProcessorFunc {
+	return map[string]PostProcessorFunc{
+		"url": func(text string, s, e int) (int, int) {
+			if s > 4 && text[s-5:s] == "link:" { // asciidoc URLs
+				url := text[s:e]
+				idx := strings.LastIndex(url, "[")
+				if idx > -1 {
+					e -= len(url) - idx
+				}
+			}
+			for e > 1 && is_punctuation(char_at(text, e)) { // remove trailing punctuation
+				e--
+			}
+			// truncate url at closing bracket/quote
+			if s > 0 && e <= len(text) && closing_bracket_for(char_at(text, s-1)) != "" {
+				q := closing_bracket_for(char_at(text, s-1))
+				idx := strings.Index(text[s:], q)
+				if idx > 0 {
+					e = s + idx
+				}
+			}
+			// reStructuredText URLs
+			if e > 3 && text[e-2:e] == "`_" {
+				e -= 2
+			}
+			return s, e
+		},
+
+		"brackets": matching_remover("(", "{", "[", "<"),
+		"quotes":   matching_remover("'", `"`, "“", "‘"),
+		"ip": func(text string, s, e int) (int, int) {
+			addr := ipaddr.NewHostName(text[s:e])
+			if !addr.IsAddress() {
+				return -1, -1
+			}
+			return s, e
+		},
+	}
+}}).Get
+
+type KittyOpts struct {
+	Url_prefixes              *utils.Set[string]
+	Select_by_word_characters string
+}
+
+func read_relevant_kitty_opts(path string) KittyOpts {
+	ans := KittyOpts{Select_by_word_characters: kitty.KittyConfigDefaults.Select_by_word_characters}
+	handle_line := func(key, val string) error {
+		switch key {
+		case "url_prefixes":
+			ans.Url_prefixes = utils.NewSetWithItems(strings.Split(val, " ")...)
+		case "select_by_word_characters":
+			ans.Select_by_word_characters = strings.TrimSpace(val)
+		}
+		return nil
+	}
+	cp := config.ConfigParser{LineHandler: handle_line}
+	cp.ParseFiles(path)
+	if ans.Url_prefixes == nil {
+		ans.Url_prefixes = utils.NewSetWithItems(kitty.KittyConfigDefaults.Url_prefixes...)
+	}
+	return ans
+}
+
+var RelevantKittyOpts = (&utils.Once[KittyOpts]{Run: func() KittyOpts {
+	return read_relevant_kitty_opts(filepath.Join(utils.ConfigDir(), "kitty.conf"))
+}}).Get
+
+func functions_for(opts *Options) (pattern string, post_processors []PostProcessorFunc) {
+	switch opts.Type {
+	case "url":
+		var url_prefixes *utils.Set[string]
+		if opts.UrlPrefixes == "default" {
+			url_prefixes = RelevantKittyOpts().Url_prefixes
+		} else {
+			url_prefixes = utils.NewSetWithItems(strings.Split(opts.UrlPrefixes, ",")...)
+		}
+		pattern = fmt.Sprintf(`(?:%s)://[^%s]{3,}`, strings.Join(url_prefixes.AsSlice(), "|"), URL_DELIMITERS)
+		post_processors = append(post_processors, PostProcessorMap()["url"])
+	case "path":
+		pattern = path_regex()
+		post_processors = append(post_processors, PostProcessorMap()["brackets"], PostProcessorMap()["quotes"])
+	case "line":
+		pattern = "(?m)^\\s*(.+)[\\s\x00]*$"
+	case "hash":
+		pattern = "[0-9a-f][0-9a-f\r]{6,127}"
+	case "ip":
+		pattern = (
+		// IPv4 with no validation
+		`((?:\d{1,3}\.){3}\d{1,3}` + "|" +
+			// IPv6 with no validation
+			`(?:[a-fA-F0-9]{0,4}:){2,7}[a-fA-F0-9]{1,4})`)
+		post_processors = append(post_processors, PostProcessorMap()["ip"])
+	case "word":
+		chars := opts.WordCharacters
+		if chars == "" {
+			chars = RelevantKittyOpts().Select_by_word_characters
+		}
+		chars = regexp.QuoteMeta(chars)
+		pattern = fmt.Sprintf(`(?u)[%s\pL\pN]{%d,}`, chars, opts.MinimumMatchLength)
+		post_processors = append(post_processors, PostProcessorMap()["brackets"], PostProcessorMap()["quotes"])
+	default:
+		pattern = opts.Regex
+		if opts.Type == "linenum" {
+			if pattern == DEFAULT_REGEX {
+				pattern = default_linenum_regex()
+			}
+		}
+	}
+	return
+}
+
+func mark(r *regexp.Regexp, post_processors []PostProcessorFunc, text string, opts *Options) (ans []Mark) {
+	sanitize_pat := regexp.MustCompile("[\r\n\x00]")
+	names := r.SubexpNames()
+	for i, v := range r.FindAllStringSubmatchIndex(text, -1) {
+		match_start, match_end := v[0], v[1]
+		for match_end > match_start+1 && text[match_end-1] == 0 {
+			match_end--
+		}
+		full_match := text[match_start:match_end]
+		if len([]rune(full_match)) < opts.MinimumMatchLength {
+			continue
+		}
+		for _, f := range post_processors {
+			match_start, match_end = f(text, match_start, match_end)
+			if match_start < 0 {
+				break
+			}
+		}
+		if match_start < 0 {
+			continue
+		}
+		full_match = sanitize_pat.ReplaceAllLiteralString(text[match_start:match_end], "")
+		gd := make(map[string]string, len(names))
+		for x, name := range names {
+			if name != "" {
+				idx := 2 * x
+				if s, e := v[idx], v[idx]+1; s > -1 && e > -1 {
+					s = utils.Max(s, match_start)
+					e = utils.Min(e, match_end)
+					gd[name] = sanitize_pat.ReplaceAllLiteralString(text[s:e], "")
+				}
+			}
+		}
+		ans = append(ans, Mark{
+			Index: i, Start: match_start, End: match_end, Text: full_match, Groupdict: gd,
+		})
+	}
+	return
+}
+
+func find_marks(text string, opts *Options) (ans []Mark, index_map map[int]*Mark, err error) {
+	text, hyperlinks := process_escape_codes(text)
+	pattern, post_processors := functions_for(opts)
+	if opts.Type == "hyperlink" {
+		ans = hyperlinks
+	} else {
+		r, err := regexp.Compile(pattern)
+		if err != nil {
+			return nil, nil, fmt.Errorf("Failed to compile the regex pattern: %#v with error: %w", pattern, err)
+		}
+		ans = mark(r, post_processors, text, opts)
+	}
+	if len(ans) == 0 {
+		none_of := "matches"
+		switch opts.Type {
+		case "urls":
+			none_of = "URLs"
+		case "hyperlinks":
+			none_of = "hyperlinks"
+		}
+		return nil, nil, fmt.Errorf("No %s found", none_of)
+	}
+	largest_index := ans[len(ans)-1].Index
+	offset := utils.Max(0, opts.HintsOffset)
+	index_map = make(map[int]*Mark, len(ans))
+	for _, m := range ans {
+		if opts.Ascending {
+			m.Index += offset
+		} else {
+			m.Index = largest_index - m.Index + offset
+		}
+		index_map[m.Index] = &m
+	}
+	return
+}

tools/cmd/hints/url_regex.go

@@ -0,0 +1,5 @@
+// generated by gen-wcwidth.py, do not edit
+
+package hints
+
+const URL_DELIMITERS = `\x00-\x09\x0b-\x0c\x0e-\x20\x7f-\xa0\xad\u0600-\u0605\u061c\u06dd\u070f\u0890-\u0891\u08e2\u1680\u180e\u2000-\u200f\u2028-\u202f\u205f-\u2064\u2066-\u206f\u3000\ud800-\uf8ff\ufeff\ufff9-\ufffb\U000110bd\U000110cd\U00013430-\U0001343f\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0001\U000e0020-\U000e007f\U000f0000-\U000ffffd\U00100000-\U0010fffd`

tools/cmd/tool/main.go

@@ -10,6 +10,7 @@ import (
 	"kitty/tools/cmd/at"
 	"kitty/tools/cmd/clipboard"
 	"kitty/tools/cmd/edit_in_kitty"
+	"kitty/tools/cmd/hints"
 	"kitty/tools/cmd/hyperlinked_grep"
 	"kitty/tools/cmd/icat"
 	"kitty/tools/cmd/pytest"
@@ -42,6 +43,8 @@ func KittyToolEntryPoints(root *cli.Command) {
 	hyperlinked_grep.EntryPoint(root)
 	// ask
 	ask.EntryPoint(root)
+	// hints
+	hints.EntryPoint(root)
 	// __pytest__
 	pytest.EntryPoint(root)
 	// __hold_till_enter__

tools/tui/ui_kitten.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 
+	"kitty/tools/cli"
 	"kitty/tools/utils"
 
 	"github.com/jamesruan/go-rfc1924/base85"
@@ -37,3 +38,9 @@ func KittenOutputSerializer() func(any) (string, error) {
 		return utils.UnsafeBytesToString(data), nil
 	}
 }
+
+func ReportError(err error) {
+	cli.ShowError(err)
+	os.Stdout.WriteString("\x1bP@kitty-overlay-ready|\x1b\\")
+	HoldTillEnter(false)
+}

tools/utils/regexp.go

@@ -25,24 +25,23 @@ func MustCompile(pat string) *regexp.Regexp {
 	return pat_cache.MustGetOrCreate(pat, regexp.MustCompile)
 }
 
-func ReplaceAll(pat, str string, repl func(full_match string, groupdict map[string]SubMatch) string) string {
+func ReplaceAll(cpat *regexp.Regexp, str string, repl func(full_match string, groupdict map[string]SubMatch) string) string {
-	cpat := MustCompile(pat)
 	result := strings.Builder{}
 	result.Grow(len(str) + 256)
 	last_index := 0
 	matches := cpat.FindAllStringSubmatchIndex(str, -1)
 	names := cpat.SubexpNames()
+	groupdict := make(map[string]SubMatch, len(names))
 	for _, v := range matches {
 		match_start, match_end := v[0], v[1]
 		full_match := str[match_start:match_end]
-		groupdict := make(map[string]SubMatch, len(names))
+		for k := range groupdict {
-		for i, name := range names {
+			delete(groupdict, k)
-			if i == 0 {
-				continue
 		}
+		for i, name := range names {
 			idx := 2 * i
 			if v[idx] > -1 && v[idx+1] > -1 {
-				groupdict[name] = SubMatch{Text: str[v[idx]:v[idx+1]], Start: v[idx] - match_start, End: v[idx+1] - match_start}
+				groupdict[name] = SubMatch{Text: str[v[idx]:v[idx+1]], Start: v[idx], End: v[idx+1]}
 			}
 		}
 		result.WriteString(str[last_index:match_start])

tools/utils/set.go

@@ -55,6 +55,10 @@ func (self *Set[T]) Iterable() map[T]struct{} {
 	return self.items
 }
 
+func (self *Set[T]) AsSlice() []T {
+	return maps.Keys(self.items)
+}
+
 func (self *Set[T]) Intersect(other *Set[T]) (ans *Set[T]) {
 	if self.Len() < other.Len() {
 		ans = NewSet[T](self.Len())