Finish implementation of config file parsing

Still needs tests

gen-go-code.py

@@ -326,12 +326,39 @@ def generate_conf_parser(kitten: str, defn: Definition) -> None:
         print(gen_go_code(defn))
 
 
+def generate_extra_cli_parser(name: str, spec: str) -> None:
+    print('import "kitty/tools/cli"')
+    go_opts = tuple(go_options_for_seq(parse_option_spec(spec)[0]))
+    print(f'type {name}_options struct ''{')
+    for opt in go_opts:
+        print(opt.struct_declaration())
+    print('}')
+    print(f'func parse_{name}_args(args []string) (*{name}_options, []string, error) ''{')
+    print(f'root := cli.Command{{Name: `{name}` }}')
+    for opt in go_opts:
+        print(opt.as_option('root'))
+    print('cmd, err := root.ParseArgs(args)')
+    print('if err != nil { return nil, nil, err }')
+    print(f'var opts {name}_options')
+    print('err = cmd.GetOptionValues(&opts)')
+    print('if err != nil { return nil, nil, err }')
+    print('return &opts, cmd.Args, nil')
+    print('}')
+
+
 def kitten_clis() -> None:
-    from kittens.runner import get_kitten_conf_docs
+    from kittens.runner import get_kitten_conf_docs, get_kitten_extra_cli_parsers
     for kitten in wrapped_kittens():
         defn = get_kitten_conf_docs(kitten)
         if defn is not None:
             generate_conf_parser(kitten, defn)
+        ecp = get_kitten_extra_cli_parsers(kitten)
+        if ecp:
+            for name, spec in ecp.items():
+                with replace_if_needed(f'tools/cmd/{kitten}/{name}_cli_generated.go'):
+                    print(f'package {kitten}')
+                    generate_extra_cli_parser(name, spec)
+
         with replace_if_needed(f'tools/cmd/{kitten}/cli_generated.go'):
             od = []
             kcd = kitten_cli_docs(kitten)

kittens/runner.py

@@ -179,6 +179,14 @@ def get_kitten_conf_docs(kitten: str) -> Optional[Definition]:
     return cast(Definition, ans)
 
 
+def get_kitten_extra_cli_parsers(kitten: str) -> Dict[str,str]:
+    setattr(sys, 'extra_cli_parsers', {})
+    run_kitten(kitten, run_name='__extra_cli_parsers__')
+    ans = getattr(sys, 'extra_cli_parsers')
+    delattr(sys, 'extra_cli_parsers')
+    return cast(Dict[str, str], ans)
+
+
 def main() -> None:
     try:
         args = sys.argv[1:]

kittens/ssh/main.py

@@ -724,3 +724,6 @@ elif __name__ == '__wrapper_of__':
 elif __name__ == '__conf__':
     from .options.definition import definition
     sys.options_definition = definition  # type: ignore
+elif __name__ == '__extra_cli_parsers__':
+    from .copy import option_text
+    setattr(sys, 'extra_cli_parsers', {'copy': option_text()})  # type: ignore

kitty/conf/generate.py

@@ -444,7 +444,7 @@ def write_output(loc: str, defn: Definition) -> None:
 
 def go_type_data(parser_func: ParserFuncType, ctype: str) -> Tuple[str, str]:
     if ctype:
-        return f'*{ctype}', f'New{ctype}(val)'
+        return f'*{ctype}', f'Parse{ctype}(val)'
     p = parser_func.__name__
     if p == 'int':
         return 'int64', 'strconv.ParseInt(val, 10, 64)'
@@ -539,12 +539,16 @@ def gen_go_code(defn: Definition) -> str:
     a('default: return fmt.Errorf("Unknown configuration key: %#v", key)')
     for oname, pname in go_parsers.items():
         ol = oname.lower()
+        is_multiple = oname in multiopts
         a(f'case "{ol}":')
-        a(f'var temp_val {go_types[oname]}')
+        if is_multiple:
+            a(f'var temp_val []{go_types[oname]}')
+        else:
+            a(f'var temp_val {go_types[oname]}')
         a(f'temp_val, err = {pname}')
         a(f'if err != nil {{ return fmt.Errorf("Failed to parse {ol} = %#v with error: %w", val, err) }}')
-        if oname in multiopts:
+        if is_multiple:
-            a(f'c.{oname} = append(c.{oname}, temp_val)')
+            a(f'c.{oname} = append(c.{oname}, temp_val...)')
         else:
             a(f'c.{oname} = temp_val')
     a('}')

tools/cmd/ssh/config.go

@@ -3,8 +3,16 @@
 package ssh
 
 import (
+	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
+
+	"kitty/tools/utils/paths"
+	"kitty/tools/utils/shlex"
+
+	"golang.org/x/sys/unix"
 )
 
 var _ = fmt.Print
@@ -19,9 +27,9 @@ type CopyInstruction struct {
 	exclude_patterns    []string
 }
 
-func NewEnvInstruction(spec string) (ei *EnvInstruction, err error) {
+func ParseEnvInstruction(spec string) (ans []*EnvInstruction, err error) {
 	const COPY_FROM_LOCAL string = "_kitty_copy_env_var_"
-	ei = &EnvInstruction{}
+	ei := &EnvInstruction{}
 	found := false
 	ei.key, ei.val, found = strings.Cut(spec, "=")
 	ei.key = strings.TrimSpace(ei.key)
@@ -37,10 +45,98 @@ func NewEnvInstruction(spec string) (ei *EnvInstruction, err error) {
 	if ei.key == "" {
 		err = fmt.Errorf("The env directive must not be empty")
 	}
+	ans = []*EnvInstruction{ei}
 	return
 }
 
-func NewCopyInstruction(spec string) (ci *CopyInstruction, err error) {
+var paths_ctx *paths.Ctx
-	ci = &CopyInstruction{}
+
+func resolve_file_spec(spec string, is_glob bool) ([]string, error) {
+	if paths_ctx == nil {
+		paths_ctx = &paths.Ctx{}
+	}
+	ans := os.ExpandEnv(paths_ctx.ExpandHome(spec))
+	if !filepath.IsAbs(ans) {
+		ans = paths_ctx.AbspathFromHome(ans)
+	}
+	if is_glob {
+		files, err := filepath.Glob(ans)
+		if err != nil {
+			return nil, err
+		}
+		if len(files) == 0 {
+			return nil, fmt.Errorf("%s does not exist", spec)
+		}
+		return files, nil
+	}
+	err := unix.Access(ans, unix.R_OK)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("%s does not exist", spec)
+		}
+		return nil, fmt.Errorf("Cannot read from: %s with error: %w", spec, err)
+	}
+	return []string{ans}, nil
+}
+
+func get_arcname(loc, dest, home string) (arcname string) {
+	if dest != "" {
+		arcname = dest
+	} else {
+		arcname = filepath.Clean(loc)
+		if filepath.HasPrefix(arcname, home) {
+			ra, err := filepath.Rel(home, arcname)
+			if err == nil {
+				arcname = ra
+			}
+		}
+	}
+	prefix := "home/"
+	if strings.HasPrefix(arcname, "/") {
+		prefix = "root"
+	}
+	return prefix + arcname
+}
+
+func ParseCopyInstruction(spec string) (ans []*CopyInstruction, err error) {
+	args, err := shlex.Split(spec)
+	if err != nil {
+		return nil, err
+	}
+	opts, args, err := parse_copy_args(args)
+	if err != nil {
+		return nil, err
+	}
+	locations := make([]string, 0, len(args))
+	for _, arg := range args {
+		locs, err := resolve_file_spec(arg, opts.Glob)
+		if err != nil {
+			return nil, err
+		}
+		locations = append(locations, locs...)
+	}
+	if len(locations) == 0 {
+		return nil, fmt.Errorf("No files to copy specified")
+	}
+	if len(locations) > 1 && opts.Dest != "" {
+		return nil, fmt.Errorf("Specifying a remote location with more than one file is not supported")
+	}
+	home := paths_ctx.HomePath()
+	ans = make([]*CopyInstruction, 0, len(locations))
+	for _, loc := range locations {
+		ci := CopyInstruction{local_path: loc, exclude_patterns: opts.Exclude}
+		if opts.SymlinkStrategy != "preserve" {
+			ci.local_path, err = filepath.EvalSymlinks(loc)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to resolve symlinks in %#v with error: %w", loc, err)
+			}
+		}
+		if opts.SymlinkStrategy == "resolve" {
+			ci.arcname = get_arcname(ci.local_path, opts.Dest, home)
+		} else {
+			ci.arcname = get_arcname(loc, opts.Dest, home)
+		}
+		ans = append(ans, &ci)
+	}
 	return
 }

tools/utils/config.go

@@ -4,8 +4,11 @@ package utils
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
-	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -16,9 +19,34 @@ func StringToBool(x string) bool {
 	return x == "y" || x == "yes" || x == "true"
 }
 
-func ParseConfData(src io.Reader, callback func(key, val string, line int)) error {
+type ConfigLine struct {
-	scanner := bufio.NewScanner(src)
+	Src_file, Line string
+	Line_number    int
+	Err            error
+}
+
+type ConfigParser struct {
+	BadLines    []ConfigLine
+	LineHandler func(key, val string) error
+}
+
+type Scanner interface {
+	Scan() bool
+	Text() string
+	Err() error
+}
+
+func (self *ConfigParser) parse(scanner Scanner, name, base_path_for_includes string) error {
 	lnum := 0
+	make_absolute := func(path string) (string, error) {
+		if path == "" {
+			return "", fmt.Errorf("Empty include paths not allowed")
+		}
+		if !filepath.IsAbs(path) {
+			path = filepath.Join(base_path_for_includes, path)
+		}
+		return path, nil
+	}
 	for scanner.Scan() {
 		line := strings.TrimLeft(scanner.Text(), " ")
 		lnum++
@@ -26,7 +54,93 @@ func ParseConfData(src io.Reader, callback func(key, val string, line int)) erro
 			continue
 		}
 		key, val, _ := strings.Cut(line, " ")
-		callback(key, val, lnum)
+		switch key {
+		default:
+			err := self.LineHandler(key, val)
+			if err != nil {
+				self.BadLines = append(self.BadLines, ConfigLine{Src_file: name, Line: line, Line_number: lnum, Err: err})
+			}
+		case "include", "globinclude", "envinclude":
+			var includes []string
+			switch key {
+			case "include":
+				aval, err := make_absolute(val)
+				if err == nil {
+					includes = []string{aval}
+				}
+			case "globinclude":
+				aval, err := make_absolute(val)
+				if err == nil {
+					matches, err := filepath.Glob(aval)
+					if err == nil {
+						includes = matches
+					}
+				}
+			case "envinclude":
+				for _, x := range os.Environ() {
+					key, eval, _ := strings.Cut(x, "=")
+					is_match, err := filepath.Match(val, key)
+					if is_match && err == nil {
+						escanner := bufio.NewScanner(strings.NewReader(eval))
+						err := self.parse(escanner, "<env var: "+key+">", base_path_for_includes)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+			if len(includes) > 0 {
+				for _, incpath := range includes {
+					raw, err := os.ReadFile(incpath)
+					if err == nil {
+						escanner := bufio.NewScanner(strings.NewReader(UnsafeBytesToString(raw)))
+						err := self.parse(escanner, incpath, filepath.Dir(incpath))
+						if err != nil {
+							return err
+						}
+					} else if !errors.Is(err, fs.ErrNotExist) {
+						return fmt.Errorf("Failed to process include %#v with error: %w", incpath, err)
+					}
+				}
+			}
+		}
 	}
-	return scanner.Err()
+	return nil
+}
+
+func (self *ConfigParser) ParseFile(path string) error {
+	apath, err := filepath.Abs(path)
+	if err == nil {
+		path = apath
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	return self.parse(scanner, f.Name(), filepath.Dir(f.Name()))
+}
+
+type LinesScanner struct {
+	lines []string
+}
+
+func (self *LinesScanner) Scan() bool {
+	return len(self.lines) > 0
+}
+
+func (self *LinesScanner) Text() string {
+	ans := self.lines[0]
+	self.lines = self.lines[1:]
+	return ans
+}
+
+func (self *LinesScanner) Err() error {
+	return nil
+}
+
+func (self *ConfigParser) ParseOverrides(overrides ...string) error {
+	s := LinesScanner{lines: overrides}
+	return self.parse(&s, "<overrides>", ConfigDir())
 }

tools/utils/paths/well_known.go

@@ -0,0 +1,65 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package paths
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"kitty/tools/utils"
+)
+
+var _ = fmt.Print
+
+type Ctx struct {
+	home, cwd string
+}
+
+func (ctx *Ctx) SetHome(val string) {
+	ctx.home = val
+}
+
+func (ctx *Ctx) SetCwd(val string) {
+	ctx.cwd = val
+}
+
+func (ctx *Ctx) HomePath() (ans string) {
+	ans = ctx.home
+	if ans == "" {
+		ans = utils.Expanduser("~")
+	}
+	return
+}
+
+func (ctx *Ctx) CwdPath() (ans string) {
+	ans = ctx.cwd
+	if ans == "" {
+		var err error
+		ans, err = os.Getwd()
+		if err != nil {
+			ans = "."
+		}
+	}
+	return
+}
+
+func abspath(path, base string) (ans string) {
+	return filepath.Join(base, path)
+}
+
+func (ctx *Ctx) Abspath(path string) (ans string) {
+	return abspath(path, ctx.CwdPath())
+}
+
+func (ctx *Ctx) AbspathFromHome(path string) (ans string) {
+	return abspath(path, ctx.HomePath())
+}
+
+func (ctx *Ctx) ExpandHome(path string) (ans string) {
+	if strings.HasPrefix(path, "~/") {
+		return ctx.AbspathFromHome(path)
+	}
+	return path
+}