From 747411be006b7cbd4aedb31d635631012df98fba Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 18 Feb 2023 16:52:16 +0530 Subject: [PATCH] Finish implementation of config file parsing Still needs tests --- gen-go-code.py | 29 +++++++- kittens/runner.py | 8 +++ kittens/ssh/main.py | 3 + kitty/conf/generate.py | 12 ++-- tools/cmd/ssh/config.go | 104 +++++++++++++++++++++++++-- tools/utils/config.go | 124 ++++++++++++++++++++++++++++++-- tools/utils/paths/well_known.go | 65 +++++++++++++++++ 7 files changed, 331 insertions(+), 14 deletions(-) create mode 100644 tools/utils/paths/well_known.go diff --git a/gen-go-code.py b/gen-go-code.py index 5de53d009..f51194a61 100755 --- a/gen-go-code.py +++ b/gen-go-code.py @@ -326,12 +326,39 @@ def generate_conf_parser(kitten: str, defn: Definition) -> None: print(gen_go_code(defn)) +def generate_extra_cli_parser(name: str, spec: str) -> None: + print('import "kitty/tools/cli"') + go_opts = tuple(go_options_for_seq(parse_option_spec(spec)[0])) + print(f'type {name}_options struct ''{') + for opt in go_opts: + print(opt.struct_declaration()) + print('}') + print(f'func parse_{name}_args(args []string) (*{name}_options, []string, error) ''{') + print(f'root := cli.Command{{Name: `{name}` }}') + for opt in go_opts: + print(opt.as_option('root')) + print('cmd, err := root.ParseArgs(args)') + print('if err != nil { return nil, nil, err }') + print(f'var opts {name}_options') + print('err = cmd.GetOptionValues(&opts)') + print('if err != nil { return nil, nil, err }') + print('return &opts, cmd.Args, nil') + print('}') + + def kitten_clis() -> None: - from kittens.runner import get_kitten_conf_docs + from kittens.runner import get_kitten_conf_docs, get_kitten_extra_cli_parsers for kitten in wrapped_kittens(): defn = get_kitten_conf_docs(kitten) if defn is not None: generate_conf_parser(kitten, defn) + ecp = get_kitten_extra_cli_parsers(kitten) + if ecp: + for name, spec in ecp.items(): + with replace_if_needed(f'tools/cmd/{kitten}/{name}_cli_generated.go'): + print(f'package {kitten}') + generate_extra_cli_parser(name, spec) + with replace_if_needed(f'tools/cmd/{kitten}/cli_generated.go'): od = [] kcd = kitten_cli_docs(kitten) diff --git a/kittens/runner.py b/kittens/runner.py index 656afb6d1..3cee69fbd 100644 --- a/kittens/runner.py +++ b/kittens/runner.py @@ -179,6 +179,14 @@ def get_kitten_conf_docs(kitten: str) -> Optional[Definition]: return cast(Definition, ans) +def get_kitten_extra_cli_parsers(kitten: str) -> Dict[str,str]: + setattr(sys, 'extra_cli_parsers', {}) + run_kitten(kitten, run_name='__extra_cli_parsers__') + ans = getattr(sys, 'extra_cli_parsers') + delattr(sys, 'extra_cli_parsers') + return cast(Dict[str, str], ans) + + def main() -> None: try: args = sys.argv[1:] diff --git a/kittens/ssh/main.py b/kittens/ssh/main.py index 585892f3e..7f08d7dca 100644 --- a/kittens/ssh/main.py +++ b/kittens/ssh/main.py @@ -724,3 +724,6 @@ elif __name__ == '__wrapper_of__': elif __name__ == '__conf__': from .options.definition import definition sys.options_definition = definition # type: ignore +elif __name__ == '__extra_cli_parsers__': + from .copy import option_text + setattr(sys, 'extra_cli_parsers', {'copy': option_text()}) # type: ignore diff --git a/kitty/conf/generate.py b/kitty/conf/generate.py index 460cc7aae..42813365a 100644 --- a/kitty/conf/generate.py +++ b/kitty/conf/generate.py @@ -444,7 +444,7 @@ def write_output(loc: str, defn: Definition) -> None: def go_type_data(parser_func: ParserFuncType, ctype: str) -> Tuple[str, str]: if ctype: - return f'*{ctype}', f'New{ctype}(val)' + return f'*{ctype}', f'Parse{ctype}(val)' p = parser_func.__name__ if p == 'int': return 'int64', 'strconv.ParseInt(val, 10, 64)' @@ -539,12 +539,16 @@ def gen_go_code(defn: Definition) -> str: a('default: return fmt.Errorf("Unknown configuration key: %#v", key)') for oname, pname in go_parsers.items(): ol = oname.lower() + is_multiple = oname in multiopts a(f'case "{ol}":') - a(f'var temp_val {go_types[oname]}') + if is_multiple: + a(f'var temp_val []{go_types[oname]}') + else: + a(f'var temp_val {go_types[oname]}') a(f'temp_val, err = {pname}') a(f'if err != nil {{ return fmt.Errorf("Failed to parse {ol} = %#v with error: %w", val, err) }}') - if oname in multiopts: - a(f'c.{oname} = append(c.{oname}, temp_val)') + if is_multiple: + a(f'c.{oname} = append(c.{oname}, temp_val...)') else: a(f'c.{oname} = temp_val') a('}') diff --git a/tools/cmd/ssh/config.go b/tools/cmd/ssh/config.go index 669c9bc33..61df05778 100644 --- a/tools/cmd/ssh/config.go +++ b/tools/cmd/ssh/config.go @@ -3,8 +3,16 @@ package ssh import ( + "errors" "fmt" + "os" + "path/filepath" "strings" + + "kitty/tools/utils/paths" + "kitty/tools/utils/shlex" + + "golang.org/x/sys/unix" ) var _ = fmt.Print @@ -19,9 +27,9 @@ type CopyInstruction struct { exclude_patterns []string } -func NewEnvInstruction(spec string) (ei *EnvInstruction, err error) { +func ParseEnvInstruction(spec string) (ans []*EnvInstruction, err error) { const COPY_FROM_LOCAL string = "_kitty_copy_env_var_" - ei = &EnvInstruction{} + ei := &EnvInstruction{} found := false ei.key, ei.val, found = strings.Cut(spec, "=") ei.key = strings.TrimSpace(ei.key) @@ -37,10 +45,98 @@ func NewEnvInstruction(spec string) (ei *EnvInstruction, err error) { if ei.key == "" { err = fmt.Errorf("The env directive must not be empty") } + ans = []*EnvInstruction{ei} return } -func NewCopyInstruction(spec string) (ci *CopyInstruction, err error) { - ci = &CopyInstruction{} +var paths_ctx *paths.Ctx + +func resolve_file_spec(spec string, is_glob bool) ([]string, error) { + if paths_ctx == nil { + paths_ctx = &paths.Ctx{} + } + ans := os.ExpandEnv(paths_ctx.ExpandHome(spec)) + if !filepath.IsAbs(ans) { + ans = paths_ctx.AbspathFromHome(ans) + } + if is_glob { + files, err := filepath.Glob(ans) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, fmt.Errorf("%s does not exist", spec) + } + return files, nil + } + err := unix.Access(ans, unix.R_OK) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("%s does not exist", spec) + } + return nil, fmt.Errorf("Cannot read from: %s with error: %w", spec, err) + } + return []string{ans}, nil +} + +func get_arcname(loc, dest, home string) (arcname string) { + if dest != "" { + arcname = dest + } else { + arcname = filepath.Clean(loc) + if filepath.HasPrefix(arcname, home) { + ra, err := filepath.Rel(home, arcname) + if err == nil { + arcname = ra + } + } + } + prefix := "home/" + if strings.HasPrefix(arcname, "/") { + prefix = "root" + } + return prefix + arcname +} + +func ParseCopyInstruction(spec string) (ans []*CopyInstruction, err error) { + args, err := shlex.Split(spec) + if err != nil { + return nil, err + } + opts, args, err := parse_copy_args(args) + if err != nil { + return nil, err + } + locations := make([]string, 0, len(args)) + for _, arg := range args { + locs, err := resolve_file_spec(arg, opts.Glob) + if err != nil { + return nil, err + } + locations = append(locations, locs...) + } + if len(locations) == 0 { + return nil, fmt.Errorf("No files to copy specified") + } + if len(locations) > 1 && opts.Dest != "" { + return nil, fmt.Errorf("Specifying a remote location with more than one file is not supported") + } + home := paths_ctx.HomePath() + ans = make([]*CopyInstruction, 0, len(locations)) + for _, loc := range locations { + ci := CopyInstruction{local_path: loc, exclude_patterns: opts.Exclude} + if opts.SymlinkStrategy != "preserve" { + ci.local_path, err = filepath.EvalSymlinks(loc) + if err != nil { + return nil, fmt.Errorf("Failed to resolve symlinks in %#v with error: %w", loc, err) + } + } + if opts.SymlinkStrategy == "resolve" { + ci.arcname = get_arcname(ci.local_path, opts.Dest, home) + } else { + ci.arcname = get_arcname(loc, opts.Dest, home) + } + ans = append(ans, &ci) + } return } diff --git a/tools/utils/config.go b/tools/utils/config.go index 5fdd93368..a46b6d9d5 100644 --- a/tools/utils/config.go +++ b/tools/utils/config.go @@ -4,8 +4,11 @@ package utils import ( "bufio" + "errors" "fmt" - "io" + "io/fs" + "os" + "path/filepath" "strings" ) @@ -16,9 +19,34 @@ func StringToBool(x string) bool { return x == "y" || x == "yes" || x == "true" } -func ParseConfData(src io.Reader, callback func(key, val string, line int)) error { - scanner := bufio.NewScanner(src) +type ConfigLine struct { + Src_file, Line string + Line_number int + Err error +} + +type ConfigParser struct { + BadLines []ConfigLine + LineHandler func(key, val string) error +} + +type Scanner interface { + Scan() bool + Text() string + Err() error +} + +func (self *ConfigParser) parse(scanner Scanner, name, base_path_for_includes string) error { lnum := 0 + make_absolute := func(path string) (string, error) { + if path == "" { + return "", fmt.Errorf("Empty include paths not allowed") + } + if !filepath.IsAbs(path) { + path = filepath.Join(base_path_for_includes, path) + } + return path, nil + } for scanner.Scan() { line := strings.TrimLeft(scanner.Text(), " ") lnum++ @@ -26,7 +54,93 @@ func ParseConfData(src io.Reader, callback func(key, val string, line int)) erro continue } key, val, _ := strings.Cut(line, " ") - callback(key, val, lnum) + switch key { + default: + err := self.LineHandler(key, val) + if err != nil { + self.BadLines = append(self.BadLines, ConfigLine{Src_file: name, Line: line, Line_number: lnum, Err: err}) + } + case "include", "globinclude", "envinclude": + var includes []string + switch key { + case "include": + aval, err := make_absolute(val) + if err == nil { + includes = []string{aval} + } + case "globinclude": + aval, err := make_absolute(val) + if err == nil { + matches, err := filepath.Glob(aval) + if err == nil { + includes = matches + } + } + case "envinclude": + for _, x := range os.Environ() { + key, eval, _ := strings.Cut(x, "=") + is_match, err := filepath.Match(val, key) + if is_match && err == nil { + escanner := bufio.NewScanner(strings.NewReader(eval)) + err := self.parse(escanner, "", base_path_for_includes) + if err != nil { + return err + } + } + } + } + if len(includes) > 0 { + for _, incpath := range includes { + raw, err := os.ReadFile(incpath) + if err == nil { + escanner := bufio.NewScanner(strings.NewReader(UnsafeBytesToString(raw))) + err := self.parse(escanner, incpath, filepath.Dir(incpath)) + if err != nil { + return err + } + } else if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("Failed to process include %#v with error: %w", incpath, err) + } + } + } + } } - return scanner.Err() + return nil +} + +func (self *ConfigParser) ParseFile(path string) error { + apath, err := filepath.Abs(path) + if err == nil { + path = apath + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + scanner := bufio.NewScanner(f) + return self.parse(scanner, f.Name(), filepath.Dir(f.Name())) +} + +type LinesScanner struct { + lines []string +} + +func (self *LinesScanner) Scan() bool { + return len(self.lines) > 0 +} + +func (self *LinesScanner) Text() string { + ans := self.lines[0] + self.lines = self.lines[1:] + return ans +} + +func (self *LinesScanner) Err() error { + return nil +} + +func (self *ConfigParser) ParseOverrides(overrides ...string) error { + s := LinesScanner{lines: overrides} + return self.parse(&s, "", ConfigDir()) } diff --git a/tools/utils/paths/well_known.go b/tools/utils/paths/well_known.go new file mode 100644 index 000000000..40490405f --- /dev/null +++ b/tools/utils/paths/well_known.go @@ -0,0 +1,65 @@ +// License: GPLv3 Copyright: 2023, Kovid Goyal, + +package paths + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "kitty/tools/utils" +) + +var _ = fmt.Print + +type Ctx struct { + home, cwd string +} + +func (ctx *Ctx) SetHome(val string) { + ctx.home = val +} + +func (ctx *Ctx) SetCwd(val string) { + ctx.cwd = val +} + +func (ctx *Ctx) HomePath() (ans string) { + ans = ctx.home + if ans == "" { + ans = utils.Expanduser("~") + } + return +} + +func (ctx *Ctx) CwdPath() (ans string) { + ans = ctx.cwd + if ans == "" { + var err error + ans, err = os.Getwd() + if err != nil { + ans = "." + } + } + return +} + +func abspath(path, base string) (ans string) { + return filepath.Join(base, path) +} + +func (ctx *Ctx) Abspath(path string) (ans string) { + return abspath(path, ctx.CwdPath()) +} + +func (ctx *Ctx) AbspathFromHome(path string) (ans string) { + return abspath(path, ctx.HomePath()) +} + +func (ctx *Ctx) ExpandHome(path string) (ans string) { + if strings.HasPrefix(path, "~/") { + return ctx.AbspathFromHome(path) + } + return path +}