From c2e549b79cff107bc1d45dce28a993e1a55bafb4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Mar 2023 16:43:23 +0530 Subject: [PATCH] Implement syntax highlighting --- go.mod | 2 + go.sum | 7 ++ tools/cmd/diff/collect.go | 53 +++++++--- tools/cmd/diff/highlight.go | 187 +++++++++++++++++++++++++++++++++++ tools/cmd/diff/render.go | 6 +- tools/cmd/diff/ui.go | 25 ++++- tools/utils/cache.go | 14 +++ tools/utils/style/wrapper.go | 4 + 8 files changed, 280 insertions(+), 18 deletions(-) create mode 100644 tools/cmd/diff/highlight.go diff --git a/go.mod b/go.mod index 3ea0d23dc..3cd35541f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924 + github.com/alecthomas/chroma/v2 v2.7.0 github.com/bmatcuk/doublestar v1.3.4 github.com/disintegration/imaging v1.6.2 github.com/google/go-cmp v0.5.9 @@ -17,6 +18,7 @@ require ( ) require ( + github.com/dlclark/regexp2 v1.4.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect diff --git a/go.sum b/go.sum index 8abe1fc47..8b92d698c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924 h1:DG4UyTVIujioxwJc8Zj8Nabz1L1wTgQ/xNBSQDfdP3I= github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924/go.mod h1:+NaH2gLeY6RPBPPQf4aRotPPStg+eXc8f9ZaE4vRfD4= +github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink= +github.com/alecthomas/chroma/v2 v2.7.0 h1:hm1rY6c/Ob4eGclpQ7X/A3yhqBOZNUTk9q+yhyLIViI= +github.com/alecthomas/chroma/v2 v2.7.0/go.mod h1:yrkMI9807G1ROx13fhe1v6PN2DDeaR73L3d+1nmYQtw= +github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -7,6 +11,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= +github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -14,6 +20,7 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/jamesruan/go-rfc1924 v0.0.0-20170108144916-2767ca7c638f h1:Ko4+g6K16vSyUrtd/pPXuQnWsiHe5BYptEtTxfwYwCc= github.com/jamesruan/go-rfc1924 v0.0.0-20170108144916-2767ca7c638f/go.mod h1:eHzfhOKbTGJEGPSdMHzU6jft192tHHt2Bu2vIZArvC0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= diff --git a/tools/cmd/diff/collect.go b/tools/cmd/diff/collect.go index 9f3e43376..732c03e7d 100644 --- a/tools/cmd/diff/collect.go +++ b/tools/cmd/diff/collect.go @@ -6,11 +6,12 @@ import ( "crypto/md5" "fmt" "io/fs" - "kitty/tools/utils" "os" "path/filepath" "strings" "unicode/utf8" + + "kitty/tools/utils" ) var _ = fmt.Print @@ -18,6 +19,7 @@ var path_name_map, remote_dirs map[string]string var mimetypes_cache, data_cache, hash_cache *utils.LRUCache[string, string] var lines_cache *utils.LRUCache[string, []string] +var highlighted_lines_cache *utils.LRUCache[string, []string] var is_text_cache *utils.LRUCache[string, bool] func init_caches() { @@ -28,6 +30,7 @@ func init_caches() { data_cache = utils.NewLRUCache[string, string](sz) is_text_cache = utils.NewLRUCache[string, bool](sz) lines_cache = utils.NewLRUCache[string, []string](sz) + highlighted_lines_cache = utils.NewLRUCache[string, []string](sz) hash_cache = utils.NewLRUCache[string, string](sz) } @@ -100,34 +103,55 @@ func hash_for_path(path string) (string, error) { } -func sanitize(x string) string { - x = strings.ReplaceAll(x, "\r\n", "⏎\n") +func sanitize_control_codes(x string) string { return utils.SanitizeControlCodes(x, "░") } +func sanitize_tabs_and_carriage_returns(x string) string { + return strings.NewReplacer("\t", conf.Replace_tab_by, "\r", "⏎").Replace(x) +} + +func sanitize(x string) string { + x = sanitize_control_codes(x) + return sanitize_tabs_and_carriage_returns(x) +} + +func text_to_lines(text string) []string { + lines := make([]string, 0, 512) + splitlines_like_git(text, false, func(line string) { lines = append(lines, line) }) + return lines +} + func lines_for_path(path string) ([]string, error) { return lines_cache.GetOrCreate(path, func(path string) ([]string, error) { ans, err := data_for_path(path) if err != nil { return nil, err } - ans = sanitize(strings.ReplaceAll(ans, "\t", conf.Replace_tab_by)) - lines := make([]string, 0, 256) - splitlines_like_git(ans, false, func(line string) { lines = append(lines, line) }) - return lines, nil + return text_to_lines(sanitize(ans)), nil }) } +func highlighted_lines_for_path(path string) ([]string, error) { + if ans, found := highlighted_lines_cache.Get(path); found && ans != nil { + return ans, nil + } + return lines_for_path(path) +} + type Collection struct { changes, renames, type_map map[string]string adds, removes *utils.Set[string] all_paths []string + paths_to_highlight *utils.Set[string] added_count, removed_count int } func (self *Collection) add_change(left, right string) { self.changes[left] = right self.all_paths = append(self.all_paths, left) + self.paths_to_highlight.Add(left) + self.paths_to_highlight.Add(right) self.type_map[left] = `diff` } @@ -140,6 +164,7 @@ func (self *Collection) add_rename(left, right string) { func (self *Collection) add_add(right string) { self.adds.Add(right) self.all_paths = append(self.all_paths, right) + self.paths_to_highlight.Add(right) self.type_map[right] = `add` if is_path_text(right) { num, _ := lines_for_path(right) @@ -150,6 +175,7 @@ func (self *Collection) add_add(right string) { func (self *Collection) add_removal(left string) { self.removes.Add(left) self.all_paths = append(self.all_paths, left) + self.paths_to_highlight.Add(left) self.type_map[left] = `removal` if is_path_text(left) { num, _ := lines_for_path(left) @@ -303,12 +329,13 @@ func (self *Collection) collect_files(left, right string) error { func create_collection(left, right string) (ans *Collection, err error) { ans = &Collection{ - changes: make(map[string]string), - renames: make(map[string]string), - type_map: make(map[string]string), - adds: utils.NewSet[string](32), - removes: utils.NewSet[string](32), - all_paths: make([]string, 0, 32), + changes: make(map[string]string), + renames: make(map[string]string), + type_map: make(map[string]string), + adds: utils.NewSet[string](32), + removes: utils.NewSet[string](32), + paths_to_highlight: utils.NewSet[string](32), + all_paths: make([]string, 0, 32), } left_stat, err := os.Stat(left) if err != nil { diff --git a/tools/cmd/diff/highlight.go b/tools/cmd/diff/highlight.go new file mode 100644 index 000000000..21dd8ecc3 --- /dev/null +++ b/tools/cmd/diff/highlight.go @@ -0,0 +1,187 @@ +// License: GPLv3 Copyright: 2023, Kovid Goyal, + +package diff + +import ( + "errors" + "fmt" + "io" + "kitty/tools/utils" + "kitty/tools/utils/images" + "path/filepath" + "strings" + + "github.com/alecthomas/chroma/v2" + "github.com/alecthomas/chroma/v2/lexers" + "github.com/alecthomas/chroma/v2/styles" +) + +var _ = fmt.Print +var ErrNoLexer = errors.New("No lexer available for this format") +var DefaultStyle = (&utils.Once[*chroma.Style]{Run: func() *chroma.Style { + // Default style generated by python style.py default pygments.styles.default.DefaultStyle + // with https://raw.githubusercontent.com/alecthomas/chroma/master/_tools/style.py + return styles.Register(chroma.MustNewStyle("default", chroma.StyleEntries{ + chroma.TextWhitespace: "#bbbbbb", + chroma.Comment: "italic #3D7B7B", + chroma.CommentPreproc: "noitalic #9C6500", + chroma.Keyword: "bold #008000", + chroma.KeywordPseudo: "nobold", + chroma.KeywordType: "nobold #B00040", + chroma.Operator: "#666666", + chroma.OperatorWord: "bold #AA22FF", + chroma.NameBuiltin: "#008000", + chroma.NameFunction: "#0000FF", + chroma.NameClass: "bold #0000FF", + chroma.NameNamespace: "bold #0000FF", + chroma.NameException: "bold #CB3F38", + chroma.NameVariable: "#19177C", + chroma.NameConstant: "#880000", + chroma.NameLabel: "#767600", + chroma.NameEntity: "bold #717171", + chroma.NameAttribute: "#687822", + chroma.NameTag: "bold #008000", + chroma.NameDecorator: "#AA22FF", + chroma.LiteralString: "#BA2121", + chroma.LiteralStringDoc: "italic", + chroma.LiteralStringInterpol: "bold #A45A77", + chroma.LiteralStringEscape: "bold #AA5D1F", + chroma.LiteralStringRegex: "#A45A77", + chroma.LiteralStringSymbol: "#19177C", + chroma.LiteralStringOther: "#008000", + chroma.LiteralNumber: "#666666", + chroma.GenericHeading: "bold #000080", + chroma.GenericSubheading: "bold #800080", + chroma.GenericDeleted: "#A00000", + chroma.GenericInserted: "#008400", + chroma.GenericError: "#E40000", + chroma.GenericEmph: "italic", + chroma.GenericStrong: "bold", + chroma.GenericPrompt: "bold #000080", + chroma.GenericOutput: "#717171", + chroma.GenericTraceback: "#04D", + chroma.Error: "border:#FF0000", + chroma.Background: " bg:#f8f8f8", + })) +}}).Get + +// Clear the background colour. +func clear_background(style *chroma.Style) *chroma.Style { + builder := style.Builder() + bg := builder.Get(chroma.Background) + bg.Background = 0 + bg.NoInherit = true + builder.AddEntry(chroma.Background, bg) + style, _ = builder.Build() + return style +} + +const SGR_PREFIX = "\033[" +const SGR_SUFFIX = "m" + +func ansi_formatter(w io.Writer, style *chroma.Style, it chroma.Iterator) error { + style = clear_background(style) + before, after := make([]byte, 0, 64), make([]byte, 0, 64) + for token := it(); token != chroma.EOF; token = it() { + entry := style.Get(token.Type) + before, after = before[:0], after[:0] + if !entry.IsZero() { + if entry.Bold == chroma.Yes { + before = append(before, '1', ';') + after = append(after, '2', '2', ';') + } + if entry.Underline == chroma.Yes { + before = append(before, '4', ';') + after = append(after, '2', '4', ';') + } + if entry.Italic == chroma.Yes { + before = append(before, '3', ';') + after = append(after, '2', '3', ';') + } + if entry.Colour.IsSet() { + before = append(before, fmt.Sprintf("38:2:%d:%d:%d;", entry.Colour.Red(), entry.Colour.Green(), entry.Colour.Blue())...) + after = append(after, '3', '9', ';') + } + } + if len(before) > 1 { + w.Write(utils.UnsafeStringToBytes(SGR_PREFIX)) + w.Write(before[:len(before)-1]) + w.Write(utils.UnsafeStringToBytes(SGR_SUFFIX)) + } + w.Write(utils.UnsafeStringToBytes(token.Value)) + if len(after) > 1 { + w.Write(utils.UnsafeStringToBytes(SGR_PREFIX)) + w.Write(after[:len(after)-1]) + w.Write(utils.UnsafeStringToBytes(SGR_SUFFIX)) + } + } + return nil +} + +func highlight_file(path string) (highlighted string, err error) { + filename_for_detection := filepath.Base(path) + ext := filepath.Ext(filename_for_detection) + if ext != "" { + ext = strings.ToLower(ext[1:]) + r := conf.Syntax_aliases[ext] + if r != "" { + filename_for_detection = "file." + r + } + } + text, err := data_for_path(path) + if err != nil { + return "", err + } + text = sanitize_control_codes(text) + lexer := lexers.Match(filename_for_detection) + if lexer == nil { + if err == nil { + lexer = lexers.Analyse(text) + } + } + if lexer == nil { + return "", fmt.Errorf("Cannot highlight %#v: %w", path, ErrNoLexer) + } + lexer = chroma.Coalesce(lexer) + name := conf.Pygments_style + const DEFAULT_LIGHT_THEME = "borland" + if name == "default" { + DefaultStyle() + } + style := styles.Get(name) + if style == nil { + if conf.Background.IsDark() && !conf.Foreground.IsDark() { + style = styles.Get("monokai") + if style == nil { + style = styles.Get("github-dark") + } + } else { + style = styles.Get(DEFAULT_LIGHT_THEME) + } + if style == nil { + style = styles.Fallback + } + } + iterator, err := lexer.Tokenise(nil, text) + if err != nil { + return "", err + } + formatter := chroma.FormatterFunc(ansi_formatter) + w := strings.Builder{} + w.Grow(len(text) * 2) + err = formatter.Format(&w, style, iterator) + return w.String(), err +} + +func highlight_all(paths []string) { + ctx := images.Context{} + ctx.Parallel(0, len(paths), func(nums <-chan int) { + for i := range nums { + path := paths[i] + raw, err := highlight_file(path) + if err == nil { + highlighted_lines_cache.Set(path, text_to_lines(sanitize_tabs_and_carriage_returns(raw))) + } + } + }) +} diff --git a/tools/cmd/diff/render.go b/tools/cmd/diff/render.go index 1743e0069..2d547af4e 100644 --- a/tools/cmd/diff/render.go +++ b/tools/cmd/diff/render.go @@ -367,13 +367,13 @@ func lines_for_diff(left_path string, right_path string, patch *Patch, columns, available_cols := columns/2 - margin_size data := DiffData{left_path: left_path, right_path: right_path, available_cols: available_cols, margin_size: margin_size} if left_path != "" { - data.left_lines, err = lines_for_path(left_path) + data.left_lines, err = highlighted_lines_for_path(left_path) if err != nil { return } } if right_path != "" { - data.right_lines, err = lines_for_path(right_path) + data.right_lines, err = highlighted_lines_for_path(right_path) if err != nil { return } @@ -405,7 +405,7 @@ func all_lines(path string, columns, margin_size int, is_add bool, ans []*Logica if !is_add { ltype = `remove` } - lines, err := lines_for_path(path) + lines, err := highlighted_lines_for_path(path) if err != nil { return nil, err } diff --git a/tools/cmd/diff/ui.go b/tools/cmd/diff/ui.go index 172cce5a2..31f0f8f06 100644 --- a/tools/cmd/diff/ui.go +++ b/tools/cmd/diff/ui.go @@ -4,11 +4,13 @@ package diff import ( "fmt" + "strconv" + "strings" + "kitty/tools/config" "kitty/tools/tui/graphics" "kitty/tools/tui/loop" - "strconv" - "strings" + "kitty/tools/utils" ) var _ = fmt.Print @@ -127,11 +129,23 @@ func (self *Handler) on_wakeup() error { } } +func (self *Handler) highlight_all() { + text_files := utils.Filter(self.collection.paths_to_highlight.AsSlice(), is_path_text) + go func() { + r := AsyncResult{rtype: HIGHLIGHT} + highlight_all(text_files) + self.async_results <- r + self.lp.WakeupMainThread() + }() + +} + func (self *Handler) handle_async_result(r AsyncResult) error { switch r.rtype { case COLLECTION: self.collection = r.collection self.generate_diff() + self.highlight_all() case DIFF: self.diff_map = r.diff_map self.calculate_statistics() @@ -147,6 +161,13 @@ func (self *Handler) handle_async_result(r AsyncResult) error { // } self.draw_screen() case HIGHLIGHT: + if self.diff_map != nil && self.collection != nil { + err := self.render_diff() + if err != nil { + return err + } + self.draw_screen() + } } return nil } diff --git a/tools/utils/cache.go b/tools/utils/cache.go index e2aa92200..70fa7bf5d 100644 --- a/tools/utils/cache.go +++ b/tools/utils/cache.go @@ -22,6 +22,20 @@ func NewLRUCache[K comparable, V any](max_size int) *LRUCache[K, V] { return &ans } +func (self *LRUCache[K, V]) Get(key K) (ans V, found bool) { + self.lock.RLock() + ans, found = self.data[key] + self.lock.RUnlock() + return +} + +func (self *LRUCache[K, V]) Set(key K, val V) { + self.lock.RLock() + self.data[key] = val + self.lock.RUnlock() + return +} + func (self *LRUCache[K, V]) GetOrCreate(key K, create func(key K) (V, error)) (V, error) { self.lock.RLock() ans, found := self.data[key] diff --git a/tools/utils/style/wrapper.go b/tools/utils/style/wrapper.go index 9e8fc3fcf..99c8fc877 100644 --- a/tools/utils/style/wrapper.go +++ b/tools/utils/style/wrapper.go @@ -82,6 +82,10 @@ func (self *RGBA) AsRGB() uint32 { return uint32(self.Blue) | (uint32(self.Green) << 8) | (uint32(self.Red) << 16) } +func (self *RGBA) IsDark() bool { + return self.Red < 155 && self.Green < 155 && self.Blue < 155 +} + func (self *RGBA) FromRGB(col uint32) { self.Red = uint8((col >> 16) & 0xff) self.Green = uint8((col >> 8) & 0xff)