Implement syntax highlighting

go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924
+	github.com/alecthomas/chroma/v2 v2.7.0
 	github.com/bmatcuk/doublestar v1.3.4
 	github.com/disintegration/imaging v1.6.2
 	github.com/google/go-cmp v0.5.9
@@ -17,6 +18,7 @@ require (
 )
 
 require (
+	github.com/dlclark/regexp2 v1.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect

go.sum

@@ -1,5 +1,9 @@
 github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924 h1:DG4UyTVIujioxwJc8Zj8Nabz1L1wTgQ/xNBSQDfdP3I=
 github.com/ALTree/bigfloat v0.0.0-20220102081255-38c8b72a9924/go.mod h1:+NaH2gLeY6RPBPPQf4aRotPPStg+eXc8f9ZaE4vRfD4=
+github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink=
+github.com/alecthomas/chroma/v2 v2.7.0 h1:hm1rY6c/Ob4eGclpQ7X/A3yhqBOZNUTk9q+yhyLIViI=
+github.com/alecthomas/chroma/v2 v2.7.0/go.mod h1:yrkMI9807G1ROx13fhe1v6PN2DDeaR73L3d+1nmYQtw=
+github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
 github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
 github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -7,6 +11,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
+github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -14,6 +20,7 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/jamesruan/go-rfc1924 v0.0.0-20170108144916-2767ca7c638f h1:Ko4+g6K16vSyUrtd/pPXuQnWsiHe5BYptEtTxfwYwCc=
 github.com/jamesruan/go-rfc1924 v0.0.0-20170108144916-2767ca7c638f/go.mod h1:eHzfhOKbTGJEGPSdMHzU6jft192tHHt2Bu2vIZArvC0=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=

tools/cmd/diff/collect.go

@@ -6,11 +6,12 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io/fs"
-	"kitty/tools/utils"
 	"os"
 	"path/filepath"
 	"strings"
 	"unicode/utf8"
+
+	"kitty/tools/utils"
 )
 
 var _ = fmt.Print
@@ -18,6 +19,7 @@ var path_name_map, remote_dirs map[string]string
 
 var mimetypes_cache, data_cache, hash_cache *utils.LRUCache[string, string]
 var lines_cache *utils.LRUCache[string, []string]
+var highlighted_lines_cache *utils.LRUCache[string, []string]
 var is_text_cache *utils.LRUCache[string, bool]
 
 func init_caches() {
@@ -28,6 +30,7 @@ func init_caches() {
 	data_cache = utils.NewLRUCache[string, string](sz)
 	is_text_cache = utils.NewLRUCache[string, bool](sz)
 	lines_cache = utils.NewLRUCache[string, []string](sz)
+	highlighted_lines_cache = utils.NewLRUCache[string, []string](sz)
 	hash_cache = utils.NewLRUCache[string, string](sz)
 }
 
@@ -100,34 +103,55 @@ func hash_for_path(path string) (string, error) {
 
 }
 
-func sanitize(x string) string {
+func sanitize_control_codes(x string) string {
-	x = strings.ReplaceAll(x, "\r\n", "⏎\n")
 	return utils.SanitizeControlCodes(x, "░")
 }
 
+func sanitize_tabs_and_carriage_returns(x string) string {
+	return strings.NewReplacer("\t", conf.Replace_tab_by, "\r", "⏎").Replace(x)
+}
+
+func sanitize(x string) string {
+	x = sanitize_control_codes(x)
+	return sanitize_tabs_and_carriage_returns(x)
+}
+
+func text_to_lines(text string) []string {
+	lines := make([]string, 0, 512)
+	splitlines_like_git(text, false, func(line string) { lines = append(lines, line) })
+	return lines
+}
+
 func lines_for_path(path string) ([]string, error) {
 	return lines_cache.GetOrCreate(path, func(path string) ([]string, error) {
 		ans, err := data_for_path(path)
 		if err != nil {
 			return nil, err
 		}
-		ans = sanitize(strings.ReplaceAll(ans, "\t", conf.Replace_tab_by))
+		return text_to_lines(sanitize(ans)), nil
-		lines := make([]string, 0, 256)
-		splitlines_like_git(ans, false, func(line string) { lines = append(lines, line) })
-		return lines, nil
 	})
 }
 
+func highlighted_lines_for_path(path string) ([]string, error) {
+	if ans, found := highlighted_lines_cache.Get(path); found && ans != nil {
+		return ans, nil
+	}
+	return lines_for_path(path)
+}
+
 type Collection struct {
 	changes, renames, type_map map[string]string
 	adds, removes              *utils.Set[string]
 	all_paths                  []string
+	paths_to_highlight         *utils.Set[string]
 	added_count, removed_count int
 }
 
 func (self *Collection) add_change(left, right string) {
 	self.changes[left] = right
 	self.all_paths = append(self.all_paths, left)
+	self.paths_to_highlight.Add(left)
+	self.paths_to_highlight.Add(right)
 	self.type_map[left] = `diff`
 }
 
@@ -140,6 +164,7 @@ func (self *Collection) add_rename(left, right string) {
 func (self *Collection) add_add(right string) {
 	self.adds.Add(right)
 	self.all_paths = append(self.all_paths, right)
+	self.paths_to_highlight.Add(right)
 	self.type_map[right] = `add`
 	if is_path_text(right) {
 		num, _ := lines_for_path(right)
@@ -150,6 +175,7 @@ func (self *Collection) add_add(right string) {
 func (self *Collection) add_removal(left string) {
 	self.removes.Add(left)
 	self.all_paths = append(self.all_paths, left)
+	self.paths_to_highlight.Add(left)
 	self.type_map[left] = `removal`
 	if is_path_text(left) {
 		num, _ := lines_for_path(left)
@@ -308,6 +334,7 @@ func create_collection(left, right string) (ans *Collection, err error) {
 		type_map:           make(map[string]string),
 		adds:               utils.NewSet[string](32),
 		removes:            utils.NewSet[string](32),
+		paths_to_highlight: utils.NewSet[string](32),
 		all_paths:          make([]string, 0, 32),
 	}
 	left_stat, err := os.Stat(left)

tools/cmd/diff/highlight.go

@@ -0,0 +1,187 @@
+// License: GPLv3 Copyright: 2023, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package diff
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"kitty/tools/utils"
+	"kitty/tools/utils/images"
+	"path/filepath"
+	"strings"
+
+	"github.com/alecthomas/chroma/v2"
+	"github.com/alecthomas/chroma/v2/lexers"
+	"github.com/alecthomas/chroma/v2/styles"
+)
+
+var _ = fmt.Print
+var ErrNoLexer = errors.New("No lexer available for this format")
+var DefaultStyle = (&utils.Once[*chroma.Style]{Run: func() *chroma.Style {
+	// Default style generated by python style.py default pygments.styles.default.DefaultStyle
+	// with https://raw.githubusercontent.com/alecthomas/chroma/master/_tools/style.py
+	return styles.Register(chroma.MustNewStyle("default", chroma.StyleEntries{
+		chroma.TextWhitespace:        "#bbbbbb",
+		chroma.Comment:               "italic #3D7B7B",
+		chroma.CommentPreproc:        "noitalic #9C6500",
+		chroma.Keyword:               "bold #008000",
+		chroma.KeywordPseudo:         "nobold",
+		chroma.KeywordType:           "nobold #B00040",
+		chroma.Operator:              "#666666",
+		chroma.OperatorWord:          "bold #AA22FF",
+		chroma.NameBuiltin:           "#008000",
+		chroma.NameFunction:          "#0000FF",
+		chroma.NameClass:             "bold #0000FF",
+		chroma.NameNamespace:         "bold #0000FF",
+		chroma.NameException:         "bold #CB3F38",
+		chroma.NameVariable:          "#19177C",
+		chroma.NameConstant:          "#880000",
+		chroma.NameLabel:             "#767600",
+		chroma.NameEntity:            "bold #717171",
+		chroma.NameAttribute:         "#687822",
+		chroma.NameTag:               "bold #008000",
+		chroma.NameDecorator:         "#AA22FF",
+		chroma.LiteralString:         "#BA2121",
+		chroma.LiteralStringDoc:      "italic",
+		chroma.LiteralStringInterpol: "bold #A45A77",
+		chroma.LiteralStringEscape:   "bold #AA5D1F",
+		chroma.LiteralStringRegex:    "#A45A77",
+		chroma.LiteralStringSymbol:   "#19177C",
+		chroma.LiteralStringOther:    "#008000",
+		chroma.LiteralNumber:         "#666666",
+		chroma.GenericHeading:        "bold #000080",
+		chroma.GenericSubheading:     "bold #800080",
+		chroma.GenericDeleted:        "#A00000",
+		chroma.GenericInserted:       "#008400",
+		chroma.GenericError:          "#E40000",
+		chroma.GenericEmph:           "italic",
+		chroma.GenericStrong:         "bold",
+		chroma.GenericPrompt:         "bold #000080",
+		chroma.GenericOutput:         "#717171",
+		chroma.GenericTraceback:      "#04D",
+		chroma.Error:                 "border:#FF0000",
+		chroma.Background:            " bg:#f8f8f8",
+	}))
+}}).Get
+
+// Clear the background colour.
+func clear_background(style *chroma.Style) *chroma.Style {
+	builder := style.Builder()
+	bg := builder.Get(chroma.Background)
+	bg.Background = 0
+	bg.NoInherit = true
+	builder.AddEntry(chroma.Background, bg)
+	style, _ = builder.Build()
+	return style
+}
+
+const SGR_PREFIX = "\033["
+const SGR_SUFFIX = "m"
+
+func ansi_formatter(w io.Writer, style *chroma.Style, it chroma.Iterator) error {
+	style = clear_background(style)
+	before, after := make([]byte, 0, 64), make([]byte, 0, 64)
+	for token := it(); token != chroma.EOF; token = it() {
+		entry := style.Get(token.Type)
+		before, after = before[:0], after[:0]
+		if !entry.IsZero() {
+			if entry.Bold == chroma.Yes {
+				before = append(before, '1', ';')
+				after = append(after, '2', '2', ';')
+			}
+			if entry.Underline == chroma.Yes {
+				before = append(before, '4', ';')
+				after = append(after, '2', '4', ';')
+			}
+			if entry.Italic == chroma.Yes {
+				before = append(before, '3', ';')
+				after = append(after, '2', '3', ';')
+			}
+			if entry.Colour.IsSet() {
+				before = append(before, fmt.Sprintf("38:2:%d:%d:%d;", entry.Colour.Red(), entry.Colour.Green(), entry.Colour.Blue())...)
+				after = append(after, '3', '9', ';')
+			}
+		}
+		if len(before) > 1 {
+			w.Write(utils.UnsafeStringToBytes(SGR_PREFIX))
+			w.Write(before[:len(before)-1])
+			w.Write(utils.UnsafeStringToBytes(SGR_SUFFIX))
+		}
+		w.Write(utils.UnsafeStringToBytes(token.Value))
+		if len(after) > 1 {
+			w.Write(utils.UnsafeStringToBytes(SGR_PREFIX))
+			w.Write(after[:len(after)-1])
+			w.Write(utils.UnsafeStringToBytes(SGR_SUFFIX))
+		}
+	}
+	return nil
+}
+
+func highlight_file(path string) (highlighted string, err error) {
+	filename_for_detection := filepath.Base(path)
+	ext := filepath.Ext(filename_for_detection)
+	if ext != "" {
+		ext = strings.ToLower(ext[1:])
+		r := conf.Syntax_aliases[ext]
+		if r != "" {
+			filename_for_detection = "file." + r
+		}
+	}
+	text, err := data_for_path(path)
+	if err != nil {
+		return "", err
+	}
+	text = sanitize_control_codes(text)
+	lexer := lexers.Match(filename_for_detection)
+	if lexer == nil {
+		if err == nil {
+			lexer = lexers.Analyse(text)
+		}
+	}
+	if lexer == nil {
+		return "", fmt.Errorf("Cannot highlight %#v: %w", path, ErrNoLexer)
+	}
+	lexer = chroma.Coalesce(lexer)
+	name := conf.Pygments_style
+	const DEFAULT_LIGHT_THEME = "borland"
+	if name == "default" {
+		DefaultStyle()
+	}
+	style := styles.Get(name)
+	if style == nil {
+		if conf.Background.IsDark() && !conf.Foreground.IsDark() {
+			style = styles.Get("monokai")
+			if style == nil {
+				style = styles.Get("github-dark")
+			}
+		} else {
+			style = styles.Get(DEFAULT_LIGHT_THEME)
+		}
+		if style == nil {
+			style = styles.Fallback
+		}
+	}
+	iterator, err := lexer.Tokenise(nil, text)
+	if err != nil {
+		return "", err
+	}
+	formatter := chroma.FormatterFunc(ansi_formatter)
+	w := strings.Builder{}
+	w.Grow(len(text) * 2)
+	err = formatter.Format(&w, style, iterator)
+	return w.String(), err
+}
+
+func highlight_all(paths []string) {
+	ctx := images.Context{}
+	ctx.Parallel(0, len(paths), func(nums <-chan int) {
+		for i := range nums {
+			path := paths[i]
+			raw, err := highlight_file(path)
+			if err == nil {
+				highlighted_lines_cache.Set(path, text_to_lines(sanitize_tabs_and_carriage_returns(raw)))
+			}
+		}
+	})
+}

tools/cmd/diff/render.go

@@ -367,13 +367,13 @@ func lines_for_diff(left_path string, right_path string, patch *Patch, columns,
 	available_cols := columns/2 - margin_size
 	data := DiffData{left_path: left_path, right_path: right_path, available_cols: available_cols, margin_size: margin_size}
 	if left_path != "" {
-		data.left_lines, err = lines_for_path(left_path)
+		data.left_lines, err = highlighted_lines_for_path(left_path)
 		if err != nil {
 			return
 		}
 	}
 	if right_path != "" {
-		data.right_lines, err = lines_for_path(right_path)
+		data.right_lines, err = highlighted_lines_for_path(right_path)
 		if err != nil {
 			return
 		}
@@ -405,7 +405,7 @@ func all_lines(path string, columns, margin_size int, is_add bool, ans []*Logica
 	if !is_add {
 		ltype = `remove`
 	}
-	lines, err := lines_for_path(path)
+	lines, err := highlighted_lines_for_path(path)
 	if err != nil {
 		return nil, err
 	}

tools/cmd/diff/ui.go

@@ -4,11 +4,13 @@ package diff
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
+
 	"kitty/tools/config"
 	"kitty/tools/tui/graphics"
 	"kitty/tools/tui/loop"
-	"strconv"
+	"kitty/tools/utils"
-	"strings"
 )
 
 var _ = fmt.Print
@@ -127,11 +129,23 @@ func (self *Handler) on_wakeup() error {
 	}
 }
 
+func (self *Handler) highlight_all() {
+	text_files := utils.Filter(self.collection.paths_to_highlight.AsSlice(), is_path_text)
+	go func() {
+		r := AsyncResult{rtype: HIGHLIGHT}
+		highlight_all(text_files)
+		self.async_results <- r
+		self.lp.WakeupMainThread()
+	}()
+
+}
+
 func (self *Handler) handle_async_result(r AsyncResult) error {
 	switch r.rtype {
 	case COLLECTION:
 		self.collection = r.collection
 		self.generate_diff()
+		self.highlight_all()
 	case DIFF:
 		self.diff_map = r.diff_map
 		self.calculate_statistics()
@@ -147,6 +161,13 @@ func (self *Handler) handle_async_result(r AsyncResult) error {
 		// }
 		self.draw_screen()
 	case HIGHLIGHT:
+		if self.diff_map != nil && self.collection != nil {
+			err := self.render_diff()
+			if err != nil {
+				return err
+			}
+			self.draw_screen()
+		}
 	}
 	return nil
 }

tools/utils/cache.go

@@ -22,6 +22,20 @@ func NewLRUCache[K comparable, V any](max_size int) *LRUCache[K, V] {
 	return &ans
 }
 
+func (self *LRUCache[K, V]) Get(key K) (ans V, found bool) {
+	self.lock.RLock()
+	ans, found = self.data[key]
+	self.lock.RUnlock()
+	return
+}
+
+func (self *LRUCache[K, V]) Set(key K, val V) {
+	self.lock.RLock()
+	self.data[key] = val
+	self.lock.RUnlock()
+	return
+}
+
 func (self *LRUCache[K, V]) GetOrCreate(key K, create func(key K) (V, error)) (V, error) {
 	self.lock.RLock()
 	ans, found := self.data[key]

tools/utils/style/wrapper.go

@@ -82,6 +82,10 @@ func (self *RGBA) AsRGB() uint32 {
 	return uint32(self.Blue) | (uint32(self.Green) << 8) | (uint32(self.Red) << 16)
 }
 
+func (self *RGBA) IsDark() bool {
+	return self.Red < 155 && self.Green < 155 && self.Blue < 155
+}
+
 func (self *RGBA) FromRGB(col uint32) {
 	self.Red = uint8((col >> 16) & 0xff)
 	self.Green = uint8((col >> 8) & 0xff)