More work on getting images to display in diff

tools/cmd/diff/main.go

@@ -149,6 +149,7 @@ func main(_ *cli.Command, opts_ *Options, args []string) (rc int, err error) {
 	lp.OnFinalize = func() string {
 		lp.SetCursorVisible(true)
 		lp.SetCursorShape(loop.BLOCK_CURSOR, true)
+		h.finalize()
 		return ""
 	}
 	lp.OnResize = h.on_resize

tools/cmd/diff/ui.go

@@ -94,9 +94,28 @@ func (self *Handler) update_screen_size(sz loop.ScreenSize) {
 	self.screen_size.cell_width = int(sz.CellWidth)
 }
 
+func (self *Handler) on_escape_code(etype loop.EscapeCodeType, payload []byte) error {
+	switch etype {
+	case loop.APC:
+		gc := graphics.GraphicsCommandFromAPC(payload)
+		if gc != nil {
+			if !image_collection.HandleGraphicsCommand(gc) {
+				self.draw_screen()
+			}
+		}
+	}
+	return nil
+}
+
+func (self *Handler) finalize() {
+	image_collection.Finalize(self.lp)
+}
+
 func (self *Handler) initialize() {
 	self.rl = readline.New(self.lp, readline.RlInit{DontMarkPrompts: true, Prompt: "/"})
+	self.lp.OnEscapeCode = self.on_escape_code
 	image_collection = graphics.NewImageCollection()
+	image_collection.Initialize(self.lp)
 	self.current_context_count = opts.Context
 	if self.current_context_count < 0 {
 		self.current_context_count = int(conf.Num_context_lines)
@@ -291,11 +310,28 @@ func (self *Handler) render_diff() (err error) {
 	return nil
 }
 
+func (self *Handler) draw_image(key string, num_rows, starting_row int) {
+	image_collection.PlaceImageSubRect(self.lp, key, self.images_resized_to, 0, self.screen_size.cell_height*starting_row, -1, -1)
+}
+
+func (self *Handler) draw_image_pair(ll *LogicalLine, starting_row int) {
+	if ll.left_image.key != "" {
+		self.lp.MoveCursorHorizontally(self.logical_lines.margin_size)
+		self.draw_image(ll.left_image.key, ll.left_image.count, starting_row)
+		self.lp.QueueWriteString("\r")
+	}
+	if ll.right_image.key != "" {
+		self.lp.MoveCursorHorizontally(self.logical_lines.margin_size + self.logical_lines.columns/2)
+		self.draw_image(ll.left_image.key, ll.left_image.count, starting_row)
+		self.lp.QueueWriteString("\r")
+	}
+}
+
 func (self *Handler) draw_screen() {
 	self.lp.StartAtomicUpdate()
 	defer self.lp.EndAtomicUpdate()
 	self.resize_all_images_if_needed()
-	image_collection.DeleteAllPlacements(self.lp)
+	image_collection.DeleteAllVisiblePlacements(self.lp)
 	lp.MoveCursorTo(1, 1)
 	lp.ClearToEndOfScreen()
 	if self.logical_lines == nil || self.diff_map == nil || self.collection == nil {
@@ -303,8 +339,15 @@ func (self *Handler) draw_screen() {
 		return
 	}
 	pos := self.scroll_pos
+	seen_images := utils.NewSet[int]()
 	for num_written := 0; num_written < self.screen_size.num_lines; num_written++ {
+		ll := self.logical_lines.At(pos.logical_line)
+		is_image := ll != nil && ll.line_type == IMAGE_LINE
 		sl := self.logical_lines.ScreenLineAt(pos)
+		if is_image && seen_images.Has(pos.logical_line) {
+			seen_images.Add(pos.logical_line)
+			self.draw_image_pair(ll, pos.screen_line)
+		}
 		if self.current_search != nil {
 			sl = self.current_search.markup_line(sl, pos)
 		}

tools/tui/graphics/collection.go

@@ -5,11 +5,17 @@ package graphics
 import (
 	"errors"
 	"fmt"
+	"os"
+	"strings"
 	"sync"
 	"sync/atomic"
 
+	"kitty/tools/tty"
+	"kitty/tools/tui"
 	"kitty/tools/tui/loop"
+	"kitty/tools/utils"
 	"kitty/tools/utils/images"
+	"kitty/tools/utils/shm"
 
 	"golang.org/x/exp/maps"
 )
@@ -18,6 +24,27 @@ var _ = fmt.Print
 
 type Size struct{ Width, Height int }
 
+type rendering struct {
+	img      *images.ImageData
+	image_id uint32
+}
+
+type temp_resource struct {
+	path string
+	mmap shm.MMap
+}
+
+func (self *temp_resource) remove() {
+	if self.path != "" {
+		os.Remove(self.path)
+		self.path = ""
+	}
+	if self.mmap != nil {
+		self.mmap.Unlink()
+		self.mmap = nil
+	}
+}
+
 type Image struct {
 	src struct {
 		path   string
@@ -25,13 +52,24 @@ type Image struct {
 		size   Size
 		loaded bool
 	}
-	renderings map[Size]*images.ImageData
+	renderings map[Size]*rendering
 	err        error
 }
 
+func NewImage() *Image {
+	return &Image{
+		renderings: make(map[Size]*rendering),
+	}
+}
+
 type ImageCollection struct {
-	Shm_supported, Files_supported atomic.Bool
+	Shm_supported, Files_supported      atomic.Bool
-	mutex                          sync.Mutex
+	detection_file_id, detection_shm_id uint32
+	temp_file_map                       map[uint32]*temp_resource
+	running_in_tmux                     bool
+
+	mutex            sync.Mutex
+	image_id_counter uint32
 
 	images map[string]*Image
 }
@@ -51,7 +89,7 @@ func (self *ImageCollection) GetSizeIfAvailable(key string, page_size Size) (Siz
 	if ans == nil {
 		return Size{}, ErrNotFound
 	}
-	return Size{ans.Width, ans.Height}, img.err
+	return Size{ans.img.Width, ans.img.Height}, img.err
 }
 
 func (self *ImageCollection) ResolutionOf(key string) Size {
@@ -71,7 +109,7 @@ func (self *ImageCollection) AddPaths(paths ...string) {
 	defer self.mutex.Unlock()
 	for _, path := range paths {
 		if self.images[path] == nil {
-			i := &Image{}
+			i := NewImage()
 			i.src.path = path
 			self.images[path] = i
 		}
@@ -85,11 +123,11 @@ func (self *Image) ResizeForPageSize(width, height int) {
 	}
 	final_width, final_height := images.FitImage(self.src.size.Width, self.src.size.Height, width, height)
 	if final_width == self.src.size.Width && final_height == self.src.data.Height {
-		self.renderings[sz] = self.src.data
+		self.renderings[sz] = &rendering{img: self.src.data}
 		return
 	}
 	x_frac, y_frac := float64(final_width)/float64(self.src.size.Width), float64(final_height)/float64(self.src.size.Height)
-	self.renderings[sz] = self.src.data.Resize(x_frac, y_frac)
+	self.renderings[sz] = &rendering{img: self.src.data.Resize(x_frac, y_frac)}
 }
 
 func (self *ImageCollection) ResizeForPageSize(width, height int) {
@@ -106,12 +144,137 @@ func (self *ImageCollection) ResizeForPageSize(width, height int) {
 	})
 }
 
-func (self *ImageCollection) DeleteAllPlacements(lp *loop.Loop) {
+func (self *ImageCollection) DeleteAllVisiblePlacements(lp *loop.Loop) {
 	g := &GraphicsCommand{}
 	g.SetAction(GRT_action_delete).SetDelete(GRT_delete_visible)
 	g.WriteWithPayloadToLoop(lp, nil)
 }
 
+func (self *ImageCollection) PlaceImageSubRect(lp *loop.Loop, key string, page_size Size, left, top, width, height int) {
+	self.mutex.Lock()
+	defer self.mutex.Unlock()
+	img := self.images[key]
+	if img == nil {
+		return
+	}
+	r := img.renderings[page_size]
+	if r == nil {
+		return
+	}
+	if r.image_id == 0 {
+		self.transmit_rendering(lp, r)
+	}
+	if width < 0 {
+		width = r.img.Width
+	}
+	if height < 0 {
+		height = r.img.Height
+	}
+	width = utils.Max(0, utils.Min(r.img.Width-left, width))
+	height = utils.Max(0, utils.Min(r.img.Height-top, height))
+	gc := &GraphicsCommand{}
+	gc.SetAction(GRT_action_display).SetLeftEdge(uint64(left)).SetTopEdge(uint64(top)).SetWidth(uint64(width)).SetHeight(uint64(height))
+	gc.SetImageId(r.image_id).SetPlacementId(1)
+	gc.WriteWithPayloadToLoop(lp, nil)
+}
+
+func (self *ImageCollection) Initialize(lp *loop.Loop) {
+	tmux := tui.TmuxSocketAddress()
+	if tmux != "" && tui.TmuxAllowPassthrough() == nil {
+		self.running_in_tmux = true
+	}
+	g := func(t GRT_t, payload string) uint32 {
+		self.image_id_counter++
+		g1 := &GraphicsCommand{}
+		g1.SetTransmission(t).SetAction(GRT_action_query).SetImageId(self.image_id_counter).SetDataWidth(1).SetDataHeight(1).SetFormat(
+			GRT_format_rgb).SetDataSize(uint64(len(payload)))
+		g1.WriteWithPayloadToLoop(lp, utils.UnsafeStringToBytes(payload))
+		return self.image_id_counter
+	}
+	tf, err := images.CreateTempInRAM()
+	if err == nil {
+		tf.Write([]byte{1, 2, 3})
+		tf.Close()
+		self.detection_file_id = g(GRT_transmission_tempfile, tf.Name())
+		self.temp_file_map[self.detection_file_id] = &temp_resource{path: tf.Name()}
+	}
+	sf, err := shm.CreateTemp("icat-", 3)
+	if err == nil {
+		copy(sf.Slice(), []byte{1, 2, 3})
+		sf.Close()
+		self.detection_shm_id = g(GRT_transmission_sharedmem, sf.Name())
+		self.temp_file_map[self.detection_shm_id] = &temp_resource{mmap: sf}
+	}
+}
+
+func (self *ImageCollection) Finalize(lp *loop.Loop) {
+	for _, tr := range self.temp_file_map {
+		tr.remove()
+	}
+}
+
+var DebugPrintln = tty.DebugPrintln
+
+func (self *ImageCollection) mark_img_as_needing_transmission(id uint32) bool {
+	self.mutex.Lock()
+	defer self.mutex.Unlock()
+
+	for _, img := range self.images {
+		for _, r := range img.renderings {
+			if r.image_id == id {
+				r.image_id = 0
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Handle graphics response. Returns false if an image needs re-transmission because
+// the terminal replied with ENOENT for a placement
+func (self *ImageCollection) HandleGraphicsCommand(gc *GraphicsCommand) bool {
+	switch gc.ImageId() {
+	case self.detection_file_id:
+		if gc.ResponseMessage() == "OK" {
+			self.Files_supported.Store(true)
+		} else {
+			if tr := self.temp_file_map[gc.ImageId()]; tr != nil {
+				tr.remove()
+			}
+		}
+		delete(self.temp_file_map, gc.ImageId())
+		self.detection_file_id = 0
+		return true
+	case self.detection_shm_id:
+		if gc.ResponseMessage() == "OK" {
+			self.Shm_supported.Store(true)
+		} else {
+			if tr := self.temp_file_map[gc.ImageId()]; tr != nil {
+				tr.remove()
+			}
+		}
+		delete(self.temp_file_map, gc.ImageId())
+		self.detection_shm_id = 0
+		return true
+	}
+	if is_transmission_response := gc.PlacementId() == 0; is_transmission_response {
+		if gc.ResponseMessage() != "OK" {
+			// this should never happen but lets cleanup anyway
+			if tr := self.temp_file_map[gc.ImageId()]; tr != nil {
+				tr.remove()
+				delete(self.temp_file_map, gc.ImageId())
+			}
+		}
+		return true
+	}
+	if gc.ResponseMessage() != "OK" && gc.PlacementId() != 0 {
+		if self.mark_img_as_needing_transmission(gc.ImageId()) {
+			return false
+		}
+	}
+	return true
+}
+
 func (self *ImageCollection) LoadAll() {
 	self.mutex.Lock()
 	defer self.mutex.Unlock()
@@ -133,9 +296,111 @@ func (self *ImageCollection) LoadAll() {
 func NewImageCollection(paths ...string) *ImageCollection {
 	items := make(map[string]*Image, len(paths))
 	for _, path := range paths {
-		i := &Image{}
+		i := NewImage()
 		i.src.path = path
 		items[path] = i
 	}
-	return &ImageCollection{images: items}
+	return &ImageCollection{images: items, temp_file_map: make(map[uint32]*temp_resource)}
+}
+
+func (self *ImageCollection) new_graphics_command() *GraphicsCommand {
+	gc := GraphicsCommand{}
+	if self.running_in_tmux {
+		gc.WrapPrefix = "\033Ptmux;"
+		gc.WrapSuffix = "\033\\"
+		gc.EncodeSerializedDataFunc = func(x string) string { return strings.ReplaceAll(x, "\033", "\033\033") }
+	}
+	return &gc
+}
+
+func transmit_by_escape_code(lp *loop.Loop, image_id uint32, temp_file_map map[uint32]*temp_resource, frame *images.ImageFrame, gc *GraphicsCommand) {
+	gc.SetTransmission(GRT_transmission_direct)
+	gc.WriteWithPayloadToLoop(lp, frame.Data())
+}
+
+func transmit_by_shm(lp *loop.Loop, image_id uint32, temp_file_map map[uint32]*temp_resource, frame *images.ImageFrame, gc *GraphicsCommand) {
+	mmap, err := frame.DataAsSHM("kdiff-img-*")
+	if err != nil {
+		transmit_by_escape_code(lp, image_id, temp_file_map, frame, gc)
+		return
+	}
+	mmap.Close()
+	temp_file_map[image_id] = &temp_resource{mmap: mmap}
+	gc.SetTransmission(GRT_transmission_sharedmem)
+	gc.WriteWithPayloadToLoop(lp, utils.UnsafeStringToBytes(mmap.Name()))
+}
+
+func transmit_by_file(lp *loop.Loop, image_id uint32, temp_file_map map[uint32]*temp_resource, frame *images.ImageFrame, gc *GraphicsCommand) {
+	f, err := images.CreateTempInRAM()
+	if err != nil {
+		transmit_by_escape_code(lp, image_id, temp_file_map, frame, gc)
+		return
+	}
+	defer f.Close()
+	temp_file_map[image_id] = &temp_resource{path: f.Name()}
+	_, err = f.Write(frame.Data())
+	if err != nil {
+		transmit_by_escape_code(lp, image_id, temp_file_map, frame, gc)
+		return
+	}
+	gc.SetTransmission(GRT_transmission_tempfile)
+	gc.WriteWithPayloadToLoop(lp, utils.UnsafeStringToBytes(f.Name()))
+}
+
+func (self *ImageCollection) transmit_rendering(lp *loop.Loop, r *rendering) {
+	if r.image_id == 0 {
+		self.image_id_counter++
+		r.image_id = self.image_id_counter
+	}
+	is_animated := len(r.img.Frames) > 0
+	transmit := transmit_by_escape_code
+	if self.Shm_supported.Load() {
+		transmit = transmit_by_shm
+	} else if self.Files_supported.Load() {
+		transmit = transmit_by_file
+	}
+
+	frame_control_cmd := self.new_graphics_command()
+	frame_control_cmd.SetAction(GRT_action_animate).SetImageId(r.image_id)
+	for frame_num, frame := range r.img.Frames {
+		gc := self.new_graphics_command()
+		gc.SetImageId(r.image_id)
+		gc.SetDataWidth(uint64(frame.Width)).SetDataHeight(uint64(frame.Height))
+		if frame.Is_opaque {
+			gc.SetFormat(GRT_format_rgb)
+		}
+		switch frame_num {
+		case 0:
+			gc.SetAction(GRT_action_transmit)
+			gc.SetCursorMovement(GRT_cursor_static)
+		default:
+			gc.SetAction(GRT_action_frame)
+			gc.SetGap(frame.Delay_ms)
+			if frame.Compose_onto > 0 {
+				gc.SetOverlaidFrame(uint64(frame.Compose_onto))
+			}
+			gc.SetLeftEdge(uint64(frame.Left)).SetTopEdge(uint64(frame.Top))
+		}
+		transmit(lp, r.image_id, self.temp_file_map, frame, gc)
+		if is_animated {
+			switch frame_num {
+			case 0:
+				// set gap for the first frame and number of loops for the animation
+				c := frame_control_cmd
+				c.SetTargetFrame(uint64(frame.Number))
+				c.SetGap(int32(frame.Delay_ms))
+				c.SetNumberOfLoops(1)
+				c.WriteWithPayloadToLoop(lp, nil)
+			case 1:
+				c := frame_control_cmd
+				c.SetAnimationControl(2) // set animation to loading mode
+				c.WriteWithPayloadToLoop(lp, nil)
+			}
+		}
+	}
+	if is_animated {
+		c := frame_control_cmd
+		c.SetAnimationControl(3) // set animation to normal mode
+		c.WriteWithPayloadToLoop(lp, nil)
+	}
 }

tools/tui/graphics/command.go

@@ -497,7 +497,11 @@ func (self *GraphicsCommand) serialize_non_default_fields() (ans []string) {
 }
 
 func (self GraphicsCommand) String() string {
-	return "GraphicsCommand(" + strings.Join(self.serialize_non_default_fields(), ", ") + ")"
+	ans := "GraphicsCommand(" + strings.Join(self.serialize_non_default_fields(), ", ")
+	if self.response_message != "" {
+		ans += fmt.Sprintf(", response=%#v", self.response_message)
+	}
+	return ans + ")"
 }
 
 func (self *GraphicsCommand) serialize_to(buf io.StringWriter, chunk string) (err error) {
@@ -921,15 +925,6 @@ func (self *GraphicsCommand) SetTopEdge(y uint64) *GraphicsCommand {
 	return self
 }
 
-func (self *GraphicsCommand) SourceLeftEdge() uint64 {
-	return self.X
-}
-
-func (self *GraphicsCommand) SetSourceLeftEdge(x uint64) *GraphicsCommand {
-	self.X = x
-	return self
-}
-
 func (self *GraphicsCommand) XOffset() uint64 {
 	return self.X
 }

tools/utils/images/loading.go

@@ -51,6 +51,75 @@ type ImageFrame struct {
 	Img                      image.Image
 }
 
+func (self *ImageFrame) DataAsSHM(pattern string) (ans shm.MMap, err error) {
+	bytes_per_pixel := 4
+	if self.Is_opaque {
+		bytes_per_pixel = 3
+	}
+	ans, err = shm.CreateTemp(pattern, uint64(self.Width*self.Height*bytes_per_pixel))
+	if err != nil {
+		return nil, err
+	}
+	switch img := self.Img.(type) {
+	case *NRGB:
+		if bytes_per_pixel == 3 {
+			copy(ans.Slice(), img.Pix)
+			return
+		}
+	case *image.NRGBA:
+		if bytes_per_pixel == 4 {
+			copy(ans.Slice(), img.Pix)
+			return
+		}
+	}
+	dest_rect := image.Rect(0, 0, self.Width, self.Height)
+	var final_img image.Image
+	switch bytes_per_pixel {
+	case 3:
+		rgb := &NRGB{Pix: ans.Slice(), Stride: bytes_per_pixel * self.Width, Rect: dest_rect}
+		final_img = rgb
+	case 4:
+		rgba := &image.NRGBA{Pix: ans.Slice(), Stride: bytes_per_pixel * self.Width, Rect: dest_rect}
+		final_img = rgba
+	}
+	ctx := Context{}
+	ctx.PasteCenter(final_img, self.Img, nil)
+	return
+
+}
+
+func (self *ImageFrame) Data() (ans []byte) {
+	bytes_per_pixel := 4
+	if self.Is_opaque {
+		bytes_per_pixel = 3
+	}
+	switch img := self.Img.(type) {
+	case *NRGB:
+		if bytes_per_pixel == 3 {
+			return img.Pix
+		}
+	case *image.NRGBA:
+		if bytes_per_pixel == 4 {
+			return img.Pix
+		}
+	}
+	dest_rect := image.Rect(0, 0, self.Width, self.Height)
+	var final_img image.Image
+	switch bytes_per_pixel {
+	case 3:
+		rgb := NewNRGB(dest_rect)
+		final_img = rgb
+		ans = rgb.Pix
+	case 4:
+		rgba := image.NewNRGBA(dest_rect)
+		final_img = rgba
+		ans = rgba.Pix
+	}
+	ctx := Context{}
+	ctx.PasteCenter(final_img, self.Img, nil)
+	return
+}
+
 type ImageData struct {
 	Width, Height    int
 	Format_uppercase string