From d66da811dbbfe709a21691631b28095b31b0f74e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 26 Mar 2023 21:57:30 +0530 Subject: [PATCH] More work on getting images to display in diff --- tools/cmd/diff/main.go | 1 + tools/cmd/diff/ui.go | 45 ++++- tools/tui/graphics/collection.go | 285 +++++++++++++++++++++++++++++-- tools/tui/graphics/command.go | 15 +- tools/utils/images/loading.go | 69 ++++++++ 5 files changed, 394 insertions(+), 21 deletions(-) diff --git a/tools/cmd/diff/main.go b/tools/cmd/diff/main.go index b4814a859..cb15502da 100644 --- a/tools/cmd/diff/main.go +++ b/tools/cmd/diff/main.go @@ -149,6 +149,7 @@ func main(_ *cli.Command, opts_ *Options, args []string) (rc int, err error) { lp.OnFinalize = func() string { lp.SetCursorVisible(true) lp.SetCursorShape(loop.BLOCK_CURSOR, true) + h.finalize() return "" } lp.OnResize = h.on_resize diff --git a/tools/cmd/diff/ui.go b/tools/cmd/diff/ui.go index bb576fc4c..8f5087067 100644 --- a/tools/cmd/diff/ui.go +++ b/tools/cmd/diff/ui.go @@ -94,9 +94,28 @@ func (self *Handler) update_screen_size(sz loop.ScreenSize) { self.screen_size.cell_width = int(sz.CellWidth) } +func (self *Handler) on_escape_code(etype loop.EscapeCodeType, payload []byte) error { + switch etype { + case loop.APC: + gc := graphics.GraphicsCommandFromAPC(payload) + if gc != nil { + if !image_collection.HandleGraphicsCommand(gc) { + self.draw_screen() + } + } + } + return nil +} + +func (self *Handler) finalize() { + image_collection.Finalize(self.lp) +} + func (self *Handler) initialize() { self.rl = readline.New(self.lp, readline.RlInit{DontMarkPrompts: true, Prompt: "/"}) + self.lp.OnEscapeCode = self.on_escape_code image_collection = graphics.NewImageCollection() + image_collection.Initialize(self.lp) self.current_context_count = opts.Context if self.current_context_count < 0 { self.current_context_count = int(conf.Num_context_lines) @@ -291,11 +310,28 @@ func (self *Handler) render_diff() (err error) { return nil } +func (self *Handler) draw_image(key string, num_rows, starting_row int) { + image_collection.PlaceImageSubRect(self.lp, key, self.images_resized_to, 0, self.screen_size.cell_height*starting_row, -1, -1) +} + +func (self *Handler) draw_image_pair(ll *LogicalLine, starting_row int) { + if ll.left_image.key != "" { + self.lp.MoveCursorHorizontally(self.logical_lines.margin_size) + self.draw_image(ll.left_image.key, ll.left_image.count, starting_row) + self.lp.QueueWriteString("\r") + } + if ll.right_image.key != "" { + self.lp.MoveCursorHorizontally(self.logical_lines.margin_size + self.logical_lines.columns/2) + self.draw_image(ll.left_image.key, ll.left_image.count, starting_row) + self.lp.QueueWriteString("\r") + } +} + func (self *Handler) draw_screen() { self.lp.StartAtomicUpdate() defer self.lp.EndAtomicUpdate() self.resize_all_images_if_needed() - image_collection.DeleteAllPlacements(self.lp) + image_collection.DeleteAllVisiblePlacements(self.lp) lp.MoveCursorTo(1, 1) lp.ClearToEndOfScreen() if self.logical_lines == nil || self.diff_map == nil || self.collection == nil { @@ -303,8 +339,15 @@ func (self *Handler) draw_screen() { return } pos := self.scroll_pos + seen_images := utils.NewSet[int]() for num_written := 0; num_written < self.screen_size.num_lines; num_written++ { + ll := self.logical_lines.At(pos.logical_line) + is_image := ll != nil && ll.line_type == IMAGE_LINE sl := self.logical_lines.ScreenLineAt(pos) + if is_image && seen_images.Has(pos.logical_line) { + seen_images.Add(pos.logical_line) + self.draw_image_pair(ll, pos.screen_line) + } if self.current_search != nil { sl = self.current_search.markup_line(sl, pos) } diff --git a/tools/tui/graphics/collection.go b/tools/tui/graphics/collection.go index aeb4b251c..689250bfe 100644 --- a/tools/tui/graphics/collection.go +++ b/tools/tui/graphics/collection.go @@ -5,11 +5,17 @@ package graphics import ( "errors" "fmt" + "os" + "strings" "sync" "sync/atomic" + "kitty/tools/tty" + "kitty/tools/tui" "kitty/tools/tui/loop" + "kitty/tools/utils" "kitty/tools/utils/images" + "kitty/tools/utils/shm" "golang.org/x/exp/maps" ) @@ -18,6 +24,27 @@ var _ = fmt.Print type Size struct{ Width, Height int } +type rendering struct { + img *images.ImageData + image_id uint32 +} + +type temp_resource struct { + path string + mmap shm.MMap +} + +func (self *temp_resource) remove() { + if self.path != "" { + os.Remove(self.path) + self.path = "" + } + if self.mmap != nil { + self.mmap.Unlink() + self.mmap = nil + } +} + type Image struct { src struct { path string @@ -25,13 +52,24 @@ type Image struct { size Size loaded bool } - renderings map[Size]*images.ImageData + renderings map[Size]*rendering err error } +func NewImage() *Image { + return &Image{ + renderings: make(map[Size]*rendering), + } +} + type ImageCollection struct { - Shm_supported, Files_supported atomic.Bool - mutex sync.Mutex + Shm_supported, Files_supported atomic.Bool + detection_file_id, detection_shm_id uint32 + temp_file_map map[uint32]*temp_resource + running_in_tmux bool + + mutex sync.Mutex + image_id_counter uint32 images map[string]*Image } @@ -51,7 +89,7 @@ func (self *ImageCollection) GetSizeIfAvailable(key string, page_size Size) (Siz if ans == nil { return Size{}, ErrNotFound } - return Size{ans.Width, ans.Height}, img.err + return Size{ans.img.Width, ans.img.Height}, img.err } func (self *ImageCollection) ResolutionOf(key string) Size { @@ -71,7 +109,7 @@ func (self *ImageCollection) AddPaths(paths ...string) { defer self.mutex.Unlock() for _, path := range paths { if self.images[path] == nil { - i := &Image{} + i := NewImage() i.src.path = path self.images[path] = i } @@ -85,11 +123,11 @@ func (self *Image) ResizeForPageSize(width, height int) { } final_width, final_height := images.FitImage(self.src.size.Width, self.src.size.Height, width, height) if final_width == self.src.size.Width && final_height == self.src.data.Height { - self.renderings[sz] = self.src.data + self.renderings[sz] = &rendering{img: self.src.data} return } x_frac, y_frac := float64(final_width)/float64(self.src.size.Width), float64(final_height)/float64(self.src.size.Height) - self.renderings[sz] = self.src.data.Resize(x_frac, y_frac) + self.renderings[sz] = &rendering{img: self.src.data.Resize(x_frac, y_frac)} } func (self *ImageCollection) ResizeForPageSize(width, height int) { @@ -106,12 +144,137 @@ func (self *ImageCollection) ResizeForPageSize(width, height int) { }) } -func (self *ImageCollection) DeleteAllPlacements(lp *loop.Loop) { +func (self *ImageCollection) DeleteAllVisiblePlacements(lp *loop.Loop) { g := &GraphicsCommand{} g.SetAction(GRT_action_delete).SetDelete(GRT_delete_visible) g.WriteWithPayloadToLoop(lp, nil) } +func (self *ImageCollection) PlaceImageSubRect(lp *loop.Loop, key string, page_size Size, left, top, width, height int) { + self.mutex.Lock() + defer self.mutex.Unlock() + img := self.images[key] + if img == nil { + return + } + r := img.renderings[page_size] + if r == nil { + return + } + if r.image_id == 0 { + self.transmit_rendering(lp, r) + } + if width < 0 { + width = r.img.Width + } + if height < 0 { + height = r.img.Height + } + width = utils.Max(0, utils.Min(r.img.Width-left, width)) + height = utils.Max(0, utils.Min(r.img.Height-top, height)) + gc := &GraphicsCommand{} + gc.SetAction(GRT_action_display).SetLeftEdge(uint64(left)).SetTopEdge(uint64(top)).SetWidth(uint64(width)).SetHeight(uint64(height)) + gc.SetImageId(r.image_id).SetPlacementId(1) + gc.WriteWithPayloadToLoop(lp, nil) +} + +func (self *ImageCollection) Initialize(lp *loop.Loop) { + tmux := tui.TmuxSocketAddress() + if tmux != "" && tui.TmuxAllowPassthrough() == nil { + self.running_in_tmux = true + } + g := func(t GRT_t, payload string) uint32 { + self.image_id_counter++ + g1 := &GraphicsCommand{} + g1.SetTransmission(t).SetAction(GRT_action_query).SetImageId(self.image_id_counter).SetDataWidth(1).SetDataHeight(1).SetFormat( + GRT_format_rgb).SetDataSize(uint64(len(payload))) + g1.WriteWithPayloadToLoop(lp, utils.UnsafeStringToBytes(payload)) + return self.image_id_counter + } + tf, err := images.CreateTempInRAM() + if err == nil { + tf.Write([]byte{1, 2, 3}) + tf.Close() + self.detection_file_id = g(GRT_transmission_tempfile, tf.Name()) + self.temp_file_map[self.detection_file_id] = &temp_resource{path: tf.Name()} + } + sf, err := shm.CreateTemp("icat-", 3) + if err == nil { + copy(sf.Slice(), []byte{1, 2, 3}) + sf.Close() + self.detection_shm_id = g(GRT_transmission_sharedmem, sf.Name()) + self.temp_file_map[self.detection_shm_id] = &temp_resource{mmap: sf} + } +} + +func (self *ImageCollection) Finalize(lp *loop.Loop) { + for _, tr := range self.temp_file_map { + tr.remove() + } +} + +var DebugPrintln = tty.DebugPrintln + +func (self *ImageCollection) mark_img_as_needing_transmission(id uint32) bool { + self.mutex.Lock() + defer self.mutex.Unlock() + + for _, img := range self.images { + for _, r := range img.renderings { + if r.image_id == id { + r.image_id = 0 + return true + } + } + } + return false +} + +// Handle graphics response. Returns false if an image needs re-transmission because +// the terminal replied with ENOENT for a placement +func (self *ImageCollection) HandleGraphicsCommand(gc *GraphicsCommand) bool { + switch gc.ImageId() { + case self.detection_file_id: + if gc.ResponseMessage() == "OK" { + self.Files_supported.Store(true) + } else { + if tr := self.temp_file_map[gc.ImageId()]; tr != nil { + tr.remove() + } + } + delete(self.temp_file_map, gc.ImageId()) + self.detection_file_id = 0 + return true + case self.detection_shm_id: + if gc.ResponseMessage() == "OK" { + self.Shm_supported.Store(true) + } else { + if tr := self.temp_file_map[gc.ImageId()]; tr != nil { + tr.remove() + } + } + delete(self.temp_file_map, gc.ImageId()) + self.detection_shm_id = 0 + return true + } + if is_transmission_response := gc.PlacementId() == 0; is_transmission_response { + if gc.ResponseMessage() != "OK" { + // this should never happen but lets cleanup anyway + if tr := self.temp_file_map[gc.ImageId()]; tr != nil { + tr.remove() + delete(self.temp_file_map, gc.ImageId()) + } + } + return true + } + if gc.ResponseMessage() != "OK" && gc.PlacementId() != 0 { + if self.mark_img_as_needing_transmission(gc.ImageId()) { + return false + } + } + return true +} + func (self *ImageCollection) LoadAll() { self.mutex.Lock() defer self.mutex.Unlock() @@ -133,9 +296,111 @@ func (self *ImageCollection) LoadAll() { func NewImageCollection(paths ...string) *ImageCollection { items := make(map[string]*Image, len(paths)) for _, path := range paths { - i := &Image{} + i := NewImage() i.src.path = path items[path] = i } - return &ImageCollection{images: items} + return &ImageCollection{images: items, temp_file_map: make(map[uint32]*temp_resource)} +} + +func (self *ImageCollection) new_graphics_command() *GraphicsCommand { + gc := GraphicsCommand{} + if self.running_in_tmux { + gc.WrapPrefix = "\033Ptmux;" + gc.WrapSuffix = "\033\\" + gc.EncodeSerializedDataFunc = func(x string) string { return strings.ReplaceAll(x, "\033", "\033\033") } + } + return &gc +} + +func transmit_by_escape_code(lp *loop.Loop, image_id uint32, temp_file_map map[uint32]*temp_resource, frame *images.ImageFrame, gc *GraphicsCommand) { + gc.SetTransmission(GRT_transmission_direct) + gc.WriteWithPayloadToLoop(lp, frame.Data()) +} + +func transmit_by_shm(lp *loop.Loop, image_id uint32, temp_file_map map[uint32]*temp_resource, frame *images.ImageFrame, gc *GraphicsCommand) { + mmap, err := frame.DataAsSHM("kdiff-img-*") + if err != nil { + transmit_by_escape_code(lp, image_id, temp_file_map, frame, gc) + return + } + mmap.Close() + temp_file_map[image_id] = &temp_resource{mmap: mmap} + gc.SetTransmission(GRT_transmission_sharedmem) + gc.WriteWithPayloadToLoop(lp, utils.UnsafeStringToBytes(mmap.Name())) +} + +func transmit_by_file(lp *loop.Loop, image_id uint32, temp_file_map map[uint32]*temp_resource, frame *images.ImageFrame, gc *GraphicsCommand) { + f, err := images.CreateTempInRAM() + if err != nil { + transmit_by_escape_code(lp, image_id, temp_file_map, frame, gc) + return + } + defer f.Close() + temp_file_map[image_id] = &temp_resource{path: f.Name()} + _, err = f.Write(frame.Data()) + if err != nil { + transmit_by_escape_code(lp, image_id, temp_file_map, frame, gc) + return + } + gc.SetTransmission(GRT_transmission_tempfile) + gc.WriteWithPayloadToLoop(lp, utils.UnsafeStringToBytes(f.Name())) +} + +func (self *ImageCollection) transmit_rendering(lp *loop.Loop, r *rendering) { + if r.image_id == 0 { + self.image_id_counter++ + r.image_id = self.image_id_counter + } + is_animated := len(r.img.Frames) > 0 + transmit := transmit_by_escape_code + if self.Shm_supported.Load() { + transmit = transmit_by_shm + } else if self.Files_supported.Load() { + transmit = transmit_by_file + } + + frame_control_cmd := self.new_graphics_command() + frame_control_cmd.SetAction(GRT_action_animate).SetImageId(r.image_id) + for frame_num, frame := range r.img.Frames { + gc := self.new_graphics_command() + gc.SetImageId(r.image_id) + gc.SetDataWidth(uint64(frame.Width)).SetDataHeight(uint64(frame.Height)) + if frame.Is_opaque { + gc.SetFormat(GRT_format_rgb) + } + switch frame_num { + case 0: + gc.SetAction(GRT_action_transmit) + gc.SetCursorMovement(GRT_cursor_static) + default: + gc.SetAction(GRT_action_frame) + gc.SetGap(frame.Delay_ms) + if frame.Compose_onto > 0 { + gc.SetOverlaidFrame(uint64(frame.Compose_onto)) + } + gc.SetLeftEdge(uint64(frame.Left)).SetTopEdge(uint64(frame.Top)) + } + transmit(lp, r.image_id, self.temp_file_map, frame, gc) + if is_animated { + switch frame_num { + case 0: + // set gap for the first frame and number of loops for the animation + c := frame_control_cmd + c.SetTargetFrame(uint64(frame.Number)) + c.SetGap(int32(frame.Delay_ms)) + c.SetNumberOfLoops(1) + c.WriteWithPayloadToLoop(lp, nil) + case 1: + c := frame_control_cmd + c.SetAnimationControl(2) // set animation to loading mode + c.WriteWithPayloadToLoop(lp, nil) + } + } + } + if is_animated { + c := frame_control_cmd + c.SetAnimationControl(3) // set animation to normal mode + c.WriteWithPayloadToLoop(lp, nil) + } } diff --git a/tools/tui/graphics/command.go b/tools/tui/graphics/command.go index e4a6dd5b7..7aa07f70d 100644 --- a/tools/tui/graphics/command.go +++ b/tools/tui/graphics/command.go @@ -497,7 +497,11 @@ func (self *GraphicsCommand) serialize_non_default_fields() (ans []string) { } func (self GraphicsCommand) String() string { - return "GraphicsCommand(" + strings.Join(self.serialize_non_default_fields(), ", ") + ")" + ans := "GraphicsCommand(" + strings.Join(self.serialize_non_default_fields(), ", ") + if self.response_message != "" { + ans += fmt.Sprintf(", response=%#v", self.response_message) + } + return ans + ")" } func (self *GraphicsCommand) serialize_to(buf io.StringWriter, chunk string) (err error) { @@ -921,15 +925,6 @@ func (self *GraphicsCommand) SetTopEdge(y uint64) *GraphicsCommand { return self } -func (self *GraphicsCommand) SourceLeftEdge() uint64 { - return self.X -} - -func (self *GraphicsCommand) SetSourceLeftEdge(x uint64) *GraphicsCommand { - self.X = x - return self -} - func (self *GraphicsCommand) XOffset() uint64 { return self.X } diff --git a/tools/utils/images/loading.go b/tools/utils/images/loading.go index 1132b8ee7..229950144 100644 --- a/tools/utils/images/loading.go +++ b/tools/utils/images/loading.go @@ -51,6 +51,75 @@ type ImageFrame struct { Img image.Image } +func (self *ImageFrame) DataAsSHM(pattern string) (ans shm.MMap, err error) { + bytes_per_pixel := 4 + if self.Is_opaque { + bytes_per_pixel = 3 + } + ans, err = shm.CreateTemp(pattern, uint64(self.Width*self.Height*bytes_per_pixel)) + if err != nil { + return nil, err + } + switch img := self.Img.(type) { + case *NRGB: + if bytes_per_pixel == 3 { + copy(ans.Slice(), img.Pix) + return + } + case *image.NRGBA: + if bytes_per_pixel == 4 { + copy(ans.Slice(), img.Pix) + return + } + } + dest_rect := image.Rect(0, 0, self.Width, self.Height) + var final_img image.Image + switch bytes_per_pixel { + case 3: + rgb := &NRGB{Pix: ans.Slice(), Stride: bytes_per_pixel * self.Width, Rect: dest_rect} + final_img = rgb + case 4: + rgba := &image.NRGBA{Pix: ans.Slice(), Stride: bytes_per_pixel * self.Width, Rect: dest_rect} + final_img = rgba + } + ctx := Context{} + ctx.PasteCenter(final_img, self.Img, nil) + return + +} + +func (self *ImageFrame) Data() (ans []byte) { + bytes_per_pixel := 4 + if self.Is_opaque { + bytes_per_pixel = 3 + } + switch img := self.Img.(type) { + case *NRGB: + if bytes_per_pixel == 3 { + return img.Pix + } + case *image.NRGBA: + if bytes_per_pixel == 4 { + return img.Pix + } + } + dest_rect := image.Rect(0, 0, self.Width, self.Height) + var final_img image.Image + switch bytes_per_pixel { + case 3: + rgb := NewNRGB(dest_rect) + final_img = rgb + ans = rgb.Pix + case 4: + rgba := image.NewNRGBA(dest_rect) + final_img = rgba + ans = rgba.Pix + } + ctx := Context{} + ctx.PasteCenter(final_img, self.Img, nil) + return +} + type ImageData struct { Width, Height int Format_uppercase string