Start work on a SHM implementation in Go

tools/utils/shm/shm.go

@@ -0,0 +1,151 @@
+// License: GPLv3 Copyright: 2022, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package shm
+
+import (
+	"errors"
+	"fmt"
+	"math/rand"
+	"os"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sys/unix"
+)
+
+var _ = fmt.Print
+var ErrPatternHasSeparator = errors.New("The specified pattern has file path separators in it")
+var ErrPatternTooLong = errors.New("The specified pattern for the SHM name is too long")
+
+type ErrNotSupported struct {
+	err error
+}
+
+func (self *ErrNotSupported) Error() string {
+	return fmt.Sprintf("POSIX shared memory not supported on this platform: with underlying error: %v", self.err)
+}
+
+// prefix_and_suffix splits pattern by the last wildcard "*", if applicable,
+// returning prefix as the part before "*" and suffix as the part after "*".
+func prefix_and_suffix(pattern string) (prefix, suffix string, err error) {
+	for i := 0; i < len(pattern); i++ {
+		if os.IsPathSeparator(pattern[i]) {
+			return "", "", ErrPatternHasSeparator
+		}
+	}
+	if pos := strings.LastIndexByte(pattern, '*'); pos != -1 {
+		prefix, suffix = pattern[:pos], pattern[pos+1:]
+	} else {
+		prefix = pattern
+	}
+	return prefix, suffix, nil
+}
+
+func next_random() string {
+	num := rand.Uint32()
+	return strconv.FormatUint(uint64(num), 16)
+}
+
+type MMap interface {
+	Close() error
+	Unlink() error
+	Slice() []byte
+	Name() string
+}
+
+type ProtectionFlags int
+
+const (
+	READ ProtectionFlags = iota
+	COPY
+	RDWR
+	EXEC
+	ANON
+)
+
+func mmap(sz int, inprot ProtectionFlags, anonymous bool, fd int, off int64) ([]byte, error) {
+	flags := unix.MAP_SHARED
+	prot := unix.PROT_READ
+	switch {
+	case inprot&COPY != 0:
+		prot |= unix.PROT_WRITE
+		flags = unix.MAP_PRIVATE
+	case inprot&RDWR != 0:
+		prot |= unix.PROT_WRITE
+	}
+	if inprot&EXEC != 0 {
+		prot |= unix.PROT_EXEC
+	}
+	if anonymous {
+		flags |= unix.MAP_ANON
+	}
+
+	b, err := unix.Mmap(fd, off, sz, prot, flags)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+type file_based_mmap struct {
+	f        *os.File
+	region   []byte
+	unlinked bool
+}
+
+func file_mmap(f *os.File, size uint64, access ProtectionFlags, truncate bool) (MMap, error) {
+	if truncate {
+		err := truncate_or_unlink(f, size)
+		if err != nil {
+			return nil, err
+		}
+	}
+	region, err := mmap(int(size), access, false, int(f.Fd()), 0)
+	if err != nil {
+		f.Close()
+		os.Remove(f.Name())
+		return nil, err
+	}
+	return &file_based_mmap{f: f, region: region}, nil
+}
+
+func (self *file_based_mmap) Name() string {
+	return self.f.Name()
+}
+
+func (self *file_based_mmap) Slice() []byte {
+	return self.region
+}
+
+func (self *file_based_mmap) Close() error {
+	err := self.f.Close()
+	self.region = nil
+	return err
+}
+
+func (self *file_based_mmap) Unlink() (err error) {
+	if self.unlinked {
+		return nil
+	}
+	self.unlinked = true
+	return os.Remove(self.f.Name())
+}
+
+func CreateTemp(pattern string, size uint64) (MMap, error) {
+	return create_temp(pattern, size)
+}
+
+func truncate_or_unlink(ans *os.File, size uint64) (err error) {
+	for {
+		err = unix.Ftruncate(int(ans.Fd()), int64(size))
+		if !errors.Is(err, unix.EINTR) {
+			break
+		}
+	}
+	if err != nil {
+		ans.Close()
+		os.Remove(ans.Name())
+		return fmt.Errorf("Failed to ftruncate() SHM file %s to size: %d with error: %w", ans.Name(), size, err)
+	}
+	return
+}

tools/utils/shm/shm_fs.go

@@ -0,0 +1,36 @@
+// License: GPLv3 Copyright: 2022, Kovid Goyal, <kovid at kovidgoyal.net>
+//go:build linux || netbsd
+
+package shm
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+var _ = fmt.Print
+
+func create_temp(pattern string, size uint64) (MMap, error) {
+	ans, err := os.CreateTemp(SHM_DIR, pattern)
+	if err != nil {
+		return nil, err
+	}
+	return file_mmap(ans, size, RDWR, true)
+}
+
+func Open(name string) (MMap, error) {
+	if !filepath.IsAbs(name) {
+		name = filepath.Join(SHM_DIR, name)
+	}
+	ans, err := os.OpenFile(name, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	s, err := os.Stat(name)
+	if err != nil {
+		ans.Close()
+		return nil, err
+	}
+	return file_mmap(ans, uint64(s.Size()), READ, false)
+}

tools/utils/shm/shm_openbsd.go

@@ -0,0 +1,40 @@
+// License: GPLv3 Copyright: 2022, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package shm
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+var _ = fmt.Print
+
+const SHM_DIR = "/tmp"
+
+func create_temp(pattern string) (*os.File, error) {
+	ans, err := os.CreateTemp(SHM_DIR, pattern)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}
+
+func Open(name string) (*os.File, error) {
+	if !filepath.IsAbs(name) {
+		name = filepath.Join(SHM_DIR, name)
+	}
+	ans, err := os.OpenFile(name, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	return ans, nil
+}
+
+func Unlink(name string) error {
+	if !filepath.IsAbs(name) {
+		name = filepath.Join(SHM_DIR, name)
+	}
+	return os.Remove(name)
+
+}

tools/utils/shm/shm_syscall.go

@@ -0,0 +1,152 @@
+// License: GPLv3 Copyright: 2022, Kovid Goyal, <kovid at kovidgoyal.net>
+//go:build darwin || freebsd
+
+package shm
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"strings"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+var _ = fmt.Print
+
+// ByteSliceFromString makes a zero terminated byte slice from the string
+func ByteSliceFromString(s string) []byte {
+	a := make([]byte, len(s)+1)
+	copy(a, s)
+	return a
+}
+
+func BytePtrFromString(s string) *byte {
+	a := ByteSliceFromString(s)
+	return &a[0]
+}
+
+func shm_unlink(name string) (err error) {
+	bname := BytePtrFromString(name)
+	for {
+		_, _, errno := unix.Syscall(unix.SYS_SHM_OPEN, uintptr(unsafe.Pointer(bname)), 0, 0)
+		if errno != unix.EINTR {
+			if errno != 0 {
+				err = fmt.Errorf("shm_unlink() failed with error: %w", errno)
+			}
+			break
+		}
+	}
+	return
+}
+
+func shm_open(name string, flags, perm int) (ans *os.File, err error) {
+	bname := BytePtrFromString(name)
+	var fd uintptr
+	var errno unix.Errno
+	for {
+		fd, _, errno = unix.Syscall(unix.SYS_SHM_OPEN, uintptr(unsafe.Pointer(bname)), uintptr(flags), uintptr(perm))
+		if errno != unix.EINTR {
+			if errno != 0 {
+				err = fmt.Errorf("shm_open() failed with error: %w", errno)
+			}
+			break
+		}
+	}
+	if err == nil {
+		ans = os.NewFile(fd, name)
+	}
+	return
+}
+
+type syscall_based_mmap struct {
+	f        *os.File
+	region   []byte
+	unlinked bool
+}
+
+func syscall_mmap(f *os.File, size uint64, access ProtectionFlags, truncate bool) (MMap, error) {
+	if truncate {
+		err := truncate_or_unlink(f, size)
+		if err != nil {
+			return nil, fmt.Errorf("truncate failed with error: %w", err)
+		}
+	}
+	region, err := mmap(int(size), access, false, int(f.Fd()), 0)
+	if err != nil {
+		f.Close()
+		os.Remove(f.Name())
+		return nil, fmt.Errorf("mmap failed with error: %w", err)
+	}
+	return &syscall_based_mmap{f: f, region: region}, nil
+}
+
+func (self *syscall_based_mmap) Name() string {
+	return self.f.Name()
+}
+
+func (self *syscall_based_mmap) Slice() []byte {
+	return self.region
+}
+
+func (self *syscall_based_mmap) Close() error {
+	err := self.f.Close()
+	self.region = nil
+	return err
+}
+
+func (self *syscall_based_mmap) Unlink() (err error) {
+	if self.unlinked {
+		return nil
+	}
+	self.unlinked = true
+	return shm_unlink(self.Name())
+}
+
+func create_temp(pattern string, size uint64) (ans MMap, err error) {
+	var prefix, suffix string
+	prefix, suffix, err = prefix_and_suffix(pattern)
+	if err != nil {
+		return
+	}
+	if SHM_REQUIRED_PREFIX != "" && !strings.HasPrefix(pattern, SHM_REQUIRED_PREFIX) {
+		// FreeBSD requires name to start with /
+		prefix = SHM_REQUIRED_PREFIX + prefix
+	}
+	var f *os.File
+	try := 0
+	for {
+		name := prefix + next_random() + suffix
+		if len(name) > SHM_NAME_MAX {
+			return nil, ErrPatternTooLong
+		}
+		f, err = shm_open(name, os.O_EXCL|os.O_CREATE|os.O_RDWR, 0600)
+		if err != nil && (errors.Is(err, fs.ErrExist) || errors.Unwrap(err) == unix.EEXIST) {
+			try += 1
+			if try > 10000 {
+				return nil, &os.PathError{Op: "createtemp", Path: prefix + "*" + suffix, Err: ErrExist}
+			}
+			continue
+		}
+		break
+	}
+	if err != nil {
+		return nil, err
+	}
+	return syscall_mmap(f, size, RDWR, true)
+}
+
+func Open(name string) (MMap, error) {
+	ans, err := shm_open(name, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	s, err := os.Stat(name)
+	if err != nil {
+		ans.Close()
+		return nil, err
+	}
+	return syscall_mmap(ans, uint64(s.Size()), READ, false)
+}

tools/utils/shm/shm_test.go

@@ -0,0 +1,43 @@
+// License: GPLv3 Copyright: 2022, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package shm
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"math/rand"
+	"os"
+	"reflect"
+	"testing"
+)
+
+var _ = fmt.Print
+
+func TestSHM(t *testing.T) {
+	data := make([]byte, 13347)
+	rand.Read(data)
+	mm, err := CreateTemp("test-kitty-shm-", uint64(len(data)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	copy(mm.Slice(), data)
+	mm.Close()
+
+	g, err := Open(mm.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	data2 := g.Slice()
+	if !reflect.DeepEqual(data, data2) {
+		t.Fatalf("Could not read back written data: Written data length: %d Read data length: %d", len(data), len(data2))
+	}
+	g.Close()
+	g.Unlink()
+	_, err = os.Stat(mm.Name())
+	if !errors.Is(err, fs.ErrNotExist) {
+		t.Fatalf("Unlinking %s did not work", mm.Name())
+	}
+
+}

tools/utils/shm/specific_darwin.go

@@ -0,0 +1,6 @@
+// License: GPLv3 Copyright: 2022, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package shm
+
+const SHM_NAME_MAX = 30
+const SHM_REQUIRED_PREFIX = ""

tools/utils/shm/specific_freebsd.go

@@ -0,0 +1,6 @@
+// License: GPLv3 Copyright: 2022, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package shm
+
+const SHM_NAME_MAX = 1023
+const SHM_REQUIRED_PREFIX = "/"

tools/utils/shm/specific_linux.go

@@ -0,0 +1,12 @@
+// License: GPLv3 Copyright: 2022, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package shm
+
+import (
+	"fmt"
+)
+
+var _ = fmt.Print
+
+const SHM_DIR = "/dev/shm"
+const SHM_NAME_MAX = 1023

tools/utils/shm/specific_netbsd.go

@@ -0,0 +1,12 @@
+// License: GPLv3 Copyright: 2022, Kovid Goyal, <kovid at kovidgoyal.net>
+
+package shm
+
+import (
+	"fmt"
+)
+
+var _ = fmt.Print
+
+const SHM_DIR = "/var/shm"
+const SHM_NAME_MAX = 1023