Move catfile batch to a sub package of git module (#36232)

2025-12-29 10:19:42 -08:00
parent d0cb198c89
commit 0ad94dfc70
17 changed files with 500 additions and 412 deletions
@@ -0,0 +1,178 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package catfile
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"strings"
+
+	"code.gitea.io/gitea/modules/git/gitcmd"
+
+	"github.com/djherbis/buffer"
+	"github.com/djherbis/nio/v3"
+)
+
+// WriteCloserError wraps an io.WriteCloser with an additional CloseWithError function
+type WriteCloserError interface {
+	io.WriteCloser
+	CloseWithError(err error) error
+}
+
+type Batch interface {
+	Writer() WriteCloserError
+	Reader() *bufio.Reader
+	Close()
+}
+
+// batch represents an active `git cat-file --batch` or `--batch-check` invocation
+// paired with the pipes that feed/read from it. Call Close to release resources.
+type batch struct {
+	cancel context.CancelFunc
+	reader *bufio.Reader
+	writer WriteCloserError
+}
+
+// NewBatch creates a new cat-file --batch process for the provided repository path.
+// The returned Batch must be closed once the caller has finished with it.
+func NewBatch(ctx context.Context, repoPath string) (Batch, error) {
+	if err := EnsureValidGitRepository(ctx, repoPath); err != nil {
+		return nil, err
+	}
+
+	var batch batch
+	batch.writer, batch.reader, batch.cancel = catFileBatch(ctx, repoPath)
+	return &batch, nil
+}
+
+// NewBatchCheck creates a cat-file --batch-check process for the provided repository path.
+// The returned Batch must be closed once the caller has finished with it.
+func NewBatchCheck(ctx context.Context, repoPath string) (Batch, error) {
+	if err := EnsureValidGitRepository(ctx, repoPath); err != nil {
+		return nil, err
+	}
+
+	var check batch
+	check.writer, check.reader, check.cancel = catFileBatchCheck(ctx, repoPath)
+	return &check, nil
+}
+
+func (b *batch) Writer() WriteCloserError {
+	return b.writer
+}
+
+func (b *batch) Reader() *bufio.Reader {
+	return b.reader
+}
+
+// Close stops the underlying git cat-file process and releases held resources.
+func (b *batch) Close() {
+	if b == nil || b.cancel == nil {
+		return
+	}
+	b.cancel()
+	b.reader = nil
+	b.writer = nil
+	b.cancel = nil
+}
+
+// EnsureValidGitRepository runs `git rev-parse` in the repository path to make sure
+// the directory is a valid git repository. This avoids git cat-file hanging indefinitely
+// when invoked in invalid paths.
+func EnsureValidGitRepository(ctx context.Context, repoPath string) error {
+	stder := strings.Builder{}
+	err := gitcmd.NewCommand("rev-parse").
+		WithDir(repoPath).
+		WithStderr(&stder).
+		Run(ctx)
+	if err != nil {
+		return gitcmd.ConcatenateError(err, stder.String())
+	}
+	return nil
+}
+
+// catFileBatch opens git cat-file --batch in the provided repo and returns a stdin pipe,
+// a stdout reader and a cancel function.
+func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) {
+	batchStdinReader, batchStdinWriter := io.Pipe()
+	batchStdoutReader, batchStdoutWriter := nio.Pipe(buffer.New(32 * 1024))
+	ctx, ctxCancel := context.WithCancel(ctx)
+	closed := make(chan struct{})
+	cancel := func() {
+		ctxCancel()
+		_ = batchStdinWriter.Close()
+		_ = batchStdoutReader.Close()
+		<-closed
+	}
+
+	go func() {
+		<-ctx.Done()
+		cancel()
+	}()
+
+	go func() {
+		stder := strings.Builder{}
+		err := gitcmd.NewCommand("cat-file", "--batch").
+			WithDir(repoPath).
+			WithStdin(batchStdinReader).
+			WithStdout(batchStdoutWriter).
+			WithStderr(&stder).
+			WithUseContextTimeout(true).
+			Run(ctx)
+		if err != nil {
+			_ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, stder.String()))
+			_ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, stder.String()))
+		} else {
+			_ = batchStdoutWriter.Close()
+			_ = batchStdinReader.Close()
+		}
+		close(closed)
+	}()
+
+	batchReader := bufio.NewReaderSize(batchStdoutReader, 32*1024)
+	return batchStdinWriter, batchReader, cancel
+}
+
+// catFileBatchCheck opens git cat-file --batch-check in the provided repo and returns a stdin pipe,
+// a stdout reader and cancel function.
+func catFileBatchCheck(ctx context.Context, repoPath string) (WriteCloserError, *bufio.Reader, func()) {
+	batchStdinReader, batchStdinWriter := io.Pipe()
+	batchStdoutReader, batchStdoutWriter := io.Pipe()
+	ctx, ctxCancel := context.WithCancel(ctx)
+	closed := make(chan struct{})
+	cancel := func() {
+		ctxCancel()
+		_ = batchStdoutReader.Close()
+		_ = batchStdinWriter.Close()
+		<-closed
+	}
+
+	go func() {
+		<-ctx.Done()
+		cancel()
+	}()
+
+	go func() {
+		stder := strings.Builder{}
+		err := gitcmd.NewCommand("cat-file", "--batch-check").
+			WithDir(repoPath).
+			WithStdin(batchStdinReader).
+			WithStdout(batchStdoutWriter).
+			WithStderr(&stder).
+			WithUseContextTimeout(true).
+			Run(ctx)
+		if err != nil {
+			_ = batchStdoutWriter.CloseWithError(gitcmd.ConcatenateError(err, stder.String()))
+			_ = batchStdinReader.CloseWithError(gitcmd.ConcatenateError(err, stder.String()))
+		} else {
+			_ = batchStdoutWriter.Close()
+			_ = batchStdinReader.Close()
+		}
+		close(closed)
+	}()
+
+	batchReader := bufio.NewReader(batchStdoutReader)
+	return batchStdinWriter, batchReader, cancel
+}
@@ -0,0 +1,211 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package catfile
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+// ErrObjectNotFound indicates that the requested object could not be read from cat-file
+type ErrObjectNotFound struct {
+	ID string
+}
+
+func (err ErrObjectNotFound) Error() string {
+	return fmt.Sprintf("catfile: object does not exist [id: %s]", err.ID)
+}
+
+// IsErrObjectNotFound reports whether err is an ErrObjectNotFound
+func IsErrObjectNotFound(err error) bool {
+	var target ErrObjectNotFound
+	return errors.As(err, &target)
+}
+
+// ObjectFormat abstracts the minimal information needed from git.ObjectFormat.
+type ObjectFormat interface {
+	FullLength() int
+}
+
+// ReadBatchLine reads the header line from cat-file --batch. It expects the format
+// "<oid> SP <type> SP <size> LF" and leaves the reader positioned at the start of
+// the object contents (which must be fully consumed by the caller).
+func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) {
+	typ, err = rd.ReadString('\n')
+	if err != nil {
+		return sha, typ, size, err
+	}
+	if len(typ) == 1 {
+		typ, err = rd.ReadString('\n')
+		if err != nil {
+			return sha, typ, size, err
+		}
+	}
+	idx := strings.IndexByte(typ, ' ')
+	if idx < 0 {
+		return sha, typ, size, ErrObjectNotFound{}
+	}
+	sha = []byte(typ[:idx])
+	typ = typ[idx+1:]
+
+	idx = strings.IndexByte(typ, ' ')
+	if idx < 0 {
+		return sha, typ, size, ErrObjectNotFound{ID: string(sha)}
+	}
+
+	sizeStr := typ[idx+1 : len(typ)-1]
+	typ = typ[:idx]
+
+	size, err = strconv.ParseInt(sizeStr, 10, 64)
+	return sha, typ, size, err
+}
+
+// ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest.
+func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) {
+	var id string
+	var n int64
+headerLoop:
+	for {
+		line, err := rd.ReadBytes('\n')
+		if err != nil {
+			return "", err
+		}
+		n += int64(len(line))
+		idx := bytes.Index(line, []byte{' '})
+		if idx < 0 {
+			continue
+		}
+
+		if string(line[:idx]) == "object" {
+			id = string(line[idx+1 : len(line)-1])
+			break headerLoop
+		}
+	}
+
+	return id, DiscardFull(rd, size-n+1)
+}
+
+// ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the commit content.
+func ReadTreeID(rd *bufio.Reader, size int64) (string, error) {
+	var id string
+	var n int64
+headerLoop:
+	for {
+		line, err := rd.ReadBytes('\n')
+		if err != nil {
+			return "", err
+		}
+		n += int64(len(line))
+		idx := bytes.Index(line, []byte{' '})
+		if idx < 0 {
+			continue
+		}
+
+		if string(line[:idx]) == "tree" {
+			id = string(line[idx+1 : len(line)-1])
+			break headerLoop
+		}
+	}
+
+	return id, DiscardFull(rd, size-n+1)
+}
+
+// hextable helps quickly convert between binary and hex representation
+const hextable = "0123456789abcdef"
+
+// BinToHex converts a binary hash into a hex encoded one. Input and output can be the
+// same byte slice to support in-place conversion without allocations.
+func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte {
+	for i := objectFormat.FullLength()/2 - 1; i >= 0; i-- {
+		v := sha[i]
+		vhi, vlo := v>>4, v&0x0f
+		shi, slo := hextable[vhi], hextable[vlo]
+		out[i*2], out[i*2+1] = shi, slo
+	}
+	return out
+}
+
+// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream and avoids allocations
+// where possible. Each line is composed of:
+// <mode-in-ascii> SP <fname> NUL <binary HASH>
+func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
+	var readBytes []byte
+
+	readBytes, err = rd.ReadSlice('\x00')
+	if err != nil {
+		return mode, fname, sha, n, err
+	}
+	idx := bytes.IndexByte(readBytes, ' ')
+	if idx < 0 {
+		log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes)
+		return mode, fname, sha, n, ErrObjectNotFound{}
+	}
+
+	n += idx + 1
+	copy(modeBuf, readBytes[:idx])
+	if len(modeBuf) >= idx {
+		modeBuf = modeBuf[:idx]
+	} else {
+		modeBuf = append(modeBuf, readBytes[len(modeBuf):idx]...)
+	}
+	mode = modeBuf
+
+	readBytes = readBytes[idx+1:]
+	copy(fnameBuf, readBytes)
+	if len(fnameBuf) > len(readBytes) {
+		fnameBuf = fnameBuf[:len(readBytes)]
+	} else {
+		fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...)
+	}
+	for err == bufio.ErrBufferFull {
+		readBytes, err = rd.ReadSlice('\x00')
+		fnameBuf = append(fnameBuf, readBytes...)
+	}
+	n += len(fnameBuf)
+	if err != nil {
+		return mode, fname, sha, n, err
+	}
+	fnameBuf = fnameBuf[:len(fnameBuf)-1]
+	fname = fnameBuf
+
+	idx = 0
+	length := objectFormat.FullLength() / 2
+	for idx < length {
+		var read int
+		read, err = rd.Read(shaBuf[idx:length])
+		n += read
+		if err != nil {
+			return mode, fname, sha, n, err
+		}
+		idx += read
+	}
+	sha = shaBuf
+	return mode, fname, sha, n, err
+}
+
+// DiscardFull discards the requested amount of bytes from the buffered reader regardless of its internal limit.
+func DiscardFull(rd *bufio.Reader, discard int64) error {
+	if discard > math.MaxInt32 {
+		n, err := rd.Discard(math.MaxInt32)
+		discard -= int64(n)
+		if err != nil {
+			return err
+		}
+	}
+	for discard > 0 {
+		n, err := rd.Discard(int(discard))
+		discard -= int64(n)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}