submarine/internal/submodule/git/provider.go

// Package git provides a submodule.Provider backed by the git command line.
// It reads; it writes only in Restore, and uses the network only in Fetch and
// in Restore (for what is missing).
package git

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"slices"
	"strings"
	"sync"
	"time"

	"submarine/internal/submodule"
)

// maxPinBranches limits Pin.Branches, the other remote branches containing the
// pin.
const maxPinBranches = 8

// maxContainsCandidates limits how many remote branches containing the pin
// are compared when the pin is not on its target.
const maxContainsCandidates = 32

// Provider reads a superproject and its submodules with git.
type Provider struct {
	root     string        // absolute superproject root
	sem      chan struct{} // bounds concurrent Status calls
	fetchSem chan struct{} // bounds concurrent fetches
	sshEnv   []string      // extra environment for fetches
	extraEnv []string      // more environment for fetches and clones; tests

	// WatchInterval is how often Watch checks for changes.
	WatchInterval time.Duration

	mu   sync.RWMutex
	mods map[string]info // by Submodule.Path; filled by Tree
}

// info is what Status needs about a submodule beyond its path.
type info struct {
	superDir string // absolute path of the parent repository
	dir      string // absolute path of the submodule
	relPath  string // relative to superDir, slash-separated
	branch   string // branch from .gitmodules, may be "" or "."
}

// New returns a provider for the repository containing dir.
func New(dir string) (*Provider, error) {
	root, err := run(context.Background(), dir, "rev-parse", "--show-toplevel")
	if err != nil {
		return nil, fmt.Errorf("%s is not a git repository", dir)
	}
	return &Provider{
		root:          root,
		sem:           make(chan struct{}, runtime.NumCPU()),
		fetchSem:      make(chan struct{}, maxFetches),
		sshEnv:        sshBatchEnv(root),
		WatchInterval: 2 * time.Second,
	}, nil
}

func (p *Provider) Tree(ctx context.Context) (*submodule.Repo, error) {
	repo := &submodule.Repo{Path: abbrevHome(p.root)}
	if b, ok, err := try(ctx, p.root, "symbolic-ref", "--short", "-q", "HEAD"); err != nil {
		return nil, err
	} else if ok {
		repo.Branch = b
	} else {
		repo.Branch = "(detached)"
	}
	if out, ok, err := try(ctx, p.root, "rev-list", "--left-right", "--count", "HEAD...@{upstream}"); err != nil {
		return nil, err
	} else if ok {
		repo.Ahead, repo.Behind, _ = parseLeftRight(out)
	}

	repo.Root = &submodule.Submodule{Name: filepath.Base(p.root), Path: ""}
	if remote, err := remoteName(ctx, p.root); err != nil {
		return nil, err
	} else if remote != "" {
		repo.Root.URL, _, _ = try(ctx, p.root, "config", "--get", "remote."+remote+".url")
	}

	mods := map[string]info{}
	subs, err := p.build(ctx, p.root, nil, mods)
	if err != nil {
		return nil, err
	}
	repo.Submodules = subs

	p.mu.Lock()
	p.mods = mods
	p.mu.Unlock()
	return repo, nil
}

// build reads the .gitmodules of the repository at dir and recurses into
// initialized submodules.
func (p *Provider) build(ctx context.Context, dir string, parent *submodule.Submodule, mods map[string]info) ([]*submodule.Submodule, error) {
	if _, err := os.Stat(filepath.Join(dir, ".gitmodules")); err != nil {
		return nil, nil
	}
	out, _, err := try(ctx, dir, "config", "-f", ".gitmodules", "--get-regexp", `^submodule\..*\.(path|url|branch)$`)
	if err != nil {
		return nil, err
	}
	var subs []*submodule.Submodule
	for _, g := range parseGitmodules(out) {
		rel := path.Clean(g.path)
		full := rel
		if parent != nil {
			full = path.Join(parent.Path, rel)
		}
		s := &submodule.Submodule{
			Name:    path.Base(rel),
			Path:    full,
			RelPath: rel,
			URL:     g.url,
			Parent:  parent,
		}
		subDir := filepath.Join(dir, filepath.FromSlash(rel))
		mods[full] = info{superDir: dir, dir: subDir, relPath: rel, branch: g.branch}
		if initialized(subDir) {
			if s.Children, err = p.build(ctx, subDir, s, mods); err != nil {
				return nil, err
			}
		}
		subs = append(subs, s)
	}
	return subs, nil
}

func (p *Provider) Status(ctx context.Context, path string) (*submodule.Status, error) {
	select {
	case p.sem <- struct{}{}:
		defer func() { <-p.sem }()
	case <-ctx.Done():
		return nil, ctx.Err()
	}
	if path == "" {
		return p.rootStatus(ctx)
	}
	p.mu.RLock()
	in, ok := p.mods[path]
	p.mu.RUnlock()
	if !ok {
		return nil, fmt.Errorf("unknown submodule %q", path)
	}
	return status(ctx, in)
}

func status(ctx context.Context, in info) (*submodule.Status, error) {
	// pin: index gitlink, compared with HEAD
	out, err := run(ctx, in.superDir, "ls-files", "-s", "--", in.relPath)
	if err != nil {
		return nil, err
	}
	pin := parseGitlink(out)
	if pin == "" {
		return nil, errors.New("listed in .gitmodules, but has no gitlink in the superproject index (stale entry?)")
	}
	out, _, err = try(ctx, in.superDir, "ls-tree", "HEAD", "--", in.relPath)
	if err != nil {
		return nil, err
	}
	committed := parseGitlink(out)
	st := &submodule.Status{PinStaged: committed != pin}

	// presence
	if _, err := os.Stat(in.dir); errors.Is(err, fs.ErrNotExist) {
		st.Presence = submodule.DirMissing
	} else if !initialized(in.dir) {
		st.Presence = submodule.NotInitialized
	}
	if st.Presence != submodule.PresenceOK {
		st.Pinned = &submodule.Commit{SHA: pin[:min(7, len(pin))]}
		return st, nil
	}

	if gitDir, _, err := gitDirs(in.dir); err == nil {
		if fi, err := os.Stat(filepath.Join(gitDir, "FETCH_HEAD")); err == nil {
			st.FetchedAt = fi.ModTime()
		}
	}

	// checkout and worktree
	head, err := run(ctx, in.dir, "rev-parse", "HEAD")
	if err != nil {
		return nil, err
	}
	branch, onBranch, err := try(ctx, in.dir, "symbolic-ref", "--short", "-q", "HEAD")
	if err != nil {
		return nil, err
	}
	switch {
	case head != pin:
		st.Checkout = submodule.CheckoutDiffers
	case !onBranch:
		st.Checkout = submodule.CheckoutDetached
	}
	out, err = run(ctx, in.dir, "status", "--porcelain=v2", "-z", "--untracked-files=normal", "--ignore-submodules=dirty")
	if err != nil {
		return nil, err
	}
	entries := parseStatus(out)
	st.Worktree = worktreeOf(entries)
	if st.Changes, err = changes(ctx, in.dir, entries); err != nil {
		return nil, err
	}

	if err := pinState(ctx, in, st, pin, head, branch, nil); err != nil {
		return nil, err
	}
	if st.PinStaged && committed != "" {
		if err := committedState(ctx, in, st, committed, pin); err != nil {
			return nil, err
		}
	}
	return st, nil
}

// committedState fills the pin of the superproject's HEAD and how it compares
// with the staged pin. It stays empty if the commit is not on disk: the staged
// pin is what matters, so this is no reason to fail.
func committedState(ctx context.Context, in info, st *submodule.Status, committed, pin string) error {
	if _, ok, err := try(ctx, in.dir, "cat-file", "-e", committed+"^{commit}"); err != nil || !ok {
		return err
	}
	c, err := commit(ctx, in.dir, committed)
	if err != nil {
		return err
	}
	out, err := run(ctx, in.dir, "rev-list", "--left-right", "--count", committed+"..."+pin)
	if err != nil {
		return err
	}
	if st.CommittedDrift.Ahead, st.CommittedDrift.Behind, err = parseLeftRight(out); err != nil {
		return err
	}
	st.Committed = c
	if st.CommittedDrift.Ahead > 0 && st.CommittedDrift.Behind > 0 {
		if st.Bases.Committed, err = mergeBase(ctx, in.dir, committed, pin); err != nil {
			return err
		}
	}
	return nil
}

// mergeBase returns the commit where the histories of a and b split; nil if
// they have none in common.
func mergeBase(ctx context.Context, dir, a, b string) (*submodule.Commit, error) {
	base, ok, err := try(ctx, dir, "merge-base", a, b)
	if err != nil || !ok || base == "" {
		return nil, err
	}
	return commit(ctx, dir, base)
}

// rootStatus describes the superproject. Its "pin" is HEAD, compared with the
// upstream of the current branch; submodules are ignored, their rows show
// them.
func (p *Provider) rootStatus(ctx context.Context) (*submodule.Status, error) {
	in := info{superDir: p.root, dir: p.root}
	st := &submodule.Status{Root: true}
	if gitDir, _, err := gitDirs(in.dir); err == nil {
		if fi, err := os.Stat(filepath.Join(gitDir, "FETCH_HEAD")); err == nil {
			st.FetchedAt = fi.ModTime()
		}
	}

	head, err := run(ctx, in.dir, "rev-parse", "HEAD")
	if err != nil {
		return nil, err
	}
	branch, onBranch, err := try(ctx, in.dir, "symbolic-ref", "--short", "-q", "HEAD")
	if err != nil {
		return nil, err
	}
	if !onBranch {
		st.Checkout = submodule.CheckoutDetached
	}
	out, err := run(ctx, in.dir, "status", "--porcelain=v2", "-z", "--untracked-files=normal", "--ignore-submodules=all")
	if err != nil {
		return nil, err
	}
	entries := parseStatus(out)
	// a leftover of --ignore-submodules=all in older git versions would
	// report the submodules the rows already show
	entries = slices.DeleteFunc(entries, func(e entry) bool { return e.submodule })
	st.Worktree = worktreeOf(entries)
	if st.Changes, err = changes(ctx, in.dir, entries); err != nil {
		return nil, err
	}

	// the root is compared with its upstream, so that the header and the
	// detail pane agree; only without one the target is looked for
	var chosen *candidate
	if branch != "" {
		if up, ok, err := try(ctx, in.dir, "rev-parse", "--abbrev-ref", "--symbolic-full-name", branch+"@{upstream}"); err != nil {
			return nil, err
		} else if ok {
			chosen = &candidate{up, submodule.RefFromCurrentBranch}
		}
	}
	if err := pinState(ctx, in, st, head, head, branch, chosen); err != nil {
		return nil, err
	}
	return st, nil
}

// pinState fills the fields of st that describe pin: whether it is pushed, the
// ref it resolves to (chosen, if not nil), the commits and the distances: of
// the checkout to the pin and of the pin to the ref. head is the checked-out
// commit.
func pinState(ctx context.Context, in info, st *submodule.Status, pin, head, branch string, chosen *candidate) error {
	// the pin must exist locally to say anything about it
	if _, ok, err := try(ctx, in.dir, "cat-file", "-e", pin+"^{commit}"); err != nil {
		return err
	} else if !ok {
		return fmt.Errorf("pinned commit %.7s not fetched", pin)
	}

	containing, err := remoteContaining(ctx, in.dir, pin)
	if err != nil {
		return err
	}
	st.Pin.Pushed = len(containing) > 0

	var ref string
	var source submodule.RefSource
	if chosen != nil {
		ref, source = chosen.ref, chosen.source
	} else if ref, source, st.Pin.Target, err = resolve(ctx, in, pin, branch, containing); err != nil {
		return err
	}
	st.Pin.Ref, st.Pin.Source = ref, source
	for _, b := range containing {
		if b != ref && len(st.Pin.Branches) < maxPinBranches {
			st.Pin.Branches = append(st.Pin.Branches, b)
		}
	}

	// commits
	if st.Pinned, err = commit(ctx, in.dir, pin); err != nil {
		return err
	}
	st.CheckedOut = st.Pinned
	var out string
	if head != pin {
		if st.CheckedOut, err = commit(ctx, in.dir, head); err != nil {
			return err
		}
		out, err = run(ctx, in.dir, "rev-list", "--left-right", "--count", head+"..."+pin)
		if err != nil {
			return err
		}
		if st.Drift.Ahead, st.Drift.Behind, err = parseLeftRight(out); err != nil {
			return err
		}
		if st.Drift.Ahead > 0 && st.Drift.Behind > 0 {
			if st.Bases.Checkout, err = mergeBase(ctx, in.dir, pin, head); err != nil {
				return err
			}
		}
	}
	if source == submodule.RefFromSHA {
		st.Pin.Ref = st.Pinned.SHA
		return nil
	}
	if st.Tip, err = commit(ctx, in.dir, ref); err != nil {
		return err
	}
	out, err = run(ctx, in.dir, "rev-list", "--left-right", "--count", pin+"..."+ref)
	if err != nil {
		return err
	}
	if st.Pin.Ahead, st.Pin.Behind, err = parseLeftRight(out); err != nil {
		return err
	}
	if st.Pin.Ahead > 0 && st.Pin.Behind > 0 {
		if st.Bases.Tip, err = mergeBase(ctx, in.dir, pin, ref); err != nil {
			return err
		}
	}

	// the pin is on another branch than its target: where the target is
	if target := st.Pin.Target; target != "" && target != ref {
		if st.TargetTip, err = commit(ctx, in.dir, target); err != nil {
			return err
		}
		out, err = run(ctx, in.dir, "rev-list", "--left-right", "--count", target+"..."+pin)
		if err != nil {
			return err
		}
		if st.TargetDrift.Ahead, st.TargetDrift.Behind, err = parseLeftRight(out); err != nil {
			return err
		}
		if st.TargetDrift.Ahead > 0 && st.TargetDrift.Behind > 0 {
			if st.Bases.Target, err = mergeBase(ctx, in.dir, pin, target); err != nil {
				return err
			}
		}
	}
	return nil
}

// remoteContaining lists the remote branches containing pin, oldest tip
// first, without the origin/HEAD symref.
func remoteContaining(ctx context.Context, dir, pin string) ([]string, error) {
	out, err := run(ctx, dir, "for-each-ref", "--contains", pin, "--sort=committerdate", "--format=%(refname)", "refs/remotes")
	if err != nil {
		return nil, err
	}
	var containing []string
	for _, r := range strings.Fields(out) {
		if !strings.HasSuffix(r, "/HEAD") {
			containing = append(containing, strings.TrimPrefix(r, "refs/remotes/"))
		}
	}
	return containing, nil
}

// resolve finds the ref the pin is compared with, following the order in
// docs/state-model.md: the target (see targetRef) if the pin is on it, else the
// branch containing the pin that is closest to it, else the target anyway,
// the upstream of the checked-out branch, a tag, the commit itself. It also
// returns the target ("" if there is none). containing lists the remote
// branches containing the pin, oldest tip first.
func resolve(ctx context.Context, in info, pin, branch string, containing []string) (string, submodule.RefSource, string, error) {
	target, source, hasTarget, err := targetRef(ctx, in)
	if err != nil {
		return "", 0, "", err
	}
	if !hasTarget {
		target = ""
	}
	if hasTarget && slices.Contains(containing, target) {
		return target, source, target, nil
	}
	if len(containing) > 0 {
		ref, err := closestContaining(ctx, in, pin, branch, containing)
		return ref, submodule.RefFromContains, target, err
	}
	if hasTarget {
		return target, source, target, nil
	}

	// the checked-out branch depends on the machine, so it is no target
	if up, ok, err := upstream(ctx, in.dir, branch); err != nil {
		return "", 0, "", err
	} else if ok {
		return up, submodule.RefFromCurrentBranch, "", nil
	}
	if tag, ok, err := try(ctx, in.dir, "describe", "--tags", "--abbrev=0", pin); err != nil {
		return "", 0, "", err
	} else if ok && tag != "" {
		return tag, submodule.RefFromTag, "", nil
	}
	return "", submodule.RefFromSHA, "", nil
}

// upstream returns the upstream of the local branch, if there is one.
func upstream(ctx context.Context, dir, branch string) (string, bool, error) {
	if branch == "" {
		return "", false, nil
	}
	return try(ctx, dir, "rev-parse", "--abbrev-ref", "--symbolic-full-name", branch+"@{upstream}")
}

// closestContaining picks the branch the pin is closest to among the remote
// branches containing it: the fewest commits after the pin. A tie goes to the
// upstream of the checked-out branch, then to the first name in alphabetical
// order, so that the answer is the same on every machine. No branch,
// including the default branch, is preferred otherwise.
func closestContaining(ctx context.Context, in info, pin, branch string, containing []string) (string, error) {
	var best []string
	bestN := -1
	for _, c := range containing[:min(len(containing), maxContainsCandidates)] {
		out, err := run(ctx, in.dir, "rev-list", "--count", pin+".."+c)
		if err != nil {
			return "", err
		}
		var n int
		if _, err := fmt.Sscan(out, &n); err != nil {
			continue
		}
		switch {
		case bestN < 0 || n < bestN:
			best, bestN = []string{c}, n
		case n == bestN:
			best = append(best, c)
		}
	}
	if len(best) == 0 {
		return containing[0], nil
	}
	if up, ok, err := upstream(ctx, in.dir, branch); err != nil {
		return "", err
	} else if ok && slices.Contains(best, up) {
		return up, nil
	}
	slices.Sort(best)
	return best[0], nil
}

// targetRef returns the branch the pin should be on: the `branch =` of
// .gitmodules (`.` is the superproject's current branch), else the remote's
// default branch, else main or master. A declared branch that does not exist
// locally is not replaced by the default: the answer would be wrong.
func targetRef(ctx context.Context, in info) (string, submodule.RefSource, bool, error) {
	remote, err := remoteName(ctx, in.dir)
	if err != nil || remote == "" {
		return "", 0, false, err
	}
	exists := func(ref string) (bool, error) {
		_, ok, err := try(ctx, in.dir, "rev-parse", "--verify", "-q", "refs/remotes/"+ref)
		return ok, err
	}

	if b := in.branch; b != "" {
		if b == "." {
			if b, _, err = try(ctx, in.superDir, "symbolic-ref", "--short", "-q", "HEAD"); err != nil {
				return "", 0, false, err
			}
		}
		if b == "" {
			return "", 0, false, nil
		}
		ok, err := exists(remote + "/" + b)
		if err != nil || !ok {
			return "", 0, false, err
		}
		return remote + "/" + b, submodule.RefFromGitmodules, true, nil
	}

	if def, ok, err := try(ctx, in.dir, "symbolic-ref", "--short", "-q", "refs/remotes/"+remote+"/HEAD"); err != nil {
		return "", 0, false, err
	} else if ok {
		return def, submodule.RefFromDefaultBranch, true, nil
	}
	for _, name := range []string{"main", "master"} {
		if ok, err := exists(remote + "/" + name); err != nil {
			return "", 0, false, err
		} else if ok {
			return remote + "/" + name, submodule.RefFromDefaultBranch, true, nil
		}
	}
	return "", 0, false, nil
}

// candidate is a ref and the rule that proposed it.
type candidate struct {
	ref    string
	source submodule.RefSource
}

// remoteName returns "origin" if it exists, else the first remote, else "".
func remoteName(ctx context.Context, dir string) (string, error) {
	out, err := run(ctx, dir, "remote")
	if err != nil {
		return "", err
	}
	remotes := strings.Fields(out)
	for _, r := range remotes {
		if r == "origin" {
			return r, nil
		}
	}
	if len(remotes) > 0 {
		return remotes[0], nil
	}
	return "", nil
}

// changes describes the changed files of the worktree; nil if there are
// none. Line counts come from one `git diff HEAD`; if that fails (e.g. no
// commit yet) they stay zero.
func changes(ctx context.Context, dir string, entries []entry) (*submodule.Changes, error) {
	if len(entries) == 0 {
		return nil, nil
	}
	var counts map[string]numstat
	if slices.ContainsFunc(entries, func(e entry) bool { return e.tracked && !e.submodule }) {
		out, ok, err := try(ctx, dir, "diff", "HEAD", "--numstat", "-z", "-M", "--no-ext-diff", "--ignore-submodules=dirty")
		if err != nil {
			return nil, err
		}
		if ok {
			counts = parseNumstat(out)
		}
	}

	c := &submodule.Changes{Total: len(entries)}
	for _, e := range entries {
		f := submodule.FileChange{Path: e.path, OrigPath: e.orig, Kind: e.kind, Submodule: e.submodule}
		if n, ok := counts[e.path]; ok && !e.submodule {
			f.Added, f.Removed, f.Binary = n.added, n.removed, n.binary
			c.Added += n.added
			c.Removed += n.removed
		}
		if e.kind == submodule.ChangeUntracked && strings.HasSuffix(e.path, "/") {
			f.Repo = initialized(filepath.Join(dir, filepath.FromSlash(e.path)))
		}
		if e.kind != submodule.ChangeRemoved {
			if fi, err := os.Stat(filepath.Join(dir, filepath.FromSlash(e.path))); err == nil {
				f.Modified = fi.ModTime()
			}
		}
		c.Files = append(c.Files, f)
	}
	c.Sort()
	if len(c.Files) > submodule.MaxChangeFiles {
		c.Files = c.Files[:submodule.MaxChangeFiles]
	}
	return c, nil
}

func commit(ctx context.Context, dir, rev string) (*submodule.Commit, error) {
	out, err := run(ctx, dir, "log", "-1", commitFormat, rev, "--")
	if err != nil {
		return nil, err
	}
	cs := parseCommits(out)
	if len(cs) == 0 {
		return nil, fmt.Errorf("no commit %s", rev)
	}
	return &cs[0], nil
}

// initialized reports whether dir is a checked-out repository (.git file or
// directory present).
func initialized(dir string) bool {
	_, err := os.Stat(filepath.Join(dir, ".git"))
	return err == nil
}

func abbrevHome(p string) string {
	home, err := os.UserHomeDir()
	if err != nil || home == "" {
		return p
	}
	if p == home {
		return "~"
	}
	if rest, ok := strings.CutPrefix(p, home+string(filepath.Separator)); ok {
		return "~/" + filepath.ToSlash(rest)
	}
	return p
}