Add restore (x): make a repository exactly what git records

Restores the selection and all submodules below it recursively, asking
only if work would be lost. May clone/fetch missing objects (D23).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-22 12:38:29 +02:00
co-authored by Claude Opus 5.5
parent a328d3cc3a
commit 7980d3356a
16 changed files with 1260 additions and 118 deletions
+4 -2
View File
@@ -1,5 +1,6 @@
// Package git provides a read-only submodule.Provider backed by the git
// command line. It never touches the network.
// Package git provides a submodule.Provider backed by the git command line.
// It reads; it writes only in Restore, and uses the network only in Fetch and
// in Restore (for what is missing).
package git
import (
@@ -32,6 +33,7 @@ type Provider struct {
sem chan struct{} // bounds concurrent Status calls
fetchSem chan struct{} // bounds concurrent fetches
sshEnv []string // extra environment for fetches
extraEnv []string // more environment for fetches and clones; tests
// WatchInterval is how often Watch checks for changes.
WatchInterval time.Duration
+400
View File
@@ -0,0 +1,400 @@
package git
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"time"
"submarine/internal/submodule"
)
// restoreArgs are prepended to every git command of a restore: the user's
// submodule.recurse must not make checkout and reset recurse behind our back,
// the restore walks the submodules itself.
var restoreArgs = []string{"-c", "submodule.recurse=false"}
// maxWalk bounds the entries visited when looking for repositories inside an
// untracked directory.
const maxWalk = 20000
// gitlink is a submodule entry of a tree or the index.
type gitlink struct{ path, sha string }
// RestoreLoss reports what Restore(path) would discard. It walks the
// repositories on disk, not the loaded tree.
func (p *Provider) RestoreLoss(ctx context.Context, path string) (submodule.Loss, error) {
dir, target, err := p.restoreTarget(ctx, path)
if err != nil {
return submodule.Loss{}, err
}
if dir == "" { // not initialized: nothing on disk to lose
return submodule.Loss{}, nil
}
return p.lossOf(ctx, dir, target), nil
}
// Restore makes path ("" is the superproject) and all submodules below it
// exactly what git records, see docs/actions.md. A missing submodule is
// cloned and missing commits are fetched; git never prompts.
func (p *Provider) Restore(ctx context.Context, path string) error {
if path == "" {
return p.restoreRepo(ctx, p.root, "HEAD")
}
p.mu.RLock()
in, ok := p.mods[path]
p.mu.RUnlock()
if !ok {
return fmt.Errorf("unknown submodule %q", path)
}
pin, err := indexGitlink(ctx, in)
if err != nil {
return err
}
if !initialized(in.dir) {
if err := p.initSubmodules(ctx, in.superDir, []string{in.relPath}); err != nil {
return fmt.Errorf("%s: %w", path, err)
}
}
return p.restoreRepo(ctx, in.dir, pin)
}
// restoreTarget returns the directory and the commit path is restored to:
// HEAD for the superproject, else the gitlink in its parent's index. The
// directory is "" if the submodule is not initialized.
func (p *Provider) restoreTarget(ctx context.Context, path string) (dir, target string, err error) {
if path == "" {
return p.root, "HEAD", nil
}
p.mu.RLock()
in, ok := p.mods[path]
p.mu.RUnlock()
if !ok {
return "", "", fmt.Errorf("unknown submodule %q", path)
}
pin, err := indexGitlink(ctx, in)
if err != nil {
return "", "", err
}
if !initialized(in.dir) {
return "", pin, nil
}
return in.dir, pin, nil
}
func indexGitlink(ctx context.Context, in info) (string, error) {
out, err := run(ctx, in.superDir, "ls-files", "-s", "--", in.relPath)
if err != nil {
return "", err
}
pin := parseGitlink(out)
if pin == "" {
return "", errors.New("has no gitlink in the superproject index (stale .gitmodules entry?)")
}
return pin, nil
}
// --- restore -----------------------------------------------------------------
// restoreRepo restores the initialized repository at dir to target (a commit
// or "HEAD") and then every submodule that target records. Branches are never
// moved: a checkout that is not on target is detached first.
func (p *Provider) restoreRepo(ctx context.Context, dir, target string) error {
env := p.networkEnv()
g := func(args ...string) (string, error) {
return p.exec(ctx, dir, env, append(append([]string{}, restoreArgs...), args...)...)
}
sha, err := g("rev-parse", "--verify", "-q", target+"^{commit}")
if err != nil {
if sha, err = p.fetchMissing(ctx, dir, target); err != nil {
return fmt.Errorf("%s: %w", dir, err)
}
}
if gitDir, _, err := gitDirs(dir); err == nil {
p.clearOperations(ctx, dir, gitDir)
}
if head, err := g("rev-parse", "--verify", "-q", "HEAD"); err != nil || head != sha {
if _, err := g("checkout", "--force", "--detach", sha); err != nil {
return err
}
}
if _, err := g("reset", "--hard", "-q"); err != nil {
return err
}
// -ff also removes untracked repositories; ignored files stay
if _, err := g("clean", "-ffdq"); err != nil {
return err
}
if _, err := g("submodule", "sync", "-q"); err != nil {
return err
}
links, err := treeGitlinks(ctx, dir, sha)
if err != nil {
return err
}
var missing []string
for _, l := range links {
if !initialized(filepath.Join(dir, filepath.FromSlash(l.path))) {
missing = append(missing, l.path)
}
}
if len(missing) > 0 {
if err := p.initSubmodules(ctx, dir, missing); err != nil {
return err
}
}
// the children are independent repositories
var wg sync.WaitGroup
errs := make([]error, len(links))
for i, l := range links {
wg.Add(1)
go func() {
defer wg.Done()
if err := p.restoreRepo(ctx, filepath.Join(dir, filepath.FromSlash(l.path)), l.sha); err != nil {
errs[i] = err
}
}()
}
wg.Wait()
return errors.Join(errs...)
}
// networkEnv is the environment of commands that may clone or fetch.
func (p *Provider) networkEnv() []string {
return append(append([]string{}, p.sshEnv...), p.extraEnv...)
}
// initSubmodules clones or checks out the given submodules of the repository
// at dir, whatever their active and update settings say (--checkout overrides
// update = none/rebase/merge, the explicit paths override inactive).
func (p *Provider) initSubmodules(ctx context.Context, dir string, rels []string) error {
ctx, cancel := context.WithTimeout(ctx, fetchTimeout*time.Duration(len(rels)))
defer cancel()
args := append(append([]string{}, restoreArgs...), "submodule", "update", "--init", "--checkout", "--force", "-q", "--")
_, err := p.exec(ctx, dir, p.networkEnv(), append(args, rels...)...)
return err
}
// exec runs git like runEnv, but counts against the concurrency limit. Hold
// it only for the command, never while recursing.
func (p *Provider) exec(ctx context.Context, dir string, env []string, args ...string) (string, error) {
select {
case p.sem <- struct{}{}:
defer func() { <-p.sem }()
case <-ctx.Done():
return "", ctx.Err()
}
return runEnv(ctx, dir, env, args...)
}
// fetchMissing fetches all remotes of dir to get target, and returns its SHA.
func (p *Provider) fetchMissing(ctx context.Context, dir, target string) (string, error) {
fctx, cancel := context.WithTimeout(ctx, fetchTimeout)
defer cancel()
// a failing fetch is reported with the missing commit below
_, ferr := runEnv(fctx, dir, p.networkEnv(), "fetch", "--all", "--no-recurse-submodules", "--quiet")
sha, ok, err := try(ctx, dir, "rev-parse", "--verify", "-q", target+"^{commit}")
if err != nil {
return "", err
}
if !ok {
msg := fmt.Sprintf("commit %.7s is not available", target)
if ferr != nil {
msg += " (fetch failed: " + firstLine(ferr) + ")"
} else {
msg += " on any remote"
}
return "", errors.New(msg)
}
return sha, nil
}
func firstLine(err error) string {
var ge *gitError
msg := err.Error()
if errors.As(err, &ge) && ge.Stderr != "" {
msg = ge.Stderr
}
line, _, _ := strings.Cut(msg, "\n")
return strings.TrimPrefix(line, "fatal: ")
}
// clearOperations ends a merge, rebase, cherry-pick or revert in progress
// without moving any ref (--quit, not --abort).
func (p *Provider) clearOperations(ctx context.Context, dir, gitDir string) {
exists := func(name string) bool { _, err := os.Stat(filepath.Join(gitDir, name)); return err == nil }
if exists("rebase-merge") || exists("rebase-apply") {
_, _, _ = try(ctx, dir, "rebase", "--quit")
}
if exists("CHERRY_PICK_HEAD") || exists("sequencer") {
_, _, _ = try(ctx, dir, "cherry-pick", "--quit")
}
if exists("REVERT_HEAD") {
_, _, _ = try(ctx, dir, "revert", "--quit")
}
// a merge is ended by the reset --hard that follows
}
// treeGitlinks lists the submodules recorded in commit's tree, at any depth
// of directories.
func treeGitlinks(ctx context.Context, dir, commit string) ([]gitlink, error) {
out, err := run(ctx, dir, "ls-tree", "-r", "-z", commit)
if err != nil {
return nil, err
}
var links []gitlink
for _, rec := range strings.Split(out, "\x00") {
meta, path, ok := strings.Cut(rec, "\t")
if f := strings.Fields(meta); ok && len(f) == 3 && f[0] == "160000" {
links = append(links, gitlink{path: path, sha: f[2]})
}
}
return links, nil
}
// indexGitlinks lists the submodules in the index of the repository at dir.
func indexGitlinks(ctx context.Context, dir string) ([]gitlink, error) {
out, err := run(ctx, dir, "ls-files", "-s", "-z")
if err != nil {
return nil, err
}
var links []gitlink
for _, rec := range strings.Split(out, "\x00") {
meta, path, ok := strings.Cut(rec, "\t")
if f := strings.Fields(meta); ok && len(f) == 3 && f[0] == "160000" && f[2] == "0" {
links = append(links, gitlink{path: path, sha: f[1]})
}
}
return links, nil
}
// --- loss --------------------------------------------------------------------
// lossOf adds up what a restore of the repository at dir to target
// discards, including the submodules on disk below it. target is "" if it is
// not known; then commits are only judged by the refs that reach them.
func (p *Provider) lossOf(ctx context.Context, dir, target string) submodule.Loss {
var own submodule.Loss
var children []gitlink
var targets map[string]string
unknown := func() { own.Unknown++ }
func() {
select {
case p.sem <- struct{}{}:
defer func() { <-p.sem }()
case <-ctx.Done():
unknown()
return
}
out, err := run(ctx, dir, "status", "--porcelain=v2", "-z", "--untracked-files=normal", "--ignore-submodules=all")
if err != nil {
unknown()
return
}
for _, e := range parseStatus(out) {
if e.submodule {
continue // restored without loss
}
own.Files++
if e.kind == submodule.ChangeUntracked && strings.HasSuffix(e.path, "/") {
own.Repositories += standaloneRepos(ctx, filepath.Join(dir, filepath.FromSlash(e.path)))
}
}
if gitDir, _, err := gitDirs(dir); err == nil {
for _, name := range []string{"rebase-merge", "rebase-apply", "MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "sequencer"} {
if _, err := os.Stat(filepath.Join(gitDir, name)); err == nil {
own.Operations++
break
}
}
}
// commits only this checkout reaches are lost if the checkout moves
head, err := run(ctx, dir, "rev-parse", "--verify", "-q", "HEAD")
if err != nil {
unknown()
} else {
moves := target == ""
if !moves {
if sha, ok, _ := try(ctx, dir, "rev-parse", "--verify", "-q", target+"^{commit}"); ok {
moves = sha != head
} else {
moves = true
}
}
if moves {
n, ok, _ := try(ctx, dir, "rev-list", "--count", "HEAD", "--not", "--branches", "--tags", "--remotes")
if ok {
fmt.Sscan(n, &own.Commits)
} else {
unknown()
}
}
}
if children, err = indexGitlinks(ctx, dir); err != nil {
unknown()
}
if target != "" {
if links, err := treeGitlinks(ctx, dir, target); err == nil {
targets = map[string]string{}
for _, l := range links {
targets[l.path] = l.sha
}
}
}
}()
if own.Files+own.Operations+own.Commits+own.Repositories+own.Unknown > 0 {
own.Modules = 1
}
// the children on disk, concurrently
results := make([]submodule.Loss, len(children))
var wg sync.WaitGroup
for i, c := range children {
sub := filepath.Join(dir, filepath.FromSlash(c.path))
if !initialized(sub) {
continue
}
wg.Add(1)
go func() {
defer wg.Done()
results[i] = p.lossOf(ctx, sub, targets[c.path])
}()
}
wg.Wait()
for _, r := range results {
own = own.Add(r)
}
return own
}
// standaloneRepos counts the repositories with their own .git directory in
// the untracked directory dir (or dir itself). An absorbed submodule leaves
// only a .git file; its commits live on in .git/modules.
func standaloneRepos(ctx context.Context, dir string) int {
n, seen := 0, 0
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if seen++; err != nil || seen > maxWalk || ctx.Err() != nil {
return filepath.SkipAll
}
if d.IsDir() && d.Name() == ".git" {
n++
return filepath.SkipDir
}
return nil
})
return n
}
+261
View File
@@ -0,0 +1,261 @@
package git
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"submarine/internal/submodule"
)
// out runs git and returns its trimmed stdout.
func out(t *testing.T, dir string, args ...string) string {
t.Helper()
b, err := exec.Command("git", append([]string{"-C", dir}, args...)...).Output()
if err != nil {
t.Fatalf("git %v: %v", args, err)
}
return strings.TrimSpace(string(b))
}
func exists(path string) bool { _, err := os.Stat(path); return err == nil }
// assertExact checks that the superproject at dir and every submodule below
// it is initialized, on its pin and has no change at all.
func assertExact(t *testing.T, dir string) {
t.Helper()
if s := out(t, dir, "status", "--porcelain"); s != "" {
t.Errorf("%s not clean:\n%s", dir, s)
}
raw, err := exec.Command("git", "-C", dir, "submodule", "status", "--recursive").Output() // the prefix is a leading space
if err != nil {
t.Fatal(err)
}
for _, line := range strings.Split(string(raw), "\n") {
if line == "" {
continue
}
if line[0] != ' ' {
t.Errorf("submodule not exact: %s", line)
continue
}
f := strings.Fields(line)
if s := out(t, filepath.Join(dir, f[1]), "status", "--porcelain"); s != "" {
t.Errorf("%s not clean:\n%s", f[1], s)
}
}
}
func TestRestoreSuperprojectMakesEverythingExact(t *testing.T) {
p, root := ownFixture(t)
super := filepath.Join(root, "super")
ctx := context.Background()
// ignored files stay
if err := os.WriteFile(filepath.Join(super, ".git", "info", "exclude"), []byte("keep.log\n"), 0o644); err != nil {
t.Fatal(err)
}
keep := filepath.Join(super, "libs", "work", "keep.log")
// libs/work is a submodule: its own exclude file lives in its git dir
wkGit := out(t, filepath.Join(super, "libs", "work"), "rev-parse", "--absolute-git-dir")
if err := os.WriteFile(filepath.Join(wkGit, "info", "exclude"), []byte("keep.log\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(keep, []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
// a branch must not move
conflict := filepath.Join(super, "libs", "conflict")
mainTip := out(t, conflict, "rev-parse", "main")
if err := p.Restore(ctx, ""); err != nil {
t.Fatal(err)
}
assertExact(t, super)
if got := out(t, conflict, "rev-parse", "main"); got != mainTip {
t.Errorf("branch main moved: %s -> %s", mainTip, got)
}
if !exists(keep) {
t.Error("ignored file was deleted")
}
if exists(filepath.Join(conflict, ".git", "MERGE_HEAD")) || exists(filepath.Join(out(t, conflict, "rev-parse", "--absolute-git-dir"), "MERGE_HEAD")) {
t.Error("merge still in progress")
}
if !exists(filepath.Join(super, "libs", "missing", ".git")) {
t.Error("libs/missing was not initialized")
}
if out(t, super, "diff", "--cached", "--name-only") != "" {
t.Error("staged pin survived a superproject restore")
}
}
func TestRestoreSubmoduleKeepsStagedPinAsTarget(t *testing.T) {
p, root := ownFixture(t)
super := filepath.Join(root, "super")
staged := out(t, filepath.Join(super, "libs", "staged"), "rev-parse", "HEAD")
// the fixture staged the checked-out commit; move the checkout away
git(t, filepath.Join(super, "libs", "staged"), "checkout", "-q", "--detach", "HEAD~1")
if err := p.Restore(context.Background(), "libs/staged"); err != nil {
t.Fatal(err)
}
if got := out(t, filepath.Join(super, "libs", "staged"), "rev-parse", "HEAD"); got != staged {
t.Errorf("HEAD = %s, want the staged pin %s", got, staged)
}
if !strings.Contains(out(t, super, "diff", "--cached", "--name-only"), "libs/staged") {
t.Error("restoring a submodule must keep its staged pin")
}
}
func TestRestoreLoss(t *testing.T) {
p, root := ownFixture(t)
super := filepath.Join(root, "super")
ctx := context.Background()
loss := func(path string) submodule.Loss {
t.Helper()
l, err := p.RestoreLoss(ctx, path)
if err != nil {
t.Fatal(err)
}
return l
}
// off pin and missing lose nothing; neither does an unpushed pin
for _, path := range []string{"src/core", "libs/missing", "src/net", "libs/staged"} {
if l := loss(path); !l.Empty() {
t.Errorf("%s: loss %+v, want none", path, l)
}
}
// libs/work: 6 files and a directory, untracked and changed
if l := loss("libs/work"); l.Files < 6 || l.Modules != 1 {
t.Errorf("libs/work: %+v", l)
}
// a nested change counts for its ancestors
if l := loss("libs/log"); l.Files != 1 || l.Modules != 1 {
t.Errorf("libs/log: %+v, want the nested change", l)
}
// a merge in progress
if l := loss("libs/conflict"); l.Operations != 1 || l.Files == 0 {
t.Errorf("libs/conflict: %+v", l)
}
// commits: detached and unreachable are lost, on a branch they are not
ui := filepath.Join(super, "src", "ui")
git(t, ui, "checkout", "-q", "--detach")
git(t, ui, "commit", "-q", "--allow-empty", "-m", "detached work")
if l := loss("src/ui"); l.Commits != 1 {
t.Errorf("detached commit: %+v", l)
}
git(t, ui, "switch", "-q", "-c", "wip")
if l := loss("src/ui"); l.Commits != 0 {
t.Errorf("commit on a branch: %+v", l)
}
// a clone nobody added as a submodule
git(t, super, "clone", "-q", filepath.Join(root, "remotes", "core.git"), "stray")
if l := loss(""); l.Repositories != 1 {
t.Errorf("root: %+v, want one standalone repository", l)
}
}
func TestRestoreRemovesLeftoversAndEndsRebase(t *testing.T) {
p, root := ownFixture(t)
super := filepath.Join(root, "super")
ctx := context.Background()
// a standalone clone is deleted
git(t, super, "clone", "-q", filepath.Join(root, "remotes", "core.git"), "stray")
// a submodule the branch no longer has leaves its directory behind
git(t, super, "rm", "-q", "--cached", "libs/topic")
git(t, super, "commit", "-q", "-m", "Drop topic")
// a rebase in progress in vendor; the branches must survive
v := filepath.Join(super, "vendor")
base := out(t, v, "rev-parse", "HEAD")
git(t, v, "switch", "-q", "-c", "x")
os.WriteFile(filepath.Join(v, "README"), []byte("x\n"), 0o644)
git(t, v, "commit", "-q", "-am", "x")
git(t, v, "switch", "-q", "-c", "y", base)
os.WriteFile(filepath.Join(v, "README"), []byte("y\n"), 0o644)
git(t, v, "commit", "-q", "-am", "y")
yTip := out(t, v, "rev-parse", "y")
_ = exec.Command("git", "-C", v, "rebase", "x").Run() // conflicts
if !exists(filepath.Join(out(t, v, "rev-parse", "--absolute-git-dir"), "rebase-merge")) {
t.Fatal("setup: no rebase in progress")
}
if err := p.Restore(ctx, ""); err != nil {
t.Fatal(err)
}
for _, gone := range []string{"stray", "libs/topic"} {
if exists(filepath.Join(super, gone)) {
t.Errorf("%s still exists", gone)
}
}
if exists(filepath.Join(out(t, v, "rev-parse", "--absolute-git-dir"), "rebase-merge")) {
t.Error("rebase still in progress")
}
if got := out(t, v, "rev-parse", "y"); got != yTip {
t.Errorf("branch y moved: %s -> %s", yTip, got)
}
if out(t, super, "status", "--porcelain") != "" {
t.Errorf("superproject not clean:\n%s", out(t, super, "status", "--porcelain"))
}
}
func TestRestoreClonesAMissingSubmodule(t *testing.T) {
p, root := ownFixture(t)
p.extraEnv = []string{"GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=protocol.file.allow", "GIT_CONFIG_VALUE_0=always"}
super := filepath.Join(root, "super")
// no object database left: it must be cloned again
if err := os.RemoveAll(filepath.Join(super, ".git", "modules", "libs", "missing")); err != nil {
t.Fatal(err)
}
if err := p.Restore(context.Background(), "libs/missing"); err != nil {
t.Fatal(err)
}
if !exists(filepath.Join(super, "libs", "missing", ".git")) {
t.Error("not cloned")
}
if got, want := out(t, filepath.Join(super, "libs", "missing"), "rev-parse", "HEAD"), strings.Fields(out(t, super, "ls-files", "-s", "libs/missing"))[1]; got != want {
t.Errorf("HEAD = %s, want pin %s", got, want)
}
}
func TestRestoreReportsAnUnavailablePin(t *testing.T) {
p, root := ownFixture(t)
super := filepath.Join(root, "super")
// pin a commit that exists on no remote and not in the submodule
core := filepath.Join(super, "src", "core")
bogus := "0123456789abcdef0123456789abcdef01234567"
git(t, super, "update-index", "--cacheinfo", "160000,"+bogus+",src/core")
_ = core
err := p.Restore(context.Background(), "src/core")
if err == nil || !strings.Contains(err.Error(), "not available") {
t.Fatalf("err = %v, want a message about the unavailable commit", err)
}
}
func TestRestoreIgnoresUpdateNoneAndInactive(t *testing.T) {
p, root := ownFixture(t)
super := filepath.Join(root, "super")
git(t, super, "config", "submodule.libs/missing.update", "none")
git(t, super, "config", "submodule.libs/missing.active", "false")
git(t, super, "config", "submodule.active", "src")
if err := p.Restore(context.Background(), ""); err != nil {
t.Fatal(err)
}
if !exists(filepath.Join(super, "libs", "missing", ".git")) {
t.Error("libs/missing not initialized")
}
assertExact(t, super)
}