Fetch in the background and follow changes in the repositories

- On startup, after the local state is shown, fetch the superproject and
  every initialized submodule (all remotes, prune, at most 8 at once).
  Fetches never prompt; failures show git's message. --no-fetch skips it,
  f fetches again.
- The header shows "fetching n/m", then the fetch age and failures; the
  detail pane shows each submodule's last fetch and fetch errors.
- The git provider polls each repository's metadata (refs, HEAD, index,
  .gitmodules) every 2 s and reports changes. The app reloads only what
  changed, keeping old values, expansion and cursor; loads are
  single-flight with coalesced reruns. Refresh is driven by the
  repositories, so a fetch in another shell updates the view like ours.
- r reloads everything (worktree edits are not watched).
- Docs: F50-F53, N1/N8 rewritten, D18-D20 (D12 superseded).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/submodule/git/exec.go

@@ -4,9 +4,9 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"fmt"
 	"os"
 	"os/exec"
+	"slices"
 	"strings"
 )
 
@@ -18,22 +18,40 @@ var gitEnv = append(os.Environ(),
 	"LC_ALL=C",
 )
 
+// gitError is a failed git command; Stderr is git's message.
+type gitError struct {
+	Args   []string
+	Stderr string
+	Err    error
+}
+
+func (e *gitError) Error() string {
+	msg := e.Stderr
+	if msg == "" {
+		msg = e.Err.Error()
+	}
+	return "git " + strings.Join(e.Args, " ") + ": " + msg
+}
+
+func (e *gitError) Unwrap() error { return e.Err }
+
 // run executes git in dir and returns stdout with the trailing newline
 // removed. The error includes git's stderr.
 func run(ctx context.Context, dir string, args ...string) (string, error) {
+	return runEnv(ctx, dir, nil, args...)
+}
+
+// runEnv is run with additional environment variables.
+func runEnv(ctx context.Context, dir string, env []string, args ...string) (string, error) {
 	cmd := exec.CommandContext(ctx, "git", append([]string{"-C", dir}, args...)...)
-	cmd.Env = gitEnv
+	cmd.Env = slices.Concat(gitEnv, env) // a fresh slice: run is called concurrently
 	var stdout, stderr bytes.Buffer
 	cmd.Stdout, cmd.Stderr = &stdout, &stderr
 	if err := cmd.Run(); err != nil {
 		if ctx.Err() != nil {
 			return "", ctx.Err()
 		}
-		msg := strings.TrimSpace(stderr.String())
-		if msg == "" {
-			msg = err.Error()
-		}
-		return "", fmt.Errorf("git %s: %s", strings.Join(args, " "), msg)
+		return "", &gitError{Args: args, Stderr: strings.TrimSpace(stderr.String()), Err: err}
 	}
 	return strings.TrimRight(stdout.String(), "\n"), nil
 }

internal/submodule/git/fetch.go

@@ -0,0 +1,79 @@
+package git
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+)
+
+// fetchTimeout bounds a single fetch, e.g. against an unreachable host.
+const fetchTimeout = 2 * time.Minute
+
+// maxFetches bounds concurrent fetches; they wait on the network, not the
+// CPU.
+const maxFetches = 8
+
+// sshBatchEnv makes ssh fail instead of asking for a password or host key,
+// which would corrupt the terminal UI. It is nil if the user configured
+// their own ssh command.
+func sshBatchEnv(root string) []string {
+	if os.Getenv("GIT_SSH_COMMAND") != "" {
+		return nil
+	}
+	if out, ok, _ := try(context.Background(), root, "config", "core.sshCommand"); ok && out != "" {
+		return nil
+	}
+	return []string{"GIT_SSH_COMMAND=ssh -o BatchMode=yes"}
+}
+
+// Fetchable reports whether path ("" is the superproject) is a checked-out
+// repository.
+func (p *Provider) Fetchable(path string) bool {
+	_, ok := p.repoDir(path)
+	return ok
+}
+
+// repoDir returns the worktree of path if it is initialized.
+func (p *Provider) repoDir(path string) (string, bool) {
+	if path == "" {
+		return p.root, true
+	}
+	p.mu.RLock()
+	in, ok := p.mods[path]
+	p.mu.RUnlock()
+	if !ok || !initialized(in.dir) {
+		return "", false
+	}
+	return in.dir, true
+}
+
+// Fetch fetches all remotes of one repository. Submodules are fetched
+// separately, not recursively. The error is git's first line of output.
+func (p *Provider) Fetch(ctx context.Context, path string) error {
+	dir, ok := p.repoDir(path)
+	if !ok {
+		return fmt.Errorf("%s is not initialized", path)
+	}
+	select {
+	case p.fetchSem <- struct{}{}:
+		defer func() { <-p.fetchSem }()
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+	ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
+	defer cancel()
+
+	_, err := runEnv(ctx, dir, p.sshEnv, "fetch", "--all", "--prune", "--no-recurse-submodules", "--quiet")
+	var ge *gitError
+	switch {
+	case errors.Is(err, context.DeadlineExceeded):
+		return fmt.Errorf("timed out after %s", fetchTimeout)
+	case errors.As(err, &ge) && ge.Stderr != "":
+		line, _, _ := strings.Cut(ge.Stderr, "\n")
+		return errors.New(strings.TrimPrefix(line, "fatal: "))
+	}
+	return err
+}

internal/submodule/git/fetch_test.go

@@ -0,0 +1,161 @@
+package git
+
+import (
+	"context"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"submarine/internal/submodule"
+)
+
+// ownFixture builds a fixture the test may modify. It returns the provider
+// (after Tree) and the fixture root (superproject in root/super, work
+// clones of the remotes in root/work).
+func ownFixture(t *testing.T) (*Provider, string) {
+	t.Helper()
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git not available")
+	}
+	root := filepath.Join(t.TempDir(), "fx")
+	if err := makeFixture(root); err != nil {
+		t.Fatal(err)
+	}
+	p, err := New(filepath.Join(root, "super"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := p.Tree(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	return p, root
+}
+
+func git(t *testing.T, dir string, args ...string) {
+	t.Helper()
+	args = append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@t"}, args...)
+	if out, err := exec.Command("git", args...).CombinedOutput(); err != nil {
+		t.Fatalf("git %v: %v\n%s", args, err, out)
+	}
+}
+
+// pushUpstream adds a commit to a fixture remote, as if someone else pushed.
+func pushUpstream(t *testing.T, root, remote string) {
+	t.Helper()
+	work := filepath.Join(root, "work", remote)
+	git(t, work, "fetch", "-q")
+	git(t, work, "reset", "-q", "--hard", "origin/main")
+	git(t, work, "commit", "--allow-empty", "-m", "Upstream change")
+	git(t, work, "push", "-q", "origin", "main")
+}
+
+func TestFetch(t *testing.T) {
+	p, root := ownFixture(t)
+	ctx := context.Background()
+
+	before, err := p.Status(ctx, "src/core")
+	if err != nil {
+		t.Fatal(err)
+	}
+	pushUpstream(t, root, "core")
+	if err := p.Fetch(ctx, "src/core"); err != nil {
+		t.Fatalf("Fetch: %v", err)
+	}
+	after, err := p.Status(ctx, "src/core")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if after.Pin.Behind <= before.Pin.Behind {
+		t.Errorf("after fetch: ↓%d, want more than ↓%d", after.Pin.Behind, before.Pin.Behind)
+	}
+	if after.FetchedAt.IsZero() {
+		t.Error("after fetch: FetchedAt is zero")
+	}
+
+	if err := p.Fetch(ctx, ""); err != nil {
+		t.Errorf("Fetch(superproject without remotes): %v", err)
+	}
+	if p.Fetchable("libs/missing") {
+		t.Error("Fetchable(libs/missing) = true for an uninitialized submodule")
+	}
+	if err := p.Fetch(ctx, "libs/missing"); err == nil {
+		t.Error("Fetch(libs/missing): want error")
+	}
+}
+
+func TestFetchError(t *testing.T) {
+	p, root := ownFixture(t)
+	git(t, filepath.Join(root, "super", "src", "ui"), "remote", "set-url", "origin", filepath.Join(root, "nowhere.git"))
+
+	err := p.Fetch(context.Background(), "src/ui")
+	if err == nil {
+		t.Fatal("Fetch with a bad remote: want error")
+	}
+	if strings.Contains(err.Error(), "\n") || strings.HasPrefix(err.Error(), "git ") {
+		t.Errorf("error should be git's first line only, got %q", err)
+	}
+}
+
+// next waits for a change or fails after timeout.
+func next(t *testing.T, ch <-chan submodule.Change) submodule.Change {
+	t.Helper()
+	select {
+	case c := <-ch:
+		return c
+	case <-time.After(3 * time.Second):
+		t.Fatal("no change reported")
+		return submodule.Change{}
+	}
+}
+
+func TestWatch(t *testing.T) {
+	p, root := ownFixture(t)
+	super := filepath.Join(root, "super")
+	p.WatchInterval = 20 * time.Millisecond
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	ch := p.Watch(ctx)
+
+	// our own reads must not look like changes
+	for _, path := range []string{"src/core", "libs/work", "libs/conflict"} {
+		if _, err := p.Status(ctx, path); err != nil {
+			t.Fatal(err)
+		}
+	}
+	select {
+	case c := <-ch:
+		t.Fatalf("change while idle: %+v", c)
+	case <-time.After(200 * time.Millisecond):
+	}
+
+	// a fetch in another shell
+	pushUpstream(t, root, "net")
+	git(t, filepath.Join(super, "src", "net"), "fetch", "-q")
+	if c := next(t, ch); c != (submodule.Change{Path: "src/net"}) {
+		t.Errorf("after fetch: %+v, want remote change of src/net", c)
+	}
+
+	// a commit in another shell
+	git(t, filepath.Join(super, "libs", "store"), "commit", "-q", "--allow-empty", "-m", "Local")
+	if c := next(t, ch); c != (submodule.Change{Path: "libs/store", Local: true}) {
+		t.Errorf("after commit: %+v, want local change of libs/store", c)
+	}
+
+	cancel()
+	for range ch { // closed after cancel
+	}
+}
+
+func TestGitDirs(t *testing.T) {
+	p, _ := loadFixture(t)
+	gd, cd, err := gitDirs(p.root)
+	if err != nil || gd != filepath.Join(p.root, ".git") || cd != gd {
+		t.Errorf("superproject: %q, %q, %v", gd, cd, err)
+	}
+	gd, cd, err = gitDirs(filepath.Join(p.root, "src", "core"))
+	if err != nil || gd != filepath.Join(p.root, ".git", "modules", "src", "core") || cd != gd {
+		t.Errorf("submodule: %q, %q, %v", gd, cd, err)
+	}
+}

internal/submodule/git/gitdir.go

@@ -0,0 +1,43 @@
+package git
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// gitDirs returns the git directory of the worktree at dir and its common
+// directory (they differ only for linked worktrees). A submodule's .git is
+// a file "gitdir: <path>".
+func gitDirs(dir string) (gitDir, commonDir string, err error) {
+	dotGit := filepath.Join(dir, ".git")
+	fi, err := os.Stat(dotGit)
+	if err != nil {
+		return "", "", err
+	}
+	gitDir = dotGit
+	if !fi.IsDir() {
+		b, err := os.ReadFile(dotGit)
+		if err != nil {
+			return "", "", err
+		}
+		p, ok := strings.CutPrefix(strings.TrimSpace(string(b)), "gitdir:")
+		if !ok {
+			return "", "", errors.New(dotGit + ": not a gitdir file")
+		}
+		gitDir = joinPath(dir, strings.TrimSpace(p))
+	}
+	commonDir = gitDir
+	if b, err := os.ReadFile(filepath.Join(gitDir, "commondir")); err == nil {
+		commonDir = joinPath(gitDir, strings.TrimSpace(string(b)))
+	}
+	return gitDir, commonDir, nil
+}
+
+func joinPath(base, p string) string {
+	if !filepath.IsAbs(p) {
+		p = filepath.Join(base, p)
+	}
+	return filepath.Clean(p)
+}

internal/submodule/git/provider.go

@@ -14,6 +14,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"time"
 
 	"submarine/internal/submodule"
 )
@@ -27,8 +28,13 @@ const maxBetween = 50
 
 // Provider reads a superproject and its submodules with git.
 type Provider struct {
-	root string        // absolute superproject root
-	sem  chan struct{} // bounds concurrent Status calls
+	root     string        // absolute superproject root
+	sem      chan struct{} // bounds concurrent Status calls
+	fetchSem chan struct{} // bounds concurrent fetches
+	sshEnv   []string      // extra environment for fetches
+
+	// WatchInterval is how often Watch checks for changes.
+	WatchInterval time.Duration
 
 	mu   sync.RWMutex
 	mods map[string]info // by Submodule.Path; filled by Tree
@@ -49,8 +55,11 @@ func New(dir string) (*Provider, error) {
 		return nil, fmt.Errorf("%s is not a git repository", dir)
 	}
 	return &Provider{
-		root: root,
-		sem:  make(chan struct{}, runtime.NumCPU()),
+		root:          root,
+		sem:           make(chan struct{}, runtime.NumCPU()),
+		fetchSem:      make(chan struct{}, maxFetches),
+		sshEnv:        sshBatchEnv(root),
+		WatchInterval: 2 * time.Second,
 	}, nil
 }
 
@@ -168,6 +177,12 @@ func status(ctx context.Context, in info) (*submodule.Status, error) {
 		return st, nil
 	}
 
+	if gitDir, _, err := gitDirs(in.dir); err == nil {
+		if fi, err := os.Stat(filepath.Join(gitDir, "FETCH_HEAD")); err == nil {
+			st.FetchedAt = fi.ModTime()
+		}
+	}
+
 	// checkout and worktree
 	head, err := run(ctx, in.dir, "rev-parse", "HEAD")
 	if err != nil {

internal/submodule/git/provider_test.go

@@ -27,11 +27,8 @@ func TestMain(m *testing.M) {
 		}
 		defer os.RemoveAll(tmp)
 		if _, err := exec.LookPath("git"); err == nil {
-			_, file, _, _ := runtime.Caller(0)
-			script := filepath.Join(filepath.Dir(file), "..", "..", "..", "scripts", "make-fixture.sh")
-			out, err := exec.Command("bash", script, filepath.Join(tmp, "fx")).CombinedOutput()
-			if err != nil {
-				panic("make-fixture.sh: " + err.Error() + "\n" + string(out))
+			if err := makeFixture(filepath.Join(tmp, "fx")); err != nil {
+				panic(err)
 			}
 			fixtureDir = filepath.Join(tmp, "fx", "super")
 		}
@@ -40,6 +37,16 @@ func TestMain(m *testing.M) {
 	os.Exit(code)
 }
 
+// makeFixture runs scripts/make-fixture.sh; the superproject is dir/super.
+func makeFixture(dir string) error {
+	_, file, _, _ := runtime.Caller(0)
+	script := filepath.Join(filepath.Dir(file), "..", "..", "..", "scripts", "make-fixture.sh")
+	if out, err := exec.Command("bash", script, dir).CombinedOutput(); err != nil {
+		return fmt.Errorf("make-fixture.sh: %v\n%s", err, out)
+	}
+	return nil
+}
+
 func loadFixture(t *testing.T) (*Provider, *submodule.Repo) {
 	t.Helper()
 	if fixtureDir == "" {

internal/submodule/git/watch.go

@@ -0,0 +1,123 @@
+package git
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"submarine/internal/submodule"
+)
+
+// fingerprint summarizes the git metadata of one repository. Each part is
+// a list of file names with modification time and size; a change in a part
+// means something in that area changed.
+type fingerprint struct {
+	remote    string // FETCH_HEAD, remote-tracking branches, tags
+	local     string // HEAD, index, local branches, packed refs, merge state
+	structure string // .gitmodules and whether the repository is checked out
+}
+
+// Watch polls the git metadata of the superproject and all submodules every
+// WatchInterval and reports the repositories that changed. Polling (instead
+// of file system events) works on every file system, including WSL's
+// Windows drives, and needs no watch handles.
+func (p *Provider) Watch(ctx context.Context) <-chan submodule.Change {
+	ch := make(chan submodule.Change, 16)
+	go func() {
+		defer close(ch)
+		prev := p.fingerprints()
+		t := time.NewTicker(p.WatchInterval)
+		defer t.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-t.C:
+			}
+			cur := p.fingerprints()
+			for path, f := range cur {
+				old, ok := prev[path]
+				if !ok || old == f {
+					continue // new repositories only get a baseline
+				}
+				c := submodule.Change{
+					Path:      path,
+					Local:     old.local != f.local || old.structure != f.structure,
+					Structure: old.structure != f.structure,
+				}
+				select {
+				case ch <- c:
+				case <-ctx.Done():
+					return
+				}
+			}
+			prev = cur
+		}
+	}()
+	return ch
+}
+
+// fingerprints returns the fingerprint of every known repository, keyed by
+// submodule path ("" is the superproject).
+func (p *Provider) fingerprints() map[string]fingerprint {
+	dirs := map[string]string{"": p.root}
+	p.mu.RLock()
+	for path, in := range p.mods {
+		dirs[path] = in.dir
+	}
+	p.mu.RUnlock()
+
+	fps := make(map[string]fingerprint, len(dirs))
+	for path, dir := range dirs {
+		fps[path] = fingerprintOf(dir)
+	}
+	return fps
+}
+
+func fingerprintOf(dir string) fingerprint {
+	var f fingerprint
+	var structure strings.Builder
+	stamp(&structure, filepath.Join(dir, ".gitmodules"))
+	gitDir, common, err := gitDirs(dir)
+	if err != nil {
+		f.structure = structure.String() + "not checked out"
+		return f
+	}
+	f.structure = structure.String()
+
+	var remote, local strings.Builder
+	stamp(&remote, filepath.Join(gitDir, "FETCH_HEAD"))
+	stampTree(&remote, filepath.Join(common, "refs", "remotes"))
+	stampTree(&remote, filepath.Join(common, "refs", "tags"))
+
+	for _, name := range []string{"HEAD", "index", "MERGE_HEAD"} {
+		stamp(&local, filepath.Join(gitDir, name))
+	}
+	stamp(&local, filepath.Join(common, "packed-refs"))
+	stampTree(&local, filepath.Join(common, "refs", "heads"))
+	stampTree(&local, filepath.Join(common, "reftable")) // reftable ref storage
+	f.remote, f.local = remote.String(), local.String()
+	return f
+}
+
+// stamp appends the name, modification time and size of a file; nothing if
+// it doesn't exist.
+func stamp(b *strings.Builder, name string) {
+	if fi, err := os.Stat(name); err == nil {
+		fmt.Fprintf(b, "%s %d %d\n", name, fi.ModTime().UnixNano(), fi.Size())
+	}
+}
+
+// stampTree stamps every file below dir.
+func stampTree(b *strings.Builder, dir string) {
+	filepath.WalkDir(dir, func(name string, d fs.DirEntry, err error) error {
+		if err == nil && !d.IsDir() {
+			stamp(b, name)
+		}
+		return nil
+	})
+}