lvmh/daemon/spawner_test.go

package main

// spawner_test.go — clone→build→create/start pipeline, git failures, env wiring.

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/docker/docker/client"
)

func TestSpawnerStartHappyPath(t *testing.T) {
	gitLog := useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	sp, store := newTestSpawner(t, f)

	res, err := sp.Start(context.Background(), "group/project", "main")
	if err != nil {
		t.Fatalf("Start: %v", err)
	}
	if res.SessionID == "" || res.ContainerID != "" {
		t.Fatalf("Start result = %+v, want sessionId set and empty containerId", res)
	}

	job := waitJobState(t, sp, res.SessionID, stateRunning)
	if job.Repo != "group/project" || job.ContainerID == "" {
		t.Fatalf("running job = %+v, want containerId set", job)
	}

	// clone URL carries the PAT only when none is stored (it isn't here)
	calls := readGitLog(t, gitLog)
	if len(calls) != 1 || !strings.HasPrefix(calls[0], "clone --branch main -- https://gitlab.example/group/project.git ") {
		t.Fatalf("git calls = %v", calls)
	}

	// container created with the right image, env, labels, mounts, network
	creates := f.createsByName("lvmh-agent-")
	if len(creates) != 1 {
		t.Fatalf("agent containers created = %+v", creates)
	}
	c := creates[0]
	if c.Image != imageRefWorker {
		t.Fatalf("image = %q", c.Image)
	}
	if c.Labels[labelSession] != res.SessionID {
		t.Fatalf("labels = %v", c.Labels)
	}
	wantEnv := map[string]bool{
		envProviderAPIKey + "=key-123":         true,
		envToken + "=" + testToken:             true,
		"LVMH_URL=" + defaultContainerURL:      true,
		envLVMHSessionID + "=" + res.SessionID: true,
		envLVMHAgent + "=1":                    true,
		envLVMHRepo + "=group/project":         true,
	}
	for _, e := range c.Env {
		if !wantEnv[e] {
			t.Fatalf("unexpected env %q in %v", e, c.Env)
		}
		delete(wantEnv, e)
	}
	if len(wantEnv) != 0 {
		t.Fatalf("missing env %v", wantEnv)
	}
	wantBinds := []string{
		"lvmh-repo-group-project:" + workspaceMount,
		volumeSessions + ":" + sessionsMount,
	}
	if len(c.HostConfig.Binds) != 2 {
		t.Fatalf("binds = %v, want %v (no models file)", c.HostConfig.Binds, wantBinds)
	}
	for i, b := range wantBinds {
		if c.HostConfig.Binds[i] != b {
			t.Fatalf("binds = %v, want %v", c.HostConfig.Binds, wantBinds)
		}
	}
	if c.HostConfig.NetworkMode != defaultNetwork {
		t.Fatalf("network = %q", c.HostConfig.NetworkMode)
	}

	// fresh repo volume seeded from the clone via CopyToContainer (tar)
	seed := f.createsByName("lvmh-seed-")
	if len(seed) != 1 {
		t.Fatalf("seed containers = %+v", seed)
	}
	if seed[0].Image != imageRefWorker {
		t.Fatalf("seed create = %+v", seed[0])
	}
	wantSeedBinds := []string{"lvmh-repo-group-project:" + workspaceMount}
	if !reflect.DeepEqual(seed[0].HostConfig.Binds, wantSeedBinds) {
		t.Fatalf("seed binds = %v, want %v (no host-path binds)", seed[0].HostConfig.Binds, wantSeedBinds)
	}
	if len(f.archives) != 1 || f.archives[0] == 0 {
		t.Fatalf("CopyToContainer archives = %v, want one non-empty tar", f.archives)
	}

	// no build expected (fake daemon already has the image)
	if f.hasCall(http.MethodPost, "/build") {
		t.Fatal("image present but build was called")
	}
	// session volume created too
	if !f.volumeExists(volumeSessions) {
		t.Fatalf("volume %q missing", volumeSessions)
	}

	// container row persisted
	row, ok, err := store.GetContainer(res.SessionID)
	if err != nil || !ok || row.Repo != "group/project" {
		t.Fatalf("container row = %+v ok=%v err=%v", row, ok, err)
	}
	f.assertNoUnknown(t)
}

func TestSpawnerBuildsImageWhenMissing(t *testing.T) {
	useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	f.images = 0 // image absent → ensureImage must build
	sp, _ := newTestSpawner(t, f)

	res, err := sp.Start(context.Background(), "group/project", "")
	if err != nil {
		t.Fatalf("Start: %v", err)
	}
	waitJobState(t, sp, res.SessionID, stateRunning)

	builds := f.countCalls(http.MethodPost, "/build")
	if builds != 1 {
		t.Fatalf("build calls = %d, want 1", builds)
	}
	f.assertNoUnknown(t)
}

func TestSpawnerStartValidatesDockerAndDockerfile(t *testing.T) {
	f := newFakeDocker()
	f.images = 0
	sp, _ := newTestSpawner(t, f) // dockerfile exists in the fake context

	// docker reachable but image absent and no Dockerfile anywhere → clear error
	t.Setenv(envWorkerDockerfile, filepath.Join(t.TempDir(), "missing.Dockerfile"))
	sp2, err := NewSpawner(sp.store, sp.hub, "https://gitlab.example/")
	if err != nil {
		t.Fatalf("NewSpawner: %v", err)
	}
	_, err = sp2.Start(context.Background(), "group/project", "")
	if err == nil || !strings.Contains(err.Error(), "no worker Dockerfile") {
		t.Fatalf("Start without dockerfile err = %v", err)
	}
	if len(sp2.JobsSnapshot()) != 0 {
		t.Fatalf("failed Start must not leave a job: %+v", sp2.JobsSnapshot())
	}
}

func TestSpawnerRunJobErrorStates(t *testing.T) {
	cases := []struct {
		name    string
		setup   func(t *testing.T, f *fakeDocker)
		gitMode string
		wantMsg string
	}{
		{
			name:    "clone-fails",
			gitMode: fakeGitModeFail,
			wantMsg: "fatal: repository not found",
		},
		{
			name:    "noisy-clone-failure-truncated-to-tail",
			gitMode: fakeGitModeNoisy,
			wantMsg: "xxx",
		},
		{
			name: "build-fails",
			setup: func(t *testing.T, f *fakeDocker) {
				f.images = 0
				f.failBuild = true
			},
			wantMsg: "build exploded",
		},
		{
			name: "create-fails",
			setup: func(t *testing.T, f *fakeDocker) {
				f.failCreate = true
			},
			wantMsg: "create failed",
		},
		{
			name: "start-fails-and-cleans-up",
			setup: func(t *testing.T, f *fakeDocker) {
				f.failStart = true
			},
			wantMsg: "start failed",
		},
		{
			name: "volume-create-fails",
			setup: func(t *testing.T, f *fakeDocker) {
				f.failVolumeCreate = true
			},
			wantMsg: "volume create failed",
		},
		{
			name: "seed-copy-fails",
			setup: func(t *testing.T, f *fakeDocker) {
				f.failArchive = true
			},
			wantMsg: "copy into volume",
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			gitMode := tc.gitMode
			if gitMode == "" {
				gitMode = fakeGitModeOK
			}
			useFakeGit(t, gitMode)
			f := newFakeDocker()
			if tc.setup != nil {
				tc.setup(t, f)
			}
			sp, _ := newTestSpawner(t, f)
			res, err := sp.Start(context.Background(), "group/project", "")
			if err != nil {
				t.Fatalf("Start: %v", err)
			}
			job := waitJobState(t, sp, res.SessionID, stateError)
			if !strings.Contains(job.Message, tc.wantMsg) {
				t.Fatalf("job message = %q, want containing %q", job.Message, tc.wantMsg)
			}
			if job.ContainerID != "" {
				t.Fatalf("error job has containerId %q", job.ContainerID)
			}
		})
	}
}

func TestSpawnerCloneOrUpdatePullsExisting(t *testing.T) {
	gitLog := useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	sp, _ := newTestSpawner(t, f)

	// existing clone → pull --ff-only in that dir, no clone
	dir := filepath.Join(sp.reposDir, "group-project")
	if err := os.MkdirAll(filepath.Join(dir, ".git"), 0o755); err != nil {
		t.Fatalf("mkdir .git: %v", err)
	}
	if err := sp.cloneOrUpdate("group/project", "", repoSlug("group/project")); err != nil {
		t.Fatalf("cloneOrUpdate: %v", err)
	}
	calls := readGitLog(t, gitLog)
	if len(calls) != 1 || calls[0] != "pull --ff-only" {
		t.Fatalf("git calls = %v, want pull --ff-only", calls)
	}
}

func TestSpawnerCloneURLInjectsPAT(t *testing.T) {
	f := newFakeDocker()
	sp, store := newTestSpawner(t, f)

	if u, err := sp.cloneURL("group/project"); err != nil || u != "https://gitlab.example/group/project.git" {
		t.Fatalf("cloneURL without PAT = %q err=%v", u, err)
	}
	if err := store.SetSetting(settingGitLabToken, "pat-1"); err != nil {
		t.Fatalf("set token: %v", err)
	}
	u, err := sp.cloneURL("group/project")
	if err != nil {
		t.Fatalf("cloneURL: %v", err)
	}
	if u != "https://pat-1@gitlab.example/group/project.git" {
		t.Fatalf("cloneURL with PAT = %q", u)
	}
}

func TestSpawnerRemoveSession(t *testing.T) {
	useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	sp, store := newTestSpawner(t, f)

	if err := sp.RemoveSession(context.Background(), "nope"); err != errNoContainer {
		t.Fatalf("remove unknown = %v, want errNoContainer", err)
	}

	if err := store.UpsertContainer("s1", "cid-9", "group/project"); err != nil {
		t.Fatalf("upsert container: %v", err)
	}
	sp.setJob("s1", "group/project", stateRunning, "cid-9", "")
	if err := sp.RemoveSession(context.Background(), "s1"); err != nil {
		t.Fatalf("RemoveSession: %v", err)
	}
	if !f.hasCall(http.MethodPost, "/containers/cid-9/stop") || !f.hasCall(http.MethodDelete, "/containers/cid-9") {
		t.Fatal("container not stopped+removed")
	}
	if _, ok, _ := store.GetContainer("s1"); ok {
		t.Fatal("container row must be deleted")
	}
	for _, j := range sp.JobsSnapshot() {
		if j.SessionID == "s1" && j.State != stateError {
			t.Fatalf("job after removal = %+v, want error state", j)
		}
	}

	// stop failure surfaces, row kept
	if err := store.UpsertContainer("s2", "cid-10", "group/project"); err != nil {
		t.Fatalf("upsert: %v", err)
	}
	f.failStop = true
	if err := sp.RemoveSession(context.Background(), "s2"); err == nil || !strings.Contains(err.Error(), "docker stop") {
		t.Fatalf("remove with stop failure = %v", err)
	}
	f.assertNoUnknown(t)
}

func TestSpawnerSeedVolumeCancel(t *testing.T) {
	useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	f.archiveHang = true // server never answers the archive PUT
	sp, _ := newTestSpawner(t, f)
	if err := os.MkdirAll(filepath.Join(sp.reposDir, "group-project"), 0o755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(sp.reposDir, "group-project", "README.md"), []byte("x"), 0o644); err != nil {
		t.Fatal(err)
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	done := make(chan error, 1)
	go func() { done <- sp.seedVolume(ctx, "group-project", "lvmh-repo-group-project") }()
	waitFor(t, 5*time.Second, func() bool { return f.hasCallSuffix(http.MethodPut, "/archive") })
	cancel()
	select {
	case err := <-done:
		if !errors.Is(err, context.Canceled) {
			t.Fatalf("seedVolume after cancel = %v, want context.Canceled", err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("seedVolume did not return after context cancel")
	}
}

func TestSpawnerSameRepoSpawnsSerialize(t *testing.T) {
	useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	sp, _ := newTestSpawner(t, f)

	l1 := sp.slugLock("a")
	l2 := sp.slugLock("a")
	if l1 != l2 {
		t.Fatal("slugLock must return the same mutex per slug")
	}
	if sp.slugLock("b") == l1 {
		t.Fatal("slugLock must return distinct mutexes per slug")
	}

	res1, err := sp.Start(context.Background(), "group/project", "")
	if err != nil {
		t.Fatalf("Start 1: %v", err)
	}
	res2, err := sp.Start(context.Background(), "group/project", "")
	if err != nil {
		t.Fatalf("Start 2: %v", err)
	}
	waitJobState(t, sp, res1.SessionID, stateRunning)
	waitJobState(t, sp, res2.SessionID, stateRunning)
	if len(f.createsByName("lvmh-agent-")) != 2 {
		t.Fatalf("agent containers = %d, want 2", len(f.createsByName("lvmh-agent-")))
	}
}

func TestRepoSlugAndUUID(t *testing.T) {
	if got := repoSlug("a/b/c"); got != "a-b-c" {
		t.Fatalf("repoSlug = %q", got)
	}
	id := newUUID()
	if len(id) != 36 || id[8] != '-' || id[13] != '-' || id[18] != '-' || id[23] != '-' {
		t.Fatalf("newUUID shape = %q", id)
	}
	if id2 := newUUID(); id2 == id {
		t.Fatal("newUUID must not repeat")
	}
}

func TestEnvOr(t *testing.T) {
	t.Setenv("LVMH_TEST_ENV_OR", "  value  ")
	if got := envOr("LVMH_TEST_ENV_OR", "def"); got != "value" {
		t.Fatalf("envOr trimmed = %q", got)
	}
	t.Setenv("LVMH_TEST_ENV_OR", "   ")
	if got := envOr("LVMH_TEST_ENV_OR", "def"); got != "def" {
		t.Fatalf("envOr whitespace-only = %q", got)
	}
	if got := envOr("LVMH_TEST_ENV_OR_UNSET", "def"); got != "def" {
		t.Fatalf("envOr unset = %q", got)
	}
}

func TestExtractBuildError(t *testing.T) {
	cases := []struct {
		name string
		body string
		want string
	}{
		{"empty", "", ""},
		{"no-error", `{"stream":"Step 1/3"}` + "\n" + `{"stream":"done"}`, ""},
		{"error-field", `{"error":"nope"}`, "nope"},
		{"error-detail-wins", `{"errorDetail":{"message":"boom"},"error":"generic"}`, "boom"},
		{"mixed-stream", "{\"stream\":\"...\"}\n{\"error\":\"late failure\"}", "late failure"},
		{"non-json-line", "garbage line\n{\"error\":\"after garbage\"}", "after garbage"},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := extractBuildError([]byte(tc.body)); got != tc.want {
				t.Fatalf("extractBuildError(%q) = %q, want %q", tc.body, got, tc.want)
			}
		})
	}
}

func TestTarDir(t *testing.T) {
	dir := t.TempDir()
	if err := os.WriteFile(filepath.Join(dir, "one.txt"), []byte("one"), 0o644); err != nil {
		t.Fatalf("write: %v", err)
	}
	if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
		t.Fatalf("mkdir: %v", err)
	}
	if err := os.WriteFile(filepath.Join(dir, "sub", "two.txt"), []byte("two"), 0o644); err != nil {
		t.Fatalf("write: %v", err)
	}
	var buf bytes.Buffer
	if err := tarDir(&buf, dir); err != nil {
		t.Fatalf("tarDir: %v", err)
	}
	tr := tar.NewReader(bytes.NewReader(buf.Bytes()))
	names := map[string]string{}
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			t.Fatalf("tar next: %v", err)
		}
		body, _ := io.ReadAll(tr)
		names[hdr.Name] = string(body)
		if hdr.Uname != "root" || hdr.Gname != "root" {
			t.Fatalf("header %q owner = %s/%s, want root/root", hdr.Name, hdr.Uname, hdr.Gname)
		}
	}
	for name, want := range map[string]string{"one.txt": "one", "sub/two.txt": "two"} {
		if names[name] != want {
			t.Fatalf("archive missing %q (have %v)", name, names)
		}
	}
	if _, ok := names["sub/"]; !ok {
		t.Fatalf("directories must be archived with trailing slash: %v", names)
	}

	if err := tarDir(&buf, filepath.Join(dir, "does-not-exist")); err == nil {
		t.Fatal("tarDir of missing dir must fail")
	}
}

func TestGitRunSilentFailure(t *testing.T) {
	useFakeGit(t, fakeGitModeSilent)
	err := gitRun("", "clone", "x")
	if err == nil || strings.Contains(err.Error(), "clone x") {
		// silent failure surfaces the bare exec error, not a padded message
		t.Fatalf("gitRun silent failure = %v", err)
	}
}

func TestSpawnerNewBadDockerHost(t *testing.T) {
	t.Setenv("DOCKER_HOST", "http://")
	store := openTestStore(t)
	if _, err := NewSpawner(store, NewHub(store), "https://gitlab.example"); err == nil {
		t.Fatal("NewSpawner must fail on an unparseable DOCKER_HOST")
	}
}

func TestSpawnerStartDockerUnavailable(t *testing.T) {
	f := newFakeDocker()
	sp, _ := newTestSpawner(t, f)
	// point the spawner at a dead endpoint: last Start request kills nothing
	// because the client is already built; close the fake server instead.
	sp.cli.Close()
	closeDocker(t, sp)
	if _, err := sp.Start(context.Background(), "group/project", ""); err == nil || !strings.Contains(err.Error(), "docker unavailable") {
		t.Fatalf("Start with dead docker = %v, want docker unavailable", err)
	}
}

// closeDocker re-aims the spawner at a closed server (dial refused).
func closeDocker(t *testing.T, sp *Spawner) {
	t.Helper()
	dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		writeJSONNow(w, http.StatusInternalServerError, `{"message":"dead"}`)
	}))
	dead.Close() // listener gone: connections refused
	if err := sp.cli.Close(); err != nil {
		t.Fatalf("close docker client: %v", err)
	}
	cli, err := client.NewClientWithOpts(client.WithHost("tcp://" + dead.Listener.Addr().String()))
	if err != nil {
		t.Fatalf("rebuild docker client: %v", err)
	}
	sp.cli = cli
}

func TestSpawnerCloneOrUpdateErrors(t *testing.T) {
	f := newFakeDocker()
	sp, _ := newTestSpawner(t, f)

	// unparseable repo path (control char) → cloneURL parse error
	if err := sp.cloneOrUpdate("bad\nrepo", "", repoSlug("bad\nrepo")); err == nil {
		t.Fatal("cloneOrUpdate with control-char repo must fail")
	}

	// reposDir path occupied by a file → MkdirAll fails
	file := filepath.Join(t.TempDir(), "not-a-dir")
	if err := os.WriteFile(file, nil, 0o644); err != nil {
		t.Fatalf("write file: %v", err)
	}
	sp.reposDir = file
	if err := sp.cloneOrUpdate("group/project", "", "group-project"); err == nil {
		t.Fatal("cloneOrUpdate with file reposDir must fail")
	}
}

func TestSpawnerEnsureImageErrors(t *testing.T) {
	f := newFakeDocker()
	f.images = 0
	sp, _ := newTestSpawner(t, f)
	ctx := context.Background()

	// dockerfile missing on disk
	sp.dockerfile = filepath.Join(t.TempDir(), "gone.Dockerfile")
	if err := sp.ensureImage(ctx); err == nil || !strings.Contains(err.Error(), "worker Dockerfile missing") {
		t.Fatalf("ensureImage without dockerfile = %v", err)
	}

	// dockerfile exists but cannot be made relative to the build context
	sp.dockerfile = filepath.Join(t.TempDir(), "worker.Dockerfile")
	if err := os.WriteFile(sp.dockerfile, []byte("FROM scratch"), 0o644); err != nil {
		t.Fatalf("write: %v", err)
	}
	sp.buildContext = "relative-context"
	if err := sp.ensureImage(ctx); err == nil {
		t.Fatal("ensureImage with relative-context vs absolute dockerfile must fail")
	}

	// tar of a missing build context fails
	sp.buildContext = filepath.Join(t.TempDir(), "no-such-context")
	if err := sp.ensureImage(ctx); err == nil || !strings.Contains(err.Error(), "build context") {
		t.Fatalf("ensureImage with missing context = %v", err)
	}
}

func TestSpawnerSessionsVolumeCreateFails(t *testing.T) {
	useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	f.volume["lvmh-repo-group-project"] = true // repo volume exists → skip seed
	f.failVolumeCreate = true
	sp, _ := newTestSpawner(t, f)
	res, err := sp.Start(context.Background(), "group/project", "")
	if err != nil {
		t.Fatalf("Start: %v", err)
	}
	job := waitJobState(t, sp, res.SessionID, stateError)
	if !strings.Contains(job.Message, volumeSessions) {
		t.Fatalf("job message = %q, want sessions volume failure", job.Message)
	}
}

func TestSpawnerSeedVolumeCreateStartFail(t *testing.T) {
	useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	sp, _ := newTestSpawner(t, f)
	ctx := context.Background()

	f.failCreate = true
	if err := sp.seedVolume(ctx, "group-project", "lvmh-repo-group-project"); err == nil {
		t.Fatal("seedVolume with failing create must fail")
	}
	f.failCreate = false
	f.failStart = true
	if err := sp.seedVolume(ctx, "group-project", "lvmh-repo-group-project"); err == nil {
		t.Fatal("seedVolume with failing start must fail")
	}
}

func TestSpawnerStoreFailures(t *testing.T) {
	useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	sp, _ := newTestSpawner(t, f)

	// closed store: GetContainer errors
	deadStore := openTestStore(t)
	_ = deadStore.Close()
	sp.store = deadStore
	if err := sp.RemoveSession(context.Background(), "s1"); err == nil || !strings.Contains(err.Error(), "closed") {
		t.Fatalf("RemoveSession with closed store = %v", err)
	}

	// runJob completes container start but cannot persist the row
	sp2, _ := newTestSpawner(t, newFakeDocker())
	dead2 := openTestStore(t)
	_ = dead2.Close()
	sp2.store = dead2
	res, err := sp2.Start(context.Background(), "group/project", "")
	if err != nil {
		t.Fatalf("Start: %v", err)
	}
	job := waitJobState(t, sp2, res.SessionID, stateError)
	if !strings.Contains(job.Message, "not persisted") {
		t.Fatalf("job message = %q, want persistence failure", job.Message)
	}
}

func TestSpawnerCloneOrUpdatePullFails(t *testing.T) {
	useFakeGit(t, fakeGitModeFail)
	f := newFakeDocker()
	sp, _ := newTestSpawner(t, f)
	dir := filepath.Join(sp.reposDir, "group-project")
	if err := os.MkdirAll(filepath.Join(dir, ".git"), 0o755); err != nil {
		t.Fatalf("mkdir .git: %v", err)
	}
	err := sp.cloneOrUpdate("group/project", "", repoSlug("group/project"))
	if err == nil || !strings.Contains(err.Error(), "git pull") {
		t.Fatalf("cloneOrUpdate pull failure = %v, want git pull error", err)
	}
}

func TestSpawnerWorkerCreateStartFailures(t *testing.T) {
	// repo volume pre-exists → seeding skipped → failures surface at the
	// worker container create/start instead of at the seed container.
	useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	f.volume["lvmh-repo-group-project"] = true
	sp, _ := newTestSpawner(t, f)

	f.failCreate = true
	res, err := sp.Start(context.Background(), "group/project", "")
	if err != nil {
		t.Fatalf("Start: %v", err)
	}
	if job := waitJobState(t, sp, res.SessionID, stateError); !strings.Contains(job.Message, "docker create") {
		t.Fatalf("job = %+v, want docker create failure", job)
	}

	f.failCreate = false
	f.failStart = true
	res2, err := sp.Start(context.Background(), "group/project", "")
	if err != nil {
		t.Fatalf("Start 2: %v", err)
	}
	if job := waitJobState(t, sp, res2.SessionID, stateError); !strings.Contains(job.Message, "docker start") {
		t.Fatalf("job = %+v, want docker start failure", job)
	}
	// failed worker start must remove the created container
	if f.countCalls(http.MethodDelete, "/containers/") == 0 {
		t.Fatal("failed start must clean up the created container")
	}
}

func TestSpawnerBuildHTTPErrors(t *testing.T) {
	useFakeGit(t, fakeGitModeOK)
	f := newFakeDocker()
	f.images = 0
	f.failBuildHTTP = true // /build endpoint itself 500s
	sp, _ := newTestSpawner(t, f)
	res, err := sp.Start(context.Background(), "group/project", "")
	if err != nil {
		t.Fatalf("Start: %v", err)
	}
	if job := waitJobState(t, sp, res.SessionID, stateError); !strings.Contains(job.Message, "docker build") {
		t.Fatalf("job = %+v, want docker build failure", job)
	}

	// dead docker endpoint: ensureImage cannot even list images
	sp2, _ := newTestSpawner(t, newFakeDocker())
	sp2.images0AndDead(t)
	if err := sp2.ensureImage(context.Background()); err == nil {
		t.Fatal("ensureImage with dead docker must fail")
	}
}

func (sp *Spawner) images0AndDead(t *testing.T) {
	t.Helper()
	dead := httptest.NewServer(http.NotFoundHandler())
	t.Cleanup(dead.Close)
	dead.Close()
	if err := sp.cli.Close(); err != nil {
		t.Fatalf("close client: %v", err)
	}
	cli, err := client.NewClientWithOpts(client.WithHost("tcp://" + dead.Listener.Addr().String()))
	if err != nil {
		t.Fatalf("client: %v", err)
	}
	sp.cli = cli
}

// failingWriter errors on the Nth write onward.
type failingWriter struct {
	n, failAt int
}

func (w *failingWriter) Write(p []byte) (int, error) {
	w.n++
	if w.n >= w.failAt {
		return 0, fmt.Errorf("write boom %d", w.n)
	}
	return len(p), nil
}

func TestTarDirWriteErrors(t *testing.T) {
	dir := t.TempDir()
	if err := os.WriteFile(filepath.Join(dir, "f.txt"), []byte("content"), 0o644); err != nil {
		t.Fatalf("write: %v", err)
	}
	if err := tarDir(&failingWriter{failAt: 1}, dir); err == nil {
		t.Fatal("tarDir into failing writer must fail")
	}
	if err := tarDir(&failingWriter{failAt: 4}, dir); err == nil {
		t.Fatal("tarDir copy into failing writer must fail")
	}
}

func TestTarDirUnreadableFile(t *testing.T) {
	if os.Geteuid() == 0 {
		t.Skip("root ignores file permissions")
	}
	dir := t.TempDir()
	secret := filepath.Join(dir, "secret.txt")
	if err := os.WriteFile(secret, []byte("x"), 0o644); err != nil {
		t.Fatalf("write: %v", err)
	}
	if err := os.Chmod(secret, 0o000); err != nil {
		t.Fatalf("chmod: %v", err)
	}
	t.Cleanup(func() { _ = os.Chmod(secret, 0o644) })
	var buf bytes.Buffer
	if err := tarDir(&buf, dir); err == nil {
		t.Fatal("tarDir with unreadable file must fail")
	}
}

func TestGitRunUnderivableExit(t *testing.T) {
	// gitRun is the exec seam: with real git present, invoking a nonexistent
	// subcommand must surface stderr, and trimming applies to long output.
	if err := gitRun("", "version"); err != nil {
		t.Fatalf("git version: %v", err)
	}
	err := gitRun("", "this-subcommand-does-not-exist")
	if err == nil || !strings.Contains(err.Error(), "this-subcommand-does-not-exist") {
		t.Fatalf("gitRun unknown subcommand = %v", err)
	}
}