package main // docker.go — worker container spawner (clone → ensure image → run). import ( "archive/tar" "bytes" "context" "crypto/rand" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "log" "net/url" "os" "os/exec" "path/filepath" "regexp" "strings" "sync" "time" cerrdefs "github.com/containerd/errdefs" "github.com/docker/docker/api/types/build" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" ) // Spawn job states and docker wiring constants. const ( stateCloning string = "cloning" stateBuilding string = "building" stateCreating string = "creating" stateRunning string = "running" stateError string = "error" imageRefWorker string = "lvmh-worker:latest" labelSession string = "lvmh.session" volumeRepoPrefix string = "lvmh-repo-" volumeSessions string = "lvmh-sessions" volumePiCache string = "lvmh-pi-cache" // pi package cache (git:/npm:), shared across spawns cacheMount string = "/root/.pi/agent/cache" workspaceMount string = "/workspace" sessionsMount string = "/pi-sessions" envWorkerDockerfile string = "LVMH_WORKER_DOCKERFILE" defaultDockerfile string = "/app/build/docker/worker.Dockerfile" envWorkerContext string = "LVMH_WORKER_CONTEXT" envRepoDir string = "LVMH_REPO_DIR" defaultRepoDir string = "/data/repos" envContainerLVMHURL string = "LVMH_CONTAINER_LVMH_URL" defaultContainerURL string = "ws://lvmh:8686/agent/ws" envContainerNetwork string = "LVMH_CONTAINER_NETWORK" defaultNetwork string = "lvmh-net" envProviderAPIKey string = "ZAI_RENAUD_API_KEY" envLVMHSessionID string = "LVMH_SESSION_ID" envLVMHAgent string = "LVMH_AGENT" envLVMHRepo string = "LVMH_REPO" stopTimeoutSeconds int = 10 buildContextReadLimit int64 = 64 << 20 maxSpawnJobs int = 50 // jobs map pruned to this many entries ) var errNoContainer = errors.New("no container for session") var repoPathRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)+$`) // branchRe guards spawn branch names against git option injection. var branchRe = regexp.MustCompile(`^[A-Za-z0-9._/-]+$`) // validRepoPath accepts "group/project" style paths (at least two segments). func validRepoPath(repo string) bool { return repoPathRe.MatchString(repo) } // repoSlug maps a repo path to a collision-free filesystem/volume name: // the `/`→`--` form alone is ambiguous ("a/b--c" vs "a/b/c"), so the // first 6 hex chars of the sha256 of the full path disambiguate. Changing // this changes existing volume names (one-time fresh clone per repo). func repoSlug(repo string) string { sum := sha256.Sum256([]byte(repo)) return strings.ReplaceAll(repo, "/", "--") + "-" + hex.EncodeToString(sum[:3]) } // newUUID returns a random RFC 4122 v4 UUID string. func newUUID() string { var b [16]byte if _, err := rand.Read(b[:]); err != nil { // crypto/rand failure is fatal-grade; never expected in practice. panic("newUUID: crypto/rand unavailable: " + err.Error()) } b[6] = (b[6] & 0x0f) | 0x40 b[8] = (b[8] & 0x3f) | 0x80 dst := make([]byte, 36) hex.Encode(dst[0:8], b[0:4]) dst[8] = '-' hex.Encode(dst[9:13], b[4:6]) dst[13] = '-' hex.Encode(dst[14:18], b[6:8]) dst[18] = '-' hex.Encode(dst[19:23], b[8:10]) dst[23] = '-' hex.Encode(dst[24:36], b[10:16]) return string(dst) } // SpawnJob is one async spawn pipeline run, exposed via /api/spawn/status. type SpawnJob struct { Repo string `json:"repo"` State string `json:"state"` ContainerID string `json:"containerId"` SessionID string `json:"sessionId,omitempty"` Message string `json:"message,omitempty"` UpdatedAt int64 `json:"-"` } // SpawnResult is the immediate POST /api/spawn response. type SpawnResult struct { SessionID string `json:"sessionId"` ContainerID string `json:"containerId"` } // Spawner owns the docker client, per-repo clone serialization and job state. type Spawner struct { ctx context.Context // base ctx for async jobs (cancelled on shutdown) store *Store hub *Hub cli *client.Client baseURL string // gitlab base, for clone URLs reposDir string dockerfile string controlDockerfile string buildContext string containerURL string network string mu sync.Mutex jobs map[string]*SpawnJob // keyed by sessionId jobOrder []string // insertion order of jobs, for pruning slugLocks map[string]*sync.Mutex } func NewSpawner(ctx context.Context, store *Store, hub *Hub, baseURL string) (*Spawner, error) { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { return nil, fmt.Errorf("docker client: %w", err) } dockerfile := envOr(envWorkerDockerfile, defaultDockerfile) return &Spawner{ ctx: ctx, store: store, hub: hub, cli: cli, baseURL: strings.TrimRight(baseURL, "/"), reposDir: envOr(envRepoDir, defaultRepoDir), dockerfile: dockerfile, controlDockerfile: envOr(envControlDockerfile, defaultControlDockerfile), buildContext: envOr(envWorkerContext, filepath.Dir(filepath.Dir(dockerfile))), containerURL: envOr(envContainerLVMHURL, defaultContainerURL), network: envOr(envContainerNetwork, defaultNetwork), jobs: make(map[string]*SpawnJob), slugLocks: make(map[string]*sync.Mutex), }, nil } func envOr(key, def string) string { if v := strings.TrimSpace(os.Getenv(key)); v != "" { return v } return def } // JobsSnapshot returns current spawn jobs for /api/spawn/status and /ws pushes. func (s *Spawner) JobsSnapshot() []SpawnJob { s.mu.Lock() defer s.mu.Unlock() out := make([]SpawnJob, 0, len(s.jobs)) for _, j := range s.jobs { out = append(out, *j) } return out } func (s *Spawner) setJob(sessionID, repo, state, containerID, message string) { s.mu.Lock() j, ok := s.jobs[sessionID] if !ok { j = &SpawnJob{SessionID: sessionID, Repo: repo} s.jobs[sessionID] = j s.jobOrder = append(s.jobOrder, sessionID) if len(s.jobOrder) > maxSpawnJobs { oldest := s.jobOrder[0] copy(s.jobOrder, s.jobOrder[1:]) s.jobOrder = s.jobOrder[:len(s.jobOrder)-1] delete(s.jobs, oldest) } } j.Repo = repo j.State = state j.ContainerID = containerID j.Message = message j.UpdatedAt = time.Now().UnixMilli() s.mu.Unlock() s.hub.BroadcastSpawnStatus() } // deleteJob drops a session's job from the map and the insertion order. func (s *Spawner) deleteJob(sessionID string) { s.mu.Lock() if _, ok := s.jobs[sessionID]; !ok { s.mu.Unlock() return } delete(s.jobs, sessionID) for i, id := range s.jobOrder { if id == sessionID { s.jobOrder = append(s.jobOrder[:i], s.jobOrder[i+1:]...) break } } s.mu.Unlock() } // Start launches the async spawn pipeline and returns the new sessionId. func (s *Spawner) Start(ctx context.Context, repo, branch string) (SpawnResult, error) { exists, err := s.imageExists(ctx) if err != nil { return SpawnResult{}, fmt.Errorf("docker unavailable: %w", err) } if !exists { if _, statErr := os.Stat(s.dockerfile); statErr != nil { return SpawnResult{}, fmt.Errorf( "image %s not found and no worker Dockerfile at %s (set %s or run `make worker-image`)", imageRefWorker, s.dockerfile, envWorkerDockerfile) } } sessionID := newUUID() s.setJob(sessionID, repo, stateCloning, "", "") go s.runJob(repo, branch, sessionID) return SpawnResult{SessionID: sessionID}, nil } func (s *Spawner) imageExists(ctx context.Context) (bool, error) { return s.imageRefExists(ctx, imageRefWorker) } // imageRefExists reports whether ref is present in the local docker store. func (s *Spawner) imageRefExists(ctx context.Context, ref string) (bool, error) { args := filters.NewArgs(filters.Arg("reference", ref)) summaries, err := s.cli.ImageList(ctx, image.ListOptions{Filters: args}) if err != nil { return false, err } return len(summaries) > 0, nil } func (s *Spawner) slugLock(slug string) *sync.Mutex { s.mu.Lock() defer s.mu.Unlock() l, ok := s.slugLocks[slug] if !ok { l = &sync.Mutex{} s.slugLocks[slug] = l } return l } // runJob is the async clone→build→create→start pipeline. func (s *Spawner) runJob(repo, branch, sessionID string) { slug := repoSlug(repo) lock := s.slugLock(slug) lock.Lock() defer lock.Unlock() if err := s.cloneOrUpdate(s.ctx, repo, branch, slug); err != nil { s.setJob(sessionID, repo, stateError, "", err.Error()) return } s.setJob(sessionID, repo, stateBuilding, "", "") if err := s.ensureImage(s.ctx); err != nil { s.setJob(sessionID, repo, stateError, "", err.Error()) return } s.setJob(sessionID, repo, stateCreating, "", "") containerID, err := s.createAndStart(s.ctx, repo, slug, sessionID) if err != nil { s.setJob(sessionID, repo, stateError, "", err.Error()) return } if err := s.store.UpsertContainer(sessionID, containerID, repo); err != nil { s.setJob(sessionID, repo, stateError, containerID, "container started but not persisted: "+err.Error()) return } s.setJob(sessionID, repo, stateRunning, containerID, "") } // cloneOrUpdate clones the repo into reposDir/ or fast-forwards it. // Credentials never appear in the clone URL (which git persists into // .git/config); auth is passed per-invocation via http.extraHeader. func (s *Spawner) cloneOrUpdate(ctx context.Context, repo, branch, slug string) error { dir := filepath.Join(s.reposDir, slug) cloneURL, err := s.cloneURL(repo) if err != nil { return err } if st, err := os.Stat(filepath.Join(dir, ".git")); err == nil && st.IsDir() { auth := s.gitAuthArgs() if branch != "" && gitBranch(ctx, dir) != branch { if err := gitRun(ctx, dir, append(append([]string{}, auth...), "fetch", "origin", branch)...); err != nil { return fmt.Errorf("git fetch %s %s: %w", repo, branch, err) } if err := gitRun(ctx, dir, "checkout", branch); err != nil { return fmt.Errorf("git checkout %s: %w", repo, err) } } if err := gitRun(ctx, dir, append(append([]string{}, auth...), "pull", "--ff-only")...); err != nil { return fmt.Errorf("git pull %s: %w", repo, err) } return nil } if err := os.MkdirAll(s.reposDir, 0o755); err != nil { return err } args := append(s.gitAuthArgs(), "clone") if branch != "" { args = append(args, "--branch", branch) } args = append(args, "--", cloneURL, dir) if err := gitRun(ctx, "", args...); err != nil { return fmt.Errorf("git clone %s: %w", repo, err) } return nil } // cloneURL builds the clean https clone URL (never credential-bearing). func (s *Spawner) cloneURL(repo string) (string, error) { u, err := url.Parse(s.baseURL + "/" + repo + ".git") if err != nil { return "", err } return u.String(), nil } // gitAuthArgs returns per-invocation git args carrying the stored PAT via // an HTTP header, or nil when no PAT is stored. func (s *Spawner) gitAuthArgs() []string { token, ok, _ := s.store.GetSetting(settingGitLabToken) if !ok || token == "" { return nil } return []string{"-c", "http.extraHeader=Authorization: token " + token} } // gitBranch returns the checked-out branch of an existing clone, "" on failure. func gitBranch(ctx context.Context, dir string) string { out, err := exec.CommandContext(ctx, "git", "-C", dir, "rev-parse", "--abbrev-ref", "HEAD").Output() if err != nil { return "" } return strings.TrimSpace(string(out)) } func gitRun(ctx context.Context, dir string, args ...string) error { cmd := exec.CommandContext(ctx, "git", args...) if dir != "" { cmd.Dir = dir } var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out if err := cmd.Run(); err != nil { msg := strings.TrimSpace(out.String()) if len(msg) > 500 { msg = msg[len(msg)-500:] } if msg != "" { return fmt.Errorf("%w: %s", err, msg) } return err } return nil } // ensureImage builds lvmh-worker:latest when missing. func (s *Spawner) ensureImage(ctx context.Context) error { exists, err := s.imageExists(ctx) if err != nil { return err } if exists { return nil } if _, err := os.Stat(s.dockerfile); err != nil { return fmt.Errorf("worker Dockerfile missing at %s: %w", s.dockerfile, err) } return s.buildImage(ctx, s.buildContext, s.dockerfile, imageRefWorker) } // buildImage streams a tar of buildContext into docker ImageBuild, building // tag from dockerfile (relative to buildContext). Shared by the worker and // ops image paths. func (s *Spawner) buildImage(ctx context.Context, buildContext, dockerfile, tag string) error { relDockerfile, err := filepath.Rel(buildContext, dockerfile) if err != nil { return err } pr, pw := io.Pipe() tarDone := make(chan error, 1) go func() { err := tarDir(pw, buildContext) _ = pw.CloseWithError(err) tarDone <- err }() resp, buildErr := s.cli.ImageBuild(ctx, pr, build.ImageBuildOptions{ Tags: []string{tag}, Dockerfile: relDockerfile, Remove: true, }) if buildErr != nil { _ = pr.CloseWithError(buildErr) // unblock the tar goroutine } if tarErr := <-tarDone; tarErr != nil { if resp.Body != nil { _ = resp.Body.Close() } return fmt.Errorf("build context %s: %w", buildContext, tarErr) } if buildErr != nil { return fmt.Errorf("docker build: %w", buildErr) } defer resp.Body.Close() body, _ := io.ReadAll(io.LimitReader(resp.Body, buildContextReadLimit)) if msg := extractBuildError(body); msg != "" { return fmt.Errorf("docker build failed: %s", msg) } return nil } // extractBuildError scans the build output stream for an "error" field. func extractBuildError(body []byte) string { for _, line := range bytes.Split(body, []byte("\n")) { var msg struct { Error string `json:"error"` ErrorDetail struct { Message string `json:"message"` } `json:"errorDetail"` } if json.Unmarshal(line, &msg) != nil { continue } if msg.Error != "" || msg.ErrorDetail.Message != "" { if msg.ErrorDetail.Message != "" { return msg.ErrorDetail.Message } return msg.Error } } return "" } // createAndStart provisions volumes, creates and starts the worker container. // A repo-registered custom image (see /api/repos) overrides the default // worker image; the ops agent builds and registers those. func (s *Spawner) createAndStart(ctx context.Context, repo, slug, sessionID string) (string, error) { image := imageRefWorker if custom, ok, _ := s.store.GetRepoImage(repo); ok && custom != "" { exists, err := s.imageRefExists(ctx, custom) if err != nil { return "", fmt.Errorf("docker unavailable: %w", err) } if !exists { return "", fmt.Errorf("custom image %s not built (ask the ops agent to build it)", custom) } image = custom } repoVolume := volumeRepoPrefix + slug fresh, err := s.ensureRepoVolume(ctx, repoVolume) if err != nil { return "", err } if fresh { if err := s.seedVolume(ctx, slug, repoVolume, sessionID); err != nil { // drop the half-seeded volume so the next spawn retries fresh // instead of silently booting into an empty workspace. if rmErr := s.cli.VolumeRemove(context.Background(), repoVolume, true); rmErr != nil { log.Printf("spawner: remove failed seed volume %s: %v", repoVolume, rmErr) } return "", fmt.Errorf("seed %s: %w", repoVolume, err) } } if _, err := s.cli.VolumeCreate(ctx, volume.CreateOptions{Name: volumeSessions}); err != nil { return "", fmt.Errorf("volume %s: %w", volumeSessions, err) } if _, err := s.cli.VolumeCreate(ctx, volume.CreateOptions{Name: volumePiCache}); err != nil { return "", fmt.Errorf("volume %s: %w", volumePiCache, err) } binds := []string{ repoVolume + ":" + workspaceMount, volumeSessions + ":" + sessionsMount, volumePiCache + ":" + cacheMount, } cfg := &container.Config{ Image: image, Env: []string{ envProviderAPIKey + "=" + os.Getenv(envProviderAPIKey), envToken + "=" + daemonToken, "LVMH_URL=" + s.containerURL, envLVMHSessionID + "=" + sessionID, envLVMHAgent + "=1", envLVMHRepo + "=" + repo, }, Labels: map[string]string{labelSession: sessionID}, } hostCfg := &container.HostConfig{ Binds: binds, NetworkMode: container.NetworkMode(s.network), AutoRemove: false, Init: &[]bool{true}[0], // tini reaps bridge setup-hook zombies } name := "lvmh-agent-" + strings.ReplaceAll(sessionID, "-", "")[:12] created, err := s.cli.ContainerCreate(ctx, cfg, hostCfg, nil, nil, name) if err != nil { return "", fmt.Errorf("docker create: %w", err) } if err := s.cli.ContainerStart(ctx, created.ID, container.StartOptions{}); err != nil { s.removeContainer(context.Background(), created.ID) return "", fmt.Errorf("docker start: %w", err) } return created.ID, nil } // ensureRepoVolume creates the per-repo volume; reports whether it is fresh. // Only a definitive "no such volume" (docker errdefs NotFound) counts as // fresh: a transient inspect error must never re-seed over an existing // volume, so it propagates instead. func (s *Spawner) ensureRepoVolume(ctx context.Context, name string) (bool, error) { if _, err := s.cli.VolumeInspect(ctx, name); err == nil { return false, nil } else if !cerrdefs.IsNotFound(err) { return false, fmt.Errorf("volume %s: %w", name, err) } if _, err := s.cli.VolumeCreate(ctx, volume.CreateOptions{Name: name}); err != nil { return false, fmt.Errorf("volume %s: %w", name, err) } return true, nil } // seedVolume populates the fresh named volume with the cloned repo by // streaming a tar of the clone into a paused container via the docker API // (CopyToContainer). No bind mounts: bind sources are HOST paths, but the // clone lives inside the daemon container's filesystem. func (s *Spawner) seedVolume(ctx context.Context, slug, repoVolume, sessionID string) error { repoDir := filepath.Join(s.reposDir, slug) cfg := &container.Config{ Image: imageRefWorker, Cmd: []string{"true"}, // never started; created only as a volume mount point } hostCfg := &container.HostConfig{ Binds: []string{repoVolume + ":" + workspaceMount}, Init: &[]bool{true}[0], } // Unique name per session: a fixed name would conflict forever after a // crash between create and the deferred remove. unq := strings.ReplaceAll(sessionID, "-", "") if len(unq) > 8 { unq = unq[:8] } name := "lvmh-seed-" + slug + "-" + unq created, err := s.cli.ContainerCreate(ctx, cfg, hostCfg, nil, nil, name) if err != nil { return err } defer s.removeContainer(context.Background(), created.ID) pr, pw := io.Pipe() tarDone := make(chan error, 1) go func() { err := tarDir(pw, repoDir) _ = pw.CloseWithError(err) tarDone <- err }() copyErr := s.cli.CopyToContainer(ctx, created.ID, workspaceMount, pr, container.CopyToContainerOptions{}) if copyErr != nil { _ = pr.CloseWithError(copyErr) // unblock the tar goroutine } if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr } if tarErr := <-tarDone; tarErr != nil { return fmt.Errorf("tar clone: %w", tarErr) } if copyErr != nil { return fmt.Errorf("copy into volume: %w", copyErr) } return nil } func (s *Spawner) removeContainer(ctx context.Context, id string) { _ = s.cli.ContainerRemove(ctx, id, container.RemoveOptions{Force: true}) } // RemoveSession stops and removes the container spawned for a session. // Stop and remove run on background contexts on purpose: a caller hanging up // mid-request must not leave an orphaned container behind a deleted DB row. func (s *Spawner) RemoveSession(_ context.Context, sessionID string) error { row, ok, err := s.store.GetContainer(sessionID) if err != nil { return err } if !ok { return errNoContainer } stopCtx, cancel := context.WithTimeout(context.Background(), time.Duration(stopTimeoutSeconds)*time.Second) if err := s.cli.ContainerStop(stopCtx, row.ContainerID, container.StopOptions{}); err != nil { cancel() return fmt.Errorf("docker stop: %w", err) } cancel() s.removeContainer(context.Background(), row.ContainerID) if err := s.store.DeleteContainer(sessionID); err != nil { return err } s.deleteJob(sessionID) s.hub.BroadcastSpawnStatus() return nil } // tarDir writes a tar archive of dir (nothing excluded, .git included) into w. // Symlinks are archived as symlinks with their target in Linkname so docker's // untar can recreate them; only regular files carry content. func tarDir(w io.Writer, dir string) error { tw := tar.NewWriter(w) defer tw.Close() return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } rel, err := filepath.Rel(dir, path) if err != nil { return err } if rel == "." { return nil } name := filepath.ToSlash(rel) link := "" if info.Mode()&os.ModeSymlink != 0 { link, err = os.Readlink(path) if err != nil { return err } } hdr, err := tar.FileInfoHeader(info, link) if err != nil { return err } hdr.Name = name if info.IsDir() { hdr.Name += "/" } hdr.Uname = "root" hdr.Gname = "root" if err := tw.WriteHeader(hdr); err != nil { return err } if info.Mode().IsRegular() { f, err := os.Open(path) if err != nil { return err } defer f.Close() if _, err := io.Copy(tw, f); err != nil { return err } } return nil }) }