lvmh/daemon/docker.go

package main

// docker.go — worker container spawner (clone → ensure image → run).

import (
	"archive/tar"
	"bytes"
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	cerrdefs "github.com/containerd/errdefs"
	"github.com/docker/docker/api/types/build"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/api/types/image"
	"github.com/docker/docker/api/types/volume"
	"github.com/docker/docker/client"
)

// Spawn job states and docker wiring constants.
const (
	stateCloning  string = "cloning"
	stateBuilding string = "building"
	stateCreating string = "creating"
	stateRunning  string = "running"
	stateError    string = "error"

	imageRefWorker   string = "lvmh-worker:latest"
	labelSession     string = "lvmh.session"
	volumeRepoPrefix string = "lvmh-repo-"
	volumeSessions   string = "lvmh-sessions"
	volumePiCache    string = "lvmh-pi-cache" // pi package cache (git:/npm:), shared across spawns
	cacheMount       string = "/root/.pi/agent/cache"
	workspaceMount   string = "/workspace"
	sessionsMount    string = "/pi-sessions"

	envWorkerDockerfile   string = "LVMH_WORKER_DOCKERFILE"
	defaultDockerfile     string = "/app/build/docker/worker.Dockerfile"
	envWorkerContext      string = "LVMH_WORKER_CONTEXT"
	envRepoDir            string = "LVMH_REPO_DIR"
	defaultRepoDir        string = "/data/repos"
	envContainerLVMHURL   string = "LVMH_CONTAINER_LVMH_URL"
	defaultContainerURL   string = "ws://lvmh:8686/agent/ws"
	envContainerNetwork   string = "LVMH_CONTAINER_NETWORK"
	defaultNetwork        string = "lvmh-net"
	envProviderAPIKey     string = "ZAI_RENAUD_API_KEY"
	envLVMHSessionID      string = "LVMH_SESSION_ID"
	envLVMHAgent          string = "LVMH_AGENT"
	envLVMHRepo           string = "LVMH_REPO"
	stopTimeoutSeconds    int    = 10
	buildContextReadLimit int64  = 64 << 20
	maxSpawnJobs          int    = 50 // jobs map pruned to this many entries
)

var errNoContainer = errors.New("no container for session")

var repoPathRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)+$`)

// branchRe guards spawn branch names against git option injection.
var branchRe = regexp.MustCompile(`^[A-Za-z0-9._/-]+$`)

// validRepoPath accepts "group/project" style paths (at least two segments).
func validRepoPath(repo string) bool { return repoPathRe.MatchString(repo) }

// repoSlug maps a repo path to a collision-free filesystem/volume name:
// the `/`→`--` form alone is ambiguous ("a/b--c" vs "a/b/c"), so the
// first 6 hex chars of the sha256 of the full path disambiguate. Changing
// this changes existing volume names (one-time fresh clone per repo).
func repoSlug(repo string) string {
	sum := sha256.Sum256([]byte(repo))
	return strings.ReplaceAll(repo, "/", "--") + "-" + hex.EncodeToString(sum[:3])
}

// newUUID returns a random RFC 4122 v4 UUID string.
func newUUID() string {
	var b [16]byte
	if _, err := rand.Read(b[:]); err != nil {
		// crypto/rand failure is fatal-grade; never expected in practice.
		panic("newUUID: crypto/rand unavailable: " + err.Error())
	}
	b[6] = (b[6] & 0x0f) | 0x40
	b[8] = (b[8] & 0x3f) | 0x80
	dst := make([]byte, 36)
	hex.Encode(dst[0:8], b[0:4])
	dst[8] = '-'
	hex.Encode(dst[9:13], b[4:6])
	dst[13] = '-'
	hex.Encode(dst[14:18], b[6:8])
	dst[18] = '-'
	hex.Encode(dst[19:23], b[8:10])
	dst[23] = '-'
	hex.Encode(dst[24:36], b[10:16])
	return string(dst)
}

// SpawnJob is one async spawn pipeline run, exposed via /api/spawn/status.
type SpawnJob struct {
	Repo        string `json:"repo"`
	State       string `json:"state"`
	ContainerID string `json:"containerId"`
	SessionID   string `json:"sessionId,omitempty"`
	Message     string `json:"message,omitempty"`
	UpdatedAt   int64  `json:"-"`
}

// SpawnResult is the immediate POST /api/spawn response.
type SpawnResult struct {
	SessionID   string `json:"sessionId"`
	ContainerID string `json:"containerId"`
}

// Spawner owns the docker client, per-repo clone serialization and job state.
type Spawner struct {
	ctx     context.Context // base ctx for async jobs (cancelled on shutdown)
	store   *Store
	hub     *Hub
	cli     *client.Client
	baseURL string // gitlab base, for clone URLs

	reposDir          string
	dockerfile        string
	controlDockerfile string
	buildContext      string
	containerURL      string
	network           string

	mu        sync.Mutex
	jobs      map[string]*SpawnJob // keyed by sessionId
	jobOrder  []string             // insertion order of jobs, for pruning
	slugLocks map[string]*sync.Mutex
}

func NewSpawner(ctx context.Context, store *Store, hub *Hub, baseURL string) (*Spawner, error) {
	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
	if err != nil {
		return nil, fmt.Errorf("docker client: %w", err)
	}
	dockerfile := envOr(envWorkerDockerfile, defaultDockerfile)
	return &Spawner{
		ctx:               ctx,
		store:             store,
		hub:               hub,
		cli:               cli,
		baseURL:           strings.TrimRight(baseURL, "/"),
		reposDir:          envOr(envRepoDir, defaultRepoDir),
		dockerfile:        dockerfile,
		controlDockerfile: envOr(envControlDockerfile, defaultControlDockerfile),
		buildContext:      envOr(envWorkerContext, filepath.Dir(filepath.Dir(dockerfile))),
		containerURL:      envOr(envContainerLVMHURL, defaultContainerURL),
		network:           envOr(envContainerNetwork, defaultNetwork),
		jobs:              make(map[string]*SpawnJob),
		slugLocks:         make(map[string]*sync.Mutex),
	}, nil
}

func envOr(key, def string) string {
	if v := strings.TrimSpace(os.Getenv(key)); v != "" {
		return v
	}
	return def
}

// JobsSnapshot returns current spawn jobs for /api/spawn/status and /ws pushes.
func (s *Spawner) JobsSnapshot() []SpawnJob {
	s.mu.Lock()
	defer s.mu.Unlock()
	out := make([]SpawnJob, 0, len(s.jobs))
	for _, j := range s.jobs {
		out = append(out, *j)
	}
	return out
}

func (s *Spawner) setJob(sessionID, repo, state, containerID, message string) {
	s.mu.Lock()
	j, ok := s.jobs[sessionID]
	if !ok {
		j = &SpawnJob{SessionID: sessionID, Repo: repo}
		s.jobs[sessionID] = j
		s.jobOrder = append(s.jobOrder, sessionID)
		if len(s.jobOrder) > maxSpawnJobs {
			oldest := s.jobOrder[0]
			copy(s.jobOrder, s.jobOrder[1:])
			s.jobOrder = s.jobOrder[:len(s.jobOrder)-1]
			delete(s.jobs, oldest)
		}
	}
	j.Repo = repo
	j.State = state
	j.ContainerID = containerID
	j.Message = message
	j.UpdatedAt = time.Now().UnixMilli()
	s.mu.Unlock()
	s.hub.BroadcastSpawnStatus()
}

// deleteJob drops a session's job from the map and the insertion order.
func (s *Spawner) deleteJob(sessionID string) {
	s.mu.Lock()
	if _, ok := s.jobs[sessionID]; !ok {
		s.mu.Unlock()
		return
	}
	delete(s.jobs, sessionID)
	for i, id := range s.jobOrder {
		if id == sessionID {
			s.jobOrder = append(s.jobOrder[:i], s.jobOrder[i+1:]...)
			break
		}
	}
	s.mu.Unlock()
}

// Start launches the async spawn pipeline and returns the new sessionId.
func (s *Spawner) Start(ctx context.Context, repo, branch string) (SpawnResult, error) {
	exists, err := s.imageExists(ctx)
	if err != nil {
		return SpawnResult{}, fmt.Errorf("docker unavailable: %w", err)
	}
	if !exists {
		if _, statErr := os.Stat(s.dockerfile); statErr != nil {
			return SpawnResult{}, fmt.Errorf(
				"image %s not found and no worker Dockerfile at %s (set %s or run `make worker-image`)",
				imageRefWorker, s.dockerfile, envWorkerDockerfile)
		}
	}
	sessionID := newUUID()
	s.setJob(sessionID, repo, stateCloning, "", "")
	go s.runJob(repo, branch, sessionID)
	return SpawnResult{SessionID: sessionID}, nil
}

func (s *Spawner) imageExists(ctx context.Context) (bool, error) {
	return s.imageRefExists(ctx, imageRefWorker)
}

// imageRefExists reports whether ref is present in the local docker store.
func (s *Spawner) imageRefExists(ctx context.Context, ref string) (bool, error) {
	args := filters.NewArgs(filters.Arg("reference", ref))
	summaries, err := s.cli.ImageList(ctx, image.ListOptions{Filters: args})
	if err != nil {
		return false, err
	}
	return len(summaries) > 0, nil
}

func (s *Spawner) slugLock(slug string) *sync.Mutex {
	s.mu.Lock()
	defer s.mu.Unlock()
	l, ok := s.slugLocks[slug]
	if !ok {
		l = &sync.Mutex{}
		s.slugLocks[slug] = l
	}
	return l
}

// runJob is the async clone→build→create→start pipeline.
func (s *Spawner) runJob(repo, branch, sessionID string) {
	slug := repoSlug(repo)
	lock := s.slugLock(slug)
	lock.Lock()
	defer lock.Unlock()

	if err := s.cloneOrUpdate(s.ctx, repo, branch, slug); err != nil {
		s.setJob(sessionID, repo, stateError, "", err.Error())
		return
	}
	s.setJob(sessionID, repo, stateBuilding, "", "")
	if err := s.ensureImage(s.ctx); err != nil {
		s.setJob(sessionID, repo, stateError, "", err.Error())
		return
	}
	s.setJob(sessionID, repo, stateCreating, "", "")
	containerID, err := s.createAndStart(s.ctx, repo, slug, sessionID)
	if err != nil {
		s.setJob(sessionID, repo, stateError, "", err.Error())
		return
	}
	if err := s.store.UpsertContainer(sessionID, containerID, repo); err != nil {
		s.setJob(sessionID, repo, stateError, containerID, "container started but not persisted: "+err.Error())
		return
	}
	s.setJob(sessionID, repo, stateRunning, containerID, "")
}

// cloneOrUpdate clones the repo into reposDir/<slug> or fast-forwards it.
// Credentials never appear in the clone URL (which git persists into
// .git/config); auth is passed per-invocation via http.extraHeader.
func (s *Spawner) cloneOrUpdate(ctx context.Context, repo, branch, slug string) error {
	dir := filepath.Join(s.reposDir, slug)
	cloneURL, err := s.cloneURL(repo)
	if err != nil {
		return err
	}
	if st, err := os.Stat(filepath.Join(dir, ".git")); err == nil && st.IsDir() {
		auth := s.gitAuthArgs()
		if branch != "" && gitBranch(ctx, dir) != branch {
			if err := gitRun(ctx, dir, append(append([]string{}, auth...), "fetch", "origin", branch)...); err != nil {
				return fmt.Errorf("git fetch %s %s: %w", repo, branch, err)
			}
			if err := gitRun(ctx, dir, "checkout", branch); err != nil {
				return fmt.Errorf("git checkout %s: %w", repo, err)
			}
		}
		if err := gitRun(ctx, dir, append(append([]string{}, auth...), "pull", "--ff-only")...); err != nil {
			return fmt.Errorf("git pull %s: %w", repo, err)
		}
		return nil
	}
	if err := os.MkdirAll(s.reposDir, 0o755); err != nil {
		return err
	}
	args := append(s.gitAuthArgs(), "clone")
	if branch != "" {
		args = append(args, "--branch", branch)
	}
	args = append(args, "--", cloneURL, dir)
	if err := gitRun(ctx, "", args...); err != nil {
		return fmt.Errorf("git clone %s: %w", repo, err)
	}
	return nil
}

// cloneURL builds the clean https clone URL (never credential-bearing).
func (s *Spawner) cloneURL(repo string) (string, error) {
	u, err := url.Parse(s.baseURL + "/" + repo + ".git")
	if err != nil {
		return "", err
	}
	return u.String(), nil
}

// gitAuthArgs returns per-invocation git args carrying the stored PAT via
// an HTTP header, or nil when no PAT is stored.
func (s *Spawner) gitAuthArgs() []string {
	token, ok, _ := s.store.GetSetting(settingGitLabToken)
	if !ok || token == "" {
		return nil
	}
	return []string{"-c", "http.extraHeader=Authorization: token " + token}
}

// gitBranch returns the checked-out branch of an existing clone, "" on failure.
func gitBranch(ctx context.Context, dir string) string {
	out, err := exec.CommandContext(ctx, "git", "-C", dir, "rev-parse", "--abbrev-ref", "HEAD").Output()
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(out))
}

func gitRun(ctx context.Context, dir string, args ...string) error {
	cmd := exec.CommandContext(ctx, "git", args...)
	if dir != "" {
		cmd.Dir = dir
	}
	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out
	if err := cmd.Run(); err != nil {
		msg := strings.TrimSpace(out.String())
		if len(msg) > 500 {
			msg = msg[len(msg)-500:]
		}
		if msg != "" {
			return fmt.Errorf("%w: %s", err, msg)
		}
		return err
	}
	return nil
}

// ensureImage builds lvmh-worker:latest when missing.
func (s *Spawner) ensureImage(ctx context.Context) error {
	exists, err := s.imageExists(ctx)
	if err != nil {
		return err
	}
	if exists {
		return nil
	}
	if _, err := os.Stat(s.dockerfile); err != nil {
		return fmt.Errorf("worker Dockerfile missing at %s: %w", s.dockerfile, err)
	}
	return s.buildImage(ctx, s.buildContext, s.dockerfile, imageRefWorker)
}

// buildImage streams a tar of buildContext into docker ImageBuild, building
// tag from dockerfile (relative to buildContext). Shared by the worker and
// ops image paths.
func (s *Spawner) buildImage(ctx context.Context, buildContext, dockerfile, tag string) error {
	relDockerfile, err := filepath.Rel(buildContext, dockerfile)
	if err != nil {
		return err
	}
	pr, pw := io.Pipe()
	tarDone := make(chan error, 1)
	go func() {
		err := tarDir(pw, buildContext)
		_ = pw.CloseWithError(err)
		tarDone <- err
	}()
	resp, buildErr := s.cli.ImageBuild(ctx, pr, build.ImageBuildOptions{
		Tags:       []string{tag},
		Dockerfile: relDockerfile,
		Remove:     true,
	})
	if buildErr != nil {
		_ = pr.CloseWithError(buildErr) // unblock the tar goroutine
	}
	if tarErr := <-tarDone; tarErr != nil {
		if resp.Body != nil {
			_ = resp.Body.Close()
		}
		return fmt.Errorf("build context %s: %w", buildContext, tarErr)
	}
	if buildErr != nil {
		return fmt.Errorf("docker build: %w", buildErr)
	}
	defer resp.Body.Close()
	body, _ := io.ReadAll(io.LimitReader(resp.Body, buildContextReadLimit))
	if msg := extractBuildError(body); msg != "" {
		return fmt.Errorf("docker build failed: %s", msg)
	}
	return nil
}

// extractBuildError scans the build output stream for an "error" field.
func extractBuildError(body []byte) string {
	for _, line := range bytes.Split(body, []byte("\n")) {
		var msg struct {
			Error       string `json:"error"`
			ErrorDetail struct {
				Message string `json:"message"`
			} `json:"errorDetail"`
		}
		if json.Unmarshal(line, &msg) != nil {
			continue
		}
		if msg.Error != "" || msg.ErrorDetail.Message != "" {
			if msg.ErrorDetail.Message != "" {
				return msg.ErrorDetail.Message
			}
			return msg.Error
		}
	}
	return ""
}

// createAndStart provisions volumes, creates and starts the worker container.
// A repo-registered custom image (see /api/repos) overrides the default
// worker image; the ops agent builds and registers those.
func (s *Spawner) createAndStart(ctx context.Context, repo, slug, sessionID string) (string, error) {
	image := imageRefWorker
	if custom, ok, _ := s.store.GetRepoImage(repo); ok && custom != "" {
		exists, err := s.imageRefExists(ctx, custom)
		if err != nil {
			return "", fmt.Errorf("docker unavailable: %w", err)
		}
		if !exists {
			return "", fmt.Errorf("custom image %s not built (ask the ops agent to build it)", custom)
		}
		image = custom
	}
	repoVolume := volumeRepoPrefix + slug
	fresh, err := s.ensureRepoVolume(ctx, repoVolume)
	if err != nil {
		return "", err
	}
	if fresh {
		if err := s.seedVolume(ctx, slug, repoVolume, sessionID); err != nil {
			// drop the half-seeded volume so the next spawn retries fresh
			// instead of silently booting into an empty workspace.
			if rmErr := s.cli.VolumeRemove(context.Background(), repoVolume, true); rmErr != nil {
				log.Printf("spawner: remove failed seed volume %s: %v", repoVolume, rmErr)
			}
			return "", fmt.Errorf("seed %s: %w", repoVolume, err)
		}
	}
	if _, err := s.cli.VolumeCreate(ctx, volume.CreateOptions{Name: volumeSessions}); err != nil {
		return "", fmt.Errorf("volume %s: %w", volumeSessions, err)
	}
	if _, err := s.cli.VolumeCreate(ctx, volume.CreateOptions{Name: volumePiCache}); err != nil {
		return "", fmt.Errorf("volume %s: %w", volumePiCache, err)
	}

	binds := []string{
		repoVolume + ":" + workspaceMount,
		volumeSessions + ":" + sessionsMount,
		volumePiCache + ":" + cacheMount,
	}
	cfg := &container.Config{
		Image: image,
		Env: []string{
			envProviderAPIKey + "=" + os.Getenv(envProviderAPIKey),
			envToken + "=" + daemonToken,
			"LVMH_URL=" + s.containerURL,
			envLVMHSessionID + "=" + sessionID,
			envLVMHAgent + "=1",
			envLVMHRepo + "=" + repo,
		},
		Labels: map[string]string{labelSession: sessionID},
	}
	hostCfg := &container.HostConfig{
		Binds:       binds,
		NetworkMode: container.NetworkMode(s.network),
		AutoRemove:  false,
		Init:        &[]bool{true}[0], // tini reaps bridge setup-hook zombies
	}
	name := "lvmh-agent-" + strings.ReplaceAll(sessionID, "-", "")[:12]
	created, err := s.cli.ContainerCreate(ctx, cfg, hostCfg, nil, nil, name)
	if err != nil {
		return "", fmt.Errorf("docker create: %w", err)
	}
	if err := s.cli.ContainerStart(ctx, created.ID, container.StartOptions{}); err != nil {
		s.removeContainer(context.Background(), created.ID)
		return "", fmt.Errorf("docker start: %w", err)
	}
	return created.ID, nil
}

// ensureRepoVolume creates the per-repo volume; reports whether it is fresh.
// Only a definitive "no such volume" (docker errdefs NotFound) counts as
// fresh: a transient inspect error must never re-seed over an existing
// volume, so it propagates instead.
func (s *Spawner) ensureRepoVolume(ctx context.Context, name string) (bool, error) {
	if _, err := s.cli.VolumeInspect(ctx, name); err == nil {
		return false, nil
	} else if !cerrdefs.IsNotFound(err) {
		return false, fmt.Errorf("volume %s: %w", name, err)
	}
	if _, err := s.cli.VolumeCreate(ctx, volume.CreateOptions{Name: name}); err != nil {
		return false, fmt.Errorf("volume %s: %w", name, err)
	}
	return true, nil
}

// seedVolume populates the fresh named volume with the cloned repo by
// streaming a tar of the clone into a paused container via the docker API
// (CopyToContainer). No bind mounts: bind sources are HOST paths, but the
// clone lives inside the daemon container's filesystem.
func (s *Spawner) seedVolume(ctx context.Context, slug, repoVolume, sessionID string) error {
	repoDir := filepath.Join(s.reposDir, slug)
	cfg := &container.Config{
		Image: imageRefWorker,
		Cmd:   []string{"true"}, // never started; created only as a volume mount point
	}
	hostCfg := &container.HostConfig{
		Binds: []string{repoVolume + ":" + workspaceMount},
		Init:  &[]bool{true}[0],
	}
	// Unique name per session: a fixed name would conflict forever after a
	// crash between create and the deferred remove.
	unq := strings.ReplaceAll(sessionID, "-", "")
	if len(unq) > 8 {
		unq = unq[:8]
	}
	name := "lvmh-seed-" + slug + "-" + unq
	created, err := s.cli.ContainerCreate(ctx, cfg, hostCfg, nil, nil, name)
	if err != nil {
		return err
	}
	defer s.removeContainer(context.Background(), created.ID)
	pr, pw := io.Pipe()
	tarDone := make(chan error, 1)
	go func() {
		err := tarDir(pw, repoDir)
		_ = pw.CloseWithError(err)
		tarDone <- err
	}()
	copyErr := s.cli.CopyToContainer(ctx, created.ID, workspaceMount, pr, container.CopyToContainerOptions{})
	if copyErr != nil {
		_ = pr.CloseWithError(copyErr) // unblock the tar goroutine
	}
	if ctxErr := ctx.Err(); ctxErr != nil {
		return ctxErr
	}
	if tarErr := <-tarDone; tarErr != nil {
		return fmt.Errorf("tar clone: %w", tarErr)
	}
	if copyErr != nil {
		return fmt.Errorf("copy into volume: %w", copyErr)
	}
	return nil
}

func (s *Spawner) removeContainer(ctx context.Context, id string) {
	_ = s.cli.ContainerRemove(ctx, id, container.RemoveOptions{Force: true})
}

// RemoveSession stops and removes the container spawned for a session.
// Stop and remove run on background contexts on purpose: a caller hanging up
// mid-request must not leave an orphaned container behind a deleted DB row.
func (s *Spawner) RemoveSession(_ context.Context, sessionID string) error {
	row, ok, err := s.store.GetContainer(sessionID)
	if err != nil {
		return err
	}
	if !ok {
		return errNoContainer
	}
	stopCtx, cancel := context.WithTimeout(context.Background(), time.Duration(stopTimeoutSeconds)*time.Second)
	if err := s.cli.ContainerStop(stopCtx, row.ContainerID, container.StopOptions{}); err != nil {
		cancel()
		return fmt.Errorf("docker stop: %w", err)
	}
	cancel()
	s.removeContainer(context.Background(), row.ContainerID)
	if err := s.store.DeleteContainer(sessionID); err != nil {
		return err
	}
	s.deleteJob(sessionID)
	s.hub.BroadcastSpawnStatus()
	return nil
}

// tarDir writes a tar archive of dir (nothing excluded, .git included) into w.
// Symlinks are archived as symlinks with their target in Linkname so docker's
// untar can recreate them; only regular files carry content.
func tarDir(w io.Writer, dir string) error {
	tw := tar.NewWriter(w)
	defer tw.Close()
	return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		rel, err := filepath.Rel(dir, path)
		if err != nil {
			return err
		}
		if rel == "." {
			return nil
		}
		name := filepath.ToSlash(rel)
		link := ""
		if info.Mode()&os.ModeSymlink != 0 {
			link, err = os.Readlink(path)
			if err != nil {
				return err
			}
		}
		hdr, err := tar.FileInfoHeader(info, link)
		if err != nil {
			return err
		}
		hdr.Name = name
		if info.IsDir() {
			hdr.Name += "/"
		}
		hdr.Uname = "root"
		hdr.Gname = "root"
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		if info.Mode().IsRegular() {
			f, err := os.Open(path)
			if err != nil {
				return err
			}
			defer f.Close()
			if _, err := io.Copy(tw, f); err != nil {
				return err
			}
		}
		return nil
	})
}