RenewCMS/test/update_helper_test.go

package test

import (
	"RenewCMS/internal/domain/update"
	infraUpdate "RenewCMS/internal/infrastructure/update"
	"RenewCMS/internal/infrastructure/useCases"
	"crypto/ed25519"
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"strings"
	"testing"
	"time"
)

const (
	helperData      = "/var/lib/renewcms"
	helperBinary    = "/opt/renewcms/renewcms"
	helperBinaryDir = "/opt/renewcms"
	helperOld       = "v2.2.0-beta"
	helperSentinel  = "sentinel bytes that must never change"
)

var (
	helperMarker     = helperData + "/" + update.RestartRequestFileName
	helperResult     = helperData + "/" + update.ResultFileName
	helperResultTmp  = helperResult + ".tmp"
	helperUpdatesDir = helperData + "/" + update.UpdatesDirName
	helperStagedDir  = helperUpdatesDir + "/" + downloadTag
	helperAsset      = update.AssetName(downloadTag, update.Platform{OS: "linux", Arch: "amd64"})
)

type fsVerifier struct {
	files  *fakeHelperFS
	key    ed25519.PublicKey
	events *[]string
}

var _ update.SignatureVerifier = &fsVerifier{}

func (v *fsVerifier) Verify(checksumsPath string, signaturePath string) error {
	*v.events = append(*v.events, "verify "+checksumsPath)
	checksums, found := v.files.content(checksumsPath)
	if !found {
		return errors.New("checksums missing")
	}
	rawSignature, found := v.files.content(signaturePath)
	if !found {
		return errors.New("signature missing")
	}
	signature, err := base64.StdEncoding.DecodeString(strings.TrimSpace(rawSignature))
	if err != nil || !ed25519.Verify(v.key, []byte(checksums), signature) {
		return errors.New("signature mismatch")
	}
	return nil
}

type helperRig struct {
	t            *testing.T
	fs           *fakeHelperFS
	running      *fakeRunningInfoStore
	clock        *fakeClock
	service      *fakeServiceControl
	health       *fakeHealthChecker
	logger       *fakeUpdateLogger
	events       []string
	public       ed25519.PublicKey
	private      ed25519.PrivateKey
	newBinary    string
	bootVersions []string
	oldInfo      update.RunningInfo
	usecase      *useCases.ApplyRestartRequestUseCase
}

func newHelperRig(t *testing.T) *helperRig {
	t.Helper()
	public, private, err := ed25519.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}
	rig := &helperRig{
		t:            t,
		fs:           newFakeHelperFS(),
		running:      &fakeRunningInfoStore{},
		clock:        &fakeClock{now: restartBaseTime},
		health:       &fakeHealthChecker{healthy: true},
		logger:       &fakeUpdateLogger{},
		public:       public,
		private:      private,
		newBinary:    "new binary bytes 2.3.0",
		bootVersions: []string{downloadTag, helperOld},
		oldInfo:      update.RunningInfo{Version: helperOld, Pid: 100, StartedAt: restartBaseTime.Add(-time.Hour)},
	}
	rig.fs.events = &rig.events
	rig.fs.putFile(helperBinary, "old")
	rig.fs.putDir(helperData)
	rig.running.set(rig.oldInfo)
	rig.service = &fakeServiceControl{clock: rig.clock, failOn: map[int]error{}, events: &rig.events}
	rig.service.onRestart = func(call int) {
		version := rig.bootVersions[min(call, len(rig.bootVersions))-1]
		rig.running.set(update.RunningInfo{Version: version, Pid: 200 + call, StartedAt: rig.clock.Now().Add(time.Second)})
	}
	rig.usecase = rig.build()
	return rig
}

func (r *helperRig) build() *useCases.ApplyRestartRequestUseCase {
	return useCases.NewApplyRestartRequestUseCase(useCases.ApplyRestartRequestDependencies{
		Requests: infraUpdate.NewFSRestartRequestStore(r.fs, helperMarker),
		Results:  infraUpdate.NewFSResultStore(r.fs, helperResult),
		Running:  r.running,
		Files:    r.fs,
		Service:  r.service,
		Health:   r.health,
		Verifier: &fsVerifier{files: r.fs, key: r.public, events: &r.events},
		Clock:    r.clock,
		Logger:   r.logger,
		Layout:   update.HelperLayout{DataDir: helperData, BinaryPath: helperBinary},
		Platform: update.Platform{OS: "linux", Arch: "amd64"},
	})
}

func (r *helperRig) checksumsFor(binary string) string {
	return update.SHA256Hex([]byte(binary)) + "  " + helperAsset + "\n"
}

func (r *helperRig) sign(checksums string, key ed25519.PrivateKey) string {
	return base64.StdEncoding.EncodeToString(ed25519.Sign(key, []byte(checksums))) + "\n"
}

func (r *helperRig) stage() {
	checksums := r.checksumsFor(r.newBinary)
	r.fs.putFile(helperStagedDir+"/"+helperAsset, r.newBinary)
	r.fs.putFile(helperStagedDir+"/checksums.txt", checksums)
	r.fs.putFile(helperStagedDir+"/checksums.txt.sig", r.sign(checksums, r.private))
}

func (r *helperRig) markUpdate(tag string) {
	r.writeMarker(update.RestartRequest{Kind: update.RestartKindUpdate, Tag: tag, RequestedAt: restartBaseTime, RequestedBy: 1})
}

func (r *helperRig) markRestart() {
	r.bootVersions = []string{helperOld, helperOld}
	r.writeMarker(update.RestartRequest{Kind: update.RestartKindRestart, RequestedAt: restartBaseTime, RequestedBy: 1})
}

func (r *helperRig) writeMarker(request update.RestartRequest) {
	content, _ := json.Marshal(request)
	r.fs.putFile(helperMarker, string(content))
}

func (r *helperRig) apply() error {
	r.t.Helper()
	return r.usecase.Apply()
}

func (r *helperRig) result() update.Result {
	r.t.Helper()
	content, found := r.fs.content(helperResult)
	if !found {
		r.t.Fatal("update-result.json was not written")
	}
	var result update.Result
	if err := json.Unmarshal([]byte(content), &result); err != nil {
		r.t.Fatalf("result is not json: %v", err)
	}
	return result
}

func (r *helperRig) outcomes() []string {
	var outcomes []string
	for _, content := range r.fs.history[helperResult] {
		var result update.Result
		_ = json.Unmarshal([]byte(content), &result)
		outcomes = append(outcomes, result.Outcome)
	}
	return outcomes
}

func (r *helperRig) assertBinary(want string) {
	r.t.Helper()
	if content, _ := r.fs.content(helperBinary); content != want {
		r.t.Errorf("binary = %q, want %q", content, want)
	}
}

func (r *helperRig) assertUntouched() {
	r.t.Helper()
	r.assertBinary("old")
	if r.fs.exists(helperBinary + ".previous") {
		r.t.Error("no .previous may be written")
	}
	if r.service.restartCount() != 0 {
		r.t.Errorf("restarts = %d, want 0", r.service.restartCount())
	}
}

func (r *helperRig) assertOutsideUntouched(sentinelPaths ...string) {
	r.t.Helper()
	if outside := r.fs.resolvedOutside(helperData, helperBinaryDir, "/tmp"); len(outside) != 0 {
		r.t.Errorf("operations reached outside the allowed directories: %v", outside)
	}
	for _, target := range sentinelPaths {
		if content, _ := r.fs.content(target); content != helperSentinel {
			r.t.Errorf("%s changed: %q", target, content)
		}
	}
}

func (r *helperRig) assertCleaned() {
	r.t.Helper()
	if r.fs.exists(helperStagedDir) {
		r.t.Error("the staged directory must be removed")
	}
	for name := range r.fs.nodes {
		if strings.HasPrefix(name, "/tmp/renewcms-update-") {
			r.t.Errorf("the temp directory must be removed, found %s", name)
		}
	}
}

func assertInOrder(t *testing.T, events []string, wants ...string) {
	t.Helper()
	position := 0
	for _, want := range wants {
		found := false
		for ; position < len(events); position++ {
			if strings.HasPrefix(events[position], want) {
				found = true
				position++
				break
			}
		}
		if !found {
			t.Fatalf("%q is missing or out of order in:\n%s", want, strings.Join(events, "\n"))
		}
	}
}

func TestApplyPlainRestart(t *testing.T) {
	t.Run("apply-update, a plain restart", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.markRestart()
		rig.fs.putFile(helperStagedDir+"/keep", "staged")

		if err := rig.apply(); err != nil {
			t.Fatal(err)
		}

		assertInOrder(t, rig.events, "lstat "+helperMarker, "read "+helperMarker, "remove "+helperMarker, "create "+helperResultTmp, "restart", "create "+helperResultTmp)
		if got := rig.outcomes(); len(got) != 2 || got[0] != "running" || got[1] != "succeeded" {
			t.Errorf("outcomes = %v", got)
		}
		result := rig.result()
		if result.Kind != "restart" || result.FinishedAt == nil || result.FromVersion != helperOld || !result.StartedAt.Equal(restartBaseTime) {
			t.Errorf("result = %+v", result)
		}
		if rig.service.restartCount() != 1 || rig.fs.exists(helperBinary+".previous") {
			t.Error("one restart, no write next to the binary")
		}
		rig.assertBinary("old")
		if content, _ := rig.fs.content(helperStagedDir + "/keep"); content != "staged" {
			t.Error("the staged files must be untouched")
		}
		if rig.fs.exists(helperMarker) {
			t.Error("the marker must be deleted")
		}
	})

	t.Run("no marker means nothing to do", func(t *testing.T) {
		rig := newHelperRig(t)

		if err := rig.apply(); err != nil || rig.fs.exists(helperResult) || rig.service.restartCount() != 0 {
			t.Errorf("err = %v", err)
		}
	})

	t.Run("a corrupt marker or an unknown kind fails without restarting", func(t *testing.T) {
		for name, content := range map[string]string{"corrupt": "{nope", "unknown kind": `{"kind":"reboot","requested_at":"2026-09-30T09:50:00Z"}`} {
			t.Run(name, func(t *testing.T) {
				rig := newHelperRig(t)
				rig.fs.putFile(helperMarker, content)

				err := rig.apply()

				if err != nil || rig.result().Outcome != "failed" || rig.service.restartCount() != 0 || rig.fs.exists(helperMarker) {
					t.Errorf("err = %v, result = %+v", err, rig.result())
				}
			})
		}
	})
}

func TestApplyStaleRunningInfo(t *testing.T) {
	staleRows := map[string]struct {
		kind  string
		boots func(rig *helperRig, call int)
	}{
		"still holds the old pid and old started_at": {"restart", func(rig *helperRig, call int) {}},
		"holds the target version but the old pid and an old started_at": {"update", func(rig *helperRig, call int) {
			rig.running.set(update.RunningInfo{Version: downloadTag, Pid: rig.oldInfo.Pid, StartedAt: rig.oldInfo.StartedAt})
		}},
		"is missing": {"restart", func(rig *helperRig, call int) { rig.running.unreadable() }},
		"is corrupt": {"restart", func(rig *helperRig, call int) { rig.running.unreadable() }},
		"holds the old version with a new pid (for an update)": {"update", func(rig *helperRig, call int) {
			rig.running.set(update.RunningInfo{Version: helperOld, Pid: 300, StartedAt: rig.clock.Now().Add(time.Second)})
		}},
	}
	for name, row := range staleRows {
		t.Run("A stale running file never counts as success: "+name, func(t *testing.T) {
			rig := newHelperRig(t)
			if row.kind == "update" {
				rig.stage()
				rig.markUpdate(downloadTag)
			} else {
				rig.markRestart()
			}
			rig.service.onRestart = func(call int) {
				if call == 1 {
					row.boots(rig, call)
					return
				}
				rig.running.set(update.RunningInfo{Version: helperOld, Pid: 500, StartedAt: rig.clock.Now().Add(time.Second)})
			}

			err := rig.apply()

			result := rig.result()
			first := rig.service.restarts[0]
			if row.kind == "restart" {
				if err == nil || result.Outcome != "failed" || rig.service.restartCount() != 1 {
					t.Errorf("err = %v, result = %+v", err, result)
				}
				if rig.clock.Now().Sub(first) < update.RestartWindow {
					t.Errorf("gave up after %v", rig.clock.Now().Sub(first))
				}
				return
			}
			if result.Outcome != "rolled_back" || rig.service.restartCount() != 2 {
				t.Fatalf("result = %+v, restarts = %d", result, rig.service.restartCount())
			}
			if rig.service.restarts[1].Sub(first) < update.RestartWindow {
				t.Errorf("rolled back after %v, before the 60 s window", rig.service.restarts[1].Sub(first))
			}
		})
	}

	t.Run("a file with the same pid but a later started_at counts as new", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.markRestart()
		rig.service.onRestart = func(int) {
			rig.running.set(update.RunningInfo{Version: helperOld, Pid: rig.oldInfo.Pid, StartedAt: rig.clock.Now().Add(time.Second)})
		}

		if err := rig.apply(); err != nil || rig.result().Outcome != "succeeded" || rig.clock.sleeps != 0 {
			t.Errorf("err = %v, outcome = %v, sleeps = %d", err, rig.result().Outcome, rig.clock.sleeps)
		}
	})

	t.Run("success needs a healthy answer too", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.markRestart()
		rig.health.healthy = false

		err := rig.apply()

		if err == nil || rig.result().Outcome != "failed" {
			t.Errorf("err = %v, result = %+v", err, rig.result())
		}
	})
}

func TestApplyUpdate(t *testing.T) {
	t.Run("apply-update, an update, happy path", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.stage()
		rig.markUpdate(downloadTag)
		temp := "/tmp/renewcms-update-1"

		if err := rig.apply(); err != nil {
			t.Fatal(err)
		}

		assertInOrder(t, rig.events,
			"read "+helperMarker, "remove "+helperMarker, "create "+helperResultTmp,
			"lstat "+helperStagedDir+"/"+helperAsset, "tempdir",
			"copy "+helperStagedDir+"/"+helperAsset, "copy "+helperStagedDir+"/checksums.txt", "copy "+helperStagedDir+"/checksums.txt.sig",
			"verify "+temp+"/checksums.txt", "read "+temp+"/checksums.txt", "read "+temp+"/"+helperAsset,
			"copy "+helperBinary, "rename "+helperBinary+".previous.tmp",
			"copy "+temp+"/"+helperAsset, "read "+helperBinary+".new", "chmod "+helperBinary+".new", "rename "+helperBinary+".new",
			"restart", "create "+helperResultTmp, "removeall "+temp, "removeall "+helperStagedDir)
		rig.assertBinary(rig.newBinary)
		if previous, _ := rig.fs.content(helperBinary + ".previous"); previous != "old" {
			t.Errorf(".previous = %q", previous)
		}
		if rig.fs.mode(helperBinary) != 0o755 || rig.fs.mode(temp) != 0 {
			t.Errorf("binary mode = %o, temp directory must be gone", rig.fs.mode(helperBinary))
		}
		if rig.clock.sleeps != 0 || rig.service.restartCount() != 1 {
			t.Errorf("success must be declared on the first matching poll: sleeps = %d", rig.clock.sleeps)
		}
		result := rig.result()
		if result.Outcome != "succeeded" || result.Kind != "update" || result.Tag != downloadTag || result.FromVersion != helperOld {
			t.Errorf("result = %+v", result)
		}
		rig.assertCleaned()
		rig.assertOutsideUntouched()
		if rig.fs.exists(helperBinary+".new") || rig.fs.exists(helperBinary+".previous.tmp") {
			t.Error("no temp file may be left next to the binary")
		}
	})

	t.Run("the verified copy lives in a private temp directory", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.stage()
		rig.markUpdate(downloadTag)
		rig.fs.after("read", 4, func(f *fakeHelperFS) {
			if f.mode("/tmp/renewcms-update-1") != 0o700 || f.nodes["/tmp/renewcms-update-1"].owner != "root" {
				t.Errorf("temp directory mode = %o owner = %s", f.mode("/tmp/renewcms-update-1"), f.nodes["/tmp/renewcms-update-1"].owner)
			}
		})

		_ = rig.apply()
	})

	t.Run("apply-update refuses an untrusted staged binary", func(t *testing.T) {
		faults := map[string]struct {
			mutate func(*helperRig)
			reason string
		}{
			"the binary was changed after the job verified it": {func(r *helperRig) { r.fs.putFile(helperStagedDir+"/"+helperAsset, "tampered") }, "sha-256 mismatch"},
			"checksums.txt was changed": {func(r *helperRig) {
				r.fs.putFile(helperStagedDir+"/checksums.txt", r.checksumsFor("tampered"))
			}, "signature"},
			"the signature is from a different key": {func(r *helperRig) {
				_, other, _ := ed25519.GenerateKey(rand.Reader)
				r.fs.putFile(helperStagedDir+"/checksums.txt.sig", r.sign(r.checksumsFor(r.newBinary), other))
			}, "signature"},
			"the signature is missing": {func(r *helperRig) { delete(r.fs.nodes, helperStagedDir+"/checksums.txt.sig") }, "checksums.txt.sig"},
			"checksums.txt has no line for the asset": {func(r *helperRig) {
				checksums := update.SHA256Hex([]byte("x")) + "  something_else\n"
				r.fs.putFile(helperStagedDir+"/checksums.txt", checksums)
				r.fs.putFile(helperStagedDir+"/checksums.txt.sig", r.sign(checksums, r.private))
			}, "no line"},
			"the staged binary is missing": {func(r *helperRig) { delete(r.fs.nodes, helperStagedDir+"/"+helperAsset) }, helperAsset},
		}
		for name, fault := range faults {
			t.Run(name, func(t *testing.T) {
				rig := newHelperRig(t)
				rig.stage()
				rig.markUpdate(downloadTag)
				fault.mutate(rig)

				err := rig.apply()

				if err != nil || rig.result().Outcome != "failed" {
					t.Errorf("err = %v, result = %+v", err, rig.result())
				}
				rig.assertUntouched()
				rig.assertCleaned()
				if !strings.Contains(rig.logger.text(), fault.reason) {
					t.Errorf("the log must carry the reason %q: %s", fault.reason, rig.logger.text())
				}
				if raw, _ := rig.fs.content(helperResult); strings.Contains(raw, "mismatch") || strings.Contains(raw, helperData) {
					t.Errorf("the technical reason must stay out of the result file: %s", raw)
				}
			})
		}
	})
}

func TestApplyRefusesSymlinks(t *testing.T) {
	const outside = "/outside"
	type row struct {
		setup func(*helperRig)
		link  string
		keep  []string
	}
	rows := map[string]row{
		"the marker itself is a symlink": {func(r *helperRig) {
			r.fs.putFile(outside+"/secret", helperSentinel)
			r.fs.putSymlink(helperMarker, outside+"/secret")
		}, helperMarker, []string{outside + "/secret"}},
		"<data>/updates is a symlink to another directory": {func(r *helperRig) {
			r.fs.putDir(outside + "/updates/" + downloadTag)
			r.fs.putFile(outside+"/updates/"+downloadTag+"/"+helperAsset, helperSentinel)
			r.fs.putFile(outside+"/updates/"+downloadTag+"/checksums.txt", helperSentinel)
			r.fs.putFile(outside+"/updates/"+downloadTag+"/checksums.txt.sig", helperSentinel)
			r.forget(helperUpdatesDir)
			r.fs.putSymlink(helperUpdatesDir, outside+"/updates")
		}, helperUpdatesDir, []string{outside + "/updates/" + downloadTag + "/" + helperAsset}},
		"<data>/updates/<tag> is a symlink to another directory": {func(r *helperRig) {
			r.fs.putFile(outside+"/tag/"+helperAsset, helperSentinel)
			r.fs.putFile(outside+"/tag/checksums.txt", helperSentinel)
			r.fs.putFile(outside+"/tag/checksums.txt.sig", helperSentinel)
			r.forget(helperStagedDir)
			r.fs.putSymlink(helperStagedDir, outside+"/tag")
		}, helperStagedDir, []string{outside + "/tag/" + helperAsset, outside + "/tag/checksums.txt"}},
		"the staged binary is a symlink to /usr/bin/something": {func(r *helperRig) {
			r.fs.putFile("/usr/bin/something", helperSentinel)
			delete(r.fs.nodes, helperStagedDir+"/"+helperAsset)
			r.fs.putSymlink(helperStagedDir+"/"+helperAsset, "/usr/bin/something")
		}, helperStagedDir + "/" + helperAsset, []string{"/usr/bin/something"}},
		"checksums.txt is a symlink": {func(r *helperRig) {
			r.fs.putFile(outside+"/c", helperSentinel)
			delete(r.fs.nodes, helperStagedDir+"/checksums.txt")
			r.fs.putSymlink(helperStagedDir+"/checksums.txt", outside+"/c")
		}, helperStagedDir + "/checksums.txt", []string{outside + "/c"}},
		"checksums.txt.sig is a symlink": {func(r *helperRig) {
			r.fs.putFile(outside+"/s", helperSentinel)
			delete(r.fs.nodes, helperStagedDir+"/checksums.txt.sig")
			r.fs.putSymlink(helperStagedDir+"/checksums.txt.sig", outside+"/s")
		}, helperStagedDir + "/checksums.txt.sig", []string{outside + "/s"}},
		"a symlink is placed at the result path": {func(r *helperRig) {
			r.fs.putFile(outside+"/result-target", helperSentinel)
			r.fs.putSymlink(helperResult, outside+"/result-target")
		}, helperResult, []string{outside + "/result-target"}},
		"a symlink is placed at a temp file name": {func(r *helperRig) {
			r.fs.putFile(outside+"/tmp-target", helperSentinel)
			r.fs.putSymlink(helperResultTmp, outside+"/tmp-target")
		}, helperResultTmp, []string{outside + "/tmp-target"}},
	}
	for name, tc := range rows {
		t.Run("apply-update refuses symlinks: "+name, func(t *testing.T) {
			rig := newHelperRig(t)
			rig.stage()
			rig.markUpdate(downloadTag)
			tc.setup(rig)

			err := rig.apply()

			if err != nil || rig.result().Outcome != "failed" {
				t.Errorf("err = %v, result = %+v", err, rig.result())
			}
			rig.assertUntouched()
			rig.assertOutsideUntouched(tc.keep...)
			if !strings.Contains(rig.logger.text(), tc.link) {
				t.Errorf("the log must name %s: %s", tc.link, rig.logger.text())
			}
			if node := rig.fs.nodes[helperResult]; node == nil || node.kind != "file" {
				t.Error("the result must be a regular file written next to the link, not through it")
			}
			if len(rig.fs.touched("read", "copy")) > 0 {
				for _, operation := range rig.fs.touched("read", "copy") {
					if strings.HasPrefix(operation.resolved, outside) || strings.HasPrefix(operation.resolved, "/usr") {
						t.Errorf("read through a link: %+v", operation)
					}
				}
			}
		})
	}

	t.Run("A symlinked marker is removed, not followed", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.fs.putFile(outside+"/secret", helperSentinel)
		rig.fs.putSymlink(helperMarker, outside+"/secret")

		err := rig.apply()

		if err != nil || rig.fs.exists(helperMarker) || rig.result().Outcome != "failed" {
			t.Errorf("err = %v, marker present = %v", err, rig.fs.exists(helperMarker))
		}
		if content, _ := rig.fs.content(outside + "/secret"); content != helperSentinel {
			t.Error("the target must be intact")
		}
	})

	t.Run("a link found in the staged directory is unlinked, not followed, when cleaning", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.stage()
		rig.markUpdate(downloadTag)
		rig.fs.putFile(outside+"/tag/"+helperAsset, helperSentinel)
		rig.forget(helperStagedDir)
		rig.fs.putSymlink(helperStagedDir, outside+"/tag")

		_ = rig.apply()

		if rig.fs.exists(helperStagedDir) {
			t.Error("the link itself may be removed")
		}
		if content, _ := rig.fs.content(outside + "/tag/" + helperAsset); content != helperSentinel {
			t.Error("the target must be intact")
		}
	})
}

func (r *helperRig) forget(prefix string) {
	for name := range r.fs.nodes {
		if name == prefix || strings.HasPrefix(name, prefix+"/") {
			delete(r.fs.nodes, name)
		}
	}
}

func TestApplySwapIsRaceFree(t *testing.T) {
	t.Run("A file swapped after verification is not installed", func(t *testing.T) {
		swapAfter := map[string]struct {
			operation string
			call      int
		}{
			"the copy step":              {"copy", 3},
			"the signature check":        {"read", 2},
			"the hash check":             {"read", 3},
			"the .previous copy":         {"copy", 4},
			"the final copy and install": {"copy", 5},
		}
		for name, point := range swapAfter {
			t.Run(name, func(t *testing.T) {
				rig := newHelperRig(t)
				rig.stage()
				rig.markUpdate(downloadTag)
				rig.fs.after(point.operation, point.call, func(f *fakeHelperFS) {
					f.putFile(helperStagedDir+"/"+helperAsset, "attacker bytes")
				})

				err := rig.apply()

				if err != nil {
					t.Fatal(err)
				}
				rig.assertBinary(rig.newBinary)
				if rig.result().Outcome != "succeeded" {
					t.Errorf("result = %+v", rig.result())
				}
			})
		}
	})

	t.Run("a swap of the temp file in the binary's directory before the rename is caught by the re-hash", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.stage()
		rig.markUpdate(downloadTag)
		rig.fs.after("copy", 5, func(f *fakeHelperFS) {
			f.putFile(helperBinary+".new", "attacker bytes in the temp file")
		})

		err := rig.apply()

		if err != nil || rig.result().Outcome != "failed" {
			t.Errorf("err = %v, result = %+v", err, rig.result())
		}
		rig.assertBinary("old")
		if rig.service.restartCount() != 0 || rig.fs.exists(helperBinary+".new") {
			t.Error("no restart and the temp file must be removed")
		}
	})
}

func TestApplyResultWrites(t *testing.T) {
	t.Run("Result and temp files are written without following links", func(t *testing.T) {
		files := newFakeHelperFS()
		files.putFile("/outside/a", helperSentinel)
		files.putFile("/outside/b", helperSentinel)
		files.putDir(helperData)
		files.putSymlink(helperResult, "/outside/a")
		files.putSymlink(helperResultTmp, "/outside/b")
		store := infraUpdate.NewFSResultStore(files, helperResult)

		err := store.Write(update.Result{Outcome: update.OutcomeFailed, Kind: "update", StartedAt: restartBaseTime, FinishedAt: finishedAt(restartBaseTime)})

		if err != nil {
			t.Fatal(err)
		}
		if a, _ := files.content("/outside/a"); a != helperSentinel {
			t.Error("the result link target must be untouched")
		}
		if b, _ := files.content("/outside/b"); b != helperSentinel {
			t.Error("the temp link target must be untouched")
		}
		if node := files.nodes[helperResult]; node == nil || node.kind != "file" || node.mode != 0o644 {
			t.Errorf("result node = %+v", node)
		}
		if got, found := store.Read(); !found || got.Outcome != update.OutcomeFailed {
			t.Errorf("read back = %+v, %v", got, found)
		}
		assertInOrder(t, files.eventsFrom(), "lstat", "remove", "create", "rename")
	})
}

func (f *fakeHelperFS) eventsFrom() []string {
	names := make([]string, 0, len(f.ops))
	for _, operation := range f.ops {
		names = append(names, operation.name+" "+operation.path)
	}
	return names
}

func TestApplyDistrustsTheMarker(t *testing.T) {
	tags := map[string]string{
		"../../etc/passwd":                     "../../etc/passwd",
		"v2.3.0-beta/../../x":                  "v2.3.0-beta/../../x",
		"an empty string":                      "",
		"v2.1.0-beta (older than the running)": "v2.1.0-beta",
		"not-a-version":                        "not-a-version",
		"the running version itself":           helperOld,
	}
	for name, tag := range tags {
		t.Run("apply-update distrusts the marker: "+name, func(t *testing.T) {
			rig := newHelperRig(t)
			rig.stage()
			rig.fs.putFile("/var/lib/renewcms/etc/passwd", helperSentinel)
			rig.markUpdate(tag)

			err := rig.apply()

			if err != nil || rig.result().Outcome != "failed" {
				t.Errorf("err = %v, result = %+v", err, rig.result())
			}
			rig.assertUntouched()
			rig.assertOutsideUntouched()
			for _, operation := range rig.fs.ops {
				if strings.Contains(operation.path, "..") || strings.Contains(operation.path, "passwd") || strings.HasPrefix(operation.path, helperUpdatesDir) {
					t.Errorf("a distrusted tag reached the filesystem: %+v", operation)
				}
			}
			if !rig.fs.exists(helperStagedDir + "/" + helperAsset) {
				t.Error("staged files of another tag must be left alone")
			}
		})
	}
}

func TestApplyRollsBack(t *testing.T) {
	faults := map[string]func(*helperRig){
		"/healthz never answers within 60 s": func(r *helperRig) {
			r.health.healthy = false
			r.service.onRestart = func(call int) {
				if call == 2 {
					r.health.healthy = true
				}
				r.running.set(update.RunningInfo{Version: r.bootVersions[call-1], Pid: 200 + call, StartedAt: r.clock.Now().Add(time.Second)})
			}
		},
		"/healthz answers but update-running.json stays old": func(r *helperRig) {
			r.service.onRestart = func(call int) {
				if call == 2 {
					r.running.set(update.RunningInfo{Version: helperOld, Pid: 300, StartedAt: r.clock.Now().Add(time.Second)})
				}
			}
		},
		"update-running.json holds another version": func(r *helperRig) {
			r.service.onRestart = func(call int) {
				version := "v9.9.9"
				if call == 2 {
					version = helperOld
				}
				r.running.set(update.RunningInfo{Version: version, Pid: 200 + call, StartedAt: r.clock.Now().Add(time.Second)})
			}
		},
		"the service restart command fails": func(r *helperRig) {
			r.service.failOn[1] = errors.New("systemctl restart renewcms: exit status 1")
		},
	}
	for name, fault := range faults {
		t.Run("apply-update rolls back when the new version does not come up: "+name, func(t *testing.T) {
			rig := newHelperRig(t)
			rig.stage()
			rig.markUpdate(downloadTag)
			fault(rig)

			err := rig.apply()

			if err != nil {
				t.Fatalf("a rollback that worked is not an error: %v", err)
			}
			rig.assertBinary("old")
			if rig.service.restartCount() != 2 {
				t.Fatalf("restarts = %d", rig.service.restartCount())
			}
			if name != "the service restart command fails" && rig.service.restarts[1].Sub(rig.service.restarts[0]) < update.RestartWindow {
				t.Errorf("rolled back after %v", rig.service.restarts[1].Sub(rig.service.restarts[0]))
			}
			assertInOrder(t, rig.events, "rename "+helperBinary+".new", "restart", "copy "+helperBinary+".previous", "rename "+helperBinary+".restore", "restart")
			result := rig.result()
			if result.Outcome != "rolled_back" || result.FromVersion != helperOld || result.Tag != downloadTag || result.FinishedAt == nil {
				t.Errorf("result = %+v", result)
			}
			rig.assertCleaned()
			if rig.fs.exists(helperBinary + ".restore") {
				t.Error("no restore temp file may be left")
			}
		})
	}
}

func TestApplySwapFailures(t *testing.T) {
	t.Run("copying to .previous fails", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.stage()
		rig.markUpdate(downloadTag)
		rig.fs.failNth("copy", 4, errors.New("no space left on device"))

		err := rig.apply()

		if err != nil || rig.result().Outcome != "failed" {
			t.Errorf("err = %v, result = %+v", err, rig.result())
		}
		rig.assertUntouched()
	})

	t.Run("the atomic install fails", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.stage()
		rig.markUpdate(downloadTag)
		rig.fs.failNth("rename", 3, errors.New("text file busy"))

		err := rig.apply()

		if err != nil || rig.result().Outcome != "failed" || rig.service.restartCount() != 0 {
			t.Errorf("err = %v, result = %+v", err, rig.result())
		}
		rig.assertBinary("old")
		if previous, _ := rig.fs.content(helperBinary + ".previous"); previous != "old" {
			t.Errorf(".previous = %q", previous)
		}
		if rig.fs.exists(helperBinary + ".new") {
			t.Error("the temp file must be removed")
		}
	})

	t.Run("the rollback restore itself fails", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.stage()
		rig.markUpdate(downloadTag)
		rig.service.failOn[1] = errors.New("restart failed")
		rig.fs.failNth("copy", 6, errors.New("input/output error"))

		err := rig.apply()

		if err == nil || rig.result().Outcome != "failed" || rig.service.restartCount() != 1 {
			t.Errorf("err = %v, result = %+v", err, rig.result())
		}
		if !strings.Contains(rig.logger.text(), "the site may be down") {
			t.Errorf("the failure must be logged loudly: %s", rig.logger.text())
		}
	})

	t.Run("the old version never returns after rollback", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.stage()
		rig.markUpdate(downloadTag)
		rig.service.onRestart = func(call int) {
			if call == 1 {
				rig.running.set(update.RunningInfo{Version: "v9.9.9", Pid: 201, StartedAt: rig.clock.Now().Add(time.Second)})
			}
		}

		err := rig.apply()

		if err == nil || rig.result().Outcome != "failed" || rig.service.restartCount() != 2 {
			t.Errorf("err = %v, result = %+v", err, rig.result())
		}
	})
}

func TestApplyAlwaysWritesAResult(t *testing.T) {
	t.Run("a panic in the helper still leaves a failed result", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.stage()
		rig.markUpdate(downloadTag)
		rig.service.panicOn = 1

		err := rig.apply()

		result := rig.result()
		if err == nil || result.Outcome != "failed" || result.FinishedAt == nil || result.Kind != "update" || result.Tag != downloadTag || result.FromVersion != helperOld {
			t.Errorf("err = %v, result = %+v", err, result)
		}
		assertInOrder(t, rig.events, "create "+helperResultTmp, "rename "+helperResultTmp, "restart", "create "+helperResultTmp)
	})

	t.Run("a running record exists before the first service call and every write is a temp file plus a rename", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.markRestart()
		var seenBeforeRestart []string
		rig.service.onRestart = func(call int) {
			seenBeforeRestart = append([]string(nil), rig.outcomes()...)
			rig.running.set(update.RunningInfo{Version: helperOld, Pid: 300, StartedAt: rig.clock.Now().Add(time.Second)})
		}

		_ = rig.apply()

		if len(seenBeforeRestart) != 1 || seenBeforeRestart[0] != "running" {
			t.Errorf("outcomes before the restart = %v", seenBeforeRestart)
		}
		creates := rig.fs.operationCount("create")
		renames := len(rig.fs.history[helperResult])
		if creates != renames || creates != 2 {
			t.Errorf("creates = %d, renames = %d", creates, renames)
		}
	})

	t.Run("timestamps come from the clock and finished is not before started", func(t *testing.T) {
		rig := newHelperRig(t)
		rig.markRestart()
		rig.clock.onSleep = func(sleeps int) {
			if sleeps == 3 {
				rig.running.set(update.RunningInfo{Version: helperOld, Pid: 300, StartedAt: rig.clock.Now().Add(time.Second)})
			}
		}
		rig.service.onRestart = func(int) {}

		if err := rig.apply(); err != nil {
			t.Fatal(err)
		}

		result := rig.result()
		if !result.StartedAt.Equal(restartBaseTime) || result.FinishedAt == nil || !result.FinishedAt.Equal(restartBaseTime.Add(3*time.Second)) || result.FinishedAt.Before(result.StartedAt) {
			t.Errorf("result = %+v", result)
		}
	})
}