package test import ( "RenewCMS/internal/domain/update" infraUpdate "RenewCMS/internal/infrastructure/update" "RenewCMS/internal/infrastructure/useCases" "crypto/ed25519" "crypto/rand" "encoding/base64" "encoding/json" "errors" "strings" "testing" "time" ) const ( helperData = "/var/lib/renewcms" helperBinary = "/opt/renewcms/renewcms" helperBinaryDir = "/opt/renewcms" helperOld = "v2.2.0-beta" helperSentinel = "sentinel bytes that must never change" ) var ( helperMarker = helperData + "/" + update.RestartRequestFileName helperResult = helperData + "/" + update.ResultFileName helperResultTmp = helperResult + ".tmp" helperUpdatesDir = helperData + "/" + update.UpdatesDirName helperStagedDir = helperUpdatesDir + "/" + downloadTag helperAsset = update.AssetName(downloadTag, update.Platform{OS: "linux", Arch: "amd64"}) ) type fsVerifier struct { files *fakeHelperFS key ed25519.PublicKey events *[]string } var _ update.SignatureVerifier = &fsVerifier{} func (v *fsVerifier) Verify(checksumsPath string, signaturePath string) error { *v.events = append(*v.events, "verify "+checksumsPath) checksums, found := v.files.content(checksumsPath) if !found { return errors.New("checksums missing") } rawSignature, found := v.files.content(signaturePath) if !found { return errors.New("signature missing") } signature, err := base64.StdEncoding.DecodeString(strings.TrimSpace(rawSignature)) if err != nil || !ed25519.Verify(v.key, []byte(checksums), signature) { return errors.New("signature mismatch") } return nil } type helperRig struct { t *testing.T fs *fakeHelperFS running *fakeRunningInfoStore clock *fakeClock service *fakeServiceControl health *fakeHealthChecker logger *fakeUpdateLogger events []string public ed25519.PublicKey private ed25519.PrivateKey newBinary string bootVersions []string oldInfo update.RunningInfo usecase *useCases.ApplyRestartRequestUseCase } func newHelperRig(t *testing.T) *helperRig { t.Helper() public, private, err := ed25519.GenerateKey(rand.Reader) if err != nil { t.Fatal(err) } rig := &helperRig{ t: t, fs: newFakeHelperFS(), running: &fakeRunningInfoStore{}, clock: &fakeClock{now: restartBaseTime}, health: &fakeHealthChecker{healthy: true}, logger: &fakeUpdateLogger{}, public: public, private: private, newBinary: "new binary bytes 2.3.0", bootVersions: []string{downloadTag, helperOld}, oldInfo: update.RunningInfo{Version: helperOld, Pid: 100, StartedAt: restartBaseTime.Add(-time.Hour)}, } rig.fs.events = &rig.events rig.fs.putFile(helperBinary, "old") rig.fs.putDir(helperData) rig.running.set(rig.oldInfo) rig.service = &fakeServiceControl{clock: rig.clock, failOn: map[int]error{}, events: &rig.events} rig.service.onRestart = func(call int) { version := rig.bootVersions[min(call, len(rig.bootVersions))-1] rig.running.set(update.RunningInfo{Version: version, Pid: 200 + call, StartedAt: rig.clock.Now().Add(time.Second)}) } rig.usecase = rig.build() return rig } func (r *helperRig) build() *useCases.ApplyRestartRequestUseCase { return useCases.NewApplyRestartRequestUseCase(useCases.ApplyRestartRequestDependencies{ Requests: infraUpdate.NewFSRestartRequestStore(r.fs, helperMarker), Results: infraUpdate.NewFSResultStore(r.fs, helperResult), Running: r.running, Files: r.fs, Service: r.service, Health: r.health, Verifier: &fsVerifier{files: r.fs, key: r.public, events: &r.events}, Clock: r.clock, Logger: r.logger, Layout: update.HelperLayout{DataDir: helperData, BinaryPath: helperBinary}, Platform: update.Platform{OS: "linux", Arch: "amd64"}, }) } func (r *helperRig) checksumsFor(binary string) string { return update.SHA256Hex([]byte(binary)) + " " + helperAsset + "\n" } func (r *helperRig) sign(checksums string, key ed25519.PrivateKey) string { return base64.StdEncoding.EncodeToString(ed25519.Sign(key, []byte(checksums))) + "\n" } func (r *helperRig) stage() { checksums := r.checksumsFor(r.newBinary) r.fs.putFile(helperStagedDir+"/"+helperAsset, r.newBinary) r.fs.putFile(helperStagedDir+"/checksums.txt", checksums) r.fs.putFile(helperStagedDir+"/checksums.txt.sig", r.sign(checksums, r.private)) } func (r *helperRig) markUpdate(tag string) { r.writeMarker(update.RestartRequest{Kind: update.RestartKindUpdate, Tag: tag, RequestedAt: restartBaseTime, RequestedBy: 1}) } func (r *helperRig) markRestart() { r.bootVersions = []string{helperOld, helperOld} r.writeMarker(update.RestartRequest{Kind: update.RestartKindRestart, RequestedAt: restartBaseTime, RequestedBy: 1}) } func (r *helperRig) writeMarker(request update.RestartRequest) { content, _ := json.Marshal(request) r.fs.putFile(helperMarker, string(content)) } func (r *helperRig) apply() error { r.t.Helper() return r.usecase.Apply() } func (r *helperRig) result() update.Result { r.t.Helper() content, found := r.fs.content(helperResult) if !found { r.t.Fatal("update-result.json was not written") } var result update.Result if err := json.Unmarshal([]byte(content), &result); err != nil { r.t.Fatalf("result is not json: %v", err) } return result } func (r *helperRig) outcomes() []string { var outcomes []string for _, content := range r.fs.history[helperResult] { var result update.Result _ = json.Unmarshal([]byte(content), &result) outcomes = append(outcomes, result.Outcome) } return outcomes } func (r *helperRig) assertBinary(want string) { r.t.Helper() if content, _ := r.fs.content(helperBinary); content != want { r.t.Errorf("binary = %q, want %q", content, want) } } func (r *helperRig) assertUntouched() { r.t.Helper() r.assertBinary("old") if r.fs.exists(helperBinary + ".previous") { r.t.Error("no .previous may be written") } if r.service.restartCount() != 0 { r.t.Errorf("restarts = %d, want 0", r.service.restartCount()) } } func (r *helperRig) assertOutsideUntouched(sentinelPaths ...string) { r.t.Helper() if outside := r.fs.resolvedOutside(helperData, helperBinaryDir, "/tmp"); len(outside) != 0 { r.t.Errorf("operations reached outside the allowed directories: %v", outside) } for _, target := range sentinelPaths { if content, _ := r.fs.content(target); content != helperSentinel { r.t.Errorf("%s changed: %q", target, content) } } } func (r *helperRig) assertCleaned() { r.t.Helper() if r.fs.exists(helperStagedDir) { r.t.Error("the staged directory must be removed") } for name := range r.fs.nodes { if strings.HasPrefix(name, "/tmp/renewcms-update-") { r.t.Errorf("the temp directory must be removed, found %s", name) } } } func assertInOrder(t *testing.T, events []string, wants ...string) { t.Helper() position := 0 for _, want := range wants { found := false for ; position < len(events); position++ { if strings.HasPrefix(events[position], want) { found = true position++ break } } if !found { t.Fatalf("%q is missing or out of order in:\n%s", want, strings.Join(events, "\n")) } } } func TestApplyPlainRestart(t *testing.T) { t.Run("apply-update, a plain restart", func(t *testing.T) { rig := newHelperRig(t) rig.markRestart() rig.fs.putFile(helperStagedDir+"/keep", "staged") if err := rig.apply(); err != nil { t.Fatal(err) } assertInOrder(t, rig.events, "lstat "+helperMarker, "read "+helperMarker, "remove "+helperMarker, "create "+helperResultTmp, "restart", "create "+helperResultTmp) if got := rig.outcomes(); len(got) != 2 || got[0] != "running" || got[1] != "succeeded" { t.Errorf("outcomes = %v", got) } result := rig.result() if result.Kind != "restart" || result.FinishedAt == nil || result.FromVersion != helperOld || !result.StartedAt.Equal(restartBaseTime) { t.Errorf("result = %+v", result) } if rig.service.restartCount() != 1 || rig.fs.exists(helperBinary+".previous") { t.Error("one restart, no write next to the binary") } rig.assertBinary("old") if content, _ := rig.fs.content(helperStagedDir + "/keep"); content != "staged" { t.Error("the staged files must be untouched") } if rig.fs.exists(helperMarker) { t.Error("the marker must be deleted") } }) t.Run("no marker means nothing to do", func(t *testing.T) { rig := newHelperRig(t) if err := rig.apply(); err != nil || rig.fs.exists(helperResult) || rig.service.restartCount() != 0 { t.Errorf("err = %v", err) } }) t.Run("a corrupt marker or an unknown kind fails without restarting", func(t *testing.T) { for name, content := range map[string]string{"corrupt": "{nope", "unknown kind": `{"kind":"reboot","requested_at":"2026-09-30T09:50:00Z"}`} { t.Run(name, func(t *testing.T) { rig := newHelperRig(t) rig.fs.putFile(helperMarker, content) err := rig.apply() if err != nil || rig.result().Outcome != "failed" || rig.service.restartCount() != 0 || rig.fs.exists(helperMarker) { t.Errorf("err = %v, result = %+v", err, rig.result()) } }) } }) } func TestApplyStaleRunningInfo(t *testing.T) { staleRows := map[string]struct { kind string boots func(rig *helperRig, call int) }{ "still holds the old pid and old started_at": {"restart", func(rig *helperRig, call int) {}}, "holds the target version but the old pid and an old started_at": {"update", func(rig *helperRig, call int) { rig.running.set(update.RunningInfo{Version: downloadTag, Pid: rig.oldInfo.Pid, StartedAt: rig.oldInfo.StartedAt}) }}, "is missing": {"restart", func(rig *helperRig, call int) { rig.running.unreadable() }}, "is corrupt": {"restart", func(rig *helperRig, call int) { rig.running.unreadable() }}, "holds the old version with a new pid (for an update)": {"update", func(rig *helperRig, call int) { rig.running.set(update.RunningInfo{Version: helperOld, Pid: 300, StartedAt: rig.clock.Now().Add(time.Second)}) }}, } for name, row := range staleRows { t.Run("A stale running file never counts as success: "+name, func(t *testing.T) { rig := newHelperRig(t) if row.kind == "update" { rig.stage() rig.markUpdate(downloadTag) } else { rig.markRestart() } rig.service.onRestart = func(call int) { if call == 1 { row.boots(rig, call) return } rig.running.set(update.RunningInfo{Version: helperOld, Pid: 500, StartedAt: rig.clock.Now().Add(time.Second)}) } err := rig.apply() result := rig.result() first := rig.service.restarts[0] if row.kind == "restart" { if err == nil || result.Outcome != "failed" || rig.service.restartCount() != 1 { t.Errorf("err = %v, result = %+v", err, result) } if rig.clock.Now().Sub(first) < update.RestartWindow { t.Errorf("gave up after %v", rig.clock.Now().Sub(first)) } return } if result.Outcome != "rolled_back" || rig.service.restartCount() != 2 { t.Fatalf("result = %+v, restarts = %d", result, rig.service.restartCount()) } if rig.service.restarts[1].Sub(first) < update.RestartWindow { t.Errorf("rolled back after %v, before the 60 s window", rig.service.restarts[1].Sub(first)) } }) } t.Run("a file with the same pid but a later started_at counts as new", func(t *testing.T) { rig := newHelperRig(t) rig.markRestart() rig.service.onRestart = func(int) { rig.running.set(update.RunningInfo{Version: helperOld, Pid: rig.oldInfo.Pid, StartedAt: rig.clock.Now().Add(time.Second)}) } if err := rig.apply(); err != nil || rig.result().Outcome != "succeeded" || rig.clock.sleeps != 0 { t.Errorf("err = %v, outcome = %v, sleeps = %d", err, rig.result().Outcome, rig.clock.sleeps) } }) t.Run("success needs a healthy answer too", func(t *testing.T) { rig := newHelperRig(t) rig.markRestart() rig.health.healthy = false err := rig.apply() if err == nil || rig.result().Outcome != "failed" { t.Errorf("err = %v, result = %+v", err, rig.result()) } }) } func TestApplyUpdate(t *testing.T) { t.Run("apply-update, an update, happy path", func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) temp := "/tmp/renewcms-update-1" if err := rig.apply(); err != nil { t.Fatal(err) } assertInOrder(t, rig.events, "read "+helperMarker, "remove "+helperMarker, "create "+helperResultTmp, "lstat "+helperStagedDir+"/"+helperAsset, "tempdir", "copy "+helperStagedDir+"/"+helperAsset, "copy "+helperStagedDir+"/checksums.txt", "copy "+helperStagedDir+"/checksums.txt.sig", "verify "+temp+"/checksums.txt", "read "+temp+"/checksums.txt", "read "+temp+"/"+helperAsset, "copy "+helperBinary, "rename "+helperBinary+".previous.tmp", "copy "+temp+"/"+helperAsset, "read "+helperBinary+".new", "chmod "+helperBinary+".new", "rename "+helperBinary+".new", "restart", "create "+helperResultTmp, "removeall "+temp, "removeall "+helperStagedDir) rig.assertBinary(rig.newBinary) if previous, _ := rig.fs.content(helperBinary + ".previous"); previous != "old" { t.Errorf(".previous = %q", previous) } if rig.fs.mode(helperBinary) != 0o755 || rig.fs.mode(temp) != 0 { t.Errorf("binary mode = %o, temp directory must be gone", rig.fs.mode(helperBinary)) } if rig.clock.sleeps != 0 || rig.service.restartCount() != 1 { t.Errorf("success must be declared on the first matching poll: sleeps = %d", rig.clock.sleeps) } result := rig.result() if result.Outcome != "succeeded" || result.Kind != "update" || result.Tag != downloadTag || result.FromVersion != helperOld { t.Errorf("result = %+v", result) } rig.assertCleaned() rig.assertOutsideUntouched() if rig.fs.exists(helperBinary+".new") || rig.fs.exists(helperBinary+".previous.tmp") { t.Error("no temp file may be left next to the binary") } }) t.Run("the verified copy lives in a private temp directory", func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) rig.fs.after("read", 4, func(f *fakeHelperFS) { if f.mode("/tmp/renewcms-update-1") != 0o700 || f.nodes["/tmp/renewcms-update-1"].owner != "root" { t.Errorf("temp directory mode = %o owner = %s", f.mode("/tmp/renewcms-update-1"), f.nodes["/tmp/renewcms-update-1"].owner) } }) _ = rig.apply() }) t.Run("apply-update refuses an untrusted staged binary", func(t *testing.T) { faults := map[string]struct { mutate func(*helperRig) reason string }{ "the binary was changed after the job verified it": {func(r *helperRig) { r.fs.putFile(helperStagedDir+"/"+helperAsset, "tampered") }, "sha-256 mismatch"}, "checksums.txt was changed": {func(r *helperRig) { r.fs.putFile(helperStagedDir+"/checksums.txt", r.checksumsFor("tampered")) }, "signature"}, "the signature is from a different key": {func(r *helperRig) { _, other, _ := ed25519.GenerateKey(rand.Reader) r.fs.putFile(helperStagedDir+"/checksums.txt.sig", r.sign(r.checksumsFor(r.newBinary), other)) }, "signature"}, "the signature is missing": {func(r *helperRig) { delete(r.fs.nodes, helperStagedDir+"/checksums.txt.sig") }, "checksums.txt.sig"}, "checksums.txt has no line for the asset": {func(r *helperRig) { checksums := update.SHA256Hex([]byte("x")) + " something_else\n" r.fs.putFile(helperStagedDir+"/checksums.txt", checksums) r.fs.putFile(helperStagedDir+"/checksums.txt.sig", r.sign(checksums, r.private)) }, "no line"}, "the staged binary is missing": {func(r *helperRig) { delete(r.fs.nodes, helperStagedDir+"/"+helperAsset) }, helperAsset}, } for name, fault := range faults { t.Run(name, func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) fault.mutate(rig) err := rig.apply() if err != nil || rig.result().Outcome != "failed" { t.Errorf("err = %v, result = %+v", err, rig.result()) } rig.assertUntouched() rig.assertCleaned() if !strings.Contains(rig.logger.text(), fault.reason) { t.Errorf("the log must carry the reason %q: %s", fault.reason, rig.logger.text()) } if raw, _ := rig.fs.content(helperResult); strings.Contains(raw, "mismatch") || strings.Contains(raw, helperData) { t.Errorf("the technical reason must stay out of the result file: %s", raw) } }) } }) } func TestApplyRefusesSymlinks(t *testing.T) { const outside = "/outside" type row struct { setup func(*helperRig) link string keep []string } rows := map[string]row{ "the marker itself is a symlink": {func(r *helperRig) { r.fs.putFile(outside+"/secret", helperSentinel) r.fs.putSymlink(helperMarker, outside+"/secret") }, helperMarker, []string{outside + "/secret"}}, "/updates is a symlink to another directory": {func(r *helperRig) { r.fs.putDir(outside + "/updates/" + downloadTag) r.fs.putFile(outside+"/updates/"+downloadTag+"/"+helperAsset, helperSentinel) r.fs.putFile(outside+"/updates/"+downloadTag+"/checksums.txt", helperSentinel) r.fs.putFile(outside+"/updates/"+downloadTag+"/checksums.txt.sig", helperSentinel) r.forget(helperUpdatesDir) r.fs.putSymlink(helperUpdatesDir, outside+"/updates") }, helperUpdatesDir, []string{outside + "/updates/" + downloadTag + "/" + helperAsset}}, "/updates/ is a symlink to another directory": {func(r *helperRig) { r.fs.putFile(outside+"/tag/"+helperAsset, helperSentinel) r.fs.putFile(outside+"/tag/checksums.txt", helperSentinel) r.fs.putFile(outside+"/tag/checksums.txt.sig", helperSentinel) r.forget(helperStagedDir) r.fs.putSymlink(helperStagedDir, outside+"/tag") }, helperStagedDir, []string{outside + "/tag/" + helperAsset, outside + "/tag/checksums.txt"}}, "the staged binary is a symlink to /usr/bin/something": {func(r *helperRig) { r.fs.putFile("/usr/bin/something", helperSentinel) delete(r.fs.nodes, helperStagedDir+"/"+helperAsset) r.fs.putSymlink(helperStagedDir+"/"+helperAsset, "/usr/bin/something") }, helperStagedDir + "/" + helperAsset, []string{"/usr/bin/something"}}, "checksums.txt is a symlink": {func(r *helperRig) { r.fs.putFile(outside+"/c", helperSentinel) delete(r.fs.nodes, helperStagedDir+"/checksums.txt") r.fs.putSymlink(helperStagedDir+"/checksums.txt", outside+"/c") }, helperStagedDir + "/checksums.txt", []string{outside + "/c"}}, "checksums.txt.sig is a symlink": {func(r *helperRig) { r.fs.putFile(outside+"/s", helperSentinel) delete(r.fs.nodes, helperStagedDir+"/checksums.txt.sig") r.fs.putSymlink(helperStagedDir+"/checksums.txt.sig", outside+"/s") }, helperStagedDir + "/checksums.txt.sig", []string{outside + "/s"}}, "a symlink is placed at the result path": {func(r *helperRig) { r.fs.putFile(outside+"/result-target", helperSentinel) r.fs.putSymlink(helperResult, outside+"/result-target") }, helperResult, []string{outside + "/result-target"}}, "a symlink is placed at a temp file name": {func(r *helperRig) { r.fs.putFile(outside+"/tmp-target", helperSentinel) r.fs.putSymlink(helperResultTmp, outside+"/tmp-target") }, helperResultTmp, []string{outside + "/tmp-target"}}, } for name, tc := range rows { t.Run("apply-update refuses symlinks: "+name, func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) tc.setup(rig) err := rig.apply() if err != nil || rig.result().Outcome != "failed" { t.Errorf("err = %v, result = %+v", err, rig.result()) } rig.assertUntouched() rig.assertOutsideUntouched(tc.keep...) if !strings.Contains(rig.logger.text(), tc.link) { t.Errorf("the log must name %s: %s", tc.link, rig.logger.text()) } if node := rig.fs.nodes[helperResult]; node == nil || node.kind != "file" { t.Error("the result must be a regular file written next to the link, not through it") } if len(rig.fs.touched("read", "copy")) > 0 { for _, operation := range rig.fs.touched("read", "copy") { if strings.HasPrefix(operation.resolved, outside) || strings.HasPrefix(operation.resolved, "/usr") { t.Errorf("read through a link: %+v", operation) } } } }) } t.Run("A symlinked marker is removed, not followed", func(t *testing.T) { rig := newHelperRig(t) rig.fs.putFile(outside+"/secret", helperSentinel) rig.fs.putSymlink(helperMarker, outside+"/secret") err := rig.apply() if err != nil || rig.fs.exists(helperMarker) || rig.result().Outcome != "failed" { t.Errorf("err = %v, marker present = %v", err, rig.fs.exists(helperMarker)) } if content, _ := rig.fs.content(outside + "/secret"); content != helperSentinel { t.Error("the target must be intact") } }) t.Run("a link found in the staged directory is unlinked, not followed, when cleaning", func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) rig.fs.putFile(outside+"/tag/"+helperAsset, helperSentinel) rig.forget(helperStagedDir) rig.fs.putSymlink(helperStagedDir, outside+"/tag") _ = rig.apply() if rig.fs.exists(helperStagedDir) { t.Error("the link itself may be removed") } if content, _ := rig.fs.content(outside + "/tag/" + helperAsset); content != helperSentinel { t.Error("the target must be intact") } }) } func (r *helperRig) forget(prefix string) { for name := range r.fs.nodes { if name == prefix || strings.HasPrefix(name, prefix+"/") { delete(r.fs.nodes, name) } } } func TestApplySwapIsRaceFree(t *testing.T) { t.Run("A file swapped after verification is not installed", func(t *testing.T) { swapAfter := map[string]struct { operation string call int }{ "the copy step": {"copy", 3}, "the signature check": {"read", 2}, "the hash check": {"read", 3}, "the .previous copy": {"copy", 4}, "the final copy and install": {"copy", 5}, } for name, point := range swapAfter { t.Run(name, func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) rig.fs.after(point.operation, point.call, func(f *fakeHelperFS) { f.putFile(helperStagedDir+"/"+helperAsset, "attacker bytes") }) err := rig.apply() if err != nil { t.Fatal(err) } rig.assertBinary(rig.newBinary) if rig.result().Outcome != "succeeded" { t.Errorf("result = %+v", rig.result()) } }) } }) t.Run("a swap of the temp file in the binary's directory before the rename is caught by the re-hash", func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) rig.fs.after("copy", 5, func(f *fakeHelperFS) { f.putFile(helperBinary+".new", "attacker bytes in the temp file") }) err := rig.apply() if err != nil || rig.result().Outcome != "failed" { t.Errorf("err = %v, result = %+v", err, rig.result()) } rig.assertBinary("old") if rig.service.restartCount() != 0 || rig.fs.exists(helperBinary+".new") { t.Error("no restart and the temp file must be removed") } }) } func TestApplyResultWrites(t *testing.T) { t.Run("Result and temp files are written without following links", func(t *testing.T) { files := newFakeHelperFS() files.putFile("/outside/a", helperSentinel) files.putFile("/outside/b", helperSentinel) files.putDir(helperData) files.putSymlink(helperResult, "/outside/a") files.putSymlink(helperResultTmp, "/outside/b") store := infraUpdate.NewFSResultStore(files, helperResult) err := store.Write(update.Result{Outcome: update.OutcomeFailed, Kind: "update", StartedAt: restartBaseTime, FinishedAt: finishedAt(restartBaseTime)}) if err != nil { t.Fatal(err) } if a, _ := files.content("/outside/a"); a != helperSentinel { t.Error("the result link target must be untouched") } if b, _ := files.content("/outside/b"); b != helperSentinel { t.Error("the temp link target must be untouched") } if node := files.nodes[helperResult]; node == nil || node.kind != "file" || node.mode != 0o644 { t.Errorf("result node = %+v", node) } if got, found := store.Read(); !found || got.Outcome != update.OutcomeFailed { t.Errorf("read back = %+v, %v", got, found) } assertInOrder(t, files.eventsFrom(), "lstat", "remove", "create", "rename") }) } func (f *fakeHelperFS) eventsFrom() []string { names := make([]string, 0, len(f.ops)) for _, operation := range f.ops { names = append(names, operation.name+" "+operation.path) } return names } func TestApplyDistrustsTheMarker(t *testing.T) { tags := map[string]string{ "../../etc/passwd": "../../etc/passwd", "v2.3.0-beta/../../x": "v2.3.0-beta/../../x", "an empty string": "", "v2.1.0-beta (older than the running)": "v2.1.0-beta", "not-a-version": "not-a-version", "the running version itself": helperOld, } for name, tag := range tags { t.Run("apply-update distrusts the marker: "+name, func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.fs.putFile("/var/lib/renewcms/etc/passwd", helperSentinel) rig.markUpdate(tag) err := rig.apply() if err != nil || rig.result().Outcome != "failed" { t.Errorf("err = %v, result = %+v", err, rig.result()) } rig.assertUntouched() rig.assertOutsideUntouched() for _, operation := range rig.fs.ops { if strings.Contains(operation.path, "..") || strings.Contains(operation.path, "passwd") || strings.HasPrefix(operation.path, helperUpdatesDir) { t.Errorf("a distrusted tag reached the filesystem: %+v", operation) } } if !rig.fs.exists(helperStagedDir + "/" + helperAsset) { t.Error("staged files of another tag must be left alone") } }) } } func TestApplyRollsBack(t *testing.T) { faults := map[string]func(*helperRig){ "/healthz never answers within 60 s": func(r *helperRig) { r.health.healthy = false r.service.onRestart = func(call int) { if call == 2 { r.health.healthy = true } r.running.set(update.RunningInfo{Version: r.bootVersions[call-1], Pid: 200 + call, StartedAt: r.clock.Now().Add(time.Second)}) } }, "/healthz answers but update-running.json stays old": func(r *helperRig) { r.service.onRestart = func(call int) { if call == 2 { r.running.set(update.RunningInfo{Version: helperOld, Pid: 300, StartedAt: r.clock.Now().Add(time.Second)}) } } }, "update-running.json holds another version": func(r *helperRig) { r.service.onRestart = func(call int) { version := "v9.9.9" if call == 2 { version = helperOld } r.running.set(update.RunningInfo{Version: version, Pid: 200 + call, StartedAt: r.clock.Now().Add(time.Second)}) } }, "the service restart command fails": func(r *helperRig) { r.service.failOn[1] = errors.New("systemctl restart renewcms: exit status 1") }, } for name, fault := range faults { t.Run("apply-update rolls back when the new version does not come up: "+name, func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) fault(rig) err := rig.apply() if err != nil { t.Fatalf("a rollback that worked is not an error: %v", err) } rig.assertBinary("old") if rig.service.restartCount() != 2 { t.Fatalf("restarts = %d", rig.service.restartCount()) } if name != "the service restart command fails" && rig.service.restarts[1].Sub(rig.service.restarts[0]) < update.RestartWindow { t.Errorf("rolled back after %v", rig.service.restarts[1].Sub(rig.service.restarts[0])) } assertInOrder(t, rig.events, "rename "+helperBinary+".new", "restart", "copy "+helperBinary+".previous", "rename "+helperBinary+".restore", "restart") result := rig.result() if result.Outcome != "rolled_back" || result.FromVersion != helperOld || result.Tag != downloadTag || result.FinishedAt == nil { t.Errorf("result = %+v", result) } rig.assertCleaned() if rig.fs.exists(helperBinary + ".restore") { t.Error("no restore temp file may be left") } }) } } func TestApplySwapFailures(t *testing.T) { t.Run("copying to .previous fails", func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) rig.fs.failNth("copy", 4, errors.New("no space left on device")) err := rig.apply() if err != nil || rig.result().Outcome != "failed" { t.Errorf("err = %v, result = %+v", err, rig.result()) } rig.assertUntouched() }) t.Run("the atomic install fails", func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) rig.fs.failNth("rename", 3, errors.New("text file busy")) err := rig.apply() if err != nil || rig.result().Outcome != "failed" || rig.service.restartCount() != 0 { t.Errorf("err = %v, result = %+v", err, rig.result()) } rig.assertBinary("old") if previous, _ := rig.fs.content(helperBinary + ".previous"); previous != "old" { t.Errorf(".previous = %q", previous) } if rig.fs.exists(helperBinary + ".new") { t.Error("the temp file must be removed") } }) t.Run("the rollback restore itself fails", func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) rig.service.failOn[1] = errors.New("restart failed") rig.fs.failNth("copy", 6, errors.New("input/output error")) err := rig.apply() if err == nil || rig.result().Outcome != "failed" || rig.service.restartCount() != 1 { t.Errorf("err = %v, result = %+v", err, rig.result()) } if !strings.Contains(rig.logger.text(), "the site may be down") { t.Errorf("the failure must be logged loudly: %s", rig.logger.text()) } }) t.Run("the old version never returns after rollback", func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) rig.service.onRestart = func(call int) { if call == 1 { rig.running.set(update.RunningInfo{Version: "v9.9.9", Pid: 201, StartedAt: rig.clock.Now().Add(time.Second)}) } } err := rig.apply() if err == nil || rig.result().Outcome != "failed" || rig.service.restartCount() != 2 { t.Errorf("err = %v, result = %+v", err, rig.result()) } }) } func TestApplyAlwaysWritesAResult(t *testing.T) { t.Run("a panic in the helper still leaves a failed result", func(t *testing.T) { rig := newHelperRig(t) rig.stage() rig.markUpdate(downloadTag) rig.service.panicOn = 1 err := rig.apply() result := rig.result() if err == nil || result.Outcome != "failed" || result.FinishedAt == nil || result.Kind != "update" || result.Tag != downloadTag || result.FromVersion != helperOld { t.Errorf("err = %v, result = %+v", err, result) } assertInOrder(t, rig.events, "create "+helperResultTmp, "rename "+helperResultTmp, "restart", "create "+helperResultTmp) }) t.Run("a running record exists before the first service call and every write is a temp file plus a rename", func(t *testing.T) { rig := newHelperRig(t) rig.markRestart() var seenBeforeRestart []string rig.service.onRestart = func(call int) { seenBeforeRestart = append([]string(nil), rig.outcomes()...) rig.running.set(update.RunningInfo{Version: helperOld, Pid: 300, StartedAt: rig.clock.Now().Add(time.Second)}) } _ = rig.apply() if len(seenBeforeRestart) != 1 || seenBeforeRestart[0] != "running" { t.Errorf("outcomes before the restart = %v", seenBeforeRestart) } creates := rig.fs.operationCount("create") renames := len(rig.fs.history[helperResult]) if creates != renames || creates != 2 { t.Errorf("creates = %d, renames = %d", creates, renames) } }) t.Run("timestamps come from the clock and finished is not before started", func(t *testing.T) { rig := newHelperRig(t) rig.markRestart() rig.clock.onSleep = func(sleeps int) { if sleeps == 3 { rig.running.set(update.RunningInfo{Version: helperOld, Pid: 300, StartedAt: rig.clock.Now().Add(time.Second)}) } } rig.service.onRestart = func(int) {} if err := rig.apply(); err != nil { t.Fatal(err) } result := rig.result() if !result.StartedAt.Equal(restartBaseTime) || result.FinishedAt == nil || !result.FinishedAt.Equal(restartBaseTime.Add(3*time.Second)) || result.FinishedAt.Before(result.StartedAt) { t.Errorf("result = %+v", result) } }) }