diff --git a/api/dependecyInjection.go b/api/dependecyInjection.go index 135e850..03e2400 100644 --- a/api/dependecyInjection.go +++ b/api/dependecyInjection.go @@ -15,6 +15,7 @@ import ( "RenewCMS/internal/infrastructure/markdown" "RenewCMS/internal/infrastructure/persistence/models" persistence "RenewCMS/internal/infrastructure/persistence/repositories" + "RenewCMS/internal/infrastructure/scheduler" "RenewCMS/internal/infrastructure/update" "RenewCMS/internal/infrastructure/useCases" "crypto/ed25519" @@ -36,6 +37,7 @@ type UseCases struct { apiKeyAuthOnce sync.Once apiKeyAuthMiddleware func(http.Handler) http.Handler Token *jwtauth.JWTAuth + PurgeScheduler *scheduler.Runner CreateArticleUseCase *useCases.CreateArticleUseCase GetArticleUseCase *useCases.GetArticleUseCase ListArticlesUseCase *useCases.ListArticlesUseCase @@ -175,7 +177,11 @@ func getUseCases(repos *Repositories, settings config.Settings) *UseCases { keyGenerator := hashing.NewSHA256KeyGenerator() mergeMedia := useCases.NewMergeMediaUseCase(repos.ImageRepo, repos.ArticleRepo) + runPurge := useCases.NewRunPurgeUseCase(repos.ImageRepo, repos.ArticleRepo, mergeMedia, repos.PurgeSettingsRepo, time.Now, log.Default()) + purgeScheduler := useCases.NewPurgeSchedulerUseCase(repos.PurgeSettingsRepo, runPurge) + return &UseCases{ + PurgeScheduler: scheduler.NewSystemRunner(purgeScheduler, log.Default()), Token: jwtauth.New("HS256", []byte(settings.JWTSecret), nil), CreateArticleUseCase: useCases.NewCreateArticleUseCase(repos.ArticleRepo, renderer), GetArticleUseCase: useCases.NewGetArticleUseCase(repos.ArticleRepo, renderer), @@ -207,7 +213,7 @@ func getUseCases(repos *Repositories, settings config.Settings) *UseCases { GetPurgeSettingsUseCase: useCases.NewGetPurgeSettingsUseCase(repos.PurgeSettingsRepo), UpdatePurgeSettingsUseCase: useCases.NewUpdatePurgeSettingsUseCase(repos.PurgeSettingsRepo, time.Now), PreviewPurgeUseCase: useCases.NewPreviewPurgeUseCase(repos.ImageRepo, repos.ArticleRepo), - RunPurgeUseCase: useCases.NewRunPurgeUseCase(repos.ImageRepo, repos.ArticleRepo, mergeMedia, repos.PurgeSettingsRepo, time.Now, log.Default()), + RunPurgeUseCase: runPurge, GetSettingsUseCase: useCases.NewGetSettingsUseCase(settings.Version), GetUpdateStatusUseCase: useCases.NewGetUpdateStatusUseCase(repos.UpdateStore, repos.UpdateJobs, repos.UpdateHelper, repos.UpdateResults, repos.RestartRequests, settings.Version, settings.Container, time.Now), CheckForUpdateUseCase: useCases.NewCheckForUpdateUseCase(repos.ReleaseFeed, repos.UpdateStore, settings.Version, time.Now), diff --git a/cmd/server/main.go b/cmd/server/main.go index 6f08ff8..88b2bdf 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -81,7 +81,7 @@ func run() error { validator.InitValidator() server := newServer(api.InitRoutes(container, settings), settings) - return serveUntilStopped(server, settings, container.RecordRunningInfoUseCase.Record) + return serveUntilStopped(server, settings, container.RecordRunningInfoUseCase.Record, container.PurgeScheduler.Run) } func initEnvVariables() { @@ -132,9 +132,18 @@ func newServer(handler http.Handler, settings config.Settings) *http.Server { } } -func serveUntilStopped(server *http.Server, settings config.Settings, recordRunning func() error) error { +func serveUntilStopped(server *http.Server, settings config.Settings, recordRunning func() error, runScheduler func(context.Context)) error { stopSignal, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() + + schedulerDone := make(chan struct{}) + go func() { + defer close(schedulerDone) + runScheduler(stopSignal) + }() + defer func() { + stop() + <-schedulerDone + }() serverErr := make(chan error, 1) go listen(server, serverErr, settings, recordRunning) diff --git a/docs/architecture.md b/docs/architecture.md index 799264a..037e1cc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -32,10 +32,18 @@ Nothing under `internal/` imports `api/`. ### Media purge -`internal/domain/purge` holds `Settings` (automatic purge on or off, interval of 1 to 365 days, `last_purge_*` history, `enabled_at`), the `Repository` interface (`Get` returns the defaults when nothing was saved, `Save`) and the sentinels `ErrInvalidPurgeInterval`, `ErrPurgeConflict` (stale fingerprint) and `ErrPurgeBusy` (a run is already in progress). `Settings.NextPurgeAt` is `max(last_purge_at, enabled_at)` plus the interval when automatic purge is on, and null otherwise. The settings live in a single-row `purge_settings` table (`models.PurgeSettings`, `mappers.PurgeSettings*`, `persistence.PurgeSettingsRepository`), created by `AutoMigrate`. +`internal/domain/purge` holds `Settings` (automatic purge on or off, interval of 1 to 365 days, `last_purge_*` history, `enabled_at`), the `Repository` interface (`Get` returns the defaults when nothing was saved, `Save`, and `SaveLastRun`, which writes only the `last_purge_*` columns so a settings change made during a run is never overwritten) and the sentinels `ErrInvalidPurgeInterval`, `ErrPurgeConflict` (stale fingerprint) and `ErrPurgeBusy` (a run is already in progress). `Settings.NextPurgeAt` is `max(last_purge_at, enabled_at)` plus the interval when automatic purge is on, and null otherwise. The settings live in a single-row `purge_settings` table (`models.PurgeSettings`, `mappers.PurgeSettings*`, `persistence.PurgeSettingsRepository`), created by `AutoMigrate`. The use cases `GetPurgeSettings`, `UpdatePurgeSettings`, `PreviewPurge` and `RunPurge` depend only on domain interfaces, an injected clock and an injected `*log.Logger`, so a scheduler can call them. `PreviewPurge` and `RunPurge` share one plan (`purgePlan.go`): a duplicate group (same non-empty sha256) is planned only when a copy is used in an article, the most used copy is kept (ties: oldest `created_at`, then lowest id) and every other copy is merged into it. The fingerprint is a SHA-256 over the planned groups. `RunPurge` takes a non-blocking in-process lock (`ErrPurgeBusy` for a concurrent call), refuses a non-matching fingerprint with `ErrPurgeConflict`, merges group by group with `MergeMedia` (a failing group is counted and skipped), unions the `UpdatedArticleIDs` of the fully merged groups, and records `last_purge_*` after every run. +#### Automatic purge scheduler + +`PurgeSchedulerUseCase.Tick(now)` returns the `purge.RunResult` of the run, whether it ran and an error. It reads the purge settings on every call and runs `RunPurge.Run(nil)` (no fingerprint) when `Settings.NextPurgeAt()` is set and `now` is not before it. It never writes settings: only `RunPurge` records `last_purge_*`, through `SaveLastRun`, which moves the anchor forward even when nothing was purged. The runner in `internal/infrastructure/scheduler` calls `Tick` and logs one line with `groups_purged`, `merged`, `articles_updated` and `failed_groups` when a run happened. It calls `Tick` once after a startup delay of 1 minute, which catches up a run missed during downtime in a single purge, then once every hour. `api/dependecyInjection.go` builds it and `cmd/server/main.go` starts it with the signal context of the server. Changes to the settings apply on the next tick, with no restart. + +- Overlap guard: a manual run holds the `RunPurge` lock, so `Tick` returns `ErrPurgeBusy` at once. The runner treats it as a skip, logs nothing and re-evaluates at the next tick. +- Failures: any other error is logged and the loop keeps going. There is no retry inside a tick; the next hourly tick evaluates the schedule again. +- Shutdown: on SIGINT or SIGTERM the runner stops taking ticks and waits, with no time limit, for a `Tick` in progress. `main` waits for the runner to return before the database is closed. + `middleware.RequireOwnerOrAdmin` lets only the roles owner and admin through (403 otherwise, including an empty role). The media router mounts these routes after the verified chain and before the `/{id}` routes: | Route | Purpose | diff --git a/docs/configuration.md b/docs/configuration.md index 48f0638..90c7f5c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -33,6 +33,8 @@ Copy [`.env.example`](../.env.example) to `.env` to get started. When a required | `RENEWCMS_CONTAINER` | `false` | Set to `true` (case-insensitive; any other value means `false`) when the app runs in a container. Settings then tells the owner to update by pulling a new image instead of offering browser updates, and `can_apply` is always `false`. The Dockerfile and `docker-compose.yml` set it to `true`. Update checks still work. | | `UPDATE_FEED_URL` | `https://git.floriansylvain.fr` | Base URL of the Gitea server that publishes RenewCMS releases. The Settings page asks `{UPDATE_FEED_URL}/api/v1/repos/fsylvain/RenewCMS/releases` when the owner checks for updates. The last result is kept in `update-status.json`, next to `DB_FILE`. | +Automatic media purge has no environment variable. Its schedule (on or off, interval, `enabled_at`, `last_purge_at`) lives in the purge settings row of the database, is changed from the Settings page or `PATCH /v1/media/purge-settings`, and is read again at every hourly tick. + Relative paths are resolved against the process's working directory. In production, use absolute paths (see [deployment.md](deployment.md)). For example, `WorkingDirectory=/var/lib/renewcms` with `DB_FILE=./data/renewcms.db` puts the database in `/var/lib/renewcms/data`. Browser-update `--data-dir` must name that directory, because the app reads helper status and update records beside the database. It is not necessarily `WorkingDirectory` or `ReadWritePaths`. Do not change an existing `DB_FILE` merely to match the helper: that could open a different, empty database. See [browser-update setup](deployment.md#browser-updates). diff --git a/features/media-purge.feature b/features/media_purge.feature similarity index 99% rename from features/media-purge.feature rename to features/media_purge.feature index 71c5ea5..0256caa 100644 --- a/features/media-purge.feature +++ b/features/media_purge.feature @@ -8,7 +8,7 @@ Feature: Owners and admins can merge duplicate images in use, with persisted pur Given internal/domain/purge defines the entity Settings with AutoPurgeEnabled (default false), AutoPurgeIntervalDays (default 30, valid 1 to 365), LastPurgeAt, LastPurgeMerged, LastPurgeArticlesUpdated and EnabledAt (unset or 0 by default), plus a Repository - interface with Get and Save, and the sentinels ErrInvalidPurgeInterval, ErrPurgeConflict + interface with Get, Save and SaveLastRun, and the sentinels ErrInvalidPurgeInterval, ErrPurgeConflict (the library changed since the fingerprint) and ErrPurgeBusy (a run is already in progress), all asserted with errors.Is And Repository.Get returns the defaults when nothing was ever saved (a single-row table) @@ -389,7 +389,7 @@ Feature: Owners and admins can merge duplicate images in use, with persisted pur And no last_purge_* values are saved Scenario: A failing save of the run result is a logged 500 and the merges stay - Given the purge settings repository's Save fails after the merges succeeded + Given the purge settings repository's SaveLastRun fails after the merges succeeded When the client sends POST /v1/media/purge with {} Then the response status is 500 and the body is "Internal server error" And the merges that already happened stay done diff --git a/features/media-purge.qa.md b/features/media_purge.qa.md similarity index 99% rename from features/media-purge.qa.md rename to features/media_purge.qa.md index 613ba0d..c43ac3b 100644 --- a/features/media-purge.qa.md +++ b/features/media_purge.qa.md @@ -1,4 +1,4 @@ -# QA procedure: media-purge +# QA procedure: media_purge Package: `test` (black-box). New files `test/media_purge_http_test.go` and `test/media_purge_test.go` (use case level). Fake type names must be unique in the package. Go only: no SPA, no scheduler, no env var. diff --git a/features/media_purge_scheduler.feature b/features/media_purge_scheduler.feature new file mode 100644 index 0000000..29a6637 --- /dev/null +++ b/features/media_purge_scheduler.feature @@ -0,0 +1,286 @@ +Feature: Duplicate images are purged automatically on the configured schedule + + As a RenewCMS owner or admin + I want the duplicate purge to run by itself every few days once I turn it on + So that I do not have to remember to purge, and the Settings "Last purge" line stays current + + Background: + Given part 1 provides purge.Settings, purge.Repository (Get, Save and SaveLastRun) and RunPurgeUseCase.Run(fingerprint *string) + And a new use case PurgeSchedulerUseCase in internal/infrastructure/useCases depends only on purge.Repository + and a RunPurge port, takes no HTTP types and imports no persistence, mailer, models, gorm, net/http, chi or api + And PurgeSchedulerUseCase.Tick(now time.Time) (result purge.RunResult, ran bool, err error) reads the settings on every call, + where result carries groups purged, copies merged and articles updated and is empty when ran is false + And a run is due when auto_purge_enabled is true, enabled_at is set and now is not before + Settings.NextPurgeAt(), that is max(last_purge_at, enabled_at) + auto_purge_interval_days days + And a scheduled run is RunPurge.Run(nil): no fingerprint, no HTTP, no user-visible message + And a run that finds nothing to purge still records last_purge_at with merged 0 and articles_updated 0, + so the anchor moves forward (this is RunPurge's existing behaviour) + And the scheduler never writes settings itself: only RunPurge records last_purge_*, + through Repository.SaveLastRun, which writes only the last_purge_* fields and never the whole settings row + And there is no env var: the schedule (auto_purge_enabled, auto_purge_interval_days, enabled_at, + last_purge_at) lives in the purge settings row, and is read again on every tick + And a runner in the new package internal/infrastructure/scheduler calls Tick on a ticker every 1 hour, + runs one Tick after a startup delay of 1 minute, and stops when its context is cancelled + And the runner takes its clock, its tick source and its startup delay by injection, so tests use a fake clock + and hand-fed channels, with no real sleeping and no real time.Ticker + And api/dependecyInjection.go builds the runner and cmd/server/main.go only starts it + And the runner uses the *log.Logger already used by RunPurge (log.Default() in the container) + + Rule: When a run is due + + Background: + Given the fake clock reads 2026-10-05T09:00:00Z + + Scenario: Automatic purge off never runs, whatever the dates are + Given auto_purge_enabled is false, interval 1, enabled_at 2020-01-01T00:00:00Z and last_purge_at 2020-01-02T00:00:00Z + When Tick is called with the fake clock's time + Then ran is false and err is nil + And RunPurge was not called and Save was not called + + Scenario: The defaults never run + Given nothing was ever saved, so the settings are the defaults + When Tick is called + Then ran is false and RunPurge was not called + + Scenario: On, never ran, not due yet + Given auto_purge_enabled is true, interval 30, enabled_at 2026-09-20T09:00:00Z and no last_purge_at + When Tick is called + Then ran is false and RunPurge was not called + + Scenario: On, never ran, due since enabled_at plus the interval + Given auto_purge_enabled is true, interval 10, enabled_at 2026-09-25T09:00:00Z and no last_purge_at + When Tick is called + Then RunPurge is called once with no fingerprint and ran is true + + Scenario Outline: The anchor is the later of last_purge_at and enabled_at + Given auto_purge_enabled is true and interval 10 + And enabled_at is and last_purge_at is + When Tick is called at 2026-10-05T09:00:00Z + Then RunPurge is called time(s) + + Examples: + | enabled_at | last_purge_at | calls | why | + | 2026-09-01T00:00:00Z | 2026-09-20T00:00:00Z | 1 | last purge + 10 days = 2026-09-30, due | + | 2026-09-01T00:00:00Z | 2026-09-28T00:00:00Z | 0 | last purge + 10 days = 2026-10-08, not due | + | 2026-09-28T00:00:00Z | 2026-09-01T00:00:00Z | 0 | enabled_at is later, 2026-10-08, not due | + | 2026-09-20T00:00:00Z | 2026-09-01T00:00:00Z | 1 | enabled_at is later, 2026-09-30, due | + | 2026-09-25T09:00:00Z | none | 1 | exactly now is due (2026-10-05T09:00:00Z) | + | 2026-09-25T09:00:01Z | none | 0 | one second before due | + + Scenario: Due exactly at the boundary, not before + Given the next purge is 2026-10-05T09:00:00Z + When Tick is called at 2026-10-05T08:59:59Z + Then RunPurge is not called + When Tick is called at 2026-10-05T09:00:00Z + Then RunPurge is called once + + Scenario: A run advances the anchor, so the next tick is not due + Given a due run with RunPurge backed by the fake settings repository + When Tick is called twice with the same time + Then RunPurge is called once + And last_purge_at equals the fake clock's time + + Scenario: A due run that finds nothing to purge still moves the clock forward + Given a due schedule and a library with no planned group + When Tick is called + Then the stored last_purge_at is the clock's time, last_purge_merged is 0 and last_purge_articles_updated is 0 + And a second Tick at the same time does not run + And a Tick after the interval has elapsed runs again + + Scenario: A run keeps auto_purge_enabled, the interval and enabled_at + Given a due schedule + When Tick runs + Then auto_purge_enabled, auto_purge_interval_days and enabled_at are unchanged in the stored settings + + Scenario: Enabled with no enabled_at is not due + Given auto_purge_enabled is true and enabled_at is unset + When Tick is called + Then RunPurge is not called and the scheduler does not stamp enabled_at itself + + Rule: Settings changes take effect on the next tick, with no restart + + Background: + Given the fake clock reads 2026-10-05T09:00:00Z + + Scenario: Shortening the interval makes a run due on the next tick + Given auto_purge_enabled is true since 2026-09-25T09:00:00Z with interval 30 and no purge yet + And Tick did not run + When UpdatePurgeSettings sets auto_purge_interval_days to 7 + And Tick is called again + Then RunPurge is called once + + Scenario: Lengthening the interval postpones the run + Given auto_purge_enabled is true since 2026-09-20T09:00:00Z with interval 10 and no purge yet, so a run is due + When UpdatePurgeSettings sets auto_purge_interval_days to 30 + And Tick is called + Then RunPurge is not called + + Scenario: Turning it off stops runs + Given a due schedule + When UpdatePurgeSettings sets auto_purge_enabled to false + And Tick is called + Then RunPurge is not called + + Scenario: Turning it on resets the anchor to the moment of the switch + Given auto_purge_enabled was turned off, last_purge_at is 2026-09-01T00:00:00Z, the interval is 30 and the clock reads 2026-10-05T09:00:00Z + When UpdatePurgeSettings sets auto_purge_enabled to true + And Tick is called at once + Then RunPurge is not called, because enabled_at is now + When Tick is called at 2026-11-04T09:00:00Z + Then RunPurge is called once + + Scenario: The scheduler reads the settings at every tick + When Tick is called three times with different stored settings + Then Repository.Get is called three times and no value is cached between ticks + + Rule: Failures never crash the loop and are retried at the next tick + + Background: + Given a due schedule and the fake clock reads 2026-10-05T09:00:00Z + + Scenario: Reading the settings fails + Given the purge settings repository's next Get fails + When Tick is called + Then ran is false and err is the repository error + And RunPurge was not called + And the next Tick, with Get working again, runs the purge + + Scenario: RunPurge returns an error + Given RunPurge fails, for example ListDuplicates fails + When Tick is called + Then err is that error and last_purge_at is unchanged + And RunPurge was called once, not retried inside the same Tick + And the next Tick, with the failure cleared, runs the purge + + Scenario: Saving the run result fails after the merges succeeded + Given the purge settings repository's SaveLastRun fails + When Tick is called + Then err is returned and the merges stay done + And the next Tick plans again from current data and converges + + Scenario: A run that partially fails still records what it merged + Given group A merges and group B's UpdateBody fails + When Tick is called + Then ran is true and err is nil + And last_purge_at is set and last_purge_merged and last_purge_articles_updated count only group A + And group B is planned again by the next run + + Scenario: A settings change made during a run survives + Given a run is in progress and about to record its result + When UpdatePurgeSettings turns auto_purge_enabled off and sets the interval to 5 meanwhile + And the run completes + Then the stored auto_purge_enabled is false and the interval is 5 + And last_purge_at, last_purge_merged and last_purge_articles_updated hold the run's values + + Scenario: A manual run is in progress + Given a manual POST /v1/media/purge holds the RunPurge guard + When Tick is called with a due schedule + Then the scheduler does not queue or wait: it returns at once with errors.Is(err, purge.ErrPurgeBusy) + And nothing is saved by the scheduler + And the runner treats it as a skip, not a failure, and the next tick re-evaluates + When the manual run finishes and records last_purge_at + Then the next Tick finds the schedule not due and does not run + + Scenario: A failing tick does not stop the runner + Given the runner is running with a hand-fed tick channel + And the first Tick returns an error and the second returns ran true + When two ticks are fed + Then the error is written to the injected logger + And the runner is still running and the second Tick was executed + And no Tick is called between the two fed ticks (no tight retry loop) + + Scenario: A completed run logs one line with the counts + Given Tick returns ran true with groups purged 3, merged 5, articles updated 2 and failed groups 1 + When the runner receives the tick + Then exactly one line is logged: "scheduler: automatic purge completed: groups_purged=3 merged=5 articles_updated=2 failed_groups=1" + And a run that found nothing to purge logs the same line with every count at 0 + + Scenario: A due Tick returns what the run did + Given a due schedule and one planned group + When Tick is called + Then ran is true and the result holds the run's groups purged, merged and articles updated + And a Tick that is not due returns an empty result + + Scenario: A busy skip is not logged as an error + Given Tick returns ErrPurgeBusy + When the runner receives the tick + Then it logs no error line and keeps waiting for the next tick + + Rule: The runner starts, ticks and stops with the server + + Scenario: One Tick shortly after startup + Given a runner with a hand-fed startup signal standing in for the 1 minute delay + When the startup signal fires + Then Tick is called once with the fake clock's time + And no Tick is called before the signal + + Scenario: A due run is caught up at startup after downtime + Given the server was down for 40 days, the interval is 30 and last_purge_at is 40 days old + When the runner's startup Tick fires + Then RunPurge is called once, and only once however many ticks were missed + + Scenario: A Tick on each hourly tick + Given the runner is running + When three hourly ticks are fed + Then Tick is called three times, each with the fake clock's time at that tick + + Scenario: Cancelling the context stops the runner + Given the runner is idle + When its context is cancelled + Then Run returns promptly + And no further Tick is called even if a tick is fed after cancel + + Scenario: Shutdown waits for a run in progress without a time limit + Given Tick is blocked inside RunPurge (fake image repository blocked on a channel) + When the context is cancelled + Then Run does not return yet + When the blocked run is released and completes + Then Run returns after the run has recorded last_purge_at + And the database is closed only after Run has returned + + Scenario: Cancelled before the startup signal + Given the context is cancelled before the startup signal + Then Tick is never called and Run returns + + Rule: The Settings page shows the result (HTTP observable, no SPA change) + + Scenario: A scheduled run updates the Last purge values served by the settings endpoint + Given an owner, automatic purge on with interval 7, enabled_at 2026-09-20T09:00:00Z and a planned group + And the fake clock reads 2026-10-05T09:00:00Z + When Tick runs + And the client sends GET /v1/media/purge-settings with a valid cookie of the owner + Then "last_purge_at" is "2026-10-05T09:00:00Z" + And "last_purge_merged" and "last_purge_articles_updated" hold the run's counts + And "next_purge_at" is "2026-10-12T09:00:00Z" + And "auto_purge_enabled", "auto_purge_interval_days" and "enabled_at" are unchanged + + Scenario: A scheduled run with nothing to purge shows zeros, not "Never run" + Given automatic purge on and due and no planned group + When Tick runs and the client sends GET /v1/media/purge-settings + Then "last_purge_at" is "2026-10-05T09:00:00Z", "last_purge_merged" is 0 and "last_purge_articles_updated" is 0 + + Scenario: A scheduled run answers no HTTP request and returns no message + When Tick runs + Then no HTTP handler is involved and the only observable results are the stored settings, the images, the article bodies and a log line + + Rule: Layering and documentation + + Scenario: The scheduler stays inside the domain boundary + Then PurgeSchedulerUseCase imports only the stdlib, internal/domain/* and package-local use case types + And internal/infrastructure/scheduler imports no persistence, mailer, models or gorm package + And only api/dependecyInjection.go constructs it and only cmd/server/main.go starts it + + Scenario: The docs describe the scheduler and state that there is no env var + Then docs/architecture.md describes the scheduler, the hourly tick, the startup tick, the due rule, the overlap guard and the unbounded shutdown wait + And docs/configuration.md states that automatic purge has no env var and that its schedule lives in the database + And no new Go, TS, docs or config file contains a comment + + @manual + Rule: Wiring verified by hand (see the qa.md) + + Scenario: The runner starts with the real server and stops on SIGINT or SIGTERM + Given the server runs and automatic purge is on and due + When one minute elapses after startup + Then a purge ran and the Settings "Last purge" line shows the new date on the next visit + When SIGTERM is sent during a run + Then the process exits only after the run finished, and the SQLite file is not corrupted diff --git a/features/media_purge_scheduler.qa.md b/features/media_purge_scheduler.qa.md new file mode 100644 index 0000000..7adaea3 --- /dev/null +++ b/features/media_purge_scheduler.qa.md @@ -0,0 +1,56 @@ +# QA procedure: media_purge_scheduler + +Package: `test` (black-box). New file `test/media_purge_scheduler_test.go`. Go only: no SPA. No env var. Fake type names must be unique in the package. Reuse part 1's `mockPurgeSettingsRepo`, `mockImageRepo`, `mockArticleRepo`, `seedGroup` and `articleUsing` from test/media_purge_test.go and test/media_purge_http_test.go. No real sleeping, no `time.Ticker`, no `time.Sleep`, no live server, no real SQLite file. + +## Setup shared by every case +1. `validator.InitValidator()` once (only needed by the HTTP cases). +2. Fakes: + - `fakeClock` (mutable, `Now()` and `Set(t)`), starting at 2026-10-05T09:00:00Z. Pass `clock.Now` as the `now func() time.Time` of RunPurge and UpdatePurgeSettings so last_purge_at and enabled_at come from it. + - `mockPurgeSettingsRepo` with `Get`, `Save` and `SaveLastRun` (writes only the last_purge_* fields), failure switches for each, a write counter shared by Save and SaveLastRun, and a hook called before SaveLastRun writes. + - A `fakePurgeRunPort` wrapping the real `RunPurgeUseCase`. It counts calls, records the fingerprint argument (must be nil), and can be armed to return an error. A second variant uses the real RunPurge directly for the integration cases. + - Hand-fed `tickCh` and `startCh` channels passed to the runner in `internal/infrastructure/scheduler` in place of time.NewTicker and the startup delay. Use a `bytes.Buffer` logger. +3. Wire the real `PurgeSchedulerUseCase` over the settings fake, the run port and the clock. + +## Tick decisions (table-driven) +4. Off never runs: off with ancient dates; the defaults; assert ran false, err nil, zero RunPurge calls, zero Save calls. +5. Not due, due, boundary: enabled_at + interval one second after now gives no run; exactly now gives a run; assert the fingerprint argument is nil. +6. Anchor table: the six rows of the outline. Cross-check each row against `Settings.NextPurgeAt()` to prove the scheduler reuses it. +7. Enabled with unset enabled_at never runs and the scheduler never stamps it. +8. Repeat the same Tick twice with the real RunPurge: one run (anchor moved), and `last_purge_at` equals the clock. Advance the clock past the interval: a third Tick runs. +9. Empty library: due Tick records `last_purge_at` = clock, merged 0, articles_updated 0, and leaves auto_purge_enabled, interval and enabled_at untouched. +10. Settings changes: shorten the interval (via `UpdatePurgeSettings` on the same fake) and a run becomes due; lengthen it and a due run is postponed; toggle off and no run; toggle off then on gives enabled_at = clock, no run now, a run once the clock passes +30 days. Assert `Get` is called once per Tick. + +## Failures +11. `failGet`: Tick returns the error, RunPurge count 0, the next Tick with the fake healed runs. +12. RunPurge error (`ListDuplicates` failure): error returned, `last_purge_at` unchanged, one RunPurge call per Tick, recovery on the next Tick. +13. `SaveLastRun` failure after merges: error returned, merged images still gone, next Tick converges. +14. Partial failure (UpdateBody failing for group B): ran true, err nil, `last_purge_at` set, counters count group A only. +15. Busy: block a manual `RunPurge.Run(nil)` inside the image fake (started channel). Call Tick with a due schedule and assert it returns promptly (use a timeout) with `errors.Is(err, purge.ErrPurgeBusy)`, with no `Save` from the scheduler. Release; the next Tick is not due because the manual run recorded `last_purge_at`. Run with `-race`. + +## Runner +16. Feed a startup signal: exactly one Tick, with the clock's time; no Tick before the signal. +17. Feed three ticks: three Ticks. Feed none: no Tick. Assert no Tick between fed ticks. +18. Error then success from Tick: the error text is in the logger buffer, the loop continues, the second Tick executes, and `Run` has not returned. +19. Busy from Tick: no error line in the logger; the loop continues. +19b. Completed run: Tick returns ran true with a known `purge.RunResult`; the logger buffer holds exactly one line with groups_purged, merged, articles_updated and failed_groups. An empty result logs zeros. +19c. Tick result: a due Tick returns the run result; a Tick that is not due returns the empty result. +19d. Lost update: block the settings fake just before `SaveLastRun` writes, PATCH auto_purge_enabled off and the interval to 5 through `UpdatePurgeSettings`, release; the stored auto_purge_* survive and last_purge_* are the run's. The persistence repository test checks that `SaveLastRun` leaves the other columns untouched and creates the default row when none exists. +20. Cancel the context while idle: `Run` returns within a timeout; then feed a tick and assert no Tick happens. Cancel before the startup signal: no Tick at all. +21. Shutdown waits without a bound: block RunPurge in the image fake, cancel the context, assert `Run` has not returned (select with a short timeout), release, assert `Run` returns and `last_purge_at` was recorded. `-race`. +22. The runner and the use case spawn no goroutine that outlives `Run` (verify with a done channel closed on return). + +## HTTP-observable check (existing settings endpoint) +23. Mount the part 1 media router (same setup as media_purge.qa.md) over the same fakes. Seed on, interval 7, enabled_at 2026-09-20T09:00:00Z, one planned group. Tick, then GET `/v1/media/purge-settings` with the owner cookie. Assert `last_purge_at` 2026-10-05T09:00:00Z, counts equal the run, `next_purge_at` 2026-10-12T09:00:00Z, and the other fields unchanged. Repeat with an empty library: zeros and a non-null `last_purge_at`. + +## Layering guards +24. The scheduler use case imports no `net/http`, chi, gorm, `persistence`, `mailer` or `api/...`. Same grep-style test as media_purge.qa.md. `internal/infrastructure/scheduler` imports no `persistence`, `mailer`, `models` or `gorm`, and no `api/...`. +25. Grep the new and edited files (Go, docs) for comments: none; only tool directives. + +## Docs +26. `docs/architecture.md` has a scheduler paragraph (hourly tick, startup tick, due rule from `NextPurgeAt`, overlap guard from RunPurge, unbounded shutdown wait, failures retried at the next tick). `docs/configuration.md` states that there is no env var and that the schedule lives in the database. Read both files and confirm. + +## Manual (not automated) +27. Real boot with `task dev:api` or the built binary: turn automatic purge on, set interval 1 and backdate `enabled_at` in the SQLite file (or wait); about a minute after start, a purge runs, `GET /v1/media/purge-settings` shows the new `last_purge_at`, and Settings "Last purge" is updated on the next visit. +28. Send SIGINT or SIGTERM during a long run: the process exits only after the run ended, with no database error on close. +29. A scheduled run and a manual "Purge now" at the same moment: one wins, the other answers 409 busy (HTTP) or is skipped (scheduler). +30. Docker: the same behaviour in the container, with state on /data. diff --git a/internal/domain/purge/purgeDomain.go b/internal/domain/purge/purgeDomain.go index ac686f0..0ffc234 100644 --- a/internal/domain/purge/purgeDomain.go +++ b/internal/domain/purge/purgeDomain.go @@ -46,3 +46,11 @@ func (s Settings) NextPurgeAt() *time.Time { next := base.AddDate(0, 0, s.AutoPurgeIntervalDays) return &next } + +type RunResult struct { + GroupsTotal int + GroupsPurged int + Merged int + ArticlesUpdated int + FailedGroups int +} diff --git a/internal/domain/purge/purgeRepositoryInterface.go b/internal/domain/purge/purgeRepositoryInterface.go index 0c4ded0..5fcde50 100644 --- a/internal/domain/purge/purgeRepositoryInterface.go +++ b/internal/domain/purge/purgeRepositoryInterface.go @@ -1,6 +1,9 @@ package purge +import "time" + type Repository interface { Get() (Settings, error) Save(settings Settings) error + SaveLastRun(at time.Time, merged int, articlesUpdated int) error } diff --git a/internal/infrastructure/persistence/repositories/purgeSettingsRepository.go b/internal/infrastructure/persistence/repositories/purgeSettingsRepository.go index 894eb10..924c6d3 100644 --- a/internal/infrastructure/persistence/repositories/purgeSettingsRepository.go +++ b/internal/infrastructure/persistence/repositories/purgeSettingsRepository.go @@ -5,6 +5,7 @@ import ( "RenewCMS/internal/infrastructure/persistence/mappers" entity "RenewCMS/internal/infrastructure/persistence/models" "errors" + "time" "gorm.io/gorm" ) @@ -38,3 +39,19 @@ func (p *PurgeSettingsRepository) Save(settings domainPurge.Settings) error { model.ID = purgeSettingsRowID return p.db.Save(&model).Error } + +func (p *PurgeSettingsRepository) SaveLastRun(at time.Time, merged int, articlesUpdated int) error { + result := p.db.Model(&entity.PurgeSettings{}).Where("id = ?", purgeSettingsRowID).Updates(map[string]any{ + "last_purge_at": at, + "last_purge_merged": merged, + "last_purge_articles_updated": articlesUpdated, + }) + if result.Error != nil || result.RowsAffected > 0 { + return result.Error + } + settings := domainPurge.DefaultSettings() + settings.LastPurgeAt = &at + settings.LastPurgeMerged = merged + settings.LastPurgeArticlesUpdated = articlesUpdated + return p.Save(settings) +} diff --git a/internal/infrastructure/scheduler/runner.go b/internal/infrastructure/scheduler/runner.go new file mode 100644 index 0000000..3548754 --- /dev/null +++ b/internal/infrastructure/scheduler/runner.go @@ -0,0 +1,75 @@ +package scheduler + +import ( + "RenewCMS/internal/domain/purge" + "context" + "errors" + "log" + "time" +) + +const ( + TickInterval = time.Hour + StartupDelay = time.Minute +) + +type Job interface { + Tick(now time.Time) (purge.RunResult, bool, error) +} + +type Runner struct { + job Job + now func() time.Time + ticks <-chan time.Time + startup <-chan time.Time + logger *log.Logger + release func() +} + +func NewRunner(job Job, now func() time.Time, ticks <-chan time.Time, startup <-chan time.Time, logger *log.Logger) *Runner { + return &Runner{job: job, now: now, ticks: ticks, startup: startup, logger: logger, release: func() {}} +} + +func NewSystemRunner(job Job, logger *log.Logger) *Runner { + ticker := time.NewTicker(TickInterval) + startup := time.NewTimer(StartupDelay) + runner := NewRunner(job, time.Now, ticker.C, startup.C, logger) + runner.release = func() { + ticker.Stop() + startup.Stop() + } + return runner +} + +func (r *Runner) Run(ctx context.Context) { + defer r.release() + startup := r.startup + for ctx.Err() == nil { + select { + case <-ctx.Done(): + return + case <-startup: + startup = nil + r.tick(ctx) + case <-r.ticks: + r.tick(ctx) + } + } +} + +func (r *Runner) tick(ctx context.Context) { + if ctx.Err() != nil { + return + } + result, ran, err := r.job.Tick(r.now()) + if errors.Is(err, purge.ErrPurgeBusy) { + return + } + if err != nil { + r.logger.Printf("scheduler: automatic purge failed: %v", err) + return + } + if ran { + r.logger.Printf("scheduler: automatic purge completed: groups_purged=%d merged=%d articles_updated=%d failed_groups=%d", result.GroupsPurged, result.Merged, result.ArticlesUpdated, result.FailedGroups) + } +} diff --git a/internal/infrastructure/useCases/PurgeScheduler.go b/internal/infrastructure/useCases/PurgeScheduler.go new file mode 100644 index 0000000..93a6419 --- /dev/null +++ b/internal/infrastructure/useCases/PurgeScheduler.go @@ -0,0 +1,35 @@ +package useCases + +import ( + "RenewCMS/internal/domain/purge" + "time" +) + +type PurgeRunner interface { + Run(fingerprint *string) (purge.RunResult, error) +} + +type PurgeSchedulerUseCase struct { + repository purge.Repository + runner PurgeRunner +} + +func NewPurgeSchedulerUseCase(repository purge.Repository, runner PurgeRunner) *PurgeSchedulerUseCase { + return &PurgeSchedulerUseCase{repository, runner} +} + +func (u *PurgeSchedulerUseCase) Tick(now time.Time) (purge.RunResult, bool, error) { + settings, err := u.repository.Get() + if err != nil { + return purge.RunResult{}, false, err + } + next := settings.NextPurgeAt() + if next == nil || now.Before(*next) { + return purge.RunResult{}, false, nil + } + result, err := u.runner.Run(nil) + if err != nil { + return purge.RunResult{}, false, err + } + return result, true, nil +} diff --git a/internal/infrastructure/useCases/RunPurge.go b/internal/infrastructure/useCases/RunPurge.go index 6ed007a..d3fbb6e 100644 --- a/internal/infrastructure/useCases/RunPurge.go +++ b/internal/infrastructure/useCases/RunPurge.go @@ -10,14 +10,6 @@ import ( "time" ) -type RunPurgeResult struct { - GroupsTotal int - GroupsPurged int - Merged int - ArticlesUpdated int - FailedGroups int -} - type RunPurgeUseCase struct { planner *purgePlanner merge *MergeMediaUseCase @@ -37,30 +29,30 @@ func NewRunPurgeUseCase(imageRepository image.Repository, articleRepository arti } } -func (u *RunPurgeUseCase) Run(fingerprint *string) (RunPurgeResult, error) { +func (u *RunPurgeUseCase) Run(fingerprint *string) (purge.RunResult, error) { if !u.guard.TryLock() { - return RunPurgeResult{}, purge.ErrPurgeBusy + return purge.RunResult{}, purge.ErrPurgeBusy } defer u.guard.Unlock() plan, err := u.planner.plan() if err != nil { - return RunPurgeResult{}, err + return purge.RunResult{}, err } if fingerprint != nil && *fingerprint != plan.fingerprint { - return RunPurgeResult{}, purge.ErrPurgeConflict + return purge.RunResult{}, purge.ErrPurgeConflict } result := u.mergeGroups(plan) if err := u.recordRun(result); err != nil { u.logger.Printf("purge: could not record the run result: %v", err) - return RunPurgeResult{}, err + return purge.RunResult{}, err } return result, nil } -func (u *RunPurgeUseCase) mergeGroups(plan purgePlan) RunPurgeResult { - result := RunPurgeResult{GroupsTotal: len(plan.groups)} +func (u *RunPurgeUseCase) mergeGroups(plan purgePlan) purge.RunResult { + result := purge.RunResult{GroupsTotal: len(plan.groups)} updated := map[uint32]bool{} for _, group := range plan.groups { merged, err := u.merge.Merge(group.kept.ID, slices.Clone(group.mergedIds)) @@ -78,14 +70,6 @@ func (u *RunPurgeUseCase) mergeGroups(plan purgePlan) RunPurgeResult { return result } -func (u *RunPurgeUseCase) recordRun(result RunPurgeResult) error { - settings, err := u.repository.Get() - if err != nil { - return err - } - at := u.now().UTC() - settings.LastPurgeAt = &at - settings.LastPurgeMerged = result.Merged - settings.LastPurgeArticlesUpdated = result.ArticlesUpdated - return u.repository.Save(settings) +func (u *RunPurgeUseCase) recordRun(result purge.RunResult) error { + return u.repository.SaveLastRun(u.now().UTC(), result.Merged, result.ArticlesUpdated) } diff --git a/test/media_purge_http_test.go b/test/media_purge_http_test.go index a925e9a..d6959f4 100644 --- a/test/media_purge_http_test.go +++ b/test/media_purge_http_test.go @@ -24,17 +24,27 @@ const ( ) type mockPurgeSettingsRepo struct { - settings purge.Settings - fail failSwitch - getCalls int - saveCalls int + mu sync.Mutex + settings purge.Settings + fail failSwitch + getCalls int + saveCalls int + beforeRunWrite func() } func newMockPurgeSettingsRepo() *mockPurgeSettingsRepo { return &mockPurgeSettingsRepo{settings: purge.DefaultSettings()} } +func (m *mockPurgeSettingsRepo) snapshot() purge.Settings { + m.mu.Lock() + defer m.mu.Unlock() + return m.settings +} + func (m *mockPurgeSettingsRepo) Get() (purge.Settings, error) { + m.mu.Lock() + defer m.mu.Unlock() m.getCalls++ if err := m.fail.trigger("Get"); err != nil { return purge.Settings{}, err @@ -43,6 +53,8 @@ func (m *mockPurgeSettingsRepo) Get() (purge.Settings, error) { } func (m *mockPurgeSettingsRepo) Save(settings purge.Settings) error { + m.mu.Lock() + defer m.mu.Unlock() m.saveCalls++ if err := m.fail.trigger("Save"); err != nil { return err @@ -51,6 +63,25 @@ func (m *mockPurgeSettingsRepo) Save(settings purge.Settings) error { return nil } +func (m *mockPurgeSettingsRepo) SaveLastRun(at time.Time, merged int, articlesUpdated int) error { + m.mu.Lock() + m.saveCalls++ + hook := m.beforeRunWrite + m.mu.Unlock() + if hook != nil { + hook() + } + m.mu.Lock() + defer m.mu.Unlock() + if err := m.fail.trigger("SaveLastRun"); err != nil { + return err + } + m.settings.LastPurgeAt = &at + m.settings.LastPurgeMerged = merged + m.settings.LastPurgeArticlesUpdated = articlesUpdated + return nil +} + var _ purge.Repository = &mockPurgeSettingsRepo{} func purgeTime(value string) *time.Time { @@ -796,7 +827,7 @@ func TestPurgeRun(t *testing.T) { env := newPurgeEnv(t) env.seedGroup(hashA, 1, 2) env.useImages(10, 1) - env.purgeSettings.fail.failNext("Save") + env.purgeSettings.fail.failNext("SaveLastRun") expectRaw(t, env.run(t, `{}`), 500, internalError) expectIds(t, env.imageIds(), []uint32{1}) diff --git a/test/media_purge_qa_test.go b/test/media_purge_qa_test.go index ca2e28c..0d8d037 100644 --- a/test/media_purge_qa_test.go +++ b/test/media_purge_qa_test.go @@ -64,6 +64,21 @@ func TestPurgeLayering(t *testing.T) { } } }) + + t.Run("the scheduler imports only the stdlib and domain packages", func(t *testing.T) { + imports := list(t, `{{join .Imports "\n"}}`, "RenewCMS/internal/infrastructure/scheduler") + if len(imports) == 0 { + t.Fatal("expected imports") + } + for _, imported := range imports { + if strings.Contains(imported, ".") && !strings.HasPrefix(imported, "RenewCMS/internal/domain/") { + t.Errorf("unexpected import %s", imported) + } + if strings.HasPrefix(imported, "RenewCMS/") && !strings.HasPrefix(imported, "RenewCMS/internal/domain/") { + t.Errorf("scheduler must not import %s", imported) + } + } + }) } func TestPurgeSettingsRepositoryRoundTrip(t *testing.T) { @@ -107,6 +122,39 @@ func TestPurgeSettingsRepositoryRoundTrip(t *testing.T) { } }) + t.Run("SaveLastRun writes only the last_purge_* columns", func(t *testing.T) { + repo := newRepo(t) + enabled := time.Date(2026, 10, 5, 9, 0, 0, 0, time.UTC) + if err := repo.Save(domainPurge.Settings{AutoPurgeEnabled: true, AutoPurgeIntervalDays: 9, EnabledAt: &enabled}); err != nil { + t.Fatal(err) + } + at := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) + + if err := repo.SaveLastRun(at, 4, 2); err != nil { + t.Fatal(err) + } + + got, err := repo.Get() + if err != nil || !got.AutoPurgeEnabled || got.AutoPurgeIntervalDays != 9 || got.EnabledAt == nil || !got.EnabledAt.Equal(enabled) || got.LastPurgeAt == nil || !got.LastPurgeAt.Equal(at) || got.LastPurgeMerged != 4 || got.LastPurgeArticlesUpdated != 2 { + t.Errorf("unexpected settings %+v, %v", got, err) + } + }) + + t.Run("SaveLastRun with no saved row keeps the defaults", func(t *testing.T) { + repo := newRepo(t) + at := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) + + if err := repo.SaveLastRun(at, 0, 0); err != nil { + t.Fatal(err) + } + + got, err := repo.Get() + defaults := domainPurge.DefaultSettings() + if err != nil || got.AutoPurgeEnabled != defaults.AutoPurgeEnabled || got.AutoPurgeIntervalDays != defaults.AutoPurgeIntervalDays || got.LastPurgeAt == nil || !got.LastPurgeAt.Equal(at) { + t.Errorf("unexpected settings %+v, %v", got, err) + } + }) + t.Run("Repeated saves keep a single row", func(t *testing.T) { db := openInMemoryDb(t) if err := db.AutoMigrate(&models.PurgeSettings{}); err != nil { diff --git a/test/media_purge_scheduler_test.go b/test/media_purge_scheduler_test.go new file mode 100644 index 0000000..b297f24 --- /dev/null +++ b/test/media_purge_scheduler_test.go @@ -0,0 +1,782 @@ +package test + +import ( + "RenewCMS/internal/domain/purge" + "RenewCMS/internal/infrastructure/scheduler" + "RenewCMS/internal/infrastructure/useCases" + "bytes" + "context" + "errors" + "log" + "net/http" + "strings" + "sync" + "testing" + "time" +) + +const ( + schedulerClock = "2026-10-05T09:00:00Z" + schedulerWaitTimeout = 2 * time.Second + schedulerQuietPeriod = 50 * time.Millisecond +) + +type schedulerEnv struct { + *mediaEnv + runPurge *useCases.RunPurgeUseCase + scheduler *useCases.PurgeSchedulerUseCase + update *useCases.UpdatePurgeSettingsUseCase +} + +func newSchedulerEnv(t *testing.T) *schedulerEnv { + t.Helper() + env := newPurgeEnv(t) + merge := useCases.NewMergeMediaUseCase(env.images, env.articles) + now := func() time.Time { return env.clock } + runPurge := useCases.NewRunPurgeUseCase(env.images, env.articles, merge, env.purgeSettings, now, log.New(env.logs, "", 0)) + return &schedulerEnv{ + mediaEnv: env, + runPurge: runPurge, + scheduler: useCases.NewPurgeSchedulerUseCase(env.purgeSettings, runPurge), + update: useCases.NewUpdatePurgeSettingsUseCase(env.purgeSettings, now), + } +} + +func (e *schedulerEnv) configure(enabled bool, interval int, enabledAt string, lastPurgeAt string) { + settings := purge.Settings{AutoPurgeEnabled: enabled, AutoPurgeIntervalDays: interval} + if enabledAt != "" { + settings.EnabledAt = purgeTime(enabledAt) + } + if lastPurgeAt != "" { + settings.LastPurgeAt = purgeTime(lastPurgeAt) + } + e.purgeSettings.settings = settings +} + +func (e *schedulerEnv) configureDue() { + e.configure(true, 10, "2026-09-20T09:00:00Z", "") +} + +func (e *schedulerEnv) seedPlannedGroup() { + e.seedGroup(hashA, 1, 2) + e.useImages(1, 1, 2) +} + +func (e *schedulerEnv) tick(t *testing.T, at string) (bool, error) { + t.Helper() + _, ran, err := e.scheduler.Tick(*purgeTime(at)) + return ran, err +} + +func (e *schedulerEnv) expectTick(t *testing.T, at string, wantRan bool) { + t.Helper() + ran, err := e.tick(t, at) + if err != nil || ran != wantRan { + t.Fatalf("expected ran %v and no error at %s, got %v and %v", wantRan, at, ran, err) + } +} + +func (e *schedulerEnv) expectLastPurge(t *testing.T, at string, merged int, updated int) { + t.Helper() + settings := e.purgeSettings.settings + if settings.LastPurgeAt == nil || !settings.LastPurgeAt.Equal(*purgeTime(at)) || settings.LastPurgeMerged != merged || settings.LastPurgeArticlesUpdated != updated { + t.Fatalf("unexpected last purge %+v", settings) + } +} + +func TestPurgeSchedulerDecidesWhenARunIsDue(t *testing.T) { + t.Run("Automatic purge off never runs, whatever the dates are", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(false, 1, "2020-01-01T00:00:00Z", "2020-01-02T00:00:00Z") + env.seedPlannedGroup() + + env.expectTick(t, schedulerClock, false) + + if env.purgeSettings.saveCalls != 0 || len(env.images.deleteCalls) != 0 { + t.Errorf("expected nothing to run or be saved") + } + }) + + t.Run("The defaults never run", func(t *testing.T) { + env := newSchedulerEnv(t) + env.seedPlannedGroup() + + env.expectTick(t, schedulerClock, false) + + if len(env.images.deleteCalls) != 0 { + t.Errorf("expected no run") + } + }) + + t.Run("On, never ran, not due yet", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(true, 30, "2026-09-20T09:00:00Z", "") + env.seedPlannedGroup() + + env.expectTick(t, schedulerClock, false) + + if env.purgeSettings.saveCalls != 0 { + t.Errorf("expected no run") + } + }) + + t.Run("On, never ran, due since enabled_at plus the interval", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(true, 10, "2026-09-25T09:00:00Z", "") + env.seedPlannedGroup() + + env.expectTick(t, schedulerClock, true) + + if env.purgeSettings.saveCalls != 1 || len(env.images.deleteCalls) != 1 { + t.Errorf("expected exactly one run") + } + }) + + t.Run("The anchor is the later of last_purge_at and enabled_at", func(t *testing.T) { + cases := []struct { + name string + enabledAt string + lastPurgeAt string + ran bool + }{ + {"last purge plus interval is past", "2026-09-01T00:00:00Z", "2026-09-20T00:00:00Z", true}, + {"last purge plus interval is ahead", "2026-09-01T00:00:00Z", "2026-09-28T00:00:00Z", false}, + {"enabled_at is later and ahead", "2026-09-28T00:00:00Z", "2026-09-01T00:00:00Z", false}, + {"enabled_at is later and past", "2026-09-20T00:00:00Z", "2026-09-01T00:00:00Z", true}, + {"exactly now is due", "2026-09-25T09:00:00Z", "", true}, + {"one second before due", "2026-09-25T09:00:01Z", "", false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(true, 10, c.enabledAt, c.lastPurgeAt) + + env.expectTick(t, schedulerClock, c.ran) + + wantSaves := 0 + if c.ran { + wantSaves = 1 + } + if env.purgeSettings.saveCalls != wantSaves { + t.Errorf("expected %d runs, got %d", wantSaves, env.purgeSettings.saveCalls) + } + }) + } + }) + + t.Run("Due exactly at the boundary, not before", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(true, 10, "2026-09-25T09:00:00Z", "") + + env.expectTick(t, "2026-10-05T08:59:59Z", false) + env.expectTick(t, "2026-10-05T09:00:00Z", true) + }) + + t.Run("A run advances the anchor, so the next tick is not due", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + env.seedPlannedGroup() + + env.expectTick(t, schedulerClock, true) + env.expectTick(t, schedulerClock, false) + + if env.purgeSettings.saveCalls != 1 { + t.Errorf("expected one run") + } + env.expectLastPurge(t, schedulerClock, 1, 1) + }) + + t.Run("A due run that finds nothing to purge still moves the clock forward", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + + env.expectTick(t, schedulerClock, true) + + env.expectLastPurge(t, schedulerClock, 0, 0) + env.expectTick(t, schedulerClock, false) + env.expectTick(t, "2026-10-15T09:00:00Z", true) + }) + + t.Run("A run keeps auto_purge_enabled, the interval and enabled_at", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + before := env.purgeSettings.settings + + env.expectTick(t, schedulerClock, true) + + after := env.purgeSettings.settings + if !after.AutoPurgeEnabled || after.AutoPurgeIntervalDays != before.AutoPurgeIntervalDays || !after.EnabledAt.Equal(*before.EnabledAt) { + t.Errorf("unexpected settings %+v", after) + } + }) + + t.Run("Enabled with no enabled_at is not due", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(true, 1, "", "") + + env.expectTick(t, schedulerClock, false) + + if env.purgeSettings.saveCalls != 0 || env.purgeSettings.settings.EnabledAt != nil { + t.Errorf("expected nothing stamped or saved") + } + }) +} + +func TestPurgeSchedulerFollowsSettingsChanges(t *testing.T) { + updateInterval := func(env *schedulerEnv, days int) { + if _, err := env.update.Update(useCases.UpdatePurgeSettingsCommand{AutoPurgeIntervalDays: &days}); err != nil { + t.Fatalf("unexpected error %v", err) + } + } + + t.Run("Shortening the interval makes a run due on the next tick", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(true, 30, "2026-09-25T09:00:00Z", "") + env.expectTick(t, schedulerClock, false) + + updateInterval(env, 7) + + env.expectTick(t, schedulerClock, true) + }) + + t.Run("Lengthening the interval postpones the run", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(true, 10, "2026-09-20T09:00:00Z", "") + + updateInterval(env, 30) + + env.expectTick(t, schedulerClock, false) + }) + + t.Run("Turning it off stops runs", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + disabled := false + + if _, err := env.update.Update(useCases.UpdatePurgeSettingsCommand{AutoPurgeEnabled: &disabled}); err != nil { + t.Fatalf("unexpected error %v", err) + } + + env.expectTick(t, schedulerClock, false) + }) + + t.Run("Turning it on resets the anchor to the moment of the switch", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(false, 30, "2026-08-01T00:00:00Z", "2026-09-01T00:00:00Z") + enabled := true + + if _, err := env.update.Update(useCases.UpdatePurgeSettingsCommand{AutoPurgeEnabled: &enabled}); err != nil { + t.Fatalf("unexpected error %v", err) + } + + env.expectTick(t, schedulerClock, false) + env.expectTick(t, "2026-11-04T09:00:00Z", true) + }) + + t.Run("The scheduler reads the settings at every tick", func(t *testing.T) { + env := newSchedulerEnv(t) + + env.expectTick(t, schedulerClock, false) + env.configureDue() + env.expectTick(t, schedulerClock, true) + env.configure(false, 10, "2026-09-20T09:00:00Z", "") + env.expectTick(t, schedulerClock, false) + + if env.purgeSettings.getCalls < 3 { + t.Errorf("expected at least three reads, got %d", env.purgeSettings.getCalls) + } + }) +} + +func TestPurgeSchedulerReportsTheRunResult(t *testing.T) { + t.Run("A due Tick returns what the run did", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + env.seedPlannedGroup() + + result, ran, err := env.scheduler.Tick(*purgeTime(schedulerClock)) + + want := purge.RunResult{GroupsTotal: 1, GroupsPurged: 1, Merged: 1, ArticlesUpdated: 1} + if err != nil || !ran || result != want { + t.Errorf("expected %+v, got %+v, %v, %v", want, result, ran, err) + } + }) + + t.Run("A Tick that is not due returns an empty result", func(t *testing.T) { + env := newSchedulerEnv(t) + + result, ran, err := env.scheduler.Tick(*purgeTime(schedulerClock)) + + if err != nil || ran || result != (purge.RunResult{}) { + t.Errorf("unexpected %+v, %v, %v", result, ran, err) + } + }) +} + +func TestPurgeRunKeepsSettingsChangedWhileItRuns(t *testing.T) { + t.Run("A settings change made during a run survives, only last_purge_* are written by the run", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + env.seedPlannedGroup() + writing := make(chan struct{}) + release := make(chan struct{}) + env.purgeSettings.beforeRunWrite = func() { + close(writing) + <-release + } + runDone := make(chan error, 1) + go func() { + _, err := env.runPurge.Run(nil) + runDone <- err + }() + waitFor(t, writing) + disabled := false + days := 5 + if _, err := env.update.Update(useCases.UpdatePurgeSettingsCommand{AutoPurgeEnabled: &disabled, AutoPurgeIntervalDays: &days}); err != nil { + t.Fatalf("unexpected error %v", err) + } + + close(release) + if err := waitFor(t, runDone); err != nil { + t.Fatalf("unexpected run error %v", err) + } + + settings := env.purgeSettings.settings + if settings.AutoPurgeEnabled || settings.AutoPurgeIntervalDays != 5 { + t.Errorf("expected the change to survive, got %+v", settings) + } + env.expectLastPurge(t, schedulerClock, 1, 1) + }) +} + +func TestPurgeSchedulerFailures(t *testing.T) { + t.Run("Reading the settings fails", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + env.purgeSettings.fail.failNext("Get") + + ran, err := env.tick(t, schedulerClock) + + if ran || !errors.Is(err, errRepositoryFailure) { + t.Fatalf("expected the repository error, got %v and %v", ran, err) + } + if len(env.images.deleteCalls) != 0 { + t.Errorf("expected no run") + } + env.expectTick(t, schedulerClock, true) + }) + + t.Run("RunPurge returns an error", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + env.images.fail.failNext("ListDuplicates") + + ran, err := env.tick(t, schedulerClock) + + if ran || !errors.Is(err, errRepositoryFailure) { + t.Fatalf("expected the repository error, got %v and %v", ran, err) + } + if env.purgeSettings.settings.LastPurgeAt != nil || env.purgeSettings.saveCalls != 0 { + t.Errorf("expected last_purge_at unchanged") + } + env.expectTick(t, schedulerClock, true) + }) + + t.Run("Saving the run result fails after the merges succeeded", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + env.seedPlannedGroup() + env.purgeSettings.fail.failNext("SaveLastRun") + + ran, err := env.tick(t, schedulerClock) + + if ran || !errors.Is(err, errRepositoryFailure) { + t.Fatalf("expected the repository error, got %v and %v", ran, err) + } + if len(env.images.deleteCalls) != 1 { + t.Errorf("expected the merge to stay done") + } + env.expectTick(t, schedulerClock, true) + env.expectLastPurge(t, schedulerClock, 0, 0) + }) + + t.Run("A run that partially fails still records what it merged", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + env.seedGroup(hashA, 1, 2) + env.seedGroup(hashB, 3, 4) + env.useImages(20, 1, 2) + env.useImages(21, 3, 4) + env.articles.failUpdateBodyFor = map[uint32]error{21: errRepositoryFailure} + + env.expectTick(t, schedulerClock, true) + + env.expectLastPurge(t, schedulerClock, 1, 1) + env.articles.failUpdateBodyFor = nil + if plan := env.preview(t); numberOf(plan["groups"]) != 1 { + t.Errorf("expected group B to be planned again, got %v", plan) + } + }) + + t.Run("A manual run is in progress", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + env.seedPlannedGroup() + entered := make(chan struct{}) + release := make(chan struct{}) + env.images.onDelete = func(uint32) { + close(entered) + <-release + } + manualDone := make(chan error, 1) + go func() { + _, err := env.runPurge.Run(nil) + manualDone <- err + }() + waitFor(t, entered) + + ran, err := env.tick(t, schedulerClock) + + if ran || !errors.Is(err, purge.ErrPurgeBusy) { + t.Fatalf("expected ErrPurgeBusy, got %v and %v", ran, err) + } + close(release) + if err := <-manualDone; err != nil { + t.Fatalf("unexpected manual error %v", err) + } + env.expectTick(t, schedulerClock, false) + }) +} + +type schedulerJobFake struct { + mu sync.Mutex + calls []time.Time + results []schedulerJobResult + entered chan time.Time + gate chan struct{} +} + +type schedulerJobResult struct { + result purge.RunResult + ran bool + err error +} + +func newSchedulerJobFake(results ...schedulerJobResult) *schedulerJobFake { + return &schedulerJobFake{results: results, entered: make(chan time.Time, 16)} +} + +func (f *schedulerJobFake) Tick(now time.Time) (purge.RunResult, bool, error) { + f.mu.Lock() + f.calls = append(f.calls, now) + var result schedulerJobResult + if len(f.results) > 0 { + result = f.results[0] + f.results = f.results[1:] + } + gate := f.gate + f.mu.Unlock() + f.entered <- now + if gate != nil { + <-gate + } + return result.result, result.ran, result.err +} + +func (f *schedulerJobFake) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +type schedulerClockFake struct { + mu sync.Mutex + now time.Time +} + +func (c *schedulerClockFake) set(value string) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = *purgeTime(value) +} + +func (c *schedulerClockFake) read() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +type runnerHarness struct { + job *schedulerJobFake + clock *schedulerClockFake + ticks chan time.Time + startup chan time.Time + logs *bytes.Buffer + cancel context.CancelFunc + done chan struct{} +} + +func startRunner(t *testing.T, job *schedulerJobFake) *runnerHarness { + t.Helper() + h := &runnerHarness{ + job: job, + clock: &schedulerClockFake{}, + ticks: make(chan time.Time, 8), + startup: make(chan time.Time, 1), + logs: &bytes.Buffer{}, + done: make(chan struct{}), + } + h.clock.set(schedulerClock) + runner := scheduler.NewRunner(job, h.clock.read, h.ticks, h.startup, log.New(h.logs, "", 0)) + ctx, cancel := context.WithCancel(context.Background()) + h.cancel = cancel + go func() { + defer close(h.done) + runner.Run(ctx) + }() + t.Cleanup(func() { + cancel() + select { + case <-h.done: + case <-time.After(schedulerWaitTimeout): + } + }) + return h +} + +func (h *runnerHarness) expectStopped(t *testing.T) { + t.Helper() + waitFor(t, h.done) +} + +func waitFor[T any](t *testing.T, ch <-chan T) T { + t.Helper() + select { + case value := <-ch: + return value + case <-time.After(schedulerWaitTimeout): + t.Fatalf("timed out waiting") + panic("unreachable") + } +} + +func TestPurgeSchedulerRunner(t *testing.T) { + t.Run("A failing tick does not stop the runner", func(t *testing.T) { + job := newSchedulerJobFake(schedulerJobResult{err: errRepositoryFailure}, schedulerJobResult{ran: true}) + h := startRunner(t, job) + + h.ticks <- time.Time{} + waitFor(t, job.entered) + time.Sleep(schedulerQuietPeriod) + if job.callCount() != 1 { + t.Fatalf("expected no retry between ticks, got %d calls", job.callCount()) + } + h.ticks <- time.Time{} + waitFor(t, job.entered) + + h.cancel() + h.expectStopped(t) + if !strings.Contains(h.logs.String(), errRepositoryFailure.Error()) { + t.Errorf("expected the error to be logged, got %q", h.logs.String()) + } + }) + + t.Run("A busy skip is not logged as an error", func(t *testing.T) { + job := newSchedulerJobFake(schedulerJobResult{err: purge.ErrPurgeBusy}) + h := startRunner(t, job) + + h.ticks <- time.Time{} + waitFor(t, job.entered) + h.cancel() + h.expectStopped(t) + + if h.logs.Len() != 0 { + t.Errorf("expected no log line, got %q", h.logs.String()) + } + }) + + t.Run("A completed run logs one line with the counts", func(t *testing.T) { + cases := []struct { + name string + result purge.RunResult + want string + }{ + {"a run that purged", purge.RunResult{GroupsTotal: 4, GroupsPurged: 3, Merged: 5, ArticlesUpdated: 2, FailedGroups: 1}, "scheduler: automatic purge completed: groups_purged=3 merged=5 articles_updated=2 failed_groups=1\n"}, + {"an empty run", purge.RunResult{}, "scheduler: automatic purge completed: groups_purged=0 merged=0 articles_updated=0 failed_groups=0\n"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + job := newSchedulerJobFake(schedulerJobResult{result: c.result, ran: true}) + h := startRunner(t, job) + + h.ticks <- time.Time{} + waitFor(t, job.entered) + h.cancel() + h.expectStopped(t) + + if h.logs.String() != c.want { + t.Errorf("expected %q, got %q", c.want, h.logs.String()) + } + }) + } + }) + + t.Run("One Tick shortly after startup", func(t *testing.T) { + job := newSchedulerJobFake() + h := startRunner(t, job) + time.Sleep(schedulerQuietPeriod) + if job.callCount() != 0 { + t.Fatalf("expected no Tick before the signal") + } + + h.startup <- time.Time{} + + if got := waitFor(t, job.entered); !got.Equal(*purgeTime(schedulerClock)) { + t.Errorf("expected the clock's time, got %s", got) + } + time.Sleep(schedulerQuietPeriod) + if job.callCount() != 1 { + t.Errorf("expected one Tick, got %d", job.callCount()) + } + }) + + t.Run("A due run is caught up at startup after downtime", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(true, 30, "2026-08-01T00:00:00Z", "2026-08-26T09:00:00Z") + env.seedPlannedGroup() + clock := &schedulerClockFake{} + clock.set(schedulerClock) + ticks := make(chan time.Time, 8) + startup := make(chan time.Time, 1) + runner := scheduler.NewRunner(env.scheduler, clock.read, ticks, startup, log.New(env.logs, "", 0)) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + runner.Run(ctx) + }() + + startup <- time.Time{} + deadline := time.Now().Add(schedulerWaitTimeout) + for lastPurgeAt := env.purgeSettings.snapshot().LastPurgeAt; lastPurgeAt == nil || lastPurgeAt.Before(*purgeTime(schedulerClock)); lastPurgeAt = env.purgeSettings.snapshot().LastPurgeAt { + if time.Now().After(deadline) { + t.Fatalf("the startup run did not happen") + } + time.Sleep(time.Millisecond) + } + cancel() + waitFor(t, done) + + if env.purgeSettings.saveCalls != 1 { + t.Errorf("expected exactly one run, got %d", env.purgeSettings.saveCalls) + } + }) + + t.Run("A Tick on each hourly tick", func(t *testing.T) { + job := newSchedulerJobFake() + h := startRunner(t, job) + + for _, at := range []string{"2026-10-05T10:00:00Z", "2026-10-05T11:00:00Z", "2026-10-05T12:00:00Z"} { + h.clock.set(at) + h.ticks <- time.Time{} + if got := waitFor(t, job.entered); !got.Equal(*purgeTime(at)) { + t.Errorf("expected %s, got %s", at, got) + } + } + + if job.callCount() != 3 { + t.Errorf("expected three Ticks, got %d", job.callCount()) + } + }) + + t.Run("Cancelling the context stops the runner", func(t *testing.T) { + job := newSchedulerJobFake() + h := startRunner(t, job) + + h.cancel() + h.expectStopped(t) + h.ticks <- time.Time{} + time.Sleep(schedulerQuietPeriod) + + if job.callCount() != 0 { + t.Errorf("expected no Tick after cancel") + } + }) + + t.Run("Cancelled before the startup signal", func(t *testing.T) { + job := newSchedulerJobFake() + h := startRunner(t, job) + + h.cancel() + h.startup <- time.Time{} + h.expectStopped(t) + + if job.callCount() != 0 { + t.Errorf("expected no Tick") + } + }) + + t.Run("Shutdown waits for a run in progress without a time limit", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + env.seedPlannedGroup() + entered := make(chan struct{}) + release := make(chan struct{}) + env.images.onDelete = func(uint32) { + close(entered) + <-release + } + clock := &schedulerClockFake{} + clock.set(schedulerClock) + ticks := make(chan time.Time, 1) + runner := scheduler.NewRunner(env.scheduler, clock.read, ticks, make(chan time.Time), log.New(env.logs, "", 0)) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + runner.Run(ctx) + }() + ticks <- time.Time{} + waitFor(t, entered) + + cancel() + select { + case <-done: + t.Fatalf("Run returned while a purge was in progress") + case <-time.After(schedulerQuietPeriod): + } + close(release) + waitFor(t, done) + + env.expectLastPurge(t, schedulerClock, 1, 1) + }) +} + +func TestPurgeSchedulerResultIsServedBySettings(t *testing.T) { + t.Run("A scheduled run updates the Last purge values served by the settings endpoint", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configure(true, 7, "2026-09-20T09:00:00Z", "") + env.seedPlannedGroup() + + env.expectTick(t, schedulerClock, true) + rec := env.purgeSend(t, http.MethodGet, "/v1/media/purge-settings", "") + + body := decodeObject(t, rec.Body.String()) + if body["last_purge_at"] != schedulerClock || numberOf(body["last_purge_merged"]) != 1 || numberOf(body["last_purge_articles_updated"]) != 1 || body["next_purge_at"] != "2026-10-12T09:00:00Z" || body["auto_purge_enabled"] != true || numberOf(body["auto_purge_interval_days"]) != 7 || body["enabled_at"] != "2026-09-20T09:00:00Z" { + t.Errorf("unexpected body %s", rec.Body.String()) + } + }) + + t.Run("A scheduled run with nothing to purge shows zeros, not Never run", func(t *testing.T) { + env := newSchedulerEnv(t) + env.configureDue() + + env.expectTick(t, schedulerClock, true) + rec := env.purgeSend(t, http.MethodGet, "/v1/media/purge-settings", "") + + body := decodeObject(t, rec.Body.String()) + if body["last_purge_at"] != schedulerClock || numberOf(body["last_purge_merged"]) != 0 || numberOf(body["last_purge_articles_updated"]) != 0 { + t.Errorf("unexpected body %s", rec.Body.String()) + } + }) +} diff --git a/test/media_purge_test.go b/test/media_purge_test.go index cb0f0ae..7920c06 100644 --- a/test/media_purge_test.go +++ b/test/media_purge_test.go @@ -95,7 +95,7 @@ func TestRunPurgeDistinguishesBusyFromConflict(t *testing.T) { result, err := newRun(env).Run(nil) - if err != nil || result != (useCases.RunPurgeResult{GroupsTotal: 1, GroupsPurged: 1, Merged: 1}) { + if err != nil || result != (purge.RunResult{GroupsTotal: 1, GroupsPurged: 1, Merged: 1}) { t.Errorf("unexpected result %+v, %v", result, err) } if env.purgeSettings.settings.LastPurgeAt == nil || env.purgeSettings.settings.LastPurgeMerged != 1 {