diff --git a/api/dependecyInjection.go b/api/dependecyInjection.go index 6e7856f..5f5909f 100644 --- a/api/dependecyInjection.go +++ b/api/dependecyInjection.go @@ -21,6 +21,7 @@ import ( "RenewCMS/internal/infrastructure/scheduler" "RenewCMS/internal/infrastructure/storage" "RenewCMS/internal/infrastructure/update" + "RenewCMS/internal/infrastructure/usage" "RenewCMS/internal/infrastructure/useCases" "context" "crypto/ed25519" @@ -100,6 +101,9 @@ type UseCases struct { GenerateAllImageVariantsUseCase *useCases.GenerateAllImageVariantsUseCase PurgeImageVariantUseCase *useCases.PurgeImageVariantUseCase GenerateVariantsOnUploadUseCase *useCases.GenerateVariantsOnUploadUseCase + GetVariantCacheStatsUseCase *useCases.GetVariantCacheStatsUseCase + PurgeVariantCacheUseCase *useCases.PurgeVariantCacheUseCase + VariantUsageTracker *usage.Tracker } func (c *UseCases) RunSchedulers(ctx context.Context) { @@ -123,6 +127,8 @@ type Repositories struct { ImageRepo domainImage.Repository VariantRepo domainImage.VariantRepository VariantFileRepo domainImage.VariantFileRepository + VariantCacheRepo domainImage.VariantCacheRepository + VariantUsageRepo domainImage.VariantUsageRepository VariantRenderer domainImage.VariantRenderer InviteRepo domainInvite.Repository APIKeyRepo domainAPIKey.Repository @@ -252,12 +258,15 @@ func releasePublicKey() ed25519.PublicKey { func getRepositories(connection *database.Connection, settings config.Settings) *Repositories { db := connection.DB dataDir := settings.DataDir() + variantFiles := persistence.NewVariantFileRepository(db) return &Repositories{ ArticleRepo: persistence.NewArticleRepository(db), UserRepo: persistence.NewUserRepository(db), ImageRepo: persistence.NewImageRepository(db), VariantRepo: persistence.NewVariantRepository(db), - VariantFileRepo: persistence.NewVariantFileRepository(db), + VariantFileRepo: variantFiles, + VariantCacheRepo: variantFiles, + VariantUsageRepo: persistence.NewVariantUsageRepository(db), VariantRenderer: imaging.NewVariantRenderer(), InviteRepo: persistence.NewInviteRepository(db), APIKeyRepo: persistence.NewAPIKeyRepository(db), @@ -290,7 +299,8 @@ func getUseCases(repos *Repositories, imageStorage domainImage.Storage, settings purgeScheduler := useCases.NewPurgeSchedulerUseCase(repos.PurgeSettingsRepo, runPurge, recorder) activityCleanup := useCases.NewCleanupActivityUseCase(repos.AuditRepo, repos.AuditSettingsRepo, recorder) - serveVariant := useCases.NewServeVariantUseCase(repos.VariantRepo, repos.VariantFileRepo, repos.ImageRepo, imageStorage, repos.VariantRenderer, useCases.ServeVariantLimits{MaxSourceBytes: settings.ImageVariantMaxSourceBytes, MaxSourcePixels: settings.ImageVariantMaxSourcePixels, MaxConcurrentRenders: settings.ImageVariantMaxConcurrentRenders}) + usageTracker := usage.NewTracker(repos.VariantUsageRepo, time.Now) + serveVariant := useCases.NewServeVariantUseCase(repos.VariantRepo, repos.VariantFileRepo, repos.ImageRepo, imageStorage, repos.VariantRenderer, useCases.ServeVariantLimits{MaxSourceBytes: settings.ImageVariantMaxSourceBytes, MaxSourcePixels: settings.ImageVariantMaxSourcePixels, MaxConcurrentRenders: settings.ImageVariantMaxConcurrentRenders}).WithUsageTracker(usageTracker) return &UseCases{ PurgeScheduler: scheduler.NewSystemRunner(purgeScheduler, log.Default()), @@ -355,6 +365,9 @@ func getUseCases(repos *Repositories, imageStorage domainImage.Storage, settings GenerateAllImageVariantsUseCase: useCases.NewGenerateAllImageVariantsUseCase(repos.ImageRepo, repos.VariantRepo, repos.VariantFileRepo, repos.VariantRenderer, serveVariant, recorder), PurgeImageVariantUseCase: useCases.NewPurgeImageVariantUseCase(repos.ImageRepo, repos.VariantRepo, repos.VariantFileRepo, imageStorage, recorder), GenerateVariantsOnUploadUseCase: useCases.NewGenerateVariantsOnUploadUseCase(repos.VariantRepo, serveVariant, log.Default()), + GetVariantCacheStatsUseCase: useCases.NewGetVariantCacheStatsUseCase(repos.VariantRepo, repos.VariantFileRepo, repos.VariantCacheRepo, repos.ImageRepo, repos.VariantUsageRepo, usageTracker, time.Now), + PurgeVariantCacheUseCase: useCases.NewPurgeVariantCacheUseCase(repos.VariantRepo, repos.ImageRepo, repos.VariantCacheRepo, repos.VariantFileRepo, imageStorage, time.Now, recorder), + VariantUsageTracker: usageTracker, } } @@ -396,7 +409,22 @@ func InitContainer(settings config.Settings) (*UseCases, func() error) { if err := backfillImageMetadata(repos.ImageRepo, imageStorage, settings); err != nil { panic("Failed to backfill image metadata: " + err.Error()) } - return getUseCases(repos, imageStorage, settings), connection.Close + container := getUseCases(repos, imageStorage, settings) + return container, container.runUsageTracker(connection.Close) +} + +func (c *UseCases) runUsageTracker(closeDatabase func() error) func() error { + ctx, stop := context.WithCancel(context.Background()) + stopped := make(chan struct{}) + go func() { + defer close(stopped) + c.VariantUsageTracker.Run(ctx) + }() + return func() error { + stop() + <-stopped + return closeDatabase() + } } type BrowserUpdateCommands struct { diff --git a/api/handlers/imagevariant/cacheHandler.go b/api/handlers/imagevariant/cacheHandler.go new file mode 100644 index 0000000..a823c77 --- /dev/null +++ b/api/handlers/imagevariant/cacheHandler.go @@ -0,0 +1,141 @@ +package imagevariant + +import ( + "RenewCMS/api/pkg/apperr" + validator "RenewCMS/api/pkg/validator" + "RenewCMS/internal/domain/image" + "RenewCMS/internal/infrastructure/useCases" + "encoding/json" + "errors" + "math" + "net/http" + + validatorlib "github.com/go-playground/validator/v10" +) + +type cacheUnusedResponse struct { + Files int `json:"files"` + Bytes int64 `json:"bytes"` + Days int `json:"days"` +} + +type cacheVariantResponse struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Files int `json:"files"` + Bytes int64 `json:"bytes"` +} + +type cacheImageResponse struct { + ID uint32 `json:"id"` + Filename string `json:"filename"` + URL string `json:"url"` + Files int `json:"files"` + Bytes int64 `json:"bytes"` +} + +type cacheResponse struct { + GeneratedFiles int `json:"generated_files"` + DiskUsed int64 `json:"disk_used"` + VariantsDefined int `json:"variants_defined"` + VariantsEnabled int `json:"variants_enabled"` + VariantsDisabled int `json:"variants_disabled"` + RejectedRequests int `json:"rejected_requests"` + RejectedWindowDays int `json:"rejected_window_days"` + Unused cacheUnusedResponse `json:"unused"` + ByVariant []cacheVariantResponse `json:"by_variant"` + TopImages []cacheImageResponse `json:"top_images"` +} + +type cachePurgeRequest struct { + Scope string `json:"scope" validate:"required,oneof=all unused variant image"` + Variant string `json:"variant" validate:"required_if=Scope variant"` + ImageID int64 `json:"image_id" validate:"required_if=Scope image,omitempty,gt=0,lte=4294967295"` +} + +type cachePurgeResponse struct { + Scope string `json:"scope"` + PurgedFiles int `json:"purged_files"` + PurgedBytes int64 `json:"purged_bytes"` + FailedFiles int `json:"failed_files"` +} + +func (r *cachePurgeRequest) ignoreUnusedTargets() { + if r.Scope != string(image.CacheScopeVariant) { + r.Variant = "" + } + if r.Scope != string(image.CacheScopeImage) { + r.ImageID = 0 + } +} + +func cacheResponseOf(stats useCases.VariantCacheStats) cacheResponse { + response := cacheResponse{ + GeneratedFiles: stats.GeneratedFiles, + DiskUsed: stats.DiskUsed, + VariantsDefined: stats.VariantsDefined, + VariantsEnabled: stats.VariantsEnabled, + VariantsDisabled: stats.VariantsDisabled, + RejectedRequests: stats.RejectedRequests, + RejectedWindowDays: stats.RejectedWindowDays, + Unused: cacheUnusedResponse{Files: stats.Unused.Files, Bytes: stats.Unused.Bytes, Days: stats.UnusedDays}, + ByVariant: make([]cacheVariantResponse, 0, len(stats.ByVariant)), + TopImages: make([]cacheImageResponse, 0, len(stats.TopImages)), + } + for _, entry := range stats.ByVariant { + response.ByVariant = append(response.ByVariant, cacheVariantResponse{Name: entry.Name, Enabled: entry.Enabled, Files: entry.Files, Bytes: entry.Bytes}) + } + for _, top := range stats.TopImages { + response.TopImages = append(response.TopImages, cacheImageResponse{ID: top.ID, Filename: top.Filename, URL: top.URL, Files: top.Files, Bytes: top.Bytes}) + } + return response +} + +func (h *Handler) CacheStats(w http.ResponseWriter, _ *http.Request) { + stats, err := h.cacheStats.Get() + if err != nil { + h.writeError(w, "", err) + return + } + writeJSON(w, http.StatusOK, cacheResponseOf(stats)) +} + +func (h *Handler) PurgeCache(w http.ResponseWriter, r *http.Request) { + var request cachePurgeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, apperr.BodyErrorMessage, http.StatusBadRequest) + return + } + request.ignoreUnusedTargets() + if err := validator.Validate.Struct(request); err != nil { + http.Error(w, cachePurgeValidationMessage(err), http.StatusUnprocessableEntity) + return + } + + result, err := h.purgeCache.Purge(r.Context(), useCases.PurgeVariantCacheCommand{ + Scope: image.CachePurgeScope(request.Scope), + Variant: request.Variant, + ImageID: uint32(min(request.ImageID, math.MaxUint32)), + }) + if err != nil { + h.writePurgeCacheError(w, err) + return + } + writeJSON(w, http.StatusOK, cachePurgeResponse{Scope: string(result.Scope), PurgedFiles: result.Files, PurgedBytes: result.Bytes, FailedFiles: result.Failed}) +} + +func cachePurgeValidationMessage(err error) string { + var failures validatorlib.ValidationErrors + if errors.As(err, &failures) && len(failures) > 0 && failures[0].StructField() != "Scope" { + return apperr.CachePurgeTargetErrorMessage + } + return apperr.CachePurgeScopeErrorMessage +} + +func (h *Handler) writePurgeCacheError(w http.ResponseWriter, err error) { + if errors.Is(err, image.ErrImageNotFound) { + http.Error(w, apperr.ImageNotFoundErrorMessage, http.StatusNotFound) + return + } + h.writeError(w, "", err) +} diff --git a/api/handlers/imagevariant/imageVariantHandler.go b/api/handlers/imagevariant/imageVariantHandler.go index b86f3ba..1bbdd88 100644 --- a/api/handlers/imagevariant/imageVariantHandler.go +++ b/api/handlers/imagevariant/imageVariantHandler.go @@ -24,10 +24,12 @@ type Handler struct { setEnabled *useCases.SetVariantEnabledUseCase remove *useCases.DeleteVariantUseCase purge *useCases.PurgeVariantUseCase + cacheStats *useCases.GetVariantCacheStatsUseCase + purgeCache *useCases.PurgeVariantCacheUseCase } -func NewHandler(list *useCases.ListVariantsUseCase, create *useCases.CreateVariantUseCase, update *useCases.UpdateVariantUseCase, setEnabled *useCases.SetVariantEnabledUseCase, remove *useCases.DeleteVariantUseCase, purge *useCases.PurgeVariantUseCase) *Handler { - return &Handler{list, create, update, setEnabled, remove, purge} +func NewHandler(list *useCases.ListVariantsUseCase, create *useCases.CreateVariantUseCase, update *useCases.UpdateVariantUseCase, setEnabled *useCases.SetVariantEnabledUseCase, remove *useCases.DeleteVariantUseCase, purge *useCases.PurgeVariantUseCase, cacheStats *useCases.GetVariantCacheStatsUseCase, purgeCache *useCases.PurgeVariantCacheUseCase) *Handler { + return &Handler{list, create, update, setEnabled, remove, purge, cacheStats, purgeCache} } type definitionRequest struct { diff --git a/api/pkg/apperr/messages.go b/api/pkg/apperr/messages.go index 5f3612f..633968e 100644 --- a/api/pkg/apperr/messages.go +++ b/api/pkg/apperr/messages.go @@ -53,5 +53,7 @@ const VariantFormatErrorMessageFormat = "Format must be %s." const VariantSourceTooLargeErrorMessage = "The source image is too large to generate this variant." const VariantSourceUnsupportedErrorMessage = "Variants aren't generated for this format." const VariantSourceUnreadableErrorMessage = "The source image can't be read, so this variant can't be generated." +const CachePurgeScopeErrorMessage = "Scope must be all, unused, variant or image." +const CachePurgeTargetErrorMessage = "A variant name or an image id is required for this scope." const VariantUnknownMessage = "Unknown variant" const VariantUnsupportedParameterMessage = "Unsupported parameter" diff --git a/api/router.go b/api/router.go index ecab90f..ea6c351 100644 --- a/api/router.go +++ b/api/router.go @@ -286,7 +286,7 @@ func NewMediaRouter(container *UseCases, trustedProxies []string) http.Handler { func NewImageVariantRouter(container *UseCases, trustedProxies []string) http.Handler { r := chi.NewRouter() - variantHandler := imagevariant.NewHandler(container.ListVariantsUseCase, container.CreateVariantUseCase, container.UpdateVariantUseCase, container.SetVariantEnabledUseCase, container.DeleteVariantUseCase, container.PurgeVariantUseCase) + variantHandler := imagevariant.NewHandler(container.ListVariantsUseCase, container.CreateVariantUseCase, container.UpdateVariantUseCase, container.SetVariantEnabledUseCase, container.DeleteVariantUseCase, container.PurgeVariantUseCase, container.GetVariantCacheStatsUseCase, container.PurgeVariantCacheUseCase) r.Use(jwtauth.Verifier(container.Token)) r.Use(middleware.AuditContext(container.GetUserUseCase, trustedProxies)) @@ -297,6 +297,8 @@ func NewImageVariantRouter(container *UseCases, trustedProxies []string) http.Ha r.Use(middleware.RequireOwnerOrAdmin(container.GetUserUseCase)) r.Get("/", variantHandler.List) r.Post("/", variantHandler.Create) + r.Get("/cache", variantHandler.CacheStats) + r.Post("/cache/purge", variantHandler.PurgeCache) r.Put("/{name}", variantHandler.Update) r.Patch("/{name}", variantHandler.SetEnabled) r.Delete("/{name}", variantHandler.Delete) diff --git a/docs/api.md b/docs/api.md index c72c873..08c06a1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -62,7 +62,7 @@ never changes the response or the change itself. | Category | Actions | | --- | --- | | `articles` | `article.create`, `article.edit`, `article.publish`, `article.unpublish`, `article.delete` | -| `media` | `media.upload`, `media.rename`, `media.delete`, `media.merge`, `media.purge`, `media.variant_create`, `media.variant_update`, `media.variant_delete`, `media.variant_enable`, `media.variant_disable`, `media.variant_purge`, `media.variant_generate_all`, `media.variant_purge_image` | +| `media` | `media.upload`, `media.rename`, `media.delete`, `media.merge`, `media.purge`, `media.variant_create`, `media.variant_update`, `media.variant_delete`, `media.variant_enable`, `media.variant_disable`, `media.variant_purge`, `media.variant_generate_all`, `media.variant_purge_image`, `media.variant_purge_all`, `media.variant_purge_unused` | | `users` | `user.register`, `user.verify`, `user.update_profile`, `invite.create`, `invite.cancel`, `invite.renew`, `invite.accept` | | `system` | `system.media_purge` (scheduled), `system.update_checked`, `system.update_downloaded`, `system.restart_requested`, `system.activity_cleanup`, `system.activity_retention_changed` | @@ -77,6 +77,10 @@ name) records nothing. Params: `media.merge` and `media.purge` and `system.media `unsupported_format` or `failed`) and is `failed` only when nothing was generated and something failed; `media.variant_purge_image` carries `purged_files` and `purged_bytes`; both have an `image` target and are recorded only when files were generated or removed; +`media.variant_purge_all` and `media.variant_purge_unused` (no target) carry `purged_files` and +`purged_bytes`, the latter also `older_than_days`; a partial purge adds `failed_files` and stays +`success`, and a purge that removed nothing while some files failed is `failed`; nothing is recorded +when no file was removed or failed; `system.update_checked` carries `update_available`; `system.restart_requested` carries `kind` and, for an update, `tag` (it has no target). `system.update_downloaded` is recorded when the job reaches `ready` or `failed`, with the origin captured at request time; a refusal at the start is recorded @@ -110,6 +114,8 @@ this." Behaviour, caching and the public `?v=` contract are in [image-vari | `PATCH /v1/image-variants/{name}` | `{ "enabled": true }` or `{ "enabled": false }`. Keeps the generated files. | | `DELETE /v1/image-variants/{name}` | 204. Removes the definition and its generated files. | | `POST /v1/image-variants/{name}/purge` | `{ "purged_files", "purged_bytes" }`. Keeps the definition. | +| `GET /v1/image-variants/cache` | Cache statistics, see below. | +| `POST /v1/image-variants/cache/purge` | Purge generated files by scope, see below. | A variant is `{ name, width, height, fit, format, quality, generate_on_upload, enabled, generated_files, cache_size, created_at, updated_at }`. `width` and `height` are 1 to 8000 or `null` @@ -122,6 +128,27 @@ Errors are plain text: 400 for a body that is not valid JSON, 404 "Variant not f named “thumb” already exists. Choose another name." and 422 with one sentence for the first invalid field (name, width and height, fit, format, quality). +### Cache statistics and purge by scope + +`GET /v1/image-variants/cache` answers `{ generated_files, disk_used, variants_defined, +variants_enabled, variants_disabled, rejected_requests, rejected_window_days, unused: { files, bytes, +days }, by_variant: [{ name, enabled, files, bytes }], top_images: [{ id, filename, url, files, bytes }] }`. +`by_variant` lists every defined variant by bytes descending then name; `top_images` lists the ten +images using the most variant bytes (ties by id, images that no longer exist are skipped). `unused` +counts the files of enabled variants whose reference time (the last public request, or the creation +time when never requested) is older than 30 days. `rejected_requests` counts the public requests +refused with 400 over today and the six previous UTC days. Pending request tracking is flushed before +the read. Arrays are never `null`. + +`POST /v1/image-variants/cache/purge` takes `{ "scope": "all" | "unused" | "variant" | "image", +"variant": "", "image_id": }` (`variant` for the scope `variant`, `image_id` for `image`; a +target given for another scope is ignored) and answers `{ scope, purged_files, purged_bytes, +failed_files }`. A purge that fails on some files still answers 200 with `failed_files` above 0; the +failed rows are kept so the same request can be sent again. Errors are plain text: 400 for a body that +is not valid JSON, 422 "Scope must be all, unused, variant or image." or "A variant name or an image +id is required for this scope.", 404 "Variant not found" or "Image not found", 500 when the rows +cannot be listed. + ### Variants of one image Owners and admins only, session only: a request with an API key is 401. All ids are image ids. diff --git a/docs/architecture.md b/docs/architecture.md index 807477c..0ebde24 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -75,6 +75,8 @@ Producers: `ArticleChanges`, `TextChange` and `BodyChange` in `internal/domain/a Generated bytes are stored only through `image.Storage`, so they follow `STORAGE_DRIVER`, and are never rows of `images`. `ServeVariantUseCase` checks, in order: unsupported parameters, the variant (it must exist and be enabled), the source blob, SVG and animated GIF (served as they are), the size cap, the `If-None-Match` ETag, then the cached file, and generates under a `singleflight` keyed by variant, settings and source. A row whose fingerprint no longer matches, or whose blob vanished, is replaced. `UpdateVariantUseCase` purges a variant's files when a rendering setting changes; `DeleteVariantUseCase` and `PurgeVariantUseCase` purge a blob and then its row, so a failed blob delete keeps the row and a retry can finish; `mediaRemover` purges the files of a source before it deletes the source (delete and merge). +The cache screen adds three ports in `internal/domain/image`: `VariantCacheRepository` (unused totals, top sources by bytes and cursor-paged row listing, implemented by the same persistence type as `VariantFileRepository`), `VariantUsageRepository` (one batched write of request times and per-day rejection counts, and the rejection total of a window) and `VariantUsageTracker` (in-memory hits and rejections). `internal/infrastructure/usage.Tracker` implements the tracker; `GetVariantCacheStatsUseCase` and `PurgeVariantCacheUseCase` use the ports only. See [image-variants.md](image-variants.md). + ### Image metadata backfill At boot, `BackfillImageMetadataUseCase` (through `image.Repository.ListMissingMetadata` and `UpdateMetadata`) fills `filename`, `size`, `content_type`, `width`, `height` and `sha256` for image rows that lack them, reading the bytes through `image.Storage` (the files in `UPLOAD_DIR` with the local adapter, the bucket objects with the S3 adapter). It is idempotent, and a row whose file is missing or unreadable is left as it is without stopping the boot. The old `article_id` column and the `article_images` table are kept and never written. diff --git a/docs/image-variants.md b/docs/image-variants.md index d6eded9..b814b7b 100644 --- a/docs/image-variants.md +++ b/docs/image-variants.md @@ -56,6 +56,31 @@ file never takes a slot. The pixel cap defaults to 50000000 pixels. - `generate_on_upload` is stored and returned only. Generating at upload time is not part of this release. +### Request tracking and the Cache screen + +`/settings/media/cache` shows what the cache holds and purges it by scope (all, unused, one variant, +one image). It reads `GET /v1/image-variants/cache` and writes `POST /v1/image-variants/cache/purge` +([api.md](api.md)). + +Two things are tracked for it, without a database write per request. `usage.Tracker` +(`internal/infrastructure/usage`) keeps in memory, behind the `image.VariantUsageTracker` port, the +last time each (variant, source) file was requested and a counter of rejected requests per UTC day. +`ServeVariantUseCase` records a hit when a cached file is served, when a file is generated by the +request and on a 304; it counts a rejection when the request answers 400 (unknown or disabled +variant, unsupported parameter). A missing source (404), a request without `?v=` and the admin +generation routes record nothing. The tracker flushes one batch through `image.VariantUsageRepository` +every 30 seconds, as soon as 500 distinct files are pending, before `GET /v1/image-variants/cache` +reads, and on graceful shutdown; only `api/dependecyInjection.go` builds it and starts the loop. A +failed flush keeps the pending hits and counts, so nothing is lost or counted twice. The stored +`last_requested_at` of a file never moves back, a late update for a purged file changes nothing, and +day rows older than 7 days are deleted by the flush. + +A file is unused when its reference time (`last_requested_at`, or `created_at` when never requested) +is older than 30 days and its variant is enabled. A purge reads the rows in batches of 500 and keeps +the rows whose blob cannot be deleted. Each scope records its own activity action +(`media.variant_purge_all`, `media.variant_purge_unused`, `media.variant_purge`, +`media.variant_purge_image`). + ## Geometry Images are never enlarged. diff --git a/features/image_variants_cache.feature b/features/image_variants_cache.feature new file mode 100644 index 0000000..7f9ca4e --- /dev/null +++ b/features/image_variants_cache.feature @@ -0,0 +1,627 @@ +Feature: Image variants cache panel: cache statistics, purge by scope, request tracking and the Cache screen + + As an owner or admin of a RenewCMS site + I want to see what the variant cache holds and to purge all of it, the unused part, one variant or one image + So that I can reclaim disk without deleting a variant, and the activity log records every purge + + Background: + Given the spec of record is docs/design/screens/image-variants.md sections 5 and 8, its "Back-office API errors" table, the cache frames of docs/design/source/VariantsFrame.dc.html and docs/design/routes.md + And the routes the design marks "proposed" are fixed here: GET /v1/image-variants/cache and POST /v1/image-variants/cache/purge, mounted in NewImageVariantRouter in api/router.go behind the existing owner or admin middleware chain, so a request with an API key has no user_id claim and answers 401 + And features/image-variants.feature and features/image_variants_per_image_api.feature are the contract for variants, index rows and per-image purge, and are unchanged + And a cached file is one VariantFile index row, a row counts whatever the state of its fingerprint, and sizes are in bytes (1 MB = 1048576) + And the page is /settings/media/cache, a verified route rendered as CachePage inside DashboardShell, and it supersedes the aria-disabled "Cache" tab of features/image_variants_screens.feature, which becomes a link to this route + And request validation goes through validate tags and validator.Validate.Struct, an unreadable JSON body answers 400 with apperr.BodyErrorMessage, a failed validation answers 422, and error responses are plain text from http.Error + And new apperr constants are CachePurgeScopeErrorMessage = "Scope must be all, unused, variant or image." and CachePurgeTargetErrorMessage = "A variant name or an image id is required for this scope." + And request tracking has two parts: the last time each generated file was requested, and a counter of rejected requests per UTC day + And tracking never writes to the database on a request: hits are kept in memory by a thread-safe tracker behind a domain port, merged per (variant, source) key, and flushed in one batch by a background loop every 30 seconds, when 500 distinct keys are pending, on GET /v1/image-variants/cache before it reads, and on graceful shutdown + And the flush reaches the database through a domain repository port, the models are added to internal/infrastructure/persistence/database/migrate.go: a nullable last_requested_at column on ImageVariantFile and a new model ImageVariantRejectionDay with a unique day and a count + And the SQL stays portable across SQLite, MySQL/MariaDB and PostgreSQL: plain GROUP BY, ORDER BY, LIMIT and comparisons, no dialect function and no upsert syntax outside GORM's portable clauses + And the cache body is JSON with snake_case fields, arrays are never null, and the Default fixture below is used by the scenarios + And the Default fixture holds six variants, the enabled ones are card, hero, og, square and thumb, the disabled one is print, and the index holds: + | variant | files | bytes | + | hero | 244 | 88080384 | + | card | 318 | 60817408 | + | thumb | 412 | 22020096 | + | print | 74 | 22020096 | + | square | 140 | 19922944 | + | og | 96 | 9437184 | + And 312 files (42991616 bytes) of enabled variants have a reference time older than 30 days, the reference time of a row being last_requested_at when set and created_at otherwise + And the ten images using the most variant bytes are, in order: north-wall.jpg (id 1, 6 files, 6082765 bytes), lime-plaster.jpg (6, 5138022), kitchen-after.webp (5, 4613734), kitchen-before.webp (5, 4089446), floorboards.jpg (5, 3774873), IMG_4471.jpg (4, 3250585), stone-wall-detail.jpg (4, 2936012), bathroom-tiles.png (4, 2726297), roof-lantern.jpg (3, 2411724) and chimney-stack.jpg (3, 2202009) + And 37 rejected requests were counted over the last 7 UTC days + + Rule: Only owners and admins reach the cache routes + + Scenario Outline: Every cache route is guarded + When the client sends with + Then the response status is + And the plain text body is + And no index row, blob, counter or activity event is read or changed + + Examples: + | method | path | credentials | status | body | + | GET | /v1/image-variants/cache | no cookie | 401 | apperr.UnauthorizedErrorMessage | + | GET | /v1/image-variants/cache | a stale token_version cookie | 401 | apperr.UnauthorizedErrorMessage | + | GET | /v1/image-variants/cache | the cookie of an unverified user | 403 | apperr.VerificationRequiredErrorMessage | + | GET | /v1/image-variants/cache | the cookie of a plain user | 403 | apperr.OwnerOrAdminRequiredErrorMessage | + | POST | /v1/image-variants/cache/purge | the cookie of a plain user | 403 | apperr.OwnerOrAdminRequiredErrorMessage | + | GET | /v1/image-variants/cache | a valid API key | 401 | apperr.UnauthorizedErrorMessage | + | POST | /v1/image-variants/cache/purge | a valid API key | 401 | apperr.UnauthorizedErrorMessage | + + Scenario Outline: Owners and admins are both allowed + When sends GET /v1/image-variants/cache with a valid cookie + Then the response status is 200 + + Examples: + | role | + | the owner | + | an admin | + + Rule: The cache statistics + + Scenario: The Default fixture + When an admin sends GET /v1/image-variants/cache + Then the response status is 200 + And the body holds "generated_files" 1284 and "disk_used" 222298112 + And it holds "variants_defined" 6, "variants_enabled" 5 and "variants_disabled" 1 + And it holds "rejected_requests" 37 and "rejected_window_days" 7 + And it holds "unused" {"files": 312, "bytes": 42991616, "days": 30} + And "by_variant" lists every defined variant sorted by bytes descending then name ascending: hero, card, print, thumb, square, og, each as {"name", "enabled", "files", "bytes"} + And "top_images" lists the ten images above in that order, each as {"id", "filename", "url", "files", "bytes"} where url is "/static/uploadedImages/" + + Scenario: Counts are the sum of the index rows + Then "generated_files" equals the sum of the "files" of "by_variant" and "disk_used" equals the sum of their "bytes" + + Scenario: A variant without a file is listed with zero + Given the variant "banner" is defined and enabled and has no file + When an admin sends GET /v1/image-variants/cache + Then "by_variant" holds {"name": "banner", "enabled": true, "files": 0, "bytes": 0} and "variants_defined" is 7 + + Scenario: A disabled variant keeps its files in the totals + Then "by_variant" holds print with "enabled" false, "files" 74 and "bytes" 22020096 + And print is counted in "generated_files" and "disk_used" but not in "unused" + + Scenario: An image deleted behind the index is not a top image + Given index rows exist for a stored name that no image record has + When an admin sends GET /v1/image-variants/cache + Then the rows count in "generated_files", "disk_used" and "by_variant" and no entry of "top_images" names that stored name + + Scenario: Top images are capped at ten and tie-break by id + Given eleven images have variant files and images 7 and 8 have equal bytes + When an admin sends GET /v1/image-variants/cache + Then "top_images" has ten entries and image 7 comes before image 8 + + Scenario: An empty cache + Given the index holds no row + When an admin sends GET /v1/image-variants/cache + Then the response status is 200 + And "generated_files" is 0, "disk_used" is 0, "unused" is {"files": 0, "bytes": 0, "days": 30} + And the raw body holds "top_images":[] and every "by_variant" entry has files 0 + And "variants_defined" and "rejected_requests" are unchanged + + Scenario: No variant defined + Given no variant exists + When an admin sends GET /v1/image-variants/cache + Then the raw body holds "by_variant":[] and "variants_defined":0 + + Scenario: Unused counts only enabled variants older than the cutoff + Given a row of "hero" with last_requested_at 29 days ago and a row of "hero" with last_requested_at 31 days ago + And a row of "print", a disabled variant, with last_requested_at 90 days ago + And a row of "card" with no last_requested_at and a created_at 45 days ago + Then the rows of "hero" 31 days ago and of "card" are counted in "unused", and the other two are not + + Scenario: A row requested exactly 30 days ago is still in use + Given a row whose reference time is exactly the cutoff + Then it is not counted in "unused" + + Scenario: The statistics are aggregated, not read per image + When an admin sends GET /v1/image-variants/cache + Then the file repository answered the totals, the unused totals and the top images in one aggregated call each, never one query per image or per file + + Scenario: The read flushes pending tracking first + Given 12 hits and 3 rejections are pending in memory + When an admin sends GET /v1/image-variants/cache + Then one batch write happens before the read and "rejected_requests" includes the 3 pending + + Scenario: The statistics change nothing else + When an admin sends GET /v1/image-variants/cache + Then no blob is opened, no render happens, no row is deleted and no activity event is recorded + + Scenario: A failing repository answers 500 + Given the file repository fails to total + When an admin sends GET /v1/image-variants/cache + Then the response status is 500 and the plain text body is "Internal server error" + + Rule: The last time each generated file was requested is tracked without a write per hit + + Scenario Outline: What counts as a request for a file + When + Then for the pair (variant, stored name) in memory and no database write happens + + Examples: + | request | effect | + | the public client gets /static/uploadedImages/a1b2.jpg?v=thumb, cached | the last requested time is set | + | the public client gets it and the file is generated by that request | the last requested time is set | + | the public client gets it with a matching If-None-Match and receives 304 | the last requested time is set | + | the public client gets /static/uploadedImages/a1b2.jpg with no variant | nothing is recorded | + | the public client gets it with ?v=unknown | nothing is recorded | + | the public client gets it with ?v=print, a disabled variant | nothing is recorded | + | the public client gets /static/uploadedImages/missing.jpg?v=thumb | nothing is recorded | + | an admin sends POST /v1/media/1/variants/thumb | nothing is recorded | + | an admin sends GET /v1/image-variants/cache | nothing is recorded | + + Scenario: A thousand hits on one file are one pending key + When the public client requests the same cached file 1000 times + Then the database received zero writes during the requests + And after a flush exactly one batch write holds one update for that file with the time of the last hit + + Scenario: The flush writes one batch for many files + Given 40 distinct files were requested + When the tracker flushes + Then the repository received one batch of 40 updates in one call + + Scenario: Nothing pending means no write + When the tracker flushes and nothing was recorded since the last flush + Then the repository is not called + + Scenario: The flush loop and the size trigger + Given 500 distinct keys are pending + Then a flush is started without waiting for the interval + And a flush also happens at each 30 second tick and on graceful shutdown + + Scenario: A failed flush keeps the pending hits + Given the repository fails on the batch + When the tracker flushes, then the repository recovers and the tracker flushes again + Then the second batch holds the hits of both flushes with the latest time per file and nothing is lost or counted twice + + Scenario: A request is never blocked by or failed by the tracker + Given the repository fails on every batch + When the public client requests a cached file + Then the response is 200 with the file bytes + + Scenario: A stored time never moves back + Given a row with last_requested_at 10:05 + When a batch carries the time 10:01 for that file + Then last_requested_at is still 10:05 + + Scenario: A purged file ignores a late update + Given the batch carries a time for a file that was purged since + When the tracker flushes + Then no row is created and the call succeeds + + Scenario: Concurrent requests are race free + When 50 goroutines record hits and rejections while another flushes + Then the totals written equal the recorded ones, with the race detector enabled + + Rule: Rejected public requests are counted per day + + Scenario Outline: What counts as rejected + When the public client sends GET + Then + + Examples: + | url | effect | status | + | /static/uploadedImages/a1b2.jpg?v=unknown | the rejected counter of today is +1 | 400 | + | /static/uploadedImages/a1b2.jpg?v=print | the rejected counter of today is +1 | 400 | + | /static/uploadedImages/a1b2.jpg?width=400 | the rejected counter of today is +1 | 400 | + | /static/uploadedImages/a1b2.jpg?v=Thumb | the rejected counter of today is +1 | 400 | + | /static/uploadedImages/missing.jpg?v=thumb | the counter is unchanged | 404 | + | /static/uploadedImages/a1b2.jpg?v=thumb | the counter is unchanged | 200 | + | /static/uploadedImages/a1b2.jpg | the counter is unchanged | 200 | + + Scenario: A rejection is an in-memory increment + When the public client sends 1000 rejected requests + Then the database received zero writes during the requests + And after a flush one batch adds 1000 to today's day row + + Scenario: The window is today and the six previous UTC days + Given the tracker clock reads 2026-10-08 23:59 UTC and day rows of 2026-10-02 (5) and 2026-10-01 (9) + When an admin sends GET /v1/image-variants/cache + Then "rejected_requests" counts the row of 2026-10-02 and not the row of 2026-10-01 + + Scenario: A day boundary starts a new bucket + Given a rejection at 2026-10-08 23:59:59 UTC and another at 2026-10-09 00:00:00 UTC + Then they are counted in two different day rows + + Scenario: Old day rows are pruned by the flush + Given a day row older than 7 days + When the tracker flushes + Then that row is deleted and the rows of the window are kept + + Scenario: A failed flush keeps the pending rejections + Given the repository fails on the batch + When the tracker flushes, then the repository recovers and flushes again + Then the rejections are written once + + Rule: Purging the cache by scope + + Background: + Given the Default fixture with blobs in image.Storage for every row + And images 1 "north-wall.jpg" and 2 "lime-plaster.jpg" hold index rows + + Scenario: Purge all removes every generated file of every variant + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "all"} + Then the response status is 200 + And the body is {"scope": "all", "purged_files": 1284, "purged_bytes": 222298112, "failed_files": 0} + And every generated blob is deleted from image.Storage and the index holds no row + And the original blobs and every variant definition are unchanged + + Scenario: Purge all includes the files of disabled variants + Then the 74 files of "print" are among the 1284 purged + + Scenario: Purge unused removes the unused files of enabled variants only + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "unused"} + Then the response status is 200 + And the body is {"scope": "unused", "purged_files": 312, "purged_bytes": 42991616, "failed_files": 0} + And the remaining 972 rows are untouched, including any row of "print" and every row requested within 30 days + + Scenario: Purge by variant removes the files of that variant only + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "variant", "variant": "hero"} + Then the body is {"scope": "variant", "purged_files": 244, "purged_bytes": 88080384, "failed_files": 0} + And the files of the other variants are untouched + + Scenario: Purge by variant works for a disabled variant + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "variant", "variant": "print"} + Then "purged_files" is 74 + + Scenario: Purge by image removes every variant file of that image + Given image 1 holds 6 rows of 6082765 bytes + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "image", "image_id": 1} + Then the body is {"scope": "image", "purged_files": 6, "purged_bytes": 6082765, "failed_files": 0} + And the files of image 2 are untouched and the original of image 1 is unchanged + + Scenario: The next public request generates a purged file again + Given a purge with {"scope": "all"} answered 200 + When the public client sends GET /static/uploadedImages/a1b2.jpg?v=thumb + Then the response is 200 and a new blob was put + + Scenario: Purging an empty cache answers zero + Given the index holds no row + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "all"} + Then the response status is 200 and the body is {"scope": "all", "purged_files": 0, "purged_bytes": 0, "failed_files": 0} + + Scenario: A blob already missing from storage does not stop a purge + Given image.Storage reports a missing blob for one row + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "all"} + Then its row is removed and counted as purged and the others are purged + + Scenario Outline: A request that cannot be served + When an admin sends POST /v1/image-variants/cache/purge with + Then the response status is and the plain text body is + And no blob is deleted and no row is deleted and no activity event is recorded + + Examples: + | body | status | message | + | {"scope": "everything"} | 422 | apperr.CachePurgeScopeErrorMessage | + | {"scope": "ALL"} | 422 | apperr.CachePurgeScopeErrorMessage | + | {} | 422 | apperr.CachePurgeScopeErrorMessage | + | {"scope": "variant"} | 422 | apperr.CachePurgeTargetErrorMessage | + | {"scope": "variant", "variant": ""} | 422 | apperr.CachePurgeTargetErrorMessage | + | {"scope": "image"} | 422 | apperr.CachePurgeTargetErrorMessage | + | {"scope": "image", "image_id": 0} | 422 | apperr.CachePurgeTargetErrorMessage | + | {"scope": "image", "image_id": -1} | 422 | apperr.CachePurgeTargetErrorMessage | + | {"scope": "variant", "variant": "missing"} | 404 | apperr.VariantNotFoundErrorMessage | + | {"scope": "image", "image_id": 9999} | 404 | Image not found | + | not json | 400 | apperr.BodyErrorMessage | + + Scenario Outline: A target given for a scope that ignores it is ignored + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "", "variant": "hero", "image_id": 1} + Then the response status is 200 and the purged files are those of the scope "" alone + + Examples: + | scope | + | all | + | unused | + + Rule: A purge that does not complete reports what it did + + Background: + Given the Default fixture and image.Storage fails to delete the blobs of 644 rows + + Scenario: A partial purge answers 200 with the failed count + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "all"} + Then the response status is 200 + And the body is {"scope": "all", "purged_files": 640, "purged_bytes": , "failed_files": 644} + And the 644 rows are kept, so a second call can retry them + + Scenario: Retrying removes the rest + Given a partial purge answered 200 + When image.Storage recovers and the admin sends the same request again + Then "purged_files" is 644 and "failed_files" is 0 + + Scenario: A failure to list the rows answers 500 + Given the file repository fails to list the rows to purge + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "all"} + Then the response status is 500 and the plain text body is "Internal server error" + + Scenario: Rows are processed in bounded batches + When an admin sends POST /v1/image-variants/cache/purge with {"scope": "all"} over 5000 rows + Then the rows were read in batches of at most 500 and never all loaded at once + + Rule: Each purge scope is recorded in the activity log + + Background: + Given recording follows features/activity-log-producers.feature: once, from the use case, after the change, with the request origin + And the category is "media" + And the domain audit gains the actions media.variant_purge_all and media.variant_purge_unused, and reuses media.variant_purge and media.variant_purge_image + + Scenario Outline: A purge that removes files is recorded + When the owner sends POST /v1/image-variants/cache/purge with + Then one event exists with action "", target , outcome "success" and params + + Examples: + | body | action | target | params | + | {"scope": "all"} | media.variant_purge_all | none | {"purged_files": 1284, "purged_bytes": 222298112} | + | {"scope": "unused"} | media.variant_purge_unused | none | {"purged_files": 312, "purged_bytes": 42991616, "older_than_days": 30} | + | {"scope": "variant", "variant": "hero"}| media.variant_purge | {"type": "image_variant", "id": "hero", "name": "hero"} | {"purged_files": 244, "purged_bytes": 88080384} | + | {"scope": "image", "image_id": 1} | media.variant_purge_image | {"type": "image", "id": "1", "name": "north-wall.jpg"} | {"purged_files": 6, "purged_bytes": 6082765} | + + Scenario: A partial failure is a success that names the failed count + Given a purge removes 640 files and fails on 644 + When the owner sends POST /v1/image-variants/cache/purge with {"scope": "all"} + Then one event exists with action "media.variant_purge_all", outcome "success" and params {"purged_files": 640, "purged_bytes": , "failed_files": 644} + + Scenario: Every file failing is a failed event with a plain reason + Given image.Storage fails to delete every blob + When the owner sends POST /v1/image-variants/cache/purge with {"scope": "all"} + Then the response is 200 with "purged_files" 0 and "failed_files" 1284 + And one event exists with action "media.variant_purge_all", outcome "failed" and failure_reason "Something went wrong on our side. Please try again." + + Scenario Outline: Nothing removed, nothing recorded + When + Then no event is recorded + + Examples: + | request | + | the owner purges all on an empty cache | + | the owner purges unused when nothing is unused | + | the owner purges the variant "hero" and it has no file | + | the owner purges image 1 and it has no file | + | the owner sends an unknown scope | + | the owner purges an unknown variant | + | the owner purges an unknown image | + | the owner sends GET /v1/image-variants/cache | + + Scenario: A failing audit write never changes the response + Given the audit repository fails on Append + When the owner sends POST /v1/image-variants/cache/purge with {"scope": "all"} + Then the response is 200 with the same body and the files are gone + + Scenario: The events read back through the existing activity API + Given one event of each of the four scopes was recorded + When the owner sends GET /v1/activity?category=media + Then the four events are listed with their action, target and params + + Scenario Outline: The Activity screen words each scope + Given an activity item with action "", outcome "" and the params above + Then the sentence reads "" and the detail line reads "" + + Examples: + | action | outcome | sentence | detail | + | media.variant_purge_all | success | Purged all variants | Removed 1,284 files (212 MB) | + | media.variant_purge_all | failed | Failed to purge all variants | | + | media.variant_purge_unused | success | Purged unused variants | Removed 312 files (41 MB) not requested for 30 days | + | media.variant_purge_unused | failed | Failed to purge unused variants | | + | media.variant_purge | success | Purged the cache of variant “hero” | Removed 244 files (84 MB) | + | media.variant_purge | failed | Failed to purge the cache of variant “hero” | | + | media.variant_purge_image | success | Purged the variants of north-wall.jpg | Removed 6 files (5.8 MB) | + + Scenario: A partial purge adds the failed count to the detail line + Given an item "media.variant_purge_all" with params {"purged_files": 640, "purged_bytes": 111149056, "failed_files": 644} + Then the detail line reads "Removed 640 files (106 MB) · 644 failed" + + Rule: Cache screen, access and header + + Background: + Given the session user has role "owner" or "admin" from GET /v1/auth/me unless a scenario says otherwise + And the page uses Nuxt UI components only and design tokens, with no raw hex value and no arbitrary Tailwind colour + And every new English string has a French entry in web/src/i18n/messages.ts + And the stats come from GET /v1/image-variants/cache once per navigation, and every size is shown with the formatter of the Variants page (1 MB = 1048576), numbers with thousands separators + And plain-text API errors are mapped by status, never shown raw + And the desktop breadcrumb is "Settings › Media › Cache", the mobile top bar title is "Cache", and the sidebar item "Settings" is current + + Scenario Outline: A session whose role is not owner or admin gets the Forbidden state + Given the session has + When the user opens /settings/media/cache + Then the page shows the empty block with a lock icon, the title "You don't have access" and the text "Only admins can manage image variants. Ask an admin if you need one changed." + And it offers one button "Back to Home" with an arrow-left icon, which routes to /home + And no header, no tabs, no tile and no list is drawn + And + + Examples: + | session | request | + | no role | no request to GET /v1/image-variants/cache is sent | + | role "owner" but the API answers 403 | the request was sent and its 403 is handled here | + + Scenario: The header and the tabs + When the Default fixture is loaded + Then the title is "Cache" and the description is "Generated variant files. Purged files are generated again on the next request." + And two buttons are shown: "Purge unused" with a clock icon and "Purge all variants" with a trash-2 icon in the danger tone + And two tabs follow: "Variants" (icon layers, a link to /settings/media/variants) and "Cache" (icon database, active, medium weight, 2 px primary underline) + And activating "Variants" routes to /settings/media/variants, where its own "Cache" tab routes back here + + Rule: Cache screen, loaded + + Scenario: The four stat tiles + When the Default fixture is loaded + Then four tiles show, as label, value and sub text: + | label | value | sub text | + | Generated files | 1,284 | Across 6 variants | + | Disk used | 212 MB | Originals not included | + | Variants defined | 6 | 5 enabled, 1 disabled | + | Rejected requests | 37 | Last 7 days | + And the tiles form four columns on desktop and two on mobile + + Scenario: The By variant list + When the Default fixture is loaded + Then the card "By variant" lists in the API order: + | name | stats | badge | bar width of the largest | + | hero | 244 files · 84 MB | | 100 % | + | card | 318 files · 58 MB | | 69 % | + | print | 74 files · 21 MB | Disabled | 25 % | + | thumb | 412 files · 21 MB | | 25 % | + | square | 140 files · 19 MB | | 23 % | + | og | 96 files · 9 MB | | 11 % | + And the bar width is the bytes of the row over the largest bytes, 0 when the largest is 0 + And a disabled variant draws its bar in the border-accented colour and the others in the primary colour + And the names are IBM Plex Mono medium and the singular reads "1 file" + + Scenario: The Top 10 list on desktop + When the Default fixture is loaded + Then the card "Top 10 images by variant disk usage" lists ten rows numbered 1 to 10, each with a 32 x 24 thumbnail of the original, the filename, "N variants", the size and a "Purge" button with an eraser icon + And the first row reads north-wall.jpg, 6 variants, 5.8 MB and the last chimney-stack.jpg, 3 variants, 2.1 MB + And the singular reads "1 variant" + + Scenario: The Top 10 list on mobile + Given the viewport is below 640 px + When the Default fixture is loaded + Then the tiles are two columns, the two cards stack, each Top 10 row is 60 px high with the size after the count ("6 variants · 5.8 MB") and the Purge button is icon only, 44 px square, with aria-label "Purge north-wall.jpg" and a tooltip + + Scenario: The layout is two columns on desktop + Given the viewport is 640 px wide or more + Then By variant is 460 px wide and Top 10 takes the rest + + Rule: Cache screen, loading, empty and error + + Scenario: Loading + Given GET /v1/image-variants/cache is unresolved + Then at 149 ms nothing is drawn and at 150 ms the region has aria-busy and the label "Loading cache" + And the four tiles show skeletons for the value and the sub text, By variant shows 5 skeleton rows, Top 10 shows 6 skeleton rows on desktop and is hidden on mobile + And a response at 100 ms never draws skeletons + + Scenario: Empty cache + Given the stats hold zero files and variants_defined 6 + Then the tiles read "0 · Nothing cached" for Generated files, "0 MB · Nothing cached" for Disk used, "6 · 5 enabled, 1 disabled" and "37 · Last 7 days" + And an empty block with a database icon shows the title "Nothing generated yet" and the text "Variants are generated on the first request with ?v=, or from an image's Variants tab." + And "Purge unused" and "Purge all variants" are disabled and no By variant or Top 10 card is drawn + + Scenario Outline: Error loading + Given the GET answers + Then an error alert shows the icon circle-x, the title "Couldn't load cache stats", the text "Check your connection and try again." and the button "Try again" + And no tile, list or table is drawn and the alert has no close button + When the user activates "Try again" + Then GET /v1/image-variants/cache is sent again and the Default fixture renders + + Examples: + | failure | + | a 500 | + | a network error | + + Scenario: Purge unused is disabled when nothing is unused + Given the stats hold "unused" {"files": 0, "bytes": 0, "days": 30} and 1284 files + Then "Purge unused" is disabled and "Purge all variants" is enabled + + Rule: Cache screen, purge all with a typed word + + Scenario: The dialog opens with the button disabled + When the user activates "Purge all variants" + Then a dialog titled "Purge all variants?" shows "This removes 1,284 generated files (212 MB) for every variant. Originals aren't touched. Files are generated again on request." + And a field labelled "Type purge to confirm" in IBM Plex Mono is focused, "Cancel" is offered and "Purge all variants" in the danger tone is disabled + And no request is sent + + Scenario Outline: The word unlocks the button + Given the dialog is open + When the user types "" + Then the confirm button is + + Examples: + | typed | state | + | pur | disabled | + | Purge | disabled | + | purge | enabled | + | purge | enabled | + | purged | disabled | + + Scenario: Cancel and Escape send nothing + When the user activates Cancel, or presses Escape + Then the dialog closes, focus returns to "Purge all variants" and no request is sent + + Scenario: Confirming sends the purge + Given the typed word is "purge" + When the user activates "Purge all variants" + Then POST /v1/image-variants/cache/purge is sent once with {"scope": "all"} and the dialog closes + + Rule: Cache screen, purge unused + + Scenario: The dialog states the count from the stats + When the user activates "Purge unused" + Then a dialog titled "Purge unused variants?" shows "This removes 312 files (41 MB) from enabled variants that haven't been requested for 30 days. Files are generated again on request." + And it offers "Cancel" and "Purge 312 files", with no typed word + + Scenario: Confirming sends the purge + When the user activates "Purge 312 files" + Then POST /v1/image-variants/cache/purge is sent once with {"scope": "unused"} + + Rule: Cache screen, purging, done and failed + + Scenario: Purging + Given a purge request is in flight and the stats hold 1284 files + Then the page shows an info banner with a loader icon and "Purging 1,284 files" with an indeterminate bar, announced with aria-live="polite" + And the tiles and lists are dimmed to 60 % opacity, "Purge unused", "Purge all variants" and every row Purge button are disabled, and the tabs stay active + + Scenario: Purge done after purge all + Given the purge answered {"scope": "all", "purged_files": 1284, "purged_bytes": 222298112, "failed_files": 0} + Then a success toast reads "Purged 1,284 files, freed 212 MB" + And the stats are fetched again once and the empty cache state shows + + Scenario: Purge done after purge unused + Given the purge answered {"scope": "unused", "purged_files": 312, "purged_bytes": 42991616, "failed_files": 0} + Then a success toast reads "Purged 312 files, freed 41 MB" and the stats are fetched again once + + Scenario: Purge failed + Given the purge answered {"scope": "all", "purged_files": 640, "purged_bytes": 111149056, "failed_files": 644} + Then an error toast shows the title "Purge didn't finish", the text "640 of 1,284 files were removed. Try again." and the button "Try again" + And the stats are fetched again once + When the user activates "Try again" + Then the same request is sent again, without asking for the word again + + Scenario Outline: A purge request that errors + Given the POST answers + Then the banner is removed, the content is restored and a toast shows and <text> + + Examples: + | failure | title | text | + | a 400 | Check the request | A value in the request isn't valid. Reload the page and try again. | + | a 403 | You don't have access| Only admins can manage image variants. | + | a 500 | Something went wrong | The server couldn't finish. Try again. If it keeps happening, check the server log. | + | a network error | Something went wrong | The server couldn't finish. Try again. If it keeps happening, check the server log. | + + Scenario: A 401 sends the user to sign in + Given the POST answers 401 + Then the user is redirected to sign in and a toast reads "Session expired" with "Sign in again to continue." + + Rule: Cache screen, purging the variants of one image + + Scenario: A row Purge sends the image scope + When the user activates "Purge" on the north-wall.jpg row + Then POST /v1/image-variants/cache/purge is sent once with {"scope": "image", "image_id": 1} and no confirm dialog is shown + And while it runs the row shows a loader on its button and the other Purge buttons stay enabled + + Scenario: Done + Given the purge answered {"scope": "image", "purged_files": 6, "purged_bytes": 6082765, "failed_files": 0} + Then a success toast reads "Purged 6 files (5.8 MB) for north-wall.jpg" and the stats are fetched again once + + Scenario: The image no longer exists + Given the POST answers 404 + Then a toast shows "Not found" and "This variant or image no longer exists. Refresh the page." and the stats are fetched again + + Scenario: Failed + Given the purge answered {"scope": "image", "purged_files": 4, "purged_bytes": 1000, "failed_files": 2} + Then the "Purge didn't finish" toast reads "4 of 6 files were removed. Try again." with "Try again" + + Rule: Accessibility and vocabulary + + Scenario: Roles and announcements + Then the tabs have role tablist and tab, the dialog traps focus and returns it to its opener, the icon-only buttons have an aria-label and a tooltip, touch targets on mobile are at least 44 px, and every status is an icon plus text + And no rendered text contains "thumbnail preset", "version" or "rendition" + + Rule: Layering and hygiene + + Scenario: The dependency rule holds + Then internal/domain/image and internal/domain/audit import only the stdlib and other internal/domain packages + And the tracker port, the usage repository port and the cache port are declared in internal/domain/image + And the new use cases (cache stats, purge cache) and the tracker import no persistence, mailer, models, gorm, net/http, chi or api package + And the handler imports no persistence, mailer, models or gorm package + And only api/dependecyInjection.go constructs the tracker, starts its flush loop, flushes it on shutdown and wires the use cases into imagevariant.NewHandler and the serve path of the public handler + And a repository method with a single-row Find or Save keeps its current signature + And the models are listed in internal/infrastructure/persistence/database/migrate.go + And no comment is added to any file, except tool directives diff --git a/features/image_variants_cache.qa.md b/features/image_variants_cache.qa.md new file mode 100644 index 0000000..fbf285f --- /dev/null +++ b/features/image_variants_cache.qa.md @@ -0,0 +1,100 @@ +# QA procedure: image_variants_cache + +Two parts: Go for the API and tracking, Vitest for the Cache screen. No live server, no real SQLite file, no real SMTP, no `time.Sleep` (use an injected clock and manual flushes). + +## Go (package `test`, black-box) + +New files: `test/image_variants_cache_http_test.go`, `test/image_variants_cache_tracking_test.go`, `test/image_variants_cache_purge_test.go`, `test/image_variants_cache_audit_test.go`. Extend `test/image_variants_support_test.go`. Fake type names must stay unique in the package. + +### Setup shared by every case +1. `validator.InitValidator()` once. `jwtauth.New("HS256", []byte("test-secret"), nil)` stands in for `JWT_SECRET`. +2. Reuse `mockVariantRepo`, `mockVariantFileRepo`, `mockVariantStorage`, `countingRenderer`, `mockImageRepo`, the user fake, the API-key fake and the audit fakes of `test/activity_support_test.go`. Add to `mockVariantFileRepo`: `last_requested_at` per row, the aggregated totals, unused totals (cutoff, enabled variants only), top-images query (limit 10, bytes desc, id asc), batched listing for purge (batch size recorded), the batch update of requested times, with a call counter per method. Add fakes for the usage repository (`mockVariantUsageRepo`: day rows, batch writes counted, switchable failure) and a controllable clock. +3. Users: owner 1, admin 2, verified plain user 3, unverified user 4. Real signed `jwt` cookies with `user_id` and `token_version`; a stale cookie; a valid API key. +4. Router: a fresh `chi.NewRouter()` mirroring `NewImageVariantRouter` in `api/router.go` (`jwtauth.Verifier`, `AuditContext`, `RequireSignedIn`, `Authenticator`, `RequireCurrentToken`, `RequireVerifiedUser`, `RequireOwnerOrAdmin`) with `Get("/cache")` and `Post("/cache/purge")` beside the existing variant routes, mounted at `/v1/image-variants`. Mount the real public handler `staticimages.NewHandler(...)` at `/static/uploadedImages` over the same storage, variant use cases and the real tracker. +5. Seed the Default fixture of the feature (six variants, the per-variant bytes, 312 unused files with the 29/31/30 day boundary rows, the ten top images, 37 rejections over the window). Images are generated in the test with `image/png`, `image/jpeg`, `image/draw`. + +### A. Guards +6. Table of the feature: each method, path and credential gives the status and exact body constant; assert no file, storage, usage or audit fake was touched. API key gets 401. Owner and admin get 200. + +### B. Statistics +7. Default fixture: decode the body and assert every field and value, `by_variant` order and content, `top_images` order, `url`. Sum invariants. Zero-file variant listed; disabled variant in totals but not in `unused`; orphan index rows counted but absent from `top_images`; tie at the tenth place; empty cache raw body `"top_images":[]`; no variant raw `"by_variant":[]`. +8. `unused` boundaries: 29 days (in use), exactly the cutoff (in use), 31 days (unused), `created_at` fallback, disabled variant excluded. +9. Aggregation: the file fake saw one totals call, one unused call and one top-images call, regardless of the number of images. A pending-tracking fixture asserts one batch write before the read, and that the 3 pending rejections are in `rejected_requests`. A failing totals call gives 500 `Internal server error`. GET records no audit event and opens no blob. + +### C. Tracking +10. Hits: for each row of the "what counts" table drive the real public handler and assert the pending key set and zero usage writes. 1000 hits on one file: zero writes, then one flush gives one update with the last time. 40 files: one batch of 40. Nothing pending: repository not called. 500 distinct keys trigger a flush without the tick (use the injected trigger, not a timer). Failed batch then recovery: nothing lost, nothing counted twice, latest time wins. A failing repository never fails or slows the public response. The stored time never moves back. A late update for a purged file creates no row. +11. Rejections: for each row of the table drive the public handler and assert the counter and the status. 1000 rejections: zero writes, one batch adds 1000. Window: clock at 2026-10-08 23:59 UTC, rows of 2026-10-02 counted, 2026-10-01 not. Day boundary at 23:59:59 and 00:00:00 UTC gives two rows. The flush prunes rows older than 7 days. Failed flush keeps the pending counts. +12. Race (run with `-race`): 50 goroutines record hits and rejections while another flushes; totals written equal totals recorded. Shutdown: closing the container flushes once. + +### D. Purge +13. One case per scope with the numbers of the feature (all 1284 and 222298112, unused 312 and 42991616, variant hero 244 and 88080384, image 1 6 and 6082765): assert the body, each deleted blob name, each deleted row, and that originals and definitions are unchanged. Disabled variant: scope all and scope variant include it; scope unused excludes it. A following public GET renders again. +14. Empty cache returns zeros. A blob missing from storage still removes its row and counts as purged. +15. Refusals table: unknown scope, case change, empty body, missing or empty variant, missing, zero and negative `image_id`, unknown variant (404 `Variant not found`), unknown image (404 `Image not found`), invalid JSON (400 `apperr.BodyErrorMessage`). Assert no storage or row change and no audit event. Extra target fields for `all` and `unused` are ignored. +16. Partial failure: storage fails on 644 rows; 200 with `purged_files` 640, `failed_files` 644, failing rows kept; retry after recovery purges the rest. Listing failure gives 500. Batch size at most 500 over 5000 rows (assert the largest batch the fake returned). + +### E. Audit +17. One case per scope row: action, target (none for all and unused, `image_variant` for variant, `image` for image), outcome, params including `older_than_days` 30 for unused. Partial failure is a success carrying `failed_files`. All-failed is a `failed` event with the generic reason. The no-op rows record nothing. A failing audit repository leaves the response and files unchanged. `GET /v1/activity?category=media` lists the four events. Extend the `failureReasons` mapping test for the two new actions. + +### F. Mutation guard +18. `task mutate` scope is `./internal/infrastructure/useCases`, scoped with `MUTATE_FILES` to the new use case files and `MUTATE_RUN` to the new test names. Each new branch needs a killing assertion: the unused cutoff comparison (below, at and above), the enabled filter of unused, the `purged_files > 0` guard, the success versus failed outcome (`purged > 0` versus `failed > 0`), the top-10 limit and its id tie-break, the window start (7 days) and the never-moving-back time guard. Bare-bool conditions produce no mutant, so cover both branches anyway. + +### G. Layering and hygiene +19. Guard test (`go list -deps` or grep): the domain imports only the stdlib and domain packages; the new use cases and the tracker import no `persistence`, `mailer`, `models`, `gorm`, `net/http`, chi or `api`; the handler imports no `persistence`, `mailer`, `models` or `gorm`. Check `migrate.go` lists `ImageVariantRejectionDay`. `CGO_ENABLED=0 go build -tags nofrontend ./...` succeeds. No comment in new or edited files. `docs/api.md` lists the two routes, the shapes, the scopes and the errors, and `docs/architecture.md` or the variants doc describes the tracker and its 30 second flush. + +## Automated (Vitest + Vue Test Utils, `task test:web`) + +Mock `useApi`'s `request` per case (GET `/v1/image-variants/cache`, POST `/v1/image-variants/cache/purge`); no backend. Mount `CachePage` inside `DashboardShell` with a resolved verified session (id 1, username "marta", role "owner") and a test router at `/settings/media/cache`. Mock `matchMedia` for the 640 px breakpoint; run layout cases at desktop and below-640. Use fake timers for the 150 ms skeleton delay. No test freezes the system date (this screen shows no date). + +Default fixture = the Background of the feature, with the six variants, the ten top images and the numbers above. + +1. **Access, Examples-driven**: no role, role "owner", role "admin", 403 on the list. Forbidden copy and "Back to Home"; no request when the role is missing. +2. **Header and tabs**: title, description, the two buttons with icons, the two tabs; "Variants" routes to `/settings/media/variants`; the Variants page's "Cache" tab now routes here (update the existing VariantsPage spec that asserted aria-disabled); breadcrumb and mobile title. +3. **Tiles**: the four rows of the table; two columns on mobile, four on desktop. +4. **By variant**: order, stats text, the Disabled badge, bar width percentages (100, 69, 25, 25, 23, 11), the colour classes for enabled and disabled, `1 file` singular, 0 width when the largest is 0. +5. **Top 10**: ten rows, numbering, thumbnails, `6 variants`, `1 variant`, sizes; mobile 60 px rows with the size after the count and an icon-only 44 px Purge with aria-label and tooltip. +6. **Loading**: nothing at 149 ms; at 150 ms `aria-busy`, label "Loading cache", skeleton counts (5 and 6; Top 10 hidden on mobile); a response at 100 ms never draws skeletons. +7. **Empty**: tile subs "Nothing cached", empty block copy, both purge buttons disabled, no lists. **Purge unused** disabled when unused is 0. +8. **Error, Examples-driven**: 500 and network error, alert copy, no close button, Try again re-sends and renders. +9. **Purge all dialog**: copy, focus on the field, confirm disabled; the typed-word table (`pur`, `Purge`, `purge`, `purged`); Cancel and Escape send nothing and return focus; confirm sends `{"scope": "all"}` once. +10. **Purge unused dialog**: copy with 312 and 41 MB, "Purge 312 files", sends `{"scope": "unused"}`. +11. **Purging**: banner copy "Purging 1,284 files", `aria-live="polite"`, dimmed content, all purge controls disabled, tabs active. +12. **Done**: toasts "Purged 1,284 files, freed 212 MB" and "Purged 312 files, freed 41 MB"; one refetch; the empty state after purge all. +13. **Failed**: toast "Purge didn't finish", "640 of 1,284 files were removed. Try again.", one refetch, Try again re-sends the same body without the typed word. +14. **Errors, Examples-driven**: 400, 403, 500, network, 401 (redirect then toast), each with the copy of the table; banner removed and content restored. +15. **Row purge**: sends `{"scope": "image", "image_id": 1}` with no dialog; button loader; other rows enabled; toast "Purged 6 files (5.8 MB) for north-wall.jpg"; 404 toast and refetch; partial failure "4 of 6 files were removed. Try again.". +16. **Activity sentences** (extend `activitySentence.spec.ts` and `ActivityVariants.spec.ts`): the seven rows of the sentence table, the partial detail line, and `SENTENCE_ACTIONS` including the three actions. +17. **French**: `setLocale('fr')`; a catalogue test that every key used on the page and in the new sentences exists in `messages.ts`; the page renders without missing keys. +18. **Vocabulary and accessibility**: no rendered text contains "thumbnail preset", "version" or "rendition"; tabs, labels, focus return. + +## Visual (task visual:check -- image-variants, desktop and mobile, light and dark) + +NOT RUN: the user forbade running task visual:check for this slice, so every item below is reported NOT RUN. + +Reference: `docs/design/screens/image-variants.md` section 5 and the cache frame in `docs/design/source/VariantsFrame.dc.html` (lines 250-278, with `AppFrame.dc.html` for the shell). Add each state to `web/visual/states.ts` with the board's data. The gate: every non-skipped capture within 10 %, both viewports, both themes. None of these states depends on the date or time, so none is skipped. + +Board sample data the mocks must reuse: variants thumb 412 files 21 MB, card 318 and 58 MB, hero 244 and 84 MB, square 140 and 19 MB, og 96 and 9 MB, print 74 and 21 MB (disabled); totals 1,284 files and 212 MB; 6 variants, 5 enabled and 1 disabled; 37 rejected requests; unused 312 files and 41 MB; the Top 10 of the frame (north-wall.jpg 6 variants 5.8 MB, lime-plaster.jpg 6 and 4.9 MB, kitchen-after.webp 5 and 4.4 MB, kitchen-before.webp 5 and 3.9 MB, floorboards.jpg 5 and 3.6 MB, IMG_4471.jpg 4 and 3.1 MB, stone-wall-detail.jpg 4 and 2.8 MB, bathroom-tiles.png 4 and 2.6 MB, roof-lantern.jpg 3 and 2.3 MB, chimney-stack.jpg 3 and 2.1 MB); session name "Marta Alves". The mock returns the By variant rows in the board's order (hero, card, then the ties). + +- [ ] 5a Loaded +- [ ] 5b Loading +- [ ] 5c Empty cache +- [ ] 5d Purging (the frame draws "640 of 1,284 · 106 MB freed" with a 50 % bar; the draft shows an indeterminate bar and no progress line, so this state is the one at risk against the budget; see open question 1) +- [ ] 5e Purge done (empty cache; the toast) +- [ ] 5f Purge failed (toast "Purge didn't finish") +- [ ] 5g Error loading +- [ ] 5h Purge all: confirm (typed "pur", button disabled) +- [ ] 5i Purge all: confirm, typed (typed "purge") +- [ ] 5j Purge unused: confirm +- [ ] Layout from the frame, not `components.md`: tiles 4 columns and 16 px gap on desktop, 2 columns and 8 px on mobile; tile padding 16 px (12 px mobile), value 28 px (22 px mobile); two columns 460 px / rest; bars 8 px high; Top 10 rows 40 px (60 px mobile) with the Purge button 26 px high (44 px square icon-only on mobile). +- [ ] Not drawn, no visual state: the error toasts for 400, 403, 500 and 404, the row-purge toast, the row loader, the disabled Purge unused with a non-empty cache; Vitest only. +- [ ] Not covered by screenshots: keyboard and focus order (Tab reaches the buttons, the tabs, each Purge; the dialog traps focus and Escape returns it), hover and active states, motion (dialog, skeleton pulse stopped under reduced motion). + +## Manual verification + +- **Real flush loop**: `task dev:api` over a scratch SQLite file. Request `/static/uploadedImages/<storedName>?v=thumb` repeatedly, wait 30 s, then open the Cache page and check "Rejected requests" after a few `?v=unknown` requests. Check in the file that `last_requested_at` is set once per file and the day row holds the count. +- **Graceful shutdown**: record hits, stop the server with Ctrl+C within 30 s, restart and check the counts survived. +- **Real purge**: generate real variants, then purge all, unused (age a row in the file), one image, and check the files disappear from `UPLOAD_DIR` and reappear on request. +- **S3 driver** (`STORAGE_DRIVER=s3`, MinIO): purge all and by image. +- **Large cache**: 20,000 index rows; check the Cache page loads in a reasonable time and the purge keeps memory flat. +- **Other databases**: run the stats and purge against MariaDB and PostgreSQL if available (portable SQL). +- **Activity screen**: read the four new events against section 8 of the design. +- **French**: switch the language and read every state, the dialogs and the toasts. diff --git a/internal/domain/audit/auditDomain.go b/internal/domain/audit/auditDomain.go index 0452de8..0e7ab80 100644 --- a/internal/domain/audit/auditDomain.go +++ b/internal/domain/audit/auditDomain.go @@ -84,6 +84,8 @@ const ( ActionVariantPurge = "media.variant_purge" ActionVariantGenerateAll = "media.variant_generate_all" ActionVariantPurgeImage = "media.variant_purge_image" + ActionVariantPurgeAll = "media.variant_purge_all" + ActionVariantPurgeUnused = "media.variant_purge_unused" ActionUserRegister = "user.register" ActionUserVerify = "user.verify" ActionUserUpdateProfile = "user.update_profile" @@ -123,6 +125,8 @@ const ( ParamKind = "kind" ParamPurgedFiles = "purged_files" ParamPurgedBytes = "purged_bytes" + ParamFailedFiles = "failed_files" + ParamOlderThanDays = "older_than_days" ParamGenerated = "generated" ParamFailed = "failed" ParamCauses = "causes" diff --git a/internal/domain/image/imageRepositoryInterface.go b/internal/domain/image/imageRepositoryInterface.go index 29967cb..89b81cb 100644 --- a/internal/domain/image/imageRepositoryInterface.go +++ b/internal/domain/image/imageRepositoryInterface.go @@ -15,6 +15,7 @@ type Repository interface { ListDuplicates() ([]Image, error) Counts() (Counts, error) ExistsByStoredName(storedName string) (bool, error) + ListByStoredNames(storedNames []string) ([]Image, error) ListMissingMetadata() ([]Image, error) UpdateMetadata(id uint32, metadata Metadata) error } diff --git a/internal/domain/image/variantCache.go b/internal/domain/image/variantCache.go new file mode 100644 index 0000000..9adae52 --- /dev/null +++ b/internal/domain/image/variantCache.go @@ -0,0 +1,76 @@ +package image + +import ( + "errors" + "time" +) + +var ErrCachePurgeScope = errors.New("cache purge scope is not valid") + +type CachePurgeScope string + +const ( + CacheScopeAll CachePurgeScope = "all" + CacheScopeUnused CachePurgeScope = "unused" + CacheScopeVariant CachePurgeScope = "variant" + CacheScopeImage CachePurgeScope = "image" +) + +const ( + UnusedVariantDays = 30 + RejectionWindowDays = 7 + TopImagesLimit = 10 + CachePurgeBatchSize = 500 + RejectionDayLayout = "2006-01-02" +) + +func UnusedCutoff(now time.Time) time.Time { + return now.UTC().AddDate(0, 0, -UnusedVariantDays) +} + +func RejectionDay(at time.Time) time.Time { + utc := at.UTC() + return time.Date(utc.Year(), utc.Month(), utc.Day(), 0, 0, 0, 0, time.UTC) +} + +func RejectionWindowStart(now time.Time) time.Time { + return RejectionDay(now).AddDate(0, 0, -(RejectionWindowDays - 1)) +} + +func RejectionPruneBefore(now time.Time) time.Time { + return RejectionDay(now).AddDate(0, 0, -RejectionWindowDays) +} + +type UnusedFilter struct { + VariantNames []string + Cutoff time.Time +} + +type VariantFileFilter struct { + VariantName string + SourceName string + Unused *UnusedFilter +} + +type SourceTotals struct { + SourceName string + Files int + Bytes int64 +} + +type VariantHit struct { + VariantName string + SourceName string + At time.Time +} + +type RejectionCount struct { + Day time.Time + Count int +} + +type VariantUsageBatch struct { + Hits []VariantHit + Rejections []RejectionCount + PruneBefore time.Time +} diff --git a/internal/domain/image/variantRepositoryInterface.go b/internal/domain/image/variantRepositoryInterface.go index 3819f9e..9f58621 100644 --- a/internal/domain/image/variantRepositoryInterface.go +++ b/internal/domain/image/variantRepositoryInterface.go @@ -1,6 +1,9 @@ package image -import "io" +import ( + "io" + "time" +) type VariantRepository interface { List() ([]Variant, error) @@ -20,6 +23,23 @@ type VariantFileRepository interface { TotalsBySource() (map[string]VariantFileTotals, error) } +type VariantCacheRepository interface { + UnusedTotals(filter UnusedFilter) (VariantFileTotals, error) + TopSources(limit int) ([]SourceTotals, error) + ListPage(filter VariantFileFilter, afterID uint32, limit int) ([]VariantFile, error) +} + +type VariantUsageRepository interface { + Record(batch VariantUsageBatch) error + RejectedSince(day time.Time) (int, error) +} + +type VariantUsageTracker interface { + RecordHit(variantName string, sourceName string) + RecordRejection() + Flush() error +} + type RenderedVariant struct { Content []byte ContentType string diff --git a/internal/infrastructure/persistence/database/migrate.go b/internal/infrastructure/persistence/database/migrate.go index c1b374d..597aad0 100644 --- a/internal/infrastructure/persistence/database/migrate.go +++ b/internal/infrastructure/persistence/database/migrate.go @@ -7,7 +7,7 @@ import ( ) func Migrate(db *gorm.DB) error { - err := tableOptionsFor(db).AutoMigrate(&models.Article{}, &models.Image{}, &models.User{}, &models.Invite{}, &models.APIKey{}, &models.PurgeSettings{}, &models.AuditEvent{}, &models.AuditSettings{}, &models.ImageVariant{}, &models.ImageVariantFile{}) + err := tableOptionsFor(db).AutoMigrate(&models.Article{}, &models.Image{}, &models.User{}, &models.Invite{}, &models.APIKey{}, &models.PurgeSettings{}, &models.AuditEvent{}, &models.AuditSettings{}, &models.ImageVariant{}, &models.ImageVariantFile{}, &models.ImageVariantRejectionDay{}) if err != nil { return err } diff --git a/internal/infrastructure/persistence/models/imageVariantFileModel.go b/internal/infrastructure/persistence/models/imageVariantFileModel.go index 12fd260..aa9a36b 100644 --- a/internal/infrastructure/persistence/models/imageVariantFileModel.go +++ b/internal/infrastructure/persistence/models/imageVariantFileModel.go @@ -11,4 +11,12 @@ type ImageVariantFile struct { Size int64 `gorm:"not null"` Fingerprint string `gorm:"not null;size:64"` CreatedAt time.Time + + LastRequestedAt *time.Time +} + +type ImageVariantRejectionDay struct { + ID uint32 `gorm:"primaryKey;autoIncrement"` + Day string `gorm:"not null;size:10;uniqueIndex:idx_image_variant_rejection_days_day"` + Count int `gorm:"not null"` } diff --git a/internal/infrastructure/persistence/repositories/imageRepository.go b/internal/infrastructure/persistence/repositories/imageRepository.go index bcec4d9..d3eea62 100644 --- a/internal/infrastructure/persistence/repositories/imageRepository.go +++ b/internal/infrastructure/persistence/repositories/imageRepository.go @@ -107,6 +107,21 @@ func (i ImageRepository) ExistsByStoredName(storedName string) (bool, error) { return matches > 0, err } +func (i ImageRepository) ListByStoredNames(storedNames []string) ([]domain.Image, error) { + if len(storedNames) == 0 { + return []domain.Image{}, nil + } + paths := make([]string, 0, len(storedNames)) + for _, storedName := range storedNames { + paths = append(paths, domain.PublicPath(storedName)) + } + var models []entity.Image + if err := i.db.Where("path IN ?", paths).Order("id asc").Find(&models).Error; err != nil { + return nil, err + } + return imagesToDomain(models), nil +} + func (i ImageRepository) duplicatedHashes() *gorm.DB { return i.db.Model(&entity.Image{}). Select("sha256"). diff --git a/internal/infrastructure/persistence/repositories/imageVariantFileRepository.go b/internal/infrastructure/persistence/repositories/imageVariantFileRepository.go index ef1b1ff..72c96cb 100644 --- a/internal/infrastructure/persistence/repositories/imageVariantFileRepository.go +++ b/internal/infrastructure/persistence/repositories/imageVariantFileRepository.go @@ -15,6 +15,7 @@ type VariantFileRepository struct { } var _ domain.VariantFileRepository = &VariantFileRepository{} +var _ domain.VariantCacheRepository = &VariantFileRepository{} func NewVariantFileRepository(db *gorm.DB) *VariantFileRepository { return &VariantFileRepository{db} @@ -90,6 +91,76 @@ func (v *VariantFileRepository) TotalsBySource() (map[string]domain.VariantFileT return totals, nil } +const ( + unusedCondition = "variant_name IN ? AND COALESCE(last_requested_at, created_at) < ?" + topSourcesOrder = "bytes DESC, source_name ASC" +) + +func (v *VariantFileRepository) UnusedTotals(filter domain.UnusedFilter) (domain.VariantFileTotals, error) { + if len(filter.VariantNames) == 0 { + return domain.VariantFileTotals{}, nil + } + var row struct { + Files int + Bytes int64 + } + err := v.db.Model(&entity.ImageVariantFile{}). + Select("COUNT(*) AS files, COALESCE(SUM(size), 0) AS bytes"). + Where(unusedCondition, filter.VariantNames, filter.Cutoff.UTC()). + Scan(&row).Error + if err != nil { + return domain.VariantFileTotals{}, err + } + return domain.VariantFileTotals{Files: row.Files, Bytes: row.Bytes}, nil +} + +func (v *VariantFileRepository) TopSources(limit int) ([]domain.SourceTotals, error) { + var rows []struct { + SourceName string + Files int + Bytes int64 + } + err := v.db.Model(&entity.ImageVariantFile{}). + Select("source_name, COUNT(*) AS files, COALESCE(SUM(size), 0) AS bytes"). + Group("source_name"). + Order(topSourcesOrder). + Limit(limit). + Scan(&rows).Error + if err != nil { + return nil, err + } + sources := make([]domain.SourceTotals, 0, len(rows)) + for _, row := range rows { + sources = append(sources, domain.SourceTotals{SourceName: row.SourceName, Files: row.Files, Bytes: row.Bytes}) + } + return sources, nil +} + +func (v *VariantFileRepository) ListPage(filter domain.VariantFileFilter, afterID uint32, limit int) ([]domain.VariantFile, error) { + statement := v.db.Where("id > ?", afterID) + if filter.VariantName != "" { + statement = statement.Where("variant_name = ?", filter.VariantName) + } + if filter.SourceName != "" { + statement = statement.Where("source_name = ?", filter.SourceName) + } + if filter.Unused != nil { + if len(filter.Unused.VariantNames) == 0 { + return []domain.VariantFile{}, nil + } + statement = statement.Where(unusedCondition, filter.Unused.VariantNames, filter.Unused.Cutoff.UTC()) + } + var found []entity.ImageVariantFile + if err := statement.Order("id asc").Limit(limit).Find(&found).Error; err != nil { + return nil, err + } + files := make([]domain.VariantFile, 0, len(found)) + for _, model := range found { + files = append(files, mappers.VariantFileToDomain(model)) + } + return files, nil +} + func (v *VariantFileRepository) Totals() (map[string]domain.VariantFileTotals, error) { var rows []struct { VariantName string diff --git a/internal/infrastructure/persistence/repositories/imageVariantUsageRepository.go b/internal/infrastructure/persistence/repositories/imageVariantUsageRepository.go new file mode 100644 index 0000000..6f6f227 --- /dev/null +++ b/internal/infrastructure/persistence/repositories/imageVariantUsageRepository.go @@ -0,0 +1,64 @@ +package persistence + +import ( + domain "RenewCMS/internal/domain/image" + entity "RenewCMS/internal/infrastructure/persistence/models" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type VariantUsageRepository struct { + db *gorm.DB +} + +var _ domain.VariantUsageRepository = &VariantUsageRepository{} + +func NewVariantUsageRepository(db *gorm.DB) *VariantUsageRepository { + return &VariantUsageRepository{db} +} + +func (v *VariantUsageRepository) Record(batch domain.VariantUsageBatch) error { + return v.db.Transaction(func(tx *gorm.DB) error { + for _, hit := range batch.Hits { + if err := recordHit(tx, hit); err != nil { + return err + } + } + for _, rejection := range batch.Rejections { + if err := recordRejections(tx, rejection); err != nil { + return err + } + } + if batch.PruneBefore.IsZero() { + return nil + } + return tx.Where("day < ?", batch.PruneBefore.UTC().Format(domain.RejectionDayLayout)).Delete(&entity.ImageVariantRejectionDay{}).Error + }) +} + +func recordHit(tx *gorm.DB, hit domain.VariantHit) error { + at := hit.At.UTC() + return tx.Model(&entity.ImageVariantFile{}). + Where("variant_name = ? AND source_name = ?", hit.VariantName, hit.SourceName). + Where("last_requested_at IS NULL OR last_requested_at < ?", at). + Update("last_requested_at", at).Error +} + +func recordRejections(tx *gorm.DB, rejection domain.RejectionCount) error { + row := entity.ImageVariantRejectionDay{Day: rejection.Day.UTC().Format(domain.RejectionDayLayout), Count: rejection.Count} + return tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "day"}}, + DoUpdates: clause.Assignments(map[string]any{"count": gorm.Expr("? + ?", clause.Column{Table: clause.CurrentTable, Name: "count"}, rejection.Count)}), + }).Create(&row).Error +} + +func (v *VariantUsageRepository) RejectedSince(day time.Time) (int, error) { + var total int + err := v.db.Model(&entity.ImageVariantRejectionDay{}). + Select("COALESCE(SUM(count), 0)"). + Where("day >= ?", day.UTC().Format(domain.RejectionDayLayout)). + Scan(&total).Error + return total, err +} diff --git a/internal/infrastructure/usage/variantUsageTracker.go b/internal/infrastructure/usage/variantUsageTracker.go new file mode 100644 index 0000000..aad209b --- /dev/null +++ b/internal/infrastructure/usage/variantUsageTracker.go @@ -0,0 +1,150 @@ +package usage + +import ( + "RenewCMS/internal/domain/image" + "context" + "sync" + "time" +) + +const FlushKeyLimit = 500 + +func FlushInterval() time.Duration { + return 30 * time.Second +} + +type hitKey struct { + variantName string + sourceName string +} + +type Tracker struct { + repository image.VariantUsageRepository + now func() time.Time + interval time.Duration + keyLimit int + flushing sync.Mutex + pending sync.Mutex + hits map[hitKey]time.Time + rejections map[time.Time]int + wake chan struct{} +} + +var _ image.VariantUsageTracker = &Tracker{} + +func NewTracker(repository image.VariantUsageRepository, now func() time.Time) *Tracker { + return NewTrackerWith(repository, now, FlushInterval(), FlushKeyLimit) +} + +func NewTrackerWith(repository image.VariantUsageRepository, now func() time.Time, interval time.Duration, keyLimit int) *Tracker { + return &Tracker{ + repository: repository, + now: now, + interval: interval, + keyLimit: keyLimit, + hits: map[hitKey]time.Time{}, + rejections: map[time.Time]int{}, + wake: make(chan struct{}, 1), + } +} + +func (t *Tracker) RecordHit(variantName string, sourceName string) { + at := t.now().UTC() + t.pending.Lock() + defer t.pending.Unlock() + keepLatest(t.hits, hitKey{variantName, sourceName}, at) + if len(t.hits) >= t.keyLimit { + t.wakeLoop() + } +} + +func (t *Tracker) RecordRejection() { + day := image.RejectionDay(t.now()) + t.pending.Lock() + defer t.pending.Unlock() + t.rejections[day]++ +} + +func (t *Tracker) wakeLoop() { + select { + case t.wake <- struct{}{}: + default: + } +} + +func keepLatest(hits map[hitKey]time.Time, key hitKey, at time.Time) { + if current, found := hits[key]; !found || at.After(current) { + hits[key] = at + } +} + +func (t *Tracker) Flush() error { + t.flushing.Lock() + defer t.flushing.Unlock() + + hits, rejections := t.take() + if len(hits) == 0 && len(rejections) == 0 { + return nil + } + batch := image.VariantUsageBatch{ + Hits: hitsOf(hits), + Rejections: rejectionsOf(rejections), + PruneBefore: image.RejectionPruneBefore(t.now()), + } + if err := t.repository.Record(batch); err != nil { + t.restore(hits, rejections) + return err + } + return nil +} + +func (t *Tracker) take() (map[hitKey]time.Time, map[time.Time]int) { + t.pending.Lock() + defer t.pending.Unlock() + hits, rejections := t.hits, t.rejections + t.hits, t.rejections = map[hitKey]time.Time{}, map[time.Time]int{} + return hits, rejections +} + +func (t *Tracker) restore(hits map[hitKey]time.Time, rejections map[time.Time]int) { + t.pending.Lock() + defer t.pending.Unlock() + for key, at := range hits { + keepLatest(t.hits, key, at) + } + for day, count := range rejections { + t.rejections[day] += count + } +} + +func hitsOf(hits map[hitKey]time.Time) []image.VariantHit { + listed := make([]image.VariantHit, 0, len(hits)) + for key, at := range hits { + listed = append(listed, image.VariantHit{VariantName: key.variantName, SourceName: key.sourceName, At: at}) + } + return listed +} + +func rejectionsOf(rejections map[time.Time]int) []image.RejectionCount { + listed := make([]image.RejectionCount, 0, len(rejections)) + for day, count := range rejections { + listed = append(listed, image.RejectionCount{Day: day, Count: count}) + } + return listed +} + +func (t *Tracker) Run(ctx context.Context) { + ticker := time.NewTicker(t.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + _ = t.Flush() + return + case <-ticker.C: + _ = t.Flush() + case <-t.wake: + _ = t.Flush() + } + } +} diff --git a/internal/infrastructure/useCases/GetVariantCacheStats.go b/internal/infrastructure/useCases/GetVariantCacheStats.go new file mode 100644 index 0000000..c7796ff --- /dev/null +++ b/internal/infrastructure/useCases/GetVariantCacheStats.go @@ -0,0 +1,170 @@ +package useCases + +import ( + "RenewCMS/internal/domain/image" + "sort" + "time" +) + +type VariantCacheEntry struct { + Name string + Enabled bool + Files int + Bytes int64 +} + +type VariantCacheImage struct { + ID uint32 + Filename string + URL string + Files int + Bytes int64 +} + +type VariantCacheStats struct { + GeneratedFiles int + DiskUsed int64 + VariantsDefined int + VariantsEnabled int + VariantsDisabled int + RejectedRequests int + RejectedWindowDays int + Unused image.VariantFileTotals + UnusedDays int + ByVariant []VariantCacheEntry + TopImages []VariantCacheImage +} + +type GetVariantCacheStatsUseCase struct { + variants image.VariantRepository + files image.VariantFileRepository + cache image.VariantCacheRepository + images image.Repository + usage image.VariantUsageRepository + tracker image.VariantUsageTracker + now func() time.Time +} + +func NewGetVariantCacheStatsUseCase(variants image.VariantRepository, files image.VariantFileRepository, cache image.VariantCacheRepository, images image.Repository, usage image.VariantUsageRepository, tracker image.VariantUsageTracker, now func() time.Time) *GetVariantCacheStatsUseCase { + return &GetVariantCacheStatsUseCase{variants, files, cache, images, usage, tracker, now} +} + +func (u *GetVariantCacheStatsUseCase) Get() (VariantCacheStats, error) { + _ = u.tracker.Flush() + + variants, err := u.variants.List() + if err != nil { + return VariantCacheStats{}, err + } + totals, err := u.files.Totals() + if err != nil { + return VariantCacheStats{}, err + } + now := u.now() + unused, err := u.cache.UnusedTotals(image.UnusedFilter{VariantNames: enabledNames(variants), Cutoff: image.UnusedCutoff(now)}) + if err != nil { + return VariantCacheStats{}, err + } + rejected, err := u.usage.RejectedSince(image.RejectionWindowStart(now)) + if err != nil { + return VariantCacheStats{}, err + } + top, err := u.topImages() + if err != nil { + return VariantCacheStats{}, err + } + + stats := VariantCacheStats{ + VariantsDefined: len(variants), + RejectedRequests: rejected, + RejectedWindowDays: image.RejectionWindowDays, + Unused: unused, + UnusedDays: image.UnusedVariantDays, + ByVariant: cacheEntries(variants, totals), + TopImages: top, + } + for _, entry := range stats.ByVariant { + stats.GeneratedFiles += entry.Files + stats.DiskUsed += entry.Bytes + if entry.Enabled { + stats.VariantsEnabled++ + } + } + stats.VariantsDisabled = stats.VariantsDefined - stats.VariantsEnabled + return stats, nil +} + +func enabledNames(variants []image.Variant) []string { + names := []string{} + for _, variant := range variants { + if variant.Enabled { + names = append(names, variant.Name) + } + } + return names +} + +func cacheEntries(variants []image.Variant, totals map[string]image.VariantFileTotals) []VariantCacheEntry { + entries := make([]VariantCacheEntry, 0, len(variants)) + for _, variant := range variants { + own := totals[variant.Name] + entries = append(entries, VariantCacheEntry{Name: variant.Name, Enabled: variant.Enabled, Files: own.Files, Bytes: own.Bytes}) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].Bytes != entries[j].Bytes { + return entries[i].Bytes > entries[j].Bytes + } + return entries[i].Name < entries[j].Name + }) + return entries +} + +func (u *GetVariantCacheStatsUseCase) topImages() ([]VariantCacheImage, error) { + limit := image.TopImagesLimit + 1 + for { + sources, err := u.cache.TopSources(limit) + if err != nil { + return nil, err + } + ranked, err := u.rank(sources) + if err != nil { + return nil, err + } + if len(sources) < limit || topIsSettled(ranked, sources) { + return ranked[:min(len(ranked), image.TopImagesLimit)], nil + } + limit *= 2 + } +} + +func topIsSettled(ranked []VariantCacheImage, sources []image.SourceTotals) bool { + return len(ranked) >= image.TopImagesLimit && ranked[image.TopImagesLimit-1].Bytes > sources[len(sources)-1].Bytes +} + +func (u *GetVariantCacheStatsUseCase) rank(sources []image.SourceTotals) ([]VariantCacheImage, error) { + names := make([]string, 0, len(sources)) + for _, source := range sources { + names = append(names, source.SourceName) + } + found, err := u.images.ListByStoredNames(names) + if err != nil { + return nil, err + } + byName := make(map[string]image.Image, len(found)) + for _, img := range found { + byName[image.StoredName(img.Path)] = img + } + ranked := make([]VariantCacheImage, 0, len(sources)) + for _, source := range sources { + if img, known := byName[source.SourceName]; known { + ranked = append(ranked, VariantCacheImage{ID: img.ID, Filename: img.Filename, URL: image.PublicPath(source.SourceName), Files: source.Files, Bytes: source.Bytes}) + } + } + sort.SliceStable(ranked, func(i, j int) bool { + if ranked[i].Bytes != ranked[j].Bytes { + return ranked[i].Bytes > ranked[j].Bytes + } + return ranked[i].ID < ranked[j].ID + }) + return ranked, nil +} diff --git a/internal/infrastructure/useCases/PurgeVariantCache.go b/internal/infrastructure/useCases/PurgeVariantCache.go new file mode 100644 index 0000000..1957b2a --- /dev/null +++ b/internal/infrastructure/useCases/PurgeVariantCache.go @@ -0,0 +1,148 @@ +package useCases + +import ( + "RenewCMS/internal/domain/audit" + "RenewCMS/internal/domain/image" + "context" + "time" +) + +type PurgeVariantCacheCommand struct { + Scope image.CachePurgeScope + Variant string + ImageID uint32 +} + +type VariantCachePurgeResult struct { + Scope image.CachePurgeScope + Files int + Bytes int64 + Failed int +} + +type PurgeVariantCacheUseCase struct { + variants image.VariantRepository + images image.Repository + cache image.VariantCacheRepository + files variantCache + recorder audit.Recorder + now func() time.Time +} + +func NewPurgeVariantCacheUseCase(variants image.VariantRepository, images image.Repository, cache image.VariantCacheRepository, files image.VariantFileRepository, storage image.Storage, now func() time.Time, recorder audit.Recorder) *PurgeVariantCacheUseCase { + return &PurgeVariantCacheUseCase{variants, images, cache, variantCache{files, storage}, recorder, now} +} + +type cachePurgePlan struct { + filter image.VariantFileFilter + event audit.Event +} + +func (u *PurgeVariantCacheUseCase) Purge(ctx context.Context, command PurgeVariantCacheCommand) (VariantCachePurgeResult, error) { + plan, err := u.plan(command) + if err != nil { + return VariantCachePurgeResult{}, err + } + result, err := u.purgeMatching(plan.filter) + result.Scope = command.Scope + if err != nil { + recordFailure(ctx, u.recorder, plan.event, err) + return result, err + } + u.record(ctx, plan.event, result) + return result, nil +} + +func (u *PurgeVariantCacheUseCase) plan(command PurgeVariantCacheCommand) (cachePurgePlan, error) { + switch command.Scope { + case image.CacheScopeAll: + return cachePurgePlan{event: audit.NewEvent(audit.ActionVariantPurgeAll, audit.CategoryMedia)}, nil + case image.CacheScopeUnused: + return u.unusedPlan() + case image.CacheScopeVariant: + return u.variantPlan(command.Variant) + case image.CacheScopeImage: + return u.imagePlan(command.ImageID) + default: + return cachePurgePlan{}, image.ErrCachePurgeScope + } +} + +func (u *PurgeVariantCacheUseCase) unusedPlan() (cachePurgePlan, error) { + variants, err := u.variants.List() + if err != nil { + return cachePurgePlan{}, err + } + event := audit.NewEvent(audit.ActionVariantPurgeUnused, audit.CategoryMedia) + filter := image.VariantFileFilter{Unused: &image.UnusedFilter{VariantNames: enabledNames(variants), Cutoff: image.UnusedCutoff(u.now())}} + return cachePurgePlan{filter: filter, event: event}, nil +} + +func (u *PurgeVariantCacheUseCase) variantPlan(name string) (cachePurgePlan, error) { + variant, err := u.variants.Get(name) + if err != nil { + return cachePurgePlan{}, err + } + if variant.Name != name { + return cachePurgePlan{}, image.ErrVariantNotFound + } + event := audit.NewEvent(audit.ActionVariantPurge, audit.CategoryMedia) + event.Target = audit.VariantTarget(name) + return cachePurgePlan{filter: image.VariantFileFilter{VariantName: name}, event: event}, nil +} + +func (u *PurgeVariantCacheUseCase) imagePlan(id uint32) (cachePurgePlan, error) { + img, err := u.images.Get(id) + if err != nil { + return cachePurgePlan{}, err + } + event := audit.NewEvent(audit.ActionVariantPurgeImage, audit.CategoryMedia) + event.Target = imageTarget(img) + return cachePurgePlan{filter: image.VariantFileFilter{SourceName: image.StoredName(img.Path)}, event: event}, nil +} + +func (u *PurgeVariantCacheUseCase) purgeMatching(filter image.VariantFileFilter) (VariantCachePurgeResult, error) { + var result VariantCachePurgeResult + var after uint32 + for { + rows, err := u.cache.ListPage(filter, after, image.CachePurgeBatchSize) + if err != nil { + return result, err + } + for _, row := range rows { + after = row.ID + if err := u.files.purgeRow(row); err != nil { + result.Failed++ + continue + } + result.Files++ + result.Bytes += row.Size + } + if len(rows) < image.CachePurgeBatchSize { + return result, nil + } + } +} + +func (u *PurgeVariantCacheUseCase) record(ctx context.Context, event audit.Event, result VariantCachePurgeResult) { + if result.Files == 0 && result.Failed == 0 { + return + } + event.Params = cachePurgeParams(event.Action, result) + if result.Files == 0 { + failFrom(ctx, u.recorder, event, GenericFailureReason) + return + } + u.recorder.Record(ctx, event) +} + +func cachePurgeParams(action string, result VariantCachePurgeResult) map[string]any { + params := map[string]any{audit.ParamPurgedFiles: result.Files, audit.ParamPurgedBytes: result.Bytes} + if action == audit.ActionVariantPurgeUnused { + params[audit.ParamOlderThanDays] = image.UnusedVariantDays + } + if result.Failed > 0 { + params[audit.ParamFailedFiles] = result.Failed + } + return params +} diff --git a/internal/infrastructure/useCases/ServeVariant.go b/internal/infrastructure/useCases/ServeVariant.go index 2edfcb4..23e731d 100644 --- a/internal/infrastructure/useCases/ServeVariant.go +++ b/internal/infrastructure/useCases/ServeVariant.go @@ -55,6 +55,18 @@ type ServeVariantUseCase struct { maxSourcePixels int64 renderSlots chan struct{} flights singleflight.Group + tracker image.VariantUsageTracker +} + +type idleUsageTracker struct{} + +func (idleUsageTracker) RecordHit(string, string) {} +func (idleUsageTracker) RecordRejection() {} +func (idleUsageTracker) Flush() error { return nil } + +func (u *ServeVariantUseCase) WithUsageTracker(tracker image.VariantUsageTracker) *ServeVariantUseCase { + u.tracker = tracker + return u } func NewServeVariantUseCase(variants image.VariantRepository, files image.VariantFileRepository, images image.Repository, storage image.Storage, renderer image.VariantRenderer, limits ServeVariantLimits) *ServeVariantUseCase { @@ -67,10 +79,19 @@ func NewServeVariantUseCase(variants image.VariantRepository, files image.Varian maxSourceBytes: limits.MaxSourceBytes, maxSourcePixels: limits.MaxSourcePixels, renderSlots: make(chan struct{}, max(1, limits.MaxConcurrentRenders)), + tracker: idleUsageTracker{}, } } func (u *ServeVariantUseCase) Serve(request ServeVariantRequest) (ServedVariant, error) { + served, err := u.serve(request) + if errors.Is(err, image.ErrUnknownVariant) || errors.Is(err, image.ErrUnsupportedParameter) { + u.tracker.RecordRejection() + } + return served, err +} + +func (u *ServeVariantUseCase) serve(request ServeVariantRequest) (ServedVariant, error) { if parameter := unsupportedParameter(request.Parameters); parameter != "" { return ServedVariant{}, image.NewUnsupportedParameterError(parameter) } @@ -134,18 +155,26 @@ func (u *ServeVariantUseCase) serveSource(variant image.Variant, request ServeVa settings := variant.Settings() etag := image.VariantETag(settings, request.StoredName) if image.ETagMatches(request.IfNoneMatch, etag) { + u.tracker.RecordHit(variant.Name, request.StoredName) return ServedVariant{NotModified: true, ETag: etag}, nil } fingerprint := settings.Fingerprint() cached, previous, hit, err := u.lookupCached(variant.Name, request.StoredName, fingerprint, etag) if err != nil || hit { + if err == nil { + u.tracker.RecordHit(variant.Name, request.StoredName) + } return cached, err } file, err := u.generated(requestContext(request), variant, request.StoredName, fingerprint, previous) if err != nil { return ServedVariant{}, err } - return u.serveFile(file, etag) + served, err := u.serveFile(file, etag) + if err == nil { + u.tracker.RecordHit(variant.Name, request.StoredName) + } + return served, err } func requestContext(request ServeVariantRequest) context.Context { diff --git a/internal/infrastructure/useCases/variantCache.go b/internal/infrastructure/useCases/variantCache.go index 5ec0a1a..79386b1 100644 --- a/internal/infrastructure/useCases/variantCache.go +++ b/internal/infrastructure/useCases/variantCache.go @@ -1,6 +1,9 @@ package useCases -import "RenewCMS/internal/domain/image" +import ( + "RenewCMS/internal/domain/image" + "errors" +) type VariantPurgeResult struct { Files int @@ -44,7 +47,7 @@ func (c variantCache) purge(rows []image.VariantFile) (VariantPurgeResult, error } func (c variantCache) purgeRow(row image.VariantFile) error { - if err := c.storage.Delete(row.CachedName); err != nil { + if err := c.storage.Delete(row.CachedName); err != nil && !errors.Is(err, image.ErrBlobNotFound) { return err } return c.files.Delete(row.ID) diff --git a/test/image_test.go b/test/image_test.go index 1fa0a5b..38adfe8 100644 --- a/test/image_test.go +++ b/test/image_test.go @@ -5,6 +5,7 @@ import ( "RenewCMS/internal/infrastructure/useCases" "errors" "io" + "slices" "sort" "strconv" "strings" @@ -78,6 +79,8 @@ type mockImageRepo struct { deleteCalls []uint32 createCalls int onDelete func(id uint32) + + storedNameLookups int } func newMockImageRepo() *mockImageRepo { @@ -257,6 +260,20 @@ func (m *mockImageRepo) ExistsByStoredName(storedName string) (bool, error) { return false, nil } +func (m *mockImageRepo) ListByStoredNames(storedNames []string) ([]imageDomain.Image, error) { + if err := m.fail.trigger("ListByStoredNames"); err != nil { + return nil, err + } + m.storedNameLookups++ + result := make([]imageDomain.Image, 0) + for _, img := range m.sortedImages() { + if slices.Contains(storedNames, imageDomain.StoredName(img.Path)) { + result = append(result, img) + } + } + return result, nil +} + var _ imageDomain.Repository = &mockImageRepo{} func TestImageUseCases(t *testing.T) { diff --git a/test/image_variants_cache_http_test.go b/test/image_variants_cache_http_test.go new file mode 100644 index 0000000..51b6d92 --- /dev/null +++ b/test/image_variants_cache_http_test.go @@ -0,0 +1,1004 @@ +package test + +import ( + "RenewCMS/api/pkg/apperr" + domain "RenewCMS/internal/domain/image" + "errors" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" +) + +const ( + cacheStatsPath = "/v1/image-variants/cache" + cachePurgePath = "/v1/image-variants/cache/purge" +) + +func TestImageVariantsCacheGuards(t *testing.T) { + rows := []struct { + name string + method string + path string + credentials string + status int + body string + }{ + {"no cookie", http.MethodGet, cacheStatsPath, "none", 401, apperr.UnauthorizedErrorMessage}, + {"a stale token_version cookie", http.MethodGet, cacheStatsPath, "stale", 401, apperr.UnauthorizedErrorMessage}, + {"the cookie of an unverified user", http.MethodGet, cacheStatsPath, "unverified", 403, apperr.VerificationRequiredErrorMessage}, + {"the cookie of a plain user", http.MethodGet, cacheStatsPath, "plain", 403, apperr.OwnerOrAdminRequiredErrorMessage}, + {"the cookie of a plain user on purge", http.MethodPost, cachePurgePath, "plain", 403, apperr.OwnerOrAdminRequiredErrorMessage}, + {"a valid API key", http.MethodGet, cacheStatsPath, "apikey", 401, apperr.UnauthorizedErrorMessage}, + {"a valid API key on purge", http.MethodPost, cachePurgePath, "apikey", 401, apperr.UnauthorizedErrorMessage}, + } + for _, row := range rows { + t.Run("Every cache route is guarded: "+row.name, func(t *testing.T) { + env := newVariantEnv(t) + seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + seedVariantFiles(env.files, env.storage, "hero", "x", 2, 10) + callsBefore, variantCallsBefore := env.files.calls, env.variants.calls + + request := httptest.NewRequest(row.method, row.path, strings.NewReader(`{"scope": "all"}`)) + switch row.credentials { + case "stale": + request.AddCookie(env.cookie(variantStaleId, 0)) + case "unverified": + request.AddCookie(env.cookie(variantUnverifiedId, 0)) + case "plain": + request.AddCookie(env.cookie(variantPlainUserId, 0)) + case "apikey": + request.Header = bearerKeyHeader() + } + recorder := httptest.NewRecorder() + env.admin.ServeHTTP(recorder, request) + + assertStatusAndBody(t, recorder, row.status, row.body) + if env.files.calls != callsBefore || env.variants.calls != variantCallsBefore || env.files.rowCount() != 2 || len(env.storage.deletedNames()) != 0 || env.usage.callCount() != 0 || env.events() != 0 { + t.Error("an index row, blob, counter or activity event was touched") + } + }) + } + + for name, userId := range map[string]uint32{"the owner": variantOwnerId, "an admin": variantAdminId} { + t.Run("Owners and admins are both allowed: "+name, func(t *testing.T) { + env := newVariantEnv(t) + + recorder := env.send(http.MethodGet, cacheStatsPath, "", userId) + + if recorder.Code != http.StatusOK { + t.Errorf("status = %d", recorder.Code) + } + }) + } +} + +func assertNumber(t *testing.T, body map[string]any, key string, want float64) { + t.Helper() + if body[key] != want { + t.Errorf("%s = %v, want %v", key, body[key], want) + } +} + +func TestImageVariantsCacheStatistics(t *testing.T) { + t.Run("The Default fixture", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + env.seedRejectedDays(map[string]int{"2026-10-08": 30, "2026-10-05": 7, "2026-09-20": 100}) + + recorder, body := env.cacheStats() + + if recorder.Code != 200 { + t.Fatalf("status = %d %s", recorder.Code, recorder.Body.String()) + } + assertNumber(t, body, "generated_files", 1284) + assertNumber(t, body, "disk_used", 222298112) + assertNumber(t, body, "variants_defined", 6) + assertNumber(t, body, "variants_enabled", 5) + assertNumber(t, body, "variants_disabled", 1) + assertNumber(t, body, "rejected_requests", 37) + assertNumber(t, body, "rejected_window_days", 7) + if got := marshalled(t, body["unused"]); got != `{"bytes":42991616,"days":30,"files":312}` { + t.Errorf("unused = %s", got) + } + byVariant := body["by_variant"].([]any) + wantNames := []string{"hero", "card", "print", "thumb", "square", "og"} + if len(byVariant) != len(wantNames) { + t.Fatalf("by_variant = %v", byVariant) + } + for index, name := range wantNames { + entry := byVariant[index].(map[string]any) + if entry["name"] != name || entry["enabled"] != (name != "print") { + t.Errorf("by_variant[%d] = %v", index, entry) + } + } + if got := marshalled(t, byVariant[0]); got != `{"bytes":88080384,"enabled":true,"files":244,"name":"hero"}` { + t.Errorf("hero = %s", got) + } + top := body["top_images"].([]any) + if len(top) != 10 { + t.Fatalf("top_images = %d", len(top)) + } + for index, img := range cacheFixtureTopImages { + entry := top[index].(map[string]any) + if entry["id"] != float64(img.id) || entry["filename"] != img.filename || entry["url"] != "/static/uploadedImages/"+img.filename || entry["files"] != float64(img.files) || entry["bytes"] != float64(img.bytes) { + t.Errorf("top_images[%d] = %v", index, entry) + } + } + }) + + t.Run("Counts are the sum of the index rows", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + _, body := env.cacheStats() + + files, bytes := 0.0, 0.0 + for _, entry := range body["by_variant"].([]any) { + files += entry.(map[string]any)["files"].(float64) + bytes += entry.(map[string]any)["bytes"].(float64) + } + if body["generated_files"] != files || body["disk_used"] != bytes { + t.Errorf("totals = %v %v, sums = %v %v", body["generated_files"], body["disk_used"], files, bytes) + } + }) + + t.Run("A variant without a file is listed with zero", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + seedVariant(env.variants, "banner", 900, 0, domain.FitCover, domain.FormatJPEG, 80) + + _, body := env.cacheStats() + + assertNumber(t, body, "variants_defined", 7) + found := false + for _, entry := range body["by_variant"].([]any) { + if marshalled(t, entry) == `{"bytes":0,"enabled":true,"files":0,"name":"banner"}` { + found = true + } + } + if !found { + t.Errorf("by_variant = %v", body["by_variant"]) + } + }) + + t.Run("A disabled variant keeps its files in the totals", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + _, body := env.cacheStats() + + print := body["by_variant"].([]any)[2].(map[string]any) + if print["name"] != "print" || print["enabled"] != false || print["files"] != 74.0 || print["bytes"] != 22020096.0 { + t.Errorf("print = %v", print) + } + assertNumber(t, body, "generated_files", 1284) + assertNumber(t, body, "disk_used", 222298112) + if got := marshalled(t, body["unused"]); got != `{"bytes":42991616,"days":30,"files":312}` { + t.Errorf("unused = %s", got) + } + }) + + t.Run("An image deleted behind the index is not a top image", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + env.seedCacheRow("hero", "orphan-giant.jpg", 50000000, env.now) + + _, body := env.cacheStats() + + assertNumber(t, body, "generated_files", 1285) + for _, entry := range body["top_images"].([]any) { + if entry.(map[string]any)["url"] == "/static/uploadedImages/orphan-giant.jpg" { + t.Error("the orphan is listed") + } + } + if len(body["top_images"].([]any)) != 10 { + t.Errorf("top_images = %d", len(body["top_images"].([]any))) + } + }) + + t.Run("Top images are capped at ten and tie-break by id", func(t *testing.T) { + env := newVariantEnv(t) + seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + sizes := map[uint32]int64{1: 1100, 2: 1000, 3: 900, 4: 800, 5: 700, 6: 600, 7: 500, 8: 500, 9: 400, 10: 300, 11: 200} + names := map[uint32]string{7: "z7.png", 8: "a8.png"} + for id, size := range sizes { + name := names[id] + if name == "" { + name = "m" + string(rune('a'+id)) + ".png" + } + env.seedImage(id, name) + env.seedCacheRow("hero", name, size, env.now) + } + + _, body := env.cacheStats() + + top := body["top_images"].([]any) + if len(top) != 10 || top[6].(map[string]any)["id"] != 7.0 || top[7].(map[string]any)["id"] != 8.0 { + t.Errorf("top_images = %v", top) + } + }) + + seedTiedTop := func(env *variantEnv, tied map[uint32]string) { + seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + for id := uint32(1); id <= 9; id++ { + name := "m" + string(rune('a'+id)) + ".png" + env.seedImage(id, name) + env.seedCacheRow("hero", name, int64(2000-100*id), env.now) + } + for id, name := range tied { + env.seedImage(id, name) + env.seedCacheRow("hero", name, 300, env.now) + } + } + + t.Run("Exactly eleven sources tied at the cut are fetched again", func(t *testing.T) { + env := newVariantEnv(t) + seedTiedTop(env, map[uint32]string{10: "b10.png", 11: "a11.png"}) + + _, body := env.cacheStats() + + top := body["top_images"].([]any) + if len(top) != 10 || top[9].(map[string]any)["id"] != 10.0 || env.files.topCalls != 2 { + t.Errorf("top_images = %v, fetches = %d", top, env.files.topCalls) + } + }) + + t.Run("A tie across the cut widens the fetch so the lower id wins", func(t *testing.T) { + env := newVariantEnv(t) + seedTiedTop(env, map[uint32]string{10: "c10.png", 11: "b11.png", 12: "a12.png"}) + + _, body := env.cacheStats() + + top := body["top_images"].([]any) + if len(top) != 10 || top[9].(map[string]any)["id"] != 10.0 || env.files.topCalls != 2 { + t.Errorf("top_images = %v, fetches = %d", top, env.files.topCalls) + } + }) + + t.Run("An empty cache", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + for _, row := range env.files.rowsOf("hero") { + _ = env.files.Delete(row.ID) + } + for _, name := range []string{"card", "thumb", "print", "square", "og"} { + for _, row := range env.files.rowsOf(name) { + _ = env.files.Delete(row.ID) + } + } + env.seedRejectedDays(map[string]int{"2026-10-08": 37}) + + recorder, body := env.cacheStats() + + assertNumber(t, body, "generated_files", 0) + assertNumber(t, body, "disk_used", 0) + assertNumber(t, body, "variants_defined", 6) + assertNumber(t, body, "rejected_requests", 37) + if got := marshalled(t, body["unused"]); got != `{"bytes":0,"days":30,"files":0}` { + t.Errorf("unused = %s", got) + } + if !strings.Contains(recorder.Body.String(), `"top_images":[]`) { + t.Errorf("body = %s", recorder.Body.String()) + } + for _, entry := range body["by_variant"].([]any) { + if entry.(map[string]any)["files"] != 0.0 { + t.Errorf("entry = %v", entry) + } + } + }) + + t.Run("No variant defined", func(t *testing.T) { + env := newVariantEnv(t) + + recorder, body := env.cacheStats() + + if !strings.Contains(recorder.Body.String(), `"by_variant":[]`) { + t.Errorf("body = %s", recorder.Body.String()) + } + assertNumber(t, body, "variants_defined", 0) + }) + + t.Run("Unused counts only enabled variants older than the cutoff", func(t *testing.T) { + env := newVariantEnv(t) + for _, name := range []string{"hero", "card"} { + seedVariant(env.variants, name, 100, 0, domain.FitCover, domain.FormatJPEG, 80) + } + print := seedVariant(env.variants, "print", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + print.Enabled = false + env.variants.variants["print"] = print + env.seedCacheRow("hero", "recent.jpg", 10, env.now.AddDate(0, 0, -29)) + env.seedCacheRow("hero", "old.jpg", 20, env.now.AddDate(0, 0, -31)) + env.seedCacheRow("print", "disabled.jpg", 40, env.now.AddDate(0, 0, -90)) + env.seedCacheRow("card", "never-requested.jpg", 80, env.now.AddDate(0, 0, -45)) + + _, body := env.cacheStats() + + if got := marshalled(t, body["unused"]); got != `{"bytes":100,"days":30,"files":2}` { + t.Errorf("unused = %s", got) + } + }) + + t.Run("A row requested exactly 30 days ago is still in use", func(t *testing.T) { + env := newVariantEnv(t) + seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + env.seedCacheRow("hero", "edge.jpg", 10, env.now.AddDate(0, 0, -30)) + + _, body := env.cacheStats() + + if got := marshalled(t, body["unused"]); got != `{"bytes":0,"days":30,"files":0}` { + t.Errorf("unused = %s", got) + } + }) + + t.Run("The statistics are aggregated, not read per image", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + env.cacheStats() + + if env.files.totalsCalls != 1 || env.files.unusedCalls != 1 || env.files.topCalls != 1 || env.images.storedNameLookups != 1 { + t.Errorf("calls = %d totals, %d unused, %d top, %d lookups", env.files.totalsCalls, env.files.unusedCalls, env.files.topCalls, env.images.storedNameLookups) + } + }) + + t.Run("The read flushes pending tracking first", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + env.seedRejectedDays(map[string]int{"2026-10-08": 37}) + for index := 0; index < 12; index++ { + env.tracker.RecordHit("hero", "pending-"+string(rune('a'+index))) + } + for index := 0; index < 3; index++ { + env.tracker.RecordRejection() + } + + _, body := env.cacheStats() + + if env.usage.batchCount() != 1 || len(env.usage.recordedHits()) != 12 { + t.Errorf("batches = %d, hits = %d", env.usage.batchCount(), len(env.usage.recordedHits())) + } + assertNumber(t, body, "rejected_requests", 40) + }) + + t.Run("The statistics change nothing else", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + rowsBefore := env.files.rowCount() + + env.cacheStats() + + if env.storage.opens != 0 || env.renderer.renderCount() != 0 || env.files.rowCount() != rowsBefore || len(env.storage.deletedNames()) != 0 || env.events() != 0 { + t.Error("a blob was opened, a render happened, a row was deleted or an event was recorded") + } + }) + + t.Run("A failing repository answers 500", func(t *testing.T) { + env := newVariantEnv(t) + env.files.failTotals = errors.New("totals down") + + recorder, _ := env.cacheStats() + + assertStatusAndBody(t, recorder, 500, "Internal server error") + }) + + t.Run("The window is today and the six previous UTC days", func(t *testing.T) { + env := newVariantEnv(t) + env.now = time.Date(2026, time.October, 8, 23, 59, 0, 0, time.UTC) + env.seedRejectedDays(map[string]int{"2026-10-02": 5, "2026-10-01": 9}) + + _, body := env.cacheStats() + + assertNumber(t, body, "rejected_requests", 5) + }) +} + +func TestImageVariantsCacheTracking(t *testing.T) { + trackedEnv := func(t *testing.T) *variantEnv { + t.Helper() + env := newVariantEnv(t) + env.seedThumb() + env.seedPhoto() + return env + } + thumbETag := func(env *variantEnv) string { + return domain.VariantETag(env.variants.stored("thumb").Settings(), variantSourceName) + } + hitsAfterFlush := func(env *variantEnv) []domain.VariantHit { + _ = env.tracker.Flush() + return env.usage.recordedHits() + } + + t.Run("What counts as a request for a file", func(t *testing.T) { + t.Run("the public client gets the file cached", func(t *testing.T) { + env := trackedEnv(t) + env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil) + _ = env.tracker.Flush() + env.now = env.now.Add(time.Minute) + + recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil) + hits := hitsAfterFlush(env) + + if recorder.Code != 200 || len(hits) != 2 || hits[1].VariantName != "thumb" || hits[1].SourceName != variantSourceName || !hits[1].At.Equal(env.now) { + t.Errorf("status = %d, hits = %+v", recorder.Code, hits) + } + }) + + t.Run("the public client gets it and the file is generated by that request", func(t *testing.T) { + env := trackedEnv(t) + + recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil) + hits := hitsAfterFlush(env) + + if recorder.Code != 200 || env.storage.putCount() != 1 || len(hits) != 1 { + t.Errorf("status = %d, puts = %d, hits = %+v", recorder.Code, env.storage.putCount(), hits) + } + }) + + t.Run("the public client gets it with a matching If-None-Match and receives 304", func(t *testing.T) { + env := trackedEnv(t) + + recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", http.Header{"If-None-Match": []string{thumbETag(env)}}) + hits := hitsAfterFlush(env) + + if recorder.Code != 304 || len(hits) != 1 { + t.Errorf("status = %d, hits = %+v", recorder.Code, hits) + } + }) + + unrecorded := map[string]string{ + "the public client gets it with no variant": "/static/uploadedImages/" + variantSourceName, + "the public client gets it with ?v=unknown": "/static/uploadedImages/" + variantSourceName + "?v=unknown", + "the public client gets it with a disabled variant": "/static/uploadedImages/" + variantSourceName + "?v=print", + "the public client gets a missing file": "/static/uploadedImages/missing.png?v=thumb", + } + for name, url := range unrecorded { + t.Run(name, func(t *testing.T) { + env := trackedEnv(t) + print := seedVariant(env.variants, "print", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + print.Enabled = false + env.variants.variants["print"] = print + + env.get(url, nil) + hits := hitsAfterFlush(env) + + if len(hits) != 0 { + t.Errorf("hits = %+v", hits) + } + }) + } + + t.Run("an admin sends POST /v1/media/1/variants/thumb", func(t *testing.T) { + env := trackedEnv(t) + + recorder := env.sendAsAdmin(http.MethodPost, "/v1/media/1/variants/thumb", "") + hits := hitsAfterFlush(env) + + if recorder.Code != 200 || len(hits) != 0 { + t.Errorf("status = %d, hits = %+v", recorder.Code, hits) + } + }) + + t.Run("an admin sends GET /v1/image-variants/cache", func(t *testing.T) { + env := trackedEnv(t) + + env.cacheStats() + + if len(env.usage.recordedHits()) != 0 { + t.Errorf("hits = %+v", env.usage.recordedHits()) + } + }) + }) + + t.Run("A thousand hits on one file are one pending key", func(t *testing.T) { + env := trackedEnv(t) + env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil) + _ = env.tracker.Flush() + callsBefore := env.usage.callCount() + base := env.now + + for index := 1; index <= 1000; index++ { + env.now = base.Add(time.Duration(index) * time.Second) + env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil) + } + writesDuring := env.usage.callCount() - callsBefore + _ = env.tracker.Flush() + + hits := env.usage.recordedHits() + if writesDuring != 0 || env.usage.batchCount() != 2 || len(hits) != 2 || !hits[1].At.Equal(base.Add(1000*time.Second)) { + t.Errorf("writes = %d, batches = %d, hits = %+v", writesDuring, env.usage.batchCount(), hits) + } + }) + + t.Run("A request is never blocked by or failed by the tracker", func(t *testing.T) { + env := trackedEnv(t) + env.usage.fail = errUsageStoreDown + env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil) + + recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil) + + if recorder.Code != 200 || recorder.Body.Len() == 0 { + t.Errorf("status = %d, body = %d bytes", recorder.Code, recorder.Body.Len()) + } + }) +} + +func TestImageVariantsRejectionCounting(t *testing.T) { + rows := []struct { + url string + status int + counts int + }{ + {"/static/uploadedImages/" + variantSourceName + "?v=unknown", 400, 1}, + {"/static/uploadedImages/" + variantSourceName + "?v=print", 400, 1}, + {"/static/uploadedImages/" + variantSourceName + "?width=400", 400, 1}, + {"/static/uploadedImages/" + variantSourceName + "?v=Thumb", 400, 1}, + {"/static/uploadedImages/missing.png?v=thumb", 404, 0}, + {"/static/uploadedImages/" + variantSourceName + "?v=thumb", 200, 0}, + {"/static/uploadedImages/" + variantSourceName, 200, 0}, + } + setup := func(t *testing.T) *variantEnv { + t.Helper() + env := newVariantEnv(t) + env.seedThumb() + env.seedPhoto() + print := seedVariant(env.variants, "print", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + print.Enabled = false + env.variants.variants["print"] = print + return env + } + for _, row := range rows { + t.Run("What counts as rejected: "+row.url, func(t *testing.T) { + env := setup(t) + + recorder := env.get(row.url, nil) + _ = env.tracker.Flush() + + if recorder.Code != row.status || env.usage.recordedRejections() != row.counts { + t.Errorf("status = %d, rejections = %d", recorder.Code, env.usage.recordedRejections()) + } + }) + } + + t.Run("A rejection is an in-memory increment", func(t *testing.T) { + env := setup(t) + + for index := 0; index < 1000; index++ { + env.get("/static/uploadedImages/"+variantSourceName+"?v=unknown", nil) + } + writesDuring := env.usage.callCount() + _ = env.tracker.Flush() + + batch := env.usage.batches[0] + if writesDuring != 0 || len(batch.Rejections) != 1 || batch.Rejections[0].Count != 1000 || batch.Rejections[0].Day.Format(domain.RejectionDayLayout) != "2026-10-08" { + t.Errorf("writes = %d, batch = %+v", writesDuring, batch) + } + }) + + t.Run("A day boundary starts a new bucket", func(t *testing.T) { + env := setup(t) + env.now = time.Date(2026, time.October, 8, 23, 59, 59, 0, time.UTC) + env.get("/static/uploadedImages/"+variantSourceName+"?v=unknown", nil) + env.now = time.Date(2026, time.October, 9, 0, 0, 0, 0, time.UTC) + env.get("/static/uploadedImages/"+variantSourceName+"?v=unknown", nil) + + _ = env.tracker.Flush() + + if len(env.usage.batches[0].Rejections) != 2 || env.usage.days["2026-10-08"] != 1 || env.usage.days["2026-10-09"] != 1 { + t.Errorf("days = %v", env.usage.days) + } + }) + + t.Run("Old day rows are pruned by the flush", func(t *testing.T) { + env := setup(t) + env.seedRejectedDays(map[string]int{"2026-09-30": 4, "2026-10-02": 5, "2026-10-08": 1}) + env.get("/static/uploadedImages/"+variantSourceName+"?v=unknown", nil) + + _ = env.tracker.Flush() + + if _, found := env.usage.days["2026-09-30"]; found || env.usage.days["2026-10-02"] != 5 || env.usage.days["2026-10-08"] != 2 { + t.Errorf("days = %v", env.usage.days) + } + }) +} + +func TestImageVariantsCachePurge(t *testing.T) { + t.Run("Purge all removes every generated file of every variant", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + recorder := env.purgeCache(`{"scope": "all"}`) + + assertStatusAndBody(t, recorder, 200, `{"scope":"all","purged_files":1284,"purged_bytes":222298112,"failed_files":0}`) + if env.files.rowCount() != 0 || len(env.storage.deletedNames()) != 1284 || env.variants.count() != 6 { + t.Errorf("rows = %d, deleted = %d, variants = %d", env.files.rowCount(), len(env.storage.deletedNames()), env.variants.count()) + } + }) + + t.Run("Purge all includes the files of disabled variants", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + printBlobs := []string{} + for _, row := range env.files.rowsOf("print") { + printBlobs = append(printBlobs, row.CachedName) + } + + env.purgeCache(`{"scope": "all"}`) + + deleted := env.storage.deletedNames() + for _, blob := range printBlobs { + if !containsString(deleted, blob) { + t.Fatalf("%s was not deleted", blob) + } + } + if len(printBlobs) != 74 { + t.Errorf("print blobs = %d", len(printBlobs)) + } + }) + + t.Run("Purge unused removes the unused files of enabled variants only", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + recorder := env.purgeCache(`{"scope": "unused"}`) + + assertStatusAndBody(t, recorder, 200, `{"scope":"unused","purged_files":312,"purged_bytes":42991616,"failed_files":0}`) + if env.files.rowCount() != 972 || len(env.files.rowsOf("print")) != 74 { + t.Errorf("rows = %d, print = %d", env.files.rowCount(), len(env.files.rowsOf("print"))) + } + }) + + t.Run("Purge by variant removes the files of that variant only", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + recorder := env.purgeCache(`{"scope": "variant", "variant": "hero"}`) + + assertStatusAndBody(t, recorder, 200, `{"scope":"variant","purged_files":244,"purged_bytes":88080384,"failed_files":0}`) + if env.files.rowCount() != 1040 || len(env.files.rowsOf("hero")) != 0 { + t.Errorf("rows = %d", env.files.rowCount()) + } + }) + + t.Run("Purge by variant works for a disabled variant", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + recorder := env.purgeCache(`{"scope": "variant", "variant": "print"}`) + + if decodeBody(t, recorder)["purged_files"] != 74.0 { + t.Errorf("body = %s", recorder.Body.String()) + } + }) + + t.Run("Purge by image removes every variant file of that image", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + recorder := env.purgeCache(`{"scope": "image", "image_id": 1}`) + + assertStatusAndBody(t, recorder, 200, `{"scope":"image","purged_files":6,"purged_bytes":6082765,"failed_files":0}`) + remaining, _ := env.files.ListBySource("lime-plaster.jpg") + if len(remaining) != 6 || env.files.rowCount() != 1278 { + t.Errorf("lime-plaster rows = %d, rows = %d", len(remaining), env.files.rowCount()) + } + }) + + t.Run("The next public request generates a purged file again", func(t *testing.T) { + env := newVariantEnv(t) + env.seedThumb() + env.seedPhoto() + env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil) + env.purgeCache(`{"scope": "all"}`) + + recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil) + + if recorder.Code != 200 || env.storage.putCount() != 2 { + t.Errorf("status = %d, puts = %d", recorder.Code, env.storage.putCount()) + } + }) + + t.Run("Purging an empty cache answers zero", func(t *testing.T) { + env := newVariantEnv(t) + + recorder := env.purgeCache(`{"scope": "all"}`) + + assertStatusAndBody(t, recorder, 200, `{"scope":"all","purged_files":0,"purged_bytes":0,"failed_files":0}`) + }) + + t.Run("A blob already missing from storage does not stop a purge", func(t *testing.T) { + env := newVariantEnv(t) + seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + rows := seedVariantFiles(env.files, env.storage, "hero", "x", 3, 10) + env.storage.failDelete[rows[1].CachedName] = domain.ErrBlobNotFound + + recorder := env.purgeCache(`{"scope": "all"}`) + + assertStatusAndBody(t, recorder, 200, `{"scope":"all","purged_files":3,"purged_bytes":30,"failed_files":0}`) + if env.files.rowCount() != 0 { + t.Errorf("rows = %d", env.files.rowCount()) + } + }) + + invalid := []struct { + body string + status int + message string + }{ + {`{"scope": "everything"}`, 422, apperr.CachePurgeScopeErrorMessage}, + {`{"scope": "ALL"}`, 422, apperr.CachePurgeScopeErrorMessage}, + {`{}`, 422, apperr.CachePurgeScopeErrorMessage}, + {`{"scope": "variant"}`, 422, apperr.CachePurgeTargetErrorMessage}, + {`{"scope": "variant", "variant": ""}`, 422, apperr.CachePurgeTargetErrorMessage}, + {`{"scope": "image"}`, 422, apperr.CachePurgeTargetErrorMessage}, + {`{"scope": "image", "image_id": 0}`, 422, apperr.CachePurgeTargetErrorMessage}, + {`{"scope": "image", "image_id": -1}`, 422, apperr.CachePurgeTargetErrorMessage}, + {`{"scope": "variant", "variant": "missing"}`, 404, apperr.VariantNotFoundErrorMessage}, + {`{"scope": "image", "image_id": 9999}`, 404, "Image not found"}, + {`not json`, 400, apperr.BodyErrorMessage}, + } + for _, row := range invalid { + t.Run("A request that cannot be served: "+row.body, func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + recorder := env.purgeCache(row.body) + + assertStatusAndBody(t, recorder, row.status, row.message) + if env.files.rowCount() != 1284 || len(env.storage.deletedNames()) != 0 || env.events() != 0 { + t.Error("a blob or row was deleted or an event was recorded") + } + }) + } + + ignored := map[string]string{"all": `"purged_files":1284`, "unused": `"purged_files":312`} + for scope, want := range ignored { + t.Run("A target given for a scope that ignores it is ignored: "+scope, func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + recorder := env.purgeCache(`{"scope": "` + scope + `", "variant": "hero", "image_id": 1}`) + + if recorder.Code != 200 || !strings.Contains(recorder.Body.String(), want) { + t.Errorf("response = %d %s", recorder.Code, recorder.Body.String()) + } + }) + } +} + +func containsString(values []string, wanted string) bool { + for _, value := range values { + if value == wanted { + return true + } + } + return false +} + +func TestImageVariantsCachePartialPurge(t *testing.T) { + failingEnv := func(t *testing.T) (*variantEnv, int64) { + t.Helper() + env := newVariantEnv(t) + env.seedDefaultCache() + page, _ := env.files.ListPage(domain.VariantFileFilter{}, 0, 2000) + var keptBytes int64 + for index, row := range page { + if index < 644 { + env.storage.failDelete[row.CachedName] = errors.New("storage down") + continue + } + keptBytes += row.Size + } + return env, keptBytes + } + + t.Run("A partial purge answers 200 with the failed count", func(t *testing.T) { + env, purgedBytes := failingEnv(t) + + recorder := env.purgeCache(`{"scope": "all"}`) + + want := `{"scope":"all","purged_files":640,"purged_bytes":` + itoa64(purgedBytes) + `,"failed_files":644}` + assertStatusAndBody(t, recorder, 200, want) + if env.files.rowCount() != 644 { + t.Errorf("rows = %d", env.files.rowCount()) + } + }) + + t.Run("Retrying removes the rest", func(t *testing.T) { + env, _ := failingEnv(t) + env.purgeCache(`{"scope": "all"}`) + env.storage.failDelete = map[string]error{} + + recorder := env.purgeCache(`{"scope": "all"}`) + + body := decodeBody(t, recorder) + if body["purged_files"] != 644.0 || body["failed_files"] != 0.0 { + t.Errorf("body = %v", body) + } + }) + + t.Run("A failure to list the rows answers 500", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + env.files.failPage = errors.New("list down") + + recorder := env.purgeCache(`{"scope": "all"}`) + + assertStatusAndBody(t, recorder, 500, "Internal server error") + }) + + t.Run("Rows are processed in bounded batches", func(t *testing.T) { + env := newVariantEnv(t) + seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + for index := 0; index < 5000; index++ { + env.files.Save(domain.VariantFile{VariantName: "hero", SourceName: "bulk-" + itoa64(int64(index)), CachedName: "bulk-" + itoa64(int64(index)), Size: 1}) + } + + recorder := env.purgeCache(`{"scope": "all"}`) + + if recorder.Code != 200 || decodeBody(t, recorder)["purged_files"] != 5000.0 { + t.Fatalf("response = %d %s", recorder.Code, recorder.Body.String()) + } + if len(env.files.pageLimits) < 10 { + t.Errorf("pages = %d", len(env.files.pageLimits)) + } + for _, limit := range env.files.pageLimits { + if limit > 500 { + t.Errorf("page limit = %d", limit) + } + } + }) +} + +func itoa64(value int64) string { + return strconv.FormatInt(value, 10) +} + +func TestImageVariantsCachePurgeActivity(t *testing.T) { + scopes := []struct { + body string + action string + target string + params string + }{ + {`{"scope": "all"}`, "media.variant_purge_all", `{"type":"","id":"","name":""}`, `{"purged_bytes":222298112,"purged_files":1284}`}, + {`{"scope": "unused"}`, "media.variant_purge_unused", `{"type":"","id":"","name":""}`, `{"older_than_days":30,"purged_bytes":42991616,"purged_files":312}`}, + {`{"scope": "variant", "variant": "hero"}`, "media.variant_purge", `{"type":"image_variant","id":"hero","name":"hero"}`, `{"purged_bytes":88080384,"purged_files":244}`}, + {`{"scope": "image", "image_id": 1}`, "media.variant_purge_image", `{"type":"image","id":"1","name":"north-wall.jpg"}`, `{"purged_bytes":6082765,"purged_files":6}`}, + } + for _, scope := range scopes { + t.Run("A purge that removes files is recorded: "+scope.action, func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + env.purgeCache(scope.body) + + event := onlyEventOf(t, env, scope.action) + if event.Category != "media" || event.Outcome != "success" || marshalled(t, event.Target) != scope.target || marshalled(t, event.Params) != scope.params { + t.Errorf("event = %+v, target = %s, params = %s", event, marshalled(t, event.Target), marshalled(t, event.Params)) + } + }) + } + + t.Run("A partial failure is a success that names the failed count", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + page, _ := env.files.ListPage(domain.VariantFileFilter{}, 0, 2000) + var keptBytes int64 + for index, row := range page { + if index < 644 { + env.storage.failDelete[row.CachedName] = errors.New("storage down") + continue + } + keptBytes += row.Size + } + + env.purgeCache(`{"scope": "all"}`) + + event := onlyEventOf(t, env, "media.variant_purge_all") + want := `{"failed_files":644,"purged_bytes":` + itoa64(keptBytes) + `,"purged_files":640}` + if event.Outcome != "success" || marshalled(t, event.Params) != want { + t.Errorf("event = %+v, params = %s", event, marshalled(t, event.Params)) + } + }) + + t.Run("Every file failing is a failed event with a plain reason", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + page, _ := env.files.ListPage(domain.VariantFileFilter{}, 0, 2000) + for _, row := range page { + env.storage.failDelete[row.CachedName] = errors.New("storage down") + } + + recorder := env.purgeCache(`{"scope": "all"}`) + + body := decodeBody(t, recorder) + event := onlyEventOf(t, env, "media.variant_purge_all") + if recorder.Code != 200 || body["purged_files"] != 0.0 || body["failed_files"] != 1284.0 || event.Outcome != "failed" || event.FailureReason == nil || *event.FailureReason != "Something went wrong on our side. Please try again." { + t.Errorf("response = %d %v, event = %+v", recorder.Code, body, event) + } + }) + + nothing := map[string]func(*variantEnv) string{ + "the owner purges all on an empty cache": func(env *variantEnv) string { return `{"scope": "all"}` }, + "the owner purges unused when nothing is unused": func(env *variantEnv) string { + env.seedDefaultCache() + for _, row := range env.files.rowsOf("hero") { + env.files.setReference(row.ID, env.now) + } + for _, row := range env.files.rowsOf("thumb") { + env.files.setReference(row.ID, env.now) + } + return `{"scope": "unused"}` + }, + "the owner purges a variant that has no file": func(env *variantEnv) string { + seedVariant(env.variants, "banner", 100, 0, domain.FitCover, domain.FormatJPEG, 80) + return `{"scope": "variant", "variant": "banner"}` + }, + "the owner purges an image that has no file": func(env *variantEnv) string { + env.seedImage(50, "nofile.png") + return `{"scope": "image", "image_id": 50}` + }, + "the owner sends an unknown scope": func(env *variantEnv) string { return `{"scope": "everything"}` }, + "the owner purges an unknown variant": func(env *variantEnv) string { return `{"scope": "variant", "variant": "ghost"}` }, + "the owner purges an unknown image": func(env *variantEnv) string { return `{"scope": "image", "image_id": 777}` }, + } + for name, prepare := range nothing { + t.Run("Nothing removed, nothing recorded: "+name, func(t *testing.T) { + env := newVariantEnv(t) + body := prepare(env) + + env.purgeCache(body) + + if env.events() != 0 { + t.Errorf("events = %d", env.events()) + } + }) + } + + t.Run("Nothing removed, nothing recorded: the owner sends GET /v1/image-variants/cache", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + + env.send(http.MethodGet, cacheStatsPath, "", variantOwnerId) + + if env.events() != 0 { + t.Errorf("events = %d", env.events()) + } + }) + + t.Run("A failing audit write never changes the response", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + env.audit.probe.appendErr = errors.New("audit store down") + + recorder := env.purgeCache(`{"scope": "all"}`) + + assertStatusAndBody(t, recorder, 200, `{"scope":"all","purged_files":1284,"purged_bytes":222298112,"failed_files":0}`) + if env.files.rowCount() != 0 { + t.Errorf("rows = %d", env.files.rowCount()) + } + }) + + t.Run("The events read back through the existing activity API", func(t *testing.T) { + env := newVariantEnv(t) + env.seedDefaultCache() + for _, body := range []string{`{"scope": "image", "image_id": 1}`, `{"scope": "variant", "variant": "hero"}`, `{"scope": "unused"}`, `{"scope": "all"}`} { + env.purgeCache(body) + } + + recorder := env.send(http.MethodGet, "/v1/activity?category=media", "", variantOwnerId) + + items := decodeBody(t, recorder)["items"].([]any) + actions := []string{} + for _, item := range items { + actions = append(actions, item.(map[string]any)["action"].(string)) + } + want := []string{"media.variant_purge_all", "media.variant_purge_unused", "media.variant_purge", "media.variant_purge_image"} + if recorder.Code != 200 || strings.Join(actions, ",") != strings.Join(want, ",") { + t.Errorf("status = %d, actions = %v", recorder.Code, actions) + } + }) +} diff --git a/test/image_variants_cache_hygiene_test.go b/test/image_variants_cache_hygiene_test.go new file mode 100644 index 0000000..65966f9 --- /dev/null +++ b/test/image_variants_cache_hygiene_test.go @@ -0,0 +1,86 @@ +package test + +import ( + "go/parser" + "go/token" + "strings" + "testing" +) + +func TestImageVariantCacheLayering(t *testing.T) { + t.Run("The cache use cases and the tracker import no persistence, mailer, models, gorm, net/http, chi or api package", func(t *testing.T) { + files := []string{ + "internal/infrastructure/useCases/GetVariantCacheStats.go", + "internal/infrastructure/useCases/PurgeVariantCache.go", + "internal/infrastructure/usage/variantUsageTracker.go", + } + for _, file := range files { + for _, path := range importsOf(t, file) { + for _, fragment := range []string{"persistence", "mailer", "models", "gorm", "net/http", "go-chi", "RenewCMS/api"} { + if strings.Contains(path, fragment) { + t.Errorf("%s imports %s", file, path) + } + } + } + } + }) + + t.Run("The cache handler imports no persistence, mailer, models or gorm package", func(t *testing.T) { + for _, path := range importsOf(t, "api/handlers/imagevariant/cacheHandler.go") { + for _, fragment := range []string{"persistence", "mailer", "models", "gorm"} { + if strings.Contains(path, fragment) { + t.Errorf("imports %s", path) + } + } + } + }) + + t.Run("The tracker, usage and cache ports are declared in internal/domain/image", func(t *testing.T) { + declared := readRepositoryFile(t, "internal/domain/image/variantRepositoryInterface.go") + + for _, port := range []string{"type VariantUsageTracker interface", "type VariantUsageRepository interface", "type VariantCacheRepository interface"} { + if !strings.Contains(declared, port) { + t.Errorf("%s is not declared", port) + } + } + }) + + t.Run("The new models are registered in migrate.go", func(t *testing.T) { + migrate := readRepositoryFile(t, "internal/infrastructure/persistence/database/migrate.go") + + if !strings.Contains(migrate, "&models.ImageVariantRejectionDay{}") { + t.Error("the rejection day model is missing from AutoMigrate") + } + }) + + t.Run("Only the composition root constructs the tracker", func(t *testing.T) { + for _, file := range []string{"api/router.go", "cmd/server/main.go", "api/handlers/staticimages/staticImagesHandler.go", "api/handlers/imagevariant/cacheHandler.go"} { + if strings.Contains(readRepositoryFile(t, file), "NewTracker") { + t.Errorf("%s constructs the tracker", file) + } + } + }) + + t.Run("The new files carry no comment", func(t *testing.T) { + for _, relative := range []string{ + "internal/domain/image/variantCache.go", + "internal/infrastructure/usage/variantUsageTracker.go", + "internal/infrastructure/useCases/GetVariantCacheStats.go", + "internal/infrastructure/useCases/PurgeVariantCache.go", + "internal/infrastructure/persistence/repositories/imageVariantUsageRepository.go", + "api/handlers/imagevariant/cacheHandler.go", + } { + parsed, err := parser.ParseFile(token.NewFileSet(), "../"+relative, nil, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + for _, group := range parsed.Comments { + for _, comment := range group.List { + if !strings.HasPrefix(comment.Text, "//go:") { + t.Errorf("%s has comment %q", relative, comment.Text) + } + } + } + } + }) +} diff --git a/test/image_variants_cache_support_test.go b/test/image_variants_cache_support_test.go new file mode 100644 index 0000000..f2b154c --- /dev/null +++ b/test/image_variants_cache_support_test.go @@ -0,0 +1,233 @@ +package test + +import ( + domain "RenewCMS/internal/domain/image" + "errors" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "time" +) + +var errUsageStoreDown = errors.New("usage store down") + +type mockVariantUsageRepo struct { + mutex sync.Mutex + batches []domain.VariantUsageBatch + days map[string]int + fail error + calls int +} + +var _ domain.VariantUsageRepository = &mockVariantUsageRepo{} + +func newMockVariantUsageRepo() *mockVariantUsageRepo { + return &mockVariantUsageRepo{days: map[string]int{}} +} + +func (m *mockVariantUsageRepo) Record(batch domain.VariantUsageBatch) error { + m.mutex.Lock() + defer m.mutex.Unlock() + m.calls++ + if m.fail != nil { + return m.fail + } + m.batches = append(m.batches, batch) + for _, rejection := range batch.Rejections { + m.days[rejection.Day.Format(domain.RejectionDayLayout)] += rejection.Count + } + if !batch.PruneBefore.IsZero() { + for day := range m.days { + if day < batch.PruneBefore.Format(domain.RejectionDayLayout) { + delete(m.days, day) + } + } + } + return nil +} + +func (m *mockVariantUsageRepo) RejectedSince(day time.Time) (int, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + total := 0 + for stored, count := range m.days { + if stored >= day.Format(domain.RejectionDayLayout) { + total += count + } + } + return total, nil +} + +func (m *mockVariantUsageRepo) recordedHits() []domain.VariantHit { + m.mutex.Lock() + defer m.mutex.Unlock() + hits := []domain.VariantHit{} + for _, batch := range m.batches { + hits = append(hits, batch.Hits...) + } + return hits +} + +func (m *mockVariantUsageRepo) recordedRejections() int { + m.mutex.Lock() + defer m.mutex.Unlock() + total := 0 + for _, batch := range m.batches { + for _, rejection := range batch.Rejections { + total += rejection.Count + } + } + return total +} + +func (m *mockVariantUsageRepo) batchCount() int { + m.mutex.Lock() + defer m.mutex.Unlock() + return len(m.batches) +} + +func (m *mockVariantUsageRepo) callCount() int { + m.mutex.Lock() + defer m.mutex.Unlock() + return m.calls +} + +type cacheFixtureImage struct { + id uint32 + filename string + files int + bytes int64 +} + +var cacheFixtureTopImages = []cacheFixtureImage{ + {1, "north-wall.jpg", 6, 6082765}, + {2, "lime-plaster.jpg", 6, 5138022}, + {3, "kitchen-after.webp", 5, 4613734}, + {4, "kitchen-before.webp", 5, 4089446}, + {5, "floorboards.jpg", 5, 3774873}, + {6, "IMG_4471.jpg", 4, 3250585}, + {7, "stone-wall-detail.jpg", 4, 2936012}, + {8, "bathroom-tiles.png", 4, 2726297}, + {9, "roof-lantern.jpg", 3, 2411724}, + {10, "chimney-stack.jpg", 3, 2202009}, +} + +type cacheFixtureVariant struct { + name string + enabled bool + files int + bytes int64 +} + +var cacheFixtureVariants = []cacheFixtureVariant{ + {"hero", true, 244, 88080384}, + {"card", true, 318, 60817408}, + {"thumb", true, 412, 22020096}, + {"print", false, 74, 22020096}, + {"square", true, 140, 19922944}, + {"og", true, 96, 9437184}, +} + +const ( + cacheFixtureUnusedHeroRows = 200 + cacheFixtureUnusedHeroBytes = 40000000 + cacheFixtureUnusedThumbRows = 112 + cacheFixtureUnusedThumbSize = 2991616 +) + +func splitBytes(total int64, parts int) []int64 { + sizes := make([]int64, parts) + for index := range sizes { + sizes[index] = total / int64(parts) + if int64(index) < total%int64(parts) { + sizes[index]++ + } + } + return sizes +} + +func (e *variantEnv) seedCacheRow(variantName string, sourceName string, size int64, reference time.Time) domain.VariantFile { + cached := "fixture-" + variantName + "-" + sourceName + ".jpeg" + e.storage.seed(cached, []byte("generated")) + saved, _ := e.files.Save(domain.VariantFile{VariantName: variantName, SourceName: sourceName, CachedName: cached, ContentType: "image/jpeg", Size: size, Fingerprint: "fixture"}) + e.files.setReference(saved.ID, reference) + return saved +} + +func (e *variantEnv) seedDefaultCache() { + recent := e.now.AddDate(0, 0, -5) + stale := e.now.AddDate(0, 0, -31) + for _, variant := range cacheFixtureVariants { + seeded := seedVariant(e.variants, variant.name, 400, 300, domain.FitCover, domain.FormatJPEG, 80) + seeded.Enabled = variant.enabled + e.variants.variants[variant.name] = seeded + } + topBytes := map[string]int64{} + topFiles := map[string]int{} + for _, img := range cacheFixtureTopImages { + e.seedImage(img.id, img.filename) + for index, size := range splitBytes(img.bytes, img.files) { + variant := cacheFixtureVariants[index%len(cacheFixtureVariants)].name + e.seedCacheRow(variant, img.filename, size, recent) + topBytes[variant] += size + topFiles[variant]++ + } + } + for _, variant := range cacheFixtureVariants { + fillers := variant.files - topFiles[variant.name] + remaining := variant.bytes - topBytes[variant.name] + sizes := e.fillerSizes(variant.name, fillers, remaining) + for index, size := range sizes { + reference := recent + if e.isFixtureStale(variant.name, index) { + reference = stale + } + e.seedCacheRow(variant.name, "filler-"+variant.name+"-"+strconv.Itoa(index), size, reference) + } + } +} + +func (e *variantEnv) isFixtureStale(variantName string, index int) bool { + return (variantName == "hero" && index < cacheFixtureUnusedHeroRows) || (variantName == "thumb" && index < cacheFixtureUnusedThumbRows) +} + +func (e *variantEnv) fillerSizes(variantName string, fillers int, remaining int64) []int64 { + switch variantName { + case "hero": + return append(splitBytes(cacheFixtureUnusedHeroBytes, cacheFixtureUnusedHeroRows), splitBytes(remaining-cacheFixtureUnusedHeroBytes, fillers-cacheFixtureUnusedHeroRows)...) + case "thumb": + return append(splitBytes(cacheFixtureUnusedThumbSize, cacheFixtureUnusedThumbRows), splitBytes(remaining-cacheFixtureUnusedThumbSize, fillers-cacheFixtureUnusedThumbRows)...) + default: + return splitBytes(remaining, fillers) + } +} + +func (e *variantEnv) seedRejectedDays(days map[string]int) { + for day, count := range days { + e.usage.days[day] = count + } +} + +func (e *variantEnv) cacheStats() (*httptest.ResponseRecorder, map[string]any) { + e.t.Helper() + recorder := e.sendAsAdmin(http.MethodGet, "/v1/image-variants/cache", "") + if recorder.Code != http.StatusOK { + return recorder, nil + } + return recorder, decodeBody(e.t, recorder) +} + +func (e *variantEnv) purgeCache(body string) *httptest.ResponseRecorder { + e.t.Helper() + return e.send(http.MethodPost, "/v1/image-variants/cache/purge", body, variantOwnerId) +} + +func (e *variantEnv) recentRejectedFixture() { + e.seedRejectedDays(map[string]int{"2026-10-08": 30, "2026-10-05": 7, "2026-09-20": 100}) +} + +func compactJSON(body *httptest.ResponseRecorder) string { + return strings.TrimSpace(body.Body.String()) +} diff --git a/test/image_variants_env_test.go b/test/image_variants_env_test.go index af07300..7e1572c 100644 --- a/test/image_variants_env_test.go +++ b/test/image_variants_env_test.go @@ -9,6 +9,7 @@ import ( "RenewCMS/internal/domain/user" "RenewCMS/internal/infrastructure/config" "RenewCMS/internal/infrastructure/hashing" + "RenewCMS/internal/infrastructure/usage" "RenewCMS/internal/infrastructure/useCases" "bytes" "encoding/json" @@ -50,6 +51,8 @@ type variantEnv struct { logs *bytes.Buffer token *jwtauth.JWTAuth now time.Time + usage *mockVariantUsageRepo + tracker *usage.Tracker admin http.Handler public http.Handler } @@ -81,7 +84,9 @@ func newVariantEnvLimits(t *testing.T, renderer *countingRenderer, limits useCas logs: &bytes.Buffer{}, token: jwtauth.New("HS256", []byte("test-secret"), nil), now: time.Date(2026, time.October, 8, 9, 0, 0, 0, time.UTC), + usage: newMockVariantUsageRepo(), } + env.tracker = usage.NewTracker(env.usage, func() time.Time { return env.now }) env.users.users[variantOwnerId] = user.User{ID: variantOwnerId, Username: "marta", DisplayName: "Marta Alves", IsVerified: true, Role: user.RoleOwner} env.users.users[variantAdminId] = user.User{ID: variantAdminId, Username: "dev", IsVerified: true, Role: user.RoleAdmin} env.users.users[variantPlainUserId] = user.User{ID: variantPlainUserId, Username: "pat", IsVerified: true} @@ -103,7 +108,7 @@ func newVariantEnvLimits(t *testing.T, renderer *countingRenderer, limits useCas func (e *variantEnv) container(limits useCases.ServeVariantLimits) *api.UseCases { clock := func() time.Time { return e.now } recorder := e.audit.recorder - serve := useCases.NewServeVariantUseCase(e.variants, e.files, e.images, e.storage, e.renderer, limits) + serve := useCases.NewServeVariantUseCase(e.variants, e.files, e.images, e.storage, e.renderer, limits).WithUsageTracker(e.tracker) return &api.UseCases{ Token: e.token, GetUserUseCase: useCases.NewGetUserUseCase(e.users), @@ -117,10 +122,15 @@ func (e *variantEnv) container(limits useCases.ServeVariantLimits) *api.UseCases DeleteVariantUseCase: useCases.NewDeleteVariantUseCase(e.variants, e.files, e.storage, recorder), PurgeVariantUseCase: useCases.NewPurgeVariantUseCase(e.variants, e.files, e.storage, recorder), ServeVariantUseCase: serve, - CreateImageUseCase: useCases.NewCreateImageUseCase(e.images, e.storage, recorder), - ListMediaUseCase: useCases.NewListMediaUseCase(e.images, e.articles, e.users, e.files), - RenameMediaUseCase: useCases.NewRenameMediaUseCase(e.images, recorder), - GetMediaUsageUseCase: useCases.NewGetMediaUsageUseCase(e.images, e.articles), + + GetVariantCacheStatsUseCase: useCases.NewGetVariantCacheStatsUseCase(e.variants, e.files, e.files, e.images, e.usage, e.tracker, clock), + PurgeVariantCacheUseCase: useCases.NewPurgeVariantCacheUseCase(e.variants, e.images, e.files, e.files, e.storage, clock, recorder), + VariantUsageTracker: e.tracker, + + CreateImageUseCase: useCases.NewCreateImageUseCase(e.images, e.storage, recorder), + ListMediaUseCase: useCases.NewListMediaUseCase(e.images, e.articles, e.users, e.files), + RenameMediaUseCase: useCases.NewRenameMediaUseCase(e.images, recorder), + GetMediaUsageUseCase: useCases.NewGetMediaUsageUseCase(e.images, e.articles), ListImageVariantsUseCase: useCases.NewListImageVariantsUseCase(e.images, e.variants, e.files, e.renderer), GenerateImageVariantUseCase: useCases.NewGenerateImageVariantUseCase(e.images, e.variants, e.files, e.renderer, serve), diff --git a/test/image_variants_support_test.go b/test/image_variants_support_test.go index ed41a26..5913498 100644 --- a/test/image_variants_support_test.go +++ b/test/image_variants_support_test.go @@ -8,6 +8,7 @@ import ( "errors" "io" "log" + "slices" "sort" "strconv" "strings" @@ -117,12 +118,88 @@ type mockVariantFileRepo struct { calls int totalsBySourceCalls int + + references map[uint32]time.Time + failTotals error + failPage error + totalsCalls int + unusedCalls int + topCalls int + pageLimits []int } var _ domain.VariantFileRepository = &mockVariantFileRepo{} +var _ domain.VariantCacheRepository = &mockVariantFileRepo{} func newMockVariantFileRepo() *mockVariantFileRepo { - return &mockVariantFileRepo{rows: map[uint32]domain.VariantFile{}} + return &mockVariantFileRepo{rows: map[uint32]domain.VariantFile{}, references: map[uint32]time.Time{}} +} + +func (m *mockVariantFileRepo) setReference(id uint32, at time.Time) { + m.mutex.Lock() + defer m.mutex.Unlock() + m.references[id] = at +} + +func (m *mockVariantFileRepo) isUnused(row domain.VariantFile, filter domain.UnusedFilter) bool { + return slices.Contains(filter.VariantNames, row.VariantName) && m.references[row.ID].Before(filter.Cutoff) +} + +func (m *mockVariantFileRepo) UnusedTotals(filter domain.UnusedFilter) (domain.VariantFileTotals, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + m.unusedCalls++ + var totals domain.VariantFileTotals + for _, row := range m.rows { + if m.isUnused(row, filter) { + totals.Files++ + totals.Bytes += row.Size + } + } + return totals, nil +} + +func (m *mockVariantFileRepo) TopSources(limit int) ([]domain.SourceTotals, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + m.topCalls++ + bySource := map[string]domain.SourceTotals{} + for _, row := range m.rows { + current := bySource[row.SourceName] + bySource[row.SourceName] = domain.SourceTotals{SourceName: row.SourceName, Files: current.Files + 1, Bytes: current.Bytes + row.Size} + } + sources := make([]domain.SourceTotals, 0, len(bySource)) + for _, source := range bySource { + sources = append(sources, source) + } + sort.Slice(sources, func(i, j int) bool { + if sources[i].Bytes != sources[j].Bytes { + return sources[i].Bytes > sources[j].Bytes + } + return sources[i].SourceName < sources[j].SourceName + }) + return sources[:min(limit, len(sources))], nil +} + +func (m *mockVariantFileRepo) ListPage(filter domain.VariantFileFilter, afterID uint32, limit int) ([]domain.VariantFile, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + m.pageLimits = append(m.pageLimits, limit) + if m.failPage != nil { + return nil, m.failPage + } + page := []domain.VariantFile{} + for _, row := range m.rows { + if row.ID <= afterID || (filter.VariantName != "" && row.VariantName != filter.VariantName) || (filter.SourceName != "" && row.SourceName != filter.SourceName) { + continue + } + if filter.Unused != nil && !m.isUnused(row, *filter.Unused) { + continue + } + page = append(page, row) + } + sort.Slice(page, func(i, j int) bool { return page[i].ID < page[j].ID }) + return page[:min(limit, len(page))], nil } func (m *mockVariantFileRepo) Find(variantName string, sourceName string) (domain.VariantFile, bool, error) { @@ -190,6 +267,10 @@ func (m *mockVariantFileRepo) Totals() (map[string]domain.VariantFileTotals, err m.mutex.Lock() defer m.mutex.Unlock() m.calls++ + m.totalsCalls++ + if m.failTotals != nil { + return nil, m.failTotals + } totals := map[string]domain.VariantFileTotals{} for _, row := range m.rows { current := totals[row.VariantName] diff --git a/test/image_variants_usage_test.go b/test/image_variants_usage_test.go new file mode 100644 index 0000000..69d85dd --- /dev/null +++ b/test/image_variants_usage_test.go @@ -0,0 +1,443 @@ +package test + +import ( + domain "RenewCMS/internal/domain/image" + entity "RenewCMS/internal/infrastructure/persistence/models" + persistence "RenewCMS/internal/infrastructure/persistence/repositories" + "RenewCMS/internal/infrastructure/usage" + "context" + "strconv" + "sync" + "testing" + "time" + + "gorm.io/gorm" +) + +func newUsageTracker(repo domain.VariantUsageRepository, clock *time.Time) *usage.Tracker { + return usage.NewTracker(repo, func() time.Time { return *clock }) +} + +func TestVariantUsageTracker(t *testing.T) { + start := time.Date(2026, time.October, 8, 10, 0, 0, 0, time.UTC) + + t.Run("The flush writes one batch for many files", func(t *testing.T) { + repo, clock := newMockVariantUsageRepo(), start + tracker := newUsageTracker(repo, &clock) + for index := 0; index < 40; index++ { + tracker.RecordHit("thumb", "file-"+strconv.Itoa(index)+".png") + } + + if err := tracker.Flush(); err != nil { + t.Fatal(err) + } + + if repo.batchCount() != 1 || repo.callCount() != 1 || len(repo.recordedHits()) != 40 { + t.Errorf("batches = %d, calls = %d, hits = %d", repo.batchCount(), repo.callCount(), len(repo.recordedHits())) + } + }) + + t.Run("Nothing pending means no write", func(t *testing.T) { + repo, clock := newMockVariantUsageRepo(), start + tracker := newUsageTracker(repo, &clock) + + _ = tracker.Flush() + tracker.RecordHit("thumb", "a.png") + _ = tracker.Flush() + _ = tracker.Flush() + + if repo.callCount() != 1 { + t.Errorf("calls = %d", repo.callCount()) + } + }) + + t.Run("A thousand hits on one file are one update with the time of the last hit", func(t *testing.T) { + repo, clock := newMockVariantUsageRepo(), start + tracker := newUsageTracker(repo, &clock) + for index := 0; index < 1000; index++ { + clock = start.Add(time.Duration(index) * time.Second) + tracker.RecordHit("thumb", "a.png") + } + + _ = tracker.Flush() + + hits := repo.recordedHits() + if repo.callCount() != 1 || len(hits) != 1 || !hits[0].At.Equal(start.Add(999*time.Second)) { + t.Errorf("calls = %d, hits = %+v", repo.callCount(), hits) + } + }) + + t.Run("A failed flush keeps the pending hits", func(t *testing.T) { + repo, clock := newMockVariantUsageRepo(), start + tracker := newUsageTracker(repo, &clock) + repo.fail = errUsageStoreDown + tracker.RecordHit("thumb", "a.png") + if err := tracker.Flush(); err == nil { + t.Fatal("the failure was swallowed") + } + clock = start.Add(time.Minute) + tracker.RecordHit("thumb", "a.png") + tracker.RecordHit("hero", "b.png") + repo.fail = nil + + _ = tracker.Flush() + + hits := repo.recordedHits() + if repo.batchCount() != 1 || len(hits) != 2 { + t.Fatalf("batches = %d, hits = %+v", repo.batchCount(), hits) + } + for _, hit := range hits { + if !hit.At.Equal(clock) { + t.Errorf("hit = %+v", hit) + } + } + }) + + t.Run("A failed flush keeps the pending rejections", func(t *testing.T) { + repo, clock := newMockVariantUsageRepo(), start + tracker := newUsageTracker(repo, &clock) + repo.fail = errUsageStoreDown + tracker.RecordRejection() + tracker.RecordRejection() + _ = tracker.Flush() + tracker.RecordRejection() + repo.fail = nil + + _ = tracker.Flush() + _ = tracker.Flush() + + if repo.recordedRejections() != 3 || repo.batchCount() != 1 || repo.days["2026-10-08"] != 3 { + t.Errorf("rejections = %d, batches = %d, days = %v", repo.recordedRejections(), repo.batchCount(), repo.days) + } + }) + + t.Run("The flush loop and the size trigger", func(t *testing.T) { + repo, clock := newMockVariantUsageRepo(), start + tracker := usage.NewTrackerWith(repo, func() time.Time { return clock }, time.Hour, 500) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + done := make(chan struct{}) + go func() { + defer close(done) + tracker.Run(ctx) + }() + + for index := 0; index < 499; index++ { + tracker.RecordHit("thumb", "file-"+strconv.Itoa(index)+".png") + } + time.Sleep(50 * time.Millisecond) + if repo.callCount() != 0 { + t.Fatal("flushed before the size trigger") + } + tracker.RecordHit("thumb", "file-499.png") + waitUntilUsage(t, func() bool { return repo.callCount() == 1 }) + + tracker.RecordHit("thumb", "late.png") + stop() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("the loop did not stop") + } + if repo.callCount() != 2 { + t.Errorf("calls after shutdown = %d", repo.callCount()) + } + }) + + t.Run("The defaults are a thirty second interval and a 500 key trigger", func(t *testing.T) { + if usage.FlushInterval() != 30*time.Second || usage.FlushKeyLimit != 500 { + t.Errorf("interval = %v, limit = %d", usage.FlushInterval(), usage.FlushKeyLimit) + } + }) + + t.Run("The same key repeated never reaches the size trigger", func(t *testing.T) { + repo, clock := newMockVariantUsageRepo(), start + tracker := usage.NewTrackerWith(repo, func() time.Time { return clock }, time.Hour, 2) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go tracker.Run(ctx) + + for index := 0; index < 5; index++ { + tracker.RecordHit("thumb", "a.png") + } + time.Sleep(50 * time.Millisecond) + + if repo.callCount() != 0 { + t.Errorf("calls = %d", repo.callCount()) + } + }) + + t.Run("A tick flushes", func(t *testing.T) { + repo, clock := newMockVariantUsageRepo(), start + tracker := usage.NewTrackerWith(repo, func() time.Time { return clock }, 10*time.Millisecond, 500) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go tracker.Run(ctx) + + tracker.RecordHit("thumb", "a.png") + + waitUntilUsage(t, func() bool { return repo.callCount() == 1 }) + }) + + t.Run("Concurrent requests are race free", func(t *testing.T) { + repo, clock := newMockVariantUsageRepo(), start + tracker := newUsageTracker(repo, &clock) + var group sync.WaitGroup + for worker := 0; worker < 50; worker++ { + group.Add(1) + go func() { + defer group.Done() + for index := 0; index < 100; index++ { + tracker.RecordHit("thumb", "file-"+strconv.Itoa(index)+".png") + tracker.RecordRejection() + } + }() + } + flushed := make(chan struct{}) + go func() { + defer close(flushed) + for index := 0; index < 20; index++ { + _ = tracker.Flush() + } + }() + group.Wait() + <-flushed + _ = tracker.Flush() + + if repo.recordedRejections() != 5000 { + t.Errorf("rejections = %d", repo.recordedRejections()) + } + seen := map[string]bool{} + for _, hit := range repo.recordedHits() { + seen[hit.SourceName] = true + } + if len(seen) != 100 { + t.Errorf("distinct files = %d", len(seen)) + } + }) +} + +func waitUntilUsage(t *testing.T, condition func() bool) { + t.Helper() + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + if condition() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("the condition was not met in time") +} + +func requestedAt(t *testing.T, db *gorm.DB, variantName string, sourceName string) *time.Time { + t.Helper() + var row entity.ImageVariantFile + if err := db.Where("variant_name = ? AND source_name = ?", variantName, sourceName).First(&row).Error; err != nil { + t.Fatal(err) + } + return row.LastRequestedAt +} + +func TestVariantUsageRepository(t *testing.T) { + noon := time.Date(2026, time.October, 8, 10, 5, 0, 0, time.UTC) + seed := func(t *testing.T) (*gorm.DB, *persistence.VariantFileRepository, *persistence.VariantUsageRepository) { + t.Helper() + db := openActivityDatabase(t) + files := persistence.NewVariantFileRepository(db) + _, err := files.Save(domain.VariantFile{VariantName: "thumb", SourceName: "a.png", CachedName: "g1", ContentType: "image/png", Size: 10, Fingerprint: "f"}) + if err != nil { + t.Fatal(err) + } + return db, files, persistence.NewVariantUsageRepository(db) + } + + t.Run("A batch sets the last requested time of the file", func(t *testing.T) { + db, _, usageRepo := seed(t) + + err := usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}}) + + if got := requestedAt(t, db, "thumb", "a.png"); err != nil || got == nil || !got.Equal(noon) { + t.Errorf("last_requested_at = %v, err = %v", got, err) + } + }) + + t.Run("A stored time never moves back", func(t *testing.T) { + db, _, usageRepo := seed(t) + _ = usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}}) + + _ = usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon.Add(-4 * time.Minute)}}}) + + if got := requestedAt(t, db, "thumb", "a.png"); got == nil || !got.Equal(noon) { + t.Errorf("last_requested_at = %v", got) + } + }) + + t.Run("A purged file ignores a late update", func(t *testing.T) { + db, files, usageRepo := seed(t) + row, _, _ := files.Find("thumb", "a.png") + _ = files.Delete(row.ID) + + err := usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}}) + + var count int64 + db.Model(&entity.ImageVariantFile{}).Count(&count) + if err != nil || count != 0 { + t.Errorf("err = %v, rows = %d", err, count) + } + }) + + t.Run("A regenerated file keeps its last requested time", func(t *testing.T) { + db, files, usageRepo := seed(t) + _ = usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}}) + + _, _ = files.Save(domain.VariantFile{VariantName: "thumb", SourceName: "a.png", CachedName: "g2", ContentType: "image/png", Size: 20, Fingerprint: "f2"}) + + if got := requestedAt(t, db, "thumb", "a.png"); got == nil || !got.Equal(noon) { + t.Errorf("last_requested_at = %v", got) + } + }) + + t.Run("Rejections add up per day and the window sums its days", func(t *testing.T) { + _, _, usageRepo := seed(t) + day := func(value string) time.Time { + parsed, _ := time.Parse(domain.RejectionDayLayout, value) + return parsed + } + for _, batch := range [][]domain.RejectionCount{ + {{Day: day("2026-10-08"), Count: 3}, {Day: day("2026-10-02"), Count: 5}}, + {{Day: day("2026-10-08"), Count: 4}, {Day: day("2026-10-01"), Count: 9}}, + } { + if err := usageRepo.Record(domain.VariantUsageBatch{Rejections: batch}); err != nil { + t.Fatal(err) + } + } + + total, err := usageRepo.RejectedSince(domain.RejectionWindowStart(time.Date(2026, time.October, 8, 23, 59, 0, 0, time.UTC))) + + if err != nil || total != 12 { + t.Errorf("total = %d, err = %v", total, err) + } + }) + + t.Run("Old day rows are pruned by the batch", func(t *testing.T) { + db, _, usageRepo := seed(t) + day := func(value string) time.Time { + parsed, _ := time.Parse(domain.RejectionDayLayout, value) + return parsed + } + _ = usageRepo.Record(domain.VariantUsageBatch{Rejections: []domain.RejectionCount{{Day: day("2026-09-30"), Count: 1}, {Day: day("2026-10-02"), Count: 2}}}) + + _ = usageRepo.Record(domain.VariantUsageBatch{Rejections: []domain.RejectionCount{{Day: day("2026-10-08"), Count: 1}}, PruneBefore: domain.RejectionPruneBefore(noon)}) + + var days []string + db.Model(&entity.ImageVariantRejectionDay{}).Order("day").Pluck("day", &days) + if len(days) != 2 || days[0] != "2026-10-02" || days[1] != "2026-10-08" { + t.Errorf("days = %v", days) + } + }) +} + +func TestVariantCacheRepository(t *testing.T) { + now := time.Date(2026, time.October, 8, 9, 0, 0, 0, time.UTC) + cutoff := domain.UnusedCutoff(now) + setup := func(t *testing.T) (*gorm.DB, *persistence.VariantFileRepository) { + t.Helper() + db := openActivityDatabase(t) + return db, persistence.NewVariantFileRepository(db) + } + save := func(t *testing.T, db *gorm.DB, files *persistence.VariantFileRepository, variantName string, sourceName string, size int64, created time.Time, requested *time.Time) { + t.Helper() + row, err := files.Save(domain.VariantFile{VariantName: variantName, SourceName: sourceName, CachedName: variantName + sourceName, ContentType: "image/png", Size: size, Fingerprint: "f"}) + if err != nil { + t.Fatal(err) + } + db.Model(&entity.ImageVariantFile{}).Where("id = ?", row.ID).Updates(map[string]any{"created_at": created, "last_requested_at": requested}) + } + at := func(days int) *time.Time { + value := now.AddDate(0, 0, -days) + return &value + } + + t.Run("The reference time is last_requested_at when set and created_at otherwise", func(t *testing.T) { + db, files := setup(t) + save(t, db, files, "hero", "recent.png", 10, now.AddDate(0, 0, -90), at(29)) + save(t, db, files, "hero", "old.png", 20, now.AddDate(0, 0, -1), at(31)) + save(t, db, files, "print", "disabled.png", 40, now.AddDate(0, 0, -90), at(90)) + save(t, db, files, "card", "never.png", 80, now.AddDate(0, 0, -45), nil) + save(t, db, files, "card", "fresh.png", 160, now.AddDate(0, 0, -2), nil) + + totals, err := files.UnusedTotals(domain.UnusedFilter{VariantNames: []string{"hero", "card"}, Cutoff: cutoff}) + + if err != nil || totals.Files != 2 || totals.Bytes != 100 { + t.Errorf("totals = %+v, err = %v", totals, err) + } + }) + + t.Run("A row whose reference time is exactly the cutoff is in use", func(t *testing.T) { + db, files := setup(t) + save(t, db, files, "hero", "edge.png", 10, now.AddDate(0, 0, -90), &cutoff) + + totals, _ := files.UnusedTotals(domain.UnusedFilter{VariantNames: []string{"hero"}, Cutoff: cutoff}) + + if totals.Files != 0 { + t.Errorf("totals = %+v", totals) + } + }) + + t.Run("No enabled variant means nothing is unused", func(t *testing.T) { + db, files := setup(t) + save(t, db, files, "hero", "old.png", 10, now.AddDate(0, 0, -90), nil) + + totals, err := files.UnusedTotals(domain.UnusedFilter{Cutoff: cutoff}) + page, pageErr := files.ListPage(domain.VariantFileFilter{Unused: &domain.UnusedFilter{Cutoff: cutoff}}, 0, 10) + + if err != nil || pageErr != nil || totals.Files != 0 || len(page) != 0 { + t.Errorf("totals = %+v, page = %v", totals, page) + } + }) + + t.Run("The top sources are ordered by bytes then name and capped", func(t *testing.T) { + db, files := setup(t) + save(t, db, files, "hero", "b.png", 100, now, nil) + save(t, db, files, "card", "b.png", 100, now, nil) + save(t, db, files, "hero", "a.png", 200, now, nil) + save(t, db, files, "hero", "c.png", 900, now, nil) + + top, err := files.TopSources(3) + + if err != nil || len(top) != 3 || top[0].SourceName != "c.png" || top[1].SourceName != "a.png" || top[2].SourceName != "b.png" || top[2].Files != 2 || top[2].Bytes != 200 { + t.Errorf("top = %+v, err = %v", top, err) + } + }) + + t.Run("A page continues after the given id and filters by variant, source and unused", func(t *testing.T) { + db, files := setup(t) + for index := 0; index < 5; index++ { + save(t, db, files, "hero", "s"+strconv.Itoa(index)+".png", 1, now.AddDate(0, 0, -90), nil) + } + save(t, db, files, "card", "s0.png", 1, now, nil) + + first, _ := files.ListPage(domain.VariantFileFilter{VariantName: "hero"}, 0, 2) + second, _ := files.ListPage(domain.VariantFileFilter{VariantName: "hero"}, first[1].ID, 10) + bySource, _ := files.ListPage(domain.VariantFileFilter{SourceName: "s0.png"}, 0, 10) + unused, _ := files.ListPage(domain.VariantFileFilter{Unused: &domain.UnusedFilter{VariantNames: []string{"hero", "card"}, Cutoff: cutoff}}, 0, 10) + + if len(first) != 2 || len(second) != 3 || second[0].ID <= first[1].ID || len(bySource) != 2 || len(unused) != 5 { + t.Errorf("first = %d, second = %d, bySource = %d, unused = %d", len(first), len(second), len(bySource), len(unused)) + } + }) +} + +func TestImageRepositoryListByStoredNames(t *testing.T) { + repo := persistence.NewImageRepository(openActivityDatabase(t)) + for _, name := range []string{"a.png", "b.png", "c.png"} { + _, _ = repo.Create(domain.Image{Path: domain.PublicPath(name), Filename: name}) + } + + found, err := repo.ListByStoredNames([]string{"c.png", "a.png", "missing.png"}) + none, noneErr := repo.ListByStoredNames(nil) + + if err != nil || noneErr != nil || len(found) != 2 || found[0].Filename != "a.png" || found[1].Filename != "c.png" || len(none) != 0 { + t.Errorf("found = %+v, err = %v, none = %v", found, err, none) + } +} diff --git a/web/src/components/DashboardShell.vue b/web/src/components/DashboardShell.vue index 376456c..08afe89 100644 --- a/web/src/components/DashboardShell.vue +++ b/web/src/components/DashboardShell.vue @@ -78,8 +78,8 @@ const settingsChildren = computed(() => [ }, { label: t('Cache'), + to: '/settings/media/cache', icon: 'i-lucide-database', - disabled: true, active: !props.noCurrentNav && route.path === '/settings/media/cache', }, ]) diff --git a/web/src/components/__tests__/DashboardShell.spec.ts b/web/src/components/__tests__/DashboardShell.spec.ts index bc3c990..3594bcb 100644 --- a/web/src/components/__tests__/DashboardShell.spec.ts +++ b/web/src/components/__tests__/DashboardShell.spec.ts @@ -396,10 +396,9 @@ describe('DashboardShell Settings expands to the Media subpages', () => { const variants = nav.find('a[href="/settings/media/variants"]') expect(variants.text()).toContain('Variants') expect(variants.attributes('aria-current')).toBeUndefined() - const cache = nav - .findAll('[aria-disabled="true"], [disabled]') - .find((el) => el.text().includes('Cache')) - expect(cache).toBeDefined() + const cache = nav.find('a[href="/settings/media/cache"]') + expect(cache.text()).toContain('Cache') + expect(cache.attributes('aria-current')).toBeUndefined() expect(nav.find('a[href="/settings"]').attributes('aria-current')).toBe('page') }, ) @@ -414,6 +413,14 @@ describe('DashboardShell Settings expands to the Media subpages', () => { expect(nav.find('a[href="/settings"]').attributes('aria-current')).toBeUndefined() }) + it('marks Cache current and Settings not current on the cache page', async () => { + const wrapper = await mountShell('/settings/media/cache', { role: 'admin' }) + + const nav = wrapper.find('nav[aria-label="Main"]') + expect(nav.find('a[href="/settings/media/cache"]').attributes('aria-current')).toBe('page') + expect(nav.find('a[href="/settings"]').attributes('aria-current')).toBeUndefined() + }) + it('shows no Media subpages without an owner or admin role', async () => { const wrapper = await mountShell('/settings', { role: 'user' }) diff --git a/web/src/components/variants/CacheByVariantCard.vue b/web/src/components/variants/CacheByVariantCard.vue new file mode 100644 index 0000000..2a075ab --- /dev/null +++ b/web/src/components/variants/CacheByVariantCard.vue @@ -0,0 +1,59 @@ +<script setup lang="ts"> +import { t } from '@/i18n' + +import { computed } from 'vue' +import type { CacheVariantStat } from '@/composables/useVariantCache' +import { barPercent, filesAndSize } from '@/lib/variants/cacheFormat' + +const props = defineProps<{ items: CacheVariantStat[] }>() + +const largest = computed(() => Math.max(0, ...props.items.map((item) => item.bytes))) +</script> + +<template> + <UCard + data-testid="cache-by-variant" + :ui="{ + root: 'min-w-0 rounded-none border border-default bg-default ring-0 divide-y-0', + header: 'flex min-h-11 items-center border-b border-default px-4 py-0 sm:px-4 sm:py-0', + body: 'my-2 flex flex-col gap-4 px-4 py-2 sm:my-2 sm:gap-3 sm:px-4 sm:py-2', + }" + > + <template #header> + <h2 class="font-semibold text-highlighted">{{ t('By variant') }}</h2> + </template> + <div + v-for="item in items" + :key="item.name" + data-testid="cache-variant-row" + class="flex flex-col gap-1.5" + > + <div class="flex items-baseline gap-2"> + <span class="font-mono font-medium text-highlighted">{{ item.name }}</span> + <UBadge + v-if="!item.enabled" + color="neutral" + variant="outline" + icon="i-lucide-circle-off" + class="border border-accented bg-transparent text-toned ring-0" + :ui="{ base: 'h-4.5 gap-1 px-1.5 py-0 text-xs', leadingIcon: 'size-3' }" + > + {{ t('Disabled') }} + </UBadge> + <span class="flex-1" /> + <span class="text-xs text-toned">{{ filesAndSize(item.files, item.bytes) }}</span> + </div> + <UProgress + :model-value="barPercent(item.bytes, largest)" + :max="100" + :aria-label="item.name" + :ui="{ + base: 'h-2 rounded-none bg-accented', + indicator: item.enabled + ? 'rounded-none bg-primary' + : 'rounded-none bg-(--ui-border-accented)', + }" + /> + </div> + </UCard> +</template> diff --git a/web/src/components/variants/CachePurgeBanner.vue b/web/src/components/variants/CachePurgeBanner.vue new file mode 100644 index 0000000..9a291f1 --- /dev/null +++ b/web/src/components/variants/CachePurgeBanner.vue @@ -0,0 +1,34 @@ +<script setup lang="ts"> +import { formatNumber, t } from '@/i18n' + +defineProps<{ files: number }>() +</script> + +<template> + <UAlert + data-testid="cache-purge-banner" + role="status" + aria-live="polite" + color="info" + variant="soft" + icon="i-lucide-loader-circle" + :close="false" + :title="files === 1 ? t('Purging 1 file') : t('Purging {0} files', [formatNumber(files)])" + :ui="{ + root: 'items-center rounded-none px-3.5 py-3', + icon: 'size-4 animate-spin', + title: 'font-semibold', + }" + > + <template #description> + <UProgress + :model-value="null" + :max="100" + :ui="{ + base: 'h-1.5 rounded-none bg-accented', + indicator: 'rounded-none bg-info', + }" + /> + </template> + </UAlert> +</template> diff --git a/web/src/components/variants/CacheSkeleton.vue b/web/src/components/variants/CacheSkeleton.vue new file mode 100644 index 0000000..9f7371a --- /dev/null +++ b/web/src/components/variants/CacheSkeleton.vue @@ -0,0 +1,42 @@ +<script setup lang="ts"> +import { t } from '@/i18n' + +import CacheStatTile from '@/components/variants/CacheStatTile.vue' + +defineProps<{ desktop: boolean }>() + +const VARIANT_ROWS = 5 +const IMAGE_ROWS = 6 +const BAR = 'rounded-none bg-accented' +const CARD_UI = { + root: 'min-w-0 rounded-none border border-default bg-default ring-0', + body: 'flex flex-col gap-5 p-4 sm:p-4', +} +</script> + +<template> + <section :aria-label="t('Loading cache')" aria-busy="true" class="flex flex-col gap-3 sm:gap-4"> + <div class="grid grid-cols-2 gap-2 sm:grid-cols-4 sm:gap-4"> + <CacheStatTile icon="i-lucide-files" :label="t('Generated files')" loading /> + <CacheStatTile icon="i-lucide-hard-drive" :label="t('Disk used')" loading /> + <CacheStatTile icon="i-lucide-layers" :label="t('Variants defined')" loading /> + <CacheStatTile icon="i-lucide-ban" :label="t('Rejected requests')" loading /> + </div> + <div class="grid grid-cols-1 gap-4 sm:grid-cols-[460px_minmax(0,1fr)]"> + <UCard :ui="CARD_UI"> + <div v-for="row in VARIANT_ROWS" :key="row" class="flex flex-col gap-2"> + <USkeleton class="h-3.25 w-[50%]" :class="BAR" /> + <USkeleton class="h-2 w-full" :class="BAR" /> + </div> + </UCard> + <UCard v-if="desktop" :ui="CARD_UI"> + <USkeleton + v-for="row in IMAGE_ROWS" + :key="row" + class="h-3.25 w-[90%]" + :class="BAR" + /> + </UCard> + </div> + </section> +</template> diff --git a/web/src/components/variants/CacheStatTile.vue b/web/src/components/variants/CacheStatTile.vue new file mode 100644 index 0000000..1b8b9c3 --- /dev/null +++ b/web/src/components/variants/CacheStatTile.vue @@ -0,0 +1,30 @@ +<script setup lang="ts"> +defineProps<{ + icon: string + label: string + value?: string + sub?: string + loading?: boolean +}>() +</script> + +<template> + <UCard + data-testid="cache-tile" + :ui="{ + root: 'min-w-0 rounded-none border border-default bg-default ring-0', + body: 'flex min-w-0 flex-col gap-1 p-3 sm:p-4', + }" + > + <div class="flex items-center gap-1.5 text-sm text-muted"> + <UIcon :name="icon" class="size-3.5 shrink-0" /> + <span>{{ label }}</span> + </div> + <USkeleton v-if="loading" class="h-7 w-[60%] rounded-none bg-accented" /> + <span v-else class="text-stat-sm/9 font-semibold text-highlighted sm:text-2xl/9"> + {{ value }} + </span> + <USkeleton v-if="loading" class="h-2.75 w-[40%] rounded-none bg-accented" /> + <span v-else class="text-xs text-muted">{{ sub }}</span> + </UCard> +</template> diff --git a/web/src/components/variants/CacheTopImagesCard.vue b/web/src/components/variants/CacheTopImagesCard.vue new file mode 100644 index 0000000..42de24b --- /dev/null +++ b/web/src/components/variants/CacheTopImagesCard.vue @@ -0,0 +1,87 @@ +<script setup lang="ts"> +import { t } from '@/i18n' + +import MediaThumb from '@/components/MediaThumb.vue' +import type { CacheImageStat } from '@/composables/useVariantCache' +import { variantsCount } from '@/lib/variants/cacheFormat' +import { formatVariantSize } from '@/lib/variants/variantFormat' + +defineProps<{ + items: CacheImageStat[] + desktop: boolean + locked: boolean + isPurging: (id: number) => boolean +}>() + +const emit = defineEmits<{ purge: [item: CacheImageStat] }>() +</script> + +<template> + <UCard + data-testid="cache-top-images" + :ui="{ + root: 'min-w-0 rounded-none border border-default bg-default ring-0 divide-y-0', + header: 'flex min-h-11 items-center border-b border-default px-4 py-0 sm:px-4 sm:py-0', + body: 'p-0 sm:p-0', + }" + > + <template #header> + <h2 class="font-semibold text-highlighted"> + {{ t('Top 10 images by variant disk usage') }} + </h2> + </template> + <div + v-for="(item, index) in items" + :key="item.id" + data-testid="cache-image-row" + class="flex min-h-15 items-center gap-3 px-3 sm:min-h-10 sm:px-4" + :class="index > 0 ? 'border-t border-muted' : ''" + > + <span class="w-5 font-mono text-xs text-muted">{{ index + 1 }}</span> + <MediaThumb + :item="{ url: item.url, content_type: '', sha256: '' }" + size="sm" + class="h-6 w-8 border border-default" + /> + <div class="flex min-w-0 flex-1 flex-col sm:flex-row sm:gap-3"> + <span class="min-w-0 flex-1 truncate font-medium text-highlighted"> + {{ item.filename }} + </span> + <span class="whitespace-nowrap text-xs text-muted sm:w-37.5"> + {{ variantsCount(item.files) + }}{{ desktop ? '' : ` · ${formatVariantSize(item.bytes)}` }} + </span> + </div> + <span v-if="desktop" class="w-15 text-right text-toned"> + {{ formatVariantSize(item.bytes) }} + </span> + <UTooltip v-if="!desktop" :text="t('Purge {0}', [item.filename])"> + <UButton + color="neutral" + variant="outline" + icon="i-lucide-eraser" + :aria-label="t('Purge {0}', [item.filename])" + :loading="isPurging(item.id)" + :disabled="locked || isPurging(item.id)" + class="size-11 shrink-0 justify-center border border-accented bg-default text-highlighted ring-0" + :ui="{ leadingIcon: 'size-3.5' }" + @click="emit('purge', item)" + /> + </UTooltip> + <UButton + v-else + color="neutral" + variant="outline" + icon="i-lucide-eraser" + :aria-label="t('Purge {0}', [item.filename])" + :loading="isPurging(item.id)" + :disabled="locked || isPurging(item.id)" + class="h-6.5 shrink-0 gap-1.5 border border-accented bg-default px-2.5 text-xs text-highlighted ring-0" + :ui="{ leadingIcon: 'size-3.5' }" + @click="emit('purge', item)" + > + {{ t('Purge') }} + </UButton> + </div> + </UCard> +</template> diff --git a/web/src/components/variants/TypeToConfirmDialog.vue b/web/src/components/variants/TypeToConfirmDialog.vue new file mode 100644 index 0000000..90ef63f --- /dev/null +++ b/web/src/components/variants/TypeToConfirmDialog.vue @@ -0,0 +1,105 @@ +<script setup lang="ts"> +import { computed, nextTick, onMounted, ref, watch } from 'vue' + +const props = defineProps<{ + title: string + description: string + fieldLabel: string + word: string + confirmLabel: string + cancelLabel: string +}>() + +const emit = defineEmits<{ close: [confirmed: boolean] }>() + +const open = defineModel<boolean>('open', { default: true }) +const typed = ref('') +const field = ref<{ inputRef: HTMLInputElement | null } | null>(null) +let settled = false + +const matches = computed(() => typed.value === props.word) + +onMounted(async () => { + await nextTick() + await new Promise((resolve) => setTimeout(resolve)) + field.value?.inputRef?.focus() +}) + +function settle(confirmed: boolean) { + if (settled) { + return + } + settled = true + open.value = false + emit('close', confirmed) +} + +watch(open, (value) => { + if (!value) { + settle(false) + } +}) + +function onConfirm() { + if (matches.value) { + settle(true) + } +} +</script> + +<template> + <UModal + :open="open" + :title="title" + :description="description" + :close="false" + :content="{ role: 'alertdialog' }" + :ui="{ + content: + 'divide-y-0 shadow-(--shadow-modal) ring-0 sm:top-35 sm:max-w-115 sm:translate-y-0 max-sm:!inset-x-4 max-sm:!top-auto max-sm:!bottom-4 max-sm:!left-4 max-sm:!w-auto max-sm:!max-w-none max-sm:!translate-x-0 max-sm:!translate-y-0', + header: 'min-h-0 px-6 pt-5 pb-0 sm:px-6', + title: 'text-lg/6 font-semibold', + description: 'mt-1 text-sm text-muted text-pretty', + body: 'px-6 pt-5 pb-0 sm:px-6 sm:pt-5 sm:pb-0', + footer: 'px-6 py-5 sm:px-6', + }" + @update:open="open = $event" + > + <template #body> + <UFormField :label="fieldLabel" name="confirm-word"> + <UInput + ref="field" + v-model="typed" + autocomplete="off" + autocapitalize="off" + spellcheck="false" + class="w-full" + :ui="{ base: 'font-mono' }" + @keydown.enter.prevent="onConfirm" + /> + </UFormField> + </template> + <template #footer> + <div class="flex w-full flex-col-reverse gap-2 sm:flex-row sm:justify-end"> + <UButton + color="neutral" + variant="outline" + block + class="h-11 justify-center text-sm text-highlighted sm:h-8 sm:w-auto sm:px-3" + @click="settle(false)" + > + {{ cancelLabel }} + </UButton> + <UButton + color="error" + block + :disabled="!matches" + class="h-11 justify-center text-sm sm:h-8 sm:w-auto sm:px-3" + @click="onConfirm" + > + {{ confirmLabel }} + </UButton> + </div> + </template> + </UModal> +</template> diff --git a/web/src/components/variants/VariantsTabs.vue b/web/src/components/variants/VariantsTabs.vue index 0948f68..7e028f2 100644 --- a/web/src/components/variants/VariantsTabs.vue +++ b/web/src/components/variants/VariantsTabs.vue @@ -1,9 +1,26 @@ <script setup lang="ts"> import { t } from '@/i18n' +withDefaults(defineProps<{ active?: 'variants' | 'cache' }>(), { active: 'variants' }) + const TAB = 'h-9 gap-1.5 rounded-none border-b-2 px-0 py-0 text-sm hover:bg-transparent max-sm:h-11' const ACTIVE = 'border-primary font-medium text-highlighted' -const INACTIVE = 'border-transparent font-normal text-toned aria-disabled:opacity-100' +const INACTIVE = 'border-transparent font-normal text-toned' + +const tabs = [ + { + key: 'variants' as const, + to: '/settings/media/variants', + icon: 'i-lucide-layers', + label: () => t('Variants'), + }, + { + key: 'cache' as const, + to: '/settings/media/cache', + icon: 'i-lucide-database', + label: () => t('Cache'), + }, +] </script> <template> @@ -13,27 +30,20 @@ const INACTIVE = 'border-transparent font-normal text-toned aria-disabled:opacit class="flex gap-5 border-b border-default" > <UButton + v-for="tab in tabs" + :key="tab.key" role="tab" - aria-selected="true" - icon="i-lucide-layers" + :to="tab.to" + :active="false" + :aria-selected="active === tab.key ? 'true' : 'false'" + :aria-current="active === tab.key ? 'page' : undefined" + :icon="tab.icon" color="neutral" variant="ghost" - :class="[TAB, ACTIVE, '-mb-px']" + :class="[TAB, active === tab.key ? ACTIVE : INACTIVE, '-mb-px']" :ui="{ leadingIcon: 'size-3.5' }" > - {{ t('Variants') }} - </UButton> - <UButton - role="tab" - aria-selected="false" - aria-disabled="true" - icon="i-lucide-database" - color="neutral" - variant="ghost" - :class="[TAB, INACTIVE, '-mb-px']" - :ui="{ leadingIcon: 'size-3.5' }" - > - {{ t('Cache') }} + {{ tab.label() }} </UButton> </div> </template> diff --git a/web/src/composables/useVariantCache.ts b/web/src/composables/useVariantCache.ts new file mode 100644 index 0000000..8821b30 --- /dev/null +++ b/web/src/composables/useVariantCache.ts @@ -0,0 +1,65 @@ +import { useApi } from '@/composables/useApi' + +export type CachePurgeScope = 'all' | 'unused' | 'variant' | 'image' + +export interface CacheVariantStat { + name: string + enabled: boolean + files: number + bytes: number +} + +export interface CacheImageStat { + id: number + filename: string + url: string + files: number + bytes: number +} + +export interface CacheStats { + generated_files: number + disk_used: number + variants_defined: number + variants_enabled: number + variants_disabled: number + rejected_requests: number + rejected_window_days: number + unused: { files: number; bytes: number; days: number } + by_variant: CacheVariantStat[] + top_images: CacheImageStat[] +} + +export interface CachePurgeRequest { + scope: CachePurgeScope + variant?: string + image_id?: number +} + +export interface CachePurgeResult { + scope: CachePurgeScope + purged_files: number + purged_bytes: number + failed_files: number +} + +const JSON_HEADERS = { 'Content-Type': 'application/json' } +const CACHE_PATH = '/v1/image-variants/cache' + +export function useVariantCache() { + const { request } = useApi() + + function getStats(): Promise<CacheStats> { + return request<CacheStats>(CACHE_PATH) + } + + function purge(input: CachePurgeRequest): Promise<CachePurgeResult> { + return request<CachePurgeResult>(`${CACHE_PATH}/purge`, { + method: 'POST', + headers: JSON_HEADERS, + body: JSON.stringify(input), + }) + } + + return { getStats, purge } +} diff --git a/web/src/i18n/messages.ts b/web/src/i18n/messages.ts index 678f090..9fb5075 100644 --- a/web/src/i18n/messages.ts +++ b/web/src/i18n/messages.ts @@ -1422,6 +1422,62 @@ export const messages = { 'Les variantes ne sont pas générées pour ce format.', "Variant '{0}' no longer exists.": 'La variante « {0} » n’existe plus.', "Variant '{0}' is disabled.": 'La variante « {0} » est désactivée.', + 'Generated variant files. Purged files are generated again on the next request.': + 'Fichiers de variantes générés. Les fichiers purgés sont régénérés à la prochaine requête.', + 'Purge unused': 'Purger les inutilisés', + 'Purge all variants': 'Purger toutes les variantes', + 'Loading cache': 'Chargement du cache', + 'Generated files': 'Fichiers générés', + 'Disk used': 'Espace disque utilisé', + 'Variants defined': 'Variantes définies', + 'Rejected requests': 'Requêtes rejetées', + 'Across 1 variant': 'Sur 1 variante', + 'Across {0} variants': 'Sur {0} variantes', + 'Originals not included': 'Originaux non inclus', + 'Nothing cached': 'Rien en cache', + '{0} enabled, {1} disabled': '{0} activées, {1} désactivées', + 'Last {0} days': '{0} derniers jours', + 'By variant': 'Par variante', + '1 file': '1 fichier', + '1 file · {0}': '1 fichier · {0}', + '{0} files · {1}': '{0} fichiers · {1}', + 'Top 10 images by variant disk usage': 'Top 10 des images par espace disque de variantes', + 'Nothing generated yet': 'Rien n’a encore été généré', + "Variants are generated on the first request with ?v=<name>, or from an image's Variants tab.": + 'Les variantes sont générées à la première requête avec ?v=<nom>, ou depuis l’onglet Variantes d’une image.', + "Couldn't load cache stats": 'Impossible de charger les statistiques du cache', + 'Purge all variants?': 'Purger toutes les variantes ?', + "This removes 1 generated file ({0}) for every variant. Originals aren't touched. Files are generated again on request.": + 'Cela supprime 1 fichier généré ({0}) pour chaque variante. Les originaux ne sont pas touchés. Les fichiers sont régénérés à la demande.', + "This removes {0} generated files ({1}) for every variant. Originals aren't touched. Files are generated again on request.": + 'Cela supprime {0} fichiers générés ({1}) pour chaque variante. Les originaux ne sont pas touchés. Les fichiers sont régénérés à la demande.', + 'Type purge to confirm': 'Saisissez purge pour confirmer', + 'Purge unused variants?': 'Purger les variantes inutilisées ?', + "This removes 1 file ({0}) from enabled variants that haven't been requested for {1} days. Files are generated again on request.": + 'Cela supprime 1 fichier ({0}) des variantes activées qui n’ont pas été demandées depuis {1} jours. Les fichiers sont régénérés à la demande.', + "This removes {0} files ({1}) from enabled variants that haven't been requested for {2} days. Files are generated again on request.": + 'Cela supprime {0} fichiers ({1}) des variantes activées qui n’ont pas été demandées depuis {2} jours. Les fichiers sont régénérés à la demande.', + 'Purging 1 file': 'Purge de 1 fichier', + 'Purging {0} files': 'Purge de {0} fichiers', + 'Purged 1 file, freed {0}': '1 fichier purgé, {0} libérés', + 'Purged {0} files, freed {1}': '{0} fichiers purgés, {1} libérés', + "Purge didn't finish": 'La purge n’est pas terminée', + '{0} of {1} files were removed. Try again.': + '{0} fichiers sur {1} ont été supprimés. Réessayez.', + 'Purged 1 file ({0}) for {1}': '1 fichier purgé ({0}) pour {1}', + 'Purged {0} files ({1}) for {2}': '{0} fichiers purgés ({1}) pour {2}', + 'Purged all variants': 'A purgé toutes les variantes', + 'Failed to purge all variants': 'Échec de la purge de toutes les variantes', + 'Purged unused variants': 'A purgé les variantes inutilisées', + 'Failed to purge unused variants': 'Échec de la purge des variantes inutilisées', + 'Purged the cache of variant “{0}”': 'A purgé le cache de la variante « {0} »', + 'Failed to purge the cache of variant “{0}”': + 'Échec de la purge du cache de la variante « {0} »', + 'Removed 1 file ({0}) not requested for {1} days': + '1 fichier supprimé ({0}) non demandé depuis {1} jours', + 'Removed {0} files ({1}) not requested for {2} days': + '{0} fichiers supprimés ({1}) non demandés depuis {2} jours', + '{0} failed': '{0} en échec', } as const export type MessageKey = keyof typeof messages diff --git a/web/src/lib/__tests__/activityCachePurge.spec.ts b/web/src/lib/__tests__/activityCachePurge.spec.ts new file mode 100644 index 0000000..97b0551 --- /dev/null +++ b/web/src/lib/__tests__/activityCachePurge.spec.ts @@ -0,0 +1,111 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { initializeLocale } from '@/i18n' +import type { ActivityItem } from '@/lib/activity' +import { detailLine, sentenceText } from '../activitySentence' + +function purgeItem(overrides: Partial<ActivityItem>): ActivityItem { + return { + id: 1, + created_at: '2026-10-03T12:00:00Z', + action: 'media.variant_purge_all', + category: 'media', + actor: { type: 'user', id: 3, name: 'Marta Alves' }, + target: null, + outcome: 'success', + failure_reason: null, + security: false, + source: 'web', + ip: null, + user_agent: null, + params: {}, + changes: [], + ...overrides, + } +} + +const hero = { type: 'image_variant', id: 'hero', name: 'hero', exists: true } +const wall = { type: 'image', id: '1', name: 'north-wall.jpg', exists: true } + +const rows: [string, Partial<ActivityItem>, string, string | null][] = [ + [ + 'media.variant_purge_all success', + { params: { purged_files: 1284, purged_bytes: 222298112 } }, + 'Purged all variants', + 'Removed 1,284 files (212 MB)', + ], + ['media.variant_purge_all failed', { outcome: 'failed' }, 'Failed to purge all variants', null], + [ + 'media.variant_purge_unused success', + { + action: 'media.variant_purge_unused', + params: { purged_files: 312, purged_bytes: 42991616, older_than_days: 30 }, + }, + 'Purged unused variants', + 'Removed 312 files (41 MB) not requested for 30 days', + ], + [ + 'media.variant_purge_unused failed', + { action: 'media.variant_purge_unused', outcome: 'failed' }, + 'Failed to purge unused variants', + null, + ], + [ + 'media.variant_purge success', + { + action: 'media.variant_purge', + target: hero, + params: { purged_files: 244, purged_bytes: 88080384 }, + }, + 'Purged the cache of variant “hero”', + 'Removed 244 files (84 MB)', + ], + [ + 'media.variant_purge failed', + { action: 'media.variant_purge', target: hero, outcome: 'failed' }, + 'Failed to purge the cache of variant “hero”', + null, + ], + [ + 'media.variant_purge_image success', + { + action: 'media.variant_purge_image', + target: wall, + params: { purged_files: 6, purged_bytes: 6082765 }, + }, + 'Purged the variants of north-wall.jpg', + 'Removed 6 files (5.8 MB)', + ], +] + +beforeEach(() => { + localStorage.clear() + initializeLocale() +}) + +afterEach(() => { + localStorage.clear() + initializeLocale() +}) + +describe('The Activity screen words each cache purge scope', () => { + it.each(rows)('%s', (_, overrides, sentence, detail) => { + const subject = purgeItem(overrides) + + expect(sentenceText(subject)).toBe(sentence) + expect(detailLine(subject)).toBe(detail) + }) + + it('A partial purge adds the failed count to the detail line', () => { + const subject = purgeItem({ + params: { purged_files: 640, purged_bytes: 111149056, failed_files: 644 }, + }) + + expect(detailLine(subject)).toBe('Removed 640 files (106 MB) · 644 failed') + }) + + it('The singular reads 1 file', () => { + const subject = purgeItem({ params: { purged_files: 1, purged_bytes: 2048 } }) + + expect(detailLine(subject)).toBe('Removed 1 file (2 KB)') + }) +}) diff --git a/web/src/lib/__tests__/activitySentence.spec.ts b/web/src/lib/__tests__/activitySentence.spec.ts index 62fa5c7..c18f545 100644 --- a/web/src/lib/__tests__/activitySentence.spec.ts +++ b/web/src/lib/__tests__/activitySentence.spec.ts @@ -231,6 +231,9 @@ describe('The catalogue is complete in both languages', () => { 'media.merge', 'media.purge', 'media.variant_generate_all', + 'media.variant_purge', + 'media.variant_purge_all', + 'media.variant_purge_unused', 'media.variant_purge_image', 'system.media_purge', 'user.register', diff --git a/web/src/lib/activitySentence.ts b/web/src/lib/activitySentence.ts index cb85e73..45169ff 100644 --- a/web/src/lib/activitySentence.ts +++ b/web/src/lib/activitySentence.ts @@ -3,6 +3,7 @@ import type { MessageKey } from '@/i18n/messages' import type { ActivityItem } from '@/lib/activity' import { sizeLabel } from '@/lib/activityRetention' import { formatSize } from '@/lib/mediaFormat' +import { formatVariantSize } from '@/lib/variants/variantFormat' import { causeText } from '@/lib/variants/variantCauses' export interface SentenceSegment { @@ -62,6 +63,18 @@ const CATALOGUE: Record<string, Template> = { success: 'Generated all variants of {0}', failed: 'Failed to generate the variants of {0}', }, + 'media.variant_purge': { + success: 'Purged the cache of variant “{0}”', + failed: 'Failed to purge the cache of variant “{0}”', + }, + 'media.variant_purge_all': { + success: 'Purged all variants', + failed: 'Failed to purge all variants', + }, + 'media.variant_purge_unused': { + success: 'Purged unused variants', + failed: 'Failed to purge unused variants', + }, 'media.variant_purge_image': { success: 'Purged the variants of {0}', failed: 'Failed to purge the variants of {0}', @@ -262,7 +275,42 @@ function purgeImageDetail(params: Record<string, unknown>) { : t('Removed {0} files ({1})', [formatNumber(files), size]) } +const DEFAULT_UNUSED_DAYS = 30 +const CACHE_PURGE_ACTIONS = [ + 'media.variant_purge', + 'media.variant_purge_all', + 'media.variant_purge_unused', +] + +function removedFiles(files: number, size: string, days: number | null) { + if (days === null) { + return files === 1 + ? t('Removed 1 file ({0})', [size]) + : t('Removed {0} files ({1})', [formatNumber(files), size]) + } + return files === 1 + ? t('Removed 1 file ({0}) not requested for {1} days', [size, days]) + : t('Removed {0} files ({1}) not requested for {2} days', [formatNumber(files), size, days]) +} + +function cachePurgeDetail(item: ActivityItem) { + if (item.outcome === 'failed') return null + const params = item.params + const unused = item.action === 'media.variant_purge_unused' + const days = unused ? count(params.older_than_days) || DEFAULT_UNUSED_DAYS : null + const removed = removedFiles( + count(params.purged_files), + formatVariantSize(count(params.purged_bytes)), + days, + ) + const failed = count(params.failed_files) + return failed > 0 ? `${removed} · ${t('{0} failed', [formatNumber(failed)])}` : removed +} + export function detailLine(item: ActivityItem): string | null { + if (CACHE_PURGE_ACTIONS.includes(item.action)) { + return cachePurgeDetail(item) + } if (item.action === 'media.variant_generate_all') { return generateAllDetail(item.params) } diff --git a/web/src/lib/variants/cacheFormat.ts b/web/src/lib/variants/cacheFormat.ts new file mode 100644 index 0000000..d59ca87 --- /dev/null +++ b/web/src/lib/variants/cacheFormat.ts @@ -0,0 +1,26 @@ +import { formatNumber, t } from '@/i18n' +import { formatVariantSize } from '@/lib/variants/variantFormat' + +const PERCENT = 100 + +export function cacheSize(bytes: number) { + return bytes === 0 ? t('{0} MB', [0]) : formatVariantSize(bytes) +} + +export function filesCount(count: number) { + return count === 1 ? t('1 file') : t('{0} files', [formatNumber(count)]) +} + +export function filesAndSize(count: number, bytes: number) { + return count === 1 + ? t('1 file · {0}', [formatVariantSize(bytes)]) + : t('{0} files · {1}', [formatNumber(count), formatVariantSize(bytes)]) +} + +export function variantsCount(count: number) { + return count === 1 ? t('1 variant') : t('{0} variants', [formatNumber(count)]) +} + +export function barPercent(bytes: number, largest: number) { + return largest === 0 ? 0 : Math.round((bytes / largest) * PERCENT) +} diff --git a/web/src/router/index.ts b/web/src/router/index.ts index b863f72..c34bf55 100644 --- a/web/src/router/index.ts +++ b/web/src/router/index.ts @@ -111,6 +111,11 @@ const router = createRouter({ component: () => import('@/views/VariantsPage.vue'), meta: { auth: 'verified', title: 'Settings' }, }, + { + path: '/settings/media/cache', + component: () => import('@/views/CachePage.vue'), + meta: { auth: 'verified', title: 'Settings' }, + }, { path: '/:pathMatch(.*)*', component: () => import('@/views/NotFoundView.vue'), diff --git a/web/src/stores/variantCache.ts b/web/src/stores/variantCache.ts new file mode 100644 index 0000000..1f1b75f --- /dev/null +++ b/web/src/stores/variantCache.ts @@ -0,0 +1,89 @@ +import { defineStore } from 'pinia' +import { computed, ref } from 'vue' +import { ApiError } from '@/composables/useApi' +import { + useVariantCache, + type CachePurgeRequest, + type CachePurgeResult, + type CacheStats, +} from '@/composables/useVariantCache' + +export type CacheLoadState = 'loading' | 'ready' | 'error' | 'forbidden' + +export interface RunningPurge { + scope: CachePurgeRequest['scope'] + files: number +} + +const FORBIDDEN_STATUS = 403 +const UNAUTHORIZED_STATUS = 401 + +export const useVariantCacheStore = defineStore('variantCache', () => { + const api = useVariantCache() + const stats = ref<CacheStats | null>(null) + const loadState = ref<CacheLoadState>('loading') + const running = ref<RunningPurge | null>(null) + const purgingImages = ref<number[]>([]) + + const isEmpty = computed( + () => loadState.value === 'ready' && stats.value?.generated_files === 0, + ) + const isPurging = computed(() => running.value !== null) + + function isPurgingImage(id: number) { + return purgingImages.value.includes(id) + } + + async function load() { + loadState.value = 'loading' + try { + stats.value = await api.getStats() + loadState.value = 'ready' + } catch (error) { + if (error instanceof ApiError && error.status === FORBIDDEN_STATUS) { + loadState.value = 'forbidden' + return + } + if (error instanceof ApiError && error.status === UNAUTHORIZED_STATUS) { + return + } + loadState.value = 'error' + } + } + + async function refresh() { + try { + stats.value = await api.getStats() + loadState.value = 'ready' + } catch { + return + } + } + + function filesFor(request: CachePurgeRequest) { + if (request.scope === 'unused') { + return stats.value?.unused.files ?? 0 + } + return stats.value?.generated_files ?? 0 + } + + async function purge(request: CachePurgeRequest): Promise<CachePurgeResult> { + if (request.scope === 'image' && request.image_id !== undefined) { + const id = request.image_id + purgingImages.value = [...purgingImages.value, id] + try { + return await api.purge(request) + } finally { + purgingImages.value = purgingImages.value.filter((entry) => entry !== id) + } + } + running.value = { scope: request.scope, files: filesFor(request) } + try { + return await api.purge(request) + } finally { + running.value = null + } + } + + return { stats, loadState, running, isEmpty, isPurging, isPurgingImage, load, refresh, purge } +}) diff --git a/web/src/views/CachePage.vue b/web/src/views/CachePage.vue new file mode 100644 index 0000000..2ebd2e4 --- /dev/null +++ b/web/src/views/CachePage.vue @@ -0,0 +1,457 @@ +<script setup lang="ts"> +import { formatNumber, t } from '@/i18n' + +import { computed, nextTick, onBeforeUnmount, ref, watch } from 'vue' +import { useMediaQuery } from '@vueuse/core' +import { useRouter } from 'vue-router' +import { useLocalizedOverlay, useLocalizedToast } from '@/i18n/presentation' +import ConfirmDialog from '@/components/ConfirmDialog.vue' +import DashboardShell from '@/components/DashboardShell.vue' +import CacheByVariantCard from '@/components/variants/CacheByVariantCard.vue' +import CachePurgeBanner from '@/components/variants/CachePurgeBanner.vue' +import CacheSkeleton from '@/components/variants/CacheSkeleton.vue' +import CacheStatTile from '@/components/variants/CacheStatTile.vue' +import CacheTopImagesCard from '@/components/variants/CacheTopImagesCard.vue' +import TypeToConfirmDialog from '@/components/variants/TypeToConfirmDialog.vue' +import VariantsForbidden from '@/components/variants/VariantsForbidden.vue' +import VariantsTabs from '@/components/variants/VariantsTabs.vue' +import type { + CacheImageStat, + CachePurgeRequest, + CachePurgeResult, +} from '@/composables/useVariantCache' +import { useVariantToasts } from '@/composables/useVariantToasts' +import { cacheSize } from '@/lib/variants/cacheFormat' +import { isHandledByApiClient, isStatus } from '@/lib/variants/variantErrors' +import { formatVariantSize } from '@/lib/variants/variantFormat' +import { useSessionStore } from '@/stores/session' +import { useVariantCacheStore } from '@/stores/variantCache' + +const DESKTOP_QUERY = '(min-width: 40rem)' +const SKELETON_DELAY_MS = 150 +const NOT_FOUND = 404 +const CONFIRM_WORD = 'purge' + +const router = useRouter() +const session = useSessionStore() +const store = useVariantCacheStore() +const overlay = useLocalizedOverlay() +const toast = useLocalizedToast() +const { announceFailure, announceSuccess } = useVariantToasts() +const isDesktop = useMediaQuery(DESKTOP_QUERY) + +const showSkeleton = ref(false) +const purgeUnusedButton = ref<{ $el: HTMLElement } | null>(null) +const purgeAllButton = ref<{ $el: HTMLElement } | null>(null) +let skeletonTimer: ReturnType<typeof setTimeout> | null = null +let requested = false + +const sessionReady = computed(() => session.status === 'ready') +const canManage = computed( + () => session.session?.role === 'owner' || session.session?.role === 'admin', +) +const forbidden = computed( + () => (sessionReady.value && !canManage.value) || store.loadState === 'forbidden', +) +const stats = computed(() => store.stats) +const ready = computed(() => store.loadState === 'ready' && stats.value !== null) +const dimmed = computed(() => store.isPurging) +const purgeUnusedDisabled = computed( + () => !ready.value || store.isPurging || (stats.value?.unused.files ?? 0) === 0, +) +const purgeAllDisabled = computed( + () => !ready.value || store.isPurging || (stats.value?.generated_files ?? 0) === 0, +) + +function clearSkeletonTimer() { + if (skeletonTimer !== null) { + clearTimeout(skeletonTimer) + skeletonTimer = null + } +} + +async function loadStats() { + showSkeleton.value = false + clearSkeletonTimer() + skeletonTimer = setTimeout(() => { + showSkeleton.value = true + }, SKELETON_DELAY_MS) + const loading = store.load() + try { + await loading + } finally { + clearSkeletonTimer() + showSkeleton.value = false + } +} + +watch( + [sessionReady, canManage], + ([isReady, isAdmin]) => { + if (isReady && isAdmin && !requested) { + requested = true + loadStats() + } + }, + { immediate: true }, +) + +onBeforeUnmount(clearSkeletonTimer) + +async function restoreFocus(element: HTMLElement | null | undefined) { + await nextTick() + if (element?.isConnected) { + element.focus() + } +} + +function freed(result: CachePurgeResult) { + const size = formatVariantSize(result.purged_bytes) + return result.purged_files === 1 + ? t('Purged 1 file, freed {0}', [size]) + : t('Purged {0} files, freed {1}', [formatNumber(result.purged_files), size]) +} + +function purgedForImage(result: CachePurgeResult, filename: string) { + const size = formatVariantSize(result.purged_bytes) + return result.purged_files === 1 + ? t('Purged 1 file ({0}) for {1}', [size, filename]) + : t('Purged {0} files ({1}) for {2}', [formatNumber(result.purged_files), size, filename]) +} + +function announceIncomplete( + request: CachePurgeRequest, + result: CachePurgeResult, + filename: string, +) { + toast.add(() => ({ + color: 'error' as const, + icon: 'i-lucide-circle-x', + title: t("Purge didn't finish"), + description: t('{0} of {1} files were removed. Try again.', [ + formatNumber(result.purged_files), + formatNumber(result.purged_files + result.failed_files), + ]), + actions: [ + { + label: t('Try again'), + icon: 'i-lucide-rotate-cw', + color: 'neutral' as const, + variant: 'outline' as const, + onClick: () => { + void runPurge(request, filename) + }, + }, + ], + })) +} + +function announceResult(request: CachePurgeRequest, result: CachePurgeResult, filename: string) { + if (result.failed_files > 0) { + announceIncomplete(request, result, filename) + return + } + announceSuccess(() => + request.scope === 'image' ? purgedForImage(result, filename) : freed(result), + ) +} + +async function runPurge(request: CachePurgeRequest, filename = '') { + let result: CachePurgeResult + try { + result = await store.purge(request) + } catch (error) { + if (isHandledByApiClient(error)) { + return + } + announceFailure(error) + if (isStatus(error, NOT_FOUND)) { + await store.refresh() + } + return + } + announceResult(request, result, filename) + await store.refresh() +} + +async function onPurgeUnused() { + const current = stats.value + if (!current) { + return + } + const { files, bytes, days } = current.unused + const confirmed = await overlay + .create(ConfirmDialog, { + props: { + get title() { + return t('Purge unused variants?') + }, + get description() { + return files === 1 + ? t( + "This removes 1 file ({0}) from enabled variants that haven't been requested for {1} days. Files are generated again on request.", + [formatVariantSize(bytes), days], + ) + : t( + "This removes {0} files ({1}) from enabled variants that haven't been requested for {2} days. Files are generated again on request.", + [formatNumber(files), formatVariantSize(bytes), days], + ) + }, + get confirmLabel() { + return files === 1 + ? t('Purge 1 file') + : t('Purge {0} files', [formatNumber(files)]) + }, + get cancelLabel() { + return t('Cancel') + }, + get pendingLabel() { + return t('Purging…') + }, + tone: 'neutral', + action: async () => undefined, + }, + }) + .open() + restoreFocus(purgeUnusedButton.value?.$el) + if (confirmed) { + await runPurge({ scope: 'unused' }) + } +} + +async function onPurgeAll() { + const current = stats.value + if (!current) { + return + } + const files = current.generated_files + const size = formatVariantSize(current.disk_used) + const confirmed = await overlay + .create(TypeToConfirmDialog, { + props: { + get title() { + return t('Purge all variants?') + }, + get description() { + return files === 1 + ? t( + "This removes 1 generated file ({0}) for every variant. Originals aren't touched. Files are generated again on request.", + [size], + ) + : t( + "This removes {0} generated files ({1}) for every variant. Originals aren't touched. Files are generated again on request.", + [formatNumber(files), size], + ) + }, + get fieldLabel() { + return t('Type purge to confirm') + }, + word: CONFIRM_WORD, + get confirmLabel() { + return t('Purge all variants') + }, + get cancelLabel() { + return t('Cancel') + }, + }, + }) + .open() + restoreFocus(purgeAllButton.value?.$el) + if (confirmed) { + await runPurge({ scope: 'all' }) + } +} + +function onPurgeImage(item: CacheImageStat) { + return runPurge({ scope: 'image', image_id: item.id }, item.filename) +} +</script> + +<template> + <DashboardShell compact-padding :mobile-title="t('Cache')" :no-current-nav="forbidden"> + <template #breadcrumb> + <UBreadcrumb + :items="[ + { + get label() { + return t('Settings') + }, + to: '/settings', + }, + { + get label() { + return t('Media') + }, + }, + { + get label() { + return t('Cache') + }, + }, + ]" + color="neutral" + class="hidden lg:block" + :ui="{ link: 'text-sm font-medium' }" + /> + </template> + + <VariantsForbidden v-if="forbidden" @home="router.push('/home')" /> + + <div v-else-if="sessionReady" class="flex flex-col gap-4"> + <div class="flex flex-col gap-3 sm:flex-row sm:items-start sm:gap-4"> + <div class="flex min-w-0 flex-1 flex-col gap-1"> + <h1 class="text-xl/8 font-semibold text-highlighted sm:text-stat"> + {{ t('Cache') }} + </h1> + <p class="max-w-155 text-pretty text-muted"> + {{ + t( + 'Generated variant files. Purged files are generated again on the next request.', + ) + }} + </p> + </div> + <div class="flex shrink-0 flex-col gap-2 sm:flex-row"> + <UButton + ref="purgeUnusedButton" + data-testid="purge-unused" + color="neutral" + variant="outline" + icon="i-lucide-clock" + :disabled="purgeUnusedDisabled" + class="h-11 justify-center border border-accented bg-default px-3 text-sm text-highlighted ring-0 sm:h-8" + @click="onPurgeUnused" + > + {{ t('Purge unused') }} + </UButton> + <UButton + ref="purgeAllButton" + data-testid="purge-all" + color="neutral" + variant="outline" + icon="i-lucide-trash-2" + :disabled="purgeAllDisabled" + class="h-11 justify-center border border-error bg-default px-3 text-sm text-error ring-0 sm:h-8" + @click="onPurgeAll" + > + {{ t('Purge all variants') }} + </UButton> + </div> + </div> + + <VariantsTabs active="cache" /> + + <CacheSkeleton + v-if="store.loadState === 'loading' && showSkeleton" + :desktop="isDesktop" + /> + + <UAlert + v-else-if="store.loadState === 'error'" + role="alert" + color="error" + variant="soft" + orientation="vertical" + :close="false" + icon="i-lucide-circle-x" + :title="t('Couldn\'t load cache stats')" + :description="t('Check your connection and try again.')" + :ui="{ actions: 'mt-2' }" + > + <template #actions> + <UButton + color="neutral" + variant="outline" + icon="i-lucide-rotate-cw" + class="h-11 gap-1.5 border border-accented bg-default px-2.5 text-xs text-highlighted ring-0 sm:h-7" + :ui="{ leadingIcon: 'size-3.5' }" + @click="loadStats" + > + {{ t('Try again') }} + </UButton> + </template> + </UAlert> + + <template v-else-if="ready && stats"> + <CachePurgeBanner v-if="store.running" :files="store.running.files" /> + + <div + class="grid grid-cols-2 gap-2 sm:grid-cols-4 sm:gap-4" + :class="dimmed ? 'opacity-60' : ''" + > + <CacheStatTile + icon="i-lucide-files" + :label="t('Generated files')" + :value="formatNumber(stats.generated_files)" + :sub=" + store.isEmpty + ? t('Nothing cached') + : t( + stats.variants_defined === 1 + ? 'Across 1 variant' + : 'Across {0} variants', + [formatNumber(stats.variants_defined)], + ) + " + /> + <CacheStatTile + icon="i-lucide-hard-drive" + :label="t('Disk used')" + :value="cacheSize(stats.disk_used)" + :sub="store.isEmpty ? t('Nothing cached') : t('Originals not included')" + /> + <CacheStatTile + icon="i-lucide-layers" + :label="t('Variants defined')" + :value="formatNumber(stats.variants_defined)" + :sub=" + t('{0} enabled, {1} disabled', [ + formatNumber(stats.variants_enabled), + formatNumber(stats.variants_disabled), + ]) + " + /> + <CacheStatTile + icon="i-lucide-ban" + :label="t('Rejected requests')" + :value="formatNumber(stats.rejected_requests)" + :sub="t('Last {0} days', [formatNumber(stats.rejected_window_days)])" + /> + </div> + + <UEmpty + v-if="store.isEmpty" + variant="naked" + icon="i-lucide-database" + :title="t('Nothing generated yet')" + :description=" + t( + 'Variants are generated on the first request with ?v=<name>, or from an image\'s Variants tab.', + ) + " + :ui="{ + root: 'gap-3 border border-default bg-default px-6 py-14', + avatar: 'mb-0 size-11 rounded-none bg-elevated text-toned', + title: 'text-md font-semibold', + description: 'max-w-95 text-pretty text-muted', + }" + /> + + <div + v-else + class="grid grid-cols-1 items-start gap-4 sm:grid-cols-[460px_minmax(0,1fr)]" + :class="dimmed ? 'opacity-60' : ''" + > + <CacheByVariantCard :items="stats.by_variant" /> + <CacheTopImagesCard + :items="stats.top_images" + :desktop="isDesktop" + :locked="store.isPurging" + :is-purging="store.isPurgingImage" + @purge="onPurgeImage" + /> + </div> + </template> + </div> + </DashboardShell> +</template> diff --git a/web/src/views/__tests__/CachePage.spec.ts b/web/src/views/__tests__/CachePage.spec.ts new file mode 100644 index 0000000..0bd82c7 --- /dev/null +++ b/web/src/views/__tests__/CachePage.spec.ts @@ -0,0 +1,787 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { flushPromises, mount, type VueWrapper } from '@vue/test-utils' +import { createPinia, setActivePinia } from 'pinia' +import { defineComponent, h } from 'vue' +import uiPlugin from '@nuxt/ui/vue-plugin' +import UApp from '@nuxt/ui/components/App.vue' +import router from '@/router' +import { ApiError, NetworkError, useApi } from '@/composables/useApi' +import type { CacheStats } from '@/composables/useVariantCache' +import { initializeLocale } from '@/i18n' +import { useSessionStore } from '@/stores/session' +import CachePage from '../CachePage.vue' + +vi.mock('@/composables/useApi', async (importOriginal) => { + const actual = await importOriginal<typeof import('@/composables/useApi')>() + return { ...actual, useApi: vi.fn() } +}) + +const toasts = vi.hoisted(() => ({ add: vi.fn() })) + +vi.mock('@nuxt/ui/composables', async (importOriginal) => { + const actual = await importOriginal<typeof import('@nuxt/ui/composables')>() + return { ...actual, useToast: () => toasts } +}) + +const STATS = 'GET /v1/image-variants/cache' +const PURGE = 'POST /v1/image-variants/cache/purge' +const MB = 1048576 + +type Handler = (body?: Record<string, unknown>) => Promise<unknown> + +const TOP_IMAGES: [string, number, number][] = [ + ['north-wall.jpg', 6, 6082765], + ['lime-plaster.jpg', 6, 5138022], + ['kitchen-after.webp', 5, 4613734], + ['kitchen-before.webp', 5, 4089446], + ['floorboards.jpg', 5, 3774873], + ['IMG_4471.jpg', 4, 3250585], + ['stone-wall-detail.jpg', 4, 2936012], + ['bathroom-tiles.png', 4, 2726297], + ['roof-lantern.jpg', 3, 2411724], + ['chimney-stack.jpg', 3, 2202009], +] + +function stats(overrides: Partial<CacheStats> = {}): CacheStats { + return { + generated_files: 1284, + disk_used: 222298112, + variants_defined: 6, + variants_enabled: 5, + variants_disabled: 1, + rejected_requests: 37, + rejected_window_days: 7, + unused: { files: 312, bytes: 42991616, days: 30 }, + by_variant: [ + { name: 'hero', enabled: true, files: 244, bytes: 88080384 }, + { name: 'card', enabled: true, files: 318, bytes: 60817408 }, + { name: 'print', enabled: false, files: 74, bytes: 22020096 }, + { name: 'thumb', enabled: true, files: 412, bytes: 22020096 }, + { name: 'square', enabled: true, files: 140, bytes: 19922944 }, + { name: 'og', enabled: true, files: 96, bytes: 9437184 }, + ], + top_images: TOP_IMAGES.map(([filename, files, bytes], index) => ({ + id: index + 1, + filename, + url: `/static/uploadedImages/${filename}`, + files, + bytes, + })), + ...overrides, + } +} + +function emptyStats() { + return stats({ + generated_files: 0, + disk_used: 0, + unused: { files: 0, bytes: 0, days: 30 }, + by_variant: [], + top_images: [], + }) +} + +function deferred<T>() { + let resolve!: (value: T) => void + let reject!: (reason: unknown) => void + const promise = new Promise<T>((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve, reject } +} + +let requestMock: ReturnType<typeof vi.fn> +let wrapper: VueWrapper | null = null + +function setViewport(desktop: boolean) { + window.matchMedia = ((query: string) => ({ + matches: desktop, + media: query, + onchange: null, + addEventListener: () => undefined, + removeEventListener: () => undefined, + addListener: () => undefined, + removeListener: () => undefined, + dispatchEvent: () => false, + })) as unknown as typeof window.matchMedia +} + +async function mountPage( + options: { desktop?: boolean; role?: string | null; routes?: Record<string, Handler> } = {}, +) { + const { desktop = true, role = 'owner', routes = {} } = options + setViewport(desktop) + setActivePinia(createPinia()) + const session = useSessionStore() + const sessionRole = role ?? undefined + session.session = { + verified: true, + username: 'marta', + id: 1, + displayName: 'Marta Alves', + role: sessionRole as 'owner' | 'admin' | undefined, + } + session.status = 'ready' + requestMock = vi.fn((path: string, init?: RequestInit) => { + const key = `${init?.method ?? 'GET'} ${path}` + const handler = routes[key] ?? (key === STATS ? () => Promise.resolve(stats()) : null) + if (!handler) { + throw new Error(`Unexpected request: ${key}`) + } + return handler( + init?.body ? (JSON.parse(String(init.body)) as Record<string, unknown>) : undefined, + ) + }) + vi.mocked(useApi).mockReturnValue({ + getMe: vi + .fn() + .mockResolvedValue({ verified: true, username: 'marta', id: 1, role: sessionRole }), + request: requestMock, + }) + await router.push('/settings/media/cache') + const Host = defineComponent({ render: () => h(UApp, null, () => h(CachePage)) }) + wrapper = mount(Host, { attachTo: document.body, global: { plugins: [router, uiPlugin] } }) + await flushPromises() + return wrapper +} + +function calls(key: string) { + return requestMock.mock.calls.filter( + ([path, init]) => `${(init as RequestInit | undefined)?.method ?? 'GET'} ${path}` === key, + ) +} + +function bodyOf(key: string, index = 0) { + return JSON.parse(String((calls(key)[index]![1] as RequestInit).body)) as Record< + string, + unknown + > +} + +function buttonIn(root: ParentNode, text: string) { + return Array.from(root.querySelectorAll('button')).find( + (button) => button.textContent?.trim() === text, + ) +} + +function findButton(text: string) { + return buttonIn(document.body, text) +} + +function buttonByLabel(label: string) { + return document.body.querySelector<HTMLButtonElement>(`button[aria-label="${label}"]`) +} + +async function click(element: HTMLElement | null | undefined) { + element?.click() + await flushPromises() +} + +function textOf(element: Element | null | undefined) { + if (!element) { + return '' + } + const walker = document.createTreeWalker(element, NodeFilter.SHOW_TEXT) + const parts: string[] = [] + while (walker.nextNode()) { + const text = walker.currentNode.textContent?.trim() + if (text) { + parts.push(text) + } + } + return parts.join(' ') +} + +function alertDialog() { + return document.body.querySelector<HTMLElement>('[role="alertdialog"]') +} + +function tiles() { + return Array.from(document.body.querySelectorAll('[data-testid="cache-tile"]')).map((tile) => + textOf(tile), + ) +} + +function toastCalls() { + return toasts.add.mock.calls.map(([source]) => source as Record<string, unknown>) +} + +async function typeWord(value: string) { + const input = alertDialog()!.querySelector<HTMLInputElement>('input')! + input.value = value + input.dispatchEvent(new Event('input', { bubbles: true })) + await flushPromises() +} + +function confirmButton() { + return buttonIn(alertDialog()!, 'Purge all variants')! +} + +async function unmountPage() { + wrapper?.unmount() + wrapper = null + vi.useRealTimers() + document.body.innerHTML = '' + await router.push('/login') +} + +beforeEach(() => { + toasts.add.mockClear() + localStorage.clear() + initializeLocale() + vi.mocked(useApi).mockReset() +}) + +afterEach(async () => { + await unmountPage() + localStorage.clear() + initializeLocale() +}) + +describe('Rule: Cache screen, access and header', () => { + it.each([ + ['no role', null], + ['role owner but the API answers 403', 'owner'], + ])( + 'A session whose role is not owner or admin gets the Forbidden state: %s', + async (_, role) => { + await mountPage({ + role, + routes: role ? { [STATS]: () => Promise.reject(new ApiError(403)) } : {}, + }) + + expect(document.body.textContent).toContain("You don't have access") + expect(document.body.textContent).toContain( + 'Only admins can manage image variants. Ask an admin if you need one changed.', + ) + expect(findButton('Back to Home')).toBeDefined() + expect(document.body.querySelector('[role="tablist"]')).toBeNull() + expect(document.body.querySelector('h1:not([data-slot="title"])')).toBeNull() + expect(calls(STATS)).toHaveLength(role ? 1 : 0) + }, + ) + + it('The header and the tabs', async () => { + await mountPage() + + expect(document.body.querySelector('h1:not([data-slot="title"])')?.textContent).toBe( + 'Cache', + ) + expect(document.body.textContent).toContain( + 'Generated variant files. Purged files are generated again on the next request.', + ) + expect(findButton('Purge unused')).toBeDefined() + expect(findButton('Purge all variants')).toBeDefined() + const tabs = Array.from(document.body.querySelectorAll<HTMLElement>('[role="tab"]')) + expect(tabs.map((tab) => tab.textContent?.trim())).toEqual(['Variants', 'Cache']) + expect(tabs[1]!.getAttribute('aria-selected')).toBe('true') + expect(tabs[1]!.className).toContain('border-primary') + expect(tabs[0]!.getAttribute('href')).toBe('/settings/media/variants') + expect(calls(STATS)).toHaveLength(1) + }) + + it('Activating Variants routes to the variants page', async () => { + await mountPage() + + await click(document.body.querySelector<HTMLElement>('[role="tab"]')) + + await vi.waitFor(() => + expect(router.currentRoute.value.path).toBe('/settings/media/variants'), + ) + }) + + it('The desktop breadcrumb reads Settings, Media, Cache', async () => { + await mountPage() + + const items = Array.from( + document.body.querySelectorAll('nav[aria-label="breadcrumb"] li'), + ).map((item) => item.textContent?.trim()) + expect(items.filter(Boolean)).toEqual(['Settings', 'Media', 'Cache']) + }) +}) + +describe('Rule: Cache screen, loaded', () => { + it('The four stat tiles', async () => { + await mountPage() + + expect(tiles()).toEqual([ + 'Generated files 1,284 Across 6 variants', + 'Disk used 212 MB Originals not included', + 'Variants defined 6 5 enabled, 1 disabled', + 'Rejected requests 37 Last 7 days', + ]) + }) + + it('The By variant list', async () => { + await mountPage() + + const rows = Array.from(document.body.querySelectorAll('[data-testid="cache-variant-row"]')) + expect(rows.map((row) => textOf(row))).toEqual([ + 'hero 244 files · 84 MB', + 'card 318 files · 58 MB', + 'print Disabled 74 files · 21 MB', + 'thumb 412 files · 21 MB', + 'square 140 files · 19 MB', + 'og 96 files · 9 MB', + ]) + const widths = rows.map((row) => + row.querySelector('[role="progressbar"]')?.getAttribute('aria-valuenow'), + ) + expect(widths).toEqual(['100', '69', '25', '25', '23', '11']) + const printBar = rows[2]!.querySelector('[data-slot="indicator"]')! + expect(printBar.className).toContain('border-accented') + expect(rows[0]!.querySelector('[data-slot="indicator"]')!.className).toContain('bg-primary') + expect(rows[0]!.querySelector('span.font-mono')!.className).toContain('font-medium') + }) + + it('The Top 10 list on desktop', async () => { + await mountPage() + + const rows = Array.from(document.body.querySelectorAll('[data-testid="cache-image-row"]')) + expect(rows).toHaveLength(10) + expect(textOf(rows[0])).toBe('1 north-wall.jpg 6 variants 5.8 MB Purge') + expect(textOf(rows[9])).toBe('10 chimney-stack.jpg 3 variants 2.1 MB Purge') + expect(rows[0]!.querySelector('[data-testid="media-thumb"]')).not.toBeNull() + }) + + it('The Top 10 list on mobile', async () => { + await mountPage({ desktop: false }) + + const rows = Array.from(document.body.querySelectorAll('[data-testid="cache-image-row"]')) + expect(textOf(rows[0])).toBe('1 north-wall.jpg 6 variants · 5.8 MB') + const purge = buttonByLabel('Purge north-wall.jpg')! + expect(purge.textContent?.trim()).toBe('') + expect(purge.className).toContain('size-11') + }) + + it('The singular reads 1 file and 1 variant', async () => { + await mountPage({ + routes: { + [STATS]: () => + Promise.resolve( + stats({ + by_variant: [{ name: 'hero', enabled: true, files: 1, bytes: MB }], + top_images: [ + { + id: 1, + filename: 'a.jpg', + url: '/static/uploadedImages/a.jpg', + files: 1, + bytes: MB, + }, + ], + }), + ), + }, + }) + + expect(textOf(document.body.querySelector('[data-testid="cache-variant-row"]'))).toBe( + 'hero 1 file · 1 MB', + ) + expect(textOf(document.body.querySelector('[data-testid="cache-image-row"]'))).toContain( + '1 variant', + ) + }) +}) + +describe('Rule: Cache screen, loading, empty and error', () => { + it('Loading', async () => { + vi.useFakeTimers() + const pending = deferred<CacheStats>() + await mountPage({ routes: { [STATS]: () => pending.promise } }) + + await vi.advanceTimersByTimeAsync(149) + expect(document.body.querySelector('[aria-busy="true"]')).toBeNull() + await vi.advanceTimersByTimeAsync(1) + + const region = document.body.querySelector('[aria-busy="true"]')! + expect(region.getAttribute('aria-label')).toBe('Loading cache') + expect(region.querySelectorAll('[data-testid="cache-tile"]')).toHaveLength(4) + expect(region.querySelectorAll('.animate-pulse').length).toBeGreaterThan(0) + pending.resolve(stats()) + await vi.advanceTimersByTimeAsync(0) + }) + + it('A response at 100 ms never draws skeletons', async () => { + vi.useFakeTimers() + const pending = deferred<CacheStats>() + await mountPage({ routes: { [STATS]: () => pending.promise } }) + + await vi.advanceTimersByTimeAsync(100) + pending.resolve(stats()) + await vi.advanceTimersByTimeAsync(100) + + expect(document.body.querySelector('[aria-busy="true"]')).toBeNull() + expect(tiles()).toHaveLength(4) + }) + + it('Empty cache', async () => { + await mountPage({ routes: { [STATS]: () => Promise.resolve(emptyStats()) } }) + + expect(tiles()).toEqual([ + 'Generated files 0 Nothing cached', + 'Disk used 0 MB Nothing cached', + 'Variants defined 6 5 enabled, 1 disabled', + 'Rejected requests 37 Last 7 days', + ]) + expect(document.body.textContent).toContain('Nothing generated yet') + expect(document.body.textContent).toContain( + "Variants are generated on the first request with ?v=<name>, or from an image's Variants tab.", + ) + expect(findButton('Purge unused')!.disabled).toBe(true) + expect(findButton('Purge all variants')!.disabled).toBe(true) + expect(document.body.querySelector('[data-testid="cache-by-variant"]')).toBeNull() + expect(document.body.querySelector('[data-testid="cache-top-images"]')).toBeNull() + }) + + it.each([ + ['a 500', () => Promise.reject(new ApiError(500))], + ['a network error', () => Promise.reject(new NetworkError())], + ])('Error loading: %s', async (_, failure) => { + let attempt = 0 + await mountPage({ + routes: { [STATS]: () => (attempt++ === 0 ? failure() : Promise.resolve(stats())) }, + }) + + const alert = document.body.querySelector<HTMLElement>('[role="alert"]')! + expect(textOf(alert)).toContain("Couldn't load cache stats") + expect(textOf(alert)).toContain('Check your connection and try again.') + expect(alert.querySelector('[data-slot="close"]')).toBeNull() + expect(tiles()).toHaveLength(0) + + await click(buttonIn(alert, 'Try again')) + + expect(calls(STATS)).toHaveLength(2) + expect(tiles()).toHaveLength(4) + }) + + it('Purge unused is disabled when nothing is unused', async () => { + await mountPage({ + routes: { + [STATS]: () => Promise.resolve(stats({ unused: { files: 0, bytes: 0, days: 30 } })), + }, + }) + + expect(findButton('Purge unused')!.disabled).toBe(true) + expect(findButton('Purge all variants')!.disabled).toBe(false) + }) +}) + +describe('Rule: Cache screen, purge all with a typed word', () => { + it('The dialog opens with the button disabled', async () => { + await mountPage() + + await click(findButton('Purge all variants')) + + const dialog = alertDialog()! + expect(textOf(dialog)).toContain('Purge all variants?') + expect(textOf(dialog)).toContain( + "This removes 1,284 generated files (212 MB) for every variant. Originals aren't touched. Files are generated again on request.", + ) + expect(textOf(dialog)).toContain('Type purge to confirm') + expect(dialog.querySelector('input')!.className).toContain('font-mono') + expect(buttonIn(dialog, 'Cancel')).toBeDefined() + expect(confirmButton().disabled).toBe(true) + expect(calls(PURGE)).toHaveLength(0) + }) + + it.each([ + ['pur', true], + ['Purge', true], + ['purge', false], + ['purged', true], + ])('The word unlocks the button: %s', async (typed, disabled) => { + await mountPage() + await click(findButton('Purge all variants')) + + await typeWord(typed) + + expect(confirmButton().disabled).toBe(disabled) + }) + + it('Cancel sends nothing', async () => { + await mountPage() + await click(findButton('Purge all variants')) + + await click(buttonIn(alertDialog()!, 'Cancel')) + + await vi.waitFor(() => expect(alertDialog()).toBeNull()) + expect(calls(PURGE)).toHaveLength(0) + }) + + it('Confirming sends the purge', async () => { + await mountPage({ + routes: { + [PURGE]: () => + Promise.resolve({ + scope: 'all', + purged_files: 1284, + purged_bytes: 222298112, + failed_files: 0, + }), + }, + }) + await click(findButton('Purge all variants')) + await typeWord('purge') + + await click(confirmButton()) + + await vi.waitFor(() => expect(calls(PURGE)).toHaveLength(1)) + expect(bodyOf(PURGE)).toEqual({ scope: 'all' }) + }) +}) + +describe('Rule: Cache screen, purge unused', () => { + it('The dialog states the count from the stats and confirming sends the purge', async () => { + await mountPage({ + routes: { + [PURGE]: () => + Promise.resolve({ + scope: 'unused', + purged_files: 312, + purged_bytes: 42991616, + failed_files: 0, + }), + }, + }) + + await click(findButton('Purge unused')) + + const dialog = alertDialog()! + expect(textOf(dialog)).toContain('Purge unused variants?') + expect(textOf(dialog)).toContain( + "This removes 312 files (41 MB) from enabled variants that haven't been requested for 30 days. Files are generated again on request.", + ) + expect(dialog.querySelector('input')).toBeNull() + expect(buttonIn(dialog, 'Cancel')).toBeDefined() + + await click(buttonIn(dialog, 'Purge 312 files')) + + await vi.waitFor(() => expect(calls(PURGE)).toHaveLength(1)) + expect(bodyOf(PURGE)).toEqual({ scope: 'unused' }) + }) +}) + +describe('Rule: Cache screen, purging, done and failed', () => { + async function startPurgeAll(routes: Record<string, Handler>) { + await mountPage({ routes }) + await click(findButton('Purge all variants')) + await typeWord('purge') + await click(confirmButton()) + } + + it('Purging', async () => { + const pending = deferred<unknown>() + await startPurgeAll({ [PURGE]: () => pending.promise }) + + const banner = document.body.querySelector('[data-testid="cache-purge-banner"]')! + expect(textOf(banner)).toContain('Purging 1,284 files') + expect(banner.getAttribute('aria-live')).toBe('polite') + expect(banner.querySelector('[role="progressbar"]')).not.toBeNull() + expect(findButton('Purge unused')!.disabled).toBe(true) + expect(findButton('Purge all variants')!.disabled).toBe(true) + expect(buttonByLabel('Purge north-wall.jpg')!.disabled).toBe(true) + expect(document.body.querySelector('.opacity-60')).not.toBeNull() + expect(document.body.querySelector('[role="tablist"]')).not.toBeNull() + }) + + it('Purge done after purge all', async () => { + let reads = 0 + await startPurgeAll({ + [STATS]: () => Promise.resolve(reads++ === 0 ? stats() : emptyStats()), + [PURGE]: () => + Promise.resolve({ + scope: 'all', + purged_files: 1284, + purged_bytes: 222298112, + failed_files: 0, + }), + }) + + await vi.waitFor(() => expect(toastCalls()).toHaveLength(1)) + expect(toastCalls()[0]).toMatchObject({ + color: 'success', + title: 'Purged 1,284 files, freed 212 MB', + }) + await vi.waitFor(() => expect(calls(STATS)).toHaveLength(2)) + await vi.waitFor(() => expect(document.body.textContent).toContain('Nothing generated yet')) + }) + + it('Purge done after purge unused', async () => { + await mountPage({ + routes: { + [PURGE]: () => + Promise.resolve({ + scope: 'unused', + purged_files: 312, + purged_bytes: 42991616, + failed_files: 0, + }), + }, + }) + await click(findButton('Purge unused')) + await click(buttonIn(alertDialog()!, 'Purge 312 files')) + + await vi.waitFor(() => expect(toastCalls()).toHaveLength(1)) + expect(toastCalls()[0]).toMatchObject({ title: 'Purged 312 files, freed 41 MB' }) + await vi.waitFor(() => expect(calls(STATS)).toHaveLength(2)) + }) + + it('Purge failed offers Try again without asking for the word again', async () => { + await startPurgeAll({ + [PURGE]: () => + Promise.resolve({ + scope: 'all', + purged_files: 640, + purged_bytes: 111149056, + failed_files: 644, + }), + }) + + await vi.waitFor(() => expect(toastCalls()).toHaveLength(1)) + const toast = toastCalls()[0]! + expect(toast).toMatchObject({ + color: 'error', + title: "Purge didn't finish", + description: '640 of 1,284 files were removed. Try again.', + }) + await vi.waitFor(() => expect(calls(STATS)).toHaveLength(2)) + const [action] = toast.actions as { label: string; onClick: () => void }[] + expect(action!.label).toBe('Try again') + + action!.onClick() + await flushPromises() + + await vi.waitFor(() => expect(calls(PURGE)).toHaveLength(2)) + expect(bodyOf(PURGE, 1)).toEqual({ scope: 'all' }) + expect(alertDialog()).toBeNull() + }) + + it.each([ + [ + 'a 400', + new ApiError(400), + 'Check the request', + "A value in the request isn't valid. Reload the page and try again.", + ], + [ + 'a 403', + new ApiError(403), + "You don't have access", + 'Only admins can manage image variants.', + ], + [ + 'a 500', + new ApiError(500), + 'Something went wrong', + "The server couldn't finish. Try again. If it keeps happening, check the server log.", + ], + [ + 'a network error', + new NetworkError(), + 'Something went wrong', + "The server couldn't finish. Try again. If it keeps happening, check the server log.", + ], + ])('A purge request that errors: %s', async (_, failure, title, description) => { + await startPurgeAll({ [PURGE]: () => Promise.reject(failure) }) + + await vi.waitFor(() => expect(toastCalls()).toHaveLength(1)) + expect(toastCalls()[0]).toMatchObject({ color: 'error', title, description }) + expect(document.body.querySelector('[data-testid="cache-purge-banner"]')).toBeNull() + expect(tiles()).toHaveLength(4) + expect(calls(STATS)).toHaveLength(1) + }) + + it('A 401 is left to the API client', async () => { + await startPurgeAll({ [PURGE]: () => Promise.reject(new ApiError(401)) }) + + expect(toastCalls()).toHaveLength(0) + expect(document.body.querySelector('[data-testid="cache-purge-banner"]')).toBeNull() + }) +}) + +describe('Rule: Cache screen, purging the variants of one image', () => { + it('A row Purge sends the image scope and is done', async () => { + await mountPage({ + routes: { + [PURGE]: () => + Promise.resolve({ + scope: 'image', + purged_files: 6, + purged_bytes: 6082765, + failed_files: 0, + }), + }, + }) + + await click(buttonByLabel('Purge north-wall.jpg')) + + await vi.waitFor(() => expect(toastCalls()).toHaveLength(1)) + expect(bodyOf(PURGE)).toEqual({ scope: 'image', image_id: 1 }) + expect(alertDialog()).toBeNull() + expect(toastCalls()[0]).toMatchObject({ + title: 'Purged 6 files (5.8 MB) for north-wall.jpg', + }) + await vi.waitFor(() => expect(calls(STATS)).toHaveLength(2)) + }) + + it('While it runs the row shows a loader and the other Purge buttons stay enabled', async () => { + const pending = deferred<unknown>() + await mountPage({ routes: { [PURGE]: () => pending.promise } }) + + await click(buttonByLabel('Purge north-wall.jpg')) + + expect(buttonByLabel('Purge north-wall.jpg')!.querySelector('.animate-spin')).not.toBeNull() + expect(buttonByLabel('Purge lime-plaster.jpg')!.disabled).toBe(false) + expect(findButton('Purge all variants')!.disabled).toBe(false) + }) + + it('The image no longer exists', async () => { + await mountPage({ routes: { [PURGE]: () => Promise.reject(new ApiError(404)) } }) + + await click(buttonByLabel('Purge north-wall.jpg')) + + await vi.waitFor(() => expect(toastCalls()).toHaveLength(1)) + expect(toastCalls()[0]).toMatchObject({ + title: 'Not found', + description: 'This variant or image no longer exists. Refresh the page.', + }) + await vi.waitFor(() => expect(calls(STATS)).toHaveLength(2)) + }) + + it('Failed', async () => { + await mountPage({ + routes: { + [PURGE]: () => + Promise.resolve({ + scope: 'image', + purged_files: 4, + purged_bytes: 1000, + failed_files: 2, + }), + }, + }) + + await click(buttonByLabel('Purge north-wall.jpg')) + + await vi.waitFor(() => expect(toastCalls()).toHaveLength(1)) + expect(toastCalls()[0]).toMatchObject({ + title: "Purge didn't finish", + description: '4 of 6 files were removed. Try again.', + }) + }) +}) + +describe('Rule: Accessibility and vocabulary', () => { + it('Roles, labels and vocabulary', async () => { + await mountPage() + + expect(document.body.querySelector('[role="tablist"]')).not.toBeNull() + expect(document.body.querySelectorAll('[role="tab"]')).toHaveLength(2) + const text = document.body.textContent ?? '' + for (const banned of ['thumbnail preset', 'version', 'rendition']) { + expect(text).not.toContain(banned) + } + }) +}) diff --git a/web/src/views/__tests__/VariantsPage.spec.ts b/web/src/views/__tests__/VariantsPage.spec.ts index 84db948..ef06be7 100644 --- a/web/src/views/__tests__/VariantsPage.spec.ts +++ b/web/src/views/__tests__/VariantsPage.spec.ts @@ -166,19 +166,18 @@ describe('Rule: Header and tabs', () => { expect(tabs[0]!.className).toContain('font-medium') expect(iconOf(tabs[0]!)).toBe('i-lucide-layers') expect(iconOf(tabs[1]!)).toBe('i-lucide-database') - expect(tabs[1]!.getAttribute('aria-disabled')).toBe('true') + expect(tabs[1]!.getAttribute('aria-selected')).toBe('false') + expect(tabs[1]!.getAttribute('href')).toBe('/settings/media/cache') expect(tabs[1]!.className).toContain('text-toned') }) - it('Activating Cache changes nothing', async () => { + it('Activating Cache routes to the Cache page', async () => { await mountPage() - const before = router.currentRoute.value.fullPath const cache = document.body.querySelectorAll<HTMLElement>('[role="tab"]')[1]! await click(cache) - expect(router.currentRoute.value.fullPath).toBe(before) - expect(cache.getAttribute('aria-selected')).toBe('false') + await vi.waitFor(() => expect(router.currentRoute.value.path).toBe('/settings/media/cache')) expect(calls(LIST)).toHaveLength(1) }) diff --git a/web/visual/check.ts b/web/visual/check.ts index cdb2ff9..604bcb3 100644 --- a/web/visual/check.ts +++ b/web/visual/check.ts @@ -274,7 +274,20 @@ function variantsScreen(): Screen { '6d · Variant deleted', '6e · Variant disabled', ] + const cacheNames = [ + '5a · Loaded', + '5b · Loading', + '5c · Empty cache', + '5d · Purging', + '5e · Purge done', + '5f · Purge failed', + '5g · Error loading', + '5h · Purge all: confirm', + '5i · Purge all: confirm, typed', + '5j · Purge unused: confirm', + ] sources.push( + { board: variantsBoard, group: 'Cache panel', states: cacheNames, alias: {} }, { board: variantsBoard, group: 'Media library', states: libraryNames, alias: {} }, { board: variantsBoard, group: 'Image detail', states: detailNames, alias: {} }, { board: variantsBoard, group: 'Editor picker', states: editorNames, alias: {} }, @@ -283,7 +296,14 @@ function variantsScreen(): Screen { name: 'image-variants', board: variantsBoard, group: 'Variants list', - states: [...listNames, ...formNames, ...libraryNames, ...detailNames, ...editorNames], + states: [ + ...listNames, + ...formNames, + ...cacheNames, + ...libraryNames, + ...detailNames, + ...editorNames, + ], sources, } } diff --git a/web/visual/states.ts b/web/visual/states.ts index df4fa9e..f443072 100644 --- a/web/visual/states.ts +++ b/web/visual/states.ts @@ -3412,6 +3412,139 @@ const variantsStates: Record<string, VisualState> = { }), } +const cachePath = '/settings/media/cache' + +const cacheBoardImages: [string, number, number][] = [ + ['north-wall.jpg', 6, 6082765], + ['lime-plaster.jpg', 6, 5138022], + ['kitchen-after.webp', 5, 4613734], + ['kitchen-before.webp', 5, 4089446], + ['floorboards.jpg', 5, 3774873], + ['IMG_4471.jpg', 4, 3250585], + ['stone-wall-detail.jpg', 4, 2936012], + ['bathroom-tiles.png', 4, 2726297], + ['roof-lantern.jpg', 3, 2411724], + ['chimney-stack.jpg', 3, 2202009], +] + +const cacheBoardStats = { + generated_files: 1284, + disk_used: 222298112, + variants_defined: 6, + variants_enabled: 5, + variants_disabled: 1, + rejected_requests: 37, + rejected_window_days: 7, + unused: { files: 312, bytes: 42991616, days: 30 }, + by_variant: [ + { name: 'hero', enabled: true, files: 244, bytes: 88080384 }, + { name: 'card', enabled: true, files: 318, bytes: 60817408 }, + { name: 'print', enabled: false, files: 74, bytes: 22020096 }, + { name: 'thumb', enabled: true, files: 412, bytes: 22020096 }, + { name: 'square', enabled: true, files: 140, bytes: 19922944 }, + { name: 'og', enabled: true, files: 96, bytes: 9437184 }, + ], + top_images: cacheBoardImages.map(([filename, files, bytes], index) => ({ + id: index + 1, + filename, + url: `/static/uploadedImages/${filename}`, + files, + bytes, + })), +} + +const cacheEmptyStats = { + ...cacheBoardStats, + generated_files: 0, + disk_used: 0, + unused: { files: 0, bytes: 0, days: 30 }, + by_variant: [], + top_images: [], +} + +const cacheStatsKey = 'GET /v1/image-variants/cache' +const cachePurgeKey = 'POST /v1/image-variants/cache/purge' +const cachePurgeAllResult = { + scope: 'all', + purged_files: 1284, + purged_bytes: 222298112, + failed_files: 0, +} + +function cacheState( + statsMock: ApiMock = { status: 200, body: cacheBoardStats }, + overrides: Record<string, ApiMock> = {}, + steps?: VisualState['steps'], +): VisualState { + return { + path: cachePath, + api: { 'GET /v1/auth/me': variantsSession, [cacheStatsKey]: statsMock, ...overrides }, + steps, + } +} + +async function confirmPurgeAll(page: Page) { + await page.getByRole('button', { name: 'Purge all variants', exact: true }).first().click() + await page.getByRole('alertdialog').getByRole('textbox').fill('purge') + await page + .getByRole('alertdialog') + .getByRole('button', { name: 'Purge all variants', exact: true }) + .click() + await page.mouse.move(0, 0) +} + +const cacheStates: Record<string, VisualState> = { + '5a · Loaded': cacheState(), + '5b · Loading': cacheState({ status: 200, hang: true }), + '5c · Empty cache': cacheState({ status: 200, body: cacheEmptyStats }), + '5d · Purging': cacheState( + { status: 200, body: cacheBoardStats }, + { [cachePurgeKey]: { status: 200, hang: true } }, + confirmPurgeAll, + ), + '5e · Purge done': cacheState( + { + status: 200, + bodyFor: (_url, call) => (call === 0 ? cacheBoardStats : cacheEmptyStats), + }, + { [cachePurgeKey]: { status: 200, body: cachePurgeAllResult } }, + confirmPurgeAll, + ), + '5f · Purge failed': cacheState( + { status: 200, body: cacheBoardStats }, + { + [cachePurgeKey]: { + status: 200, + body: { + ...cachePurgeAllResult, + purged_files: 640, + purged_bytes: 111149056, + failed_files: 644, + }, + }, + }, + confirmPurgeAll, + ), + '5g · Error loading': cacheState({ status: 500 }), + '5h · Purge all: confirm': cacheState(undefined, {}, async (page) => { + await page.getByRole('button', { name: 'Purge all variants', exact: true }).first().click() + await page.getByRole('alertdialog').waitFor() + await page.getByRole('alertdialog').getByRole('textbox').fill('pur') + await page.mouse.move(0, 0) + }), + '5i · Purge all: confirm, typed': cacheState(undefined, {}, async (page) => { + await page.getByRole('button', { name: 'Purge all variants', exact: true }).first().click() + await page.getByRole('alertdialog').waitFor() + await page.getByRole('alertdialog').getByRole('textbox').fill('purge') + await page.mouse.move(0, 0) + }), + '5j · Purge unused: confirm': cacheState(undefined, {}, async (page) => { + await page.getByRole('button', { name: 'Purge unused', exact: true }).first().click() + await page.getByRole('alertdialog').waitFor() + await page.mouse.move(0, 0) + }), +} + const variantBoardSizes: Record<string, number> = { thumb: 18432, card: 65536, @@ -4929,6 +5062,7 @@ export const visualStates: Record<string, Record<string, VisualState>> = { }, 'image-variants': { ...variantsStates, + ...cacheStates, ...withVariantExceptions(variantsMediaStates), ...withVariantExceptions(variantsEditorStates), },