1005 lines
35 KiB
Go
1005 lines
35 KiB
Go
package test
|
|
|
|
import (
|
|
"RenewCMS/api/pkg/apperr"
|
|
domain "RenewCMS/internal/domain/image"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
cacheStatsPath = "/v1/image-variants/cache"
|
|
cachePurgePath = "/v1/image-variants/cache/purge"
|
|
)
|
|
|
|
func TestImageVariantsCacheGuards(t *testing.T) {
|
|
rows := []struct {
|
|
name string
|
|
method string
|
|
path string
|
|
credentials string
|
|
status int
|
|
body string
|
|
}{
|
|
{"no cookie", http.MethodGet, cacheStatsPath, "none", 401, apperr.UnauthorizedErrorMessage},
|
|
{"a stale token_version cookie", http.MethodGet, cacheStatsPath, "stale", 401, apperr.UnauthorizedErrorMessage},
|
|
{"the cookie of an unverified user", http.MethodGet, cacheStatsPath, "unverified", 403, apperr.VerificationRequiredErrorMessage},
|
|
{"the cookie of a plain user", http.MethodGet, cacheStatsPath, "plain", 403, apperr.OwnerOrAdminRequiredErrorMessage},
|
|
{"the cookie of a plain user on purge", http.MethodPost, cachePurgePath, "plain", 403, apperr.OwnerOrAdminRequiredErrorMessage},
|
|
{"a valid API key", http.MethodGet, cacheStatsPath, "apikey", 401, apperr.UnauthorizedErrorMessage},
|
|
{"a valid API key on purge", http.MethodPost, cachePurgePath, "apikey", 401, apperr.UnauthorizedErrorMessage},
|
|
}
|
|
for _, row := range rows {
|
|
t.Run("Every cache route is guarded: "+row.name, func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
seedVariantFiles(env.files, env.storage, "hero", "x", 2, 10)
|
|
callsBefore, variantCallsBefore := env.files.calls, env.variants.calls
|
|
|
|
request := httptest.NewRequest(row.method, row.path, strings.NewReader(`{"scope": "all"}`))
|
|
switch row.credentials {
|
|
case "stale":
|
|
request.AddCookie(env.cookie(variantStaleId, 0))
|
|
case "unverified":
|
|
request.AddCookie(env.cookie(variantUnverifiedId, 0))
|
|
case "plain":
|
|
request.AddCookie(env.cookie(variantPlainUserId, 0))
|
|
case "apikey":
|
|
request.Header = bearerKeyHeader()
|
|
}
|
|
recorder := httptest.NewRecorder()
|
|
env.admin.ServeHTTP(recorder, request)
|
|
|
|
assertStatusAndBody(t, recorder, row.status, row.body)
|
|
if env.files.calls != callsBefore || env.variants.calls != variantCallsBefore || env.files.rowCount() != 2 || len(env.storage.deletedNames()) != 0 || env.usage.callCount() != 0 || env.events() != 0 {
|
|
t.Error("an index row, blob, counter or activity event was touched")
|
|
}
|
|
})
|
|
}
|
|
|
|
for name, userId := range map[string]uint32{"the owner": variantOwnerId, "an admin": variantAdminId} {
|
|
t.Run("Owners and admins are both allowed: "+name, func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
|
|
recorder := env.send(http.MethodGet, cacheStatsPath, "", userId)
|
|
|
|
if recorder.Code != http.StatusOK {
|
|
t.Errorf("status = %d", recorder.Code)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func assertNumber(t *testing.T, body map[string]any, key string, want float64) {
|
|
t.Helper()
|
|
if body[key] != want {
|
|
t.Errorf("%s = %v, want %v", key, body[key], want)
|
|
}
|
|
}
|
|
|
|
func TestImageVariantsCacheStatistics(t *testing.T) {
|
|
t.Run("The Default fixture", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
env.seedRejectedDays(map[string]int{"2026-10-08": 30, "2026-10-05": 7, "2026-09-20": 100})
|
|
|
|
recorder, body := env.cacheStats()
|
|
|
|
if recorder.Code != 200 {
|
|
t.Fatalf("status = %d %s", recorder.Code, recorder.Body.String())
|
|
}
|
|
assertNumber(t, body, "generated_files", 1284)
|
|
assertNumber(t, body, "disk_used", 222298112)
|
|
assertNumber(t, body, "variants_defined", 6)
|
|
assertNumber(t, body, "variants_enabled", 5)
|
|
assertNumber(t, body, "variants_disabled", 1)
|
|
assertNumber(t, body, "rejected_requests", 37)
|
|
assertNumber(t, body, "rejected_window_days", 7)
|
|
if got := marshalled(t, body["unused"]); got != `{"bytes":42991616,"days":30,"files":312}` {
|
|
t.Errorf("unused = %s", got)
|
|
}
|
|
byVariant := body["by_variant"].([]any)
|
|
wantNames := []string{"hero", "card", "print", "thumb", "square", "og"}
|
|
if len(byVariant) != len(wantNames) {
|
|
t.Fatalf("by_variant = %v", byVariant)
|
|
}
|
|
for index, name := range wantNames {
|
|
entry := byVariant[index].(map[string]any)
|
|
if entry["name"] != name || entry["enabled"] != (name != "print") {
|
|
t.Errorf("by_variant[%d] = %v", index, entry)
|
|
}
|
|
}
|
|
if got := marshalled(t, byVariant[0]); got != `{"bytes":88080384,"enabled":true,"files":244,"name":"hero"}` {
|
|
t.Errorf("hero = %s", got)
|
|
}
|
|
top := body["top_images"].([]any)
|
|
if len(top) != 10 {
|
|
t.Fatalf("top_images = %d", len(top))
|
|
}
|
|
for index, img := range cacheFixtureTopImages {
|
|
entry := top[index].(map[string]any)
|
|
if entry["id"] != float64(img.id) || entry["filename"] != img.filename || entry["url"] != "/static/uploadedImages/"+img.filename || entry["files"] != float64(img.files) || entry["bytes"] != float64(img.bytes) {
|
|
t.Errorf("top_images[%d] = %v", index, entry)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("Counts are the sum of the index rows", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
files, bytes := 0.0, 0.0
|
|
for _, entry := range body["by_variant"].([]any) {
|
|
files += entry.(map[string]any)["files"].(float64)
|
|
bytes += entry.(map[string]any)["bytes"].(float64)
|
|
}
|
|
if body["generated_files"] != files || body["disk_used"] != bytes {
|
|
t.Errorf("totals = %v %v, sums = %v %v", body["generated_files"], body["disk_used"], files, bytes)
|
|
}
|
|
})
|
|
|
|
t.Run("A variant without a file is listed with zero", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
seedVariant(env.variants, "banner", 900, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
assertNumber(t, body, "variants_defined", 7)
|
|
found := false
|
|
for _, entry := range body["by_variant"].([]any) {
|
|
if marshalled(t, entry) == `{"bytes":0,"enabled":true,"files":0,"name":"banner"}` {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Errorf("by_variant = %v", body["by_variant"])
|
|
}
|
|
})
|
|
|
|
t.Run("A disabled variant keeps its files in the totals", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
print := body["by_variant"].([]any)[2].(map[string]any)
|
|
if print["name"] != "print" || print["enabled"] != false || print["files"] != 74.0 || print["bytes"] != 22020096.0 {
|
|
t.Errorf("print = %v", print)
|
|
}
|
|
assertNumber(t, body, "generated_files", 1284)
|
|
assertNumber(t, body, "disk_used", 222298112)
|
|
if got := marshalled(t, body["unused"]); got != `{"bytes":42991616,"days":30,"files":312}` {
|
|
t.Errorf("unused = %s", got)
|
|
}
|
|
})
|
|
|
|
t.Run("An image deleted behind the index is not a top image", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
env.seedCacheRow("hero", "orphan-giant.jpg", 50000000, env.now)
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
assertNumber(t, body, "generated_files", 1285)
|
|
for _, entry := range body["top_images"].([]any) {
|
|
if entry.(map[string]any)["url"] == "/static/uploadedImages/orphan-giant.jpg" {
|
|
t.Error("the orphan is listed")
|
|
}
|
|
}
|
|
if len(body["top_images"].([]any)) != 10 {
|
|
t.Errorf("top_images = %d", len(body["top_images"].([]any)))
|
|
}
|
|
})
|
|
|
|
t.Run("Top images are capped at ten and tie-break by id", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
sizes := map[uint32]int64{1: 1100, 2: 1000, 3: 900, 4: 800, 5: 700, 6: 600, 7: 500, 8: 500, 9: 400, 10: 300, 11: 200}
|
|
names := map[uint32]string{7: "z7.png", 8: "a8.png"}
|
|
for id, size := range sizes {
|
|
name := names[id]
|
|
if name == "" {
|
|
name = "m" + string(rune('a'+id)) + ".png"
|
|
}
|
|
env.seedImage(id, name)
|
|
env.seedCacheRow("hero", name, size, env.now)
|
|
}
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
top := body["top_images"].([]any)
|
|
if len(top) != 10 || top[6].(map[string]any)["id"] != 7.0 || top[7].(map[string]any)["id"] != 8.0 {
|
|
t.Errorf("top_images = %v", top)
|
|
}
|
|
})
|
|
|
|
seedTiedTop := func(env *variantEnv, tied map[uint32]string) {
|
|
seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
for id := uint32(1); id <= 9; id++ {
|
|
name := "m" + string(rune('a'+id)) + ".png"
|
|
env.seedImage(id, name)
|
|
env.seedCacheRow("hero", name, int64(2000-100*id), env.now)
|
|
}
|
|
for id, name := range tied {
|
|
env.seedImage(id, name)
|
|
env.seedCacheRow("hero", name, 300, env.now)
|
|
}
|
|
}
|
|
|
|
t.Run("Exactly eleven sources tied at the cut are fetched again", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
seedTiedTop(env, map[uint32]string{10: "b10.png", 11: "a11.png"})
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
top := body["top_images"].([]any)
|
|
if len(top) != 10 || top[9].(map[string]any)["id"] != 10.0 || env.files.topCalls != 2 {
|
|
t.Errorf("top_images = %v, fetches = %d", top, env.files.topCalls)
|
|
}
|
|
})
|
|
|
|
t.Run("A tie across the cut widens the fetch so the lower id wins", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
seedTiedTop(env, map[uint32]string{10: "c10.png", 11: "b11.png", 12: "a12.png"})
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
top := body["top_images"].([]any)
|
|
if len(top) != 10 || top[9].(map[string]any)["id"] != 10.0 || env.files.topCalls != 2 {
|
|
t.Errorf("top_images = %v, fetches = %d", top, env.files.topCalls)
|
|
}
|
|
})
|
|
|
|
t.Run("An empty cache", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
for _, row := range env.files.rowsOf("hero") {
|
|
_ = env.files.Delete(row.ID)
|
|
}
|
|
for _, name := range []string{"card", "thumb", "print", "square", "og"} {
|
|
for _, row := range env.files.rowsOf(name) {
|
|
_ = env.files.Delete(row.ID)
|
|
}
|
|
}
|
|
env.seedRejectedDays(map[string]int{"2026-10-08": 37})
|
|
|
|
recorder, body := env.cacheStats()
|
|
|
|
assertNumber(t, body, "generated_files", 0)
|
|
assertNumber(t, body, "disk_used", 0)
|
|
assertNumber(t, body, "variants_defined", 6)
|
|
assertNumber(t, body, "rejected_requests", 37)
|
|
if got := marshalled(t, body["unused"]); got != `{"bytes":0,"days":30,"files":0}` {
|
|
t.Errorf("unused = %s", got)
|
|
}
|
|
if !strings.Contains(recorder.Body.String(), `"top_images":[]`) {
|
|
t.Errorf("body = %s", recorder.Body.String())
|
|
}
|
|
for _, entry := range body["by_variant"].([]any) {
|
|
if entry.(map[string]any)["files"] != 0.0 {
|
|
t.Errorf("entry = %v", entry)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("No variant defined", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
|
|
recorder, body := env.cacheStats()
|
|
|
|
if !strings.Contains(recorder.Body.String(), `"by_variant":[]`) {
|
|
t.Errorf("body = %s", recorder.Body.String())
|
|
}
|
|
assertNumber(t, body, "variants_defined", 0)
|
|
})
|
|
|
|
t.Run("Unused counts only enabled variants older than the cutoff", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
for _, name := range []string{"hero", "card"} {
|
|
seedVariant(env.variants, name, 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
}
|
|
print := seedVariant(env.variants, "print", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
print.Enabled = false
|
|
env.variants.variants["print"] = print
|
|
env.seedCacheRow("hero", "recent.jpg", 10, env.now.AddDate(0, 0, -29))
|
|
env.seedCacheRow("hero", "old.jpg", 20, env.now.AddDate(0, 0, -31))
|
|
env.seedCacheRow("print", "disabled.jpg", 40, env.now.AddDate(0, 0, -90))
|
|
env.seedCacheRow("card", "never-requested.jpg", 80, env.now.AddDate(0, 0, -45))
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
if got := marshalled(t, body["unused"]); got != `{"bytes":100,"days":30,"files":2}` {
|
|
t.Errorf("unused = %s", got)
|
|
}
|
|
})
|
|
|
|
t.Run("A row requested exactly 30 days ago is still in use", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
env.seedCacheRow("hero", "edge.jpg", 10, env.now.AddDate(0, 0, -30))
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
if got := marshalled(t, body["unused"]); got != `{"bytes":0,"days":30,"files":0}` {
|
|
t.Errorf("unused = %s", got)
|
|
}
|
|
})
|
|
|
|
t.Run("The statistics are aggregated, not read per image", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
env.cacheStats()
|
|
|
|
if env.files.totalsCalls != 1 || env.files.unusedCalls != 1 || env.files.topCalls != 1 || env.images.storedNameLookups != 1 {
|
|
t.Errorf("calls = %d totals, %d unused, %d top, %d lookups", env.files.totalsCalls, env.files.unusedCalls, env.files.topCalls, env.images.storedNameLookups)
|
|
}
|
|
})
|
|
|
|
t.Run("The read flushes pending tracking first", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
env.seedRejectedDays(map[string]int{"2026-10-08": 37})
|
|
for index := 0; index < 12; index++ {
|
|
env.tracker.RecordHit("hero", "pending-"+string(rune('a'+index)))
|
|
}
|
|
for index := 0; index < 3; index++ {
|
|
env.tracker.RecordRejection()
|
|
}
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
if env.usage.batchCount() != 1 || len(env.usage.recordedHits()) != 12 {
|
|
t.Errorf("batches = %d, hits = %d", env.usage.batchCount(), len(env.usage.recordedHits()))
|
|
}
|
|
assertNumber(t, body, "rejected_requests", 40)
|
|
})
|
|
|
|
t.Run("The statistics change nothing else", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
rowsBefore := env.files.rowCount()
|
|
|
|
env.cacheStats()
|
|
|
|
if env.storage.opens != 0 || env.renderer.renderCount() != 0 || env.files.rowCount() != rowsBefore || len(env.storage.deletedNames()) != 0 || env.events() != 0 {
|
|
t.Error("a blob was opened, a render happened, a row was deleted or an event was recorded")
|
|
}
|
|
})
|
|
|
|
t.Run("A failing repository answers 500", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.files.failTotals = errors.New("totals down")
|
|
|
|
recorder, _ := env.cacheStats()
|
|
|
|
assertStatusAndBody(t, recorder, 500, "Internal server error")
|
|
})
|
|
|
|
t.Run("The window is today and the six previous UTC days", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.now = time.Date(2026, time.October, 8, 23, 59, 0, 0, time.UTC)
|
|
env.seedRejectedDays(map[string]int{"2026-10-02": 5, "2026-10-01": 9})
|
|
|
|
_, body := env.cacheStats()
|
|
|
|
assertNumber(t, body, "rejected_requests", 5)
|
|
})
|
|
}
|
|
|
|
func TestImageVariantsCacheTracking(t *testing.T) {
|
|
trackedEnv := func(t *testing.T) *variantEnv {
|
|
t.Helper()
|
|
env := newVariantEnv(t)
|
|
env.seedThumb()
|
|
env.seedPhoto()
|
|
return env
|
|
}
|
|
thumbETag := func(env *variantEnv) string {
|
|
return domain.VariantETag(env.variants.stored("thumb").Settings(), variantSourceName)
|
|
}
|
|
hitsAfterFlush := func(env *variantEnv) []domain.VariantHit {
|
|
_ = env.tracker.Flush()
|
|
return env.usage.recordedHits()
|
|
}
|
|
|
|
t.Run("What counts as a request for a file", func(t *testing.T) {
|
|
t.Run("the public client gets the file cached", func(t *testing.T) {
|
|
env := trackedEnv(t)
|
|
env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil)
|
|
_ = env.tracker.Flush()
|
|
env.now = env.now.Add(time.Minute)
|
|
|
|
recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil)
|
|
hits := hitsAfterFlush(env)
|
|
|
|
if recorder.Code != 200 || len(hits) != 2 || hits[1].VariantName != "thumb" || hits[1].SourceName != variantSourceName || !hits[1].At.Equal(env.now) {
|
|
t.Errorf("status = %d, hits = %+v", recorder.Code, hits)
|
|
}
|
|
})
|
|
|
|
t.Run("the public client gets it and the file is generated by that request", func(t *testing.T) {
|
|
env := trackedEnv(t)
|
|
|
|
recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil)
|
|
hits := hitsAfterFlush(env)
|
|
|
|
if recorder.Code != 200 || env.storage.putCount() != 1 || len(hits) != 1 {
|
|
t.Errorf("status = %d, puts = %d, hits = %+v", recorder.Code, env.storage.putCount(), hits)
|
|
}
|
|
})
|
|
|
|
t.Run("the public client gets it with a matching If-None-Match and receives 304", func(t *testing.T) {
|
|
env := trackedEnv(t)
|
|
|
|
recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", http.Header{"If-None-Match": []string{thumbETag(env)}})
|
|
hits := hitsAfterFlush(env)
|
|
|
|
if recorder.Code != 304 || len(hits) != 1 {
|
|
t.Errorf("status = %d, hits = %+v", recorder.Code, hits)
|
|
}
|
|
})
|
|
|
|
unrecorded := map[string]string{
|
|
"the public client gets it with no variant": "/static/uploadedImages/" + variantSourceName,
|
|
"the public client gets it with ?v=unknown": "/static/uploadedImages/" + variantSourceName + "?v=unknown",
|
|
"the public client gets it with a disabled variant": "/static/uploadedImages/" + variantSourceName + "?v=print",
|
|
"the public client gets a missing file": "/static/uploadedImages/missing.png?v=thumb",
|
|
}
|
|
for name, url := range unrecorded {
|
|
t.Run(name, func(t *testing.T) {
|
|
env := trackedEnv(t)
|
|
print := seedVariant(env.variants, "print", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
print.Enabled = false
|
|
env.variants.variants["print"] = print
|
|
|
|
env.get(url, nil)
|
|
hits := hitsAfterFlush(env)
|
|
|
|
if len(hits) != 0 {
|
|
t.Errorf("hits = %+v", hits)
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("an admin sends POST /v1/media/1/variants/thumb", func(t *testing.T) {
|
|
env := trackedEnv(t)
|
|
|
|
recorder := env.sendAsAdmin(http.MethodPost, "/v1/media/1/variants/thumb", "")
|
|
hits := hitsAfterFlush(env)
|
|
|
|
if recorder.Code != 200 || len(hits) != 0 {
|
|
t.Errorf("status = %d, hits = %+v", recorder.Code, hits)
|
|
}
|
|
})
|
|
|
|
t.Run("an admin sends GET /v1/image-variants/cache", func(t *testing.T) {
|
|
env := trackedEnv(t)
|
|
|
|
env.cacheStats()
|
|
|
|
if len(env.usage.recordedHits()) != 0 {
|
|
t.Errorf("hits = %+v", env.usage.recordedHits())
|
|
}
|
|
})
|
|
})
|
|
|
|
t.Run("A thousand hits on one file are one pending key", func(t *testing.T) {
|
|
env := trackedEnv(t)
|
|
env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil)
|
|
_ = env.tracker.Flush()
|
|
callsBefore := env.usage.callCount()
|
|
base := env.now
|
|
|
|
for index := 1; index <= 1000; index++ {
|
|
env.now = base.Add(time.Duration(index) * time.Second)
|
|
env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil)
|
|
}
|
|
writesDuring := env.usage.callCount() - callsBefore
|
|
_ = env.tracker.Flush()
|
|
|
|
hits := env.usage.recordedHits()
|
|
if writesDuring != 0 || env.usage.batchCount() != 2 || len(hits) != 2 || !hits[1].At.Equal(base.Add(1000*time.Second)) {
|
|
t.Errorf("writes = %d, batches = %d, hits = %+v", writesDuring, env.usage.batchCount(), hits)
|
|
}
|
|
})
|
|
|
|
t.Run("A request is never blocked by or failed by the tracker", func(t *testing.T) {
|
|
env := trackedEnv(t)
|
|
env.usage.fail = errUsageStoreDown
|
|
env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil)
|
|
|
|
recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil)
|
|
|
|
if recorder.Code != 200 || recorder.Body.Len() == 0 {
|
|
t.Errorf("status = %d, body = %d bytes", recorder.Code, recorder.Body.Len())
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestImageVariantsRejectionCounting(t *testing.T) {
|
|
rows := []struct {
|
|
url string
|
|
status int
|
|
counts int
|
|
}{
|
|
{"/static/uploadedImages/" + variantSourceName + "?v=unknown", 400, 1},
|
|
{"/static/uploadedImages/" + variantSourceName + "?v=print", 400, 1},
|
|
{"/static/uploadedImages/" + variantSourceName + "?width=400", 400, 1},
|
|
{"/static/uploadedImages/" + variantSourceName + "?v=Thumb", 400, 1},
|
|
{"/static/uploadedImages/missing.png?v=thumb", 404, 0},
|
|
{"/static/uploadedImages/" + variantSourceName + "?v=thumb", 200, 0},
|
|
{"/static/uploadedImages/" + variantSourceName, 200, 0},
|
|
}
|
|
setup := func(t *testing.T) *variantEnv {
|
|
t.Helper()
|
|
env := newVariantEnv(t)
|
|
env.seedThumb()
|
|
env.seedPhoto()
|
|
print := seedVariant(env.variants, "print", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
print.Enabled = false
|
|
env.variants.variants["print"] = print
|
|
return env
|
|
}
|
|
for _, row := range rows {
|
|
t.Run("What counts as rejected: "+row.url, func(t *testing.T) {
|
|
env := setup(t)
|
|
|
|
recorder := env.get(row.url, nil)
|
|
_ = env.tracker.Flush()
|
|
|
|
if recorder.Code != row.status || env.usage.recordedRejections() != row.counts {
|
|
t.Errorf("status = %d, rejections = %d", recorder.Code, env.usage.recordedRejections())
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("A rejection is an in-memory increment", func(t *testing.T) {
|
|
env := setup(t)
|
|
|
|
for index := 0; index < 1000; index++ {
|
|
env.get("/static/uploadedImages/"+variantSourceName+"?v=unknown", nil)
|
|
}
|
|
writesDuring := env.usage.callCount()
|
|
_ = env.tracker.Flush()
|
|
|
|
batch := env.usage.batches[0]
|
|
if writesDuring != 0 || len(batch.Rejections) != 1 || batch.Rejections[0].Count != 1000 || batch.Rejections[0].Day.Format(domain.RejectionDayLayout) != "2026-10-08" {
|
|
t.Errorf("writes = %d, batch = %+v", writesDuring, batch)
|
|
}
|
|
})
|
|
|
|
t.Run("A day boundary starts a new bucket", func(t *testing.T) {
|
|
env := setup(t)
|
|
env.now = time.Date(2026, time.October, 8, 23, 59, 59, 0, time.UTC)
|
|
env.get("/static/uploadedImages/"+variantSourceName+"?v=unknown", nil)
|
|
env.now = time.Date(2026, time.October, 9, 0, 0, 0, 0, time.UTC)
|
|
env.get("/static/uploadedImages/"+variantSourceName+"?v=unknown", nil)
|
|
|
|
_ = env.tracker.Flush()
|
|
|
|
if len(env.usage.batches[0].Rejections) != 2 || env.usage.days["2026-10-08"] != 1 || env.usage.days["2026-10-09"] != 1 {
|
|
t.Errorf("days = %v", env.usage.days)
|
|
}
|
|
})
|
|
|
|
t.Run("Old day rows are pruned by the flush", func(t *testing.T) {
|
|
env := setup(t)
|
|
env.seedRejectedDays(map[string]int{"2026-09-30": 4, "2026-10-02": 5, "2026-10-08": 1})
|
|
env.get("/static/uploadedImages/"+variantSourceName+"?v=unknown", nil)
|
|
|
|
_ = env.tracker.Flush()
|
|
|
|
if _, found := env.usage.days["2026-09-30"]; found || env.usage.days["2026-10-02"] != 5 || env.usage.days["2026-10-08"] != 2 {
|
|
t.Errorf("days = %v", env.usage.days)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestImageVariantsCachePurge(t *testing.T) {
|
|
t.Run("Purge all removes every generated file of every variant", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
recorder := env.purgeCache(`{"scope": "all"}`)
|
|
|
|
assertStatusAndBody(t, recorder, 200, `{"scope":"all","purged_files":1284,"purged_bytes":222298112,"failed_files":0}`)
|
|
if env.files.rowCount() != 0 || len(env.storage.deletedNames()) != 1284 || env.variants.count() != 6 {
|
|
t.Errorf("rows = %d, deleted = %d, variants = %d", env.files.rowCount(), len(env.storage.deletedNames()), env.variants.count())
|
|
}
|
|
})
|
|
|
|
t.Run("Purge all includes the files of disabled variants", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
printBlobs := []string{}
|
|
for _, row := range env.files.rowsOf("print") {
|
|
printBlobs = append(printBlobs, row.CachedName)
|
|
}
|
|
|
|
env.purgeCache(`{"scope": "all"}`)
|
|
|
|
deleted := env.storage.deletedNames()
|
|
for _, blob := range printBlobs {
|
|
if !containsString(deleted, blob) {
|
|
t.Fatalf("%s was not deleted", blob)
|
|
}
|
|
}
|
|
if len(printBlobs) != 74 {
|
|
t.Errorf("print blobs = %d", len(printBlobs))
|
|
}
|
|
})
|
|
|
|
t.Run("Purge unused removes the unused files of enabled variants only", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
recorder := env.purgeCache(`{"scope": "unused"}`)
|
|
|
|
assertStatusAndBody(t, recorder, 200, `{"scope":"unused","purged_files":312,"purged_bytes":42991616,"failed_files":0}`)
|
|
if env.files.rowCount() != 972 || len(env.files.rowsOf("print")) != 74 {
|
|
t.Errorf("rows = %d, print = %d", env.files.rowCount(), len(env.files.rowsOf("print")))
|
|
}
|
|
})
|
|
|
|
t.Run("Purge by variant removes the files of that variant only", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
recorder := env.purgeCache(`{"scope": "variant", "variant": "hero"}`)
|
|
|
|
assertStatusAndBody(t, recorder, 200, `{"scope":"variant","purged_files":244,"purged_bytes":88080384,"failed_files":0}`)
|
|
if env.files.rowCount() != 1040 || len(env.files.rowsOf("hero")) != 0 {
|
|
t.Errorf("rows = %d", env.files.rowCount())
|
|
}
|
|
})
|
|
|
|
t.Run("Purge by variant works for a disabled variant", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
recorder := env.purgeCache(`{"scope": "variant", "variant": "print"}`)
|
|
|
|
if decodeBody(t, recorder)["purged_files"] != 74.0 {
|
|
t.Errorf("body = %s", recorder.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("Purge by image removes every variant file of that image", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
recorder := env.purgeCache(`{"scope": "image", "image_id": 1}`)
|
|
|
|
assertStatusAndBody(t, recorder, 200, `{"scope":"image","purged_files":6,"purged_bytes":6082765,"failed_files":0}`)
|
|
remaining, _ := env.files.ListBySource("lime-plaster.jpg")
|
|
if len(remaining) != 6 || env.files.rowCount() != 1278 {
|
|
t.Errorf("lime-plaster rows = %d, rows = %d", len(remaining), env.files.rowCount())
|
|
}
|
|
})
|
|
|
|
t.Run("The next public request generates a purged file again", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedThumb()
|
|
env.seedPhoto()
|
|
env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil)
|
|
env.purgeCache(`{"scope": "all"}`)
|
|
|
|
recorder := env.get("/static/uploadedImages/"+variantSourceName+"?v=thumb", nil)
|
|
|
|
if recorder.Code != 200 || env.storage.putCount() != 2 {
|
|
t.Errorf("status = %d, puts = %d", recorder.Code, env.storage.putCount())
|
|
}
|
|
})
|
|
|
|
t.Run("Purging an empty cache answers zero", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
|
|
recorder := env.purgeCache(`{"scope": "all"}`)
|
|
|
|
assertStatusAndBody(t, recorder, 200, `{"scope":"all","purged_files":0,"purged_bytes":0,"failed_files":0}`)
|
|
})
|
|
|
|
t.Run("A blob already missing from storage does not stop a purge", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
rows := seedVariantFiles(env.files, env.storage, "hero", "x", 3, 10)
|
|
env.storage.failDelete[rows[1].CachedName] = domain.ErrBlobNotFound
|
|
|
|
recorder := env.purgeCache(`{"scope": "all"}`)
|
|
|
|
assertStatusAndBody(t, recorder, 200, `{"scope":"all","purged_files":3,"purged_bytes":30,"failed_files":0}`)
|
|
if env.files.rowCount() != 0 {
|
|
t.Errorf("rows = %d", env.files.rowCount())
|
|
}
|
|
})
|
|
|
|
invalid := []struct {
|
|
body string
|
|
status int
|
|
message string
|
|
}{
|
|
{`{"scope": "everything"}`, 422, apperr.CachePurgeScopeErrorMessage},
|
|
{`{"scope": "ALL"}`, 422, apperr.CachePurgeScopeErrorMessage},
|
|
{`{}`, 422, apperr.CachePurgeScopeErrorMessage},
|
|
{`{"scope": "variant"}`, 422, apperr.CachePurgeTargetErrorMessage},
|
|
{`{"scope": "variant", "variant": ""}`, 422, apperr.CachePurgeTargetErrorMessage},
|
|
{`{"scope": "image"}`, 422, apperr.CachePurgeTargetErrorMessage},
|
|
{`{"scope": "image", "image_id": 0}`, 422, apperr.CachePurgeTargetErrorMessage},
|
|
{`{"scope": "image", "image_id": -1}`, 422, apperr.CachePurgeTargetErrorMessage},
|
|
{`{"scope": "variant", "variant": "missing"}`, 404, apperr.VariantNotFoundErrorMessage},
|
|
{`{"scope": "image", "image_id": 9999}`, 404, "Image not found"},
|
|
{`not json`, 400, apperr.BodyErrorMessage},
|
|
}
|
|
for _, row := range invalid {
|
|
t.Run("A request that cannot be served: "+row.body, func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
recorder := env.purgeCache(row.body)
|
|
|
|
assertStatusAndBody(t, recorder, row.status, row.message)
|
|
if env.files.rowCount() != 1284 || len(env.storage.deletedNames()) != 0 || env.events() != 0 {
|
|
t.Error("a blob or row was deleted or an event was recorded")
|
|
}
|
|
})
|
|
}
|
|
|
|
ignored := map[string]string{"all": `"purged_files":1284`, "unused": `"purged_files":312`}
|
|
for scope, want := range ignored {
|
|
t.Run("A target given for a scope that ignores it is ignored: "+scope, func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
recorder := env.purgeCache(`{"scope": "` + scope + `", "variant": "hero", "image_id": 1}`)
|
|
|
|
if recorder.Code != 200 || !strings.Contains(recorder.Body.String(), want) {
|
|
t.Errorf("response = %d %s", recorder.Code, recorder.Body.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func containsString(values []string, wanted string) bool {
|
|
for _, value := range values {
|
|
if value == wanted {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestImageVariantsCachePartialPurge(t *testing.T) {
|
|
failingEnv := func(t *testing.T) (*variantEnv, int64) {
|
|
t.Helper()
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
page, _ := env.files.ListPage(domain.VariantFileFilter{}, 0, 2000)
|
|
var keptBytes int64
|
|
for index, row := range page {
|
|
if index < 644 {
|
|
env.storage.failDelete[row.CachedName] = errors.New("storage down")
|
|
continue
|
|
}
|
|
keptBytes += row.Size
|
|
}
|
|
return env, keptBytes
|
|
}
|
|
|
|
t.Run("A partial purge answers 200 with the failed count", func(t *testing.T) {
|
|
env, purgedBytes := failingEnv(t)
|
|
|
|
recorder := env.purgeCache(`{"scope": "all"}`)
|
|
|
|
want := `{"scope":"all","purged_files":640,"purged_bytes":` + itoa64(purgedBytes) + `,"failed_files":644}`
|
|
assertStatusAndBody(t, recorder, 200, want)
|
|
if env.files.rowCount() != 644 {
|
|
t.Errorf("rows = %d", env.files.rowCount())
|
|
}
|
|
})
|
|
|
|
t.Run("Retrying removes the rest", func(t *testing.T) {
|
|
env, _ := failingEnv(t)
|
|
env.purgeCache(`{"scope": "all"}`)
|
|
env.storage.failDelete = map[string]error{}
|
|
|
|
recorder := env.purgeCache(`{"scope": "all"}`)
|
|
|
|
body := decodeBody(t, recorder)
|
|
if body["purged_files"] != 644.0 || body["failed_files"] != 0.0 {
|
|
t.Errorf("body = %v", body)
|
|
}
|
|
})
|
|
|
|
t.Run("A failure to list the rows answers 500", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
env.files.failPage = errors.New("list down")
|
|
|
|
recorder := env.purgeCache(`{"scope": "all"}`)
|
|
|
|
assertStatusAndBody(t, recorder, 500, "Internal server error")
|
|
})
|
|
|
|
t.Run("Rows are processed in bounded batches", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
for index := 0; index < 5000; index++ {
|
|
env.files.Save(domain.VariantFile{VariantName: "hero", SourceName: "bulk-" + itoa64(int64(index)), CachedName: "bulk-" + itoa64(int64(index)), Size: 1})
|
|
}
|
|
|
|
recorder := env.purgeCache(`{"scope": "all"}`)
|
|
|
|
if recorder.Code != 200 || decodeBody(t, recorder)["purged_files"] != 5000.0 {
|
|
t.Fatalf("response = %d %s", recorder.Code, recorder.Body.String())
|
|
}
|
|
if len(env.files.pageLimits) < 10 {
|
|
t.Errorf("pages = %d", len(env.files.pageLimits))
|
|
}
|
|
for _, limit := range env.files.pageLimits {
|
|
if limit > 500 {
|
|
t.Errorf("page limit = %d", limit)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func itoa64(value int64) string {
|
|
return strconv.FormatInt(value, 10)
|
|
}
|
|
|
|
func TestImageVariantsCachePurgeActivity(t *testing.T) {
|
|
scopes := []struct {
|
|
body string
|
|
action string
|
|
target string
|
|
params string
|
|
}{
|
|
{`{"scope": "all"}`, "media.variant_purge_all", `{"type":"","id":"","name":""}`, `{"purged_bytes":222298112,"purged_files":1284}`},
|
|
{`{"scope": "unused"}`, "media.variant_purge_unused", `{"type":"","id":"","name":""}`, `{"older_than_days":30,"purged_bytes":42991616,"purged_files":312}`},
|
|
{`{"scope": "variant", "variant": "hero"}`, "media.variant_purge", `{"type":"image_variant","id":"hero","name":"hero"}`, `{"purged_bytes":88080384,"purged_files":244}`},
|
|
{`{"scope": "image", "image_id": 1}`, "media.variant_purge_image", `{"type":"image","id":"1","name":"north-wall.jpg"}`, `{"purged_bytes":6082765,"purged_files":6}`},
|
|
}
|
|
for _, scope := range scopes {
|
|
t.Run("A purge that removes files is recorded: "+scope.action, func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
env.purgeCache(scope.body)
|
|
|
|
event := onlyEventOf(t, env, scope.action)
|
|
if event.Category != "media" || event.Outcome != "success" || marshalled(t, event.Target) != scope.target || marshalled(t, event.Params) != scope.params {
|
|
t.Errorf("event = %+v, target = %s, params = %s", event, marshalled(t, event.Target), marshalled(t, event.Params))
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("A partial failure is a success that names the failed count", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
page, _ := env.files.ListPage(domain.VariantFileFilter{}, 0, 2000)
|
|
var keptBytes int64
|
|
for index, row := range page {
|
|
if index < 644 {
|
|
env.storage.failDelete[row.CachedName] = errors.New("storage down")
|
|
continue
|
|
}
|
|
keptBytes += row.Size
|
|
}
|
|
|
|
env.purgeCache(`{"scope": "all"}`)
|
|
|
|
event := onlyEventOf(t, env, "media.variant_purge_all")
|
|
want := `{"failed_files":644,"purged_bytes":` + itoa64(keptBytes) + `,"purged_files":640}`
|
|
if event.Outcome != "success" || marshalled(t, event.Params) != want {
|
|
t.Errorf("event = %+v, params = %s", event, marshalled(t, event.Params))
|
|
}
|
|
})
|
|
|
|
t.Run("Every file failing is a failed event with a plain reason", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
page, _ := env.files.ListPage(domain.VariantFileFilter{}, 0, 2000)
|
|
for _, row := range page {
|
|
env.storage.failDelete[row.CachedName] = errors.New("storage down")
|
|
}
|
|
|
|
recorder := env.purgeCache(`{"scope": "all"}`)
|
|
|
|
body := decodeBody(t, recorder)
|
|
event := onlyEventOf(t, env, "media.variant_purge_all")
|
|
if recorder.Code != 200 || body["purged_files"] != 0.0 || body["failed_files"] != 1284.0 || event.Outcome != "failed" || event.FailureReason == nil || *event.FailureReason != "Something went wrong on our side. Please try again." {
|
|
t.Errorf("response = %d %v, event = %+v", recorder.Code, body, event)
|
|
}
|
|
})
|
|
|
|
nothing := map[string]func(*variantEnv) string{
|
|
"the owner purges all on an empty cache": func(env *variantEnv) string { return `{"scope": "all"}` },
|
|
"the owner purges unused when nothing is unused": func(env *variantEnv) string {
|
|
env.seedDefaultCache()
|
|
for _, row := range env.files.rowsOf("hero") {
|
|
env.files.setReference(row.ID, env.now)
|
|
}
|
|
for _, row := range env.files.rowsOf("thumb") {
|
|
env.files.setReference(row.ID, env.now)
|
|
}
|
|
return `{"scope": "unused"}`
|
|
},
|
|
"the owner purges a variant that has no file": func(env *variantEnv) string {
|
|
seedVariant(env.variants, "banner", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
|
|
return `{"scope": "variant", "variant": "banner"}`
|
|
},
|
|
"the owner purges an image that has no file": func(env *variantEnv) string {
|
|
env.seedImage(50, "nofile.png")
|
|
return `{"scope": "image", "image_id": 50}`
|
|
},
|
|
"the owner sends an unknown scope": func(env *variantEnv) string { return `{"scope": "everything"}` },
|
|
"the owner purges an unknown variant": func(env *variantEnv) string { return `{"scope": "variant", "variant": "ghost"}` },
|
|
"the owner purges an unknown image": func(env *variantEnv) string { return `{"scope": "image", "image_id": 777}` },
|
|
}
|
|
for name, prepare := range nothing {
|
|
t.Run("Nothing removed, nothing recorded: "+name, func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
body := prepare(env)
|
|
|
|
env.purgeCache(body)
|
|
|
|
if env.events() != 0 {
|
|
t.Errorf("events = %d", env.events())
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("Nothing removed, nothing recorded: the owner sends GET /v1/image-variants/cache", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
|
|
env.send(http.MethodGet, cacheStatsPath, "", variantOwnerId)
|
|
|
|
if env.events() != 0 {
|
|
t.Errorf("events = %d", env.events())
|
|
}
|
|
})
|
|
|
|
t.Run("A failing audit write never changes the response", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
env.audit.probe.appendErr = errors.New("audit store down")
|
|
|
|
recorder := env.purgeCache(`{"scope": "all"}`)
|
|
|
|
assertStatusAndBody(t, recorder, 200, `{"scope":"all","purged_files":1284,"purged_bytes":222298112,"failed_files":0}`)
|
|
if env.files.rowCount() != 0 {
|
|
t.Errorf("rows = %d", env.files.rowCount())
|
|
}
|
|
})
|
|
|
|
t.Run("The events read back through the existing activity API", func(t *testing.T) {
|
|
env := newVariantEnv(t)
|
|
env.seedDefaultCache()
|
|
for _, body := range []string{`{"scope": "image", "image_id": 1}`, `{"scope": "variant", "variant": "hero"}`, `{"scope": "unused"}`, `{"scope": "all"}`} {
|
|
env.purgeCache(body)
|
|
}
|
|
|
|
recorder := env.send(http.MethodGet, "/v1/activity?category=media", "", variantOwnerId)
|
|
|
|
items := decodeBody(t, recorder)["items"].([]any)
|
|
actions := []string{}
|
|
for _, item := range items {
|
|
actions = append(actions, item.(map[string]any)["action"].(string))
|
|
}
|
|
want := []string{"media.variant_purge_all", "media.variant_purge_unused", "media.variant_purge", "media.variant_purge_image"}
|
|
if recorder.Code != 200 || strings.Join(actions, ",") != strings.Join(want, ",") {
|
|
t.Errorf("status = %d, actions = %v", recorder.Code, actions)
|
|
}
|
|
})
|
|
}
|