feat: image variants cache panel

This commit is contained in:
fsylvain committed 2026-10-08 21:13:32 +02:00
1 parent 951fef18ba
commit d768668014
53 files changed
+5809 -46

No files matched your search

+17
View File
@@ -5,6 +5,7 @@ import (
"RenewCMS/internal/infrastructure/useCases"
"errors"
"io"
"slices"
"sort"
"strconv"
"strings"
@@ -78,6 +79,8 @@ type mockImageRepo struct {
deleteCalls []uint32
createCalls int
onDelete func(id uint32)
storedNameLookups int
}
func newMockImageRepo() *mockImageRepo {
@@ -257,6 +260,20 @@ func (m *mockImageRepo) ExistsByStoredName(storedName string) (bool, error) {
return false, nil
}
func (m *mockImageRepo) ListByStoredNames(storedNames []string) ([]imageDomain.Image, error) {
if err := m.fail.trigger("ListByStoredNames"); err != nil {
return nil, err
}
m.storedNameLookups++
result := make([]imageDomain.Image, 0)
for _, img := range m.sortedImages() {
if slices.Contains(storedNames, imageDomain.StoredName(img.Path)) {
result = append(result, img)
}
}
return result, nil
}
var _ imageDomain.Repository = &mockImageRepo{}
func TestImageUseCases(t *testing.T) {
File diff suppressed because it is too large. Load diff
+86
View File
@@ -0,0 +1,86 @@
package test
import (
"go/parser"
"go/token"
"strings"
"testing"
)
func TestImageVariantCacheLayering(t *testing.T) {
t.Run("The cache use cases and the tracker import no persistence, mailer, models, gorm, net/http, chi or api package", func(t *testing.T) {
files := []string{
"internal/infrastructure/useCases/GetVariantCacheStats.go",
"internal/infrastructure/useCases/PurgeVariantCache.go",
"internal/infrastructure/usage/variantUsageTracker.go",
}
for _, file := range files {
for _, path := range importsOf(t, file) {
for _, fragment := range []string{"persistence", "mailer", "models", "gorm", "net/http", "go-chi", "RenewCMS/api"} {
if strings.Contains(path, fragment) {
t.Errorf("%s imports %s", file, path)
}
}
}
}
})
t.Run("The cache handler imports no persistence, mailer, models or gorm package", func(t *testing.T) {
for _, path := range importsOf(t, "api/handlers/imagevariant/cacheHandler.go") {
for _, fragment := range []string{"persistence", "mailer", "models", "gorm"} {
if strings.Contains(path, fragment) {
t.Errorf("imports %s", path)
}
}
}
})
t.Run("The tracker, usage and cache ports are declared in internal/domain/image", func(t *testing.T) {
declared := readRepositoryFile(t, "internal/domain/image/variantRepositoryInterface.go")
for _, port := range []string{"type VariantUsageTracker interface", "type VariantUsageRepository interface", "type VariantCacheRepository interface"} {
if !strings.Contains(declared, port) {
t.Errorf("%s is not declared", port)
}
}
})
t.Run("The new models are registered in migrate.go", func(t *testing.T) {
migrate := readRepositoryFile(t, "internal/infrastructure/persistence/database/migrate.go")
if !strings.Contains(migrate, "&models.ImageVariantRejectionDay{}") {
t.Error("the rejection day model is missing from AutoMigrate")
}
})
t.Run("Only the composition root constructs the tracker", func(t *testing.T) {
for _, file := range []string{"api/router.go", "cmd/server/main.go", "api/handlers/staticimages/staticImagesHandler.go", "api/handlers/imagevariant/cacheHandler.go"} {
if strings.Contains(readRepositoryFile(t, file), "NewTracker") {
t.Errorf("%s constructs the tracker", file)
}
}
})
t.Run("The new files carry no comment", func(t *testing.T) {
for _, relative := range []string{
"internal/domain/image/variantCache.go",
"internal/infrastructure/usage/variantUsageTracker.go",
"internal/infrastructure/useCases/GetVariantCacheStats.go",
"internal/infrastructure/useCases/PurgeVariantCache.go",
"internal/infrastructure/persistence/repositories/imageVariantUsageRepository.go",
"api/handlers/imagevariant/cacheHandler.go",
} {
parsed, err := parser.ParseFile(token.NewFileSet(), "../"+relative, nil, parser.ParseComments)
if err != nil {
t.Fatal(err)
}
for _, group := range parsed.Comments {
for _, comment := range group.List {
if !strings.HasPrefix(comment.Text, "//go:") {
t.Errorf("%s has comment %q", relative, comment.Text)
}
}
}
}
})
}
+233
View File
@@ -0,0 +1,233 @@
package test
import (
domain "RenewCMS/internal/domain/image"
"errors"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"time"
)
var errUsageStoreDown = errors.New("usage store down")
type mockVariantUsageRepo struct {
mutex sync.Mutex
batches []domain.VariantUsageBatch
days map[string]int
fail error
calls int
}
var _ domain.VariantUsageRepository = &mockVariantUsageRepo{}
func newMockVariantUsageRepo() *mockVariantUsageRepo {
return &mockVariantUsageRepo{days: map[string]int{}}
}
func (m *mockVariantUsageRepo) Record(batch domain.VariantUsageBatch) error {
m.mutex.Lock()
defer m.mutex.Unlock()
m.calls++
if m.fail != nil {
return m.fail
}
m.batches = append(m.batches, batch)
for _, rejection := range batch.Rejections {
m.days[rejection.Day.Format(domain.RejectionDayLayout)] += rejection.Count
}
if !batch.PruneBefore.IsZero() {
for day := range m.days {
if day < batch.PruneBefore.Format(domain.RejectionDayLayout) {
delete(m.days, day)
}
}
}
return nil
}
func (m *mockVariantUsageRepo) RejectedSince(day time.Time) (int, error) {
m.mutex.Lock()
defer m.mutex.Unlock()
total := 0
for stored, count := range m.days {
if stored >= day.Format(domain.RejectionDayLayout) {
total += count
}
}
return total, nil
}
func (m *mockVariantUsageRepo) recordedHits() []domain.VariantHit {
m.mutex.Lock()
defer m.mutex.Unlock()
hits := []domain.VariantHit{}
for _, batch := range m.batches {
hits = append(hits, batch.Hits...)
}
return hits
}
func (m *mockVariantUsageRepo) recordedRejections() int {
m.mutex.Lock()
defer m.mutex.Unlock()
total := 0
for _, batch := range m.batches {
for _, rejection := range batch.Rejections {
total += rejection.Count
}
}
return total
}
func (m *mockVariantUsageRepo) batchCount() int {
m.mutex.Lock()
defer m.mutex.Unlock()
return len(m.batches)
}
func (m *mockVariantUsageRepo) callCount() int {
m.mutex.Lock()
defer m.mutex.Unlock()
return m.calls
}
type cacheFixtureImage struct {
id uint32
filename string
files int
bytes int64
}
var cacheFixtureTopImages = []cacheFixtureImage{
{1, "north-wall.jpg", 6, 6082765},
{2, "lime-plaster.jpg", 6, 5138022},
{3, "kitchen-after.webp", 5, 4613734},
{4, "kitchen-before.webp", 5, 4089446},
{5, "floorboards.jpg", 5, 3774873},
{6, "IMG_4471.jpg", 4, 3250585},
{7, "stone-wall-detail.jpg", 4, 2936012},
{8, "bathroom-tiles.png", 4, 2726297},
{9, "roof-lantern.jpg", 3, 2411724},
{10, "chimney-stack.jpg", 3, 2202009},
}
type cacheFixtureVariant struct {
name string
enabled bool
files int
bytes int64
}
var cacheFixtureVariants = []cacheFixtureVariant{
{"hero", true, 244, 88080384},
{"card", true, 318, 60817408},
{"thumb", true, 412, 22020096},
{"print", false, 74, 22020096},
{"square", true, 140, 19922944},
{"og", true, 96, 9437184},
}
const (
cacheFixtureUnusedHeroRows = 200
cacheFixtureUnusedHeroBytes = 40000000
cacheFixtureUnusedThumbRows = 112
cacheFixtureUnusedThumbSize = 2991616
)
func splitBytes(total int64, parts int) []int64 {
sizes := make([]int64, parts)
for index := range sizes {
sizes[index] = total / int64(parts)
if int64(index) < total%int64(parts) {
sizes[index]++
}
}
return sizes
}
func (e *variantEnv) seedCacheRow(variantName string, sourceName string, size int64, reference time.Time) domain.VariantFile {
cached := "fixture-" + variantName + "-" + sourceName + ".jpeg"
e.storage.seed(cached, []byte("generated"))
saved, _ := e.files.Save(domain.VariantFile{VariantName: variantName, SourceName: sourceName, CachedName: cached, ContentType: "image/jpeg", Size: size, Fingerprint: "fixture"})
e.files.setReference(saved.ID, reference)
return saved
}
func (e *variantEnv) seedDefaultCache() {
recent := e.now.AddDate(0, 0, -5)
stale := e.now.AddDate(0, 0, -31)
for _, variant := range cacheFixtureVariants {
seeded := seedVariant(e.variants, variant.name, 400, 300, domain.FitCover, domain.FormatJPEG, 80)
seeded.Enabled = variant.enabled
e.variants.variants[variant.name] = seeded
}
topBytes := map[string]int64{}
topFiles := map[string]int{}
for _, img := range cacheFixtureTopImages {
e.seedImage(img.id, img.filename)
for index, size := range splitBytes(img.bytes, img.files) {
variant := cacheFixtureVariants[index%len(cacheFixtureVariants)].name
e.seedCacheRow(variant, img.filename, size, recent)
topBytes[variant] += size
topFiles[variant]++
}
}
for _, variant := range cacheFixtureVariants {
fillers := variant.files - topFiles[variant.name]
remaining := variant.bytes - topBytes[variant.name]
sizes := e.fillerSizes(variant.name, fillers, remaining)
for index, size := range sizes {
reference := recent
if e.isFixtureStale(variant.name, index) {
reference = stale
}
e.seedCacheRow(variant.name, "filler-"+variant.name+"-"+strconv.Itoa(index), size, reference)
}
}
}
func (e *variantEnv) isFixtureStale(variantName string, index int) bool {
return (variantName == "hero" && index < cacheFixtureUnusedHeroRows) || (variantName == "thumb" && index < cacheFixtureUnusedThumbRows)
}
func (e *variantEnv) fillerSizes(variantName string, fillers int, remaining int64) []int64 {
switch variantName {
case "hero":
return append(splitBytes(cacheFixtureUnusedHeroBytes, cacheFixtureUnusedHeroRows), splitBytes(remaining-cacheFixtureUnusedHeroBytes, fillers-cacheFixtureUnusedHeroRows)...)
case "thumb":
return append(splitBytes(cacheFixtureUnusedThumbSize, cacheFixtureUnusedThumbRows), splitBytes(remaining-cacheFixtureUnusedThumbSize, fillers-cacheFixtureUnusedThumbRows)...)
default:
return splitBytes(remaining, fillers)
}
}
func (e *variantEnv) seedRejectedDays(days map[string]int) {
for day, count := range days {
e.usage.days[day] = count
}
}
func (e *variantEnv) cacheStats() (*httptest.ResponseRecorder, map[string]any) {
e.t.Helper()
recorder := e.sendAsAdmin(http.MethodGet, "/v1/image-variants/cache", "")
if recorder.Code != http.StatusOK {
return recorder, nil
}
return recorder, decodeBody(e.t, recorder)
}
func (e *variantEnv) purgeCache(body string) *httptest.ResponseRecorder {
e.t.Helper()
return e.send(http.MethodPost, "/v1/image-variants/cache/purge", body, variantOwnerId)
}
func (e *variantEnv) recentRejectedFixture() {
e.seedRejectedDays(map[string]int{"2026-10-08": 30, "2026-10-05": 7, "2026-09-20": 100})
}
func compactJSON(body *httptest.ResponseRecorder) string {
return strings.TrimSpace(body.Body.String())
}
+15 -5
View File
@@ -9,6 +9,7 @@ import (
"RenewCMS/internal/domain/user"
"RenewCMS/internal/infrastructure/config"
"RenewCMS/internal/infrastructure/hashing"
"RenewCMS/internal/infrastructure/usage"
"RenewCMS/internal/infrastructure/useCases"
"bytes"
"encoding/json"
@@ -50,6 +51,8 @@ type variantEnv struct {
logs *bytes.Buffer
token *jwtauth.JWTAuth
now time.Time
usage *mockVariantUsageRepo
tracker *usage.Tracker
admin http.Handler
public http.Handler
}
@@ -81,7 +84,9 @@ func newVariantEnvLimits(t *testing.T, renderer *countingRenderer, limits useCas
logs: &bytes.Buffer{},
token: jwtauth.New("HS256", []byte("test-secret"), nil),
now: time.Date(2026, time.October, 8, 9, 0, 0, 0, time.UTC),
usage: newMockVariantUsageRepo(),
}
env.tracker = usage.NewTracker(env.usage, func() time.Time { return env.now })
env.users.users[variantOwnerId] = user.User{ID: variantOwnerId, Username: "marta", DisplayName: "Marta Alves", IsVerified: true, Role: user.RoleOwner}
env.users.users[variantAdminId] = user.User{ID: variantAdminId, Username: "dev", IsVerified: true, Role: user.RoleAdmin}
env.users.users[variantPlainUserId] = user.User{ID: variantPlainUserId, Username: "pat", IsVerified: true}
@@ -103,7 +108,7 @@ func newVariantEnvLimits(t *testing.T, renderer *countingRenderer, limits useCas
func (e *variantEnv) container(limits useCases.ServeVariantLimits) *api.UseCases {
clock := func() time.Time { return e.now }
recorder := e.audit.recorder
serve := useCases.NewServeVariantUseCase(e.variants, e.files, e.images, e.storage, e.renderer, limits)
serve := useCases.NewServeVariantUseCase(e.variants, e.files, e.images, e.storage, e.renderer, limits).WithUsageTracker(e.tracker)
return &api.UseCases{
Token: e.token,
GetUserUseCase: useCases.NewGetUserUseCase(e.users),
@@ -117,10 +122,15 @@ func (e *variantEnv) container(limits useCases.ServeVariantLimits) *api.UseCases
DeleteVariantUseCase: useCases.NewDeleteVariantUseCase(e.variants, e.files, e.storage, recorder),
PurgeVariantUseCase: useCases.NewPurgeVariantUseCase(e.variants, e.files, e.storage, recorder),
ServeVariantUseCase: serve,
CreateImageUseCase: useCases.NewCreateImageUseCase(e.images, e.storage, recorder),
ListMediaUseCase: useCases.NewListMediaUseCase(e.images, e.articles, e.users, e.files),
RenameMediaUseCase: useCases.NewRenameMediaUseCase(e.images, recorder),
GetMediaUsageUseCase: useCases.NewGetMediaUsageUseCase(e.images, e.articles),
GetVariantCacheStatsUseCase: useCases.NewGetVariantCacheStatsUseCase(e.variants, e.files, e.files, e.images, e.usage, e.tracker, clock),
PurgeVariantCacheUseCase: useCases.NewPurgeVariantCacheUseCase(e.variants, e.images, e.files, e.files, e.storage, clock, recorder),
VariantUsageTracker: e.tracker,
CreateImageUseCase: useCases.NewCreateImageUseCase(e.images, e.storage, recorder),
ListMediaUseCase: useCases.NewListMediaUseCase(e.images, e.articles, e.users, e.files),
RenameMediaUseCase: useCases.NewRenameMediaUseCase(e.images, recorder),
GetMediaUsageUseCase: useCases.NewGetMediaUsageUseCase(e.images, e.articles),
ListImageVariantsUseCase: useCases.NewListImageVariantsUseCase(e.images, e.variants, e.files, e.renderer),
GenerateImageVariantUseCase: useCases.NewGenerateImageVariantUseCase(e.images, e.variants, e.files, e.renderer, serve),
+82 -1
View File
@@ -8,6 +8,7 @@ import (
"errors"
"io"
"log"
"slices"
"sort"
"strconv"
"strings"
@@ -117,12 +118,88 @@ type mockVariantFileRepo struct {
calls int
totalsBySourceCalls int
references map[uint32]time.Time
failTotals error
failPage error
totalsCalls int
unusedCalls int
topCalls int
pageLimits []int
}
var _ domain.VariantFileRepository = &mockVariantFileRepo{}
var _ domain.VariantCacheRepository = &mockVariantFileRepo{}
func newMockVariantFileRepo() *mockVariantFileRepo {
return &mockVariantFileRepo{rows: map[uint32]domain.VariantFile{}}
return &mockVariantFileRepo{rows: map[uint32]domain.VariantFile{}, references: map[uint32]time.Time{}}
}
func (m *mockVariantFileRepo) setReference(id uint32, at time.Time) {
m.mutex.Lock()
defer m.mutex.Unlock()
m.references[id] = at
}
func (m *mockVariantFileRepo) isUnused(row domain.VariantFile, filter domain.UnusedFilter) bool {
return slices.Contains(filter.VariantNames, row.VariantName) && m.references[row.ID].Before(filter.Cutoff)
}
func (m *mockVariantFileRepo) UnusedTotals(filter domain.UnusedFilter) (domain.VariantFileTotals, error) {
m.mutex.Lock()
defer m.mutex.Unlock()
m.unusedCalls++
var totals domain.VariantFileTotals
for _, row := range m.rows {
if m.isUnused(row, filter) {
totals.Files++
totals.Bytes += row.Size
}
}
return totals, nil
}
func (m *mockVariantFileRepo) TopSources(limit int) ([]domain.SourceTotals, error) {
m.mutex.Lock()
defer m.mutex.Unlock()
m.topCalls++
bySource := map[string]domain.SourceTotals{}
for _, row := range m.rows {
current := bySource[row.SourceName]
bySource[row.SourceName] = domain.SourceTotals{SourceName: row.SourceName, Files: current.Files + 1, Bytes: current.Bytes + row.Size}
}
sources := make([]domain.SourceTotals, 0, len(bySource))
for _, source := range bySource {
sources = append(sources, source)
}
sort.Slice(sources, func(i, j int) bool {
if sources[i].Bytes != sources[j].Bytes {
return sources[i].Bytes > sources[j].Bytes
}
return sources[i].SourceName < sources[j].SourceName
})
return sources[:min(limit, len(sources))], nil
}
func (m *mockVariantFileRepo) ListPage(filter domain.VariantFileFilter, afterID uint32, limit int) ([]domain.VariantFile, error) {
m.mutex.Lock()
defer m.mutex.Unlock()
m.pageLimits = append(m.pageLimits, limit)
if m.failPage != nil {
return nil, m.failPage
}
page := []domain.VariantFile{}
for _, row := range m.rows {
if row.ID <= afterID || (filter.VariantName != "" && row.VariantName != filter.VariantName) || (filter.SourceName != "" && row.SourceName != filter.SourceName) {
continue
}
if filter.Unused != nil && !m.isUnused(row, *filter.Unused) {
continue
}
page = append(page, row)
}
sort.Slice(page, func(i, j int) bool { return page[i].ID < page[j].ID })
return page[:min(limit, len(page))], nil
}
func (m *mockVariantFileRepo) Find(variantName string, sourceName string) (domain.VariantFile, bool, error) {
@@ -190,6 +267,10 @@ func (m *mockVariantFileRepo) Totals() (map[string]domain.VariantFileTotals, err
m.mutex.Lock()
defer m.mutex.Unlock()
m.calls++
m.totalsCalls++
if m.failTotals != nil {
return nil, m.failTotals
}
totals := map[string]domain.VariantFileTotals{}
for _, row := range m.rows {
current := totals[row.VariantName]
+443
View File
@@ -0,0 +1,443 @@
package test
import (
domain "RenewCMS/internal/domain/image"
entity "RenewCMS/internal/infrastructure/persistence/models"
persistence "RenewCMS/internal/infrastructure/persistence/repositories"
"RenewCMS/internal/infrastructure/usage"
"context"
"strconv"
"sync"
"testing"
"time"
"gorm.io/gorm"
)
func newUsageTracker(repo domain.VariantUsageRepository, clock *time.Time) *usage.Tracker {
return usage.NewTracker(repo, func() time.Time { return *clock })
}
func TestVariantUsageTracker(t *testing.T) {
start := time.Date(2026, time.October, 8, 10, 0, 0, 0, time.UTC)
t.Run("The flush writes one batch for many files", func(t *testing.T) {
repo, clock := newMockVariantUsageRepo(), start
tracker := newUsageTracker(repo, &clock)
for index := 0; index < 40; index++ {
tracker.RecordHit("thumb", "file-"+strconv.Itoa(index)+".png")
}
if err := tracker.Flush(); err != nil {
t.Fatal(err)
}
if repo.batchCount() != 1 || repo.callCount() != 1 || len(repo.recordedHits()) != 40 {
t.Errorf("batches = %d, calls = %d, hits = %d", repo.batchCount(), repo.callCount(), len(repo.recordedHits()))
}
})
t.Run("Nothing pending means no write", func(t *testing.T) {
repo, clock := newMockVariantUsageRepo(), start
tracker := newUsageTracker(repo, &clock)
_ = tracker.Flush()
tracker.RecordHit("thumb", "a.png")
_ = tracker.Flush()
_ = tracker.Flush()
if repo.callCount() != 1 {
t.Errorf("calls = %d", repo.callCount())
}
})
t.Run("A thousand hits on one file are one update with the time of the last hit", func(t *testing.T) {
repo, clock := newMockVariantUsageRepo(), start
tracker := newUsageTracker(repo, &clock)
for index := 0; index < 1000; index++ {
clock = start.Add(time.Duration(index) * time.Second)
tracker.RecordHit("thumb", "a.png")
}
_ = tracker.Flush()
hits := repo.recordedHits()
if repo.callCount() != 1 || len(hits) != 1 || !hits[0].At.Equal(start.Add(999*time.Second)) {
t.Errorf("calls = %d, hits = %+v", repo.callCount(), hits)
}
})
t.Run("A failed flush keeps the pending hits", func(t *testing.T) {
repo, clock := newMockVariantUsageRepo(), start
tracker := newUsageTracker(repo, &clock)
repo.fail = errUsageStoreDown
tracker.RecordHit("thumb", "a.png")
if err := tracker.Flush(); err == nil {
t.Fatal("the failure was swallowed")
}
clock = start.Add(time.Minute)
tracker.RecordHit("thumb", "a.png")
tracker.RecordHit("hero", "b.png")
repo.fail = nil
_ = tracker.Flush()
hits := repo.recordedHits()
if repo.batchCount() != 1 || len(hits) != 2 {
t.Fatalf("batches = %d, hits = %+v", repo.batchCount(), hits)
}
for _, hit := range hits {
if !hit.At.Equal(clock) {
t.Errorf("hit = %+v", hit)
}
}
})
t.Run("A failed flush keeps the pending rejections", func(t *testing.T) {
repo, clock := newMockVariantUsageRepo(), start
tracker := newUsageTracker(repo, &clock)
repo.fail = errUsageStoreDown
tracker.RecordRejection()
tracker.RecordRejection()
_ = tracker.Flush()
tracker.RecordRejection()
repo.fail = nil
_ = tracker.Flush()
_ = tracker.Flush()
if repo.recordedRejections() != 3 || repo.batchCount() != 1 || repo.days["2026-10-08"] != 3 {
t.Errorf("rejections = %d, batches = %d, days = %v", repo.recordedRejections(), repo.batchCount(), repo.days)
}
})
t.Run("The flush loop and the size trigger", func(t *testing.T) {
repo, clock := newMockVariantUsageRepo(), start
tracker := usage.NewTrackerWith(repo, func() time.Time { return clock }, time.Hour, 500)
ctx, stop := context.WithCancel(context.Background())
defer stop()
done := make(chan struct{})
go func() {
defer close(done)
tracker.Run(ctx)
}()
for index := 0; index < 499; index++ {
tracker.RecordHit("thumb", "file-"+strconv.Itoa(index)+".png")
}
time.Sleep(50 * time.Millisecond)
if repo.callCount() != 0 {
t.Fatal("flushed before the size trigger")
}
tracker.RecordHit("thumb", "file-499.png")
waitUntilUsage(t, func() bool { return repo.callCount() == 1 })
tracker.RecordHit("thumb", "late.png")
stop()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("the loop did not stop")
}
if repo.callCount() != 2 {
t.Errorf("calls after shutdown = %d", repo.callCount())
}
})
t.Run("The defaults are a thirty second interval and a 500 key trigger", func(t *testing.T) {
if usage.FlushInterval() != 30*time.Second || usage.FlushKeyLimit != 500 {
t.Errorf("interval = %v, limit = %d", usage.FlushInterval(), usage.FlushKeyLimit)
}
})
t.Run("The same key repeated never reaches the size trigger", func(t *testing.T) {
repo, clock := newMockVariantUsageRepo(), start
tracker := usage.NewTrackerWith(repo, func() time.Time { return clock }, time.Hour, 2)
ctx, stop := context.WithCancel(context.Background())
defer stop()
go tracker.Run(ctx)
for index := 0; index < 5; index++ {
tracker.RecordHit("thumb", "a.png")
}
time.Sleep(50 * time.Millisecond)
if repo.callCount() != 0 {
t.Errorf("calls = %d", repo.callCount())
}
})
t.Run("A tick flushes", func(t *testing.T) {
repo, clock := newMockVariantUsageRepo(), start
tracker := usage.NewTrackerWith(repo, func() time.Time { return clock }, 10*time.Millisecond, 500)
ctx, stop := context.WithCancel(context.Background())
defer stop()
go tracker.Run(ctx)
tracker.RecordHit("thumb", "a.png")
waitUntilUsage(t, func() bool { return repo.callCount() == 1 })
})
t.Run("Concurrent requests are race free", func(t *testing.T) {
repo, clock := newMockVariantUsageRepo(), start
tracker := newUsageTracker(repo, &clock)
var group sync.WaitGroup
for worker := 0; worker < 50; worker++ {
group.Add(1)
go func() {
defer group.Done()
for index := 0; index < 100; index++ {
tracker.RecordHit("thumb", "file-"+strconv.Itoa(index)+".png")
tracker.RecordRejection()
}
}()
}
flushed := make(chan struct{})
go func() {
defer close(flushed)
for index := 0; index < 20; index++ {
_ = tracker.Flush()
}
}()
group.Wait()
<-flushed
_ = tracker.Flush()
if repo.recordedRejections() != 5000 {
t.Errorf("rejections = %d", repo.recordedRejections())
}
seen := map[string]bool{}
for _, hit := range repo.recordedHits() {
seen[hit.SourceName] = true
}
if len(seen) != 100 {
t.Errorf("distinct files = %d", len(seen))
}
})
}
func waitUntilUsage(t *testing.T, condition func() bool) {
t.Helper()
deadline := time.Now().Add(500 * time.Millisecond)
for time.Now().Before(deadline) {
if condition() {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatal("the condition was not met in time")
}
func requestedAt(t *testing.T, db *gorm.DB, variantName string, sourceName string) *time.Time {
t.Helper()
var row entity.ImageVariantFile
if err := db.Where("variant_name = ? AND source_name = ?", variantName, sourceName).First(&row).Error; err != nil {
t.Fatal(err)
}
return row.LastRequestedAt
}
func TestVariantUsageRepository(t *testing.T) {
noon := time.Date(2026, time.October, 8, 10, 5, 0, 0, time.UTC)
seed := func(t *testing.T) (*gorm.DB, *persistence.VariantFileRepository, *persistence.VariantUsageRepository) {
t.Helper()
db := openActivityDatabase(t)
files := persistence.NewVariantFileRepository(db)
_, err := files.Save(domain.VariantFile{VariantName: "thumb", SourceName: "a.png", CachedName: "g1", ContentType: "image/png", Size: 10, Fingerprint: "f"})
if err != nil {
t.Fatal(err)
}
return db, files, persistence.NewVariantUsageRepository(db)
}
t.Run("A batch sets the last requested time of the file", func(t *testing.T) {
db, _, usageRepo := seed(t)
err := usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}})
if got := requestedAt(t, db, "thumb", "a.png"); err != nil || got == nil || !got.Equal(noon) {
t.Errorf("last_requested_at = %v, err = %v", got, err)
}
})
t.Run("A stored time never moves back", func(t *testing.T) {
db, _, usageRepo := seed(t)
_ = usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}})
_ = usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon.Add(-4 * time.Minute)}}})
if got := requestedAt(t, db, "thumb", "a.png"); got == nil || !got.Equal(noon) {
t.Errorf("last_requested_at = %v", got)
}
})
t.Run("A purged file ignores a late update", func(t *testing.T) {
db, files, usageRepo := seed(t)
row, _, _ := files.Find("thumb", "a.png")
_ = files.Delete(row.ID)
err := usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}})
var count int64
db.Model(&entity.ImageVariantFile{}).Count(&count)
if err != nil || count != 0 {
t.Errorf("err = %v, rows = %d", err, count)
}
})
t.Run("A regenerated file keeps its last requested time", func(t *testing.T) {
db, files, usageRepo := seed(t)
_ = usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}})
_, _ = files.Save(domain.VariantFile{VariantName: "thumb", SourceName: "a.png", CachedName: "g2", ContentType: "image/png", Size: 20, Fingerprint: "f2"})
if got := requestedAt(t, db, "thumb", "a.png"); got == nil || !got.Equal(noon) {
t.Errorf("last_requested_at = %v", got)
}
})
t.Run("Rejections add up per day and the window sums its days", func(t *testing.T) {
_, _, usageRepo := seed(t)
day := func(value string) time.Time {
parsed, _ := time.Parse(domain.RejectionDayLayout, value)
return parsed
}
for _, batch := range [][]domain.RejectionCount{
{{Day: day("2026-10-08"), Count: 3}, {Day: day("2026-10-02"), Count: 5}},
{{Day: day("2026-10-08"), Count: 4}, {Day: day("2026-10-01"), Count: 9}},
} {
if err := usageRepo.Record(domain.VariantUsageBatch{Rejections: batch}); err != nil {
t.Fatal(err)
}
}
total, err := usageRepo.RejectedSince(domain.RejectionWindowStart(time.Date(2026, time.October, 8, 23, 59, 0, 0, time.UTC)))
if err != nil || total != 12 {
t.Errorf("total = %d, err = %v", total, err)
}
})
t.Run("Old day rows are pruned by the batch", func(t *testing.T) {
db, _, usageRepo := seed(t)
day := func(value string) time.Time {
parsed, _ := time.Parse(domain.RejectionDayLayout, value)
return parsed
}
_ = usageRepo.Record(domain.VariantUsageBatch{Rejections: []domain.RejectionCount{{Day: day("2026-09-30"), Count: 1}, {Day: day("2026-10-02"), Count: 2}}})
_ = usageRepo.Record(domain.VariantUsageBatch{Rejections: []domain.RejectionCount{{Day: day("2026-10-08"), Count: 1}}, PruneBefore: domain.RejectionPruneBefore(noon)})
var days []string
db.Model(&entity.ImageVariantRejectionDay{}).Order("day").Pluck("day", &days)
if len(days) != 2 || days[0] != "2026-10-02" || days[1] != "2026-10-08" {
t.Errorf("days = %v", days)
}
})
}
func TestVariantCacheRepository(t *testing.T) {
now := time.Date(2026, time.October, 8, 9, 0, 0, 0, time.UTC)
cutoff := domain.UnusedCutoff(now)
setup := func(t *testing.T) (*gorm.DB, *persistence.VariantFileRepository) {
t.Helper()
db := openActivityDatabase(t)
return db, persistence.NewVariantFileRepository(db)
}
save := func(t *testing.T, db *gorm.DB, files *persistence.VariantFileRepository, variantName string, sourceName string, size int64, created time.Time, requested *time.Time) {
t.Helper()
row, err := files.Save(domain.VariantFile{VariantName: variantName, SourceName: sourceName, CachedName: variantName + sourceName, ContentType: "image/png", Size: size, Fingerprint: "f"})
if err != nil {
t.Fatal(err)
}
db.Model(&entity.ImageVariantFile{}).Where("id = ?", row.ID).Updates(map[string]any{"created_at": created, "last_requested_at": requested})
}
at := func(days int) *time.Time {
value := now.AddDate(0, 0, -days)
return &value
}
t.Run("The reference time is last_requested_at when set and created_at otherwise", func(t *testing.T) {
db, files := setup(t)
save(t, db, files, "hero", "recent.png", 10, now.AddDate(0, 0, -90), at(29))
save(t, db, files, "hero", "old.png", 20, now.AddDate(0, 0, -1), at(31))
save(t, db, files, "print", "disabled.png", 40, now.AddDate(0, 0, -90), at(90))
save(t, db, files, "card", "never.png", 80, now.AddDate(0, 0, -45), nil)
save(t, db, files, "card", "fresh.png", 160, now.AddDate(0, 0, -2), nil)
totals, err := files.UnusedTotals(domain.UnusedFilter{VariantNames: []string{"hero", "card"}, Cutoff: cutoff})
if err != nil || totals.Files != 2 || totals.Bytes != 100 {
t.Errorf("totals = %+v, err = %v", totals, err)
}
})
t.Run("A row whose reference time is exactly the cutoff is in use", func(t *testing.T) {
db, files := setup(t)
save(t, db, files, "hero", "edge.png", 10, now.AddDate(0, 0, -90), &cutoff)
totals, _ := files.UnusedTotals(domain.UnusedFilter{VariantNames: []string{"hero"}, Cutoff: cutoff})
if totals.Files != 0 {
t.Errorf("totals = %+v", totals)
}
})
t.Run("No enabled variant means nothing is unused", func(t *testing.T) {
db, files := setup(t)
save(t, db, files, "hero", "old.png", 10, now.AddDate(0, 0, -90), nil)
totals, err := files.UnusedTotals(domain.UnusedFilter{Cutoff: cutoff})
page, pageErr := files.ListPage(domain.VariantFileFilter{Unused: &domain.UnusedFilter{Cutoff: cutoff}}, 0, 10)
if err != nil || pageErr != nil || totals.Files != 0 || len(page) != 0 {
t.Errorf("totals = %+v, page = %v", totals, page)
}
})
t.Run("The top sources are ordered by bytes then name and capped", func(t *testing.T) {
db, files := setup(t)
save(t, db, files, "hero", "b.png", 100, now, nil)
save(t, db, files, "card", "b.png", 100, now, nil)
save(t, db, files, "hero", "a.png", 200, now, nil)
save(t, db, files, "hero", "c.png", 900, now, nil)
top, err := files.TopSources(3)
if err != nil || len(top) != 3 || top[0].SourceName != "c.png" || top[1].SourceName != "a.png" || top[2].SourceName != "b.png" || top[2].Files != 2 || top[2].Bytes != 200 {
t.Errorf("top = %+v, err = %v", top, err)
}
})
t.Run("A page continues after the given id and filters by variant, source and unused", func(t *testing.T) {
db, files := setup(t)
for index := 0; index < 5; index++ {
save(t, db, files, "hero", "s"+strconv.Itoa(index)+".png", 1, now.AddDate(0, 0, -90), nil)
}
save(t, db, files, "card", "s0.png", 1, now, nil)
first, _ := files.ListPage(domain.VariantFileFilter{VariantName: "hero"}, 0, 2)
second, _ := files.ListPage(domain.VariantFileFilter{VariantName: "hero"}, first[1].ID, 10)
bySource, _ := files.ListPage(domain.VariantFileFilter{SourceName: "s0.png"}, 0, 10)
unused, _ := files.ListPage(domain.VariantFileFilter{Unused: &domain.UnusedFilter{VariantNames: []string{"hero", "card"}, Cutoff: cutoff}}, 0, 10)
if len(first) != 2 || len(second) != 3 || second[0].ID <= first[1].ID || len(bySource) != 2 || len(unused) != 5 {
t.Errorf("first = %d, second = %d, bySource = %d, unused = %d", len(first), len(second), len(bySource), len(unused))
}
})
}
func TestImageRepositoryListByStoredNames(t *testing.T) {
repo := persistence.NewImageRepository(openActivityDatabase(t))
for _, name := range []string{"a.png", "b.png", "c.png"} {
_, _ = repo.Create(domain.Image{Path: domain.PublicPath(name), Filename: name})
}
found, err := repo.ListByStoredNames([]string{"c.png", "a.png", "missing.png"})
none, noneErr := repo.ListByStoredNames(nil)
if err != nil || noneErr != nil || len(found) != 2 || found[0].Filename != "a.png" || found[1].Filename != "c.png" || len(none) != 0 {
t.Errorf("found = %+v, err = %v, none = %v", found, err, none)
}
}