feat: image variants cache panel
This commit is contained in:
1 parent
951fef18ba
commit
d768668014
53 files changed
+5809
-46
No files matched your search
@@ -5,6 +5,7 @@ import (
|
||||
"RenewCMS/internal/infrastructure/useCases"
|
||||
"errors"
|
||||
"io"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -78,6 +79,8 @@ type mockImageRepo struct {
|
||||
deleteCalls []uint32
|
||||
createCalls int
|
||||
onDelete func(id uint32)
|
||||
|
||||
storedNameLookups int
|
||||
}
|
||||
|
||||
func newMockImageRepo() *mockImageRepo {
|
||||
@@ -257,6 +260,20 @@ func (m *mockImageRepo) ExistsByStoredName(storedName string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (m *mockImageRepo) ListByStoredNames(storedNames []string) ([]imageDomain.Image, error) {
|
||||
if err := m.fail.trigger("ListByStoredNames"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.storedNameLookups++
|
||||
result := make([]imageDomain.Image, 0)
|
||||
for _, img := range m.sortedImages() {
|
||||
if slices.Contains(storedNames, imageDomain.StoredName(img.Path)) {
|
||||
result = append(result, img)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
var _ imageDomain.Repository = &mockImageRepo{}
|
||||
|
||||
func TestImageUseCases(t *testing.T) {
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,86 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestImageVariantCacheLayering(t *testing.T) {
|
||||
t.Run("The cache use cases and the tracker import no persistence, mailer, models, gorm, net/http, chi or api package", func(t *testing.T) {
|
||||
files := []string{
|
||||
"internal/infrastructure/useCases/GetVariantCacheStats.go",
|
||||
"internal/infrastructure/useCases/PurgeVariantCache.go",
|
||||
"internal/infrastructure/usage/variantUsageTracker.go",
|
||||
}
|
||||
for _, file := range files {
|
||||
for _, path := range importsOf(t, file) {
|
||||
for _, fragment := range []string{"persistence", "mailer", "models", "gorm", "net/http", "go-chi", "RenewCMS/api"} {
|
||||
if strings.Contains(path, fragment) {
|
||||
t.Errorf("%s imports %s", file, path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("The cache handler imports no persistence, mailer, models or gorm package", func(t *testing.T) {
|
||||
for _, path := range importsOf(t, "api/handlers/imagevariant/cacheHandler.go") {
|
||||
for _, fragment := range []string{"persistence", "mailer", "models", "gorm"} {
|
||||
if strings.Contains(path, fragment) {
|
||||
t.Errorf("imports %s", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("The tracker, usage and cache ports are declared in internal/domain/image", func(t *testing.T) {
|
||||
declared := readRepositoryFile(t, "internal/domain/image/variantRepositoryInterface.go")
|
||||
|
||||
for _, port := range []string{"type VariantUsageTracker interface", "type VariantUsageRepository interface", "type VariantCacheRepository interface"} {
|
||||
if !strings.Contains(declared, port) {
|
||||
t.Errorf("%s is not declared", port)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("The new models are registered in migrate.go", func(t *testing.T) {
|
||||
migrate := readRepositoryFile(t, "internal/infrastructure/persistence/database/migrate.go")
|
||||
|
||||
if !strings.Contains(migrate, "&models.ImageVariantRejectionDay{}") {
|
||||
t.Error("the rejection day model is missing from AutoMigrate")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Only the composition root constructs the tracker", func(t *testing.T) {
|
||||
for _, file := range []string{"api/router.go", "cmd/server/main.go", "api/handlers/staticimages/staticImagesHandler.go", "api/handlers/imagevariant/cacheHandler.go"} {
|
||||
if strings.Contains(readRepositoryFile(t, file), "NewTracker") {
|
||||
t.Errorf("%s constructs the tracker", file)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("The new files carry no comment", func(t *testing.T) {
|
||||
for _, relative := range []string{
|
||||
"internal/domain/image/variantCache.go",
|
||||
"internal/infrastructure/usage/variantUsageTracker.go",
|
||||
"internal/infrastructure/useCases/GetVariantCacheStats.go",
|
||||
"internal/infrastructure/useCases/PurgeVariantCache.go",
|
||||
"internal/infrastructure/persistence/repositories/imageVariantUsageRepository.go",
|
||||
"api/handlers/imagevariant/cacheHandler.go",
|
||||
} {
|
||||
parsed, err := parser.ParseFile(token.NewFileSet(), "../"+relative, nil, parser.ParseComments)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, group := range parsed.Comments {
|
||||
for _, comment := range group.List {
|
||||
if !strings.HasPrefix(comment.Text, "//go:") {
|
||||
t.Errorf("%s has comment %q", relative, comment.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
domain "RenewCMS/internal/domain/image"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var errUsageStoreDown = errors.New("usage store down")
|
||||
|
||||
type mockVariantUsageRepo struct {
|
||||
mutex sync.Mutex
|
||||
batches []domain.VariantUsageBatch
|
||||
days map[string]int
|
||||
fail error
|
||||
calls int
|
||||
}
|
||||
|
||||
var _ domain.VariantUsageRepository = &mockVariantUsageRepo{}
|
||||
|
||||
func newMockVariantUsageRepo() *mockVariantUsageRepo {
|
||||
return &mockVariantUsageRepo{days: map[string]int{}}
|
||||
}
|
||||
|
||||
func (m *mockVariantUsageRepo) Record(batch domain.VariantUsageBatch) error {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
m.calls++
|
||||
if m.fail != nil {
|
||||
return m.fail
|
||||
}
|
||||
m.batches = append(m.batches, batch)
|
||||
for _, rejection := range batch.Rejections {
|
||||
m.days[rejection.Day.Format(domain.RejectionDayLayout)] += rejection.Count
|
||||
}
|
||||
if !batch.PruneBefore.IsZero() {
|
||||
for day := range m.days {
|
||||
if day < batch.PruneBefore.Format(domain.RejectionDayLayout) {
|
||||
delete(m.days, day)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockVariantUsageRepo) RejectedSince(day time.Time) (int, error) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
total := 0
|
||||
for stored, count := range m.days {
|
||||
if stored >= day.Format(domain.RejectionDayLayout) {
|
||||
total += count
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (m *mockVariantUsageRepo) recordedHits() []domain.VariantHit {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
hits := []domain.VariantHit{}
|
||||
for _, batch := range m.batches {
|
||||
hits = append(hits, batch.Hits...)
|
||||
}
|
||||
return hits
|
||||
}
|
||||
|
||||
func (m *mockVariantUsageRepo) recordedRejections() int {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
total := 0
|
||||
for _, batch := range m.batches {
|
||||
for _, rejection := range batch.Rejections {
|
||||
total += rejection.Count
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (m *mockVariantUsageRepo) batchCount() int {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
return len(m.batches)
|
||||
}
|
||||
|
||||
func (m *mockVariantUsageRepo) callCount() int {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
return m.calls
|
||||
}
|
||||
|
||||
type cacheFixtureImage struct {
|
||||
id uint32
|
||||
filename string
|
||||
files int
|
||||
bytes int64
|
||||
}
|
||||
|
||||
var cacheFixtureTopImages = []cacheFixtureImage{
|
||||
{1, "north-wall.jpg", 6, 6082765},
|
||||
{2, "lime-plaster.jpg", 6, 5138022},
|
||||
{3, "kitchen-after.webp", 5, 4613734},
|
||||
{4, "kitchen-before.webp", 5, 4089446},
|
||||
{5, "floorboards.jpg", 5, 3774873},
|
||||
{6, "IMG_4471.jpg", 4, 3250585},
|
||||
{7, "stone-wall-detail.jpg", 4, 2936012},
|
||||
{8, "bathroom-tiles.png", 4, 2726297},
|
||||
{9, "roof-lantern.jpg", 3, 2411724},
|
||||
{10, "chimney-stack.jpg", 3, 2202009},
|
||||
}
|
||||
|
||||
type cacheFixtureVariant struct {
|
||||
name string
|
||||
enabled bool
|
||||
files int
|
||||
bytes int64
|
||||
}
|
||||
|
||||
var cacheFixtureVariants = []cacheFixtureVariant{
|
||||
{"hero", true, 244, 88080384},
|
||||
{"card", true, 318, 60817408},
|
||||
{"thumb", true, 412, 22020096},
|
||||
{"print", false, 74, 22020096},
|
||||
{"square", true, 140, 19922944},
|
||||
{"og", true, 96, 9437184},
|
||||
}
|
||||
|
||||
const (
|
||||
cacheFixtureUnusedHeroRows = 200
|
||||
cacheFixtureUnusedHeroBytes = 40000000
|
||||
cacheFixtureUnusedThumbRows = 112
|
||||
cacheFixtureUnusedThumbSize = 2991616
|
||||
)
|
||||
|
||||
func splitBytes(total int64, parts int) []int64 {
|
||||
sizes := make([]int64, parts)
|
||||
for index := range sizes {
|
||||
sizes[index] = total / int64(parts)
|
||||
if int64(index) < total%int64(parts) {
|
||||
sizes[index]++
|
||||
}
|
||||
}
|
||||
return sizes
|
||||
}
|
||||
|
||||
func (e *variantEnv) seedCacheRow(variantName string, sourceName string, size int64, reference time.Time) domain.VariantFile {
|
||||
cached := "fixture-" + variantName + "-" + sourceName + ".jpeg"
|
||||
e.storage.seed(cached, []byte("generated"))
|
||||
saved, _ := e.files.Save(domain.VariantFile{VariantName: variantName, SourceName: sourceName, CachedName: cached, ContentType: "image/jpeg", Size: size, Fingerprint: "fixture"})
|
||||
e.files.setReference(saved.ID, reference)
|
||||
return saved
|
||||
}
|
||||
|
||||
func (e *variantEnv) seedDefaultCache() {
|
||||
recent := e.now.AddDate(0, 0, -5)
|
||||
stale := e.now.AddDate(0, 0, -31)
|
||||
for _, variant := range cacheFixtureVariants {
|
||||
seeded := seedVariant(e.variants, variant.name, 400, 300, domain.FitCover, domain.FormatJPEG, 80)
|
||||
seeded.Enabled = variant.enabled
|
||||
e.variants.variants[variant.name] = seeded
|
||||
}
|
||||
topBytes := map[string]int64{}
|
||||
topFiles := map[string]int{}
|
||||
for _, img := range cacheFixtureTopImages {
|
||||
e.seedImage(img.id, img.filename)
|
||||
for index, size := range splitBytes(img.bytes, img.files) {
|
||||
variant := cacheFixtureVariants[index%len(cacheFixtureVariants)].name
|
||||
e.seedCacheRow(variant, img.filename, size, recent)
|
||||
topBytes[variant] += size
|
||||
topFiles[variant]++
|
||||
}
|
||||
}
|
||||
for _, variant := range cacheFixtureVariants {
|
||||
fillers := variant.files - topFiles[variant.name]
|
||||
remaining := variant.bytes - topBytes[variant.name]
|
||||
sizes := e.fillerSizes(variant.name, fillers, remaining)
|
||||
for index, size := range sizes {
|
||||
reference := recent
|
||||
if e.isFixtureStale(variant.name, index) {
|
||||
reference = stale
|
||||
}
|
||||
e.seedCacheRow(variant.name, "filler-"+variant.name+"-"+strconv.Itoa(index), size, reference)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *variantEnv) isFixtureStale(variantName string, index int) bool {
|
||||
return (variantName == "hero" && index < cacheFixtureUnusedHeroRows) || (variantName == "thumb" && index < cacheFixtureUnusedThumbRows)
|
||||
}
|
||||
|
||||
func (e *variantEnv) fillerSizes(variantName string, fillers int, remaining int64) []int64 {
|
||||
switch variantName {
|
||||
case "hero":
|
||||
return append(splitBytes(cacheFixtureUnusedHeroBytes, cacheFixtureUnusedHeroRows), splitBytes(remaining-cacheFixtureUnusedHeroBytes, fillers-cacheFixtureUnusedHeroRows)...)
|
||||
case "thumb":
|
||||
return append(splitBytes(cacheFixtureUnusedThumbSize, cacheFixtureUnusedThumbRows), splitBytes(remaining-cacheFixtureUnusedThumbSize, fillers-cacheFixtureUnusedThumbRows)...)
|
||||
default:
|
||||
return splitBytes(remaining, fillers)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *variantEnv) seedRejectedDays(days map[string]int) {
|
||||
for day, count := range days {
|
||||
e.usage.days[day] = count
|
||||
}
|
||||
}
|
||||
|
||||
func (e *variantEnv) cacheStats() (*httptest.ResponseRecorder, map[string]any) {
|
||||
e.t.Helper()
|
||||
recorder := e.sendAsAdmin(http.MethodGet, "/v1/image-variants/cache", "")
|
||||
if recorder.Code != http.StatusOK {
|
||||
return recorder, nil
|
||||
}
|
||||
return recorder, decodeBody(e.t, recorder)
|
||||
}
|
||||
|
||||
func (e *variantEnv) purgeCache(body string) *httptest.ResponseRecorder {
|
||||
e.t.Helper()
|
||||
return e.send(http.MethodPost, "/v1/image-variants/cache/purge", body, variantOwnerId)
|
||||
}
|
||||
|
||||
func (e *variantEnv) recentRejectedFixture() {
|
||||
e.seedRejectedDays(map[string]int{"2026-10-08": 30, "2026-10-05": 7, "2026-09-20": 100})
|
||||
}
|
||||
|
||||
func compactJSON(body *httptest.ResponseRecorder) string {
|
||||
return strings.TrimSpace(body.Body.String())
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"RenewCMS/internal/domain/user"
|
||||
"RenewCMS/internal/infrastructure/config"
|
||||
"RenewCMS/internal/infrastructure/hashing"
|
||||
"RenewCMS/internal/infrastructure/usage"
|
||||
"RenewCMS/internal/infrastructure/useCases"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
@@ -50,6 +51,8 @@ type variantEnv struct {
|
||||
logs *bytes.Buffer
|
||||
token *jwtauth.JWTAuth
|
||||
now time.Time
|
||||
usage *mockVariantUsageRepo
|
||||
tracker *usage.Tracker
|
||||
admin http.Handler
|
||||
public http.Handler
|
||||
}
|
||||
@@ -81,7 +84,9 @@ func newVariantEnvLimits(t *testing.T, renderer *countingRenderer, limits useCas
|
||||
logs: &bytes.Buffer{},
|
||||
token: jwtauth.New("HS256", []byte("test-secret"), nil),
|
||||
now: time.Date(2026, time.October, 8, 9, 0, 0, 0, time.UTC),
|
||||
usage: newMockVariantUsageRepo(),
|
||||
}
|
||||
env.tracker = usage.NewTracker(env.usage, func() time.Time { return env.now })
|
||||
env.users.users[variantOwnerId] = user.User{ID: variantOwnerId, Username: "marta", DisplayName: "Marta Alves", IsVerified: true, Role: user.RoleOwner}
|
||||
env.users.users[variantAdminId] = user.User{ID: variantAdminId, Username: "dev", IsVerified: true, Role: user.RoleAdmin}
|
||||
env.users.users[variantPlainUserId] = user.User{ID: variantPlainUserId, Username: "pat", IsVerified: true}
|
||||
@@ -103,7 +108,7 @@ func newVariantEnvLimits(t *testing.T, renderer *countingRenderer, limits useCas
|
||||
func (e *variantEnv) container(limits useCases.ServeVariantLimits) *api.UseCases {
|
||||
clock := func() time.Time { return e.now }
|
||||
recorder := e.audit.recorder
|
||||
serve := useCases.NewServeVariantUseCase(e.variants, e.files, e.images, e.storage, e.renderer, limits)
|
||||
serve := useCases.NewServeVariantUseCase(e.variants, e.files, e.images, e.storage, e.renderer, limits).WithUsageTracker(e.tracker)
|
||||
return &api.UseCases{
|
||||
Token: e.token,
|
||||
GetUserUseCase: useCases.NewGetUserUseCase(e.users),
|
||||
@@ -117,10 +122,15 @@ func (e *variantEnv) container(limits useCases.ServeVariantLimits) *api.UseCases
|
||||
DeleteVariantUseCase: useCases.NewDeleteVariantUseCase(e.variants, e.files, e.storage, recorder),
|
||||
PurgeVariantUseCase: useCases.NewPurgeVariantUseCase(e.variants, e.files, e.storage, recorder),
|
||||
ServeVariantUseCase: serve,
|
||||
CreateImageUseCase: useCases.NewCreateImageUseCase(e.images, e.storage, recorder),
|
||||
ListMediaUseCase: useCases.NewListMediaUseCase(e.images, e.articles, e.users, e.files),
|
||||
RenameMediaUseCase: useCases.NewRenameMediaUseCase(e.images, recorder),
|
||||
GetMediaUsageUseCase: useCases.NewGetMediaUsageUseCase(e.images, e.articles),
|
||||
|
||||
GetVariantCacheStatsUseCase: useCases.NewGetVariantCacheStatsUseCase(e.variants, e.files, e.files, e.images, e.usage, e.tracker, clock),
|
||||
PurgeVariantCacheUseCase: useCases.NewPurgeVariantCacheUseCase(e.variants, e.images, e.files, e.files, e.storage, clock, recorder),
|
||||
VariantUsageTracker: e.tracker,
|
||||
|
||||
CreateImageUseCase: useCases.NewCreateImageUseCase(e.images, e.storage, recorder),
|
||||
ListMediaUseCase: useCases.NewListMediaUseCase(e.images, e.articles, e.users, e.files),
|
||||
RenameMediaUseCase: useCases.NewRenameMediaUseCase(e.images, recorder),
|
||||
GetMediaUsageUseCase: useCases.NewGetMediaUsageUseCase(e.images, e.articles),
|
||||
|
||||
ListImageVariantsUseCase: useCases.NewListImageVariantsUseCase(e.images, e.variants, e.files, e.renderer),
|
||||
GenerateImageVariantUseCase: useCases.NewGenerateImageVariantUseCase(e.images, e.variants, e.files, e.renderer, serve),
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -117,12 +118,88 @@ type mockVariantFileRepo struct {
|
||||
calls int
|
||||
|
||||
totalsBySourceCalls int
|
||||
|
||||
references map[uint32]time.Time
|
||||
failTotals error
|
||||
failPage error
|
||||
totalsCalls int
|
||||
unusedCalls int
|
||||
topCalls int
|
||||
pageLimits []int
|
||||
}
|
||||
|
||||
var _ domain.VariantFileRepository = &mockVariantFileRepo{}
|
||||
var _ domain.VariantCacheRepository = &mockVariantFileRepo{}
|
||||
|
||||
func newMockVariantFileRepo() *mockVariantFileRepo {
|
||||
return &mockVariantFileRepo{rows: map[uint32]domain.VariantFile{}}
|
||||
return &mockVariantFileRepo{rows: map[uint32]domain.VariantFile{}, references: map[uint32]time.Time{}}
|
||||
}
|
||||
|
||||
func (m *mockVariantFileRepo) setReference(id uint32, at time.Time) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
m.references[id] = at
|
||||
}
|
||||
|
||||
func (m *mockVariantFileRepo) isUnused(row domain.VariantFile, filter domain.UnusedFilter) bool {
|
||||
return slices.Contains(filter.VariantNames, row.VariantName) && m.references[row.ID].Before(filter.Cutoff)
|
||||
}
|
||||
|
||||
func (m *mockVariantFileRepo) UnusedTotals(filter domain.UnusedFilter) (domain.VariantFileTotals, error) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
m.unusedCalls++
|
||||
var totals domain.VariantFileTotals
|
||||
for _, row := range m.rows {
|
||||
if m.isUnused(row, filter) {
|
||||
totals.Files++
|
||||
totals.Bytes += row.Size
|
||||
}
|
||||
}
|
||||
return totals, nil
|
||||
}
|
||||
|
||||
func (m *mockVariantFileRepo) TopSources(limit int) ([]domain.SourceTotals, error) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
m.topCalls++
|
||||
bySource := map[string]domain.SourceTotals{}
|
||||
for _, row := range m.rows {
|
||||
current := bySource[row.SourceName]
|
||||
bySource[row.SourceName] = domain.SourceTotals{SourceName: row.SourceName, Files: current.Files + 1, Bytes: current.Bytes + row.Size}
|
||||
}
|
||||
sources := make([]domain.SourceTotals, 0, len(bySource))
|
||||
for _, source := range bySource {
|
||||
sources = append(sources, source)
|
||||
}
|
||||
sort.Slice(sources, func(i, j int) bool {
|
||||
if sources[i].Bytes != sources[j].Bytes {
|
||||
return sources[i].Bytes > sources[j].Bytes
|
||||
}
|
||||
return sources[i].SourceName < sources[j].SourceName
|
||||
})
|
||||
return sources[:min(limit, len(sources))], nil
|
||||
}
|
||||
|
||||
func (m *mockVariantFileRepo) ListPage(filter domain.VariantFileFilter, afterID uint32, limit int) ([]domain.VariantFile, error) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
m.pageLimits = append(m.pageLimits, limit)
|
||||
if m.failPage != nil {
|
||||
return nil, m.failPage
|
||||
}
|
||||
page := []domain.VariantFile{}
|
||||
for _, row := range m.rows {
|
||||
if row.ID <= afterID || (filter.VariantName != "" && row.VariantName != filter.VariantName) || (filter.SourceName != "" && row.SourceName != filter.SourceName) {
|
||||
continue
|
||||
}
|
||||
if filter.Unused != nil && !m.isUnused(row, *filter.Unused) {
|
||||
continue
|
||||
}
|
||||
page = append(page, row)
|
||||
}
|
||||
sort.Slice(page, func(i, j int) bool { return page[i].ID < page[j].ID })
|
||||
return page[:min(limit, len(page))], nil
|
||||
}
|
||||
|
||||
func (m *mockVariantFileRepo) Find(variantName string, sourceName string) (domain.VariantFile, bool, error) {
|
||||
@@ -190,6 +267,10 @@ func (m *mockVariantFileRepo) Totals() (map[string]domain.VariantFileTotals, err
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
m.calls++
|
||||
m.totalsCalls++
|
||||
if m.failTotals != nil {
|
||||
return nil, m.failTotals
|
||||
}
|
||||
totals := map[string]domain.VariantFileTotals{}
|
||||
for _, row := range m.rows {
|
||||
current := totals[row.VariantName]
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
domain "RenewCMS/internal/domain/image"
|
||||
entity "RenewCMS/internal/infrastructure/persistence/models"
|
||||
persistence "RenewCMS/internal/infrastructure/persistence/repositories"
|
||||
"RenewCMS/internal/infrastructure/usage"
|
||||
"context"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func newUsageTracker(repo domain.VariantUsageRepository, clock *time.Time) *usage.Tracker {
|
||||
return usage.NewTracker(repo, func() time.Time { return *clock })
|
||||
}
|
||||
|
||||
func TestVariantUsageTracker(t *testing.T) {
|
||||
start := time.Date(2026, time.October, 8, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
t.Run("The flush writes one batch for many files", func(t *testing.T) {
|
||||
repo, clock := newMockVariantUsageRepo(), start
|
||||
tracker := newUsageTracker(repo, &clock)
|
||||
for index := 0; index < 40; index++ {
|
||||
tracker.RecordHit("thumb", "file-"+strconv.Itoa(index)+".png")
|
||||
}
|
||||
|
||||
if err := tracker.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if repo.batchCount() != 1 || repo.callCount() != 1 || len(repo.recordedHits()) != 40 {
|
||||
t.Errorf("batches = %d, calls = %d, hits = %d", repo.batchCount(), repo.callCount(), len(repo.recordedHits()))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Nothing pending means no write", func(t *testing.T) {
|
||||
repo, clock := newMockVariantUsageRepo(), start
|
||||
tracker := newUsageTracker(repo, &clock)
|
||||
|
||||
_ = tracker.Flush()
|
||||
tracker.RecordHit("thumb", "a.png")
|
||||
_ = tracker.Flush()
|
||||
_ = tracker.Flush()
|
||||
|
||||
if repo.callCount() != 1 {
|
||||
t.Errorf("calls = %d", repo.callCount())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("A thousand hits on one file are one update with the time of the last hit", func(t *testing.T) {
|
||||
repo, clock := newMockVariantUsageRepo(), start
|
||||
tracker := newUsageTracker(repo, &clock)
|
||||
for index := 0; index < 1000; index++ {
|
||||
clock = start.Add(time.Duration(index) * time.Second)
|
||||
tracker.RecordHit("thumb", "a.png")
|
||||
}
|
||||
|
||||
_ = tracker.Flush()
|
||||
|
||||
hits := repo.recordedHits()
|
||||
if repo.callCount() != 1 || len(hits) != 1 || !hits[0].At.Equal(start.Add(999*time.Second)) {
|
||||
t.Errorf("calls = %d, hits = %+v", repo.callCount(), hits)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("A failed flush keeps the pending hits", func(t *testing.T) {
|
||||
repo, clock := newMockVariantUsageRepo(), start
|
||||
tracker := newUsageTracker(repo, &clock)
|
||||
repo.fail = errUsageStoreDown
|
||||
tracker.RecordHit("thumb", "a.png")
|
||||
if err := tracker.Flush(); err == nil {
|
||||
t.Fatal("the failure was swallowed")
|
||||
}
|
||||
clock = start.Add(time.Minute)
|
||||
tracker.RecordHit("thumb", "a.png")
|
||||
tracker.RecordHit("hero", "b.png")
|
||||
repo.fail = nil
|
||||
|
||||
_ = tracker.Flush()
|
||||
|
||||
hits := repo.recordedHits()
|
||||
if repo.batchCount() != 1 || len(hits) != 2 {
|
||||
t.Fatalf("batches = %d, hits = %+v", repo.batchCount(), hits)
|
||||
}
|
||||
for _, hit := range hits {
|
||||
if !hit.At.Equal(clock) {
|
||||
t.Errorf("hit = %+v", hit)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("A failed flush keeps the pending rejections", func(t *testing.T) {
|
||||
repo, clock := newMockVariantUsageRepo(), start
|
||||
tracker := newUsageTracker(repo, &clock)
|
||||
repo.fail = errUsageStoreDown
|
||||
tracker.RecordRejection()
|
||||
tracker.RecordRejection()
|
||||
_ = tracker.Flush()
|
||||
tracker.RecordRejection()
|
||||
repo.fail = nil
|
||||
|
||||
_ = tracker.Flush()
|
||||
_ = tracker.Flush()
|
||||
|
||||
if repo.recordedRejections() != 3 || repo.batchCount() != 1 || repo.days["2026-10-08"] != 3 {
|
||||
t.Errorf("rejections = %d, batches = %d, days = %v", repo.recordedRejections(), repo.batchCount(), repo.days)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("The flush loop and the size trigger", func(t *testing.T) {
|
||||
repo, clock := newMockVariantUsageRepo(), start
|
||||
tracker := usage.NewTrackerWith(repo, func() time.Time { return clock }, time.Hour, 500)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
tracker.Run(ctx)
|
||||
}()
|
||||
|
||||
for index := 0; index < 499; index++ {
|
||||
tracker.RecordHit("thumb", "file-"+strconv.Itoa(index)+".png")
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if repo.callCount() != 0 {
|
||||
t.Fatal("flushed before the size trigger")
|
||||
}
|
||||
tracker.RecordHit("thumb", "file-499.png")
|
||||
waitUntilUsage(t, func() bool { return repo.callCount() == 1 })
|
||||
|
||||
tracker.RecordHit("thumb", "late.png")
|
||||
stop()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("the loop did not stop")
|
||||
}
|
||||
if repo.callCount() != 2 {
|
||||
t.Errorf("calls after shutdown = %d", repo.callCount())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("The defaults are a thirty second interval and a 500 key trigger", func(t *testing.T) {
|
||||
if usage.FlushInterval() != 30*time.Second || usage.FlushKeyLimit != 500 {
|
||||
t.Errorf("interval = %v, limit = %d", usage.FlushInterval(), usage.FlushKeyLimit)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("The same key repeated never reaches the size trigger", func(t *testing.T) {
|
||||
repo, clock := newMockVariantUsageRepo(), start
|
||||
tracker := usage.NewTrackerWith(repo, func() time.Time { return clock }, time.Hour, 2)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go tracker.Run(ctx)
|
||||
|
||||
for index := 0; index < 5; index++ {
|
||||
tracker.RecordHit("thumb", "a.png")
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
if repo.callCount() != 0 {
|
||||
t.Errorf("calls = %d", repo.callCount())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("A tick flushes", func(t *testing.T) {
|
||||
repo, clock := newMockVariantUsageRepo(), start
|
||||
tracker := usage.NewTrackerWith(repo, func() time.Time { return clock }, 10*time.Millisecond, 500)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go tracker.Run(ctx)
|
||||
|
||||
tracker.RecordHit("thumb", "a.png")
|
||||
|
||||
waitUntilUsage(t, func() bool { return repo.callCount() == 1 })
|
||||
})
|
||||
|
||||
t.Run("Concurrent requests are race free", func(t *testing.T) {
|
||||
repo, clock := newMockVariantUsageRepo(), start
|
||||
tracker := newUsageTracker(repo, &clock)
|
||||
var group sync.WaitGroup
|
||||
for worker := 0; worker < 50; worker++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for index := 0; index < 100; index++ {
|
||||
tracker.RecordHit("thumb", "file-"+strconv.Itoa(index)+".png")
|
||||
tracker.RecordRejection()
|
||||
}
|
||||
}()
|
||||
}
|
||||
flushed := make(chan struct{})
|
||||
go func() {
|
||||
defer close(flushed)
|
||||
for index := 0; index < 20; index++ {
|
||||
_ = tracker.Flush()
|
||||
}
|
||||
}()
|
||||
group.Wait()
|
||||
<-flushed
|
||||
_ = tracker.Flush()
|
||||
|
||||
if repo.recordedRejections() != 5000 {
|
||||
t.Errorf("rejections = %d", repo.recordedRejections())
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, hit := range repo.recordedHits() {
|
||||
seen[hit.SourceName] = true
|
||||
}
|
||||
if len(seen) != 100 {
|
||||
t.Errorf("distinct files = %d", len(seen))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func waitUntilUsage(t *testing.T, condition func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(500 * time.Millisecond)
|
||||
for time.Now().Before(deadline) {
|
||||
if condition() {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("the condition was not met in time")
|
||||
}
|
||||
|
||||
func requestedAt(t *testing.T, db *gorm.DB, variantName string, sourceName string) *time.Time {
|
||||
t.Helper()
|
||||
var row entity.ImageVariantFile
|
||||
if err := db.Where("variant_name = ? AND source_name = ?", variantName, sourceName).First(&row).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return row.LastRequestedAt
|
||||
}
|
||||
|
||||
func TestVariantUsageRepository(t *testing.T) {
|
||||
noon := time.Date(2026, time.October, 8, 10, 5, 0, 0, time.UTC)
|
||||
seed := func(t *testing.T) (*gorm.DB, *persistence.VariantFileRepository, *persistence.VariantUsageRepository) {
|
||||
t.Helper()
|
||||
db := openActivityDatabase(t)
|
||||
files := persistence.NewVariantFileRepository(db)
|
||||
_, err := files.Save(domain.VariantFile{VariantName: "thumb", SourceName: "a.png", CachedName: "g1", ContentType: "image/png", Size: 10, Fingerprint: "f"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return db, files, persistence.NewVariantUsageRepository(db)
|
||||
}
|
||||
|
||||
t.Run("A batch sets the last requested time of the file", func(t *testing.T) {
|
||||
db, _, usageRepo := seed(t)
|
||||
|
||||
err := usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}})
|
||||
|
||||
if got := requestedAt(t, db, "thumb", "a.png"); err != nil || got == nil || !got.Equal(noon) {
|
||||
t.Errorf("last_requested_at = %v, err = %v", got, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("A stored time never moves back", func(t *testing.T) {
|
||||
db, _, usageRepo := seed(t)
|
||||
_ = usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}})
|
||||
|
||||
_ = usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon.Add(-4 * time.Minute)}}})
|
||||
|
||||
if got := requestedAt(t, db, "thumb", "a.png"); got == nil || !got.Equal(noon) {
|
||||
t.Errorf("last_requested_at = %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("A purged file ignores a late update", func(t *testing.T) {
|
||||
db, files, usageRepo := seed(t)
|
||||
row, _, _ := files.Find("thumb", "a.png")
|
||||
_ = files.Delete(row.ID)
|
||||
|
||||
err := usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}})
|
||||
|
||||
var count int64
|
||||
db.Model(&entity.ImageVariantFile{}).Count(&count)
|
||||
if err != nil || count != 0 {
|
||||
t.Errorf("err = %v, rows = %d", err, count)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("A regenerated file keeps its last requested time", func(t *testing.T) {
|
||||
db, files, usageRepo := seed(t)
|
||||
_ = usageRepo.Record(domain.VariantUsageBatch{Hits: []domain.VariantHit{{VariantName: "thumb", SourceName: "a.png", At: noon}}})
|
||||
|
||||
_, _ = files.Save(domain.VariantFile{VariantName: "thumb", SourceName: "a.png", CachedName: "g2", ContentType: "image/png", Size: 20, Fingerprint: "f2"})
|
||||
|
||||
if got := requestedAt(t, db, "thumb", "a.png"); got == nil || !got.Equal(noon) {
|
||||
t.Errorf("last_requested_at = %v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Rejections add up per day and the window sums its days", func(t *testing.T) {
|
||||
_, _, usageRepo := seed(t)
|
||||
day := func(value string) time.Time {
|
||||
parsed, _ := time.Parse(domain.RejectionDayLayout, value)
|
||||
return parsed
|
||||
}
|
||||
for _, batch := range [][]domain.RejectionCount{
|
||||
{{Day: day("2026-10-08"), Count: 3}, {Day: day("2026-10-02"), Count: 5}},
|
||||
{{Day: day("2026-10-08"), Count: 4}, {Day: day("2026-10-01"), Count: 9}},
|
||||
} {
|
||||
if err := usageRepo.Record(domain.VariantUsageBatch{Rejections: batch}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
total, err := usageRepo.RejectedSince(domain.RejectionWindowStart(time.Date(2026, time.October, 8, 23, 59, 0, 0, time.UTC)))
|
||||
|
||||
if err != nil || total != 12 {
|
||||
t.Errorf("total = %d, err = %v", total, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("Old day rows are pruned by the batch", func(t *testing.T) {
|
||||
db, _, usageRepo := seed(t)
|
||||
day := func(value string) time.Time {
|
||||
parsed, _ := time.Parse(domain.RejectionDayLayout, value)
|
||||
return parsed
|
||||
}
|
||||
_ = usageRepo.Record(domain.VariantUsageBatch{Rejections: []domain.RejectionCount{{Day: day("2026-09-30"), Count: 1}, {Day: day("2026-10-02"), Count: 2}}})
|
||||
|
||||
_ = usageRepo.Record(domain.VariantUsageBatch{Rejections: []domain.RejectionCount{{Day: day("2026-10-08"), Count: 1}}, PruneBefore: domain.RejectionPruneBefore(noon)})
|
||||
|
||||
var days []string
|
||||
db.Model(&entity.ImageVariantRejectionDay{}).Order("day").Pluck("day", &days)
|
||||
if len(days) != 2 || days[0] != "2026-10-02" || days[1] != "2026-10-08" {
|
||||
t.Errorf("days = %v", days)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestVariantCacheRepository(t *testing.T) {
|
||||
now := time.Date(2026, time.October, 8, 9, 0, 0, 0, time.UTC)
|
||||
cutoff := domain.UnusedCutoff(now)
|
||||
setup := func(t *testing.T) (*gorm.DB, *persistence.VariantFileRepository) {
|
||||
t.Helper()
|
||||
db := openActivityDatabase(t)
|
||||
return db, persistence.NewVariantFileRepository(db)
|
||||
}
|
||||
save := func(t *testing.T, db *gorm.DB, files *persistence.VariantFileRepository, variantName string, sourceName string, size int64, created time.Time, requested *time.Time) {
|
||||
t.Helper()
|
||||
row, err := files.Save(domain.VariantFile{VariantName: variantName, SourceName: sourceName, CachedName: variantName + sourceName, ContentType: "image/png", Size: size, Fingerprint: "f"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Model(&entity.ImageVariantFile{}).Where("id = ?", row.ID).Updates(map[string]any{"created_at": created, "last_requested_at": requested})
|
||||
}
|
||||
at := func(days int) *time.Time {
|
||||
value := now.AddDate(0, 0, -days)
|
||||
return &value
|
||||
}
|
||||
|
||||
t.Run("The reference time is last_requested_at when set and created_at otherwise", func(t *testing.T) {
|
||||
db, files := setup(t)
|
||||
save(t, db, files, "hero", "recent.png", 10, now.AddDate(0, 0, -90), at(29))
|
||||
save(t, db, files, "hero", "old.png", 20, now.AddDate(0, 0, -1), at(31))
|
||||
save(t, db, files, "print", "disabled.png", 40, now.AddDate(0, 0, -90), at(90))
|
||||
save(t, db, files, "card", "never.png", 80, now.AddDate(0, 0, -45), nil)
|
||||
save(t, db, files, "card", "fresh.png", 160, now.AddDate(0, 0, -2), nil)
|
||||
|
||||
totals, err := files.UnusedTotals(domain.UnusedFilter{VariantNames: []string{"hero", "card"}, Cutoff: cutoff})
|
||||
|
||||
if err != nil || totals.Files != 2 || totals.Bytes != 100 {
|
||||
t.Errorf("totals = %+v, err = %v", totals, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("A row whose reference time is exactly the cutoff is in use", func(t *testing.T) {
|
||||
db, files := setup(t)
|
||||
save(t, db, files, "hero", "edge.png", 10, now.AddDate(0, 0, -90), &cutoff)
|
||||
|
||||
totals, _ := files.UnusedTotals(domain.UnusedFilter{VariantNames: []string{"hero"}, Cutoff: cutoff})
|
||||
|
||||
if totals.Files != 0 {
|
||||
t.Errorf("totals = %+v", totals)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("No enabled variant means nothing is unused", func(t *testing.T) {
|
||||
db, files := setup(t)
|
||||
save(t, db, files, "hero", "old.png", 10, now.AddDate(0, 0, -90), nil)
|
||||
|
||||
totals, err := files.UnusedTotals(domain.UnusedFilter{Cutoff: cutoff})
|
||||
page, pageErr := files.ListPage(domain.VariantFileFilter{Unused: &domain.UnusedFilter{Cutoff: cutoff}}, 0, 10)
|
||||
|
||||
if err != nil || pageErr != nil || totals.Files != 0 || len(page) != 0 {
|
||||
t.Errorf("totals = %+v, page = %v", totals, page)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("The top sources are ordered by bytes then name and capped", func(t *testing.T) {
|
||||
db, files := setup(t)
|
||||
save(t, db, files, "hero", "b.png", 100, now, nil)
|
||||
save(t, db, files, "card", "b.png", 100, now, nil)
|
||||
save(t, db, files, "hero", "a.png", 200, now, nil)
|
||||
save(t, db, files, "hero", "c.png", 900, now, nil)
|
||||
|
||||
top, err := files.TopSources(3)
|
||||
|
||||
if err != nil || len(top) != 3 || top[0].SourceName != "c.png" || top[1].SourceName != "a.png" || top[2].SourceName != "b.png" || top[2].Files != 2 || top[2].Bytes != 200 {
|
||||
t.Errorf("top = %+v, err = %v", top, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("A page continues after the given id and filters by variant, source and unused", func(t *testing.T) {
|
||||
db, files := setup(t)
|
||||
for index := 0; index < 5; index++ {
|
||||
save(t, db, files, "hero", "s"+strconv.Itoa(index)+".png", 1, now.AddDate(0, 0, -90), nil)
|
||||
}
|
||||
save(t, db, files, "card", "s0.png", 1, now, nil)
|
||||
|
||||
first, _ := files.ListPage(domain.VariantFileFilter{VariantName: "hero"}, 0, 2)
|
||||
second, _ := files.ListPage(domain.VariantFileFilter{VariantName: "hero"}, first[1].ID, 10)
|
||||
bySource, _ := files.ListPage(domain.VariantFileFilter{SourceName: "s0.png"}, 0, 10)
|
||||
unused, _ := files.ListPage(domain.VariantFileFilter{Unused: &domain.UnusedFilter{VariantNames: []string{"hero", "card"}, Cutoff: cutoff}}, 0, 10)
|
||||
|
||||
if len(first) != 2 || len(second) != 3 || second[0].ID <= first[1].ID || len(bySource) != 2 || len(unused) != 5 {
|
||||
t.Errorf("first = %d, second = %d, bySource = %d, unused = %d", len(first), len(second), len(bySource), len(unused))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestImageRepositoryListByStoredNames(t *testing.T) {
|
||||
repo := persistence.NewImageRepository(openActivityDatabase(t))
|
||||
for _, name := range []string{"a.png", "b.png", "c.png"} {
|
||||
_, _ = repo.Create(domain.Image{Path: domain.PublicPath(name), Filename: name})
|
||||
}
|
||||
|
||||
found, err := repo.ListByStoredNames([]string{"c.png", "a.png", "missing.png"})
|
||||
none, noneErr := repo.ListByStoredNames(nil)
|
||||
|
||||
if err != nil || noneErr != nil || len(found) != 2 || found[0].Filename != "a.png" || found[1].Filename != "c.png" || len(none) != 0 {
|
||||
t.Errorf("found = %+v, err = %v, none = %v", found, err, none)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user