feat: image variants cache panel

This commit is contained in:
fsylvain committed 2026-10-08 21:13:32 +02:00
1 parent 951fef18ba
commit d768668014
53 files changed
+5809 -46

No files matched your search

@@ -7,7 +7,7 @@ import (
)
func Migrate(db *gorm.DB) error {
err := tableOptionsFor(db).AutoMigrate(&models.Article{}, &models.Image{}, &models.User{}, &models.Invite{}, &models.APIKey{}, &models.PurgeSettings{}, &models.AuditEvent{}, &models.AuditSettings{}, &models.ImageVariant{}, &models.ImageVariantFile{})
err := tableOptionsFor(db).AutoMigrate(&models.Article{}, &models.Image{}, &models.User{}, &models.Invite{}, &models.APIKey{}, &models.PurgeSettings{}, &models.AuditEvent{}, &models.AuditSettings{}, &models.ImageVariant{}, &models.ImageVariantFile{}, &models.ImageVariantRejectionDay{})
if err != nil {
return err
}
@@ -11,4 +11,12 @@ type ImageVariantFile struct {
Size int64 `gorm:"not null"`
Fingerprint string `gorm:"not null;size:64"`
CreatedAt time.Time
LastRequestedAt *time.Time
}
type ImageVariantRejectionDay struct {
ID uint32 `gorm:"primaryKey;autoIncrement"`
Day string `gorm:"not null;size:10;uniqueIndex:idx_image_variant_rejection_days_day"`
Count int `gorm:"not null"`
}
@@ -107,6 +107,21 @@ func (i ImageRepository) ExistsByStoredName(storedName string) (bool, error) {
return matches > 0, err
}
func (i ImageRepository) ListByStoredNames(storedNames []string) ([]domain.Image, error) {
if len(storedNames) == 0 {
return []domain.Image{}, nil
}
paths := make([]string, 0, len(storedNames))
for _, storedName := range storedNames {
paths = append(paths, domain.PublicPath(storedName))
}
var models []entity.Image
if err := i.db.Where("path IN ?", paths).Order("id asc").Find(&models).Error; err != nil {
return nil, err
}
return imagesToDomain(models), nil
}
func (i ImageRepository) duplicatedHashes() *gorm.DB {
return i.db.Model(&entity.Image{}).
Select("sha256").
@@ -15,6 +15,7 @@ type VariantFileRepository struct {
}
var _ domain.VariantFileRepository = &VariantFileRepository{}
var _ domain.VariantCacheRepository = &VariantFileRepository{}
func NewVariantFileRepository(db *gorm.DB) *VariantFileRepository {
return &VariantFileRepository{db}
@@ -90,6 +91,76 @@ func (v *VariantFileRepository) TotalsBySource() (map[string]domain.VariantFileT
return totals, nil
}
const (
unusedCondition = "variant_name IN ? AND COALESCE(last_requested_at, created_at) < ?"
topSourcesOrder = "bytes DESC, source_name ASC"
)
func (v *VariantFileRepository) UnusedTotals(filter domain.UnusedFilter) (domain.VariantFileTotals, error) {
if len(filter.VariantNames) == 0 {
return domain.VariantFileTotals{}, nil
}
var row struct {
Files int
Bytes int64
}
err := v.db.Model(&entity.ImageVariantFile{}).
Select("COUNT(*) AS files, COALESCE(SUM(size), 0) AS bytes").
Where(unusedCondition, filter.VariantNames, filter.Cutoff.UTC()).
Scan(&row).Error
if err != nil {
return domain.VariantFileTotals{}, err
}
return domain.VariantFileTotals{Files: row.Files, Bytes: row.Bytes}, nil
}
func (v *VariantFileRepository) TopSources(limit int) ([]domain.SourceTotals, error) {
var rows []struct {
SourceName string
Files int
Bytes int64
}
err := v.db.Model(&entity.ImageVariantFile{}).
Select("source_name, COUNT(*) AS files, COALESCE(SUM(size), 0) AS bytes").
Group("source_name").
Order(topSourcesOrder).
Limit(limit).
Scan(&rows).Error
if err != nil {
return nil, err
}
sources := make([]domain.SourceTotals, 0, len(rows))
for _, row := range rows {
sources = append(sources, domain.SourceTotals{SourceName: row.SourceName, Files: row.Files, Bytes: row.Bytes})
}
return sources, nil
}
func (v *VariantFileRepository) ListPage(filter domain.VariantFileFilter, afterID uint32, limit int) ([]domain.VariantFile, error) {
statement := v.db.Where("id > ?", afterID)
if filter.VariantName != "" {
statement = statement.Where("variant_name = ?", filter.VariantName)
}
if filter.SourceName != "" {
statement = statement.Where("source_name = ?", filter.SourceName)
}
if filter.Unused != nil {
if len(filter.Unused.VariantNames) == 0 {
return []domain.VariantFile{}, nil
}
statement = statement.Where(unusedCondition, filter.Unused.VariantNames, filter.Unused.Cutoff.UTC())
}
var found []entity.ImageVariantFile
if err := statement.Order("id asc").Limit(limit).Find(&found).Error; err != nil {
return nil, err
}
files := make([]domain.VariantFile, 0, len(found))
for _, model := range found {
files = append(files, mappers.VariantFileToDomain(model))
}
return files, nil
}
func (v *VariantFileRepository) Totals() (map[string]domain.VariantFileTotals, error) {
var rows []struct {
VariantName string
@@ -0,0 +1,64 @@
package persistence
import (
domain "RenewCMS/internal/domain/image"
entity "RenewCMS/internal/infrastructure/persistence/models"
"time"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
type VariantUsageRepository struct {
db *gorm.DB
}
var _ domain.VariantUsageRepository = &VariantUsageRepository{}
func NewVariantUsageRepository(db *gorm.DB) *VariantUsageRepository {
return &VariantUsageRepository{db}
}
func (v *VariantUsageRepository) Record(batch domain.VariantUsageBatch) error {
return v.db.Transaction(func(tx *gorm.DB) error {
for _, hit := range batch.Hits {
if err := recordHit(tx, hit); err != nil {
return err
}
}
for _, rejection := range batch.Rejections {
if err := recordRejections(tx, rejection); err != nil {
return err
}
}
if batch.PruneBefore.IsZero() {
return nil
}
return tx.Where("day < ?", batch.PruneBefore.UTC().Format(domain.RejectionDayLayout)).Delete(&entity.ImageVariantRejectionDay{}).Error
})
}
func recordHit(tx *gorm.DB, hit domain.VariantHit) error {
at := hit.At.UTC()
return tx.Model(&entity.ImageVariantFile{}).
Where("variant_name = ? AND source_name = ?", hit.VariantName, hit.SourceName).
Where("last_requested_at IS NULL OR last_requested_at < ?", at).
Update("last_requested_at", at).Error
}
func recordRejections(tx *gorm.DB, rejection domain.RejectionCount) error {
row := entity.ImageVariantRejectionDay{Day: rejection.Day.UTC().Format(domain.RejectionDayLayout), Count: rejection.Count}
return tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "day"}},
DoUpdates: clause.Assignments(map[string]any{"count": gorm.Expr("? + ?", clause.Column{Table: clause.CurrentTable, Name: "count"}, rejection.Count)}),
}).Create(&row).Error
}
func (v *VariantUsageRepository) RejectedSince(day time.Time) (int, error) {
var total int
err := v.db.Model(&entity.ImageVariantRejectionDay{}).
Select("COALESCE(SUM(count), 0)").
Where("day >= ?", day.UTC().Format(domain.RejectionDayLayout)).
Scan(&total).Error
return total, err
}
@@ -0,0 +1,150 @@
package usage
import (
"RenewCMS/internal/domain/image"
"context"
"sync"
"time"
)
const FlushKeyLimit = 500
func FlushInterval() time.Duration {
return 30 * time.Second
}
type hitKey struct {
variantName string
sourceName string
}
type Tracker struct {
repository image.VariantUsageRepository
now func() time.Time
interval time.Duration
keyLimit int
flushing sync.Mutex
pending sync.Mutex
hits map[hitKey]time.Time
rejections map[time.Time]int
wake chan struct{}
}
var _ image.VariantUsageTracker = &Tracker{}
func NewTracker(repository image.VariantUsageRepository, now func() time.Time) *Tracker {
return NewTrackerWith(repository, now, FlushInterval(), FlushKeyLimit)
}
func NewTrackerWith(repository image.VariantUsageRepository, now func() time.Time, interval time.Duration, keyLimit int) *Tracker {
return &Tracker{
repository: repository,
now: now,
interval: interval,
keyLimit: keyLimit,
hits: map[hitKey]time.Time{},
rejections: map[time.Time]int{},
wake: make(chan struct{}, 1),
}
}
func (t *Tracker) RecordHit(variantName string, sourceName string) {
at := t.now().UTC()
t.pending.Lock()
defer t.pending.Unlock()
keepLatest(t.hits, hitKey{variantName, sourceName}, at)
if len(t.hits) >= t.keyLimit {
t.wakeLoop()
}
}
func (t *Tracker) RecordRejection() {
day := image.RejectionDay(t.now())
t.pending.Lock()
defer t.pending.Unlock()
t.rejections[day]++
}
func (t *Tracker) wakeLoop() {
select {
case t.wake <- struct{}{}:
default:
}
}
func keepLatest(hits map[hitKey]time.Time, key hitKey, at time.Time) {
if current, found := hits[key]; !found || at.After(current) {
hits[key] = at
}
}
func (t *Tracker) Flush() error {
t.flushing.Lock()
defer t.flushing.Unlock()
hits, rejections := t.take()
if len(hits) == 0 && len(rejections) == 0 {
return nil
}
batch := image.VariantUsageBatch{
Hits: hitsOf(hits),
Rejections: rejectionsOf(rejections),
PruneBefore: image.RejectionPruneBefore(t.now()),
}
if err := t.repository.Record(batch); err != nil {
t.restore(hits, rejections)
return err
}
return nil
}
func (t *Tracker) take() (map[hitKey]time.Time, map[time.Time]int) {
t.pending.Lock()
defer t.pending.Unlock()
hits, rejections := t.hits, t.rejections
t.hits, t.rejections = map[hitKey]time.Time{}, map[time.Time]int{}
return hits, rejections
}
func (t *Tracker) restore(hits map[hitKey]time.Time, rejections map[time.Time]int) {
t.pending.Lock()
defer t.pending.Unlock()
for key, at := range hits {
keepLatest(t.hits, key, at)
}
for day, count := range rejections {
t.rejections[day] += count
}
}
func hitsOf(hits map[hitKey]time.Time) []image.VariantHit {
listed := make([]image.VariantHit, 0, len(hits))
for key, at := range hits {
listed = append(listed, image.VariantHit{VariantName: key.variantName, SourceName: key.sourceName, At: at})
}
return listed
}
func rejectionsOf(rejections map[time.Time]int) []image.RejectionCount {
listed := make([]image.RejectionCount, 0, len(rejections))
for day, count := range rejections {
listed = append(listed, image.RejectionCount{Day: day, Count: count})
}
return listed
}
func (t *Tracker) Run(ctx context.Context) {
ticker := time.NewTicker(t.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
_ = t.Flush()
return
case <-ticker.C:
_ = t.Flush()
case <-t.wake:
_ = t.Flush()
}
}
}
@@ -0,0 +1,170 @@
package useCases
import (
"RenewCMS/internal/domain/image"
"sort"
"time"
)
type VariantCacheEntry struct {
Name string
Enabled bool
Files int
Bytes int64
}
type VariantCacheImage struct {
ID uint32
Filename string
URL string
Files int
Bytes int64
}
type VariantCacheStats struct {
GeneratedFiles int
DiskUsed int64
VariantsDefined int
VariantsEnabled int
VariantsDisabled int
RejectedRequests int
RejectedWindowDays int
Unused image.VariantFileTotals
UnusedDays int
ByVariant []VariantCacheEntry
TopImages []VariantCacheImage
}
type GetVariantCacheStatsUseCase struct {
variants image.VariantRepository
files image.VariantFileRepository
cache image.VariantCacheRepository
images image.Repository
usage image.VariantUsageRepository
tracker image.VariantUsageTracker
now func() time.Time
}
func NewGetVariantCacheStatsUseCase(variants image.VariantRepository, files image.VariantFileRepository, cache image.VariantCacheRepository, images image.Repository, usage image.VariantUsageRepository, tracker image.VariantUsageTracker, now func() time.Time) *GetVariantCacheStatsUseCase {
return &GetVariantCacheStatsUseCase{variants, files, cache, images, usage, tracker, now}
}
func (u *GetVariantCacheStatsUseCase) Get() (VariantCacheStats, error) {
_ = u.tracker.Flush()
variants, err := u.variants.List()
if err != nil {
return VariantCacheStats{}, err
}
totals, err := u.files.Totals()
if err != nil {
return VariantCacheStats{}, err
}
now := u.now()
unused, err := u.cache.UnusedTotals(image.UnusedFilter{VariantNames: enabledNames(variants), Cutoff: image.UnusedCutoff(now)})
if err != nil {
return VariantCacheStats{}, err
}
rejected, err := u.usage.RejectedSince(image.RejectionWindowStart(now))
if err != nil {
return VariantCacheStats{}, err
}
top, err := u.topImages()
if err != nil {
return VariantCacheStats{}, err
}
stats := VariantCacheStats{
VariantsDefined: len(variants),
RejectedRequests: rejected,
RejectedWindowDays: image.RejectionWindowDays,
Unused: unused,
UnusedDays: image.UnusedVariantDays,
ByVariant: cacheEntries(variants, totals),
TopImages: top,
}
for _, entry := range stats.ByVariant {
stats.GeneratedFiles += entry.Files
stats.DiskUsed += entry.Bytes
if entry.Enabled {
stats.VariantsEnabled++
}
}
stats.VariantsDisabled = stats.VariantsDefined - stats.VariantsEnabled
return stats, nil
}
func enabledNames(variants []image.Variant) []string {
names := []string{}
for _, variant := range variants {
if variant.Enabled {
names = append(names, variant.Name)
}
}
return names
}
func cacheEntries(variants []image.Variant, totals map[string]image.VariantFileTotals) []VariantCacheEntry {
entries := make([]VariantCacheEntry, 0, len(variants))
for _, variant := range variants {
own := totals[variant.Name]
entries = append(entries, VariantCacheEntry{Name: variant.Name, Enabled: variant.Enabled, Files: own.Files, Bytes: own.Bytes})
}
sort.Slice(entries, func(i, j int) bool {
if entries[i].Bytes != entries[j].Bytes {
return entries[i].Bytes > entries[j].Bytes
}
return entries[i].Name < entries[j].Name
})
return entries
}
func (u *GetVariantCacheStatsUseCase) topImages() ([]VariantCacheImage, error) {
limit := image.TopImagesLimit + 1
for {
sources, err := u.cache.TopSources(limit)
if err != nil {
return nil, err
}
ranked, err := u.rank(sources)
if err != nil {
return nil, err
}
if len(sources) < limit || topIsSettled(ranked, sources) {
return ranked[:min(len(ranked), image.TopImagesLimit)], nil
}
limit *= 2
}
}
func topIsSettled(ranked []VariantCacheImage, sources []image.SourceTotals) bool {
return len(ranked) >= image.TopImagesLimit && ranked[image.TopImagesLimit-1].Bytes > sources[len(sources)-1].Bytes
}
func (u *GetVariantCacheStatsUseCase) rank(sources []image.SourceTotals) ([]VariantCacheImage, error) {
names := make([]string, 0, len(sources))
for _, source := range sources {
names = append(names, source.SourceName)
}
found, err := u.images.ListByStoredNames(names)
if err != nil {
return nil, err
}
byName := make(map[string]image.Image, len(found))
for _, img := range found {
byName[image.StoredName(img.Path)] = img
}
ranked := make([]VariantCacheImage, 0, len(sources))
for _, source := range sources {
if img, known := byName[source.SourceName]; known {
ranked = append(ranked, VariantCacheImage{ID: img.ID, Filename: img.Filename, URL: image.PublicPath(source.SourceName), Files: source.Files, Bytes: source.Bytes})
}
}
sort.SliceStable(ranked, func(i, j int) bool {
if ranked[i].Bytes != ranked[j].Bytes {
return ranked[i].Bytes > ranked[j].Bytes
}
return ranked[i].ID < ranked[j].ID
})
return ranked, nil
}
@@ -0,0 +1,148 @@
package useCases
import (
"RenewCMS/internal/domain/audit"
"RenewCMS/internal/domain/image"
"context"
"time"
)
type PurgeVariantCacheCommand struct {
Scope image.CachePurgeScope
Variant string
ImageID uint32
}
type VariantCachePurgeResult struct {
Scope image.CachePurgeScope
Files int
Bytes int64
Failed int
}
type PurgeVariantCacheUseCase struct {
variants image.VariantRepository
images image.Repository
cache image.VariantCacheRepository
files variantCache
recorder audit.Recorder
now func() time.Time
}
func NewPurgeVariantCacheUseCase(variants image.VariantRepository, images image.Repository, cache image.VariantCacheRepository, files image.VariantFileRepository, storage image.Storage, now func() time.Time, recorder audit.Recorder) *PurgeVariantCacheUseCase {
return &PurgeVariantCacheUseCase{variants, images, cache, variantCache{files, storage}, recorder, now}
}
type cachePurgePlan struct {
filter image.VariantFileFilter
event audit.Event
}
func (u *PurgeVariantCacheUseCase) Purge(ctx context.Context, command PurgeVariantCacheCommand) (VariantCachePurgeResult, error) {
plan, err := u.plan(command)
if err != nil {
return VariantCachePurgeResult{}, err
}
result, err := u.purgeMatching(plan.filter)
result.Scope = command.Scope
if err != nil {
recordFailure(ctx, u.recorder, plan.event, err)
return result, err
}
u.record(ctx, plan.event, result)
return result, nil
}
func (u *PurgeVariantCacheUseCase) plan(command PurgeVariantCacheCommand) (cachePurgePlan, error) {
switch command.Scope {
case image.CacheScopeAll:
return cachePurgePlan{event: audit.NewEvent(audit.ActionVariantPurgeAll, audit.CategoryMedia)}, nil
case image.CacheScopeUnused:
return u.unusedPlan()
case image.CacheScopeVariant:
return u.variantPlan(command.Variant)
case image.CacheScopeImage:
return u.imagePlan(command.ImageID)
default:
return cachePurgePlan{}, image.ErrCachePurgeScope
}
}
func (u *PurgeVariantCacheUseCase) unusedPlan() (cachePurgePlan, error) {
variants, err := u.variants.List()
if err != nil {
return cachePurgePlan{}, err
}
event := audit.NewEvent(audit.ActionVariantPurgeUnused, audit.CategoryMedia)
filter := image.VariantFileFilter{Unused: &image.UnusedFilter{VariantNames: enabledNames(variants), Cutoff: image.UnusedCutoff(u.now())}}
return cachePurgePlan{filter: filter, event: event}, nil
}
func (u *PurgeVariantCacheUseCase) variantPlan(name string) (cachePurgePlan, error) {
variant, err := u.variants.Get(name)
if err != nil {
return cachePurgePlan{}, err
}
if variant.Name != name {
return cachePurgePlan{}, image.ErrVariantNotFound
}
event := audit.NewEvent(audit.ActionVariantPurge, audit.CategoryMedia)
event.Target = audit.VariantTarget(name)
return cachePurgePlan{filter: image.VariantFileFilter{VariantName: name}, event: event}, nil
}
func (u *PurgeVariantCacheUseCase) imagePlan(id uint32) (cachePurgePlan, error) {
img, err := u.images.Get(id)
if err != nil {
return cachePurgePlan{}, err
}
event := audit.NewEvent(audit.ActionVariantPurgeImage, audit.CategoryMedia)
event.Target = imageTarget(img)
return cachePurgePlan{filter: image.VariantFileFilter{SourceName: image.StoredName(img.Path)}, event: event}, nil
}
func (u *PurgeVariantCacheUseCase) purgeMatching(filter image.VariantFileFilter) (VariantCachePurgeResult, error) {
var result VariantCachePurgeResult
var after uint32
for {
rows, err := u.cache.ListPage(filter, after, image.CachePurgeBatchSize)
if err != nil {
return result, err
}
for _, row := range rows {
after = row.ID
if err := u.files.purgeRow(row); err != nil {
result.Failed++
continue
}
result.Files++
result.Bytes += row.Size
}
if len(rows) < image.CachePurgeBatchSize {
return result, nil
}
}
}
func (u *PurgeVariantCacheUseCase) record(ctx context.Context, event audit.Event, result VariantCachePurgeResult) {
if result.Files == 0 && result.Failed == 0 {
return
}
event.Params = cachePurgeParams(event.Action, result)
if result.Files == 0 {
failFrom(ctx, u.recorder, event, GenericFailureReason)
return
}
u.recorder.Record(ctx, event)
}
func cachePurgeParams(action string, result VariantCachePurgeResult) map[string]any {
params := map[string]any{audit.ParamPurgedFiles: result.Files, audit.ParamPurgedBytes: result.Bytes}
if action == audit.ActionVariantPurgeUnused {
params[audit.ParamOlderThanDays] = image.UnusedVariantDays
}
if result.Failed > 0 {
params[audit.ParamFailedFiles] = result.Failed
}
return params
}
@@ -55,6 +55,18 @@ type ServeVariantUseCase struct {
maxSourcePixels int64
renderSlots chan struct{}
flights singleflight.Group
tracker image.VariantUsageTracker
}
type idleUsageTracker struct{}
func (idleUsageTracker) RecordHit(string, string) {}
func (idleUsageTracker) RecordRejection() {}
func (idleUsageTracker) Flush() error { return nil }
func (u *ServeVariantUseCase) WithUsageTracker(tracker image.VariantUsageTracker) *ServeVariantUseCase {
u.tracker = tracker
return u
}
func NewServeVariantUseCase(variants image.VariantRepository, files image.VariantFileRepository, images image.Repository, storage image.Storage, renderer image.VariantRenderer, limits ServeVariantLimits) *ServeVariantUseCase {
@@ -67,10 +79,19 @@ func NewServeVariantUseCase(variants image.VariantRepository, files image.Varian
maxSourceBytes: limits.MaxSourceBytes,
maxSourcePixels: limits.MaxSourcePixels,
renderSlots: make(chan struct{}, max(1, limits.MaxConcurrentRenders)),
tracker: idleUsageTracker{},
}
}
func (u *ServeVariantUseCase) Serve(request ServeVariantRequest) (ServedVariant, error) {
served, err := u.serve(request)
if errors.Is(err, image.ErrUnknownVariant) || errors.Is(err, image.ErrUnsupportedParameter) {
u.tracker.RecordRejection()
}
return served, err
}
func (u *ServeVariantUseCase) serve(request ServeVariantRequest) (ServedVariant, error) {
if parameter := unsupportedParameter(request.Parameters); parameter != "" {
return ServedVariant{}, image.NewUnsupportedParameterError(parameter)
}
@@ -134,18 +155,26 @@ func (u *ServeVariantUseCase) serveSource(variant image.Variant, request ServeVa
settings := variant.Settings()
etag := image.VariantETag(settings, request.StoredName)
if image.ETagMatches(request.IfNoneMatch, etag) {
u.tracker.RecordHit(variant.Name, request.StoredName)
return ServedVariant{NotModified: true, ETag: etag}, nil
}
fingerprint := settings.Fingerprint()
cached, previous, hit, err := u.lookupCached(variant.Name, request.StoredName, fingerprint, etag)
if err != nil || hit {
if err == nil {
u.tracker.RecordHit(variant.Name, request.StoredName)
}
return cached, err
}
file, err := u.generated(requestContext(request), variant, request.StoredName, fingerprint, previous)
if err != nil {
return ServedVariant{}, err
}
return u.serveFile(file, etag)
served, err := u.serveFile(file, etag)
if err == nil {
u.tracker.RecordHit(variant.Name, request.StoredName)
}
return served, err
}
func requestContext(request ServeVariantRequest) context.Context {
@@ -1,6 +1,9 @@
package useCases
import "RenewCMS/internal/domain/image"
import (
"RenewCMS/internal/domain/image"
"errors"
)
type VariantPurgeResult struct {
Files int
@@ -44,7 +47,7 @@ func (c variantCache) purge(rows []image.VariantFile) (VariantPurgeResult, error
}
func (c variantCache) purgeRow(row image.VariantFile) error {
if err := c.storage.Delete(row.CachedName); err != nil {
if err := c.storage.Delete(row.CachedName); err != nil && !errors.Is(err, image.ErrBlobNotFound) {
return err
}
return c.files.Delete(row.ID)