feat: image variants cache panel
This commit is contained in:
1 parent
951fef18ba
commit
d768668014
53 files changed
+5809
-46
No files matched your search
@@ -84,6 +84,8 @@ const (
|
||||
ActionVariantPurge = "media.variant_purge"
|
||||
ActionVariantGenerateAll = "media.variant_generate_all"
|
||||
ActionVariantPurgeImage = "media.variant_purge_image"
|
||||
ActionVariantPurgeAll = "media.variant_purge_all"
|
||||
ActionVariantPurgeUnused = "media.variant_purge_unused"
|
||||
ActionUserRegister = "user.register"
|
||||
ActionUserVerify = "user.verify"
|
||||
ActionUserUpdateProfile = "user.update_profile"
|
||||
@@ -123,6 +125,8 @@ const (
|
||||
ParamKind = "kind"
|
||||
ParamPurgedFiles = "purged_files"
|
||||
ParamPurgedBytes = "purged_bytes"
|
||||
ParamFailedFiles = "failed_files"
|
||||
ParamOlderThanDays = "older_than_days"
|
||||
ParamGenerated = "generated"
|
||||
ParamFailed = "failed"
|
||||
ParamCauses = "causes"
|
||||
|
||||
@@ -15,6 +15,7 @@ type Repository interface {
|
||||
ListDuplicates() ([]Image, error)
|
||||
Counts() (Counts, error)
|
||||
ExistsByStoredName(storedName string) (bool, error)
|
||||
ListByStoredNames(storedNames []string) ([]Image, error)
|
||||
ListMissingMetadata() ([]Image, error)
|
||||
UpdateMetadata(id uint32, metadata Metadata) error
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package image
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrCachePurgeScope = errors.New("cache purge scope is not valid")
|
||||
|
||||
type CachePurgeScope string
|
||||
|
||||
const (
|
||||
CacheScopeAll CachePurgeScope = "all"
|
||||
CacheScopeUnused CachePurgeScope = "unused"
|
||||
CacheScopeVariant CachePurgeScope = "variant"
|
||||
CacheScopeImage CachePurgeScope = "image"
|
||||
)
|
||||
|
||||
const (
|
||||
UnusedVariantDays = 30
|
||||
RejectionWindowDays = 7
|
||||
TopImagesLimit = 10
|
||||
CachePurgeBatchSize = 500
|
||||
RejectionDayLayout = "2006-01-02"
|
||||
)
|
||||
|
||||
func UnusedCutoff(now time.Time) time.Time {
|
||||
return now.UTC().AddDate(0, 0, -UnusedVariantDays)
|
||||
}
|
||||
|
||||
func RejectionDay(at time.Time) time.Time {
|
||||
utc := at.UTC()
|
||||
return time.Date(utc.Year(), utc.Month(), utc.Day(), 0, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
func RejectionWindowStart(now time.Time) time.Time {
|
||||
return RejectionDay(now).AddDate(0, 0, -(RejectionWindowDays - 1))
|
||||
}
|
||||
|
||||
func RejectionPruneBefore(now time.Time) time.Time {
|
||||
return RejectionDay(now).AddDate(0, 0, -RejectionWindowDays)
|
||||
}
|
||||
|
||||
type UnusedFilter struct {
|
||||
VariantNames []string
|
||||
Cutoff time.Time
|
||||
}
|
||||
|
||||
type VariantFileFilter struct {
|
||||
VariantName string
|
||||
SourceName string
|
||||
Unused *UnusedFilter
|
||||
}
|
||||
|
||||
type SourceTotals struct {
|
||||
SourceName string
|
||||
Files int
|
||||
Bytes int64
|
||||
}
|
||||
|
||||
type VariantHit struct {
|
||||
VariantName string
|
||||
SourceName string
|
||||
At time.Time
|
||||
}
|
||||
|
||||
type RejectionCount struct {
|
||||
Day time.Time
|
||||
Count int
|
||||
}
|
||||
|
||||
type VariantUsageBatch struct {
|
||||
Hits []VariantHit
|
||||
Rejections []RejectionCount
|
||||
PruneBefore time.Time
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
package image
|
||||
|
||||
import "io"
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
type VariantRepository interface {
|
||||
List() ([]Variant, error)
|
||||
@@ -20,6 +23,23 @@ type VariantFileRepository interface {
|
||||
TotalsBySource() (map[string]VariantFileTotals, error)
|
||||
}
|
||||
|
||||
type VariantCacheRepository interface {
|
||||
UnusedTotals(filter UnusedFilter) (VariantFileTotals, error)
|
||||
TopSources(limit int) ([]SourceTotals, error)
|
||||
ListPage(filter VariantFileFilter, afterID uint32, limit int) ([]VariantFile, error)
|
||||
}
|
||||
|
||||
type VariantUsageRepository interface {
|
||||
Record(batch VariantUsageBatch) error
|
||||
RejectedSince(day time.Time) (int, error)
|
||||
}
|
||||
|
||||
type VariantUsageTracker interface {
|
||||
RecordHit(variantName string, sourceName string)
|
||||
RecordRejection()
|
||||
Flush() error
|
||||
}
|
||||
|
||||
type RenderedVariant struct {
|
||||
Content []byte
|
||||
ContentType string
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
)
|
||||
|
||||
func Migrate(db *gorm.DB) error {
|
||||
err := tableOptionsFor(db).AutoMigrate(&models.Article{}, &models.Image{}, &models.User{}, &models.Invite{}, &models.APIKey{}, &models.PurgeSettings{}, &models.AuditEvent{}, &models.AuditSettings{}, &models.ImageVariant{}, &models.ImageVariantFile{})
|
||||
err := tableOptionsFor(db).AutoMigrate(&models.Article{}, &models.Image{}, &models.User{}, &models.Invite{}, &models.APIKey{}, &models.PurgeSettings{}, &models.AuditEvent{}, &models.AuditSettings{}, &models.ImageVariant{}, &models.ImageVariantFile{}, &models.ImageVariantRejectionDay{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -11,4 +11,12 @@ type ImageVariantFile struct {
|
||||
Size int64 `gorm:"not null"`
|
||||
Fingerprint string `gorm:"not null;size:64"`
|
||||
CreatedAt time.Time
|
||||
|
||||
LastRequestedAt *time.Time
|
||||
}
|
||||
|
||||
type ImageVariantRejectionDay struct {
|
||||
ID uint32 `gorm:"primaryKey;autoIncrement"`
|
||||
Day string `gorm:"not null;size:10;uniqueIndex:idx_image_variant_rejection_days_day"`
|
||||
Count int `gorm:"not null"`
|
||||
}
|
||||
@@ -107,6 +107,21 @@ func (i ImageRepository) ExistsByStoredName(storedName string) (bool, error) {
|
||||
return matches > 0, err
|
||||
}
|
||||
|
||||
func (i ImageRepository) ListByStoredNames(storedNames []string) ([]domain.Image, error) {
|
||||
if len(storedNames) == 0 {
|
||||
return []domain.Image{}, nil
|
||||
}
|
||||
paths := make([]string, 0, len(storedNames))
|
||||
for _, storedName := range storedNames {
|
||||
paths = append(paths, domain.PublicPath(storedName))
|
||||
}
|
||||
var models []entity.Image
|
||||
if err := i.db.Where("path IN ?", paths).Order("id asc").Find(&models).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return imagesToDomain(models), nil
|
||||
}
|
||||
|
||||
func (i ImageRepository) duplicatedHashes() *gorm.DB {
|
||||
return i.db.Model(&entity.Image{}).
|
||||
Select("sha256").
|
||||
|
||||
@@ -15,6 +15,7 @@ type VariantFileRepository struct {
|
||||
}
|
||||
|
||||
var _ domain.VariantFileRepository = &VariantFileRepository{}
|
||||
var _ domain.VariantCacheRepository = &VariantFileRepository{}
|
||||
|
||||
func NewVariantFileRepository(db *gorm.DB) *VariantFileRepository {
|
||||
return &VariantFileRepository{db}
|
||||
@@ -90,6 +91,76 @@ func (v *VariantFileRepository) TotalsBySource() (map[string]domain.VariantFileT
|
||||
return totals, nil
|
||||
}
|
||||
|
||||
const (
|
||||
unusedCondition = "variant_name IN ? AND COALESCE(last_requested_at, created_at) < ?"
|
||||
topSourcesOrder = "bytes DESC, source_name ASC"
|
||||
)
|
||||
|
||||
func (v *VariantFileRepository) UnusedTotals(filter domain.UnusedFilter) (domain.VariantFileTotals, error) {
|
||||
if len(filter.VariantNames) == 0 {
|
||||
return domain.VariantFileTotals{}, nil
|
||||
}
|
||||
var row struct {
|
||||
Files int
|
||||
Bytes int64
|
||||
}
|
||||
err := v.db.Model(&entity.ImageVariantFile{}).
|
||||
Select("COUNT(*) AS files, COALESCE(SUM(size), 0) AS bytes").
|
||||
Where(unusedCondition, filter.VariantNames, filter.Cutoff.UTC()).
|
||||
Scan(&row).Error
|
||||
if err != nil {
|
||||
return domain.VariantFileTotals{}, err
|
||||
}
|
||||
return domain.VariantFileTotals{Files: row.Files, Bytes: row.Bytes}, nil
|
||||
}
|
||||
|
||||
func (v *VariantFileRepository) TopSources(limit int) ([]domain.SourceTotals, error) {
|
||||
var rows []struct {
|
||||
SourceName string
|
||||
Files int
|
||||
Bytes int64
|
||||
}
|
||||
err := v.db.Model(&entity.ImageVariantFile{}).
|
||||
Select("source_name, COUNT(*) AS files, COALESCE(SUM(size), 0) AS bytes").
|
||||
Group("source_name").
|
||||
Order(topSourcesOrder).
|
||||
Limit(limit).
|
||||
Scan(&rows).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sources := make([]domain.SourceTotals, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
sources = append(sources, domain.SourceTotals{SourceName: row.SourceName, Files: row.Files, Bytes: row.Bytes})
|
||||
}
|
||||
return sources, nil
|
||||
}
|
||||
|
||||
func (v *VariantFileRepository) ListPage(filter domain.VariantFileFilter, afterID uint32, limit int) ([]domain.VariantFile, error) {
|
||||
statement := v.db.Where("id > ?", afterID)
|
||||
if filter.VariantName != "" {
|
||||
statement = statement.Where("variant_name = ?", filter.VariantName)
|
||||
}
|
||||
if filter.SourceName != "" {
|
||||
statement = statement.Where("source_name = ?", filter.SourceName)
|
||||
}
|
||||
if filter.Unused != nil {
|
||||
if len(filter.Unused.VariantNames) == 0 {
|
||||
return []domain.VariantFile{}, nil
|
||||
}
|
||||
statement = statement.Where(unusedCondition, filter.Unused.VariantNames, filter.Unused.Cutoff.UTC())
|
||||
}
|
||||
var found []entity.ImageVariantFile
|
||||
if err := statement.Order("id asc").Limit(limit).Find(&found).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files := make([]domain.VariantFile, 0, len(found))
|
||||
for _, model := range found {
|
||||
files = append(files, mappers.VariantFileToDomain(model))
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (v *VariantFileRepository) Totals() (map[string]domain.VariantFileTotals, error) {
|
||||
var rows []struct {
|
||||
VariantName string
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
domain "RenewCMS/internal/domain/image"
|
||||
entity "RenewCMS/internal/infrastructure/persistence/models"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
type VariantUsageRepository struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
var _ domain.VariantUsageRepository = &VariantUsageRepository{}
|
||||
|
||||
func NewVariantUsageRepository(db *gorm.DB) *VariantUsageRepository {
|
||||
return &VariantUsageRepository{db}
|
||||
}
|
||||
|
||||
func (v *VariantUsageRepository) Record(batch domain.VariantUsageBatch) error {
|
||||
return v.db.Transaction(func(tx *gorm.DB) error {
|
||||
for _, hit := range batch.Hits {
|
||||
if err := recordHit(tx, hit); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, rejection := range batch.Rejections {
|
||||
if err := recordRejections(tx, rejection); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if batch.PruneBefore.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return tx.Where("day < ?", batch.PruneBefore.UTC().Format(domain.RejectionDayLayout)).Delete(&entity.ImageVariantRejectionDay{}).Error
|
||||
})
|
||||
}
|
||||
|
||||
func recordHit(tx *gorm.DB, hit domain.VariantHit) error {
|
||||
at := hit.At.UTC()
|
||||
return tx.Model(&entity.ImageVariantFile{}).
|
||||
Where("variant_name = ? AND source_name = ?", hit.VariantName, hit.SourceName).
|
||||
Where("last_requested_at IS NULL OR last_requested_at < ?", at).
|
||||
Update("last_requested_at", at).Error
|
||||
}
|
||||
|
||||
func recordRejections(tx *gorm.DB, rejection domain.RejectionCount) error {
|
||||
row := entity.ImageVariantRejectionDay{Day: rejection.Day.UTC().Format(domain.RejectionDayLayout), Count: rejection.Count}
|
||||
return tx.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "day"}},
|
||||
DoUpdates: clause.Assignments(map[string]any{"count": gorm.Expr("? + ?", clause.Column{Table: clause.CurrentTable, Name: "count"}, rejection.Count)}),
|
||||
}).Create(&row).Error
|
||||
}
|
||||
|
||||
func (v *VariantUsageRepository) RejectedSince(day time.Time) (int, error) {
|
||||
var total int
|
||||
err := v.db.Model(&entity.ImageVariantRejectionDay{}).
|
||||
Select("COALESCE(SUM(count), 0)").
|
||||
Where("day >= ?", day.UTC().Format(domain.RejectionDayLayout)).
|
||||
Scan(&total).Error
|
||||
return total, err
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package usage
|
||||
|
||||
import (
|
||||
"RenewCMS/internal/domain/image"
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const FlushKeyLimit = 500
|
||||
|
||||
func FlushInterval() time.Duration {
|
||||
return 30 * time.Second
|
||||
}
|
||||
|
||||
type hitKey struct {
|
||||
variantName string
|
||||
sourceName string
|
||||
}
|
||||
|
||||
type Tracker struct {
|
||||
repository image.VariantUsageRepository
|
||||
now func() time.Time
|
||||
interval time.Duration
|
||||
keyLimit int
|
||||
flushing sync.Mutex
|
||||
pending sync.Mutex
|
||||
hits map[hitKey]time.Time
|
||||
rejections map[time.Time]int
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
var _ image.VariantUsageTracker = &Tracker{}
|
||||
|
||||
func NewTracker(repository image.VariantUsageRepository, now func() time.Time) *Tracker {
|
||||
return NewTrackerWith(repository, now, FlushInterval(), FlushKeyLimit)
|
||||
}
|
||||
|
||||
func NewTrackerWith(repository image.VariantUsageRepository, now func() time.Time, interval time.Duration, keyLimit int) *Tracker {
|
||||
return &Tracker{
|
||||
repository: repository,
|
||||
now: now,
|
||||
interval: interval,
|
||||
keyLimit: keyLimit,
|
||||
hits: map[hitKey]time.Time{},
|
||||
rejections: map[time.Time]int{},
|
||||
wake: make(chan struct{}, 1),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) RecordHit(variantName string, sourceName string) {
|
||||
at := t.now().UTC()
|
||||
t.pending.Lock()
|
||||
defer t.pending.Unlock()
|
||||
keepLatest(t.hits, hitKey{variantName, sourceName}, at)
|
||||
if len(t.hits) >= t.keyLimit {
|
||||
t.wakeLoop()
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) RecordRejection() {
|
||||
day := image.RejectionDay(t.now())
|
||||
t.pending.Lock()
|
||||
defer t.pending.Unlock()
|
||||
t.rejections[day]++
|
||||
}
|
||||
|
||||
func (t *Tracker) wakeLoop() {
|
||||
select {
|
||||
case t.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func keepLatest(hits map[hitKey]time.Time, key hitKey, at time.Time) {
|
||||
if current, found := hits[key]; !found || at.After(current) {
|
||||
hits[key] = at
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tracker) Flush() error {
|
||||
t.flushing.Lock()
|
||||
defer t.flushing.Unlock()
|
||||
|
||||
hits, rejections := t.take()
|
||||
if len(hits) == 0 && len(rejections) == 0 {
|
||||
return nil
|
||||
}
|
||||
batch := image.VariantUsageBatch{
|
||||
Hits: hitsOf(hits),
|
||||
Rejections: rejectionsOf(rejections),
|
||||
PruneBefore: image.RejectionPruneBefore(t.now()),
|
||||
}
|
||||
if err := t.repository.Record(batch); err != nil {
|
||||
t.restore(hits, rejections)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Tracker) take() (map[hitKey]time.Time, map[time.Time]int) {
|
||||
t.pending.Lock()
|
||||
defer t.pending.Unlock()
|
||||
hits, rejections := t.hits, t.rejections
|
||||
t.hits, t.rejections = map[hitKey]time.Time{}, map[time.Time]int{}
|
||||
return hits, rejections
|
||||
}
|
||||
|
||||
func (t *Tracker) restore(hits map[hitKey]time.Time, rejections map[time.Time]int) {
|
||||
t.pending.Lock()
|
||||
defer t.pending.Unlock()
|
||||
for key, at := range hits {
|
||||
keepLatest(t.hits, key, at)
|
||||
}
|
||||
for day, count := range rejections {
|
||||
t.rejections[day] += count
|
||||
}
|
||||
}
|
||||
|
||||
func hitsOf(hits map[hitKey]time.Time) []image.VariantHit {
|
||||
listed := make([]image.VariantHit, 0, len(hits))
|
||||
for key, at := range hits {
|
||||
listed = append(listed, image.VariantHit{VariantName: key.variantName, SourceName: key.sourceName, At: at})
|
||||
}
|
||||
return listed
|
||||
}
|
||||
|
||||
func rejectionsOf(rejections map[time.Time]int) []image.RejectionCount {
|
||||
listed := make([]image.RejectionCount, 0, len(rejections))
|
||||
for day, count := range rejections {
|
||||
listed = append(listed, image.RejectionCount{Day: day, Count: count})
|
||||
}
|
||||
return listed
|
||||
}
|
||||
|
||||
func (t *Tracker) Run(ctx context.Context) {
|
||||
ticker := time.NewTicker(t.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = t.Flush()
|
||||
return
|
||||
case <-ticker.C:
|
||||
_ = t.Flush()
|
||||
case <-t.wake:
|
||||
_ = t.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package useCases
|
||||
|
||||
import (
|
||||
"RenewCMS/internal/domain/image"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type VariantCacheEntry struct {
|
||||
Name string
|
||||
Enabled bool
|
||||
Files int
|
||||
Bytes int64
|
||||
}
|
||||
|
||||
type VariantCacheImage struct {
|
||||
ID uint32
|
||||
Filename string
|
||||
URL string
|
||||
Files int
|
||||
Bytes int64
|
||||
}
|
||||
|
||||
type VariantCacheStats struct {
|
||||
GeneratedFiles int
|
||||
DiskUsed int64
|
||||
VariantsDefined int
|
||||
VariantsEnabled int
|
||||
VariantsDisabled int
|
||||
RejectedRequests int
|
||||
RejectedWindowDays int
|
||||
Unused image.VariantFileTotals
|
||||
UnusedDays int
|
||||
ByVariant []VariantCacheEntry
|
||||
TopImages []VariantCacheImage
|
||||
}
|
||||
|
||||
type GetVariantCacheStatsUseCase struct {
|
||||
variants image.VariantRepository
|
||||
files image.VariantFileRepository
|
||||
cache image.VariantCacheRepository
|
||||
images image.Repository
|
||||
usage image.VariantUsageRepository
|
||||
tracker image.VariantUsageTracker
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewGetVariantCacheStatsUseCase(variants image.VariantRepository, files image.VariantFileRepository, cache image.VariantCacheRepository, images image.Repository, usage image.VariantUsageRepository, tracker image.VariantUsageTracker, now func() time.Time) *GetVariantCacheStatsUseCase {
|
||||
return &GetVariantCacheStatsUseCase{variants, files, cache, images, usage, tracker, now}
|
||||
}
|
||||
|
||||
func (u *GetVariantCacheStatsUseCase) Get() (VariantCacheStats, error) {
|
||||
_ = u.tracker.Flush()
|
||||
|
||||
variants, err := u.variants.List()
|
||||
if err != nil {
|
||||
return VariantCacheStats{}, err
|
||||
}
|
||||
totals, err := u.files.Totals()
|
||||
if err != nil {
|
||||
return VariantCacheStats{}, err
|
||||
}
|
||||
now := u.now()
|
||||
unused, err := u.cache.UnusedTotals(image.UnusedFilter{VariantNames: enabledNames(variants), Cutoff: image.UnusedCutoff(now)})
|
||||
if err != nil {
|
||||
return VariantCacheStats{}, err
|
||||
}
|
||||
rejected, err := u.usage.RejectedSince(image.RejectionWindowStart(now))
|
||||
if err != nil {
|
||||
return VariantCacheStats{}, err
|
||||
}
|
||||
top, err := u.topImages()
|
||||
if err != nil {
|
||||
return VariantCacheStats{}, err
|
||||
}
|
||||
|
||||
stats := VariantCacheStats{
|
||||
VariantsDefined: len(variants),
|
||||
RejectedRequests: rejected,
|
||||
RejectedWindowDays: image.RejectionWindowDays,
|
||||
Unused: unused,
|
||||
UnusedDays: image.UnusedVariantDays,
|
||||
ByVariant: cacheEntries(variants, totals),
|
||||
TopImages: top,
|
||||
}
|
||||
for _, entry := range stats.ByVariant {
|
||||
stats.GeneratedFiles += entry.Files
|
||||
stats.DiskUsed += entry.Bytes
|
||||
if entry.Enabled {
|
||||
stats.VariantsEnabled++
|
||||
}
|
||||
}
|
||||
stats.VariantsDisabled = stats.VariantsDefined - stats.VariantsEnabled
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func enabledNames(variants []image.Variant) []string {
|
||||
names := []string{}
|
||||
for _, variant := range variants {
|
||||
if variant.Enabled {
|
||||
names = append(names, variant.Name)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func cacheEntries(variants []image.Variant, totals map[string]image.VariantFileTotals) []VariantCacheEntry {
|
||||
entries := make([]VariantCacheEntry, 0, len(variants))
|
||||
for _, variant := range variants {
|
||||
own := totals[variant.Name]
|
||||
entries = append(entries, VariantCacheEntry{Name: variant.Name, Enabled: variant.Enabled, Files: own.Files, Bytes: own.Bytes})
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
if entries[i].Bytes != entries[j].Bytes {
|
||||
return entries[i].Bytes > entries[j].Bytes
|
||||
}
|
||||
return entries[i].Name < entries[j].Name
|
||||
})
|
||||
return entries
|
||||
}
|
||||
|
||||
func (u *GetVariantCacheStatsUseCase) topImages() ([]VariantCacheImage, error) {
|
||||
limit := image.TopImagesLimit + 1
|
||||
for {
|
||||
sources, err := u.cache.TopSources(limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ranked, err := u.rank(sources)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(sources) < limit || topIsSettled(ranked, sources) {
|
||||
return ranked[:min(len(ranked), image.TopImagesLimit)], nil
|
||||
}
|
||||
limit *= 2
|
||||
}
|
||||
}
|
||||
|
||||
func topIsSettled(ranked []VariantCacheImage, sources []image.SourceTotals) bool {
|
||||
return len(ranked) >= image.TopImagesLimit && ranked[image.TopImagesLimit-1].Bytes > sources[len(sources)-1].Bytes
|
||||
}
|
||||
|
||||
func (u *GetVariantCacheStatsUseCase) rank(sources []image.SourceTotals) ([]VariantCacheImage, error) {
|
||||
names := make([]string, 0, len(sources))
|
||||
for _, source := range sources {
|
||||
names = append(names, source.SourceName)
|
||||
}
|
||||
found, err := u.images.ListByStoredNames(names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byName := make(map[string]image.Image, len(found))
|
||||
for _, img := range found {
|
||||
byName[image.StoredName(img.Path)] = img
|
||||
}
|
||||
ranked := make([]VariantCacheImage, 0, len(sources))
|
||||
for _, source := range sources {
|
||||
if img, known := byName[source.SourceName]; known {
|
||||
ranked = append(ranked, VariantCacheImage{ID: img.ID, Filename: img.Filename, URL: image.PublicPath(source.SourceName), Files: source.Files, Bytes: source.Bytes})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(ranked, func(i, j int) bool {
|
||||
if ranked[i].Bytes != ranked[j].Bytes {
|
||||
return ranked[i].Bytes > ranked[j].Bytes
|
||||
}
|
||||
return ranked[i].ID < ranked[j].ID
|
||||
})
|
||||
return ranked, nil
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package useCases
|
||||
|
||||
import (
|
||||
"RenewCMS/internal/domain/audit"
|
||||
"RenewCMS/internal/domain/image"
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type PurgeVariantCacheCommand struct {
|
||||
Scope image.CachePurgeScope
|
||||
Variant string
|
||||
ImageID uint32
|
||||
}
|
||||
|
||||
type VariantCachePurgeResult struct {
|
||||
Scope image.CachePurgeScope
|
||||
Files int
|
||||
Bytes int64
|
||||
Failed int
|
||||
}
|
||||
|
||||
type PurgeVariantCacheUseCase struct {
|
||||
variants image.VariantRepository
|
||||
images image.Repository
|
||||
cache image.VariantCacheRepository
|
||||
files variantCache
|
||||
recorder audit.Recorder
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewPurgeVariantCacheUseCase(variants image.VariantRepository, images image.Repository, cache image.VariantCacheRepository, files image.VariantFileRepository, storage image.Storage, now func() time.Time, recorder audit.Recorder) *PurgeVariantCacheUseCase {
|
||||
return &PurgeVariantCacheUseCase{variants, images, cache, variantCache{files, storage}, recorder, now}
|
||||
}
|
||||
|
||||
type cachePurgePlan struct {
|
||||
filter image.VariantFileFilter
|
||||
event audit.Event
|
||||
}
|
||||
|
||||
func (u *PurgeVariantCacheUseCase) Purge(ctx context.Context, command PurgeVariantCacheCommand) (VariantCachePurgeResult, error) {
|
||||
plan, err := u.plan(command)
|
||||
if err != nil {
|
||||
return VariantCachePurgeResult{}, err
|
||||
}
|
||||
result, err := u.purgeMatching(plan.filter)
|
||||
result.Scope = command.Scope
|
||||
if err != nil {
|
||||
recordFailure(ctx, u.recorder, plan.event, err)
|
||||
return result, err
|
||||
}
|
||||
u.record(ctx, plan.event, result)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (u *PurgeVariantCacheUseCase) plan(command PurgeVariantCacheCommand) (cachePurgePlan, error) {
|
||||
switch command.Scope {
|
||||
case image.CacheScopeAll:
|
||||
return cachePurgePlan{event: audit.NewEvent(audit.ActionVariantPurgeAll, audit.CategoryMedia)}, nil
|
||||
case image.CacheScopeUnused:
|
||||
return u.unusedPlan()
|
||||
case image.CacheScopeVariant:
|
||||
return u.variantPlan(command.Variant)
|
||||
case image.CacheScopeImage:
|
||||
return u.imagePlan(command.ImageID)
|
||||
default:
|
||||
return cachePurgePlan{}, image.ErrCachePurgeScope
|
||||
}
|
||||
}
|
||||
|
||||
func (u *PurgeVariantCacheUseCase) unusedPlan() (cachePurgePlan, error) {
|
||||
variants, err := u.variants.List()
|
||||
if err != nil {
|
||||
return cachePurgePlan{}, err
|
||||
}
|
||||
event := audit.NewEvent(audit.ActionVariantPurgeUnused, audit.CategoryMedia)
|
||||
filter := image.VariantFileFilter{Unused: &image.UnusedFilter{VariantNames: enabledNames(variants), Cutoff: image.UnusedCutoff(u.now())}}
|
||||
return cachePurgePlan{filter: filter, event: event}, nil
|
||||
}
|
||||
|
||||
func (u *PurgeVariantCacheUseCase) variantPlan(name string) (cachePurgePlan, error) {
|
||||
variant, err := u.variants.Get(name)
|
||||
if err != nil {
|
||||
return cachePurgePlan{}, err
|
||||
}
|
||||
if variant.Name != name {
|
||||
return cachePurgePlan{}, image.ErrVariantNotFound
|
||||
}
|
||||
event := audit.NewEvent(audit.ActionVariantPurge, audit.CategoryMedia)
|
||||
event.Target = audit.VariantTarget(name)
|
||||
return cachePurgePlan{filter: image.VariantFileFilter{VariantName: name}, event: event}, nil
|
||||
}
|
||||
|
||||
func (u *PurgeVariantCacheUseCase) imagePlan(id uint32) (cachePurgePlan, error) {
|
||||
img, err := u.images.Get(id)
|
||||
if err != nil {
|
||||
return cachePurgePlan{}, err
|
||||
}
|
||||
event := audit.NewEvent(audit.ActionVariantPurgeImage, audit.CategoryMedia)
|
||||
event.Target = imageTarget(img)
|
||||
return cachePurgePlan{filter: image.VariantFileFilter{SourceName: image.StoredName(img.Path)}, event: event}, nil
|
||||
}
|
||||
|
||||
func (u *PurgeVariantCacheUseCase) purgeMatching(filter image.VariantFileFilter) (VariantCachePurgeResult, error) {
|
||||
var result VariantCachePurgeResult
|
||||
var after uint32
|
||||
for {
|
||||
rows, err := u.cache.ListPage(filter, after, image.CachePurgeBatchSize)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
for _, row := range rows {
|
||||
after = row.ID
|
||||
if err := u.files.purgeRow(row); err != nil {
|
||||
result.Failed++
|
||||
continue
|
||||
}
|
||||
result.Files++
|
||||
result.Bytes += row.Size
|
||||
}
|
||||
if len(rows) < image.CachePurgeBatchSize {
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (u *PurgeVariantCacheUseCase) record(ctx context.Context, event audit.Event, result VariantCachePurgeResult) {
|
||||
if result.Files == 0 && result.Failed == 0 {
|
||||
return
|
||||
}
|
||||
event.Params = cachePurgeParams(event.Action, result)
|
||||
if result.Files == 0 {
|
||||
failFrom(ctx, u.recorder, event, GenericFailureReason)
|
||||
return
|
||||
}
|
||||
u.recorder.Record(ctx, event)
|
||||
}
|
||||
|
||||
func cachePurgeParams(action string, result VariantCachePurgeResult) map[string]any {
|
||||
params := map[string]any{audit.ParamPurgedFiles: result.Files, audit.ParamPurgedBytes: result.Bytes}
|
||||
if action == audit.ActionVariantPurgeUnused {
|
||||
params[audit.ParamOlderThanDays] = image.UnusedVariantDays
|
||||
}
|
||||
if result.Failed > 0 {
|
||||
params[audit.ParamFailedFiles] = result.Failed
|
||||
}
|
||||
return params
|
||||
}
|
||||
@@ -55,6 +55,18 @@ type ServeVariantUseCase struct {
|
||||
maxSourcePixels int64
|
||||
renderSlots chan struct{}
|
||||
flights singleflight.Group
|
||||
tracker image.VariantUsageTracker
|
||||
}
|
||||
|
||||
type idleUsageTracker struct{}
|
||||
|
||||
func (idleUsageTracker) RecordHit(string, string) {}
|
||||
func (idleUsageTracker) RecordRejection() {}
|
||||
func (idleUsageTracker) Flush() error { return nil }
|
||||
|
||||
func (u *ServeVariantUseCase) WithUsageTracker(tracker image.VariantUsageTracker) *ServeVariantUseCase {
|
||||
u.tracker = tracker
|
||||
return u
|
||||
}
|
||||
|
||||
func NewServeVariantUseCase(variants image.VariantRepository, files image.VariantFileRepository, images image.Repository, storage image.Storage, renderer image.VariantRenderer, limits ServeVariantLimits) *ServeVariantUseCase {
|
||||
@@ -67,10 +79,19 @@ func NewServeVariantUseCase(variants image.VariantRepository, files image.Varian
|
||||
maxSourceBytes: limits.MaxSourceBytes,
|
||||
maxSourcePixels: limits.MaxSourcePixels,
|
||||
renderSlots: make(chan struct{}, max(1, limits.MaxConcurrentRenders)),
|
||||
tracker: idleUsageTracker{},
|
||||
}
|
||||
}
|
||||
|
||||
func (u *ServeVariantUseCase) Serve(request ServeVariantRequest) (ServedVariant, error) {
|
||||
served, err := u.serve(request)
|
||||
if errors.Is(err, image.ErrUnknownVariant) || errors.Is(err, image.ErrUnsupportedParameter) {
|
||||
u.tracker.RecordRejection()
|
||||
}
|
||||
return served, err
|
||||
}
|
||||
|
||||
func (u *ServeVariantUseCase) serve(request ServeVariantRequest) (ServedVariant, error) {
|
||||
if parameter := unsupportedParameter(request.Parameters); parameter != "" {
|
||||
return ServedVariant{}, image.NewUnsupportedParameterError(parameter)
|
||||
}
|
||||
@@ -134,18 +155,26 @@ func (u *ServeVariantUseCase) serveSource(variant image.Variant, request ServeVa
|
||||
settings := variant.Settings()
|
||||
etag := image.VariantETag(settings, request.StoredName)
|
||||
if image.ETagMatches(request.IfNoneMatch, etag) {
|
||||
u.tracker.RecordHit(variant.Name, request.StoredName)
|
||||
return ServedVariant{NotModified: true, ETag: etag}, nil
|
||||
}
|
||||
fingerprint := settings.Fingerprint()
|
||||
cached, previous, hit, err := u.lookupCached(variant.Name, request.StoredName, fingerprint, etag)
|
||||
if err != nil || hit {
|
||||
if err == nil {
|
||||
u.tracker.RecordHit(variant.Name, request.StoredName)
|
||||
}
|
||||
return cached, err
|
||||
}
|
||||
file, err := u.generated(requestContext(request), variant, request.StoredName, fingerprint, previous)
|
||||
if err != nil {
|
||||
return ServedVariant{}, err
|
||||
}
|
||||
return u.serveFile(file, etag)
|
||||
served, err := u.serveFile(file, etag)
|
||||
if err == nil {
|
||||
u.tracker.RecordHit(variant.Name, request.StoredName)
|
||||
}
|
||||
return served, err
|
||||
}
|
||||
|
||||
func requestContext(request ServeVariantRequest) context.Context {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package useCases
|
||||
|
||||
import "RenewCMS/internal/domain/image"
|
||||
import (
|
||||
"RenewCMS/internal/domain/image"
|
||||
"errors"
|
||||
)
|
||||
|
||||
type VariantPurgeResult struct {
|
||||
Files int
|
||||
@@ -44,7 +47,7 @@ func (c variantCache) purge(rows []image.VariantFile) (VariantPurgeResult, error
|
||||
}
|
||||
|
||||
func (c variantCache) purgeRow(row image.VariantFile) error {
|
||||
if err := c.storage.Delete(row.CachedName); err != nil {
|
||||
if err := c.storage.Delete(row.CachedName); err != nil && !errors.Is(err, image.ErrBlobNotFound) {
|
||||
return err
|
||||
}
|
||||
return c.files.Delete(row.ID)
|
||||
|
||||
Reference in new issue
Block a user