feat: media duplicate purge, part 1 of 3: persisted purge settings and the purge API

internal/domain/purge/purgeDomain.go

@@ -0,0 +1,48 @@
+package purge
+
+import (
+	"errors"
+	"time"
+)
+
+var ErrInvalidPurgeInterval = errors.New("invalid purge interval")
+var ErrPurgeConflict = errors.New("the library changed since the fingerprint")
+var ErrPurgeBusy = errors.New("a purge is already running")
+
+const (
+	DefaultIntervalDays = 30
+	MinIntervalDays     = 1
+	MaxIntervalDays     = 365
+)
+
+type Settings struct {
+	AutoPurgeEnabled         bool       `json:"auto_purge_enabled"`
+	AutoPurgeIntervalDays    int        `json:"auto_purge_interval_days"`
+	LastPurgeAt              *time.Time `json:"last_purge_at"`
+	LastPurgeMerged          int        `json:"last_purge_merged"`
+	LastPurgeArticlesUpdated int        `json:"last_purge_articles_updated"`
+	EnabledAt                *time.Time `json:"enabled_at"`
+}
+
+func DefaultSettings() Settings {
+	return Settings{AutoPurgeIntervalDays: DefaultIntervalDays}
+}
+
+func ValidateIntervalDays(days int) error {
+	if days < MinIntervalDays || days > MaxIntervalDays {
+		return ErrInvalidPurgeInterval
+	}
+	return nil
+}
+
+func (s Settings) NextPurgeAt() *time.Time {
+	if !s.AutoPurgeEnabled || s.EnabledAt == nil {
+		return nil
+	}
+	base := *s.EnabledAt
+	if s.LastPurgeAt != nil && s.LastPurgeAt.After(base) {
+		base = *s.LastPurgeAt
+	}
+	next := base.AddDate(0, 0, s.AutoPurgeIntervalDays)
+	return &next
+}

internal/domain/purge/purgeRepositoryInterface.go

@@ -0,0 +1,6 @@
+package purge
+
+type Repository interface {
+	Get() (Settings, error)
+	Save(settings Settings) error
+}

internal/infrastructure/persistence/mappers/purgeSettingsMapper.go

@@ -0,0 +1,28 @@
+package mappers
+
+import (
+	"RenewCMS/internal/domain/purge"
+	"RenewCMS/internal/infrastructure/persistence/models"
+)
+
+func PurgeSettingsToDomain(m models.PurgeSettings) purge.Settings {
+	return purge.Settings{
+		AutoPurgeEnabled:         m.AutoPurgeEnabled,
+		AutoPurgeIntervalDays:    m.AutoPurgeIntervalDays,
+		LastPurgeAt:              m.LastPurgeAt,
+		LastPurgeMerged:          m.LastPurgeMerged,
+		LastPurgeArticlesUpdated: m.LastPurgeArticlesUpdated,
+		EnabledAt:                m.EnabledAt,
+	}
+}
+
+func PurgeSettingsToModel(s purge.Settings) models.PurgeSettings {
+	return models.PurgeSettings{
+		AutoPurgeEnabled:         s.AutoPurgeEnabled,
+		AutoPurgeIntervalDays:    s.AutoPurgeIntervalDays,
+		LastPurgeAt:              s.LastPurgeAt,
+		LastPurgeMerged:          s.LastPurgeMerged,
+		LastPurgeArticlesUpdated: s.LastPurgeArticlesUpdated,
+		EnabledAt:                s.EnabledAt,
+	}
+}

internal/infrastructure/persistence/models/purgeSettingsModel.go

@@ -0,0 +1,13 @@
+package models
+
+import "time"
+
+type PurgeSettings struct {
+	ID                       uint32 `gorm:"primaryKey"`
+	AutoPurgeEnabled         bool   `gorm:"not null"`
+	AutoPurgeIntervalDays    int    `gorm:"not null"`
+	LastPurgeAt              *time.Time
+	LastPurgeMerged          int `gorm:"not null"`
+	LastPurgeArticlesUpdated int `gorm:"not null"`
+	EnabledAt                *time.Time
+}

internal/infrastructure/persistence/repositories/purgeSettingsRepository.go

@@ -0,0 +1,40 @@
+package persistence
+
+import (
+	domainPurge "RenewCMS/internal/domain/purge"
+	"RenewCMS/internal/infrastructure/persistence/mappers"
+	entity "RenewCMS/internal/infrastructure/persistence/models"
+	"errors"
+
+	"gorm.io/gorm"
+)
+
+const purgeSettingsRowID uint32 = 1
+
+type PurgeSettingsRepository struct {
+	db *gorm.DB
+}
+
+var _ domainPurge.Repository = &PurgeSettingsRepository{}
+
+func NewPurgeSettingsRepository(db *gorm.DB) *PurgeSettingsRepository {
+	return &PurgeSettingsRepository{db}
+}
+
+func (p *PurgeSettingsRepository) Get() (domainPurge.Settings, error) {
+	var model entity.PurgeSettings
+	err := p.db.First(&model, purgeSettingsRowID).Error
+	if errors.Is(err, gorm.ErrRecordNotFound) {
+		return domainPurge.DefaultSettings(), nil
+	}
+	if err != nil {
+		return domainPurge.Settings{}, err
+	}
+	return mappers.PurgeSettingsToDomain(model), nil
+}
+
+func (p *PurgeSettingsRepository) Save(settings domainPurge.Settings) error {
+	model := mappers.PurgeSettingsToModel(settings)
+	model.ID = purgeSettingsRowID
+	return p.db.Save(&model).Error
+}

internal/infrastructure/useCases/GetPurgeSettings.go

@@ -0,0 +1,15 @@
+package useCases
+
+import "RenewCMS/internal/domain/purge"
+
+type GetPurgeSettingsUseCase struct {
+	repository purge.Repository
+}
+
+func NewGetPurgeSettingsUseCase(repository purge.Repository) *GetPurgeSettingsUseCase {
+	return &GetPurgeSettingsUseCase{repository}
+}
+
+func (g *GetPurgeSettingsUseCase) Get() (purge.Settings, error) {
+	return g.repository.Get()
+}

internal/infrastructure/useCases/MergeMedia.go

@@ -10,8 +10,9 @@ import (
 )
 
 type MergeMediaResult struct {
-	ArticlesUpdated int
-	Merged          int
+	ArticlesUpdated   int
+	Merged            int
+	UpdatedArticleIDs []uint32
 }
 
 type MergeMediaUseCase struct {
@@ -57,7 +58,15 @@ func (u *MergeMediaUseCase) Merge(keepId uint32, mergeIds []uint32) (MergeMediaR
 		}
 	}
 
-	return MergeMediaResult{ArticlesUpdated: len(rewrites), Merged: len(copies)}, nil
+	return MergeMediaResult{ArticlesUpdated: len(rewrites), Merged: len(copies), UpdatedArticleIDs: rewrittenIds(rewrites)}, nil
+}
+
+func rewrittenIds(rewrites []bodyRewrite) []uint32 {
+	ids := make([]uint32, 0, len(rewrites))
+	for _, plan := range rewrites {
+		ids = append(ids, plan.articleId)
+	}
+	return ids
 }
 
 func validateMergeRequest(keepId uint32, mergeIds []uint32) error {

internal/infrastructure/useCases/PreviewPurge.go

@@ -0,0 +1,61 @@
+package useCases
+
+import (
+	"RenewCMS/internal/domain/article"
+	"RenewCMS/internal/domain/image"
+)
+
+const purgePreviewListLimit = 5
+
+type PurgePreviewGroup struct {
+	KeepID   uint32
+	Filename string
+	URL      string
+	UsedIn   int
+	Merged   int
+}
+
+type PurgePreview struct {
+	Groups      int
+	Copies      int
+	Articles    int
+	List        []PurgePreviewGroup
+	More        int
+	Fingerprint string
+}
+
+type PreviewPurgeUseCase struct {
+	planner *purgePlanner
+}
+
+func NewPreviewPurgeUseCase(imageRepository image.Repository, articleRepository article.Repository) *PreviewPurgeUseCase {
+	return &PreviewPurgeUseCase{&purgePlanner{imageRepository, articleRepository}}
+}
+
+func (u *PreviewPurgeUseCase) Preview() (PurgePreview, error) {
+	plan, err := u.planner.plan()
+	if err != nil {
+		return PurgePreview{}, err
+	}
+
+	shown := min(len(plan.groups), purgePreviewListLimit)
+	list := make([]PurgePreviewGroup, 0, shown)
+	for _, group := range plan.groups[:shown] {
+		list = append(list, PurgePreviewGroup{
+			KeepID:   group.kept.ID,
+			Filename: group.kept.Filename,
+			URL:      group.kept.Path,
+			UsedIn:   group.keptUsage,
+			Merged:   len(group.mergedIds),
+		})
+	}
+
+	return PurgePreview{
+		Groups:      len(plan.groups),
+		Copies:      plan.copies,
+		Articles:    plan.articles,
+		List:        list,
+		More:        len(plan.groups) - shown,
+		Fingerprint: plan.fingerprint,
+	}, nil
+}

internal/infrastructure/useCases/RunPurge.go

@@ -0,0 +1,91 @@
+package useCases
+
+import (
+	"RenewCMS/internal/domain/article"
+	"RenewCMS/internal/domain/image"
+	"RenewCMS/internal/domain/purge"
+	"log"
+	"slices"
+	"sync"
+	"time"
+)
+
+type RunPurgeResult struct {
+	GroupsTotal     int
+	GroupsPurged    int
+	Merged          int
+	ArticlesUpdated int
+	FailedGroups    int
+}
+
+type RunPurgeUseCase struct {
+	planner    *purgePlanner
+	merge      *MergeMediaUseCase
+	repository purge.Repository
+	now        func() time.Time
+	logger     *log.Logger
+	guard      sync.Mutex
+}
+
+func NewRunPurgeUseCase(imageRepository image.Repository, articleRepository article.Repository, merge *MergeMediaUseCase, repository purge.Repository, now func() time.Time, logger *log.Logger) *RunPurgeUseCase {
+	return &RunPurgeUseCase{
+		planner:    &purgePlanner{imageRepository, articleRepository},
+		merge:      merge,
+		repository: repository,
+		now:        now,
+		logger:     logger,
+	}
+}
+
+func (u *RunPurgeUseCase) Run(fingerprint *string) (RunPurgeResult, error) {
+	if !u.guard.TryLock() {
+		return RunPurgeResult{}, purge.ErrPurgeBusy
+	}
+	defer u.guard.Unlock()
+
+	plan, err := u.planner.plan()
+	if err != nil {
+		return RunPurgeResult{}, err
+	}
+	if fingerprint != nil && *fingerprint != plan.fingerprint {
+		return RunPurgeResult{}, purge.ErrPurgeConflict
+	}
+
+	result := u.mergeGroups(plan)
+	if err := u.recordRun(result); err != nil {
+		u.logger.Printf("purge: could not record the run result: %v", err)
+		return RunPurgeResult{}, err
+	}
+	return result, nil
+}
+
+func (u *RunPurgeUseCase) mergeGroups(plan purgePlan) RunPurgeResult {
+	result := RunPurgeResult{GroupsTotal: len(plan.groups)}
+	updated := map[uint32]bool{}
+	for _, group := range plan.groups {
+		merged, err := u.merge.Merge(group.kept.ID, slices.Clone(group.mergedIds))
+		if err != nil {
+			result.FailedGroups++
+			continue
+		}
+		result.GroupsPurged++
+		result.Merged += merged.Merged
+		for _, id := range merged.UpdatedArticleIDs {
+			updated[id] = true
+		}
+	}
+	result.ArticlesUpdated = len(updated)
+	return result
+}
+
+func (u *RunPurgeUseCase) recordRun(result RunPurgeResult) error {
+	settings, err := u.repository.Get()
+	if err != nil {
+		return err
+	}
+	at := u.now().UTC()
+	settings.LastPurgeAt = &at
+	settings.LastPurgeMerged = result.Merged
+	settings.LastPurgeArticlesUpdated = result.ArticlesUpdated
+	return u.repository.Save(settings)
+}

internal/infrastructure/useCases/UpdatePurgeSettings.go

@@ -0,0 +1,49 @@
+package useCases
+
+import (
+	"RenewCMS/internal/domain/purge"
+	"time"
+)
+
+type UpdatePurgeSettingsCommand struct {
+	AutoPurgeEnabled      *bool
+	AutoPurgeIntervalDays *int
+}
+
+type UpdatePurgeSettingsUseCase struct {
+	repository purge.Repository
+	now        func() time.Time
+}
+
+func NewUpdatePurgeSettingsUseCase(repository purge.Repository, now func() time.Time) *UpdatePurgeSettingsUseCase {
+	return &UpdatePurgeSettingsUseCase{repository, now}
+}
+
+func (u *UpdatePurgeSettingsUseCase) Update(command UpdatePurgeSettingsCommand) (purge.Settings, error) {
+	if command.AutoPurgeIntervalDays != nil {
+		if err := purge.ValidateIntervalDays(*command.AutoPurgeIntervalDays); err != nil {
+			return purge.Settings{}, err
+		}
+	}
+
+	settings, err := u.repository.Get()
+	if err != nil {
+		return purge.Settings{}, err
+	}
+
+	if command.AutoPurgeEnabled != nil {
+		if *command.AutoPurgeEnabled && !settings.AutoPurgeEnabled {
+			enabledAt := u.now().UTC()
+			settings.EnabledAt = &enabledAt
+		}
+		settings.AutoPurgeEnabled = *command.AutoPurgeEnabled
+	}
+	if command.AutoPurgeIntervalDays != nil {
+		settings.AutoPurgeIntervalDays = *command.AutoPurgeIntervalDays
+	}
+
+	if err := u.repository.Save(settings); err != nil {
+		return purge.Settings{}, err
+	}
+	return settings, nil
+}

internal/infrastructure/useCases/purgePlan.go

@@ -0,0 +1,162 @@
+package useCases
+
+import (
+	"RenewCMS/internal/domain/article"
+	"RenewCMS/internal/domain/image"
+	"crypto/sha256"
+	"encoding/hex"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+type purgeGroup struct {
+	kept      image.Image
+	keptUsage int
+	mergedIds []uint32
+}
+
+type purgePlan struct {
+	groups      []purgeGroup
+	copies      int
+	articles    int
+	fingerprint string
+}
+
+type purgePlanner struct {
+	imageRepository   image.Repository
+	articleRepository article.Repository
+}
+
+func (p *purgePlanner) plan() (purgePlan, error) {
+	duplicates, err := p.imageRepository.ListDuplicates()
+	if err != nil {
+		return purgePlan{}, err
+	}
+	articles, err := p.articleRepository.FindByFilters(article.Filters{})
+	if err != nil {
+		return purgePlan{}, err
+	}
+
+	groups := []purgeGroup{}
+	copies := 0
+	touched := map[uint32]bool{}
+	for _, members := range groupByHash(duplicates) {
+		group, planned := planGroup(members, articles)
+		if !planned {
+			continue
+		}
+		groups = append(groups, group)
+		copies += len(group.mergedIds)
+		for _, id := range articlesUsingCopies(group, members, articles) {
+			touched[id] = true
+		}
+	}
+	slices.SortFunc(groups, func(a, b purgeGroup) int { return int(a.kept.ID) - int(b.kept.ID) })
+
+	return purgePlan{groups: groups, copies: copies, articles: len(touched), fingerprint: fingerprintOf(groups)}, nil
+}
+
+func groupByHash(images []image.Image) [][]image.Image {
+	byHash := map[string][]image.Image{}
+	hashes := []string{}
+	for _, img := range images {
+		if img.SHA256 == "" {
+			continue
+		}
+		if _, seen := byHash[img.SHA256]; !seen {
+			hashes = append(hashes, img.SHA256)
+		}
+		byHash[img.SHA256] = append(byHash[img.SHA256], img)
+	}
+	groups := make([][]image.Image, 0, len(hashes))
+	for _, hash := range hashes {
+		groups = append(groups, byHash[hash])
+	}
+	return groups
+}
+
+func usageOf(img image.Image, articles []article.Article) int {
+	storedName := image.StoredName(img.Path)
+	used := 0
+	for _, a := range articles {
+		if strings.Contains(a.Body, storedName) {
+			used++
+		}
+	}
+	return used
+}
+
+func planGroup(members []image.Image, articles []article.Article) (purgeGroup, bool) {
+	usage := make(map[uint32]int, len(members))
+	anyUsed := false
+	for _, member := range members {
+		usage[member.ID] = usageOf(member, articles)
+		anyUsed = anyUsed || usage[member.ID] > 0
+	}
+	if !anyUsed {
+		return purgeGroup{}, false
+	}
+
+	kept := members[0]
+	for _, member := range members[1:] {
+		if isBetterToKeep(member, kept, usage) {
+			kept = member
+		}
+	}
+
+	mergedIds := []uint32{}
+	for _, member := range members {
+		if member.ID != kept.ID {
+			mergedIds = append(mergedIds, member.ID)
+		}
+	}
+	slices.Sort(mergedIds)
+	return purgeGroup{kept: kept, keptUsage: usage[kept.ID], mergedIds: mergedIds}, true
+}
+
+func isBetterToKeep(candidate, current image.Image, usage map[uint32]int) bool {
+	if usage[candidate.ID] != usage[current.ID] {
+		return usage[candidate.ID] > usage[current.ID]
+	}
+	if !candidate.CreatedAt.Equal(current.CreatedAt) {
+		return candidate.CreatedAt.Before(current.CreatedAt)
+	}
+	return candidate.ID < current.ID
+}
+
+func articlesUsingCopies(group purgeGroup, members []image.Image, articles []article.Article) []uint32 {
+	mergedNames := []string{}
+	for _, member := range members {
+		if member.ID != group.kept.ID {
+			mergedNames = append(mergedNames, image.StoredName(member.Path))
+		}
+	}
+	ids := []uint32{}
+	for _, a := range articles {
+		for _, name := range mergedNames {
+			if strings.Contains(a.Body, name) {
+				ids = append(ids, a.ID)
+				break
+			}
+		}
+	}
+	return ids
+}
+
+func fingerprintOf(groups []purgeGroup) string {
+	var builder strings.Builder
+	for _, group := range groups {
+		builder.WriteString(strconv.FormatUint(uint64(group.kept.ID), 10))
+		builder.WriteString(":")
+		for position, id := range group.mergedIds {
+			if position > 0 {
+				builder.WriteString(",")
+			}
+			builder.WriteString(strconv.FormatUint(uint64(id), 10))
+		}
+		builder.WriteString(";")
+	}
+	sum := sha256.Sum256([]byte(builder.String()))
+	return hex.EncodeToString(sum[:])
+}