feat: setup logs retention by max file size

internal/domain/audit/auditDomain.go

@@ -105,6 +105,8 @@ const (
 	ParamRemoved         = "removed"
 	ParamFrom            = "from"
 	ParamTo              = "to"
+	ParamMaxGBFrom       = "max_gb_from"
+	ParamMaxGBTo         = "max_gb_to"
 	ParamMerged          = "merged"
 	ParamArticlesUpdated = "articles_updated"
 	ParamFailedGroups    = "failed_groups"

internal/domain/audit/auditRepositoryInterface.go

@@ -7,10 +7,12 @@ type Repository interface {
 	List(filter Filter) (Page, error)
 	Summary(query SummaryQuery) (Summary, error)
 	DeleteOlderThan(cutoff time.Time) (int, error)
+	DeleteOldest(limit int, before time.Time) (int, error)
+	SizeBytes() (int64, error)
 }
 
 type SettingsRepository interface {
 	Get() (Settings, error)
-	SaveRetention(days int) error
+	SaveRetention(days int, maxGB int) error
 	SaveLastCleanup(at time.Time, removed int) error
 }

internal/domain/audit/auditSettings.go

@@ -7,6 +7,9 @@ const (
 	ForeverRetentionDays = 0
 	CleanupHour          = 3
 	KeepRecentHours      = 24
+	NoSizeLimitGB        = 0
+	BytesPerGB           = int64(1) << 30
+	RowOverheadBytes     = int64(64)
 )
 
 func RetentionOptions() []int {
@@ -22,8 +25,23 @@ func ValidateRetentionDays(days int) error {
 	return ErrInvalidRetention
 }
 
+func SizeOptions() []int {
+	return []int{NoSizeLimitGB, 1, 5, 10, 25, 50}
+}
+
+func ValidateRetentionMaxGB(maxGB int) error {
+	for _, option := range SizeOptions() {
+		if option == maxGB {
+			return nil
+		}
+	}
+	return ErrInvalidRetention
+}
+
 type Settings struct {
 	RetentionDays      int        `json:"activity_retention_days"`
+	RetentionMaxGB     int        `json:"activity_retention_max_gb"`
+	LogSizeBytes       int64      `json:"activity_log_size_bytes"`
 	LastCleanupAt      *time.Time `json:"activity_last_cleanup_at"`
 	LastCleanupRemoved int        `json:"activity_last_cleanup_removed"`
 	NextCleanupAt      *time.Time `json:"activity_next_cleanup_at"`
@@ -33,6 +51,14 @@ func DefaultSettings() Settings {
 	return Settings{RetentionDays: DefaultRetentionDays}
 }
 
+func (s Settings) HasRetentionLimit() bool {
+	return s.RetentionDays != ForeverRetentionDays || s.RetentionMaxGB != NoSizeLimitGB
+}
+
+func (s Settings) MaxLogSizeBytes() int64 {
+	return int64(s.RetentionMaxGB) * BytesPerGB
+}
+
 func (s Settings) CleanupDue(now time.Time) bool {
 	todayRun := time.Date(now.Year(), now.Month(), now.Day(), CleanupHour, 0, 0, 0, now.Location())
 	if now.Before(todayRun) {
@@ -43,7 +69,7 @@ func (s Settings) CleanupDue(now time.Time) bool {
 
 func (s Settings) WithNextCleanup(now time.Time) Settings {
 	s.NextCleanupAt = nil
-	if s.RetentionDays == ForeverRetentionDays {
+	if !s.HasRetentionLimit() {
 		return s
 	}
 	next := time.Date(now.Year(), now.Month(), now.Day(), CleanupHour, 0, 0, 0, now.Location())
@@ -62,6 +88,10 @@ func ClampCutoff(now time.Time, cutoff time.Time) time.Time {
 	return cutoff
 }
 
+func GuardCutoff(now time.Time) time.Time {
+	return ClampCutoff(now, now)
+}
+
 func RetentionCutoff(now time.Time, retentionDays int) time.Time {
 	return ClampCutoff(now, now.Add(-time.Duration(retentionDays)*24*time.Hour))
 }

internal/infrastructure/persistence/mappers/auditMapper.go

@@ -83,6 +83,7 @@ func encodeJSON(value any, empty string) (string, error) {
 func AuditSettingsToDomain(m models.AuditSettings) audit.Settings {
 	return audit.Settings{
 		RetentionDays:      m.RetentionDays,
+		RetentionMaxGB:     m.RetentionMaxGB,
 		LastCleanupAt:      m.LastCleanupAt,
 		LastCleanupRemoved: m.LastCleanupRemoved,
 	}
@@ -91,6 +92,7 @@ func AuditSettingsToDomain(m models.AuditSettings) audit.Settings {
 func AuditSettingsToModel(s audit.Settings) models.AuditSettings {
 	return models.AuditSettings{
 		RetentionDays:      s.RetentionDays,
+		RetentionMaxGB:     s.RetentionMaxGB,
 		LastCleanupAt:      s.LastCleanupAt,
 		LastCleanupRemoved: s.LastCleanupRemoved,
 	}

internal/infrastructure/persistence/models/auditSettingsModel.go

@@ -5,6 +5,7 @@ import "time"
 type AuditSettings struct {
 	ID                 uint32 `gorm:"primaryKey"`
 	RetentionDays      int    `gorm:"not null"`
+	RetentionMaxGB     int    `gorm:"not null;default:0"`
 	LastCleanupAt      *time.Time
 	LastCleanupRemoved int `gorm:"not null"`
 }

internal/infrastructure/persistence/repositories/auditEventRepository.go

@@ -4,6 +4,7 @@ import (
 	"RenewCMS/internal/domain/audit"
 	"RenewCMS/internal/infrastructure/persistence/mappers"
 	entity "RenewCMS/internal/infrastructure/persistence/models"
+	"slices"
 	"strings"
 	"time"
 
@@ -186,3 +187,49 @@ func (a *AuditEventRepository) DeleteOlderThan(cutoff time.Time) (int, error) {
 	result := a.db.Where("created_at < ?", cutoff.UTC()).Delete(&entity.AuditEvent{})
 	return int(result.RowsAffected), result.Error
 }
+
+const (
+	deleteChunkSize = 500
+	payloadSizeSQL  = `COALESCE(SUM(
+		OCTET_LENGTH(action) + OCTET_LENGTH(actor_name) + OCTET_LENGTH(target_type) +
+		OCTET_LENGTH(target_id) + OCTET_LENGTH(target_name) + COALESCE(OCTET_LENGTH(failure_reason), 0) +
+		COALESCE(OCTET_LENGTH(ip), 0) + COALESCE(OCTET_LENGTH(user_agent), 0) +
+		OCTET_LENGTH(params) + OCTET_LENGTH(changes)), 0) AS payload, COUNT(*) AS total`
+)
+
+type payloadRow struct {
+	Payload int64
+	Total   int64
+}
+
+func (a *AuditEventRepository) SizeBytes() (int64, error) {
+	var row payloadRow
+	if err := a.db.Model(&entity.AuditEvent{}).Select(payloadSizeSQL).Scan(&row).Error; err != nil {
+		return 0, err
+	}
+	return row.Payload + row.Total*audit.RowOverheadBytes, nil
+}
+
+func (a *AuditEventRepository) DeleteOldest(limit int, before time.Time) (int, error) {
+	if limit < 1 {
+		return 0, nil
+	}
+	var ids []uint64
+	err := a.db.Model(&entity.AuditEvent{}).
+		Where("created_at < ?", before.UTC()).
+		Order("created_at ASC, id ASC").
+		Limit(limit).
+		Pluck("id", &ids).Error
+	if err != nil {
+		return 0, err
+	}
+	removed := 0
+	for chunk := range slices.Chunk(ids, deleteChunkSize) {
+		result := a.db.Where("id IN ?", chunk).Delete(&entity.AuditEvent{})
+		if result.Error != nil {
+			return removed, result.Error
+		}
+		removed += int(result.RowsAffected)
+	}
+	return removed, nil
+}

internal/infrastructure/persistence/repositories/auditSettingsRepository.go

@@ -34,9 +34,10 @@ func (a *AuditSettingsRepository) Get() (audit.Settings, error) {
 	return mappers.AuditSettingsToDomain(model), nil
 }
 
-func (a *AuditSettingsRepository) SaveRetention(days int) error {
-	return a.update(map[string]any{"retention_days": days}, func(settings *audit.Settings) {
+func (a *AuditSettingsRepository) SaveRetention(days int, maxGB int) error {
+	return a.update(map[string]any{"retention_days": days, "retention_max_gb": maxGB}, func(settings *audit.Settings) {
 		settings.RetentionDays = days
+		settings.RetentionMaxGB = maxGB
 	})
 }
 

internal/infrastructure/useCases/ActivityRetention.go

@@ -8,11 +8,12 @@ import (
 
 type GetActivityRetentionUseCase struct {
 	settings audit.SettingsRepository
+	events   audit.Repository
 	now      func() time.Time
 }
 
-func NewGetActivityRetentionUseCase(settings audit.SettingsRepository, now func() time.Time) *GetActivityRetentionUseCase {
-	return &GetActivityRetentionUseCase{settings, now}
+func NewGetActivityRetentionUseCase(settings audit.SettingsRepository, events audit.Repository, now func() time.Time) *GetActivityRetentionUseCase {
+	return &GetActivityRetentionUseCase{settings, events, now}
 }
 
 func (g *GetActivityRetentionUseCase) GetActivityRetention() (audit.Settings, error) {
@@ -20,38 +21,80 @@ func (g *GetActivityRetentionUseCase) GetActivityRetention() (audit.Settings, er
 	if err != nil {
 		return audit.Settings{}, err
 	}
+	current.LogSizeBytes, err = g.events.SizeBytes()
+	if err != nil {
+		return audit.Settings{}, err
+	}
 	return current.WithNextCleanup(g.now()), nil
 }
 
+type UpdateActivityRetentionCommand struct {
+	Days  *int
+	MaxGB *int
+}
+
 type UpdateActivityRetentionUseCase struct {
 	settings audit.SettingsRepository
+	events   audit.Repository
 	recorder audit.Recorder
 	now      func() time.Time
 }
 
-func NewUpdateActivityRetentionUseCase(settings audit.SettingsRepository, recorder audit.Recorder, now func() time.Time) *UpdateActivityRetentionUseCase {
-	return &UpdateActivityRetentionUseCase{settings, recorder, now}
+func NewUpdateActivityRetentionUseCase(settings audit.SettingsRepository, events audit.Repository, recorder audit.Recorder, now func() time.Time) *UpdateActivityRetentionUseCase {
+	return &UpdateActivityRetentionUseCase{settings, events, recorder, now}
 }
 
-func (u *UpdateActivityRetentionUseCase) UpdateActivityRetention(ctx context.Context, days int) (audit.Settings, error) {
-	if err := audit.ValidateRetentionDays(days); err != nil {
-		return audit.Settings{}, err
+func (u *UpdateActivityRetentionUseCase) UpdateActivityRetention(ctx context.Context, command UpdateActivityRetentionCommand) (audit.Settings, error) {
+	if command.Days == nil && command.MaxGB == nil {
+		return audit.Settings{}, audit.ErrInvalidRetention
+	}
+	if command.Days != nil {
+		if err := audit.ValidateRetentionDays(*command.Days); err != nil {
+			return audit.Settings{}, err
+		}
+	}
+	if command.MaxGB != nil {
+		if err := audit.ValidateRetentionMaxGB(*command.MaxGB); err != nil {
+			return audit.Settings{}, err
+		}
 	}
 	current, err := u.settings.Get()
 	if err != nil {
 		return audit.Settings{}, err
 	}
-	if current.RetentionDays == days {
+	size, err := u.events.SizeBytes()
+	if err != nil {
+		return audit.Settings{}, err
+	}
+	current.LogSizeBytes = size
+
+	updated := current
+	if command.Days != nil {
+		updated.RetentionDays = *command.Days
+	}
+	if command.MaxGB != nil {
+		updated.RetentionMaxGB = *command.MaxGB
+	}
+	daysChanged := updated.RetentionDays != current.RetentionDays
+	sizeChanged := updated.RetentionMaxGB != current.RetentionMaxGB
+	if !daysChanged && !sizeChanged {
 		return current.WithNextCleanup(u.now()), nil
 	}
-	if err := u.settings.SaveRetention(days); err != nil {
+	if err := u.settings.SaveRetention(updated.RetentionDays, updated.RetentionMaxGB); err != nil {
 		return audit.Settings{}, err
 	}
 
 	event := audit.NewEvent(audit.ActionRetentionChanged, audit.CategorySystem)
-	event.Params = map[string]any{audit.ParamFrom: current.RetentionDays, audit.ParamTo: days}
+	event.Params = map[string]any{}
+	if daysChanged {
+		event.Params[audit.ParamFrom] = current.RetentionDays
+		event.Params[audit.ParamTo] = updated.RetentionDays
+	}
+	if sizeChanged {
+		event.Params[audit.ParamMaxGBFrom] = current.RetentionMaxGB
+		event.Params[audit.ParamMaxGBTo] = updated.RetentionMaxGB
+	}
 	u.recorder.Record(ctx, event)
 
-	current.RetentionDays = days
-	return current.WithNextCleanup(u.now()), nil
+	return updated.WithNextCleanup(u.now()), nil
 }

internal/infrastructure/useCases/CleanupActivity.go

@@ -6,14 +6,22 @@ import (
 	"time"
 )
 
+const defaultCleanupBatchSize = 1000
+
 type CleanupActivityUseCase struct {
-	events   audit.Repository
-	settings audit.SettingsRepository
-	recorder audit.Recorder
+	events    audit.Repository
+	settings  audit.SettingsRepository
+	recorder  audit.Recorder
+	batchSize int
 }
 
 func NewCleanupActivityUseCase(events audit.Repository, settings audit.SettingsRepository, recorder audit.Recorder) *CleanupActivityUseCase {
-	return &CleanupActivityUseCase{events, settings, recorder}
+	return &CleanupActivityUseCase{events, settings, recorder, defaultCleanupBatchSize}
+}
+
+func (c *CleanupActivityUseCase) WithBatchSize(batchSize int) *CleanupActivityUseCase {
+	c.batchSize = batchSize
+	return c
 }
 
 func (c *CleanupActivityUseCase) Tick(now time.Time) (audit.CleanupResult, bool, error) {
@@ -21,13 +29,23 @@ func (c *CleanupActivityUseCase) Tick(now time.Time) (audit.CleanupResult, bool,
 	if err != nil {
 		return audit.CleanupResult{}, false, err
 	}
-	if settings.RetentionDays == audit.ForeverRetentionDays || !settings.CleanupDue(now) {
+	if !settings.HasRetentionLimit() || !settings.CleanupDue(now) {
 		return audit.CleanupResult{}, false, nil
 	}
 
-	removed, err := c.events.DeleteOlderThan(audit.RetentionCutoff(now, settings.RetentionDays))
-	if err != nil {
-		return audit.CleanupResult{}, false, err
+	removed := 0
+	if settings.RetentionDays != audit.ForeverRetentionDays {
+		removed, err = c.events.DeleteOlderThan(audit.RetentionCutoff(now, settings.RetentionDays))
+		if err != nil {
+			return audit.CleanupResult{}, false, err
+		}
+	}
+	if settings.RetentionMaxGB != audit.NoSizeLimitGB {
+		trimmed, err := c.trimToSize(now, settings.MaxLogSizeBytes())
+		if err != nil {
+			return audit.CleanupResult{}, false, err
+		}
+		removed += trimmed
 	}
 	if err := c.settings.SaveLastCleanup(now, removed); err != nil {
 		return audit.CleanupResult{}, false, err
@@ -38,6 +56,28 @@ func (c *CleanupActivityUseCase) Tick(now time.Time) (audit.CleanupResult, bool,
 	return audit.CleanupResult{Removed: removed}, true, nil
 }
 
+func (c *CleanupActivityUseCase) trimToSize(now time.Time, limitBytes int64) (int, error) {
+	guard := audit.GuardCutoff(now)
+	removed := 0
+	for {
+		size, err := c.events.SizeBytes()
+		if err != nil {
+			return removed, err
+		}
+		if size <= limitBytes {
+			return removed, nil
+		}
+		batch, err := c.events.DeleteOldest(c.batchSize, guard)
+		if err != nil {
+			return removed, err
+		}
+		if batch == 0 {
+			return removed, nil
+		}
+		removed += batch
+	}
+}
+
 func (c *CleanupActivityUseCase) recordCleanup(removed int) {
 	event := audit.NewEvent(audit.ActionCleanup, audit.CategorySystem)
 	event.Actor = audit.SystemActor()