RenewCMS/test/article_localization_storage_persistence_test.go

package test

import (
	domainArticle "RenewCMS/internal/domain/article"
	authordomain "RenewCMS/internal/domain/author"
	"RenewCMS/internal/domain/language"
	"RenewCMS/internal/infrastructure/persistence/database"
	entity "RenewCMS/internal/infrastructure/persistence/models"
	persistence "RenewCMS/internal/infrastructure/persistence/repositories"
	"RenewCMS/internal/infrastructure/useCases"
	"context"
	"errors"
	"fmt"
	"slices"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"gorm.io/gorm"
)

const (
	storiesBackfillMarker     = "article_stories"
	languageSuggestionsMarker = "site_language_suggestions"
	storyLanguageIndexName    = "idx_articles_story_lang"
	siteSlugIndexName         = "idx_articles_site_slug"
	retiredIDsMarker          = "article_retired_ids"
	slugPerLanguageMarker     = "article_slug_index_per_language"
	injectedBackfillFailure   = "injected backfill failure"
	raceCreators              = 8
	raceRetries               = 60
	raceRetriesPerWriter      = 4
	raceRetryBudget           = raceRetriesPerWriter * (raceCreators + 1)
	storyRollbackCallback     = "test:fail-credit-write"
	storyRollbackFailure      = "injected credit failure"
	racePauseBetweenRetries   = 10 * time.Millisecond
)

func localeOf(value string) *string {
	return &value
}

func storageDefaultSite(t *testing.T, db *gorm.DB) entity.Site {
	t.Helper()
	var site entity.Site
	if err := db.Where("is_default = ?", true).First(&site).Error; err != nil {
		t.Fatalf("default site: %v", err)
	}
	return site
}

func storageSetLocale(t *testing.T, db *gorm.DB, siteID uint32, locale *string) {
	t.Helper()
	if err := db.Model(&entity.Site{}).Where("id = ?", siteID).UpdateColumn("locale", locale).Error; err != nil {
		t.Fatalf("set locale: %v", err)
	}
}

func regressToPreLocalizationSchema(t *testing.T, db *gorm.DB) {
	t.Helper()
	migrator := db.Migrator()
	for _, name := range []string{storyLanguageIndexName, siteSlugIndexName} {
		if migrator.HasIndex(&entity.Article{}, name) {
			if err := migrator.DropIndex(&entity.Article{}, name); err != nil {
				t.Fatalf("drop %s: %v", name, err)
			}
		}
	}
	if err := db.Exec("CREATE UNIQUE INDEX idx_articles_site_slug ON articles (site_id, slug)").Error; err != nil {
		t.Fatalf("recreate the pre-change slug index: %v", err)
	}
	statements := []string{
		"UPDATE articles SET story_id = 0, lang = ''",
		"DELETE FROM stories",
		"DELETE FROM site_languages",
		"DELETE FROM migration_markers WHERE name IN ('" + storiesBackfillMarker + "', '" + languageSuggestionsMarker + "', '" + slugPerLanguageMarker + "', '" + retiredIDsMarker + "')",
	}
	for _, statement := range statements {
		if err := db.Exec(statement).Error; err != nil {
			t.Fatalf("%s: %v", statement, err)
		}
	}
}

func seedPreLocalizationArticles(t *testing.T, db *gorm.DB, siteID uint32, softDeleted ...uint32) {
	t.Helper()
	for id := uint32(1); id <= 3; id++ {
		row := entity.Article{StoryID: fixtureStory(), Title: fmt.Sprintf("Article %d", id), Body: fmt.Sprintf("Body %d", id), SiteID: siteID, Slug: fmt.Sprintf("article-%d", id), IsOnline: id == 3}
		if err := db.Create(&row).Error; err != nil {
			t.Fatalf("seed article %d: %v", id, err)
		}
	}
	for _, id := range softDeleted {
		if err := db.Model(&entity.Article{}).Where("id = ?", id).UpdateColumn("slug", domainArticle.Tombstone(id)).Error; err != nil {
			t.Fatal(err)
		}
		if err := db.Delete(&entity.Article{}, id).Error; err != nil {
			t.Fatal(err)
		}
	}
	regressToPreLocalizationSchema(t, db)
}

func legacyDatabase(t *testing.T, open func(*testing.T) *gorm.DB, locale *string, softDeleted ...uint32) (*gorm.DB, entity.Site) {
	t.Helper()
	db := open(t)
	site := storageDefaultSite(t, db)
	storageSetLocale(t, db, site.ID, locale)
	seedPreLocalizationArticles(t, db, site.ID, softDeleted...)
	return db, site
}

func migrateStorage(t *testing.T, db *gorm.DB) {
	t.Helper()
	if err := database.Migrate(db, migrationPublicURL); err != nil {
		t.Fatalf("migrate: %v", err)
	}
}

func storedArticles(t *testing.T, db *gorm.DB) []entity.Article {
	t.Helper()
	var rows []entity.Article
	if err := db.Unscoped().Order("id ASC").Find(&rows).Error; err != nil {
		t.Fatal(err)
	}
	return rows
}

func storedArticle(t *testing.T, db *gorm.DB, id uint32) entity.Article {
	t.Helper()
	var row entity.Article
	if err := db.Unscoped().First(&row, id).Error; err != nil {
		t.Fatal(err)
	}
	return row
}

func storageSnapshot(t *testing.T, db *gorm.DB) string {
	t.Helper()
	var stories []entity.Story
	var languages []entity.SiteLanguage
	var markers []entity.MigrationMarker
	for _, target := range []any{&stories, &languages, &markers} {
		if err := db.Order("1").Find(target).Error; err != nil {
			t.Fatal(err)
		}
	}
	indexes, err := db.Migrator().GetIndexes(&entity.Article{})
	if err != nil {
		t.Fatal(err)
	}
	names := []string{}
	for _, index := range indexes {
		unique, _ := index.Unique()
		names = append(names, fmt.Sprintf("%s:%v:%v", index.Name(), unique, index.Columns()))
	}
	slices.Sort(names)
	return fmt.Sprintf("%v|%v|%v|%v|%v", storedArticles(t, db), stories, languages, markers, names)
}

func markerRecorded(t *testing.T, db *gorm.DB, name string) bool {
	t.Helper()
	var count int64
	if err := db.Model(&entity.MigrationMarker{}).Where("name = ?", name).Count(&count).Error; err != nil {
		t.Fatal(err)
	}
	return count > 0
}

func createThroughUseCase(db *gorm.DB) *useCases.CreateArticleUseCase {
	return useCases.NewCreateArticleUseCase(persistence.NewTransactor(db), &fakeBodyRenderer{}, discardRecorder, persistence.NewSiteRepository(db), persistence.NewAuthorRepository(db))
}

func createTitled(db *gorm.DB, title string) (domainArticle.Article, error) {
	return createThroughUseCase(db).CreateArticle(context.Background(), useCases.CreateArticleCommand{Title: title, Caller: ownerCaller(1)})
}

func newStorageArticle(siteID uint32, lang string, slug string, storyID uint32) domainArticle.Article {
	return domainArticle.Article{Title: "Article " + slug, Body: "Body", SiteID: siteID, Lang: lang, Slug: slug, StoryID: storyID, SEODismissed: []string{}}
}

func runArticleLocalizationStorage(t *testing.T, open func(*testing.T) *gorm.DB) {
	t.Run("Migration of an existing site and its articles", func(t *testing.T) {
		db, site := legacyDatabase(t, open, localeOf("fr-CA"))
		before := storedArticles(t, db)

		migrateStorage(t, db)

		after := storedArticles(t, db)
		seen := map[uint32]bool{}
		for index, row := range after {
			if row.ID != before[index].ID || row.Title != before[index].Title || row.Body != before[index].Body || row.Slug != before[index].Slug || row.IsOnline != before[index].IsOnline || !row.CreatedAt.Equal(before[index].CreatedAt) {
				t.Errorf("article %d changed: %+v", row.ID, row)
			}
			if row.Lang != "" || row.StoryID == 0 || seen[row.StoryID] || row.ContentRevision != 1 {
				t.Errorf("article %d must have lang '', its own story and revision 1: %+v", row.ID, row)
			}
			seen[row.StoryID] = true
		}
		var languages []entity.SiteLanguage
		if err := db.Where("site_id = ?", site.ID).Find(&languages).Error; err != nil {
			t.Fatal(err)
		}
		if len(languages) != 1 || languages[0].Lang != "fr-CA" || !languages[0].IsDefault || !languages[0].Enabled || languages[0].Confirmed {
			t.Errorf("expected one unconfirmed default fr-CA row, got %+v", languages)
		}
		if !markerRecorded(t, db, storiesBackfillMarker) {
			t.Error("the story backfill marker must be recorded")
		}
	})

	t.Run("Bare language fallback is stored as the suggestion", func(t *testing.T) {
		db, site := legacyDatabase(t, open, localeOf("fr-BE"))

		migrateStorage(t, db)

		languages, err := persistence.NewSiteLanguageRepository(db).ListBySite(site.ID)
		if err != nil || len(languages) != 1 || languages[0].Tag != "fr" || languages[0].Confirmed {
			t.Errorf("expected one unconfirmed fr row, got %+v %v", languages, err)
		}
	})

	t.Run("Site without a usable locale gets no suggestion row", func(t *testing.T) {
		for _, locale := range []*string{nil, localeOf("xx-YY")} {
			db, site := legacyDatabase(t, open, locale)

			migrateStorage(t, db)

			var count int64
			if err := db.Model(&entity.SiteLanguage{}).Where("site_id = ?", site.ID).Count(&count).Error; err != nil || count != 0 {
				t.Errorf("expected no row, got %d %v", count, err)
			}
		}
	})

	t.Run("Migration runs twice", func(t *testing.T) {
		db, _ := legacyDatabase(t, open, localeOf("fr-CA"))
		migrateStorage(t, db)
		first := storageSnapshot(t, db)

		migrateStorage(t, db)

		if second := storageSnapshot(t, db); second != first {
			t.Errorf("the second migration changed the database:\n%s\n%s", first, second)
		}
	})

	t.Run("Migration is atomic", func(t *testing.T) {
		db, _ := legacyDatabase(t, open, localeOf("fr-CA"))
		updates := 0
		const callbackName = "test:fail-second-story-assignment"
		err := db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) {
			if tx.Statement.Table != "articles" {
				return
			}
			updates++
			if updates == 2 {
				_ = tx.AddError(errors.New(injectedBackfillFailure))
			}
		})
		if err != nil {
			t.Fatal(err)
		}

		failure := database.Migrate(db, migrationPublicURL)

		if failure == nil {
			t.Fatal("the injected failure must abort the migration")
		}
		for _, row := range storedArticles(t, db) {
			if row.StoryID != 0 {
				t.Errorf("article %d kept a story_id after the rollback", row.ID)
			}
		}
		var stories int64
		if err := db.Model(&entity.Story{}).Count(&stories).Error; err != nil || stories != 0 {
			t.Errorf("the rolled back stories must not exist: %d %v", stories, err)
		}
		if markerRecorded(t, db, storiesBackfillMarker) {
			t.Error("the marker must not be recorded")
		}
		if err := db.Callback().Update().Remove(callbackName); err != nil {
			t.Fatal(err)
		}

		migrateStorage(t, db)

		for _, row := range storedArticles(t, db) {
			if row.StoryID == 0 {
				t.Errorf("article %d must have a story after the retry", row.ID)
			}
		}
		if !markerRecorded(t, db, storiesBackfillMarker) {
			t.Error("the marker must be recorded after the retry")
		}
	})

	t.Run("Soft-deleted articles are migrated", func(t *testing.T) {
		db, _ := legacyDatabase(t, open, localeOf("fr-CA"), 2)

		migrateStorage(t, db)

		for _, row := range storedArticles(t, db) {
			if row.StoryID == 0 {
				t.Errorf("article %d has no story", row.ID)
			}
			if row.ID == 2 && (!row.DeletedAt.Valid || row.Slug != domainArticle.Tombstone(2)) {
				t.Errorf("the tombstone must be unchanged: %+v", row)
			}
		}
	})

	t.Run("Slug is unique per site and language", func(t *testing.T) {
		db := open(t)
		articles := persistence.NewArticleRepository(db)
		site := storageDefaultSite(t, db)
		other := entity.Site{Name: "Other", Key: "other", BaseURL: "https://other.example.com", DefaultRobots: "index"}
		if err := db.Create(&other).Error; err != nil {
			t.Fatal(err)
		}
		if _, err := articles.Create(newStorageArticle(site.ID, "fr", "bonjour", 0)); err != nil {
			t.Fatalf("first create: %v", err)
		}

		_, duplicate := articles.Create(newStorageArticle(site.ID, "fr", "bonjour", 0))
		_, otherLanguage := articles.Create(newStorageArticle(site.ID, "en", "bonjour", 0))
		_, otherSite := articles.Create(newStorageArticle(other.ID, "fr", "bonjour", 0))

		if !errors.Is(duplicate, domainArticle.ErrSlugTaken) {
			t.Errorf("expected ErrSlugTaken, got %v", duplicate)
		}
		if otherLanguage != nil || otherSite != nil {
			t.Errorf("language and site variants must be stored: %v %v", otherLanguage, otherSite)
		}
	})

	t.Run("Existing unassigned articles keep the current per-site slug rule", func(t *testing.T) {
		db := open(t)
		articles := persistence.NewArticleRepository(db)
		site := storageDefaultSite(t, db)
		if _, err := articles.Create(newStorageArticle(site.ID, "", "same", 0)); err != nil {
			t.Fatal(err)
		}
		second, err := articles.Create(newStorageArticle(site.ID, "", "other", 0))
		if err != nil {
			t.Fatal(err)
		}

		_, updateErr := articles.Update(second.ID, domainArticle.Update{Slug: localeOf("same")})

		if !errors.Is(updateErr, domainArticle.ErrSlugTaken) {
			t.Errorf("expected ErrSlugTaken, got %v", updateErr)
		}
	})

	t.Run("One version per language in a story", func(t *testing.T) {
		db := open(t)
		articles := persistence.NewArticleRepository(db)
		site := storageDefaultSite(t, db)
		story := entity.Story{}
		if err := db.Create(&story).Error; err != nil {
			t.Fatal(err)
		}
		if _, err := articles.Create(newStorageArticle(site.ID, "fr", "un", story.ID)); err != nil {
			t.Fatal(err)
		}

		_, duplicate := articles.Create(newStorageArticle(site.ID, "fr", "deux", story.ID))
		_, otherLanguage := articles.Create(newStorageArticle(site.ID, "en", "three", story.ID))

		if duplicate == nil {
			t.Error("a second fr version in the same story must be rejected")
		}
		if otherLanguage != nil {
			t.Errorf("an en version must be stored: %v", otherLanguage)
		}
	})

	t.Run("Unassigned articles never collide on language", func(t *testing.T) {
		db := open(t)
		articles := persistence.NewArticleRepository(db)
		site := storageDefaultSite(t, db)
		stories := map[uint32]bool{}
		for index := 0; index < 3; index++ {
			created, err := articles.Create(newStorageArticle(site.ID, "", fmt.Sprintf("slug-%d", index), 0))
			if err != nil || created.StoryID == 0 || stories[created.StoryID] || created.Lang != "" {
				t.Fatalf("article %d: %+v %v", index, created, err)
			}
			stories[created.StoryID] = true
		}
	})

	t.Run("Creating an article before the default is confirmed", func(t *testing.T) {
		db, _ := legacyDatabase(t, open, localeOf("fr-CA"))
		migrateStorage(t, db)

		created, err := createTitled(db, "Bonjour le monde")
		stored := storedArticle(t, db, created.ID)

		if err != nil || stored.Lang != "" || stored.StoryID == 0 || stored.ContentRevision != 1 || created.ContentRevision != 1 {
			t.Errorf("expected lang '' with a new story and revision 1, got %+v %v", stored, err)
		}
	})

	t.Run("Creating an article after the default is confirmed", func(t *testing.T) {
		db, site := legacyDatabase(t, open, localeOf("fr-CA"))
		migrateStorage(t, db)
		if err := persistence.NewSiteLanguageRepository(db).ConfirmDefault(site.ID); err != nil {
			t.Fatal(err)
		}

		created, err := createTitled(db, "Bonjour le monde")
		stored := storedArticle(t, db, created.ID)

		if err != nil || stored.Lang != "fr-CA" {
			t.Fatalf("expected lang fr-CA, got %+v %v", stored, err)
		}
		for _, row := range storedArticles(t, db) {
			if row.ID != stored.ID && row.StoryID == stored.StoryID {
				t.Errorf("the new story must be distinct from article %d", row.ID)
			}
		}
	})

	t.Run("Create racing a confirmation never strands an article", func(t *testing.T) {
		db, site := legacyDatabase(t, open, localeOf("fr-CA"))
		migrateStorage(t, db)
		languages := persistence.NewSiteLanguageRepository(db)
		var group sync.WaitGroup
		var retries atomic.Int64
		start := make(chan struct{})
		retry := func(work func() error) error {
			var err error
			for attempt := 0; attempt < raceRetries; attempt++ {
				if err = work(); err == nil {
					return nil
				}
				retries.Add(1)
				time.Sleep(racePauseBetweenRetries)
			}
			return err
		}
		failures := make(chan error, raceCreators+1)
		for index := 0; index < raceCreators; index++ {
			group.Add(1)
			go func() {
				defer group.Done()
				<-start
				failures <- retry(func() error {
					_, err := createTitled(db, fmt.Sprintf("Race article %d", index))
					return err
				})
			}()
		}
		group.Add(1)
		go func() {
			defer group.Done()
			<-start
			failures <- retry(func() error { return languages.ConfirmDefault(site.ID) })
		}()
		close(start)
		group.Wait()
		close(failures)

		for err := range failures {
			if err != nil {
				t.Fatalf("a racing write never succeeded: %v", err)
			}
		}
		t.Logf("busy retries: %d of a budget of %d", retries.Load(), raceRetryBudget)
		if retries.Load() > raceRetryBudget {
			t.Errorf("%d retries exceed the budget of %d: the lock is not serializing the writers", retries.Load(), raceRetryBudget)
		}
		rows := storedArticles(t, db)
		if len(rows) != 3+raceCreators {
			t.Fatalf("expected %d articles, got %d", 3+raceCreators, len(rows))
		}
		for _, row := range rows {
			if row.Lang != "fr-CA" {
				t.Errorf("article %d ended with lang %q", row.ID, row.Lang)
			}
		}
	})

	t.Run("A failed create leaves no orphan story", func(t *testing.T) {
		db := open(t)
		site := storageDefaultSite(t, db)
		var before int64
		if err := db.Model(&entity.Story{}).Count(&before).Error; err != nil {
			t.Fatal(err)
		}
		err := db.Callback().Create().Before("gorm:create").Register(storyRollbackCallback, func(tx *gorm.DB) {
			if tx.Statement.Table == "article_authors" {
				_ = tx.AddError(errors.New(storyRollbackFailure))
			}
		})
		if err != nil {
			t.Fatal(err)
		}
		defer func() {
			if err := db.Callback().Create().Remove(storyRollbackCallback); err != nil {
				t.Fatal(err)
			}
		}()
		draft := newStorageArticle(site.ID, "", "orphan", 0)
		draft.Authors = []authordomain.Byline{{ID: 1}}

		_, createErr := persistence.NewArticleRepository(db).Create(draft)

		var after int64
		if err := db.Model(&entity.Story{}).Count(&after).Error; err != nil {
			t.Fatal(err)
		}
		if createErr == nil || after != before {
			t.Errorf("the story must roll back with the failed create: error %v, stories %d then %d", createErr, before, after)
		}
	})

	t.Run("Migrating twice keeps the same slug in two languages of one site", func(t *testing.T) {
		db := open(t)
		site := storageDefaultSite(t, db)
		articles := persistence.NewArticleRepository(db)
		for _, lang := range []string{"fr", "en"} {
			if _, err := articles.Create(newStorageArticle(site.ID, lang, "bonjour", 0)); err != nil {
				t.Fatalf("create %s: %v", lang, err)
			}
		}

		migrateStorage(t, db)
		migrateStorage(t, db)

		for _, row := range storedArticles(t, db) {
			if row.Slug != "bonjour" {
				t.Errorf("article %d (%s) was renamed to %q", row.ID, row.Lang, row.Slug)
			}
		}
	})

	t.Run("Assigning a language is idempotent", func(t *testing.T) {
		db, site := legacyDatabase(t, open, localeOf("fr-CA"))
		migrateStorage(t, db)
		languages := persistence.NewSiteLanguageRepository(db)
		if err := languages.ConfirmDefault(site.ID); err != nil {
			t.Fatal(err)
		}
		before := storageSnapshot(t, db)

		err := languages.ConfirmDefault(site.ID)

		if err != nil || storageSnapshot(t, db) != before {
			t.Errorf("the second assignment must change nothing: %v", err)
		}
		for _, row := range storedArticles(t, db) {
			if row.Lang != "fr-CA" {
				t.Errorf("article %d has lang %q", row.ID, row.Lang)
			}
		}
	})

	t.Run("Confirming a site without a default language is refused", func(t *testing.T) {
		db := open(t)
		site := storageDefaultSite(t, db)

		err := persistence.NewSiteLanguageRepository(db).ConfirmDefault(site.ID)

		if !errors.Is(err, language.ErrNoDefaultLanguage) {
			t.Errorf("expected ErrNoDefaultLanguage, got %v", err)
		}
	})

	t.Run("Unique indexes replace the per-site slug index", func(t *testing.T) {
		db := open(t)
		for _, name := range []string{storyLanguageIndexName, siteSlugIndexName} {
			if !db.Migrator().HasIndex(&entity.Article{}, name) {
				t.Errorf("index %s is missing", name)
			}
		}
		indexes, err := db.Migrator().GetIndexes(&entity.Article{})
		if err != nil {
			t.Fatal(err)
		}
		for _, index := range indexes {
			if index.Name() == siteSlugIndexName && !slices.Equal(slices.Sorted(slices.Values(index.Columns())), []string{"lang", "site_id", "slug"}) {
				t.Errorf("the slug index must cover (site_id, lang, slug), got %v", index.Columns())
			}
		}
	})

	t.Run("The per-language slug index is recorded and survives a second migration", func(t *testing.T) {
		db := open(t)
		if !markerRecorded(t, db, slugPerLanguageMarker) {
			t.Error("the per-language slug index marker must be recorded")
		}

		migrateStorage(t, db)

		if !db.Migrator().HasIndex(&entity.Article{}, siteSlugIndexName) || !markerRecorded(t, db, slugPerLanguageMarker) {
			t.Error("a second migration must keep the index and its marker")
		}
	})

	t.Run("A missing per-language slug index is recreated once the marker exists", func(t *testing.T) {
		db := open(t)
		if err := db.Migrator().DropIndex(&entity.Article{}, siteSlugIndexName); err != nil {
			t.Fatal(err)
		}

		migrateStorage(t, db)

		if !db.Migrator().HasIndex(&entity.Article{}, siteSlugIndexName) {
			t.Error("the slug index must be recreated")
		}
	})
}

func TestArticleLocalizationStorageSQLite(t *testing.T) {
	runArticleLocalizationStorage(t, openContractSQLite)
}

func TestArticleLocalizationStorageMySQL(t *testing.T) {
	runArticleLocalizationStorage(t, openContractMySQL)
}

func TestArticleLocalizationStoragePostgres(t *testing.T) {
	runArticleLocalizationStorage(t, openContractPostgres)
}