package test import ( domainArticle "RenewCMS/internal/domain/article" authordomain "RenewCMS/internal/domain/author" "RenewCMS/internal/domain/localization" "RenewCMS/internal/infrastructure/persistence/database" entity "RenewCMS/internal/infrastructure/persistence/models" persistence "RenewCMS/internal/infrastructure/persistence/repositories" "RenewCMS/internal/infrastructure/useCases" "context" "errors" "fmt" "slices" "sync" "sync/atomic" "testing" "time" "gorm.io/gorm" ) const ( storiesBackfillMarker = "article_stories" languageSuggestionsMarker = "site_language_suggestions" storyLanguageIndexName = "idx_articles_story_lang" siteSlugIndexName = "idx_articles_site_slug" retiredIDsMarker = "article_retired_ids" slugPerLanguageMarker = "article_slug_index_per_language" injectedBackfillFailure = "injected backfill failure" raceCreators = 8 raceRetries = 60 raceRetriesPerWriter = 4 raceRetryBudget = raceRetriesPerWriter * (raceCreators + 1) storyRollbackCallback = "test:fail-credit-write" storyRollbackFailure = "injected credit failure" racePauseBetweenRetries = 10 * time.Millisecond ) func localeOf(value string) *string { return &value } func storageDefaultSite(t *testing.T, db *gorm.DB) entity.Site { t.Helper() var site entity.Site if err := db.Where("is_default = ?", true).First(&site).Error; err != nil { t.Fatalf("default site: %v", err) } return site } func storageSetLocale(t *testing.T, db *gorm.DB, siteID uint32, locale *string) { t.Helper() if err := db.Model(&entity.Site{}).Where("id = ?", siteID).UpdateColumn("locale", locale).Error; err != nil { t.Fatalf("set locale: %v", err) } } func regressToPreLocalizationSchema(t *testing.T, db *gorm.DB) { t.Helper() migrator := db.Migrator() for _, name := range []string{storyLanguageIndexName, siteSlugIndexName} { if migrator.HasIndex(&entity.Article{}, name) { if err := migrator.DropIndex(&entity.Article{}, name); err != nil { t.Fatalf("drop %s: %v", name, err) } } } if err := db.Exec("CREATE UNIQUE INDEX idx_articles_site_slug ON articles (site_id, slug)").Error; err != nil { t.Fatalf("recreate the pre-change slug index: %v", err) } statements := []string{ "UPDATE articles SET story_id = 0, lang = ''", "DELETE FROM stories", "DELETE FROM site_languages", "DELETE FROM migration_markers WHERE name IN ('" + storiesBackfillMarker + "', '" + languageSuggestionsMarker + "', '" + slugPerLanguageMarker + "', '" + retiredIDsMarker + "')", } for _, statement := range statements { if err := db.Exec(statement).Error; err != nil { t.Fatalf("%s: %v", statement, err) } } } func seedPreLocalizationArticles(t *testing.T, db *gorm.DB, siteID uint32, softDeleted ...uint32) { t.Helper() for id := uint32(1); id <= 3; id++ { row := entity.Article{StoryID: fixtureStory(), Title: fmt.Sprintf("Article %d", id), Body: fmt.Sprintf("Body %d", id), SiteID: siteID, Slug: fmt.Sprintf("article-%d", id), IsOnline: id == 3} if err := db.Create(&row).Error; err != nil { t.Fatalf("seed article %d: %v", id, err) } } for _, id := range softDeleted { if err := db.Model(&entity.Article{}).Where("id = ?", id).UpdateColumn("slug", domainArticle.Tombstone(id)).Error; err != nil { t.Fatal(err) } if err := db.Delete(&entity.Article{}, id).Error; err != nil { t.Fatal(err) } } regressToPreLocalizationSchema(t, db) } func legacyDatabase(t *testing.T, open func(*testing.T) *gorm.DB, locale *string, softDeleted ...uint32) (*gorm.DB, entity.Site) { t.Helper() db := open(t) site := storageDefaultSite(t, db) storageSetLocale(t, db, site.ID, locale) seedPreLocalizationArticles(t, db, site.ID, softDeleted...) return db, site } func migrateStorage(t *testing.T, db *gorm.DB) { t.Helper() if err := database.Migrate(db, migrationPublicURL); err != nil { t.Fatalf("migrate: %v", err) } } func storedArticles(t *testing.T, db *gorm.DB) []entity.Article { t.Helper() var rows []entity.Article if err := db.Unscoped().Order("id ASC").Find(&rows).Error; err != nil { t.Fatal(err) } return rows } func storedArticle(t *testing.T, db *gorm.DB, id uint32) entity.Article { t.Helper() var row entity.Article if err := db.Unscoped().First(&row, id).Error; err != nil { t.Fatal(err) } return row } func storageSnapshot(t *testing.T, db *gorm.DB) string { t.Helper() var stories []entity.Story var languages []entity.SiteLanguage var markers []entity.MigrationMarker for _, target := range []any{&stories, &languages, &markers} { if err := db.Order("1").Find(target).Error; err != nil { t.Fatal(err) } } indexes, err := db.Migrator().GetIndexes(&entity.Article{}) if err != nil { t.Fatal(err) } names := []string{} for _, index := range indexes { unique, _ := index.Unique() names = append(names, fmt.Sprintf("%s:%v:%v", index.Name(), unique, index.Columns())) } slices.Sort(names) return fmt.Sprintf("%v|%v|%v|%v|%v", storedArticles(t, db), stories, languages, markers, names) } func markerRecorded(t *testing.T, db *gorm.DB, name string) bool { t.Helper() var count int64 if err := db.Model(&entity.MigrationMarker{}).Where("name = ?", name).Count(&count).Error; err != nil { t.Fatal(err) } return count > 0 } func createThroughUseCase(db *gorm.DB) *useCases.CreateArticleUseCase { return useCases.NewCreateArticleUseCase(persistence.NewTransactor(db), &fakeBodyRenderer{}, discardRecorder, persistence.NewSiteRepository(db), persistence.NewAuthorRepository(db)) } func createTitled(db *gorm.DB, title string) (domainArticle.Article, error) { return createThroughUseCase(db).CreateArticle(context.Background(), useCases.CreateArticleCommand{Title: title, Caller: ownerCaller(1)}) } func confirmDefaultLanguage(db *gorm.DB, siteID uint32) error { var found entity.Site if err := db.First(&found, siteID).Error; err != nil { return err } var seeded entity.SiteLanguage if err := db.Where("site_id = ? AND is_default = ?", siteID, true).First(&seeded).Error; err != nil { return err } setup := useCases.NewApplyLanguageSetupUseCase(persistence.NewTransactor(db), discardRecorder) _, err := setup.Apply(context.Background(), []localization.SetupChoice{{Key: found.Key, Lang: seeded.Lang}}) return err } func newStorageArticle(siteID uint32, lang string, slug string, storyID uint32) domainArticle.Article { return domainArticle.Article{Title: "Article " + slug, Body: "Body", SiteID: siteID, Lang: lang, Slug: slug, StoryID: storyID, SEODismissed: []string{}} } func runArticleLocalizationStorage(t *testing.T, open func(*testing.T) *gorm.DB) { t.Run("Migration of an existing site and its articles", func(t *testing.T) { db, site := legacyDatabase(t, open, localeOf("fr-CA")) before := storedArticles(t, db) migrateStorage(t, db) after := storedArticles(t, db) seen := map[uint32]bool{} for index, row := range after { if row.ID != before[index].ID || row.Title != before[index].Title || row.Body != before[index].Body || row.Slug != before[index].Slug || row.IsOnline != before[index].IsOnline || !row.CreatedAt.Equal(before[index].CreatedAt) { t.Errorf("article %d changed: %+v", row.ID, row) } if row.Lang != "" || row.StoryID == 0 || seen[row.StoryID] || row.ContentRevision != 1 { t.Errorf("article %d must have lang '', its own story and revision 1: %+v", row.ID, row) } seen[row.StoryID] = true } var languages []entity.SiteLanguage if err := db.Where("site_id = ?", site.ID).Find(&languages).Error; err != nil { t.Fatal(err) } if len(languages) != 1 || languages[0].Lang != "fr-CA" || !languages[0].IsDefault || !languages[0].Enabled || languages[0].Confirmed { t.Errorf("expected one unconfirmed default fr-CA row, got %+v", languages) } if !markerRecorded(t, db, storiesBackfillMarker) { t.Error("the story backfill marker must be recorded") } }) t.Run("Bare language fallback is stored as the suggestion", func(t *testing.T) { db, site := legacyDatabase(t, open, localeOf("fr-BE")) migrateStorage(t, db) languages, err := persistence.NewLocalizationRepository(db).Rows(site.ID) if err != nil || len(languages) != 1 || languages[0].Tag != "fr" || languages[0].Confirmed { t.Errorf("expected one unconfirmed fr row, got %+v %v", languages, err) } }) t.Run("Site without a usable locale gets no suggestion row", func(t *testing.T) { for _, locale := range []*string{nil, localeOf("xx-YY")} { db, site := legacyDatabase(t, open, locale) migrateStorage(t, db) var count int64 if err := db.Model(&entity.SiteLanguage{}).Where("site_id = ?", site.ID).Count(&count).Error; err != nil || count != 0 { t.Errorf("expected no row, got %d %v", count, err) } } }) t.Run("Migration runs twice", func(t *testing.T) { db, _ := legacyDatabase(t, open, localeOf("fr-CA")) migrateStorage(t, db) first := storageSnapshot(t, db) migrateStorage(t, db) if second := storageSnapshot(t, db); second != first { t.Errorf("the second migration changed the database:\n%s\n%s", first, second) } }) t.Run("Migration is atomic", func(t *testing.T) { db, _ := legacyDatabase(t, open, localeOf("fr-CA")) updates := 0 const callbackName = "test:fail-second-story-assignment" err := db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) { if tx.Statement.Table != "articles" { return } updates++ if updates == 2 { _ = tx.AddError(errors.New(injectedBackfillFailure)) } }) if err != nil { t.Fatal(err) } failure := database.Migrate(db, migrationPublicURL) if failure == nil { t.Fatal("the injected failure must abort the migration") } for _, row := range storedArticles(t, db) { if row.StoryID != 0 { t.Errorf("article %d kept a story_id after the rollback", row.ID) } } var stories int64 if err := db.Model(&entity.Story{}).Count(&stories).Error; err != nil || stories != 0 { t.Errorf("the rolled back stories must not exist: %d %v", stories, err) } if markerRecorded(t, db, storiesBackfillMarker) { t.Error("the marker must not be recorded") } if err := db.Callback().Update().Remove(callbackName); err != nil { t.Fatal(err) } migrateStorage(t, db) for _, row := range storedArticles(t, db) { if row.StoryID == 0 { t.Errorf("article %d must have a story after the retry", row.ID) } } if !markerRecorded(t, db, storiesBackfillMarker) { t.Error("the marker must be recorded after the retry") } }) t.Run("Soft-deleted articles are migrated", func(t *testing.T) { db, _ := legacyDatabase(t, open, localeOf("fr-CA"), 2) migrateStorage(t, db) for _, row := range storedArticles(t, db) { if row.StoryID == 0 { t.Errorf("article %d has no story", row.ID) } if row.ID == 2 && (!row.DeletedAt.Valid || row.Slug != domainArticle.Tombstone(2)) { t.Errorf("the tombstone must be unchanged: %+v", row) } } }) t.Run("Slug is unique per site and language", func(t *testing.T) { db := open(t) articles := persistence.NewArticleRepository(db) site := storageDefaultSite(t, db) other := entity.Site{Name: "Other", Key: "other", BaseURL: "https://other.example.com", DefaultRobots: "index"} if err := db.Create(&other).Error; err != nil { t.Fatal(err) } if _, err := articles.Create(newStorageArticle(site.ID, "fr", "bonjour", 0)); err != nil { t.Fatalf("first create: %v", err) } _, duplicate := articles.Create(newStorageArticle(site.ID, "fr", "bonjour", 0)) _, otherLanguage := articles.Create(newStorageArticle(site.ID, "en", "bonjour", 0)) _, otherSite := articles.Create(newStorageArticle(other.ID, "fr", "bonjour", 0)) if !errors.Is(duplicate, domainArticle.ErrSlugTaken) { t.Errorf("expected ErrSlugTaken, got %v", duplicate) } if otherLanguage != nil || otherSite != nil { t.Errorf("language and site variants must be stored: %v %v", otherLanguage, otherSite) } }) t.Run("Existing unassigned articles keep the current per-site slug rule", func(t *testing.T) { db := open(t) articles := persistence.NewArticleRepository(db) site := storageDefaultSite(t, db) if _, err := articles.Create(newStorageArticle(site.ID, "", "same", 0)); err != nil { t.Fatal(err) } second, err := articles.Create(newStorageArticle(site.ID, "", "other", 0)) if err != nil { t.Fatal(err) } _, updateErr := articles.Update(second.ID, domainArticle.Update{Slug: localeOf("same")}) if !errors.Is(updateErr, domainArticle.ErrSlugTaken) { t.Errorf("expected ErrSlugTaken, got %v", updateErr) } }) t.Run("One version per language in a story", func(t *testing.T) { db := open(t) articles := persistence.NewArticleRepository(db) site := storageDefaultSite(t, db) story := entity.Story{} if err := db.Create(&story).Error; err != nil { t.Fatal(err) } if _, err := articles.Create(newStorageArticle(site.ID, "fr", "un", story.ID)); err != nil { t.Fatal(err) } _, duplicate := articles.Create(newStorageArticle(site.ID, "fr", "deux", story.ID)) _, otherLanguage := articles.Create(newStorageArticle(site.ID, "en", "three", story.ID)) if duplicate == nil { t.Error("a second fr version in the same story must be rejected") } if otherLanguage != nil { t.Errorf("an en version must be stored: %v", otherLanguage) } }) t.Run("Unassigned articles never collide on language", func(t *testing.T) { db := open(t) articles := persistence.NewArticleRepository(db) site := storageDefaultSite(t, db) stories := map[uint32]bool{} for index := 0; index < 3; index++ { created, err := articles.Create(newStorageArticle(site.ID, "", fmt.Sprintf("slug-%d", index), 0)) if err != nil || created.StoryID == 0 || stories[created.StoryID] || created.Lang != "" { t.Fatalf("article %d: %+v %v", index, created, err) } stories[created.StoryID] = true } }) t.Run("Creating an article before the default is confirmed", func(t *testing.T) { db, _ := legacyDatabase(t, open, localeOf("fr-CA")) migrateStorage(t, db) created, err := createTitled(db, "Bonjour le monde") stored := storedArticle(t, db, created.ID) if err != nil || stored.Lang != "" || stored.StoryID == 0 || stored.ContentRevision != 1 || created.ContentRevision != 1 { t.Errorf("expected lang '' with a new story and revision 1, got %+v %v", stored, err) } }) t.Run("Creating an article after the default is confirmed", func(t *testing.T) { db, site := legacyDatabase(t, open, localeOf("fr-CA")) migrateStorage(t, db) if err := confirmDefaultLanguage(db, site.ID); err != nil { t.Fatal(err) } created, err := createTitled(db, "Bonjour le monde") stored := storedArticle(t, db, created.ID) if err != nil || stored.Lang != "fr-CA" { t.Fatalf("expected lang fr-CA, got %+v %v", stored, err) } for _, row := range storedArticles(t, db) { if row.ID != stored.ID && row.StoryID == stored.StoryID { t.Errorf("the new story must be distinct from article %d", row.ID) } } }) t.Run("Create racing a confirmation never strands an article", func(t *testing.T) { db, site := legacyDatabase(t, open, localeOf("fr-CA")) migrateStorage(t, db) var group sync.WaitGroup var retries atomic.Int64 start := make(chan struct{}) retry := func(work func() error) error { var err error for attempt := 0; attempt < raceRetries; attempt++ { if err = work(); err == nil { return nil } retries.Add(1) time.Sleep(racePauseBetweenRetries) } return err } failures := make(chan error, raceCreators+1) for index := 0; index < raceCreators; index++ { group.Add(1) go func() { defer group.Done() <-start failures <- retry(func() error { _, err := createTitled(db, fmt.Sprintf("Race article %d", index)) return err }) }() } group.Add(1) go func() { defer group.Done() <-start failures <- retry(func() error { return confirmDefaultLanguage(db, site.ID) }) }() close(start) group.Wait() close(failures) for err := range failures { if err != nil { t.Fatalf("a racing write never succeeded: %v", err) } } t.Logf("busy retries: %d of a budget of %d", retries.Load(), raceRetryBudget) if retries.Load() > raceRetryBudget { t.Errorf("%d retries exceed the budget of %d: the lock is not serializing the writers", retries.Load(), raceRetryBudget) } rows := storedArticles(t, db) if len(rows) != 3+raceCreators { t.Fatalf("expected %d articles, got %d", 3+raceCreators, len(rows)) } for _, row := range rows { if row.Lang != "fr-CA" { t.Errorf("article %d ended with lang %q", row.ID, row.Lang) } } }) t.Run("A failed create leaves no orphan story", func(t *testing.T) { db := open(t) site := storageDefaultSite(t, db) var before int64 if err := db.Model(&entity.Story{}).Count(&before).Error; err != nil { t.Fatal(err) } err := db.Callback().Create().Before("gorm:create").Register(storyRollbackCallback, func(tx *gorm.DB) { if tx.Statement.Table == "article_authors" { _ = tx.AddError(errors.New(storyRollbackFailure)) } }) if err != nil { t.Fatal(err) } defer func() { if err := db.Callback().Create().Remove(storyRollbackCallback); err != nil { t.Fatal(err) } }() draft := newStorageArticle(site.ID, "", "orphan", 0) draft.Authors = []authordomain.Byline{{ID: 1}} _, createErr := persistence.NewArticleRepository(db).Create(draft) var after int64 if err := db.Model(&entity.Story{}).Count(&after).Error; err != nil { t.Fatal(err) } if createErr == nil || after != before { t.Errorf("the story must roll back with the failed create: error %v, stories %d then %d", createErr, before, after) } }) t.Run("Migrating twice keeps the same slug in two languages of one site", func(t *testing.T) { db := open(t) site := storageDefaultSite(t, db) articles := persistence.NewArticleRepository(db) for _, lang := range []string{"fr", "en"} { if _, err := articles.Create(newStorageArticle(site.ID, lang, "bonjour", 0)); err != nil { t.Fatalf("create %s: %v", lang, err) } } migrateStorage(t, db) migrateStorage(t, db) for _, row := range storedArticles(t, db) { if row.Slug != "bonjour" { t.Errorf("article %d (%s) was renamed to %q", row.ID, row.Lang, row.Slug) } } }) t.Run("Assigning a language is idempotent", func(t *testing.T) { db, site := legacyDatabase(t, open, localeOf("fr-CA")) migrateStorage(t, db) if err := confirmDefaultLanguage(db, site.ID); err != nil { t.Fatal(err) } before := storageSnapshot(t, db) err := confirmDefaultLanguage(db, site.ID) if err != nil || storageSnapshot(t, db) != before { t.Errorf("the second assignment must change nothing: %v", err) } for _, row := range storedArticles(t, db) { if row.Lang != "fr-CA" { t.Errorf("article %d has lang %q", row.ID, row.Lang) } } }) t.Run("Unique indexes replace the per-site slug index", func(t *testing.T) { db := open(t) for _, name := range []string{storyLanguageIndexName, siteSlugIndexName} { if !db.Migrator().HasIndex(&entity.Article{}, name) { t.Errorf("index %s is missing", name) } } indexes, err := db.Migrator().GetIndexes(&entity.Article{}) if err != nil { t.Fatal(err) } for _, index := range indexes { if index.Name() == siteSlugIndexName && !slices.Equal(slices.Sorted(slices.Values(index.Columns())), []string{"lang", "site_id", "slug"}) { t.Errorf("the slug index must cover (site_id, lang, slug), got %v", index.Columns()) } } }) t.Run("The per-language slug index is recorded and survives a second migration", func(t *testing.T) { db := open(t) if !markerRecorded(t, db, slugPerLanguageMarker) { t.Error("the per-language slug index marker must be recorded") } migrateStorage(t, db) if !db.Migrator().HasIndex(&entity.Article{}, siteSlugIndexName) || !markerRecorded(t, db, slugPerLanguageMarker) { t.Error("a second migration must keep the index and its marker") } }) t.Run("A missing per-language slug index is recreated once the marker exists", func(t *testing.T) { db := open(t) if err := db.Migrator().DropIndex(&entity.Article{}, siteSlugIndexName); err != nil { t.Fatal(err) } migrateStorage(t, db) if !db.Migrator().HasIndex(&entity.Article{}, siteSlugIndexName) { t.Error("the slug index must be recreated") } }) } func TestArticleLocalizationStorageSQLite(t *testing.T) { runArticleLocalizationStorage(t, openContractSQLite) } func TestArticleLocalizationStorageMySQL(t *testing.T) { runArticleLocalizationStorage(t, openContractMySQL) } func TestArticleLocalizationStoragePostgres(t *testing.T) { runArticleLocalizationStorage(t, openContractPostgres) }