refac: migration crash, real-database tests, site deleted during create, ruels out of the repo, lists

test/article_create_language_test.go

@@ -81,6 +81,17 @@ func TestCreateArticleStartingLanguage(t *testing.T) {
 		}
 	})
 
+	t.Run("A site deleted after its lookup refuses the creation with a specific error", func(t *testing.T) {
+		create, transactor, articles := newCreateLanguageFixture(map[uint32]string{createLanguageSiteID: confirmedDefaultTag})
+		transactor.languages.removed = map[uint32]bool{createLanguageSiteID: true}
+
+		_, err := create.CreateArticle(context.Background(), createLanguageCommand())
+
+		if !errors.Is(err, article.ErrTargetSiteDeleted) || errors.Is(err, site.ErrSiteNotFound) || articles.createCalled {
+			t.Errorf("expected ErrTargetSiteDeleted without a create, got %v", err)
+		}
+	})
+
 	t.Run("The command site decides the site and the language lock", func(t *testing.T) {
 		create, transactor, articles := newCreateLanguageFixture(map[uint32]string{createLanguageSiteID: confirmedDefaultTag, createLanguageOtherSiteID: "de"})
 		create = useCases.NewCreateArticleUseCase(transactor, &fakeBodyRenderer{}, discardRecorder, createLanguageSites(), plainAuthors())

test/article_create_site_deleted_http_test.go

@@ -0,0 +1,28 @@
+package test
+
+import (
+	"RenewCMS/api/pkg/apperr"
+	"RenewCMS/internal/domain/audit"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestArticleCreateSiteDeletedHTTP(t *testing.T) {
+	t.Run("A site deleted between its lookup and the insert answers 409 target_site_deleted", func(t *testing.T) {
+		w := newCreateSiteWorld(t)
+		w.transactor.removed = map[uint32]bool{createSiteNotes: true}
+		before := w.snapshot()
+
+		response := w.send(storyWriter, http.MethodPost, createSitePath, `{"title":"Plain","site":"notes"}`)
+
+		if response.Code != http.StatusConflict || strings.TrimSpace(response.text()) != apperr.TargetSiteDeletedErrorMessage {
+			t.Fatalf("expected 409 %q, got %d %q", apperr.TargetSiteDeletedErrorMessage, response.Code, response.text())
+		}
+		w.expectUnchanged(before)
+		events := w.eventsOf(audit.ActionArticleCreate)
+		if len(events) != 1 || events[0].Outcome != audit.OutcomeFailed || events[0].FailureReason == nil || *events[0].FailureReason != "The site was deleted while the article was being created." {
+			t.Errorf("one failed article.create event with a plain reason expected: %+v", events)
+		}
+	})
+}

test/article_create_support_test.go

@@ -3,6 +3,7 @@ package test
 import (
 	"RenewCMS/internal/domain/article"
 	"RenewCMS/internal/domain/language"
+	"RenewCMS/internal/domain/site"
 	"RenewCMS/internal/domain/transaction"
 	"context"
 	"sync/atomic"
@@ -12,6 +13,7 @@ type mockStartingLanguages struct {
 	confirmed map[uint32]string
 	locked    []uint32
 	failWith  error
+	removed   map[uint32]bool
 }
 
 var _ language.Repository = &mockStartingLanguages{}
@@ -20,16 +22,23 @@ func (m *mockStartingLanguages) ListBySite(siteID uint32) ([]language.SiteLangua
 	return nil, nil
 }
 
-func (m *mockStartingLanguages) ConfirmDefault(siteID uint32) error {
+func (m *mockStartingLanguages) MarkConfirmed(siteID uint32, tag string) error {
 	return nil
 }
 
-func (m *mockStartingLanguages) LockStartingLanguage(siteID uint32) (string, error) {
+func (m *mockStartingLanguages) LockSiteLanguages(siteID uint32) ([]language.SiteLanguage, error) {
 	m.locked = append(m.locked, siteID)
 	if m.failWith != nil {
-		return "", m.failWith
+		return nil, m.failWith
 	}
-	return m.confirmed[siteID], nil
+	if m.removed[siteID] {
+		return nil, site.ErrSiteNotFound
+	}
+	tag, confirmed := m.confirmed[siteID]
+	if !confirmed || tag == language.Unassigned {
+		return nil, nil
+	}
+	return []language.SiteLanguage{{SiteID: siteID, Tag: tag, IsDefault: true, Enabled: true, Confirmed: true}}, nil
 }
 
 type mockCreateTransactor struct {

test/article_list_seo_scale_test.go

@@ -0,0 +1,167 @@
+package test
+
+import (
+	domainArticle "RenewCMS/internal/domain/article"
+	entity "RenewCMS/internal/infrastructure/persistence/models"
+	persistence "RenewCMS/internal/infrastructure/persistence/repositories"
+	"RenewCMS/internal/infrastructure/useCases"
+	"fmt"
+	"testing"
+
+	"gorm.io/gorm"
+)
+
+const (
+	seoScaleVersions      = 40_001
+	seoScaleBatch         = 500
+	seoScaleHiddenEvery   = 1000
+	seoScalePageSize      = 25
+	seoScaleStoryOffset   = 500_000
+	seoScaleTitledEvery   = 7
+	seoScaleTitleFragment = "Lighthouse"
+)
+
+type countingStoryRepository struct {
+	*persistence.ArticleRepository
+	titleSearches int
+	pageLoads     int
+}
+
+func (r *countingStoryRepository) FindTitleMatches(q string, readable domainArticle.StoryFilters) ([]uint32, error) {
+	r.titleSearches++
+	return r.ArticleRepository.FindTitleMatches(q, readable)
+}
+
+func (r *countingStoryRepository) FindPage(filters domainArticle.Filters) (domainArticle.PageResult, error) {
+	r.pageLoads++
+	return r.ArticleRepository.FindPage(filters)
+}
+
+func seoScaleTitle(index int) string {
+	if index%seoScaleTitledEvery == 0 {
+		return fmt.Sprintf("%s %d", seoScaleTitleFragment, index)
+	}
+	return fmt.Sprintf("Harbour %d", index)
+}
+
+func seedSEOScale(t *testing.T, db *gorm.DB, siteID uint32) {
+	t.Helper()
+	rows := make([]entity.Article, 0, seoScaleVersions)
+	for index := 0; index < seoScaleVersions; index++ {
+		rows = append(rows, entity.Article{
+			Title:    seoScaleTitle(index),
+			Body:     "<p>Short body.</p>",
+			SiteID:   siteID,
+			Slug:     fmt.Sprintf("scale-%d", index),
+			StoryID:  uint32(seoScaleStoryOffset + index),
+			IsOnline: true,
+			Noindex:  index%seoScaleHiddenEvery == 0,
+		})
+	}
+	if err := db.CreateInBatches(rows, seoScaleBatch).Error; err != nil {
+		t.Fatalf("bulk insert: %v", err)
+	}
+}
+
+func expectedSEOScale(hidden bool, titled bool) int {
+	total := 0
+	for index := 0; index < seoScaleVersions; index++ {
+		if (index%seoScaleHiddenEvery == 0) == hidden && (!titled || index%seoScaleTitledEvery == 0) {
+			total++
+		}
+	}
+	return total
+}
+
+func TestArticleStoryListSEOFilterAtScaleSQLite(t *testing.T) {
+	db := openContractSQLite(t)
+	site := storageDefaultSite(t, db)
+	seedSEOScale(t, db, site.ID)
+	repository := &countingStoryRepository{ArticleRepository: persistence.NewArticleRepository(db)}
+	list := useCases.NewListArticlesUseCase(repository, &fakeBodyRenderer{}, persistence.NewSiteRepository(db), persistence.NewLocalizationRepository(db))
+
+	cases := []struct {
+		name   string
+		seo    string
+		q      string
+		hidden bool
+		titled bool
+	}{
+		{"More versions with suggestions than the database accepts bind parameters", domainArticle.SEOStatusSuggestions, "", false, false},
+		{"The hidden versions among them", domainArticle.SEOStatusHidden, "", true, false},
+		{"The SEO filter combines with a title search run once", domainArticle.SEOStatusSuggestions, seoScaleTitleFragment, false, true},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			repository.titleSearches, repository.pageLoads = 0, 0
+			want := expectedSEOScale(c.hidden, c.titled)
+
+			page, err := list.ListStoriesPage(useCases.ListStoriesCommand{Caller: ownerCaller(1), Page: 2, Size: seoScalePageSize, SEO: c.seo, Q: c.q})
+
+			if err != nil {
+				t.Fatalf("list: %v", err)
+			}
+			if page.Total != want || page.Counts.All != want {
+				t.Errorf("total %d, counts %+v, want %d", page.Total, page.Counts, want)
+			}
+			if len(page.Rows) != min(seoScalePageSize, max(want-seoScalePageSize, 0)) {
+				t.Errorf("rows = %d", len(page.Rows))
+			}
+			for _, row := range page.Rows {
+				if row.SEO.Status != c.seo || row.Article.Body == "" {
+					t.Errorf("row %d: status %q body %q", row.Article.ID, row.SEO.Status, row.Article.Body)
+				}
+			}
+			wantSearches := 0
+			if c.q != "" {
+				wantSearches = 1
+			}
+			if repository.titleSearches != wantSearches || repository.pageLoads != 1 {
+				t.Errorf("title searches %d, page loads %d per request", repository.titleSearches, repository.pageLoads)
+			}
+		})
+	}
+}
+
+func TestArticleRepositoryListByStoriesSkipsBodiesAndUnreadableDrafts(t *testing.T) {
+	db := openContractSQLite(t)
+	site := storageDefaultSite(t, db)
+	story := fixtureStory()
+	versions := []entity.Article{
+		{Title: "Published", Body: "<p>Published body</p>", SiteID: site.ID, Lang: "en", StoryID: story, Slug: uniqueTestSlug(), IsOnline: true, CreatedBy: localizationOtherUserID},
+		{Title: "Own draft", Body: "<p>Own body</p>", SiteID: site.ID, Lang: "fr", StoryID: story, Slug: uniqueTestSlug(), CreatedBy: localizationWriterID},
+		{Title: "Foreign draft", Body: "<p>Foreign body</p>", SiteID: site.ID, Lang: "de", StoryID: story, Slug: uniqueTestSlug(), CreatedBy: localizationOtherUserID},
+	}
+	if err := db.Create(&versions).Error; err != nil {
+		t.Fatal(err)
+	}
+	repository := persistence.NewArticleRepository(db)
+	writer := localizationWriterID
+	cases := []struct {
+		name     string
+		readable domainArticle.StoryFilters
+		titles   []string
+	}{
+		{"An editor reads every version", domainArticle.StoryFilters{}, []string{"Published", "Own draft", "Foreign draft"}},
+		{"An own-scope caller reads the published versions and their drafts", domainArticle.StoryFilters{VisibleTo: &writer}, []string{"Published", "Own draft"}},
+		{"A reader reads the published versions only", domainArticle.StoryFilters{OnlyOnline: true}, []string{"Published"}},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			loaded, err := repository.ListByStories([]uint32{story}, c.readable)
+			if err != nil {
+				t.Fatal(err)
+			}
+			titles := []string{}
+			for _, version := range loaded {
+				titles = append(titles, version.Title)
+				if version.Body != "" {
+					t.Errorf("version %q carries its body", version.Title)
+				}
+			}
+			if fmt.Sprint(titles) != fmt.Sprint(c.titles) {
+				t.Errorf("titles = %v, want %v", titles, c.titles)
+			}
+		})
+	}
+}

test/article_localization_storage_persistence_test.go

@@ -176,6 +176,10 @@ func createTitled(db *gorm.DB, title string) (domainArticle.Article, error) {
 	return createThroughUseCase(db).CreateArticle(context.Background(), useCases.CreateArticleCommand{Title: title, Caller: ownerCaller(1)})
 }
 
+func confirmDefaultLanguage(db *gorm.DB, siteID uint32) error {
+	return useCases.NewConfirmDefaultLanguageUseCase(persistence.NewTransactor(db)).Confirm(context.Background(), siteID)
+}
+
 func newStorageArticle(siteID uint32, lang string, slug string, storyID uint32) domainArticle.Article {
 	return domainArticle.Article{Title: "Article " + slug, Body: "Body", SiteID: siteID, Lang: lang, Slug: slug, StoryID: storyID, SEODismissed: []string{}}
 }
@@ -406,7 +410,7 @@ func runArticleLocalizationStorage(t *testing.T, open func(*testing.T) *gorm.DB)
 	t.Run("Creating an article after the default is confirmed", func(t *testing.T) {
 		db, site := legacyDatabase(t, open, localeOf("fr-CA"))
 		migrateStorage(t, db)
-		if err := persistence.NewSiteLanguageRepository(db).ConfirmDefault(site.ID); err != nil {
+		if err := confirmDefaultLanguage(db, site.ID); err != nil {
 			t.Fatal(err)
 		}
 
@@ -426,7 +430,6 @@ func runArticleLocalizationStorage(t *testing.T, open func(*testing.T) *gorm.DB)
 	t.Run("Create racing a confirmation never strands an article", func(t *testing.T) {
 		db, site := legacyDatabase(t, open, localeOf("fr-CA"))
 		migrateStorage(t, db)
-		languages := persistence.NewSiteLanguageRepository(db)
 		var group sync.WaitGroup
 		var retries atomic.Int64
 		start := make(chan struct{})
@@ -457,7 +460,7 @@ func runArticleLocalizationStorage(t *testing.T, open func(*testing.T) *gorm.DB)
 		go func() {
 			defer group.Done()
 			<-start
-			failures <- retry(func() error { return languages.ConfirmDefault(site.ID) })
+			failures <- retry(func() error { return confirmDefaultLanguage(db, site.ID) })
 		}()
 		close(start)
 		group.Wait()
@@ -540,13 +543,12 @@ func runArticleLocalizationStorage(t *testing.T, open func(*testing.T) *gorm.DB)
 	t.Run("Assigning a language is idempotent", func(t *testing.T) {
 		db, site := legacyDatabase(t, open, localeOf("fr-CA"))
 		migrateStorage(t, db)
-		languages := persistence.NewSiteLanguageRepository(db)
-		if err := languages.ConfirmDefault(site.ID); err != nil {
+		if err := confirmDefaultLanguage(db, site.ID); err != nil {
 			t.Fatal(err)
 		}
 		before := storageSnapshot(t, db)
 
-		err := languages.ConfirmDefault(site.ID)
+		err := confirmDefaultLanguage(db, site.ID)
 
 		if err != nil || storageSnapshot(t, db) != before {
 			t.Errorf("the second assignment must change nothing: %v", err)
@@ -562,7 +564,7 @@ func runArticleLocalizationStorage(t *testing.T, open func(*testing.T) *gorm.DB)
 		db := open(t)
 		site := storageDefaultSite(t, db)
 
-		err := persistence.NewSiteLanguageRepository(db).ConfirmDefault(site.ID)
+		err := confirmDefaultLanguage(db, site.ID)
 
 		if !errors.Is(err, language.ErrNoDefaultLanguage) {
 			t.Errorf("expected ErrNoDefaultLanguage, got %v", err)

test/article_test.go

@@ -199,11 +199,16 @@ func (m *mockArticleRepo) CountStories(filters article.StoryFilters) (article.St
 	return article.StoryCounts{}, m.fail.trigger("CountStories")
 }
 
-func (m *mockArticleRepo) ListByStories(storyIDs []uint32) ([]article.Article, error) {
+func (m *mockArticleRepo) ListByStories(storyIDs []uint32, readable article.StoryFilters) ([]article.Article, error) {
 	m.listCalls++
 	return []article.Article{}, m.fail.trigger("ListByStories")
 }
 
+func (m *mockArticleRepo) FindTitleMatches(q string, readable article.StoryFilters) ([]uint32, error) {
+	m.listCalls++
+	return []uint32{}, m.fail.trigger("FindTitleMatches")
+}
+
 func (m *mockArticleRepo) CountVisibleTo(userID uint32) (article.Counts, error) {
 	if err := m.fail.trigger("CountVisibleTo"); err != nil {
 		return article.Counts{}, err

test/localization_concurrency_persistence_test.go

@@ -0,0 +1,312 @@
+package test
+
+import (
+	domainArticle "RenewCMS/internal/domain/article"
+	"RenewCMS/internal/domain/localization"
+	"RenewCMS/internal/domain/site"
+	"RenewCMS/internal/domain/transaction"
+	"RenewCMS/internal/infrastructure/persistence/database"
+	entity "RenewCMS/internal/infrastructure/persistence/models"
+	persistence "RenewCMS/internal/infrastructure/persistence/repositories"
+	"RenewCMS/internal/infrastructure/useCases"
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+const (
+	nowaitRefusalBudget    = 3 * time.Second
+	setupRaceRounds        = 4
+	setupRaceAttempts      = 80
+	setupRacePause         = 5 * time.Millisecond
+	mysqlShortLockWait     = "SET SESSION innodb_lock_wait_timeout = 1"
+	postgresShortLockWait  = "SET LOCAL lock_timeout = '300ms'"
+	pendingSiteDefaultLang = "en"
+)
+
+type localizationCase struct {
+	name string
+	run  func(t *testing.T, db *gorm.DB)
+}
+
+type vanishingSites struct {
+	*persistence.SiteRepository
+	db *gorm.DB
+}
+
+func (v *vanishingSites) GetByKey(key string) (site.Site, error) {
+	found, err := v.SiteRepository.GetByKey(key)
+	if err == nil {
+		err = v.db.Delete(&entity.Site{}, found.ID).Error
+	}
+	return found, err
+}
+
+func pendingSite(t *testing.T, db *gorm.DB, key string) site.Site {
+	t.Helper()
+	return seedLocalizedSite(t, db, key, []entity.SiteLanguage{{Lang: pendingSiteDefaultLang, IsDefault: true, Enabled: true}}, nil)
+}
+
+func setupChoicesFor(keys ...string) []localization.SetupChoice {
+	choices := []localization.SetupChoice{{Key: "default", Lang: pendingSiteDefaultLang}}
+	for _, key := range keys {
+		choices = append(choices, localization.SetupChoice{Key: key, Lang: pendingSiteDefaultLang})
+	}
+	return choices
+}
+
+func createOnSite(db *gorm.DB, key string, title string) (domainArticle.Article, error) {
+	return createThroughUseCase(db).CreateArticle(context.Background(), useCases.CreateArticleCommand{Title: title, Site: &key, Caller: ownerCaller(1)})
+}
+
+func retryWhile(attempts int, retryable func(error) bool, work func() error) error {
+	var err error
+	for attempt := 0; attempt < attempts; attempt++ {
+		if err = work(); err == nil || !retryable(err) {
+			return err
+		}
+		time.Sleep(setupRacePause)
+	}
+	return err
+}
+
+func isContention(err error) bool {
+	return errors.Is(err, transaction.ErrLockContention) || errors.Is(err, localization.ErrSetupInProgress)
+}
+
+func articlesOnSite(t *testing.T, db *gorm.DB, siteID uint32) []entity.Article {
+	t.Helper()
+	var rows []entity.Article
+	if err := db.Unscoped().Where("site_id = ?", siteID).Order("id ASC").Find(&rows).Error; err != nil {
+		t.Fatal(err)
+	}
+	return rows
+}
+
+func holdSiteLock(t *testing.T, db *gorm.DB, siteID uint32, then func(transaction.Repositories) error) (func() error, error) {
+	t.Helper()
+	held := make(chan struct{})
+	release := make(chan struct{})
+	finished := make(chan error, 1)
+	go func() {
+		finished <- persistence.NewTransactor(db).Run(context.Background(), func(tx transaction.Repositories) error {
+			if _, err := tx.Languages.LockSiteLanguages(siteID); err != nil {
+				return err
+			}
+			close(held)
+			<-release
+			return then(tx)
+		})
+	}()
+	var once sync.Once
+	finish := func() error {
+		once.Do(func() { close(release) })
+		return <-finished
+	}
+	select {
+	case <-held:
+		t.Cleanup(func() { once.Do(func() { close(release) }) })
+		return finish, nil
+	case err := <-finished:
+		return nil, err
+	}
+}
+
+func lockSiteRow(tx *gorm.DB, siteID uint32) error {
+	var locked entity.Site
+	return tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&locked, siteID).Error
+}
+
+func shortLockWait(db *gorm.DB) string {
+	if db.Name() == database.DriverMySQL {
+		return mysqlShortLockWait
+	}
+	return postgresShortLockWait
+}
+
+func skipWithoutRowLocks(t *testing.T, db *gorm.DB) {
+	t.Helper()
+	if db.Name() == database.DriverSQLite {
+		t.Skip("SQLite ignores FOR UPDATE; its single writer is covered by the race cases")
+	}
+}
+
+func localizationConcurrencyCases() []localizationCase {
+	return []localizationCase{
+		{"An article create whose site is deleted after the lookup is refused with target site deleted", func(t *testing.T, db *gorm.DB) {
+			gone := pendingSite(t, db, "vanishing")
+			create := useCases.NewCreateArticleUseCase(persistence.NewTransactor(db), &fakeBodyRenderer{}, discardRecorder, &vanishingSites{persistence.NewSiteRepository(db), db}, persistence.NewAuthorRepository(db))
+			key := gone.Key
+
+			_, err := create.CreateArticle(context.Background(), useCases.CreateArticleCommand{Title: "Vanishing site", Site: &key, Caller: ownerCaller(1)})
+
+			if !errors.Is(err, domainArticle.ErrTargetSiteDeleted) {
+				t.Fatalf("expected ErrTargetSiteDeleted, got %v", err)
+			}
+			if rows := articlesOnSite(t, db, gone.ID); len(rows) != 0 {
+				t.Errorf("an article was stored on the deleted site: %+v", rows)
+			}
+		}},
+		{"Duplicate addresses and languages rejected by the server become domain errors", func(t *testing.T, db *gorm.DB) {
+			owner := seedLocalizedSite(t, db, "unique", nil, nil)
+			articles := persistence.NewArticleRepository(db)
+			first, err := articles.Create(newStorageArticle(owner.ID, "en", "same-address", 0))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			_, slugErr := articles.Create(newStorageArticle(owner.ID, "en", "same-address", 0))
+			_, languageErr := articles.Create(newStorageArticle(owner.ID, "en", "other-address", first.StoryID))
+			duplicate := entity.Article{Title: "Raw", SiteID: owner.ID, Lang: "en", Slug: "same-address", StoryID: fixtureStory()}
+			rawErr := db.Create(&duplicate).Error
+
+			if !errors.Is(slugErr, domainArticle.ErrSlugTaken) {
+				t.Errorf("duplicate slug: %v", slugErr)
+			}
+			if !errors.Is(languageErr, domainArticle.ErrLanguageTaken) {
+				t.Errorf("duplicate story language: %v", languageErr)
+			}
+			if !database.IsUniqueViolation(db, rawErr) || database.IsLockContention(db, rawErr) {
+				t.Errorf("the raw duplicate must be a unique violation: %v", rawErr)
+			}
+			if rows := articlesOnSite(t, db, owner.ID); len(rows) != 1 {
+				t.Errorf("rejected writes left %d rows", len(rows))
+			}
+		}},
+		{"Language setup does not wait for a site held by an article create", func(t *testing.T, db *gorm.DB) {
+			skipWithoutRowLocks(t, db)
+			held := pendingSite(t, db, "held")
+			apply := useCases.NewApplyLanguageSetupUseCase(persistence.NewTransactor(db), discardRecorder)
+			finish, err := holdSiteLock(t, db, held.ID, func(tx transaction.Repositories) error {
+				_, err := tx.Articles.Create(newStorageArticle(held.ID, "", "held-create", 0))
+				return err
+			})
+			if err != nil {
+				t.Fatalf("hold: %v", err)
+			}
+
+			started := time.Now()
+			_, refused := apply.Apply(context.Background(), setupChoicesFor(held.Key))
+			waited := time.Since(started)
+			holdErr := finish()
+
+			if !errors.Is(refused, localization.ErrSetupInProgress) || !errors.Is(refused, transaction.ErrLockContention) {
+				t.Fatalf("expected a NOWAIT refusal, got %v", refused)
+			}
+			if waited > nowaitRefusalBudget {
+				t.Errorf("the setup waited %v for the lock", waited)
+			}
+			if holdErr != nil {
+				t.Fatalf("the held create failed: %v", holdErr)
+			}
+			result, err := apply.Apply(context.Background(), setupChoicesFor(held.Key))
+			if err != nil || result.Assigned != 1 {
+				t.Fatalf("retry = %+v, %v", result, err)
+			}
+			for _, row := range articlesOnSite(t, db, held.ID) {
+				if row.Lang != pendingSiteDefaultLang {
+					t.Errorf("article %d kept lang %q", row.ID, row.Lang)
+				}
+			}
+		}},
+		{"Language setup racing an article create never strands an article", func(t *testing.T, db *gorm.DB) {
+			apply := useCases.NewApplyLanguageSetupUseCase(persistence.NewTransactor(db), discardRecorder)
+			for round := 0; round < setupRaceRounds; round++ {
+				racing := pendingSite(t, db, fmt.Sprintf("racing%d", round))
+				start := make(chan struct{})
+				var group sync.WaitGroup
+				var createErr, setupErr error
+				group.Add(2)
+				go func() {
+					defer group.Done()
+					<-start
+					createErr = retryWhile(setupRaceAttempts, func(error) bool { return true }, func() error {
+						_, err := createOnSite(db, racing.Key, fmt.Sprintf("Racing article %d", round))
+						return err
+					})
+				}()
+				go func() {
+					defer group.Done()
+					<-start
+					setupErr = retryWhile(setupRaceAttempts, isContention, func() error {
+						_, err := apply.Apply(context.Background(), setupChoicesFor(racing.Key))
+						return err
+					})
+				}()
+				close(start)
+				group.Wait()
+
+				if createErr != nil || setupErr != nil {
+					t.Fatalf("round %d: create %v, setup %v", round, createErr, setupErr)
+				}
+				rows := articlesOnSite(t, db, racing.ID)
+				if len(rows) != 1 || rows[0].Lang != pendingSiteDefaultLang {
+					t.Fatalf("round %d: articles %+v", round, rows)
+				}
+			}
+		}},
+		{"A lock wait timeout from the server is lock contention", func(t *testing.T, db *gorm.DB) {
+			skipWithoutRowLocks(t, db)
+			held := pendingSite(t, db, "waiting")
+			finish, err := holdSiteLock(t, db, held.ID, func(transaction.Repositories) error { return nil })
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			waitErr := db.Transaction(func(tx *gorm.DB) error {
+				if err := tx.Exec(shortLockWait(db)).Error; err != nil {
+					return err
+				}
+				return lockSiteRow(tx, held.ID)
+			})
+			if err := finish(); err != nil {
+				t.Fatal(err)
+			}
+
+			if waitErr == nil || !database.IsLockContention(db, waitErr) {
+				t.Errorf("expected lock contention, got %v", waitErr)
+			}
+		}},
+		{"A deadlock from the server is lock contention", func(t *testing.T, db *gorm.DB) {
+			skipWithoutRowLocks(t, db)
+			first := pendingSite(t, db, "deadfirst")
+			second := pendingSite(t, db, "deadsecond")
+			bothLocked := sync.WaitGroup{}
+			bothLocked.Add(2)
+			failures := make(chan error, 2)
+			crossLock := func(own uint32, other uint32) {
+				failures <- db.Transaction(func(tx *gorm.DB) error {
+					err := lockSiteRow(tx, own)
+					bothLocked.Done()
+					if err != nil {
+						return err
+					}
+					bothLocked.Wait()
+					return lockSiteRow(tx, other)
+				})
+			}
+			go crossLock(first.ID, second.ID)
+			go crossLock(second.ID, first.ID)
+
+			contended := 0
+			for range 2 {
+				if err := <-failures; err != nil {
+					if !database.IsLockContention(db, err) {
+						t.Errorf("unexpected error: %v", err)
+					}
+					contended++
+				}
+			}
+
+			if contended == 0 {
+				t.Errorf("one of the crossed transactions must be chosen as the deadlock victim")
+			}
+		}},
+	}
+}

test/localization_persistence_test.go

@@ -79,10 +79,7 @@ func TestLocalizationPersistence(t *testing.T) {
 		{name: "mysql", open: openContractMySQL},
 		{name: "postgres", open: openContractPostgres},
 	}
-	cases := []struct {
-		name string
-		run  func(t *testing.T, db *gorm.DB)
-	}{
+	cases := []localizationCase{
 		{"Persistence additions are safe to migrate", func(t *testing.T, db *gorm.DB) {
 			migrator := db.Migrator()
 			created := seedLocalizedSite(t, db, "migrating", []entity.SiteLanguage{{Lang: "en", IsDefault: true, Enabled: true, Confirmed: true}}, nil)
@@ -272,6 +269,7 @@ func TestLocalizationPersistence(t *testing.T) {
 			}
 		}},
 	}
+	cases = append(cases, localizationConcurrencyCases()...)
 	for _, backend := range backends {
 		for _, c := range cases {
 			t.Run(backend.name+"/"+c.name, func(t *testing.T) {

test/lock_contention_driver_test.go

@@ -76,3 +76,32 @@ func TestTransactorClassifiesLockContention(t *testing.T) {
 		}
 	})
 }
+
+func TestDriverUniqueViolation(t *testing.T) {
+	cases := []struct {
+		name   string
+		driver string
+		err    error
+		want   bool
+	}{
+		{"sqlite unique index", database.DriverSQLite, codedSQLiteError{2067}, true},
+		{"sqlite primary key", database.DriverSQLite, fmt.Errorf("wrapped: %w", codedSQLiteError{1555}), true},
+		{"sqlite not null", database.DriverSQLite, codedSQLiteError{1299}, false},
+		{"sqlite busy", database.DriverSQLite, codedSQLiteError{5}, false},
+		{"sqlite plain error", database.DriverSQLite, errors.New("UNIQUE constraint failed"), false},
+		{"mysql duplicate entry", database.DriverMySQL, &mysqldriver.MySQLError{Number: 1062}, true},
+		{"mysql duplicate entry with key name", database.DriverMySQL, fmt.Errorf("wrapped: %w", &mysqldriver.MySQLError{Number: 1586}), true},
+		{"mysql deadlock", database.DriverMySQL, &mysqldriver.MySQLError{Number: 1213}, false},
+		{"postgres unique violation", database.DriverPostgres, &pgconn.PgError{Code: "23505"}, true},
+		{"postgres foreign key violation", database.DriverPostgres, &pgconn.PgError{Code: "23503"}, false},
+		{"unknown driver", "oracle", &pgconn.PgError{Code: "23505"}, false},
+		{"nil error", database.DriverPostgres, nil, false},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if got := database.IsUniqueViolationFor(c.driver, c.err); got != c.want {
+				t.Errorf("got %v, want %v", got, c.want)
+			}
+		})
+	}
+}

test/source_revision_backfill_test.go

@@ -0,0 +1,132 @@
+package test
+
+import (
+	"RenewCMS/internal/infrastructure/persistence/database"
+	entity "RenewCMS/internal/infrastructure/persistence/models"
+	"path/filepath"
+	"testing"
+
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+const (
+	nullableSourceRevisionColumn = "ALTER TABLE articles ADD COLUMN source_revision BIGINT"
+	legacyRevisionArticles       = 3
+)
+
+func reopenableSQLite(t *testing.T) (*gorm.DB, func() *gorm.DB) {
+	t.Helper()
+	path := filepath.ToSlash(filepath.Join(t.TempDir(), "revisions.db"))
+	connect := func() *gorm.DB {
+		connection, err := database.Connect(database.Config{Driver: database.DriverSQLite, Source: path})
+		if err != nil {
+			t.Fatalf("open sqlite: %v", err)
+		}
+		connection.DB.Logger = logger.Discard
+		t.Cleanup(func() { _ = connection.Close() })
+		return connection.DB
+	}
+	return connect(), connect
+}
+
+func reopenableMySQL(t *testing.T) (*gorm.DB, func() *gorm.DB) {
+	t.Helper()
+	server := connectMySQLTestServer(t)
+	config := server.createDatabase(t)
+	connect := func() *gorm.DB { return server.connect(t, config).DB }
+	return connect(), connect
+}
+
+func reopenablePostgres(t *testing.T) (*gorm.DB, func() *gorm.DB) {
+	t.Helper()
+	server := connectPostgresTestServer(t)
+	config := server.createDatabase(t)
+	connect := func() *gorm.DB { return server.connect(t, config).DB }
+	return connect(), connect
+}
+
+func regressToNullableSourceRevision(t *testing.T, db *gorm.DB) {
+	t.Helper()
+	if err := db.Migrator().DropColumn(&entity.Article{}, "source_revision"); err != nil {
+		t.Fatalf("drop source_revision: %v", err)
+	}
+	if err := db.Exec(nullableSourceRevisionColumn).Error; err != nil {
+		t.Fatalf("add nullable source_revision: %v", err)
+	}
+}
+
+func nullSourceRevisions(t *testing.T, db *gorm.DB) int64 {
+	t.Helper()
+	var total int64
+	if err := db.Unscoped().Model(&entity.Article{}).Where("source_revision IS NULL").Count(&total).Error; err != nil {
+		t.Fatal(err)
+	}
+	return total
+}
+
+func runSourceRevisionBackfill(t *testing.T, open func(t *testing.T) (*gorm.DB, func() *gorm.DB)) {
+	t.Run("Articles stored before source revisions existed migrate twice", func(t *testing.T) {
+		db, reconnect := open(t)
+		if err := database.Migrate(db, migrationPublicURL); err != nil {
+			t.Fatalf("first migrate: %v", err)
+		}
+		for index := 0; index < legacyRevisionArticles; index++ {
+			row := entity.Article{Title: "Legacy", Body: "Body", StoryID: fixtureStory(), Slug: uniqueTestSlug(), SourceRevision: 7}
+			if err := db.Create(&row).Error; err != nil {
+				t.Fatal(err)
+			}
+		}
+		regressToNullableSourceRevision(t, db)
+		if nulls := nullSourceRevisions(t, db); nulls != legacyRevisionArticles {
+			t.Fatalf("expected %d NULL source revisions, got %d", legacyRevisionArticles, nulls)
+		}
+		db = reconnect()
+
+		if err := database.Migrate(db, migrationPublicURL); err != nil {
+			t.Fatalf("migrate over NULL source revisions: %v", err)
+		}
+
+		if nulls := nullSourceRevisions(t, db); nulls != 0 {
+			t.Errorf("%d NULL source revisions survived", nulls)
+		}
+		var revisions []uint32
+		if err := db.Unscoped().Model(&entity.Article{}).Order("id ASC").Pluck("source_revision", &revisions).Error; err != nil {
+			t.Fatal(err)
+		}
+		if len(revisions) != legacyRevisionArticles {
+			t.Fatalf("revisions = %v", revisions)
+		}
+		for _, revision := range revisions {
+			if revision != 0 {
+				t.Errorf("backfilled source revision = %d, want 0", revision)
+			}
+		}
+		if columns, err := db.Migrator().ColumnTypes(&entity.Article{}); err == nil {
+			for _, column := range columns {
+				if nullable, known := column.Nullable(); column.Name() == "source_revision" && known && nullable {
+					t.Errorf("source_revision is still nullable")
+				}
+			}
+		}
+		db = reconnect()
+		if err := database.Migrate(db, migrationPublicURL); err != nil {
+			t.Fatalf("second migrate: %v", err)
+		}
+		if nulls := nullSourceRevisions(t, db); nulls != 0 {
+			t.Errorf("second migrate left %d NULL source revisions", nulls)
+		}
+	})
+}
+
+func TestSourceRevisionBackfillSQLite(t *testing.T) {
+	runSourceRevisionBackfill(t, reopenableSQLite)
+}
+
+func TestSourceRevisionBackfillMySQL(t *testing.T) {
+	runSourceRevisionBackfill(t, reopenableMySQL)
+}
+
+func TestSourceRevisionBackfillPostgres(t *testing.T) {
+	runSourceRevisionBackfill(t, reopenablePostgres)
+}

test/story_support_test.go

@@ -63,6 +63,7 @@ type storyTransactor struct {
 	runs     int
 	external map[uint32]article.Article
 	starting map[uint32]string
+	removed  map[uint32]bool
 }
 
 var _ transaction.Transactor = &storyTransactor{}
@@ -77,7 +78,7 @@ func (s *storyTransactor) Run(ctx context.Context, work func(transaction.Reposit
 	if s.starting != nil {
 		starting = s.starting
 	}
-	err := work(transaction.Repositories{Articles: w.articles, Sites: w.sites, Authors: w.authors, Localization: w.rows, Languages: &mockStartingLanguages{confirmed: starting}})
+	err := work(transaction.Repositories{Articles: w.articles, Sites: w.sites, Authors: w.authors, Localization: w.rows, Languages: &mockStartingLanguages{confirmed: starting, removed: s.removed}})
 	if err != nil {
 		w.articles.articles = articles
 		w.articles.credits = credits