From d752a61e52c358840a7f2bf244bc00b597aefe30 Mon Sep 17 00:00:00 2001 From: Florian Sylvain Date: Tue, 6 Oct 2026 04:08:33 +0200 Subject: [PATCH] feat: optional MySQL/MariaDB database driver for deployments that cannot use local SQLite --- .env.example | 11 + Taskfile.yml | 41 ++ api/dependecyInjection.go | 32 +- cmd/server/main.go | 14 +- docker-compose.mysql.yml | 51 ++ docker.env.example | 11 + docs/architecture.md | 16 +- docs/configuration.md | 30 +- docs/deployment.md | 31 +- docs/development.md | 1 + docs/docker.md | 18 +- features/mysql_database_driver.feature | 160 +++++ features/mysql_database_driver.qa.md | 42 ++ go.mod | 5 +- go.sum | 6 + .../infrastructure/config/databaseSettings.go | 122 ++++ internal/infrastructure/config/settings.go | 12 + .../persistence/database/backupRetention.go | 55 ++ .../persistence/database/driver.go | 24 + .../persistence/database/dumpBackup.go | 79 +++ .../persistence/database/migrate.go | 2 +- .../persistence/database/mysql.go | 86 +++ .../persistence/database/mysqlBackup.go | 131 ++++ .../persistence/database/sqlDump.go | 170 ++++++ .../persistence/database/sqliteBackup.go | 47 +- .../persistence/models/apiKeyModel.go | 2 +- .../persistence/models/imageModel.go | 2 +- .../persistence/models/inviteModel.go | 2 +- .../persistence/models/userModel.go | 12 +- .../repositories/userRepository.go | 14 +- test/mysql_database_driver_test.go | 559 ++++++++++++++++++ test/mysql_server_test.go | 311 ++++++++++ test/repository_contract_test.go | 368 ++++++++++++ test/sql_dump_backup_test.go | 328 ++++++++++ test/sqlite_schema_unchanged_test.go | 177 ++++++ 35 files changed, 2896 insertions(+), 76 deletions(-) create mode 100644 docker-compose.mysql.yml create mode 100644 features/mysql_database_driver.feature create mode 100644 features/mysql_database_driver.qa.md create mode 100644 internal/infrastructure/config/databaseSettings.go create mode 100644 internal/infrastructure/persistence/database/backupRetention.go create mode 100644 internal/infrastructure/persistence/database/dumpBackup.go create mode 100644 internal/infrastructure/persistence/database/mysql.go create mode 100644 internal/infrastructure/persistence/database/mysqlBackup.go create mode 100644 internal/infrastructure/persistence/database/sqlDump.go create mode 100644 test/mysql_database_driver_test.go create mode 100644 test/mysql_server_test.go create mode 100644 test/repository_contract_test.go create mode 100644 test/sql_dump_backup_test.go create mode 100644 test/sqlite_schema_unchanged_test.go diff --git a/.env.example b/.env.example index 5780c38..9ad5bf2 100644 --- a/.env.example +++ b/.env.example @@ -16,3 +16,14 @@ AUTH_RATE_LIMIT_WINDOW_SECONDS=60 TRUSTED_PROXIES= UPDATE_FEED_URL=https://git.floriansylvain.fr RENEWCMS_CONTAINER=false +DB_DRIVER=sqlite +DATA_DIR= +DB_HOST= +DB_PORT=3306 +DB_USER= +DB_PASSWORD= +DB_NAME= +DB_TLS=false +DB_MAX_OPEN_CONNS=10 +DB_MAX_IDLE_CONNS=5 +DB_CONN_MAX_LIFETIME_SECONDS=300 diff --git a/Taskfile.yml b/Taskfile.yml index 063a5bd..048165a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -10,6 +10,7 @@ vars: env: CGO_ENABLED: '0' + __COMPAT_LAYER: RunAsInvoker tasks: default: @@ -46,6 +47,46 @@ tasks: cmds: - go test -tags nofrontend ./test/... -count=1 {{.CLI_ARGS}} + test:mysql: + desc: Run the MySQL tests against real servers (mysql:8.4 then mariadb:11 in Docker, removed afterwards) + cmds: + - cmd: | + PASSWORD=renewcms-test-root + STATUS=0 + run_engine() { + NAME="renewcms-test-$1" + IMAGE="$2" + docker rm -f "$NAME" >/dev/null 2>&1 + docker run -d --name "$NAME" -e MYSQL_ROOT_PASSWORD="$PASSWORD" -p 127.0.0.1::3306 "$IMAGE" >/dev/null || return 1 + MAPPING="$(docker port "$NAME" 3306/tcp)" + PORT="${MAPPING##*:}" + READY=no + TRIES=0 + while [ "$TRIES" -lt 90 ]; do + if docker exec "$NAME" sh -c "\$(command -v mariadb || command -v mysql) -h127.0.0.1 -uroot -p$PASSWORD -e 'SELECT 1'" >/dev/null 2>&1; then + READY=yes + break + fi + TRIES=$((TRIES + 1)) + sleep 2 + done + if [ "$READY" != yes ]; then + echo "$IMAGE did not become ready" >&2 + docker logs "$NAME" 2>&1 | tail -n 20 >&2 + docker rm -f "$NAME" >/dev/null 2>&1 + return 1 + fi + echo "== $IMAGE on 127.0.0.1:$PORT" + RENEWCMS_TEST_MYSQL_DSN="root:$PASSWORD@tcp(127.0.0.1:$PORT)/mysql" go test -tags nofrontend ./test/... -run MySQL -count=1 {{.CLI_ARGS}} + RESULT=$? + docker rm -f "$NAME" >/dev/null 2>&1 + return $RESULT + } + run_engine mysql mysql:8.4 || STATUS=1 + run_engine mariadb mariadb:11 || STATUS=1 + exit $STATUS + silent: true + cover: desc: Go test coverage of the use cases → tmp/coverage.out cmds: diff --git a/api/dependecyInjection.go b/api/dependecyInjection.go index d74426f..d737914 100644 --- a/api/dependecyInjection.go +++ b/api/dependecyInjection.go @@ -117,8 +117,38 @@ func openImageStorage(settings config.Settings) domainImage.Storage { return imageStorage } +func databaseConfig(settings config.Settings) database.Config { + switch settings.Database.Driver { + case config.DatabaseDriverMySQL: + return mysqlDatabaseConfig(settings.Database) + case "", config.DatabaseDriverSQLite: + return database.Config{Driver: database.DriverSQLite, Source: settings.DBFile} + default: + return database.Config{Driver: settings.Database.Driver} + } +} + +func mysqlDatabaseConfig(settings config.DatabaseSettings) database.Config { + return database.Config{ + Driver: database.DriverMySQL, + Source: database.MySQLDSN(database.MySQLConfig{ + Host: settings.Host, + Port: settings.Port, + User: settings.User, + Password: settings.Password, + Name: settings.Name, + TLS: settings.TLS, + }), + Pool: &database.PoolSettings{ + MaxOpenConns: settings.MaxOpenConns, + MaxIdleConns: settings.MaxIdleConns, + ConnMaxLifetime: settings.ConnMaxLifetime, + }, + } +} + func openDatabase(settings config.Settings) *database.Connection { - connection, err := database.Connect(database.Config{Driver: database.DriverSQLite, Source: settings.DBFile}) + connection, err := database.Connect(databaseConfig(settings)) if err != nil { panic("Unable to open the database: " + err.Error()) } diff --git a/cmd/server/main.go b/cmd/server/main.go index 88b2bdf..dd48f26 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -12,6 +12,7 @@ import ( "net/http" "os" "os/signal" + "slices" "syscall" "time" @@ -44,6 +45,12 @@ var envDefaults = map[string]string{ "ENVIRONMENT": "production", "PORT": "8080", "DB_FILE": "./data/renewcms.db", + config.EnvDatabaseDriver: config.DefaultDatabaseDriver, + "DB_PORT": config.DefaultDatabasePort, + "DB_TLS": config.DefaultDatabaseTLS, + "DB_MAX_OPEN_CONNS": config.DefaultMaxOpenConns, + "DB_MAX_IDLE_CONNS": config.DefaultMaxIdleConns, + "DB_CONN_MAX_LIFETIME_SECONDS": config.DefaultConnMaxLifetimeSeconds, "UPLOAD_DIR": "./data/uploads", "AUTH_RATE_LIMIT_MAX_ATTEMPTS": "5", "AUTH_RATE_LIMIT_WINDOW_SECONDS": "60", @@ -108,10 +115,9 @@ func applyEnvDefaults() { } func ensureRequiredEnvVars() { - for _, envVar := range requiredEnvVars { - if _, isSet := os.LookupEnv(envVar); !isSet { - panic(fmt.Sprintf("Environment variable %s is not set", envVar)) - } + required := slices.Concat(requiredEnvVars, config.DatabaseRequiredEnvVars(os.Getenv(config.EnvDatabaseDriver))) + if err := config.MissingEnvVarError(required, os.LookupEnv); err != nil { + panic(err.Error()) } } diff --git a/docker-compose.mysql.yml b/docker-compose.mysql.yml new file mode 100644 index 0000000..eff3371 --- /dev/null +++ b/docker-compose.mysql.yml @@ -0,0 +1,51 @@ +services: + mysql: + image: mysql:8.4 + restart: unless-stopped + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?set MYSQL_ROOT_PASSWORD} + MYSQL_DATABASE: renewcms + MYSQL_USER: renewcms + MYSQL_PASSWORD: ${DB_PASSWORD:?set DB_PASSWORD} + volumes: + - mysql-data:/var/lib/mysql + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u root -p\"$$MYSQL_ROOT_PASSWORD\" --silent"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + + renewcms: + image: ${RENEWCMS_IMAGE:-git.floriansylvain.fr/fsylvain/renewcms:beta} + restart: unless-stopped + depends_on: + mysql: + condition: service_healthy + env_file: + - docker.env + environment: + DB_DRIVER: mysql + DB_HOST: mysql + DB_PORT: "3306" + DB_NAME: renewcms + DB_USER: renewcms + DB_PASSWORD: ${DB_PASSWORD:?set DB_PASSWORD} + DATA_DIR: /data + UPLOAD_DIR: /data/uploads + RENEWCMS_CONTAINER: "true" + ports: + - "8080:8080" + volumes: + - renewcms-data:/data + stop_grace_period: 15s + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + +volumes: + renewcms-data: + mysql-data: diff --git a/docker.env.example b/docker.env.example index 8fffdb5..19572bb 100644 --- a/docker.env.example +++ b/docker.env.example @@ -11,3 +11,14 @@ CORS_ALLOWED_ORIGINS= AUTH_RATE_LIMIT_MAX_ATTEMPTS=5 AUTH_RATE_LIMIT_WINDOW_SECONDS=60 TRUSTED_PROXIES= +DB_DRIVER=sqlite +DATA_DIR= +DB_HOST= +DB_PORT=3306 +DB_USER= +DB_PASSWORD= +DB_NAME= +DB_TLS=false +DB_MAX_OPEN_CONNS=10 +DB_MAX_IDLE_CONNS=5 +DB_CONN_MAX_LIFETIME_SECONDS=300 diff --git a/docs/architecture.md b/docs/architecture.md index 555d902..0d1a656 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -80,6 +80,18 @@ bin/renewcms ── /v1/* API Uploaded images are addressed by a stored name (`.`). `CreateImageUseCase` calls `Storage.Put` (which generates the name), then `Repository.Create` with `Path = /static/uploadedImages/`, and deletes the blob again if the row cannot be created. `DeleteMediaUseCase` and `MergeMediaUseCase` remove the blob first and the row second (`mediaRemover`), so a failed blob delete keeps the row and the delete can be retried. `GET /static/uploadedImages/` is served by `OpenMediaUseCase` through `api/handlers/staticimages`, which uses `http.ServeContent` when the blob is seekable and streams it otherwise; `ErrBlobNotFound` maps to 404 and any other error to 500. -SQL in repositories must run on SQLite, MySQL/MariaDB and PostgreSQL: substring searches use `LIKE ... ESCAPE '!'` through `containsPattern` instead of `instr`. `Settings.DataDir()` is where update state, staging and backups live; it is derived from `DB_FILE` today and does not depend on where the database is. +SQL in repositories must run on SQLite, MySQL/MariaDB and PostgreSQL: substring searches use `LIKE ... ESCAPE '!'` through `containsPattern` instead of `instr`. `Settings.DataDir()` is where update state, staging and backups live; it is `DATA_DIR` when set and the directory of `DB_FILE` otherwise, so it does not depend on where the database is. -Adding a database driver: implement `database.Driver` (`Open`, and `NewBackup` returning a `domain/update.DatabaseBackup`), register it in `drivers`, then select it in `openDatabase` in the composition root (and add the env vars to `main.go`, `.env.example` and `configuration.md`). Adding a storage backend: implement `image.Storage`, add a case to `storage.Open`, and select it in `openImageStorage`. No use case, handler or repository changes. +Two drivers are registered in `database.drivers`: `sqlite` (default) and `mysql` (MySQL 8 and MariaDB 11, through `gorm.io/driver/mysql`, pure Go so `CGO_ENABLED=0` and the six release targets still build). `DB_DRIVER` picks one in `databaseConfig` (composition root); an unknown name fails in `database.Connect` with `unsupported database driver ""`. `database.MySQLDSN` builds the DSN from separate variables, and the driver redacts the password from every error and names only host and port. `Connect` applies the optional `PoolSettings` to any driver. What keeps the repositories identical on both engines, all without touching the domain, use cases or handlers: + +- Indexed and unique strings carry `size:` tags in the models (a MySQL index needs a bounded `varchar`). SQLite maps them to the same `text` column, so an existing SQLite file migrates with no schema or row change (`test/sqlite_schema_unchanged_test.go` builds a file from the old model definitions and compares `sqlite_master` and every row). +- `Migrate` creates MySQL tables with `utf8mb4_bin` (`gorm:table_options`), so `Bob` and `bob` are distinct like in SQLite. The DSN also sets `collation=utf8mb4_bin`. +- The DSN sets `clientFoundRows=true`: an update that matches a row but changes nothing still reports one affected row, as in SQLite. `InviteRepository.Renew`, `deleteOrNotFound` and `PurgeSettingsRepository.SaveLastRun` depend on that. +- The Go driver sends a zero `time.Time` as `0000-00-00`, which MySQL's default `NO_ZERO_DATE` rejects, so each session sets `sql_mode` to the server default minus `NO_ZERO_DATE`. Reading `0000-00-00` gives the zero time back. +- `UserRepository.Create` returns the filled model instead of re-reading it, because MySQL has no `RETURNING`. + +Backups are a `domain/update.DatabaseBackup` per driver. SQLite uses `VACUUM INTO`. MySQL uses `DumpBackup`, a pure Go logical dump (`SHOW CREATE TABLE` plus every row, strings escaped and binary values as hex, NULL and zero dates kept) written inside one repeatable-read transaction to `DATA_DIR/backups/renewcms-