feat: optional MySQL/MariaDB database driver for deployments that cannot use local SQLite

This commit is contained in:
fsylvain committed 2026-10-06 04:08:33 +02:00
1 parent 81e299f09a
commit d752a61e52
35 files changed
+2896 -76

No files matched your search

+11
View File
@@ -16,3 +16,14 @@ AUTH_RATE_LIMIT_WINDOW_SECONDS=60
TRUSTED_PROXIES=
UPDATE_FEED_URL=https://git.floriansylvain.fr
RENEWCMS_CONTAINER=false
DB_DRIVER=sqlite
DATA_DIR=
DB_HOST=
DB_PORT=3306
DB_USER=
DB_PASSWORD=
DB_NAME=
DB_TLS=false
DB_MAX_OPEN_CONNS=10
DB_MAX_IDLE_CONNS=5
DB_CONN_MAX_LIFETIME_SECONDS=300
+41
View File
@@ -10,6 +10,7 @@ vars:
env:
CGO_ENABLED: '0'
__COMPAT_LAYER: RunAsInvoker
tasks:
default:
@@ -46,6 +47,46 @@ tasks:
cmds:
- go test -tags nofrontend ./test/... -count=1 {{.CLI_ARGS}}
test:mysql:
desc: Run the MySQL tests against real servers (mysql:8.4 then mariadb:11 in Docker, removed afterwards)
cmds:
- cmd: |
PASSWORD=renewcms-test-root
STATUS=0
run_engine() {
NAME="renewcms-test-$1"
IMAGE="$2"
docker rm -f "$NAME" >/dev/null 2>&1
docker run -d --name "$NAME" -e MYSQL_ROOT_PASSWORD="$PASSWORD" -p 127.0.0.1::3306 "$IMAGE" >/dev/null || return 1
MAPPING="$(docker port "$NAME" 3306/tcp)"
PORT="${MAPPING##*:}"
READY=no
TRIES=0
while [ "$TRIES" -lt 90 ]; do
if docker exec "$NAME" sh -c "\$(command -v mariadb || command -v mysql) -h127.0.0.1 -uroot -p$PASSWORD -e 'SELECT 1'" >/dev/null 2>&1; then
READY=yes
break
fi
TRIES=$((TRIES + 1))
sleep 2
done
if [ "$READY" != yes ]; then
echo "$IMAGE did not become ready" >&2
docker logs "$NAME" 2>&1 | tail -n 20 >&2
docker rm -f "$NAME" >/dev/null 2>&1
return 1
fi
echo "== $IMAGE on 127.0.0.1:$PORT"
RENEWCMS_TEST_MYSQL_DSN="root:$PASSWORD@tcp(127.0.0.1:$PORT)/mysql" go test -tags nofrontend ./test/... -run MySQL -count=1 {{.CLI_ARGS}}
RESULT=$?
docker rm -f "$NAME" >/dev/null 2>&1
return $RESULT
}
run_engine mysql mysql:8.4 || STATUS=1
run_engine mariadb mariadb:11 || STATUS=1
exit $STATUS
silent: true
cover:
desc: Go test coverage of the use cases → tmp/coverage.out
cmds:
+31 -1
View File
@@ -117,8 +117,38 @@ func openImageStorage(settings config.Settings) domainImage.Storage {
return imageStorage
}
func databaseConfig(settings config.Settings) database.Config {
switch settings.Database.Driver {
case config.DatabaseDriverMySQL:
return mysqlDatabaseConfig(settings.Database)
case "", config.DatabaseDriverSQLite:
return database.Config{Driver: database.DriverSQLite, Source: settings.DBFile}
default:
return database.Config{Driver: settings.Database.Driver}
}
}
func mysqlDatabaseConfig(settings config.DatabaseSettings) database.Config {
return database.Config{
Driver: database.DriverMySQL,
Source: database.MySQLDSN(database.MySQLConfig{
Host: settings.Host,
Port: settings.Port,
User: settings.User,
Password: settings.Password,
Name: settings.Name,
TLS: settings.TLS,
}),
Pool: &database.PoolSettings{
MaxOpenConns: settings.MaxOpenConns,
MaxIdleConns: settings.MaxIdleConns,
ConnMaxLifetime: settings.ConnMaxLifetime,
},
}
}
func openDatabase(settings config.Settings) *database.Connection {
connection, err := database.Connect(database.Config{Driver: database.DriverSQLite, Source: settings.DBFile})
connection, err := database.Connect(databaseConfig(settings))
if err != nil {
panic("Unable to open the database: " + err.Error())
}
+10 -4
View File
@@ -12,6 +12,7 @@ import (
"net/http"
"os"
"os/signal"
"slices"
"syscall"
"time"
@@ -44,6 +45,12 @@ var envDefaults = map[string]string{
"ENVIRONMENT": "production",
"PORT": "8080",
"DB_FILE": "./data/renewcms.db",
config.EnvDatabaseDriver: config.DefaultDatabaseDriver,
"DB_PORT": config.DefaultDatabasePort,
"DB_TLS": config.DefaultDatabaseTLS,
"DB_MAX_OPEN_CONNS": config.DefaultMaxOpenConns,
"DB_MAX_IDLE_CONNS": config.DefaultMaxIdleConns,
"DB_CONN_MAX_LIFETIME_SECONDS": config.DefaultConnMaxLifetimeSeconds,
"UPLOAD_DIR": "./data/uploads",
"AUTH_RATE_LIMIT_MAX_ATTEMPTS": "5",
"AUTH_RATE_LIMIT_WINDOW_SECONDS": "60",
@@ -108,10 +115,9 @@ func applyEnvDefaults() {
}
func ensureRequiredEnvVars() {
for _, envVar := range requiredEnvVars {
if _, isSet := os.LookupEnv(envVar); !isSet {
panic(fmt.Sprintf("Environment variable %s is not set", envVar))
}
required := slices.Concat(requiredEnvVars, config.DatabaseRequiredEnvVars(os.Getenv(config.EnvDatabaseDriver)))
if err := config.MissingEnvVarError(required, os.LookupEnv); err != nil {
panic(err.Error())
}
}
+51
View File
@@ -0,0 +1,51 @@
services:
mysql:
image: mysql:8.4
restart: unless-stopped
environment:
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?set MYSQL_ROOT_PASSWORD}
MYSQL_DATABASE: renewcms
MYSQL_USER: renewcms
MYSQL_PASSWORD: ${DB_PASSWORD:?set DB_PASSWORD}
volumes:
- mysql-data:/var/lib/mysql
healthcheck:
test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u root -p\"$$MYSQL_ROOT_PASSWORD\" --silent"]
interval: 10s
timeout: 5s
retries: 10
start_period: 30s
renewcms:
image: ${RENEWCMS_IMAGE:-git.floriansylvain.fr/fsylvain/renewcms:beta}
restart: unless-stopped
depends_on:
mysql:
condition: service_healthy
env_file:
- docker.env
environment:
DB_DRIVER: mysql
DB_HOST: mysql
DB_PORT: "3306"
DB_NAME: renewcms
DB_USER: renewcms
DB_PASSWORD: ${DB_PASSWORD:?set DB_PASSWORD}
DATA_DIR: /data
UPLOAD_DIR: /data/uploads
RENEWCMS_CONTAINER: "true"
ports:
- "8080:8080"
volumes:
- renewcms-data:/data
stop_grace_period: 15s
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/healthz"]
interval: 30s
timeout: 5s
retries: 3
start_period: 10s
volumes:
renewcms-data:
mysql-data:
+11
View File
@@ -11,3 +11,14 @@ CORS_ALLOWED_ORIGINS=
AUTH_RATE_LIMIT_MAX_ATTEMPTS=5
AUTH_RATE_LIMIT_WINDOW_SECONDS=60
TRUSTED_PROXIES=
DB_DRIVER=sqlite
DATA_DIR=
DB_HOST=
DB_PORT=3306
DB_USER=
DB_PASSWORD=
DB_NAME=
DB_TLS=false
DB_MAX_OPEN_CONNS=10
DB_MAX_IDLE_CONNS=5
DB_CONN_MAX_LIFETIME_SECONDS=300
+14 -2
View File
@@ -80,6 +80,18 @@ bin/renewcms ── /v1/* API
Uploaded images are addressed by a stored name (`<uuid>.<ext>`). `CreateImageUseCase` calls `Storage.Put` (which generates the name), then `Repository.Create` with `Path = /static/uploadedImages/<name>`, and deletes the blob again if the row cannot be created. `DeleteMediaUseCase` and `MergeMediaUseCase` remove the blob first and the row second (`mediaRemover`), so a failed blob delete keeps the row and the delete can be retried. `GET /static/uploadedImages/<name>` is served by `OpenMediaUseCase` through `api/handlers/staticimages`, which uses `http.ServeContent` when the blob is seekable and streams it otherwise; `ErrBlobNotFound` maps to 404 and any other error to 500.
SQL in repositories must run on SQLite, MySQL/MariaDB and PostgreSQL: substring searches use `LIKE ... ESCAPE '!'` through `containsPattern` instead of `instr`. `Settings.DataDir()` is where update state, staging and backups live; it is derived from `DB_FILE` today and does not depend on where the database is.
SQL in repositories must run on SQLite, MySQL/MariaDB and PostgreSQL: substring searches use `LIKE ... ESCAPE '!'` through `containsPattern` instead of `instr`. `Settings.DataDir()` is where update state, staging and backups live; it is `DATA_DIR` when set and the directory of `DB_FILE` otherwise, so it does not depend on where the database is.
Adding a database driver: implement `database.Driver` (`Open`, and `NewBackup` returning a `domain/update.DatabaseBackup`), register it in `drivers`, then select it in `openDatabase` in the composition root (and add the env vars to `main.go`, `.env.example` and `configuration.md`). Adding a storage backend: implement `image.Storage`, add a case to `storage.Open`, and select it in `openImageStorage`. No use case, handler or repository changes.
Two drivers are registered in `database.drivers`: `sqlite` (default) and `mysql` (MySQL 8 and MariaDB 11, through `gorm.io/driver/mysql`, pure Go so `CGO_ENABLED=0` and the six release targets still build). `DB_DRIVER` picks one in `databaseConfig` (composition root); an unknown name fails in `database.Connect` with `unsupported database driver "<name>"`. `database.MySQLDSN` builds the DSN from separate variables, and the driver redacts the password from every error and names only host and port. `Connect` applies the optional `PoolSettings` to any driver. What keeps the repositories identical on both engines, all without touching the domain, use cases or handlers:
- Indexed and unique strings carry `size:` tags in the models (a MySQL index needs a bounded `varchar`). SQLite maps them to the same `text` column, so an existing SQLite file migrates with no schema or row change (`test/sqlite_schema_unchanged_test.go` builds a file from the old model definitions and compares `sqlite_master` and every row).
- `Migrate` creates MySQL tables with `utf8mb4_bin` (`gorm:table_options`), so `Bob` and `bob` are distinct like in SQLite. The DSN also sets `collation=utf8mb4_bin`.
- The DSN sets `clientFoundRows=true`: an update that matches a row but changes nothing still reports one affected row, as in SQLite. `InviteRepository.Renew`, `deleteOrNotFound` and `PurgeSettingsRepository.SaveLastRun` depend on that.
- The Go driver sends a zero `time.Time` as `0000-00-00`, which MySQL's default `NO_ZERO_DATE` rejects, so each session sets `sql_mode` to the server default minus `NO_ZERO_DATE`. Reading `0000-00-00` gives the zero time back.
- `UserRepository.Create` returns the filled model instead of re-reading it, because MySQL has no `RETURNING`.
Backups are a `domain/update.DatabaseBackup` per driver. SQLite uses `VACUUM INTO`. MySQL uses `DumpBackup`, a pure Go logical dump (`SHOW CREATE TABLE` plus every row, strings escaped and binary values as hex, NULL and zero dates kept) written inside one repeatable-read transaction to `DATA_DIR/backups/renewcms-<time>.sql.partial` and renamed once complete. Any failure removes the partial file and returns an error, so the update download fails instead of claiming a backup. Both keep the newest 3 files (`pruneBackups`).
`task test` runs everything that needs no server. The repository contract tests (`test/repository_contract_test.go`) run on SQLite always, and on MySQL when `RENEWCMS_TEST_MYSQL_DSN` points at a server (admin DSN; each test creates and drops its own database). `task test:mysql` starts `mysql:8.4` and then `mariadb:11` in Docker and runs the `MySQL` tests against each.
Adding a database driver: implement `database.Driver` (`Open`, and `NewBackup` returning a `domain/update.DatabaseBackup`), register it in `drivers`, then select it in `databaseConfig` in the composition root (and add the env vars to `main.go`, `.env.example` and `configuration.md`). Adding a storage backend: implement `image.Storage`, add a case to `storage.Open`, and select it in `openImageStorage`. No use case, handler or repository changes.
+27 -3
View File
@@ -24,17 +24,41 @@ Copy [`.env.example`](../.env.example) to `.env` to get started. When a required
|------------------------|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `ENVIRONMENT` | `production` | `production` or `development`. In `production` the auth cookie is `Secure`, so it is only sent over HTTPS. `development` turns on HTTP request logging. |
| `PORT` | `8080` | Port the HTTP server listens on (all interfaces). |
| `DB_FILE` | `./data/renewcms.db` | Path to the SQLite database file. Missing parent directories are created. The schema is migrated automatically at startup. |
| `DB_DRIVER` | `sqlite` | `sqlite` or `mysql` (MySQL and MariaDB). Any other value stops the server with `unsupported database driver "<value>"`. See [Database](#database). |
| `DB_FILE` | `./data/renewcms.db` | Path to the SQLite database file. Missing parent directories are created. The schema is migrated automatically at startup. Ignored when `DB_DRIVER=mysql`. |
| `DATA_DIR` | directory of `DB_FILE` | Directory for update state, staged updates and database backups (`backups/`). Set it with `DB_DRIVER=mysql`, where there is no database file to sit beside (the default still follows `DB_FILE`). With SQLite, leave it unset to keep everything beside the database. |
| `UPLOAD_DIR` | `./data/uploads` | Directory where uploaded images are stored. |
| `CORS_ALLOWED_ORIGINS` | *(empty)* | Semicolon-separated list of origins allowed to call the API. When empty, no CORS headers are sent. That is the right setting whenever the SPA is served by RenewCMS itself (production) or through the Vite proxy (development). |
| `AUTH_RATE_LIMIT_MAX_ATTEMPTS` | `5` | Maximum number of requests a single client IP may send to `POST /v1/auth/login`, `POST /v1/auth/password-reset` or the invite-accept pair (`GET /v1/auth/invite` and `POST /v1/auth/invite/accept`, sharing one counter) within the window, tracked independently per route. |
| `AUTH_RATE_LIMIT_WINDOW_SECONDS` | `60` | Length, in seconds, of the fixed rate-limit window for the routes above. |
| `TRUSTED_PROXIES` | *(empty)* | Semicolon-separated list of exact IP addresses (no CIDR) of reverse proxies allowed to supply the client IP via `X-Real-IP` for the rate limiting of the routes above. When a request's connection address is not in this list, any `X-Real-IP` or `X-Forwarded-For` header it sends is ignored and the connection address is used instead. See [deployment.md](deployment.md). |
| `RENEWCMS_CONTAINER` | `false` | Set to `true` (case-insensitive; any other value means `false`) when the app runs in a container. Settings then tells the owner to update by pulling a new image instead of offering browser updates, and `can_apply` is always `false`. The Dockerfile and `docker-compose.yml` set it to `true`. Update checks still work. |
| `UPDATE_FEED_URL` | `https://git.floriansylvain.fr` | Base URL of the Gitea server that publishes RenewCMS releases. The Settings page asks `{UPDATE_FEED_URL}/api/v1/repos/fsylvain/RenewCMS/releases` when the owner checks for updates. The last result is kept in `update-status.json`, next to `DB_FILE`. |
| `UPDATE_FEED_URL` | `https://git.floriansylvain.fr` | Base URL of the Gitea server that publishes RenewCMS releases. The Settings page asks `{UPDATE_FEED_URL}/api/v1/repos/fsylvain/RenewCMS/releases` when the owner checks for updates. The last result is kept in `update-status.json`, in the data directory. |
Automatic media purge has no environment variable. Its schedule (on or off, interval, `enabled_at`, `last_purge_at`) lives in the purge settings row of the database, is changed from the Settings page or `PATCH /v1/media/purge-settings`, and is read again at every hourly tick.
## Database
SQLite is the default and needs nothing but `DB_FILE`. To run against MySQL 8 or MariaDB 11 instead, set `DB_DRIVER=mysql` and the connection variables below. They are only read (and the four required ones only checked) when `DB_DRIVER=mysql`. A required variable that is not set stops the server with `Environment variable DB_HOST is not set` (or `DB_USER`, `DB_PASSWORD`, `DB_NAME`).
| Name | Default | Description |
|--------------------------------|----------|----------------------------------------------------------------------------------------------------------------------------------|
| `DB_HOST` | required | Server host name or IP address. |
| `DB_PORT` | `3306` | Server port. |
| `DB_USER` | required | Database user. It needs full rights on `DB_NAME`, because the schema is created and migrated at startup. |
| `DB_PASSWORD` | required | Database password. It is never logged and never appears in an error message. |
| `DB_NAME` | required | Name of an existing database. The server does not create it. |
| `DB_TLS` | `false` | `false`, `true`, `skip-verify` or `preferred`. Any other value stops the server with an error naming `DB_TLS` and these values. `true` verifies the server certificate; `skip-verify` encrypts without verifying it. |
| `DB_MAX_OPEN_CONNS` | `10` | Maximum open connections in the pool. `0` means unlimited. |
| `DB_MAX_IDLE_CONNS` | `5` | Maximum idle connections kept in the pool. |
| `DB_CONN_MAX_LIFETIME_SECONDS` | `300` | Seconds after which a connection is closed and replaced. `0` keeps connections forever. Keep it below the server's `wait_timeout`. |
A non numeric (or negative) pool value stops the server with an error naming the variable. There is no DSN variable: the connection string is built from the variables above, with `parseTime=true`, `loc=UTC`, `charset=utf8mb4`, `collation=utf8mb4_bin` and `clientFoundRows=true`. Connecting uses a 5 second timeout, so a wrong host, port, password or database name stops startup quickly with `Unable to open the database`, naming the host and port but never the password.
Why those settings: tables are created with the `utf8mb4_bin` collation so that usernames and emails compare case-sensitively exactly like in SQLite (`Bob` and `bob` are different users). The collation of `DB_NAME` itself does not matter, but tables that already exist keep the collation they have. `clientFoundRows` makes an update that does not change a row still count as matched, which the invite and purge-settings code relies on. Each session also drops the server's `NO_ZERO_DATE` SQL mode, because the Go driver sends a zero time as `0000-00-00`. The other strict modes stay on.
With MySQL the data directory (`DATA_DIR`) holds update state and backups only. The pre-update backup is a `renewcms-<time>.sql` logical dump (schema and rows, written by RenewCMS itself, no `mysqldump` needed) that keeps the newest 3. If the dump fails, the update download fails with it. The dump holds your data, such as password hashes, but not the connection password.
Relative paths are resolved against the process's working directory. In production, use absolute paths (see [deployment.md](deployment.md)).
For example, `WorkingDirectory=/var/lib/renewcms` with `DB_FILE=./data/renewcms.db` puts the database in `/var/lib/renewcms/data`. Browser-update `--data-dir` must name that directory, because the app reads helper status and update records beside the database. It is not necessarily `WorkingDirectory` or `ReadWritePaths`. Do not change an existing `DB_FILE` merely to match the helper: that could open a different, empty database. See [browser-update setup](deployment.md#browser-updates).
@@ -42,7 +66,7 @@ For example, `WorkingDirectory=/var/lib/renewcms` with `DB_FILE=./data/renewcms.
## Adding a variable
1. Read it in the code where it is needed (infrastructure or `api/`, never `internal/domain`).
2. Register it in [`cmd/server/main.go`](../cmd/server/main.go): add it to `requiredEnvVars` if it is required, or to `envDefaults` if it has a default.
2. Register it in [`cmd/server/main.go`](../cmd/server/main.go): add it to `requiredEnvVars` if it is required (or to `DatabaseRequiredEnvVars` in `internal/infrastructure/config` if only one database driver needs it), or to `envDefaults` if it has a default.
3. Add it to [`.env.example`](../.env.example) and to the tables above.
Never hardcode a secret, and never commit `.env`.
+30 -1
View File
@@ -8,7 +8,7 @@ RenewCMS ships as **one self-contained executable**. It serves:
| `/healthz` | Liveness check, returns `200 ok` |
| `/*` | The Vue SPA (embedded), with fallback to `index.html` for client-side routes |
Runtime state includes the SQLite database (`DB_FILE`), uploads (`UPLOAD_DIR`), and update records, staged binaries and backups beside the database. Running a release binary needs no Go, Bun or Node installation. Browser updates additionally require Linux with systemd and the helper setup below.
Runtime state includes the SQLite database (`DB_FILE`, or a MySQL/MariaDB server, see [Using MySQL or MariaDB](#using-mysql-or-mariadb)), uploads (`UPLOAD_DIR`), and update records, staged binaries and backups beside the database. Running a release binary needs no Go, Bun or Node installation. Browser updates additionally require Linux with systemd and the helper setup below.
To run it in a container instead, see [docker.md](docker.md). The published image `git.floriansylvain.fr/fsylvain/renewcms` is upgraded with `docker compose pull && docker compose up -d`. Tags are `vX.Y.Z`, `X.Y.Z`, `X.Y` and `latest` for a stable release, and `vX.Y.Z-<pre>`, `X.Y.Z-<pre>` and the moving `<pre-name>` (`beta`, `rc`) for a pre-release, which never moves `latest`. Maintainers publish with `task docker:publish` after `docker login git.floriansylvain.fr`; there is no CI. See [Maintainer: publishing](docker.md#maintainer-publishing).
@@ -276,8 +276,37 @@ RenewCMS sends `Content-Security-Policy` (including `frame-ancestors 'none'`), `
The CSP allows Google Fonts (`fonts.googleapis.com`, `fonts.gstatic.com`) and `https:` images, because the SPA loads its fonts from Google and articles may embed remote images. `style-src` keeps `'unsafe-inline'` for the article editor; `script-src` is `'self'` only.
## Using MySQL or MariaDB
SQLite is the default. To use an existing MySQL 8 or MariaDB 11 server instead, create an empty database and a user with full rights on it, then set in `/etc/renewcms/renewcms.env`:
```sh
DB_DRIVER=mysql
DB_HOST=db.example.com
DB_USER=renewcms
DB_PASSWORD=<generated password>
DB_NAME=renewcms
DB_TLS=true
DATA_DIR=/var/lib/renewcms
```
`DB_FILE` is then ignored. `DATA_DIR` is where update state, staged updates and backups live, so give it the same value as the helper's `--data-dir` (see [browser updates](#browser-updates)). Use `DB_TLS=true` (or `preferred`) whenever the server is not on the same host. Every variable is described in [configuration.md](configuration.md#database). The schema is created and migrated at startup, so the first start on an empty database needs nothing else.
### Moving an existing SQLite site to MySQL
There is no migration tool. The manual procedure, with the site stopped:
1. Take a backup of the data directory (database file and uploads).
2. Start RenewCMS once on the empty MySQL database so it creates the schema, then stop it.
3. Export the rows from the SQLite file with `sqlite3 renewcms.db .dump`, keep only the `INSERT INTO` statements for `articles`, `images`, `users`, `invites`, `api_keys` and `purge_settings`, and import them into MySQL. Quoting and date formats differ between the two engines, so check a sample first: SQLite writes times such as `2026-01-02 03:04:05.123456789+00:00`, and the `+00:00` suffix must be removed for MariaDB and older MySQL versions. An unset time (`0001-01-01 00:00:00`) is valid as it is in MySQL and is read back as unset.
4. Copy `uploads/` to the new `UPLOAD_DIR`, set `DB_DRIVER=mysql` and the connection variables, and start the service.
Keep the SQLite file until you have checked the site.
## Backups
With MySQL or MariaDB, back the database up with your server's own tools (for example `mysqldump --single-transaction`), plus the uploads directory. RenewCMS also writes a logical dump to `DATA_DIR/backups` before each update download. The rest of this section is about SQLite.
Everything worth backing up is under the data directory. Take a consistent SQLite copy while the service is running:
```sh
+1
View File
@@ -55,6 +55,7 @@ If your editor reports `pattern all:dist: no matching files found`, either run `
```sh
task --list # everything available
task test # Go tests; extra args pass through: task test -- -run TestCreateArticle -v
task test:mysql # MySQL tests against real servers in Docker (mysql:8.4, then mariadb:11)
task cover # coverage of the use cases → tmp/coverage.out
task mutate # gremlins mutation testing → tmp/gremlins.json
task test:web # SPA component tests (Vitest)
+17 -1
View File
@@ -62,7 +62,7 @@ The container receives SIGTERM on stop, and RenewCMS shuts down gracefully withi
## Data volume
State lives on the named volume `renewcms-data` (Compose prefixes it with the project name, usually `renewcms_renewcms-data`; check with `docker volume ls`). It holds the SQLite database, uploads, and update records.
State lives on the named volume `renewcms-data` (Compose prefixes it with the project name, usually `renewcms_renewcms-data`; check with `docker volume ls`). It holds the SQLite database (unless you use MySQL), uploads, and update records.
- `docker compose down` removes the container and keeps the volume. The data survives and `docker compose up -d` brings it back.
- **`docker compose down -v` deletes the volume and all data.** The next start is an empty instance.
@@ -86,6 +86,22 @@ docker run --rm -v VOLUME:/data -v "$PWD":/backup alpine sh -c "tar xzf /backup/
docker compose up -d
```
## MySQL or MariaDB
SQLite on the `/data` volume is the default. [`docker-compose.mysql.yml`](../docker-compose.mysql.yml) runs RenewCMS next to a MySQL 8.4 container instead:
```sh
export MYSQL_ROOT_PASSWORD=<generated>
export DB_PASSWORD=<generated>
docker compose -f docker-compose.mysql.yml up -d
```
It starts the `mysql` service with a named volume `mysql-data` and a healthcheck, and starts `renewcms` only once the database is healthy (`depends_on` with `service_healthy`). RenewCMS gets `DB_DRIVER=mysql`, `DB_HOST=mysql`, `DB_NAME`, `DB_USER` and `DB_PASSWORD` from the compose file and the rest from `docker.env`. `DATA_DIR=/data` keeps update state and the `.sql` pre-update backups on the `renewcms-data` volume, which still holds the uploads. Replace the placeholders; never commit real passwords.
To use MariaDB, change the `mysql` service image to `mariadb:11`, its `MYSQL_*` variables to `MARIADB_*` and its healthcheck to `healthcheck.sh --connect --innodb_initialized`. To use a server outside Compose, drop the `mysql` service and point `DB_HOST` at it. For a server on another machine set `DB_TLS=true`.
Back the database up with `docker compose -f docker-compose.mysql.yml exec mysql mysqldump ...` (a `tar` copy of the volume is not consistent while it runs), and the `renewcms-data` volume as described above. Moving an existing SQLite volume is a manual procedure, see [deployment.md](deployment.md#moving-an-existing-sqlite-site-to-mysql).
## Reverse proxy and TLS
With `ENVIRONMENT=production` the site must be served over HTTPS, so put a reverse proxy in front and set `PUBLIC_URL` to the public `https://` address. Bind the published port to the host only when the proxy runs on the same host, by changing `ports` to `"127.0.0.1:8080:8080"`.
+160
View File
@@ -0,0 +1,160 @@
Feature: Optional MySQL/MariaDB database driver
An operator who cannot use a local SQLite file runs RenewCMS against MySQL or
MariaDB by setting environment variables. SQLite remains the default and
behaves exactly as before.
Background:
Given the database drivers are registered by name in database.drivers
And the database is chosen by the DB_DRIVER environment variable
And repositories, use cases and handlers are unchanged
Scenario Outline: Driver selection
Given DB_DRIVER is "<value>"
When the server starts
Then the outcome is "<outcome>"
Examples:
| value | outcome |
| unset | the sqlite driver opens DB_FILE |
| sqlite | the sqlite driver opens DB_FILE |
| mysql | the mysql driver opens the DB_HOST connection |
| oracle | startup fails with: unsupported database driver "oracle" |
Scenario Outline: Missing MySQL variable
Given DB_DRIVER is "mysql"
And "<missing>" is not set
When the environment is validated
Then startup fails with "Environment variable <missing> is not set"
Examples:
| missing |
| DB_HOST |
| DB_USER |
| DB_PASSWORD |
| DB_NAME |
Scenario: SQLite does not require MySQL variables
Given DB_DRIVER is "sqlite"
And DB_HOST, DB_USER, DB_PASSWORD and DB_NAME are not set
When the environment is validated
Then validation succeeds
Scenario Outline: DSN built from variables
Given DB_HOST "<host>", DB_PORT "<port>", DB_USER "app", DB_NAME "cms", DB_TLS "<tls>"
When the MySQL DSN is built
Then the DSN targets "<host>:<expectedPort>" and database "cms"
And it contains parseTime=true, loc=UTC, charset=utf8mb4, clientFoundRows=true
And it contains tls="<tls>"
Examples:
| host | port | expectedPort | tls |
| db.local | | 3306 | false |
| db.local | 3307 | 3307 | true |
| mysql | 3306 | 3306 | skip-verify |
Scenario: Invalid TLS option is rejected
Given DB_DRIVER is "mysql" and DB_TLS is "maybe"
When the settings are read
Then startup fails naming DB_TLS and the accepted values
Scenario Outline: Fail fast when the server cannot be used
Given DB_DRIVER is "mysql" and "<fault>"
When the server starts
Then startup stops with "Unable to open the database"
And the message names the host and port
And the message does not contain the DB_PASSWORD value
Examples:
| fault |
| nothing listens on DB_PORT |
| the password is wrong |
| the database does not exist |
Scenario: Data directory with MySQL
Given DB_DRIVER is "mysql" and DATA_DIR is "/data"
Then update state, staging and backups live in "/data"
And DB_FILE is ignored
Scenario Outline: DataDir resolution
Given DB_FILE "<dbfile>" and DATA_DIR "<datadir>"
Then Settings.DataDir() is "<result>"
Examples:
| dbfile | datadir | result |
| ./data/renewcms.db | | ./data |
| /var/lib/cms/cms.db | | /var/lib/cms |
| ./data/renewcms.db | /srv/x | /srv/x |
Scenario: Connection pool
Given DB_MAX_OPEN_CONNS 4, DB_MAX_IDLE_CONNS 2 and DB_CONN_MAX_LIFETIME_SECONDS 60
When the MySQL connection is opened
Then the pool limits are 4 open, 2 idle and 60 seconds
Scenario: Migration on MySQL
Given an empty MySQL database
When database.Migrate runs
Then the tables articles, images, users, invites, api_keys and purge_settings exist
And the unique indexes on users and api_keys exist
And the first user lacking a role becomes "owner" and the others "admin"
Scenario: SQLite schema is untouched by the model changes
Given a SQLite file created by the models before this change, holding rows
When database.Migrate runs
Then the schema and every row are unchanged
Scenario Outline: Repository behaviour is the same on both drivers
Given the "<driver>" driver
Then the repository contract "<case>" holds
Examples:
| driver | case |
| sqlite | create user and read it back with its id and timestamps |
| mysql | create user and read it back with its id and timestamps |
| sqlite | username "Bob" and "bob" are distinct users |
| mysql | username "Bob" and "bob" are distinct users |
| sqlite | duplicate email returns an error and stores nothing |
| mysql | duplicate email returns an error and stores nothing |
| sqlite | unknown id returns domain ErrUserNotFound |
| mysql | unknown id returns domain ErrUserNotFound |
| sqlite | accept invite twice: the second returns ErrInviteNotFound |
| mysql | accept invite twice: the second returns ErrInviteNotFound |
| sqlite | accept invite with a taken username leaves the invite unaccepted |
| mysql | accept invite with a taken username leaves the invite unaccepted |
| sqlite | renew invite with unchanged values still succeeds |
| mysql | renew invite with unchanged values still succeeds |
| sqlite | duplicate image hashes are grouped and counted |
| mysql | duplicate image hashes are grouped and counted |
| sqlite | media search treats % and _ literally |
| mysql | media search treats % and _ literally |
| sqlite | article body of 1 MB round-trips |
| mysql | article body of 1 MB round-trips |
| sqlite | zero reset expiration round-trips as a zero time |
| mysql | zero reset expiration round-trips as a zero time |
| sqlite | purge settings save, then SaveLastRun with identical values succeeds |
| mysql | purge settings save, then SaveLastRun with identical values succeeds |
Scenario Outline: Backup before an update
Given the "<driver>" driver with a populated database
When the pre-update backup runs
Then a file "<pattern>" exists in DATA_DIR/backups
And only the 3 newest backups are kept
And a failure returns an error, so the update download fails instead of claiming a backup
Examples:
| driver | pattern |
| sqlite | renewcms-<time>.db |
| mysql | renewcms-<time>.sql |
Scenario: MySQL backup content
Given a MySQL database with 2 users and 1 article
When the pre-update backup runs
Then the dump contains the schema and the rows of every table
And it does not contain the DB_PASSWORD value
And replaying it into an empty database restores the same rows
Scenario: Secrets are never exposed
Given DB_DRIVER is "mysql"
Then no log line, error message or file in the repository contains the DB_PASSWORD value
Scenario: Static builds
Then the module builds with CGO_ENABLED=0 for the 6 task release targets
+42
View File
@@ -0,0 +1,42 @@
# QA: MySQL/MariaDB database driver
Backend only. Go tests in test/, black-box, no live server by default.
Decisions: separate DB_* vars, new DATA_DIR (DB_FILE ignored under mysql), pure-Go logical dump backup, `task test:mysql` against mysql:8.4 and mariadb:11, SQLite-to-MySQL data migration documented as manual export/import only.
## Automated, no server (task test)
1. Registry: Connect with Driver "sqlite" works against a temp file; Driver "oracle" returns `unsupported database driver "oracle"`; the registry contains sqlite and mysql.
2. Env validation: table-driven over the Driver selection and Missing MySQL variable scenarios. SQLite needs no DB_* vars. DB_DRIVER defaults to sqlite. An invalid DB_TLS or a non-numeric pool var returns an error naming the variable.
3. DSN builder: table-driven over the DSN scenario. Assert host:port, database, parseTime, loc=UTC, charset=utf8mb4, clientFoundRows, tls. Assert that the redacted form used in errors never contains the password.
4. Settings.DataDir: table-driven (DB_FILE only, DATA_DIR overrides).
5. Fail fast without a server: point mysql at a closed local port with a 1s timeout. Assert the error mentions the host and port, and not the password.
6. Pool settings: assert sql.DB stats MaxOpenConnections via a stub opener, or via the real mysql test below.
7. SQLite regression: build a database file using the pre-change schema (a fixture created with the old model definitions), run Migrate, and assert that the schema (sqlite_master) and the rows are unchanged. The existing suite must pass unchanged.
8. Backup (SQLite): existing tests unchanged.
9. Dump writer: unit-test the SQL dump formatter with a fake row source (quoting, NULL, time, bool, binary-safe strings, 3-file retention, existing-file handling, failure leaves no partial file).
## Automated, real server (task test:mysql)
Setup: the task runs `docker run` for mysql:8.4 and, in a second pass, mariadb:11, waits for readiness, exports RENEWCMS_TEST_MYSQL_DSN, runs `go test -tags nofrontend ./test/... -run MySQL -count=1`, then removes the container. Tests call t.Skip when the var is unset, so `task test` skips them. The repository tests are parameterised over both drivers (SQLite temp file and the MySQL DSN, each with a fresh database), and run the "Repository behaviour is the same on both drivers" Examples.
10. Migrate on an empty database; table and index existence via information_schema; the backfill role rules.
11. User create/read, case-sensitive username and email, duplicate email, ErrUserNotFound.
12. Invite accept twice, accept with a taken username (invite stays unaccepted), renew with unchanged values.
13. Image duplicates grouping and counts, search with % and _, filename backfill listing.
14. Article body 1 MB, title lookup, online filter.
15. API key unique hashed_key, revoke, last_used.
16. Zero-time round trip, UTC times.
17. Purge settings upsert.
18. Backup: run on a populated MySQL database, assert the .sql file, the 3-file retention, a replay into a second empty database giving identical rows, and that the file contains no password.
19. Wrong password and missing database fail with the "Unable to open the database" prefix and no password in the message.
## Build and lint
20. `task lint:go` and `task test` pass. `task test:mysql` passes on MySQL 8.4 and MariaDB 11 (record both; NOT RUN if Docker is unavailable).
21. `CGO_ENABLED=0 go build -tags nofrontend ./...` passes. Cross-compile the 6 `task release` targets (e.g. `GOOS=linux GOARCH=arm64`) with CGO_ENABLED=0.
22. `task mutate` impact: none, because it only targets useCases. No change.
23. No comment lines in the added Go files (the hook enforces this).
## Manual (needs Compose and real SMTP)
24. docker-compose.mysql.yml example: renewcms plus a mysql service with a named volume and a healthcheck, `depends_on: condition: service_healthy`, DB_DRIVER=mysql, DB_HOST=mysql, DATA_DIR=/data. `docker compose up`: /healthz returns 200, register, log in, create an article, restart the stack, and the data persists.
25. Stop the mysql container and start renewcms: it exits with a clear error and no password in the logs. docker.env.example holds placeholders only.
26. In the Docker image (non-root uid 10001), trigger the backup path and confirm the .sql file lands under /data/backups.
27. Docs: configuration.md, docker.md, deployment.md, `.env.example`, `docker.env.example` and the architecture.md "Storage and database seams" section list the new vars and the manual SQLite to MySQL export/import note.
28. Verify the SQLite default end to end with no new vars set: same behaviour as before.
+4 -1
View File
@@ -9,16 +9,20 @@ require (
github.com/go-chi/cors v1.2.2
github.com/go-chi/jwtauth/v5 v5.3.3
github.com/go-playground/validator/v10 v10.28.0
github.com/go-sql-driver/mysql v1.8.1
github.com/google/uuid v1.6.0
github.com/joho/godotenv v1.5.1
github.com/lestrrat-go/jwx/v2 v2.1.6
golang.org/x/crypto v0.57.0
golang.org/x/image v0.46.0
golang.org/x/net v0.59.0
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gorm.io/driver/mysql v1.6.0
gorm.io/gorm v1.31.1
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
@@ -35,7 +39,6 @@ require (
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/httprc v1.0.6 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/jwx/v2 v2.1.6 // indirect
github.com/lestrrat-go/option v1.0.1 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
+6
View File
@@ -1,3 +1,5 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/MadAppGang/httplog v1.3.0 h1:1XU54TO8kiqTeO+7oZLKAM3RP/cJ7SadzslRcKspVHo=
github.com/MadAppGang/httplog v1.3.0/go.mod h1:gpYEdkjh/Cda6YxtDy4AB7KY+fR7mb3SqBZw74A5hJ4=
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4=
@@ -31,6 +33,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688=
github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
@@ -105,6 +109,8 @@ gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkp
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg=
gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
@@ -0,0 +1,122 @@
package config
import (
"fmt"
"os"
"slices"
"strconv"
"strings"
"time"
)
const (
DatabaseDriverSQLite = "sqlite"
DatabaseDriverMySQL = "mysql"
DefaultDatabaseDriver = DatabaseDriverSQLite
DefaultDatabasePort = "3306"
DefaultDatabaseTLS = "false"
DefaultMaxOpenConns = "10"
DefaultMaxIdleConns = "5"
DefaultConnMaxLifetimeSeconds = "300"
EnvDatabaseDriver = "DB_DRIVER"
envDatabaseHost = "DB_HOST"
envDatabasePort = "DB_PORT"
envDatabaseUser = "DB_USER"
envDatabasePassword = "DB_PASSWORD"
envDatabaseName = "DB_NAME"
envDatabaseTLS = "DB_TLS"
envDatabaseMaxOpenConns = "DB_MAX_OPEN_CONNS"
envDatabaseMaxIdleConns = "DB_MAX_IDLE_CONNS"
envDatabaseConnMaxLifetimeSecs = "DB_CONN_MAX_LIFETIME_SECONDS"
)
var mysqlRequiredEnvVars = []string{envDatabaseHost, envDatabaseUser, envDatabasePassword, envDatabaseName}
var DatabaseTLSModes = []string{"false", "true", "skip-verify", "preferred"}
type DatabaseSettings struct {
Driver string
Host string
Port string
User string
Password string
Name string
TLS string
MaxOpenConns int
MaxIdleConns int
ConnMaxLifetime time.Duration
}
func DatabaseRequiredEnvVars(driver string) []string {
if driver == DatabaseDriverMySQL {
return slices.Clone(mysqlRequiredEnvVars)
}
return nil
}
func MissingEnvVarError(names []string, lookup func(string) (string, bool)) error {
for _, name := range names {
if _, isSet := lookup(name); !isSet {
return fmt.Errorf("Environment variable %s is not set", name)
}
}
return nil
}
func databaseDriverFromEnv() string {
return envOrDefault(EnvDatabaseDriver, DefaultDatabaseDriver)
}
func databaseSettingsFromEnv() (DatabaseSettings, error) {
driver := databaseDriverFromEnv()
if driver != DatabaseDriverMySQL {
return DatabaseSettings{Driver: driver}, nil
}
tls := envOrDefault(envDatabaseTLS, DefaultDatabaseTLS)
if !slices.Contains(DatabaseTLSModes, tls) {
return DatabaseSettings{}, fmt.Errorf("%s must be one of %s, got %q", envDatabaseTLS, strings.Join(DatabaseTLSModes, ", "), tls)
}
maxOpen, err := nonNegativeIntEnv(envDatabaseMaxOpenConns, DefaultMaxOpenConns)
if err != nil {
return DatabaseSettings{}, err
}
maxIdle, err := nonNegativeIntEnv(envDatabaseMaxIdleConns, DefaultMaxIdleConns)
if err != nil {
return DatabaseSettings{}, err
}
lifetimeSeconds, err := nonNegativeIntEnv(envDatabaseConnMaxLifetimeSecs, DefaultConnMaxLifetimeSeconds)
if err != nil {
return DatabaseSettings{}, err
}
return DatabaseSettings{
Driver: driver,
Host: os.Getenv(envDatabaseHost),
Port: envOrDefault(envDatabasePort, DefaultDatabasePort),
User: os.Getenv(envDatabaseUser),
Password: os.Getenv(envDatabasePassword),
Name: os.Getenv(envDatabaseName),
TLS: tls,
MaxOpenConns: maxOpen,
MaxIdleConns: maxIdle,
ConnMaxLifetime: time.Duration(lifetimeSeconds) * time.Second,
}, nil
}
func envOrDefault(name string, fallback string) string {
if value := os.Getenv(name); value != "" {
return value
}
return fallback
}
func nonNegativeIntEnv(name string, fallback string) (int, error) {
raw := envOrDefault(name, fallback)
value, err := strconv.Atoi(raw)
if err != nil || value < 0 {
return 0, fmt.Errorf("%s must be a non-negative whole number, got %q", name, raw)
}
return value, nil
}
@@ -28,6 +28,8 @@ type Settings struct {
UploadDir string
JWTSecret string
DBFile string
Database DatabaseSettings
DataDirPath string
Port string
Smtp SmtpSettings
AuthRateLimit AuthRateLimitSettings
@@ -38,6 +40,9 @@ type Settings struct {
}
func (s Settings) DataDir() string {
if s.DataDirPath != "" {
return s.DataDirPath
}
return filepath.Dir(s.DBFile)
}
@@ -61,6 +66,11 @@ func NewSettingsFromEnv() (Settings, error) {
return Settings{}, err
}
database, err := databaseSettingsFromEnv()
if err != nil {
return Settings{}, err
}
return Settings{
Environment: os.Getenv("ENVIRONMENT"),
CorsAllowedOrigins: parseSemicolonSeparatedList(os.Getenv("CORS_ALLOWED_ORIGINS")),
@@ -68,6 +78,8 @@ func NewSettingsFromEnv() (Settings, error) {
UploadDir: os.Getenv("UPLOAD_DIR"),
JWTSecret: os.Getenv("JWT_SECRET"),
DBFile: os.Getenv("DB_FILE"),
Database: database,
DataDirPath: os.Getenv("DATA_DIR"),
Port: os.Getenv("PORT"),
Smtp: SmtpSettings{
Login: os.Getenv("SMTP_LOGIN"),
@@ -0,0 +1,55 @@
package database
import (
"os"
"path/filepath"
"slices"
"strings"
"time"
)
type backupFile struct {
path string
modified time.Time
}
func newBackupTarget(directory string, now time.Time, extension string) (string, error) {
if err := os.MkdirAll(directory, os.ModePerm); err != nil {
return "", err
}
target := filepath.Join(directory, backupFilePrefix+now.UTC().Format(backupTimeLayout)+extension)
if _, err := os.Stat(target); err == nil {
return "", errBackupExists
}
return target, nil
}
func pruneBackups(directory string, extension string) error {
entries, err := os.ReadDir(directory)
if err != nil {
return err
}
files := make([]backupFile, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), extension) {
continue
}
info, err := entry.Info()
if err != nil {
return err
}
files = append(files, backupFile{path: filepath.Join(directory, entry.Name()), modified: info.ModTime()})
}
slices.SortFunc(files, func(a, b backupFile) int {
if byTime := b.modified.Compare(a.modified); byTime != 0 {
return byTime
}
return strings.Compare(b.path, a.path)
})
for _, stale := range files[min(backupsToKeep, len(files)):] {
if err := os.Remove(stale.path); err != nil {
return err
}
}
return nil
}
@@ -8,9 +8,16 @@ import (
"gorm.io/gorm"
)
type PoolSettings struct {
MaxOpenConns int
MaxIdleConns int
ConnMaxLifetime time.Duration
}
type Config struct {
Driver string
Source string
Pool *PoolSettings
}
type Driver interface {
@@ -20,6 +27,7 @@ type Driver interface {
var drivers = map[string]Driver{
DriverSQLite: sqliteDriver{},
DriverMySQL: mysqlDriver{},
}
type Connection struct {
@@ -36,9 +44,25 @@ func Connect(config Config) (*Connection, error) {
if err != nil {
return nil, err
}
if config.Pool != nil {
if err := applyPool(db, *config.Pool); err != nil {
return nil, err
}
}
return &Connection{DB: db, driver: driver}, nil
}
func applyPool(db *gorm.DB, pool PoolSettings) error {
sqlDb, err := db.DB()
if err != nil {
return err
}
sqlDb.SetMaxOpenConns(pool.MaxOpenConns)
sqlDb.SetMaxIdleConns(pool.MaxIdleConns)
sqlDb.SetConnMaxLifetime(pool.ConnMaxLifetime)
return nil
}
func (c *Connection) NewBackup(directory string, now func() time.Time) domainUpdate.DatabaseBackup {
return c.driver.NewBackup(c.DB, directory, now)
}
@@ -0,0 +1,79 @@
package database
import (
domain "RenewCMS/internal/domain/update"
"bufio"
"os"
"time"
)
const (
dumpFileExtension = ".sql"
dumpPartialFileExtension = ".partial"
)
type DumpBackup struct {
open func() (DumpSource, error)
directory string
now func() time.Time
}
var _ domain.DatabaseBackup = &DumpBackup{}
func NewDumpBackup(open func() (DumpSource, error), directory string, now func() time.Time) *DumpBackup {
return &DumpBackup{open: open, directory: directory, now: now}
}
func (b *DumpBackup) Backup() (string, error) {
target, err := newBackupTarget(b.directory, b.now(), dumpFileExtension)
if err != nil {
return "", err
}
if err := b.writeAtomically(target); err != nil {
return "", err
}
if err := pruneBackups(b.directory, dumpFileExtension); err != nil {
return "", err
}
return target, nil
}
func (b *DumpBackup) writeAtomically(target string) error {
partial := target + dumpPartialFileExtension
if err := b.writePartial(partial); err != nil {
_ = os.Remove(partial)
return err
}
if err := os.Rename(partial, target); err != nil {
_ = os.Remove(partial)
return err
}
return nil
}
func (b *DumpBackup) writePartial(partial string) error {
source, err := b.open()
if err != nil {
return err
}
defer source.Close()
file, err := os.OpenFile(partial, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return err
}
buffered := bufio.NewWriter(file)
if err := WriteDump(buffered, source); err != nil {
_ = file.Close()
return err
}
if err := buffered.Flush(); err != nil {
_ = file.Close()
return err
}
if err := file.Sync(); err != nil {
_ = file.Close()
return err
}
return file.Close()
}
@@ -7,7 +7,7 @@ import (
)
func Migrate(db *gorm.DB) error {
err := db.AutoMigrate(&models.Article{}, &models.Image{}, &models.User{}, &models.Invite{}, &models.APIKey{}, &models.PurgeSettings{})
err := tableOptionsFor(db).AutoMigrate(&models.Article{}, &models.Image{}, &models.User{}, &models.Invite{}, &models.APIKey{}, &models.PurgeSettings{})
if err != nil {
return err
}
@@ -0,0 +1,86 @@
package database
import (
domainUpdate "RenewCMS/internal/domain/update"
"fmt"
"net"
"strings"
"time"
mysqldriver "github.com/go-sql-driver/mysql"
"gorm.io/driver/mysql"
"gorm.io/gorm"
)
const (
DriverMySQL = "mysql"
mysqlCharset = "utf8mb4"
mysqlLocation = "UTC"
mysqlCollation = "utf8mb4_bin"
mysqlConnectTimeout = 5 * time.Second
mysqlSessionSQLMode = "'ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'"
mysqlTableOptionsKey = "gorm:table_options"
redactedPassword = "***"
)
type mysqlDriver struct{}
type MySQLConfig struct {
Host string
Port string
User string
Password string
Name string
TLS string
}
func MySQLDSN(config MySQLConfig) string {
dsn := mysqldriver.NewConfig()
dsn.User = config.User
dsn.Passwd = config.Password
dsn.Net = "tcp"
dsn.Addr = net.JoinHostPort(config.Host, config.Port)
dsn.DBName = config.Name
dsn.TLSConfig = config.TLS
dsn.ParseTime = true
dsn.ClientFoundRows = true
dsn.Collation = mysqlCollation
dsn.Timeout = mysqlConnectTimeout
dsn.Params = map[string]string{
"charset": mysqlCharset,
"loc": mysqlLocation,
"sql_mode": mysqlSessionSQLMode,
}
return dsn.FormatDSN()
}
func (mysqlDriver) Open(source string) (*gorm.DB, error) {
parsed, err := mysqldriver.ParseDSN(source)
if err != nil {
return nil, fmt.Errorf("the mysql connection settings are invalid")
}
db, err := gorm.Open(mysql.Open(source), &gorm.Config{})
if err != nil {
return nil, fmt.Errorf("cannot use the mysql server at %s: %s", parsed.Addr, redact(err.Error(), parsed.Passwd))
}
return db, nil
}
func (mysqlDriver) NewBackup(db *gorm.DB, directory string, now func() time.Time) domainUpdate.DatabaseBackup {
return NewMySQLBackup(db, directory, now)
}
func redact(message string, secret string) string {
if secret == "" {
return message
}
return strings.ReplaceAll(message, secret, redactedPassword)
}
func tableOptionsFor(db *gorm.DB) *gorm.DB {
if db.Dialector.Name() == DriverMySQL {
return db.Set(mysqlTableOptionsKey, "DEFAULT CHARSET="+mysqlCharset+" COLLATE="+mysqlCollation)
}
return db
}
@@ -0,0 +1,131 @@
package database
import (
"database/sql"
"fmt"
"time"
"gorm.io/gorm"
)
const (
dumpTimeLayout = "2006-01-02 15:04:05.000000"
zeroDumpTime = "0000-00-00 00:00:00"
)
func NewMySQLBackup(db *gorm.DB, directory string, now func() time.Time) *DumpBackup {
return NewDumpBackup(func() (DumpSource, error) { return openMySQLDumpSource(db) }, directory, now)
}
type mysqlDumpSource struct {
tx *gorm.DB
}
var _ DumpSource = &mysqlDumpSource{}
func openMySQLDumpSource(db *gorm.DB) (*mysqlDumpSource, error) {
tx := db.Begin(&sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
if tx.Error != nil {
return nil, tx.Error
}
return &mysqlDumpSource{tx: tx}, nil
}
func (s *mysqlDumpSource) Tables() ([]string, error) {
var tables []string
err := s.tx.Raw("SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE' ORDER BY table_name").Scan(&tables).Error
return tables, err
}
func (s *mysqlDumpSource) CreateStatement(table string) (string, error) {
var name, statement string
row := s.tx.Raw("SHOW CREATE TABLE " + quoteIdentifier(table)).Row()
if err := row.Scan(&name, &statement); err != nil {
return "", err
}
return statement, nil
}
func (s *mysqlDumpSource) Rows(table string) (DumpRows, error) {
rows, err := s.tx.Raw("SELECT * FROM " + quoteIdentifier(table)).Rows()
if err != nil {
return nil, err
}
return &mysqlDumpRows{rows: rows}, nil
}
func (s *mysqlDumpSource) Close() error {
return s.tx.Rollback().Error
}
type mysqlDumpRows struct {
rows *sql.Rows
}
var _ DumpRows = &mysqlDumpRows{}
func (r *mysqlDumpRows) Columns() ([]DumpColumn, error) {
types, err := r.rows.ColumnTypes()
if err != nil {
return nil, err
}
columns := make([]DumpColumn, len(types))
for index, columnType := range types {
columns[index] = DumpColumn{Name: columnType.Name(), Type: columnType.DatabaseTypeName()}
}
return columns, nil
}
func (r *mysqlDumpRows) Next() bool {
return r.rows.Next()
}
func (r *mysqlDumpRows) Values() ([][]byte, error) {
columnCount, err := r.rows.Columns()
if err != nil {
return nil, err
}
scanned := make([]any, len(columnCount))
targets := make([]any, len(columnCount))
for index := range scanned {
targets[index] = &scanned[index]
}
if err := r.rows.Scan(targets...); err != nil {
return nil, err
}
values := make([][]byte, len(scanned))
for index, value := range scanned {
values[index] = dumpBytes(value)
}
return values, nil
}
func (r *mysqlDumpRows) Err() error {
return r.rows.Err()
}
func (r *mysqlDumpRows) Close() error {
return r.rows.Close()
}
func dumpBytes(value any) []byte {
switch typed := value.(type) {
case nil:
return nil
case []byte:
return append([]byte{}, typed...)
case string:
return []byte(typed)
case time.Time:
return []byte(formatDumpTime(typed))
default:
return []byte(fmt.Sprint(typed))
}
}
func formatDumpTime(value time.Time) string {
if value.IsZero() {
return zeroDumpTime
}
return value.UTC().Format(dumpTimeLayout)
}
@@ -0,0 +1,170 @@
package database
import (
"encoding/hex"
"fmt"
"io"
"regexp"
"strings"
)
type DumpColumn struct {
Name string
Type string
}
type DumpRows interface {
Columns() ([]DumpColumn, error)
Next() bool
Values() ([][]byte, error)
Err() error
Close() error
}
type DumpSource interface {
Tables() ([]string, error)
CreateStatement(table string) (string, error)
Rows(table string) (DumpRows, error)
Close() error
}
var dumpPreamble = []string{
"SET NAMES utf8mb4;",
"SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO';",
"SET foreign_key_checks = 0;",
}
var dumpPostamble = []string{
"SET foreign_key_checks = 1;",
}
var plainNumber = regexp.MustCompile(`^-?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?$`)
var textEscapes = strings.NewReplacer(
`\`, `\\`,
`'`, `\'`,
"\x00", `\0`,
"\n", `\n`,
"\r", `\r`,
"\x1a", `\Z`,
)
func WriteDump(output io.Writer, source DumpSource) error {
if err := writeStatements(output, dumpPreamble); err != nil {
return err
}
tables, err := source.Tables()
if err != nil {
return err
}
for _, table := range tables {
if err := writeTable(output, source, table); err != nil {
return err
}
}
return writeStatements(output, dumpPostamble)
}
func writeStatements(output io.Writer, statements []string) error {
for _, statement := range statements {
if _, err := fmt.Fprintln(output, statement); err != nil {
return err
}
}
return nil
}
func writeTable(output io.Writer, source DumpSource, table string) error {
create, err := source.CreateStatement(table)
if err != nil {
return err
}
if _, err := fmt.Fprintf(output, "%s;\n", create); err != nil {
return err
}
rows, err := source.Rows(table)
if err != nil {
return err
}
defer rows.Close()
return writeRows(output, table, rows)
}
func writeRows(output io.Writer, table string, rows DumpRows) error {
columns, err := rows.Columns()
if err != nil {
return err
}
insertPrefix := insertStatementPrefix(table, columns)
for rows.Next() {
values, err := rows.Values()
if err != nil {
return err
}
if _, err := fmt.Fprintf(output, "%s%s);\n", insertPrefix, joinLiterals(columns, values)); err != nil {
return err
}
}
return rows.Err()
}
func insertStatementPrefix(table string, columns []DumpColumn) string {
names := make([]string, len(columns))
for index, column := range columns {
names[index] = quoteIdentifier(column.Name)
}
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (", quoteIdentifier(table), strings.Join(names, ","))
}
func joinLiterals(columns []DumpColumn, values [][]byte) string {
literals := make([]string, len(columns))
for index, column := range columns {
literals[index] = sqlLiteral(column.Type, values[index])
}
return strings.Join(literals, ",")
}
func quoteIdentifier(name string) string {
return "`" + strings.ReplaceAll(name, "`", "``") + "`"
}
func sqlLiteral(databaseType string, value []byte) string {
if value == nil {
return "NULL"
}
switch {
case isNumericType(databaseType) && plainNumber.Match(value):
return string(value)
case isBinaryType(databaseType):
return hexLiteral(value)
default:
return "'" + textEscapes.Replace(string(value)) + "'"
}
}
func hexLiteral(value []byte) string {
if len(value) == 0 {
return "''"
}
return "0x" + hex.EncodeToString(value)
}
func isNumericType(databaseType string) bool {
upper := strings.ToUpper(databaseType)
for _, suffix := range []string{"INT", "DECIMAL", "NUMERIC", "FLOAT", "DOUBLE", "YEAR"} {
if strings.HasSuffix(upper, suffix) {
return true
}
}
return false
}
func isBinaryType(databaseType string) bool {
upper := strings.ToUpper(databaseType)
for _, suffix := range []string{"BLOB", "BINARY", "BIT", "GEOMETRY"} {
if strings.HasSuffix(upper, suffix) {
return true
}
}
return false
}
@@ -4,9 +4,6 @@ import (
domain "RenewCMS/internal/domain/update"
"errors"
"os"
"path/filepath"
"sort"
"strings"
"time"
"gorm.io/gorm"
@@ -34,54 +31,16 @@ func NewSQLiteBackup(db *gorm.DB, directory string, now func() time.Time) *SQLit
}
func (b *SQLiteBackup) Backup() (string, error) {
if err := os.MkdirAll(b.directory, os.ModePerm); err != nil {
target, err := newBackupTarget(b.directory, b.now(), backupFileExtension)
if err != nil {
return "", err
}
target := filepath.Join(b.directory, backupFilePrefix+b.now().UTC().Format(backupTimeLayout)+backupFileExtension)
if _, err := os.Stat(target); err == nil {
return "", errBackupExists
}
if err := b.db.Exec("VACUUM INTO ?", target).Error; err != nil {
_ = os.Remove(target)
return "", err
}
if err := b.prune(); err != nil {
if err := pruneBackups(b.directory, backupFileExtension); err != nil {
return "", err
}
return target, nil
}
type backupFile struct {
path string
modified time.Time
}
func (b *SQLiteBackup) prune() error {
entries, err := os.ReadDir(b.directory)
if err != nil {
return err
}
files := make([]backupFile, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), backupFileExtension) {
continue
}
info, err := entry.Info()
if err != nil {
return err
}
files = append(files, backupFile{path: filepath.Join(b.directory, entry.Name()), modified: info.ModTime()})
}
sort.Slice(files, func(i, j int) bool {
if files[i].modified.Equal(files[j].modified) {
return files[i].path > files[j].path
}
return files[i].modified.After(files[j].modified)
})
for _, stale := range files[min(backupsToKeep, len(files)):] {
if err := os.Remove(stale.path); err != nil {
return err
}
}
return nil
}
@@ -7,7 +7,7 @@ type APIKey struct {
UserID uint32 `gorm:"index;not null"`
Name string `gorm:"not null"`
Prefix string `gorm:"not null"`
HashedKey string `gorm:"uniqueIndex;not null"`
HashedKey string `gorm:"uniqueIndex;not null;size:64"`
ExpiresAt *time.Time
RevokedAt *time.Time
LastUsedAt *time.Time
@@ -15,7 +15,7 @@ type Image struct {
ContentType string
Width int
Height int
SHA256 string `gorm:"column:sha256;index"`
SHA256 string `gorm:"column:sha256;index;size:64"`
UploadedByID uint32
CreatedAt time.Time `gorm:"autoCreateTime"`
UpdatedAt time.Time `gorm:"autoUpdateTime"`
@@ -4,7 +4,7 @@ import "time"
type Invite struct {
ID uint32 `gorm:"primaryKey;autoIncrement"`
Email string `gorm:"index;not null"`
Email string `gorm:"index;not null;size:255"`
HashedCode string `gorm:"not null"`
ExpiresAt time.Time
InvitedBy uint32 `gorm:"not null"`
@@ -6,14 +6,14 @@ import (
type User struct {
ID uint32 `gorm:"primaryKey;autoIncrement"`
Username string `gorm:"unique;not null"`
Username string `gorm:"unique;not null;size:255"`
Password string `gorm:"not null"`
PasswordResetCode *string `gorm:"unique"`
Email string `gorm:"unique;not null"`
DisplayName string `gorm:"not null;default:''"`
PasswordResetCode *string `gorm:"unique;size:255"`
Email string `gorm:"unique;not null;size:255"`
DisplayName string `gorm:"not null;default:'';size:255"`
IsVerified bool `gorm:"default=false;not null"`
Role string `gorm:"not null;default:''"`
VerificationCode *string `gorm:"unique"`
Role string `gorm:"not null;default:'';size:32"`
VerificationCode *string `gorm:"unique;size:255"`
VerificationExpiration time.Time
PasswordResetExpiration time.Time
TokenVersion uint32 `gorm:"default:0;not null"`
@@ -28,7 +28,7 @@ func (u *UserRepository) Get(id uint32) (domainUser.User, error) {
}
func (u *UserRepository) Create(user domainUser.User) (domainUser.User, error) {
creationResult := u.db.Create(&entity.User{
createdUser := entity.User{
Username: user.Username,
Password: user.Password,
PasswordResetCode: mappers.DomainToNullableString(user.PasswordResetCode),
@@ -37,16 +37,12 @@ func (u *UserRepository) Create(user domainUser.User) (domainUser.User, error) {
Role: string(user.Role),
VerificationCode: mappers.DomainToNullableString(user.VerificationCode),
VerificationExpiration: user.VerificationExpiration,
})
if creationResult.Error != nil {
return domainUser.User{}, creationResult.Error
}
if err := u.db.Create(&createdUser).Error; err != nil {
return domainUser.User{}, err
}
var createdUser entity.User
creationResult.Scan(&createdUser)
return mappers.UserToDomain(createdUser),
nil
return mappers.UserToDomain(createdUser), nil
}
func (u *UserRepository) Delete(id uint32) error {
+559
View File
@@ -0,0 +1,559 @@
package test
import (
"RenewCMS/internal/infrastructure/config"
"RenewCMS/internal/infrastructure/persistence/database"
"bytes"
"context"
"database/sql"
"fmt"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
mysqldriver "github.com/go-sql-driver/mysql"
)
const mysqlTestPassword = "s3cr3t/Value@42:x"
func setMySQLEnv(t *testing.T) {
t.Helper()
setBaseSettingsEnv(t)
t.Setenv("DB_DRIVER", "mysql")
t.Setenv("DB_HOST", "db.local")
t.Setenv("DB_USER", "app")
t.Setenv("DB_PASSWORD", mysqlTestPassword)
t.Setenv("DB_NAME", "cms")
for _, optional := range []string{"DB_PORT", "DB_TLS", "DB_MAX_OPEN_CONNS", "DB_MAX_IDLE_CONNS", "DB_CONN_MAX_LIFETIME_SECONDS", "DATA_DIR"} {
t.Setenv(optional, "")
}
}
func lookupFrom(values map[string]string) func(string) (string, bool) {
return func(name string) (string, bool) {
value, found := values[name]
return value, found
}
}
func TestMySQLDriverSelection(t *testing.T) {
t.Run("Driver selection", func(t *testing.T) {
cases := []struct {
value string
set bool
driver string
}{
{"", false, "sqlite"},
{"sqlite", true, "sqlite"},
{"mysql", true, "mysql"},
{"oracle", true, "oracle"},
}
for _, tc := range cases {
t.Run(fmt.Sprintf("DB_DRIVER %q", tc.value), func(t *testing.T) {
setMySQLEnv(t)
if tc.set {
t.Setenv("DB_DRIVER", tc.value)
} else {
t.Setenv("DB_DRIVER", "")
}
settings, err := config.NewSettingsFromEnv()
if err != nil {
t.Fatalf("settings: %v", err)
}
if settings.Database.Driver != tc.driver {
t.Errorf("expected driver %q, got %q", tc.driver, settings.Database.Driver)
}
})
}
})
t.Run("an unregistered driver fails with its name", func(t *testing.T) {
_, err := database.Connect(database.Config{Driver: "oracle", Source: "x"})
if err == nil || err.Error() != `unsupported database driver "oracle"` {
t.Errorf("unexpected error %v", err)
}
})
}
func TestMySQLEnvironmentValidation(t *testing.T) {
complete := map[string]string{"DB_HOST": "h", "DB_USER": "u", "DB_PASSWORD": "p", "DB_NAME": "n"}
t.Run("Missing MySQL variable", func(t *testing.T) {
for _, missing := range []string{"DB_HOST", "DB_USER", "DB_PASSWORD", "DB_NAME"} {
t.Run(missing, func(t *testing.T) {
values := map[string]string{}
for name, value := range complete {
if name != missing {
values[name] = value
}
}
err := config.MissingEnvVarError(config.DatabaseRequiredEnvVars("mysql"), lookupFrom(values))
expected := "Environment variable " + missing + " is not set"
if err == nil || err.Error() != expected {
t.Errorf("expected %q, got %v", expected, err)
}
})
}
})
t.Run("SQLite does not require MySQL variables", func(t *testing.T) {
for _, driver := range []string{"sqlite", "", "oracle"} {
if err := config.MissingEnvVarError(config.DatabaseRequiredEnvVars(driver), lookupFrom(nil)); err != nil {
t.Errorf("driver %q: expected success, got %v", driver, err)
}
}
})
t.Run("all MySQL variables set passes validation", func(t *testing.T) {
if err := config.MissingEnvVarError(config.DatabaseRequiredEnvVars("mysql"), lookupFrom(complete)); err != nil {
t.Errorf("expected success, got %v", err)
}
})
}
func TestMySQLSettingsFromEnv(t *testing.T) {
t.Run("defaults apply when the optional variables are empty", func(t *testing.T) {
setMySQLEnv(t)
settings, err := config.NewSettingsFromEnv()
if err != nil {
t.Fatalf("settings: %v", err)
}
database := settings.Database
if database.Port != "3306" || database.TLS != "false" || database.MaxOpenConns != 10 || database.MaxIdleConns != 5 || database.ConnMaxLifetime != 300*time.Second {
t.Errorf("unexpected defaults %+v", database)
}
if database.Host != "db.local" || database.User != "app" || database.Password != mysqlTestPassword || database.Name != "cms" {
t.Errorf("unexpected connection fields %+v", database)
}
})
t.Run("Invalid TLS option is rejected", func(t *testing.T) {
setMySQLEnv(t)
t.Setenv("DB_TLS", "maybe")
_, err := config.NewSettingsFromEnv()
if err == nil {
t.Fatal("expected an error")
}
for _, expected := range []string{"DB_TLS", "false", "true", "skip-verify", "preferred"} {
if !strings.Contains(err.Error(), expected) {
t.Errorf("error %q must mention %q", err, expected)
}
}
})
t.Run("every accepted TLS option is read", func(t *testing.T) {
for _, mode := range []string{"false", "true", "skip-verify", "preferred"} {
setMySQLEnv(t)
t.Setenv("DB_TLS", mode)
settings, err := config.NewSettingsFromEnv()
if err != nil || settings.Database.TLS != mode {
t.Errorf("mode %q: settings %+v, err %v", mode, settings.Database, err)
}
}
})
t.Run("a non numeric pool variable names the variable", func(t *testing.T) {
for _, name := range []string{"DB_MAX_OPEN_CONNS", "DB_MAX_IDLE_CONNS", "DB_CONN_MAX_LIFETIME_SECONDS"} {
setMySQLEnv(t)
t.Setenv(name, "many")
_, err := config.NewSettingsFromEnv()
if err == nil || !strings.Contains(err.Error(), name) {
t.Errorf("%s: expected an error naming it, got %v", name, err)
}
}
})
t.Run("a negative pool variable is rejected", func(t *testing.T) {
setMySQLEnv(t)
t.Setenv("DB_MAX_OPEN_CONNS", "-1")
if _, err := config.NewSettingsFromEnv(); err == nil {
t.Error("expected an error")
}
})
t.Run("zero is an accepted pool value", func(t *testing.T) {
setMySQLEnv(t)
t.Setenv("DB_MAX_OPEN_CONNS", "0")
settings, err := config.NewSettingsFromEnv()
if err != nil || settings.Database.MaxOpenConns != 0 {
t.Errorf("unexpected settings %+v, err %v", settings.Database, err)
}
})
t.Run("pool variables are read", func(t *testing.T) {
setMySQLEnv(t)
t.Setenv("DB_MAX_OPEN_CONNS", "4")
t.Setenv("DB_MAX_IDLE_CONNS", "2")
t.Setenv("DB_CONN_MAX_LIFETIME_SECONDS", "60")
t.Setenv("DB_PORT", "3307")
settings, err := config.NewSettingsFromEnv()
database := settings.Database
if err != nil || database.MaxOpenConns != 4 || database.MaxIdleConns != 2 || database.ConnMaxLifetime != 60*time.Second || database.Port != "3307" {
t.Errorf("unexpected settings %+v, err %v", database, err)
}
})
t.Run("SQLite ignores the MySQL variables", func(t *testing.T) {
setMySQLEnv(t)
t.Setenv("DB_DRIVER", "sqlite")
t.Setenv("DB_TLS", "maybe")
t.Setenv("DB_MAX_OPEN_CONNS", "many")
settings, err := config.NewSettingsFromEnv()
if err != nil || settings.Database.Driver != "sqlite" || settings.Database.Host != "" {
t.Errorf("unexpected settings %+v, err %v", settings.Database, err)
}
})
}
func TestMySQLDSN(t *testing.T) {
t.Run("DSN built from variables", func(t *testing.T) {
cases := []struct {
host, port, expectedPort, tls string
}{
{"db.local", "", "3306", "false"},
{"db.local", "3307", "3307", "true"},
{"mysql", "3306", "3306", "skip-verify"},
}
for _, tc := range cases {
t.Run(tc.host+":"+tc.port+" tls "+tc.tls, func(t *testing.T) {
port := tc.port
if port == "" {
port = config.DefaultDatabasePort
}
dsn := database.MySQLDSN(database.MySQLConfig{Host: tc.host, Port: port, User: "app", Password: "pw", Name: "cms", TLS: tc.tls})
parsed, err := mysqldriver.ParseDSN(dsn)
if err != nil {
t.Fatalf("parse: %v", err)
}
if parsed.Addr != tc.host+":"+tc.expectedPort || parsed.DBName != "cms" || parsed.User != "app" {
t.Errorf("unexpected target %q db %q user %q", parsed.Addr, parsed.DBName, parsed.User)
}
for _, fragment := range []string{"parseTime=true", "loc=UTC", "charset=utf8mb4", "clientFoundRows=true", "tls=" + tc.tls} {
if !strings.Contains(dsn, fragment) {
t.Errorf("dsn must contain %q", fragment)
}
}
if parsed.Timeout <= 0 || parsed.Timeout > 10*time.Second {
t.Errorf("connect timeout must be short, got %v", parsed.Timeout)
}
})
}
})
t.Run("a password with reserved characters survives the DSN", func(t *testing.T) {
dsn := database.MySQLDSN(database.MySQLConfig{Host: "db.local", Port: "3306", User: "app", Password: mysqlTestPassword, Name: "cms", TLS: "false"})
parsed, err := mysqldriver.ParseDSN(dsn)
if err != nil || parsed.Passwd != mysqlTestPassword || parsed.DBName != "cms" || parsed.Addr != "db.local:3306" {
t.Errorf("unexpected parse %+v err %v", parsed, err)
}
})
t.Run("the connection uses a binary collation so Bob and bob stay distinct", func(t *testing.T) {
parsed, _ := mysqldriver.ParseDSN(database.MySQLDSN(database.MySQLConfig{Host: "h", Port: "1", User: "u", Name: "n", TLS: "false"}))
if parsed.Collation != "utf8mb4_bin" {
t.Errorf("unexpected collation %q", parsed.Collation)
}
})
}
func TestMySQLDataDir(t *testing.T) {
t.Run("DataDir resolution", func(t *testing.T) {
cases := []struct{ dbFile, dataDir, result string }{
{"./data/renewcms.db", "", "./data"},
{"/var/lib/cms/cms.db", "", "/var/lib/cms"},
{"./data/renewcms.db", "/srv/x", "/srv/x"},
}
for _, tc := range cases {
t.Run(tc.dbFile+" "+tc.dataDir, func(t *testing.T) {
settings := config.Settings{DBFile: tc.dbFile, DataDirPath: tc.dataDir}
if got := settings.DataDir(); filepath.Clean(got) != filepath.Clean(tc.result) {
t.Errorf("expected %q, got %q", tc.result, got)
}
})
}
})
t.Run("Data directory with MySQL", func(t *testing.T) {
setMySQLEnv(t)
t.Setenv("DATA_DIR", "/data")
t.Setenv("DB_FILE", "./elsewhere/ignored.db")
settings, err := config.NewSettingsFromEnv()
if err != nil || filepath.Clean(settings.DataDir()) != filepath.Clean("/data") {
t.Errorf("unexpected data dir %q err %v", settings.DataDir(), err)
}
})
}
func TestMySQLConnectionPool(t *testing.T) {
t.Run("Connection pool", func(t *testing.T) {
pool := database.PoolSettings{MaxOpenConns: 4, MaxIdleConns: 2, ConnMaxLifetime: 60 * time.Second}
connection, err := database.Connect(database.Config{Driver: database.DriverSQLite, Source: filepath.ToSlash(filepath.Join(t.TempDir(), "pool.db")), Pool: &pool})
if err != nil {
t.Fatalf("connect: %v", err)
}
defer connection.Close()
sqlDb, _ := connection.DB.DB()
held := make([]*sql.Conn, 0, 4)
for range 4 {
conn, err := sqlDb.Conn(context.Background())
if err != nil {
t.Fatalf("conn: %v", err)
}
held = append(held, conn)
}
blocked, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
_, fifthErr := sqlDb.Conn(blocked)
for _, conn := range held {
_ = conn.Close()
}
stats := sqlDb.Stats()
if stats.MaxOpenConnections != 4 {
t.Errorf("expected 4 open connections at most, got %d", stats.MaxOpenConnections)
}
if fifthErr == nil {
t.Error("a fifth concurrent connection must wait for the pool")
}
if stats.Idle != 2 {
t.Errorf("expected 2 idle connections, got %d", stats.Idle)
}
})
t.Run("the connection lifetime closes old connections", func(t *testing.T) {
pool := database.PoolSettings{MaxOpenConns: 1, MaxIdleConns: 1, ConnMaxLifetime: 20 * time.Millisecond}
connection, err := database.Connect(database.Config{Driver: database.DriverSQLite, Source: filepath.ToSlash(filepath.Join(t.TempDir(), "life.db")), Pool: &pool})
if err != nil {
t.Fatalf("connect: %v", err)
}
defer connection.Close()
sqlDb, _ := connection.DB.DB()
_ = sqlDb.Ping()
time.Sleep(100 * time.Millisecond)
_ = sqlDb.Ping()
if sqlDb.Stats().MaxLifetimeClosed == 0 {
t.Error("an expired connection must be closed")
}
})
t.Run("settings without a pool leave the defaults untouched", func(t *testing.T) {
connection, err := database.Connect(database.Config{Driver: database.DriverSQLite, Source: filepath.ToSlash(filepath.Join(t.TempDir(), "plain.db"))})
if err != nil {
t.Fatalf("connect: %v", err)
}
defer connection.Close()
sqlDb, _ := connection.DB.DB()
if sqlDb.Stats().MaxOpenConnections != 0 {
t.Errorf("expected an unlimited pool, got %d", sqlDb.Stats().MaxOpenConnections)
}
})
}
func closedLocalPort(t *testing.T) string {
t.Helper()
return freeTCPPort(t)
}
func mysqlDSNWithTimeout(config database.MySQLConfig, timeout time.Duration) string {
parsed, _ := mysqldriver.ParseDSN(database.MySQLDSN(config))
parsed.Timeout = timeout
return parsed.FormatDSN()
}
func TestMySQLFailFast(t *testing.T) {
t.Run("nothing listens on DB_PORT", func(t *testing.T) {
port := closedLocalPort(t)
dsn := mysqlDSNWithTimeout(database.MySQLConfig{Host: "127.0.0.1", Port: port, User: "app", Password: mysqlTestPassword, Name: "cms", TLS: "false"}, time.Second)
started := time.Now()
_, err := database.Connect(database.Config{Driver: database.DriverMySQL, Source: dsn})
if err == nil {
t.Fatal("expected an error")
}
if !strings.Contains(err.Error(), "127.0.0.1:"+port) {
t.Errorf("the error must name the host and port: %v", err)
}
if strings.Contains(err.Error(), mysqlTestPassword) {
t.Errorf("the error must not contain the password: %v", err)
}
if elapsed := time.Since(started); elapsed > 4*time.Second {
t.Errorf("startup must fail fast, took %v", elapsed)
}
})
t.Run("an empty password leaves the error message untouched", func(t *testing.T) {
port := closedLocalPort(t)
dsn := mysqlDSNWithTimeout(database.MySQLConfig{Host: "127.0.0.1", Port: port, User: "app", Name: "cms", TLS: "false"}, time.Second)
_, err := database.Connect(database.Config{Driver: database.DriverMySQL, Source: dsn})
if err == nil {
t.Fatal("expected an error")
}
if strings.Contains(err.Error(), "***") {
t.Errorf("an empty password must not be redacted into the message: %v", err)
}
if !strings.Contains(err.Error(), "127.0.0.1:"+port) {
t.Errorf("the error must name the host and port: %v", err)
}
})
t.Run("an unusable DSN is rejected without echoing it", func(t *testing.T) {
_, err := database.Connect(database.Config{Driver: database.DriverMySQL, Source: "user:" + mysqlTestPassword + "@broken"})
if err == nil || strings.Contains(err.Error(), mysqlTestPassword) {
t.Errorf("unexpected error %v", err)
}
})
}
func startupEnvironment(t *testing.T, extra ...string) []string {
t.Helper()
dataDir := t.TempDir()
return append(emptyEnvironment(),
append([]string{
"ENVIRONMENT=development", "PORT=" + freeTCPPort(t), "PUBLIC_URL=http://localhost", "JWT_SECRET=test-secret-for-startup",
"SMTP_LOGIN=a@example.com", "SMTP_SENDER=a@example.com", "SMTP_PASSWORD=x", "SMTP_HOST=localhost", "SMTP_PORT=2525",
"DB_FILE=" + filepath.Join(dataDir, "renewcms.db"), "UPLOAD_DIR=" + filepath.Join(dataDir, "uploads"),
"UPDATE_FEED_URL=https://git.example",
}, extra...)...)
}
func runServerWithEnvironment(t *testing.T, binary string, environment []string) (string, int) {
t.Helper()
command := exec.Command(binary)
command.Dir = t.TempDir()
command.Env = environment
var output bytes.Buffer
command.Stdout, command.Stderr = &output, &output
err := command.Run()
if exitErr, ok := err.(*exec.ExitError); ok {
return output.String(), exitErr.ExitCode()
}
if err != nil {
t.Fatal(err)
}
return output.String(), 0
}
func TestMySQLStartupFailures(t *testing.T) {
binary := buildServerBinary(t, "")
t.Run("an unsupported driver stops the server", func(t *testing.T) {
output, code := runServerWithEnvironment(t, binary, startupEnvironment(t, "DB_DRIVER=oracle"))
if code == 0 || !strings.Contains(output, `unsupported database driver "oracle"`) {
t.Errorf("code %d output %q", code, output)
}
})
t.Run("Missing MySQL variable stops the server", func(t *testing.T) {
complete := map[string]string{"DB_HOST": "127.0.0.1", "DB_USER": "app", "DB_PASSWORD": mysqlTestPassword, "DB_NAME": "cms"}
for missing := range complete {
t.Run(missing, func(t *testing.T) {
extra := []string{"DB_DRIVER=mysql"}
for name, value := range complete {
if name != missing {
extra = append(extra, name+"="+value)
}
}
output, code := runServerWithEnvironment(t, binary, startupEnvironment(t, extra...))
if code == 0 || !strings.Contains(output, "Environment variable "+missing+" is not set") {
t.Errorf("code %d output %q", code, output)
}
if strings.Contains(output, mysqlTestPassword) {
t.Errorf("the output must not contain the password: %q", output)
}
})
}
})
t.Run("an invalid DB_TLS stops the server naming the variable", func(t *testing.T) {
output, code := runServerWithEnvironment(t, binary, startupEnvironment(t,
"DB_DRIVER=mysql", "DB_HOST=127.0.0.1", "DB_USER=app", "DB_PASSWORD="+mysqlTestPassword, "DB_NAME=cms", "DB_TLS=maybe"))
if code == 0 || !strings.Contains(output, "DB_TLS") || strings.Contains(output, mysqlTestPassword) {
t.Errorf("code %d output %q", code, output)
}
})
t.Run("Fail fast when the server cannot be used", func(t *testing.T) {
port := closedLocalPort(t)
output, code := runServerWithEnvironment(t, binary, startupEnvironment(t,
"DB_DRIVER=mysql", "DB_HOST=127.0.0.1", "DB_PORT="+port, "DB_USER=app", "DB_PASSWORD="+mysqlTestPassword, "DB_NAME=cms"))
if code == 0 || !strings.Contains(output, "Unable to open the database") {
t.Errorf("code %d output %q", code, output)
}
if !strings.Contains(output, "127.0.0.1:"+port) {
t.Errorf("the message must name the host and port: %q", output)
}
if strings.Contains(output, mysqlTestPassword) {
t.Errorf("the message must not contain the password: %q", output)
}
})
t.Run("the SQLite default starts without any MySQL variable", func(t *testing.T) {
output, code := runServerWithEnvironment(t, binary, startupEnvironment(t, "PORT=99999"))
bindFailed := strings.Contains(output, "invalid port")
if code == 0 || !bindFailed || strings.Contains(output, "Environment variable DB_") {
t.Errorf("startup must get as far as binding the port, code %d output %q", code, output)
}
})
}
func TestMySQLSettingsAreNeverPrinted(t *testing.T) {
t.Run("Secrets are never exposed", func(t *testing.T) {
setMySQLEnv(t)
settings, err := config.NewSettingsFromEnv()
if err != nil {
t.Fatal(err)
}
dsn := mysqlDSNWithTimeout(database.MySQLConfig{Host: "127.0.0.1", Port: closedLocalPort(t), User: settings.Database.User, Password: settings.Database.Password, Name: settings.Database.Name, TLS: settings.Database.TLS}, time.Second)
_, err = database.Connect(database.Config{Driver: database.DriverMySQL, Source: dsn})
if err == nil || strings.Contains(err.Error(), settings.Database.Password) {
t.Errorf("unexpected error %v", err)
}
})
}
+311
View File
@@ -0,0 +1,311 @@
package test
import (
domainArticle "RenewCMS/internal/domain/article"
domainInvite "RenewCMS/internal/domain/invite"
"RenewCMS/internal/infrastructure/persistence/database"
"RenewCMS/internal/infrastructure/persistence/models"
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
mysqldriver "github.com/go-sql-driver/mysql"
"gorm.io/gorm"
)
func migratedMySQL(t *testing.T) (*mysqlTestServer, database.MySQLConfig, *gorm.DB) {
t.Helper()
server := connectMySQLTestServer(t)
config := server.createDatabase(t)
connection := server.connect(t, config)
if err := database.Migrate(connection.DB); err != nil {
t.Fatalf("migrate: %v", err)
}
return server, config, connection.DB
}
func columnCount(t *testing.T, db *gorm.DB, query string, arguments ...any) int {
t.Helper()
var count int
if err := db.Raw(query, arguments...).Scan(&count).Error; err != nil {
t.Fatalf("query %q: %v", query, err)
}
return count
}
func TestMySQLMigration(t *testing.T) {
t.Run("Migration on MySQL", func(t *testing.T) {
_, _, db := migratedMySQL(t)
for _, table := range []string{"articles", "images", "users", "invites", "api_keys", "purge_settings"} {
if !db.Migrator().HasTable(table) {
t.Errorf("table %s must exist", table)
}
}
uniqueColumns := map[string][]string{
"users": {"username", "email", "password_reset_code", "verification_code"},
"api_keys": {"hashed_key"},
}
for table, columns := range uniqueColumns {
for _, column := range columns {
count := columnCount(t, db, "SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ? AND non_unique = 0 AND seq_in_index = 1", table, column)
if count == 0 {
t.Errorf("a unique index on %s.%s must exist", table, column)
}
}
}
binaryTables := columnCount(t, db, "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = DATABASE() AND table_collation = 'utf8mb4_bin'")
if binaryTables != 6 {
t.Errorf("every table must use the utf8mb4_bin collation, got %d of 6", binaryTables)
}
})
t.Run("migrating twice changes nothing", func(t *testing.T) {
_, _, db := migratedMySQL(t)
if err := database.Migrate(db); err != nil {
t.Errorf("the second migration must succeed: %v", err)
}
})
t.Run("the first user lacking a role becomes owner and the others admin", func(t *testing.T) {
_, _, db := migratedMySQL(t)
for _, name := range []string{"first", "second", "third"} {
insert := db.Exec("INSERT INTO users (username, password, email, role, is_verified, token_version) VALUES (?, 'x', ?, '', 1, 0)", name, name+"@x.example")
if insert.Error != nil {
t.Fatalf("insert %s: %v", name, insert.Error)
}
}
if err := database.Migrate(db); err != nil {
t.Fatalf("migrate: %v", err)
}
roles := map[string]string{}
var users []models.User
db.Find(&users)
for _, user := range users {
roles[user.Username] = user.Role
}
if roles["first"] != "owner" || roles["second"] != "admin" || roles["third"] != "admin" {
t.Errorf("unexpected roles %v", roles)
}
})
}
func TestMySQLTimes(t *testing.T) {
t.Run("times are stored and read back in UTC", func(t *testing.T) {
_, _, db := migratedMySQL(t)
repos := newContractRepositories(db)
zone := time.FixedZone("plus2", 2*3600)
expiration := time.Date(2030, 5, 6, 7, 8, 9, 0, zone)
user := contractUser("zoned", "zoned@x.example")
user.VerificationExpiration = expiration
created := mustCreateUser(t, repos, user)
read, err := repos.users.Get(created.ID)
if err != nil || !read.VerificationExpiration.Equal(expiration) || read.VerificationExpiration.Location() != time.UTC {
t.Errorf("expected %v in UTC, got %v (%v) err %v", expiration, read.VerificationExpiration, read.VerificationExpiration.Location(), err)
}
})
t.Run("sessions allow the zero date the driver sends for a zero time", func(t *testing.T) {
_, _, db := migratedMySQL(t)
var mode string
if err := db.Raw("SELECT @@SESSION.sql_mode").Scan(&mode).Error; err != nil {
t.Fatalf("read sql_mode: %v", err)
}
if strings.Contains(mode, "NO_ZERO_DATE") || !strings.Contains(mode, "STRICT_TRANS_TABLES") {
t.Errorf("unexpected session sql_mode %q", mode)
}
})
}
func populateForBackup(t *testing.T, repos contractRepositories) {
t.Helper()
trickyName := "o'brien \"quoted\" back\\slash\nline é😀"
mustCreateUser(t, repos, contractUser(trickyName, "tricky@x.example"))
second := contractUser("plain", "plain@x.example")
second.PasswordResetCode = "reset-code"
mustCreateUser(t, repos, second)
if _, err := repos.articles.Create(domainArticle.Article{Title: "Title", Body: "line1\nline2 'q' \"d\" \\ \x00 end", IsOnline: true}); err != nil {
t.Fatalf("create article: %v", err)
}
mustCreateImage(t, repos, "pic.png", strings.Repeat("c", 64))
if _, err := repos.invites.Create(domainInvite.Invite{Email: "i@x.example", HashedCode: "h", ExpiresAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC), InvitedBy: 1}); err != nil {
t.Fatalf("create invite: %v", err)
}
}
func steppingClock(start time.Time) func() time.Time {
current := start
return func() time.Time {
current = current.Add(time.Second)
return current
}
}
func TestMySQLBackup(t *testing.T) {
t.Run("Backup before an update", func(t *testing.T) {
_, _, db := migratedMySQL(t)
populateForBackup(t, newContractRepositories(db))
directory := filepath.Join(t.TempDir(), "backups")
backup := database.NewMySQLBackup(db, directory, steppingClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)))
var paths []string
for range 5 {
path, err := backup.Backup()
if err != nil {
t.Fatalf("backup: %v", err)
}
paths = append(paths, path)
}
entries, _ := os.ReadDir(directory)
if len(entries) != 3 {
t.Fatalf("only the 3 newest backups are kept, got %d", len(entries))
}
for _, path := range paths[2:] {
if filepath.Ext(path) != ".sql" || !strings.HasPrefix(filepath.Base(path), "renewcms-") {
t.Errorf("unexpected backup name %q", path)
}
if _, err := os.Stat(path); err != nil {
t.Errorf("the newest backup %q must remain: %v", path, err)
}
}
})
t.Run("a failure returns an error and leaves no file", func(t *testing.T) {
server := connectMySQLTestServer(t)
connection := server.connect(t, server.createDatabase(t))
directory := filepath.Join(t.TempDir(), "backups")
backup := database.NewMySQLBackup(connection.DB, directory, time.Now)
if err := connection.Close(); err != nil {
t.Fatalf("close: %v", err)
}
path, err := backup.Backup()
if err == nil || path != "" {
t.Errorf("expected a failure, got %q %v", path, err)
}
if entries, _ := os.ReadDir(directory); len(entries) != 0 {
t.Errorf("no file may remain, got %d entries", len(entries))
}
})
t.Run("MySQL backup content", func(t *testing.T) {
server, _, db := migratedMySQL(t)
repos := newContractRepositories(db)
populateForBackup(t, repos)
directory := filepath.Join(t.TempDir(), "backups")
backup := database.NewMySQLBackup(db, directory, time.Now)
path, err := backup.Backup()
if err != nil {
t.Fatalf("backup: %v", err)
}
content, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
dump := string(content)
for _, table := range []string{"articles", "images", "users", "invites", "api_keys", "purge_settings"} {
if !strings.Contains(dump, "CREATE TABLE `"+table+"`") {
t.Errorf("the dump must contain the schema of %s", table)
}
}
for _, table := range []string{"articles", "images", "users", "invites"} {
if !strings.Contains(dump, "INSERT INTO `"+table+"`") {
t.Errorf("the dump must contain rows of %s", table)
}
}
if password := server.config.Password; password != "" && strings.Contains(dump, password) {
t.Errorf("the dump must not contain the DB_PASSWORD value")
}
restoredConfig := server.createDatabase(t)
replayDump(t, restoredConfig, dump)
restored := server.connect(t, restoredConfig)
restoredPath, err := database.NewMySQLBackup(restored.DB, filepath.Join(t.TempDir(), "again"), time.Now).Backup()
if err != nil {
t.Fatalf("backup of the restored database: %v", err)
}
restoredContent, _ := os.ReadFile(restoredPath)
if line := firstDifferingLine(withoutRedundantCharset(dump), withoutRedundantCharset(string(restoredContent))); line != "" {
t.Errorf("replaying the dump must restore the same schema and rows, first difference: %s", line)
}
restoredUsers := newContractRepositories(restored.DB).users.GetAll()
if len(restoredUsers) != 2 || restoredUsers[0].Username != "o'brien \"quoted\" back\\slash\nline é😀" || restoredUsers[1].PasswordResetCode != "reset-code" {
t.Errorf("unexpected restored users %+v", restoredUsers)
}
restoredArticles, _ := newContractRepositories(restored.DB).articles.Get(1)
if restoredArticles.Body != "line1\nline2 'q' \"d\" \\ \x00 end" {
t.Errorf("unexpected restored article body %q", restoredArticles.Body)
}
})
}
func replayDump(t *testing.T, config database.MySQLConfig, dump string) {
t.Helper()
parsed, err := mysqldriver.ParseDSN(database.MySQLDSN(config))
if err != nil {
t.Fatal(err)
}
parsed.MultiStatements = true
replay, err := sql.Open("mysql", parsed.FormatDSN())
if err != nil {
t.Fatal(err)
}
defer replay.Close()
if _, err := replay.Exec(dump); err != nil {
t.Fatalf("replay: %v", err)
}
}
func TestMySQLUsersAreCaseSensitiveThroughTheDomain(t *testing.T) {
t.Run("Bob and bob can both register", func(t *testing.T) {
_, _, db := migratedMySQL(t)
repos := newContractRepositories(db)
mustCreateUser(t, repos, contractUser("Bob", "b1@x.example"))
other, err := repos.users.Create(contractUser("bob", "b2@x.example"))
_, missing := repos.users.GetByUsername("BOB")
if err != nil || other.Username != "bob" || missing == nil {
t.Errorf("created %+v err %v, uppercase lookup err %v", other, err, missing)
}
if _, err := repos.users.GetByEmail("B1@X.EXAMPLE"); err == nil {
t.Error("emails are compared exactly like SQLite does")
}
})
}
func firstDifferingLine(expected string, actual string) string {
expectedLines, actualLines := strings.Split(expected, "\n"), strings.Split(actual, "\n")
for index := 0; index < max(len(expectedLines), len(actualLines)); index++ {
var want, got string
if index < len(expectedLines) {
want = expectedLines[index]
}
if index < len(actualLines) {
got = actualLines[index]
}
if want != got {
return fmt.Sprintf("line %d: want %q got %q", index+1, want, got)
}
}
return ""
}
func withoutRedundantCharset(dump string) string {
return strings.ReplaceAll(dump, " CHARACTER SET utf8mb4", "")
}
+368
View File
@@ -0,0 +1,368 @@
package test
import (
domainArticle "RenewCMS/internal/domain/article"
domainImage "RenewCMS/internal/domain/image"
domainInvite "RenewCMS/internal/domain/invite"
domainPurge "RenewCMS/internal/domain/purge"
domainUser "RenewCMS/internal/domain/user"
"RenewCMS/internal/infrastructure/persistence/database"
persistence "RenewCMS/internal/infrastructure/persistence/repositories"
"database/sql"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
mysqldriver "github.com/go-sql-driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
const mysqlTestDSNVariable = "RENEWCMS_TEST_MYSQL_DSN"
var mysqlTestDatabaseCounter atomic.Int64
type contractRepositories struct {
db *gorm.DB
users *persistence.UserRepository
invites *persistence.InviteRepository
images *persistence.ImageRepository
articles *persistence.ArticleRepository
purge *persistence.PurgeSettingsRepository
}
type contractBackend struct {
name string
open func(t *testing.T) *gorm.DB
}
func openContractSQLite(t *testing.T) *gorm.DB {
t.Helper()
path := filepath.ToSlash(filepath.Join(t.TempDir(), "contract.db"))
connection, err := database.Connect(database.Config{Driver: database.DriverSQLite, Source: path})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
connection.DB.Logger = logger.Discard
if err := database.Migrate(connection.DB); err != nil {
t.Fatalf("migrate sqlite: %v", err)
}
t.Cleanup(func() { _ = connection.Close() })
return connection.DB
}
type mysqlTestServer struct {
admin *sql.DB
config database.MySQLConfig
}
func connectMySQLTestServer(t *testing.T) *mysqlTestServer {
t.Helper()
dsn := os.Getenv(mysqlTestDSNVariable)
if dsn == "" {
t.Skip(mysqlTestDSNVariable + " is not set; run task test:mysql to test against a real server")
}
parsed, err := mysqldriver.ParseDSN(dsn)
if err != nil {
t.Fatalf("parse %s: %v", mysqlTestDSNVariable, err)
}
host, port, err := net.SplitHostPort(parsed.Addr)
if err != nil {
t.Fatalf("split %q: %v", parsed.Addr, err)
}
admin, err := sql.Open("mysql", dsn)
if err != nil {
t.Fatalf("open admin: %v", err)
}
t.Cleanup(func() { _ = admin.Close() })
return &mysqlTestServer{admin: admin, config: database.MySQLConfig{Host: host, Port: port, User: parsed.User, Password: parsed.Passwd, TLS: "false"}}
}
func (s *mysqlTestServer) createDatabase(t *testing.T) database.MySQLConfig {
t.Helper()
name := fmt.Sprintf("renewcms_test_%d_%d", time.Now().UnixNano()%1_000_000_000, mysqlTestDatabaseCounter.Add(1))
if _, err := s.admin.Exec("CREATE DATABASE " + name); err != nil {
t.Fatalf("create database: %v", err)
}
t.Cleanup(func() { _, _ = s.admin.Exec("DROP DATABASE IF EXISTS " + name) })
config := s.config
config.Name = name
return config
}
func (s *mysqlTestServer) connect(t *testing.T, config database.MySQLConfig) *database.Connection {
t.Helper()
connection, err := database.Connect(database.Config{Driver: database.DriverMySQL, Source: database.MySQLDSN(config)})
if err != nil {
t.Fatalf("connect mysql: %v", err)
}
connection.DB.Logger = logger.Discard
t.Cleanup(func() { _ = connection.Close() })
return connection
}
func openContractMySQL(t *testing.T) *gorm.DB {
t.Helper()
server := connectMySQLTestServer(t)
connection := server.connect(t, server.createDatabase(t))
if err := database.Migrate(connection.DB); err != nil {
t.Fatalf("migrate mysql: %v", err)
}
return connection.DB
}
func newContractRepositories(db *gorm.DB) contractRepositories {
return contractRepositories{
db: db,
users: persistence.NewUserRepository(db),
invites: persistence.NewInviteRepository(db),
images: persistence.NewImageRepository(db),
articles: persistence.NewArticleRepository(db),
purge: persistence.NewPurgeSettingsRepository(db),
}
}
func contractUser(username string, email string) domainUser.User {
return domainUser.User{Username: username, Password: "hashed", Email: email, Role: domainUser.RoleAdmin, IsVerified: true}
}
func contractInvite(t *testing.T, repos contractRepositories, email string) domainInvite.Invite {
t.Helper()
created, err := repos.invites.Create(domainInvite.Invite{
Email: email,
HashedCode: "hashed-code",
ExpiresAt: time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC),
InvitedBy: 1,
CreatedAt: time.Date(2029, 12, 30, 3, 4, 5, 0, time.UTC),
})
if err != nil {
t.Fatalf("create invite: %v", err)
}
return created
}
func mustCreateUser(t *testing.T, repos contractRepositories, user domainUser.User) domainUser.User {
t.Helper()
created, err := repos.users.Create(user)
if err != nil {
t.Fatalf("create user %q: %v", user.Username, err)
}
return created
}
func mustCreateImage(t *testing.T, repos contractRepositories, filename string, hash string) domainImage.Image {
t.Helper()
created, err := repos.images.Create(domainImage.Image{Path: "/static/uploadedImages/" + filename, Filename: filename, SHA256: hash, ContentType: "image/png", Size: 3})
if err != nil {
t.Fatalf("create image %q: %v", filename, err)
}
return created
}
func runRepositoryContract(t *testing.T, backend contractBackend) {
cases := []struct {
name string
run func(t *testing.T, repos contractRepositories)
}{
{`create user and read it back with its id and timestamps`, func(t *testing.T, repos contractRepositories) {
created := mustCreateUser(t, repos, contractUser("Alice", "alice@x.example"))
read, err := repos.users.Get(created.ID)
if err != nil || created.ID == 0 || read.ID != created.ID || read.Username != "Alice" || read.Email != "alice@x.example" {
t.Fatalf("created %+v read %+v err %v", created, read, err)
}
if created.CreatedAt.IsZero() || read.CreatedAt.IsZero() || read.UpdatedAt.IsZero() {
t.Errorf("timestamps must be filled: created %v read %v / %v", created.CreatedAt, read.CreatedAt, read.UpdatedAt)
}
if difference := read.CreatedAt.Sub(created.CreatedAt).Abs(); difference > time.Second {
t.Errorf("the stored timestamp %v must match the returned one %v", read.CreatedAt, created.CreatedAt)
}
if time.Since(read.CreatedAt).Abs() > time.Minute {
t.Errorf("the timestamp %v must be the current instant", read.CreatedAt)
}
}},
{`username "Bob" and "bob" are distinct users`, func(t *testing.T, repos contractRepositories) {
upper := mustCreateUser(t, repos, contractUser("Bob", "upper@x.example"))
lower := mustCreateUser(t, repos, contractUser("bob", "lower@x.example"))
foundUpper, upperErr := repos.users.GetByUsername("Bob")
foundLower, lowerErr := repos.users.GetByUsername("bob")
if upperErr != nil || lowerErr != nil || foundUpper.ID != upper.ID || foundLower.ID != lower.ID || upper.ID == lower.ID {
t.Errorf("Bob %+v (%v) and bob %+v (%v) must be distinct", foundUpper, upperErr, foundLower, lowerErr)
}
}},
{`duplicate email returns an error and stores nothing`, func(t *testing.T, repos contractRepositories) {
mustCreateUser(t, repos, contractUser("first", "same@x.example"))
_, err := repos.users.Create(contractUser("second", "same@x.example"))
if err == nil {
t.Error("expected an error for the duplicate email")
}
if all := repos.users.GetAll(); len(all) != 1 || all[0].Username != "first" {
t.Errorf("nothing may be stored for the failed create, got %+v", all)
}
}},
{`unknown id returns domain ErrUserNotFound`, func(t *testing.T, repos contractRepositories) {
_, err := repos.users.Get(4242)
if !errors.Is(err, domainUser.ErrUserNotFound) {
t.Errorf("expected ErrUserNotFound, got %v", err)
}
}},
{`accept invite twice: the second returns ErrInviteNotFound`, func(t *testing.T, repos contractRepositories) {
pending := contractInvite(t, repos, "new@x.example")
accepted := time.Now()
first, firstErr := repos.invites.AcceptWithUser(pending.ID, accepted, contractUser("lea", "new@x.example"))
_, secondErr := repos.invites.AcceptWithUser(pending.ID, accepted, contractUser("lea2", "other@x.example"))
if firstErr != nil || first.ID == 0 {
t.Fatalf("first accept: %+v %v", first, firstErr)
}
if !errors.Is(secondErr, domainInvite.ErrInviteNotFound) {
t.Errorf("expected ErrInviteNotFound, got %v", secondErr)
}
if len(repos.users.GetAll()) != 1 {
t.Errorf("only the first accept may create a user")
}
}},
{`accept invite with a taken username leaves the invite unaccepted`, func(t *testing.T, repos contractRepositories) {
mustCreateUser(t, repos, contractUser("marta", "marta@x.example"))
pending := contractInvite(t, repos, "new@x.example")
_, err := repos.invites.AcceptWithUser(pending.ID, time.Now(), contractUser("marta", "new@x.example"))
if err == nil {
t.Fatal("expected the taken username to be refused")
}
invites := repos.invites.GetAll()
if len(invites) != 1 || invites[0].AcceptedAt != nil {
t.Errorf("the invite must stay pending, got %+v", invites)
}
}},
{`renew invite with unchanged values still succeeds`, func(t *testing.T, repos contractRepositories) {
pending := contractInvite(t, repos, "new@x.example")
renewed, err := repos.invites.Renew(pending.ID, pending.HashedCode, pending.ExpiresAt, pending.CreatedAt)
if err != nil || renewed.ID != pending.ID || !renewed.ExpiresAt.Equal(pending.ExpiresAt) {
t.Errorf("renew with identical values: %+v %v", renewed, err)
}
}},
{`duplicate image hashes are grouped and counted`, func(t *testing.T, repos contractRepositories) {
first := strings.Repeat("a", 64)
second := strings.Repeat("b", 64)
mustCreateImage(t, repos, "a1.png", first)
mustCreateImage(t, repos, "a2.png", first)
mustCreateImage(t, repos, "b1.png", second)
mustCreateImage(t, repos, "none1.png", "")
mustCreateImage(t, repos, "none2.png", "")
counts, countsErr := repos.images.Counts()
duplicates, listErr := repos.images.ListDuplicates()
if countsErr != nil || counts.All != 5 || counts.DuplicateGroups != 1 {
t.Errorf("counts %+v err %v", counts, countsErr)
}
if listErr != nil || len(duplicates) != 2 || duplicates[0].Filename != "a1.png" || duplicates[1].Filename != "a2.png" {
t.Errorf("duplicates %+v err %v", duplicates, listErr)
}
}},
{`media search treats % and _ literally`, func(t *testing.T, repos contractRepositories) {
for _, filename := range []string{"100%.png", "a_b.png", "axb.png", "50x.png", "Plain.png"} {
mustCreateImage(t, repos, filename, "")
}
percent, percentErr := repos.images.List(domainImage.SearchQuery{Search: "%"})
underscore, underscoreErr := repos.images.List(domainImage.SearchQuery{Search: "_"})
caseless, caselessErr := repos.images.List(domainImage.SearchQuery{Search: "PLAIN"})
if percentErr != nil || len(percent) != 1 || percent[0].Filename != "100%.png" {
t.Errorf("percent search %+v err %v", percent, percentErr)
}
if underscoreErr != nil || len(underscore) != 1 || underscore[0].Filename != "a_b.png" {
t.Errorf("underscore search %+v err %v", underscore, underscoreErr)
}
if caselessErr != nil || len(caseless) != 1 || caseless[0].Filename != "Plain.png" {
t.Errorf("case insensitive search %+v err %v", caseless, caselessErr)
}
}},
{`article body of 1 MB round-trips`, func(t *testing.T, repos contractRepositories) {
body := "é€😀" + strings.Repeat("0123456789abcdef", 1<<16)
created, err := repos.articles.Create(domainArticle.Article{Title: "Big", Body: body, IsOnline: true})
if err != nil {
t.Fatalf("create: %v", err)
}
read, err := repos.articles.Get(created.ID)
if err != nil || read.Body != body || !read.IsOnline {
t.Errorf("the body must round-trip, got %d bytes err %v", len(read.Body), err)
}
found, err := repos.articles.FindByBodyContaining("€😀0123")
if err != nil || len(found) != 1 {
t.Errorf("body search found %d err %v", len(found), err)
}
}},
{`zero reset expiration round-trips as a zero time`, func(t *testing.T, repos contractRepositories) {
created := mustCreateUser(t, repos, contractUser("zero", "zero@x.example"))
fresh, freshErr := repos.users.Get(created.ID)
withCode, codeErr := repos.users.UpdatePasswordResetCode(created.ID, "code", time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC))
reset, resetErr := repos.users.ResetPassword(created.ID, "new-hash")
reread, rereadErr := repos.users.Get(created.ID)
if freshErr != nil || !fresh.PasswordResetExpiration.IsZero() || !fresh.VerificationExpiration.IsZero() {
t.Errorf("fresh user expirations %v / %v err %v", fresh.PasswordResetExpiration, fresh.VerificationExpiration, freshErr)
}
if codeErr != nil || withCode.PasswordResetExpiration.IsZero() {
t.Errorf("the expiration must be stored: %+v %v", withCode, codeErr)
}
if resetErr != nil || rereadErr != nil || !reset.PasswordResetExpiration.IsZero() || !reread.PasswordResetExpiration.IsZero() || reread.TokenVersion != 1 {
t.Errorf("the zero expiration must round-trip: %+v %v %v", reread, resetErr, rereadErr)
}
}},
{`purge settings save, then SaveLastRun with identical values succeeds`, func(t *testing.T, repos contractRepositories) {
enabledAt := time.Date(2030, 1, 2, 3, 4, 5, 0, time.UTC)
lastRun := time.Date(2030, 2, 3, 4, 5, 6, 0, time.UTC)
saved := domainPurge.Settings{AutoPurgeEnabled: true, AutoPurgeIntervalDays: 7, EnabledAt: &enabledAt}
if err := repos.purge.Save(saved); err != nil {
t.Fatalf("save: %v", err)
}
firstErr := repos.purge.SaveLastRun(lastRun, 3, 4)
secondErr := repos.purge.SaveLastRun(lastRun, 3, 4)
got, err := repos.purge.Get()
if firstErr != nil || secondErr != nil || err != nil {
t.Fatalf("SaveLastRun errors %v / %v, get %v", firstErr, secondErr, err)
}
if !got.AutoPurgeEnabled || got.AutoPurgeIntervalDays != 7 || got.LastPurgeMerged != 3 || got.LastPurgeArticlesUpdated != 4 || got.LastPurgeAt == nil || !got.LastPurgeAt.Equal(lastRun) {
t.Errorf("the settings must survive an identical SaveLastRun, got %+v", got)
}
}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
tc.run(t, newContractRepositories(backend.open(t)))
})
}
}
func TestSQLiteRepositoryContract(t *testing.T) {
runRepositoryContract(t, contractBackend{name: "sqlite", open: openContractSQLite})
}
func TestMySQLRepositoryContract(t *testing.T) {
runRepositoryContract(t, contractBackend{name: "mysql", open: openContractMySQL})
}
+328
View File
@@ -0,0 +1,328 @@
package test
import (
"RenewCMS/internal/infrastructure/persistence/database"
"bytes"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
type fakeDumpTable struct {
create string
columns []database.DumpColumn
rows [][][]byte
}
type fakeDumpSource struct {
order []string
tables map[string]fakeDumpTable
failOn string
closed bool
rowsError error
}
var _ database.DumpSource = &fakeDumpSource{}
type fakeDumpRows struct {
table fakeDumpTable
index int
failed error
}
var _ database.DumpRows = &fakeDumpRows{}
func (s *fakeDumpSource) Tables() ([]string, error) { return s.order, nil }
func (s *fakeDumpSource) CreateStatement(table string) (string, error) {
if s.failOn == table {
return "", errors.New("show create failed")
}
return s.tables[table].create, nil
}
func (s *fakeDumpSource) Rows(table string) (database.DumpRows, error) {
return &fakeDumpRows{table: s.tables[table], index: -1, failed: s.rowsError}, nil
}
func (s *fakeDumpSource) Close() error {
s.closed = true
return nil
}
func (r *fakeDumpRows) Columns() ([]database.DumpColumn, error) { return r.table.columns, nil }
func (r *fakeDumpRows) Next() bool {
r.index++
return r.index < len(r.table.rows)
}
func (r *fakeDumpRows) Values() ([][]byte, error) { return r.table.rows[r.index], nil }
func (r *fakeDumpRows) Err() error { return r.failed }
func (r *fakeDumpRows) Close() error { return nil }
func newFakeDumpSource() *fakeDumpSource {
return &fakeDumpSource{
order: []string{"notes", "plain"},
tables: map[string]fakeDumpTable{
"notes": {
create: "CREATE TABLE `notes` (\n `id` int unsigned NOT NULL,\n `body` longtext,\n `blob` longblob,\n `at` datetime(3),\n `ok` tinyint(1)\n)",
columns: []database.DumpColumn{
{Name: "id", Type: "UNSIGNED INT"},
{Name: "body", Type: "LONGTEXT"},
{Name: "blob", Type: "LONGBLOB"},
{Name: "at", Type: "DATETIME"},
{Name: "ok", Type: "TINYINT"},
},
rows: [][][]byte{
{[]byte("1"), []byte("it's a \"test\"\nline\\two\r\x00\x1a"), []byte{0x00, 0xff, 0x10}, []byte("2026-01-02 03:04:05.123000"), []byte("1")},
{[]byte("2"), nil, []byte{}, []byte("0000-00-00 00:00:00"), []byte("0")},
},
},
"plain": {
create: "CREATE TABLE `plain` (\n `a` int\n)",
columns: []database.DumpColumn{{Name: "a", Type: "INT"}},
},
},
}
}
func TestDumpWriter(t *testing.T) {
t.Run("the dump holds the schema and the rows of every table", func(t *testing.T) {
var output bytes.Buffer
err := database.WriteDump(&output, newFakeDumpSource())
if err != nil {
t.Fatalf("write: %v", err)
}
dump := output.String()
for _, expected := range []string{
"CREATE TABLE `notes` (",
"CREATE TABLE `plain` (",
"INSERT INTO `notes` (`id`,`body`,`blob`,`at`,`ok`) VALUES (1,'it\\'s a \"test\"\\nline\\\\two\\r\\0\\Z',0x00ff10,'2026-01-02 03:04:05.123000',1);",
"INSERT INTO `notes` (`id`,`body`,`blob`,`at`,`ok`) VALUES (2,NULL,'','0000-00-00 00:00:00',0);",
"SET sql_mode = 'NO_AUTO_VALUE_ON_ZERO';",
} {
if !strings.Contains(dump, expected) {
t.Errorf("the dump must contain %q, got:\n%s", expected, dump)
}
}
if strings.Contains(dump, "INSERT INTO `plain`") {
t.Error("an empty table has no rows to insert")
}
})
t.Run("every statement after the schema stays on one line", func(t *testing.T) {
var output bytes.Buffer
_ = database.WriteDump(&output, newFakeDumpSource())
for _, line := range strings.Split(output.String(), "\n") {
if strings.HasPrefix(line, "INSERT") && !strings.HasSuffix(line, ");") {
t.Errorf("insert split over several lines: %q", line)
}
}
})
t.Run("a failing table stops the dump with the error", func(t *testing.T) {
source := newFakeDumpSource()
source.failOn = "plain"
err := database.WriteDump(&bytes.Buffer{}, source)
if err == nil || err.Error() != "show create failed" {
t.Errorf("unexpected error %v", err)
}
})
t.Run("a row error stops the dump with the error", func(t *testing.T) {
source := newFakeDumpSource()
source.rowsError = errors.New("connection lost")
err := database.WriteDump(&bytes.Buffer{}, source)
if err == nil || err.Error() != "connection lost" {
t.Errorf("unexpected error %v", err)
}
})
}
type dumpBackupRig struct {
directory string
source *fakeDumpSource
clock time.Time
backup *database.DumpBackup
}
func newDumpBackupRig(t *testing.T) *dumpBackupRig {
t.Helper()
rig := &dumpBackupRig{
directory: filepath.Join(t.TempDir(), "nested", "backups"),
source: newFakeDumpSource(),
clock: time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC),
}
rig.backup = database.NewDumpBackup(func() (database.DumpSource, error) { return rig.source, nil }, rig.directory, func() time.Time {
rig.clock = rig.clock.Add(time.Second)
return rig.clock
})
return rig
}
func (r *dumpBackupRig) names(t *testing.T) []string {
t.Helper()
entries, err := os.ReadDir(r.directory)
if err != nil {
t.Fatalf("read backups: %v", err)
}
names := make([]string, 0, len(entries))
for _, entry := range entries {
names = append(names, entry.Name())
}
return names
}
func TestDumpBackup(t *testing.T) {
t.Run("a file renewcms-<time>.sql exists in the backups directory", func(t *testing.T) {
rig := newDumpBackupRig(t)
path, err := rig.backup.Backup()
if err != nil {
t.Fatalf("backup: %v", err)
}
name := filepath.Base(path)
if filepath.Dir(path) != rig.directory || !strings.HasPrefix(name, "renewcms-20260304T050608") || !strings.HasSuffix(name, ".sql") {
t.Errorf("unexpected backup path %q", path)
}
content, err := os.ReadFile(path)
if err != nil || !strings.Contains(string(content), "INSERT INTO `notes`") {
t.Errorf("unexpected content %q err %v", content, err)
}
if !rig.source.closed {
t.Error("the source must be released")
}
})
t.Run("only the 3 newest backups are kept", func(t *testing.T) {
rig := newDumpBackupRig(t)
var paths []string
for range 5 {
path, err := rig.backup.Backup()
if err != nil {
t.Fatalf("backup: %v", err)
}
paths = append(paths, path)
}
names := rig.names(t)
if len(names) != 3 {
t.Fatalf("expected 3 backups, got %v", names)
}
for _, newest := range paths[2:] {
if _, err := os.Stat(newest); err != nil {
t.Errorf("the newest backup %q must remain: %v", newest, err)
}
}
})
t.Run("a failure returns an error and leaves no file behind", func(t *testing.T) {
rig := newDumpBackupRig(t)
rig.source.failOn = "plain"
path, err := rig.backup.Backup()
if err == nil || path != "" {
t.Fatalf("expected a failure, got path %q err %v", path, err)
}
if names := rig.names(t); len(names) != 0 {
t.Errorf("no partial file may remain, got %v", names)
}
})
t.Run("a source that cannot be opened is a failure", func(t *testing.T) {
directory := filepath.Join(t.TempDir(), "backups")
backup := database.NewDumpBackup(func() (database.DumpSource, error) { return nil, errors.New("no connection") }, directory, time.Now)
path, err := backup.Backup()
if err == nil || path != "" {
t.Errorf("expected a failure, got path %q err %v", path, err)
}
if entries, _ := os.ReadDir(directory); len(entries) != 0 {
t.Errorf("no file may remain, got %d entries", len(entries))
}
})
t.Run("a backup taken at the same instant is refused instead of overwritten", func(t *testing.T) {
directory := filepath.Join(t.TempDir(), "backups")
fixed := func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) }
backup := database.NewDumpBackup(func() (database.DumpSource, error) { return newFakeDumpSource(), nil }, directory, fixed)
if _, err := backup.Backup(); err != nil {
t.Fatalf("first backup: %v", err)
}
_, err := backup.Backup()
if err == nil {
t.Error("expected the second backup to fail")
}
})
}
func TestDumpBackupFiles(t *testing.T) {
t.Run("backups sharing a modification time are pruned by name, oldest name first", func(t *testing.T) {
rig := newDumpBackupRig(t)
if err := os.MkdirAll(rig.directory, os.ModePerm); err != nil {
t.Fatalf("mkdir: %v", err)
}
shared := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
for _, name := range []string{"renewcms-1.sql", "renewcms-2.sql", "renewcms-3.sql", "renewcms-4.sql"} {
path := filepath.Join(rig.directory, name)
if err := os.WriteFile(path, []byte("x"), 0o600); err != nil {
t.Fatalf("write: %v", err)
}
if err := os.Chtimes(path, shared, shared); err != nil {
t.Fatalf("chtimes: %v", err)
}
}
if _, err := rig.backup.Backup(); err != nil {
t.Fatalf("backup: %v", err)
}
names := rig.names(t)
if len(names) != 3 || !strings.HasPrefix(names[0], "renewcms-2026") || names[1] != "renewcms-3.sql" || names[2] != "renewcms-4.sql" {
t.Errorf("the new backup and the two highest names must survive, got %v", names)
}
})
t.Run("a target that cannot be renamed into is a failure and leaves no partial file", func(t *testing.T) {
rig := newDumpBackupRig(t)
blocker := ""
rig.backup = database.NewDumpBackup(func() (database.DumpSource, error) {
blocker = filepath.Join(rig.directory, "renewcms-"+rig.clock.UTC().Format("20060102T150405.000000000Z")+".sql")
if err := os.MkdirAll(filepath.Join(blocker, "inner"), os.ModePerm); err != nil {
t.Fatalf("block target: %v", err)
}
return rig.source, nil
}, rig.directory, func() time.Time {
rig.clock = rig.clock.Add(time.Second)
return rig.clock
})
path, err := rig.backup.Backup()
if err == nil || path != "" {
t.Fatalf("expected a failure, got path %q err %v", path, err)
}
for _, name := range rig.names(t) {
if strings.HasSuffix(name, ".partial") {
t.Errorf("the partial file %q must be removed", name)
}
}
})
}
+177
View File
@@ -0,0 +1,177 @@
package test
import (
"RenewCMS/internal/infrastructure/persistence/database"
"fmt"
"path/filepath"
"reflect"
"testing"
"time"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
type legacyArticle struct {
gorm.Model
ID uint32 `gorm:"primary_key;auto_increment;not_null"`
Title string
Body string
IsOnline bool `gorm:"not_null;default:false"`
CreatedAt time.Time `gorm:"autoCreateTime"`
UpdatedAt time.Time `gorm:"autoUpdateTime"`
}
func (legacyArticle) TableName() string { return "articles" }
type legacyImage struct {
gorm.Model
ID uint32 `gorm:"primary_key;auto_increment;not_null"`
Path string
Filename string
Size int64
ContentType string
Width int
Height int
SHA256 string `gorm:"column:sha256;index"`
UploadedByID uint32
CreatedAt time.Time `gorm:"autoCreateTime"`
UpdatedAt time.Time `gorm:"autoUpdateTime"`
}
func (legacyImage) TableName() string { return "images" }
type legacyUser struct {
ID uint32 `gorm:"primaryKey;autoIncrement"`
Username string `gorm:"unique;not null"`
Password string `gorm:"not null"`
PasswordResetCode *string `gorm:"unique"`
Email string `gorm:"unique;not null"`
DisplayName string `gorm:"not null;default:''"`
IsVerified bool `gorm:"default=false;not null"`
Role string `gorm:"not null;default:''"`
VerificationCode *string `gorm:"unique"`
VerificationExpiration time.Time
PasswordResetExpiration time.Time
TokenVersion uint32 `gorm:"default:0;not null"`
CreatedAt time.Time `gorm:"autoCreateTime"`
UpdatedAt time.Time `gorm:"autoUpdateTime"`
}
func (legacyUser) TableName() string { return "users" }
type legacyInvite struct {
ID uint32 `gorm:"primaryKey;autoIncrement"`
Email string `gorm:"index;not null"`
HashedCode string `gorm:"not null"`
ExpiresAt time.Time
InvitedBy uint32 `gorm:"not null"`
AcceptedAt *time.Time
CreatedAt time.Time `gorm:"autoCreateTime"`
}
func (legacyInvite) TableName() string { return "invites" }
type legacyAPIKey struct {
ID uint32 `gorm:"primaryKey;autoIncrement"`
UserID uint32 `gorm:"index;not null"`
Name string `gorm:"not null"`
Prefix string `gorm:"not null"`
HashedKey string `gorm:"uniqueIndex;not null"`
ExpiresAt *time.Time
RevokedAt *time.Time
LastUsedAt *time.Time
CreatedAt time.Time `gorm:"autoCreateTime"`
}
func (legacyAPIKey) TableName() string { return "api_keys" }
type legacyPurgeSettings struct {
ID uint32 `gorm:"primaryKey"`
AutoPurgeEnabled bool `gorm:"not null"`
AutoPurgeIntervalDays int `gorm:"not null"`
LastPurgeAt *time.Time
LastPurgeMerged int `gorm:"not null"`
LastPurgeArticlesUpdated int `gorm:"not null"`
EnabledAt *time.Time
}
func (legacyPurgeSettings) TableName() string { return "purge_settings" }
type sqliteSnapshot struct {
schema map[string]string
rows map[string][]map[string]any
}
func takeSqliteSnapshot(t *testing.T, db *gorm.DB) sqliteSnapshot {
t.Helper()
snapshot := sqliteSnapshot{schema: map[string]string{}, rows: map[string][]map[string]any{}}
var objects []struct{ Name, Sql string }
if err := db.Raw("SELECT name, sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name").Scan(&objects).Error; err != nil {
t.Fatalf("read schema: %v", err)
}
for _, object := range objects {
snapshot.schema[object.Name] = object.Sql
}
for _, table := range []string{"articles", "images", "users", "invites", "api_keys", "purge_settings"} {
var rows []map[string]any
if err := db.Raw(fmt.Sprintf("SELECT * FROM %s ORDER BY 1", table)).Scan(&rows).Error; err != nil {
t.Fatalf("read %s: %v", table, err)
}
snapshot.rows[table] = rows
}
return snapshot
}
func seedLegacyRows(t *testing.T, db *gorm.DB) {
t.Helper()
at := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
reset := "reset-1"
seeds := []any{
&legacyArticle{Title: "First", Body: "Body", IsOnline: true},
&legacyImage{Path: "/static/uploadedImages/a.png", Filename: "a.png", Size: 12, ContentType: "image/png", Width: 1, Height: 2, SHA256: "abc", UploadedByID: 1},
&legacyUser{Username: "Bob", Password: "x", Email: "bob@x.example", DisplayName: "Bob", IsVerified: true, Role: "owner", PasswordResetCode: &reset, VerificationExpiration: at},
&legacyUser{Username: "bob", Password: "y", Email: "bob2@x.example", Role: "admin"},
&legacyInvite{Email: "new@x.example", HashedCode: "h", ExpiresAt: at, InvitedBy: 1},
&legacyAPIKey{UserID: 1, Name: "ci", Prefix: "rcms_abc", HashedKey: "0123456789abcdef", ExpiresAt: &at},
&legacyPurgeSettings{ID: 1, AutoPurgeEnabled: true, AutoPurgeIntervalDays: 7, LastPurgeAt: &at},
}
for _, seed := range seeds {
if err := db.Create(seed).Error; err != nil {
t.Fatalf("seed %T: %v", seed, err)
}
}
}
func TestSQLiteSchemaIsUntouchedByTheModelChanges(t *testing.T) {
t.Run("SQLite schema is untouched by the model changes", func(t *testing.T) {
path := filepath.ToSlash(filepath.Join(t.TempDir(), "legacy.db"))
db, err := gorm.Open(sqlite.Open(database.SQLiteDSN(path)), &gorm.Config{Logger: logger.Discard})
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(func() {
if sqlDB, err := db.DB(); err == nil {
_ = sqlDB.Close()
}
})
if err := db.AutoMigrate(&legacyArticle{}, &legacyImage{}, &legacyUser{}, &legacyInvite{}, &legacyAPIKey{}, &legacyPurgeSettings{}); err != nil {
t.Fatalf("legacy migrate: %v", err)
}
seedLegacyRows(t, db)
before := takeSqliteSnapshot(t, db)
if err := database.Migrate(db); err != nil {
t.Fatalf("migrate: %v", err)
}
after := takeSqliteSnapshot(t, db)
if !reflect.DeepEqual(before.schema, after.schema) {
t.Errorf("schema changed:\nbefore %v\nafter %v", before.schema, after.schema)
}
if !reflect.DeepEqual(before.rows, after.rows) {
t.Errorf("rows changed:\nbefore %v\nafter %v", before.rows, after.rows)
}
})
}