feat: optional S3-compatible storage for uploaded images, instead of the local upload directory

This commit is contained in:
fsylvain committed 2026-10-06 12:13:24 +02:00
1 parent d752a61e52
commit eb9bfabed5
23 files changed
+1962 -7

No files matched your search

+9
View File
@@ -27,3 +27,12 @@ DB_TLS=false
DB_MAX_OPEN_CONNS=10
DB_MAX_IDLE_CONNS=5
DB_CONN_MAX_LIFETIME_SECONDS=300
STORAGE_DRIVER=local
S3_BUCKET=
S3_REGION=us-east-1
S3_ENDPOINT=
S3_ACCESS_KEY_ID=
S3_SECRET_ACCESS_KEY=
S3_KEY_PREFIX=
S3_PATH_STYLE=false
S3_TLS=true
+41
View File
@@ -87,6 +87,47 @@ tasks:
exit $STATUS
silent: true
test:s3:
desc: Run the S3 storage integration test against a throwaway SeaweedFS in Docker (skipped when Docker is unavailable)
cmds:
- cmd: |
if ! docker info >/dev/null 2>&1; then
echo "Docker is not available, skipping the S3 integration test"
exit 0
fi
export MSYS_NO_PATHCONV=1
NAME=renewcms-test-seaweedfs
IMAGE=chrislusf/seaweedfs:4.48
BUCKET=renewcms-test
ACCESS_KEY=renewcms-test
SECRET_KEY=renewcms-test-secret
docker rm -f "$NAME" >/dev/null 2>&1
docker run -d --name "$NAME" -e AWS_ACCESS_KEY_ID="$ACCESS_KEY" -e AWS_SECRET_ACCESS_KEY="$SECRET_KEY" -p 127.0.0.1::8333 "$IMAGE" server -dir=/data -s3 -s3.port=8333 >/dev/null || exit 1
MAPPING="$(docker port "$NAME" 8333/tcp | head -n 1)"
PORT="${MAPPING##*:}"
READY=no
TRIES=0
while [ "$TRIES" -lt 60 ]; do
if docker run --rm --network "container:$NAME" --entrypoint sh "$IMAGE" -c "echo 's3.bucket.create -name $BUCKET' | timeout 30 weed shell -master=127.0.0.1:9333 && echo s3.bucket.list | timeout 30 weed shell -master=127.0.0.1:9333 | grep -q $BUCKET" >/dev/null 2>&1; then
READY=yes
break
fi
TRIES=$((TRIES + 1))
sleep 2
done
if [ "$READY" != yes ]; then
echo "SeaweedFS did not become ready" >&2
docker logs "$NAME" 2>&1 | tail -n 20 >&2
docker rm -f "$NAME" >/dev/null 2>&1
exit 1
fi
echo "== SeaweedFS on 127.0.0.1:$PORT"
RENEWCMS_TEST_S3_ENDPOINT="http://127.0.0.1:$PORT" RENEWCMS_TEST_S3_BUCKET="$BUCKET" RENEWCMS_TEST_S3_ACCESS_KEY="$ACCESS_KEY" RENEWCMS_TEST_S3_SECRET_KEY="$SECRET_KEY" go test -tags nofrontend ./test/... -run S3StorageIntegration -count=1 -v {{.CLI_ARGS}}
RESULT=$?
docker rm -f "$NAME" >/dev/null 2>&1
exit $RESULT
silent: true
cover:
desc: Go test coverage of the use cases → tmp/coverage.out
cmds:
+32 -2
View File
@@ -109,8 +109,28 @@ const (
backupsDirName = "backups"
)
func imageStorageConfig(settings config.Settings) storage.Config {
if settings.Storage.Driver == config.StorageDriverS3 {
s3 := settings.Storage.S3
return storage.Config{
Driver: storage.DriverS3,
S3: storage.S3Config{
Bucket: s3.Bucket,
Region: s3.Region,
Endpoint: s3.Endpoint,
AccessKeyID: s3.AccessKeyID,
SecretAccessKey: s3.SecretAccessKey,
KeyPrefix: s3.KeyPrefix,
PathStyle: s3.PathStyle,
TLS: s3.TLS,
},
}
}
return storage.Config{Driver: storage.DriverLocal, Location: settings.UploadDir}
}
func openImageStorage(settings config.Settings) domainImage.Storage {
imageStorage, err := storage.Open(storage.Config{Driver: storage.DriverLocal, Location: settings.UploadDir})
imageStorage, err := storage.Open(imageStorageConfig(settings))
if err != nil {
panic("Unable to open the image storage: " + err.Error())
}
@@ -268,11 +288,21 @@ func startUpdateDownloadUseCase(repos *Repositories, settings config.Settings) *
})
}
func backfillImageMetadata(imageRepo domainImage.Repository, imageStorage domainImage.Storage, settings config.Settings) error {
run := func(readable domainImage.Storage) error {
return useCases.NewBackfillImageMetadataUseCase(imageRepo, readable).Run()
}
if settings.Storage.Driver == config.StorageDriverS3 {
return storage.RunBackfillReportingReads(imageStorage, log.Default(), run)
}
return run(imageStorage)
}
func InitContainer(settings config.Settings) (*UseCases, func() error) {
imageStorage := openImageStorage(settings)
connection := openDatabase(settings)
repos := getRepositories(connection, settings)
if err := useCases.NewBackfillImageMetadataUseCase(repos.ImageRepo, imageStorage).Run(); err != nil {
if err := backfillImageMetadata(repos.ImageRepo, imageStorage, settings); err != nil {
panic("Failed to backfill image metadata: " + err.Error())
}
return getUseCases(repos, imageStorage, settings), connection.Close
+9 -1
View File
@@ -52,6 +52,10 @@ var envDefaults = map[string]string{
"DB_MAX_IDLE_CONNS": config.DefaultMaxIdleConns,
"DB_CONN_MAX_LIFETIME_SECONDS": config.DefaultConnMaxLifetimeSeconds,
"UPLOAD_DIR": "./data/uploads",
config.EnvStorageDriver: config.DefaultStorageDriver,
"S3_REGION": config.DefaultS3Region,
"S3_PATH_STYLE": config.DefaultS3PathStyle,
"S3_TLS": config.DefaultS3TLS,
"AUTH_RATE_LIMIT_MAX_ATTEMPTS": "5",
"AUTH_RATE_LIMIT_WINDOW_SECONDS": "60",
"TRUSTED_PROXIES": "",
@@ -115,7 +119,11 @@ func applyEnvDefaults() {
}
func ensureRequiredEnvVars() {
required := slices.Concat(requiredEnvVars, config.DatabaseRequiredEnvVars(os.Getenv(config.EnvDatabaseDriver)))
required := slices.Concat(
requiredEnvVars,
config.DatabaseRequiredEnvVars(os.Getenv(config.EnvDatabaseDriver)),
config.StorageRequiredEnvVars(os.Getenv(config.EnvStorageDriver)),
)
if err := config.MissingEnvVarError(required, os.LookupEnv); err != nil {
panic(err.Error())
}
+56
View File
@@ -0,0 +1,56 @@
services:
seaweedfs:
image: ${SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.48}
restart: unless-stopped
command: ["server", "-dir=/data", "-s3", "-s3.port=8333"]
environment:
AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-renewcms}
AWS_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY}
volumes:
- seaweedfs-data:/data
seaweedfs-init:
image: ${SEAWEEDFS_IMAGE:-chrislusf/seaweedfs:4.48}
restart: "no"
depends_on:
- seaweedfs
entrypoint: ["sh", "-c"]
command:
- |
until echo "s3.bucket.create -name renewcms-media" | timeout 30 weed shell -master=seaweedfs:9333 && echo "s3.bucket.list" | timeout 30 weed shell -master=seaweedfs:9333 | grep -q renewcms-media; do
sleep 2
done
renewcms:
image: ${RENEWCMS_IMAGE:-git.floriansylvain.fr/fsylvain/renewcms:beta}
restart: unless-stopped
depends_on:
seaweedfs-init:
condition: service_completed_successfully
env_file:
- docker.env
environment:
STORAGE_DRIVER: s3
S3_ENDPOINT: http://seaweedfs:8333
S3_BUCKET: renewcms-media
S3_REGION: us-east-1
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-renewcms}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY}
S3_PATH_STYLE: "true"
DB_FILE: /data/renewcms.db
RENEWCMS_CONTAINER: "true"
ports:
- "8080:8080"
volumes:
- renewcms-data:/data
stop_grace_period: 15s
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/healthz"]
interval: 30s
timeout: 5s
retries: 3
start_period: 10s
volumes:
renewcms-data:
seaweedfs-data:
+9
View File
@@ -22,3 +22,12 @@ DB_TLS=false
DB_MAX_OPEN_CONNS=10
DB_MAX_IDLE_CONNS=5
DB_CONN_MAX_LIFETIME_SECONDS=300
STORAGE_DRIVER=local
S3_BUCKET=
S3_REGION=us-east-1
S3_ENDPOINT=
S3_ACCESS_KEY_ID=
S3_SECRET_ACCESS_KEY=
S3_KEY_PREFIX=
S3_PATH_STYLE=false
S3_TLS=true
+12 -1
View File
@@ -57,7 +57,7 @@ The use cases `GetPurgeSettings`, `UpdatePurgeSettings`, `PreviewPurge` and `Run
### Image metadata backfill
At boot, `BackfillImageMetadataUseCase` (through `image.Repository.ListMissingMetadata` and `UpdateMetadata`) fills `filename`, `size`, `content_type`, `width`, `height` and `sha256` for image rows that lack them, reading the bytes through `image.Storage` (the files in `UPLOAD_DIR` with the local adapter). It is idempotent, and a row whose file is missing or unreadable is left as it is without stopping the boot. The old `article_id` column and the `article_images` table are kept and never written.
At boot, `BackfillImageMetadataUseCase` (through `image.Repository.ListMissingMetadata` and `UpdateMetadata`) fills `filename`, `size`, `content_type`, `width`, `height` and `sha256` for image rows that lack them, reading the bytes through `image.Storage` (the files in `UPLOAD_DIR` with the local adapter, the bucket objects with the S3 adapter). It is idempotent, and a row whose file is missing or unreadable is left as it is without stopping the boot. The old `article_id` column and the `article_images` table are kept and never written.
## Frontend ↔ backend
@@ -94,4 +94,15 @@ Backups are a `domain/update.DatabaseBackup` per driver. SQLite uses `VACUUM INT
`task test` runs everything that needs no server. The repository contract tests (`test/repository_contract_test.go`) run on SQLite always, and on MySQL when `RENEWCMS_TEST_MYSQL_DSN` points at a server (admin DSN; each test creates and drops its own database). `task test:mysql` starts `mysql:8.4` and then `mariadb:11` in Docker and runs the `MySQL` tests against each.
Two storage adapters implement `image.Storage`: `local` (default, files in `UPLOAD_DIR`) and `s3` (`storage.S3Storage`, `aws-sdk-go-v2` over plain HTTPS, pure Go so `CGO_ENABLED=0` still holds). `STORAGE_DRIVER` picks one in `imageStorageConfig` (composition root, from `config.StorageSettings`); `storage.Open` fails with `unsupported storage driver "<name>"` for anything else, and `openImageStorage` panics with the error. The S3 adapter works as follows, with no change to the domain, use cases, repositories or the `staticimages` handler:
- `Open` calls `GetObject` and returns a `Blob` whose `Content` is an `io.ReadSeeker` (`s3Reader`): `Size` and `ModTime` come from `ContentLength` and `LastModified`, the first bytes are read from that response and every `Seek` to another offset makes the next `Read` issue a ranged `GetObject`. `http.ServeContent` therefore answers `Range` (206, 416) and `If-Modified-Since` (304) with no `ETag`. A 404 maps to `image.ErrBlobNotFound`; any other failure is a plain error (500 in the handler).
- `Put` does one `PutObject` of `<S3_KEY_PREFIX><uuid>.<ext>` with the upload's `Content-Type`. `Delete` treats 404 as success. Names that are not plain file names are never requested, as in the local adapter.
- Each call has a 30 second timeout to get a response (the body then streams without a deadline) and the SDK default retries. Request checksums are only sent when required, so older S3-compatible servers work.
- `storage.Open` checks the bucket with `HeadBucket`. The failure text names the bucket and the reason (404, 403, other status or transport error) and never the credentials.
- At boot, `RunBackfillReportingReads` wraps the storage in a counter for the `BackfillImageMetadataUseCase` run and logs how many images it read; the use case itself is unchanged.
- Update backups (`DATA_DIR`) never include uploads, whatever the driver.
`task test:s3` starts SeaweedFS in Docker and runs `TestS3StorageIntegration` against it (skipped without Docker, and not part of `task test`). Everything else uses an in-process fake S3 server listening on `127.0.0.1`.
Adding a database driver: implement `database.Driver` (`Open`, and `NewBackup` returning a `domain/update.DatabaseBackup`), register it in `drivers`, then select it in `databaseConfig` in the composition root (and add the env vars to `main.go`, `.env.example` and `configuration.md`). Adding a storage backend: implement `image.Storage`, add a case to `storage.Open`, and select it in `openImageStorage`. No use case, handler or repository changes.
+34 -1
View File
@@ -27,7 +27,8 @@ Copy [`.env.example`](../.env.example) to `.env` to get started. When a required
| `DB_DRIVER` | `sqlite` | `sqlite` or `mysql` (MySQL and MariaDB). Any other value stops the server with `unsupported database driver "<value>"`. See [Database](#database). |
| `DB_FILE` | `./data/renewcms.db` | Path to the SQLite database file. Missing parent directories are created. The schema is migrated automatically at startup. Ignored when `DB_DRIVER=mysql`. |
| `DATA_DIR` | directory of `DB_FILE` | Directory for update state, staged updates and database backups (`backups/`). Set it with `DB_DRIVER=mysql`, where there is no database file to sit beside (the default still follows `DB_FILE`). With SQLite, leave it unset to keep everything beside the database. |
| `UPLOAD_DIR` | `./data/uploads` | Directory where uploaded images are stored. |
| `STORAGE_DRIVER` | `local` | `local` (files in `UPLOAD_DIR`) or `s3` (an S3-compatible bucket). Any other value stops the server with `unsupported storage driver "<value>"`. See [Image storage](#image-storage). |
| `UPLOAD_DIR` | `./data/uploads` | Directory where uploaded images are stored. Ignored, without error, when `STORAGE_DRIVER=s3`. |
| `CORS_ALLOWED_ORIGINS` | *(empty)* | Semicolon-separated list of origins allowed to call the API. When empty, no CORS headers are sent. That is the right setting whenever the SPA is served by RenewCMS itself (production) or through the Vite proxy (development). |
| `AUTH_RATE_LIMIT_MAX_ATTEMPTS` | `5` | Maximum number of requests a single client IP may send to `POST /v1/auth/login`, `POST /v1/auth/password-reset` or the invite-accept pair (`GET /v1/auth/invite` and `POST /v1/auth/invite/accept`, sharing one counter) within the window, tracked independently per route. |
| `AUTH_RATE_LIMIT_WINDOW_SECONDS` | `60` | Length, in seconds, of the fixed rate-limit window for the routes above. |
@@ -37,6 +38,38 @@ Copy [`.env.example`](../.env.example) to `.env` to get started. When a required
Automatic media purge has no environment variable. Its schedule (on or off, interval, `enabled_at`, `last_purge_at`) lives in the purge settings row of the database, is changed from the Settings page or `PATCH /v1/media/purge-settings`, and is read again at every hourly tick.
## Image storage
By default uploaded images are files in `UPLOAD_DIR`. To keep them in an S3-compatible bucket (AWS S3, MinIO, Ceph, Cloudflare R2, Backblaze B2 and similar), set `STORAGE_DRIVER=s3` and the variables below. They are only read when `STORAGE_DRIVER=s3`. Image URLs do not change: the app still serves `/static/uploadedImages/<name>`, reading the object from the bucket (the bucket stays private, and `Range` and `If-Modified-Since` requests keep working; there is no `ETag`).
| Name | Default | Description |
|------------------------|-------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `S3_BUCKET` | required | Name of an existing bucket. A missing value stops the server with `Environment variable S3_BUCKET is not set`. |
| `S3_REGION` | `us-east-1` | Bucket region. Most S3-compatible servers accept the default. |
| `S3_ENDPOINT` | *(empty)* | Host (with optional port) or full URL of an S3-compatible server. Leave empty for AWS S3. |
| `S3_ACCESS_KEY_ID` | *(empty)* | Access key. Set it together with `S3_SECRET_ACCESS_KEY`, or set neither. Setting only one stops the server with an error naming the missing one. |
| `S3_SECRET_ACCESS_KEY` | *(empty)* | Secret key. Never logged and never in an error message. When both keys are empty, the default AWS credential chain is used (`AWS_*` variables, shared credentials file, instance or task role). |
| `S3_KEY_PREFIX` | *(empty)* | Prefix put before every object key, for example `uploads/`. Include the trailing slash. |
| `S3_PATH_STYLE` | `false` | `true` addresses the bucket as `endpoint/bucket` instead of `bucket.endpoint`. SeaweedFS, MinIO and most self-hosted servers need `true`. |
| `S3_TLS` | `true` | Only used when `S3_ENDPOINT` has no `http://` or `https://` prefix: `true` means `https`, `false` means `http`. |
At startup the server checks the bucket (`HeadBucket`). A missing bucket, refused credentials or an unreachable server stop startup with `Unable to open the image storage: S3 bucket "<name>" is not usable: <reason>`; the error never contains the secret. Each S3 call has a 30 second timeout and the SDK default retries. At boot, images whose metadata is missing are read back from the bucket to fill it; when there were some, one log line says how many.
### Moving existing uploads to a bucket
There is no automatic migration and the database does not change: image rows only hold the stored name, so the same names must exist in the bucket. With the site stopped, copy the files under the same names, under `S3_KEY_PREFIX` if you set one, then set `STORAGE_DRIVER=s3` and restart:
```sh
aws s3 sync ./data/uploads s3://my-bucket/uploads/
rclone sync ./data/uploads remote:my-bucket/uploads
```
Keep `UPLOAD_DIR` until you have checked the site. Moving back is the same copy in the other direction.
### Backups
The pre-update backup covers the database only (and `DATA_DIR` state). Neither the update backup nor the data directory includes uploads, so back the bucket up with your provider (versioning, replication) or with `aws s3 sync` / `mc mirror`.
## Database
SQLite is the default and needs nothing but `DB_FILE`. To run against MySQL 8 or MariaDB 11 instead, set `DB_DRIVER=mysql` and the connection variables below. They are only read (and the four required ones only checked) when `DB_DRIVER=mysql`. A required variable that is not set stops the server with `Environment variable DB_HOST is not set` (or `DB_USER`, `DB_PASSWORD`, `DB_NAME`).
+1 -1
View File
@@ -8,7 +8,7 @@ RenewCMS ships as **one self-contained executable**. It serves:
| `/healthz` | Liveness check, returns `200 ok` |
| `/*` | The Vue SPA (embedded), with fallback to `index.html` for client-side routes |
Runtime state includes the SQLite database (`DB_FILE`, or a MySQL/MariaDB server, see [Using MySQL or MariaDB](#using-mysql-or-mariadb)), uploads (`UPLOAD_DIR`), and update records, staged binaries and backups beside the database. Running a release binary needs no Go, Bun or Node installation. Browser updates additionally require Linux with systemd and the helper setup below.
Runtime state includes the SQLite database (`DB_FILE`, or a MySQL/MariaDB server, see [Using MySQL or MariaDB](#using-mysql-or-mariadb)), uploads (`UPLOAD_DIR`, or an S3-compatible bucket with `STORAGE_DRIVER=s3`, see [configuration.md](configuration.md#image-storage)), and update records, staged binaries and backups beside the database. Running a release binary needs no Go, Bun or Node installation. Browser updates additionally require Linux with systemd and the helper setup below.
To run it in a container instead, see [docker.md](docker.md). The published image `git.floriansylvain.fr/fsylvain/renewcms` is upgraded with `docker compose pull && docker compose up -d`. Tags are `vX.Y.Z`, `X.Y.Z`, `X.Y` and `latest` for a stable release, and `vX.Y.Z-<pre>`, `X.Y.Z-<pre>` and the moving `<pre-name>` (`beta`, `rc`) for a pre-release, which never moves `latest`. Maintainers publish with `task docker:publish` after `docker login git.floriansylvain.fr`; there is no CI. See [Maintainer: publishing](docker.md#maintainer-publishing).
+13
View File
@@ -102,6 +102,19 @@ To use MariaDB, change the `mysql` service image to `mariadb:11`, its `MYSQL_*`
Back the database up with `docker compose -f docker-compose.mysql.yml exec mysql mysqldump ...` (a `tar` copy of the volume is not consistent while it runs), and the `renewcms-data` volume as described above. Moving an existing SQLite volume is a manual procedure, see [deployment.md](deployment.md#moving-an-existing-sqlite-site-to-mysql).
## S3-compatible storage (SeaweedFS)
[`docker-compose.s3.yml`](../docker-compose.s3.yml) runs RenewCMS with `STORAGE_DRIVER=s3` next to a [SeaweedFS](https://github.com/seaweedfs/seaweedfs) server (Apache 2.0, image `chrislusf/seaweedfs`, override with `SEAWEEDFS_IMAGE`) with its S3 gateway enabled, and a one-shot `seaweedfs-init` step that creates the bucket with `weed shell`:
```sh
export S3_SECRET_ACCESS_KEY=<generated>
docker compose -f docker-compose.s3.yml up -d
```
RenewCMS gets `S3_ENDPOINT=http://seaweedfs:8333`, `S3_PATH_STYLE=true`, the bucket name and the access key (`S3_ACCESS_KEY_ID`, default `renewcms`, and `S3_SECRET_ACCESS_KEY`) from the compose file, and the rest from `docker.env`. SeaweedFS reads the same key pair as its admin identity. The `renewcms-data` volume then holds the SQLite database and update state but no uploads; the images live in the `seaweedfs-data` volume. The bucket is not published and the app serves the images. SeaweedFS is used because MinIO stopped publishing community images. To use AWS S3 or another provider, drop the `seaweedfs` services and set the `S3_*` variables in `docker.env` (see [configuration.md](configuration.md#image-storage)). The volume backup above covers the database only, so back the bucket up separately. To move existing uploads see [configuration.md](configuration.md#moving-existing-uploads-to-a-bucket).
`task test:s3` starts a throwaway SeaweedFS and runs the storage integration test against it; it skips with a message when Docker is not available.
## Reverse proxy and TLS
With `ENVIRONMENT=production` the site must be served over HTTPS, so put a reverse proxy in front and set `PUBLIC_URL` to the public `https://` address. Bind the published port to the host only when the proxy runs on the same host, by changing `ports` to `"127.0.0.1:8080:8080"`.
+143
View File
@@ -0,0 +1,143 @@
Feature: Uploaded images can be stored in an S3-compatible bucket
As an operator whose host disk is ephemeral or who runs several hosts
I want to set STORAGE_DRIVER=s3 and bucket settings
So that uploads live in the bucket while images are still served by the app
at /static/uploadedImages/<name>, with unchanged URLs
Background:
Given STORAGE_DRIVER defaults to "local" and local disk behaves exactly as today
And an S3 storage implements image.Storage (Put, Open, Delete) in internal/infrastructure/storage
And storage.Open has a case for the driver "s3" and openImageStorage selects it from the settings
And no domain, use case, repository or staticimages handler file changes
And the bucket is faked by an in-process httptest S3 server listening on 127.0.0.1
And Open returns a Blob whose Content is an io.ReadSeeker, so http.ServeContent handles Range and conditional requests
And the Blob ModTime and Size come from the S3 object metadata
And no ETag is emitted and conditional requests are validated by Last-Modified only
Scenario Outline: Startup configuration validation
Given STORAGE_DRIVER is <driver> and the S3 variables are <vars>
When the storage settings are validated
Then the result is <result>
Examples:
| driver | vars | result |
| (unset) | none | driver local, no S3 variable required |
| local | S3_BUCKET set | driver local, S3 variables ignored |
| s3 | S3_BUCKET and S3_REGION set | valid, default AWS credential chain |
| s3 | S3_BUCKET set, S3_REGION unset | region defaults to us-east-1 |
| s3 | S3_REGION set, S3_BUCKET unset | error "Environment variable S3_BUCKET is not set" |
| s3 | key id set, secret unset | error naming S3_SECRET_ACCESS_KEY |
| s3 | secret set, key id unset | error naming S3_ACCESS_KEY_ID |
| s3 | UPLOAD_DIR set | valid, UPLOAD_DIR ignored without error |
| gcs | none | error unsupported storage driver "gcs" |
Scenario: The registry opens the S3 storage after a bucket check
Given the fake S3 answers HEAD bucket with 200
When storage.Open is called with driver "s3"
Then it returns a storage and no error
Scenario Outline: Startup fails fast with a clear secret-free error
Given the fake S3 answers HEAD bucket with <status>
When storage.Open is called with driver "s3" and secret access key "TOPSECRET"
Then it returns an error mentioning the bucket name
And the error text does not contain "TOPSECRET"
And openImageStorage panics with that same text
Examples:
| status |
| 404 |
| 403 |
| 500 |
Scenario: Put stores the object under the prefix with its content type
Given S3_KEY_PREFIX is "uploads/"
When Put is called with stored name "a1b2.png", content type "image/png" and 3 bytes
Then the fake S3 holds key "uploads/a1b2.png" with those bytes and Content-Type "image/png"
Scenario: Put failure is returned as an error
Given the fake S3 answers PUT with 500
When Put is called
Then Put returns a non-nil error that is not image.ErrBlobNotFound
Scenario: Open returns content, size and modification time
Given the fake S3 holds "uploads/a1b2.png" with 10 bytes and a known Last-Modified
When Open("a1b2.png") is called
Then the Blob has Size 10 and that ModTime
And its Content implements io.ReadSeeker and reads the 10 bytes
Scenario: Open of a missing key is not found
Given the fake S3 answers GET with 404 NoSuchKey
When Open("missing.png") is called
Then the result is image.ErrBlobNotFound, asserted with errors.Is
Scenario: Open with a backend failure is not a not-found
Given the fake S3 answers GET with 500
When Open is called
Then the error is non-nil and is not image.ErrBlobNotFound
Scenario Outline: Delete is idempotent
Given the fake S3 <state>
When Delete("a1b2.png") is called
Then it returns no error
Examples:
| state |
| holds the key |
| does not hold the key (404) |
Scenario: A backend failure on Delete is returned
Given the fake S3 answers DELETE with 500
When Delete is called
Then it returns a non-nil error
Scenario Outline: Serving /static/uploadedImages/<name> from the bucket
Given the bucket holds "a1b2.png" with 10 bytes and a known Last-Modified
When <method> /static/uploadedImages/a1b2.png is requested <with>
Then the status is <status>
And the response has <result>
Examples:
| method | with | status | result |
| GET | no extra header | 200 | Content-Type image/png, Content-Length 10, Last-Modified, Accept-Ranges bytes, Cache-Control "public, no-cache", X-Content-Type-Options nosniff |
| HEAD | no extra header | 200 | the same headers and an empty body |
| GET | If-Modified-Since = Last-Modified | 304 | an empty body |
| GET | Range bytes=0-3 | 206 | Content-Range "bytes 0-3/10" and Content-Length 4 |
| GET | Range bytes=50-60 | 416 | Content-Range "bytes */10" |
Scenario: SVG from the bucket keeps its security headers
Given the bucket holds "logo.svg"
When GET /static/uploadedImages/logo.svg is requested
Then Content-Type is image/svg+xml
And Content-Security-Policy is "default-src 'none'; style-src 'unsafe-inline'; sandbox"
Scenario: Missing object is 404 and a backend failure is 500
Given the bucket does not hold "missing.png"
When GET /static/uploadedImages/missing.png is requested
Then the status is 404
Given the fake S3 answers GET with 500
When GET /static/uploadedImages/a1b2.png is requested
Then the status is 500 and the plain text body is "Unable to read the file"
Scenario: Path traversal never reaches another key
When GET /static/uploadedImages/../secret.png is requested
Then the key requested from the fake S3 is the prefix plus "secret.png"
Scenario: Handler as-is for a non-seekable blob
Given a storage fake returning a Blob whose Content is not an io.ReadSeeker, with Size 10 and a ModTime
When GET and HEAD /static/uploadedImages/a1b2.png are requested
Then GET returns 200 with Content-Length 10, Last-Modified and the body
And HEAD returns 200 with the same headers and an empty body
And no Range or ETag behaviour is specified for this path
Scenario: Boot backfill logs how many images it read from the bucket
Given the driver is s3 and 3 stored images lack metadata
When InitContainer runs the boot backfill
Then a log line reports 3 images read from storage and contains no credential
And backfill behaviour and errors are unchanged
And no cap is applied and BackfillImageMetadataUseCase is not modified
Scenario: The local driver is unchanged
Given STORAGE_DRIVER is unset and UPLOAD_DIR is a temp directory
When an image is uploaded and requested
Then it is stored in UPLOAD_DIR and served exactly as before
+26
View File
@@ -0,0 +1,26 @@
# QA: S3 image storage
All tests live in test/ (package test). No cloud account, no real SMTP, no real SQLite file. Test names carry "S3Storage".
Settled decisions: boot backfill gets a log line only, wired in the composition root, with no cap and no use case change. No ETag is emitted; the staticimages handler is unchanged and conditional requests are validated by Last-Modified only.
## Automated
1. Config validation: table test over STORAGE_DRIVER and S3_* lookup functions (the MissingEnvVarError pattern, a `StorageRequiredEnvVars(driver)` helper like `DatabaseRequiredEnvVars`). Covers the validation table, UPLOAD_DIR ignored, unknown driver error.
2. Registry: storage.Open with "local" returns local storage, "s3" returns the S3 storage after HEAD bucket, anything else returns the "unsupported storage driver" error.
3. Fake S3: an httptest.Server on 127.0.0.1 only (never bind :port) implementing HEAD bucket, PUT, GET with Range, HEAD and DELETE object, with injectable statuses and recorded keys, headers and bodies.
4. Adapter tests against the fake: the Put, Open (found, 404, 500) and Delete scenarios, HEAD bucket 404/403/500, prefix handling, ModTime and Size from metadata, Content as io.ReadSeeker (seek then read issues a ranged GET), secret absent from error text.
5. Handler tests: staticimages.NewHandler over OpenMediaUseCase over the S3 storage on the fake server, mounted on a fresh chi.NewRouter() at /static/uploadedImages/*. Cover GET, HEAD, 304 via If-Modified-Since, 206, 416, SVG CSP, 404, 500, traversal.
6. Handler regression: a storage fake returning a non-seekable Blob. GET and HEAD behave exactly as today.
7. Backfill log: the boot log line for the number of images read, captured with a log writer, containing no credential.
8. Local regression: all existing image tests pass unchanged.
9. Gates: `task test` and `task lint:go` pass. `task mutate` is not part of this slice, but the new storage package becomes a target if the mutation scope is widened. The 404 to ErrBlobNotFound and idempotent-Delete branches need killing tests.
## Optional integration (task test:s3)
Start a SeaweedFS container with Docker, create a bucket, set RENEWCMS_TEST_S3_ENDPOINT and run the adapter tests against it. Skip with a message when Docker is unavailable. Not part of `task test`.
## Manual
- Run docker-compose.s3.yml, upload an image in the SPA, confirm the object is in the bucket and /static/uploadedImages/<name> serves it, including after the container is recreated.
- Real AWS S3, R2, B2 or Scaleway smoke test, and an instance-role credential check.
- Start with a wrong bucket and confirm a clear, secret-free startup error.
- Sync an old UPLOAD_DIR with `aws s3 sync` or `mc mirror` and confirm old images load.
- Confirm docs describe: variables (configuration.md, .env.example, docker.env.example, Docker docs), UPLOAD_DIR ignored with s3, manual migration, update backups covering the DB only and not the bucket, and the updated architecture.md "Storage and database seams" section.
+18
View File
@@ -4,6 +4,10 @@ go 1.27.1
require (
github.com/MadAppGang/httplog v1.3.0
github.com/aws/aws-sdk-go-v2 v1.47.1
github.com/aws/aws-sdk-go-v2/config v1.33.6
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0
github.com/glebarez/sqlite v1.11.0
github.com/go-chi/chi/v5 v5.2.3
github.com/go-chi/cors v1.2.2
@@ -24,6 +28,20 @@ require (
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect
github.com/aws/smithy-go v1.28.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.18.0 // indirect
+36
View File
@@ -4,6 +4,42 @@ github.com/MadAppGang/httplog v1.3.0 h1:1XU54TO8kiqTeO+7oZLKAM3RP/cJ7SadzslRcKsp
github.com/MadAppGang/httplog v1.3.0/go.mod h1:gpYEdkjh/Cda6YxtDy4AB7KY+fR7mb3SqBZw74A5hJ4=
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4=
github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w=
github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw=
github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE=
github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw=
github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM=
github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ=
github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -29,6 +29,7 @@ type Settings struct {
JWTSecret string
DBFile string
Database DatabaseSettings
Storage StorageSettings
DataDirPath string
Port string
Smtp SmtpSettings
@@ -71,6 +72,11 @@ func NewSettingsFromEnv() (Settings, error) {
return Settings{}, err
}
storage, err := storageSettingsFromEnv()
if err != nil {
return Settings{}, err
}
return Settings{
Environment: os.Getenv("ENVIRONMENT"),
CorsAllowedOrigins: parseSemicolonSeparatedList(os.Getenv("CORS_ALLOWED_ORIGINS")),
@@ -79,6 +85,7 @@ func NewSettingsFromEnv() (Settings, error) {
JWTSecret: os.Getenv("JWT_SECRET"),
DBFile: os.Getenv("DB_FILE"),
Database: database,
Storage: storage,
DataDirPath: os.Getenv("DATA_DIR"),
Port: os.Getenv("PORT"),
Smtp: SmtpSettings{
@@ -0,0 +1,114 @@
package config
import (
"fmt"
"os"
"slices"
"strconv"
)
const (
StorageDriverLocal = "local"
StorageDriverS3 = "s3"
DefaultStorageDriver = StorageDriverLocal
DefaultS3Region = "us-east-1"
DefaultS3PathStyle = "false"
DefaultS3TLS = "true"
EnvStorageDriver = "STORAGE_DRIVER"
envS3Bucket = "S3_BUCKET"
envS3Region = "S3_REGION"
envS3Endpoint = "S3_ENDPOINT"
envS3AccessKeyID = "S3_ACCESS_KEY_ID"
envS3SecretKey = "S3_SECRET_ACCESS_KEY"
envS3KeyPrefix = "S3_KEY_PREFIX"
envS3PathStyle = "S3_PATH_STYLE"
envS3TLS = "S3_TLS"
)
var s3RequiredEnvVars = []string{envS3Bucket}
type S3Settings struct {
Bucket string
Region string
Endpoint string
AccessKeyID string
SecretAccessKey string
KeyPrefix string
PathStyle bool
TLS bool
}
type StorageSettings struct {
Driver string
S3 S3Settings
}
func StorageRequiredEnvVars(driver string) []string {
if driver == StorageDriverS3 {
return slices.Clone(s3RequiredEnvVars)
}
return nil
}
func storageSettingsFromEnv() (StorageSettings, error) {
driver := envOrDefault(EnvStorageDriver, DefaultStorageDriver)
switch driver {
case StorageDriverLocal:
return StorageSettings{Driver: driver}, nil
case StorageDriverS3:
s3, err := s3SettingsFromEnv()
if err != nil {
return StorageSettings{}, err
}
return StorageSettings{Driver: driver, S3: s3}, nil
default:
return StorageSettings{}, fmt.Errorf("unsupported storage driver %q", driver)
}
}
func s3SettingsFromEnv() (S3Settings, error) {
if err := MissingEnvVarError(s3RequiredEnvVars, nonEmptyEnvLookup); err != nil {
return S3Settings{}, err
}
accessKeyID := os.Getenv(envS3AccessKeyID)
secretKey := os.Getenv(envS3SecretKey)
if accessKeyID != "" && secretKey == "" {
return S3Settings{}, fmt.Errorf("Environment variable %s is not set (it is required when %s is set)", envS3SecretKey, envS3AccessKeyID)
}
if secretKey != "" && accessKeyID == "" {
return S3Settings{}, fmt.Errorf("Environment variable %s is not set (it is required when %s is set)", envS3AccessKeyID, envS3SecretKey)
}
pathStyle, err := boolEnv(envS3PathStyle, DefaultS3PathStyle)
if err != nil {
return S3Settings{}, err
}
tls, err := boolEnv(envS3TLS, DefaultS3TLS)
if err != nil {
return S3Settings{}, err
}
return S3Settings{
Bucket: os.Getenv(envS3Bucket),
Region: envOrDefault(envS3Region, DefaultS3Region),
Endpoint: os.Getenv(envS3Endpoint),
AccessKeyID: accessKeyID,
SecretAccessKey: secretKey,
KeyPrefix: os.Getenv(envS3KeyPrefix),
PathStyle: pathStyle,
TLS: tls,
}, nil
}
func nonEmptyEnvLookup(name string) (string, bool) {
value := os.Getenv(name)
return value, value != ""
}
func boolEnv(name string, fallback string) (bool, error) {
raw := envOrDefault(name, fallback)
value, err := strconv.ParseBool(raw)
if err != nil {
return false, fmt.Errorf("%s must be true or false, got %q", name, raw)
}
return value, nil
}
@@ -0,0 +1,42 @@
package storage
import (
domain "RenewCMS/internal/domain/image"
"sync/atomic"
)
type ReadCountingStorage struct {
domain.Storage
opened atomic.Int64
}
var _ domain.Storage = &ReadCountingStorage{}
func NewReadCountingStorage(inner domain.Storage) *ReadCountingStorage {
return &ReadCountingStorage{Storage: inner}
}
func (s *ReadCountingStorage) Open(storedName string) (domain.Blob, error) {
blob, err := s.Storage.Open(storedName)
if err == nil {
s.opened.Add(1)
}
return blob, err
}
func (s *ReadCountingStorage) Opened() int {
return int(s.opened.Load())
}
type Logger interface {
Printf(format string, args ...any)
}
func RunBackfillReportingReads(inner domain.Storage, logger Logger, run func(domain.Storage) error) error {
counting := NewReadCountingStorage(inner)
err := run(counting)
if opened := counting.Opened(); opened > 0 {
logger.Printf("image metadata backfill read %d images from the bucket", opened)
}
return err
}
@@ -0,0 +1,83 @@
package storage
import (
"context"
"errors"
"fmt"
"io"
)
type rangeOpener func(offset int64) (io.ReadCloser, context.CancelFunc, error)
type s3Reader struct {
size int64
position int64
bodyOffset int64
body io.ReadCloser
cancel context.CancelFunc
reopen rangeOpener
}
func newS3Reader(size int64, body io.ReadCloser, cancel context.CancelFunc, reopen rangeOpener) *s3Reader {
return &s3Reader{size: size, body: body, cancel: cancel, reopen: reopen}
}
func (r *s3Reader) Read(p []byte) (int, error) {
if r.position >= r.size {
return 0, io.EOF
}
if r.body == nil || r.bodyOffset != r.position {
if err := r.openAtPosition(); err != nil {
return 0, err
}
}
n, err := r.body.Read(p)
r.position += int64(n)
r.bodyOffset = r.position
return n, err
}
func (r *s3Reader) openAtPosition() error {
r.release()
body, cancel, err := r.reopen(r.position)
if err != nil {
return err
}
r.body, r.cancel, r.bodyOffset = body, cancel, r.position
return nil
}
func (r *s3Reader) Seek(offset int64, whence int) (int64, error) {
var target int64
switch whence {
case io.SeekStart:
target = offset
case io.SeekCurrent:
target = r.position + offset
case io.SeekEnd:
target = r.size + offset
default:
return 0, errors.New("invalid seek whence")
}
if target < 0 {
return 0, fmt.Errorf("negative seek position %d", target)
}
r.position = target
return target, nil
}
func (r *s3Reader) Close() error {
r.release()
return nil
}
func (r *s3Reader) release() {
if r.body != nil {
_ = r.body.Close()
r.body = nil
}
if r.cancel != nil {
r.cancel()
r.cancel = nil
}
}
@@ -0,0 +1,211 @@
package storage
import (
domain "RenewCMS/internal/domain/image"
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
)
const s3CallTimeout = 30 * time.Second
type S3Config struct {
Bucket string
Region string
Endpoint string
AccessKeyID string
SecretAccessKey string
KeyPrefix string
PathStyle bool
TLS bool
MaxAttempts int
}
type S3Storage struct {
client *s3.Client
bucket string
keyPrefix string
}
var _ domain.Storage = &S3Storage{}
func OpenS3Storage(config S3Config) (*S3Storage, error) {
client, err := newS3Client(config)
if err != nil {
return nil, fmt.Errorf("unable to configure the S3 storage for bucket %q: %s", config.Bucket, redact(err, config))
}
store := &S3Storage{client: client, bucket: config.Bucket, keyPrefix: config.KeyPrefix}
if err := store.checkBucket(config); err != nil {
return nil, err
}
return store, nil
}
func newS3Client(config S3Config) (*s3.Client, error) {
options := []func(*awsconfig.LoadOptions) error{awsconfig.WithRegion(config.Region)}
if config.AccessKeyID != "" {
options = append(options, awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(config.AccessKeyID, config.SecretAccessKey, "")))
}
loaded, err := awsconfig.LoadDefaultConfig(context.Background(), options...)
if err != nil {
return nil, err
}
loaded.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
loaded.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
if config.MaxAttempts > 0 {
loaded.RetryMaxAttempts = config.MaxAttempts
}
endpoint := s3Endpoint(config)
return s3.NewFromConfig(loaded, func(o *s3.Options) {
o.UsePathStyle = config.PathStyle
if endpoint != "" {
o.BaseEndpoint = aws.String(endpoint)
}
}), nil
}
func s3Endpoint(config S3Config) string {
if config.Endpoint == "" || strings.Contains(config.Endpoint, "://") {
return config.Endpoint
}
if config.TLS {
return "https://" + config.Endpoint
}
return "http://" + config.Endpoint
}
func (s *S3Storage) checkBucket(config S3Config) error {
ctx, cancel := context.WithTimeout(context.Background(), s3CallTimeout)
defer cancel()
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)})
if err == nil {
return nil
}
return fmt.Errorf("S3 bucket %q is not usable: %s", s.bucket, bucketFailureReason(err, config))
}
func bucketFailureReason(err error, config S3Config) string {
switch statusOf(err) {
case http.StatusNotFound:
return "the bucket does not exist at the configured endpoint and region"
case http.StatusForbidden:
return "access denied, check the credentials and the bucket policy"
case 0:
return redact(err, config)
default:
return fmt.Sprintf("unexpected response from the storage service (HTTP %d)", statusOf(err))
}
}
func redact(err error, config S3Config) string {
text := err.Error()
for _, secret := range []string{config.SecretAccessKey, config.AccessKeyID} {
if secret != "" {
text = strings.ReplaceAll(text, secret, "***")
}
}
return text
}
func statusOf(err error) int {
var responseErr *awshttp.ResponseError
if errors.As(err, &responseErr) {
return responseErr.HTTPStatusCode()
}
return 0
}
func isNotFound(err error) bool {
return statusOf(err) == http.StatusNotFound
}
func (s *S3Storage) key(storedName string) string {
return s.keyPrefix + storedName
}
func (s *S3Storage) Put(contentType string, content io.Reader) (string, error) {
data, err := io.ReadAll(content)
if err != nil {
return "", err
}
storedName := uuid.NewString() + contentTypeExtensions[contentType]
ctx, cancel := context.WithTimeout(context.Background(), s3CallTimeout)
defer cancel()
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.key(storedName)),
Body: bytes.NewReader(data),
ContentType: aws.String(contentType),
})
if err != nil {
return "", fmt.Errorf("unable to store the image in the S3 bucket: %w", err)
}
return storedName, nil
}
func (s *S3Storage) Open(storedName string) (domain.Blob, error) {
if !isPlainName(storedName) {
return domain.Blob{}, domain.ErrBlobNotFound
}
key := s.key(storedName)
output, cancel, err := s.getObject(key, "")
if err != nil {
if isNotFound(err) {
return domain.Blob{}, domain.ErrBlobNotFound
}
return domain.Blob{}, fmt.Errorf("unable to read the image from the S3 bucket: %w", err)
}
size := aws.ToInt64(output.ContentLength)
reader := newS3Reader(size, output.Body, cancel, func(offset int64) (io.ReadCloser, context.CancelFunc, error) {
ranged, rangedCancel, err := s.getObject(key, fmt.Sprintf("bytes=%d-", offset))
if err != nil {
return nil, nil, fmt.Errorf("unable to read the image from the S3 bucket: %w", err)
}
return ranged.Body, rangedCancel, nil
})
return domain.Blob{Content: reader, Size: size, ModTime: aws.ToTime(output.LastModified)}, nil
}
func (s *S3Storage) getObject(key string, byteRange string) (*s3.GetObjectOutput, context.CancelFunc, error) {
ctx, cancel := context.WithCancel(context.Background())
deadline := time.AfterFunc(s3CallTimeout, cancel)
input := &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}
if byteRange != "" {
input.Range = aws.String(byteRange)
}
output, err := s.client.GetObject(ctx, input)
deadline.Stop()
if err != nil {
cancel()
return nil, nil, err
}
return output, cancel, nil
}
func (s *S3Storage) Delete(storedName string) error {
if !isPlainName(storedName) {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), s3CallTimeout)
defer cancel()
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(s.key(storedName)),
})
if err != nil && !isNotFound(err) {
return fmt.Errorf("unable to delete the image from the S3 bucket: %w", err)
}
return nil
}
+7 -1
View File
@@ -5,17 +5,23 @@ import (
"fmt"
)
const DriverLocal = "local"
const (
DriverLocal = "local"
DriverS3 = "s3"
)
type Config struct {
Driver string
Location string
S3 S3Config
}
func Open(config Config) (domain.Storage, error) {
switch config.Driver {
case DriverLocal:
return NewLocalStorage(config.Location), nil
case DriverS3:
return OpenS3Storage(config.S3)
default:
return nil, fmt.Errorf("unsupported storage driver %q", config.Driver)
}
+441
View File
@@ -0,0 +1,441 @@
package test
import (
"RenewCMS/api/handlers/staticimages"
validator "RenewCMS/api/pkg/validator"
domainImage "RenewCMS/internal/domain/image"
"RenewCMS/internal/infrastructure/config"
"RenewCMS/internal/infrastructure/storage"
"RenewCMS/internal/infrastructure/useCases"
"bytes"
"errors"
"log"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/go-chi/chi/v5"
)
const s3ObjectBody = "0123456789"
func newS3StorageRouter(store domainImage.Storage) *chi.Mux {
router := chi.NewRouter()
router.Mount("/static/uploadedImages", staticimages.NewHandler(useCases.NewOpenMediaUseCase(store)))
return router
}
func serveS3Storage(router http.Handler, method string, target string, headers map[string]string) *httptest.ResponseRecorder {
request := httptest.NewRequest(method, target, nil)
for name, value := range headers {
request.Header.Set(name, value)
}
recorder := httptest.NewRecorder()
router.ServeHTTP(recorder, request)
return recorder
}
func clearStorageEnv(t *testing.T) {
t.Helper()
setBaseSettingsEnv(t)
for _, name := range []string{"STORAGE_DRIVER", "S3_BUCKET", "S3_REGION", "S3_ENDPOINT", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "S3_KEY_PREFIX", "S3_PATH_STYLE", "S3_TLS"} {
t.Setenv(name, "")
}
}
func TestS3StorageAcceptance(t *testing.T) {
validator.InitValidator()
t.Run("Startup configuration validation", func(t *testing.T) {
examples := []struct {
name string
env map[string]string
wantError string
check func(t *testing.T, settings config.Settings)
}{
{
name: "(unset) with no S3 variable",
env: map[string]string{},
check: func(t *testing.T, settings config.Settings) {
if settings.Storage.Driver != "local" {
t.Errorf("driver = %q, want local", settings.Storage.Driver)
}
},
},
{
name: "local with S3_BUCKET set",
env: map[string]string{"STORAGE_DRIVER": "local", "S3_BUCKET": "media"},
check: func(t *testing.T, settings config.Settings) {
if settings.Storage.Driver != "local" || settings.Storage.S3.Bucket != "" {
t.Errorf("S3 variables must be ignored, got %+v", settings.Storage)
}
},
},
{
name: "s3 with S3_BUCKET and S3_REGION set",
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "S3_REGION": "eu-west-3"},
check: func(t *testing.T, settings config.Settings) {
s3 := settings.Storage.S3
if s3.Bucket != "media" || s3.Region != "eu-west-3" || s3.AccessKeyID != "" || s3.SecretAccessKey != "" {
t.Errorf("unexpected settings %+v", s3)
}
},
},
{
name: "s3 with S3_BUCKET set and S3_REGION unset",
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media"},
check: func(t *testing.T, settings config.Settings) {
if settings.Storage.S3.Region != "us-east-1" {
t.Errorf("region = %q, want us-east-1", settings.Storage.S3.Region)
}
},
},
{
name: "s3 with S3_REGION set and S3_BUCKET unset",
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_REGION": "eu-west-3"},
wantError: "Environment variable S3_BUCKET is not set",
},
{
name: "s3 with key id set and secret unset",
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "S3_ACCESS_KEY_ID": "id"},
wantError: "S3_SECRET_ACCESS_KEY",
},
{
name: "s3 with secret set and key id unset",
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "S3_SECRET_ACCESS_KEY": "secret"},
wantError: "S3_ACCESS_KEY_ID",
},
{
name: "s3 with UPLOAD_DIR set",
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "UPLOAD_DIR": "./data/uploads"},
check: func(t *testing.T, settings config.Settings) {
if settings.Storage.Driver != "s3" {
t.Errorf("driver = %q, want s3", settings.Storage.Driver)
}
},
},
{
name: "gcs with no variable",
env: map[string]string{"STORAGE_DRIVER": "gcs"},
wantError: `unsupported storage driver "gcs"`,
},
}
for _, example := range examples {
t.Run(example.name, func(t *testing.T) {
clearStorageEnv(t)
for name, value := range example.env {
t.Setenv(name, value)
}
settings, err := config.NewSettingsFromEnv()
if example.wantError != "" {
if err == nil || !strings.Contains(err.Error(), example.wantError) {
t.Fatalf("expected error containing %q, got %v", example.wantError, err)
}
return
}
if err != nil {
t.Fatalf("unexpected error %v", err)
}
example.check(t, settings)
})
}
})
t.Run("The registry opens the S3 storage after a bucket check", func(t *testing.T) {
fake := newFakeS3Server(t)
store, err := storage.Open(storage.Config{Driver: "s3", S3: fake.config("")})
if err != nil || store == nil {
t.Fatalf("expected a storage and no error, got %v, %v", store, err)
}
})
t.Run("Startup fails fast with a clear secret-free error", func(t *testing.T) {
for _, status := range []int{http.StatusNotFound, http.StatusForbidden, http.StatusInternalServerError} {
fake := newFakeS3Server(t)
fake.setBucketStatus(status)
_, err := storage.Open(storage.Config{Driver: "s3", S3: fake.config("")})
if err == nil {
t.Fatalf("status %d: expected an error", status)
}
if !strings.Contains(err.Error(), s3TestBucket) {
t.Errorf("status %d: error must mention the bucket, got %q", status, err.Error())
}
if strings.Contains(err.Error(), s3TestSecret) {
t.Errorf("status %d: error leaks the secret: %q", status, err.Error())
}
}
})
t.Run("Put stores the object under the prefix with its content type", func(t *testing.T) {
fake := newFakeS3Server(t)
store := openFakeS3Storage(t, fake, "uploads/")
name, err := store.Put("image/png", bytes.NewReader([]byte{1, 2, 3}))
if err != nil || !strings.HasSuffix(name, ".png") {
t.Fatalf("Put returned %q, %v", name, err)
}
object, ok := fake.object("uploads/" + name)
if !ok || !bytes.Equal(object.data, []byte{1, 2, 3}) || object.contentType != "image/png" {
t.Errorf("unexpected stored object %+v (found %v)", object, ok)
}
})
t.Run("Put failure is returned as an error", func(t *testing.T) {
fake := newFakeS3Server(t)
store := openFakeS3Storage(t, fake, "")
fake.failMethod(http.MethodPut, http.StatusInternalServerError)
_, err := store.Put("image/png", bytes.NewReader([]byte{1}))
if err == nil || errors.Is(err, domainImage.ErrBlobNotFound) {
t.Errorf("expected a plain error, got %v", err)
}
})
t.Run("Open returns content, size and modification time", func(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("uploads/a1b2.png", []byte(s3ObjectBody), "image/png")
store := openFakeS3Storage(t, fake, "uploads/")
blob, err := store.Open("a1b2.png")
if err != nil {
t.Fatalf("Open failed: %v", err)
}
if blob.Size != 10 || !blob.ModTime.Equal(s3TestModTime) {
t.Errorf("unexpected size %d or modtime %v", blob.Size, blob.ModTime)
}
if got := readAllBlob(t, blob); string(got) != s3ObjectBody {
t.Errorf("unexpected content %q", got)
}
})
t.Run("Open returns a Content that implements io.ReadSeeker", func(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte(s3ObjectBody), "image/png")
blob, err := openFakeS3Storage(t, fake, "").Open("a1b2.png")
if err != nil {
t.Fatalf("Open failed: %v", err)
}
defer blob.Content.Close()
if _, ok := blob.Content.(interface {
Seek(int64, int) (int64, error)
}); !ok {
t.Error("Content must be seekable")
}
})
t.Run("Open of a missing key is not found", func(t *testing.T) {
fake := newFakeS3Server(t)
_, err := openFakeS3Storage(t, fake, "").Open("missing.png")
if !errors.Is(err, domainImage.ErrBlobNotFound) {
t.Errorf("expected ErrBlobNotFound, got %v", err)
}
})
t.Run("Open with a backend failure is not a not-found", func(t *testing.T) {
fake := newFakeS3Server(t)
store := openFakeS3Storage(t, fake, "")
fake.failMethod(http.MethodGet, http.StatusInternalServerError)
_, err := store.Open("a1b2.png")
if err == nil || errors.Is(err, domainImage.ErrBlobNotFound) {
t.Errorf("expected a plain error, got %v", err)
}
})
t.Run("Delete is idempotent", func(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte(s3ObjectBody), "image/png")
store := openFakeS3Storage(t, fake, "")
if err := store.Delete("a1b2.png"); err != nil {
t.Fatalf("deleting an existing key failed: %v", err)
}
if _, ok := fake.object("a1b2.png"); ok {
t.Error("the key must be gone")
}
fake.failMethod(http.MethodDelete, http.StatusNotFound)
if err := store.Delete("a1b2.png"); err != nil {
t.Errorf("deleting a missing key must not fail, got %v", err)
}
})
t.Run("A backend failure on Delete is returned", func(t *testing.T) {
fake := newFakeS3Server(t)
store := openFakeS3Storage(t, fake, "")
fake.failMethod(http.MethodDelete, http.StatusInternalServerError)
if err := store.Delete("a1b2.png"); err == nil {
t.Error("expected an error")
}
})
t.Run("Serving /static/uploadedImages/<name> from the bucket", func(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte(s3ObjectBody), "image/png")
router := newS3StorageRouter(openFakeS3Storage(t, fake, ""))
lastModified := s3TestModTime.UTC().Format(http.TimeFormat)
assertCommonHeaders := func(t *testing.T, recorder *httptest.ResponseRecorder) {
t.Helper()
header := recorder.Header()
if header.Get("Content-Type") != "image/png" || header.Get("Last-Modified") != lastModified ||
header.Get("Accept-Ranges") != "bytes" || header.Get("Cache-Control") != "public, no-cache" ||
header.Get("X-Content-Type-Options") != "nosniff" || header.Get("Content-Length") != "10" {
t.Errorf("unexpected headers %v", header)
}
if header.Get("ETag") != "" {
t.Errorf("no ETag expected, got %q", header.Get("ETag"))
}
}
t.Run("GET with no extra header", func(t *testing.T) {
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", nil)
if recorder.Code != http.StatusOK || recorder.Body.String() != s3ObjectBody {
t.Fatalf("got %d %q", recorder.Code, recorder.Body.String())
}
assertCommonHeaders(t, recorder)
})
t.Run("HEAD with no extra header", func(t *testing.T) {
recorder := serveS3Storage(router, http.MethodHead, "/static/uploadedImages/a1b2.png", nil)
if recorder.Code != http.StatusOK || recorder.Body.Len() != 0 {
t.Fatalf("got %d with %d body bytes", recorder.Code, recorder.Body.Len())
}
assertCommonHeaders(t, recorder)
})
t.Run("GET with If-Modified-Since equal to Last-Modified", func(t *testing.T) {
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"If-Modified-Since": lastModified})
if recorder.Code != http.StatusNotModified || recorder.Body.Len() != 0 {
t.Errorf("got %d with %d body bytes", recorder.Code, recorder.Body.Len())
}
})
t.Run("GET with Range bytes=0-3", func(t *testing.T) {
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"Range": "bytes=0-3"})
if recorder.Code != http.StatusPartialContent || recorder.Body.String() != "0123" {
t.Fatalf("got %d %q", recorder.Code, recorder.Body.String())
}
if recorder.Header().Get("Content-Range") != "bytes 0-3/10" || recorder.Header().Get("Content-Length") != "4" {
t.Errorf("unexpected headers %v", recorder.Header())
}
})
t.Run("GET with Range bytes=4-6 reads the middle through a ranged request", func(t *testing.T) {
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"Range": "bytes=4-6"})
if recorder.Code != http.StatusPartialContent || recorder.Body.String() != "456" {
t.Fatalf("got %d %q", recorder.Code, recorder.Body.String())
}
})
t.Run("GET with Range bytes=50-60", func(t *testing.T) {
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"Range": "bytes=50-60"})
if recorder.Code != http.StatusRequestedRangeNotSatisfiable || recorder.Header().Get("Content-Range") != "bytes */10" {
t.Errorf("got %d %v", recorder.Code, recorder.Header())
}
})
})
t.Run("SVG from the bucket keeps its security headers", func(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("logo.svg", []byte("<svg/>"), "image/svg+xml")
router := newS3StorageRouter(openFakeS3Storage(t, fake, ""))
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/logo.svg", nil)
if recorder.Header().Get("Content-Type") != "image/svg+xml" {
t.Errorf("unexpected Content-Type %q", recorder.Header().Get("Content-Type"))
}
if recorder.Header().Get("Content-Security-Policy") != "default-src 'none'; style-src 'unsafe-inline'; sandbox" {
t.Errorf("unexpected CSP %q", recorder.Header().Get("Content-Security-Policy"))
}
})
t.Run("Missing object is 404 and a backend failure is 500", func(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte(s3ObjectBody), "image/png")
router := newS3StorageRouter(openFakeS3Storage(t, fake, ""))
if recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/missing.png", nil); recorder.Code != http.StatusNotFound {
t.Errorf("expected 404, got %d", recorder.Code)
}
fake.failMethod(http.MethodGet, http.StatusInternalServerError)
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", nil)
if recorder.Code != http.StatusInternalServerError || strings.TrimSpace(recorder.Body.String()) != "Unable to read the file" {
t.Errorf("got %d %q", recorder.Code, recorder.Body.String())
}
})
t.Run("Path traversal never reaches another key", func(t *testing.T) {
fake := newFakeS3Server(t)
router := newS3StorageRouter(openFakeS3Storage(t, fake, "uploads/"))
serveS3Storage(router, http.MethodGet, "/static/uploadedImages/../secret.png", nil)
keys := fake.requestedKeys(http.MethodGet)
if len(keys) != 1 || keys[0] != "uploads/secret.png" {
t.Errorf("unexpected requested keys %v", keys)
}
})
t.Run("Handler as-is for a non-seekable blob", func(t *testing.T) {
store := streamOnlyStorage{data: []byte(s3ObjectBody)}
get := serveFrom(t, store, http.MethodGet, "/remote.png")
if get.Code != http.StatusOK || get.Body.String() != s3ObjectBody || get.Header().Get("Content-Length") != "10" || get.Header().Get("Last-Modified") == "" {
t.Errorf("unexpected GET %d %q %v", get.Code, get.Body.String(), get.Header())
}
head := serveFrom(t, store, http.MethodHead, "/remote.png")
if head.Code != http.StatusOK || head.Body.Len() != 0 || head.Header().Get("Content-Length") != "10" || head.Header().Get("Last-Modified") == "" {
t.Errorf("unexpected HEAD %d %q %v", head.Code, head.Body.String(), head.Header())
}
})
t.Run("Boot backfill logs how many images it read from the bucket", func(t *testing.T) {
repo, store := newMockImageRepo(), newMockImageStorage()
for id, name := range []string{"one.png", "two.png", "three.png"} {
store.blobs[name] = pngBytes(t, id+1, 1)
repo.images[uint32(id+1)] = domainImage.Image{ID: uint32(id + 1), Path: domainImage.PublicPath(name)}
}
repo.images[4] = domainImage.Image{ID: 4, Path: domainImage.PublicPath("missing.png")}
var logged bytes.Buffer
err := storage.RunBackfillReportingReads(store, log.New(&logged, "", 0), func(readable domainImage.Storage) error {
return useCases.NewBackfillImageMetadataUseCase(repo, readable).Run()
})
if err != nil {
t.Fatalf("backfill failed: %v", err)
}
if !strings.Contains(logged.String(), "read 3 images") || strings.Contains(logged.String(), s3TestSecret) {
t.Errorf("unexpected log %q", logged.String())
}
if repo.images[1].Filename != "one.png" || repo.images[4].Filename != "" {
t.Errorf("backfill behaviour changed: %+v", repo.images)
}
failure := errors.New("repository down")
repo.fail.failNextWith("ListMissingMetadata", failure)
var silent bytes.Buffer
err = storage.RunBackfillReportingReads(store, log.New(&silent, "", 0), func(readable domainImage.Storage) error {
return useCases.NewBackfillImageMetadataUseCase(repo, readable).Run()
})
if !errors.Is(err, failure) || silent.Len() != 0 {
t.Errorf("errors must be returned unchanged and nothing logged for zero reads, got %v %q", err, silent.String())
}
})
t.Run("The local driver is unchanged", func(t *testing.T) {
clearStorageEnv(t)
settings, err := config.NewSettingsFromEnv()
if err != nil || settings.Storage.Driver != "local" {
t.Fatalf("expected the local driver, got %+v, %v", settings.Storage, err)
}
uploadDir := t.TempDir()
store, err := storage.Open(storage.Config{Driver: storage.DriverLocal, Location: uploadDir})
if err != nil {
t.Fatalf("Open failed: %v", err)
}
name, err := store.Put("image/png", bytes.NewReader([]byte(s3ObjectBody)))
if err != nil {
t.Fatalf("Put failed: %v", err)
}
if data, err := os.ReadFile(filepath.Join(uploadDir, name)); err != nil || string(data) != s3ObjectBody {
t.Fatalf("expected the file in UPLOAD_DIR, got %q, %v", data, err)
}
recorder := serveS3Storage(newS3StorageRouter(store), http.MethodGet, "/static/uploadedImages/"+name, nil)
if recorder.Code != http.StatusOK || recorder.Body.String() != s3ObjectBody || recorder.Header().Get("Content-Type") != "image/png" {
t.Errorf("got %d %q %v", recorder.Code, recorder.Body.String(), recorder.Header())
}
})
}
+71
View File
@@ -0,0 +1,71 @@
package test
import (
domainImage "RenewCMS/internal/domain/image"
"RenewCMS/internal/infrastructure/storage"
"bytes"
"errors"
"io"
"os"
"testing"
)
const (
envTestS3Endpoint = "RENEWCMS_TEST_S3_ENDPOINT"
envTestS3Bucket = "RENEWCMS_TEST_S3_BUCKET"
envTestS3AccessKey = "RENEWCMS_TEST_S3_ACCESS_KEY"
envTestS3SecretKey = "RENEWCMS_TEST_S3_SECRET_KEY"
)
func TestS3StorageIntegration(t *testing.T) {
endpoint := os.Getenv(envTestS3Endpoint)
if endpoint == "" {
t.Skipf("%s is not set, run task test:s3 to start a throwaway S3 server", envTestS3Endpoint)
}
store, err := storage.Open(storage.Config{Driver: storage.DriverS3, S3: storage.S3Config{
Bucket: os.Getenv(envTestS3Bucket),
Region: "us-east-1",
Endpoint: endpoint,
AccessKeyID: os.Getenv(envTestS3AccessKey),
SecretAccessKey: os.Getenv(envTestS3SecretKey),
KeyPrefix: "integration/",
PathStyle: true,
}})
if err != nil {
t.Fatalf("storage.Open failed: %v", err)
}
name, err := store.Put("image/png", bytes.NewReader([]byte("0123456789")))
if err != nil {
t.Fatalf("Put failed: %v", err)
}
blob, err := store.Open(name)
if err != nil {
t.Fatalf("Open failed: %v", err)
}
if blob.Size != 10 || blob.ModTime.IsZero() {
t.Errorf("unexpected blob metadata size=%d modtime=%v", blob.Size, blob.ModTime)
}
seeker, ok := blob.Content.(io.ReadSeeker)
if !ok {
t.Fatal("Content is not an io.ReadSeeker")
}
if _, err := seeker.Seek(4, io.SeekStart); err != nil {
t.Fatalf("Seek failed: %v", err)
}
chunk := make([]byte, 3)
if _, err := io.ReadFull(seeker, chunk); err != nil || string(chunk) != "456" {
t.Errorf("ranged read = %q, %v", chunk, err)
}
_ = blob.Content.Close()
if err := store.Delete(name); err != nil {
t.Fatalf("Delete failed: %v", err)
}
if err := store.Delete(name); err != nil {
t.Errorf("deleting a missing key must not fail, got %v", err)
}
if _, err := store.Open(name); !errors.Is(err, domainImage.ErrBlobNotFound) {
t.Errorf("expected ErrBlobNotFound after delete, got %v", err)
}
}
+547
View File
@@ -0,0 +1,547 @@
package test
import (
domainImage "RenewCMS/internal/domain/image"
"RenewCMS/internal/infrastructure/config"
"RenewCMS/internal/infrastructure/storage"
"bytes"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"testing"
"time"
)
const (
s3TestBucket = "media"
s3TestSecret = "TOPSECRET"
s3TestAccessKey = "AKIATESTKEY"
)
var s3TestModTime = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
type fakeS3Object struct {
data []byte
contentType string
modTime time.Time
}
type fakeS3Request struct {
method string
key string
rng string
}
type fakeS3Server struct {
server *httptest.Server
mu sync.Mutex
objects map[string]fakeS3Object
bucketStatus int
failures map[string]int
requests []fakeS3Request
}
func newFakeS3Server(t *testing.T) *fakeS3Server {
t.Helper()
fake := &fakeS3Server{
objects: map[string]fakeS3Object{},
bucketStatus: http.StatusOK,
failures: map[string]int{},
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
fake.server = httptest.NewUnstartedServer(http.HandlerFunc(fake.handle))
fake.server.Listener = listener
fake.server.Start()
t.Cleanup(fake.server.Close)
return fake
}
func (f *fakeS3Server) put(key string, data []byte, contentType string) {
f.mu.Lock()
defer f.mu.Unlock()
f.objects[key] = fakeS3Object{data: data, contentType: contentType, modTime: s3TestModTime}
}
func (f *fakeS3Server) object(key string) (fakeS3Object, bool) {
f.mu.Lock()
defer f.mu.Unlock()
object, ok := f.objects[key]
return object, ok
}
func (f *fakeS3Server) failMethod(method string, status int) {
f.mu.Lock()
defer f.mu.Unlock()
f.failures[method] = status
}
func (f *fakeS3Server) setBucketStatus(status int) {
f.mu.Lock()
defer f.mu.Unlock()
f.bucketStatus = status
}
func (f *fakeS3Server) recorded() []fakeS3Request {
f.mu.Lock()
defer f.mu.Unlock()
return append([]fakeS3Request(nil), f.requests...)
}
func (f *fakeS3Server) requestedKeys(method string) []string {
var keys []string
for _, request := range f.recorded() {
if request.method == method {
keys = append(keys, request.key)
}
}
return keys
}
func (f *fakeS3Server) handle(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
defer f.mu.Unlock()
trimmed := strings.TrimPrefix(r.URL.Path, "/"+s3TestBucket)
if r.URL.Path != "/"+s3TestBucket && trimmed == r.URL.Path {
http.Error(w, "unknown bucket", http.StatusNotFound)
return
}
key := strings.TrimPrefix(trimmed, "/")
if key == "" {
w.WriteHeader(f.bucketStatus)
return
}
f.requests = append(f.requests, fakeS3Request{method: r.Method, key: key, rng: r.Header.Get("Range")})
if status, failing := f.failures[r.Method]; failing {
writeFakeS3Error(w, r, status, "InternalError")
return
}
switch r.Method {
case http.MethodPut:
data, _ := io.ReadAll(r.Body)
f.objects[key] = fakeS3Object{data: data, contentType: r.Header.Get("Content-Type"), modTime: s3TestModTime}
w.WriteHeader(http.StatusOK)
case http.MethodDelete:
delete(f.objects, key)
w.WriteHeader(http.StatusNoContent)
case http.MethodGet, http.MethodHead:
object, ok := f.objects[key]
if !ok {
writeFakeS3Error(w, r, http.StatusNotFound, "NoSuchKey")
return
}
writeFakeS3Object(w, r, object)
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
}
func writeFakeS3Error(w http.ResponseWriter, r *http.Request, status int, code string) {
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(status)
if r.Method != http.MethodHead {
_, _ = fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?><Error><Code>%s</Code><Message>%s</Message></Error>`, code, code)
}
}
func writeFakeS3Object(w http.ResponseWriter, r *http.Request, object fakeS3Object) {
data := object.data
status := http.StatusOK
w.Header().Set("Content-Type", object.contentType)
w.Header().Set("Last-Modified", object.modTime.UTC().Format(http.TimeFormat))
if rng := r.Header.Get("Range"); rng != "" {
var start int
spec := strings.TrimPrefix(rng, "bytes=")
bounds := strings.SplitN(spec, "-", 2)
start, _ = strconv.Atoi(bounds[0])
end := len(data) - 1
if bounds[1] != "" {
end, _ = strconv.Atoi(bounds[1])
}
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(data)))
data = data[start : end+1]
status = http.StatusPartialContent
}
w.Header().Set("Content-Length", strconv.Itoa(len(data)))
w.WriteHeader(status)
if r.Method != http.MethodHead {
_, _ = w.Write(data)
}
}
func (f *fakeS3Server) config(prefix string) storage.S3Config {
return storage.S3Config{
Bucket: s3TestBucket,
Region: "us-east-1",
Endpoint: f.server.URL,
AccessKeyID: s3TestAccessKey,
SecretAccessKey: s3TestSecret,
KeyPrefix: prefix,
PathStyle: true,
MaxAttempts: 1,
}
}
func openFakeS3Storage(t *testing.T, fake *fakeS3Server, prefix string) domainImage.Storage {
t.Helper()
store, err := storage.Open(storage.Config{Driver: storage.DriverS3, S3: fake.config(prefix)})
if err != nil {
t.Fatalf("storage.Open failed: %v", err)
}
return store
}
func readAllBlob(t *testing.T, blob domainImage.Blob) []byte {
t.Helper()
defer blob.Content.Close()
data, err := io.ReadAll(blob.Content)
if err != nil {
t.Fatalf("read failed: %v", err)
}
return data
}
func TestS3StorageSeekThenReadUsesRangedRequests(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte("0123456789"), "image/png")
store := openFakeS3Storage(t, fake, "")
blob, err := store.Open("a1b2.png")
if err != nil {
t.Fatalf("Open failed: %v", err)
}
defer blob.Content.Close()
seeker, ok := blob.Content.(io.ReadSeeker)
if !ok {
t.Fatal("Content is not an io.ReadSeeker")
}
end, err := seeker.Seek(0, io.SeekEnd)
if err != nil || end != 10 {
t.Fatalf("Seek to end = %d, %v", end, err)
}
if _, err := seeker.Seek(4, io.SeekStart); err != nil {
t.Fatalf("Seek failed: %v", err)
}
chunk := make([]byte, 3)
if _, err := io.ReadFull(seeker, chunk); err != nil || string(chunk) != "456" {
t.Fatalf("read after seek = %q, %v", chunk, err)
}
rest, _ := io.ReadAll(seeker)
if string(rest) != "789" {
t.Errorf("remaining bytes = %q", rest)
}
var ranges []string
for _, request := range fake.recorded() {
if request.method == http.MethodGet {
ranges = append(ranges, request.rng)
}
}
if len(ranges) != 2 || ranges[0] != "" || ranges[1] != "bytes=4-" {
t.Errorf("unexpected GET ranges %q", ranges)
}
if _, err := seeker.Seek(-1, io.SeekStart); err == nil {
t.Error("a negative seek must fail")
}
}
func TestS3StorageNamesThatAreNotPlainAreNeverRequested(t *testing.T) {
fake := newFakeS3Server(t)
store := openFakeS3Storage(t, fake, "uploads/")
for _, name := range []string{"", ".", "..", "../x.png", "a/b.png"} {
if _, err := store.Open(name); !errors.Is(err, domainImage.ErrBlobNotFound) {
t.Errorf("Open(%q) = %v, want ErrBlobNotFound", name, err)
}
if err := store.Delete(name); err != nil {
t.Errorf("Delete(%q) = %v, want nil", name, err)
}
}
if requests := fake.recorded(); len(requests) != 0 {
t.Errorf("no object request expected, got %v", requests)
}
}
func TestS3StorageErrorsNeverContainTheSecret(t *testing.T) {
fake := newFakeS3Server(t)
store := openFakeS3Storage(t, fake, "")
fake.failMethod(http.MethodPut, http.StatusInternalServerError)
fake.failMethod(http.MethodGet, http.StatusInternalServerError)
fake.failMethod(http.MethodDelete, http.StatusInternalServerError)
_, putErr := store.Put("image/png", bytes.NewReader([]byte("abc")))
_, openErr := store.Open("a1b2.png")
deleteErr := store.Delete("a1b2.png")
for name, err := range map[string]error{"Put": putErr, "Open": openErr, "Delete": deleteErr} {
if err == nil {
t.Fatalf("%s must fail", name)
}
if strings.Contains(err.Error(), s3TestSecret) {
t.Errorf("%s error leaks the secret: %v", name, err)
}
}
}
func TestS3StorageOpenWithUnreachableEndpointIsSecretFree(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
endpoint := "http://" + listener.Addr().String()
_ = listener.Close()
_, err = storage.Open(storage.Config{Driver: storage.DriverS3, S3: storage.S3Config{
Bucket: s3TestBucket, Region: "us-east-1", Endpoint: endpoint, PathStyle: true, MaxAttempts: 1,
AccessKeyID: s3TestAccessKey, SecretAccessKey: s3TestSecret,
}})
if err == nil || !strings.Contains(err.Error(), s3TestBucket) || strings.Contains(err.Error(), s3TestSecret) {
t.Errorf("expected a bucket-named, secret-free error, got %v", err)
}
}
func TestS3StorageEndpointWithoutSchemeFollowsTheTLSSetting(t *testing.T) {
fake := newFakeS3Server(t)
settings := fake.config("")
settings.Endpoint = strings.TrimPrefix(fake.server.URL, "http://")
settings.TLS = false
store, err := storage.Open(storage.Config{Driver: storage.DriverS3, S3: settings})
if err != nil {
t.Fatalf("plain HTTP endpoint without scheme must open, got %v", err)
}
if _, err := store.Open("missing.png"); !errors.Is(err, domainImage.ErrBlobNotFound) {
t.Errorf("expected not found, got %v", err)
}
}
func TestS3StorageSettingsFromEnv(t *testing.T) {
setBaseSettingsEnv(t)
for _, name := range []string{"STORAGE_DRIVER", "S3_BUCKET", "S3_REGION", "S3_ENDPOINT", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "S3_KEY_PREFIX", "S3_PATH_STYLE", "S3_TLS"} {
t.Setenv(name, "")
}
t.Run("Every S3 variable is read", func(t *testing.T) {
t.Setenv("STORAGE_DRIVER", "s3")
t.Setenv("S3_BUCKET", "media")
t.Setenv("S3_REGION", "eu-west-3")
t.Setenv("S3_ENDPOINT", "minio:9000")
t.Setenv("S3_ACCESS_KEY_ID", "id")
t.Setenv("S3_SECRET_ACCESS_KEY", "secret")
t.Setenv("S3_KEY_PREFIX", "cms/")
t.Setenv("S3_PATH_STYLE", "true")
t.Setenv("S3_TLS", "false")
settings, err := config.NewSettingsFromEnv()
if err != nil {
t.Fatalf("unexpected error %v", err)
}
want := config.S3Settings{Bucket: "media", Region: "eu-west-3", Endpoint: "minio:9000", AccessKeyID: "id", SecretAccessKey: "secret", KeyPrefix: "cms/", PathStyle: true, TLS: false}
if settings.Storage.Driver != "s3" || settings.Storage.S3 != want {
t.Errorf("unexpected storage settings %+v", settings.Storage)
}
})
t.Run("Defaults apply", func(t *testing.T) {
t.Setenv("STORAGE_DRIVER", "s3")
t.Setenv("S3_BUCKET", "media")
settings, err := config.NewSettingsFromEnv()
if err != nil {
t.Fatalf("unexpected error %v", err)
}
s3 := settings.Storage.S3
if s3.Region != "us-east-1" || s3.PathStyle || !s3.TLS || s3.KeyPrefix != "" || s3.Endpoint != "" {
t.Errorf("unexpected defaults %+v", s3)
}
})
t.Run("Invalid booleans are rejected", func(t *testing.T) {
t.Setenv("STORAGE_DRIVER", "s3")
t.Setenv("S3_BUCKET", "media")
t.Setenv("S3_PATH_STYLE", "maybe")
if _, err := config.NewSettingsFromEnv(); err == nil || !strings.Contains(err.Error(), "S3_PATH_STYLE") {
t.Errorf("expected an S3_PATH_STYLE error, got %v", err)
}
t.Setenv("S3_PATH_STYLE", "")
t.Setenv("S3_TLS", "maybe")
if _, err := config.NewSettingsFromEnv(); err == nil || !strings.Contains(err.Error(), "S3_TLS") {
t.Errorf("expected an S3_TLS error, got %v", err)
}
})
t.Run("Required variables by driver", func(t *testing.T) {
if got := config.StorageRequiredEnvVars("s3"); len(got) != 1 || got[0] != "S3_BUCKET" {
t.Errorf("unexpected required variables %v", got)
}
if got := config.StorageRequiredEnvVars("local"); len(got) != 0 {
t.Errorf("local requires nothing, got %v", got)
}
})
}
func countS3Gets(fake *fakeS3Server) int {
return len(fake.requestedKeys(http.MethodGet))
}
func openFakeS3Seeker(t *testing.T, fake *fakeS3Server, name string) io.ReadSeekCloser {
t.Helper()
store := openFakeS3Storage(t, fake, "")
blob, err := store.Open(name)
if err != nil {
t.Fatalf("Open failed: %v", err)
}
seeker, ok := blob.Content.(io.ReadSeekCloser)
if !ok {
t.Fatal("Content is not an io.ReadSeekCloser")
}
return seeker
}
func TestS3StorageSequentialReadUsesASingleRequest(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte("0123456789"), "image/png")
seeker := openFakeS3Seeker(t, fake, "a1b2.png")
defer seeker.Close()
first := make([]byte, 4)
if _, err := io.ReadFull(seeker, first); err != nil || string(first) != "0123" {
t.Fatalf("first read = %q, %v", first, err)
}
rest, err := io.ReadAll(seeker)
if err != nil || string(rest) != "456789" {
t.Fatalf("rest = %q, %v", rest, err)
}
if gets := countS3Gets(fake); gets != 1 {
t.Errorf("sequential read must use 1 GET, got %d", gets)
}
}
func TestS3StorageReadAtEndReturnsEOFWithoutRequest(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte("0123456789"), "image/png")
seeker := openFakeS3Seeker(t, fake, "a1b2.png")
defer seeker.Close()
if _, err := seeker.Seek(0, io.SeekEnd); err != nil {
t.Fatalf("Seek failed: %v", err)
}
n, err := seeker.Read(make([]byte, 4))
if n != 0 || err != io.EOF {
t.Errorf("Read at end = %d, %v, want 0, EOF", n, err)
}
if gets := countS3Gets(fake); gets != 1 {
t.Errorf("reading at the end must not issue a request, got %d GETs", gets)
}
}
func TestS3StorageSeekWhenceVariants(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte("0123456789"), "image/png")
seeker := openFakeS3Seeker(t, fake, "a1b2.png")
defer seeker.Close()
if pos, err := seeker.Seek(-3, io.SeekEnd); err != nil || pos != 7 {
t.Fatalf("SeekEnd(-3) = %d, %v", pos, err)
}
if pos, err := seeker.Seek(-2, io.SeekCurrent); err != nil || pos != 5 {
t.Fatalf("SeekCurrent(-2) = %d, %v", pos, err)
}
chunk := make([]byte, 2)
if _, err := io.ReadFull(seeker, chunk); err != nil || string(chunk) != "56" {
t.Errorf("read after seeks = %q, %v", chunk, err)
}
if _, err := seeker.Seek(0, 99); err == nil {
t.Error("an invalid whence must fail")
}
}
func TestS3StorageReadAfterCloseReopensTheObject(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte("0123456789"), "image/png")
seeker := openFakeS3Seeker(t, fake, "a1b2.png")
head := make([]byte, 3)
if _, err := io.ReadFull(seeker, head); err != nil {
t.Fatalf("read failed: %v", err)
}
if err := seeker.Close(); err != nil {
t.Fatalf("Close failed: %v", err)
}
rest, err := io.ReadAll(seeker)
if err != nil || string(rest) != "3456789" {
t.Errorf("read after Close = %q, %v", rest, err)
}
_ = seeker.Close()
if gets := countS3Gets(fake); gets != 2 {
t.Errorf("expected 2 GETs, got %d", gets)
}
}
func TestS3StorageMaxAttemptsLimitsRetries(t *testing.T) {
fake := newFakeS3Server(t)
store := openFakeS3Storage(t, fake, "")
fake.failMethod(http.MethodGet, http.StatusInternalServerError)
if _, err := store.Open("a1b2.png"); err == nil {
t.Fatal("Open must fail")
}
if gets := countS3Gets(fake); gets != 1 {
t.Errorf("MaxAttempts=1 must issue exactly 1 GET, got %d", gets)
}
}
func TestS3StorageTLSEndpointWithoutSchemeUsesHTTPS(t *testing.T) {
fake := newFakeS3Server(t)
settings := fake.config("")
settings.Endpoint = strings.TrimPrefix(fake.server.URL, "http://")
settings.TLS = true
_, err := storage.Open(storage.Config{Driver: storage.DriverS3, S3: settings})
if err == nil || !strings.Contains(err.Error(), s3TestBucket) {
t.Errorf("https against a plain HTTP server must fail naming the bucket, got %v", err)
}
}
func TestS3StorageRedactsSecretsFoundInTransportErrors(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
endpoint := "http://" + listener.Addr().String() + "/" + s3TestAccessKey + "/" + s3TestSecret
_ = listener.Close()
_, err = storage.Open(storage.Config{Driver: storage.DriverS3, S3: storage.S3Config{
Bucket: s3TestBucket, Region: "us-east-1", Endpoint: endpoint, PathStyle: true, MaxAttempts: 1,
AccessKeyID: s3TestAccessKey, SecretAccessKey: s3TestSecret,
}})
if err == nil {
t.Fatal("expected an error")
}
if strings.Contains(err.Error(), s3TestSecret) || strings.Contains(err.Error(), s3TestAccessKey) {
t.Errorf("credentials leaked: %v", err)
}
if !strings.Contains(err.Error(), "***") {
t.Errorf("expected redaction markers, got %v", err)
}
}
func TestS3StorageReadFailsWhenTheRangedRequestFails(t *testing.T) {
fake := newFakeS3Server(t)
fake.put("a1b2.png", []byte("0123456789"), "image/png")
seeker := openFakeS3Seeker(t, fake, "a1b2.png")
defer seeker.Close()
if _, err := seeker.Seek(4, io.SeekStart); err != nil {
t.Fatalf("Seek failed: %v", err)
}
fake.failMethod(http.MethodGet, http.StatusInternalServerError)
n, err := seeker.Read(make([]byte, 3))
if err == nil || n != 0 {
t.Errorf("Read = %d, %v, want a failure with no bytes", n, err)
}
}