442 lines
17 KiB
Go
442 lines
17 KiB
Go
package test
|
|
|
|
import (
|
|
"RenewCMS/api/handlers/staticimages"
|
|
validator "RenewCMS/api/pkg/validator"
|
|
domainImage "RenewCMS/internal/domain/image"
|
|
"RenewCMS/internal/infrastructure/config"
|
|
"RenewCMS/internal/infrastructure/storage"
|
|
"RenewCMS/internal/infrastructure/useCases"
|
|
"bytes"
|
|
"errors"
|
|
"log"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
)
|
|
|
|
const s3ObjectBody = "0123456789"
|
|
|
|
func newS3StorageRouter(store domainImage.Storage) *chi.Mux {
|
|
router := chi.NewRouter()
|
|
router.Mount("/static/uploadedImages", staticimages.NewHandler(useCases.NewOpenMediaUseCase(store), newInertServeVariantUseCase(store)))
|
|
return router
|
|
}
|
|
|
|
func serveS3Storage(router http.Handler, method string, target string, headers map[string]string) *httptest.ResponseRecorder {
|
|
request := httptest.NewRequest(method, target, nil)
|
|
for name, value := range headers {
|
|
request.Header.Set(name, value)
|
|
}
|
|
recorder := httptest.NewRecorder()
|
|
router.ServeHTTP(recorder, request)
|
|
return recorder
|
|
}
|
|
|
|
func clearStorageEnv(t *testing.T) {
|
|
t.Helper()
|
|
setBaseSettingsEnv(t)
|
|
for _, name := range []string{"STORAGE_DRIVER", "S3_BUCKET", "S3_REGION", "S3_ENDPOINT", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "S3_KEY_PREFIX", "S3_PATH_STYLE", "S3_TLS"} {
|
|
t.Setenv(name, "")
|
|
}
|
|
}
|
|
|
|
func TestS3StorageAcceptance(t *testing.T) {
|
|
validator.InitValidator()
|
|
|
|
t.Run("Startup configuration validation", func(t *testing.T) {
|
|
examples := []struct {
|
|
name string
|
|
env map[string]string
|
|
wantError string
|
|
check func(t *testing.T, settings config.Settings)
|
|
}{
|
|
{
|
|
name: "(unset) with no S3 variable",
|
|
env: map[string]string{},
|
|
check: func(t *testing.T, settings config.Settings) {
|
|
if settings.Storage.Driver != "local" {
|
|
t.Errorf("driver = %q, want local", settings.Storage.Driver)
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "local with S3_BUCKET set",
|
|
env: map[string]string{"STORAGE_DRIVER": "local", "S3_BUCKET": "media"},
|
|
check: func(t *testing.T, settings config.Settings) {
|
|
if settings.Storage.Driver != "local" || settings.Storage.S3.Bucket != "" {
|
|
t.Errorf("S3 variables must be ignored, got %+v", settings.Storage)
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "s3 with S3_BUCKET and S3_REGION set",
|
|
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "S3_REGION": "eu-west-3"},
|
|
check: func(t *testing.T, settings config.Settings) {
|
|
s3 := settings.Storage.S3
|
|
if s3.Bucket != "media" || s3.Region != "eu-west-3" || s3.AccessKeyID != "" || s3.SecretAccessKey != "" {
|
|
t.Errorf("unexpected settings %+v", s3)
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "s3 with S3_BUCKET set and S3_REGION unset",
|
|
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media"},
|
|
check: func(t *testing.T, settings config.Settings) {
|
|
if settings.Storage.S3.Region != "us-east-1" {
|
|
t.Errorf("region = %q, want us-east-1", settings.Storage.S3.Region)
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "s3 with S3_REGION set and S3_BUCKET unset",
|
|
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_REGION": "eu-west-3"},
|
|
wantError: "Environment variable S3_BUCKET is not set",
|
|
},
|
|
{
|
|
name: "s3 with key id set and secret unset",
|
|
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "S3_ACCESS_KEY_ID": "id"},
|
|
wantError: "S3_SECRET_ACCESS_KEY",
|
|
},
|
|
{
|
|
name: "s3 with secret set and key id unset",
|
|
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "S3_SECRET_ACCESS_KEY": "secret"},
|
|
wantError: "S3_ACCESS_KEY_ID",
|
|
},
|
|
{
|
|
name: "s3 with UPLOAD_DIR set",
|
|
env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "UPLOAD_DIR": "./data/uploads"},
|
|
check: func(t *testing.T, settings config.Settings) {
|
|
if settings.Storage.Driver != "s3" {
|
|
t.Errorf("driver = %q, want s3", settings.Storage.Driver)
|
|
}
|
|
},
|
|
},
|
|
{
|
|
name: "gcs with no variable",
|
|
env: map[string]string{"STORAGE_DRIVER": "gcs"},
|
|
wantError: `unsupported storage driver "gcs"`,
|
|
},
|
|
}
|
|
for _, example := range examples {
|
|
t.Run(example.name, func(t *testing.T) {
|
|
clearStorageEnv(t)
|
|
for name, value := range example.env {
|
|
t.Setenv(name, value)
|
|
}
|
|
settings, err := config.NewSettingsFromEnv()
|
|
if example.wantError != "" {
|
|
if err == nil || !strings.Contains(err.Error(), example.wantError) {
|
|
t.Fatalf("expected error containing %q, got %v", example.wantError, err)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("unexpected error %v", err)
|
|
}
|
|
example.check(t, settings)
|
|
})
|
|
}
|
|
})
|
|
|
|
t.Run("The registry opens the S3 storage after a bucket check", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
store, err := storage.Open(storage.Config{Driver: "s3", S3: fake.config("")})
|
|
if err != nil || store == nil {
|
|
t.Fatalf("expected a storage and no error, got %v, %v", store, err)
|
|
}
|
|
})
|
|
|
|
t.Run("Startup fails fast with a clear secret-free error", func(t *testing.T) {
|
|
for _, status := range []int{http.StatusNotFound, http.StatusForbidden, http.StatusInternalServerError} {
|
|
fake := newFakeS3Server(t)
|
|
fake.setBucketStatus(status)
|
|
_, err := storage.Open(storage.Config{Driver: "s3", S3: fake.config("")})
|
|
if err == nil {
|
|
t.Fatalf("status %d: expected an error", status)
|
|
}
|
|
if !strings.Contains(err.Error(), s3TestBucket) {
|
|
t.Errorf("status %d: error must mention the bucket, got %q", status, err.Error())
|
|
}
|
|
if strings.Contains(err.Error(), s3TestSecret) {
|
|
t.Errorf("status %d: error leaks the secret: %q", status, err.Error())
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("Put stores the object under the prefix with its content type", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
store := openFakeS3Storage(t, fake, "uploads/")
|
|
name, err := store.Put("image/png", bytes.NewReader([]byte{1, 2, 3}))
|
|
if err != nil || !strings.HasSuffix(name, ".png") {
|
|
t.Fatalf("Put returned %q, %v", name, err)
|
|
}
|
|
object, ok := fake.object("uploads/" + name)
|
|
if !ok || !bytes.Equal(object.data, []byte{1, 2, 3}) || object.contentType != "image/png" {
|
|
t.Errorf("unexpected stored object %+v (found %v)", object, ok)
|
|
}
|
|
})
|
|
|
|
t.Run("Put failure is returned as an error", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
store := openFakeS3Storage(t, fake, "")
|
|
fake.failMethod(http.MethodPut, http.StatusInternalServerError)
|
|
_, err := store.Put("image/png", bytes.NewReader([]byte{1}))
|
|
if err == nil || errors.Is(err, domainImage.ErrBlobNotFound) {
|
|
t.Errorf("expected a plain error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("Open returns content, size and modification time", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
fake.put("uploads/a1b2.png", []byte(s3ObjectBody), "image/png")
|
|
store := openFakeS3Storage(t, fake, "uploads/")
|
|
blob, err := store.Open("a1b2.png")
|
|
if err != nil {
|
|
t.Fatalf("Open failed: %v", err)
|
|
}
|
|
if blob.Size != 10 || !blob.ModTime.Equal(s3TestModTime) {
|
|
t.Errorf("unexpected size %d or modtime %v", blob.Size, blob.ModTime)
|
|
}
|
|
if got := readAllBlob(t, blob); string(got) != s3ObjectBody {
|
|
t.Errorf("unexpected content %q", got)
|
|
}
|
|
})
|
|
|
|
t.Run("Open returns a Content that implements io.ReadSeeker", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
fake.put("a1b2.png", []byte(s3ObjectBody), "image/png")
|
|
blob, err := openFakeS3Storage(t, fake, "").Open("a1b2.png")
|
|
if err != nil {
|
|
t.Fatalf("Open failed: %v", err)
|
|
}
|
|
defer blob.Content.Close()
|
|
if _, ok := blob.Content.(interface {
|
|
Seek(int64, int) (int64, error)
|
|
}); !ok {
|
|
t.Error("Content must be seekable")
|
|
}
|
|
})
|
|
|
|
t.Run("Open of a missing key is not found", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
_, err := openFakeS3Storage(t, fake, "").Open("missing.png")
|
|
if !errors.Is(err, domainImage.ErrBlobNotFound) {
|
|
t.Errorf("expected ErrBlobNotFound, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("Open with a backend failure is not a not-found", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
store := openFakeS3Storage(t, fake, "")
|
|
fake.failMethod(http.MethodGet, http.StatusInternalServerError)
|
|
_, err := store.Open("a1b2.png")
|
|
if err == nil || errors.Is(err, domainImage.ErrBlobNotFound) {
|
|
t.Errorf("expected a plain error, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("Delete is idempotent", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
fake.put("a1b2.png", []byte(s3ObjectBody), "image/png")
|
|
store := openFakeS3Storage(t, fake, "")
|
|
if err := store.Delete("a1b2.png"); err != nil {
|
|
t.Fatalf("deleting an existing key failed: %v", err)
|
|
}
|
|
if _, ok := fake.object("a1b2.png"); ok {
|
|
t.Error("the key must be gone")
|
|
}
|
|
fake.failMethod(http.MethodDelete, http.StatusNotFound)
|
|
if err := store.Delete("a1b2.png"); err != nil {
|
|
t.Errorf("deleting a missing key must not fail, got %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("A backend failure on Delete is returned", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
store := openFakeS3Storage(t, fake, "")
|
|
fake.failMethod(http.MethodDelete, http.StatusInternalServerError)
|
|
if err := store.Delete("a1b2.png"); err == nil {
|
|
t.Error("expected an error")
|
|
}
|
|
})
|
|
|
|
t.Run("Serving /static/uploadedImages/<name> from the bucket", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
fake.put("a1b2.png", []byte(s3ObjectBody), "image/png")
|
|
router := newS3StorageRouter(openFakeS3Storage(t, fake, ""))
|
|
lastModified := s3TestModTime.UTC().Format(http.TimeFormat)
|
|
|
|
assertCommonHeaders := func(t *testing.T, recorder *httptest.ResponseRecorder) {
|
|
t.Helper()
|
|
header := recorder.Header()
|
|
if header.Get("Content-Type") != "image/png" || header.Get("Last-Modified") != lastModified ||
|
|
header.Get("Accept-Ranges") != "bytes" || header.Get("Cache-Control") != "public, no-cache" ||
|
|
header.Get("X-Content-Type-Options") != "nosniff" || header.Get("Content-Length") != "10" {
|
|
t.Errorf("unexpected headers %v", header)
|
|
}
|
|
if header.Get("ETag") != "" {
|
|
t.Errorf("no ETag expected, got %q", header.Get("ETag"))
|
|
}
|
|
}
|
|
|
|
t.Run("GET with no extra header", func(t *testing.T) {
|
|
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", nil)
|
|
if recorder.Code != http.StatusOK || recorder.Body.String() != s3ObjectBody {
|
|
t.Fatalf("got %d %q", recorder.Code, recorder.Body.String())
|
|
}
|
|
assertCommonHeaders(t, recorder)
|
|
})
|
|
|
|
t.Run("HEAD with no extra header", func(t *testing.T) {
|
|
recorder := serveS3Storage(router, http.MethodHead, "/static/uploadedImages/a1b2.png", nil)
|
|
if recorder.Code != http.StatusOK || recorder.Body.Len() != 0 {
|
|
t.Fatalf("got %d with %d body bytes", recorder.Code, recorder.Body.Len())
|
|
}
|
|
assertCommonHeaders(t, recorder)
|
|
})
|
|
|
|
t.Run("GET with If-Modified-Since equal to Last-Modified", func(t *testing.T) {
|
|
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"If-Modified-Since": lastModified})
|
|
if recorder.Code != http.StatusNotModified || recorder.Body.Len() != 0 {
|
|
t.Errorf("got %d with %d body bytes", recorder.Code, recorder.Body.Len())
|
|
}
|
|
})
|
|
|
|
t.Run("GET with Range bytes=0-3", func(t *testing.T) {
|
|
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"Range": "bytes=0-3"})
|
|
if recorder.Code != http.StatusPartialContent || recorder.Body.String() != "0123" {
|
|
t.Fatalf("got %d %q", recorder.Code, recorder.Body.String())
|
|
}
|
|
if recorder.Header().Get("Content-Range") != "bytes 0-3/10" || recorder.Header().Get("Content-Length") != "4" {
|
|
t.Errorf("unexpected headers %v", recorder.Header())
|
|
}
|
|
})
|
|
|
|
t.Run("GET with Range bytes=4-6 reads the middle through a ranged request", func(t *testing.T) {
|
|
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"Range": "bytes=4-6"})
|
|
if recorder.Code != http.StatusPartialContent || recorder.Body.String() != "456" {
|
|
t.Fatalf("got %d %q", recorder.Code, recorder.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("GET with Range bytes=50-60", func(t *testing.T) {
|
|
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"Range": "bytes=50-60"})
|
|
if recorder.Code != http.StatusRequestedRangeNotSatisfiable || recorder.Header().Get("Content-Range") != "bytes */10" {
|
|
t.Errorf("got %d %v", recorder.Code, recorder.Header())
|
|
}
|
|
})
|
|
})
|
|
|
|
t.Run("SVG from the bucket keeps its security headers", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
fake.put("logo.svg", []byte("<svg/>"), "image/svg+xml")
|
|
router := newS3StorageRouter(openFakeS3Storage(t, fake, ""))
|
|
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/logo.svg", nil)
|
|
if recorder.Header().Get("Content-Type") != "image/svg+xml" {
|
|
t.Errorf("unexpected Content-Type %q", recorder.Header().Get("Content-Type"))
|
|
}
|
|
if recorder.Header().Get("Content-Security-Policy") != "default-src 'none'; style-src 'unsafe-inline'; sandbox" {
|
|
t.Errorf("unexpected CSP %q", recorder.Header().Get("Content-Security-Policy"))
|
|
}
|
|
})
|
|
|
|
t.Run("Missing object is 404 and a backend failure is 500", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
fake.put("a1b2.png", []byte(s3ObjectBody), "image/png")
|
|
router := newS3StorageRouter(openFakeS3Storage(t, fake, ""))
|
|
if recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/missing.png", nil); recorder.Code != http.StatusNotFound {
|
|
t.Errorf("expected 404, got %d", recorder.Code)
|
|
}
|
|
fake.failMethod(http.MethodGet, http.StatusInternalServerError)
|
|
recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", nil)
|
|
if recorder.Code != http.StatusInternalServerError || strings.TrimSpace(recorder.Body.String()) != "Unable to read the file" {
|
|
t.Errorf("got %d %q", recorder.Code, recorder.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("Path traversal never reaches another key", func(t *testing.T) {
|
|
fake := newFakeS3Server(t)
|
|
router := newS3StorageRouter(openFakeS3Storage(t, fake, "uploads/"))
|
|
serveS3Storage(router, http.MethodGet, "/static/uploadedImages/../secret.png", nil)
|
|
keys := fake.requestedKeys(http.MethodGet)
|
|
if len(keys) != 1 || keys[0] != "uploads/secret.png" {
|
|
t.Errorf("unexpected requested keys %v", keys)
|
|
}
|
|
})
|
|
|
|
t.Run("Handler as-is for a non-seekable blob", func(t *testing.T) {
|
|
store := streamOnlyStorage{data: []byte(s3ObjectBody)}
|
|
get := serveFrom(t, store, http.MethodGet, "/remote.png")
|
|
if get.Code != http.StatusOK || get.Body.String() != s3ObjectBody || get.Header().Get("Content-Length") != "10" || get.Header().Get("Last-Modified") == "" {
|
|
t.Errorf("unexpected GET %d %q %v", get.Code, get.Body.String(), get.Header())
|
|
}
|
|
head := serveFrom(t, store, http.MethodHead, "/remote.png")
|
|
if head.Code != http.StatusOK || head.Body.Len() != 0 || head.Header().Get("Content-Length") != "10" || head.Header().Get("Last-Modified") == "" {
|
|
t.Errorf("unexpected HEAD %d %q %v", head.Code, head.Body.String(), head.Header())
|
|
}
|
|
})
|
|
|
|
t.Run("Boot backfill logs how many images it read from the bucket", func(t *testing.T) {
|
|
repo, store := newMockImageRepo(), newMockImageStorage()
|
|
for id, name := range []string{"one.png", "two.png", "three.png"} {
|
|
store.blobs[name] = pngBytes(t, id+1, 1)
|
|
repo.images[uint32(id+1)] = domainImage.Image{ID: uint32(id + 1), Path: domainImage.PublicPath(name)}
|
|
}
|
|
repo.images[4] = domainImage.Image{ID: 4, Path: domainImage.PublicPath("missing.png")}
|
|
|
|
var logged bytes.Buffer
|
|
err := storage.RunBackfillReportingReads(store, log.New(&logged, "", 0), func(readable domainImage.Storage) error {
|
|
return useCases.NewBackfillImageMetadataUseCase(repo, readable).Run()
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("backfill failed: %v", err)
|
|
}
|
|
if !strings.Contains(logged.String(), "read 3 images") || strings.Contains(logged.String(), s3TestSecret) {
|
|
t.Errorf("unexpected log %q", logged.String())
|
|
}
|
|
if repo.images[1].Filename != "one.png" || repo.images[4].Filename != "" {
|
|
t.Errorf("backfill behaviour changed: %+v", repo.images)
|
|
}
|
|
|
|
failure := errors.New("repository down")
|
|
repo.fail.failNextWith("ListMissingMetadata", failure)
|
|
var silent bytes.Buffer
|
|
err = storage.RunBackfillReportingReads(store, log.New(&silent, "", 0), func(readable domainImage.Storage) error {
|
|
return useCases.NewBackfillImageMetadataUseCase(repo, readable).Run()
|
|
})
|
|
if !errors.Is(err, failure) || silent.Len() != 0 {
|
|
t.Errorf("errors must be returned unchanged and nothing logged for zero reads, got %v %q", err, silent.String())
|
|
}
|
|
})
|
|
|
|
t.Run("The local driver is unchanged", func(t *testing.T) {
|
|
clearStorageEnv(t)
|
|
settings, err := config.NewSettingsFromEnv()
|
|
if err != nil || settings.Storage.Driver != "local" {
|
|
t.Fatalf("expected the local driver, got %+v, %v", settings.Storage, err)
|
|
}
|
|
uploadDir := t.TempDir()
|
|
store, err := storage.Open(storage.Config{Driver: storage.DriverLocal, Location: uploadDir})
|
|
if err != nil {
|
|
t.Fatalf("Open failed: %v", err)
|
|
}
|
|
name, err := store.Put("image/png", bytes.NewReader([]byte(s3ObjectBody)))
|
|
if err != nil {
|
|
t.Fatalf("Put failed: %v", err)
|
|
}
|
|
if data, err := os.ReadFile(filepath.Join(uploadDir, name)); err != nil || string(data) != s3ObjectBody {
|
|
t.Fatalf("expected the file in UPLOAD_DIR, got %q, %v", data, err)
|
|
}
|
|
recorder := serveS3Storage(newS3StorageRouter(store), http.MethodGet, "/static/uploadedImages/"+name, nil)
|
|
if recorder.Code != http.StatusOK || recorder.Body.String() != s3ObjectBody || recorder.Header().Get("Content-Type") != "image/png" {
|
|
t.Errorf("got %d %q %v", recorder.Code, recorder.Body.String(), recorder.Header())
|
|
}
|
|
})
|
|
}
|