package test import ( "RenewCMS/api/handlers/staticimages" validator "RenewCMS/api/pkg/validator" domainImage "RenewCMS/internal/domain/image" "RenewCMS/internal/infrastructure/config" "RenewCMS/internal/infrastructure/storage" "RenewCMS/internal/infrastructure/useCases" "bytes" "errors" "log" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "testing" "github.com/go-chi/chi/v5" ) const s3ObjectBody = "0123456789" func newS3StorageRouter(store domainImage.Storage) *chi.Mux { router := chi.NewRouter() router.Mount("/static/uploadedImages", staticimages.NewHandler(useCases.NewOpenMediaUseCase(store), newInertServeVariantUseCase(store))) return router } func serveS3Storage(router http.Handler, method string, target string, headers map[string]string) *httptest.ResponseRecorder { request := httptest.NewRequest(method, target, nil) for name, value := range headers { request.Header.Set(name, value) } recorder := httptest.NewRecorder() router.ServeHTTP(recorder, request) return recorder } func clearStorageEnv(t *testing.T) { t.Helper() setBaseSettingsEnv(t) for _, name := range []string{"STORAGE_DRIVER", "S3_BUCKET", "S3_REGION", "S3_ENDPOINT", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "S3_KEY_PREFIX", "S3_PATH_STYLE", "S3_TLS"} { t.Setenv(name, "") } } func TestS3StorageAcceptance(t *testing.T) { validator.InitValidator() t.Run("Startup configuration validation", func(t *testing.T) { examples := []struct { name string env map[string]string wantError string check func(t *testing.T, settings config.Settings) }{ { name: "(unset) with no S3 variable", env: map[string]string{}, check: func(t *testing.T, settings config.Settings) { if settings.Storage.Driver != "local" { t.Errorf("driver = %q, want local", settings.Storage.Driver) } }, }, { name: "local with S3_BUCKET set", env: map[string]string{"STORAGE_DRIVER": "local", "S3_BUCKET": "media"}, check: func(t *testing.T, settings config.Settings) { if settings.Storage.Driver != "local" || settings.Storage.S3.Bucket != "" { t.Errorf("S3 variables must be ignored, got %+v", settings.Storage) } }, }, { name: "s3 with S3_BUCKET and S3_REGION set", env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "S3_REGION": "eu-west-3"}, check: func(t *testing.T, settings config.Settings) { s3 := settings.Storage.S3 if s3.Bucket != "media" || s3.Region != "eu-west-3" || s3.AccessKeyID != "" || s3.SecretAccessKey != "" { t.Errorf("unexpected settings %+v", s3) } }, }, { name: "s3 with S3_BUCKET set and S3_REGION unset", env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media"}, check: func(t *testing.T, settings config.Settings) { if settings.Storage.S3.Region != "us-east-1" { t.Errorf("region = %q, want us-east-1", settings.Storage.S3.Region) } }, }, { name: "s3 with S3_REGION set and S3_BUCKET unset", env: map[string]string{"STORAGE_DRIVER": "s3", "S3_REGION": "eu-west-3"}, wantError: "Environment variable S3_BUCKET is not set", }, { name: "s3 with key id set and secret unset", env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "S3_ACCESS_KEY_ID": "id"}, wantError: "S3_SECRET_ACCESS_KEY", }, { name: "s3 with secret set and key id unset", env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "S3_SECRET_ACCESS_KEY": "secret"}, wantError: "S3_ACCESS_KEY_ID", }, { name: "s3 with UPLOAD_DIR set", env: map[string]string{"STORAGE_DRIVER": "s3", "S3_BUCKET": "media", "UPLOAD_DIR": "./data/uploads"}, check: func(t *testing.T, settings config.Settings) { if settings.Storage.Driver != "s3" { t.Errorf("driver = %q, want s3", settings.Storage.Driver) } }, }, { name: "gcs with no variable", env: map[string]string{"STORAGE_DRIVER": "gcs"}, wantError: `unsupported storage driver "gcs"`, }, } for _, example := range examples { t.Run(example.name, func(t *testing.T) { clearStorageEnv(t) for name, value := range example.env { t.Setenv(name, value) } settings, err := config.NewSettingsFromEnv() if example.wantError != "" { if err == nil || !strings.Contains(err.Error(), example.wantError) { t.Fatalf("expected error containing %q, got %v", example.wantError, err) } return } if err != nil { t.Fatalf("unexpected error %v", err) } example.check(t, settings) }) } }) t.Run("The registry opens the S3 storage after a bucket check", func(t *testing.T) { fake := newFakeS3Server(t) store, err := storage.Open(storage.Config{Driver: "s3", S3: fake.config("")}) if err != nil || store == nil { t.Fatalf("expected a storage and no error, got %v, %v", store, err) } }) t.Run("Startup fails fast with a clear secret-free error", func(t *testing.T) { for _, status := range []int{http.StatusNotFound, http.StatusForbidden, http.StatusInternalServerError} { fake := newFakeS3Server(t) fake.setBucketStatus(status) _, err := storage.Open(storage.Config{Driver: "s3", S3: fake.config("")}) if err == nil { t.Fatalf("status %d: expected an error", status) } if !strings.Contains(err.Error(), s3TestBucket) { t.Errorf("status %d: error must mention the bucket, got %q", status, err.Error()) } if strings.Contains(err.Error(), s3TestSecret) { t.Errorf("status %d: error leaks the secret: %q", status, err.Error()) } } }) t.Run("Put stores the object under the prefix with its content type", func(t *testing.T) { fake := newFakeS3Server(t) store := openFakeS3Storage(t, fake, "uploads/") name, err := store.Put("image/png", bytes.NewReader([]byte{1, 2, 3})) if err != nil || !strings.HasSuffix(name, ".png") { t.Fatalf("Put returned %q, %v", name, err) } object, ok := fake.object("uploads/" + name) if !ok || !bytes.Equal(object.data, []byte{1, 2, 3}) || object.contentType != "image/png" { t.Errorf("unexpected stored object %+v (found %v)", object, ok) } }) t.Run("Put failure is returned as an error", func(t *testing.T) { fake := newFakeS3Server(t) store := openFakeS3Storage(t, fake, "") fake.failMethod(http.MethodPut, http.StatusInternalServerError) _, err := store.Put("image/png", bytes.NewReader([]byte{1})) if err == nil || errors.Is(err, domainImage.ErrBlobNotFound) { t.Errorf("expected a plain error, got %v", err) } }) t.Run("Open returns content, size and modification time", func(t *testing.T) { fake := newFakeS3Server(t) fake.put("uploads/a1b2.png", []byte(s3ObjectBody), "image/png") store := openFakeS3Storage(t, fake, "uploads/") blob, err := store.Open("a1b2.png") if err != nil { t.Fatalf("Open failed: %v", err) } if blob.Size != 10 || !blob.ModTime.Equal(s3TestModTime) { t.Errorf("unexpected size %d or modtime %v", blob.Size, blob.ModTime) } if got := readAllBlob(t, blob); string(got) != s3ObjectBody { t.Errorf("unexpected content %q", got) } }) t.Run("Open returns a Content that implements io.ReadSeeker", func(t *testing.T) { fake := newFakeS3Server(t) fake.put("a1b2.png", []byte(s3ObjectBody), "image/png") blob, err := openFakeS3Storage(t, fake, "").Open("a1b2.png") if err != nil { t.Fatalf("Open failed: %v", err) } defer blob.Content.Close() if _, ok := blob.Content.(interface { Seek(int64, int) (int64, error) }); !ok { t.Error("Content must be seekable") } }) t.Run("Open of a missing key is not found", func(t *testing.T) { fake := newFakeS3Server(t) _, err := openFakeS3Storage(t, fake, "").Open("missing.png") if !errors.Is(err, domainImage.ErrBlobNotFound) { t.Errorf("expected ErrBlobNotFound, got %v", err) } }) t.Run("Open with a backend failure is not a not-found", func(t *testing.T) { fake := newFakeS3Server(t) store := openFakeS3Storage(t, fake, "") fake.failMethod(http.MethodGet, http.StatusInternalServerError) _, err := store.Open("a1b2.png") if err == nil || errors.Is(err, domainImage.ErrBlobNotFound) { t.Errorf("expected a plain error, got %v", err) } }) t.Run("Delete is idempotent", func(t *testing.T) { fake := newFakeS3Server(t) fake.put("a1b2.png", []byte(s3ObjectBody), "image/png") store := openFakeS3Storage(t, fake, "") if err := store.Delete("a1b2.png"); err != nil { t.Fatalf("deleting an existing key failed: %v", err) } if _, ok := fake.object("a1b2.png"); ok { t.Error("the key must be gone") } fake.failMethod(http.MethodDelete, http.StatusNotFound) if err := store.Delete("a1b2.png"); err != nil { t.Errorf("deleting a missing key must not fail, got %v", err) } }) t.Run("A backend failure on Delete is returned", func(t *testing.T) { fake := newFakeS3Server(t) store := openFakeS3Storage(t, fake, "") fake.failMethod(http.MethodDelete, http.StatusInternalServerError) if err := store.Delete("a1b2.png"); err == nil { t.Error("expected an error") } }) t.Run("Serving /static/uploadedImages/ from the bucket", func(t *testing.T) { fake := newFakeS3Server(t) fake.put("a1b2.png", []byte(s3ObjectBody), "image/png") router := newS3StorageRouter(openFakeS3Storage(t, fake, "")) lastModified := s3TestModTime.UTC().Format(http.TimeFormat) assertCommonHeaders := func(t *testing.T, recorder *httptest.ResponseRecorder) { t.Helper() header := recorder.Header() if header.Get("Content-Type") != "image/png" || header.Get("Last-Modified") != lastModified || header.Get("Accept-Ranges") != "bytes" || header.Get("Cache-Control") != "public, no-cache" || header.Get("X-Content-Type-Options") != "nosniff" || header.Get("Content-Length") != "10" { t.Errorf("unexpected headers %v", header) } if header.Get("ETag") != "" { t.Errorf("no ETag expected, got %q", header.Get("ETag")) } } t.Run("GET with no extra header", func(t *testing.T) { recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", nil) if recorder.Code != http.StatusOK || recorder.Body.String() != s3ObjectBody { t.Fatalf("got %d %q", recorder.Code, recorder.Body.String()) } assertCommonHeaders(t, recorder) }) t.Run("HEAD with no extra header", func(t *testing.T) { recorder := serveS3Storage(router, http.MethodHead, "/static/uploadedImages/a1b2.png", nil) if recorder.Code != http.StatusOK || recorder.Body.Len() != 0 { t.Fatalf("got %d with %d body bytes", recorder.Code, recorder.Body.Len()) } assertCommonHeaders(t, recorder) }) t.Run("GET with If-Modified-Since equal to Last-Modified", func(t *testing.T) { recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"If-Modified-Since": lastModified}) if recorder.Code != http.StatusNotModified || recorder.Body.Len() != 0 { t.Errorf("got %d with %d body bytes", recorder.Code, recorder.Body.Len()) } }) t.Run("GET with Range bytes=0-3", func(t *testing.T) { recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"Range": "bytes=0-3"}) if recorder.Code != http.StatusPartialContent || recorder.Body.String() != "0123" { t.Fatalf("got %d %q", recorder.Code, recorder.Body.String()) } if recorder.Header().Get("Content-Range") != "bytes 0-3/10" || recorder.Header().Get("Content-Length") != "4" { t.Errorf("unexpected headers %v", recorder.Header()) } }) t.Run("GET with Range bytes=4-6 reads the middle through a ranged request", func(t *testing.T) { recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"Range": "bytes=4-6"}) if recorder.Code != http.StatusPartialContent || recorder.Body.String() != "456" { t.Fatalf("got %d %q", recorder.Code, recorder.Body.String()) } }) t.Run("GET with Range bytes=50-60", func(t *testing.T) { recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", map[string]string{"Range": "bytes=50-60"}) if recorder.Code != http.StatusRequestedRangeNotSatisfiable || recorder.Header().Get("Content-Range") != "bytes */10" { t.Errorf("got %d %v", recorder.Code, recorder.Header()) } }) }) t.Run("SVG from the bucket keeps its security headers", func(t *testing.T) { fake := newFakeS3Server(t) fake.put("logo.svg", []byte(""), "image/svg+xml") router := newS3StorageRouter(openFakeS3Storage(t, fake, "")) recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/logo.svg", nil) if recorder.Header().Get("Content-Type") != "image/svg+xml" { t.Errorf("unexpected Content-Type %q", recorder.Header().Get("Content-Type")) } if recorder.Header().Get("Content-Security-Policy") != "default-src 'none'; style-src 'unsafe-inline'; sandbox" { t.Errorf("unexpected CSP %q", recorder.Header().Get("Content-Security-Policy")) } }) t.Run("Missing object is 404 and a backend failure is 500", func(t *testing.T) { fake := newFakeS3Server(t) fake.put("a1b2.png", []byte(s3ObjectBody), "image/png") router := newS3StorageRouter(openFakeS3Storage(t, fake, "")) if recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/missing.png", nil); recorder.Code != http.StatusNotFound { t.Errorf("expected 404, got %d", recorder.Code) } fake.failMethod(http.MethodGet, http.StatusInternalServerError) recorder := serveS3Storage(router, http.MethodGet, "/static/uploadedImages/a1b2.png", nil) if recorder.Code != http.StatusInternalServerError || strings.TrimSpace(recorder.Body.String()) != "Unable to read the file" { t.Errorf("got %d %q", recorder.Code, recorder.Body.String()) } }) t.Run("Path traversal never reaches another key", func(t *testing.T) { fake := newFakeS3Server(t) router := newS3StorageRouter(openFakeS3Storage(t, fake, "uploads/")) serveS3Storage(router, http.MethodGet, "/static/uploadedImages/../secret.png", nil) keys := fake.requestedKeys(http.MethodGet) if len(keys) != 1 || keys[0] != "uploads/secret.png" { t.Errorf("unexpected requested keys %v", keys) } }) t.Run("Handler as-is for a non-seekable blob", func(t *testing.T) { store := streamOnlyStorage{data: []byte(s3ObjectBody)} get := serveFrom(t, store, http.MethodGet, "/remote.png") if get.Code != http.StatusOK || get.Body.String() != s3ObjectBody || get.Header().Get("Content-Length") != "10" || get.Header().Get("Last-Modified") == "" { t.Errorf("unexpected GET %d %q %v", get.Code, get.Body.String(), get.Header()) } head := serveFrom(t, store, http.MethodHead, "/remote.png") if head.Code != http.StatusOK || head.Body.Len() != 0 || head.Header().Get("Content-Length") != "10" || head.Header().Get("Last-Modified") == "" { t.Errorf("unexpected HEAD %d %q %v", head.Code, head.Body.String(), head.Header()) } }) t.Run("Boot backfill logs how many images it read from the bucket", func(t *testing.T) { repo, store := newMockImageRepo(), newMockImageStorage() for id, name := range []string{"one.png", "two.png", "three.png"} { store.blobs[name] = pngBytes(t, id+1, 1) repo.images[uint32(id+1)] = domainImage.Image{ID: uint32(id + 1), Path: domainImage.PublicPath(name)} } repo.images[4] = domainImage.Image{ID: 4, Path: domainImage.PublicPath("missing.png")} var logged bytes.Buffer err := storage.RunBackfillReportingReads(store, log.New(&logged, "", 0), func(readable domainImage.Storage) error { return useCases.NewBackfillImageMetadataUseCase(repo, readable).Run() }) if err != nil { t.Fatalf("backfill failed: %v", err) } if !strings.Contains(logged.String(), "read 3 images") || strings.Contains(logged.String(), s3TestSecret) { t.Errorf("unexpected log %q", logged.String()) } if repo.images[1].Filename != "one.png" || repo.images[4].Filename != "" { t.Errorf("backfill behaviour changed: %+v", repo.images) } failure := errors.New("repository down") repo.fail.failNextWith("ListMissingMetadata", failure) var silent bytes.Buffer err = storage.RunBackfillReportingReads(store, log.New(&silent, "", 0), func(readable domainImage.Storage) error { return useCases.NewBackfillImageMetadataUseCase(repo, readable).Run() }) if !errors.Is(err, failure) || silent.Len() != 0 { t.Errorf("errors must be returned unchanged and nothing logged for zero reads, got %v %q", err, silent.String()) } }) t.Run("The local driver is unchanged", func(t *testing.T) { clearStorageEnv(t) settings, err := config.NewSettingsFromEnv() if err != nil || settings.Storage.Driver != "local" { t.Fatalf("expected the local driver, got %+v, %v", settings.Storage, err) } uploadDir := t.TempDir() store, err := storage.Open(storage.Config{Driver: storage.DriverLocal, Location: uploadDir}) if err != nil { t.Fatalf("Open failed: %v", err) } name, err := store.Put("image/png", bytes.NewReader([]byte(s3ObjectBody))) if err != nil { t.Fatalf("Put failed: %v", err) } if data, err := os.ReadFile(filepath.Join(uploadDir, name)); err != nil || string(data) != s3ObjectBody { t.Fatalf("expected the file in UPLOAD_DIR, got %q, %v", data, err) } recorder := serveS3Storage(newS3StorageRouter(store), http.MethodGet, "/static/uploadedImages/"+name, nil) if recorder.Code != http.StatusOK || recorder.Body.String() != s3ObjectBody || recorder.Header().Get("Content-Type") != "image/png" { t.Errorf("got %d %q %v", recorder.Code, recorder.Body.String(), recorder.Header()) } }) }