RenewCMS/test/image_variants_serving_http_test.go

package test

import (
	domain "RenewCMS/internal/domain/image"
	"bytes"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"
)

const (
	thumbURL        = "/static/uploadedImages/a1b2.png?v=thumb"
	unknownMessage  = "Unknown variant"
	unsupportedText = "Unsupported parameter"
	cacheHeader     = "public, no-cache"
	privateBlobName = "a1b2.png"
)

func servingEnv(t *testing.T) *variantEnv {
	t.Helper()
	env := newVariantEnv(t)
	env.seedThumb()
	env.seedPhoto()
	return env
}

func assertRequestError(t *testing.T, recorder *httptest.ResponseRecorder, code string, message string, detail string) {
	t.Helper()
	if recorder.Code != 400 || !strings.HasPrefix(recorder.Header().Get("Content-Type"), "application/json") {
		t.Fatalf("response = %d %q", recorder.Code, recorder.Header().Get("Content-Type"))
	}
	var body struct {
		Status  int    `json:"status"`
		Error   string `json:"error"`
		Message string `json:"message"`
		Detail  string `json:"detail"`
	}
	if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
		t.Fatalf("body = %q", recorder.Body.String())
	}
	if body.Status != 400 || body.Error != code || body.Message != message || body.Detail != detail {
		t.Errorf("body = %+v", body)
	}
}

func TestImageVariantsFirstAndCachedRequests(t *testing.T) {
	t.Run("The first request generates and caches", func(t *testing.T) {
		env := servingEnv(t)
		original := append([]byte(nil), env.storage.blobs[privateBlobName]...)

		recorder := env.get(thumbURL, nil)

		if recorder.Code != 200 || recorder.Header().Get("Content-Type") != "image/jpeg" {
			t.Fatalf("response = %d %q", recorder.Code, recorder.Header().Get("Content-Type"))
		}
		width, height := decodedSize(t, recorder.Body.Bytes())
		_, format := decodeImage(t, recorder.Body.Bytes())
		if width != 400 || height != 300 || format != "jpeg" {
			t.Errorf("output = %dx%d %s", width, height, format)
		}
		rows := env.files.rowsOf("thumb")
		if env.storage.putCount() != 1 || len(rows) != 1 || rows[0].Size != int64(recorder.Body.Len()) {
			t.Errorf("puts = %d, rows = %+v", env.storage.putCount(), rows)
		}
		if !bytes.Equal(env.storage.blobs[privateBlobName], original) {
			t.Error("the original blob changed")
		}
	})

	t.Run("The second request serves the cached file without rendering", func(t *testing.T) {
		env := servingEnv(t)
		first := env.get(thumbURL, nil)

		second := env.get(thumbURL, nil)

		if !bytes.Equal(first.Body.Bytes(), second.Body.Bytes()) || env.renderer.renderCount() != 1 || env.storage.putCount() != 1 {
			t.Errorf("renders = %d, puts = %d", env.renderer.renderCount(), env.storage.putCount())
		}
	})

	t.Run("A request without ?v= is the unchanged original", func(t *testing.T) {
		env := servingEnv(t)

		recorder := env.get("/static/uploadedImages/a1b2.png", nil)

		if recorder.Code != 200 || recorder.Header().Get("Content-Type") != "image/png" || recorder.Header().Get("Cache-Control") != cacheHeader || recorder.Header().Get("X-Content-Type-Options") != "nosniff" || recorder.Header().Get("ETag") != "" {
			t.Errorf("headers = %v", recorder.Header())
		}
		if !bytes.Equal(recorder.Body.Bytes(), env.storage.blobs[privateBlobName]) || env.renderer.renderCount() != 0 {
			t.Error("the original was altered or the renderer ran")
		}
	})

	t.Run("Other query parameters are ignored", func(t *testing.T) {
		env := servingEnv(t)

		recorder := env.get(thumbURL+"&t=123", nil)

		if recorder.Code != 200 || recorder.Header().Get("Content-Type") != "image/jpeg" {
			t.Errorf("response = %d", recorder.Code)
		}
	})

	t.Run("A HEAD request answers the headers of the variant", func(t *testing.T) {
		env := servingEnv(t)
		request := httptest.NewRequest(http.MethodHead, thumbURL, nil)
		recorder := httptest.NewRecorder()

		env.public.ServeHTTP(recorder, request)

		if recorder.Code != 200 || recorder.Header().Get("Content-Type") != "image/jpeg" || recorder.Body.Len() != 0 {
			t.Errorf("response = %d %q", recorder.Code, recorder.Header().Get("Content-Type"))
		}
	})
}

func TestImageVariantsETag(t *testing.T) {
	t.Run("The ETag derives from the variant settings", func(t *testing.T) {
		env := servingEnv(t)
		first := env.get(thumbURL, nil)
		etag := first.Header().Get("ETag")

		again := env.get(thumbURL, nil)
		conditional := env.get(thumbURL, http.Header{"If-None-Match": []string{etag}})

		if len(etag) < 3 || !strings.HasPrefix(etag, `"`) || !strings.HasSuffix(etag, `"`) || again.Header().Get("ETag") != etag {
			t.Errorf("etag = %q then %q", etag, again.Header().Get("ETag"))
		}
		if conditional.Code != 304 || conditional.Body.Len() != 0 || env.renderer.renderCount() != 1 {
			t.Errorf("conditional = %d %q, renders = %d", conditional.Code, conditional.Body.String(), env.renderer.renderCount())
		}
	})

	t.Run("A 304 does not touch the cache", func(t *testing.T) {
		env := servingEnv(t)
		etag := env.get(thumbURL, nil).Header().Get("ETag")
		callsBefore := env.files.calls

		env.get(thumbURL, http.Header{"If-None-Match": []string{etag}})

		if env.files.calls != callsBefore {
			t.Error("the file index was read")
		}
	})

	changes := []struct {
		setting string
		body    string
		changes bool
	}{
		{"quality", `{"width": 400, "height": 300, "fit": "cover", "format": "jpeg", "quality": 70}`, true},
		{"width", `{"width": 500, "height": 300, "fit": "cover", "format": "jpeg", "quality": 80}`, true},
		{"height", `{"width": 400, "height": 200, "fit": "cover", "format": "jpeg", "quality": 80}`, true},
		{"fit", `{"width": 400, "height": 300, "fit": "inside", "format": "jpeg", "quality": 80}`, true},
		{"format", `{"width": 400, "height": 300, "fit": "cover", "format": "png"}`, true},
		{"generate_on_upload", `{"width": 400, "height": 300, "fit": "cover", "format": "jpeg", "quality": 80, "generate_on_upload": true}`, false},
	}
	for _, row := range changes {
		t.Run("The ETag changes when a setting changes and stays when a switch changes: "+row.setting, func(t *testing.T) {
			env := servingEnv(t)
			before := env.get(thumbURL, nil).Header().Get("ETag")

			update := env.sendAsAdmin(http.MethodPut, "/v1/image-variants/thumb", row.body)
			after := env.get(thumbURL, nil).Header().Get("ETag")

			if update.Code != 200 || (before != after) != row.changes {
				t.Errorf("update = %d, before = %q, after = %q", update.Code, before, after)
			}
		})
	}

	t.Run("A stale If-None-Match after an edit gets a fresh 200", func(t *testing.T) {
		env := servingEnv(t)
		stale := env.get(thumbURL, nil).Header().Get("ETag")
		env.sendAsAdmin(http.MethodPut, "/v1/image-variants/thumb", `{"width": 400, "height": 300, "fit": "cover", "format": "jpeg", "quality": 70}`)

		recorder := env.get(thumbURL, http.Header{"If-None-Match": []string{stale}})

		if recorder.Code != 200 || recorder.Header().Get("ETag") == stale || recorder.Body.Len() == 0 {
			t.Errorf("response = %d %q", recorder.Code, recorder.Header().Get("ETag"))
		}
	})

	t.Run("Different images get different ETags for the same variant", func(t *testing.T) {
		env := servingEnv(t)
		env.seedSource("c3d4.png", encodePNG(t, variantSolidImage(900, 700, variantBlue)))

		first := env.get(thumbURL, nil).Header().Get("ETag")
		second := env.get("/static/uploadedImages/c3d4.png?v=thumb", nil).Header().Get("ETag")

		if first == second {
			t.Error("both images share one ETag")
		}
	})

	t.Run("Editing a variant regenerates on the next request", func(t *testing.T) {
		env := servingEnv(t)
		env.get(thumbURL, nil)
		oldName := env.files.rowsOf("thumb")[0].CachedName

		env.sendAsAdmin(http.MethodPut, "/v1/image-variants/thumb", `{"width": 400, "height": 300, "fit": "cover", "format": "jpeg", "quality": 70}`)
		env.get(thumbURL, nil)

		rows := env.files.rowsOf("thumb")
		if env.storage.has(oldName) || len(rows) != 1 || rows[0].CachedName == oldName || env.storage.putCount() != 2 {
			t.Errorf("rows = %+v, puts = %d", rows, env.storage.putCount())
		}
	})

	t.Run("A row with an old fingerprint is replaced and its blob removed", func(t *testing.T) {
		env := servingEnv(t)
		env.storage.seed("old.jpeg", []byte("old"))
		env.files.Save(domain.VariantFile{VariantName: "thumb", SourceName: privateBlobName, CachedName: "old.jpeg", ContentType: "image/jpeg", Size: 3, Fingerprint: "outdated"})

		recorder := env.get(thumbURL, nil)

		if recorder.Code != 200 || env.storage.has("old.jpeg") || env.files.rowsOf("thumb")[0].CachedName == "old.jpeg" {
			t.Error("the stale row was served or kept")
		}
	})

	t.Run("A cached file that vanished from storage is generated again", func(t *testing.T) {
		env := servingEnv(t)
		env.get(thumbURL, nil)
		vanished := env.files.rowsOf("thumb")[0].CachedName
		delete(env.storage.blobs, vanished)

		recorder := env.get(thumbURL, nil)

		rows := env.files.rowsOf("thumb")
		if recorder.Code != 200 || recorder.Body.Len() == 0 || len(rows) != 1 || rows[0].CachedName == vanished || !env.storage.has(rows[0].CachedName) {
			t.Errorf("response = %d, rows = %+v", recorder.Code, rows)
		}
	})
}

func TestImageVariantsConcurrency(t *testing.T) {
	t.Run("Concurrent first requests generate once", func(t *testing.T) {
		const clients = 20
		env := servingEnv(t)
		env.renderer.onRender = func() {
			for env.storage.openCountOf(privateBlobName) < clients {
				runtime.Gosched()
			}
		}
		bodies := make([][]byte, clients)
		codes := make([]int, clients)
		start := make(chan struct{})
		var finished sync.WaitGroup
		for index := 0; index < clients; index++ {
			finished.Add(1)
			go func() {
				defer finished.Done()
				<-start
				recorder := env.get(thumbURL, nil)
				codes[index], bodies[index] = recorder.Code, recorder.Body.Bytes()
			}()
		}
		close(start)
		finished.Wait()

		for index := range bodies {
			if codes[index] != 200 || !bytes.Equal(bodies[index], bodies[0]) {
				t.Fatalf("client %d got %d", index, codes[index])
			}
		}
		if env.renderer.renderCount() != 1 || env.storage.putCount() != 1 {
			t.Errorf("renders = %d, puts = %d", env.renderer.renderCount(), env.storage.putCount())
		}
	})

	t.Run("Different variants and different images are not serialised together", func(t *testing.T) {
		env := servingEnv(t)
		seedVariant(env.variants, "card", 200, 0, domain.FitInside, domain.FormatJPEG, 80)
		env.seedSource("c3d4.png", encodePNG(t, variantSolidImage(900, 700, variantBlue)))
		urls := []string{
			"/static/uploadedImages/a1b2.png?v=thumb",
			"/static/uploadedImages/a1b2.png?v=card",
			"/static/uploadedImages/c3d4.png?v=thumb",
			"/static/uploadedImages/c3d4.png?v=card",
		}
		rendezvous := newRendezvous(len(urls))
		env.renderer.onRender = rendezvous.arrive

		var finished sync.WaitGroup
		codes := make([]int, len(urls))
		for index, url := range urls {
			finished.Add(1)
			go func() {
				defer finished.Done()
				codes[index] = env.get(url, nil).Code
			}()
		}
		finished.Wait()

		if !rendezvous.allInFlight() {
			t.Error("the renders were serialised")
		}
		for _, code := range codes {
			if code != 200 {
				t.Errorf("codes = %v", codes)
			}
		}
	})

	t.Run("A failed generation is not cached and is retried", func(t *testing.T) {
		env := servingEnv(t)
		env.renderer.failFirst = true

		first := env.get(thumbURL, nil)
		blobsAfterFailure, rowsAfterFailure := env.storage.putCount(), env.files.rowCount()
		second := env.get(thumbURL, nil)

		if first.Code != 500 || blobsAfterFailure != 0 || rowsAfterFailure != 0 || second.Code != 200 {
			t.Errorf("statuses = %d then %d, puts = %d, rows = %d", first.Code, second.Code, blobsAfterFailure, rowsAfterFailure)
		}
	})
}

type rendezvous struct {
	expected int
	mutex    sync.Mutex
	arrived  int
	all      chan struct{}
	timedOut bool
}

func newRendezvous(expected int) *rendezvous {
	return &rendezvous{expected: expected, all: make(chan struct{})}
}

func (r *rendezvous) arrive() {
	r.mutex.Lock()
	r.arrived++
	if r.arrived == r.expected {
		close(r.all)
	}
	r.mutex.Unlock()
	select {
	case <-r.all:
	case <-time.After(10 * time.Second):
		r.mutex.Lock()
		r.timedOut = true
		r.mutex.Unlock()
	}
}

func (r *rendezvous) allInFlight() bool {
	r.mutex.Lock()
	defer r.mutex.Unlock()
	return r.arrived == r.expected && !r.timedOut
}

func TestImageVariantsRequestContract(t *testing.T) {
	rows := []struct {
		name, url, code, message, detail string
	}{
		{"unknown name", "/static/uploadedImages/a1b2.png?v=banner", "unknown_variant", unknownMessage, "banner"},
		{"uppercase name", "/static/uploadedImages/a1b2.png?v=Thumb", "unknown_variant", unknownMessage, "Thumb"},
		{"empty name", "/static/uploadedImages/a1b2.png?v=", "unknown_variant", unknownMessage, ""},
		{"disabled variant", "/static/uploadedImages/a1b2.png?v=hero", "unknown_variant", unknownMessage, "hero"},
		{"width", "/static/uploadedImages/a1b2.png?width=400", "unsupported_parameter", unsupportedText, "width"},
		{"height", "/static/uploadedImages/a1b2.png?height=300", "unsupported_parameter", unsupportedText, "height"},
		{"format", "/static/uploadedImages/a1b2.png?format=webp", "unsupported_parameter", unsupportedText, "format"},
		{"quality", "/static/uploadedImages/a1b2.png?quality=50", "unsupported_parameter", unsupportedText, "quality"},
		{"variant and width", "/static/uploadedImages/a1b2.png?v=thumb&width=400", "unsupported_parameter", unsupportedText, "width"},
		{"unknown variant and quality", "/static/uploadedImages/a1b2.png?v=banner&quality=50", "unsupported_parameter", unsupportedText, "quality"},
		{"parameter before missing source", "/static/uploadedImages/missing.png?width=400", "unsupported_parameter", unsupportedText, "width"},
		{"unknown variant before missing source", "/static/uploadedImages/missing.png?v=banner", "unknown_variant", unknownMessage, "banner"},
	}
	for _, row := range rows {
		t.Run("A request outside the contract answers 400 with the JSON error body: "+row.name, func(t *testing.T) {
			env := servingEnv(t)
			hero := seedVariant(env.variants, "hero", 100, 0, domain.FitCover, domain.FormatJPEG, 80)
			hero.Enabled = false
			env.variants.variants["hero"] = hero

			recorder := env.get(row.url, nil)

			assertRequestError(t, recorder, row.code, row.message, row.detail)
			if env.renderer.renderCount() != 0 || env.storage.putCount() != 0 {
				t.Error("something was generated")
			}
		})
	}

	t.Run("A disabled variant serves again once enabled, from its kept cache", func(t *testing.T) {
		env := servingEnv(t)
		env.get(thumbURL, nil)
		env.sendAsAdmin(http.MethodPatch, "/v1/image-variants/thumb", `{"enabled": false}`)

		disabled := env.get(thumbURL, nil)
		env.sendAsAdmin(http.MethodPatch, "/v1/image-variants/thumb", `{"enabled": true}`)
		enabled := env.get(thumbURL, nil)

		assertRequestError(t, disabled, "unknown_variant", unknownMessage, "thumb")
		if enabled.Code != 200 || env.storage.putCount() != 1 {
			t.Errorf("response = %d, puts = %d", enabled.Code, env.storage.putCount())
		}
	})

	t.Run("A deleted variant answers 400", func(t *testing.T) {
		env := servingEnv(t)
		env.get(thumbURL, nil)
		env.sendAsAdmin(http.MethodDelete, "/v1/image-variants/thumb", "")

		recorder := env.get(thumbURL, nil)

		assertRequestError(t, recorder, "unknown_variant", unknownMessage, "thumb")
	})
}

func TestImageVariantsMissingAndUnreadableSources(t *testing.T) {
	urls := []string{
		"/static/uploadedImages/missing.png?v=thumb",
		"/static/uploadedImages/../secret.png?v=thumb",
		"/static/uploadedImages/nested/a1b2.png?v=thumb",
	}
	for _, url := range urls {
		t.Run("A missing source answers 404: "+url, func(t *testing.T) {
			env := servingEnv(t)

			recorder := env.get(url, nil)

			if recorder.Code != 404 || env.renderer.renderCount() != 0 || env.storage.putCount() != 0 {
				t.Errorf("response = %d", recorder.Code)
			}
		})
	}

	t.Run("A source that cannot be decoded answers 500 and caches nothing", func(t *testing.T) {
		env := servingEnv(t)
		env.seedSource("bad.png", []byte("these bytes are not an image"))

		recorder := env.get("/static/uploadedImages/bad.png?v=thumb", nil)

		if recorder.Code != 500 || env.storage.putCount() != 0 || env.files.rowCount() != 0 {
			t.Errorf("response = %d", recorder.Code)
		}
	})
}

func TestImageVariantsSourceCap(t *testing.T) {
	for _, row := range []struct {
		size   int
		status int
	}{{1000, 200}, {1001, 413}} {
		t.Run("A source beyond IMAGE_VARIANT_MAX_SOURCE_BYTES answers 413: "+http.StatusText(row.status), func(t *testing.T) {
			env := newVariantEnvWith(t, newCountingRenderer(), 1000)
			env.seedThumb()
			env.seedSource("big.png", padSizedPNG(t, row.size))

			recorder := env.get("/static/uploadedImages/big.png?v=thumb", nil)

			if recorder.Code != row.status {
				t.Errorf("status = %d", recorder.Code)
			}
		})
	}

	t.Run("The 413 body is plain text and nothing is generated", func(t *testing.T) {
		env := newVariantEnvWith(t, newCountingRenderer(), 1000)
		env.seedThumb()
		env.seedSource("big.png", padSizedPNG(t, 1001))

		recorder := env.get("/static/uploadedImages/big.png?v=thumb", nil)

		assertStatusAndBody(t, recorder, 413, "The source image is too large to generate this variant.")
		if env.renderer.renderCount() != 0 || env.storage.putCount() != 0 {
			t.Error("something was generated")
		}
	})

	t.Run("The cap never affects the original", func(t *testing.T) {
		env := newVariantEnvWith(t, newCountingRenderer(), 1000)
		content := padSizedPNG(t, 1001)
		env.seedSource("big.png", content)

		recorder := env.get("/static/uploadedImages/big.png", nil)

		if recorder.Code != 200 || !bytes.Equal(recorder.Body.Bytes(), content) {
			t.Errorf("status = %d", recorder.Code)
		}
	})
}

func TestImageVariantsAsIsSources(t *testing.T) {
	const svg = `<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10"></svg>`

	t.Run("An SVG with a valid variant is served unchanged", func(t *testing.T) {
		env := servingEnv(t)
		env.seedSource("e5f6.svg", []byte(svg))

		recorder := env.get("/static/uploadedImages/e5f6.svg?v=thumb", nil)

		if recorder.Code != 200 || recorder.Body.String() != svg || recorder.Header().Get("Content-Type") != "image/svg+xml" || recorder.Header().Get("Content-Security-Policy") != "default-src 'none'; style-src 'unsafe-inline'; sandbox" {
			t.Errorf("response = %d %v", recorder.Code, recorder.Header())
		}
		if env.renderer.renderCount() != 0 || env.storage.putCount() != 0 || env.files.rowCount() != 0 {
			t.Error("something was generated")
		}
	})

	t.Run("An SVG with an unknown variant is still a 400", func(t *testing.T) {
		env := servingEnv(t)
		env.seedSource("e5f6.svg", []byte(svg))

		recorder := env.get("/static/uploadedImages/e5f6.svg?v=banner", nil)

		assertRequestError(t, recorder, "unknown_variant", unknownMessage, "banner")
	})

	t.Run("An SVG larger than the cap is still served", func(t *testing.T) {
		env := newVariantEnvWith(t, newCountingRenderer(), 10)
		env.seedThumb()
		env.seedSource("e5f6.svg", []byte(svg))

		recorder := env.get("/static/uploadedImages/e5f6.svg?v=thumb", nil)

		if recorder.Code != 200 || recorder.Body.String() != svg {
			t.Errorf("response = %d", recorder.Code)
		}
	})

	t.Run("An animated GIF is served unchanged", func(t *testing.T) {
		env := servingEnv(t)
		animation := animatedGIF(t)
		env.seedSource("anim.gif", animation)

		recorder := env.get("/static/uploadedImages/anim.gif?v=thumb", nil)

		if recorder.Code != 200 || !bytes.Equal(recorder.Body.Bytes(), animation) || recorder.Header().Get("Content-Type") != "image/gif" || env.files.rowCount() != 0 || env.renderer.renderCount() != 0 {
			t.Errorf("response = %d %q", recorder.Code, recorder.Header().Get("Content-Type"))
		}
	})

	t.Run("A single-frame GIF is treated like any raster source", func(t *testing.T) {
		env := servingEnv(t)
		env.seedSource("still.gif", stillGIF(t))

		recorder := env.get("/static/uploadedImages/still.gif?v=thumb", nil)

		if recorder.Code != 200 || recorder.Header().Get("Content-Type") != "image/jpeg" || env.renderer.renderCount() != 1 {
			t.Errorf("response = %d %q", recorder.Code, recorder.Header().Get("Content-Type"))
		}
	})

	t.Run("A stored GIF is served with the image/gif content type", func(t *testing.T) {
		env := servingEnv(t)
		env.seedSource("anim.gif", animatedGIF(t))

		recorder := env.get("/static/uploadedImages/anim.gif", nil)

		if recorder.Header().Get("Content-Type") != "image/gif" {
			t.Errorf("content type = %q", recorder.Header().Get("Content-Type"))
		}
	})
}

func TestImageVariantsServeFormats(t *testing.T) {
	t.Run("A webp variant is served as image/webp with the right dimensions", func(t *testing.T) {
		env := servingEnv(t)
		seedVariant(env.variants, "wp", 200, 0, domain.FitCover, domain.FormatWebP, 70)

		recorder := env.get("/static/uploadedImages/a1b2.png?v=wp", nil)

		width, height := webpSize(t, recorder.Body.Bytes())
		if recorder.Header().Get("Content-Type") != "image/webp" || width != 200 || height != 160 {
			t.Errorf("response = %q %dx%d", recorder.Header().Get("Content-Type"), width, height)
		}
	})

	t.Run("The content type follows the generated file, not the source", func(t *testing.T) {
		env := servingEnv(t)
		seedVariant(env.variants, "logo", 100, 0, domain.FitCover, domain.FormatPNG, 0)

		recorder := env.get("/static/uploadedImages/a1b2.png?v=logo", nil)

		if recorder.Header().Get("Content-Type") != "image/png" || !bytes.HasPrefix(recorder.Body.Bytes(), []byte("\x89PNG")) {
			t.Errorf("content type = %q", recorder.Header().Get("Content-Type"))
		}
	})
}