feat: image variants engine

2026-10-08 02:33:32 +02:00
1 parent f0845de643
commit f1d56ef50e
@@ -0,0 +1,444 @@
+package test
+
+import (
+	domain "RenewCMS/internal/domain/image"
+	"RenewCMS/internal/infrastructure/config"
+	persistence "RenewCMS/internal/infrastructure/persistence/repositories"
+	"RenewCMS/internal/infrastructure/useCases"
+	"bytes"
+	"context"
+	"encoding/binary"
+	"hash/crc32"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+const (
+	tooLargeBody       = "The source image is too large to generate this variant."
+	pngSignatureLength = 8
+	pngChunkTypeLength = 4
+	pngHeaderLength    = 13
+	gateTimeout        = 5 * time.Second
+	settleDelay        = 150 * time.Millisecond
+)
+
+func declaringPNG(t *testing.T, width int, height int) []byte {
+	t.Helper()
+	content := solidPNG(t, 1, 1)
+	headerStart := pngSignatureLength + pngChunkTypeLength
+	binary.BigEndian.PutUint32(content[headerStart+pngChunkTypeLength:], uint32(width))
+	binary.BigEndian.PutUint32(content[headerStart+pngChunkTypeLength+4:], uint32(height))
+	checksumStart := headerStart + pngChunkTypeLength + pngHeaderLength
+	binary.BigEndian.PutUint32(content[checksumStart:], crc32.ChecksumIEEE(content[headerStart:checksumStart]))
+	return content
+}
+
+func pixelLimitedEnv(t *testing.T, pixels int64) *variantEnv {
+	t.Helper()
+	env := newVariantEnvLimits(t, newCountingRenderer(), useCases.ServeVariantLimits{MaxSourceBytes: variantDefaultCap, MaxSourcePixels: pixels, MaxConcurrentRenders: variantDefaultRenders})
+	env.seedThumb()
+	return env
+}
+
+func TestImageVariantsSourcePixelCap(t *testing.T) {
+	rows := []struct {
+		width, height, status int
+		real                  bool
+	}{
+		{100, 100, 200, true},
+		{101, 100, 413, true},
+		{30000, 30000, 413, false},
+	}
+	for _, row := range rows {
+		t.Run("A source beyond IMAGE_VARIANT_MAX_SOURCE_PIXELS answers 413 without decoding it: "+strconv.Itoa(row.width)+"x"+strconv.Itoa(row.height), func(t *testing.T) {
+			env := pixelLimitedEnv(t, 10000)
+			content := declaringPNG(t, row.width, row.height)
+			if row.real {
+				content = solidPNG(t, row.width, row.height)
+			}
+			env.seedSource("wide.png", content)
+
+			recorder := env.get("/static/uploadedImages/wide.png?v=thumb", nil)
+
+			if recorder.Code != row.status {
+				t.Fatalf("status = %d", recorder.Code)
+			}
+			if row.status == 413 && (env.renderer.renderCount() != 0 || env.storage.putCount() != 0 || env.files.rowCount() != 0) {
+				t.Errorf("renders = %d, puts = %d, rows = %d", env.renderer.renderCount(), env.storage.putCount(), env.files.rowCount())
+			}
+		})
+	}
+
+	t.Run("The pixel 413 body is plain text and nothing is generated", func(t *testing.T) {
+		env := pixelLimitedEnv(t, 10000)
+		env.seedSource("wide.png", declaringPNG(t, 30000, 30000))
+
+		recorder := env.get("/static/uploadedImages/wide.png?v=thumb", nil)
+
+		assertStatusAndBody(t, recorder, 413, tooLargeBody)
+		if !strings.HasPrefix(recorder.Header().Get("Content-Type"), "text/plain") || env.renderer.renderCount() != 0 || env.storage.putCount() != 0 {
+			t.Errorf("content type = %q, renders = %d", recorder.Header().Get("Content-Type"), env.renderer.renderCount())
+		}
+	})
+
+	t.Run("The pixel cap never affects the original", func(t *testing.T) {
+		env := pixelLimitedEnv(t, 10000)
+		content := declaringPNG(t, 30000, 30000)
+		env.seedSource("wide.png", content)
+
+		recorder := env.get("/static/uploadedImages/wide.png", nil)
+
+		if recorder.Code != 200 || !bytes.Equal(recorder.Body.Bytes(), content) {
+			t.Errorf("status = %d", recorder.Code)
+		}
+	})
+
+	t.Run("The pixel cap never affects an SVG", func(t *testing.T) {
+		env := pixelLimitedEnv(t, 1)
+		svg := `<svg xmlns="http://www.w3.org/2000/svg" width="9000" height="9000"></svg>`
+		env.seedSource("e5f6.svg", []byte(svg))
+
+		recorder := env.get("/static/uploadedImages/e5f6.svg?v=thumb", nil)
+
+		if recorder.Code != 200 || recorder.Body.String() != svg || env.renderer.renderCount() != 0 {
+			t.Errorf("status = %d", recorder.Code)
+		}
+	})
+
+	t.Run("A header that cannot be read answers 500 and caches nothing", func(t *testing.T) {
+		env := pixelLimitedEnv(t, 10000)
+		env.seedSource("bad.png", []byte("these bytes are not an image"))
+
+		recorder := env.get("/static/uploadedImages/bad.png?v=thumb", nil)
+
+		if recorder.Code != 500 || env.renderer.renderCount() != 0 || env.storage.putCount() != 0 || env.files.rowCount() != 0 {
+			t.Errorf("status = %d, renders = %d", recorder.Code, env.renderer.renderCount())
+		}
+	})
+}
+
+func TestImageVariantsKnownSource(t *testing.T) {
+	t.Run("An image record with that stored name is a source", func(t *testing.T) {
+		env := servingEnv(t)
+
+		if recorder := env.get(thumbURL, nil); recorder.Code != 200 {
+			t.Errorf("status = %d", recorder.Code)
+		}
+	})
+
+	for _, name := range []string{"gen-1.jpg", "orphan.png"} {
+		t.Run("A blob without an image record answers 404 and nothing is read: "+name, func(t *testing.T) {
+			env := servingEnv(t)
+			env.storage.seed(name, encodePNG(t, variantSolidImage(10, 10, variantRed)))
+
+			recorder := env.get("/static/uploadedImages/"+name+"?v=thumb", nil)
+
+			if recorder.Code != 404 || env.storage.openCountOf(name) != 0 || env.renderer.renderCount() != 0 || env.storage.putCount() != 0 || env.files.rowCount() != 0 {
+				t.Errorf("status = %d, opens = %d", recorder.Code, env.storage.openCountOf(name))
+			}
+		})
+	}
+
+	t.Run("The record lookup comes after the parameter checks and before any storage read", func(t *testing.T) {
+		env := servingEnv(t)
+		env.storage.seed("orphan.png", []byte("orphan"))
+
+		unknown := env.get("/static/uploadedImages/orphan.png?v=banner", nil)
+		orphan := env.get("/static/uploadedImages/orphan.png?v=thumb", nil)
+
+		assertRequestError(t, unknown, "unknown_variant", unknownMessage, "banner")
+		if orphan.Code != 404 || env.storage.openCountOf("orphan.png") != 0 {
+			t.Errorf("status = %d, opens = %d", orphan.Code, env.storage.openCountOf("orphan.png"))
+		}
+	})
+
+	t.Run("The original of any stored blob is still served without a variant", func(t *testing.T) {
+		env := servingEnv(t)
+		env.storage.seed("gen-1.jpg", []byte("generated bytes"))
+
+		recorder := env.get("/static/uploadedImages/gen-1.jpg", nil)
+
+		if recorder.Code != 200 || recorder.Body.String() != "generated bytes" {
+			t.Errorf("status = %d", recorder.Code)
+		}
+	})
+
+	t.Run("A failing record lookup answers 500 and reads nothing", func(t *testing.T) {
+		env := servingEnv(t)
+		env.images.fail.failNext("ExistsByStoredName")
+
+		recorder := env.get(thumbURL, nil)
+
+		if recorder.Code != 500 || env.storage.openCountOf(variantSourceName) != 0 || env.renderer.renderCount() != 0 {
+			t.Errorf("status = %d", recorder.Code)
+		}
+	})
+}
+
+func TestImageRepositoryExistsByStoredName(t *testing.T) {
+	repo := persistence.NewImageRepository(openActivityDatabase(t))
+	if _, err := repo.Create(domain.Image{Path: domain.PublicPath("a1b2.png"), Filename: "photo.png"}); err != nil {
+		t.Fatal(err)
+	}
+
+	for name, want := range map[string]bool{"a1b2.png": true, "a1b2": false, "b2.png": false, "gen-1.jpg": false, "": false} {
+		found, err := repo.ExistsByStoredName(name)
+		if err != nil || found != want {
+			t.Errorf("%q: found = %v, err = %v", name, found, err)
+		}
+	}
+}
+
+type renderGate struct {
+	mutex    sync.Mutex
+	inFlight int
+	peak     int
+	started  chan struct{}
+	release  chan struct{}
+}
+
+func newRenderGate() *renderGate {
+	return &renderGate{started: make(chan struct{}, 64), release: make(chan struct{}, 64)}
+}
+
+func (g *renderGate) hold() {
+	g.mutex.Lock()
+	g.inFlight++
+	g.peak = max(g.peak, g.inFlight)
+	g.mutex.Unlock()
+	g.started <- struct{}{}
+	<-g.release
+	g.mutex.Lock()
+	g.inFlight--
+	g.mutex.Unlock()
+}
+
+func (g *renderGate) running() int {
+	g.mutex.Lock()
+	defer g.mutex.Unlock()
+	return g.inFlight
+}
+
+func (g *renderGate) peakRunning() int {
+	g.mutex.Lock()
+	defer g.mutex.Unlock()
+	return g.peak
+}
+
+func (g *renderGate) awaitStarts(t *testing.T, count int) {
+	t.Helper()
+	for index := 0; index < count; index++ {
+		select {
+		case <-g.started:
+		case <-time.After(gateTimeout):
+			t.Fatalf("only %d renders started, want %d", index, count)
+		}
+	}
+}
+
+func (g *renderGate) letRun(count int) {
+	for index := 0; index < count; index++ {
+		g.release <- struct{}{}
+	}
+}
+
+func concurrencyEnv(t *testing.T, slots int, sources int) (*variantEnv, *renderGate, []string) {
+	t.Helper()
+	renderer := newCountingRenderer()
+	env := newVariantEnvLimits(t, renderer, useCases.ServeVariantLimits{MaxSourceBytes: variantDefaultCap, MaxSourcePixels: variantDefaultPixels, MaxConcurrentRenders: slots})
+	env.seedThumb()
+	gate := newRenderGate()
+	urls := make([]string, 0, sources)
+	for index := 0; index < sources; index++ {
+		name := "src" + strconv.Itoa(index) + ".png"
+		env.seedSource(name, encodePNG(t, variantSolidImage(60, 40, variantRed)))
+		urls = append(urls, "/static/uploadedImages/"+name+"?v=thumb")
+	}
+	renderer.onRender = gate.hold
+	return env, gate, urls
+}
+
+func getAsync(env *variantEnv, ctx context.Context, url string, codes chan<- int) {
+	go func() {
+		request := httptest.NewRequest(http.MethodGet, url, nil).WithContext(ctx)
+		recorder := httptest.NewRecorder()
+		env.public.ServeHTTP(recorder, request)
+		codes <- recorder.Code
+	}()
+}
+
+func TestImageVariantsBoundedRenders(t *testing.T) {
+	t.Run("At most IMAGE_VARIANT_MAX_CONCURRENT_RENDERS renders run at once", func(t *testing.T) {
+		env, gate, urls := concurrencyEnv(t, 2, 5)
+		codes := make(chan int, len(urls))
+		for _, url := range urls {
+			getAsync(env, context.Background(), url, codes)
+		}
+
+		gate.awaitStarts(t, 2)
+		time.Sleep(settleDelay)
+		if gate.running() != 2 || env.renderer.renderCount() != 2 {
+			t.Fatalf("in flight = %d, renders = %d", gate.running(), env.renderer.renderCount())
+		}
+		gate.letRun(len(urls))
+		for range urls {
+			select {
+			case code := <-codes:
+				if code != 200 {
+					t.Errorf("status = %d", code)
+				}
+			case <-time.After(gateTimeout):
+				t.Fatal("a request never finished")
+			}
+		}
+		if gate.peakRunning() != 2 || env.renderer.renderCount() != 5 {
+			t.Errorf("peak = %d, renders = %d", gate.peakRunning(), env.renderer.renderCount())
+		}
+	})
+
+	t.Run("A cached file never takes a render slot", func(t *testing.T) {
+		env, gate, urls := concurrencyEnv(t, 1, 2)
+		gate.letRun(1)
+		warm := make(chan int, 1)
+		getAsync(env, context.Background(), urls[0], warm)
+		gate.awaitStarts(t, 1)
+		if code := <-warm; code != 200 {
+			t.Fatalf("warm-up status = %d", code)
+		}
+		blocked := make(chan int, 1)
+		getAsync(env, context.Background(), urls[1], blocked)
+		gate.awaitStarts(t, 1)
+
+		cached := make(chan int, 1)
+		getAsync(env, context.Background(), urls[0], cached)
+
+		select {
+		case code := <-cached:
+			if code != 200 {
+				t.Errorf("status = %d", code)
+			}
+		case <-time.After(gateTimeout):
+			t.Fatal("the cached request waited for a slot")
+		}
+		gate.letRun(1)
+		<-blocked
+	})
+
+	t.Run("A waiting request whose client has gone does not render", func(t *testing.T) {
+		env, gate, urls := concurrencyEnv(t, 1, 2)
+		holder := make(chan int, 1)
+		getAsync(env, context.Background(), urls[0], holder)
+		gate.awaitStarts(t, 1)
+		ctx, cancel := context.WithCancel(context.Background())
+		waiter := make(chan int, 1)
+		getAsync(env, ctx, urls[1], waiter)
+		time.Sleep(settleDelay)
+
+		cancel()
+
+		select {
+		case <-waiter:
+		case <-time.After(gateTimeout):
+			t.Fatal("the waiting request kept waiting")
+		}
+		gate.letRun(1)
+		<-holder
+		if env.renderer.renderCount() != 1 || env.storage.putCount() != 1 || env.files.rowCount() != 1 {
+			t.Errorf("renders = %d, puts = %d, rows = %d", env.renderer.renderCount(), env.storage.putCount(), env.files.rowCount())
+		}
+	})
+
+	t.Run("Identical concurrent requests still share one render and one slot", func(t *testing.T) {
+		env, gate, urls := concurrencyEnv(t, 1, 1)
+		codes := make(chan int, 20)
+		for index := 0; index < 20; index++ {
+			getAsync(env, context.Background(), urls[0], codes)
+		}
+
+		gate.awaitStarts(t, 1)
+		time.Sleep(settleDelay)
+		gate.letRun(20)
+		var served atomic.Int32
+		for index := 0; index < 20; index++ {
+			if <-codes == 200 {
+				served.Add(1)
+			}
+		}
+
+		if served.Load() != 20 || env.renderer.renderCount() != 1 || gate.peakRunning() != 1 {
+			t.Errorf("served = %d, renders = %d, peak = %d", served.Load(), env.renderer.renderCount(), gate.peakRunning())
+		}
+	})
+}
+
+func TestImageVariantPixelAndConcurrencySettings(t *testing.T) {
+	rows := []struct {
+		name    string
+		env     string
+		value   string
+		unset   bool
+		want    int64
+		failure bool
+	}{
+		{"pixels unset", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "", true, 50000000, false},
+		{"pixels 2048", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "2048", false, 2048, false},
+		{"pixels 0", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "0", false, 0, true},
+		{"pixels -5", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "-5", false, 0, true},
+		{"pixels many", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "many", false, 0, true},
+		{"renders unset", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "", true, 4, false},
+		{"renders 1", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "1", false, 1, false},
+		{"renders 0", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "0", false, 0, true},
+		{"renders -1", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "-1", false, 0, true},
+		{"renders many", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "many", false, 0, true},
+	}
+	for _, row := range rows {
+		t.Run("The env var is loaded: "+row.name, func(t *testing.T) {
+			setBaseSettingsEnv(t)
+			if row.unset {
+				t.Setenv(row.env, "")
+			} else {
+				t.Setenv(row.env, row.value)
+			}
+
+			settings, err := config.NewSettingsFromEnv()
+
+			if row.failure {
+				if err == nil || !strings.Contains(err.Error(), row.env) {
+					t.Errorf("err = %v", err)
+				}
+				return
+			}
+			got := settings.ImageVariantMaxSourcePixels
+			if row.env == "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS" {
+				got = int64(settings.ImageVariantMaxConcurrentRenders)
+			}
+			if err != nil || got != row.want {
+				t.Errorf("value = %d, err = %v", got, err)
+			}
+		})
+	}
+
+	t.Run("Both defaults are listed in envDefaults, .env.example and docs/configuration.md", func(t *testing.T) {
+		defaults := between(readRepositoryFile(t, "cmd/server/main.go"), "var envDefaults = map[string]string{", "\n}")
+		required := between(readRepositoryFile(t, "cmd/server/main.go"), "var requiredEnvVars = []string{", "\n}")
+		for _, name := range []string{"EnvImageVariantMaxSourcePixels", "EnvImageVariantMaxConcurrentRenders"} {
+			if !strings.Contains(defaults, "config."+name) || strings.Contains(required, name) {
+				t.Errorf("%s is misplaced", name)
+			}
+		}
+		for _, document := range []string{".env.example", "docs/configuration.md"} {
+			content := readRepositoryFile(t, document)
+			for _, name := range []string{"IMAGE_VARIANT_MAX_SOURCE_PIXELS", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS"} {
+				if !strings.Contains(content, name) {
+					t.Errorf("%s does not mention %s", document, name)
+				}
+			}
+		}
+	})
+}