RenewCMS/test/image_variants_limits_test.go

package test

import (
	domain "RenewCMS/internal/domain/image"
	"RenewCMS/internal/infrastructure/config"
	persistence "RenewCMS/internal/infrastructure/persistence/repositories"
	"RenewCMS/internal/infrastructure/useCases"
	"bytes"
	"context"
	"encoding/binary"
	"hash/crc32"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

const (
	tooLargeBody       = "The source image is too large to generate this variant."
	pngSignatureLength = 8
	pngChunkTypeLength = 4
	pngHeaderLength    = 13
	gateTimeout        = 5 * time.Second
	settleDelay        = 150 * time.Millisecond
)

func declaringPNG(t *testing.T, width int, height int) []byte {
	t.Helper()
	content := solidPNG(t, 1, 1)
	headerStart := pngSignatureLength + pngChunkTypeLength
	binary.BigEndian.PutUint32(content[headerStart+pngChunkTypeLength:], uint32(width))
	binary.BigEndian.PutUint32(content[headerStart+pngChunkTypeLength+4:], uint32(height))
	checksumStart := headerStart + pngChunkTypeLength + pngHeaderLength
	binary.BigEndian.PutUint32(content[checksumStart:], crc32.ChecksumIEEE(content[headerStart:checksumStart]))
	return content
}

func pixelLimitedEnv(t *testing.T, pixels int64) *variantEnv {
	t.Helper()
	env := newVariantEnvLimits(t, newCountingRenderer(), useCases.ServeVariantLimits{MaxSourceBytes: variantDefaultCap, MaxSourcePixels: pixels, MaxConcurrentRenders: variantDefaultRenders})
	env.seedThumb()
	return env
}

func TestImageVariantsSourcePixelCap(t *testing.T) {
	rows := []struct {
		width, height, status int
		real                  bool
	}{
		{100, 100, 200, true},
		{101, 100, 413, true},
		{30000, 30000, 413, false},
	}
	for _, row := range rows {
		t.Run("A source beyond IMAGE_VARIANT_MAX_SOURCE_PIXELS answers 413 without decoding it: "+strconv.Itoa(row.width)+"x"+strconv.Itoa(row.height), func(t *testing.T) {
			env := pixelLimitedEnv(t, 10000)
			content := declaringPNG(t, row.width, row.height)
			if row.real {
				content = solidPNG(t, row.width, row.height)
			}
			env.seedSource("wide.png", content)

			recorder := env.get("/static/uploadedImages/wide.png?v=thumb", nil)

			if recorder.Code != row.status {
				t.Fatalf("status = %d", recorder.Code)
			}
			if row.status == 413 && (env.renderer.renderCount() != 0 || env.storage.putCount() != 0 || env.files.rowCount() != 0) {
				t.Errorf("renders = %d, puts = %d, rows = %d", env.renderer.renderCount(), env.storage.putCount(), env.files.rowCount())
			}
		})
	}

	t.Run("The pixel 413 body is plain text and nothing is generated", func(t *testing.T) {
		env := pixelLimitedEnv(t, 10000)
		env.seedSource("wide.png", declaringPNG(t, 30000, 30000))

		recorder := env.get("/static/uploadedImages/wide.png?v=thumb", nil)

		assertStatusAndBody(t, recorder, 413, tooLargeBody)
		if !strings.HasPrefix(recorder.Header().Get("Content-Type"), "text/plain") || env.renderer.renderCount() != 0 || env.storage.putCount() != 0 {
			t.Errorf("content type = %q, renders = %d", recorder.Header().Get("Content-Type"), env.renderer.renderCount())
		}
	})

	t.Run("The pixel cap never affects the original", func(t *testing.T) {
		env := pixelLimitedEnv(t, 10000)
		content := declaringPNG(t, 30000, 30000)
		env.seedSource("wide.png", content)

		recorder := env.get("/static/uploadedImages/wide.png", nil)

		if recorder.Code != 200 || !bytes.Equal(recorder.Body.Bytes(), content) {
			t.Errorf("status = %d", recorder.Code)
		}
	})

	t.Run("The pixel cap never affects an SVG", func(t *testing.T) {
		env := pixelLimitedEnv(t, 1)
		svg := `<svg xmlns="http://www.w3.org/2000/svg" width="9000" height="9000"></svg>`
		env.seedSource("e5f6.svg", []byte(svg))

		recorder := env.get("/static/uploadedImages/e5f6.svg?v=thumb", nil)

		if recorder.Code != 200 || recorder.Body.String() != svg || env.renderer.renderCount() != 0 {
			t.Errorf("status = %d", recorder.Code)
		}
	})

	t.Run("A header that cannot be read answers 500 and caches nothing", func(t *testing.T) {
		env := pixelLimitedEnv(t, 10000)
		env.seedSource("bad.png", []byte("these bytes are not an image"))

		recorder := env.get("/static/uploadedImages/bad.png?v=thumb", nil)

		if recorder.Code != 500 || env.renderer.renderCount() != 0 || env.storage.putCount() != 0 || env.files.rowCount() != 0 {
			t.Errorf("status = %d, renders = %d", recorder.Code, env.renderer.renderCount())
		}
	})
}

func TestImageVariantsKnownSource(t *testing.T) {
	t.Run("An image record with that stored name is a source", func(t *testing.T) {
		env := servingEnv(t)

		if recorder := env.get(thumbURL, nil); recorder.Code != 200 {
			t.Errorf("status = %d", recorder.Code)
		}
	})

	for _, name := range []string{"gen-1.jpg", "orphan.png"} {
		t.Run("A blob without an image record answers 404 and nothing is read: "+name, func(t *testing.T) {
			env := servingEnv(t)
			env.storage.seed(name, encodePNG(t, variantSolidImage(10, 10, variantRed)))

			recorder := env.get("/static/uploadedImages/"+name+"?v=thumb", nil)

			if recorder.Code != 404 || env.storage.openCountOf(name) != 0 || env.renderer.renderCount() != 0 || env.storage.putCount() != 0 || env.files.rowCount() != 0 {
				t.Errorf("status = %d, opens = %d", recorder.Code, env.storage.openCountOf(name))
			}
		})
	}

	t.Run("The record lookup comes after the parameter checks and before any storage read", func(t *testing.T) {
		env := servingEnv(t)
		env.storage.seed("orphan.png", []byte("orphan"))

		unknown := env.get("/static/uploadedImages/orphan.png?v=banner", nil)
		orphan := env.get("/static/uploadedImages/orphan.png?v=thumb", nil)

		assertRequestError(t, unknown, "unknown_variant", unknownMessage, "banner")
		if orphan.Code != 404 || env.storage.openCountOf("orphan.png") != 0 {
			t.Errorf("status = %d, opens = %d", orphan.Code, env.storage.openCountOf("orphan.png"))
		}
	})

	t.Run("The original of any stored blob is still served without a variant", func(t *testing.T) {
		env := servingEnv(t)
		env.storage.seed("gen-1.jpg", []byte("generated bytes"))

		recorder := env.get("/static/uploadedImages/gen-1.jpg", nil)

		if recorder.Code != 200 || recorder.Body.String() != "generated bytes" {
			t.Errorf("status = %d", recorder.Code)
		}
	})

	t.Run("A failing record lookup answers 500 and reads nothing", func(t *testing.T) {
		env := servingEnv(t)
		env.images.fail.failNext("ExistsByStoredName")

		recorder := env.get(thumbURL, nil)

		if recorder.Code != 500 || env.storage.openCountOf(variantSourceName) != 0 || env.renderer.renderCount() != 0 {
			t.Errorf("status = %d", recorder.Code)
		}
	})
}

func TestImageRepositoryExistsByStoredName(t *testing.T) {
	repo := persistence.NewImageRepository(openActivityDatabase(t))
	if _, err := repo.Create(domain.Image{Path: domain.PublicPath("a1b2.png"), Filename: "photo.png"}); err != nil {
		t.Fatal(err)
	}

	for name, want := range map[string]bool{"a1b2.png": true, "a1b2": false, "b2.png": false, "gen-1.jpg": false, "": false} {
		found, err := repo.ExistsByStoredName(name)
		if err != nil || found != want {
			t.Errorf("%q: found = %v, err = %v", name, found, err)
		}
	}
}

type renderGate struct {
	mutex    sync.Mutex
	inFlight int
	peak     int
	started  chan struct{}
	release  chan struct{}
}

func newRenderGate() *renderGate {
	return &renderGate{started: make(chan struct{}, 64), release: make(chan struct{}, 64)}
}

func (g *renderGate) hold() {
	g.mutex.Lock()
	g.inFlight++
	g.peak = max(g.peak, g.inFlight)
	g.mutex.Unlock()
	g.started <- struct{}{}
	<-g.release
	g.mutex.Lock()
	g.inFlight--
	g.mutex.Unlock()
}

func (g *renderGate) running() int {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	return g.inFlight
}

func (g *renderGate) peakRunning() int {
	g.mutex.Lock()
	defer g.mutex.Unlock()
	return g.peak
}

func (g *renderGate) awaitStarts(t *testing.T, count int) {
	t.Helper()
	for index := 0; index < count; index++ {
		select {
		case <-g.started:
		case <-time.After(gateTimeout):
			t.Fatalf("only %d renders started, want %d", index, count)
		}
	}
}

func (g *renderGate) letRun(count int) {
	for index := 0; index < count; index++ {
		g.release <- struct{}{}
	}
}

func concurrencyEnv(t *testing.T, slots int, sources int) (*variantEnv, *renderGate, []string) {
	t.Helper()
	renderer := newCountingRenderer()
	env := newVariantEnvLimits(t, renderer, useCases.ServeVariantLimits{MaxSourceBytes: variantDefaultCap, MaxSourcePixels: variantDefaultPixels, MaxConcurrentRenders: slots})
	env.seedThumb()
	gate := newRenderGate()
	urls := make([]string, 0, sources)
	for index := 0; index < sources; index++ {
		name := "src" + strconv.Itoa(index) + ".png"
		env.seedSource(name, encodePNG(t, variantSolidImage(60, 40, variantRed)))
		urls = append(urls, "/static/uploadedImages/"+name+"?v=thumb")
	}
	renderer.onRender = gate.hold
	return env, gate, urls
}

func getAsync(env *variantEnv, ctx context.Context, url string, codes chan<- int) {
	go func() {
		request := httptest.NewRequest(http.MethodGet, url, nil).WithContext(ctx)
		recorder := httptest.NewRecorder()
		env.public.ServeHTTP(recorder, request)
		codes <- recorder.Code
	}()
}

func TestImageVariantsBoundedRenders(t *testing.T) {
	t.Run("At most IMAGE_VARIANT_MAX_CONCURRENT_RENDERS renders run at once", func(t *testing.T) {
		env, gate, urls := concurrencyEnv(t, 2, 5)
		codes := make(chan int, len(urls))
		for _, url := range urls {
			getAsync(env, context.Background(), url, codes)
		}

		gate.awaitStarts(t, 2)
		time.Sleep(settleDelay)
		if gate.running() != 2 || env.renderer.renderCount() != 2 {
			t.Fatalf("in flight = %d, renders = %d", gate.running(), env.renderer.renderCount())
		}
		gate.letRun(len(urls))
		for range urls {
			select {
			case code := <-codes:
				if code != 200 {
					t.Errorf("status = %d", code)
				}
			case <-time.After(gateTimeout):
				t.Fatal("a request never finished")
			}
		}
		if gate.peakRunning() != 2 || env.renderer.renderCount() != 5 {
			t.Errorf("peak = %d, renders = %d", gate.peakRunning(), env.renderer.renderCount())
		}
	})

	t.Run("A cached file never takes a render slot", func(t *testing.T) {
		env, gate, urls := concurrencyEnv(t, 1, 2)
		gate.letRun(1)
		warm := make(chan int, 1)
		getAsync(env, context.Background(), urls[0], warm)
		gate.awaitStarts(t, 1)
		if code := <-warm; code != 200 {
			t.Fatalf("warm-up status = %d", code)
		}
		blocked := make(chan int, 1)
		getAsync(env, context.Background(), urls[1], blocked)
		gate.awaitStarts(t, 1)

		cached := make(chan int, 1)
		getAsync(env, context.Background(), urls[0], cached)

		select {
		case code := <-cached:
			if code != 200 {
				t.Errorf("status = %d", code)
			}
		case <-time.After(gateTimeout):
			t.Fatal("the cached request waited for a slot")
		}
		gate.letRun(1)
		<-blocked
	})

	t.Run("A waiting request whose client has gone does not render", func(t *testing.T) {
		env, gate, urls := concurrencyEnv(t, 1, 2)
		holder := make(chan int, 1)
		getAsync(env, context.Background(), urls[0], holder)
		gate.awaitStarts(t, 1)
		ctx, cancel := context.WithCancel(context.Background())
		waiter := make(chan int, 1)
		getAsync(env, ctx, urls[1], waiter)
		time.Sleep(settleDelay)

		cancel()

		select {
		case <-waiter:
		case <-time.After(gateTimeout):
			t.Fatal("the waiting request kept waiting")
		}
		gate.letRun(1)
		<-holder
		if env.renderer.renderCount() != 1 || env.storage.putCount() != 1 || env.files.rowCount() != 1 {
			t.Errorf("renders = %d, puts = %d, rows = %d", env.renderer.renderCount(), env.storage.putCount(), env.files.rowCount())
		}
	})

	t.Run("Identical concurrent requests still share one render and one slot", func(t *testing.T) {
		env, gate, urls := concurrencyEnv(t, 1, 1)
		codes := make(chan int, 20)
		for index := 0; index < 20; index++ {
			getAsync(env, context.Background(), urls[0], codes)
		}

		gate.awaitStarts(t, 1)
		time.Sleep(settleDelay)
		gate.letRun(20)
		var served atomic.Int32
		for index := 0; index < 20; index++ {
			if <-codes == 200 {
				served.Add(1)
			}
		}

		if served.Load() != 20 || env.renderer.renderCount() != 1 || gate.peakRunning() != 1 {
			t.Errorf("served = %d, renders = %d, peak = %d", served.Load(), env.renderer.renderCount(), gate.peakRunning())
		}
	})
}

func TestImageVariantPixelAndConcurrencySettings(t *testing.T) {
	rows := []struct {
		name    string
		env     string
		value   string
		unset   bool
		want    int64
		failure bool
	}{
		{"pixels unset", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "", true, 50000000, false},
		{"pixels 2048", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "2048", false, 2048, false},
		{"pixels 0", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "0", false, 0, true},
		{"pixels -5", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "-5", false, 0, true},
		{"pixels many", "IMAGE_VARIANT_MAX_SOURCE_PIXELS", "many", false, 0, true},
		{"renders unset", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "", true, 4, false},
		{"renders 1", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "1", false, 1, false},
		{"renders 0", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "0", false, 0, true},
		{"renders -1", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "-1", false, 0, true},
		{"renders many", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS", "many", false, 0, true},
	}
	for _, row := range rows {
		t.Run("The env var is loaded: "+row.name, func(t *testing.T) {
			setBaseSettingsEnv(t)
			if row.unset {
				t.Setenv(row.env, "")
			} else {
				t.Setenv(row.env, row.value)
			}

			settings, err := config.NewSettingsFromEnv()

			if row.failure {
				if err == nil || !strings.Contains(err.Error(), row.env) {
					t.Errorf("err = %v", err)
				}
				return
			}
			got := settings.ImageVariantMaxSourcePixels
			if row.env == "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS" {
				got = int64(settings.ImageVariantMaxConcurrentRenders)
			}
			if err != nil || got != row.want {
				t.Errorf("value = %d, err = %v", got, err)
			}
		})
	}

	t.Run("Both defaults are listed in envDefaults, .env.example and docs/configuration.md", func(t *testing.T) {
		defaults := between(readRepositoryFile(t, "cmd/server/main.go"), "var envDefaults = map[string]string{", "\n}")
		required := between(readRepositoryFile(t, "cmd/server/main.go"), "var requiredEnvVars = []string{", "\n}")
		for _, name := range []string{"EnvImageVariantMaxSourcePixels", "EnvImageVariantMaxConcurrentRenders"} {
			if !strings.Contains(defaults, "config."+name) || strings.Contains(required, name) {
				t.Errorf("%s is misplaced", name)
			}
		}
		for _, document := range []string{".env.example", "docs/configuration.md"} {
			content := readRepositoryFile(t, document)
			for _, name := range []string{"IMAGE_VARIANT_MAX_SOURCE_PIXELS", "IMAGE_VARIANT_MAX_CONCURRENT_RENDERS"} {
				if !strings.Contains(content, name) {
					t.Errorf("%s does not mention %s", document, name)
				}
			}
		}
	})
}