pad/internal/server/middleware_query_test.go

package server

import (
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"strings"
	"testing"

	"github.com/PerpetualSoftware/pad/internal/models"
	"github.com/PerpetualSoftware/pad/internal/store"
	"github.com/PerpetualSoftware/pad/internal/store/storetest"
)

// badQueryVal is the canonically escaped form of an invalid-UTF-8 byte, in a
// query VALUE. Unlike the path (see badPathSeg), the case of the hex digits
// is not load-bearing here — url.ParseQuery decodes "%ff" and "%FF"
// identically, and TestValidateQueryRejectsUnbindableQueryText drives both to
// pin that rather than assume it.
const badQueryVal = "bad-%FF-x"

// assertQueryVectorIntact fails if the request Go builds from target does not
// actually deliver invalid query text to a handler — i.e. if the premise of
// the assertions below has stopped holding. Without this, a future change to
// how net/url decodes a query would silently turn every case in this file
// into a test of nothing that still passes.
//
// It asserts against r.URL.Query(), because that is what handlers read and
// therefore what "reaches the store" means for the query string.
func assertQueryVectorIntact(t *testing.T, target string) {
	t.Helper()
	req := httptest.NewRequest("GET", target, nil)
	for key, values := range req.URL.Query() {
		if !bindableText(key) {
			return
		}
		for _, v := range values {
			if !bindableText(v) {
				return
			}
		}
	}
	t.Fatalf("premise broken: %q parses to a query whose keys and values are all "+
		"already valid text (%#v); this test can no longer exercise BUG-2784's vector",
		target, req.URL.Query())
}

// TestValidateQueryRejectsUnbindableQueryText drives the REAL server rather
// than ValidateQuery directly, so it vouches for the middleware's BINDING as
// well as its logic — a direct call would pass even if nobody had wired it
// into the chain (CONVE-19).
//
// It does NOT reproduce the unfixed 500; no test here can, because every test
// runs the fixed server. The pre-fix distribution is recorded on ValidateQuery
// and what keeps it reachable as evidence is the mutation matrix, since
// unwiring ValidateQuery fails TestValidateQueryPostgresNoInternalError.
func TestValidateQueryRejectsUnbindableQueryText(t *testing.T) {
	t.Parallel()
	srv := testServer(t)
	ws := createWSForTest(t, srv)

	cases := []struct {
		name   string
		method string
		target string
	}{
		// The parseItemListParams funnel: a DECLARED parameter, and an
		// undeclared one that becomes a field filter only because the
		// wildcard branch exists. Both reach a text comparison.
		{"items list, declared param", "GET", "/api/v1/workspaces/" + ws + "/items?search=" + badQueryVal},
		{"items list, undeclared param", "GET", "/api/v1/workspaces/" + ws + "/items?email=" + badQueryVal},
		{"collection items", "GET", "/api/v1/workspaces/" + ws + "/collections/tasks/items?status=" + badQueryVal},
		// The discrete point-of-use sites the sweep found.
		{"search q", "GET", "/api/v1/search?q=" + badQueryVal},
		{"activity actor", "GET", "/api/v1/workspaces/" + ws + "/activity?actor=" + badQueryVal},
		{"attachments collection", "GET", "/api/v1/workspaces/" + ws + "/attachments?collection=" + badQueryVal},
		{"items-index collection", "GET", "/api/v1/workspaces/" + ws + "/items-index?collection=" + badQueryVal},
		// A NUL is valid UTF-8 and refused by Postgres, so the rule covers
		// it too; this case survives dropping the ContainsRune half of
		// bindableText only if that half is still there.
		{"NUL value", "GET", "/api/v1/workspaces/" + ws + "/items?search=bad-%00-x"},
		// The KEY half. Precautionary — no key was observed to 500 — so this
		// case is what makes the key check non-vacuous.
		{"invalid key", "GET", "/api/v1/workspaces/" + ws + "/items?bad-%FF-key=x"},
		{"NUL key", "GET", "/api/v1/workspaces/" + ws + "/items?bad-%00-key=x"},
		// Lowercase escape: same byte, and the answer must not depend on the
		// case of a hex digit.
		{"lowercase %ff", "GET", "/api/v1/workspaces/" + ws + "/items?search=bad-%ff-x"},
		// Not percent-encoded at all: a client may put the raw byte in the
		// query string. This is the case url.ParseQuery never decodes, and
		// it is caught by the raw check before the fast path returns.
		{"raw unescaped byte", "GET", "/api/v1/workspaces/" + ws + "/items?search=bad-\xff-x"},
		// Second and later values of a repeated key, and later pairs
		// generally: a check that stopped at the first pair would pass the
		// cases above and fail these.
		{"second pair", "GET", "/api/v1/workspaces/" + ws + "/items?status=open&search=" + badQueryVal},
		{"repeated key, second value", "GET", "/api/v1/workspaces/" + ws + "/items?search=fine&search=" + badQueryVal},
		// Methods other than GET route through the same chain.
		{"PATCH", "PATCH", "/api/v1/workspaces/" + ws + "/items/anything?search=" + badQueryVal},
		{"POST", "POST", "/api/v1/workspaces/" + ws + "/collections/tasks/items?search=" + badQueryVal},
		// Not an /api/v1 route: ValidateQuery is on the ROOT router. Fails if
		// someone moves the Use() into the API group.
		{"non-API route (SPA catch-all)", "GET", "/anything?search=" + badQueryVal},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			assertQueryVectorIntact(t, tc.target)
			rr := doRequest(srv, tc.method, tc.target, nil)
			if rr.Code != http.StatusBadRequest {
				t.Fatalf("%s %s: expected 400, got %d: %s",
					tc.method, tc.target, rr.Code, rr.Body.String())
			}
			if code := pathErrorCode(t, rr); code != "invalid_query" {
				t.Fatalf("%s %s: expected error code invalid_query, got %q",
					tc.method, tc.target, code)
			}
		})
	}
}

// TestValidateQueryAllowsValidText is the control leg, and it is the half
// that matters most here: the objection to validating the query string at
// the transport was that its legitimate value domain is much wider than the
// path's. It is — and every one of these must still work.
//
// A middleware that answered 400 to any query with a percent-escape, or to
// anything non-ASCII, would pass the test above and break real clients.
func TestValidateQueryAllowsValidText(t *testing.T) {
	t.Parallel()
	srv := testServer(t)
	ws := createWSForTest(t, srv)

	for _, tc := range []struct {
		name   string
		target string
		// handlerRefuses: the middleware must let the request through, and the
		// HANDLER refuses it for its own reason. Asserted by the error code,
		// which says whose refusal it is.
		handlerRefuses string
	}{
		{"no query at all", "/api/v1/workspaces/" + ws + "/items", ""},
		{"empty query", "/api/v1/workspaces/" + ws + "/items?", ""},
		{"plain ascii", "/api/v1/workspaces/" + ws + "/items?search=hello", ""},
		{"escaped accented text", "/api/v1/workspaces/" + ws + "/items?search=caf%C3%A9", ""},
		{"unescaped accented text", "/api/v1/workspaces/" + ws + "/items?search=café", ""},
		{"escaped emoji", "/api/v1/workspaces/" + ws + "/items?search=%F0%9F%9A%80", ""},
		{"plus as space", "/api/v1/workspaces/" + ws + "/items?search=two+words", ""},
		{"escaped space", "/api/v1/workspaces/" + ws + "/items?search=two%20words", ""},
		{"escaped ampersand in value", "/api/v1/workspaces/" + ws + "/items?search=a%26b", ""},
		{"escaped equals in value", "/api/v1/workspaces/" + ws + "/items?search=a%3Db", ""},
		{"empty value", "/api/v1/workspaces/" + ws + "/items?search=", ""},
		{"key with no equals", "/api/v1/workspaces/" + ws + "/items?search", ""},
		{"repeated key", "/api/v1/workspaces/" + ws + "/items?search=a&search=b", ""},
		// Valid UTF-8, so the middleware passes it; since BUG-3480 the list
		// handler refuses a filter key that is not a field key (letters,
		// digits, '_', '-'), which the store used to drop, answering the
		// unfiltered list.
		{"non-ascii key", "/api/v1/workspaces/" + ws + "/items?caf%C3%A9=x", "validation_error"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			rr := doRequest(srv, "GET", tc.target, nil)
			if tc.handlerRefuses != "" {
				if rr.Code != http.StatusBadRequest || pathErrorCode(t, rr) != tc.handlerRefuses {
					t.Fatalf("GET %s: expected the handler's 400 %s, got %d: %s", tc.target, tc.handlerRefuses, rr.Code, rr.Body.String())
				}
				return
			}
			if rr.Code != http.StatusOK {
				t.Fatalf("GET %s: expected 200, got %d: %s", tc.target, rr.Code, rr.Body.String())
			}
		})
	}
}

// TestValidateQueryDeliversEscapedUnicodeToTheHandler is the control the
// status-code table above cannot give: a 200 proves the request was not
// refused, not that the parameter still means what the client sent. This
// asserts the decoded value reached the handler intact by requiring it to
// MATCH an item it can only match if it did.
func TestValidateQueryDeliversEscapedUnicodeToTheHandler(t *testing.T) {
	t.Parallel()
	srv := testServer(t)
	ws := createWSForTest(t, srv)

	create := func(title string) models.Item {
		t.Helper()
		rr := doRequest(srv, "POST", "/api/v1/workspaces/"+ws+"/collections/tasks/items",
			map[string]interface{}{"title": title})
		if rr.Code != http.StatusCreated {
			t.Fatalf("create item %q: %d %s", title, rr.Code, rr.Body.String())
		}
		var it models.Item
		parseJSON(t, rr, &it)
		return it
	}

	created := create("Le café serves 🚀 rockets")
	// A DECOY that must not match. Without it this test is vacuous: with one
	// item in the workspace, a handler that ignored `search` entirely — or a
	// middleware that stripped the query — would return that item and pass
	// every assertion below. The decoy is what makes "the value reached the
	// handler" distinguishable from "the endpoint returned everything".
	decoy := create("Plain teahouse, no pastries")

	// Premise: the decoy is visible to THIS endpoint when nothing filters it
	// out. If the unfiltered list did not return both, the exact-set
	// assertions below would be satisfied by an endpoint that simply never
	// returns the decoy, and the decoy would prove nothing.
	rr := doRequest(srv, "GET", "/api/v1/workspaces/"+ws+"/items", nil)
	var all []models.Item
	parseJSON(t, rr, &all)
	if len(all) != 2 {
		t.Fatalf("premise broken: the unfiltered list returned %d items, want the 2 just "+
			"created — the decoy cannot discriminate a filtered result from an unfiltered one",
			len(all))
	}

	// NOT an emoji case, deliberately, though the status-code table above has
	// one. `?search=%F0%9F%9A%80` matches zero items — and so does the same
	// emoji sent UNESCAPED, against the same item, which is what identifies
	// the cause: the search index does not tokenize it. That is a property of
	// search, not of percent-decoding, so an emoji here would fail for a
	// reason this test is not about. Multi-byte coverage comes from café,
	// whose escape sequence is exactly as multi-byte as the emoji's.
	for _, tc := range []struct{ name, query string }{
		{"escaped accented text", "search=caf%C3%A9"},
		{"escaped space in a phrase", "search=serves%20rockets"},
		{"plus-encoded space in a phrase", "search=serves+rockets"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			rr := doRequest(srv, "GET", "/api/v1/workspaces/"+ws+"/items?"+tc.query, nil)
			if rr.Code != http.StatusOK {
				t.Fatalf("expected 200, got %d: %s", rr.Code, rr.Body.String())
			}
			var got []models.Item
			parseJSON(t, rr, &got)
			// The EXACT result set, not merely "contains". Asserting only
			// membership would pass for a handler that returned every item.
			if len(got) != 1 || got[0].ID != created.ID {
				ids := make([]string, len(got))
				for i, it := range got {
					ids[i] = it.Title
				}
				t.Fatalf("?%s returned %d items (%v), want exactly the one it names (%q). "+
					"Returning the decoy %q too means the filter was not applied; returning "+
					"none means the decoded value did not reach the handler intact.",
					tc.query, len(got), ids, created.Title, decoy.Title)
			}
		})
	}
}

// TestValidateQueryRunsAfterValidatePath pins the ORDERING decision in
// setupRouter. A request bad in both places is answered for its path, which
// is the more specific fault; swapping the two Use() calls flips the code and
// fails here.
func TestValidateQueryRunsAfterValidatePath(t *testing.T) {
	t.Parallel()
	srv := testServer(t)
	target := "/api/v1/workspaces/" + badPathSeg + "/items?search=" + badQueryVal
	assertPathVectorIntact(t, target)
	assertQueryVectorIntact(t, target)

	rr := doRequest(srv, "GET", target, nil)
	if rr.Code != http.StatusBadRequest {
		t.Fatalf("expected 400, got %d: %s", rr.Code, rr.Body.String())
	}
	if code := pathErrorCode(t, rr); code != "invalid_path" {
		t.Fatalf("a request bad in both path and query must be answered for the path: got %q", code)
	}
}

// TestValidateQueryRejectionLooksLikeEveryOtherAPIError pins the response
// SHAPE, not just the status: this rejection short-circuits above the
// /api/v1 group's cors.Handler and jsonContentType, so a naive
// implementation answers a JSON body typed text/plain and, on a
// cross-origin deployment, with no CORS headers — which the browser refuses
// to let the page read at all.
// Each header is compared against the SAME endpoint answered normally, so
// the assertion is parity with the API's own responses rather than a list of
// header names copied from a spec — the same construction the path half uses.
func TestValidateQueryRejectionLooksLikeEveryOtherAPIError(t *testing.T) {
	t.Parallel()
	const allowed = "https://app.example.com"

	srv := testServer(t)
	srv.SetCORSOrigins(allowed)
	ws := createWSForTest(t, srv)

	get := func(target, origin string) *httptest.ResponseRecorder {
		req := httptest.NewRequest("GET", target, nil)
		req.RemoteAddr = "10.9.9.9:1"
		if origin != "" {
			req.Header.Set("Origin", origin)
		}
		rec := httptest.NewRecorder()
		srv.ServeHTTP(rec, req)
		return rec
	}

	badTarget := "/api/v1/workspaces/" + ws + "/items?search=" + badQueryVal
	okTarget := "/api/v1/workspaces/" + ws + "/items?search=ordinary-value"
	compared := []string{
		"Content-Type",
		"Access-Control-Allow-Origin",
		"Access-Control-Allow-Credentials",
		"Vary",
	}

	for _, origin := range []string{allowed, "https://evil.example.com"} {
		bad := get(badTarget, origin)
		ok := get(okTarget, origin)
		if bad.Code != http.StatusBadRequest {
			t.Fatalf("origin %s: expected 400, got %d: %s", origin, bad.Code, bad.Body.String())
		}
		if ok.Code != http.StatusOK {
			t.Fatalf("origin %s: control expected 200, got %d: %s", origin, ok.Code, ok.Body.String())
		}
		for _, h := range compared {
			if bad.Header().Get(h) != ok.Header().Get(h) {
				t.Errorf("origin %s: %s differs between the rejection and an ordinary "+
					"response: rejection %q, ordinary %q",
					origin, h, bad.Header().Get(h), ok.Header().Get(h))
			}
		}
	}

	// Sanity: the parity above is worthless if the ordinary response carries
	// no CORS header to match in the first place.
	if got := get(okTarget, allowed).Header().Get("Access-Control-Allow-Origin"); got != allowed {
		t.Fatalf("premise broken: an ordinary response to an allowed origin carries "+
			"Access-Control-Allow-Origin %q, so header parity proves nothing", got)
	}
}

// TestValidateQueryPostgresNoInternalError is the dialect half: it drives the
// vector on the backend where the 500 occurred, against a real Postgres
// store, and requires 400. Unwiring ValidateQuery fails this test, which is
// what keeps the measured 500 reachable as evidence.
//
// Skips unless PAD_TEST_POSTGRES_URL is set; runs under `make test-pg`.
func TestValidateQueryPostgresNoInternalError(t *testing.T) {
	if os.Getenv("PAD_TEST_POSTGRES_URL") == "" {
		t.Skip("PAD_TEST_POSTGRES_URL not set — the 500 only reproduces on Postgres")
	}
	s := storetest.NewPostgres(t)
	srv := New(s)
	t.Cleanup(func() { srv.Stop() })
	if srv.store.D().Driver() != store.DriverPostgres {
		t.Fatalf("expected a Postgres store, got %s", srv.store.D().Driver())
	}

	ws := createWSForTest(t, srv)
	for _, target := range []string{
		"/api/v1/workspaces/" + ws + "/items?search=" + badQueryVal,
		"/api/v1/workspaces/" + ws + "/items?email=" + badQueryVal,
		"/api/v1/workspaces/" + ws + "/items?search=bad-%00-x",
		"/api/v1/workspaces/" + ws + "/collections/tasks/items?status=" + badQueryVal,
		"/api/v1/search?q=" + badQueryVal,
		"/api/v1/workspaces/" + ws + "/activity?actor=" + badQueryVal,
		"/api/v1/workspaces/" + ws + "/attachments?collection=" + badQueryVal,
		"/api/v1/workspaces/" + ws + "/items-index?collection=" + badQueryVal,
	} {
		assertQueryVectorIntact(t, target)
		rr := doRequest(srv, "GET", target, nil)
		if rr.Code != http.StatusBadRequest {
			t.Fatalf("GET %s on Postgres: expected 400, got %d: %s",
				target, rr.Code, rr.Body.String())
		}
	}

	// Control: the same store answers the same endpoints normally, so the
	// 400s above are the middleware's judgement and not a broken fixture.
	rr := doRequest(srv, "GET", "/api/v1/workspaces/"+ws+"/items?search=ordinary-value", nil)
	if rr.Code != http.StatusOK {
		t.Fatalf("GET valid search on Postgres: expected 200, got %d: %s", rr.Code, rr.Body.String())
	}
}

func TestValidQueryText(t *testing.T) {
	t.Parallel()
	cases := []struct {
		name string
		in   string
		want bool
	}{
		{"empty", "", true},
		{"plain", "search=hello&status=open", true},
		{"escaped utf8 value", "search=caf%C3%A9", true},
		{"literal utf8 value", "search=café", true},
		{"escaped emoji", "search=%F0%9F%9A%80", true},
		{"plus", "search=two+words", true},
		{"escaped separators", "search=a%26b%3Dc", true},
		{"escaped key", "caf%C3%A9=x", true},
		{"no equals", "search", true},

		{"escaped lone 0xff value", "search=bad-%FF-x", false},
		{"escaped lone 0xff value, lowercase", "search=bad-%ff-x", false},
		{"raw lone 0xff value", "search=bad-\xff-x", false},
		{"escaped NUL value", "search=bad-%00-x", false},
		{"raw NUL value", "search=bad-\x00-x", false},
		{"escaped truncated sequence", "search=bad-%C3(-x", false},
		{"escaped surrogate half", "search=%ED%A0%80", false},
		{"escaped overlong", "search=%C0%AF", false},
		{"bad key", "bad-%FF-key=x", false},
		{"bad value in a later pair", "status=open&search=bad-%FF-x", false},
		{"bad second value of a repeated key", "search=fine&search=bad-%FF-x", false},

		// A malformed escape is dropped by url.ParseQuery, and by
		// r.URL.Query(), so no handler can see it. The pairs that DO parse
		// still decide the answer — both directions.
		{"malformed escape, rest fine", "search=100%off&status=open", true},
		{"malformed escape, rest bad", "search=100%off&status=bad-%FF-x", false},

		// The raw check is STRICTER than the reachable set, deliberately: the
		// bad byte here sits in a pair url.ParseQuery drops, so no handler
		// could see it, and the request is refused anyway. Pinned because it
		// is documented behaviour, not an accident — see validQueryText.
		{"raw NUL inside a dropped pair", "search=ok&ignored=\x00%", false},
		{"raw 0xff inside a dropped pair", "search=ok&ignored=\xff%", false},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got := validQueryText(c.in); got != c.want {
				t.Errorf("validQueryText(%q) = %v, want %v", c.in, got, c.want)
			}
		})
	}
}

// TestValidQueryTextFastPathIsNeverMorePermissive is the differential check
// behind the '%' fast path.
//
// AN EARLIER VERSION OF THIS TEST CLAIMED EQUIVALENCE — that the fast path
// "can never disagree" with decoding every pair and checking each one — and
// that claim is FALSE. `ignored=\xff;bad` disagrees: url.ParseQuery refuses
// an unescaped ';' ("invalid semicolon separator in query"), drops every
// pair, and the per-pair comparator then finds nothing to object to and
// returns true, while validQueryText rejects on the raw check. The old
// corpus happened not to contain a semicolon, so a false contract passed.
//
// The property that is actually true, and the one worth pinning, is
// ONE-DIRECTIONAL: the fast path is never MORE PERMISSIVE than the per-pair
// check. That is the safety direction — it may not let through anything the
// decode path would reject. The other direction is the deliberate extra
// strictness validQueryText documents, and the semicolon case is now in the
// corpus so the case that falsified the old contract exercises the new one.
func TestValidQueryTextFastPathIsNeverMorePermissive(t *testing.T) {
	t.Parallel()
	slow := func(rawQuery string) bool {
		q, _ := url.ParseQuery(rawQuery)
		for key, values := range q {
			if !bindableText(key) {
				return false
			}
			for _, v := range values {
				if !bindableText(v) {
					return false
				}
			}
		}
		return true
	}
	corpus := []string{
		"", "a=b", "a=b&c=d", "search=café", "search=two+words", "a", "a=", "=b",
		"a=b&a=c", "&&", "a=b&", "search=a b", "x=\xff", "\xff=x", "x=a\x00b",
		"a=b&c=\xc3", "søk=svar", "a=%", "s=100%off",
		// The cases that falsified the old equivalence contract: ParseQuery
		// drops every pair on an unescaped ';', so the per-pair check has
		// nothing left to reject and returns true while the raw check
		// rejects. Permitted under the one-directional property, and a
		// regression to equivalence fails here.
		"ignored=\xff;bad", "a=b;c=\xff", "x=\x00;y", "a=b;c=d",
	}
	sawStricter := false
	for _, raw := range corpus {
		if strings.Contains(raw, "%") {
			continue
		}
		fast, perPair := validQueryText(raw), slow(raw)
		if fast && !perPair {
			t.Errorf("fast path is MORE PERMISSIVE for %q: validQueryText=true, per-pair=false — "+
				"it let through something the decode path rejects", raw)
		}
		if !fast && perPair {
			sawStricter = true
		}
	}
	// The corpus must actually contain a case where the two differ, or the
	// assertion above is satisfied by an equivalence that would make this
	// test's whole premise pointless — which is exactly the state the old
	// version was in.
	if !sawStricter {
		t.Error("premise broken: no corpus entry exercises the stricter-than-per-pair " +
			"direction, so this test would pass against a fast path that simply mirrored " +
			"the decode path and the one-directional contract is untested")
	}
}