mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-11-01 00:42:13 +00:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Josh Siviour
|
0fd85d117b | ||
|
Ben Duncan
|
51d7ac91f5 | ||
|
Ben Duncan
|
707dbefa13 | ||
|
Ben Duncan
|
4050b63ada | ||
|
Ben Duncan
|
955714f557 | ||
|
Josh Siviour
|
2e770e8859 | ||
|
Ben Duncan
|
f3ee105ed4 | ||
|
Ben Duncan
|
00070b1ecd | ||
|
Josh Siviour
|
9405d2c233 | ||
|
Josh Siviour
|
a431fd3385 |
@@ -0,0 +1,110 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="580" viewBox="0 0 900 580">
|
||||
<defs>
|
||||
<linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#0a0a0a"/>
|
||||
<stop offset="100%" style="stop-color:#0d1117"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="vpc-grad" x1="0%" y1="0%" x2="0%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#58a6ff14"/>
|
||||
<stop offset="100%" style="stop-color:#58a6ff05"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="node-grad" x1="0%" y1="0%" x2="0%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#3fb9501f"/>
|
||||
<stop offset="100%" style="stop-color:#3fb95008"/>
|
||||
</linearGradient>
|
||||
<linearGradient id="svc-grad" x1="0%" y1="0%" x2="0%" y2="100%">
|
||||
<stop offset="0%" style="stop-color:#a855f71f"/>
|
||||
<stop offset="100%" style="stop-color:#a855f708"/>
|
||||
</linearGradient>
|
||||
<marker id="arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
|
||||
<path d="M0,0 L10,5 L0,10 z" fill="#71717a"/>
|
||||
</marker>
|
||||
</defs>
|
||||
<rect width="900" height="580" fill="url(#bg)" rx="12"/>
|
||||
<text x="24" y="30" font-family="'Segoe UI', system-ui, sans-serif" font-size="11" fill="#a1a1aa" letter-spacing="2.5" font-weight="600">SPINIFEX ON VULTR · ONE NODE OR THREE</text>
|
||||
<text x="876" y="30" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#71717a" text-anchor="end">--nodes 1 or 3 · vx1-g-16c-64g or bare metal</text>
|
||||
<rect x="30" y="52" width="250" height="56" rx="8" fill="#e4e4e70d" stroke="#e4e4e7" stroke-width="1.1" stroke-opacity="0.6"/>
|
||||
<text x="155.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#e4e4e7" text-anchor="middle" font-weight="700">Your machine</text>
|
||||
<text x="155.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="middle">deploy-vultr.sh · OpenTofu or Terraform</text>
|
||||
<rect x="325" y="52" width="250" height="56" rx="8" fill="#3fb9500d" stroke="#3fb950" stroke-width="1.1" stroke-opacity="0.6"/>
|
||||
<text x="450.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" text-anchor="middle" font-weight="700">Internet</text>
|
||||
<text x="450.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#7ee787" text-anchor="middle">customers reach your EC2 and load balancers</text>
|
||||
<rect x="620" y="52" width="250" height="56" rx="8" fill="#58a6ff12" stroke="#58a6ff" stroke-width="1.1" stroke-opacity="0.6"/>
|
||||
<text x="745.0" y="76" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#79c0ff" text-anchor="middle" font-weight="700">Vultr API</text>
|
||||
<text x="745.0" y="94" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a5d6ff" text-anchor="middle">VPC · servers · reserved IPs</text>
|
||||
<rect x="30" y="160" width="840" height="400" rx="10" fill="url(#vpc-grad)" stroke="#58a6ff" stroke-width="1.2" stroke-opacity="0.55" stroke-dasharray="6,4"/>
|
||||
<text x="50" y="184" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#79c0ff" font-weight="700">Vultr VPC</text>
|
||||
<text x="850" y="184" font-family="'SF Mono', ui-monospace, monospace" font-size="10" fill="#58a6ff" text-anchor="end">private network 10.10.8.0/24</text>
|
||||
<line x1="155" y1="108" x2="155" y2="158" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
|
||||
<text x="165" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="start">Terraform builds · SSH installs</text>
|
||||
<line x1="450" y1="108" x2="450" y2="158" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
|
||||
<text x="460" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="start">reserved IPs</text>
|
||||
<line x1="745" y1="158" x2="745" y2="110" stroke="#71717a" stroke-width="1.2" opacity="0.8" marker-end="url(#arrow)"/>
|
||||
<text x="735" y="138" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#a1a1aa" text-anchor="end">Spinifex calls the API</text>
|
||||
<rect x="60" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6"/>
|
||||
<text x="78" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node1</text>
|
||||
<text x="292" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">Ubuntu 24.04 · Spinifex</text>
|
||||
<line x1="74" y1="237" x2="296" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
|
||||
<rect x="76" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="113.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
|
||||
<rect x="160" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="190.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
|
||||
<rect x="230" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="260.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
|
||||
<text x="185" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
|
||||
<rect x="325" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" stroke-dasharray="5,3"/>
|
||||
<text x="343" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node2</text>
|
||||
<text x="557" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">with --nodes 3</text>
|
||||
<line x1="339" y1="237" x2="561" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
|
||||
<rect x="341" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="378.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
|
||||
<rect x="425" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="455.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
|
||||
<rect x="495" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="525.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
|
||||
<text x="450" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
|
||||
<rect x="590" y="205" width="250" height="118" rx="8" fill="url(#node-grad)" stroke="#3fb950" stroke-width="1.2" stroke-opacity="0.6" stroke-dasharray="5,3"/>
|
||||
<text x="608" y="229" font-family="'SF Mono', ui-monospace, monospace" font-size="12.5" fill="#3fb950" font-weight="700">node3</text>
|
||||
<text x="822" y="229" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#7ee787" text-anchor="end">with --nodes 3</text>
|
||||
<line x1="604" y1="237" x2="826" y2="237" stroke="#3fb950" stroke-opacity="0.3" stroke-width="0.6"/>
|
||||
<rect x="606" y="250" width="74" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="643.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">AWS API</text>
|
||||
<rect x="690" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="720.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">EBS</text>
|
||||
<rect x="760" y="250" width="60" height="24" rx="5" fill="#3fb9501a" stroke="#3fb950" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="790.0" y="266" font-family="'SF Mono', ui-monospace, monospace" font-size="10.5" fill="#7ee787" text-anchor="middle">S3</text>
|
||||
<text x="715" y="300" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#a1a1aa" text-anchor="middle">VPC networking · your instances</text>
|
||||
<line x1="310" y1="264" x2="325" y2="264" stroke="#58a6ff" stroke-width="1.6" stroke-dasharray="3,3" opacity="0.9"/>
|
||||
<line x1="575" y1="264" x2="590" y2="264" stroke="#58a6ff" stroke-width="1.6" stroke-dasharray="3,3" opacity="0.9"/>
|
||||
<text x="450" y="348" font-family="'Segoe UI', system-ui, sans-serif" font-size="10" fill="#79c0ff" text-anchor="middle">nodes cluster over the private network: control plane, EBS replicas and S3 shards</text>
|
||||
<rect x="60" y="370" width="780" height="166" rx="8" fill="url(#svc-grad)" stroke="#a855f7" stroke-width="1.2" stroke-opacity="0.55"/>
|
||||
<text x="80" y="394" font-family="'SF Mono', ui-monospace, monospace" font-size="12" fill="#c084fc" font-weight="700">AWS services across the fleet</text>
|
||||
<text x="820" y="394" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#d8b4fe" text-anchor="end">spread across the nodes; with three, moved to a survivor if one fails</text>
|
||||
<line x1="76" y1="402" x2="824" y2="402" stroke="#a855f7" stroke-opacity="0.3" stroke-width="0.6"/>
|
||||
<rect x="76" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="119.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EC2</text>
|
||||
<text x="119.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">instances</text>
|
||||
<rect x="169" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="212.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">ALB / NLB</text>
|
||||
<text x="212.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">load balancers</text>
|
||||
<rect x="262" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="305.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EKS</text>
|
||||
<text x="305.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">Kubernetes</text>
|
||||
<rect x="355" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="398.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">ECS</text>
|
||||
<text x="398.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">containers</text>
|
||||
<rect x="448" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="491.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">RDS</text>
|
||||
<text x="491.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">databases</text>
|
||||
<rect x="541" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="584.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">S3</text>
|
||||
<text x="584.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">object storage</text>
|
||||
<rect x="634" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="677.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">EBS</text>
|
||||
<text x="677.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">volumes</text>
|
||||
<rect x="727" y="418" width="86" height="52" rx="6" fill="#a855f71a" stroke="#a855f7" stroke-opacity="0.5" stroke-width="0.6"/>
|
||||
<text x="770.0" y="440" font-family="'SF Mono', ui-monospace, monospace" font-size="11.5" fill="#d8b4fe" text-anchor="middle" font-weight="700">IAM</text>
|
||||
<text x="770.0" y="457" font-family="'Segoe UI', system-ui, sans-serif" font-size="9" fill="#a1a1aa" text-anchor="middle">access</text>
|
||||
<text x="450" y="500" font-family="'Segoe UI', system-ui, sans-serif" font-size="10.5" fill="#a1a1aa" text-anchor="middle">Use the AWS CLI, SDKs and Terraform against your own endpoint, unmodified.</text>
|
||||
<text x="450" y="520" font-family="'Segoe UI', system-ui, sans-serif" font-size="9.5" fill="#71717a" text-anchor="middle">Public addresses are Vultr reserved IPs, attached to the node each resource runs on.</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 13 KiB |
@@ -213,3 +213,28 @@ func TestADR0001_InterimDaemonDirectionCoversRuntimeDaemon(t *testing.T) {
|
||||
t.Error("gateway -> runtime/host/process is rejected, but it is not the daemon")
|
||||
}
|
||||
}
|
||||
|
||||
// TestADR0002_S7_RequestRecordRejectsOutsiders: the request record is readable
|
||||
// by the gateway and ingress/aws only.
|
||||
func TestADR0002_S7_RequestRecordRejectsOutsiders(t *testing.T) {
|
||||
record, ok := classify("spinifex/ingress/aws/reqctx")
|
||||
if !ok || !record.requestRecord {
|
||||
t.Fatalf("ingress/aws/reqctx is not classified as the request record: %+v", record)
|
||||
}
|
||||
r := ruleByID("S7_RequestRecordPrivateToIngress")
|
||||
for name, wantRejected := range map[string]bool{
|
||||
"domains/ecs/awsapi": true,
|
||||
"runtime/roles/awsgw": true,
|
||||
"operator/admin": true,
|
||||
"gateway": false,
|
||||
"ingress/aws/dispatch": false,
|
||||
} {
|
||||
from, ok := classify(fullPath(name))
|
||||
if !ok {
|
||||
t.Fatalf("%s is unclassified", name)
|
||||
}
|
||||
if got := r.forbids(from, record, name); got != wantRejected {
|
||||
t.Errorf("%s -> ingress/aws/reqctx rejected=%v, want %v", name, got, wantRejected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,6 +43,8 @@ type class struct {
|
||||
domain string // owning domain for domain-shaped and agent packages
|
||||
kind domainKind
|
||||
daemon bool // runtime/daemon, the interim daemon -> gateway -> handlers root
|
||||
// requestRecord marks ingress/aws/reqctx, the request record ADR-0002:S7 keeps private to ingress.
|
||||
requestRecord bool
|
||||
}
|
||||
|
||||
func (c class) domainShaped() bool { return c.kind != notDomain }
|
||||
@@ -103,7 +105,7 @@ func classifyApp(app []string) (class, bool) {
|
||||
case "foundation":
|
||||
return class{layer: layerFoundation}, true
|
||||
case "ingress":
|
||||
return class{layer: layerIngress}, true
|
||||
return class{layer: layerIngress, requestRecord: len(app) >= 3 && app[1] == "aws" && app[2] == "reqctx"}, true
|
||||
case "runtime":
|
||||
return class{layer: layerRuntime, daemon: len(app) >= 2 && app[1] == "daemon"}, true
|
||||
case "operator":
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package layering
|
||||
|
||||
import "slices"
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// rule is one ADR-0001 dependency rule over a classified import edge.
|
||||
type rule struct {
|
||||
@@ -151,6 +154,18 @@ var rules = []rule{
|
||||
(importsDomainImplementation(t) || t.layer == layerAgents)
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "S7_RequestRecordPrivateToIngress",
|
||||
clause: "ADR-0002:S7: the request record is private to ingress, so no domain, runtime or operator package reads it; only the gateway and packages under ingress/aws may import ingress/aws/reqctx",
|
||||
hint: "receive what you need through dispatch.Invocation instead of the request record",
|
||||
subject: func(_ class, _ string) bool { return true },
|
||||
forbids: func(_, t class, fromName string) bool {
|
||||
if !t.requestRecord {
|
||||
return false
|
||||
}
|
||||
return fromName != "gateway" && !strings.HasPrefix(fromName, "ingress/aws/")
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "Interim_DaemonGatewayHandlersDirection",
|
||||
clause: "during migration the existing daemon -> gateway -> handlers rule remains the guardrail (Q-20, Q-29)",
|
||||
|
||||
@@ -101,6 +101,24 @@ sqs_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
|
||||
# sqs_same <pid> <starttime>: the recorded process is still alive.
|
||||
sqs_same() { [ -n "$2" ] && sqs_alive "$1" && [ "$(sqs_starttime "$1")" = "$2" ]; }
|
||||
|
||||
# sqs_write_config <addr> <nkey> <file>: the dedicated server's config. Its
|
||||
# max_payload admits a 1 MiB SQS message with its headers (ADR-0011:S4); the
|
||||
# 1MB NATS default does not.
|
||||
sqs_write_config() {
|
||||
cat > "$3" <<EOF
|
||||
listen: "$1:$SQS_PORT"
|
||||
server_name: "sqs-store"
|
||||
max_payload: 2MiB
|
||||
jetstream {
|
||||
store_dir: "$SQS_DATA"
|
||||
sync_interval: always
|
||||
}
|
||||
authorization {
|
||||
users = [ { nkey: "$2" } ]
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# sqs_prepare: validate the contract, check and mount the data disk, write the
|
||||
# server config and mark the store ready. Returns non-zero, unready, on any refusal.
|
||||
sqs_prepare() {
|
||||
@@ -117,17 +135,7 @@ sqs_prepare() {
|
||||
# The server checks the config, including the NKey checksum, before the
|
||||
# disk is touched.
|
||||
umask 077
|
||||
cat > "$SQS_RUN/nats.conf" <<EOF
|
||||
listen: "$addr:$SQS_PORT"
|
||||
server_name: "sqs-store"
|
||||
jetstream {
|
||||
store_dir: "$SQS_DATA"
|
||||
sync_interval: always
|
||||
}
|
||||
authorization {
|
||||
users = [ { nkey: "$nkey" } ]
|
||||
}
|
||||
EOF
|
||||
sqs_write_config "$addr" "$nkey" "$SQS_RUN/nats.conf" || return 1
|
||||
if ! "$NATS_BIN" -t -c "$SQS_RUN/nats.conf" > "$SQS_RUN/config-test.out" 2>&1; then
|
||||
sqs_fatal "server config test failed: $(sed 's/U[A-Z2-7]\{55\}/<nkey>/g' "$SQS_RUN/config-test.out" | tr '\n' ' ')"
|
||||
return 1
|
||||
|
||||
+1
-1
@@ -342,7 +342,7 @@ A release without a complete image removes any previously installed one, so a la
|
||||
|
||||
At boot the guest reads three firmware config values: the data disk's serial, an IPv4 address on one of its interfaces, and one NATS user NKey public key.
|
||||
It mounts exactly one matching disk at `/data` after `e2fsck -p`, refuses a missing, duplicate, unformatted or unrepairable disk, and never formats.
|
||||
JetStream keeps its data on `/data` with `sync_interval: always` (ADR-0011:S3), and the server listens only on that address and accepts only that key.
|
||||
JetStream keeps its data on `/data` with `sync_interval: always` (ADR-0011:S3), `max_payload: 2MiB` admits a full 1 MiB SQS message with its headers (ADR-0011:S4), and the server listens only on that address and accepts only that key.
|
||||
On shutdown the store is stopped before `/data` is unmounted; a store that survives SIGKILL leaves every mount in place.
|
||||
|
||||
A launch uses the versioned subject `system.LaunchSQSStore.v1`, which a daemon without this code does not serve, so it fails with no responder instead of booting another guest.
|
||||
|
||||
@@ -367,6 +367,7 @@ while the branch is in flight; they do not describe the umbrella repository's
|
||||
| `spinifex#1351` | `spinifex/gateway/{bedrock,bedrockruntime,bedrockagent,bedrockagentruntime}.go` (`Bedrock_Request`, `BedrockRuntime_Request`, `BedrockAgent_Request`, `BedrockAgentRuntime_Request`, the four route tables), `spinifex/domains/ochre/authz.go`, and the 13 `Bedrock*` fields of `GatewayConfig` | `spinifex/domains/ochre/awsapi` (`NewRegistration`, `Deps`, `Quota`, `OperationInventory`, `ServiceName`), composed by `spinifex/runtime/roles/awsgw` | The four Bedrock services register through the ADR-0002 seam as one registration, per ADR-0002:S1 (one per signing service) and the ADR-0001 amendment of 2026-10-06: the service is `bedrock` and `bedrock-runtime`, `bedrock-agent` and `bedrock-agent-runtime` are aliases. A request signed with a sub-service scope is dispatched to that sub-service whatever its path, so a path owned by another sub-service answers `InvalidActionException` as before; only the shared `bedrock` scope is split by path. The pipeline characterisation tests committed first (status, error type and envelope per service, metering order, nil-capability answers) pass unchanged apart from setup. The 13 gateway fields become `awsapi.Deps`: `Credentials`, `Endpoints`, `Access`, `Recorder`, `Logging`, `Provisioned`, `Guardrails`, `Embedder`, `KB`, `DataSources`, `Vector` and `Quota`, with the documented nil fallbacks (no grants denies every model, a nil store errors rather than panics); `BedrockAccessAdmin` had no reader outside one test and is gone, and `Deps.NATS` is dropped because the nil-NATS answer is given by the gateway-wide gate before dispatch. `Logging`, `Provisioned`, `Guardrails`, `KB` and `DataSources` are the concrete exported store types, a temporary edge until Ochre exposes capability interfaces for them. The wire-typed action functions stay in the `domains/ochre` root and the SDK-typed adapters move; `authz.go` moves with them, which removes the `domains/ochre` to `ingress/aws/bodyscope` allowlist row, the only allowlist change. The legacy switch cases, the `supportedServices` and `jsonErrorService` entries and the path-based sub-service rewrite in `GetService` are removed, so `ec2` is the only service left on the legacy table; the rewrite's rows are pinned in `awsapi` as `resolveSubservice` and dispatch tests. The registration declares the AWS JSON 1.1 envelope it had and `cmd/aws-model-coverage` is not wired, so no coverage allowlist row is added. Recorded divergences, not changed by this move: Bedrock is restJson1 in AWS but answers with the JSON 1.1 envelope (`application/x-amz-json-1.1`, `__type` body), and CreateGuardrail answers 200 where AWS answers 202; both are tracked by `mulga-xug8b`. No change to `dispatch.Actor` or `dispatch.Invocation`. |
|
||||
| `spinifex#1354` | `spinifex/gateway/ec2.go` (the EC2 action table, `ec2Handler`, `ec2Actions`, the dispatch pipeline), `ec2_idempotency.go`, `ec2_validation.go`, and the EC2 token-bucket fields of `GatewayConfig` | `spinifex/domains/ec2/awsapi` (`Handler`, `NewHandler`, `Deps`, `Quota`, `Dispatch`, `ActionNames`, `InputHasField`), composed by `spinifex/gateway/ec2.go` | Slice 9a of the EC2 move: the capabilities first, with no behaviour change. The EC2 action table, idempotency and validation move to `domains/ec2/awsapi` behind `Deps{NATS, IAM, Quota, ActiveNodes, ExpectedNodes, NodeIDs, AZ, Endpoint, InstanceStatus}`; the gateway keeps the composition and `EC2_Request`, still on the legacy switch until slice 9b registers EC2 through the ADR-0002 seam. `Quota` is declared in `awsapi` because `domains/admission/quota` imports the EC2 action packages and cannot be imported back, and the gateway wraps `quota.Service` to supply the NATS instance-type resolver its retype gate takes. The instance and spot-instance adapters take the two-method `InstanceProfiles` interface (`ResolveInstanceProfile`, `CanonicalResourceARN`) instead of `iamdomain.IAMService`, which pays off the `spotinstance` to `domains/admission/quota` and `domains/iam` allowlist rows. Allowlist changes are 20 deletions and no additions: the 18 `gateway` to `domains/ec2/awsapi/*` rows and those 2. The pipeline characterisation tests committed first (error envelopes, check order, gateway-local actions, idempotent replay, PassRole, ImportKeyPair padding, launch-template expansion before authorization, RunInstances instance-profile refusal order) pass unchanged. Package-local tests drive every routed action through `Dispatch` against a stub daemon. No change to `dispatch.Actor` or `dispatch.Invocation`. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and MaxResults errors, a no-policy user refused with UnauthorizedOperation (including on the gateway-local DescribeRegions), an assumed-role session allowed and refused per its policy, and one RunInstances to running then terminated. |
|
||||
| `spinifex#1355` | `spinifex/gateway/ec2.go` (`EC2_Request` in production, the legacy service switch, its `supportedServices` and JSON tables) and the `Quota` field of `GatewayConfig` | `spinifex/domains/ec2/awsapi` (`NewRegistration`, `OperationInventory`, `ServiceName`), composed by `spinifex/runtime/roles/awsgw` (`ec2.go`: `newEC2Registration`, `advertisedEndpoint`; `ec2quota`) | Slice 9b of the EC2 move: EC2 registers through the ADR-0002 seam as service `ec2` and the legacy gateway switch is empty. The EC2 query-protocol error envelope is kept through `ErrorEnvelopeXML`, and the policy denial match is now `*dispatch.PolicyDenialError`. The characterisation tests committed first also pin the Authorize service name and action, the trace span and audit action, the throttle key (account plus action) and the cluster-unavailable gate, which runs before policy and action lookup for gateway-local and NATS-backed actions alike. They and the 9a tests pass unchanged apart from setup: STS and EC2 test setups now register the services they sign as, and the two tests that claimed a legacy service name are removed because no legacy name remains. `EC2_Request` survives only as a test shim. Allowlist changes are two deletions (the `gateway` to `domains/ec2/awsapi` and `domains/ec2/awsapi/instance` rows) and one addition, the per-service model-coverage row `cmd/aws-model-coverage` to `domains/ec2/awsapi`. `startInstanceStatus` stays in `awsgw.go` because ADR-0007:S6 records the raw instance-state uses by file. No change to `dispatch.Actor` or `dispatch.Invocation`. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a no-policy user refused with UnauthorizedOperation (including on the gateway-local DescribeRegions), an assumed-role session allowed and refused per its policy, and one RunInstances to running then terminated. The gateway request logs carry `service=ec2` and the per-action label. Not exercised live: a throttled request (the default limits are not reachable from one client; the throttle key is pinned by the characterisation tests) and an agent-signed EC2 call. |
|
||||
| `spinifex#1363` | `spinifex/gateway` context keys (`ctxService`, `ctxRegion`, `ctxAction`, the principal and access key id keys, and the unread `ctxQueryArgs`), the duplicate `callerPrincipal` and `principalContext` types, `gateway/caller_arn.go` (`buildCallerARN`), and the direct `policyauth`, `accountadmission` and NATS-connection reads in the auth pipeline | `spinifex/ingress/aws/reqctx` (the request record), `credverify.CallerARN`, and the gateway-consumed capabilities `PolicyEvaluator`, `AccountAdmission` and `Readiness`, composed by `spinifex/runtime/roles/awsgw` (`clusterready`) | Slice 11a of the gateway removal: the request-context record and the ADR-0002:S8 capabilities, with no behaviour change. The record is a copy-on-write value under one context key; the principal is sealed once and a second `WithPrincipal` keeps the first caller. The layering rule `S7_RequestRecordPrivateToIngress` lets only `gateway` and `ingress/aws/*` import `reqctx`, with a fault-injection test, and slice 11c moves the package to `ingress/aws/internal/reqctx` and drops the rule. The caller ARN is built once in `credverify` (root cases kept) and filled on `Principal.CallerARN`. The policy evaluator, account admission (`credential.ErrAccountNotActive`, with `accountadmission.ErrNotActive` an alias) and cluster readiness (`Available() bool`) are interfaces the gateway consumes and fails closed on when nil. The step 1 characterisation tests (throttle against both readiness gates and the policy gate for ECS and IAM, `/admin` skipping `GetService` and the second gate, the nil-connection difference between the two gates, five error writers each minting a request id that differs from the audit record, and the credential-scope region reaching only the span) pass unchanged apart from setup. No allowlist rows change, and there is no change to `dispatch.Actor` or `dispatch.Invocation`. Debt for slice 11c: the test-only `defaultCapabilities` seam (a package variable set by a test `init` so about 60 gateway test files can build a config from `IAMService` alone), the nil-connection case of the early readiness gate, the remaining `NATSConn` readers (`DiscoverActiveNodes` and the field that `awsgw` and the integration harness set), and the `IAMService`, `STSService`, `Config`, `RootCAs` and `InternalSuffix` fields of `GatewayConfig`, which no production code reads. Live on env14: DescribeRegions, DescribeAvailabilityZones and DescribeAccountAttributes, an idempotent CreateVolume replay and its parameter mismatch, the DryRun and InvalidVolume.NotFound errors, a bad access key refused with InvalidClientTokenId on a Query service and InvalidClientTokenIdException on a JSON service, a no-policy user refused with UnauthorizedOperation, and an assumed-role session allowed and refused per its policy. Not exercised live: a suspended-account refusal, a throttled request and the NATS-down 503, which the step 1 tests pin. |
|
||||
| `spinifex#1334` | `spinifex/gateway/oidc_discovery.go` (`eksAccountKV`, the discovery-document and JWKS read and construction) | `spinifex/domains/eks/cluster` (`OIDCPublisher`, `OIDCDiscoveryDocument`), composed by `spinifex/runtime/roles/awsgw` | The EKS domain now reads its own account bucket and builds the per-cluster discovery document and JWKS responses (ADR-0005:S4), and the gateway only maps the unauthenticated `/oidc/eks/...` routes onto an `OIDCPublisher` interface it is handed. Responses are byte-identical, pinned by `oidc_discovery_current_behaviour_test.go`, whose assertions are unchanged. The gateway no longer imports `domains/eks/cluster` in production code. `STS` keeps its own `ClusterJWKSResolver` (`OIDCJWKSReader`), untouched. Recorded debt, not repaired here: a stored JWKS is served verbatim with 200 even when it is not valid JSON. |
|
||||
The EC2 contract row is intentionally different from the directory moves: it creates
|
||||
a compatibility boundary. `ec2.cmd.*` retains its deployed one-token NATS
|
||||
|
||||
@@ -150,13 +150,17 @@ aws ec2 describe-instance-types \
|
||||
--query "InstanceTypes[?InstanceType=='t3.small']"
|
||||
```
|
||||
|
||||
Show capacity (how many of each type can still be launched):
|
||||
List only the types that can still be launched:
|
||||
|
||||
```bash
|
||||
aws ec2 describe-instance-types \
|
||||
--filters Name=capacity,Values=true
|
||||
```
|
||||
|
||||
Each type appears once, whatever its free capacity.
|
||||
This filter is a Spinifex extension and AWS does not declare it, so it is ignored by a region that does not implement it.
|
||||
AWS reports exhaustion at launch instead, with `InsufficientInstanceCapacity`, and so does Spinifex.
|
||||
|
||||
## SSH
|
||||
|
||||
To SSH into instances via their public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster).
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
The console's tokens, primitives and the rules that hold pages to them.
|
||||
The live catalogue is the `/design` route: every token and primitive below rendered in both themes.
|
||||
It is served by the Vite dev server only and is absent from release builds; a release build answers it with "Page not found".
|
||||
The rules on it are proposed as `PROP-CONSOLEUI-001` in `docs/development/proposals/console-design-system/` in the mulga repository, and bind nobody until accepted.
|
||||
The rules are accepted as ADR-0016, `docs/adr/0016-console-design-system.md` in the mulga repository, and every console change follows them; cite a clause as `ADR-0016:S10`.
|
||||
|
||||
Source: `spinifex/runtime/roles/spinifexui/frontend/`.
|
||||
Tokens are in `src/styles.css`, primitives in `src/components/ui/`, and the catalogue in `src/routes/_auth/-components/design/`.
|
||||
@@ -118,14 +118,17 @@ The Go console serves `script-src 'self'; style-src 'self'`.
|
||||
`src/components/ui/` holds the shadcn primitives on Base UI.
|
||||
Added in Phase 1: breadcrumb, checkbox, command (`cmdk`), dialog, hover-card, popover, resizable (`react-resizable-panels`), scroll-area, slider, switch, table, toggle and toggle-group.
|
||||
`button-group` joins buttons edge to edge, as the instance power buttons are.
|
||||
`radio-group` is one of a few exclusive choices inside a form, such as a load balancer's type; a raw `<input type="radio">` is not used.
|
||||
|
||||
A dialog only confirms (ADR-0016:S3): a destructive action, a typed phrase, or a secret shown once.
|
||||
A form never opens in one.
|
||||
Every dialog closes on Escape.
|
||||
⌘↵ or Ctrl+Enter submits it: an `AlertDialog` clicks its action unless that is disabled, and a `Dialog` submits its form, including from a textarea where Enter alone adds a line.
|
||||
Enter in a typed confirmation confirms once the phrase matches.
|
||||
The confirm button shows the shortcut, ⌘↵ on a Mac and Ctrl ↵ elsewhere (`src/lib/shortcuts.ts`), and names it in `aria-keyshortcuts`.
|
||||
|
||||
A disabled control stays visible and shows `cursor: not-allowed`, so it says why it does nothing rather than vanishing.
|
||||
An on/off setting is a `Switch`, orange when on; a `Checkbox` only picks items from a list, such as table rows, subnets or security groups.
|
||||
An on/off setting is a `Switch`, orange when on; a `Checkbox` only selects table rows, and a multi-select picker such as subnets, security groups or load balancer targets is a `SwitchList`.
|
||||
|
||||
`InfoTip` (`src/components/info-tip.tsx`) is UniFi's (i): a small button beside a label that opens a short explanation on hover or click, with `InfoList` for the facts inside it.
|
||||
`PanelField` takes it through `info`, so the login user names the image's distribution and the image names its AMI details.
|
||||
@@ -139,7 +142,7 @@ Charts are uPlot instead (see Charts).
|
||||
|
||||
## Enforcement
|
||||
|
||||
Four lint layers keep pages on the system.
|
||||
Three lint rules and three source tests keep pages on the system.
|
||||
|
||||
| Rule | Where | What it catches |
|
||||
|---|---|---|
|
||||
@@ -152,6 +155,23 @@ Four lint layers keep pages on the system.
|
||||
The table test allows two files, each with its reason: the catalogue's specimen of the `ui/table` primitives, and the RDS parameter editor, an editable grid.
|
||||
Every other list is a `DataTable`, and the test also fails when an allowed file stops building a table so the list cannot go stale.
|
||||
|
||||
`src/routes/design-rules.test.ts` reads the source of every route and shared component and holds one test per rule, named by its ADR-0016 clause, so a failure names the rule it breaks:
|
||||
|
||||
| Clause | Fails when |
|
||||
|---|---|
|
||||
| S3 | a file that imports a dialog renders a form field or `<form>`, or a `create-*` route does not redirect to its list with the panel open |
|
||||
| S4 | a file renders `FormActions` without `FormLock` |
|
||||
| S5 | anything but `DataTable` uses a checkbox |
|
||||
| S6 | a file renders `FormActions` without `CliCommandPanel` |
|
||||
| S9 | a file renders `ResourceView` without `PageHeading` |
|
||||
| S10 | a route declares its own `selected`, `tab` or `create` search field instead of spreading `panelSearchFields` |
|
||||
| S12 | a panel's tabs do not start with Overview, or a retired `DetailCard`, `DetailRow` or `ListCard` comes back |
|
||||
| S13 | a component outside `ui/` renders a raw `<input>`, `<select>` or `<textarea>` |
|
||||
| S15 | a route imports the `ui/alert` primitive instead of using `ErrorBanner` |
|
||||
|
||||
An exception is named in that rule's allowlist with its reason, and the test fails when an allowed file stops needing it.
|
||||
S11, S14 and S16 are held by review, since none has a shape in the source a test can read without false alarms.
|
||||
|
||||
## Shell
|
||||
|
||||
`src/lib/nav.ts` is the one description of the console's navigation.
|
||||
@@ -251,6 +271,14 @@ Modelled on UniFi's console: a click on a row opens the resource in a panel besi
|
||||
- **Loading.** A panel or page still loading shows `PanelSpinner`, a centred spinner on the background, not a grey skeleton block.
|
||||
- **`DetailCard`, `DetailRow` and `ListCard` are retired.** Every resource page uses the panel sections instead.
|
||||
|
||||
### Forms from a resource's panel
|
||||
|
||||
A form that acts on an open resource, such as attaching a volume, scaling a node group or adding a listener, opens in `FormPanel` (`src/components/form-panel.tsx`), never in a dialog.
|
||||
From `lg` it lies over the resource's panel at the same right edge and the create panel's width; below `lg` it covers the page, with a back arrow.
|
||||
It takes Escape from the panel beneath it, takes focus on open and hands it back to the button that opened it, and keeps a failure inside itself.
|
||||
Each such form is its own `*-form.tsx` file with `FormLock`, `CliCommandPanel` and `FormActions`, like a create form.
|
||||
It is held in the details component's state rather than the URL, because a form opened from one row of a tab has no list route to carry it, so Back closes the resource's panel and the form with it.
|
||||
|
||||
## Create flows
|
||||
|
||||
Every create flow opens in the side panel over its list, through `CreatePanel`, and never in a dialog or on a page of its own.
|
||||
|
||||
@@ -41,8 +41,6 @@ resources:
|
||||
- [Limits](#limits)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Spinifex is an open-source infrastructure platform that brings core AWS services to bare-metal, edge, and on-prem environments.
|
||||
@@ -69,8 +67,6 @@ It deletes the reserved IP again when the resource goes away.
|
||||
The key is copied to the nodes over SSH and stored at `/etc/spinifex/vultr/api-key`, readable only by the Spinifex daemon.
|
||||
It is never written to the Terraform state or to the instance metadata.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
You need:
|
||||
@@ -81,7 +77,7 @@ You need:
|
||||
The nodes call the API to manage their addresses, and their addresses do not exist until the deploy creates them.
|
||||
You replace `0.0.0.0/0` with the real addresses in [step 4](#4-restrict-the-api-key).
|
||||
- **OpenTofu or Terraform**, version 1.5 or newer. Install [OpenTofu](https://opentofu.org/docs/intro/install/) or [Terraform](https://developer.hashicorp.com/terraform/install); the deploy script uses whichever it finds, preferring OpenTofu.
|
||||
- **The AWS CLI v2** and `python3`.
|
||||
- **The [AWS CLI v2](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)** and `python3`.
|
||||
- **An SSH key pair** at `~/.ssh/id_ed25519`.
|
||||
Create one with `ssh-keygen -t ed25519` if you do not have one.
|
||||
The deploy installs the public key on every node.
|
||||
@@ -91,8 +87,7 @@ The API key page, with the key and the temporary `0.0.0.0/0` entry in its access
|
||||
<p align="center">
|
||||
<img src="../../.github/assets/images/vultr/vultr-api-keys.png" alt="Vultr API Access page, showing an API key and an access control list that allows 0.0.0.0/0" width="900">
|
||||
</p>
|
||||
|
||||
---
|
||||
<p align="center"><em>Create a Vultr API key in the web console, with 0.0.0.0/0 allowed until the deploy finishes.</em></p>
|
||||
|
||||
## Instructions
|
||||
|
||||
@@ -112,8 +107,6 @@ cd scripts/terraform/vultr-spx
|
||||
> `git switch dev` gives you the deploy script for it, and `--channel dev` in [step 3](#3-deploy) installs the matching Spinifex build, the pre-release published from every change to `dev`.
|
||||
> Once Vultr support is in a release, skip `git switch dev` and pass `--channel latest`.
|
||||
|
||||
---
|
||||
|
||||
## 2. Choose a deployment
|
||||
|
||||
| Use | Nodes | Plan | Cost while running |
|
||||
@@ -130,14 +123,17 @@ Spinifex keeps 2 of its 16 vCPUs for the host and its own services and offers th
|
||||
|
||||
Bare metal takes longer to deliver than a VM, and Vultr bills a bare-metal server from the moment it is created, whether it is running or stopped.
|
||||
|
||||
---
|
||||
|
||||
## 3. Deploy
|
||||
|
||||
`deploy-vultr.sh` is a helper script around Terraform or OpenTofu and the official [Vultr provider](https://registry.terraform.io/providers/vultr/vultr/latest), so the Vultr side of the deployment is infrastructure as code you can read, keep and change in `scripts/terraform/vultr-spx/`.
|
||||
OpenTofu or Terraform creates a private Vultr VPC and the nodes.
|
||||
The script then installs Spinifex on every node over SSH and forms the cluster.
|
||||
|
||||
<p align="center">
|
||||
<img src="../../.github/assets/diagrams/vultr-deployment.svg" alt="Spinifex on Vultr: deploy-vultr.sh builds a Vultr VPC with one or three nodes, which cluster over the private network, call the Vultr API for reserved IPs, and run EC2, load balancers, EKS, ECS, RDS, S3, EBS and IAM across the fleet" width="900">
|
||||
</p>
|
||||
<p align="center"><em>One node or three in a private Vultr VPC, running AWS services across the fleet.</em></p>
|
||||
|
||||
Each node runs **Ubuntu 24.04 LTS** as its host operating system, and Spinifex runs on top of it.
|
||||
Vultr installs the operating system when it creates the node, so there is no host to prepare first.
|
||||
|
||||
@@ -176,8 +172,6 @@ api_key_allowlist=203.0.113.10/32
|
||||
|
||||
A three-node deployment lists `node2` and `node3` as well, and all three addresses in `api_key_allowlist`.
|
||||
|
||||
---
|
||||
|
||||
## 4. Restrict the API key
|
||||
|
||||
Now that the node addresses are known, close the API key to everything else.
|
||||
@@ -189,8 +183,6 @@ In the Vultr portal, open your user menu, then **API Access**, and under **Acces
|
||||
|
||||
`./deploy-vultr.sh status` prints the node addresses again whenever you need them.
|
||||
|
||||
---
|
||||
|
||||
## 5. Check it worked
|
||||
|
||||
```bash
|
||||
@@ -206,6 +198,7 @@ These are the servers that run Spinifex, and the AWS workloads and services you
|
||||
<p align="center">
|
||||
<img src="../../.github/assets/images/vultr/vultr-instances.png" alt="Vultr Instances page, showing Spinifex nodes named spinifex-deployment-nodeN with the spinifex and deployment tags" width="900">
|
||||
</p>
|
||||
<p align="center"><em>Spinifex nodes in the Vultr console, tagged with their deployment.</em></p>
|
||||
|
||||
To log in to a node:
|
||||
|
||||
@@ -220,9 +213,15 @@ For a three-node deployment, check that every node has joined:
|
||||
./deploy-vultr.sh ssh 'spx get nodes'
|
||||
```
|
||||
|
||||
Expect three nodes with `Ready` status.
|
||||
Expect three nodes with `Ready` status:
|
||||
|
||||
---
|
||||
```text
|
||||
NAME │ STATUS │ ROLES │ IP │ REGION │ AZ │ UPTIME │ VMs │ SERVICES
|
||||
──────────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
node1 │ Ready │ nats:leader │ 10.10.8.5 │ us-east-1 │ us-east-1a │ 12m │ 0 │ nats,predastore,viperblock,daemon,awsgw,vpcd,ui
|
||||
node2 │ Ready │ nats:follower │ 10.10.8.4 │ us-east-1 │ us-east-1a │ 11m │ 0 │ nats,predastore,viperblock,daemon,awsgw,vpcd,ui
|
||||
node3 │ Ready │ nats:follower │ 10.10.8.3 │ us-east-1 │ us-east-1a │ 11m │ 0 │ nats,predastore,viperblock,daemon,awsgw,vpcd,ui
|
||||
```
|
||||
|
||||
## 6. Obtain AWS credentials
|
||||
|
||||
@@ -265,11 +264,10 @@ If this returns a list of available instance types, your installation is working
|
||||
<p align="center">
|
||||
<img src="../../.github/assets/spinifex-ui.jpg" alt="Spinifex Web UI" width="900">
|
||||
</p>
|
||||
<p align="center"><em>The Spinifex web console, on port 3000 of node 1.</em></p>
|
||||
|
||||
**Congratulations, Spinifex is installed.**
|
||||
|
||||
---
|
||||
|
||||
## 7. Set up your cluster
|
||||
|
||||
Spinifex is running, but it holds nothing yet: no machine images, no networks, no instances.
|
||||
@@ -277,11 +275,10 @@ Spinifex is running, but it holds nothing yet: no machine images, no networks, n
|
||||
Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance.
|
||||
An instance you launch into a public subnet gets a Vultr reserved IP, which you can see in the Vultr portal under **Network → Reserved IPs**.
|
||||
|
||||
---
|
||||
|
||||
## 8. Test resiliency
|
||||
|
||||
A three-node deployment keeps running when a node fails.
|
||||
A three-node deployment is for high availability: it protects your workloads against any one machine going down.
|
||||
The three nodes form one cluster, a lightweight AWS region on Vultr, and the cluster and everything on it keep running when a node fails.
|
||||
Spinifex restarts the failed node's instances on the surviving nodes, and moves each instance's Vultr reserved IP with it, so its public address keeps working.
|
||||
To watch it happen, run a load-balanced website across the three nodes, then power one of them off.
|
||||
|
||||
@@ -374,8 +371,6 @@ Call another node meanwhile by adding `--endpoint-url https://<node2-address>:99
|
||||
|
||||
When you are done, remove the workbook with `tofu destroy` and the same `-var` options before [removing the deployment](#removing-the-deployment).
|
||||
|
||||
---
|
||||
|
||||
## Removing the deployment
|
||||
|
||||
Remove anything you built on the cluster first, then:
|
||||
@@ -390,21 +385,12 @@ When it finishes, nothing of the deployment is left, and if anything is, it list
|
||||
|
||||
After removing the deployment, remove the node addresses from the API key's **Access Control** list, since Vultr may give them to someone else.
|
||||
|
||||
---
|
||||
|
||||
## Limits
|
||||
|
||||
- **One or three nodes.** Other cluster sizes are not supported.
|
||||
- **16 vCPU per node is the recommended minimum.** Spinifex keeps 1 CPU on a node with up to 8, 2 on one with up to 16 and 4 on one with up to 63, and offers the rest to instances one to one, so a 16-vCPU node offers 14 and an 8-vCPU node 7.
|
||||
- **16 vCPU per node is the recommended minimum.** On a 16-vCPU node, Spinifex keeps 2 vCPU for the OS and its own services, and the other 14 are available for your EC2 instances and AWS services.
|
||||
- **A deleted reserved IP keeps its quota slot for 24 hours**, by Vultr's documentation, so creating and deleting many addresses in a day can reach the account's limit.
|
||||
- **A three-node deployment survives one failed node at a time.**
|
||||
With two of the three nodes down, the cluster loses quorum, and in our test the instances on the remaining node went to `error` and stayed there after a second node returned.
|
||||
- **A recovered instance can share a node with another member of its spread placement group**, when only one node has room for it.
|
||||
It stays there after the failed node returns.
|
||||
- **Disassociating an Elastic IP leaves the instance with no public address**, where AWS would give it a new one.
|
||||
Stopping and starting the instance gets it one.
|
||||
|
||||
---
|
||||
- **A three-node deployment survives one failed node at a time.** The cluster needs a majority of its nodes, two of three, online to keep running.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
// Package gwfix holds runtime-free fixtures for gateway-level tests: SigV4
|
||||
// signing helpers, identity/session doubles and policy builders. It must not
|
||||
// import runtime/ or operator/ so gateway tests stay free of them.
|
||||
package gwfix
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
awsv1 "github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
AccessKey = "AKIAIOSFODNN7EXAMPLE" //nolint:gosec // AWS documentation example key
|
||||
SecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" //nolint:gosec // AWS documentation example secret
|
||||
Region = "us-east-1"
|
||||
Service = "ec2"
|
||||
|
||||
SessionAKID = "ASIATESTSESSIONAAAAA"
|
||||
SessionToken = "wire-session-token-plain"
|
||||
RoleARN = "arn:aws:iam::123456789012:role/test-role"
|
||||
AssumedARN = "arn:aws:sts::123456789012:assumed-role/test-role/test-session"
|
||||
|
||||
PrincipalTypeUser = "user"
|
||||
PrincipalTypeAssumedRole = "assumed-role"
|
||||
PrincipalTypeRoot = "root"
|
||||
)
|
||||
|
||||
// IAM implements iamdomain.IAMService for auth tests. It embeds the interface
|
||||
// so it satisfies the full contract; only the methods the SigV4 path drives are
|
||||
// wired, and any other method nil-panics to surface an unexpected call.
|
||||
type IAM struct {
|
||||
iamdomain.IAMService
|
||||
|
||||
AccessKeys map[string]*iamdomain.AccessKey
|
||||
MasterKey []byte
|
||||
|
||||
// Accounts overrides the status the auth gate reads. An empty map means
|
||||
// every account is ACTIVE.
|
||||
Accounts map[string]*iamdomain.Account
|
||||
|
||||
// GetUserFn overrides the synthetic user record, for a test that cares what
|
||||
// aws:userid resolves to or that it fails to resolve at all.
|
||||
GetUserFn func(accountID string, input *iam.GetUserInput) (*iam.GetUserOutput, error)
|
||||
}
|
||||
|
||||
func (m *IAM) LookupAccessKey(accessKeyID string) (*iamdomain.AccessKey, error) {
|
||||
ak, ok := m.AccessKeys[accessKeyID]
|
||||
if !ok {
|
||||
return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
|
||||
}
|
||||
return ak, nil
|
||||
}
|
||||
|
||||
func (m *IAM) DecryptSecret(ciphertext string) (string, error) {
|
||||
return iamdomain.DecryptSecret(ciphertext, m.MasterKey)
|
||||
}
|
||||
|
||||
func (m *IAM) GetAccount(accountID string) (*iamdomain.Account, error) {
|
||||
if account, ok := m.Accounts[accountID]; ok {
|
||||
return account, nil
|
||||
}
|
||||
return &iamdomain.Account{AccountID: accountID, Status: iamdomain.AccountStatusActive}, nil
|
||||
}
|
||||
|
||||
// GetUser answers with a unique ID derived from the name, so aws:userid resolves
|
||||
// on the authorization path without every test defining a user record.
|
||||
func (m *IAM) GetUser(accountID string, input *iam.GetUserInput) (*iam.GetUserOutput, error) {
|
||||
if m.GetUserFn != nil {
|
||||
return m.GetUserFn(accountID, input)
|
||||
}
|
||||
name := awsv1.StringValue(input.UserName)
|
||||
return &iam.GetUserOutput{User: &iam.User{
|
||||
UserName: awsv1.String(name),
|
||||
UserId: awsv1.String("AIDA" + strings.ToUpper(name)),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// PolicyIAM adds configurable user and role policy lookups to IAM.
|
||||
type PolicyIAM struct {
|
||||
IAM
|
||||
|
||||
GetUserPoliciesFn func(accountID, userName string) ([]iamdomain.PolicyDocument, error)
|
||||
GetRolePoliciesFn func(accountID, roleName string) ([]iamdomain.PolicyDocument, error)
|
||||
}
|
||||
|
||||
func (m *PolicyIAM) GetUserPolicies(accountID, userName string) ([]iamdomain.PolicyDocument, error) {
|
||||
if m.GetUserPoliciesFn != nil {
|
||||
return m.GetUserPoliciesFn(accountID, userName)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *PolicyIAM) GetRolePolicies(accountID, roleName string) ([]iamdomain.PolicyDocument, error) {
|
||||
if m.GetRolePoliciesFn != nil {
|
||||
return m.GetRolePoliciesFn(accountID, roleName)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// ErrorLookupIAM returns a configurable error from LookupAccessKey.
|
||||
type ErrorLookupIAM struct {
|
||||
IAM
|
||||
|
||||
Err error
|
||||
}
|
||||
|
||||
func (m *ErrorLookupIAM) LookupAccessKey(_ string) (*iamdomain.AccessKey, error) {
|
||||
return nil, m.Err
|
||||
}
|
||||
|
||||
// MasterKey is a fixed 32-byte key for deterministic secret encryption in tests.
|
||||
var MasterKey = bytes.Repeat([]byte{0x5a}, 32)
|
||||
|
||||
// STS implements the session-credential half of stsdomain.STSService.
|
||||
type STS struct {
|
||||
stsdomain.STSService
|
||||
|
||||
Sessions map[string]*stsdomain.SessionCredential
|
||||
Tokens map[string]string // AKID → plaintext wire token for HMAC equivalence
|
||||
LookupErr error
|
||||
Lookups atomic.Int32 // counts LookupSessionCredential calls for negative-side-effect assertions
|
||||
|
||||
PrincipalErr error // verdict or fault the continuity check returns
|
||||
PrincipalChecks atomic.Int32 // counts VerifySessionPrincipal calls, to pin its ordering
|
||||
}
|
||||
|
||||
func (m *STS) LookupSessionCredential(accessKeyID string) (*stsdomain.SessionCredential, error) {
|
||||
m.Lookups.Add(1)
|
||||
if m.LookupErr != nil {
|
||||
return nil, m.LookupErr
|
||||
}
|
||||
if !strings.HasPrefix(accessKeyID, "ASIA") {
|
||||
return nil, nil
|
||||
}
|
||||
cred, ok := m.Sessions[accessKeyID]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
return cred, nil
|
||||
}
|
||||
|
||||
// VerifySessionToken uses plaintext equality rather than HMAC to keep tests
|
||||
// simple while preserving the contract shape.
|
||||
func (m *STS) VerifySessionToken(cred *stsdomain.SessionCredential, wireToken string) bool {
|
||||
if cred == nil {
|
||||
return false
|
||||
}
|
||||
want, ok := m.Tokens[cred.AccessKeyID]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return want == wireToken
|
||||
}
|
||||
|
||||
// VerifySessionPrincipal defaults to "still the same principal"; PrincipalErr
|
||||
// drives the rejection-verdict and dependency-fault branches.
|
||||
func (m *STS) VerifySessionPrincipal(cred *stsdomain.SessionCredential) (*stsdomain.SessionPrincipal, error) {
|
||||
m.PrincipalChecks.Add(1)
|
||||
if m.PrincipalErr != nil {
|
||||
return nil, m.PrincipalErr
|
||||
}
|
||||
return StubSessionPrincipal(cred)
|
||||
}
|
||||
|
||||
// StubSessionPrincipal is the shared "principal unchanged" answer for the
|
||||
// session doubles. It derives RoleName from the underlying role ARN rather than
|
||||
// the session name, because that is what the real implementation returns: a
|
||||
// stub echoing SessionName would let a door read the wrong field and stay green.
|
||||
func StubSessionPrincipal(cred *stsdomain.SessionCredential) (*stsdomain.SessionPrincipal, error) {
|
||||
if cred == nil {
|
||||
return nil, errors.New("nil session credential")
|
||||
}
|
||||
roleName := cred.UnderlyingRoleARN
|
||||
if idx := strings.LastIndex(roleName, "/"); idx >= 0 {
|
||||
roleName = roleName[idx+1:]
|
||||
}
|
||||
return &stsdomain.SessionPrincipal{
|
||||
UserID: cred.UserID,
|
||||
RoleID: cred.RoleID,
|
||||
RoleName: roleName,
|
||||
RoleARN: cred.UnderlyingRoleARN,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SignRequest signs req in place with the aws-sdk-go-v2 v4 signer — the same
|
||||
// path production callers use via gwsign — so the gateway verifies a real SDK
|
||||
// signature. body must match what the middleware reads from r.Body so the payload
|
||||
// hash agrees. An optional signingTime pins the clock for skew tests.
|
||||
func SignRequest(t *testing.T, req *http.Request, body []byte, accessKey, secret string, signingTime ...time.Time) {
|
||||
t.Helper()
|
||||
when := time.Now().UTC()
|
||||
if len(signingTime) > 0 {
|
||||
when = signingTime[0]
|
||||
}
|
||||
SignRequestAs(t, req, body, accessKey, secret, Service, when)
|
||||
}
|
||||
|
||||
// SignRequestAs signs with an explicit credential-scope service, so a test
|
||||
// can drive a scope the gateway does not serve.
|
||||
func SignRequestAs(t *testing.T, req *http.Request, body []byte, accessKey, secret, service string, when time.Time) {
|
||||
t.Helper()
|
||||
sum := sha256.Sum256(body)
|
||||
payloadHash := hex.EncodeToString(sum[:])
|
||||
// gwsign sets this header so the SDK signs it; sigv4 reproduces it when
|
||||
// reconstructing the canonical request.
|
||||
req.Header.Set("X-Amz-Content-Sha256", payloadHash)
|
||||
require.NoError(t, v4.NewSigner().SignHTTP(context.Background(),
|
||||
aws.Credentials{AccessKeyID: accessKey, SecretAccessKey: secret},
|
||||
req, payloadHash, service, Region, when))
|
||||
}
|
||||
|
||||
// SignSessionRequest sets X-Amz-Security-Token before signing so it's included
|
||||
// in SignedHeaders, matching what the AWS CLI/SDK do with session credentials.
|
||||
func SignSessionRequest(t *testing.T, req *http.Request, body []byte, accessKey, secret, sessionToken string) {
|
||||
t.Helper()
|
||||
req.Header.Set("X-Amz-Security-Token", sessionToken)
|
||||
SignRequest(t, req, body, accessKey, secret)
|
||||
}
|
||||
|
||||
// AllowPolicy returns a single-statement Allow policy granting action on "*".
|
||||
func AllowPolicy(action string) []iamdomain.PolicyDocument {
|
||||
return AllowPolicyResource(action, "*")
|
||||
}
|
||||
|
||||
// AllowPolicyResource returns a single-statement Allow policy for action on resource.
|
||||
func AllowPolicyResource(action, resource string) []iamdomain.PolicyDocument {
|
||||
return []iamdomain.PolicyDocument{{
|
||||
Version: "2012-10-17",
|
||||
Statement: []iamdomain.Statement{{
|
||||
Effect: "Allow",
|
||||
Action: iamdomain.StringOrArr{action},
|
||||
Resource: iamdomain.StringOrArr{resource},
|
||||
}},
|
||||
}}
|
||||
}
|
||||
|
||||
// AssumedRoleSessionCred builds an assumed-role SessionCredential.
|
||||
// SecretEncrypted is filled in by the gateway builder.
|
||||
func AssumedRoleSessionCred(sessionName, underlyingRoleARN, accountID string) *stsdomain.SessionCredential {
|
||||
return &stsdomain.SessionCredential{
|
||||
AccessKeyID: SessionAKID,
|
||||
AccountID: accountID,
|
||||
PrincipalType: PrincipalTypeAssumedRole,
|
||||
AssumedRoleARN: "arn:aws:sts::" + accountID + ":assumed-role/role/" + sessionName,
|
||||
UnderlyingRoleARN: underlyingRoleARN,
|
||||
SessionName: sessionName,
|
||||
ExpiresAt: time.Now().UTC().Add(time.Hour),
|
||||
CreatedAt: time.Now().UTC().Add(-time.Minute),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package gwfix
|
||||
|
||||
import (
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
|
||||
)
|
||||
|
||||
var _ credential.Verifier = (*Verifier)(nil)
|
||||
|
||||
// Verifier authenticates a long-lived access key as the IAM user the key names.
|
||||
// It decides nothing else: no status, session or account check, so a test that
|
||||
// asserts a credential verdict must use the real verifier instead.
|
||||
type Verifier struct {
|
||||
IAM iamdomain.IAMService
|
||||
}
|
||||
|
||||
func (v *Verifier) Authenticate(req credential.Request, verifySig func(secret string) error) (credential.Authenticated, *credential.Failure) {
|
||||
ak, err := v.IAM.LookupAccessKey(req.AccessKeyID)
|
||||
if err != nil {
|
||||
return credential.Authenticated{}, &credential.Failure{Code: awserrors.ErrorInvalidClientTokenId}
|
||||
}
|
||||
secret, err := v.IAM.DecryptSecret(ak.SecretAccessKey)
|
||||
if err != nil {
|
||||
return credential.Authenticated{}, &credential.Failure{Code: awserrors.ErrorInternalError}
|
||||
}
|
||||
if err := verifySig(secret); err != nil {
|
||||
return credential.Authenticated{}, &credential.Failure{Code: awserrors.ErrorSignatureDoesNotMatch, SignatureMismatch: true}
|
||||
}
|
||||
return credential.Authenticated{AccountID: ak.AccountID, Handle: ak}, nil
|
||||
}
|
||||
|
||||
func (v *Verifier) Resolve(_ credential.Request, auth credential.Authenticated) (credential.Principal, *credential.Failure) {
|
||||
ak, ok := auth.Handle.(*iamdomain.AccessKey)
|
||||
if !ok {
|
||||
return credential.Principal{}, &credential.Failure{Code: awserrors.ErrorInternalError}
|
||||
}
|
||||
return credential.Principal{
|
||||
Identity: ak.UserName,
|
||||
AccountID: ak.AccountID,
|
||||
PrincipalType: PrincipalTypeUser,
|
||||
CallerARN: "arn:aws:iam::" + ak.AccountID + ":user/" + ak.UserName,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package gwtest
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// AssertDenied matches on the resolved code, not the message: a policy denial
|
||||
// names the principal, action and resource ahead of the code.
|
||||
func AssertDenied(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
code, ok := awserrors.ResolveErrorCode(err)
|
||||
require.True(t, ok, "unresolvable error code: %v", err)
|
||||
assert.Equal(t, awserrors.ErrorAccessDenied, code)
|
||||
}
|
||||
|
||||
// AssertUnauthorized asserts EC2's policy denial, UnauthorizedOperation.
|
||||
func AssertUnauthorized(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorUnauthorizedOperation, err.Error())
|
||||
}
|
||||
|
||||
// AssertPermitted asserts the policy gate passed. The request then fails on the
|
||||
// absent NATS connection rather than on authorization, so a denial cannot hide
|
||||
// behind a not-found.
|
||||
func AssertPermitted(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
}
|
||||
|
||||
// AssertGatePassed asserts the policy gate ran and allowed the request, whatever
|
||||
// the handler then did with it. A denial cannot hide behind a later error.
|
||||
func (h *Harness) AssertGatePassed(t *testing.T) {
|
||||
t.Helper()
|
||||
require.True(t, h.GateCalled, "policy gate was never consulted")
|
||||
require.NoError(t, h.LastGateErr)
|
||||
}
|
||||
|
||||
// AssertNotDenied asserts the request failed, but not with a policy denial.
|
||||
func AssertNotDenied(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
code, _ := awserrors.ResolveErrorCode(err)
|
||||
assert.NotEqual(t, awserrors.ErrorAccessDenied, code)
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
// Package gwtest builds the invocation a registered AWS service receives from
|
||||
// the ingress pipeline, so a service's handler tests run in the service's own
|
||||
// package with real policy evaluation and no gateway.
|
||||
package gwtest
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/pipeline"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/policyauth"
|
||||
)
|
||||
|
||||
const (
|
||||
// AccountID is the account the default caller belongs to.
|
||||
AccountID = "123456789012"
|
||||
// Region is the region the default harness serves.
|
||||
Region = "ap-southeast-2"
|
||||
)
|
||||
|
||||
// policyIAM resolves fixed policy documents for every user and role. Only the
|
||||
// two methods the policy evaluator reads are implemented; any other call on
|
||||
// the embedded nil interface panics, which is the point.
|
||||
type policyIAM struct {
|
||||
iamdomain.IAMService
|
||||
|
||||
resolve func(accountID, name string) ([]iamdomain.PolicyDocument, error)
|
||||
}
|
||||
|
||||
func (p policyIAM) GetUserPolicies(accountID, name string) ([]iamdomain.PolicyDocument, error) {
|
||||
return p.resolve(accountID, name)
|
||||
}
|
||||
|
||||
func (p policyIAM) GetRolePolicies(accountID, name string) ([]iamdomain.PolicyDocument, error) {
|
||||
return p.resolve(accountID, name)
|
||||
}
|
||||
|
||||
// Statement is one policy statement naming a single action and resource.
|
||||
func Statement(effect, action, resource string) iamdomain.Statement {
|
||||
return iamdomain.Statement{
|
||||
Effect: effect,
|
||||
Action: iamdomain.StringOrArr{action},
|
||||
Resource: iamdomain.StringOrArr{resource},
|
||||
}
|
||||
}
|
||||
|
||||
// Harness invokes a registration as the pipeline would for one authenticated
|
||||
// user, authorizing through the real policy evaluator.
|
||||
type Harness struct {
|
||||
// AccountID is the caller's account as the invocation carries it.
|
||||
AccountID string
|
||||
Region string
|
||||
Policy pipeline.PolicyEvaluator
|
||||
// Caller overrides the default caller, an IAM user named alice.
|
||||
Caller *credential.Principal
|
||||
// LastGateErr is the verdict of the most recent Authorize call, and
|
||||
// GateCalled whether the last Dispatch reached the gate at all.
|
||||
LastGateErr error
|
||||
GateCalled bool
|
||||
}
|
||||
|
||||
// Scoped returns a harness whose caller holds one policy of statements.
|
||||
func Scoped(statements ...iamdomain.Statement) *Harness {
|
||||
return FromDocs([]iamdomain.PolicyDocument{{Version: "2012-10-17", Statement: statements}})
|
||||
}
|
||||
|
||||
// FromDocs returns a harness whose caller holds exactly these policy documents.
|
||||
func FromDocs(docs []iamdomain.PolicyDocument) *Harness {
|
||||
return FromResolver(func(_, _ string) ([]iamdomain.PolicyDocument, error) { return docs, nil })
|
||||
}
|
||||
|
||||
// FromResolver returns a harness whose caller's policies come from resolve,
|
||||
// which sees every user or role lookup, so a test can count them.
|
||||
func FromResolver(resolve func(accountID, name string) ([]iamdomain.PolicyDocument, error)) *Harness {
|
||||
return &Harness{
|
||||
AccountID: AccountID,
|
||||
Region: Region,
|
||||
Policy: policyauth.New(policyIAM{resolve: resolve}),
|
||||
}
|
||||
}
|
||||
|
||||
// AllowAll returns a harness whose caller may do anything.
|
||||
func AllowAll() *Harness {
|
||||
return Scoped(Statement("Allow", "*", "*"))
|
||||
}
|
||||
|
||||
// Dispatch hands r to reg as an invocation for service, authorizing through
|
||||
// the pipeline's own gate over h's policy.
|
||||
func (h *Harness) Dispatch(reg dispatch.Registration, service string, w http.ResponseWriter, r *http.Request) error {
|
||||
caller := credential.Principal{
|
||||
AccountID: h.AccountID,
|
||||
Identity: "alice",
|
||||
PrincipalType: stsdomain.PrincipalTypeUser,
|
||||
CallerARN: "arn:aws:iam::" + h.AccountID + ":user/alice",
|
||||
}
|
||||
if h.Caller != nil {
|
||||
caller = *h.Caller
|
||||
}
|
||||
h.LastGateErr, h.GateCalled = nil, false
|
||||
return reg.Dispatch(w, dispatch.Invocation{
|
||||
Request: r,
|
||||
AccountID: h.AccountID,
|
||||
Service: service,
|
||||
Region: h.Region,
|
||||
Actor: func() (dispatch.Actor, error) { return pipeline.ActorFor(caller, "") },
|
||||
Authorize: func(svc, action string, resources []string, actionKeys iampolicy.ConditionKeys) error {
|
||||
h.GateCalled = true
|
||||
h.LastGateErr = pipeline.Authorize(h.Policy, r, caller, true, svc, action, resources, actionKeys)
|
||||
return h.LastGateErr
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// PassRoleTo is a PassRole statement on every role, conditioned on the
|
||||
// consuming service the way AWS's documented examples scope it.
|
||||
func PassRoleTo(effect, operator, service string) iamdomain.Statement {
|
||||
s := Statement(effect, "iam:PassRole", "*")
|
||||
s.Condition = map[string]map[string]iamdomain.ConditionValue{
|
||||
operator: {iampolicy.KeyPassedToService: {service}},
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -111,6 +111,8 @@ check "config listens only on the supplied address" \
|
||||
check "config admits only the supplied user NKey" 'grep -q "users = \[ { nkey: \"$NKEY\" } \]" "$T/run/nats.conf"'
|
||||
check "config syncs JetStream on the data disk" \
|
||||
'grep -q "store_dir: \"$T/data\"" "$T/run/nats.conf" && grep -q "sync_interval: always" "$T/run/nats.conf"'
|
||||
check "config admits a full SQS message (2MiB max_payload)" \
|
||||
'grep -c "max_payload" "$T/run/nats.conf" | grep -qx 1 && grep -qx "max_payload: 2MiB" "$T/run/nats.conf"'
|
||||
check "config is tested by the server before ready" 'grep -qx "nats-server -t -c $T/run/nats.conf" "$T/calls.log"'
|
||||
check "no auth material on the console" '! grep -q "$NKEY" "$T/console"'
|
||||
check "prepare reports the disk and config, not a listener" 'grep -q "prepared: $T/dev/vdb mounted" "$T/console" && ! grep -qi "listening" "$T/console"'
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
. "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// acmRequest builds the request an ACM JSON 1.1 client sends.
|
||||
func acmRequest(target, body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
if target != "" {
|
||||
req.Header.Set("X-Amz-Target", target)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// dispatchACM drives the registration with no NATS connection. A permitted request
|
||||
// therefore fails at the NATS-availability guard rather than on authorization,
|
||||
// which is what proves the policy check ran ahead of the certificate existing.
|
||||
func dispatchACM(t *testing.T, h *gwtest.Harness, action, body string) error {
|
||||
t.Helper()
|
||||
return h.Dispatch(NewRegistration(Deps{}), ServiceName, httptest.NewRecorder(), acmRequest("CertificateManager."+action, body))
|
||||
}
|
||||
|
||||
func TestACMRequest_MissingTarget(t *testing.T) {
|
||||
err := gwtest.AllowAll().Dispatch(NewRegistration(Deps{}), ServiceName, httptest.NewRecorder(), acmRequest("", ""))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMissingAction, err.Error())
|
||||
}
|
||||
|
||||
func TestACMRequest_UnknownAction(t *testing.T) {
|
||||
err := dispatchACM(t, gwtest.AllowAll(), "BogusAction", "{}")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidAction, err.Error())
|
||||
}
|
||||
|
||||
// A known action with no NATS connection passes routing + policy and fails at
|
||||
// the NATS-availability guard.
|
||||
func TestACMRequest_KnownActionNoNATS(t *testing.T) {
|
||||
err := dispatchACM(t, gwtest.AllowAll(), "ListCertificates", "{}")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
}
|
||||
|
||||
func acmCertARN(id string) string {
|
||||
return "arn:aws:acm:" + gwtest.Region + ":" + gwtest.AccountID + ":certificate/" + id
|
||||
}
|
||||
|
||||
// TestACMRequest_ScopedDenyFires is the bypass this work closes. An operator
|
||||
// fences a production certificate; before the resolver the fence was inert and
|
||||
// DeleteCertificate against it was permitted with nothing logged.
|
||||
func TestACMRequest_ScopedDenyFires(t *testing.T) {
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:*", "*"),
|
||||
gwtest.Statement("Deny", "acm:DeleteCertificate", acmCertARN("prod-1111")),
|
||||
)
|
||||
|
||||
gwtest.AssertDenied(t, dispatchACM(t, gw, "DeleteCertificate", `{"CertificateArn":"`+acmCertARN("prod-1111")+`"}`))
|
||||
gwtest.AssertPermitted(t, dispatchACM(t, gw, "DeleteCertificate", `{"CertificateArn":"`+acmCertARN("dev-2222")+`"}`))
|
||||
}
|
||||
|
||||
// TestACMRequest_ScopedAllowGrants is the other half: a least-privilege policy
|
||||
// used to deny everything, so the only working policy shape was Resource "*".
|
||||
// The sibling is denied with AccessDenied and not a ResourceNotFound, which is
|
||||
// what proves the gate runs ahead of existence.
|
||||
func TestACMRequest_ScopedAllowGrants(t *testing.T) {
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:DescribeCertificate", acmCertARN("dev-2222")),
|
||||
)
|
||||
|
||||
gwtest.AssertPermitted(t, dispatchACM(t, gw, "DescribeCertificate", `{"CertificateArn":"`+acmCertARN("dev-2222")+`"}`))
|
||||
gwtest.AssertDenied(t, dispatchACM(t, gw, "DescribeCertificate", `{"CertificateArn":"`+acmCertARN("prod-1111")+`"}`))
|
||||
}
|
||||
|
||||
// A caller-supplied region or account in the ARN would let a request slide out
|
||||
// from under a Deny scoped to the real one.
|
||||
func TestACMRequest_CallerSuppliedAnchorIsIgnored(t *testing.T) {
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:*", "*"),
|
||||
gwtest.Statement("Deny", "acm:DeleteCertificate", acmCertARN("prod-1111")),
|
||||
)
|
||||
|
||||
gwtest.AssertDenied(t, dispatchACM(t, gw, "DeleteCertificate",
|
||||
`{"CertificateArn":"arn:aws:acm:us-east-1:999999999999:certificate/prod-1111"}`))
|
||||
}
|
||||
|
||||
// ImportCertificate creates unless it names a certificate to replace, so a grant
|
||||
// on the type covers the create and a fence on one certificate covers only the
|
||||
// re-import that names it.
|
||||
func TestACMRequest_ImportResolvesBothShapes(t *testing.T) {
|
||||
create := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:ImportCertificate", acmCertARN("*")),
|
||||
)
|
||||
gwtest.AssertPermitted(t, dispatchACM(t, create, "ImportCertificate", `{"Certificate":"pem"}`))
|
||||
|
||||
fenced := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:*", "*"),
|
||||
gwtest.Statement("Deny", "acm:ImportCertificate", acmCertARN("prod-1111")),
|
||||
)
|
||||
gwtest.AssertDenied(t, dispatchACM(t, fenced, "ImportCertificate",
|
||||
`{"CertificateArn":"`+acmCertARN("prod-1111")+`","Certificate":"pem"}`))
|
||||
gwtest.AssertPermitted(t, dispatchACM(t, fenced, "ImportCertificate", `{"Certificate":"pem"}`))
|
||||
}
|
||||
|
||||
// RequestCertificate has no id at gate time, so it resolves the type.
|
||||
func TestACMRequest_RequestCertificateResolvesTheType(t *testing.T) {
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:RequestCertificate", acmCertARN("*")),
|
||||
)
|
||||
gwtest.AssertPermitted(t, dispatchACM(t, gw, "RequestCertificate", `{"DomainName":"example.com"}`))
|
||||
|
||||
scoped := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:RequestCertificate", acmCertARN("prod-1111")),
|
||||
)
|
||||
gwtest.AssertDenied(t, dispatchACM(t, scoped, "RequestCertificate", `{"DomainName":"example.com"}`))
|
||||
}
|
||||
|
||||
// A grant scoped to one certificate does not become a licence to mint new ones
|
||||
// by naming that certificate in a field RequestCertificate discards.
|
||||
func TestACMRequest_RequestCertificateIgnoresACertificateArnInTheBody(t *testing.T) {
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:*", acmCertARN("team-a-1")),
|
||||
)
|
||||
|
||||
gwtest.AssertDenied(t, dispatchACM(t, gw, "RequestCertificate",
|
||||
`{"DomainName":"evil.example","CertificateArn":"`+acmCertARN("team-a-1")+`"}`))
|
||||
}
|
||||
|
||||
// ListCertificates is account-level, so a grant scoped to one certificate does
|
||||
// not reach it.
|
||||
func TestACMRequest_ListIsAccountLevel(t *testing.T) {
|
||||
scoped := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:ListCertificates", acmCertARN("dev-2222")),
|
||||
)
|
||||
gwtest.AssertDenied(t, dispatchACM(t, scoped, "ListCertificates", `{}`))
|
||||
|
||||
wide := gwtest.Scoped(gwtest.Statement("Allow", "acm:ListCertificates", "*"))
|
||||
gwtest.AssertPermitted(t, dispatchACM(t, wide, "ListCertificates", `{}`))
|
||||
}
|
||||
|
||||
// The property the per-service rollout rests on: every policy that works today
|
||||
// carries Resource "*", and passing a real ARN cannot turn it into a denial.
|
||||
func TestACMRequest_WildcardPolicyStillPermitsEveryAction(t *testing.T) {
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "acm:*", "*"))
|
||||
|
||||
for _, action := range ScopedActions() {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
gwtest.AssertPermitted(t, dispatchACM(t, gw, action, `{"CertificateArn":"`+acmCertARN("aaaa-1111")+`"}`))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An absent identifier authorizes account-wide, so the handler still reports its
|
||||
// own validation fault rather than the gate converting it into a denial.
|
||||
func TestACMRequest_AbsentIdentifierStaysTheHandlersFault(t *testing.T) {
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:*", "*"),
|
||||
gwtest.Statement("Deny", "acm:DeleteCertificate", acmCertARN("prod-1111")),
|
||||
)
|
||||
|
||||
gwtest.AssertPermitted(t, dispatchACM(t, gw, "DeleteCertificate", `{}`))
|
||||
}
|
||||
|
||||
// A body the gate cannot parse authorizes account-wide for the same reason.
|
||||
func TestACMRequest_UnparseableBodyStaysTheHandlersFault(t *testing.T) {
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "acm:*", "*"),
|
||||
gwtest.Statement("Deny", "acm:DeleteCertificate", acmCertARN("prod-1111")),
|
||||
)
|
||||
|
||||
gwtest.AssertPermitted(t, dispatchACM(t, gw, "DeleteCertificate", "{not json"))
|
||||
}
|
||||
|
||||
// A body spelling one field two ways is rejected rather than authorized against
|
||||
// whichever spelling the case fold happened to keep.
|
||||
func TestACMRequest_FieldSpelledTwoWaysIsRejected(t *testing.T) {
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "acm:*", "*"))
|
||||
|
||||
// Repeated: the fold's disagreement is random, the rejection must not be.
|
||||
for range 50 {
|
||||
err := dispatchACM(t, gw, "DeleteCertificate",
|
||||
`{"CertificateArn":"`+acmCertARN("dev-2222")+`","certificateArn":"`+acmCertARN("prod-1111")+`"}`)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// A body past the signed path's cap cannot be used to bypass the gate or to
|
||||
// exhaust memory: it is rejected before either the gate or the handler.
|
||||
func TestACMRequest_OversizedBodyIsRejected(t *testing.T) {
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "acm:*", "*"))
|
||||
body := `{"CertificateArn":"` + strings.Repeat("a", sigv4.MaxPayloadLen) + `"}`
|
||||
|
||||
err := dispatchACM(t, gw, "DeleteCertificate", body)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorRequestEntityTooLarge, err.Error())
|
||||
}
|
||||
|
||||
// The account-ID guard moved above the policy gate, which used to reach its own
|
||||
// InternalError first. That is the code the caller has always seen.
|
||||
func TestACMRequest_MissingAccountID(t *testing.T) {
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "acm:*", "*"))
|
||||
gw.AccountID = ""
|
||||
|
||||
err := dispatchACM(t, gw, "ListCertificates", `{}`)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInternalError, err.Error())
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
ec2awsapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// dispatchEC2 drives the registration with no NATS connection. A permitted request
|
||||
// therefore reaches the handler and fails there, which is what proves the policy
|
||||
// check ran ahead of the resource existing at all.
|
||||
func dispatchEC2(t *testing.T, h *gwtest.Harness, body string) error {
|
||||
t.Helper()
|
||||
return dispatchEC2With(t, h, ec2awsapi.NewRegistration(ec2awsapi.Deps{}), body)
|
||||
}
|
||||
|
||||
func dispatchEC2With(t *testing.T, h *gwtest.Harness, reg dispatch.Registration, body string) error {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
return h.Dispatch(reg, ec2awsapi.ServiceName, httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
// TestEC2Request_ScopedDenyFires is the bypass this work closes. An operator
|
||||
// fences a production instance; before the resolver the fence was inert and
|
||||
// TerminateInstances against it was permitted with nothing logged.
|
||||
func TestEC2Request_ScopedDenyFires(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:TerminateInstances", "arn:aws:ec2:*:*:instance/i-prod"),
|
||||
)
|
||||
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=TerminateInstances&InstanceId.1=i-prod"))
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=TerminateInstances&InstanceId.1=i-dev"))
|
||||
}
|
||||
|
||||
// TestEC2Request_ScopedAllowGrants is the other half: a least-privilege policy
|
||||
// used to deny everything, so the only working policy shape was Resource "*".
|
||||
func TestEC2Request_ScopedAllowGrants(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:TerminateInstances", "arn:aws:ec2:*:*:instance/i-dev"),
|
||||
)
|
||||
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=TerminateInstances&InstanceId.1=i-dev"))
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=TerminateInstances&InstanceId.1=i-prod"))
|
||||
}
|
||||
|
||||
// TestEC2Request_DenyOneMemberFailsTheBatch pins the AWS combining rule: a batch
|
||||
// containing one fenced instance fails whole rather than terminating the rest.
|
||||
func TestEC2Request_DenyOneMemberFailsTheBatch(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:TerminateInstances", "arn:aws:ec2:*:*:instance/i-prod"),
|
||||
)
|
||||
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h,
|
||||
"Action=TerminateInstances&InstanceId.1=i-dev&InstanceId.2=i-prod&InstanceId.3=i-other"))
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h,
|
||||
"Action=TerminateInstances&InstanceId.1=i-dev&InstanceId.2=i-other"))
|
||||
}
|
||||
|
||||
// TestEC2Request_MultiResourceParity covers a resource beyond the primary one.
|
||||
// A tenant constraining launches by instance would otherwise have the AMI and
|
||||
// subnet unchecked, which is a smaller instance of the same bug.
|
||||
func TestEC2Request_MultiResourceParity(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:RunInstances", "arn:aws:ec2:*:*:image/ami-restricted"),
|
||||
gwtest.Statement("Deny", "ec2:RunInstances", "arn:aws:ec2:*:*:subnet/subnet-restricted"),
|
||||
)
|
||||
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=RunInstances&ImageId=ami-restricted&MinCount=1&MaxCount=1"))
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h,
|
||||
"Action=RunInstances&ImageId=ami-allowed&NetworkInterface.1.SubnetId=subnet-restricted&MinCount=1&MaxCount=1"))
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=RunInstances&ImageId=ami-allowed&MinCount=1&MaxCount=1"))
|
||||
}
|
||||
|
||||
func TestEC2Request_UsesCanonicalParsedResources(t *testing.T) {
|
||||
t.Run("lower camel location name", func(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:ModifyInstanceAttribute", "arn:aws:ec2:*:*:instance/i-prod"),
|
||||
)
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=ModifyInstanceAttribute&instanceId=i-prod"))
|
||||
})
|
||||
|
||||
t.Run("location name list wrapper", func(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:TerminateInstances", "arn:aws:ec2:*:*:instance/i-prod"),
|
||||
)
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=TerminateInstances&InstanceId.InstanceId.1=i-prod"))
|
||||
})
|
||||
|
||||
t.Run("handler field precedence", func(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:DeleteKeyPair", "arn:aws:ec2:*:*:key-pair/key-prod"),
|
||||
)
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=DeleteKeyPair&KeyName=dev&KeyPairId=key-prod"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestEC2Request_CreateRouteChecksNATGateway(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:CreateRoute", "arn:aws:ec2:*:*:natgateway/nat-prod"),
|
||||
)
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=CreateRoute&RouteTableId=rtb-dev&NatGatewayId=nat-prod"))
|
||||
}
|
||||
|
||||
func TestEC2Request_RequestSpotInstancesChecksLaunchResources(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:RequestSpotInstances", "arn:aws:ec2:*:*:subnet/subnet-prod"),
|
||||
)
|
||||
body := "Action=RequestSpotInstances&LaunchSpecification.ImageId=ami-1&" +
|
||||
"LaunchSpecification.InstanceType=t3.micro&LaunchSpecification.SubnetId=subnet-prod"
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, body))
|
||||
}
|
||||
|
||||
// TestEC2Request_GetSecurityGroupsForVpcIsVpcScoped pins the one read action
|
||||
// that names a resource: registered unscoped, a policy fencing one VPC would be
|
||||
// bypassed by asking about another.
|
||||
func TestEC2Request_GetSecurityGroupsForVpcIsVpcScoped(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:GetSecurityGroupsForVpc", "arn:aws:ec2:*:*:vpc/vpc-prod"),
|
||||
)
|
||||
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=GetSecurityGroupsForVpc&VpcId=vpc-prod"))
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=GetSecurityGroupsForVpc&VpcId=vpc-dev"))
|
||||
}
|
||||
|
||||
// TestEC2Request_UpdateSecurityGroupRuleDescriptionsIsGroupScoped pins that
|
||||
// both description writers name the group they mutate: registered unscoped, a
|
||||
// policy fencing one security group would not apply to them.
|
||||
func TestEC2Request_UpdateSecurityGroupRuleDescriptionsIsGroupScoped(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:UpdateSecurityGroupRuleDescriptions*", "arn:aws:ec2:*:*:security-group/sg-prod"),
|
||||
)
|
||||
|
||||
for _, action := range []string{"UpdateSecurityGroupRuleDescriptionsIngress", "UpdateSecurityGroupRuleDescriptionsEgress"} {
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action="+action+"&GroupId=sg-prod"))
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action="+action+"&GroupId=sg-dev"))
|
||||
}
|
||||
}
|
||||
|
||||
// TestEC2Request_ModifySecurityGroupRulesIsGroupScoped pins that the modify
|
||||
// names the group it mutates, so a policy fencing one security group applies.
|
||||
func TestEC2Request_ModifySecurityGroupRulesIsGroupScoped(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:ModifySecurityGroupRules", "arn:aws:ec2:*:*:security-group/sg-prod"),
|
||||
)
|
||||
|
||||
const update = "&SecurityGroupRule.1.SecurityGroupRuleId=sgr-0123456789abcdef0" +
|
||||
"&SecurityGroupRule.1.SecurityGroupRule.IpProtocol=tcp" +
|
||||
"&SecurityGroupRule.1.SecurityGroupRule.FromPort=22&SecurityGroupRule.1.SecurityGroupRule.ToPort=22" +
|
||||
"&SecurityGroupRule.1.SecurityGroupRule.CidrIpv4=10.0.0.0/24"
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=ModifySecurityGroupRules&GroupId=sg-prod"+update))
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=ModifySecurityGroupRules&GroupId=sg-dev"+update))
|
||||
}
|
||||
|
||||
func TestEC2Request_RejectsOversizedResourceList(t *testing.T) {
|
||||
var body strings.Builder
|
||||
body.WriteString("Action=TerminateInstances")
|
||||
for i := 1; i <= query.MaxSliceLen+1; i++ {
|
||||
body.WriteString("&InstanceId.")
|
||||
body.WriteString(strconv.Itoa(i))
|
||||
body.WriteString("=i-")
|
||||
body.WriteString(strconv.Itoa(i))
|
||||
}
|
||||
|
||||
h := gwtest.Scoped(gwtest.Statement("Allow", "ec2:*", "*"))
|
||||
err := dispatchEC2(t, h, body.String())
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMalformedQueryString, err.Error())
|
||||
}
|
||||
|
||||
// TestEC2Request_AccountWideAllowIsUnaffected is the property the remaining
|
||||
// per-service beads rest on: passing a real ARN where "*" was passed cannot
|
||||
// withdraw access that works today.
|
||||
func TestEC2Request_AccountWideAllowIsUnaffected(t *testing.T) {
|
||||
for _, policy := range []iamdomain.Statement{
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Allow", "*", "*"),
|
||||
} {
|
||||
h := gwtest.Scoped(policy)
|
||||
for _, body := range []string{
|
||||
"Action=TerminateInstances&InstanceId.1=i-abc",
|
||||
"Action=RunInstances&ImageId=ami-1&SubnetId=subnet-1&KeyName=k1",
|
||||
"Action=CreateTags&ResourceId.1=i-abc&Tag.1.Key=Name&Tag.1.Value=x",
|
||||
"Action=CreateTags&ResourceId.1=nope-abc&Tag.1.Key=Name&Tag.1.Value=x",
|
||||
"Action=DeleteVolume&VolumeId=vol-1",
|
||||
"Action=DescribeInstances",
|
||||
} {
|
||||
t.Run(body, func(t *testing.T) {
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, body))
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEC2Request_MissingIdentifierStaysAValidationFault: the gate must not turn
|
||||
// a malformed request into a denial. With an account-wide grant the absent
|
||||
// identifier resolves to "*" and the handler reports its own fault.
|
||||
func TestEC2Request_MissingIdentifierStaysAValidationFault(t *testing.T) {
|
||||
h := gwtest.Scoped(gwtest.Statement("Allow", "ec2:*", "*"))
|
||||
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=TerminateInstances"))
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=DeleteVolume"))
|
||||
}
|
||||
|
||||
// TestEC2Request_IdentifierIsAValueNotAPattern: an id of literally * builds an
|
||||
// ARN ending /*, which neither matches a scoped Deny on a real instance nor
|
||||
// widens a grant to one.
|
||||
func TestEC2Request_IdentifierIsAValueNotAPattern(t *testing.T) {
|
||||
fenced := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:TerminateInstances", "arn:aws:ec2:*:*:instance/i-prod"),
|
||||
)
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, fenced, "Action=TerminateInstances&InstanceId.1=*"))
|
||||
|
||||
// An id containing / is not truncated on the way to the evaluator. The
|
||||
// reverted STS work built an ARN from the segment after the final /, so a
|
||||
// Deny on i-prod fenced i-prod/admin and a grant fenced the wrong object.
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, fenced, "Action=TerminateInstances&InstanceId.1=i-prod/admin"))
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, fenced, "Action=TerminateInstances&InstanceId.1=i-prod"))
|
||||
}
|
||||
|
||||
// TestEC2Request_ARNIsBuiltFromTheNodeRegion: the credential-scope region is
|
||||
// caller-supplied and never validated, so an ARN built from it would let a
|
||||
// caller sign for another region and slide out from under a Deny scoped to the
|
||||
// real one.
|
||||
func TestEC2Request_ARNIsBuiltFromTheNodeRegion(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:TerminateInstances", "arn:aws:ec2:"+gwtest.Region+":*:instance/i-prod"),
|
||||
)
|
||||
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=TerminateInstances&InstanceId.1=i-prod"))
|
||||
}
|
||||
|
||||
// profileIAMService resolves every instance profile to one holding role app, or
|
||||
// fails resolution with resolveErr.
|
||||
type profileIAMService struct {
|
||||
resolveErr error
|
||||
}
|
||||
|
||||
func (s *profileIAMService) ResolveInstanceProfile(accountID, _ string) (*iamdomain.InstanceProfile, error) {
|
||||
if s.resolveErr != nil {
|
||||
return nil, s.resolveErr
|
||||
}
|
||||
return &iamdomain.InstanceProfile{AccountID: accountID, RoleName: "app"}, nil
|
||||
}
|
||||
|
||||
func (s *profileIAMService) CanonicalResourceARN(accountID string, _ arn.IAMResourceType, name string) (string, error) {
|
||||
return "arn:aws:iam::" + accountID + ":role/" + name, nil
|
||||
}
|
||||
|
||||
func passRoleDeniedHarness(t *testing.T, resolveErr error) (*gwtest.Harness, dispatch.Registration) {
|
||||
t.Helper()
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "iam:PassRole", "*"),
|
||||
)
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
return h, ec2awsapi.NewRegistration(ec2awsapi.Deps{NATS: nc, IAM: &profileIAMService{resolveErr: resolveErr}})
|
||||
}
|
||||
|
||||
// An iam:PassRole denial raised during dispatch is EC2's UnauthorizedOperation,
|
||||
// the same as a denial at the action gate.
|
||||
func TestEC2Request_PassRoleDenialIsUnauthorizedOperation(t *testing.T) {
|
||||
h, reg := passRoleDeniedHarness(t, nil)
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2With(t, h, reg,
|
||||
"Action=AssociateIamInstanceProfile&InstanceId=i-dev&IamInstanceProfile.Name=web"))
|
||||
}
|
||||
|
||||
// Only the gateway's own policy denial is translated: an AccessDenied from
|
||||
// instance-profile resolution reaches the caller unchanged.
|
||||
func TestEC2Request_ResolverAccessDeniedPassesThrough(t *testing.T) {
|
||||
h, reg := passRoleDeniedHarness(t, errors.New(awserrors.ErrorAccessDenied))
|
||||
err := dispatchEC2With(t, h, reg, "Action=AssociateIamInstanceProfile&InstanceId=i-dev&IamInstanceProfile.Name=web")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func assertDryRun(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorDryRunOperation, err.Error())
|
||||
}
|
||||
|
||||
// A caller the policy refuses is denied rather than told the request would
|
||||
// have succeeded.
|
||||
func TestEC2Request_DryRunDeniedCaller(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
gwtest.Statement("Deny", "ec2:CreateVpc", "*"),
|
||||
)
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=CreateVpc&CidrBlock=10.0.0.0/16&DryRun=true"))
|
||||
assertDryRun(t, dispatchEC2(t, h, "Action=CreateSecurityGroup&GroupName=g&GroupDescription=d&DryRun=true"))
|
||||
}
|
||||
|
||||
func TestEC2Request_DryRunFalseDispatches(t *testing.T) {
|
||||
h := gwtest.AllowAll()
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=CreateVpc&CidrBlock=10.0.0.0/16&DryRun=false"))
|
||||
}
|
||||
+9
-11
@@ -1,30 +1,28 @@
|
||||
//test:in-package — drives EC2_Request through the gateway's unexported test
|
||||
// helpers (scopedPolicyGateway, dispatchEC2).
|
||||
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
)
|
||||
|
||||
// The PowerUserAccess shape: an Allow on everything except the listed actions.
|
||||
func TestEC2Request_AllowNotActionGrantsTheComplement(t *testing.T) {
|
||||
gw := scopedPolicyGateway(iamdomain.Statement{
|
||||
h := gwtest.Scoped(iamdomain.Statement{
|
||||
Effect: "Allow",
|
||||
NotAction: iamdomain.StringOrArr{"iam:*", "ec2:TerminateInstances"},
|
||||
Resource: iamdomain.StringOrArr{"*"},
|
||||
})
|
||||
|
||||
assertPermitted(t, dispatchEC2(t, gw, "Action=StopInstances&InstanceId.1=i-dev"))
|
||||
assertUnauthorized(t, dispatchEC2(t, gw, "Action=TerminateInstances&InstanceId.1=i-dev"))
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=StopInstances&InstanceId.1=i-dev"))
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=TerminateInstances&InstanceId.1=i-dev"))
|
||||
}
|
||||
|
||||
// A Deny on NotResource fences every instance but the named one.
|
||||
func TestEC2Request_DenyNotResourceFencesTheComplement(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "ec2:*", "*"),
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ec2:*", "*"),
|
||||
iamdomain.Statement{
|
||||
Effect: "Deny",
|
||||
Action: iamdomain.StringOrArr{"ec2:TerminateInstances"},
|
||||
@@ -32,6 +30,6 @@ func TestEC2Request_DenyNotResourceFencesTheComplement(t *testing.T) {
|
||||
},
|
||||
)
|
||||
|
||||
assertPermitted(t, dispatchEC2(t, gw, "Action=TerminateInstances&InstanceId.1=i-dev"))
|
||||
assertUnauthorized(t, dispatchEC2(t, gw, "Action=TerminateInstances&InstanceId.1=i-prod"))
|
||||
gwtest.AssertPermitted(t, dispatchEC2(t, h, "Action=TerminateInstances&InstanceId.1=i-dev"))
|
||||
gwtest.AssertUnauthorized(t, dispatchEC2(t, h, "Action=TerminateInstances&InstanceId.1=i-prod"))
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
instancesvc "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
filterutil "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -43,18 +44,24 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes
|
||||
}
|
||||
|
||||
var allInstanceTypes []*ec2.InstanceTypeInfo
|
||||
availability := make(map[string]int64)
|
||||
answered := make(map[string]bool, len(frames))
|
||||
usable := 0
|
||||
for _, frame := range frames {
|
||||
var nodeOutput ec2.DescribeInstanceTypesOutput
|
||||
var nodeOutput instancesvc.DescribeInstanceTypesNodeOutput
|
||||
if json.Unmarshal(frame.Data, &nodeOutput) == nil {
|
||||
allInstanceTypes = append(allInstanceTypes, nodeOutput.InstanceTypes...)
|
||||
for name, count := range nodeOutput.Availability {
|
||||
availability[name] += count
|
||||
}
|
||||
usable++
|
||||
if frame.NodeID != "" {
|
||||
answered[frame.NodeID] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// capacity=true filter shows all slots (including duplicates) across nodes.
|
||||
// capacity=true reports cluster-wide free slots per type.
|
||||
showCapacity := false
|
||||
for _, f := range input.Filters {
|
||||
if f.Name != nil && *f.Name == "capacity" {
|
||||
@@ -76,17 +83,19 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes
|
||||
|
||||
var finalInstanceTypes []*ec2.InstanceTypeInfo
|
||||
if showCapacity {
|
||||
finalInstanceTypes = allInstanceTypes
|
||||
} else {
|
||||
seen := make(map[string]bool)
|
||||
for _, it := range allInstanceTypes {
|
||||
if it != nil && it.InstanceType != nil {
|
||||
if !seen[*it.InstanceType] {
|
||||
seen[*it.InstanceType] = true
|
||||
finalInstanceTypes = append(finalInstanceTypes, it)
|
||||
}
|
||||
}
|
||||
// No node answered, so nothing is known about capacity. Reporting an
|
||||
// empty list here reads as "no capacity anywhere", which is what made
|
||||
// the console refuse every launch.
|
||||
if usable == 0 {
|
||||
return nil, awserrors.WithRetryAfter(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable,
|
||||
"cannot determine cluster capacity: no node answered the describe fan-out"),
|
||||
describeIncompleteRetryAfter,
|
||||
)
|
||||
}
|
||||
finalInstanceTypes = dedupeByName(allInstanceTypes, availability)
|
||||
} else {
|
||||
finalInstanceTypes = dedupeByName(allInstanceTypes, nil)
|
||||
}
|
||||
|
||||
if len(requestedTypes) > 0 {
|
||||
@@ -128,8 +137,8 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes
|
||||
|
||||
output := &ec2.DescribeInstanceTypesOutput{InstanceTypes: finalInstanceTypes}
|
||||
|
||||
// Unpaged unless asked: the capacity listing the UI reads has one entry per
|
||||
// free slot and does not follow NextToken.
|
||||
// Unpaged unless asked. The listing is one entry per instance type, which is
|
||||
// inside the page cap AWS declares, so a caller that wants pages asks.
|
||||
if input.MaxResults != nil || aws.StringValue(input.NextToken) != "" {
|
||||
page, nextToken, err := pageByOffset(finalInstanceTypes, input.NextToken, input.MaxResults)
|
||||
if err != nil {
|
||||
@@ -143,6 +152,28 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// dedupeByName returns one entry per instance type name. Every node describes
|
||||
// the types it hosts, so the fan-out carries a copy per node.
|
||||
//
|
||||
// A non-nil availability keeps only the types with a free slot, which is the
|
||||
// capacity request. The count itself is not carried here: the reply is bounded
|
||||
// by the catalogue, never by how much capacity is free.
|
||||
func dedupeByName(types []*ec2.InstanceTypeInfo, availability map[string]int64) []*ec2.InstanceTypeInfo {
|
||||
seen := make(map[string]bool, len(types))
|
||||
deduped := make([]*ec2.InstanceTypeInfo, 0, len(types))
|
||||
for _, it := range types {
|
||||
if it == nil || it.InstanceType == nil || seen[*it.InstanceType] {
|
||||
continue
|
||||
}
|
||||
if availability != nil && availability[*it.InstanceType] <= 0 {
|
||||
continue
|
||||
}
|
||||
seen[*it.InstanceType] = true
|
||||
deduped = append(deduped, it)
|
||||
}
|
||||
return deduped
|
||||
}
|
||||
|
||||
// missingInstanceTypes returns the requested types absent from found, in the
|
||||
// caller's order and without repeats.
|
||||
func missingInstanceTypes(found []*ec2.InstanceTypeInfo, requested []*string) []string {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
instancesvc "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -87,52 +88,105 @@ func TestDescribeInstanceTypes_DeduplicatesAcrossNodes(t *testing.T) {
|
||||
assert.Equal(t, 1, seen["m5.large"])
|
||||
}
|
||||
|
||||
func TestDescribeInstanceTypes_CapacityFilterShowsDuplicates(t *testing.T) {
|
||||
// Every node describes the types it hosts, so a four-node cluster sends four
|
||||
// copies. The listing reports each type once.
|
||||
func TestDescribeInstanceTypes_CapacityReportsEachTypeOncePerCluster(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, nc := startTestNATSServer(t)
|
||||
|
||||
// Node 1
|
||||
nc.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
|
||||
data, _ := json.Marshal(&ec2.DescribeInstanceTypesOutput{
|
||||
InstanceTypes: []*ec2.InstanceTypeInfo{
|
||||
{InstanceType: aws.String("t3.micro")},
|
||||
},
|
||||
})
|
||||
msg.Respond(data)
|
||||
msg.Respond(capacityPayload(t, map[string]int64{"t3.micro": 1}))
|
||||
})
|
||||
|
||||
// Node 2 reports same type
|
||||
// Node 2 reports the same type
|
||||
nc2, err := nats.Connect(nc.ConnectedUrl())
|
||||
require.NoError(t, err)
|
||||
defer nc2.Close()
|
||||
|
||||
nc2.Subscribe("ec2.DescribeInstanceTypes", func(msg *nats.Msg) {
|
||||
data, _ := json.Marshal(&ec2.DescribeInstanceTypesOutput{
|
||||
InstanceTypes: []*ec2.InstanceTypeInfo{
|
||||
{InstanceType: aws.String("t3.micro")},
|
||||
},
|
||||
})
|
||||
msg.Respond(data)
|
||||
msg.Respond(capacityPayload(t, map[string]int64{"t3.micro": 1}))
|
||||
})
|
||||
|
||||
nc.Flush()
|
||||
nc2.Flush()
|
||||
|
||||
// With capacity filter = true, duplicates should be preserved
|
||||
input := &ec2.DescribeInstanceTypesInput{
|
||||
Filters: []*ec2.Filter{
|
||||
{
|
||||
Name: aws.String("capacity"),
|
||||
Values: []*string{aws.String("true")},
|
||||
},
|
||||
},
|
||||
}
|
||||
output, err := DescribeInstanceTypes(context.Background(), input, nc, 2, nil, "")
|
||||
output, err := DescribeInstanceTypes(context.Background(), capacityInput(), nc, 2, nil, "")
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, output)
|
||||
// Both slots should appear since capacity filter is enabled
|
||||
assert.Len(t, output.InstanceTypes, 2)
|
||||
assert.Len(t, output.InstanceTypes, 1)
|
||||
assert.Equal(t, "t3.micro", aws.StringValue(output.InstanceTypes[0].InstanceType))
|
||||
}
|
||||
|
||||
// The reply is bounded by the catalogue, so its size is the same whether the
|
||||
// cluster is idle or full. This is the property the per-slot encoding lost: at
|
||||
// 44,346 free slots the same shape exceeded the NATS maximum payload and the
|
||||
// console was told the cluster had no compute at all.
|
||||
func TestDescribeInstanceTypes_CapacityIsBoundedByTypeCountNotFreeSlots(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for name, slots := range map[string]int64{"idle": 44346, "nearly full": 1} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, nc := startTestNATSServer(t)
|
||||
|
||||
payload := capacityPayload(t, map[string]int64{"t3.micro": slots})
|
||||
require.Less(t, len(payload), 4096, "the node reply must not grow with free capacity")
|
||||
|
||||
subscribeAsNode(t, nc, "ec2.DescribeInstanceTypes", "n1", payload)
|
||||
|
||||
output, err := DescribeInstanceTypes(context.Background(), capacityInput(), nc, 1, nil, "")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, output.InstanceTypes, 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A type the node could not admit is absent from the listing, so the console
|
||||
// does not offer a type the cluster would refuse.
|
||||
func TestDescribeInstanceTypes_CapacityOmitsTypesWithNoFreeSlot(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, nc := startTestNATSServer(t)
|
||||
|
||||
subscribeAsNode(t, nc, "ec2.DescribeInstanceTypes", "n1",
|
||||
capacityPayload(t, map[string]int64{"t3.micro": 4, "x2gd.metal": 0}))
|
||||
|
||||
output, err := DescribeInstanceTypes(context.Background(), capacityInput(), nc, 1, nil, "")
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, output.InstanceTypes, 1)
|
||||
assert.Equal(t, "t3.micro", aws.StringValue(output.InstanceTypes[0].InstanceType))
|
||||
}
|
||||
|
||||
// Nothing is known about capacity when no node answered, so the request fails
|
||||
// rather than reporting an empty list the console reads as no capacity.
|
||||
func TestDescribeInstanceTypes_CapacityWithNoResponsesIsRetryable(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, nc := startTestNATSServer(t)
|
||||
|
||||
_, err := DescribeInstanceTypes(context.Background(), capacityInput(), nc, 0, nil, "")
|
||||
|
||||
code, _, ok := awserrors.ResolveErrorDetail(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, awserrors.ErrorServiceUnavailable, code)
|
||||
}
|
||||
|
||||
func capacityInput() *ec2.DescribeInstanceTypesInput {
|
||||
return &ec2.DescribeInstanceTypesInput{
|
||||
Filters: []*ec2.Filter{{Name: aws.String("capacity"), Values: []*string{aws.String("true")}}},
|
||||
}
|
||||
}
|
||||
|
||||
func capacityPayload(t *testing.T, availability map[string]int64) []byte {
|
||||
t.Helper()
|
||||
out := &instancesvc.DescribeInstanceTypesNodeOutput{Availability: availability}
|
||||
for name := range availability {
|
||||
out.InstanceTypes = append(out.InstanceTypes, &ec2.InstanceTypeInfo{InstanceType: aws.String(name)})
|
||||
}
|
||||
data, err := json.Marshal(out)
|
||||
require.NoError(t, err)
|
||||
return data
|
||||
}
|
||||
|
||||
func TestDescribeInstanceTypes_CapacityFilterFalseDeduplicates(t *testing.T) {
|
||||
|
||||
@@ -17,7 +17,7 @@ type InstanceService interface {
|
||||
RunInstances(ctx context.Context, input *ec2.RunInstancesInput, accountID string) (*ec2.Reservation, error)
|
||||
DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error)
|
||||
DescribeInstanceStatus(ctx context.Context, input *ec2.DescribeInstanceStatusInput, accountID string) (*ec2.DescribeInstanceStatusOutput, error)
|
||||
DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypesInput, accountID string) (*ec2.DescribeInstanceTypesOutput, error)
|
||||
DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypesInput, accountID string) (*DescribeInstanceTypesNodeOutput, error)
|
||||
DescribeInstanceAttribute(ctx context.Context, input *ec2.DescribeInstanceAttributeInput, accountID string) (*ec2.DescribeInstanceAttributeOutput, error)
|
||||
DescribeStoppedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error)
|
||||
DescribeTerminatedInstances(ctx context.Context, input *ec2.DescribeInstancesInput, accountID string) (*ec2.DescribeInstancesOutput, error)
|
||||
@@ -49,11 +49,30 @@ type TerminateStoppedInstanceOutput struct {
|
||||
InstanceID string `json:"instanceId"`
|
||||
}
|
||||
|
||||
// DescribeInstanceTypesNodeOutput is one node's reply to the describe fan-out.
|
||||
// Availability is a count per instance type, so the reply is bounded by the
|
||||
// number of types and never by how much capacity is free.
|
||||
type DescribeInstanceTypesNodeOutput struct {
|
||||
InstanceTypes []*ec2.InstanceTypeInfo `json:"InstanceTypes"`
|
||||
|
||||
// Set only for the capacity request, keyed by instance type name. A type
|
||||
// present in InstanceTypes with no entry here has no free slot on the node.
|
||||
Availability map[string]int64 `json:"Availability,omitempty"`
|
||||
}
|
||||
|
||||
// InstanceTypeAvailability pairs an instance type with the number of instances
|
||||
// of it the node can currently admit.
|
||||
type InstanceTypeAvailability struct {
|
||||
Info *ec2.InstanceTypeInfo
|
||||
Available int64
|
||||
}
|
||||
|
||||
// ResourceCapacityProvider exposes per-node instance-type availability.
|
||||
// GetAvailableInstanceTypeInfos gates on free capacity; GetSupportedInstanceTypeInfos
|
||||
// returns all configured types regardless of allocation.
|
||||
// GetInstanceTypeAvailability counts the free slots per type;
|
||||
// GetSupportedInstanceTypeInfos returns all configured types regardless of
|
||||
// allocation.
|
||||
type ResourceCapacityProvider interface {
|
||||
GetAvailableInstanceTypeInfos(showCapacity bool) []*ec2.InstanceTypeInfo
|
||||
GetInstanceTypeAvailability() []InstanceTypeAvailability
|
||||
GetSupportedInstanceTypeInfos() []*ec2.InstanceTypeInfo
|
||||
}
|
||||
|
||||
|
||||
@@ -2141,9 +2141,9 @@ func (s *InstanceServiceImpl) DescribeInstances(ctx context.Context, input *ec2.
|
||||
}
|
||||
|
||||
// DescribeInstanceTypes returns supported instance types. With filter
|
||||
// capacity=true, expands each type to one entry per free slot for cluster-wide
|
||||
// capacity aggregation.
|
||||
func (s *InstanceServiceImpl) DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypesInput, _ string) (*ec2.DescribeInstanceTypesOutput, error) {
|
||||
// capacity=true, each admissible type is returned once with the number of
|
||||
// instances of it this node can take, for cluster-wide capacity aggregation.
|
||||
func (s *InstanceServiceImpl) DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypesInput, _ string) (*DescribeInstanceTypesNodeOutput, error) {
|
||||
slog.InfoContext(ctx, "Processing DescribeInstanceTypes request from this node")
|
||||
|
||||
if s.resourceMgr == nil {
|
||||
@@ -2162,14 +2162,30 @@ func (s *InstanceServiceImpl) DescribeInstanceTypes(ctx context.Context, input *
|
||||
}
|
||||
}
|
||||
|
||||
var filteredTypes []*ec2.InstanceTypeInfo
|
||||
if showCapacity {
|
||||
filteredTypes = s.resourceMgr.GetAvailableInstanceTypeInfos(true)
|
||||
} else {
|
||||
filteredTypes = s.resourceMgr.GetSupportedInstanceTypeInfos()
|
||||
if !showCapacity {
|
||||
types := s.resourceMgr.GetSupportedInstanceTypeInfos()
|
||||
slog.InfoContext(ctx, "DescribeInstanceTypes completed", "count", len(types), "showCapacity", false)
|
||||
return &DescribeInstanceTypesNodeOutput{InstanceTypes: types}, nil
|
||||
}
|
||||
slog.InfoContext(ctx, "DescribeInstanceTypes completed", "count", len(filteredTypes), "showCapacity", showCapacity)
|
||||
return &ec2.DescribeInstanceTypesOutput{InstanceTypes: filteredTypes}, nil
|
||||
|
||||
avail := s.resourceMgr.GetInstanceTypeAvailability()
|
||||
out := &DescribeInstanceTypesNodeOutput{
|
||||
InstanceTypes: make([]*ec2.InstanceTypeInfo, 0, len(avail)),
|
||||
Availability: make(map[string]int64, len(avail)),
|
||||
}
|
||||
totalSlots := int64(0)
|
||||
for _, a := range avail {
|
||||
if a.Info == nil || a.Info.InstanceType == nil || a.Available <= 0 {
|
||||
continue
|
||||
}
|
||||
out.InstanceTypes = append(out.InstanceTypes, a.Info)
|
||||
out.Availability[*a.Info.InstanceType] = a.Available
|
||||
totalSlots += a.Available
|
||||
}
|
||||
|
||||
slog.InfoContext(ctx, "DescribeInstanceTypes completed", "count", len(out.InstanceTypes),
|
||||
"total_available_slots", totalSlots, "showCapacity", true)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DescribeStoppedInstances returns stopped instances from shared KV.
|
||||
|
||||
@@ -693,6 +693,7 @@ type fakeResourceCapacityProvider struct {
|
||||
allocated []*ec2.InstanceTypeInfo
|
||||
deallocated []*ec2.InstanceTypeInfo
|
||||
canAllocFn func(*ec2.InstanceTypeInfo, int) int
|
||||
availableFn func(*ec2.InstanceTypeInfo) int64
|
||||
|
||||
reservationAllocErr error
|
||||
reservationAllocated []*ec2.InstanceTypeInfo
|
||||
@@ -704,10 +705,18 @@ type fakeResourceCapacityProvider struct {
|
||||
|
||||
func (f *fakeResourceCapacityProvider) HostUnderMemoryPressure() bool { return f.memPressure }
|
||||
|
||||
func (f *fakeResourceCapacityProvider) GetAvailableInstanceTypeInfos(showCapacity bool) []*ec2.InstanceTypeInfo {
|
||||
func (f *fakeResourceCapacityProvider) GetInstanceTypeAvailability() []InstanceTypeAvailability {
|
||||
f.calls++
|
||||
f.gotShowCap = showCapacity
|
||||
return f.types
|
||||
f.gotShowCap = true
|
||||
avail := make([]InstanceTypeAvailability, 0, len(f.types))
|
||||
for _, it := range f.types {
|
||||
slots := int64(1)
|
||||
if f.availableFn != nil {
|
||||
slots = f.availableFn(it)
|
||||
}
|
||||
avail = append(avail, InstanceTypeAvailability{Info: it, Available: slots})
|
||||
}
|
||||
return avail
|
||||
}
|
||||
|
||||
func (f *fakeResourceCapacityProvider) GetSupportedInstanceTypeInfos() []*ec2.InstanceTypeInfo {
|
||||
@@ -792,16 +801,20 @@ func TestDescribeInstanceTypes_ReturnsSupportedByDefault(t *testing.T) {
|
||||
assert.Equal(t, 0, prov.calls, "capacity-gated path must not be hit when capacity filter absent")
|
||||
}
|
||||
|
||||
// The free slots cross as a number against each type, so the reply is sized by
|
||||
// the catalogue however much capacity the node has.
|
||||
func TestDescribeInstanceTypes_CapacityFilterHitsAvailable(t *testing.T) {
|
||||
prov := &fakeResourceCapacityProvider{
|
||||
types: []*ec2.InstanceTypeInfo{
|
||||
{InstanceType: aws.String("t3.micro")},
|
||||
{InstanceType: aws.String("t3.micro")},
|
||||
},
|
||||
supportedTypes: []*ec2.InstanceTypeInfo{
|
||||
{InstanceType: aws.String("t3.micro")},
|
||||
{InstanceType: aws.String("m5.large")},
|
||||
},
|
||||
availableFn: func(it *ec2.InstanceTypeInfo) int64 {
|
||||
if aws.StringValue(it.InstanceType) == "t3.micro" {
|
||||
return 874
|
||||
}
|
||||
return 3
|
||||
},
|
||||
}
|
||||
svc := &InstanceServiceImpl{resourceMgr: prov}
|
||||
|
||||
@@ -812,7 +825,8 @@ func TestDescribeInstanceTypes_CapacityFilterHitsAvailable(t *testing.T) {
|
||||
}
|
||||
out, err := svc.DescribeInstanceTypes(context.Background(), input, "")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, out.InstanceTypes, 2, "capacity=true must use the per-slot list")
|
||||
require.Len(t, out.InstanceTypes, 2, "each admissible type is named once")
|
||||
assert.Equal(t, map[string]int64{"t3.micro": 874, "m5.large": 3}, out.Availability)
|
||||
assert.Equal(t, 1, prov.calls, "capacity-gated path must be hit when capacity=true")
|
||||
assert.True(t, prov.gotShowCap, "capacity=true filter must reach the provider")
|
||||
assert.Equal(t, 0, prov.supportedCalls, "supported-types path must not be hit when capacity=true")
|
||||
|
||||
@@ -41,7 +41,7 @@ func (s *NATSInstanceService) DescribeInstances(_ context.Context, _ *ec2.Descri
|
||||
|
||||
// DescribeInstanceTypes is a fan-out operation; use the gateway's scatter-gather
|
||||
// helper instead of this single-node client.
|
||||
func (s *NATSInstanceService) DescribeInstanceTypes(_ context.Context, _ *ec2.DescribeInstanceTypesInput, _ string) (*ec2.DescribeInstanceTypesOutput, error) {
|
||||
func (s *NATSInstanceService) DescribeInstanceTypes(_ context.Context, _ *ec2.DescribeInstanceTypesInput, _ string) (*DescribeInstanceTypesNodeOutput, error) {
|
||||
return nil, fmt.Errorf("DescribeInstanceTypes is fan-out; use gateway scatter-gather, not NATSInstanceService")
|
||||
}
|
||||
|
||||
|
||||
@@ -680,7 +680,9 @@ func (s *SnapshotServiceImpl) snapshotInUseByVolumes(ctx context.Context, snapsh
|
||||
}
|
||||
inUse := false
|
||||
for _, volume := range volumes {
|
||||
if volume.SnapshotID != snapshotID {
|
||||
// A deleting volume's teardown removes only its own objects and never
|
||||
// reads the snapshot again, so it no longer holds the snapshot's chunks.
|
||||
if volume.SnapshotID != snapshotID || volume.State == string(ebsprovider.VolumeStateDeleting) {
|
||||
continue
|
||||
}
|
||||
if volume.TenantID == accountID {
|
||||
|
||||
@@ -505,6 +505,35 @@ func TestDeleteSnapshot_InUseByVolume(t *testing.T) {
|
||||
assert.Len(t, result.Snapshots, 1)
|
||||
}
|
||||
|
||||
// TestDeleteSnapshot_CloneAlreadyDeleting tests that a clone whose deletion was
|
||||
// accepted no longer holds the snapshot, so deleting a restored resource and then
|
||||
// its snapshot succeeds without waiting for the clone's teardown.
|
||||
func TestDeleteSnapshot_CloneAlreadyDeleting(t *testing.T) {
|
||||
svc, store := setupTestSnapshotService(t)
|
||||
|
||||
createTestVolume(t, svc, store, "vol-source", 50)
|
||||
snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{
|
||||
VolumeId: aws.String("vol-source"),
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
seedVolumeDocument(t, store, ebsmetadata.Volume{
|
||||
VolumeID: "vol-cloned", CapacityGiB: 50, SnapshotID: *snap.SnapshotId,
|
||||
State: string(ebsprovider.VolumeStateDeleting),
|
||||
})
|
||||
|
||||
_, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{
|
||||
SnapshotId: snap.SnapshotId,
|
||||
}, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = svc.DescribeSnapshots(context.Background(), &ec2.DescribeSnapshotsInput{
|
||||
SnapshotIds: []*string{snap.SnapshotId},
|
||||
}, testAccountID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), awserrors.ErrorInvalidSnapshotNotFound)
|
||||
}
|
||||
|
||||
// TestDeleteSnapshot_InUseByAnotherAccountsVolume tests that the refusal
|
||||
// never names a volume the caller does not own, since the dependency scan is
|
||||
// whole-cluster and an endpoint never confirms another account's IDs.
|
||||
|
||||
+57
-45
@@ -1,27 +1,36 @@
|
||||
//test:in-package — drives ECR dispatch through the gateway's unexported test
|
||||
// helpers (setupECRRequest, policyMockIAMService) and auth context keys.
|
||||
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/pipeline"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// dispatchECR drives the gateway at an action whose handler is the 501 stub, so
|
||||
// ecrRequest builds the request an ECR JSON 1.1 client sends.
|
||||
func ecrRequest(target, body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
if target != "" {
|
||||
req.Header.Set("X-Amz-Target", target)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// dispatchECR drives the registration at an action whose handler is the 501 stub, so
|
||||
// a permitted request fails there rather than on authorization.
|
||||
func dispatchECR(t *testing.T, gw *GatewayConfig, action, body string) error {
|
||||
func dispatchECR(t *testing.T, h *gwtest.Harness, action, body string) error {
|
||||
t.Helper()
|
||||
return gw.serveECR(httptest.NewRecorder(), setupECRRequest(awsapi.TargetPrefix+"."+action, body))
|
||||
return h.Dispatch(awsapi.NewRegistration(awsapi.Deps{}), awsapi.ServiceName, httptest.NewRecorder(), ecrRequest(awsapi.TargetPrefix+"."+action, body))
|
||||
}
|
||||
|
||||
func assertECRPermitted(t *testing.T, err error) {
|
||||
@@ -33,32 +42,32 @@ func assertECRPermitted(t *testing.T, err error) {
|
||||
// TestECRRequest_ScopedDenyFires is the bypass this work closes. An operator
|
||||
// fences a production repository; before the resolver the fence was inert.
|
||||
func TestECRRequest_ScopedDenyFires(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "ecr:*", "*"),
|
||||
statement("Deny", "ecr:BatchCheckLayerAvailability", "arn:aws:ecr:*:*:repository/prod"),
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecr:*", "*"),
|
||||
gwtest.Statement("Deny", "ecr:BatchCheckLayerAvailability", "arn:aws:ecr:*:*:repository/prod"),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchECR(t, gw, "BatchCheckLayerAvailability", `{"repositoryName":"prod"}`))
|
||||
gwtest.AssertDenied(t, dispatchECR(t, gw, "BatchCheckLayerAvailability", `{"repositoryName":"prod"}`))
|
||||
assertECRPermitted(t, dispatchECR(t, gw, "BatchCheckLayerAvailability", `{"repositoryName":"dev"}`))
|
||||
}
|
||||
|
||||
// TestECRRequest_ScopedAllowGrants is the other half: a least-privilege policy
|
||||
// used to deny everything, so the only working policy shape was Resource "*".
|
||||
func TestECRRequest_ScopedAllowGrants(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "ecr:BatchCheckLayerAvailability", "arn:aws:ecr:*:*:repository/dev"),
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecr:BatchCheckLayerAvailability", "arn:aws:ecr:*:*:repository/dev"),
|
||||
)
|
||||
|
||||
assertECRPermitted(t, dispatchECR(t, gw, "BatchCheckLayerAvailability", `{"repositoryName":"dev"}`))
|
||||
assertDenied(t, dispatchECR(t, gw, "BatchCheckLayerAvailability", `{"repositoryName":"prod"}`))
|
||||
gwtest.AssertDenied(t, dispatchECR(t, gw, "BatchCheckLayerAvailability", `{"repositoryName":"prod"}`))
|
||||
}
|
||||
|
||||
// A body spelling one field two ways is rejected rather than authorized against
|
||||
// whichever spelling the case fold happened to keep.
|
||||
func TestECRRequest_FieldSpelledTwoWaysIsRejected(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "ecr:*", "*"),
|
||||
statement("Deny", "ecr:BatchCheckLayerAvailability", "arn:aws:ecr:*:*:repository/prod"),
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecr:*", "*"),
|
||||
gwtest.Statement("Deny", "ecr:BatchCheckLayerAvailability", "arn:aws:ecr:*:*:repository/prod"),
|
||||
)
|
||||
|
||||
// Repeated: the fold's disagreement is random, the rejection must not be.
|
||||
@@ -73,37 +82,37 @@ func TestECRRequest_FieldSpelledTwoWaysIsRejected(t *testing.T) {
|
||||
// DescribeRepositories naming no repository enumerates the whole registry, so a
|
||||
// grant scoped to one repository does not reach it.
|
||||
func TestECRRequest_OmittedRepositoryListEnumeratesTheRegistry(t *testing.T) {
|
||||
scoped := scopedPolicyGateway(
|
||||
statement("Allow", "ecr:DescribeRepositories", "arn:aws:ecr:*:*:repository/dev"),
|
||||
scoped := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecr:DescribeRepositories", "arn:aws:ecr:*:*:repository/dev"),
|
||||
)
|
||||
assertDenied(t, dispatchECR(t, scoped, "DescribeRepositories", `{}`))
|
||||
assertDenied(t, dispatchECR(t, scoped, "DescribeRepositories", `{"repositoryNames":[]}`))
|
||||
gwtest.AssertDenied(t, dispatchECR(t, scoped, "DescribeRepositories", `{}`))
|
||||
gwtest.AssertDenied(t, dispatchECR(t, scoped, "DescribeRepositories", `{"repositoryNames":[]}`))
|
||||
|
||||
wide := scopedPolicyGateway(
|
||||
statement("Allow", "ecr:DescribeRepositories", "arn:aws:ecr:*:*:repository/*"),
|
||||
wide := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecr:DescribeRepositories", "arn:aws:ecr:*:*:repository/*"),
|
||||
)
|
||||
assertNotDenied(t, dispatchECR(t, wide, "DescribeRepositories", `{}`))
|
||||
gwtest.AssertNotDenied(t, dispatchECR(t, wide, "DescribeRepositories", `{}`))
|
||||
}
|
||||
|
||||
// A caller-supplied registryId would let a request slide out from under a Deny
|
||||
// scoped to the real account, so the ARN is built under the caller's.
|
||||
func TestECRRequest_RegistryIDInTheBodyIsIgnored(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "ecr:*", "*"),
|
||||
statement("Deny", "ecr:BatchCheckLayerAvailability",
|
||||
"arn:aws:ecr:"+authzRegion+":"+authzAccountID+":repository/prod"),
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecr:*", "*"),
|
||||
gwtest.Statement("Deny", "ecr:BatchCheckLayerAvailability",
|
||||
"arn:aws:ecr:"+gwtest.Region+":"+gwtest.AccountID+":repository/prod"),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchECR(t, gw, "BatchCheckLayerAvailability",
|
||||
gwtest.AssertDenied(t, dispatchECR(t, gw, "BatchCheckLayerAvailability",
|
||||
`{"registryId":"999999999999","repositoryName":"prod"}`))
|
||||
}
|
||||
|
||||
// A body the gate cannot parse authorizes account-wide, and the handler still
|
||||
// reports its own fault.
|
||||
func TestECRRequest_UnparseableBodyStaysTheHandlersFault(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "ecr:*", "*"),
|
||||
statement("Deny", "ecr:BatchCheckLayerAvailability", "arn:aws:ecr:*:*:repository/prod"),
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecr:*", "*"),
|
||||
gwtest.Statement("Deny", "ecr:BatchCheckLayerAvailability", "arn:aws:ecr:*:*:repository/prod"),
|
||||
)
|
||||
|
||||
assertECRPermitted(t, dispatchECR(t, gw, "BatchCheckLayerAvailability", "{not json"))
|
||||
@@ -112,7 +121,7 @@ func TestECRRequest_UnparseableBodyStaysTheHandlersFault(t *testing.T) {
|
||||
// A body past the signed path's cap cannot be used to bypass the gate or to
|
||||
// exhaust memory: it is rejected before either the gate or the handler.
|
||||
func TestECRRequest_OversizedBodyIsRejected(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "ecr:*", "*"))
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "ecr:*", "*"))
|
||||
body := `{"repositoryName":"` + strings.Repeat("a", sigv4.MaxPayloadLen) + `"}`
|
||||
|
||||
err := dispatchECR(t, gw, "BatchCheckLayerAvailability", body)
|
||||
@@ -123,17 +132,22 @@ func TestECRRequest_OversizedBodyIsRejected(t *testing.T) {
|
||||
// The property the per-service rollout rests on: passing a real ARN where "*"
|
||||
// was passed before cannot withdraw access a working policy already grants.
|
||||
func TestECRRequest_AccountWideGrantStillPermitsEveryAction(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "ecr:*", "*"))
|
||||
req := withTestIdentity(httptest.NewRequest(http.MethodPost, "/", nil).
|
||||
WithContext(withPrincipalField(context.Background(), fieldAccountID, authzAccountID)))
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "ecr:*", "*"))
|
||||
req := httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
caller := credential.Principal{
|
||||
AccountID: gwtest.AccountID,
|
||||
Identity: "alice",
|
||||
PrincipalType: stsdomain.PrincipalTypeUser,
|
||||
CallerARN: "arn:aws:iam::" + gwtest.AccountID + ":user/alice",
|
||||
}
|
||||
body := []byte(`{"repositoryName":"app","repositoryNames":["app"],` +
|
||||
`"resourceArn":"arn:aws:ecr:` + authzRegion + `:` + authzAccountID + `:repository/app"}`)
|
||||
`"resourceArn":"arn:aws:ecr:` + gwtest.Region + `:` + gwtest.AccountID + `:repository/app"}`)
|
||||
|
||||
for _, action := range awsapi.ScopedActions() {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
resources, err := awsapi.ResourceARNs(action, authzRegion, authzAccountID, body)
|
||||
resources, err := awsapi.ResourceARNs(action, gwtest.Region, gwtest.AccountID, body)
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, gw.checkPolicyResources(req, "ecr", action, resources))
|
||||
assert.NoError(t, pipeline.Authorize(gw.Policy, req, caller, true, "ecr", action, resources, nil))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -141,12 +155,10 @@ func TestECRRequest_AccountWideGrantStillPermitsEveryAction(t *testing.T) {
|
||||
// The account-ID read moved above the gate, which used to reject a missing
|
||||
// account itself. InternalError is the code the caller has always seen.
|
||||
func TestECRRequest_MissingAccountIDReturnsInternalError(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "ecr:*", "*"))
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("{}"))
|
||||
req.Header.Set("X-Amz-Target", awsapi.TargetPrefix+".ListRepositories")
|
||||
req = withTestIdentity(req.WithContext(context.WithValue(req.Context(), ctxService, "ecr")))
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "ecr:*", "*"))
|
||||
gw.AccountID = ""
|
||||
|
||||
err := gw.serveECR(httptest.NewRecorder(), req)
|
||||
err := dispatchECR(t, gw, "ListRepositories", "{}")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInternalError, err.Error())
|
||||
}
|
||||
+12
-18
@@ -1,4 +1,4 @@
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
|
||||
// newRepoLifecycleGateway wires the repo create/describe/delete + manifest
|
||||
// subjects against an embedded KV-backed service, returning a gateway client.
|
||||
func newRepoLifecycleGateway(t *testing.T) (*GatewayConfig, *nats.Conn) {
|
||||
func newRepoLifecycleGateway(t *testing.T) (*ecrGW, *nats.Conn) {
|
||||
t.Helper()
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
js := testutil.NewJetStream(t, nc)
|
||||
@@ -30,28 +30,24 @@ func newRepoLifecycleGateway(t *testing.T) (*GatewayConfig, *nats.Conn) {
|
||||
serveECRMeta(t, nc, ecr.SubjectRepoDelete, svc.RepoDelete)
|
||||
serveECRMeta(t, nc, ecr.SubjectManifestPut, svc.ManifestPut)
|
||||
serveECRMeta(t, nc, ecr.SubjectManifestList, svc.ManifestList)
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
NATSConn: nc, Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}), awsapi.Deps{Repository: awsapi.NewRepositoryActionService(ecr.NewNATSMetaStore(nc), awsapi.RepositoryEndpoint{
|
||||
gw := newECRGW(nc, awsapi.Deps{Repository: awsapi.NewRepositoryActionService(ecr.NewNATSMetaStore(nc), awsapi.RepositoryEndpoint{
|
||||
Region: ecrTestRegion, ServicesDomain: ecrTestSuffix,
|
||||
})})
|
||||
return gw, nc
|
||||
}
|
||||
|
||||
func ecrRepositoryRequest(t *testing.T, gw *GatewayConfig, action, body string) (*httptest.ResponseRecorder, error) {
|
||||
func ecrRepositoryRequest(t *testing.T, gw *ecrGW, action, body string) (*httptest.ResponseRecorder, error) {
|
||||
t.Helper()
|
||||
req := setupECRRequest(awsapi.TargetPrefix+"."+action, body)
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
w := httptest.NewRecorder()
|
||||
return w, gw.serveECR(w, req.WithContext(ctx))
|
||||
return w, gw.serveECR(w, req)
|
||||
}
|
||||
|
||||
func createRepo(t *testing.T, gw *GatewayConfig, body string) (*httptest.ResponseRecorder, error) {
|
||||
func createRepo(t *testing.T, gw *ecrGW, body string) (*httptest.ResponseRecorder, error) {
|
||||
return ecrRepositoryRequest(t, gw, "CreateRepository", body)
|
||||
}
|
||||
|
||||
func deleteRepo(t *testing.T, gw *GatewayConfig, body string) (*httptest.ResponseRecorder, error) {
|
||||
func deleteRepo(t *testing.T, gw *ecrGW, body string) (*httptest.ResponseRecorder, error) {
|
||||
return ecrRepositoryRequest(t, gw, "DeleteRepository", body)
|
||||
}
|
||||
|
||||
@@ -202,9 +198,11 @@ func TestCreateRepository_NoAccountAndMalformed(t *testing.T) {
|
||||
|
||||
req := noAccountRequest(`{"repositoryName":"team/app"}`)
|
||||
req.Header.Set("X-Amz-Target", awsapi.TargetPrefix+".CreateRepository")
|
||||
gw.h.AccountID = ""
|
||||
err := gw.serveECR(httptest.NewRecorder(), req)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "InternalError", err.Error())
|
||||
gw.h.AccountID = ecrTestAccount
|
||||
|
||||
_, err = createRepo(t, gw, `{`)
|
||||
require.Error(t, err)
|
||||
@@ -216,9 +214,11 @@ func TestDeleteRepository_NoAccountAndMalformed(t *testing.T) {
|
||||
|
||||
req := noAccountRequest(`{"repositoryName":"team/app"}`)
|
||||
req.Header.Set("X-Amz-Target", awsapi.TargetPrefix+".DeleteRepository")
|
||||
gw.h.AccountID = ""
|
||||
err := gw.serveECR(httptest.NewRecorder(), req)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "InternalError", err.Error())
|
||||
gw.h.AccountID = ecrTestAccount
|
||||
|
||||
_, err = deleteRepo(t, gw, `{`)
|
||||
require.Error(t, err)
|
||||
@@ -287,24 +287,18 @@ func TestECRRequest_CreateDeleteDispatched(t *testing.T) {
|
||||
gw, _ := newRepoLifecycleGateway(t)
|
||||
|
||||
create := setupECRRequest("AmazonEC2ContainerRegistry_V20150921.CreateRepository", `{"repositoryName":"team/app"}`)
|
||||
create = create.WithContext(withPrincipalField(create.Context(), fieldAccountID, ecrTestAccount))
|
||||
wc := httptest.NewRecorder()
|
||||
require.NoError(t, gw.serveECR(wc, create))
|
||||
assert.Equal(t, http.StatusOK, wc.Code)
|
||||
|
||||
del := setupECRRequest("AmazonEC2ContainerRegistry_V20150921.DeleteRepository", `{"repositoryName":"team/app"}`)
|
||||
del = del.WithContext(withPrincipalField(del.Context(), fieldAccountID, ecrTestAccount))
|
||||
wd := httptest.NewRecorder()
|
||||
require.NoError(t, gw.serveECR(wd, del))
|
||||
assert.Equal(t, http.StatusOK, wd.Code)
|
||||
}
|
||||
|
||||
func TestRepositoryActions_MissingComposition(t *testing.T) {
|
||||
gw := withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion,
|
||||
DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
})
|
||||
gw := newECRGW(nil, awsapi.Deps{})
|
||||
|
||||
_, err := ecrRepositoryRequest(t, gw, "CreateRepository", `{"repositoryName":"team/app"}`)
|
||||
require.Error(t, err)
|
||||
+8
-15
@@ -1,4 +1,4 @@
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -41,7 +41,7 @@ func serveECRMeta[I any, O any](t *testing.T, nc *nats.Conn, subject string, fn
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
|
||||
func newDescribeReposGateway(t *testing.T, repos ...string) *GatewayConfig {
|
||||
func newDescribeReposGateway(t *testing.T, repos ...string) *ecrGW {
|
||||
t.Helper()
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
js := testutil.NewJetStream(t, nc)
|
||||
@@ -54,20 +54,16 @@ func newDescribeReposGateway(t *testing.T, repos ...string) *GatewayConfig {
|
||||
for _, r := range repos {
|
||||
require.NoError(t, store.PutRepo(context.Background(), ecrTestAccount, ecr.RepoMeta{Name: r, CreatedAt: time.Now()}))
|
||||
}
|
||||
return withECR(withVerifier(&GatewayConfig{
|
||||
NATSConn: nc, Region: ecrTestRegion, InternalSuffix: ecrTestSuffix, DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}), awsapi.Deps{Repository: awsapi.NewRepositoryActionService(store, awsapi.RepositoryEndpoint{
|
||||
return newECRGW(nc, awsapi.Deps{Repository: awsapi.NewRepositoryActionService(store, awsapi.RepositoryEndpoint{
|
||||
Region: ecrTestRegion, ServicesDomain: ecrTestSuffix,
|
||||
})})
|
||||
}
|
||||
|
||||
func describeReposRequest(t *testing.T, gw *GatewayConfig, body string) *httptest.ResponseRecorder {
|
||||
func describeReposRequest(t *testing.T, gw *ecrGW, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := setupECRRequest(awsapi.TargetPrefix+".DescribeRepositories", body)
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
w := httptest.NewRecorder()
|
||||
require.NoError(t, gw.serveECR(w, req.WithContext(ctx)))
|
||||
require.NoError(t, gw.serveECR(w, req))
|
||||
return w
|
||||
}
|
||||
|
||||
@@ -149,8 +145,7 @@ func TestDescribeRepositories_NameFilter(t *testing.T) {
|
||||
func TestDescribeRepositories_MissingNamedRepo(t *testing.T) {
|
||||
gw := newDescribeReposGateway(t, "team/app")
|
||||
req := setupECRRequest(awsapi.TargetPrefix+".DescribeRepositories", `{"repositoryNames":["team/ghost"]}`)
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
err := gw.serveECR(httptest.NewRecorder(), req.WithContext(ctx))
|
||||
err := gw.serveECR(httptest.NewRecorder(), req)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "RepositoryNotFoundException", awserrors.ValidErrorCodeFromError(err))
|
||||
}
|
||||
@@ -158,8 +153,7 @@ func TestDescribeRepositories_MissingNamedRepo(t *testing.T) {
|
||||
func TestDescribeRepositories_CrossAccountDenied(t *testing.T) {
|
||||
gw := newDescribeReposGateway(t, "team/app")
|
||||
req := setupECRRequest(awsapi.TargetPrefix+".DescribeRepositories", `{"registryId":"999999999999"}`)
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
err := gw.serveECR(httptest.NewRecorder(), req.WithContext(ctx))
|
||||
err := gw.serveECR(httptest.NewRecorder(), req)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "AccessDenied", err.Error())
|
||||
}
|
||||
@@ -167,8 +161,7 @@ func TestDescribeRepositories_CrossAccountDenied(t *testing.T) {
|
||||
func TestECRRequest_DescribeRepositoriesDispatched(t *testing.T) {
|
||||
gw := newDescribeReposGateway(t, "team/app")
|
||||
req := setupECRRequest("AmazonEC2ContainerRegistry_V20150921.DescribeRepositories", "{}")
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
w := httptest.NewRecorder()
|
||||
require.NoError(t, gw.serveECR(w, req.WithContext(ctx)))
|
||||
require.NoError(t, gw.serveECR(w, req))
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestECRRequest_MissingTarget(t *testing.T) {
|
||||
gw := newECRGW(nil, awsapi.Deps{})
|
||||
err := gw.serveECR(httptest.NewRecorder(), setupECRRequest("", ""))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMissingAction, err.Error())
|
||||
}
|
||||
|
||||
func TestECRRequest_UnknownAction(t *testing.T) {
|
||||
gw := newECRGW(nil, awsapi.Deps{})
|
||||
err := gw.serveECR(httptest.NewRecorder(),
|
||||
setupECRRequest("AmazonEC2ContainerRegistry_V20150921.MadeUpAction", "{}"))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidAction, err.Error())
|
||||
}
|
||||
|
||||
// A registered-but-unimplemented action resolves to the 501 stub until its
|
||||
// handler lands. Repository actions may use a transitional gateway adapter and
|
||||
// image actions a composed registry capability, so ListRepositories stands in
|
||||
// as a still-stubbed action.
|
||||
func TestECRRequest_KnownActionNotImplemented(t *testing.T) {
|
||||
gw := newECRGW(nil, awsapi.Deps{})
|
||||
err := gw.serveECR(httptest.NewRecorder(),
|
||||
setupECRRequest("AmazonEC2ContainerRegistry_V20150921.ListRepositories", "{}"))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorNotImplemented, err.Error())
|
||||
}
|
||||
+12
-15
@@ -1,4 +1,4 @@
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -21,18 +21,13 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newImageGateway wires a GatewayConfig with an in-memory OCI registry (memory
|
||||
// newImageGateway wires the ECR registration over an in-memory OCI registry (memory
|
||||
// object store + memory meta) and its composed ECR JSON image-action service.
|
||||
func newImageGateway(t *testing.T) *GatewayConfig {
|
||||
func newImageGateway(t *testing.T) *ecrGW {
|
||||
t.Helper()
|
||||
meta := ecrdomain.NewMemoryMetaStore()
|
||||
reg := ecrregistry.NewRegistry(objectstore.NewMemoryObjectStore(), meta, ecrTestAccount)
|
||||
gw := withECR(withVerifier(&GatewayConfig{
|
||||
Region: ecrTestRegion,
|
||||
InternalSuffix: ecrTestSuffix,
|
||||
DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
}), awsapi.Deps{Registry: awsapi.NewRegistryActionService(reg, reg, reg, reg), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(meta, reg)})
|
||||
gw := newECRGW(nil, awsapi.Deps{Registry: awsapi.NewRegistryActionService(reg, reg, reg, reg), LifecyclePreview: awsapi.NewLifecyclePreviewActionService(meta, reg)})
|
||||
imageRegistries.Store(gw, reg)
|
||||
return gw
|
||||
}
|
||||
@@ -40,21 +35,21 @@ func newImageGateway(t *testing.T) *GatewayConfig {
|
||||
// imageRegistries maps a newImageGateway config to its registry so seeds can reach it.
|
||||
var imageRegistries sync.Map
|
||||
|
||||
func registryOf(gw *GatewayConfig) *ecrregistry.Registry {
|
||||
func registryOf(gw *ecrGW) *ecrregistry.Registry {
|
||||
reg, _ := imageRegistries.Load(gw)
|
||||
return reg.(*ecrregistry.Registry)
|
||||
}
|
||||
|
||||
// seedGatewayRepo creates the repository metadata so push/PutImage handlers —
|
||||
// which require an existing repository — accept writes. Idempotent.
|
||||
func seedGatewayRepo(t *testing.T, gw *GatewayConfig, repo string) {
|
||||
func seedGatewayRepo(t *testing.T, gw *ecrGW, repo string) {
|
||||
t.Helper()
|
||||
require.NoError(t, registryOf(gw).Meta.PutRepo(context.Background(), ecrTestAccount, ecrdomain.RepoMeta{Name: repo}))
|
||||
}
|
||||
|
||||
// seedTaggedImage stores a layerless manifest under repo:tag, returning its
|
||||
// digest. A unique manifest per tag yields a unique digest.
|
||||
func seedTaggedImage(t *testing.T, gw *GatewayConfig, repo, tag string) string {
|
||||
func seedTaggedImage(t *testing.T, gw *ecrGW, repo, tag string) string {
|
||||
t.Helper()
|
||||
seedGatewayRepo(t, gw, repo)
|
||||
manifest := fmt.Appendf(nil,
|
||||
@@ -67,10 +62,10 @@ func seedTaggedImage(t *testing.T, gw *GatewayConfig, repo, tag string) string {
|
||||
func imageReq(t *testing.T, action, body string) *http.Request {
|
||||
t.Helper()
|
||||
req := setupECRRequest(awsapi.TargetPrefix+"."+action, body)
|
||||
return req.WithContext(withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount))
|
||||
return req
|
||||
}
|
||||
|
||||
func callImage(t *testing.T, gw *GatewayConfig, action, body string) (*httptest.ResponseRecorder, error) {
|
||||
func callImage(t *testing.T, gw *ecrGW, action, body string) (*httptest.ResponseRecorder, error) {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
return w, gw.serveECR(w, imageReq(t, action, body))
|
||||
@@ -305,12 +300,14 @@ func TestImageHandlers_GuardRails(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"repositoryName":"team/app"}`))
|
||||
req.Header.Set("X-Amz-Target", awsapi.TargetPrefix+".ListImages")
|
||||
gw.h.AccountID = ""
|
||||
err = gw.serveECR(w, req)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "InternalError", err.Error())
|
||||
gw.h.AccountID = ecrTestAccount
|
||||
|
||||
// A missing registry-action composition -> ServerInternal.
|
||||
bare := withVerifier(&GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()})
|
||||
bare := newECRGW(nil, awsapi.Deps{})
|
||||
err = bare.serveECR(httptest.NewRecorder(), imageReq(t, "ListImages", `{"repositoryName":"team/app"}`))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, "ServerInternal", err.Error())
|
||||
+3
-8
@@ -1,12 +1,12 @@
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/ecr"
|
||||
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -91,12 +91,7 @@ func TestLifecyclePreview_Errors(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLifecyclePreview_MissingComposition(t *testing.T) {
|
||||
gw := &GatewayConfig{
|
||||
Region: ecrTestRegion,
|
||||
DisableLogging: true,
|
||||
IAMService: allowAllIAMService(),
|
||||
Credentials: credverify.New(allowAllIAMService(), nil),
|
||||
}
|
||||
gw := newECRGW(nil, awsapi.Deps{})
|
||||
|
||||
_, err := callImage(t, gw, "StartLifecyclePolicyPreview", previewBody("team/app", previewExpireOldest))
|
||||
require.Error(t, err)
|
||||
@@ -0,0 +1,48 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
const (
|
||||
ecrTestRegion = gwtest.Region
|
||||
ecrTestSuffix = "spinifex.internal"
|
||||
ecrTestAccount = "000000000001"
|
||||
)
|
||||
|
||||
// ecrGW is the ECR registration invoked as the pipeline would for one
|
||||
// authenticated user of ecrTestAccount.
|
||||
type ecrGW struct {
|
||||
h *gwtest.Harness
|
||||
reg dispatch.Registration
|
||||
}
|
||||
|
||||
// newECRGW registers ECR over deps, relaying onto nc unless deps names its own.
|
||||
func newECRGW(nc *nats.Conn, deps awsapi.Deps) *ecrGW {
|
||||
if deps.NATS == nil {
|
||||
deps.NATS = nc
|
||||
}
|
||||
h := gwtest.AllowAll()
|
||||
h.AccountID = ecrTestAccount
|
||||
return &ecrGW{h: h, reg: awsapi.NewRegistration(deps)}
|
||||
}
|
||||
|
||||
func (g *ecrGW) serveECR(w http.ResponseWriter, r *http.Request) error {
|
||||
return g.h.Dispatch(g.reg, awsapi.ServiceName, w, r)
|
||||
}
|
||||
|
||||
// setupECRRequest builds an ECR control-plane request for target.
|
||||
func setupECRRequest(target, body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
if target != "" {
|
||||
req.Header.Set("X-Amz-Target", target)
|
||||
}
|
||||
return req
|
||||
}
|
||||
+3
-5
@@ -1,4 +1,4 @@
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@@ -11,12 +11,11 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func putTagMutability(t *testing.T, gw *GatewayConfig, body string) (*httptest.ResponseRecorder, error) {
|
||||
func putTagMutability(t *testing.T, gw *ecrGW, body string) (*httptest.ResponseRecorder, error) {
|
||||
t.Helper()
|
||||
req := setupECRRequest("AmazonEC2ContainerRegistry_V20150921.PutImageTagMutability", body)
|
||||
ctx := withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount)
|
||||
w := httptest.NewRecorder()
|
||||
return w, gw.serveECR(w, req.WithContext(ctx))
|
||||
return w, gw.serveECR(w, req)
|
||||
}
|
||||
|
||||
type tagMutabilityOut struct {
|
||||
@@ -106,7 +105,6 @@ func TestECRRequest_PutImageTagMutabilityDispatched(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
req := setupECRRequest("AmazonEC2ContainerRegistry_V20150921.PutImageTagMutability", `{"repositoryName":"team/app","imageTagMutability":"IMMUTABLE"}`)
|
||||
req = req.WithContext(withPrincipalField(req.Context(), fieldAccountID, ecrTestAccount))
|
||||
w := httptest.NewRecorder()
|
||||
require.NoError(t, gw.serveECR(w, req))
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
@@ -0,0 +1,236 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ecs"
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
ecsawsapi "github.com/mulgadc/spinifex/spinifex/domains/ecs/awsapi"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const ecsPassRoleTargetARN = "arn:aws:iam::123456789012:role/ecs-task-role"
|
||||
|
||||
// dispatchECS drives the registration with no NATS connection. A permitted request
|
||||
// therefore reaches the handler and fails there, which is what proves the policy
|
||||
// check ran ahead of the resource existing at all.
|
||||
func dispatchECS(t *testing.T, h *gwtest.Harness, action, body string) error {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
req.Header.Set("X-Amz-Target", ecsawsapi.TargetPrefix+"."+action)
|
||||
return h.Dispatch(ecsawsapi.NewRegistration(ecsawsapi.Deps{}), ecsawsapi.ServiceName, httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
func ecsResourceARN(resource string) string {
|
||||
return "arn:aws:ecs:" + gwtest.Region + ":" + gwtest.AccountID + ":" + resource
|
||||
}
|
||||
|
||||
// TestECSRequest_ScopedDenyFires is the bypass this work closes. An operator
|
||||
// fences a production service; before the resolver the fence was inert and
|
||||
// DeleteService against it was permitted with nothing logged.
|
||||
func TestECSRequest_ScopedDenyFires(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecs:*", "*"),
|
||||
gwtest.Statement("Deny", "ecs:DeleteService", "arn:aws:ecs:*:*:service/prod/web"),
|
||||
)
|
||||
|
||||
gwtest.AssertDenied(t, dispatchECS(t, h, "DeleteService", `{"cluster":"prod","service":"web"}`))
|
||||
require.Error(t, dispatchECS(t, h, "DeleteService", `{"cluster":"prod","service":"api"}`))
|
||||
h.AssertGatePassed(t)
|
||||
require.Error(t, dispatchECS(t, h, "DeleteService", `{"cluster":"dev","service":"web"}`))
|
||||
h.AssertGatePassed(t)
|
||||
}
|
||||
|
||||
// TestECSRequest_ScopedAllowGrants is the other half: a least-privilege policy
|
||||
// used to deny everything, so the only working policy shape was Resource "*".
|
||||
func TestECSRequest_ScopedAllowGrants(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecs:UpdateCluster", "arn:aws:ecs:*:*:cluster/dev"),
|
||||
)
|
||||
|
||||
err := dispatchECS(t, h, "UpdateCluster", `{"cluster":"dev"}`)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorNotImplemented, err.Error())
|
||||
gwtest.AssertDenied(t, dispatchECS(t, h, "UpdateCluster", `{"cluster":"prod"}`))
|
||||
}
|
||||
|
||||
// An omitted cluster is the default cluster, both at the gate and in the
|
||||
// handler, so a fence on it fires.
|
||||
func TestECSRequest_OmittedClusterIsFencedAsDefault(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecs:*", "*"),
|
||||
gwtest.Statement("Deny", "ecs:UpdateCluster", ecsResourceARN("cluster/default")),
|
||||
)
|
||||
|
||||
gwtest.AssertDenied(t, dispatchECS(t, h, "UpdateCluster", `{}`))
|
||||
gwtest.AssertDenied(t, dispatchECS(t, h, "UpdateCluster", `{"cluster":"default"}`))
|
||||
require.Error(t, dispatchECS(t, h, "UpdateCluster", `{"cluster":"prod"}`))
|
||||
h.AssertGatePassed(t)
|
||||
}
|
||||
|
||||
// A body the gate cannot parse authorizes account-wide, and the handler still
|
||||
// reports its own validation fault.
|
||||
func TestECSRequest_UnparseableBodyStaysTheHandlersFault(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecs:*", "*"),
|
||||
gwtest.Statement("Deny", "ecs:UpdateCluster", ecsResourceARN("cluster/prod")),
|
||||
)
|
||||
|
||||
err := dispatchECS(t, h, "UpdateCluster", "{not json")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorNotImplemented, err.Error())
|
||||
}
|
||||
|
||||
// A body spelling one field two ways is the bypass a case-folded lookup opens:
|
||||
// the handler reads the last spelling in document order, the gate's map cannot,
|
||||
// so the request is rejected rather than authorized against the wrong cluster.
|
||||
func TestECSRequest_FieldSpelledTwoWaysIsRejected(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecs:*", "*"),
|
||||
gwtest.Statement("Deny", "ecs:DeleteCluster", ecsResourceARN("cluster/prod")),
|
||||
)
|
||||
|
||||
// Repeated: the fold's disagreement is random, the rejection must not be.
|
||||
for range 50 {
|
||||
err := dispatchECS(t, h, "DeleteCluster", `{"cluster":"dev","Cluster":"prod"}`)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// A describe naming no cluster describes the default one in the handler, so a
|
||||
// fence on the default cluster fires against an empty list too.
|
||||
func TestECSRequest_OmittedClusterListIsFencedAsDefault(t *testing.T) {
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "ecs:*", "*"),
|
||||
gwtest.Statement("Deny", "ecs:DescribeClusters", ecsResourceARN("cluster/default")),
|
||||
)
|
||||
|
||||
gwtest.AssertDenied(t, dispatchECS(t, h, "DescribeClusters", `{}`))
|
||||
gwtest.AssertDenied(t, dispatchECS(t, h, "DescribeClusters", `{"clusters":[]}`))
|
||||
require.Error(t, dispatchECS(t, h, "DescribeClusters", `{"clusters":["prod"]}`))
|
||||
h.AssertGatePassed(t)
|
||||
}
|
||||
|
||||
// A body past the signed path's cap cannot be used to bypass the gate or to
|
||||
// exhaust memory: it is rejected before either the gate or the handler.
|
||||
func TestECSRequest_OversizedBodyIsRejected(t *testing.T) {
|
||||
h := gwtest.Scoped(gwtest.Statement("Allow", "ecs:*", "*"))
|
||||
body := `{"cluster":"` + strings.Repeat("a", sigv4.MaxPayloadLen) + `"}`
|
||||
|
||||
err := dispatchECS(t, h, "UpdateCluster", body)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorRequestEntityTooLarge, err.Error())
|
||||
}
|
||||
|
||||
// The property the per-service rollout rests on: passing a real ARN where "*"
|
||||
// was passed before cannot withdraw access a working policy already grants.
|
||||
func TestECSRequest_AccountWideGrantStillPermitsEveryAction(t *testing.T) {
|
||||
h := gwtest.Scoped(gwtest.Statement("Allow", "ecs:*", "*"))
|
||||
body := `{"cluster":"prod","service":"web","serviceName":"web","task":"t-1",` +
|
||||
`"containerInstance":"ci-1","taskDefinition":"app:1","name":"cp","capacityProvider":"cp",` +
|
||||
`"clusterName":"prod","resourceArn":"` + ecsResourceARN("cluster/prod") + `"}`
|
||||
|
||||
for _, action := range ecsawsapi.ScopedActions() {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
require.Error(t, dispatchECS(t, h, action, body))
|
||||
h.AssertGatePassed(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The account-ID read moved above the gate, which used to reject a missing
|
||||
// account itself. InternalError is the code the caller has always seen.
|
||||
func TestECSRequest_MissingAccountIDReturnsInternalError(t *testing.T) {
|
||||
h := gwtest.Scoped(gwtest.Statement("Allow", "ecs:*", "*"))
|
||||
h.AccountID = ""
|
||||
|
||||
err := dispatchECS(t, h, "ListClusters", "{}")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInternalError, err.Error())
|
||||
}
|
||||
|
||||
func registerTaskDefinitionBody(t *testing.T, taskRoleARN string) string {
|
||||
t.Helper()
|
||||
input := &ecs.RegisterTaskDefinitionInput{Family: aws.String("web")}
|
||||
if taskRoleARN != "" {
|
||||
input.TaskRoleArn = aws.String(taskRoleARN)
|
||||
}
|
||||
body, err := json.Marshal(input)
|
||||
require.NoError(t, err)
|
||||
return string(body)
|
||||
}
|
||||
|
||||
// allowRegisterTaskDefAndPassRole grants ecs:RegisterTaskDefinition
|
||||
// unconditionally (the action-level check always runs first) plus iam:PassRole
|
||||
// scoped to passRoleResource, so tests can isolate the resource-level PassRole
|
||||
// decision from the action-level one.
|
||||
func allowRegisterTaskDefAndPassRole(passRoleResource string) []iamdomain.PolicyDocument {
|
||||
return []iamdomain.PolicyDocument{{
|
||||
Version: "2012-10-17",
|
||||
Statement: []iamdomain.Statement{
|
||||
{Effect: "Allow", Action: iamdomain.StringOrArr{"ecs:RegisterTaskDefinition"}, Resource: iamdomain.StringOrArr{"*"}},
|
||||
{Effect: "Allow", Action: iamdomain.StringOrArr{"iam:PassRole"}, Resource: iamdomain.StringOrArr{passRoleResource}},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestECSRegisterTaskDefinition_DeniedWithoutPassRole(t *testing.T) {
|
||||
// ecs:RegisterTaskDefinition is granted; iam:PassRole is scoped to a
|
||||
// different role, so the target ARN is specifically denied.
|
||||
h := gwtest.FromDocs(allowRegisterTaskDefAndPassRole("arn:aws:iam::123456789012:role/some-other-role"))
|
||||
|
||||
err := dispatchECS(t, h, "RegisterTaskDefinition", registerTaskDefinitionBody(t, ecsPassRoleTargetARN))
|
||||
require.Error(t, err)
|
||||
gwtest.AssertDenied(t, err)
|
||||
}
|
||||
|
||||
func TestECSRegisterTaskDefinition_AllowedWithPassRole(t *testing.T) {
|
||||
h := gwtest.FromDocs(allowRegisterTaskDefAndPassRole(ecsPassRoleTargetARN))
|
||||
|
||||
err := dispatchECS(t, h, "RegisterTaskDefinition", registerTaskDefinitionBody(t, ecsPassRoleTargetARN))
|
||||
// PassRole is granted, so the request proceeds past the check to the (nil,
|
||||
// disconnected) NATS dispatch; whatever fails downstream, it must not be
|
||||
// the PassRole denial.
|
||||
require.Error(t, err)
|
||||
h.AssertGatePassed(t)
|
||||
}
|
||||
|
||||
func TestECSRegisterTaskDefinition_NoRoleSkipsPassRole(t *testing.T) {
|
||||
// No iam:PassRole grant at all — if the gateway still consulted PassRole
|
||||
// for a roleless task definition this would deny it.
|
||||
h := gwtest.Scoped(gwtest.Statement("Allow", "ecs:RegisterTaskDefinition", "*"))
|
||||
|
||||
err := dispatchECS(t, h, "RegisterTaskDefinition", registerTaskDefinitionBody(t, ""))
|
||||
require.Error(t, err) // still fails downstream (nil NATS conn), just not on PassRole
|
||||
h.AssertGatePassed(t)
|
||||
}
|
||||
|
||||
// Current behaviour, not a statement of what AWS does: ECS resolves nothing, so
|
||||
// a role ARN that is malformed, in another account or absent from the store
|
||||
// reaches PassRole as written and is accepted when the grant covers it.
|
||||
func TestECSRegisterTaskDefinition_UnresolvedRoleARNsAreNotRejected(t *testing.T) {
|
||||
for name, roleARN := range map[string]string{
|
||||
"malformed": "not-an-arn",
|
||||
"nonexistent": "arn:aws:iam::123456789012:role/no-such-role",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := gwtest.FromDocs(allowRegisterTaskDefAndPassRole("*"))
|
||||
|
||||
err := dispatchECS(t, h, "RegisterTaskDefinition", registerTaskDefinitionBody(t, roleARN))
|
||||
require.Error(t, err) // fails downstream on the nil NATS conn
|
||||
h.AssertGatePassed(t)
|
||||
code, _ := awserrors.ResolveErrorCode(err)
|
||||
assert.NotEqual(t, awserrors.ErrorInvalidParameterValue, code)
|
||||
})
|
||||
}
|
||||
}
|
||||
+174
-149
@@ -1,14 +1,9 @@
|
||||
//test:in-package — drives EKS_Request through the gateway's unexported test
|
||||
// helpers (withTestIdentity, policyMockIAMService) and auth context keys.
|
||||
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -17,57 +12,93 @@ import (
|
||||
"time"
|
||||
|
||||
eksv1 "github.com/mulgadc/spinifex/contracts/eks/v1"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/eks/access"
|
||||
eksawsapi "github.com/mulgadc/spinifex/spinifex/domains/eks/awsapi"
|
||||
ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/passrole"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/pipeline"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// eksGateway is the policy harness plus the NATS connection the registration is
|
||||
// composed with; a nil connection fails the actions that reach the daemon.
|
||||
type eksGateway struct {
|
||||
*gwtest.Harness
|
||||
|
||||
nc *nats.Conn
|
||||
roles passrole.Resolver
|
||||
}
|
||||
|
||||
func scopedEKS(statements ...iamdomain.Statement) *eksGateway {
|
||||
return &eksGateway{Harness: gwtest.Scoped(statements...)}
|
||||
}
|
||||
|
||||
// as returns the same gateway acting as caller, in caller's own account.
|
||||
func (g *eksGateway) as(caller credential.Principal) *eksGateway {
|
||||
h := *g.Harness
|
||||
h.AccountID = caller.AccountID
|
||||
h.Caller = &caller
|
||||
return &eksGateway{Harness: &h, nc: g.nc}
|
||||
}
|
||||
|
||||
func (g *eksGateway) serve(w http.ResponseWriter, r *http.Request) error {
|
||||
return g.Dispatch(eksawsapi.NewRegistration(eksawsapi.Deps{NATS: g.nc, Roles: g.roles}), eksawsapi.ServiceName, w, r)
|
||||
}
|
||||
|
||||
func assertNotDenied(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
code, _ := awserrors.ResolveErrorCode(err)
|
||||
assert.NotEqual(t, awserrors.ErrorAccessDenied, code)
|
||||
}
|
||||
|
||||
// dispatchEKS drives the gateway with no NATS connection. A permitted request
|
||||
// therefore reaches the NATS guard and fails there, which is what proves the
|
||||
// policy check ran ahead of the resource existing at all.
|
||||
func dispatchEKS(t *testing.T, gw *GatewayConfig, method, path, body string) error {
|
||||
func dispatchEKS(t *testing.T, gw *eksGateway, method, path, body string) error {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(method, path, strings.NewReader(body))
|
||||
ctx := context.WithValue(req.Context(), ctxService, "eks")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, authzAccountID)
|
||||
req = withTestIdentity(req.WithContext(ctx))
|
||||
return gw.serveEKS(httptest.NewRecorder(), req)
|
||||
return gw.serve(httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
func eksARN(resource string) string {
|
||||
return "arn:aws:eks:" + authzRegion + ":" + authzAccountID + ":" + resource
|
||||
return "arn:aws:eks:" + gwtest.Region + ":" + gwtest.AccountID + ":" + resource
|
||||
}
|
||||
|
||||
// TestEKSRequest_ScopedDenyFires is the bypass this work closes. An operator
|
||||
// fences a production cluster; before the resolver the fence was inert and
|
||||
// DeleteCluster against it was permitted with nothing logged.
|
||||
func TestEKSRequest_ScopedDenyFires(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "eks:*", "*"),
|
||||
statement("Deny", "eks:DeleteCluster", "arn:aws:eks:*:*:cluster/prod"),
|
||||
gw := scopedEKS(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Deny", "eks:DeleteCluster", "arn:aws:eks:*:*:cluster/prod"),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod", ""))
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/dev", ""))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod", ""))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/dev", ""))
|
||||
}
|
||||
|
||||
// TestEKSRequest_ScopedAllowGrants is the other half: a least-privilege policy
|
||||
// used to deny everything, so the only working policy shape was Resource "*".
|
||||
func TestEKSRequest_ScopedAllowGrants(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "eks:DeleteCluster", "arn:aws:eks:*:*:cluster/dev"),
|
||||
gw := scopedEKS(
|
||||
gwtest.Statement("Allow", "eks:DeleteCluster", "arn:aws:eks:*:*:cluster/dev"),
|
||||
)
|
||||
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/dev", ""))
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod", ""))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/dev", ""))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod", ""))
|
||||
}
|
||||
|
||||
// The bypass this work closes. A tenant pastes the ARN out of DescribeNodegroup
|
||||
@@ -75,94 +106,94 @@ func TestEKSRequest_ScopedAllowGrants(t *testing.T) {
|
||||
// wildcarded spelling that this exact-ARN pattern could not match, so the fence
|
||||
// was inert and the delete went through.
|
||||
func TestEKSRequest_NodegroupScopeMatchesExactStoredARN(t *testing.T) {
|
||||
stored := arn.FormatEKSNodegroup(authzRegion, authzAccountID, "prod", "workers",
|
||||
arn.EKSNodegroupDiscriminator(authzAccountID, "prod", "workers"))
|
||||
stored := arn.FormatEKSNodegroup(gwtest.Region, gwtest.AccountID, "prod", "workers",
|
||||
arn.EKSNodegroupDiscriminator(gwtest.AccountID, "prod", "workers"))
|
||||
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "eks:*", "*"),
|
||||
statement("Deny", "eks:DeleteNodegroup", stored),
|
||||
gw := scopedEKS(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Deny", "eks:DeleteNodegroup", stored),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod/node-groups/workers", ""))
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod/node-groups/batch", ""))
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/dev/node-groups/workers", ""))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod/node-groups/workers", ""))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod/node-groups/batch", ""))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/dev/node-groups/workers", ""))
|
||||
}
|
||||
|
||||
// The AWS-documented policy spelling wildcards the discriminator, and was the
|
||||
// only spelling that worked before. It must keep working.
|
||||
func TestEKSRequest_NodegroupScopeMatchesWildcardUUID(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "eks:*", "*"),
|
||||
statement("Deny", "eks:DeleteNodegroup", "arn:aws:eks:*:*:nodegroup/prod/workers/*"),
|
||||
gw := scopedEKS(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Deny", "eks:DeleteNodegroup", "arn:aws:eks:*:*:nodegroup/prod/workers/*"),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod/node-groups/workers", ""))
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod/node-groups/batch", ""))
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/dev/node-groups/workers", ""))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod/node-groups/workers", ""))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/prod/node-groups/batch", ""))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, "/clusters/dev/node-groups/workers", ""))
|
||||
}
|
||||
|
||||
// CreateCluster names its cluster in the body, so the body read has to happen
|
||||
// before the gate for a fence on a name prefix to fire at all.
|
||||
func TestEKSRequest_CreateClusterScopesFromBody(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "eks:*", "*"),
|
||||
statement("Deny", "eks:CreateCluster", "arn:aws:eks:*:*:cluster/prod-*"),
|
||||
gw := scopedEKS(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Deny", "eks:CreateCluster", "arn:aws:eks:*:*:cluster/prod-*"),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodPost, "/clusters", `{"name":"prod-1"}`))
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodPost, "/clusters", `{"name":"dev-1"}`))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodPost, "/clusters", `{"name":"prod-1"}`))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodPost, "/clusters", `{"name":"dev-1"}`))
|
||||
// An unreadable body authorizes account-wide and stays the handler's fault.
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodPost, "/clusters", "{not json"))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodPost, "/clusters", "{not json"))
|
||||
}
|
||||
|
||||
// The access-entry principal arrives percent-encoded in the path; the ARN the
|
||||
// gate builds must be the one the handler's own builder produces.
|
||||
func TestEKSRequest_AccessEntryPathIsUnescaped(t *testing.T) {
|
||||
const principal = "arn:aws:iam::123456789012:role/app/admin"
|
||||
entry := arn.FormatEKSAccessEntry(authzRegion, authzAccountID, "prod", access.PrincipalARNHash(principal))
|
||||
entry := arn.FormatEKSAccessEntry(gwtest.Region, gwtest.AccountID, "prod", access.PrincipalARNHash(principal))
|
||||
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "eks:*", "*"),
|
||||
statement("Deny", "eks:DeleteAccessEntry", entry),
|
||||
gw := scopedEKS(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Deny", "eks:DeleteAccessEntry", entry),
|
||||
)
|
||||
|
||||
path := "/clusters/prod/access-entries/" + url.PathEscape(principal)
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodDelete, path, ""))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodDelete, path, ""))
|
||||
|
||||
other := "/clusters/prod/access-entries/" + url.PathEscape("arn:aws:iam::123456789012:role/app/reader")
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, other, ""))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodDelete, other, ""))
|
||||
}
|
||||
|
||||
// A tag request names its resource by ARN. The handler ignores that ARN's
|
||||
// account and works in the caller's, so the gate must too.
|
||||
func TestEKSRequest_TagResourceScopesTheNamedResource(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "eks:*", "*"),
|
||||
statement("Deny", "eks:TagResource", eksARN("cluster/prod")),
|
||||
gw := scopedEKS(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Deny", "eks:TagResource", eksARN("cluster/prod")),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodPost,
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodPost,
|
||||
"/tags/"+url.PathEscape(eksARN("cluster/prod")), `{"tags":{"k":"v"}}`))
|
||||
assertPermitted(t, dispatchEKS(t, gw, http.MethodPost,
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodPost,
|
||||
"/tags/"+url.PathEscape(eksARN("cluster/dev")), `{"tags":{"k":"v"}}`))
|
||||
// The same cluster spelled under another account is still the caller's.
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodPost,
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodPost,
|
||||
"/tags/"+url.PathEscape("arn:aws:eks:us-east-1:999999999999:cluster/prod"), `{"tags":{"k":"v"}}`))
|
||||
}
|
||||
|
||||
// The property the per-service rollout rests on: passing a real ARN where "*"
|
||||
// was passed before cannot withdraw access a working policy already grants.
|
||||
func TestEKSRequest_AccountWideGrantStillPermitsEveryAction(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "eks:*", "*"))
|
||||
req := withTestIdentity(httptest.NewRequest(http.MethodGet, "/clusters", nil).
|
||||
WithContext(withPrincipalField(context.Background(), fieldAccountID, authzAccountID)))
|
||||
gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
req := httptest.NewRequest(http.MethodGet, "/clusters", nil)
|
||||
caller := userContext(gwtest.AccountID, "alice")
|
||||
|
||||
for _, action := range eksawsapi.ScopedActions() {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
resources, err := eksawsapi.ResourceARNs(action, authzRegion, authzAccountID,
|
||||
resources, err := eksawsapi.ResourceARNs(action, gwtest.Region, gwtest.AccountID,
|
||||
[]string{"prod", "workers", "extra"}, []byte(`{"name":"prod","nodegroupName":"workers","addonName":"coredns","principalArn":"arn:aws:iam::123456789012:role/app"}`))
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, gw.checkPolicyResources(req, "eks", action, resources))
|
||||
assert.NoError(t, pipeline.Authorize(gw.Policy, req, caller, true, "eks", action, resources, nil))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -171,31 +202,22 @@ func TestEKSRequest_AccountWideGrantStillPermitsEveryAction(t *testing.T) {
|
||||
// evaluated as permitted and returned another tenant's cluster state. The
|
||||
// principal gate now rejects the tenant ahead of the policy check.
|
||||
func TestEKSRequest_InternalRoutesRejectTenantPrincipal(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "eks:*", "*"))
|
||||
gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodGet, "/clusters/prod/internal-addons/999988887777", ""))
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodGet,
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodGet, "/clusters/prod/internal-addons/999988887777", ""))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodGet,
|
||||
"/clusters/prod/internal-recovery/999988887777/i-0abc", ""))
|
||||
// Its own account is no different: a tenant is not a control-plane VM.
|
||||
assertDenied(t, dispatchEKS(t, gw, http.MethodGet, "/clusters/prod/internal-addons/"+authzAccountID, ""))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodGet, "/clusters/prod/internal-addons/"+gwtest.AccountID, ""))
|
||||
}
|
||||
|
||||
// dispatchEKSAsCPAgent drives the gateway with the auth context a CP VM's IMDS
|
||||
// credentials actually produce, so the gate is exercised through eksCaller's
|
||||
// field mapping rather than a hand-built Caller.
|
||||
func dispatchEKSAsCPAgent(t *testing.T, gw *GatewayConfig, path, instanceID string) error {
|
||||
func dispatchEKSAsCPAgent(t *testing.T, gw *eksGateway, path, instanceID string) error {
|
||||
t.Helper()
|
||||
roleName := ekscluster.CPInstanceRoleName
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "eks")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, awsidentifiers.GlobalAccountID)
|
||||
ctx = withPrincipalField(ctx, fieldIdentity, instanceID)
|
||||
ctx = withPrincipalField(ctx, fieldPrincipalType, principalTypeAssumedRole)
|
||||
ctx = withPrincipalField(ctx, fieldUnderlyingRoleARN,
|
||||
"arn:aws:iam::"+awsidentifiers.GlobalAccountID+":role/"+roleName)
|
||||
ctx = withPrincipalField(ctx, fieldAssumedRoleARN,
|
||||
"arn:aws:sts::"+awsidentifiers.GlobalAccountID+":assumed-role/"+roleName+"/"+instanceID)
|
||||
return gw.serveEKS(httptest.NewRecorder(), req.WithContext(ctx))
|
||||
return gw.as(cpAgentContext(instanceID)).serve(httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
// The denial tests above all fail on the first clause of the class gate, so on
|
||||
@@ -206,7 +228,7 @@ func TestEKSRequest_InternalRoutesAdmitTheCPAgentPrincipal(t *testing.T) {
|
||||
const cpInstanceID = "i-cp0000000000001"
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{
|
||||
Bucket: ekscluster.AccountBucketName(authzAccountID),
|
||||
Bucket: ekscluster.AccountBucketName(gwtest.AccountID),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, ekscluster.PutClusterMeta(t.Context(), kv, &ekscluster.ClusterMeta{
|
||||
@@ -214,29 +236,29 @@ func TestEKSRequest_InternalRoutesAdmitTheCPAgentPrincipal(t *testing.T) {
|
||||
ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
|
||||
}))
|
||||
|
||||
gw := scopedPolicyGateway(statement("Allow", "eks:*", "*"))
|
||||
gw.NATSConn = nc
|
||||
gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gw.nc = nc
|
||||
|
||||
// No eks daemon is subscribed, so the handler fails on the absent responder.
|
||||
// Anything but AccessDenied means the gate and the policy check both passed.
|
||||
assertNotDenied(t, dispatchEKSAsCPAgent(t, gw,
|
||||
"/clusters/prod/internal-addons/"+authzAccountID, cpInstanceID))
|
||||
"/clusters/prod/internal-addons/"+gwtest.AccountID, cpInstanceID))
|
||||
assertNotDenied(t, dispatchEKSAsCPAgent(t, gw,
|
||||
"/clusters/prod/internal-recovery/"+authzAccountID+"/"+cpInstanceID, cpInstanceID))
|
||||
"/clusters/prod/internal-recovery/"+gwtest.AccountID+"/"+cpInstanceID, cpInstanceID))
|
||||
|
||||
// Same credentials, a cluster this VM does not serve: the binding still bites.
|
||||
assertDenied(t, dispatchEKSAsCPAgent(t, gw,
|
||||
"/clusters/other/internal-addons/"+authzAccountID, cpInstanceID))
|
||||
gwtest.AssertDenied(t, dispatchEKSAsCPAgent(t, gw,
|
||||
"/clusters/other/internal-addons/"+gwtest.AccountID, cpInstanceID))
|
||||
}
|
||||
|
||||
// The account-ID read moved above the gate, which used to reject a missing
|
||||
// account itself. InternalError is the code the caller has always seen.
|
||||
func TestEKSRequest_MissingAccountIDReturnsInternalError(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "eks:*", "*"))
|
||||
req := withTestIdentity(httptest.NewRequest(http.MethodGet, "/clusters", nil).
|
||||
WithContext(context.WithValue(context.Background(), ctxService, "eks")))
|
||||
gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gw.AccountID = ""
|
||||
req := httptest.NewRequest(http.MethodGet, "/clusters", nil)
|
||||
|
||||
err := gw.serveEKS(httptest.NewRecorder(), req)
|
||||
err := gw.serve(httptest.NewRecorder(), req)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInternalError, err.Error())
|
||||
}
|
||||
@@ -244,7 +266,7 @@ func TestEKSRequest_MissingAccountIDReturnsInternalError(t *testing.T) {
|
||||
// publishFixture seeds two clusters in the caller's account and one in another,
|
||||
// each served by its own control-plane VM, and subscribes to every EKS subject.
|
||||
type publishFixture struct {
|
||||
gw *GatewayConfig
|
||||
gw *eksGateway
|
||||
nc *nats.Conn
|
||||
sub *nats.Subscription
|
||||
}
|
||||
@@ -261,8 +283,8 @@ func newPublishFixture(t *testing.T) *publishFixture {
|
||||
t.Helper()
|
||||
_, nc, js := testutil.StartTestJetStream(t)
|
||||
for _, c := range []struct{ acct, cluster, member string }{
|
||||
{authzAccountID, "alpha", publishCPAlpha},
|
||||
{authzAccountID, "beta", publishCPBeta},
|
||||
{gwtest.AccountID, "alpha", publishCPAlpha},
|
||||
{gwtest.AccountID, "beta", publishCPBeta},
|
||||
{publishOtherAcct, "gamma", publishCPGamma},
|
||||
} {
|
||||
kv, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, c.acct)
|
||||
@@ -276,17 +298,17 @@ func newPublishFixture(t *testing.T) *publishFixture {
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
gw := scopedPolicyGateway(statement("Allow", "eks:*", "*"))
|
||||
gw.NATSConn = nc
|
||||
gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gw.nc = nc
|
||||
return &publishFixture{gw: gw, nc: nc, sub: sub}
|
||||
}
|
||||
|
||||
// publish POSTs body to cluster's internal-publish route under ctx's identity.
|
||||
func (f *publishFixture) publish(t *testing.T, ctx context.Context, cluster, body string) (*httptest.ResponseRecorder, error) {
|
||||
func (f *publishFixture) publish(t *testing.T, caller credential.Principal, cluster, body string) (*httptest.ResponseRecorder, error) {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "/clusters/"+cluster+"/internal-publish", strings.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
return rec, f.gw.serveEKS(rec, req.WithContext(ctx))
|
||||
return rec, f.gw.as(caller).serve(rec, req)
|
||||
}
|
||||
|
||||
// requireNothingPublished proves a denial stopped the relay, not just the reply.
|
||||
@@ -298,24 +320,27 @@ func (f *publishFixture) requireNothingPublished(t *testing.T) {
|
||||
require.Zero(t, pending, "relayed onto eks.>")
|
||||
}
|
||||
|
||||
// cpAgentContext is the auth context a CP VM's IMDS credentials produce.
|
||||
func cpAgentContext(instanceID string) context.Context {
|
||||
// cpAgentContext is the caller a CP VM's IMDS credentials produce.
|
||||
func cpAgentContext(instanceID string) credential.Principal {
|
||||
roleName := ekscluster.CPInstanceRoleName
|
||||
ctx := context.WithValue(context.Background(), ctxService, "eks")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, awsidentifiers.GlobalAccountID)
|
||||
ctx = withPrincipalField(ctx, fieldIdentity, instanceID)
|
||||
ctx = withPrincipalField(ctx, fieldPrincipalType, principalTypeAssumedRole)
|
||||
ctx = withPrincipalField(ctx, fieldUnderlyingRoleARN,
|
||||
"arn:aws:iam::"+awsidentifiers.GlobalAccountID+":role/"+roleName)
|
||||
return withPrincipalField(ctx, fieldAssumedRoleARN,
|
||||
"arn:aws:sts::"+awsidentifiers.GlobalAccountID+":assumed-role/"+roleName+"/"+instanceID)
|
||||
assumed := "arn:aws:sts::" + awsidentifiers.GlobalAccountID + ":assumed-role/" + roleName + "/" + instanceID
|
||||
return credential.Principal{
|
||||
AccountID: awsidentifiers.GlobalAccountID,
|
||||
Identity: instanceID,
|
||||
PrincipalType: dispatch.PrincipalTypeAssumedRole,
|
||||
UnderlyingRoleARN: "arn:aws:iam::" + awsidentifiers.GlobalAccountID + ":role/" + roleName,
|
||||
AssumedRoleARN: assumed,
|
||||
CallerARN: assumed,
|
||||
}
|
||||
}
|
||||
|
||||
func userContext(accountID, name string) context.Context {
|
||||
ctx := context.WithValue(context.Background(), ctxService, "eks")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, accountID)
|
||||
ctx = withPrincipalField(ctx, fieldIdentity, name)
|
||||
return withPrincipalField(ctx, fieldPrincipalType, principalTypeUser)
|
||||
func userContext(accountID, name string) credential.Principal {
|
||||
return credential.Principal{
|
||||
AccountID: accountID,
|
||||
Identity: name,
|
||||
PrincipalType: stsdomain.PrincipalTypeUser,
|
||||
CallerARN: "arn:aws:iam::" + accountID + ":user/" + name,
|
||||
}
|
||||
}
|
||||
|
||||
// publishChannel is one body shape PublishInternal relays, and where it lands.
|
||||
@@ -347,7 +372,7 @@ func publishChannels() []publishChannel {
|
||||
|
||||
const publishPayload = `{"healthz":"ok", "node_count":1,"ts":42}`
|
||||
|
||||
func publishBody(accountID string, ch publishChannel) string {
|
||||
func relayBody(accountID string, ch publishChannel) string {
|
||||
return `{"accountId":"` + accountID + `",` + ch.fields + `,"payload":` + publishPayload + `}`
|
||||
}
|
||||
|
||||
@@ -357,14 +382,14 @@ func TestEKSRequest_PublishInternalRelaysForTheCallersOwnCluster(t *testing.T) {
|
||||
f := newPublishFixture(t)
|
||||
for _, ch := range publishChannels() {
|
||||
t.Run(ch.name, func(t *testing.T) {
|
||||
rec, err := f.publish(t, cpAgentContext(publishCPAlpha), "alpha", publishBody(authzAccountID, ch))
|
||||
rec, err := f.publish(t, cpAgentContext(publishCPAlpha), "alpha", relayBody(gwtest.AccountID, ch))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.JSONEq(t, `{}`, rec.Body.String())
|
||||
|
||||
msg, err := f.sub.NextMsg(2 * time.Second)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ch.subject(authzAccountID, "alpha"), msg.Subject)
|
||||
assert.Equal(t, ch.subject(gwtest.AccountID, "alpha"), msg.Subject)
|
||||
assert.Equal(t, publishPayload, string(msg.Data))
|
||||
f.requireNothingPublished(t)
|
||||
})
|
||||
@@ -379,18 +404,18 @@ func TestEKSRequest_PublishInternalDeniesAClusterTheCallerDoesNotServe(t *testin
|
||||
targets := []struct {
|
||||
name, account, cluster string
|
||||
}{
|
||||
{"its own account, another cluster", authzAccountID, "beta"},
|
||||
{"its own account, another cluster", gwtest.AccountID, "beta"},
|
||||
{"another account's cluster", publishOtherAcct, "gamma"},
|
||||
{"its own cluster's name in another account", publishOtherAcct, "alpha"},
|
||||
{"a cluster that does not exist", authzAccountID, "delta"},
|
||||
{"a cluster that does not exist", gwtest.AccountID, "delta"},
|
||||
{"an account with no clusters", publishNoClusters, "alpha"},
|
||||
{"an account no bucket can be named for", publishOtherAcct + ".gamma", "alpha"},
|
||||
{"a cluster no key can be named for", authzAccountID, "*"},
|
||||
{"a cluster no key can be named for", gwtest.AccountID, "*"},
|
||||
}
|
||||
for _, ch := range publishChannels() {
|
||||
for _, target := range targets {
|
||||
t.Run(ch.name+"/"+target.name, func(t *testing.T) {
|
||||
_, err := f.publish(t, cpAgentContext(publishCPAlpha), url.PathEscape(target.cluster), publishBody(target.account, ch))
|
||||
_, err := f.publish(t, cpAgentContext(publishCPAlpha), url.PathEscape(target.cluster), relayBody(target.account, ch))
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorAccessDenied, awserrors.ValidErrorCodeFromError(err))
|
||||
assert.Equal(t, errors.New(awserrors.ErrorAccessDenied), err)
|
||||
@@ -407,29 +432,29 @@ func TestEKSRequest_PublishInternalRedirectionIsDenied(t *testing.T) {
|
||||
state := publishChannels()[0]
|
||||
|
||||
// URL names the caller's cluster, body names another account.
|
||||
_, err := f.publish(t, cpAgentContext(publishCPAlpha), "alpha", publishBody(publishOtherAcct, state))
|
||||
assertDenied(t, err)
|
||||
_, err := f.publish(t, cpAgentContext(publishCPAlpha), "alpha", relayBody(publishOtherAcct, state))
|
||||
gwtest.AssertDenied(t, err)
|
||||
f.requireNothingPublished(t)
|
||||
|
||||
// encoding/json folds case, so a re-cased key is the account the handler reads.
|
||||
_, err = f.publish(t, cpAgentContext(publishCPAlpha), "gamma",
|
||||
`{"AccountID":"`+publishOtherAcct+`","channel":"state","payload":{"x":1}}`)
|
||||
assertDenied(t, err)
|
||||
gwtest.AssertDenied(t, err)
|
||||
f.requireNothingPublished(t)
|
||||
|
||||
// A repeated key decodes to its last value, which is what the gate binds.
|
||||
_, err = f.publish(t, cpAgentContext(publishCPAlpha), "alpha",
|
||||
`{"accountId":"`+authzAccountID+`","accountId":"`+publishOtherAcct+`","channel":"state","payload":{"x":1}}`)
|
||||
assertDenied(t, err)
|
||||
`{"accountId":"`+gwtest.AccountID+`","accountId":"`+publishOtherAcct+`","channel":"state","payload":{"x":1}}`)
|
||||
gwtest.AssertDenied(t, err)
|
||||
f.requireNothingPublished(t)
|
||||
|
||||
// The body has no cluster field; the route's cluster is the only one used.
|
||||
_, err = f.publish(t, cpAgentContext(publishCPAlpha), "alpha",
|
||||
`{"accountId":"`+authzAccountID+`","clusterName":"beta","channel":"state","payload":{"x":1}}`)
|
||||
`{"accountId":"`+gwtest.AccountID+`","clusterName":"beta","channel":"state","payload":{"x":1}}`)
|
||||
require.NoError(t, err)
|
||||
msg, err := f.sub.NextMsg(2 * time.Second)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ekscluster.StateSubject(authzAccountID, "alpha"), msg.Subject)
|
||||
assert.Equal(t, ekscluster.StateSubject(gwtest.AccountID, "alpha"), msg.Subject)
|
||||
f.requireNothingPublished(t)
|
||||
}
|
||||
|
||||
@@ -439,17 +464,17 @@ func TestEKSRequest_PublishInternalRejectsNonCPCallers(t *testing.T) {
|
||||
f := newPublishFixture(t)
|
||||
callers := []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
ctx credential.Principal
|
||||
}{
|
||||
{"tenant user, own account", userContext(authzAccountID, "alice")},
|
||||
{"tenant user, own account", userContext(gwtest.AccountID, "alice")},
|
||||
{"tenant user, another account", userContext(publishOtherAcct, "mallory")},
|
||||
{"static system credentials", userContext(awsidentifiers.GlobalAccountID, "admin")},
|
||||
}
|
||||
for _, ch := range publishChannels() {
|
||||
for _, caller := range callers {
|
||||
t.Run(ch.name+"/"+caller.name, func(t *testing.T) {
|
||||
_, err := f.publish(t, caller.ctx, "alpha", publishBody(authzAccountID, ch))
|
||||
assertDenied(t, err)
|
||||
_, err := f.publish(t, caller.ctx, "alpha", relayBody(gwtest.AccountID, ch))
|
||||
gwtest.AssertDenied(t, err)
|
||||
f.requireNothingPublished(t)
|
||||
})
|
||||
}
|
||||
@@ -460,11 +485,11 @@ func TestEKSRequest_PublishInternalRejectsNonCPCallers(t *testing.T) {
|
||||
// granting one principal a group named for its cluster, and a stand-in STS
|
||||
// verifier that records which cluster each review was pinned to.
|
||||
type tokenReviewFixture struct {
|
||||
gw *GatewayConfig
|
||||
gw *eksGateway
|
||||
verified chan string
|
||||
}
|
||||
|
||||
const tokenReviewPrincipal = "arn:aws:iam::" + authzAccountID + ":role/admin"
|
||||
const tokenReviewPrincipal = "arn:aws:iam::" + gwtest.AccountID + ":role/admin"
|
||||
|
||||
var tokenReviewToken = "k8s-aws-v1." + base64.RawURLEncoding.EncodeToString([]byte("https://sts/?Action=GetCallerIdentity"))
|
||||
|
||||
@@ -474,11 +499,11 @@ func newTokenReviewFixture(t *testing.T) *tokenReviewFixture {
|
||||
js, err := jetstream.New(f.nc)
|
||||
require.NoError(t, err)
|
||||
for _, c := range []struct{ acct, cluster string }{
|
||||
{authzAccountID, "alpha"}, {authzAccountID, "beta"}, {publishOtherAcct, "gamma"},
|
||||
{gwtest.AccountID, "alpha"}, {gwtest.AccountID, "beta"}, {publishOtherAcct, "gamma"},
|
||||
} {
|
||||
kv, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, c.acct)
|
||||
require.NoError(t, err)
|
||||
_, err = access.New(authzRegion).Create(t.Context(), kv, c.acct, access.Spec{
|
||||
_, err = access.New(gwtest.Region).Create(t.Context(), kv, c.acct, access.Spec{
|
||||
Cluster: c.cluster, PrincipalARN: tokenReviewPrincipal,
|
||||
Groups: []string{"group-" + c.cluster}, Type: access.EntryTypeStandard,
|
||||
})
|
||||
@@ -491,7 +516,7 @@ func newTokenReviewFixture(t *testing.T) *tokenReviewFixture {
|
||||
_ = json.Unmarshal(m.Data, &req)
|
||||
verified <- req.ClusterName
|
||||
resp, _ := json.Marshal(ekscluster.TokenVerifyResponse{
|
||||
AccountID: authzAccountID, ARN: tokenReviewPrincipal, UserID: "AROAEXAMPLE:session",
|
||||
AccountID: gwtest.AccountID, ARN: tokenReviewPrincipal, UserID: "AROAEXAMPLE:session",
|
||||
})
|
||||
_ = m.Respond(resp)
|
||||
})
|
||||
@@ -500,15 +525,15 @@ func newTokenReviewFixture(t *testing.T) *tokenReviewFixture {
|
||||
return &tokenReviewFixture{gw: f.gw, verified: verified}
|
||||
}
|
||||
|
||||
func tokenReviewBody(accountID string) string {
|
||||
func reviewBody(accountID string) string {
|
||||
return `{"accountId":"` + accountID + `","token":"` + tokenReviewToken + `"}`
|
||||
}
|
||||
|
||||
func (f *tokenReviewFixture) review(t *testing.T, ctx context.Context, cluster, body string) (*httptest.ResponseRecorder, error) {
|
||||
func (f *tokenReviewFixture) review(t *testing.T, caller credential.Principal, cluster, body string) (*httptest.ResponseRecorder, error) {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "/clusters/"+cluster+"/token-review", strings.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
return rec, f.gw.serveEKS(rec, req.WithContext(ctx))
|
||||
return rec, f.gw.as(caller).serve(rec, req)
|
||||
}
|
||||
|
||||
func (f *tokenReviewFixture) nextVerified(t *testing.T) string {
|
||||
@@ -545,7 +570,7 @@ func requireIndistinguishableDenial(t *testing.T, rec *httptest.ResponseRecorder
|
||||
// identity it always did, verified against that cluster.
|
||||
func TestEKSRequest_WebhookTokenReviewResolvesForTheCallersOwnCluster(t *testing.T) {
|
||||
f := newTokenReviewFixture(t)
|
||||
rec, err := f.review(t, cpAgentContext(publishCPAlpha), "alpha", tokenReviewBody(authzAccountID))
|
||||
rec, err := f.review(t, cpAgentContext(publishCPAlpha), "alpha", reviewBody(gwtest.AccountID))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.JSONEq(t, `{"authenticated":true,"username":"`+tokenReviewPrincipal+`",`+
|
||||
@@ -562,17 +587,17 @@ func TestEKSRequest_WebhookTokenReviewDeniesAClusterTheCallerDoesNotServe(t *tes
|
||||
targets := []struct {
|
||||
name, account, cluster string
|
||||
}{
|
||||
{"its own account, another cluster", authzAccountID, "beta"},
|
||||
{"its own account, another cluster", gwtest.AccountID, "beta"},
|
||||
{"another account's cluster", publishOtherAcct, "gamma"},
|
||||
{"its own cluster's name in another account", publishOtherAcct, "alpha"},
|
||||
{"a cluster that does not exist", authzAccountID, "delta"},
|
||||
{"a cluster that does not exist", gwtest.AccountID, "delta"},
|
||||
{"an account with no clusters", publishNoClusters, "alpha"},
|
||||
{"an account no bucket can be named for", publishOtherAcct + ".gamma", "alpha"},
|
||||
{"a cluster no key can be named for", authzAccountID, "*"},
|
||||
{"a cluster no key can be named for", gwtest.AccountID, "*"},
|
||||
}
|
||||
for _, target := range targets {
|
||||
t.Run(target.name, func(t *testing.T) {
|
||||
rec, err := f.review(t, cpAgentContext(publishCPAlpha), url.PathEscape(target.cluster), tokenReviewBody(target.account))
|
||||
rec, err := f.review(t, cpAgentContext(publishCPAlpha), url.PathEscape(target.cluster), reviewBody(target.account))
|
||||
requireIndistinguishableDenial(t, rec, err)
|
||||
f.requireNothingVerified(t)
|
||||
})
|
||||
@@ -584,11 +609,11 @@ func TestEKSRequest_WebhookTokenReviewDeniesAClusterTheCallerDoesNotServe(t *tes
|
||||
func TestEKSRequest_WebhookTokenReviewRedirectionIsDenied(t *testing.T) {
|
||||
f := newTokenReviewFixture(t)
|
||||
cases := []struct{ name, cluster, body string }{
|
||||
{"URL names its cluster, body another account", "alpha", tokenReviewBody(publishOtherAcct)},
|
||||
{"URL names its cluster, body another account", "alpha", reviewBody(publishOtherAcct)},
|
||||
{"re-cased key the handler folds onto accountId", "gamma",
|
||||
`{"AccountID":"` + publishOtherAcct + `","token":"` + tokenReviewToken + `"}`},
|
||||
{"repeated key, whose last value the handler reads", "gamma",
|
||||
`{"accountId":"` + authzAccountID + `","accountId":"` + publishOtherAcct + `","token":"` + tokenReviewToken + `"}`},
|
||||
`{"accountId":"` + gwtest.AccountID + `","accountId":"` + publishOtherAcct + `","token":"` + tokenReviewToken + `"}`},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -599,7 +624,7 @@ func TestEKSRequest_WebhookTokenReviewRedirectionIsDenied(t *testing.T) {
|
||||
}
|
||||
|
||||
rec, err := f.review(t, cpAgentContext(publishCPAlpha), "alpha",
|
||||
`{"accountId":"`+authzAccountID+`","clusterName":"beta","token":"`+tokenReviewToken+`"}`)
|
||||
`{"accountId":"`+gwtest.AccountID+`","clusterName":"beta","token":"`+tokenReviewToken+`"}`)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, rec.Body.String(), `"group-alpha"`)
|
||||
assert.Equal(t, "alpha", f.nextVerified(t))
|
||||
@@ -611,16 +636,16 @@ func TestEKSRequest_WebhookTokenReviewRejectsNonCPCallers(t *testing.T) {
|
||||
f := newTokenReviewFixture(t)
|
||||
cases := []struct {
|
||||
name, cluster, account string
|
||||
ctx context.Context
|
||||
ctx credential.Principal
|
||||
}{
|
||||
{"tenant, its own cluster", "alpha", authzAccountID, userContext(authzAccountID, "alice")},
|
||||
{"tenant, another account's cluster", "gamma", publishOtherAcct, userContext(authzAccountID, "alice")},
|
||||
{"other account's tenant, this cluster", "alpha", authzAccountID, userContext(publishOtherAcct, "mallory")},
|
||||
{"static system credentials", "alpha", authzAccountID, userContext(awsidentifiers.GlobalAccountID, "admin")},
|
||||
{"tenant, its own cluster", "alpha", gwtest.AccountID, userContext(gwtest.AccountID, "alice")},
|
||||
{"tenant, another account's cluster", "gamma", publishOtherAcct, userContext(gwtest.AccountID, "alice")},
|
||||
{"other account's tenant, this cluster", "alpha", gwtest.AccountID, userContext(publishOtherAcct, "mallory")},
|
||||
{"static system credentials", "alpha", gwtest.AccountID, userContext(awsidentifiers.GlobalAccountID, "admin")},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rec, err := f.review(t, tc.ctx, tc.cluster, tokenReviewBody(tc.account))
|
||||
rec, err := f.review(t, tc.ctx, tc.cluster, reviewBody(tc.account))
|
||||
requireIndistinguishableDenial(t, rec, err)
|
||||
f.requireNothingVerified(t)
|
||||
})
|
||||
@@ -631,12 +656,12 @@ func TestEKSRequest_WebhookTokenReviewRejectsNonCPCallers(t *testing.T) {
|
||||
// tenant sending one is refused on its class before the body matters.
|
||||
func TestEKSRequest_WebhookTokenReviewMalformedBody(t *testing.T) {
|
||||
f := newTokenReviewFixture(t)
|
||||
for _, body := range []string{`{not-json`, `{"token":"` + tokenReviewToken + `"}`, `{"accountId":"` + authzAccountID + `","token":""}`} {
|
||||
for _, body := range []string{`{not-json`, `{"token":"` + tokenReviewToken + `"}`, `{"accountId":"` + gwtest.AccountID + `","token":""}`} {
|
||||
_, err := f.review(t, cpAgentContext(publishCPAlpha), "alpha", body)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidParameterValue, awserrors.ValidErrorCodeFromError(err), "body %q", body)
|
||||
|
||||
rec, err := f.review(t, userContext(authzAccountID, "alice"), "alpha", body)
|
||||
rec, err := f.review(t, userContext(gwtest.AccountID, "alice"), "alpha", body)
|
||||
requireIndistinguishableDenial(t, rec, err)
|
||||
}
|
||||
f.requireNothingVerified(t)
|
||||
@@ -0,0 +1,243 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/auth"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/passrole"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
eksNodeRoleARN = "arn:aws:iam::" + gwtest.AccountID + ":role/eks/node-role"
|
||||
foreignAccountID = "999999999999"
|
||||
)
|
||||
|
||||
// roleStore resolves role names to the ARNs in roles, keyed by account then
|
||||
// name, the way the IAM store does.
|
||||
type roleStore struct {
|
||||
roles map[string]map[string]string
|
||||
canonicalErr error
|
||||
}
|
||||
|
||||
func (s *roleStore) ResolveRole(accountID, roleARN string) (passrole.Role, error) {
|
||||
roleAccount, name, err := auth.ParseRoleARN(roleARN)
|
||||
if err != nil {
|
||||
return passrole.Role{}, passrole.ErrMalformed
|
||||
}
|
||||
if roleAccount != accountID {
|
||||
return passrole.Role{}, passrole.ErrForeignAccount
|
||||
}
|
||||
canonical, err := s.canonical(roleAccount, name)
|
||||
if err != nil {
|
||||
if awserrors.IsErrorCode(err, awserrors.ErrorIAMNoSuchEntity) {
|
||||
return passrole.Role{}, passrole.ErrNotFound
|
||||
}
|
||||
return passrole.Role{}, err
|
||||
}
|
||||
if canonical != roleARN {
|
||||
return passrole.Role{}, passrole.ErrNotFound
|
||||
}
|
||||
return passrole.Role{ARN: canonical, Name: name}, nil
|
||||
}
|
||||
|
||||
func (s *roleStore) canonical(account, name string) (string, error) {
|
||||
if s.canonicalErr != nil {
|
||||
return "", s.canonicalErr
|
||||
}
|
||||
if roleARN, ok := s.roles[account][name]; ok {
|
||||
return roleARN, nil
|
||||
}
|
||||
return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
|
||||
}
|
||||
|
||||
// The foreign account holds a role of the same name, so only the account
|
||||
// check stops its ARN from resolving.
|
||||
func eksPassRole(statements ...iamdomain.Statement) *eksGateway {
|
||||
return eksPassRoleFaulting(nil, statements...)
|
||||
}
|
||||
|
||||
// eksPassRoleFaulting is eksPassRole with every role lookup failing with
|
||||
// canonicalErr when it is non-nil.
|
||||
func eksPassRoleFaulting(canonicalErr error, statements ...iamdomain.Statement) *eksGateway {
|
||||
store := &roleStore{canonicalErr: canonicalErr, roles: map[string]map[string]string{
|
||||
gwtest.AccountID: {"node-role": eksNodeRoleARN},
|
||||
foreignAccountID: {"node-role": "arn:aws:iam::" + foreignAccountID + ":role/eks/node-role"},
|
||||
}}
|
||||
g := scopedEKS(statements...)
|
||||
g.roles = store
|
||||
return g
|
||||
}
|
||||
|
||||
func createNodegroupBody(nodeRole string) string {
|
||||
return `{"nodegroupName":"workers","nodeRole":"` + nodeRole + `"}`
|
||||
}
|
||||
|
||||
func createAddonBody(roleARN string) string {
|
||||
return `{"addonName":"aws-load-balancer-controller","serviceAccountRoleArn":"` + roleARN + `"}`
|
||||
}
|
||||
|
||||
func updateAddonBody(roleARN string) string {
|
||||
return `{"serviceAccountRoleArn":"` + roleARN + `"}`
|
||||
}
|
||||
|
||||
// The AWS capture of 2026-10-30 refuses a foreign or non-role ARN on CreateCluster
|
||||
// and CreateNodegroup with AccessDeniedException before PassRole is evaluated.
|
||||
func assertCrossAccountPassRole(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
code, ok := awserrors.ResolveErrorCode(err)
|
||||
require.True(t, ok, "unresolvable error code: %v", err)
|
||||
assert.Equal(t, awserrors.ErrorAccessDeniedException, code)
|
||||
assert.Contains(t, err.Error(), "Cross-account pass role is not allowed.")
|
||||
}
|
||||
|
||||
func assertInvalidParameter(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
code, ok := awserrors.ResolveErrorCode(err)
|
||||
require.True(t, ok, "unresolvable error code: %v", err)
|
||||
assert.Equal(t, awserrors.ErrorEKSInvalidParameter, code)
|
||||
}
|
||||
|
||||
func TestEKSCreateAddon_NoRoleSkipsPassRole(t *testing.T) {
|
||||
gw := eksPassRole(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/addons", createAddonBody("")))
|
||||
}
|
||||
|
||||
func TestEKSCreateAddon_PassRole(t *testing.T) {
|
||||
body := createAddonBody(eksNodeRoleARN)
|
||||
|
||||
denied := eksPassRole(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, denied, http.MethodPost, "/clusters/prod/addons", body))
|
||||
|
||||
allowed := eksPassRole(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Allow", "iam:PassRole", eksNodeRoleARN),
|
||||
)
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, allowed, http.MethodPost, "/clusters/prod/addons", body))
|
||||
}
|
||||
|
||||
func TestEKSCreateCluster_PassRole(t *testing.T) {
|
||||
body := `{"name":"prod","roleArn":"` + eksNodeRoleARN + `"}`
|
||||
|
||||
denied := eksPassRole(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, denied, http.MethodPost, "/clusters", body))
|
||||
|
||||
allowed := eksPassRole(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Allow", "iam:PassRole", eksNodeRoleARN),
|
||||
)
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, allowed, http.MethodPost, "/clusters", body))
|
||||
}
|
||||
|
||||
func TestEKSCreateNodegroup_AllowedWithPassRoleOnRole(t *testing.T) {
|
||||
gw := eksPassRole(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Allow", "iam:PassRole", eksNodeRoleARN),
|
||||
)
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups", createNodegroupBody(eksNodeRoleARN)))
|
||||
}
|
||||
|
||||
// eks:* alone must not let a caller launch nodes under a role it cannot pass.
|
||||
func TestEKSCreateNodegroup_DeniedWithoutPassRole(t *testing.T) {
|
||||
gw := eksPassRole(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups", createNodegroupBody(eksNodeRoleARN)))
|
||||
}
|
||||
|
||||
func TestEKSCreateNodegroup_ForeignAccountRoleIsRefused(t *testing.T) {
|
||||
gw := eksPassRole(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Allow", "iam:PassRole", "*"),
|
||||
)
|
||||
err := dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups",
|
||||
createNodegroupBody("arn:aws:iam::"+foreignAccountID+":role/eks/node-role"))
|
||||
assertCrossAccountPassRole(t, err)
|
||||
}
|
||||
|
||||
// A grant scoped to a path the role is not under must not be satisfied by
|
||||
// spelling the role's name under that path.
|
||||
func TestEKSCreateNodegroup_InventedPathIsRefused(t *testing.T) {
|
||||
gw := eksPassRole(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Allow", "iam:PassRole", "arn:aws:iam::"+gwtest.AccountID+":role/sandbox/*"),
|
||||
)
|
||||
err := dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups",
|
||||
createNodegroupBody("arn:aws:iam::"+gwtest.AccountID+":role/sandbox/node-role"))
|
||||
assertInvalidParameter(t, err)
|
||||
}
|
||||
|
||||
// A store fault is not evidence the role is absent, so it must not surface as
|
||||
// a caller error or fall through to dispatch.
|
||||
func TestEKSCreateNodegroup_LookupFaultPassesThrough(t *testing.T) {
|
||||
storeErr := errors.New("get role: nats: timeout")
|
||||
gw := eksPassRoleFaulting(storeErr,
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Allow", "iam:PassRole", "*"),
|
||||
)
|
||||
err := dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups", createNodegroupBody(eksNodeRoleARN))
|
||||
require.ErrorIs(t, err, storeErr)
|
||||
}
|
||||
|
||||
func TestEKSCreateNodegroup_NoRoleSkipsPassRole(t *testing.T) {
|
||||
gw := eksPassRole(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups", `{"nodegroupName":"workers"}`))
|
||||
}
|
||||
|
||||
func TestEKSCreateNodegroup_NonRoleARNIsRefused(t *testing.T) {
|
||||
gw := eksPassRole(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Allow", "iam:PassRole", "*"),
|
||||
)
|
||||
err := dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups",
|
||||
createNodegroupBody("arn:aws:iam::"+gwtest.AccountID+":user/node-role"))
|
||||
assertCrossAccountPassRole(t, err)
|
||||
}
|
||||
|
||||
// Current behaviour: a store fault on the role lookup is returned before the
|
||||
// PassRole check, so a caller without iam:PassRole sees the fault, not a denial.
|
||||
func TestEKSCreateNodegroup_OutageIsReportedToACallerWithoutPassRole(t *testing.T) {
|
||||
storeErr := errors.New("get role: nats: timeout")
|
||||
gw := eksPassRoleFaulting(storeErr, gwtest.Statement("Allow", "eks:*", "*"))
|
||||
err := dispatchEKS(t, gw, http.MethodPost, "/clusters/prod/node-groups", createNodegroupBody(eksNodeRoleARN))
|
||||
require.ErrorIs(t, err, storeErr)
|
||||
}
|
||||
|
||||
// An unknown role is reported only to a caller allowed to pass it, so the
|
||||
// reply cannot be used to enumerate roles.
|
||||
func TestEKSCreateNodegroup_UnknownRole(t *testing.T) {
|
||||
body := createNodegroupBody("arn:aws:iam::" + gwtest.AccountID + ":role/ghost")
|
||||
|
||||
denied := eksPassRole(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, denied, http.MethodPost, "/clusters/prod/node-groups", body))
|
||||
|
||||
allowed := eksPassRole(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Allow", "iam:PassRole", "*"),
|
||||
)
|
||||
assertInvalidParameter(t, dispatchEKS(t, allowed, http.MethodPost, "/clusters/prod/node-groups", body))
|
||||
}
|
||||
|
||||
func TestEKSUpdateAddon_NoRoleSkipsPassRole(t *testing.T) {
|
||||
gw := eksPassRole(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, gw, http.MethodPost,
|
||||
"/clusters/prod/addons/aws-load-balancer-controller/update", updateAddonBody("")))
|
||||
}
|
||||
|
||||
func TestEKSUpdateAddon_PassRole(t *testing.T) {
|
||||
body := updateAddonBody(eksNodeRoleARN)
|
||||
const path = "/clusters/prod/addons/aws-load-balancer-controller/update"
|
||||
|
||||
denied := eksPassRole(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gwtest.AssertDenied(t, dispatchEKS(t, denied, http.MethodPost, path, body))
|
||||
|
||||
allowed := eksPassRole(
|
||||
gwtest.Statement("Allow", "eks:*", "*"),
|
||||
gwtest.Statement("Allow", "iam:PassRole", eksNodeRoleARN),
|
||||
)
|
||||
gwtest.AssertPermitted(t, dispatchEKS(t, allowed, http.MethodPost, path, body))
|
||||
}
|
||||
+28
-13
@@ -1,4 +1,4 @@
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,15 +9,38 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/eks"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
eksawsapi "github.com/mulgadc/spinifex/spinifex/domains/eks/awsapi"
|
||||
ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestEKSRequest_UnknownActionReturnsInvalidAction(t *testing.T) {
|
||||
gw := &eksGateway{Harness: gwtest.AllowAll()}
|
||||
gw.AccountID = "111122223333"
|
||||
req := httptest.NewRequest(http.MethodGet, "/totally-unknown", nil)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.serve(w, req)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidAction, err.Error())
|
||||
}
|
||||
|
||||
func TestEKSRequest_MissingNATSReturnsServerInternal(t *testing.T) {
|
||||
gw := &eksGateway{Harness: gwtest.AllowAll()}
|
||||
gw.AccountID = "111122223333"
|
||||
req := httptest.NewRequest(http.MethodGet, "/clusters", nil)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
err := gw.serve(w, req)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
}
|
||||
|
||||
// The body the CP agent receives is built by the SDK REST-JSON marshaller,
|
||||
// which ignores the json tags: field names are Go names and empty strings are
|
||||
// sent. The agent decodes it only because encoding/json matches case-insensitively.
|
||||
@@ -56,19 +79,11 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
gw := scopedPolicyGateway(statement("Allow", "eks:*", "*"))
|
||||
gw.NATSConn = nc
|
||||
gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
|
||||
gw.nc = nc
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/clusters/prod/internal-addons/"+authzAccountID, nil)
|
||||
ctx := context.WithValue(req.Context(), ctxService, "eks")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, awsidentifiers.GlobalAccountID)
|
||||
ctx = withPrincipalField(ctx, fieldIdentity, cpInstanceID)
|
||||
ctx = withPrincipalField(ctx, fieldPrincipalType, principalTypeAssumedRole)
|
||||
ctx = withPrincipalField(ctx, fieldUnderlyingRoleARN,
|
||||
"arn:aws:iam::"+awsidentifiers.GlobalAccountID+":role/"+ekscluster.CPInstanceRoleName)
|
||||
ctx = withPrincipalField(ctx, fieldAssumedRoleARN,
|
||||
"arn:aws:sts::"+awsidentifiers.GlobalAccountID+":assumed-role/"+ekscluster.CPInstanceRoleName+"/"+cpInstanceID)
|
||||
require.NoError(t, gw.serveEKS(rec, req.WithContext(ctx)))
|
||||
require.NoError(t, gw.as(cpAgentContext(cpInstanceID)).serve(rec, req))
|
||||
|
||||
assert.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Equal(t, `{"Addons":[`+
|
||||
+133
-64
@@ -1,94 +1,102 @@
|
||||
//test:in-package — drives the ELBv2 service through the gateway's unexported test
|
||||
// helpers (scopedPolicyGateway, withTestIdentity) and auth context keys.
|
||||
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/elbv2/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func elbv2LB(name, id string) string {
|
||||
return arn.FormatELBv2LoadBalancer(authzRegion, authzAccountID, name, id, "application")
|
||||
}
|
||||
|
||||
func elbv2TG(name, id string) string {
|
||||
return arn.FormatELBv2TargetGroup(authzRegion, authzAccountID, name, id)
|
||||
}
|
||||
|
||||
func elbv2Listener(lbName, lbID, id string) string {
|
||||
return arn.FormatELBv2Listener(authzRegion, authzAccountID, lbName, lbID, id, "application")
|
||||
}
|
||||
|
||||
// dispatchELBv2 drives the service with no NATS connection. A permitted request
|
||||
// therefore reaches the NATS guard and fails there, which is what proves the
|
||||
// policy check ran ahead of the resource having to exist.
|
||||
func dispatchELBv2(t *testing.T, gw *GatewayConfig, body string) error {
|
||||
func dispatchELBv2(t *testing.T, h *gwtest.Harness, body string) error {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
ctx := context.WithValue(req.Context(), ctxService, "elasticloadbalancing")
|
||||
ctx = withPrincipalField(ctx, fieldAccountID, authzAccountID)
|
||||
return gw.serveELBv2(httptest.NewRecorder(), withTestIdentity(req.WithContext(ctx)))
|
||||
return h.Dispatch(awsapi.NewRegistration(awsapi.Deps{}), awsapi.ServiceName, httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
func TestELBv2Request_MissingAction(t *testing.T) {
|
||||
err := dispatchELBv2(t, gwtest.AllowAll(), "")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMissingAction, err.Error())
|
||||
}
|
||||
|
||||
func TestELBv2Request_UnknownAction(t *testing.T) {
|
||||
err := dispatchELBv2(t, gwtest.AllowAll(), "Action=FakeAction")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidAction, err.Error())
|
||||
}
|
||||
|
||||
func elbv2LB(name, id string) string {
|
||||
return arn.FormatELBv2LoadBalancer(gwtest.Region, gwtest.AccountID, name, id, "application")
|
||||
}
|
||||
|
||||
func elbv2TG(name, id string) string {
|
||||
return arn.FormatELBv2TargetGroup(gwtest.Region, gwtest.AccountID, name, id)
|
||||
}
|
||||
|
||||
func elbv2Listener(lbName, lbID, id string) string {
|
||||
return arn.FormatELBv2Listener(gwtest.Region, gwtest.AccountID, lbName, lbID, id, "application")
|
||||
}
|
||||
|
||||
// TestELBv2Request_ScopedDenyFires is the bypass this work closes. An operator
|
||||
// fences a production load balancer; before the resolver the fence was inert
|
||||
// and DeleteLoadBalancer against it was permitted with nothing logged.
|
||||
func TestELBv2Request_ScopedDenyFires(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "elasticloadbalancing:*", "*"),
|
||||
statement("Deny", "elasticloadbalancing:DeleteLoadBalancer", elbv2LB("prod", "abc123")),
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "elasticloadbalancing:*", "*"),
|
||||
gwtest.Statement("Deny", "elasticloadbalancing:DeleteLoadBalancer", elbv2LB("prod", "abc123")),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchELBv2(t, gw, "Action=DeleteLoadBalancer&LoadBalancerArn="+elbv2LB("prod", "abc123")))
|
||||
assertPermitted(t, dispatchELBv2(t, gw, "Action=DeleteLoadBalancer&LoadBalancerArn="+elbv2LB("dev", "def456")))
|
||||
gwtest.AssertDenied(t, dispatchELBv2(t, gw, "Action=DeleteLoadBalancer&LoadBalancerArn="+elbv2LB("prod", "abc123")))
|
||||
gwtest.AssertPermitted(t, dispatchELBv2(t, gw, "Action=DeleteLoadBalancer&LoadBalancerArn="+elbv2LB("dev", "def456")))
|
||||
}
|
||||
|
||||
// TestELBv2Request_ScopedAllowGrants is the other half: a least-privilege
|
||||
// policy used to deny everything, so the only working shape was Resource "*".
|
||||
// The sibling is denied rather than not-found, because the gate runs first.
|
||||
func TestELBv2Request_ScopedAllowGrants(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "elasticloadbalancing:RegisterTargets", elbv2TG("web", "def456")),
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "elasticloadbalancing:RegisterTargets", elbv2TG("web", "def456")),
|
||||
)
|
||||
|
||||
assertPermitted(t, dispatchELBv2(t, gw, "Action=RegisterTargets&TargetGroupArn="+elbv2TG("web", "def456")))
|
||||
assertDenied(t, dispatchELBv2(t, gw, "Action=RegisterTargets&TargetGroupArn="+elbv2TG("other", "def456")))
|
||||
gwtest.AssertPermitted(t, dispatchELBv2(t, gw, "Action=RegisterTargets&TargetGroupArn="+elbv2TG("web", "def456")))
|
||||
gwtest.AssertDenied(t, dispatchELBv2(t, gw, "Action=RegisterTargets&TargetGroupArn="+elbv2TG("other", "def456")))
|
||||
}
|
||||
|
||||
// A create-time fence is what a name-prefix convention depends on, and it can
|
||||
// only fire because the create resolves to the name it asks for.
|
||||
func TestELBv2Request_CreateScopedByName(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "elasticloadbalancing:*", "*"),
|
||||
statement("Deny", "elasticloadbalancing:CreateLoadBalancer",
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "elasticloadbalancing:*", "*"),
|
||||
gwtest.Statement("Deny", "elasticloadbalancing:CreateLoadBalancer",
|
||||
"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/prod-*"),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchELBv2(t, gw, "Action=CreateLoadBalancer&Name=prod-web"))
|
||||
assertPermitted(t, dispatchELBv2(t, gw, "Action=CreateLoadBalancer&Name=dev-web"))
|
||||
gwtest.AssertDenied(t, dispatchELBv2(t, gw, "Action=CreateLoadBalancer&Name=prod-web"))
|
||||
gwtest.AssertPermitted(t, dispatchELBv2(t, gw, "Action=CreateLoadBalancer&Name=dev-web"))
|
||||
}
|
||||
|
||||
// A listener is denied on its target group even though its load balancer is
|
||||
// permitted: every resource AWS evaluates is evaluated here too.
|
||||
func TestELBv2Request_MultiResourceParity(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "elasticloadbalancing:*", "*"),
|
||||
statement("Deny", "elasticloadbalancing:CreateListener", elbv2TG("restricted", "1")),
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "elasticloadbalancing:*", "*"),
|
||||
gwtest.Statement("Deny", "elasticloadbalancing:CreateListener", elbv2TG("restricted", "1")),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchELBv2(t, gw,
|
||||
gwtest.AssertDenied(t, dispatchELBv2(t, gw,
|
||||
"Action=CreateListener&LoadBalancerArn="+elbv2LB("prod", "abc123")+
|
||||
"&DefaultActions.member.1.Type=forward&DefaultActions.member.1.TargetGroupArn="+elbv2TG("restricted", "1")))
|
||||
assertPermitted(t, dispatchELBv2(t, gw,
|
||||
gwtest.AssertPermitted(t, dispatchELBv2(t, gw,
|
||||
"Action=CreateListener&LoadBalancerArn="+elbv2LB("prod", "abc123")+
|
||||
"&DefaultActions.member.1.Type=forward&DefaultActions.member.1.TargetGroupArn="+elbv2TG("allowed", "2")))
|
||||
}
|
||||
@@ -96,29 +104,29 @@ func TestELBv2Request_MultiResourceParity(t *testing.T) {
|
||||
// A redirect-only default action names no target group, so the call stays
|
||||
// scoped to the load balancer rather than widening to "*".
|
||||
func TestELBv2Request_RedirectActionDoesNotWiden(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "elasticloadbalancing:CreateListener", elbv2LB("prod", "abc123")),
|
||||
statement("Allow", "elasticloadbalancing:CreateListener",
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "elasticloadbalancing:CreateListener", elbv2LB("prod", "abc123")),
|
||||
gwtest.Statement("Allow", "elasticloadbalancing:CreateListener",
|
||||
"arn:aws:elasticloadbalancing:*:*:listener/app/prod/abc123/*"),
|
||||
)
|
||||
|
||||
assertPermitted(t, dispatchELBv2(t, gw,
|
||||
gwtest.AssertPermitted(t, dispatchELBv2(t, gw,
|
||||
"Action=CreateListener&LoadBalancerArn="+elbv2LB("prod", "abc123")+
|
||||
"&DefaultActions.member.1.Type=redirect"))
|
||||
}
|
||||
|
||||
// One fenced member fails the batch whole, which is AWS's combining rule.
|
||||
func TestELBv2Request_DenyOneTagMemberFailsTheBatch(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "elasticloadbalancing:*", "*"),
|
||||
statement("Deny", "elasticloadbalancing:AddTags", elbv2LB("prod", "abc123")),
|
||||
gw := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "elasticloadbalancing:*", "*"),
|
||||
gwtest.Statement("Deny", "elasticloadbalancing:AddTags", elbv2LB("prod", "abc123")),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchELBv2(t, gw,
|
||||
gwtest.AssertDenied(t, dispatchELBv2(t, gw,
|
||||
"Action=AddTags&ResourceArns.member.1="+elbv2TG("web", "1")+
|
||||
"&ResourceArns.member.2="+elbv2LB("prod", "abc123")+
|
||||
"&Tags.member.1.Key=env&Tags.member.1.Value=prod"))
|
||||
assertPermitted(t, dispatchELBv2(t, gw,
|
||||
gwtest.AssertPermitted(t, dispatchELBv2(t, gw,
|
||||
"Action=AddTags&ResourceArns.member.1="+elbv2TG("web", "1")+
|
||||
"&Tags.member.1.Key=env&Tags.member.1.Value=prod"))
|
||||
}
|
||||
@@ -126,17 +134,17 @@ func TestELBv2Request_DenyOneTagMemberFailsTheBatch(t *testing.T) {
|
||||
// An ARN naming another account is rejected at the gate rather than authorized
|
||||
// verbatim and left for the handler to fail as a not-found.
|
||||
func TestELBv2Request_ForeignAccountARNRejected(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "elasticloadbalancing:*", "*"))
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "elasticloadbalancing:*", "*"))
|
||||
|
||||
err := dispatchELBv2(t, gw, "Action=DeleteLoadBalancer&LoadBalancerArn="+
|
||||
arn.FormatELBv2LoadBalancer(authzRegion, "999999999999", "prod", "abc123", "application"))
|
||||
arn.FormatELBv2LoadBalancer(gwtest.Region, "999999999999", "prod", "abc123", "application"))
|
||||
require.Error(t, err)
|
||||
code, ok := awserrors.ResolveErrorCode(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
|
||||
|
||||
err = dispatchELBv2(t, gw, "Action=DeleteLoadBalancer&LoadBalancerArn="+
|
||||
arn.FormatELBv2LoadBalancer("us-east-1", authzAccountID, "prod", "abc123", "application"))
|
||||
arn.FormatELBv2LoadBalancer("us-east-1", gwtest.AccountID, "prod", "abc123", "application"))
|
||||
require.Error(t, err)
|
||||
code, ok = awserrors.ResolveErrorCode(err)
|
||||
require.True(t, ok)
|
||||
@@ -146,22 +154,22 @@ func TestELBv2Request_ForeignAccountARNRejected(t *testing.T) {
|
||||
// An absent identifier authorizes account-wide, leaving the validation fault
|
||||
// where it belongs: with the handler.
|
||||
func TestELBv2Request_AbsentIdentifierStaysHandlerFault(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "elasticloadbalancing:*", "*"))
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "elasticloadbalancing:*", "*"))
|
||||
|
||||
assertPermitted(t, dispatchELBv2(t, gw, "Action=DeleteLoadBalancer"))
|
||||
assertDenied(t, dispatchELBv2(t,
|
||||
scopedPolicyGateway(statement("Allow", "elasticloadbalancing:*", elbv2LB("prod", "abc123"))),
|
||||
gwtest.AssertPermitted(t, dispatchELBv2(t, gw, "Action=DeleteLoadBalancer"))
|
||||
gwtest.AssertDenied(t, dispatchELBv2(t,
|
||||
gwtest.Scoped(gwtest.Statement("Allow", "elasticloadbalancing:*", elbv2LB("prod", "abc123"))),
|
||||
"Action=DeleteLoadBalancer"))
|
||||
}
|
||||
|
||||
// A missing account ID returns InternalError, which is the code the policy gate
|
||||
// returned before the read was hoisted above it.
|
||||
func TestELBv2Request_MissingAccountIDIsInternalError(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "elasticloadbalancing:*", "*"))
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "elasticloadbalancing:*", "*"))
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=DescribeLoadBalancers"))
|
||||
ctx := context.WithValue(req.Context(), ctxService, "elasticloadbalancing")
|
||||
err := gw.serveELBv2(httptest.NewRecorder(), withTestIdentity(req.WithContext(ctx)))
|
||||
gw.AccountID = ""
|
||||
|
||||
err := dispatchELBv2(t, gw, "Action=DescribeLoadBalancers")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInternalError, err.Error())
|
||||
@@ -170,12 +178,12 @@ func TestELBv2Request_MissingAccountIDIsInternalError(t *testing.T) {
|
||||
// Non-regression: an account-wide Allow still permits every scoped action once
|
||||
// a real ARN is passed, so scoping did not quietly deny working policies.
|
||||
func TestELBv2Request_AccountWideAllowStillPermitsEveryScopedAction(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "elasticloadbalancing:*", "*"))
|
||||
gw := gwtest.Scoped(gwtest.Statement("Allow", "elasticloadbalancing:*", "*"))
|
||||
|
||||
lb := elbv2LB("prod", "abc123")
|
||||
tg := elbv2TG("web", "def456")
|
||||
listener := elbv2Listener("prod", "abc123", "l1")
|
||||
rule := arn.FormatELBv2Resource(authzRegion, authzAccountID, arn.ELBv2ListenerRule, "app/prod/abc123/l1/r1")
|
||||
rule := arn.FormatELBv2Resource(gwtest.Region, gwtest.AccountID, arn.ELBv2ListenerRule, "app/prod/abc123/l1/r1")
|
||||
|
||||
bodies := map[string]string{
|
||||
"CreateLoadBalancer": "Action=CreateLoadBalancer&Name=prod",
|
||||
@@ -211,7 +219,7 @@ func TestELBv2Request_AccountWideAllowStillPermitsEveryScopedAction(t *testing.T
|
||||
}
|
||||
|
||||
// Every scoped action is covered, so a new one cannot land here untested.
|
||||
for _, action := range elbv2ServedActions() {
|
||||
for _, action := range awsapi.ActionNames() {
|
||||
if _, unscoped := elbv2UnscopedActions[action]; unscoped {
|
||||
continue
|
||||
}
|
||||
@@ -220,7 +228,7 @@ func TestELBv2Request_AccountWideAllowStillPermitsEveryScopedAction(t *testing.T
|
||||
|
||||
for action, body := range bodies {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
assertPermitted(t, dispatchELBv2(t, gw, body))
|
||||
gwtest.AssertPermitted(t, dispatchELBv2(t, gw, body))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -237,3 +245,64 @@ var elbv2UnscopedActions = map[string]struct{}{
|
||||
"LBAgentHeartbeat": {},
|
||||
"GetLBConfig": {},
|
||||
}
|
||||
|
||||
func TestELBv2ActionsMap_AllActionsRegistered(t *testing.T) {
|
||||
expectedActions := []string{
|
||||
"CreateLoadBalancer",
|
||||
"DeleteLoadBalancer",
|
||||
"DescribeLoadBalancers",
|
||||
"CreateTargetGroup",
|
||||
"ModifyTargetGroup",
|
||||
"DeleteTargetGroup",
|
||||
"DescribeTargetGroups",
|
||||
"RegisterTargets",
|
||||
"DeregisterTargets",
|
||||
"DescribeTargetHealth",
|
||||
"CreateListener",
|
||||
"DeleteListener",
|
||||
"ModifyListener",
|
||||
"DescribeListeners",
|
||||
"DescribeTags",
|
||||
"AddTags",
|
||||
"RemoveTags",
|
||||
"SetSecurityGroups",
|
||||
"SetIpAddressType",
|
||||
"SetSubnets",
|
||||
"LBAgentHeartbeat",
|
||||
"GetLBConfig",
|
||||
"ModifyTargetGroupAttributes",
|
||||
"DescribeTargetGroupAttributes",
|
||||
"ModifyLoadBalancerAttributes",
|
||||
"DescribeLoadBalancerAttributes",
|
||||
"DescribeListenerAttributes",
|
||||
"ModifyListenerAttributes",
|
||||
"CreateRule",
|
||||
"ModifyRule",
|
||||
"DeleteRule",
|
||||
"DescribeRules",
|
||||
"SetRulePriorities",
|
||||
"AddListenerCertificates",
|
||||
"RemoveListenerCertificates",
|
||||
"DescribeListenerCertificates",
|
||||
"DescribeSSLPolicies",
|
||||
"DescribeAccountLimits",
|
||||
}
|
||||
|
||||
served := awsapi.ActionNames()
|
||||
for _, action := range expectedActions {
|
||||
assert.Contains(t, served, action, "action %q should be served", action)
|
||||
}
|
||||
|
||||
assert.Len(t, served, len(expectedActions), "ELBv2 should serve exactly %d actions", len(expectedActions))
|
||||
}
|
||||
|
||||
func TestELBv2ActionsMap_UnknownActionNotRegistered(t *testing.T) {
|
||||
unknownActions := []string{
|
||||
"ModifyLoadBalancer",
|
||||
"RunInstances",
|
||||
}
|
||||
|
||||
for _, action := range unknownActions {
|
||||
assert.NotContains(t, awsapi.ActionNames(), action, "action %q should not be served", action)
|
||||
}
|
||||
}
|
||||
+126
-7
@@ -1,19 +1,138 @@
|
||||
//test:in-package — exercises IAM_Request with policy and auth context helpers.
|
||||
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
. "github.com/mulgadc/spinifex/spinifex/domains/iam/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
authzAccountID = gwtest.AccountID
|
||||
ec2ServicePrincipal = "ec2.amazonaws.com"
|
||||
ecsTasksServicePrincipal = "ecs-tasks.amazonaws.com"
|
||||
)
|
||||
|
||||
var statement = gwtest.Statement
|
||||
|
||||
type iamAuthzPolicyService struct {
|
||||
iamdomain.IAMService
|
||||
|
||||
arns map[string]string
|
||||
canonicalErr error
|
||||
deleteRoleErr error
|
||||
deleted []string
|
||||
}
|
||||
|
||||
func (s *iamAuthzPolicyService) CanonicalResourceARN(_ string, _ arn.IAMResourceType, name string) (string, error) {
|
||||
if s.canonicalErr != nil {
|
||||
return "", s.canonicalErr
|
||||
}
|
||||
resource, ok := s.arns[name]
|
||||
if !ok {
|
||||
return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
|
||||
}
|
||||
return resource, nil
|
||||
}
|
||||
|
||||
func (s *iamAuthzPolicyService) DeleteRole(_ string, input *iam.DeleteRoleInput) (*iam.DeleteRoleOutput, error) {
|
||||
s.deleted = append(s.deleted, *input.RoleName)
|
||||
if s.deleteRoleErr != nil {
|
||||
return nil, s.deleteRoleErr
|
||||
}
|
||||
return &iam.DeleteRoleOutput{}, nil
|
||||
}
|
||||
|
||||
// iamGateway pairs a caller holding statements with the service they act on.
|
||||
type iamGateway struct {
|
||||
h *gwtest.Harness
|
||||
svc iamdomain.IAMService
|
||||
}
|
||||
|
||||
func iamScopedGateway(resources map[string]string, statements ...iamdomain.Statement) (*iamGateway, *iamAuthzPolicyService) {
|
||||
svc := &iamAuthzPolicyService{arns: resources}
|
||||
return &iamGateway{h: gwtest.Scoped(statements...), svc: svc}, svc
|
||||
}
|
||||
|
||||
func dispatchIAM(t *testing.T, gw *iamGateway, body string) error {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
return gw.h.Dispatch(NewRegistration(Deps{IAM: gw.svc}), ServiceName, httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
func assertDenied(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
gwtest.AssertDenied(t, err)
|
||||
}
|
||||
|
||||
func TestIAMRequest_ScopedDenyFires(t *testing.T) {
|
||||
gw, svc := iamScopedGateway(map[string]string{
|
||||
"prod": "arn:aws:iam::123456789012:role/service-roles/prod",
|
||||
"dev": "arn:aws:iam::123456789012:role/service-roles/dev",
|
||||
},
|
||||
statement("Allow", "iam:*", "*"),
|
||||
statement("Deny", "iam:DeleteRole", "arn:aws:iam::*:role/service-roles/prod"),
|
||||
)
|
||||
|
||||
err := dispatchIAM(t, gw, "Action=DeleteRole&RoleName=prod")
|
||||
assertDenied(t, err)
|
||||
require.NoError(t, dispatchIAM(t, gw, "Action=DeleteRole&RoleName=dev"))
|
||||
assert.Equal(t, []string{"dev"}, svc.deleted)
|
||||
}
|
||||
|
||||
func TestIAMRequest_ScopedAllowGrants(t *testing.T) {
|
||||
gw, svc := iamScopedGateway(map[string]string{
|
||||
"prod": "arn:aws:iam::123456789012:role/service-roles/prod",
|
||||
"dev": "arn:aws:iam::123456789012:role/service-roles/dev",
|
||||
}, statement("Allow", "iam:DeleteRole", "arn:aws:iam::*:role/service-roles/dev"))
|
||||
|
||||
require.NoError(t, dispatchIAM(t, gw, "Action=DeleteRole&RoleName=dev"))
|
||||
err := dispatchIAM(t, gw, "Action=DeleteRole&RoleName=prod")
|
||||
assertDenied(t, err)
|
||||
assert.Equal(t, []string{"dev"}, svc.deleted)
|
||||
}
|
||||
|
||||
func TestIAMRequest_MissingTargetAndLookupFailure(t *testing.T) {
|
||||
t.Run("account-wide grant reaches handler no-such-entity", func(t *testing.T) {
|
||||
gw, svc := iamScopedGateway(nil, statement("Allow", "iam:*", "*"))
|
||||
svc.deleteRoleErr = errors.New(awserrors.ErrorIAMNoSuchEntity)
|
||||
err := dispatchIAM(t, gw, "Action=DeleteRole&RoleName=missing")
|
||||
require.EqualError(t, err, awserrors.ErrorIAMNoSuchEntity)
|
||||
})
|
||||
|
||||
t.Run("scoped grant denies unresolved target", func(t *testing.T) {
|
||||
gw, _ := iamScopedGateway(nil,
|
||||
statement("Allow", "iam:DeleteRole", "arn:aws:iam::*:role/missing"))
|
||||
err := dispatchIAM(t, gw, "Action=DeleteRole&RoleName=missing")
|
||||
assertDenied(t, err)
|
||||
})
|
||||
|
||||
t.Run("storage failure fails closed", func(t *testing.T) {
|
||||
gw, svc := iamScopedGateway(nil, statement("Allow", "iam:*", "*"))
|
||||
svc.canonicalErr = errors.New("storage unavailable")
|
||||
err := dispatchIAM(t, gw, "Action=DeleteRole&RoleName=app")
|
||||
require.EqualError(t, err, awserrors.ErrorInternalError)
|
||||
assert.Empty(t, svc.deleted)
|
||||
})
|
||||
}
|
||||
|
||||
func TestIAMRequest_MissingIdentifierStaysValidationFault(t *testing.T) {
|
||||
gw, _ := iamScopedGateway(nil, statement("Allow", "iam:*", "*"))
|
||||
err := dispatchIAM(t, gw, "Action=DeleteRole")
|
||||
require.EqualError(t, err, awserrors.ErrorMissingParameter)
|
||||
}
|
||||
|
||||
const (
|
||||
passRoleProfileARN = "arn:aws:iam::" + authzAccountID + ":instance-profile/web"
|
||||
passRoleAdminARN = "arn:aws:iam::" + authzAccountID + ":role/ops/admin"
|
||||
@@ -41,13 +160,13 @@ func (s *addRoleIAMService) AddRoleToInstanceProfile(_ string, input *iam.AddRol
|
||||
return &iam.AddRoleToInstanceProfileOutput{}, nil
|
||||
}
|
||||
|
||||
func addRoleGateway(statements ...iamdomain.Statement) (*GatewayConfig, *addRoleIAMService) {
|
||||
func addRoleGateway(statements ...iamdomain.Statement) (*iamGateway, *addRoleIAMService) {
|
||||
base, inner := iamScopedGateway(map[string]string{
|
||||
"web": passRoleProfileARN,
|
||||
"admin": passRoleAdminARN,
|
||||
}, statements...)
|
||||
svc := &addRoleIAMService{iamAuthzPolicyService: *inner}
|
||||
base.IAMService = svc
|
||||
base.svc = svc
|
||||
return base, svc
|
||||
}
|
||||
|
||||
@@ -89,14 +208,14 @@ func TestAddRoleToInstanceProfile_PassRoleOnCanonicalPath(t *testing.T) {
|
||||
func TestAddRoleToInstanceProfile_PassedToServiceIsEC2(t *testing.T) {
|
||||
ec2Grant, svc := addRoleGateway(
|
||||
statement("Allow", "iam:AddRoleToInstanceProfile", "*"),
|
||||
passRoleTo("Allow", "StringEquals", ec2ServicePrincipal),
|
||||
gwtest.PassRoleTo("Allow", "StringEquals", ec2ServicePrincipal),
|
||||
)
|
||||
require.NoError(t, dispatchIAM(t, ec2Grant, addAdminToWeb))
|
||||
assert.Equal(t, []string{"admin"}, svc.added)
|
||||
|
||||
ecsGrant, _ := addRoleGateway(
|
||||
statement("Allow", "iam:AddRoleToInstanceProfile", "*"),
|
||||
passRoleTo("Allow", "StringEquals", ecsTasksServicePrincipal),
|
||||
gwtest.PassRoleTo("Allow", "StringEquals", ecsTasksServicePrincipal),
|
||||
)
|
||||
assertDenied(t, dispatchIAM(t, ecsGrant, addAdminToWeb))
|
||||
}
|
||||
+55
-69
@@ -1,20 +1,18 @@
|
||||
//test:in-package — drives the four Bedrock-family dispatchers through the
|
||||
// gateway's unexported test helpers (withTestIdentity, policyMockIAMService)
|
||||
// and auth context keys.
|
||||
|
||||
package gateway
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
ochreawsapi "github.com/mulgadc/spinifex/spinifex/domains/ochre/awsapi"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ochre/awsapi"
|
||||
stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/pipeline"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -23,129 +21,112 @@ import (
|
||||
// dispatcher with no stores behind it. A request that clears the policy gate
|
||||
// reaches the handler and fails there, which is what proves the gate ran
|
||||
// ahead of the resource existing.
|
||||
func dispatchBedrockFamily(t *testing.T, gw *GatewayConfig, service, method, path, body string) error {
|
||||
func dispatchBedrockFamily(t *testing.T, h *gwtest.Harness, service, method, path, body string) error {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(method, path, strings.NewReader(body))
|
||||
ctx := context.WithValue(req.Context(), ctxService, service)
|
||||
req = withTestIdentity(req.WithContext(withPrincipalField(ctx, fieldAccountID, authzAccountID)))
|
||||
return serveBedrock(gw, service, httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
func serveBedrock(gw *GatewayConfig, service string, w http.ResponseWriter, req *http.Request) error {
|
||||
if gw.Services == nil {
|
||||
b := dispatch.NewBuilder()
|
||||
if err := b.Register(ochreawsapi.NewRegistration(ochreawsapi.Deps{})); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
gw.Services = b.Build()
|
||||
}
|
||||
entry, ok := gw.registered(service)
|
||||
if !ok {
|
||||
panic(service + " is not registered on this gateway")
|
||||
}
|
||||
return gw.dispatchRegistered(entry, w, req)
|
||||
return h.Dispatch(awsapi.NewRegistration(awsapi.Deps{}), service, httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
func bedrockScopeARN(resource string) string {
|
||||
return "arn:aws:bedrock:" + authzRegion + ":" + authzAccountID + ":" + resource
|
||||
return "arn:aws:bedrock:" + gwtest.Region + ":" + gwtest.AccountID + ":" + resource
|
||||
}
|
||||
|
||||
// TestBedrockRequest_ScopedDenyFires is the bypass this work closes. An operator
|
||||
// fences a production guardrail; before the resolver the fence was inert and
|
||||
// DeleteGuardrail against it was permitted with nothing logged.
|
||||
func TestBedrockRequest_ScopedDenyFires(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "bedrock:*", "*"),
|
||||
statement("Deny", "bedrock:DeleteGuardrail", bedrockScopeARN("guardrail/gr-prod")),
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "bedrock:*", "*"),
|
||||
gwtest.Statement("Deny", "bedrock:DeleteGuardrail", bedrockScopeARN("guardrail/gr-prod")),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchBedrockFamily(t, gw, "bedrock", http.MethodDelete, "/guardrails/gr-prod", ""))
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, gw, "bedrock", http.MethodDelete, "/guardrails/gr-dev", ""))
|
||||
gwtest.AssertDenied(t, dispatchBedrockFamily(t, h, "bedrock", http.MethodDelete, "/guardrails/gr-prod", ""))
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, h, "bedrock", http.MethodDelete, "/guardrails/gr-dev", ""))
|
||||
}
|
||||
|
||||
// TestBedrockRequest_ScopedAllowGrants is the other half: a least-privilege
|
||||
// policy used to deny everything, so the only working shape was Resource "*".
|
||||
func TestBedrockRequest_ScopedAllowGrants(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "bedrock:GetProvisionedModelThroughput", bedrockScopeARN("provisioned-model/pm-dev")),
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "bedrock:GetProvisionedModelThroughput", bedrockScopeARN("provisioned-model/pm-dev")),
|
||||
)
|
||||
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, gw, "bedrock",
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, h, "bedrock",
|
||||
http.MethodGet, "/provisioned-model-throughput/pm-dev", ""))
|
||||
assertDenied(t, dispatchBedrockFamily(t, gw, "bedrock",
|
||||
gwtest.AssertDenied(t, dispatchBedrockFamily(t, h, "bedrock",
|
||||
http.MethodGet, "/provisioned-model-throughput/pm-prod", ""))
|
||||
}
|
||||
|
||||
// The foundation model a commitment names arrives in the body, so the body read
|
||||
// has to happen before the gate for a fence on the model to fire at all.
|
||||
func TestBedrockRequest_CreateProvisionedThroughputScopesFromBody(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "bedrock:*", "*"),
|
||||
statement("Deny", "bedrock:CreateProvisionedModelThroughput",
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "bedrock:*", "*"),
|
||||
gwtest.Statement("Deny", "bedrock:CreateProvisionedModelThroughput",
|
||||
"arn:aws:bedrock:*::foundation-model/anthropic.*"),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchBedrockFamily(t, gw, "bedrock", http.MethodPost,
|
||||
gwtest.AssertDenied(t, dispatchBedrockFamily(t, h, "bedrock", http.MethodPost,
|
||||
"/provisioned-model-throughput", `{"modelId":"anthropic.claude-v2"}`))
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, gw, "bedrock", http.MethodPost,
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, h, "bedrock", http.MethodPost,
|
||||
"/provisioned-model-throughput", `{"modelId":"meta.llama3"}`))
|
||||
// An unreadable body authorizes account-wide and stays the handler's fault.
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, gw, "bedrock", http.MethodPost,
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, h, "bedrock", http.MethodPost,
|
||||
"/provisioned-model-throughput", "{not json"))
|
||||
}
|
||||
|
||||
// bedrock-runtime names its model in the path, so a fence on a model prefix
|
||||
// fences inference against it.
|
||||
func TestBedrockRuntimeRequest_ScopedDenyFires(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "bedrock:*", "*"),
|
||||
statement("Deny", "bedrock:InvokeModel", "arn:aws:bedrock:*::foundation-model/anthropic.*"),
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "bedrock:*", "*"),
|
||||
gwtest.Statement("Deny", "bedrock:InvokeModel", "arn:aws:bedrock:*::foundation-model/anthropic.*"),
|
||||
)
|
||||
|
||||
assertDenied(t, dispatchBedrockFamily(t, gw, "bedrock-runtime",
|
||||
gwtest.AssertDenied(t, dispatchBedrockFamily(t, h, "bedrock-runtime",
|
||||
http.MethodPost, "/model/anthropic.claude-v2/invoke", `{}`))
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, gw, "bedrock-runtime",
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, h, "bedrock-runtime",
|
||||
http.MethodPost, "/model/meta.llama3/invoke", `{}`))
|
||||
}
|
||||
|
||||
// bedrock-agent's data-source and ingestion-job actions are evaluated against
|
||||
// the knowledge base, so a fence on the base covers them.
|
||||
func TestBedrockAgentRequest_ScopedDenyCoversTheWholeKnowledgeBase(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "bedrock:*", "*"),
|
||||
statement("Deny", "bedrock:StartIngestionJob", bedrockScopeARN("knowledge-base/kb-prod")),
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "bedrock:*", "*"),
|
||||
gwtest.Statement("Deny", "bedrock:StartIngestionJob", bedrockScopeARN("knowledge-base/kb-prod")),
|
||||
)
|
||||
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, gw, "bedrock-agent", http.MethodPut,
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, h, "bedrock-agent", http.MethodPut,
|
||||
"/knowledgebases/kb-dev/datasources/ds-1/ingestionjobs/", `{}`))
|
||||
assertDenied(t, dispatchBedrockFamily(t, gw, "bedrock-agent", http.MethodPut,
|
||||
gwtest.AssertDenied(t, dispatchBedrockFamily(t, h, "bedrock-agent", http.MethodPut,
|
||||
"/knowledgebases/kb-prod/datasources/ds-1/ingestionjobs/", `{}`))
|
||||
}
|
||||
|
||||
// RetrieveAndGenerate names its knowledge base in the body, two levels down.
|
||||
func TestBedrockAgentRuntimeRequest_RetrieveAndGenerateScopesFromBody(t *testing.T) {
|
||||
gw := scopedPolicyGateway(
|
||||
statement("Allow", "bedrock:*", "*"),
|
||||
statement("Deny", "bedrock:RetrieveAndGenerate", bedrockScopeARN("knowledge-base/kb-prod")),
|
||||
h := gwtest.Scoped(
|
||||
gwtest.Statement("Allow", "bedrock:*", "*"),
|
||||
gwtest.Statement("Deny", "bedrock:RetrieveAndGenerate", bedrockScopeARN("knowledge-base/kb-prod")),
|
||||
)
|
||||
body := func(kb string) string {
|
||||
return `{"input":{"text":"q"},"retrieveAndGenerateConfiguration":{"type":"KNOWLEDGE_BASE",` +
|
||||
`"knowledgeBaseConfiguration":{"knowledgeBaseId":"` + kb + `","modelArn":"meta.llama3"}}}`
|
||||
}
|
||||
|
||||
assertDenied(t, dispatchBedrockFamily(t, gw, "bedrock-agent-runtime",
|
||||
gwtest.AssertDenied(t, dispatchBedrockFamily(t, h, "bedrock-agent-runtime",
|
||||
http.MethodPost, "/retrieveAndGenerate", body("kb-prod")))
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, gw, "bedrock-agent-runtime",
|
||||
assertBedrockPermitted(t, dispatchBedrockFamily(t, h, "bedrock-agent-runtime",
|
||||
http.MethodPost, "/retrieveAndGenerate", body("kb-dev")))
|
||||
}
|
||||
|
||||
// A body past the signed path's cap cannot be used to bypass the gate or to
|
||||
// exhaust memory: it is rejected before either the gate or the handler.
|
||||
func TestBedrockRequest_OversizedBodyIsRejected(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "bedrock:*", "*"))
|
||||
h := gwtest.Scoped(gwtest.Statement("Allow", "bedrock:*", "*"))
|
||||
body := `{"modelId":"` + strings.Repeat("a", sigv4.MaxPayloadLen) + `"}`
|
||||
|
||||
err := dispatchBedrockFamily(t, gw, "bedrock", http.MethodPost,
|
||||
err := dispatchBedrockFamily(t, h, "bedrock", http.MethodPost,
|
||||
"/provisioned-model-throughput", body)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorRequestEntityTooLarge, err.Error())
|
||||
@@ -154,19 +135,24 @@ func TestBedrockRequest_OversizedBodyIsRejected(t *testing.T) {
|
||||
// The property the per-service rollout rests on: passing a real ARN where "*"
|
||||
// was passed before cannot withdraw access a working policy already grants.
|
||||
func TestBedrockFamily_AccountWideGrantStillPermitsEveryAction(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "bedrock:*", "*"))
|
||||
req := withTestIdentity(httptest.NewRequest(http.MethodGet, "/", nil).
|
||||
WithContext(withPrincipalField(context.Background(), fieldAccountID, authzAccountID)))
|
||||
h := gwtest.Scoped(gwtest.Statement("Allow", "bedrock:*", "*"))
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
caller := credential.Principal{
|
||||
AccountID: gwtest.AccountID,
|
||||
Identity: "alice",
|
||||
PrincipalType: stsdomain.PrincipalTypeUser,
|
||||
CallerARN: "arn:aws:iam::" + gwtest.AccountID + ":user/alice",
|
||||
}
|
||||
body := []byte(`{"modelId":"meta.llama3","retrieveAndGenerateConfiguration":` +
|
||||
`{"knowledgeBaseConfiguration":{"knowledgeBaseId":"kb-1"}}}`)
|
||||
|
||||
for _, service := range []string{"bedrock", "bedrock-runtime", "bedrock-agent", "bedrock-agent-runtime"} {
|
||||
for _, action := range ochreawsapi.ScopedActions(service) {
|
||||
for _, action := range awsapi.ScopedActions(service) {
|
||||
t.Run(service+"/"+action, func(t *testing.T) {
|
||||
resources, err := ochreawsapi.ResourceARNs(service, action, authzRegion, authzAccountID,
|
||||
resources, err := awsapi.ResourceARNs(service, action, gwtest.Region, gwtest.AccountID,
|
||||
[]string{"id-1", "id-2", "id-3"}, body)
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, gw.checkPolicyResources(req, service, action, resources))
|
||||
assert.NoError(t, pipeline.Authorize(h.Policy, req, caller, true, service, action, resources, nil))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -175,7 +161,8 @@ func TestBedrockFamily_AccountWideGrantStillPermitsEveryAction(t *testing.T) {
|
||||
// The account-ID read moved above the gate, which used to reject a missing
|
||||
// account itself. InternalError is the code the caller has always seen.
|
||||
func TestBedrockFamily_MissingAccountIDReturnsInternalError(t *testing.T) {
|
||||
gw := scopedPolicyGateway(statement("Allow", "bedrock:*", "*"))
|
||||
h := gwtest.Scoped(gwtest.Statement("Allow", "bedrock:*", "*"))
|
||||
h.AccountID = ""
|
||||
dispatchers := map[string]struct {
|
||||
method string
|
||||
path string
|
||||
@@ -189,8 +176,7 @@ func TestBedrockFamily_MissingAccountIDReturnsInternalError(t *testing.T) {
|
||||
for service, d := range dispatchers {
|
||||
t.Run(service, func(t *testing.T) {
|
||||
req := httptest.NewRequest(d.method, d.path, strings.NewReader("{}"))
|
||||
req = withTestIdentity(req.WithContext(context.WithValue(req.Context(), ctxService, service)))
|
||||
err := serveBedrock(gw, service, httptest.NewRecorder(), req)
|
||||
err := h.Dispatch(awsapi.NewRegistration(awsapi.Deps{}), service, httptest.NewRecorder(), req)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInternalError, err.Error())
|
||||
})
|
||||
@@ -0,0 +1,359 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit/gwtest"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
rdsawsapi "github.com/mulgadc/spinifex/spinifex/domains/rds/awsapi"
|
||||
rdsinstance "github.com/mulgadc/spinifex/spinifex/domains/rds/instance"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const rdsTestAccountID = gwtest.AccountID
|
||||
|
||||
// Counts policy resolutions, so a test can tell an authorized request from one
|
||||
// that failed some other guard before the evaluator was ever reached.
|
||||
type rdsPolicyProbe struct{ consulted int }
|
||||
|
||||
// serveRDS dispatches body to a registration with no NATS connection, as the
|
||||
// pipeline would for the harness's caller.
|
||||
func serveRDS(h *gwtest.Harness, body string) error {
|
||||
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
return h.Dispatch(rdsawsapi.NewRegistration(rdsawsapi.Deps{}), rdsawsapi.ServiceName, httptest.NewRecorder(), req)
|
||||
}
|
||||
|
||||
// A harness whose IAM service answers with policies, so the RDS dispatch path
|
||||
// runs a real evaluation for either principal branch.
|
||||
func newRDSPolicyHarness(policies []iamdomain.PolicyDocument) (*gwtest.Harness, *rdsPolicyProbe) {
|
||||
return newRDSHarnessWithResolver(func(_, _ string) ([]iamdomain.PolicyDocument, error) {
|
||||
return policies, nil
|
||||
})
|
||||
}
|
||||
|
||||
func newRDSHarnessWithResolver(
|
||||
resolve func(accountID, name string) ([]iamdomain.PolicyDocument, error),
|
||||
) (*gwtest.Harness, *rdsPolicyProbe) {
|
||||
probe := &rdsPolicyProbe{}
|
||||
// Both branches are wired: an assumed-role principal resolves by role, and a
|
||||
// test that only stubbed users would miss it entirely.
|
||||
return gwtest.FromResolver(func(accountID, name string) ([]iamdomain.PolicyDocument, error) {
|
||||
probe.consulted++
|
||||
return resolve(accountID, name)
|
||||
}), probe
|
||||
}
|
||||
|
||||
// An in-guest agent's request: an assumed-role session under the system account
|
||||
// whose underlying role is the DB VM instance role.
|
||||
func asRDSAgent(h *gwtest.Harness, sessionName string) {
|
||||
roleARN := "arn:aws:iam::" + awsidentifiers.GlobalAccountID + ":role/" + rdsinstance.InstanceRoleName
|
||||
assumed := "arn:aws:sts::" + awsidentifiers.GlobalAccountID + ":assumed-role/" + rdsinstance.InstanceRoleName + "/" + sessionName
|
||||
h.AccountID = awsidentifiers.GlobalAccountID
|
||||
h.Caller = &credential.Principal{
|
||||
AccountID: awsidentifiers.GlobalAccountID,
|
||||
Identity: sessionName,
|
||||
PrincipalType: "assumed-role",
|
||||
UnderlyingRoleARN: roleARN,
|
||||
AssumedRoleARN: assumed,
|
||||
CallerARN: assumed,
|
||||
}
|
||||
}
|
||||
|
||||
func allowRDSPolicy(action string) []iamdomain.PolicyDocument {
|
||||
return allowRDSPolicyResource(action, "*")
|
||||
}
|
||||
|
||||
func allowRDSPolicyResource(action, resource string) []iamdomain.PolicyDocument {
|
||||
return []iamdomain.PolicyDocument{{
|
||||
Version: "2012-10-17",
|
||||
Statement: []iamdomain.Statement{{
|
||||
Effect: "Allow",
|
||||
Action: iamdomain.StringOrArr{action},
|
||||
Resource: iamdomain.StringOrArr{resource},
|
||||
}},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestRDSRequest_MissingAction(t *testing.T) {
|
||||
err := serveRDS(gwtest.AllowAll(), "")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMissingAction, err.Error())
|
||||
}
|
||||
|
||||
func TestRDSRequest_UnknownAction(t *testing.T) {
|
||||
err := serveRDS(gwtest.AllowAll(), "Action=FakeAction")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidAction, err.Error())
|
||||
}
|
||||
|
||||
// A malformed percent-encoding is a client error, not an internal one.
|
||||
func TestRDSRequest_MalformedQueryString(t *testing.T) {
|
||||
err := serveRDS(gwtest.AllowAll(), "Action=%zz")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorMalformedQueryString, err.Error())
|
||||
}
|
||||
|
||||
// A known action still needs a NATS connection to reach the control plane, so
|
||||
// the disconnected case must fail rather than answer from the gateway alone.
|
||||
func TestRDSRequest_NoNATSConn(t *testing.T) {
|
||||
err := serveRDS(gwtest.AllowAll(), "Action=DescribeDBInstances")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
}
|
||||
|
||||
// The headline of this surface: rds:* on * is the natural admin grant, and it
|
||||
// must not reach an internal agent action — GetDBBootstrapConfig returns the
|
||||
// database master password.
|
||||
func TestRDSRequest_InternalActionsDeniedToCustomerAdmin(t *testing.T) {
|
||||
h, _ := newRDSPolicyHarness(allowRDSPolicy("rds:*"))
|
||||
for _, action := range rdsinstance.InternalAgentActions {
|
||||
t.Run(action, func(t *testing.T) {
|
||||
err := serveRDS(h, "Action="+action)
|
||||
require.Error(t, err)
|
||||
gwtest.AssertDenied(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A DB VM's own session passes the class gate and is evaluated against the
|
||||
// four-action role policy, which admits it: the only thing left to fail on is the
|
||||
// control plane, absent here.
|
||||
func TestRDSRequest_InternalActionAdmitsTheAgentPrincipal(t *testing.T) {
|
||||
h, probe := newRDSPolicyHarness(allowRDSPolicy("rds:GetDBBootstrapConfig"))
|
||||
asRDSAgent(h, "i-0abc123")
|
||||
err := serveRDS(h, "Action=GetDBBootstrapConfig")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 1, probe.consulted, "the agent's role policy must be evaluated, not bypassed")
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error(),
|
||||
"the agent must be authorized and fail only on the missing NATS connection")
|
||||
}
|
||||
|
||||
// The instance role grants only the internal actions, so the same credentials
|
||||
// that register an instance cannot describe the account's fleet.
|
||||
func TestRDSRequest_AgentRoleCannotCallCustomerActions(t *testing.T) {
|
||||
h, _ := newRDSPolicyHarness(allowRDSPolicy("rds:GetDBBootstrapConfig"))
|
||||
asRDSAgent(h, "i-0abc123")
|
||||
err := serveRDS(h, "Action=DescribeDBInstances")
|
||||
require.Error(t, err)
|
||||
gwtest.AssertDenied(t, err)
|
||||
}
|
||||
|
||||
// A policy scoped to db:prod-* has to actually restrict: checked against "*" it
|
||||
// would read as a restriction and permit the whole account.
|
||||
func TestRDSRequest_ResourceScopedPolicyRestrictsByIdentifier(t *testing.T) {
|
||||
h, probe := newRDSPolicyHarness(allowRDSPolicyResource("rds:*", "arn:aws:rds:*:*:db:prod-*"))
|
||||
|
||||
err := serveRDS(h, "Action=DeleteDBInstance&DBInstanceIdentifier=prod-orders")
|
||||
require.Error(t, err)
|
||||
// The probe is what makes the pass meaningful: a guard that failed before the
|
||||
// evaluator would leave the same error with no policy consulted at all.
|
||||
assert.Equal(t, 1, probe.consulted)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error(),
|
||||
"the in-scope instance must be authorized and fail only on the missing NATS connection")
|
||||
|
||||
err = serveRDS(h, "Action=DeleteDBInstance&DBInstanceIdentifier=dev-orders")
|
||||
require.Error(t, err)
|
||||
gwtest.AssertDenied(t, err)
|
||||
}
|
||||
|
||||
func TestRDSRequest_SnapshotActionsAuthorizeSourceAndTarget(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
action string
|
||||
query string
|
||||
resources iamdomain.StringOrArr
|
||||
}{
|
||||
{
|
||||
name: "create snapshot",
|
||||
action: "rds:CreateDBSnapshot",
|
||||
query: "Action=CreateDBSnapshot&DBInstanceIdentifier=orders-db" +
|
||||
"&DBSnapshotIdentifier=orders-db-nightly",
|
||||
resources: iamdomain.StringOrArr{
|
||||
rdsinstance.FormatARN(rdsinstance.ResourceKindDBInstance, "ap-southeast-2", rdsTestAccountID, "orders-db"),
|
||||
rdsinstance.FormatARN(rdsinstance.ResourceKindDBSnapshot, "ap-southeast-2", rdsTestAccountID, "orders-db-nightly"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "restore snapshot",
|
||||
action: "rds:RestoreDBInstanceFromDBSnapshot",
|
||||
query: "Action=RestoreDBInstanceFromDBSnapshot&DBSnapshotIdentifier=orders-db-nightly" +
|
||||
"&DBInstanceIdentifier=orders-db-restored",
|
||||
resources: iamdomain.StringOrArr{
|
||||
rdsinstance.FormatARN(rdsinstance.ResourceKindDBSnapshot, "ap-southeast-2", rdsTestAccountID, "orders-db-nightly"),
|
||||
rdsinstance.FormatARN(rdsinstance.ResourceKindDBInstance, "ap-southeast-2", rdsTestAccountID, "orders-db-restored"),
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
policies := []iamdomain.PolicyDocument{{
|
||||
Version: "2012-10-17",
|
||||
Statement: []iamdomain.Statement{{
|
||||
Effect: "Allow", Action: iamdomain.StringOrArr{tt.action}, Resource: tt.resources,
|
||||
}},
|
||||
}}
|
||||
h, probe := newRDSPolicyHarness(policies)
|
||||
err := serveRDS(h, tt.query)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 1, probe.consulted)
|
||||
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRDSRequest_SnapshotActionsUseOnePolicySnapshot(t *testing.T) {
|
||||
source := rdsinstance.FormatARN(rdsinstance.ResourceKindDBInstance, "ap-southeast-2", rdsTestAccountID, "orders-db")
|
||||
target := rdsinstance.FormatARN(rdsinstance.ResourceKindDBSnapshot, "ap-southeast-2", rdsTestAccountID, "orders-db-nightly")
|
||||
resolution := 0
|
||||
h, probe := newRDSHarnessWithResolver(func(_, _ string) ([]iamdomain.PolicyDocument, error) {
|
||||
resolution++
|
||||
if resolution == 1 {
|
||||
return allowRDSPolicyResource("rds:CreateDBSnapshot", source), nil
|
||||
}
|
||||
return allowRDSPolicyResource("rds:CreateDBSnapshot", target), nil
|
||||
})
|
||||
|
||||
err := serveRDS(h, "Action=CreateDBSnapshot&DBInstanceIdentifier=orders-db&DBSnapshotIdentifier=orders-db-nightly")
|
||||
require.Error(t, err)
|
||||
gwtest.AssertDenied(t, err)
|
||||
assert.Equal(t, 1, probe.consulted)
|
||||
}
|
||||
|
||||
func TestRDSRequest_SnapshotActionsHonorTargetDeny(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
action string
|
||||
query string
|
||||
target string
|
||||
}{
|
||||
{
|
||||
name: "create snapshot", action: "rds:CreateDBSnapshot",
|
||||
query: "Action=CreateDBSnapshot&DBInstanceIdentifier=orders-db" +
|
||||
"&DBSnapshotIdentifier=orders-db-nightly",
|
||||
target: rdsinstance.FormatARN(rdsinstance.ResourceKindDBSnapshot, "ap-southeast-2", rdsTestAccountID, "orders-db-nightly"),
|
||||
},
|
||||
{
|
||||
name: "restore snapshot", action: "rds:RestoreDBInstanceFromDBSnapshot",
|
||||
query: "Action=RestoreDBInstanceFromDBSnapshot&DBSnapshotIdentifier=orders-db-nightly" +
|
||||
"&DBInstanceIdentifier=orders-db-restored",
|
||||
target: rdsinstance.FormatARN(rdsinstance.ResourceKindDBInstance, "ap-southeast-2", rdsTestAccountID, "orders-db-restored"),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
policies := []iamdomain.PolicyDocument{{
|
||||
Version: "2012-10-17",
|
||||
Statement: []iamdomain.Statement{
|
||||
{Effect: "Allow", Action: iamdomain.StringOrArr{tt.action}, Resource: iamdomain.StringOrArr{"*"}},
|
||||
{Effect: "Deny", Action: iamdomain.StringOrArr{tt.action}, Resource: iamdomain.StringOrArr{tt.target}},
|
||||
},
|
||||
}}
|
||||
h, probe := newRDSPolicyHarness(policies)
|
||||
err := serveRDS(h, tt.query)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 1, probe.consulted)
|
||||
gwtest.AssertDenied(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Creates and plural describes name no one resource, so a policy scoped to a
|
||||
// single instance must not be read as permitting them.
|
||||
func TestRDSRequest_UnscopedActionsEvaluateAgainstAnyResource(t *testing.T) {
|
||||
h, _ := newRDSPolicyHarness(allowRDSPolicyResource("rds:*", "arn:aws:rds:*:*:db:prod-orders"))
|
||||
err := serveRDS(h, "Action=CreateDBInstance&DBInstanceIdentifier=prod-orders")
|
||||
require.Error(t, err)
|
||||
gwtest.AssertDenied(t, err)
|
||||
}
|
||||
|
||||
// Cross-account references are refused at the ARN, not merely unauthorized:
|
||||
// snapshot sharing is out of v1, and an evaluator that sees a foreign account is
|
||||
// one policy away from allowing it.
|
||||
func TestRDSRequest_CrossAccountARNIsRejectedNotEvaluated(t *testing.T) {
|
||||
h, _ := newRDSHarnessWithResolver(func(_, _ string) ([]iamdomain.PolicyDocument, error) {
|
||||
t.Error("a cross-account ARN must never reach policy evaluation")
|
||||
return nil, nil
|
||||
})
|
||||
foreign := rdsinstance.FormatARN(rdsinstance.ResourceKindDBInstance, "ap-southeast-2", "999988887777", "orders-db")
|
||||
err := serveRDS(h, "Action=ListTagsForResource&ResourceName="+foreign)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorInvalidParameterValue, awserrors.ValidErrorCodeFromError(err))
|
||||
}
|
||||
|
||||
// A request with no resolvable identity is denied outright: there is no
|
||||
// principal to evaluate, so the gate refuses before consulting any policy.
|
||||
func TestRDSRequest_NoIdentityDenied(t *testing.T) {
|
||||
h, probe := newRDSPolicyHarness(nil)
|
||||
h.Caller = &credential.Principal{AccountID: rdsTestAccountID}
|
||||
|
||||
err := serveRDS(h, "Action=DescribeDBInstances")
|
||||
require.Error(t, err)
|
||||
gwtest.AssertDenied(t, err)
|
||||
assert.Zero(t, probe.consulted, "a request with no identity must not be evaluated at all")
|
||||
}
|
||||
|
||||
// RDS carries its scopes on the dispatch table itself, so the two cannot
|
||||
// disagree the way the other services' can. What can still go wrong is an action
|
||||
// landing with no scopes and nobody noticing it now evaluates against "*", so
|
||||
// the gate is the account-level half: every entry here is a written decision
|
||||
// that the action addresses no single resource.
|
||||
var rdsAccountLevelActions = map[string]string{
|
||||
// Creates: the resource does not exist yet, which is how AWS evaluates them.
|
||||
"CreateDBInstance": "the instance does not exist until this call makes it",
|
||||
"CreateDBSubnetGroup": "the subnet group does not exist until this call makes it",
|
||||
"CreateDBParameterGroup": "the parameter group does not exist until this call makes it",
|
||||
|
||||
// Describes that filter rather than address one resource.
|
||||
"DescribeDBInstances": "filters the account's instances",
|
||||
"DescribeDBSnapshots": "filters the account's snapshots",
|
||||
"DescribeDBInstanceAutomatedBackups": "filters the account's automated backups",
|
||||
"DescribeDBSubnetGroups": "filters the account's subnet groups",
|
||||
"DescribeDBParameterGroups": "filters the account's parameter groups",
|
||||
"DescribeEvents": "the event ring is per-account and a filter names no one resource",
|
||||
|
||||
// Catalogs read no per-account state at all.
|
||||
"DescribeDBEngineVersions": "a static engine catalog, not tenant state",
|
||||
"DescribeOrderableDBInstanceOptions": "cluster capability, not tenant state",
|
||||
|
||||
// Internal agent actions. AuthorizeCaller refuses these to every customer
|
||||
// principal by class before any policy is evaluated, so a resource scope
|
||||
// would narrow nothing.
|
||||
"RegisterDBInstance": "agent-only, gated by principal class",
|
||||
"SubmitDBStateChange": "agent-only, gated by principal class",
|
||||
"PollDBCommands": "agent-only, gated by principal class",
|
||||
"GetDBBootstrapConfig": "agent-only, gated by principal class",
|
||||
"AcknowledgeDBBootstrap": "agent-only, gated by principal class",
|
||||
}
|
||||
|
||||
// TestRDSScopeTableIsExhaustive is what stops the next RDS action being added
|
||||
// with a silent account-wide grant. Every served action is either scoped or
|
||||
// carries a written reason for being account-level, and an action in both sets
|
||||
// or in neither fails.
|
||||
func TestRDSScopeTableIsExhaustive(t *testing.T) {
|
||||
unsupported := rdsawsapi.UnsupportedActionNames()
|
||||
|
||||
for _, action := range rdsawsapi.ActionNames() {
|
||||
if slices.Contains(unsupported, action) {
|
||||
continue
|
||||
}
|
||||
_, accountLevel := rdsAccountLevelActions[action]
|
||||
assert.NotEqual(t, rdsawsapi.HasScope(action), accountLevel,
|
||||
"rds action %q must either carry a scope in domains/rds/awsapi/handler.go or a written reason "+
|
||||
"in rdsAccountLevelActions, not both and not neither", action)
|
||||
}
|
||||
|
||||
for action := range rdsAccountLevelActions {
|
||||
assert.True(t, rdsawsapi.HasAction(action),
|
||||
"rdsAccountLevelActions names %q, which RDS does not serve: remove it", action)
|
||||
assert.NotContains(t, unsupported, action,
|
||||
"rdsAccountLevelActions names %q, which is recognised but not offered: remove it", action)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
testAccount = "123456789012"
|
||||
testRegion = "ap-southeast-2"
|
||||
testEndpoint = "https://sqs.example:9999"
|
||||
testQueueURL = testEndpoint + "/" + testAccount + "/q"
|
||||
)
|
||||
|
||||
// recordingMessages captures every send so a test can tell a refused request
|
||||
// from one that reached the message owner.
|
||||
type recordingMessages struct {
|
||||
sent []message.SendInput
|
||||
err error
|
||||
}
|
||||
|
||||
func (m *recordingMessages) Send(_ context.Context, in message.SendInput) (message.SendResult, error) {
|
||||
m.sent = append(m.sent, in)
|
||||
if m.err != nil {
|
||||
return message.SendResult{}, m.err
|
||||
}
|
||||
return message.SendResult{MessageID: "id-1", MD5OfBody: "md5"}, nil
|
||||
}
|
||||
|
||||
// countingQueues fails the test's expectations by counting any owner call.
|
||||
type countingQueues struct{ calls int }
|
||||
|
||||
func (q *countingQueues) Create(context.Context, queue.CreateRequest) (queue.Record, error) {
|
||||
q.calls++
|
||||
return queue.Record{}, queue.ErrUnavailable
|
||||
}
|
||||
|
||||
func (q *countingQueues) Resolve(context.Context, string, string, string) (queue.Record, error) {
|
||||
q.calls++
|
||||
return queue.Record{}, queue.ErrUnavailable
|
||||
}
|
||||
|
||||
func (q *countingQueues) List(context.Context, queue.ListRequest) (queue.ListResult, error) {
|
||||
q.calls++
|
||||
return queue.ListResult{}, queue.ErrUnavailable
|
||||
}
|
||||
|
||||
func (q *countingQueues) SetAttributes(context.Context, string, string, string, map[string]string) (queue.Record, error) {
|
||||
q.calls++
|
||||
return queue.Record{}, queue.ErrUnavailable
|
||||
}
|
||||
|
||||
func (q *countingQueues) Tag(context.Context, string, string, string, map[string]string) (queue.Record, error) {
|
||||
q.calls++
|
||||
return queue.Record{}, queue.ErrUnavailable
|
||||
}
|
||||
|
||||
func (q *countingQueues) Untag(context.Context, string, string, string, []string) (queue.Record, error) {
|
||||
q.calls++
|
||||
return queue.Record{}, queue.ErrUnavailable
|
||||
}
|
||||
|
||||
func (q *countingQueues) Tags(context.Context, string, string, string) (map[string]string, error) {
|
||||
q.calls++
|
||||
return nil, queue.ErrUnavailable
|
||||
}
|
||||
|
||||
type harness struct {
|
||||
msgs *recordingMessages
|
||||
queues *countingQueues
|
||||
ops *stubMessageOps
|
||||
authorized int
|
||||
}
|
||||
|
||||
func newHarness() *harness {
|
||||
return &harness{msgs: &recordingMessages{}, queues: &countingQueues{}, ops: &stubMessageOps{}}
|
||||
}
|
||||
|
||||
// call dispatches one operation as an authenticated caller who is allowed
|
||||
// everything, returning the response recorder and the dispatch error.
|
||||
func (h *harness) call(t *testing.T, op, body string) (*httptest.ResponseRecorder, error) {
|
||||
t.Helper()
|
||||
reg := awsapi.NewRegistration(awsapi.Deps{
|
||||
Queues: h.queues, Messages: h.msgs, Receives: h.ops, Deletes: h.ops, Visibility: h.ops, Endpoint: testEndpoint,
|
||||
})
|
||||
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
if op != "" {
|
||||
r.Header.Set("X-Amz-Target", awsapi.TargetPrefix+op)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
err := reg.Dispatch(w, dispatch.Invocation{
|
||||
Request: r, AccountID: testAccount, Region: testRegion, Service: awsapi.ServiceName,
|
||||
Authorize: func(string, string, []string, iampolicy.ConditionKeys) error {
|
||||
h.authorized++
|
||||
return nil
|
||||
},
|
||||
Actor: func() (dispatch.Actor, error) { return dispatch.Actor{UserID: "AIDAALICE"}, nil },
|
||||
})
|
||||
return w, err
|
||||
}
|
||||
|
||||
func code(t *testing.T, err error) string {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
c, ok := awserrors.ResolveErrorCode(err)
|
||||
require.True(t, ok, "unresolved error %v", err)
|
||||
return c
|
||||
}
|
||||
|
||||
func sendBody(rawBody string) string {
|
||||
return `{"QueueUrl":"` + testQueueURL + `","MessageBody":` + rawBody + `}`
|
||||
}
|
||||
|
||||
// Every operation reads exactly one JSON object; nothing but whitespace may
|
||||
// follow it, and the owner is never called otherwise.
|
||||
func TestDecode_StrictSingleObject(t *testing.T) {
|
||||
ops := []string{awsapi.OpCreateQueue, awsapi.OpGetQueueURL, awsapi.OpListQueues, awsapi.OpSendMessage}
|
||||
for _, op := range ops {
|
||||
for _, body := range []string{
|
||||
`{"QueueName":"q"}}`,
|
||||
`{"QueueName":"q"}]`,
|
||||
`{"QueueName":"q"}{}`,
|
||||
`{"QueueName":"q"} 1`,
|
||||
`null`,
|
||||
`[]`,
|
||||
`"q"`,
|
||||
`{"QueueName":`,
|
||||
} {
|
||||
h := newHarness()
|
||||
_, err := h.call(t, op, body)
|
||||
assert.Equal(t, awserrors.ErrorSerializationException, code(t, err), "%s %s", op, body)
|
||||
assert.Zero(t, h.queues.calls, "%s %s", op, body)
|
||||
assert.Empty(t, h.msgs.sent, "%s %s", op, body)
|
||||
}
|
||||
}
|
||||
|
||||
h := newHarness()
|
||||
_, err := h.call(t, awsapi.OpSendMessage, " \n"+sendBody(`"x"`)+" \t\r\n")
|
||||
require.NoError(t, err, "surrounding whitespace is accepted")
|
||||
require.Len(t, h.msgs.sent, 1)
|
||||
}
|
||||
|
||||
func TestSendMessage_QueueURLBoundary(t *testing.T) {
|
||||
for _, tc := range []struct{ raw, want string }{
|
||||
{"http://sqs.example:9999/" + testAccount + "/q", awserrors.ErrorSQSInvalidAddress},
|
||||
{"https://sqs.example/" + testAccount + "/q", awserrors.ErrorSQSInvalidAddress},
|
||||
{"https://other.example:9999/" + testAccount + "/q", awserrors.ErrorSQSInvalidAddress},
|
||||
{"https://sqs.example:9999/210987654321/q", awserrors.ErrorSQSInvalidAddress},
|
||||
{testQueueURL + "?x=1", awserrors.ErrorSQSInvalidAddress},
|
||||
{testQueueURL + "?", awserrors.ErrorSQSInvalidAddress},
|
||||
{testQueueURL + "#f", awserrors.ErrorSQSInvalidAddress},
|
||||
{"https://u@sqs.example:9999/" + testAccount + "/q", awserrors.ErrorSQSInvalidAddress},
|
||||
{testQueueURL + "/extra", awserrors.ErrorSQSInvalidAddress},
|
||||
{testEndpoint + "/" + testAccount, awserrors.ErrorSQSInvalidAddress},
|
||||
{"sqs.example:9999/" + testAccount + "/q", awserrors.ErrorSQSInvalidAddress},
|
||||
{testEndpoint + "/" + testAccount + "/bad.name", awserrors.ErrorSQSQueueDoesNotExist},
|
||||
{testEndpoint + "/" + testAccount + "/" + strings.Repeat("a", 81), awserrors.ErrorSQSQueueDoesNotExist},
|
||||
} {
|
||||
raw, want := tc.raw, tc.want
|
||||
h := newHarness()
|
||||
body := `{"QueueUrl":"` + raw + `","MessageBody":"x"}`
|
||||
_, err := h.call(t, awsapi.OpSendMessage, body)
|
||||
assert.Equal(t, want, code(t, err), raw)
|
||||
assert.Empty(t, h.msgs.sent, raw)
|
||||
assert.Zero(t, h.authorized, "%s: refused before authorizing", raw)
|
||||
}
|
||||
|
||||
h := newHarness()
|
||||
_, err := h.call(t, awsapi.OpSendMessage, `{"QueueUrl":"HTTPS://SQS.EXAMPLE:9999/`+testAccount+`/q","MessageBody":"x"}`)
|
||||
require.NoError(t, err, "scheme and host compare case-insensitively")
|
||||
require.Len(t, h.msgs.sent, 1)
|
||||
assert.Equal(t, "q", h.msgs.sent[0].Queue)
|
||||
}
|
||||
|
||||
// Unsupported members are refused when present at all, empty included.
|
||||
func TestSendMessage_UnsupportedMembersByPresence(t *testing.T) {
|
||||
for _, member := range []string{
|
||||
`"MessageGroupId":""`,
|
||||
`"MessageGroupId":"g"`,
|
||||
`"MessageDeduplicationId":""`,
|
||||
`"MessageDeduplicationId":"d"`,
|
||||
`"MessageAttributes":{"a":{"DataType":"Number","StringValue":"1"}}`,
|
||||
`"MessageSystemAttributes":{"AWSTraceHeader":{"DataType":"String","StringValue":"v"}}`,
|
||||
} {
|
||||
h := newHarness()
|
||||
body := `{"QueueUrl":"` + testQueueURL + `","MessageBody":"x",` + member + `}`
|
||||
_, err := h.call(t, awsapi.OpSendMessage, body)
|
||||
assert.Equal(t, awserrors.ErrorUnsupportedOperation, code(t, err), member)
|
||||
assert.Empty(t, h.msgs.sent, member)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendMessage_OwnerErrorsMapped(t *testing.T) {
|
||||
for ownerErr, want := range map[error]string{
|
||||
message.ErrQueueDoesNotExist: awserrors.ErrorSQSQueueDoesNotExist,
|
||||
message.ErrUnavailable: awserrors.ErrorServiceUnavailableException,
|
||||
message.ErrInvalidContents: awserrors.ErrorSQSInvalidMessageContents,
|
||||
message.ErrInvalidParameter: awserrors.ErrorInvalidParameterValueException,
|
||||
errors.New("unexpected"): awserrors.ErrorInternalFailure,
|
||||
} {
|
||||
h := newHarness()
|
||||
h.msgs.err = ownerErr
|
||||
w, err := h.call(t, awsapi.OpSendMessage, sendBody(`"x"`))
|
||||
assert.Equal(t, want, code(t, err), ownerErr.Error())
|
||||
assert.Empty(t, w.Body.String(), "%s: no success body", ownerErr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"unicode/utf16"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
)
|
||||
|
||||
// errMalformedText marks JSON text encoding/json would silently repair:
|
||||
// malformed UTF-8 or an unpaired surrogate escape.
|
||||
var errMalformedText = errors.New("malformed UTF-8 or UTF-16")
|
||||
|
||||
// decodeMessageAttributes reads a MessageAttributes object exactly: names
|
||||
// and members keep their order and spelling, and an unknown or repeated
|
||||
// name or member, which encoding/json would resolve silently, is refused. A JSON
|
||||
// structure error is a serialization fault; a bad value is the owner's
|
||||
// ErrInvalidParameter, which a batch reports per entry.
|
||||
func decodeMessageAttributes(raw json.RawMessage) ([]message.Attribute, error) {
|
||||
if raw == nil || string(raw) == "null" {
|
||||
return nil, nil
|
||||
}
|
||||
if err := checkJSONText(raw); err != nil {
|
||||
if errors.Is(err, errMalformedText) {
|
||||
return nil, fmt.Errorf("%w: message attribute names and strings must be well-formed UTF-8 or UTF-16", message.ErrInvalidParameter)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
members, err := objectMembers(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]message.Attribute, 0, len(members))
|
||||
for _, m := range members {
|
||||
v, err := decodeAttributeValue(m.name, m.value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, message.Attribute{Name: m.name, Value: v})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func decodeAttributeValue(name string, raw json.RawMessage) (message.AttributeValue, error) {
|
||||
members, err := objectMembers(raw)
|
||||
if err != nil {
|
||||
return message.AttributeValue{}, err
|
||||
}
|
||||
var v message.AttributeValue
|
||||
for _, m := range members {
|
||||
if string(m.value) == "null" {
|
||||
continue
|
||||
}
|
||||
var err error
|
||||
switch m.name {
|
||||
case "DataType":
|
||||
err = json.Unmarshal(m.value, &v.DataType)
|
||||
case "StringValue":
|
||||
var s string
|
||||
if err = json.Unmarshal(m.value, &s); err == nil {
|
||||
v.StringValue = &s
|
||||
}
|
||||
case "BinaryValue":
|
||||
v.BinaryValue, err = decodeBlob(m.value)
|
||||
case "StringListValues":
|
||||
v.StringListValues = []string{}
|
||||
err = json.Unmarshal(m.value, &v.StringListValues)
|
||||
case "BinaryListValues":
|
||||
v.BinaryListValues = [][]byte{}
|
||||
err = json.Unmarshal(m.value, &v.BinaryListValues)
|
||||
default:
|
||||
return message.AttributeValue{}, fmt.Errorf("%w: message attribute %q has an unknown member", message.ErrInvalidParameter, name)
|
||||
}
|
||||
if err != nil {
|
||||
return message.AttributeValue{}, errors.New(awserrors.ErrorSerializationException)
|
||||
}
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// decodeBlob decodes a JSON base64 blob strictly; an empty string is a
|
||||
// present, empty value.
|
||||
func decodeBlob(raw json.RawMessage) ([]byte, error) {
|
||||
var s string
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b, err := base64.StdEncoding.Strict().DecodeString(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b == nil {
|
||||
b = []byte{}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
type member struct {
|
||||
name string
|
||||
value json.RawMessage
|
||||
}
|
||||
|
||||
// objectMembers lists a JSON object's members in order, refusing a repeated
|
||||
// name, which encoding/json would resolve silently to the last.
|
||||
func objectMembers(raw json.RawMessage) ([]member, error) {
|
||||
serr := errors.New(awserrors.ErrorSerializationException)
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
if t, err := dec.Token(); err != nil || t != json.Delim('{') {
|
||||
return nil, serr
|
||||
}
|
||||
var out []member
|
||||
seen := map[string]bool{}
|
||||
for dec.More() {
|
||||
t, err := dec.Token()
|
||||
if err != nil {
|
||||
return nil, serr
|
||||
}
|
||||
name, ok := t.(string)
|
||||
if !ok {
|
||||
return nil, serr
|
||||
}
|
||||
if seen[name] {
|
||||
return nil, fmt.Errorf("%w: message attribute JSON repeats the name %q", message.ErrInvalidParameter, name)
|
||||
}
|
||||
seen[name] = true
|
||||
var v json.RawMessage
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
return nil, serr
|
||||
}
|
||||
out = append(out, member{name: name, value: v})
|
||||
}
|
||||
if t, err := dec.Token(); err != nil || t != json.Delim('}') {
|
||||
return nil, serr
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// checkJSONText refuses JSON whose strings encoding/json would repair to
|
||||
// U+FFFD: malformed UTF-8, or a \u escape of an unpaired surrogate. A
|
||||
// literal or escaped U+FFFD and a valid escaped pair are kept.
|
||||
func checkJSONText(raw []byte) error {
|
||||
if !utf8.Valid(raw) {
|
||||
return errMalformedText
|
||||
}
|
||||
for i := 0; i < len(raw); i++ {
|
||||
if raw[i] != '\\' {
|
||||
continue
|
||||
}
|
||||
i++
|
||||
if i >= len(raw) || raw[i] != 'u' {
|
||||
continue
|
||||
}
|
||||
r, ok := hex4(raw, i+1)
|
||||
if !ok {
|
||||
return errors.New(awserrors.ErrorSerializationException)
|
||||
}
|
||||
i += 4
|
||||
switch {
|
||||
case utf16.IsSurrogate(r) && r < 0xDC00:
|
||||
if i+6 >= len(raw) || raw[i+1] != '\\' || raw[i+2] != 'u' {
|
||||
return errMalformedText
|
||||
}
|
||||
lo, ok := hex4(raw, i+3)
|
||||
if !ok || lo < 0xDC00 || lo > 0xDFFF {
|
||||
return errMalformedText
|
||||
}
|
||||
i += 6
|
||||
case utf16.IsSurrogate(r):
|
||||
return errMalformedText
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
type changeVisibilityBatchInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
Entries *[]changeVisibilityBatchEntry `json:"Entries"`
|
||||
}
|
||||
|
||||
type changeVisibilityBatchEntry struct {
|
||||
ID *string `json:"Id"`
|
||||
ReceiptHandle *string `json:"ReceiptHandle"`
|
||||
VisibilityTimeout *int `json:"VisibilityTimeout"`
|
||||
}
|
||||
|
||||
// changeMessageVisibilityBatch moves each entry's deadline under the
|
||||
// ChangeMessageVisibility permission. The batch is validated before any
|
||||
// change; each entry's outcome, uncertain ones included, is reported under
|
||||
// its own Id. Handles bind their queue, so a replaced queue is never changed.
|
||||
func changeMessageVisibilityBatch(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in changeVisibilityBatchInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
if in.Entries == nil {
|
||||
return missing("Entries")
|
||||
}
|
||||
entries := *in.Entries
|
||||
ids := make([]*string, len(entries))
|
||||
for i, e := range entries {
|
||||
ids[i] = e.ID
|
||||
}
|
||||
if err := validateBatchIDs(ids); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, e := range entries {
|
||||
if e.ReceiptHandle == nil || *e.ReceiptHandle == "" {
|
||||
return missing(fmt.Sprintf("Entries.%d.ReceiptHandle", i+1))
|
||||
}
|
||||
if e.VisibilityTimeout == nil {
|
||||
return missing(fmt.Sprintf("Entries.%d.VisibilityTimeout", i+1))
|
||||
}
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := authorize(inv, actionChangeMessageVisibility, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Visibility == nil || d.Queues == nil {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
if _, err := resolveBatchQueue(d, inv, name); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
out := deleteBatchOutput{Successful: []batchSuccess{}, Failed: []batchFailure{}}
|
||||
for _, e := range entries {
|
||||
if v := *e.VisibilityTimeout; v < 0 || v > message.MaxVisibilitySeconds {
|
||||
out.Failed = append(out.Failed, batchFailure{ID: *e.ID, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorInvalidParameterValueException],
|
||||
Message: fmt.Sprintf("VisibilityTimeout must be between 0 and %d.", message.MaxVisibilitySeconds)})
|
||||
continue
|
||||
}
|
||||
err := d.Visibility.ChangeVisibility(ctx, message.ChangeVisibilityInput{
|
||||
Account: inv.AccountID, Region: inv.Region, Queue: name,
|
||||
ReceiptHandle: *e.ReceiptHandle, VisibilityTimeout: *e.VisibilityTimeout,
|
||||
})
|
||||
if err == nil {
|
||||
out.Successful = append(out.Successful, batchSuccess{ID: *e.ID})
|
||||
continue
|
||||
}
|
||||
slog.DebugContext(ctx, "SQS ChangeMessageVisibilityBatch entry failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "sqs.batch_entry_id", *e.ID, "err", err)
|
||||
out.Failed = append(out.Failed, visibilityEntryFailure(*e.ID, err))
|
||||
}
|
||||
return writeJSON(w, out)
|
||||
}
|
||||
|
||||
// visibilityEntryFailure reports one entry's error under its query code, an
|
||||
// uncertain write before any client fault so it stays retryable; the
|
||||
// message never carries the receipt handle.
|
||||
func visibilityEntryFailure(id string, err error) batchFailure {
|
||||
switch {
|
||||
case errors.Is(err, message.ErrQueueDoesNotExist):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorSQSQueueDoesNotExist], Message: "The specified queue doesn't exist."}
|
||||
case errors.Is(err, message.ErrUnavailable):
|
||||
return batchFailure{ID: id, Code: queryErrorCodes[awserrors.ErrorServiceUnavailableException], Message: "The visibility change could not be confirmed; retry this entry."}
|
||||
case errors.Is(err, message.ErrReceiptInvalid):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: awserrors.ErrorSQSReceiptHandleIsInvalid, Message: "The specified receipt handle isn't valid."}
|
||||
case errors.Is(err, message.ErrNotInflight):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorSQSMessageNotInflight], Message: "The message referred to isn't in flight."}
|
||||
case errors.Is(err, message.ErrVisibilityCap):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorInvalidParameterValueException], Message: "VisibilityTimeout would extend past 12 hours from this receive."}
|
||||
case errors.Is(err, message.ErrInvalidParameter):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorInvalidParameterValueException], Message: err.Error()}
|
||||
}
|
||||
return batchFailure{ID: id, Code: awserrors.ErrorInternalFailure, Message: "The visibility change failed."}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// handleVisibility answers each change with the error keyed by its handle.
|
||||
type handleVisibility struct {
|
||||
errs map[string]error
|
||||
got []message.ChangeVisibilityInput
|
||||
}
|
||||
|
||||
func (v *handleVisibility) ChangeVisibility(_ context.Context, in message.ChangeVisibilityInput) error {
|
||||
v.got = append(v.got, in)
|
||||
return v.errs[in.ReceiptHandle]
|
||||
}
|
||||
|
||||
func callChangeBatch(t *testing.T, qs awsapi.Queues, vis awsapi.Visibility, deny bool, body string) (*httptest.ResponseRecorder, *batchCall, error) {
|
||||
t.Helper()
|
||||
reg := awsapi.NewRegistration(awsapi.Deps{Queues: qs, Visibility: vis, Endpoint: testEndpoint})
|
||||
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
r.Header.Set("X-Amz-Target", awsapi.TargetPrefix+awsapi.OpChangeVisibilityBatch)
|
||||
w := httptest.NewRecorder()
|
||||
c := &batchCall{}
|
||||
err := reg.Dispatch(w, dispatch.Invocation{
|
||||
Request: r, AccountID: testAccount, Region: testRegion, Service: awsapi.ServiceName,
|
||||
Authorize: func(_ string, action string, resources []string, _ iampolicy.ConditionKeys) error {
|
||||
c.actions = append(c.actions, action)
|
||||
c.resources = append(c.resources, resources...)
|
||||
if deny {
|
||||
return errors.New(awserrors.ErrorAccessDeniedException)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Actor: func() (dispatch.Actor, error) { return dispatch.Actor{UserID: "AIDAALICE"}, nil },
|
||||
})
|
||||
return w, c, err
|
||||
}
|
||||
|
||||
func TestChangeVisibilityBatch_EntryOutcomes(t *testing.T) {
|
||||
vis := &handleVisibility{errs: map[string]error{
|
||||
"h-forged": message.ErrReceiptInvalid,
|
||||
"h-stale": message.ErrNotInflight,
|
||||
"h-cap": message.ErrVisibilityCap,
|
||||
"h-uncertain": fmt.Errorf("%w: outcome unknown", message.ErrUnavailable),
|
||||
"h-both": errors.Join(message.ErrReceiptInvalid, message.ErrUnavailable),
|
||||
"h-gone": message.ErrQueueDoesNotExist,
|
||||
"h-odd": errors.New("boom"),
|
||||
}}
|
||||
w, c, err := callChangeBatch(t, &pinQueues{}, vis, false, batchReq(
|
||||
`{"Id":"a","ReceiptHandle":"h-ok","VisibilityTimeout":7},{"Id":"b","ReceiptHandle":"h-forged","VisibilityTimeout":0},`+
|
||||
`{"Id":"c","ReceiptHandle":"h-stale","VisibilityTimeout":0},{"Id":"d","ReceiptHandle":"h-cap","VisibilityTimeout":43200},`+
|
||||
`{"Id":"e","ReceiptHandle":"h-uncertain","VisibilityTimeout":0},{"Id":"f","ReceiptHandle":"h-both","VisibilityTimeout":0},`+
|
||||
`{"Id":"g","ReceiptHandle":"h-gone","VisibilityTimeout":0},{"Id":"h","ReceiptHandle":"h-odd","VisibilityTimeout":0},`+
|
||||
`{"Id":"i","ReceiptHandle":"h-ok","VisibilityTimeout":43201},{"Id":"j","ReceiptHandle":"h-ok","VisibilityTimeout":-1}`))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"ChangeMessageVisibility"}, c.actions, "authorized once, as ChangeMessageVisibility")
|
||||
assert.Equal(t, []string{"arn:aws:sqs:" + testRegion + ":" + testAccount + ":q"}, c.resources)
|
||||
require.Len(t, vis.got, 8, "out-of-range timeouts never reach the owner")
|
||||
assert.Equal(t, message.ChangeVisibilityInput{Account: testAccount, Region: testRegion, Queue: "q", ReceiptHandle: "h-ok", VisibilityTimeout: 7}, vis.got[0])
|
||||
|
||||
var out batchReply
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &out))
|
||||
assert.Equal(t, []map[string]any{{"Id": "a"}}, out.Successful)
|
||||
retry := "The visibility change could not be confirmed; retry this entry."
|
||||
invalidTimeout := "VisibilityTimeout must be between 0 and 43200."
|
||||
assert.Equal(t, []map[string]any{
|
||||
{"Id": "b", "SenderFault": true, "Code": "ReceiptHandleIsInvalid", "Message": "The specified receipt handle isn't valid."},
|
||||
{"Id": "c", "SenderFault": true, "Code": "AWS.SimpleQueueService.MessageNotInflight", "Message": "The message referred to isn't in flight."},
|
||||
{"Id": "d", "SenderFault": true, "Code": "InvalidParameterValue", "Message": "VisibilityTimeout would extend past 12 hours from this receive."},
|
||||
{"Id": "e", "SenderFault": false, "Code": "ServiceUnavailable", "Message": retry},
|
||||
{"Id": "f", "SenderFault": false, "Code": "ServiceUnavailable", "Message": retry},
|
||||
{"Id": "g", "SenderFault": true, "Code": "AWS.SimpleQueueService.NonExistentQueue", "Message": "The specified queue doesn't exist."},
|
||||
{"Id": "h", "SenderFault": false, "Code": "InternalFailure", "Message": "The visibility change failed."},
|
||||
{"Id": "i", "SenderFault": true, "Code": "InvalidParameterValue", "Message": invalidTimeout},
|
||||
{"Id": "j", "SenderFault": true, "Code": "InvalidParameterValue", "Message": invalidTimeout},
|
||||
}, out.Failed)
|
||||
assert.NotContains(t, w.Body.String(), "h-", "no receipt handle is echoed")
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
const actionChangeMessageVisibility = "ChangeMessageVisibility"
|
||||
|
||||
type changeVisibilityInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
ReceiptHandle *string `json:"ReceiptHandle"`
|
||||
VisibilityTimeout *int `json:"VisibilityTimeout"`
|
||||
}
|
||||
|
||||
// changeMessageVisibility moves the deadline of the message a receipt handle
|
||||
// names. The handle is authenticated only after authorizing, as for delete.
|
||||
func changeMessageVisibility(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in changeVisibilityInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
if in.ReceiptHandle == nil || *in.ReceiptHandle == "" {
|
||||
return missing("ReceiptHandle")
|
||||
}
|
||||
if in.VisibilityTimeout == nil {
|
||||
return missing("VisibilityTimeout")
|
||||
}
|
||||
if v := *in.VisibilityTimeout; v < 0 || v > message.MaxVisibilitySeconds {
|
||||
return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "VisibilityTimeout must be between 0 and %d.", message.MaxVisibilitySeconds)
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := authorize(inv, actionChangeMessageVisibility, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Visibility == nil {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
err = d.Visibility.ChangeVisibility(ctx, message.ChangeVisibilityInput{
|
||||
Account: inv.AccountID, Region: inv.Region, Queue: name,
|
||||
ReceiptHandle: *in.ReceiptHandle, VisibilityTimeout: *in.VisibilityTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "SQS ChangeMessageVisibility failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return messageError(err)
|
||||
}
|
||||
return writeJSON(w, struct{}{})
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
// Actions authorized on the queue ARN. Tagging at creation also needs
|
||||
// sqs:TagQueue.
|
||||
const (
|
||||
actionCreateQueue = "CreateQueue"
|
||||
actionTagQueue = "TagQueue"
|
||||
)
|
||||
|
||||
type createQueueInput struct {
|
||||
QueueName *string `json:"QueueName"`
|
||||
Attributes map[string]string `json:"Attributes"`
|
||||
Tags map[string]string `json:"tags"`
|
||||
}
|
||||
|
||||
type createQueueOutput struct {
|
||||
QueueURL string `json:"QueueUrl"`
|
||||
}
|
||||
|
||||
// createQueue validates the whole request, authorizes it on the queue ARN it
|
||||
// names, then creates the queue or returns the matching existing one.
|
||||
func createQueue(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
ctx := inv.Request.Context()
|
||||
var in createQueueInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueName == nil || *in.QueueName == "" {
|
||||
return missing("QueueName")
|
||||
}
|
||||
name := *in.QueueName
|
||||
if err := queue.ValidateName(name); err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
if _, err := queue.ValidateCreate(in.Attributes); err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
if err := queue.ValidateTags(in.Tags, true); err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
|
||||
resource := queueARN(inv, name)
|
||||
if err := authorize(inv, actionCreateQueue, resource); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(in.Tags) > 0 {
|
||||
if err := authorize(inv, actionTagQueue, resource); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if d.Queues == nil || d.Endpoint == "" {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
rec, err := d.Queues.Create(ctx, queue.CreateRequest{
|
||||
Account: inv.AccountID,
|
||||
Region: inv.Region,
|
||||
Name: name,
|
||||
Attributes: in.Attributes,
|
||||
Tags: in.Tags,
|
||||
})
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "SQS CreateQueue failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return ownerError(err)
|
||||
}
|
||||
slog.InfoContext(ctx, "SQS queue created or matched", "sqs.queue_name", rec.Name,
|
||||
"cloud.account.id", rec.Account, "sqs.queue_generation", rec.Generation)
|
||||
return writeJSON(w, createQueueOutput{QueueURL: queueURL(d.Endpoint, rec.Account, rec.Name)})
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
// MaxBatchEntries is the most entries one batch request may carry.
|
||||
const MaxBatchEntries = 10
|
||||
|
||||
// batchEntryID is the documented entry Id syntax: up to 80 alphanumeric,
|
||||
// hyphen or underscore characters.
|
||||
var batchEntryID = regexp.MustCompile(`^[A-Za-z0-9_-]{1,80}$`)
|
||||
|
||||
type deleteBatchInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
Entries *[]deleteBatchEntry `json:"Entries"`
|
||||
}
|
||||
|
||||
type deleteBatchEntry struct {
|
||||
ID *string `json:"Id"`
|
||||
ReceiptHandle *string `json:"ReceiptHandle"`
|
||||
}
|
||||
|
||||
type batchSuccess struct {
|
||||
ID string `json:"Id"`
|
||||
}
|
||||
|
||||
type batchFailure struct {
|
||||
ID string `json:"Id"`
|
||||
SenderFault bool `json:"SenderFault"`
|
||||
Code string `json:"Code"`
|
||||
Message string `json:"Message,omitempty"`
|
||||
}
|
||||
|
||||
type deleteBatchOutput struct {
|
||||
Successful []batchSuccess `json:"Successful"`
|
||||
Failed []batchFailure `json:"Failed"`
|
||||
}
|
||||
|
||||
// deleteMessageBatch deletes each entry's message under the DeleteMessage
|
||||
// permission. The whole batch is validated before any delete; each entry's
|
||||
// outcome, uncertain ones included, is reported under its own Id.
|
||||
func deleteMessageBatch(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in deleteBatchInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
if in.Entries == nil {
|
||||
return missing("Entries")
|
||||
}
|
||||
entries := *in.Entries
|
||||
if err := validateBatch(entries); err != nil {
|
||||
return err
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := authorize(inv, actionDeleteMessage, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Deletes == nil {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
out := deleteBatchOutput{Successful: []batchSuccess{}, Failed: []batchFailure{}}
|
||||
for i, e := range entries {
|
||||
err := d.Deletes.Delete(ctx, message.DeleteInput{
|
||||
Account: inv.AccountID, Region: inv.Region, Queue: name, ReceiptHandle: *e.ReceiptHandle,
|
||||
})
|
||||
if err == nil {
|
||||
out.Successful = append(out.Successful, batchSuccess{ID: *e.ID})
|
||||
continue
|
||||
}
|
||||
if i == 0 && errors.Is(err, message.ErrQueueDoesNotExist) {
|
||||
return messageError(err)
|
||||
}
|
||||
slog.DebugContext(ctx, "SQS DeleteMessageBatch entry failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "sqs.batch_entry_id", *e.ID, "err", err)
|
||||
out.Failed = append(out.Failed, batchEntryFailure(*e.ID, err))
|
||||
}
|
||||
return writeJSON(w, out)
|
||||
}
|
||||
|
||||
// validateBatch rejects the whole request before any side effect: no
|
||||
// entries, too many, a malformed or repeated Id, or a missing handle.
|
||||
func validateBatch(entries []deleteBatchEntry) error {
|
||||
ids := make([]*string, len(entries))
|
||||
for i, e := range entries {
|
||||
ids[i] = e.ID
|
||||
}
|
||||
if err := validateBatchIDs(ids); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, e := range entries {
|
||||
if e.ReceiptHandle == nil || *e.ReceiptHandle == "" {
|
||||
return missing(fmt.Sprintf("Entries.%d.ReceiptHandle", i+1))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateBatchIDs checks a batch's entry count and that every entry Id is
|
||||
// well formed and distinct.
|
||||
func validateBatchIDs(ids []*string) error {
|
||||
if len(ids) == 0 {
|
||||
return errors.New(awserrors.ErrorSQSEmptyBatchRequest)
|
||||
}
|
||||
if len(ids) > MaxBatchEntries {
|
||||
return awserrors.Errorf(awserrors.ErrorSQSTooManyEntriesInBatchRequest, "Maximum number of entries per request are %d. You have sent %d.", MaxBatchEntries, len(ids))
|
||||
}
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
if id == nil || !batchEntryID.MatchString(*id) {
|
||||
return errors.New(awserrors.ErrorSQSInvalidBatchEntryId)
|
||||
}
|
||||
if _, dup := seen[*id]; dup {
|
||||
return awserrors.Errorf(awserrors.ErrorSQSBatchEntryIdsNotDistinct, "Id %s repeated.", *id)
|
||||
}
|
||||
seen[*id] = struct{}{}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// batchEntryFailure reports one entry's error under its query code, in
|
||||
// messageError's precedence so an uncertain delete stays retryable; the
|
||||
// message never carries the receipt handle.
|
||||
func batchEntryFailure(id string, err error) batchFailure {
|
||||
switch {
|
||||
case errors.Is(err, message.ErrQueueDoesNotExist):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorSQSQueueDoesNotExist], Message: "The specified queue doesn't exist."}
|
||||
case errors.Is(err, message.ErrUnavailable):
|
||||
return batchFailure{ID: id, Code: queryErrorCodes[awserrors.ErrorServiceUnavailableException], Message: "The delete could not be confirmed; retry this entry."}
|
||||
case errors.Is(err, message.ErrReceiptInvalid):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: awserrors.ErrorSQSReceiptHandleIsInvalid, Message: "The specified receipt handle isn't valid."}
|
||||
}
|
||||
return batchFailure{ID: id, Code: awserrors.ErrorInternalFailure, Message: "The delete failed."}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// handleDeleter answers each delete with the error keyed by its handle.
|
||||
type handleDeleter struct {
|
||||
errs map[string]error
|
||||
handles []string
|
||||
}
|
||||
|
||||
func (d *handleDeleter) Delete(_ context.Context, in message.DeleteInput) error {
|
||||
d.handles = append(d.handles, in.ReceiptHandle)
|
||||
return d.errs[in.ReceiptHandle]
|
||||
}
|
||||
|
||||
type batchCall struct {
|
||||
actions []string
|
||||
resources []string
|
||||
}
|
||||
|
||||
func callBatch(t *testing.T, del awsapi.Deletes, deny bool, body string) (*httptest.ResponseRecorder, *batchCall, error) {
|
||||
t.Helper()
|
||||
reg := awsapi.NewRegistration(awsapi.Deps{Queues: &countingQueues{}, Deletes: del, Endpoint: testEndpoint})
|
||||
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
r.Header.Set("X-Amz-Target", awsapi.TargetPrefix+awsapi.OpDeleteMessageBatch)
|
||||
w := httptest.NewRecorder()
|
||||
c := &batchCall{}
|
||||
err := reg.Dispatch(w, dispatch.Invocation{
|
||||
Request: r, AccountID: testAccount, Region: testRegion, Service: awsapi.ServiceName,
|
||||
Authorize: func(_ string, action string, resources []string, _ iampolicy.ConditionKeys) error {
|
||||
c.actions = append(c.actions, action)
|
||||
c.resources = append(c.resources, resources...)
|
||||
if deny {
|
||||
return errors.New(awserrors.ErrorAccessDeniedException)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Actor: func() (dispatch.Actor, error) { return dispatch.Actor{UserID: "AIDAALICE"}, nil },
|
||||
})
|
||||
return w, c, err
|
||||
}
|
||||
|
||||
func batchReq(entries string) string {
|
||||
return `{"QueueUrl":"` + testQueueURL + `","Entries":[` + entries + `]}`
|
||||
}
|
||||
|
||||
type batchReply struct {
|
||||
Successful []map[string]any `json:"Successful"`
|
||||
Failed []map[string]any `json:"Failed"`
|
||||
}
|
||||
|
||||
func TestDeleteMessageBatch_EntryOutcomes(t *testing.T) {
|
||||
del := &handleDeleter{errs: map[string]error{
|
||||
"h-forged": message.ErrReceiptInvalid,
|
||||
"h-uncertain": message.ErrUnavailable,
|
||||
"h-gone": message.ErrQueueDoesNotExist,
|
||||
"h-odd": errors.New("boom"),
|
||||
}}
|
||||
w, c, err := callBatch(t, del, false, batchReq(
|
||||
`{"Id":"a","ReceiptHandle":"h-ok"},{"Id":"b","ReceiptHandle":"h-forged"},{"Id":"c","ReceiptHandle":"h-uncertain"},`+
|
||||
`{"Id":"d","ReceiptHandle":"h-gone"},{"Id":"e","ReceiptHandle":"h-odd"},{"Id":"f","ReceiptHandle":"h-ok2"}`))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"DeleteMessage"}, c.actions, "authorized once, as DeleteMessage")
|
||||
assert.Equal(t, []string{"arn:aws:sqs:" + testRegion + ":" + testAccount + ":q"}, c.resources)
|
||||
assert.Equal(t, []string{"h-ok", "h-forged", "h-uncertain", "h-gone", "h-odd", "h-ok2"}, del.handles, "every entry is tried in order")
|
||||
|
||||
var out batchReply
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &out))
|
||||
assert.Equal(t, []map[string]any{{"Id": "a"}, {"Id": "f"}}, out.Successful)
|
||||
assert.Equal(t, []map[string]any{
|
||||
{"Id": "b", "SenderFault": true, "Code": "ReceiptHandleIsInvalid", "Message": "The specified receipt handle isn't valid."},
|
||||
{"Id": "c", "SenderFault": false, "Code": "ServiceUnavailable", "Message": "The delete could not be confirmed; retry this entry."},
|
||||
{"Id": "d", "SenderFault": true, "Code": "AWS.SimpleQueueService.NonExistentQueue", "Message": "The specified queue doesn't exist."},
|
||||
{"Id": "e", "SenderFault": false, "Code": "InternalFailure", "Message": "The delete failed."},
|
||||
}, out.Failed)
|
||||
assert.NotContains(t, w.Body.String(), "h-", "no receipt handle is echoed")
|
||||
}
|
||||
|
||||
// An unknown deletion outcome stays retryable even when the owner's error
|
||||
// also carries a client-fault sentinel.
|
||||
func TestDeleteMessageBatch_UncertainOutranksInvalidHandle(t *testing.T) {
|
||||
del := &handleDeleter{errs: map[string]error{
|
||||
"h-joined": errors.Join(message.ErrReceiptInvalid, message.ErrUnavailable),
|
||||
"h-wrapped": fmt.Errorf("confirm: %w: %w", message.ErrUnavailable, message.ErrReceiptInvalid),
|
||||
"h-forged": message.ErrReceiptInvalid,
|
||||
}}
|
||||
w, _, err := callBatch(t, del, false, batchReq(
|
||||
`{"Id":"joined","ReceiptHandle":"h-joined"},{"Id":"wrapped","ReceiptHandle":"h-wrapped"},{"Id":"forged","ReceiptHandle":"h-forged"}`))
|
||||
require.NoError(t, err)
|
||||
var out batchReply
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &out))
|
||||
assert.Empty(t, out.Successful)
|
||||
assert.Equal(t, []map[string]any{
|
||||
{"Id": "joined", "SenderFault": false, "Code": "ServiceUnavailable", "Message": "The delete could not be confirmed; retry this entry."},
|
||||
{"Id": "wrapped", "SenderFault": false, "Code": "ServiceUnavailable", "Message": "The delete could not be confirmed; retry this entry."},
|
||||
{"Id": "forged", "SenderFault": true, "Code": "ReceiptHandleIsInvalid", "Message": "The specified receipt handle isn't valid."},
|
||||
}, out.Failed)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
// sqs:ListQueues defines no resource type, so it is authorized on "*".
|
||||
const actionListQueues = "ListQueues"
|
||||
|
||||
type listQueuesInput struct {
|
||||
QueueNamePrefix *string `json:"QueueNamePrefix"`
|
||||
NextToken *string `json:"NextToken"`
|
||||
MaxResults *int `json:"MaxResults"`
|
||||
}
|
||||
|
||||
type listQueuesOutput struct {
|
||||
QueueURLs []string `json:"QueueUrls,omitempty"`
|
||||
NextToken string `json:"NextToken,omitempty"`
|
||||
}
|
||||
|
||||
// listQueues lists the caller's active queues in its own account and Region.
|
||||
// Ordering, the token format and listing consistency are provisional.
|
||||
func listQueues(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in listQueuesInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
req := queue.ListRequest{Account: inv.AccountID, Region: inv.Region, MaxResults: in.MaxResults}
|
||||
if in.QueueNamePrefix != nil {
|
||||
req.Prefix = *in.QueueNamePrefix
|
||||
}
|
||||
if in.NextToken != nil {
|
||||
req.NextToken = *in.NextToken
|
||||
}
|
||||
if err := req.Validate(); err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
if err := authorize(inv, actionListQueues, "*"); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Queues == nil || d.Endpoint == "" {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
res, err := d.Queues.List(ctx, req)
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "SQS ListQueues failed", "cloud.account.id", inv.AccountID, "err", err)
|
||||
return ownerError(err)
|
||||
}
|
||||
out := listQueuesOutput{NextToken: res.NextToken}
|
||||
for _, r := range res.Queues {
|
||||
out.QueueURLs = append(out.QueueURLs, queueURL(d.Endpoint, r.Account, r.Name))
|
||||
}
|
||||
return writeJSON(w, out)
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
const (
|
||||
actionGetQueueURL = "GetQueueUrl"
|
||||
actionGetQueueAttributes = "GetQueueAttributes"
|
||||
)
|
||||
|
||||
type getQueueURLInput struct {
|
||||
QueueName *string `json:"QueueName"`
|
||||
QueueOwnerAWSAccountID *string `json:"QueueOwnerAWSAccountId"`
|
||||
}
|
||||
|
||||
type getQueueURLOutput struct {
|
||||
QueueURL string `json:"QueueUrl"`
|
||||
}
|
||||
|
||||
// getQueueURL returns the URL of an active queue in the caller's account.
|
||||
// Queue policies are not supported, so naming another owner is refused before
|
||||
// any read, with the code AWS returns for another account's queue URL.
|
||||
func getQueueURL(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in getQueueURLInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueName == nil || *in.QueueName == "" {
|
||||
return missing("QueueName")
|
||||
}
|
||||
name := *in.QueueName
|
||||
if err := queue.ValidateName(name); err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
if owner := in.QueueOwnerAWSAccountID; owner != nil && *owner != "" && *owner != inv.AccountID {
|
||||
return invalidAddress(d)
|
||||
}
|
||||
if err := authorize(inv, actionGetQueueURL, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
rec, err := resolve(d, inv, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeJSON(w, getQueueURLOutput{QueueURL: queueURL(d.Endpoint, rec.Account, rec.Name)})
|
||||
}
|
||||
|
||||
type getQueueAttributesInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
AttributeNames []string `json:"AttributeNames"`
|
||||
}
|
||||
|
||||
type getQueueAttributesOutput struct {
|
||||
Attributes map[string]string `json:"Attributes,omitempty"`
|
||||
}
|
||||
|
||||
// getQueueAttributes returns the named attributes of a queue in the caller's
|
||||
// account. Message counts, and All which includes them, are refused until the
|
||||
// counts can be read.
|
||||
func getQueueAttributes(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in getQueueAttributesInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := queue.ValidateAttributeNames(in.AttributeNames); err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
if err := authorize(inv, actionGetQueueAttributes, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
rec, err := resolve(d, inv, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attrs, err := rec.AttributesFor(in.AttributeNames)
|
||||
if err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
return writeJSON(w, getQueueAttributesOutput{Attributes: attrs})
|
||||
}
|
||||
|
||||
func resolve(d Deps, inv dispatch.Invocation, name string) (queue.Record, error) {
|
||||
if d.Queues == nil || d.Endpoint == "" {
|
||||
return queue.Record{}, errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
rec, err := d.Queues.Resolve(ctx, inv.AccountID, inv.Region, name)
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "SQS queue lookup failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return queue.Record{}, ownerError(err)
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// parseQueueURL returns the queue name a URL of the form
|
||||
// <endpoint>/<account>/<name> names in the caller's account. Only the
|
||||
// configured endpoint's scheme and host are accepted; no alias host is.
|
||||
func parseQueueURL(d Deps, inv dispatch.Invocation, raw string) (string, error) {
|
||||
ep, err := url.Parse(d.Endpoint)
|
||||
if err != nil || ep.Host == "" {
|
||||
return "", errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || !strings.EqualFold(u.Scheme, ep.Scheme) || !strings.EqualFold(u.Host, ep.Host) ||
|
||||
u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
|
||||
return "", invalidAddress(d)
|
||||
}
|
||||
rest, ok := strings.CutPrefix(u.Path, ep.Path+"/")
|
||||
if !ok {
|
||||
return "", invalidAddress(d)
|
||||
}
|
||||
account, name, ok := strings.Cut(rest, "/")
|
||||
if !ok || strings.Contains(name, "/") || account != inv.AccountID {
|
||||
return "", invalidAddress(d)
|
||||
}
|
||||
if queue.ValidateName(name) != nil {
|
||||
return "", errors.New(awserrors.ErrorSQSQueueDoesNotExist)
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
func invalidAddress(d Deps) error {
|
||||
return awserrors.Errorf(awserrors.ErrorSQSInvalidAddress, "The address %s/ is not valid for this endpoint.", d.Endpoint)
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The gateway tests run on an IP endpoint, so a named host's case rule and a
|
||||
// path-prefixed endpoint are checked here.
|
||||
func TestParseQueueURL_NamedEndpoint(t *testing.T) {
|
||||
inv := dispatch.Invocation{AccountID: "123456789012"}
|
||||
|
||||
for endpoint, forms := range map[string][]string{
|
||||
"https://sqs.example:9999": {"https://sqs.example:9999/123456789012/q", "https://SQS.Example:9999/123456789012/q"},
|
||||
"https://edge.example/sqs-api": {"https://edge.example/sqs-api/123456789012/q"},
|
||||
} {
|
||||
for _, raw := range forms {
|
||||
name, err := parseQueueURL(Deps{Endpoint: endpoint}, inv, raw)
|
||||
require.NoError(t, err, raw)
|
||||
assert.Equal(t, "q", name)
|
||||
}
|
||||
}
|
||||
|
||||
for _, raw := range []string{
|
||||
"https://edge.example/123456789012/q",
|
||||
"https://edge.example/other/123456789012/q",
|
||||
} {
|
||||
_, err := parseQueueURL(Deps{Endpoint: "https://edge.example/sqs-api"}, inv, raw)
|
||||
assert.Error(t, err, raw)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
const (
|
||||
actionReceiveMessage = "ReceiveMessage"
|
||||
actionDeleteMessage = "DeleteMessage"
|
||||
)
|
||||
|
||||
// System attribute names a message can carry. Only the first four are ever
|
||||
// present: send refuses trace headers and FIFO, and redrive is unsupported.
|
||||
const (
|
||||
sysAll = "All"
|
||||
sysSenderID = "SenderId"
|
||||
sysSentTimestamp = "SentTimestamp"
|
||||
sysReceiveCount = "ApproximateReceiveCount"
|
||||
sysFirstReceive = "ApproximateFirstReceiveTimestamp"
|
||||
sysSequenceNumber = "SequenceNumber"
|
||||
sysDeduplicationID = "MessageDeduplicationId"
|
||||
sysGroupID = "MessageGroupId"
|
||||
sysTraceHeader = "AWSTraceHeader"
|
||||
sysDeadLetterSourceQARN = "DeadLetterQueueSourceArn"
|
||||
)
|
||||
|
||||
var systemAttributeNames = map[string]bool{
|
||||
sysAll: true, sysSenderID: true, sysSentTimestamp: true, sysReceiveCount: true,
|
||||
sysFirstReceive: true, sysSequenceNumber: true, sysDeduplicationID: true,
|
||||
sysGroupID: true, sysTraceHeader: true, sysDeadLetterSourceQARN: true,
|
||||
}
|
||||
|
||||
type receiveMessageInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
// AttributeNames is the discontinued spelling of
|
||||
// MessageSystemAttributeNames; both are honoured.
|
||||
AttributeNames []string `json:"AttributeNames"`
|
||||
MessageSystemAttributeNames []string `json:"MessageSystemAttributeNames"`
|
||||
MessageAttributeNames []string `json:"MessageAttributeNames"`
|
||||
MaxNumberOfMessages *int `json:"MaxNumberOfMessages"`
|
||||
VisibilityTimeout *int `json:"VisibilityTimeout"`
|
||||
WaitTimeSeconds *int `json:"WaitTimeSeconds"`
|
||||
ReceiveRequestAttemptID *string `json:"ReceiveRequestAttemptId"`
|
||||
}
|
||||
|
||||
type receivedMessage struct {
|
||||
MessageID string `json:"MessageId"`
|
||||
ReceiptHandle string `json:"ReceiptHandle"`
|
||||
MD5OfBody string `json:"MD5OfBody"`
|
||||
Body string `json:"Body"`
|
||||
Attributes map[string]string `json:"Attributes,omitempty"`
|
||||
MessageAttributes map[string]wireAttributeOut `json:"MessageAttributes,omitempty"`
|
||||
MD5OfMessageAttributes string `json:"MD5OfMessageAttributes,omitempty"`
|
||||
}
|
||||
|
||||
// wireAttributeOut is one returned message attribute; exactly one value
|
||||
// member is set, since send refuses a missing or empty value.
|
||||
type wireAttributeOut struct {
|
||||
DataType string `json:"DataType"`
|
||||
StringValue *string `json:"StringValue,omitempty"`
|
||||
BinaryValue []byte `json:"BinaryValue,omitempty"`
|
||||
}
|
||||
|
||||
type receiveMessageOutput struct {
|
||||
Messages []receivedMessage `json:"Messages,omitempty"`
|
||||
}
|
||||
|
||||
// receiveMessage takes up to MaxNumberOfMessages messages in flight from a
|
||||
// queue in the caller's account, long polling for the request's wait or the
|
||||
// queue's default; a cancelled request ends the wait.
|
||||
func receiveMessage(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in receiveMessageInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
if in.ReceiveRequestAttemptID != nil {
|
||||
return awserrors.Errorf(awserrors.ErrorUnsupportedOperation, "ReceiveRequestAttemptId applies only to FIFO queues, which are not supported.")
|
||||
}
|
||||
want, err := systemAttributes(in.AttributeNames, in.MessageSystemAttributeNames)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sel, err := message.ParseAttributeSelector(in.MessageAttributeNames)
|
||||
if err != nil {
|
||||
return messageError(err)
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req := message.ReceiveInput{
|
||||
Account: inv.AccountID, Region: inv.Region, Queue: name,
|
||||
MaxNumberOfMessages: in.MaxNumberOfMessages, VisibilityTimeout: in.VisibilityTimeout,
|
||||
WaitTimeSeconds: in.WaitTimeSeconds,
|
||||
}
|
||||
if err := message.ValidateReceive(req); err != nil {
|
||||
return messageError(err)
|
||||
}
|
||||
if err := authorize(inv, actionReceiveMessage, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Receives == nil || d.Endpoint == "" {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
got, err := d.Receives.Receive(ctx, req)
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "SQS ReceiveMessage failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return messageError(err)
|
||||
}
|
||||
out := receiveMessageOutput{}
|
||||
for _, m := range got {
|
||||
attrs := m.Attributes.Select(sel)
|
||||
out.Messages = append(out.Messages, receivedMessage{
|
||||
MessageID: m.MessageID, ReceiptHandle: m.ReceiptHandle, MD5OfBody: m.MD5OfBody,
|
||||
Body: string(m.Body), Attributes: attributesOf(m, want),
|
||||
MessageAttributes: messageAttributesOut(attrs), MD5OfMessageAttributes: attrs.MD5(),
|
||||
})
|
||||
}
|
||||
return writeJSON(w, out)
|
||||
}
|
||||
|
||||
// systemAttributes is the set of system attributes asked for across both
|
||||
// spellings. An unknown name is InvalidAttributeName, provisionally.
|
||||
func systemAttributes(lists ...[]string) (map[string]bool, error) {
|
||||
want := map[string]bool{}
|
||||
for _, names := range lists {
|
||||
for _, n := range names {
|
||||
if !systemAttributeNames[n] {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorSQSInvalidAttributeName, "Unknown Attribute %s.", n)
|
||||
}
|
||||
want[n] = true
|
||||
}
|
||||
}
|
||||
return want, nil
|
||||
}
|
||||
|
||||
func attributesOf(m message.Received, want map[string]bool) map[string]string {
|
||||
all := want[sysAll]
|
||||
out := map[string]string{}
|
||||
if all || want[sysSenderID] {
|
||||
out[sysSenderID] = m.SenderID
|
||||
}
|
||||
if all || want[sysSentTimestamp] {
|
||||
out[sysSentTimestamp] = strconv.FormatInt(m.SentAt.UnixMilli(), 10)
|
||||
}
|
||||
if all || want[sysReceiveCount] {
|
||||
out[sysReceiveCount] = strconv.FormatUint(m.ReceiveCount, 10)
|
||||
}
|
||||
if all || want[sysFirstReceive] {
|
||||
out[sysFirstReceive] = strconv.FormatInt(m.FirstReceive.UnixMilli(), 10)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// messageAttributesOut renders the selected attributes; the digest covers
|
||||
// only these, provisionally until compared against AWS.
|
||||
func messageAttributesOut(a message.Attributes) map[string]wireAttributeOut {
|
||||
if a.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]wireAttributeOut, a.Len())
|
||||
for _, at := range a.All() {
|
||||
out[at.Name] = wireAttributeOut{DataType: at.Value.DataType, StringValue: at.Value.StringValue, BinaryValue: at.Value.BinaryValue}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type deleteMessageInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
ReceiptHandle *string `json:"ReceiptHandle"`
|
||||
}
|
||||
|
||||
// deleteMessage tombstones the message a receipt handle names. The handle is
|
||||
// authenticated against the queue only after authorizing, so an unauthorized
|
||||
// caller learns nothing about it.
|
||||
func deleteMessage(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in deleteMessageInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
if in.ReceiptHandle == nil || *in.ReceiptHandle == "" {
|
||||
return missing("ReceiptHandle")
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := authorize(inv, actionDeleteMessage, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Deletes == nil {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
err = d.Deletes.Delete(ctx, message.DeleteInput{
|
||||
Account: inv.AccountID, Region: inv.Region, Queue: name, ReceiptHandle: *in.ReceiptHandle,
|
||||
})
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "SQS DeleteMessage failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return messageError(err)
|
||||
}
|
||||
return writeJSON(w, struct{}{})
|
||||
}
|
||||
|
||||
// messageError maps a message owner error to its SQS wire code; the
|
||||
// parameter, 12-hour limit and unsupported codes are provisional.
|
||||
func messageError(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, message.ErrQueueDoesNotExist):
|
||||
return errors.New(awserrors.ErrorSQSQueueDoesNotExist)
|
||||
case errors.Is(err, message.ErrUnavailable):
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
case errors.Is(err, message.ErrReceiptInvalid):
|
||||
return errors.New(awserrors.ErrorSQSReceiptHandleIsInvalid)
|
||||
case errors.Is(err, message.ErrNotInflight):
|
||||
return errors.New(awserrors.ErrorSQSMessageNotInflight)
|
||||
case errors.Is(err, message.ErrVisibilityCap):
|
||||
return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "VisibilityTimeout would extend past 12 hours from this receive.")
|
||||
case errors.Is(err, message.ErrInvalidParameter):
|
||||
return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "%s", err.Error())
|
||||
case errors.Is(err, message.ErrUnsupported):
|
||||
return awserrors.Errorf(awserrors.ErrorUnsupportedOperation, "%s", err.Error())
|
||||
}
|
||||
return errors.New(awserrors.ErrorInternalFailure)
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// stubMessageOps answers receives with got and records each request.
|
||||
type stubMessageOps struct {
|
||||
got []message.Received
|
||||
err error
|
||||
receives []message.ReceiveInput
|
||||
deletes []message.DeleteInput
|
||||
changes []message.ChangeVisibilityInput
|
||||
}
|
||||
|
||||
func (s *stubMessageOps) ChangeVisibility(_ context.Context, in message.ChangeVisibilityInput) error {
|
||||
s.changes = append(s.changes, in)
|
||||
return s.err
|
||||
}
|
||||
|
||||
func (s *stubMessageOps) Receive(_ context.Context, in message.ReceiveInput) ([]message.Received, error) {
|
||||
s.receives = append(s.receives, in)
|
||||
return s.got, s.err
|
||||
}
|
||||
|
||||
func (s *stubMessageOps) Delete(_ context.Context, in message.DeleteInput) error {
|
||||
s.deletes = append(s.deletes, in)
|
||||
return s.err
|
||||
}
|
||||
|
||||
func recvReq(rest string) string { return `{"QueueUrl":"` + testQueueURL + `"` + rest + `}` }
|
||||
|
||||
var sample = message.Received{
|
||||
MessageID: "m-1", ReceiptHandle: "h-1", Body: []byte("hi é"), MD5OfBody: "abc",
|
||||
SenderID: "AIDAALICE", SentAt: time.UnixMilli(1_700_000_000_123).UTC(), ReceiveCount: 3,
|
||||
FirstReceive: time.UnixMilli(1_700_000_005_456).UTC(),
|
||||
}
|
||||
|
||||
type wireMessage struct {
|
||||
MessageID string `json:"MessageId"`
|
||||
ReceiptHandle string `json:"ReceiptHandle"`
|
||||
MD5OfBody string `json:"MD5OfBody"`
|
||||
Body string `json:"Body"`
|
||||
Attributes map[string]string `json:"Attributes"`
|
||||
}
|
||||
|
||||
func received(t *testing.T, h *harness, body string) []wireMessage {
|
||||
t.Helper()
|
||||
w, err := h.call(t, awsapi.OpReceiveMessage, body)
|
||||
require.NoError(t, err)
|
||||
var out struct {
|
||||
Messages []wireMessage `json:"Messages"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &out))
|
||||
return out.Messages
|
||||
}
|
||||
|
||||
func TestReceiveMessage_RendersMessageAndAskedAttributes(t *testing.T) {
|
||||
all := map[string]string{
|
||||
"SenderId": "AIDAALICE", "SentTimestamp": "1700000000123",
|
||||
"ApproximateReceiveCount": "3", "ApproximateFirstReceiveTimestamp": "1700000005456",
|
||||
}
|
||||
for name, tc := range map[string]struct {
|
||||
rest string
|
||||
want map[string]string
|
||||
}{
|
||||
"none asked": {"", nil},
|
||||
"all": {`,"MessageSystemAttributeNames":["All"]`, all},
|
||||
"legacy all": {`,"AttributeNames":["All"]`, all},
|
||||
"one": {`,"MessageSystemAttributeNames":["SentTimestamp"]`, map[string]string{"SentTimestamp": "1700000000123"}},
|
||||
"both spellings": {`,"AttributeNames":["SenderId"],"MessageSystemAttributeNames":["ApproximateReceiveCount"]`, map[string]string{"SenderId": "AIDAALICE", "ApproximateReceiveCount": "3"}},
|
||||
"never carried": {`,"MessageSystemAttributeNames":["SequenceNumber","MessageGroupId","MessageDeduplicationId","AWSTraceHeader","DeadLetterQueueSourceArn"]`, nil},
|
||||
"first receive": {`,"MessageSystemAttributeNames":["ApproximateFirstReceiveTimestamp"]`, map[string]string{"ApproximateFirstReceiveTimestamp": "1700000005456"}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
h := newHarness()
|
||||
h.ops.got = []message.Received{sample}
|
||||
got := received(t, h, recvReq(tc.rest))
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, wireMessage{MessageID: "m-1", ReceiptHandle: "h-1", MD5OfBody: "abc", Body: "hi é", Attributes: tc.want}, got[0])
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An outage that wraps an invalid handle is still an outage, never a claim
|
||||
// that the handle is bad.
|
||||
func TestMessageOps_OwnerErrorsMapped(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
err error
|
||||
want string
|
||||
}{
|
||||
{message.ErrQueueDoesNotExist, awserrors.ErrorSQSQueueDoesNotExist},
|
||||
{message.ErrReceiptInvalid, awserrors.ErrorSQSReceiptHandleIsInvalid},
|
||||
{message.ErrNotInflight, awserrors.ErrorSQSMessageNotInflight},
|
||||
{message.ErrVisibilityCap, awserrors.ErrorInvalidParameterValueException},
|
||||
{fmt.Errorf("%w: %w", message.ErrUnavailable, message.ErrNotInflight), awserrors.ErrorServiceUnavailableException},
|
||||
{fmt.Errorf("%w: %w", message.ErrUnavailable, message.ErrReceiptInvalid), awserrors.ErrorServiceUnavailableException},
|
||||
{message.ErrUnavailable, awserrors.ErrorServiceUnavailableException},
|
||||
{message.ErrInvalidParameter, awserrors.ErrorInvalidParameterValueException},
|
||||
{message.ErrUnsupported, awserrors.ErrorUnsupportedOperation},
|
||||
{errors.New("unexpected"), awserrors.ErrorInternalFailure},
|
||||
} {
|
||||
for _, op := range []struct{ name, body string }{
|
||||
{awsapi.OpReceiveMessage, recvReq("")},
|
||||
{awsapi.OpDeleteMessage, `{"QueueUrl":"` + testQueueURL + `","ReceiptHandle":"h"}`},
|
||||
{awsapi.OpChangeVisibility, `{"QueueUrl":"` + testQueueURL + `","ReceiptHandle":"h","VisibilityTimeout":5}`},
|
||||
} {
|
||||
h := newHarness()
|
||||
h.ops.err = tc.err
|
||||
w, err := h.call(t, op.name, op.body)
|
||||
assert.Equal(t, tc.want, code(t, err), "%s: %v", op.name, tc.err)
|
||||
assert.Empty(t, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
// Package awsapi is the public SQS adapter: AWS JSON 1.0 with query-compatible
|
||||
// errors, served under the sqs signing service. It parses and validates each
|
||||
// request, authorizes it on the queue ARN, then runs the queue owner.
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/sigv4"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
|
||||
)
|
||||
|
||||
// ServiceName is the SigV4 credential-scope service SQS clients sign for.
|
||||
const ServiceName = "sqs"
|
||||
|
||||
// TargetPrefix is the X-Amz-Target prefix of every SQS JSON 1.0 operation.
|
||||
const TargetPrefix = "AmazonSQS."
|
||||
|
||||
// ErrorNamespace is the Smithy namespace of the errors the SQS model defines.
|
||||
const ErrorNamespace = "com.amazonaws.sqs"
|
||||
|
||||
// Operation names the registration dispatches.
|
||||
const (
|
||||
OpCreateQueue = "CreateQueue"
|
||||
OpGetQueueURL = "GetQueueUrl"
|
||||
OpGetQueueAttributes = "GetQueueAttributes"
|
||||
OpListQueues = "ListQueues"
|
||||
OpSetQueueAttributes = "SetQueueAttributes"
|
||||
OpTagQueue = "TagQueue"
|
||||
OpUntagQueue = "UntagQueue"
|
||||
OpListQueueTags = "ListQueueTags"
|
||||
OpSendMessage = "SendMessage"
|
||||
OpReceiveMessage = "ReceiveMessage"
|
||||
OpDeleteMessage = "DeleteMessage"
|
||||
OpChangeVisibility = "ChangeMessageVisibility"
|
||||
OpDeleteMessageBatch = "DeleteMessageBatch"
|
||||
OpSendMessageBatch = "SendMessageBatch"
|
||||
OpChangeVisibilityBatch = "ChangeMessageVisibilityBatch"
|
||||
)
|
||||
|
||||
// Queues is the queue owner capability the actions run against.
|
||||
type Queues interface {
|
||||
Create(ctx context.Context, req queue.CreateRequest) (queue.Record, error)
|
||||
Resolve(ctx context.Context, account, region, name string) (queue.Record, error)
|
||||
List(ctx context.Context, req queue.ListRequest) (queue.ListResult, error)
|
||||
SetAttributes(ctx context.Context, account, region, name string, attrs map[string]string) (queue.Record, error)
|
||||
Tag(ctx context.Context, account, region, name string, tags map[string]string) (queue.Record, error)
|
||||
Untag(ctx context.Context, account, region, name string, keys []string) (queue.Record, error)
|
||||
Tags(ctx context.Context, account, region, name string) (map[string]string, error)
|
||||
}
|
||||
|
||||
var _ Queues = (*queue.Store)(nil)
|
||||
|
||||
// Messages is the message owner capability sends run against.
|
||||
type Messages interface {
|
||||
Send(ctx context.Context, in message.SendInput) (message.SendResult, error)
|
||||
}
|
||||
|
||||
var _ Messages = (*message.Sender)(nil)
|
||||
|
||||
// Receives is the message owner capability receives run against.
|
||||
type Receives interface {
|
||||
Receive(ctx context.Context, in message.ReceiveInput) ([]message.Received, error)
|
||||
}
|
||||
|
||||
var _ Receives = (*message.Receiver)(nil)
|
||||
|
||||
// Deletes is the message owner capability deletes run against.
|
||||
type Deletes interface {
|
||||
Delete(ctx context.Context, in message.DeleteInput) error
|
||||
}
|
||||
|
||||
var _ Deletes = (*message.Deleter)(nil)
|
||||
|
||||
// Visibility is the message owner capability visibility changes run against.
|
||||
type Visibility interface {
|
||||
ChangeVisibility(ctx context.Context, in message.ChangeVisibilityInput) error
|
||||
}
|
||||
|
||||
var _ Visibility = (*message.VisibilityChanger)(nil)
|
||||
|
||||
// Deps are the capabilities the adapter dispatches to. A nil Queues answers
|
||||
// every valid, authorized request with ServiceUnavailable.
|
||||
type Deps struct {
|
||||
Queues Queues
|
||||
// Messages runs sends; nil answers a valid, authorized send with
|
||||
// ServiceUnavailable.
|
||||
Messages Messages
|
||||
// Receives and Deletes run ReceiveMessage and DeleteMessage; nil answers
|
||||
// a valid, authorized request with ServiceUnavailable.
|
||||
Receives Receives
|
||||
Deletes Deletes
|
||||
// Visibility runs ChangeMessageVisibility; nil answers a valid,
|
||||
// authorized request with ServiceUnavailable.
|
||||
Visibility Visibility
|
||||
// Endpoint is the base URL queue URLs are built on, with no trailing
|
||||
// slash, e.g. https://sqs.example:9999.
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
type handler func(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error
|
||||
|
||||
var operations = map[string]handler{
|
||||
OpCreateQueue: createQueue,
|
||||
OpGetQueueURL: getQueueURL,
|
||||
OpGetQueueAttributes: getQueueAttributes,
|
||||
OpListQueues: listQueues,
|
||||
OpSetQueueAttributes: setQueueAttributes,
|
||||
OpTagQueue: tagQueue,
|
||||
OpUntagQueue: untagQueue,
|
||||
OpListQueueTags: listQueueTags,
|
||||
OpSendMessage: sendMessage,
|
||||
OpReceiveMessage: receiveMessage,
|
||||
OpDeleteMessage: deleteMessage,
|
||||
OpChangeVisibility: changeMessageVisibility,
|
||||
OpDeleteMessageBatch: deleteMessageBatch,
|
||||
OpSendMessageBatch: sendMessageBatch,
|
||||
OpChangeVisibilityBatch: changeMessageVisibilityBatch,
|
||||
}
|
||||
|
||||
// OperationNames lists every operation the registration dispatches, sorted.
|
||||
func OperationNames() []string {
|
||||
names := make([]string, 0, len(operations))
|
||||
for n := range operations {
|
||||
names = append(names, n)
|
||||
}
|
||||
slices.Sort(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// queryErrorCodes are the legacy query codes the SQS model gives error shapes
|
||||
// whose name differs, as aws-sdk-go v1 records them, and the Coral framework
|
||||
// errors' query codes as captured from AWS.
|
||||
var queryErrorCodes = map[string]string{
|
||||
awserrors.ErrorSQSQueueDoesNotExist: "AWS.SimpleQueueService.NonExistentQueue",
|
||||
awserrors.ErrorSQSQueueNameExists: "QueueAlreadyExists",
|
||||
awserrors.ErrorSQSQueueDeletedRecently: "AWS.SimpleQueueService.QueueDeletedRecently",
|
||||
awserrors.ErrorSQSMessageNotInflight: "AWS.SimpleQueueService.MessageNotInflight",
|
||||
awserrors.ErrorSQSEmptyBatchRequest: "AWS.SimpleQueueService.EmptyBatchRequest",
|
||||
awserrors.ErrorSQSTooManyEntriesInBatchRequest: "AWS.SimpleQueueService.TooManyEntriesInBatchRequest",
|
||||
awserrors.ErrorSQSBatchEntryIdsNotDistinct: "AWS.SimpleQueueService.BatchEntryIdsNotDistinct",
|
||||
awserrors.ErrorSQSInvalidBatchEntryId: "AWS.SimpleQueueService.InvalidBatchEntryId",
|
||||
awserrors.ErrorSQSBatchRequestTooLong: "AWS.SimpleQueueService.BatchRequestTooLong",
|
||||
awserrors.ErrorUnsupportedOperation: "AWS.SimpleQueueService.UnsupportedOperation",
|
||||
awserrors.ErrorInvalidParameterValueException: "InvalidParameterValue",
|
||||
awserrors.ErrorMissingRequiredParameterException: "MissingParameter",
|
||||
awserrors.ErrorSerializationException: "MalformedInput",
|
||||
awserrors.ErrorUnknownOperationException: "InvalidAction",
|
||||
awserrors.ErrorAccessDeniedException: "AccessDenied",
|
||||
awserrors.ErrorServiceUnavailableException: "ServiceUnavailable",
|
||||
}
|
||||
|
||||
// coralErrors are raised by the AWS framework rather than the SQS model.
|
||||
var coralErrors = map[string]string{
|
||||
awserrors.ErrorInvalidParameterValueException: envelope.CoralNamespace,
|
||||
awserrors.ErrorMissingRequiredParameterException: envelope.CoralNamespace,
|
||||
awserrors.ErrorSerializationException: envelope.CoralNamespace,
|
||||
awserrors.ErrorUnknownOperationException: envelope.CoralNamespace,
|
||||
awserrors.ErrorAccessDeniedException: envelope.CoralNamespace,
|
||||
awserrors.ErrorServiceUnavailableException: envelope.CoralNamespace,
|
||||
}
|
||||
|
||||
// NewRegistration returns the public SQS JSON 1.0 registration.
|
||||
func NewRegistration(deps Deps) dispatch.Registration {
|
||||
return dispatch.Registration{
|
||||
Service: ServiceName,
|
||||
ResolveAction: resolveAction,
|
||||
Dispatch: deps.dispatch,
|
||||
Errors: dispatch.ErrorEnvelopeJSON10Query,
|
||||
QueryErrorCodes: queryErrorCodes,
|
||||
ErrorNamespace: ErrorNamespace,
|
||||
ErrorNamespaces: coralErrors,
|
||||
Inventory: dispatch.Inventory{Registered: OperationNames()},
|
||||
}
|
||||
}
|
||||
|
||||
// resolveAction accepts only "AmazonSQS.<Operation>"; anything else, the
|
||||
// query protocol included, names no operation.
|
||||
func resolveAction(r *http.Request) string {
|
||||
op, ok := strings.CutPrefix(r.Header.Get("X-Amz-Target"), TargetPrefix)
|
||||
if !ok || strings.Contains(op, ".") {
|
||||
return ""
|
||||
}
|
||||
return op
|
||||
}
|
||||
|
||||
func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
|
||||
r := inv.Request
|
||||
h, ok := operations[resolveAction(r)]
|
||||
if !ok {
|
||||
return errors.New(awserrors.ErrorUnknownOperationException)
|
||||
}
|
||||
if inv.AccountID == "" || inv.Region == "" {
|
||||
slog.ErrorContext(r.Context(), "SQS dispatch: no account or region in the invocation")
|
||||
return errors.New(awserrors.ErrorInternalFailure)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
|
||||
if err != nil {
|
||||
return errors.New(awserrors.ErrorSerializationException)
|
||||
}
|
||||
if int64(len(body)) > sigv4.MaxPayloadLen {
|
||||
return errors.New(awserrors.ErrorRequestEntityTooLarge)
|
||||
}
|
||||
return h(d, w, inv, body)
|
||||
}
|
||||
|
||||
// decode reads exactly one JSON object into v: null, any other value, or
|
||||
// anything but whitespace after the object is a SerializationException.
|
||||
// Unknown members are ignored, provisionally until compared against AWS.
|
||||
func decode(body []byte, v any) error {
|
||||
trimmed := bytes.TrimSpace(body)
|
||||
if len(trimmed) == 0 {
|
||||
trimmed = []byte("{}")
|
||||
}
|
||||
if trimmed[0] != '{' {
|
||||
return errors.New(awserrors.ErrorSerializationException)
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(trimmed))
|
||||
if err := dec.Decode(v); err != nil {
|
||||
return errors.New(awserrors.ErrorSerializationException)
|
||||
}
|
||||
if _, err := dec.Token(); !errors.Is(err, io.EOF) {
|
||||
return errors.New(awserrors.ErrorSerializationException)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func missing(param string) error {
|
||||
return awserrors.Errorf(awserrors.ErrorMissingRequiredParameterException, "The request must contain the parameter %s.", param)
|
||||
}
|
||||
|
||||
// authorize reports a denial as SQS's framework AccessDenied. The tag keys
|
||||
// AWS defines for these actions are not passed: the evaluator does not enforce
|
||||
// them and fails closed on any statement that names one.
|
||||
func authorize(inv dispatch.Invocation, action, resource string) error {
|
||||
err := inv.Authorize(ServiceName, action, []string{resource}, nil)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if code, ok := awserrors.ResolveErrorCode(err); ok && code == awserrors.ErrorAccessDenied {
|
||||
return errors.New(awserrors.ErrorAccessDeniedException)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// ownerError maps a queue owner error to its SQS wire code. Codes not
|
||||
// characterised against AWS yet are provisional.
|
||||
func ownerError(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, queue.ErrQueueDoesNotExist):
|
||||
return errors.New(awserrors.ErrorSQSQueueDoesNotExist)
|
||||
case errors.Is(err, queue.ErrQueueNameExists):
|
||||
return errors.New(awserrors.ErrorSQSQueueNameExists)
|
||||
case errors.Is(err, queue.ErrQueueDeletedRecently):
|
||||
return errors.New(awserrors.ErrorSQSQueueDeletedRecently)
|
||||
case errors.Is(err, queue.ErrInvalidAttributeName):
|
||||
return awserrors.Errorf(awserrors.ErrorSQSInvalidAttributeName, "%s", err.Error())
|
||||
case errors.Is(err, queue.ErrInvalidAttributeValue):
|
||||
return awserrors.Errorf(awserrors.ErrorSQSInvalidAttributeValue, "%s", err.Error())
|
||||
case errors.Is(err, queue.ErrUnsupportedAttribute):
|
||||
return awserrors.Errorf(awserrors.ErrorUnsupportedOperation, "%s", err.Error())
|
||||
case errors.Is(err, queue.ErrInvalidQueueName), errors.Is(err, queue.ErrInvalidTag),
|
||||
errors.Is(err, queue.ErrInvalidListRequest):
|
||||
return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "%s", err.Error())
|
||||
case errors.Is(err, queue.ErrUnavailable):
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
return errors.New(awserrors.ErrorInternalFailure)
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, v any) error {
|
||||
out, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return errors.New(awserrors.ErrorInternalFailure)
|
||||
}
|
||||
w.Header().Set("Content-Type", envelope.JSON10ContentType)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := w.Write(out); err != nil {
|
||||
slog.Error("SQS: failed to write response", "err", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func queueURL(endpoint, account, name string) string {
|
||||
return endpoint + "/" + account + "/" + name
|
||||
}
|
||||
|
||||
func queueARN(inv dispatch.Invocation, name string) string {
|
||||
return arn.FormatSQSQueue(inv.Region, inv.AccountID, name)
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/sqs"
|
||||
sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
|
||||
)
|
||||
|
||||
const otherAccount = "210987654321"
|
||||
|
||||
// sdkApp serves the SQS registration to the AWS SDK over the real queue and
|
||||
// message owners. The caller's account and the actions it may call are set
|
||||
// per request; ingress authentication itself is not under test here.
|
||||
type sdkApp struct {
|
||||
url string
|
||||
|
||||
mu sync.Mutex
|
||||
account string
|
||||
denied map[string]bool
|
||||
asked []string
|
||||
}
|
||||
|
||||
func newSDKApp(t *testing.T) *sdkApp {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
_, _, cpJS := testutil.StartTestJetStream(t)
|
||||
qs, err := queue.Open(ctx, cpJS, 3, 2)
|
||||
require.NoError(t, err)
|
||||
_, _, storeJS := testutil.StartTestJetStream(t)
|
||||
ms, err := sqsjs.New(ctx, storeJS, sqsjs.Config{MessageStreams: 3, StateBuckets: 2, DuplicateWindow: time.Minute})
|
||||
require.NoError(t, err)
|
||||
keys, err := message.NewReceiptKeys(1, map[uint32][]byte{1: bytes.Repeat([]byte{9}, message.MinReceiptKeyLen)})
|
||||
require.NoError(t, err)
|
||||
|
||||
a := &sdkApp{account: testAccount, denied: map[string]bool{}}
|
||||
var reg dispatch.Registration
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
a.mu.Lock()
|
||||
account := a.account
|
||||
a.mu.Unlock()
|
||||
err := reg.Dispatch(w, dispatch.Invocation{
|
||||
Request: r, AccountID: account, Region: testRegion, Service: awsapi.ServiceName,
|
||||
Authorize: a.authorize,
|
||||
Actor: func() (dispatch.Actor, error) { return dispatch.Actor{UserID: "AIDAPOLLER", AccountID: account}, nil },
|
||||
})
|
||||
if err != nil {
|
||||
writeError(w, reg, err)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
a.url = srv.URL
|
||||
reg = awsapi.NewRegistration(awsapi.Deps{
|
||||
Queues: qs, Messages: &message.Sender{Store: ms, Queues: qs},
|
||||
Receives: &message.Receiver{Store: ms, Queues: qs, Keys: keys},
|
||||
Deletes: &message.Deleter{Store: ms, Queues: qs, Keys: keys},
|
||||
Visibility: &message.VisibilityChanger{Store: ms, Queues: qs, Keys: keys},
|
||||
Endpoint: srv.URL,
|
||||
})
|
||||
return a
|
||||
}
|
||||
|
||||
func (a *sdkApp) authorize(_, action string, resources []string, _ iampolicy.ConditionKeys) error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
for _, r := range resources {
|
||||
a.asked = append(a.asked, action+" "+r)
|
||||
}
|
||||
if a.denied[action] {
|
||||
return &dispatch.PolicyDenialError{Action: action}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *sdkApp) as(account string, denied ...string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.account, a.denied, a.asked = account, map[string]bool{}, nil
|
||||
for _, d := range denied {
|
||||
a.denied[d] = true
|
||||
}
|
||||
}
|
||||
|
||||
func (a *sdkApp) client() *sqs.Client {
|
||||
return sqs.New(sqs.Options{
|
||||
BaseEndpoint: aws.String(a.url), Region: testRegion, RetryMaxAttempts: 1,
|
||||
Credentials: aws.AnonymousCredentials{},
|
||||
})
|
||||
}
|
||||
|
||||
// writeError renders a dispatch error as the JSON 1.0 query-compatible body
|
||||
// the SDK decodes, using the registration's legacy query codes.
|
||||
func writeError(w http.ResponseWriter, reg dispatch.Registration, err error) {
|
||||
code, msg, _ := awserrors.ResolveErrorDetail(err)
|
||||
if code == "" {
|
||||
code = err.Error()
|
||||
}
|
||||
meta := awserrors.LookupErrorMessage(awsapi.ServiceName, code)
|
||||
if msg == "" {
|
||||
msg = meta.Message
|
||||
}
|
||||
qc := code
|
||||
if q, ok := reg.QueryErrorCodes[code]; ok {
|
||||
qc = q
|
||||
}
|
||||
envelope.WriteJSON(w, envelope.ProtocolJSON10Query, envelope.JSONError{
|
||||
Code: code, Verbatim: true, Message: msg, Status: meta.HTTPCode, Namespace: reg.ErrorNamespace, QueryCode: qc,
|
||||
})
|
||||
}
|
||||
|
||||
func apiCode(t *testing.T, err error) string {
|
||||
t.Helper()
|
||||
var ae smithy.APIError
|
||||
require.ErrorAs(t, err, &ae)
|
||||
return ae.ErrorCode()
|
||||
}
|
||||
|
||||
// TestSDK_LambdaPollerPath drives the calls an event source mapping makes, in
|
||||
// order, through the registration and the real queue and message owners.
|
||||
func TestSDK_LambdaPollerPath(t *testing.T) {
|
||||
a := newSDKApp(t)
|
||||
c := a.client()
|
||||
ctx := context.Background()
|
||||
|
||||
created, err := c.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("lambda-src"),
|
||||
Attributes: map[string]string{"VisibilityTimeout": "45"}})
|
||||
require.NoError(t, err)
|
||||
got, err := c.GetQueueUrl(ctx, &sqs.GetQueueUrlInput{QueueName: aws.String("lambda-src")})
|
||||
require.NoError(t, err)
|
||||
u := aws.ToString(got.QueueUrl)
|
||||
require.Equal(t, aws.ToString(created.QueueUrl), u)
|
||||
|
||||
attrs, err := c.GetQueueAttributes(ctx, &sqs.GetQueueAttributesInput{QueueUrl: aws.String(u),
|
||||
AttributeNames: []sqstypes.QueueAttributeName{sqstypes.QueueAttributeNameQueueArn, sqstypes.QueueAttributeNameVisibilityTimeout}})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "arn:aws:sqs:"+testRegion+":"+testAccount+":lambda-src", attrs.Attributes["QueueArn"])
|
||||
assert.Equal(t, "45", attrs.Attributes["VisibilityTimeout"])
|
||||
|
||||
entries := make([]sqstypes.SendMessageBatchRequestEntry, 3)
|
||||
for i := range entries {
|
||||
entries[i] = sqstypes.SendMessageBatchRequestEntry{Id: aws.String("e" + strconv.Itoa(i)), MessageBody: aws.String("job-" + strconv.Itoa(i)),
|
||||
MessageAttributes: map[string]sqstypes.MessageAttributeValue{"kind": {DataType: aws.String("String"), StringValue: aws.String("job")}}}
|
||||
}
|
||||
sent, err := c.SendMessageBatch(ctx, &sqs.SendMessageBatchInput{QueueUrl: aws.String(u), Entries: entries})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, sent.Successful, 3)
|
||||
require.Empty(t, sent.Failed)
|
||||
|
||||
recv, err := c.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(u), MaxNumberOfMessages: 10,
|
||||
WaitTimeSeconds: 20, MessageAttributeNames: []string{"All"},
|
||||
MessageSystemAttributeNames: []sqstypes.MessageSystemAttributeName{sqstypes.MessageSystemAttributeNameAll}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, recv.Messages, 3)
|
||||
bodies := map[string]bool{}
|
||||
for _, m := range recv.Messages {
|
||||
bodies[aws.ToString(m.Body)] = true
|
||||
assert.Equal(t, "job", aws.ToString(m.MessageAttributes["kind"].StringValue))
|
||||
assert.NotEmpty(t, m.Attributes["SenderId"])
|
||||
assert.Equal(t, "1", m.Attributes["ApproximateReceiveCount"])
|
||||
assert.NotEmpty(t, m.Attributes["ApproximateFirstReceiveTimestamp"])
|
||||
}
|
||||
assert.Equal(t, map[string]bool{"job-0": true, "job-1": true, "job-2": true}, bodies)
|
||||
|
||||
// A failed record goes back to the queue; the rest are deleted in a batch.
|
||||
failed := recv.Messages[0]
|
||||
vis, err := c.ChangeMessageVisibilityBatch(ctx, &sqs.ChangeMessageVisibilityBatchInput{QueueUrl: aws.String(u),
|
||||
Entries: []sqstypes.ChangeMessageVisibilityBatchRequestEntry{{Id: aws.String("f"), ReceiptHandle: failed.ReceiptHandle, VisibilityTimeout: 0}}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, vis.Successful, 1)
|
||||
del := make([]sqstypes.DeleteMessageBatchRequestEntry, 0, 2)
|
||||
for i, m := range recv.Messages[1:] {
|
||||
del = append(del, sqstypes.DeleteMessageBatchRequestEntry{Id: aws.String("d" + strconv.Itoa(i)), ReceiptHandle: m.ReceiptHandle})
|
||||
}
|
||||
dr, err := c.DeleteMessageBatch(ctx, &sqs.DeleteMessageBatchInput{QueueUrl: aws.String(u), Entries: del})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, dr.Successful, 2)
|
||||
require.Empty(t, dr.Failed)
|
||||
|
||||
again, err := c.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(u), MaxNumberOfMessages: 10, WaitTimeSeconds: 1})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, again.Messages, 1, "only the released record is redelivered")
|
||||
assert.Equal(t, aws.ToString(failed.Body), aws.ToString(again.Messages[0].Body))
|
||||
_, err = c.DeleteMessage(ctx, &sqs.DeleteMessageInput{QueueUrl: aws.String(u), ReceiptHandle: again.Messages[0].ReceiptHandle})
|
||||
require.NoError(t, err)
|
||||
|
||||
start := time.Now()
|
||||
empty, err := c.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(u), WaitTimeSeconds: 2})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, empty.Messages)
|
||||
assert.GreaterOrEqual(t, time.Since(start), 2*time.Second, "an empty long poll waits its full time")
|
||||
}
|
||||
|
||||
// TestSDK_AuthorizationAndIsolation checks that each call is authorized on the
|
||||
// queue ARN, that a denial changes nothing, and that one account cannot reach
|
||||
// another account's queue by URL.
|
||||
func TestSDK_AuthorizationAndIsolation(t *testing.T) {
|
||||
a := newSDKApp(t)
|
||||
c := a.client()
|
||||
ctx := context.Background()
|
||||
created, err := c.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("jobs")})
|
||||
require.NoError(t, err)
|
||||
u := aws.ToString(created.QueueUrl)
|
||||
arn := "arn:aws:sqs:" + testRegion + ":" + testAccount + ":jobs"
|
||||
|
||||
a.as(testAccount, "SendMessage")
|
||||
_, err = c.SendMessage(ctx, &sqs.SendMessageInput{QueueUrl: aws.String(u), MessageBody: aws.String("x")})
|
||||
assert.Equal(t, awserrors.ErrorAccessDenied, apiCode(t, err))
|
||||
assert.Equal(t, []string{"SendMessage " + arn}, a.asked)
|
||||
|
||||
a.as(otherAccount)
|
||||
_, err = c.SendMessage(ctx, &sqs.SendMessageInput{QueueUrl: aws.String(u), MessageBody: aws.String("x")})
|
||||
assert.Equal(t, awserrors.ErrorSQSInvalidAddress, apiCode(t, err))
|
||||
assert.Empty(t, a.asked, "a foreign queue URL is refused before authorization")
|
||||
_, err = c.GetQueueUrl(ctx, &sqs.GetQueueUrlInput{QueueName: aws.String("jobs")})
|
||||
var missing *sqstypes.QueueDoesNotExist
|
||||
assert.ErrorAs(t, err, &missing, "another account sees no queue named jobs")
|
||||
|
||||
a.as(testAccount)
|
||||
out, err := c.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(u), WaitTimeSeconds: 0})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, out.Messages, "neither refused send reached the queue")
|
||||
assert.Equal(t, []string{"ReceiveMessage " + arn}, a.asked)
|
||||
}
|
||||
|
||||
// TestSDK_DeleteBatchReportsEachEntry checks that a batch with a valid and a
|
||||
// tampered handle deletes the valid one and reports the other as failed.
|
||||
func TestSDK_DeleteBatchReportsEachEntry(t *testing.T) {
|
||||
a := newSDKApp(t)
|
||||
c := a.client()
|
||||
ctx := context.Background()
|
||||
created, err := c.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("jobs")})
|
||||
require.NoError(t, err)
|
||||
u := aws.ToString(created.QueueUrl)
|
||||
for _, b := range []string{"a", "b"} {
|
||||
_, err = c.SendMessage(ctx, &sqs.SendMessageInput{QueueUrl: aws.String(u), MessageBody: aws.String(b)})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
recv, err := c.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(u), MaxNumberOfMessages: 10, VisibilityTimeout: 1})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, recv.Messages, 2)
|
||||
kept := recv.Messages[1]
|
||||
// Flip a middle character, not the last, whose low bits base64 may drop.
|
||||
h := []byte(aws.ToString(kept.ReceiptHandle))
|
||||
i := len(h) / 2
|
||||
if h[i] == 'A' {
|
||||
h[i] = 'B'
|
||||
} else {
|
||||
h[i] = 'A'
|
||||
}
|
||||
tampered := string(h)
|
||||
require.NotEqual(t, aws.ToString(kept.ReceiptHandle), tampered)
|
||||
|
||||
dr, err := c.DeleteMessageBatch(ctx, &sqs.DeleteMessageBatchInput{QueueUrl: aws.String(u), Entries: []sqstypes.DeleteMessageBatchRequestEntry{
|
||||
{Id: aws.String("ok"), ReceiptHandle: recv.Messages[0].ReceiptHandle},
|
||||
{Id: aws.String("bad"), ReceiptHandle: aws.String(tampered)},
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, dr.Successful, 1)
|
||||
assert.Equal(t, "ok", aws.ToString(dr.Successful[0].Id))
|
||||
require.Len(t, dr.Failed, 1)
|
||||
assert.Equal(t, "bad", aws.ToString(dr.Failed[0].Id))
|
||||
assert.True(t, dr.Failed[0].SenderFault)
|
||||
assert.Contains(t, aws.ToString(dr.Failed[0].Code), "ReceiptHandleIsInvalid")
|
||||
assert.NotContains(t, aws.ToString(dr.Failed[0].Message), tampered, "a handle is never echoed")
|
||||
|
||||
again, err := c.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(u), MaxNumberOfMessages: 10, WaitTimeSeconds: 3})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, again.Messages, 1, "only the entry that failed to delete returns")
|
||||
assert.Equal(t, aws.ToString(kept.Body), aws.ToString(again.Messages[0].Body))
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
// MaxBatchPayload is the most the bodies and message attributes of one
|
||||
// batch may total, in bytes.
|
||||
const MaxBatchPayload = message.MaxMessageSize
|
||||
|
||||
type sendBatchInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
Entries *[]sendBatchEntry `json:"Entries"`
|
||||
}
|
||||
|
||||
type sendBatchEntry struct {
|
||||
ID *string `json:"Id"`
|
||||
MessageBody json.RawMessage `json:"MessageBody"`
|
||||
DelaySeconds *int `json:"DelaySeconds"`
|
||||
// MessageAttributes is decoded exactly; the features after it are
|
||||
// refused, never stored inert, and only checked for presence.
|
||||
MessageAttributes json.RawMessage `json:"MessageAttributes"`
|
||||
MessageSystemAttributes map[string]json.RawMessage `json:"MessageSystemAttributes"`
|
||||
MessageGroupID *string `json:"MessageGroupId"`
|
||||
MessageDeduplicationID *string `json:"MessageDeduplicationId"`
|
||||
}
|
||||
|
||||
type sendBatchSuccess struct {
|
||||
ID string `json:"Id"`
|
||||
MessageID string `json:"MessageId"`
|
||||
MD5OfMessageBody string `json:"MD5OfMessageBody"`
|
||||
MD5OfMessageAttributes string `json:"MD5OfMessageAttributes,omitempty"`
|
||||
}
|
||||
|
||||
type sendBatchOutput struct {
|
||||
Successful []sendBatchSuccess `json:"Successful"`
|
||||
Failed []batchFailure `json:"Failed"`
|
||||
}
|
||||
|
||||
// sendPlan is one entry after validation: a request to send, or the failure
|
||||
// that entry reports without reaching the owner.
|
||||
type sendPlan struct {
|
||||
id string
|
||||
req message.SendInput
|
||||
fail *batchFailure
|
||||
}
|
||||
|
||||
// sendMessageBatch appends each entry under the SendMessage permission. The
|
||||
// batch shape, Ids and total size are checked before any append; each entry's
|
||||
// outcome, uncertain ones included, is reported under its own Id.
|
||||
func sendMessageBatch(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in sendBatchInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
if in.Entries == nil {
|
||||
return missing("Entries")
|
||||
}
|
||||
entries := *in.Entries
|
||||
ids := make([]*string, len(entries))
|
||||
for i, e := range entries {
|
||||
ids[i] = e.ID
|
||||
}
|
||||
if err := validateBatchIDs(ids); err != nil {
|
||||
return err
|
||||
}
|
||||
plans, err := planSends(entries)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := authorize(inv, actionSendMessage, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Messages == nil || d.Queues == nil || d.Endpoint == "" {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
// Every entry is pinned to the queue the name resolves to now, so a
|
||||
// queue replaced under the same name mid-batch is never written to.
|
||||
pin, err := resolveBatchQueue(d, inv, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if inv.Actor == nil {
|
||||
slog.ErrorContext(ctx, "SQS SendMessageBatch: no caller identity in the invocation")
|
||||
return errors.New(awserrors.ErrorInternalFailure)
|
||||
}
|
||||
actor, err := inv.Actor()
|
||||
if err != nil || actor.UserID == "" {
|
||||
slog.ErrorContext(ctx, "SQS SendMessageBatch: caller identity unresolved", "cloud.account.id", inv.AccountID, "err", err)
|
||||
return errors.New(awserrors.ErrorInternalFailure)
|
||||
}
|
||||
out := sendBatchOutput{Successful: []sendBatchSuccess{}, Failed: []batchFailure{}}
|
||||
for _, p := range plans {
|
||||
if p.fail != nil {
|
||||
out.Failed = append(out.Failed, *p.fail)
|
||||
continue
|
||||
}
|
||||
p.req.Account, p.req.Region, p.req.Queue, p.req.Sender = inv.AccountID, inv.Region, name, actor.UserID
|
||||
p.req.QueueID = pin
|
||||
res, err := d.Messages.Send(ctx, p.req)
|
||||
if err == nil {
|
||||
out.Successful = append(out.Successful, sendBatchSuccess{
|
||||
ID: p.id, MessageID: res.MessageID, MD5OfMessageBody: res.MD5OfBody, MD5OfMessageAttributes: res.MD5OfMessageAttributes,
|
||||
})
|
||||
continue
|
||||
}
|
||||
slog.DebugContext(ctx, "SQS SendMessageBatch entry failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "sqs.batch_entry_id", p.id, "err", err)
|
||||
out.Failed = append(out.Failed, sendEntryFailure(p.id, err))
|
||||
}
|
||||
return writeJSON(w, out)
|
||||
}
|
||||
|
||||
// planSends decodes and validates every entry. A malformed body, malformed
|
||||
// attribute JSON or a batch over MaxBatchPayload fails the request; an entry
|
||||
// the owner would refuse becomes that entry's failure.
|
||||
func planSends(entries []sendBatchEntry) ([]sendPlan, error) {
|
||||
plans := make([]sendPlan, len(entries))
|
||||
total := 0
|
||||
for i, e := range entries {
|
||||
p := sendPlan{id: *e.ID}
|
||||
plans[i] = p
|
||||
if e.MessageBody == nil || string(e.MessageBody) == "null" {
|
||||
f := sendEntryFailure(p.id, message.ErrBodyRequired)
|
||||
plans[i].fail = &f
|
||||
continue
|
||||
}
|
||||
body, err := decodeBody(e.MessageBody)
|
||||
if err != nil {
|
||||
if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorSQSInvalidMessageContents {
|
||||
return nil, err
|
||||
}
|
||||
plans[i].fail = &batchFailure{ID: p.id, SenderFault: true, Code: awserrors.ErrorSQSInvalidMessageContents, Message: "The message body is not well-formed UTF-8 or UTF-16."}
|
||||
continue
|
||||
}
|
||||
total += len(body)
|
||||
req := message.SendInput{Body: body, DelaySeconds: e.DelaySeconds, MessageGroupID: e.MessageGroupID, MessageDeduplicationID: e.MessageDeduplicationID}
|
||||
attrs, err := decodeMessageAttributes(e.MessageAttributes)
|
||||
if err != nil {
|
||||
if !errors.Is(err, message.ErrInvalidParameter) && !errors.Is(err, message.ErrUnsupported) {
|
||||
return nil, err
|
||||
}
|
||||
f := sendEntryFailure(p.id, err)
|
||||
plans[i].fail = &f
|
||||
continue
|
||||
}
|
||||
req.MessageAttributes = attrs
|
||||
// Decoded attributes count toward the batch total with the body even
|
||||
// when the owner will refuse them below.
|
||||
total += message.SuppliedSize(attrs)
|
||||
if len(e.MessageSystemAttributes) > 0 {
|
||||
req.MessageSystemAttributes = map[string]message.AttributeValue{}
|
||||
for k := range e.MessageSystemAttributes {
|
||||
req.MessageSystemAttributes[k] = message.AttributeValue{}
|
||||
}
|
||||
}
|
||||
if err := message.ValidateMessage(req); err != nil {
|
||||
f := sendEntryFailure(p.id, err)
|
||||
plans[i].fail = &f
|
||||
continue
|
||||
}
|
||||
plans[i].req = req
|
||||
}
|
||||
if total > MaxBatchPayload {
|
||||
return nil, awserrors.Errorf(awserrors.ErrorSQSBatchRequestTooLong, "Batch requests cannot be longer than %d bytes. You have sent %d bytes.", MaxBatchPayload, total)
|
||||
}
|
||||
return plans, nil
|
||||
}
|
||||
|
||||
// sendEntryFailure reports one entry's error under its query code, an
|
||||
// uncertain append before any client fault so it stays retryable. Owner
|
||||
// messages name offsets and limits, never the body.
|
||||
func sendEntryFailure(id string, err error) batchFailure {
|
||||
switch {
|
||||
case errors.Is(err, message.ErrQueueDoesNotExist):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorSQSQueueDoesNotExist], Message: "The specified queue doesn't exist."}
|
||||
case errors.Is(err, message.ErrUnavailable):
|
||||
return batchFailure{ID: id, Code: queryErrorCodes[awserrors.ErrorServiceUnavailableException], Message: "The send could not be confirmed; a retry may store this message twice."}
|
||||
case errors.Is(err, message.ErrBodyRequired):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorMissingRequiredParameterException], Message: "The request must contain the parameter MessageBody."}
|
||||
case errors.Is(err, message.ErrInvalidContents):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: awserrors.ErrorSQSInvalidMessageContents, Message: err.Error()}
|
||||
case errors.Is(err, message.ErrInvalidParameter):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorInvalidParameterValueException], Message: err.Error()}
|
||||
case errors.Is(err, message.ErrUnsupported):
|
||||
return batchFailure{ID: id, SenderFault: true, Code: queryErrorCodes[awserrors.ErrorUnsupportedOperation], Message: err.Error()}
|
||||
}
|
||||
return batchFailure{ID: id, Code: awserrors.ErrorInternalFailure, Message: "The send failed."}
|
||||
}
|
||||
|
||||
// resolveBatchQueue resolves a batch's queue once, before any entry runs,
|
||||
// and fails closed unless the record is exactly the requested queue.
|
||||
func resolveBatchQueue(d Deps, inv dispatch.Invocation, name string) (message.QueueID, error) {
|
||||
ctx := inv.Request.Context()
|
||||
rec, err := d.Queues.Resolve(ctx, inv.AccountID, inv.Region, name)
|
||||
if err != nil {
|
||||
return "", ownerError(err)
|
||||
}
|
||||
ns := rec.Namespace()
|
||||
if err := ns.Validate(); err != nil || rec.Account != inv.AccountID || rec.Region != inv.Region || rec.Name != name {
|
||||
slog.ErrorContext(ctx, "SQS batch: resolved queue record is not the requested queue", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return "", errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
return ns.Queue, nil
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package awsapi_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/bluebottle/pkg/iampolicy"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// bodySender answers each send with the error keyed by its body.
|
||||
type bodySender struct {
|
||||
errs map[string]error
|
||||
sent []message.SendInput
|
||||
}
|
||||
|
||||
func (s *bodySender) Send(_ context.Context, in message.SendInput) (message.SendResult, error) {
|
||||
s.sent = append(s.sent, in)
|
||||
if err := s.errs[string(in.Body)]; err != nil {
|
||||
return message.SendResult{}, err
|
||||
}
|
||||
return message.SendResult{MessageID: "id-" + string(in.Body), MD5OfBody: "md5-" + string(in.Body)}, nil
|
||||
}
|
||||
|
||||
// pinQueues resolves every name to one queue, or fails with err; alter,
|
||||
// when set, corrupts the record it returns.
|
||||
type pinQueues struct {
|
||||
countingQueues
|
||||
|
||||
err error
|
||||
alter func(*queue.Record)
|
||||
}
|
||||
|
||||
func (q *pinQueues) Resolve(_ context.Context, account, region, name string) (queue.Record, error) {
|
||||
q.calls++
|
||||
if q.err != nil {
|
||||
return queue.Record{}, q.err
|
||||
}
|
||||
r := queue.Record{QueueID: "qid-" + name, Account: account, Region: region, Name: name, State: queue.StateActive}
|
||||
if q.alter != nil {
|
||||
q.alter(&r)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func callSendBatch(t *testing.T, msgs awsapi.Messages, deny bool, body string) (*httptest.ResponseRecorder, *batchCall, error) {
|
||||
t.Helper()
|
||||
return callSendBatchOn(t, &pinQueues{}, msgs, deny, body)
|
||||
}
|
||||
|
||||
func callSendBatchOn(t *testing.T, qs awsapi.Queues, msgs awsapi.Messages, deny bool, body string) (*httptest.ResponseRecorder, *batchCall, error) {
|
||||
t.Helper()
|
||||
reg := awsapi.NewRegistration(awsapi.Deps{Queues: qs, Messages: msgs, Endpoint: testEndpoint})
|
||||
r := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||
r.Header.Set("X-Amz-Target", awsapi.TargetPrefix+awsapi.OpSendMessageBatch)
|
||||
w := httptest.NewRecorder()
|
||||
c := &batchCall{}
|
||||
err := reg.Dispatch(w, dispatch.Invocation{
|
||||
Request: r, AccountID: testAccount, Region: testRegion, Service: awsapi.ServiceName,
|
||||
Authorize: func(_ string, action string, resources []string, _ iampolicy.ConditionKeys) error {
|
||||
c.actions = append(c.actions, action)
|
||||
c.resources = append(c.resources, resources...)
|
||||
if deny {
|
||||
return errors.New(awserrors.ErrorAccessDeniedException)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Actor: func() (dispatch.Actor, error) { return dispatch.Actor{UserID: "AIDAALICE"}, nil },
|
||||
})
|
||||
return w, c, err
|
||||
}
|
||||
|
||||
func sendBatchReq(entries string) string {
|
||||
return `{"QueueUrl":"` + testQueueURL + `","Entries":[` + entries + `]}`
|
||||
}
|
||||
|
||||
func TestSendMessageBatch_EntryOutcomes(t *testing.T) {
|
||||
msgs := &bodySender{errs: map[string]error{
|
||||
"uncertain": fmt.Errorf("%w: append: lost", message.ErrUnavailable),
|
||||
"both": errors.Join(message.ErrInvalidParameter, message.ErrUnavailable),
|
||||
"gone": message.ErrQueueDoesNotExist,
|
||||
"toolong": fmt.Errorf("%w: message must be shorter than 1025 bytes", message.ErrInvalidParameter),
|
||||
"odd": errors.New("boom"),
|
||||
}}
|
||||
w, c, err := callSendBatch(t, msgs, false, sendBatchReq(
|
||||
`{"Id":"a","MessageBody":"ok","DelaySeconds":5},{"Id":"b","MessageBody":"uncertain"},{"Id":"c","MessageBody":"both"},`+
|
||||
`{"Id":"d","MessageBody":"gone"},{"Id":"e","MessageBody":"toolong"},{"Id":"f","MessageBody":"odd"},{"Id":"g","MessageBody":"ok2"}`))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"SendMessage"}, c.actions, "authorized once, as SendMessage")
|
||||
assert.Equal(t, []string{"arn:aws:sqs:" + testRegion + ":" + testAccount + ":q"}, c.resources)
|
||||
require.Len(t, msgs.sent, 7, "every entry is tried")
|
||||
first := msgs.sent[0]
|
||||
assert.Equal(t, []any{testAccount, testRegion, "q", "AIDAALICE", "ok", 5}, []any{first.Account, first.Region, first.Queue, first.Sender, string(first.Body), *first.DelaySeconds})
|
||||
assert.Nil(t, msgs.sent[1].DelaySeconds, "an absent delay leaves the queue's default")
|
||||
|
||||
var out struct {
|
||||
Successful []map[string]any `json:"Successful"`
|
||||
Failed []map[string]any `json:"Failed"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &out))
|
||||
assert.Equal(t, []map[string]any{
|
||||
{"Id": "a", "MessageId": "id-ok", "MD5OfMessageBody": "md5-ok"},
|
||||
{"Id": "g", "MessageId": "id-ok2", "MD5OfMessageBody": "md5-ok2"},
|
||||
}, out.Successful)
|
||||
retry := "The send could not be confirmed; a retry may store this message twice."
|
||||
assert.Equal(t, []map[string]any{
|
||||
{"Id": "b", "SenderFault": false, "Code": "ServiceUnavailable", "Message": retry},
|
||||
{"Id": "c", "SenderFault": false, "Code": "ServiceUnavailable", "Message": retry},
|
||||
{"Id": "d", "SenderFault": true, "Code": "AWS.SimpleQueueService.NonExistentQueue", "Message": "The specified queue doesn't exist."},
|
||||
{"Id": "e", "SenderFault": true, "Code": "InvalidParameterValue", "Message": "message: invalid parameter value: message must be shorter than 1025 bytes"},
|
||||
{"Id": "f", "SenderFault": false, "Code": "InternalFailure", "Message": "The send failed."},
|
||||
}, out.Failed)
|
||||
}
|
||||
|
||||
// The queue is resolved once after authorization; a name that resolves to
|
||||
// nothing fails the request before any append.
|
||||
func TestSendMessageBatch_QueueResolvedOnceBeforeAnyAppend(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
err error
|
||||
want string
|
||||
}{
|
||||
"missing": {queue.ErrQueueDoesNotExist, awserrors.ErrorSQSQueueDoesNotExist},
|
||||
"unavailable": {queue.ErrUnavailable, awserrors.ErrorServiceUnavailableException},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
msgs := &bodySender{}
|
||||
_, c, err := callSendBatchOn(t, &pinQueues{err: tc.err}, msgs, false, sendBatchReq(`{"Id":"x","MessageBody":"x"},{"Id":"y","MessageBody":"y"}`))
|
||||
assert.Equal(t, tc.want, code(t, err))
|
||||
assert.Len(t, c.actions, 1, "authorized before the queue is looked up")
|
||||
assert.Empty(t, msgs.sent)
|
||||
})
|
||||
}
|
||||
|
||||
qs := &pinQueues{}
|
||||
msgs := &bodySender{}
|
||||
_, _, err := callSendBatchOn(t, qs, msgs, false, sendBatchReq(`{"Id":"x","MessageBody":"x"},{"Id":"y","MessageBody":"y"}`))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, qs.calls)
|
||||
require.Len(t, msgs.sent, 2)
|
||||
for _, in := range msgs.sent {
|
||||
assert.Equal(t, message.QueueID("qid-q"), in.QueueID, "every entry is pinned to the resolved queue")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
const actionSendMessage = "SendMessage"
|
||||
|
||||
type sendMessageInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
MessageBody json.RawMessage `json:"MessageBody"`
|
||||
DelaySeconds *int `json:"DelaySeconds"`
|
||||
// MessageAttributes is decoded exactly; the features after it are
|
||||
// refused, never stored inert, and only checked for presence.
|
||||
MessageAttributes json.RawMessage `json:"MessageAttributes"`
|
||||
MessageSystemAttributes map[string]json.RawMessage `json:"MessageSystemAttributes"`
|
||||
MessageGroupID *string `json:"MessageGroupId"`
|
||||
MessageDeduplicationID *string `json:"MessageDeduplicationId"`
|
||||
}
|
||||
|
||||
type sendMessageOutput struct {
|
||||
MessageID string `json:"MessageId"`
|
||||
MD5OfMessageBody string `json:"MD5OfMessageBody"`
|
||||
MD5OfMessageAttributes string `json:"MD5OfMessageAttributes,omitempty"`
|
||||
}
|
||||
|
||||
// sendMessage appends one message to a queue in the caller's account. The
|
||||
// request is validated before authorizing; the size limit needs the queue, so
|
||||
// it is checked after. SenderId is the authenticated caller's aws:userid.
|
||||
func sendMessage(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in sendMessageInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
if in.MessageBody == nil || string(in.MessageBody) == "null" {
|
||||
return missing("MessageBody")
|
||||
}
|
||||
msgBody, err := decodeBody(in.MessageBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req := message.SendInput{
|
||||
Account: inv.AccountID, Region: inv.Region, Queue: name,
|
||||
Body: msgBody, DelaySeconds: in.DelaySeconds,
|
||||
}
|
||||
if req.MessageAttributes, err = decodeMessageAttributes(in.MessageAttributes); err != nil {
|
||||
return attributeError(err)
|
||||
}
|
||||
if len(in.MessageSystemAttributes) > 0 {
|
||||
req.MessageSystemAttributes = map[string]message.AttributeValue{}
|
||||
}
|
||||
for k := range in.MessageSystemAttributes {
|
||||
req.MessageSystemAttributes[k] = message.AttributeValue{}
|
||||
}
|
||||
req.MessageGroupID = in.MessageGroupID
|
||||
req.MessageDeduplicationID = in.MessageDeduplicationID
|
||||
if err := message.ValidateMessage(req); err != nil {
|
||||
return sendError(err)
|
||||
}
|
||||
if err := authorize(inv, actionSendMessage, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Messages == nil || d.Endpoint == "" {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
if inv.Actor == nil {
|
||||
slog.ErrorContext(ctx, "SQS SendMessage: no caller identity in the invocation")
|
||||
return errors.New(awserrors.ErrorInternalFailure)
|
||||
}
|
||||
actor, err := inv.Actor()
|
||||
if err != nil || actor.UserID == "" {
|
||||
slog.ErrorContext(ctx, "SQS SendMessage: caller identity unresolved", "cloud.account.id", inv.AccountID, "err", err)
|
||||
return errors.New(awserrors.ErrorInternalFailure)
|
||||
}
|
||||
req.Sender = actor.UserID
|
||||
res, err := d.Messages.Send(ctx, req)
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "SQS SendMessage failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return sendError(err)
|
||||
}
|
||||
return writeJSON(w, sendMessageOutput{MessageID: res.MessageID, MD5OfMessageBody: res.MD5OfBody, MD5OfMessageAttributes: res.MD5OfMessageAttributes})
|
||||
}
|
||||
|
||||
// attributeError passes a serialization fault through and maps an owner
|
||||
// validation error to its wire code.
|
||||
func attributeError(err error) error {
|
||||
if errors.Is(err, message.ErrInvalidParameter) || errors.Is(err, message.ErrUnsupported) {
|
||||
return sendError(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// sendError maps a message owner error to its SQS wire code. The empty-body,
|
||||
// unsupported-feature and oversize codes are provisional until the AWS baseline.
|
||||
func sendError(err error) error {
|
||||
switch {
|
||||
// An uncertain outcome is never reported as a settled refusal.
|
||||
case errors.Is(err, message.ErrUnavailable):
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
case errors.Is(err, message.ErrQueueDoesNotExist):
|
||||
return errors.New(awserrors.ErrorSQSQueueDoesNotExist)
|
||||
case errors.Is(err, message.ErrBodyRequired):
|
||||
return missing("MessageBody")
|
||||
case errors.Is(err, message.ErrInvalidContents):
|
||||
return awserrors.Errorf(awserrors.ErrorSQSInvalidMessageContents, "%s", err.Error())
|
||||
case errors.Is(err, message.ErrInvalidParameter):
|
||||
return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "%s", err.Error())
|
||||
case errors.Is(err, message.ErrUnsupported):
|
||||
return awserrors.Errorf(awserrors.ErrorUnsupportedOperation, "%s", err.Error())
|
||||
}
|
||||
return errors.New(awserrors.ErrorInternalFailure)
|
||||
}
|
||||
|
||||
// decodeBody decodes the MessageBody JSON string without the replacements
|
||||
// encoding/json makes: malformed UTF-8 and unpaired \u surrogate escapes are
|
||||
// refused rather than stored as U+FFFD. The code is provisional until baseline.
|
||||
func decodeBody(raw json.RawMessage) ([]byte, error) {
|
||||
invalid := awserrors.Errorf(awserrors.ErrorSQSInvalidMessageContents, "The message body is not well-formed UTF-8 or UTF-16.")
|
||||
if len(raw) < 2 || raw[0] != '"' || raw[len(raw)-1] != '"' {
|
||||
return nil, errors.New(awserrors.ErrorSerializationException)
|
||||
}
|
||||
if err := checkJSONText(raw); err != nil {
|
||||
if errors.Is(err, errMalformedText) {
|
||||
return nil, invalid
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var s string
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return nil, errors.New(awserrors.ErrorSerializationException)
|
||||
}
|
||||
return []byte(s), nil
|
||||
}
|
||||
|
||||
func hex4(b []byte, at int) (rune, bool) {
|
||||
if at+4 > len(b) {
|
||||
return 0, false
|
||||
}
|
||||
var r rune
|
||||
for _, c := range b[at : at+4] {
|
||||
switch {
|
||||
case c >= '0' && c <= '9':
|
||||
r = r<<4 | rune(c-'0')
|
||||
case c >= 'a' && c <= 'f':
|
||||
r = r<<4 | rune(c-'a'+10)
|
||||
case c >= 'A' && c <= 'F':
|
||||
r = r<<4 | rune(c-'A'+10)
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
return r, true
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
const actionSetQueueAttributes = "SetQueueAttributes"
|
||||
|
||||
type setQueueAttributesInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
Attributes map[string]string `json:"Attributes"`
|
||||
}
|
||||
|
||||
// setQueueAttributes updates a queue in the caller's account. The whole
|
||||
// attribute set is validated before authorizing; an empty set is refused as
|
||||
// missing, which is provisional until the AWS baseline.
|
||||
func setQueueAttributes(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in setQueueAttributesInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
if len(in.Attributes) == 0 {
|
||||
return missing("Attributes")
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := queue.ValidateSet(in.Attributes); err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
if err := authorize(inv, actionSetQueueAttributes, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Queues == nil || d.Endpoint == "" {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
if _, err := d.Queues.SetAttributes(ctx, inv.AccountID, inv.Region, name, in.Attributes); err != nil {
|
||||
slog.DebugContext(ctx, "SQS SetQueueAttributes failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return ownerError(err)
|
||||
}
|
||||
return writeJSON(w, struct{}{})
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package awsapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
)
|
||||
|
||||
const (
|
||||
actionUntagQueue = "UntagQueue"
|
||||
actionListQueueTags = "ListQueueTags"
|
||||
)
|
||||
|
||||
type tagQueueInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
Tags map[string]string `json:"Tags"`
|
||||
}
|
||||
|
||||
type untagQueueInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
TagKeys []string `json:"TagKeys"`
|
||||
}
|
||||
|
||||
// tagQueue adds or overwrites tags on a queue in the caller's account. Tag
|
||||
// condition keys are not passed: the IAM evaluator fails closed on them. An
|
||||
// empty tag set is refused as missing, which is provisional.
|
||||
func tagQueue(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in tagQueueInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
name, err := tagTarget(d, inv, in.QueueURL, len(in.Tags), "Tags")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := queue.ValidateTags(in.Tags, false); err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
return mutateTags(d, w, inv, actionTagQueue, name, func() error {
|
||||
_, err := d.Queues.Tag(inv.Request.Context(), inv.AccountID, inv.Region, name, in.Tags)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// untagQueue removes tag keys from a queue in the caller's account; keys the
|
||||
// queue does not carry are ignored.
|
||||
func untagQueue(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in untagQueueInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
name, err := tagTarget(d, inv, in.QueueURL, len(in.TagKeys), "TagKeys")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := queue.ValidateTagKeys(in.TagKeys); err != nil {
|
||||
return ownerError(err)
|
||||
}
|
||||
return mutateTags(d, w, inv, actionUntagQueue, name, func() error {
|
||||
_, err := d.Queues.Untag(inv.Request.Context(), inv.AccountID, inv.Region, name, in.TagKeys)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func tagTarget(d Deps, inv dispatch.Invocation, rawURL *string, n int, member string) (string, error) {
|
||||
if rawURL == nil || *rawURL == "" {
|
||||
return "", missing("QueueUrl")
|
||||
}
|
||||
if n == 0 {
|
||||
return "", missing(member)
|
||||
}
|
||||
return parseQueueURL(d, inv, *rawURL)
|
||||
}
|
||||
|
||||
func mutateTags(d Deps, w http.ResponseWriter, inv dispatch.Invocation, action, name string, run func() error) error {
|
||||
if err := authorize(inv, action, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Queues == nil || d.Endpoint == "" {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
if err := run(); err != nil {
|
||||
slog.DebugContext(inv.Request.Context(), "SQS "+action+" failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return ownerError(err)
|
||||
}
|
||||
return writeJSON(w, struct{}{})
|
||||
}
|
||||
|
||||
type listQueueTagsInput struct {
|
||||
QueueURL *string `json:"QueueUrl"`
|
||||
}
|
||||
|
||||
type listQueueTagsOutput struct {
|
||||
Tags map[string]string `json:"Tags,omitempty"`
|
||||
}
|
||||
|
||||
// listQueueTags returns the tags of a queue in the caller's account. A queue
|
||||
// with no tags answers with no Tags member, which is provisional.
|
||||
func listQueueTags(d Deps, w http.ResponseWriter, inv dispatch.Invocation, body []byte) error {
|
||||
var in listQueueTagsInput
|
||||
if err := decode(body, &in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.QueueURL == nil || *in.QueueURL == "" {
|
||||
return missing("QueueUrl")
|
||||
}
|
||||
name, err := parseQueueURL(d, inv, *in.QueueURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := authorize(inv, actionListQueueTags, queueARN(inv, name)); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Queues == nil || d.Endpoint == "" {
|
||||
return errors.New(awserrors.ErrorServiceUnavailableException)
|
||||
}
|
||||
ctx := inv.Request.Context()
|
||||
tags, err := d.Queues.Tags(ctx, inv.AccountID, inv.Region, name)
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "SQS ListQueueTags failed", "sqs.queue_name", name, "cloud.account.id", inv.AccountID, "err", err)
|
||||
return ownerError(err)
|
||||
}
|
||||
return writeJSON(w, listQueueTagsOutput{Tags: tags})
|
||||
}
|
||||
@@ -0,0 +1,494 @@
|
||||
package message
|
||||
|
||||
import (
|
||||
"crypto/md5" //nolint:gosec // MD5OfMessageAttributes is an AWS wire checksum, not a security control.
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Message attribute limits from the SQS developer guide.
|
||||
const (
|
||||
MaxAttributes = 10
|
||||
MaxAttributeNameLen = 256
|
||||
MaxAttributeTypeLen = 256
|
||||
maxNumberDigits = 38
|
||||
minNumberExponent = -128
|
||||
maxNumberExponent = 126
|
||||
attributeCodecFormat = 1
|
||||
)
|
||||
|
||||
// Transport types of the attribute digest and encoding.
|
||||
const (
|
||||
transportString byte = 1
|
||||
transportBinary byte = 2
|
||||
)
|
||||
|
||||
// ErrAttributesCorrupt reports stored attribute bytes that do not decode.
|
||||
var ErrAttributesCorrupt = errors.New("message: stored message attributes are corrupt")
|
||||
|
||||
// Attribute is one named message attribute as supplied by a client, kept
|
||||
// as a list so a decoder can report a repeated name instead of losing it.
|
||||
type Attribute struct {
|
||||
Name string
|
||||
Value AttributeValue
|
||||
}
|
||||
|
||||
// AttributeValue is one attribute's members. A nil StringValue or
|
||||
// BinaryValue was not supplied; a non-nil empty one was supplied empty.
|
||||
// The list members are reserved by AWS and refused whenever present.
|
||||
type AttributeValue struct {
|
||||
DataType string
|
||||
StringValue *string
|
||||
BinaryValue []byte
|
||||
StringListValues []string
|
||||
BinaryListValues [][]byte
|
||||
}
|
||||
|
||||
// Attributes is a validated attribute set, sorted by name.
|
||||
type Attributes struct {
|
||||
list []Attribute
|
||||
}
|
||||
|
||||
// NewAttributes validates up to MaxAttributes attributes and sorts them.
|
||||
// Errors name the attribute, never its value.
|
||||
func NewAttributes(in []Attribute) (Attributes, error) {
|
||||
if len(in) == 0 {
|
||||
return Attributes{}, nil
|
||||
}
|
||||
if len(in) > MaxAttributes {
|
||||
return Attributes{}, fmt.Errorf("%w: a message can have at most %d message attributes, got %d", ErrInvalidParameter, MaxAttributes, len(in))
|
||||
}
|
||||
seen := make(map[string]struct{}, len(in))
|
||||
out := make([]Attribute, 0, len(in))
|
||||
for _, a := range in {
|
||||
if err := validateAttributeName(a.Name); err != nil {
|
||||
return Attributes{}, err
|
||||
}
|
||||
if _, dup := seen[a.Name]; dup {
|
||||
return Attributes{}, fmt.Errorf("%w: message attribute name %q is repeated", ErrInvalidParameter, a.Name)
|
||||
}
|
||||
seen[a.Name] = struct{}{}
|
||||
v, err := validateAttributeValue(a.Name, a.Value)
|
||||
if err != nil {
|
||||
return Attributes{}, err
|
||||
}
|
||||
out = append(out, Attribute{Name: a.Name, Value: v})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
attrs := Attributes{list: out}
|
||||
// No queue accepts more, and it keeps every encoded length within uint32.
|
||||
if attrs.Size() > MaxMessageSize {
|
||||
return Attributes{}, fmt.Errorf("%w: message attributes are %d bytes, over the %d byte maximum", ErrInvalidParameter, attrs.Size(), MaxMessageSize)
|
||||
}
|
||||
return attrs, nil
|
||||
}
|
||||
|
||||
// Len is the number of attributes.
|
||||
func (a Attributes) Len() int { return len(a.list) }
|
||||
|
||||
// All returns copies of the attributes in name order.
|
||||
func (a Attributes) All() []Attribute {
|
||||
out := make([]Attribute, len(a.list))
|
||||
for i, at := range a.list {
|
||||
out[i] = Attribute{Name: at.Name, Value: copyValue(at.Value)}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Size is the bytes the attributes count toward a message's size limit:
|
||||
// every name, data type and value.
|
||||
func (a Attributes) Size() int {
|
||||
n := 0
|
||||
for _, at := range a.list {
|
||||
n += len(at.Name) + len(at.Value.DataType) + len(valueBytes(at.Value))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// SuppliedSize measures attributes as supplied, valid or not: every name,
|
||||
// data type and value, list members included. For a valid set it equals
|
||||
// Attributes.Size.
|
||||
func SuppliedSize(in []Attribute) int {
|
||||
n := 0
|
||||
for _, a := range in {
|
||||
n += len(a.Name) + len(a.Value.DataType) + len(a.Value.BinaryValue)
|
||||
if a.Value.StringValue != nil {
|
||||
n += len(*a.Value.StringValue)
|
||||
}
|
||||
for _, s := range a.Value.StringListValues {
|
||||
n += len(s)
|
||||
}
|
||||
for _, b := range a.Value.BinaryListValues {
|
||||
n += len(b)
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// MD5 is MD5OfMessageAttributes by the developer guide's algorithm, or ""
|
||||
// when there are no attributes.
|
||||
func (a Attributes) MD5() string {
|
||||
if len(a.list) == 0 {
|
||||
return ""
|
||||
}
|
||||
sum := md5.Sum(a.digestBuffer()) //nolint:gosec // AWS wire checksum.
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Encode is the store-neutral form: a format byte then the digest buffer,
|
||||
// which is length-prefixed and so decodes unambiguously. Nil when empty.
|
||||
func (a Attributes) Encode() []byte {
|
||||
if len(a.list) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]byte{attributeCodecFormat}, a.digestBuffer()...)
|
||||
}
|
||||
|
||||
// DecodeAttributes reverses Encode and revalidates the result, so stored
|
||||
// bytes that would not pass a send are ErrAttributesCorrupt.
|
||||
func DecodeAttributes(b []byte) (Attributes, error) {
|
||||
if len(b) == 0 {
|
||||
return Attributes{}, nil
|
||||
}
|
||||
if b[0] != attributeCodecFormat {
|
||||
return Attributes{}, fmt.Errorf("%w: unknown format %d", ErrAttributesCorrupt, b[0])
|
||||
}
|
||||
r := b[1:]
|
||||
var in []Attribute
|
||||
for len(r) > 0 {
|
||||
name, rest, ok := readField(r)
|
||||
if !ok {
|
||||
return Attributes{}, fmt.Errorf("%w: truncated name", ErrAttributesCorrupt)
|
||||
}
|
||||
dataType, rest, ok := readField(rest)
|
||||
if !ok || len(rest) == 0 {
|
||||
return Attributes{}, fmt.Errorf("%w: truncated data type", ErrAttributesCorrupt)
|
||||
}
|
||||
transport := rest[0]
|
||||
value, rest, ok := readField(rest[1:])
|
||||
if !ok {
|
||||
return Attributes{}, fmt.Errorf("%w: truncated value", ErrAttributesCorrupt)
|
||||
}
|
||||
v := AttributeValue{DataType: string(dataType)}
|
||||
switch transport {
|
||||
case transportString:
|
||||
s := string(value)
|
||||
v.StringValue = &s
|
||||
case transportBinary:
|
||||
v.BinaryValue = value
|
||||
default:
|
||||
return Attributes{}, fmt.Errorf("%w: unknown transport type %d", ErrAttributesCorrupt, transport)
|
||||
}
|
||||
in = append(in, Attribute{Name: string(name), Value: v})
|
||||
r = rest
|
||||
}
|
||||
attrs, err := NewAttributes(in)
|
||||
if err != nil {
|
||||
return Attributes{}, fmt.Errorf("%w: %w", ErrAttributesCorrupt, err)
|
||||
}
|
||||
for i := range in {
|
||||
if in[i].Name != attrs.list[i].Name {
|
||||
return Attributes{}, fmt.Errorf("%w: attributes out of order", ErrAttributesCorrupt)
|
||||
}
|
||||
}
|
||||
return attrs, nil
|
||||
}
|
||||
|
||||
// CheckMessageSize applies the combined limit: body bytes plus attribute
|
||||
// bytes must not exceed the queue's maximum message size.
|
||||
func CheckMessageSize(bodyLen int, attrs Attributes, maxSize int) error {
|
||||
if total := bodyLen + attrs.Size(); total > maxSize {
|
||||
return fmt.Errorf("%w: message body and attributes are %d bytes, over the queue's limit of %d", ErrInvalidParameter, total, maxSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// encodePayload frames a schema-2 body record: the attribute encoding's
|
||||
// length, the encoding, then the body.
|
||||
func encodePayload(attrs Attributes, body []byte) []byte {
|
||||
enc := attrs.Encode()
|
||||
out := make([]byte, 0, 4+len(enc)+len(body))
|
||||
out = appendField(out, enc)
|
||||
return append(out, body...)
|
||||
}
|
||||
|
||||
// decodePayload reverses encodePayload and checks the attributes against
|
||||
// the digest the send recorded.
|
||||
func decodePayload(payload []byte, md5a string) ([]byte, Attributes, error) {
|
||||
enc, body, ok := readField(payload)
|
||||
if !ok {
|
||||
return nil, Attributes{}, fmt.Errorf("%w: truncated payload", ErrAttributesCorrupt)
|
||||
}
|
||||
attrs, err := DecodeAttributes(enc)
|
||||
if err != nil {
|
||||
return nil, Attributes{}, err
|
||||
}
|
||||
if attrs.MD5() != md5a {
|
||||
return nil, Attributes{}, fmt.Errorf("%w: attributes do not match their recorded digest", ErrAttributesCorrupt)
|
||||
}
|
||||
return body, attrs, nil
|
||||
}
|
||||
|
||||
func (a Attributes) digestBuffer() []byte {
|
||||
var buf []byte
|
||||
for _, at := range a.list {
|
||||
buf = appendField(buf, []byte(at.Name))
|
||||
buf = appendField(buf, []byte(at.Value.DataType))
|
||||
if at.Value.StringValue != nil {
|
||||
buf = append(buf, transportString)
|
||||
} else {
|
||||
buf = append(buf, transportBinary)
|
||||
}
|
||||
buf = appendField(buf, valueBytes(at.Value))
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
func appendField(buf, b []byte) []byte {
|
||||
buf = binary.BigEndian.AppendUint32(buf, uint32(len(b))) //nolint:gosec // NewAttributes bounds every field by MaxMessageSize.
|
||||
return append(buf, b...)
|
||||
}
|
||||
|
||||
func readField(b []byte) (field, rest []byte, ok bool) {
|
||||
if len(b) < 4 {
|
||||
return nil, nil, false
|
||||
}
|
||||
n := binary.BigEndian.Uint32(b)
|
||||
if int64(n) > int64(len(b)-4) {
|
||||
return nil, nil, false
|
||||
}
|
||||
return b[4 : 4+n], b[4+n:], true
|
||||
}
|
||||
|
||||
func valueBytes(v AttributeValue) []byte {
|
||||
if v.StringValue != nil {
|
||||
return []byte(*v.StringValue)
|
||||
}
|
||||
return v.BinaryValue
|
||||
}
|
||||
|
||||
func copyValue(v AttributeValue) AttributeValue {
|
||||
out := AttributeValue{DataType: v.DataType}
|
||||
if v.StringValue != nil {
|
||||
s := *v.StringValue
|
||||
out.StringValue = &s
|
||||
}
|
||||
if v.BinaryValue != nil {
|
||||
out.BinaryValue = append([]byte{}, v.BinaryValue...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// validateAttributeName applies the documented name rules: 1 to 256 of
|
||||
// A-Z a-z 0-9 _ - ., no leading, trailing or repeated period, and no
|
||||
// AWS. or Amazon. prefix in any casing.
|
||||
func validateAttributeName(name string) error {
|
||||
if name == "" {
|
||||
return fmt.Errorf("%w: message attribute name must not be empty", ErrInvalidParameter)
|
||||
}
|
||||
if len(name) > MaxAttributeNameLen {
|
||||
return fmt.Errorf("%w: message attribute name is longer than %d characters", ErrInvalidParameter, MaxAttributeNameLen)
|
||||
}
|
||||
for i := 0; i < len(name); i++ {
|
||||
if !isNameByte(name[i]) {
|
||||
return fmt.Errorf("%w: message attribute name may contain only A-Z, a-z, 0-9, underscore, hyphen and period", ErrInvalidParameter)
|
||||
}
|
||||
}
|
||||
lower := strings.ToLower(name)
|
||||
switch {
|
||||
case strings.HasPrefix(lower, "aws.") || strings.HasPrefix(lower, "amazon."):
|
||||
return fmt.Errorf("%w: message attribute name %q uses a reserved prefix", ErrInvalidParameter, name)
|
||||
case name[0] == '.' || name[len(name)-1] == '.':
|
||||
return fmt.Errorf("%w: message attribute name %q starts or ends with a period", ErrInvalidParameter, name)
|
||||
case strings.Contains(name, ".."):
|
||||
return fmt.Errorf("%w: message attribute name %q has consecutive periods", ErrInvalidParameter, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateAttributeValue checks the data type, then that exactly the value
|
||||
// member its base type uses is present and valid.
|
||||
func validateAttributeValue(name string, v AttributeValue) (AttributeValue, error) {
|
||||
if v.StringListValues != nil || v.BinaryListValues != nil {
|
||||
return AttributeValue{}, fmt.Errorf("%w: message attribute %q list values", ErrUnsupported, name)
|
||||
}
|
||||
base, err := attributeBaseType(name, v.DataType)
|
||||
if err != nil {
|
||||
return AttributeValue{}, err
|
||||
}
|
||||
if base == "Binary" {
|
||||
switch {
|
||||
case v.StringValue != nil:
|
||||
return AttributeValue{}, fmt.Errorf("%w: message attribute %q of type Binary must not have a StringValue", ErrInvalidParameter, name)
|
||||
case v.BinaryValue == nil:
|
||||
return AttributeValue{}, fmt.Errorf("%w: message attribute %q is missing its BinaryValue", ErrInvalidParameter, name)
|
||||
case len(v.BinaryValue) == 0:
|
||||
return AttributeValue{}, fmt.Errorf("%w: message attribute %q has an empty BinaryValue", ErrInvalidParameter, name)
|
||||
}
|
||||
return copyValue(v), nil
|
||||
}
|
||||
switch {
|
||||
case v.BinaryValue != nil:
|
||||
return AttributeValue{}, fmt.Errorf("%w: message attribute %q of type %s must not have a BinaryValue", ErrInvalidParameter, name, base)
|
||||
case v.StringValue == nil:
|
||||
return AttributeValue{}, fmt.Errorf("%w: message attribute %q is missing its StringValue", ErrInvalidParameter, name)
|
||||
case *v.StringValue == "":
|
||||
return AttributeValue{}, fmt.Errorf("%w: message attribute %q has an empty StringValue", ErrInvalidParameter, name)
|
||||
}
|
||||
if err := validateContents([]byte(*v.StringValue)); err != nil {
|
||||
return AttributeValue{}, fmt.Errorf("%w: message attribute %q has a StringValue with a character SQS does not allow", ErrInvalidParameter, name)
|
||||
}
|
||||
if base == "Number" && !validNumber(*v.StringValue) {
|
||||
return AttributeValue{}, fmt.Errorf("%w: message attribute %q is not a number within 38 digits and 10^-128 to 10^126", ErrInvalidParameter, name)
|
||||
}
|
||||
return copyValue(v), nil
|
||||
}
|
||||
|
||||
// attributeBaseType returns String, Number or Binary from a data type with
|
||||
// an optional ".label" suffix; the label follows the body's character rules.
|
||||
func attributeBaseType(name, dataType string) (string, error) {
|
||||
if dataType == "" {
|
||||
return "", fmt.Errorf("%w: message attribute %q is missing its DataType", ErrInvalidParameter, name)
|
||||
}
|
||||
if utf8.RuneCountInString(dataType) > MaxAttributeTypeLen {
|
||||
return "", fmt.Errorf("%w: message attribute %q has a DataType longer than %d characters", ErrInvalidParameter, name, MaxAttributeTypeLen)
|
||||
}
|
||||
if err := validateContents([]byte(dataType)); err != nil {
|
||||
return "", fmt.Errorf("%w: message attribute %q has a DataType with a character SQS does not allow", ErrInvalidParameter, name)
|
||||
}
|
||||
base, label, custom := strings.Cut(dataType, ".")
|
||||
if base != "String" && base != "Number" && base != "Binary" {
|
||||
return "", fmt.Errorf("%w: message attribute %q has a DataType that is not String, Number or Binary", ErrInvalidParameter, name)
|
||||
}
|
||||
if custom && label == "" {
|
||||
return "", fmt.Errorf("%w: message attribute %q has an empty custom DataType label", ErrInvalidParameter, name)
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// validNumber accepts a signed decimal with an optional exponent of up to
|
||||
// 38 significant digits whose magnitude is zero or within 10^-128..10^126.
|
||||
func validNumber(s string) bool {
|
||||
if s != "" && (s[0] == '+' || s[0] == '-') {
|
||||
s = s[1:]
|
||||
}
|
||||
mant, exp := s, 0
|
||||
if i := strings.IndexAny(s, "eE"); i >= 0 {
|
||||
mant = s[:i]
|
||||
e, neg := s[i+1:], false
|
||||
if e != "" && (e[0] == '+' || e[0] == '-') {
|
||||
e, neg = e[1:], e[0] == '-'
|
||||
}
|
||||
// Past six digits any non-zero mantissa is out of range anyway.
|
||||
e = strings.TrimLeft(e, "0")
|
||||
if !allDigits(e) || s[i+1:] == "" || strings.HasSuffix(s, "+") || strings.HasSuffix(s, "-") || len(e) > 6 {
|
||||
return false
|
||||
}
|
||||
if e != "" {
|
||||
n, err := strconv.Atoi(e)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
exp = n
|
||||
}
|
||||
if neg {
|
||||
exp = -exp
|
||||
}
|
||||
}
|
||||
intPart, frac, _ := strings.Cut(mant, ".")
|
||||
if intPart == "" && frac == "" || !allDigits(intPart) || !allDigits(frac) {
|
||||
return false
|
||||
}
|
||||
digits := strings.TrimLeft(intPart+frac, "0")
|
||||
if digits == "" {
|
||||
return true
|
||||
}
|
||||
// The leading significant digit's power of ten.
|
||||
lead := exp + len(intPart) - 1 - (len(intPart+frac) - len(digits))
|
||||
digits = strings.TrimRight(digits, "0")
|
||||
if len(digits) > maxNumberDigits {
|
||||
return false
|
||||
}
|
||||
return lead >= minNumberExponent && (lead < maxNumberExponent || lead == maxNumberExponent && digits == "1")
|
||||
}
|
||||
|
||||
func isNameByte(c byte) bool {
|
||||
return c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '_' || c == '-' || c == '.'
|
||||
}
|
||||
|
||||
func allDigits(s string) bool {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] < '0' || s[i] > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// AttributeSelector is a parsed ReceiveMessage MessageAttributeNames list.
|
||||
// The zero value selects nothing.
|
||||
type AttributeSelector struct {
|
||||
all bool
|
||||
names map[string]struct{}
|
||||
prefixes []string
|
||||
}
|
||||
|
||||
// ParseAttributeSelector reads names, each All, .*, an attribute name, or
|
||||
// an attribute name followed by .* for every attribute under that dotted
|
||||
// prefix. A malformed entry is ErrInvalidParameter rather than ignored.
|
||||
func ParseAttributeSelector(names []string) (AttributeSelector, error) {
|
||||
var s AttributeSelector
|
||||
for _, n := range names {
|
||||
switch {
|
||||
case n == "All" || n == ".*":
|
||||
s.all = true
|
||||
case strings.HasSuffix(n, ".*"):
|
||||
p := strings.TrimSuffix(n, ".*")
|
||||
if err := validateAttributeName(p); err != nil {
|
||||
return AttributeSelector{}, err
|
||||
}
|
||||
s.prefixes = append(s.prefixes, p+".")
|
||||
default:
|
||||
if err := validateAttributeName(n); err != nil {
|
||||
return AttributeSelector{}, err
|
||||
}
|
||||
if s.names == nil {
|
||||
s.names = map[string]struct{}{}
|
||||
}
|
||||
s.names[n] = struct{}{}
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s AttributeSelector) selects(name string) bool {
|
||||
if s.all {
|
||||
return true
|
||||
}
|
||||
if _, ok := s.names[name]; ok {
|
||||
return true
|
||||
}
|
||||
for _, p := range s.prefixes {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Select is the attributes s names, still in name order.
|
||||
func (a Attributes) Select(s AttributeSelector) Attributes {
|
||||
var out []Attribute
|
||||
for _, at := range a.list {
|
||||
if s.selects(at.Name) {
|
||||
out = append(out, at)
|
||||
}
|
||||
}
|
||||
return Attributes{list: out}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package message_test
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func str(name, dataType, v string) message.Attribute {
|
||||
return message.Attribute{Name: name, Value: message.AttributeValue{DataType: dataType, StringValue: &v}}
|
||||
}
|
||||
|
||||
func bin(name, dataType string, v []byte) message.Attribute {
|
||||
return message.Attribute{Name: name, Value: message.AttributeValue{DataType: dataType, BinaryValue: v}}
|
||||
}
|
||||
|
||||
func mustAttrs(t *testing.T, in ...message.Attribute) message.Attributes {
|
||||
t.Helper()
|
||||
a, err := message.NewAttributes(in)
|
||||
require.NoError(t, err)
|
||||
return a
|
||||
}
|
||||
|
||||
// The expected digests come from a separate implementation of the
|
||||
// developer guide's algorithm, not from AWS; they are provisional.
|
||||
func TestAttributesMD5_DocumentedAlgorithmVectors(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
in []message.Attribute
|
||||
want string
|
||||
}{
|
||||
"single string": {[]message.Attribute{str("a", "String", "x")}, "a512aac9938034acb0b3c3d965944bf1"},
|
||||
"three types": {[]message.Attribute{
|
||||
str("Name", "String", "Alice"), str("Count", "Number", "42"), bin("Blob", "Binary", []byte{0, 1, 2, 255}),
|
||||
}, "655be3ff51c2255322ad67ede1a57544"},
|
||||
"custom suffix": {[]message.Attribute{str("n", "Number.int", "7")}, "f3c2a2801c523d26c8996134de718052"},
|
||||
"no suffix": {[]message.Attribute{str("n", "Number", "7")}, "3a734e6fc44468814b287b503a526bff"},
|
||||
"utf-8 value": {[]message.Attribute{str("greet", "String", "héllo 世界 😀")}, "d3d5cb26f19f5ac251fb595d2aad84eb"},
|
||||
"byte-wise order": {[]message.Attribute{str("b", "String", "1"), str("B", "String", "2"), str("a", "String", "3")}, "cd783096648921dba095ec2755c7ef34"},
|
||||
"guide example": {[]message.Attribute{
|
||||
str("Author", "String", "John Grisham"), str("Title", "String", "The Whistler"), str("WeeksOn", "Number", "6"),
|
||||
}, "d25a6aea97eb8f585bfa92d314504a92"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.want, mustAttrs(t, tc.in...).MD5())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttributesMD5_IndependentOfSuppliedOrder(t *testing.T) {
|
||||
a := mustAttrs(t, str("z", "String", "1"), str("m", "Number", "2"), bin("a", "Binary", []byte{9}))
|
||||
b := mustAttrs(t, bin("a", "Binary", []byte{9}), str("z", "String", "1"), str("m", "Number", "2"))
|
||||
assert.Equal(t, a.MD5(), b.MD5())
|
||||
names := []string{}
|
||||
for _, at := range a.All() {
|
||||
names = append(names, at.Name)
|
||||
}
|
||||
assert.Equal(t, []string{"a", "m", "z"}, names)
|
||||
}
|
||||
|
||||
func TestAttributes_EncodeRoundTrip(t *testing.T) {
|
||||
raw := []byte{0, 0xff, 0x80, 'x'}
|
||||
a := mustAttrs(t, str("s", "String.custom label", "héllo"), str("n", "Number.float", "-1.5e3"), bin("b", "Binary.gz", raw))
|
||||
enc := a.Encode()
|
||||
sum := md5.Sum(enc[1:])
|
||||
assert.Equal(t, a.MD5(), hex.EncodeToString(sum[:]), "the encoding carries the digested bytes")
|
||||
|
||||
got, err := message.DecodeAttributes(enc)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, a.All(), got.All())
|
||||
assert.Equal(t, a.MD5(), got.MD5())
|
||||
for _, at := range got.All() {
|
||||
if at.Name == "b" {
|
||||
assert.Nil(t, at.Value.StringValue, "binary stays binary")
|
||||
assert.Equal(t, raw, at.Value.BinaryValue)
|
||||
} else {
|
||||
assert.Nil(t, at.Value.BinaryValue, "string stays string")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAttributes_Names(t *testing.T) {
|
||||
for _, ok := range []string{"a", "A-b_c.d", strings.Repeat("n", 256), "awsx", "Amazonian", "x.AWS.y", "0"} {
|
||||
_, err := message.NewAttributes([]message.Attribute{str(ok, "String", "v")})
|
||||
assert.NoError(t, err, ok)
|
||||
}
|
||||
for _, bad := range []string{
|
||||
"", strings.Repeat("n", 257), "has space", "é", "a/b", "AWS.x", "aws.x", "aWs.x", "Amazon.x", "AMAZON.x",
|
||||
".a", "a.", "a..b", "a\x00",
|
||||
} {
|
||||
_, err := message.NewAttributes([]message.Attribute{str(bad, "String", "v")})
|
||||
assert.ErrorIs(t, err, message.ErrInvalidParameter, "%q", bad)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAttributes_DataTypes(t *testing.T) {
|
||||
for _, ok := range []string{"String", "Number", "Binary", "String.x", "Number.int", "Binary.png", "String.a.b", "String.label with space", "String." + strings.Repeat("é", 249)} {
|
||||
v := message.AttributeValue{DataType: ok}
|
||||
if strings.HasPrefix(ok, "Binary") {
|
||||
v.BinaryValue = []byte{1}
|
||||
} else {
|
||||
s := "1"
|
||||
v.StringValue = &s
|
||||
}
|
||||
_, err := message.NewAttributes([]message.Attribute{{Name: "a", Value: v}})
|
||||
assert.NoError(t, err, ok)
|
||||
}
|
||||
for _, bad := range []string{"", "string", "STRING", "Strings", "Text", "String.", ".String", "String" + "\x01", "String." + strings.Repeat("x", 250)} {
|
||||
_, err := message.NewAttributes([]message.Attribute{str("a", bad, "1")})
|
||||
assert.ErrorIs(t, err, message.ErrInvalidParameter, "%q", bad)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAttributes_ValuePresence(t *testing.T) {
|
||||
empty, one := "", "1"
|
||||
for name, v := range map[string]message.AttributeValue{
|
||||
"string missing": {DataType: "String"},
|
||||
"string empty": {DataType: "String", StringValue: &empty},
|
||||
"number missing": {DataType: "Number"},
|
||||
"binary missing": {DataType: "Binary"},
|
||||
"binary empty": {DataType: "Binary", BinaryValue: []byte{}},
|
||||
"string with binary": {DataType: "String", StringValue: &one, BinaryValue: []byte{1}},
|
||||
"binary with string": {DataType: "Binary", StringValue: &one, BinaryValue: []byte{1}},
|
||||
"binary given only a string": {DataType: "Binary", StringValue: &one},
|
||||
"string given only binary": {DataType: "String", BinaryValue: []byte{1}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := message.NewAttributes([]message.Attribute{{Name: "a", Value: v}})
|
||||
assert.ErrorIs(t, err, message.ErrInvalidParameter)
|
||||
})
|
||||
}
|
||||
// Missing and empty are told apart.
|
||||
_, missingErr := message.NewAttributes([]message.Attribute{{Name: "a", Value: message.AttributeValue{DataType: "String"}}})
|
||||
_, emptyErr := message.NewAttributes([]message.Attribute{{Name: "a", Value: message.AttributeValue{DataType: "String", StringValue: &empty}}})
|
||||
assert.Contains(t, missingErr.Error(), "missing")
|
||||
assert.Contains(t, emptyErr.Error(), "empty")
|
||||
}
|
||||
|
||||
func TestNewAttributes_StringValueCharacters(t *testing.T) {
|
||||
for _, ok := range []string{"tab\there", "line\nfeed", "\r", "�", "😀", ""} {
|
||||
_, err := message.NewAttributes([]message.Attribute{str("a", "String", ok)})
|
||||
assert.NoError(t, err, "%q", ok)
|
||||
}
|
||||
for _, bad := range []string{"\x00", "\x1f", "", "\xff", "a\xc3"} {
|
||||
_, err := message.NewAttributes([]message.Attribute{str("a", "String", bad)})
|
||||
require.ErrorIs(t, err, message.ErrInvalidParameter, "%q", bad)
|
||||
assert.NotContains(t, err.Error(), bad)
|
||||
}
|
||||
// A binary value carries any bytes.
|
||||
_, err := message.NewAttributes([]message.Attribute{bin("a", "Binary", []byte{0, 0xff, 0xfe})})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestNewAttributes_Numbers(t *testing.T) {
|
||||
for _, ok := range []string{
|
||||
"0", "-0", "+1", "42", "-3.25", ".5", "5.", "1e10", "1E-10", "1e+5", "00012", "0.000",
|
||||
strings.Repeat("9", 38), "1" + strings.Repeat("0", 50), "0." + strings.Repeat("0", 127) + "1",
|
||||
"1e126", "1.000e126", "-1e126", "1e-128", "9.99e125", "100e124", "0e999999",
|
||||
} {
|
||||
_, err := message.NewAttributes([]message.Attribute{str("n", "Number", ok)})
|
||||
assert.NoError(t, err, ok)
|
||||
}
|
||||
for _, bad := range []string{
|
||||
"", "-", "+", ".", "abc", "1a", "1.2.3", "1e", "1e+", "e5", "--1", "1 ", " 1", "0x10", "Infinity", "NaN", "1,000",
|
||||
strings.Repeat("9", 39), "1." + strings.Repeat("1", 38), "1.1e126", "2e126", "1e127", "11e125", "9.9e-129", "1e-129",
|
||||
"0." + strings.Repeat("0", 128) + "1", "1e9999999",
|
||||
} {
|
||||
_, err := message.NewAttributes([]message.Attribute{str("n", "Number", bad)})
|
||||
assert.ErrorIs(t, err, message.ErrInvalidParameter, "%q", bad)
|
||||
}
|
||||
// A custom suffix keeps Number validation.
|
||||
_, err := message.NewAttributes([]message.Attribute{str("n", "Number.int", "x")})
|
||||
assert.ErrorIs(t, err, message.ErrInvalidParameter)
|
||||
}
|
||||
|
||||
func TestNewAttributes_CountAndDuplicates(t *testing.T) {
|
||||
var ten []message.Attribute
|
||||
for i := range 10 {
|
||||
ten = append(ten, str(fmt.Sprintf("a%d", i), "String", "v"))
|
||||
}
|
||||
a, err := message.NewAttributes(ten)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 10, a.Len())
|
||||
|
||||
_, err = message.NewAttributes(append(ten, str("a10", "String", "v")))
|
||||
assert.ErrorIs(t, err, message.ErrInvalidParameter)
|
||||
|
||||
_, err = message.NewAttributes([]message.Attribute{str("dup", "String", "1"), bin("dup", "Binary", []byte{1})})
|
||||
assert.ErrorIs(t, err, message.ErrInvalidParameter)
|
||||
// Names are case-sensitive, so these are distinct.
|
||||
_, err = message.NewAttributes([]message.Attribute{str("Dup", "String", "1"), str("dup", "String", "1")})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestNewAttributes_ErrorsNeverCarryValues(t *testing.T) {
|
||||
secret := "s3cr3t-value"
|
||||
for _, in := range []message.Attribute{
|
||||
str("n", "Number", secret),
|
||||
str("s", "String", secret+"\x00"),
|
||||
{Name: "b", Value: message.AttributeValue{DataType: "Binary", StringValue: &secret, BinaryValue: []byte(secret)}},
|
||||
{Name: "l", Value: message.AttributeValue{DataType: "String", StringValue: &secret, StringListValues: []string{secret}}},
|
||||
} {
|
||||
_, err := message.NewAttributes([]message.Attribute{in})
|
||||
require.Error(t, err)
|
||||
assert.NotContains(t, err.Error(), secret)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttributes_SizeAndCombinedLimit(t *testing.T) {
|
||||
a := mustAttrs(t, str("ab", "String", "héllo"), bin("c", "Binary.gz", []byte{1, 2, 3}))
|
||||
// name + data type + value bytes: (2+6+6) + (1+9+3).
|
||||
assert.Equal(t, 27, a.Size())
|
||||
|
||||
const limit = message.MaxMessageSize
|
||||
assert.NoError(t, message.CheckMessageSize(limit-27, a, limit))
|
||||
assert.ErrorIs(t, message.CheckMessageSize(limit-26, a, limit), message.ErrInvalidParameter)
|
||||
assert.NoError(t, message.CheckMessageSize(limit, mustAttrs(t), limit))
|
||||
assert.ErrorIs(t, message.CheckMessageSize(1024, a, 1024), message.ErrInvalidParameter)
|
||||
|
||||
big := mustAttrs(t, bin("b", "Binary", make([]byte, limit-len("b")-len("Binary"))))
|
||||
assert.Equal(t, limit, big.Size())
|
||||
assert.NoError(t, message.CheckMessageSize(0, big, limit))
|
||||
assert.ErrorIs(t, message.CheckMessageSize(1, big, limit), message.ErrInvalidParameter)
|
||||
|
||||
_, err := message.NewAttributes([]message.Attribute{bin("b", "Binary", make([]byte, limit-len("b")-len("Binary")+1))})
|
||||
assert.ErrorIs(t, err, message.ErrInvalidParameter, "a set no queue could accept is refused outright")
|
||||
}
|
||||
|
||||
func selectedNames(a message.Attributes) []string {
|
||||
var out []string
|
||||
for _, at := range a.All() {
|
||||
out = append(out, at.Name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAttributeSelector_Selects(t *testing.T) {
|
||||
set := mustAttrs(t, str("app.id", "String", "1"), str("app.x.y", "String", "2"), str("apple", "String", "3"), str("b", "String", "4"))
|
||||
for name, tc := range map[string]struct {
|
||||
in []string
|
||||
want []string
|
||||
}{
|
||||
"nothing asked": {nil, nil},
|
||||
"All": {[]string{"All"}, []string{"app.id", "app.x.y", "apple", "b"}},
|
||||
"dot star": {[]string{".*"}, []string{"app.id", "app.x.y", "apple", "b"}},
|
||||
"prefix stops at dot": {[]string{"app.*"}, []string{"app.id", "app.x.y"}},
|
||||
"nested prefix": {[]string{"app.x.*"}, []string{"app.x.y"}},
|
||||
"exact": {[]string{"apple", "b"}, []string{"apple", "b"}},
|
||||
"exact is not prefix": {[]string{"app"}, nil},
|
||||
"case sensitive": {[]string{"B", "APP.*"}, nil},
|
||||
"repeated selector": {[]string{"b", "b"}, []string{"b"}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s, err := message.ParseAttributeSelector(tc.in)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.want, selectedNames(set.Select(s)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttributes_SelectDigestsSubset(t *testing.T) {
|
||||
set := mustAttrs(t, bin("a.bin", "Binary", []byte{1, 2}), str("b", "String", "x"))
|
||||
s, err := message.ParseAttributeSelector([]string{"a.*"})
|
||||
require.NoError(t, err)
|
||||
sub := set.Select(s)
|
||||
assert.Equal(t, mustAttrs(t, bin("a.bin", "Binary", []byte{1, 2})).MD5(), sub.MD5())
|
||||
assert.NotEqual(t, set.MD5(), sub.MD5())
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
package message
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// StateVersion is the schema version of the state records a claim writes.
|
||||
const StateVersion = 1
|
||||
|
||||
// MaxVisibilityTimeout is the longest visibility a receive may set, 12 hours.
|
||||
const MaxVisibilityTimeout = 12 * time.Hour
|
||||
|
||||
// MetaRc is the receive count a DLQ move carries in (Sqs-Rc); absent otherwise.
|
||||
const MetaRc = "Rc"
|
||||
|
||||
// State record values of s.
|
||||
const (
|
||||
StatusInflight = "inflight"
|
||||
StatusMoving = "moving"
|
||||
StatusDeleted = "deleted"
|
||||
StatusExpired = "expired"
|
||||
StatusMoved = "moved"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNotVisible reports a message that is delayed, in flight or being
|
||||
// moved: it exists but cannot be received now.
|
||||
ErrNotVisible = errors.New("message: not visible")
|
||||
// ErrGone reports a message that is deleted, expired, moved or past
|
||||
// retention, or whose body is no longer stored.
|
||||
ErrGone = errors.New("message: gone")
|
||||
// ErrClaimLost reports that another receiver's claim won the race.
|
||||
ErrClaimLost = errors.New("message: claim lost to another receiver")
|
||||
// ErrLeaseExpired reports a claim acknowledged only after its visibility
|
||||
// deadline: it counted as a receive, but the message is visible again.
|
||||
ErrLeaseExpired = errors.New("message: claim acknowledged after its deadline")
|
||||
// ErrNotOurs reports a record or body of a newer schema, or a state this
|
||||
// release does not know; it is never received, swept or expired here.
|
||||
ErrNotOurs = errors.New("message: record is not this release's to act on")
|
||||
)
|
||||
|
||||
// Claimer performs the receive claim (lifecycle contract M3) on one stored
|
||||
// message. It uses only the store interface.
|
||||
type Claimer struct {
|
||||
Store Store
|
||||
// Now defaults to the wall clock.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// ClaimInput names one stored message and the receive's parameters.
|
||||
type ClaimInput struct {
|
||||
Namespace Namespace
|
||||
Position Position
|
||||
// Visibility is the receive's visibility timeout, 0 to 12 hours.
|
||||
Visibility time.Duration
|
||||
// Retention is the queue's MessageRetentionPeriod, counted from the
|
||||
// original enqueue time.
|
||||
Retention time.Duration
|
||||
}
|
||||
|
||||
// Claim is a message taken in flight by an acknowledged claim. Generation is
|
||||
// the receipt generation the receipt handle must be issued for.
|
||||
type Claim struct {
|
||||
// Message.Body is the message body alone, without stored attributes.
|
||||
Message Message
|
||||
Attributes Attributes
|
||||
MessageID string
|
||||
ReceiveCount uint64
|
||||
Generation uint64
|
||||
Deadline time.Time
|
||||
ReceivedAt time.Time
|
||||
FirstReceive time.Time
|
||||
Revision Revision
|
||||
}
|
||||
|
||||
// Claim takes one message in flight. It answers only after the store
|
||||
// acknowledges the conditional write; a write whose outcome is unknown is
|
||||
// ErrUnavailable and is never resent, so it can count as a receive that
|
||||
// returned nothing, and the message returns at its deadline.
|
||||
func (c *Claimer) Claim(ctx context.Context, in ClaimInput) (Claim, error) {
|
||||
if c == nil || c.Store == nil {
|
||||
return Claim{}, fmt.Errorf("%w: claimer is not configured", ErrUnavailable)
|
||||
}
|
||||
if err := in.Namespace.Validate(); err != nil {
|
||||
return Claim{}, err
|
||||
}
|
||||
if in.Position == 0 {
|
||||
return Claim{}, fmt.Errorf("%w: position 0", ErrInvalidArgument)
|
||||
}
|
||||
if in.Visibility < 0 || in.Visibility > MaxVisibilityTimeout || in.Visibility%time.Second != 0 {
|
||||
return Claim{}, fmt.Errorf("%w: visibility timeout must be whole seconds from 0 to 43200", ErrInvalidParameter)
|
||||
}
|
||||
if in.Retention <= 0 {
|
||||
return Claim{}, fmt.Errorf("%w: queue retention %s is not positive", ErrUnavailable, in.Retention)
|
||||
}
|
||||
now := time.Now
|
||||
if c.Now != nil {
|
||||
now = c.Now
|
||||
}
|
||||
|
||||
msg, err := c.Store.Get(ctx, in.Namespace, in.Position)
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
return Claim{}, ErrGone
|
||||
}
|
||||
if err != nil {
|
||||
return Claim{}, fmt.Errorf("%w: read message: %w", ErrUnavailable, err)
|
||||
}
|
||||
h, err := readHeaders(msg.Meta)
|
||||
if err != nil {
|
||||
return Claim{}, err
|
||||
}
|
||||
// The stored payload is split before any write, so a corrupt one is
|
||||
// never taken in flight.
|
||||
var attrs Attributes
|
||||
if h.v > MetaVersionPlain {
|
||||
if msg.Body, attrs, err = decodePayload(msg.Body, h.md5a); err != nil {
|
||||
return Claim{}, fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
}
|
||||
cur, stateErr := c.Store.GetState(ctx, in.Namespace, in.Position)
|
||||
// The claim's time is taken after both reads, so a slow read cannot spend
|
||||
// the visibility the record is about to grant.
|
||||
t := now().UnixMilli()
|
||||
if t >= h.enq+in.Retention.Milliseconds() {
|
||||
return Claim{}, ErrGone
|
||||
}
|
||||
var rec stateRecord
|
||||
switch {
|
||||
case errors.Is(stateErr, ErrNotFound):
|
||||
if t < h.vis {
|
||||
return Claim{}, ErrNotVisible
|
||||
}
|
||||
rec = stateRecord{c: h.rc}
|
||||
cur = State{}
|
||||
case stateErr != nil:
|
||||
return Claim{}, fmt.Errorf("%w: read state: %w", ErrUnavailable, stateErr)
|
||||
default:
|
||||
if rec, err = decodeState(cur.Value); err != nil {
|
||||
return Claim{}, err
|
||||
}
|
||||
switch {
|
||||
case rec.s == StatusInflight && rec.d > t, rec.s == StatusMoving:
|
||||
return Claim{}, ErrNotVisible
|
||||
case rec.s != StatusInflight:
|
||||
return Claim{}, ErrGone
|
||||
}
|
||||
}
|
||||
if rec.c == math.MaxUint64 || rec.g == math.MaxUint64 {
|
||||
return Claim{}, fmt.Errorf("%w: receive count or generation exhausted", ErrUnavailable)
|
||||
}
|
||||
|
||||
rec.v = StateVersion
|
||||
rec.s = StatusInflight
|
||||
rec.c++
|
||||
rec.g++
|
||||
rec.d = t + in.Visibility.Milliseconds()
|
||||
rec.r = t
|
||||
if rec.f == 0 {
|
||||
rec.f = t
|
||||
}
|
||||
value, err := rec.encode()
|
||||
if err != nil {
|
||||
return Claim{}, fmt.Errorf("%w: encode state: %w", ErrUnavailable, err)
|
||||
}
|
||||
rev, err := c.Store.PutState(ctx, in.Namespace, in.Position, value, cur.Revision)
|
||||
if errors.Is(err, ErrRevisionMismatch) {
|
||||
return Claim{}, ErrClaimLost
|
||||
}
|
||||
if err != nil {
|
||||
return Claim{}, fmt.Errorf("%w: claim outcome unknown: %w", ErrUnavailable, err)
|
||||
}
|
||||
if err := c.recheckBody(ctx, in, rev); err != nil {
|
||||
return Claim{}, err
|
||||
}
|
||||
if in.Visibility > 0 && now().UnixMilli() >= rec.d {
|
||||
return Claim{}, ErrLeaseExpired
|
||||
}
|
||||
return Claim{
|
||||
Message: msg, Attributes: attrs, MessageID: h.id, ReceiveCount: rec.c, Generation: rec.g,
|
||||
Deadline: time.UnixMilli(rec.d).UTC(), ReceivedAt: time.UnixMilli(rec.r).UTC(), FirstReceive: time.UnixMilli(rec.f).UTC(),
|
||||
Revision: rev,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// recheckBody confirms the body still exists after every acknowledged claim:
|
||||
// purge and removal delete bodies without first moving the state record, so
|
||||
// a raced deletion is seen here and the claim just written is withdrawn.
|
||||
func (c *Claimer) recheckBody(ctx context.Context, in ClaimInput, rev Revision) error {
|
||||
_, err := c.Store.Get(ctx, in.Namespace, in.Position)
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case errors.Is(err, ErrNotFound):
|
||||
if derr := c.Store.DeleteState(ctx, in.Namespace, in.Position, rev); derr != nil {
|
||||
return fmt.Errorf("%w: withdraw claim on removed message: %w", ErrGone, derr)
|
||||
}
|
||||
return ErrGone
|
||||
default:
|
||||
return fmt.Errorf("%w: confirm message after claim: %w", ErrUnavailable, err)
|
||||
}
|
||||
}
|
||||
|
||||
var md5Hex = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
||||
|
||||
type headers struct {
|
||||
v int
|
||||
id string
|
||||
enq, vis int64
|
||||
rc uint64
|
||||
md5a string
|
||||
}
|
||||
|
||||
// readHeaders validates the body metadata a claim depends on. A newer schema
|
||||
// is not this release's; a malformed known field fails closed.
|
||||
func readHeaders(meta map[string]string) (headers, error) {
|
||||
v, err := strconv.Atoi(meta[MetaV])
|
||||
switch {
|
||||
case err != nil || v < 1:
|
||||
return headers{}, fmt.Errorf("%w: message schema version %q", ErrUnavailable, meta[MetaV])
|
||||
case v > MetaVersion:
|
||||
return headers{}, ErrNotOurs
|
||||
}
|
||||
h := headers{v: v, id: meta[MetaID]}
|
||||
if h.id == "" {
|
||||
return headers{}, fmt.Errorf("%w: message has no ID", ErrUnavailable)
|
||||
}
|
||||
if h.enq, err = strconv.ParseInt(meta[MetaEnq], 10, 64); err != nil || h.enq <= 0 {
|
||||
return headers{}, fmt.Errorf("%w: message enqueue time %q", ErrUnavailable, meta[MetaEnq])
|
||||
}
|
||||
if h.vis, err = strconv.ParseInt(meta[MetaVis], 10, 64); err != nil || h.vis < h.enq {
|
||||
return headers{}, fmt.Errorf("%w: message delay-until %q", ErrUnavailable, meta[MetaVis])
|
||||
}
|
||||
if meta[MetaSender] == "" {
|
||||
return headers{}, fmt.Errorf("%w: message has no sender", ErrUnavailable)
|
||||
}
|
||||
if !md5Hex.MatchString(meta[MetaMD5]) {
|
||||
return headers{}, fmt.Errorf("%w: message body MD5 %q", ErrUnavailable, meta[MetaMD5])
|
||||
}
|
||||
md5a, hasA := meta[MetaMD5A]
|
||||
switch {
|
||||
case v > MetaVersionPlain && !md5Hex.MatchString(md5a):
|
||||
return headers{}, fmt.Errorf("%w: message attributes MD5 %q", ErrUnavailable, md5a)
|
||||
case v == MetaVersionPlain && hasA:
|
||||
return headers{}, fmt.Errorf("%w: schema %d message carries an attributes MD5", ErrUnavailable, v)
|
||||
}
|
||||
h.md5a = md5a
|
||||
if raw, ok := meta[MetaRc]; ok {
|
||||
if h.rc, err = strconv.ParseUint(raw, 10, 64); err != nil {
|
||||
return headers{}, fmt.Errorf("%w: carried receive count %q", ErrUnavailable, raw)
|
||||
}
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// stateRecord is the lifecycle contract's state record. raw keeps every
|
||||
// field, so a rewrite preserves fields a newer release added.
|
||||
type stateRecord struct {
|
||||
v int
|
||||
s string
|
||||
c, g uint64
|
||||
d, r, f int64
|
||||
raw map[string]json.RawMessage
|
||||
}
|
||||
|
||||
var stateKnown = map[string]bool{"v": true, "s": true, "c": true, "g": true, "d": true, "r": true, "f": true}
|
||||
|
||||
// decodeState reads a stored state record. Undecodable or inconsistent
|
||||
// records fail closed; a newer version or unknown s is not ours.
|
||||
func decodeState(b []byte) (stateRecord, error) {
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(b, &raw); err != nil || raw == nil {
|
||||
return stateRecord{}, fmt.Errorf("%w: state record is not a JSON object", ErrUnavailable)
|
||||
}
|
||||
var known struct {
|
||||
V *int `json:"v"`
|
||||
S *string `json:"s"`
|
||||
C uint64 `json:"c"`
|
||||
G uint64 `json:"g"`
|
||||
D int64 `json:"d"`
|
||||
R int64 `json:"r"`
|
||||
F int64 `json:"f"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &known); err != nil {
|
||||
return stateRecord{}, fmt.Errorf("%w: state record field: %w", ErrUnavailable, err)
|
||||
}
|
||||
switch {
|
||||
case known.V == nil || *known.V < 1:
|
||||
return stateRecord{}, fmt.Errorf("%w: state record has no valid version", ErrUnavailable)
|
||||
case *known.V > StateVersion:
|
||||
return stateRecord{}, ErrNotOurs
|
||||
case known.S == nil:
|
||||
return stateRecord{}, fmt.Errorf("%w: state record has no state", ErrUnavailable)
|
||||
}
|
||||
rec := stateRecord{v: *known.V, s: *known.S, c: known.C, g: known.G, d: known.D, r: known.R, f: known.F, raw: raw}
|
||||
switch rec.s {
|
||||
case StatusInflight:
|
||||
if rec.c == 0 || rec.g == 0 || rec.d <= 0 || rec.r <= 0 || rec.f <= 0 || rec.f > rec.r {
|
||||
return stateRecord{}, fmt.Errorf("%w: in-flight state record is incomplete", ErrUnavailable)
|
||||
}
|
||||
case StatusMoving, StatusDeleted, StatusExpired, StatusMoved:
|
||||
default:
|
||||
return stateRecord{}, ErrNotOurs
|
||||
}
|
||||
return rec, nil
|
||||
}
|
||||
|
||||
// encode writes the known fields over the preserved raw record.
|
||||
func (r stateRecord) encode() ([]byte, error) {
|
||||
out := make(map[string]json.RawMessage, len(r.raw)+len(stateKnown))
|
||||
for k, v := range r.raw {
|
||||
if !stateKnown[k] {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
set := func(k string, v any) error {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out[k] = b
|
||||
return nil
|
||||
}
|
||||
for _, f := range []struct {
|
||||
k string
|
||||
v any
|
||||
}{{"v", r.v}, {"s", r.s}, {"c", r.c}, {"g", r.g}, {"d", r.d}, {"r", r.r}, {"f", r.f}} {
|
||||
if err := set(f.k, f.v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
enc := json.NewEncoder(&buf)
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return bytes.TrimRight(buf.Bytes(), "\n"), nil
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package message_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"maps"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var claimNow = time.UnixMilli(1_800_000_000_000)
|
||||
|
||||
// stateStore holds one message and its state record in memory, with
|
||||
// injectable read and write failures; other Store methods are unused.
|
||||
type stateStore struct {
|
||||
message.Store
|
||||
|
||||
msg *message.Message
|
||||
state []byte
|
||||
rev message.Revision
|
||||
getErr error
|
||||
stateErr error
|
||||
putErr error
|
||||
puts int
|
||||
// recheck, when set, answers every body read after the first.
|
||||
recheck error
|
||||
deleteErr error
|
||||
gets int
|
||||
deletes int
|
||||
}
|
||||
|
||||
func (s *stateStore) Get(_ context.Context, _ message.Namespace, pos message.Position) (message.Message, error) {
|
||||
s.gets++
|
||||
if s.gets > 1 && s.recheck != nil {
|
||||
return message.Message{}, s.recheck
|
||||
}
|
||||
if s.getErr != nil {
|
||||
return message.Message{}, s.getErr
|
||||
}
|
||||
if s.msg == nil || s.msg.Position != pos {
|
||||
return message.Message{}, message.ErrNotFound
|
||||
}
|
||||
return *s.msg, nil
|
||||
}
|
||||
|
||||
func (s *stateStore) GetState(context.Context, message.Namespace, message.Position) (message.State, error) {
|
||||
if s.stateErr != nil {
|
||||
return message.State{}, s.stateErr
|
||||
}
|
||||
if s.rev == 0 {
|
||||
return message.State{}, message.ErrNotFound
|
||||
}
|
||||
return message.State{Value: s.state, Revision: s.rev}, nil
|
||||
}
|
||||
|
||||
func (s *stateStore) PutState(_ context.Context, _ message.Namespace, _ message.Position, v []byte, expected message.Revision) (message.Revision, error) {
|
||||
s.puts++
|
||||
if s.putErr != nil {
|
||||
return 0, s.putErr
|
||||
}
|
||||
if expected != s.rev {
|
||||
return 0, message.ErrRevisionMismatch
|
||||
}
|
||||
s.state, s.rev = v, s.rev+1
|
||||
return s.rev, nil
|
||||
}
|
||||
|
||||
func (s *stateStore) DeleteState(_ context.Context, _ message.Namespace, _ message.Position, expected message.Revision) error {
|
||||
s.deletes++
|
||||
if s.deleteErr != nil {
|
||||
return s.deleteErr
|
||||
}
|
||||
if expected != s.rev {
|
||||
return message.ErrRevisionMismatch
|
||||
}
|
||||
s.state, s.rev = nil, 0
|
||||
return nil
|
||||
}
|
||||
|
||||
func ms(t time.Time) string { return strconv.FormatInt(t.UnixMilli(), 10) }
|
||||
|
||||
func storedMessage(enq, vis time.Time) *message.Message {
|
||||
return &message.Message{Position: 7, Body: []byte("hello"), Meta: map[string]string{
|
||||
message.MetaV: "1", message.MetaID: "id-1", message.MetaEnq: ms(enq), message.MetaVis: ms(vis),
|
||||
message.MetaSender: "AIDAEXAMPLE", message.MetaMD5: "5d41402abc4b2a76b9719d911017c592",
|
||||
}}
|
||||
}
|
||||
|
||||
func claimer(s *stateStore, now time.Time) *message.Claimer {
|
||||
return &message.Claimer{Store: s, Now: func() time.Time { return now }}
|
||||
}
|
||||
|
||||
func claimIn(vt time.Duration) message.ClaimInput {
|
||||
return message.ClaimInput{Namespace: sendNS, Position: 7, Visibility: vt, Retention: 4 * 24 * time.Hour}
|
||||
}
|
||||
|
||||
func decoded(t *testing.T, b []byte) map[string]any {
|
||||
t.Helper()
|
||||
var m map[string]any
|
||||
require.NoError(t, json.Unmarshal(b, &m))
|
||||
return m
|
||||
}
|
||||
|
||||
func TestClaimFirstReceive(t *testing.T) {
|
||||
s := &stateStore{msg: storedMessage(claimNow.Add(-time.Minute), claimNow.Add(-time.Minute))}
|
||||
got, err := claimer(s, claimNow).Claim(context.Background(), claimIn(30*time.Second))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "id-1", got.MessageID)
|
||||
assert.Equal(t, []byte("hello"), got.Message.Body)
|
||||
assert.Equal(t, uint64(1), got.ReceiveCount)
|
||||
assert.Equal(t, uint64(1), got.Generation)
|
||||
assert.Equal(t, claimNow.Add(30*time.Second).UnixMilli(), got.Deadline.UnixMilli())
|
||||
assert.Equal(t, claimNow.UnixMilli(), got.ReceivedAt.UnixMilli())
|
||||
assert.Equal(t, claimNow.UnixMilli(), got.FirstReceive.UnixMilli())
|
||||
assert.Equal(t, message.Revision(1), got.Revision)
|
||||
n := float64(claimNow.UnixMilli())
|
||||
assert.Equal(t, map[string]any{"v": 1.0, "s": "inflight", "c": 1.0, "g": 1.0, "d": n + 30_000, "r": n, "f": n}, decoded(t, s.state))
|
||||
}
|
||||
|
||||
func TestClaimDelayAndRetention(t *testing.T) {
|
||||
s := &stateStore{msg: storedMessage(claimNow.Add(-time.Minute), claimNow.Add(time.Millisecond))}
|
||||
_, err := claimer(s, claimNow).Claim(context.Background(), claimIn(time.Second))
|
||||
require.ErrorIs(t, err, message.ErrNotVisible)
|
||||
_, err = claimer(s, claimNow.Add(time.Millisecond)).Claim(context.Background(), claimIn(time.Second))
|
||||
require.NoError(t, err, "visible exactly at delay-until")
|
||||
|
||||
old := &stateStore{msg: storedMessage(claimNow.Add(-4*24*time.Hour), claimNow.Add(-4*24*time.Hour))}
|
||||
_, err = claimer(old, claimNow).Claim(context.Background(), claimIn(time.Second))
|
||||
require.ErrorIs(t, err, message.ErrGone, "expired exactly at enqueue plus retention")
|
||||
assert.Zero(t, old.puts)
|
||||
_, err = claimer(old, claimNow.Add(-time.Millisecond)).Claim(context.Background(), claimIn(time.Second))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func inflight(c, g uint64, d, r, f time.Time) []byte {
|
||||
b, _ := json.Marshal(map[string]any{"v": 1, "s": "inflight", "c": c, "g": g, "d": d.UnixMilli(), "r": r.UnixMilli(), "f": f.UnixMilli()})
|
||||
return b
|
||||
}
|
||||
|
||||
func TestClaimInflightAndExpiry(t *testing.T) {
|
||||
first := claimNow.Add(-time.Hour)
|
||||
s := &stateStore{msg: storedMessage(first, first), state: inflight(2, 5, claimNow.Add(time.Millisecond), first, first), rev: 9}
|
||||
_, err := claimer(s, claimNow).Claim(context.Background(), claimIn(time.Second))
|
||||
require.ErrorIs(t, err, message.ErrNotVisible)
|
||||
assert.Zero(t, s.puts)
|
||||
|
||||
got, err := claimer(s, claimNow.Add(time.Millisecond)).Claim(context.Background(), claimIn(time.Second))
|
||||
require.NoError(t, err, "visible again exactly at the deadline")
|
||||
assert.Equal(t, uint64(3), got.ReceiveCount)
|
||||
assert.Equal(t, uint64(6), got.Generation)
|
||||
assert.Equal(t, first.UnixMilli(), got.FirstReceive.UnixMilli(), "first receive time is kept")
|
||||
assert.Equal(t, claimNow.Add(time.Millisecond).UnixMilli(), got.ReceivedAt.UnixMilli())
|
||||
assert.Equal(t, message.Revision(10), got.Revision)
|
||||
}
|
||||
|
||||
func TestClaimRefusesWhatIsNotOurs(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
state string
|
||||
meta map[string]string
|
||||
wantErr error
|
||||
}{
|
||||
"unknown state": {state: `{"v":1,"s":"archived"}`, wantErr: message.ErrNotOurs},
|
||||
"newer state version": {state: `{"v":2,"s":"inflight","c":1,"g":1,"d":1,"r":1,"f":1}`, wantErr: message.ErrNotOurs},
|
||||
"newer body version": {meta: map[string]string{message.MetaV: strconv.Itoa(message.MetaVersion + 1)}, wantErr: message.ErrNotOurs},
|
||||
"attributes without MD5": {meta: map[string]string{message.MetaV: "2"}, wantErr: message.ErrUnavailable},
|
||||
"plain with attrs MD5": {meta: map[string]string{message.MetaMD5A: strings.Repeat("0", 32)}, wantErr: message.ErrUnavailable},
|
||||
"not an object": {state: `[]`, wantErr: message.ErrUnavailable},
|
||||
"null": {state: `null`, wantErr: message.ErrUnavailable},
|
||||
"no version": {state: `{"s":"deleted"}`, wantErr: message.ErrUnavailable},
|
||||
"zero version": {state: `{"v":0,"s":"deleted"}`, wantErr: message.ErrUnavailable},
|
||||
"no state": {state: `{"v":1}`, wantErr: message.ErrUnavailable},
|
||||
"wrong field type": {state: `{"v":1,"s":"inflight","c":"1"}`, wantErr: message.ErrUnavailable},
|
||||
"inflight without count": {state: `{"v":1,"s":"inflight","g":1,"d":1,"r":1,"f":1}`, wantErr: message.ErrUnavailable},
|
||||
"inflight without gen": {state: `{"v":1,"s":"inflight","c":1,"d":1,"r":1,"f":1}`, wantErr: message.ErrUnavailable},
|
||||
"inflight without dead": {state: `{"v":1,"s":"inflight","c":1,"g":1,"r":1,"f":1}`, wantErr: message.ErrUnavailable},
|
||||
"inflight without recv": {state: `{"v":1,"s":"inflight","c":1,"g":1,"d":1,"f":1}`, wantErr: message.ErrUnavailable},
|
||||
"inflight without first": {state: `{"v":1,"s":"inflight","c":1,"g":1,"d":1,"r":1}`, wantErr: message.ErrUnavailable},
|
||||
"first after receive": {state: `{"v":1,"s":"inflight","c":1,"g":1,"d":1,"r":1,"f":2}`, wantErr: message.ErrUnavailable},
|
||||
"exhausted generation": {state: `{"v":1,"s":"inflight","c":1,"g":18446744073709551615,"d":1,"r":1,"f":1}`, wantErr: message.ErrUnavailable},
|
||||
"body without version": {meta: map[string]string{message.MetaV: ""}, wantErr: message.ErrUnavailable},
|
||||
"body without id": {meta: map[string]string{message.MetaID: ""}, wantErr: message.ErrUnavailable},
|
||||
"body without enqueue": {meta: map[string]string{message.MetaEnq: "x"}, wantErr: message.ErrUnavailable},
|
||||
"delay before enqueue": {meta: map[string]string{message.MetaVis: "1"}, wantErr: message.ErrUnavailable},
|
||||
"malformed carried count": {meta: map[string]string{message.MetaRc: "-1"}, wantErr: message.ErrUnavailable},
|
||||
"body without sender": {meta: map[string]string{message.MetaSender: ""}, wantErr: message.ErrUnavailable},
|
||||
"body without md5": {meta: map[string]string{message.MetaMD5: ""}, wantErr: message.ErrUnavailable},
|
||||
"body with short md5": {meta: map[string]string{message.MetaMD5: "5d41402abc4b2a76"}, wantErr: message.ErrUnavailable},
|
||||
"body with upper md5": {meta: map[string]string{message.MetaMD5: "5D41402ABC4B2A76B9719D911017C592"}, wantErr: message.ErrUnavailable},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s := &stateStore{msg: storedMessage(claimNow.Add(-time.Minute), claimNow.Add(-time.Minute))}
|
||||
maps.Copy(s.msg.Meta, tc.meta)
|
||||
if tc.state != "" {
|
||||
s.state, s.rev = []byte(tc.state), 3
|
||||
}
|
||||
_, err := claimer(s, claimNow).Claim(context.Background(), claimIn(time.Second))
|
||||
require.ErrorIs(t, err, tc.wantErr)
|
||||
assert.Zero(t, s.puts, "nothing is written")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimPreservesUnknownFields(t *testing.T) {
|
||||
first := claimNow.Add(-time.Hour)
|
||||
f := ms(first)
|
||||
b := []byte(`{"v":1,"s":"inflight","c":1,"g":1,"d":` + f + `,"r":` + f + `,"f":` + f + `,"grp":{"future":[1,"x"]},"zz":"<kept>"}`)
|
||||
s := &stateStore{msg: storedMessage(first, first), state: b, rev: 2}
|
||||
|
||||
_, err := claimer(s, claimNow).Claim(context.Background(), claimIn(time.Second))
|
||||
require.NoError(t, err)
|
||||
var got map[string]json.RawMessage
|
||||
require.NoError(t, json.Unmarshal(s.state, &got))
|
||||
assert.JSONEq(t, `{"future":[1,"x"]}`, string(got["grp"]))
|
||||
assert.Equal(t, `"<kept>"`, string(got["zz"]), "unknown fields are kept byte for byte")
|
||||
assert.Equal(t, `2`, string(got["g"]))
|
||||
}
|
||||
|
||||
func TestClaimWriteOutcomes(t *testing.T) {
|
||||
enq := claimNow.Add(-time.Minute)
|
||||
lost := &stateStore{msg: storedMessage(enq, enq), putErr: message.ErrRevisionMismatch}
|
||||
_, err := claimer(lost, claimNow).Claim(context.Background(), claimIn(time.Second))
|
||||
require.ErrorIs(t, err, message.ErrClaimLost)
|
||||
|
||||
unknown := &stateStore{msg: storedMessage(enq, enq), putErr: errors.New("nats: timeout")}
|
||||
_, err = claimer(unknown, claimNow).Claim(context.Background(), claimIn(time.Second))
|
||||
require.ErrorIs(t, err, message.ErrUnavailable)
|
||||
assert.Equal(t, 1, unknown.puts, "an unknown outcome is never resent")
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package message
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// deleteAttempts bounds retries after a definite revision conflict.
|
||||
const deleteAttempts = 5
|
||||
|
||||
// DeleteInput is one DeleteMessage request after wire decoding.
|
||||
type DeleteInput struct {
|
||||
Account, Region, Queue string
|
||||
ReceiptHandle string
|
||||
}
|
||||
|
||||
// Deleter is the DeleteMessage owner core. It writes the deleted tombstone
|
||||
// only; removing the body and then the state record is the sweeper's.
|
||||
type Deleter struct {
|
||||
Store Store
|
||||
Queues ReceiveQueues
|
||||
Keys *ReceiptKeys
|
||||
}
|
||||
|
||||
// Delete tombstones the message a current handle names. An older handle, or
|
||||
// a message already tombstoned or gone, succeeds without writing; a handle
|
||||
// that does not authenticate for this queue is ErrReceiptInvalid.
|
||||
func (d *Deleter) Delete(ctx context.Context, in DeleteInput) error {
|
||||
if d == nil || d.Store == nil || d.Queues == nil {
|
||||
return fmt.Errorf("%w: deleter is not configured", ErrUnavailable)
|
||||
}
|
||||
if err := d.Keys.Check(); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
t, err := d.Queues.ReceiveTarget(ctx, in.Account, in.Region, in.Queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if t.Account != in.Account {
|
||||
return fmt.Errorf("%w: resolved queue belongs to another account", ErrUnavailable)
|
||||
}
|
||||
if err := t.Namespace.Validate(); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
rc, err := d.Keys.Decode(in.ReceiptHandle, t.Namespace.Queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for range deleteAttempts {
|
||||
cur, rec, done, err := d.current(ctx, t.Namespace, rc)
|
||||
if done || err != nil {
|
||||
return err
|
||||
}
|
||||
rec.s = StatusDeleted
|
||||
value, err := rec.encode()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: encode tombstone: %w", ErrUnavailable, err)
|
||||
}
|
||||
_, err = d.Store.PutState(ctx, t.Namespace, rc.Position, value, cur.Revision)
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case errors.Is(err, ErrRevisionMismatch):
|
||||
continue
|
||||
}
|
||||
return d.settle(ctx, t.Namespace, rc, err)
|
||||
}
|
||||
return fmt.Errorf("%w: delete kept conflicting", ErrUnavailable)
|
||||
}
|
||||
|
||||
// current reads the state a handle names. done reports an outcome that needs
|
||||
// no write: no record, a tombstone, or a newer receive.
|
||||
func (d *Deleter) current(ctx context.Context, ns Namespace, rc Receipt) (State, stateRecord, bool, error) {
|
||||
cur, err := d.Store.GetState(ctx, ns, rc.Position)
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
return State{}, stateRecord{}, true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return State{}, stateRecord{}, true, fmt.Errorf("%w: read state: %w", ErrUnavailable, err)
|
||||
}
|
||||
rec, err := decodeState(cur.Value)
|
||||
if err != nil {
|
||||
return State{}, stateRecord{}, true, err
|
||||
}
|
||||
switch {
|
||||
case rec.s == StatusDeleted || rec.s == StatusExpired || rec.s == StatusMoved:
|
||||
return State{}, stateRecord{}, true, nil
|
||||
case rec.g > rc.Generation:
|
||||
return State{}, stateRecord{}, true, nil
|
||||
case rec.g < rc.Generation:
|
||||
return State{}, stateRecord{}, true, fmt.Errorf("%w: state is older than the handle", ErrUnavailable)
|
||||
case rec.s == StatusMoving:
|
||||
return State{}, stateRecord{}, true, fmt.Errorf("%w: message is being moved", ErrUnavailable)
|
||||
}
|
||||
return cur, rec, false, nil
|
||||
}
|
||||
|
||||
// settle decides a tombstone write whose reply was lost. A tombstone or a
|
||||
// newer receive now in place is success either way; a record still at this
|
||||
// handle's generation may yet take the write, so the outcome is unknown.
|
||||
func (d *Deleter) settle(ctx context.Context, ns Namespace, rc Receipt, cause error) error {
|
||||
_, _, done, err := d.current(ctx, ns, rc)
|
||||
if err == nil && done {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%w: delete outcome unknown: %w", ErrUnavailable, cause)
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package jetstream_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func strAttr(name, dataType, v string) message.Attribute {
|
||||
return message.Attribute{Name: name, Value: message.AttributeValue{DataType: dataType, StringValue: &v}}
|
||||
}
|
||||
|
||||
func binAttr(name, dataType string, v []byte) message.Attribute {
|
||||
return message.Attribute{Name: name, Value: message.AttributeValue{DataType: dataType, BinaryValue: v}}
|
||||
}
|
||||
|
||||
func md5Hex(b []byte) string {
|
||||
sum := md5.Sum(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func TestSendWithAttributesStoresVersionedRecordAndReceivesThem(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
in := input("orders", "hello")
|
||||
in.MessageAttributes = []message.Attribute{strAttr("Name", "String", "Alice"), binAttr("Blob", "Binary.gz", []byte{0, 1, 2, 255})}
|
||||
res, err := e.sender(nil).Send(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
// From a separate implementation of the documented algorithm, not AWS.
|
||||
assert.Equal(t, "7ba25f6d674965cfffd95c5a417584dc", res.MD5OfMessageAttributes)
|
||||
assert.Equal(t, md5Hex([]byte("hello")), res.MD5OfBody)
|
||||
|
||||
raw, err := e.store.Get(context.Background(), q.Namespace(), res.Position)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, strconv.Itoa(message.MetaVersion), raw.Meta[message.MetaV], "a reader without attribute support refuses this record")
|
||||
assert.Equal(t, res.MD5OfMessageAttributes, raw.Meta[message.MetaMD5A])
|
||||
assert.NotEqual(t, []byte("hello"), raw.Body, "the stored payload is not a plain body")
|
||||
assert.True(t, bytes.HasSuffix(raw.Body, []byte("hello")))
|
||||
|
||||
got, err := e.receiver(nil).Receive(context.Background(), recv("orders", 1))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, []byte("hello"), got[0].Body)
|
||||
assert.Equal(t, res.MD5OfBody, got[0].MD5OfBody)
|
||||
assert.Equal(t, res.MD5OfMessageAttributes, got[0].MD5OfMessageAttributes)
|
||||
want, err := message.NewAttributes(in.MessageAttributes)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, want.All(), got[0].Attributes.All())
|
||||
}
|
||||
|
||||
func TestSendWithoutAttributesKeepsPlainRecord(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
res := e.send(t, "orders", "plain", nil)
|
||||
assert.Empty(t, res.MD5OfMessageAttributes)
|
||||
|
||||
raw, err := e.store.Get(context.Background(), q.Namespace(), res.Position)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, strconv.Itoa(message.MetaVersionPlain), raw.Meta[message.MetaV])
|
||||
assert.NotContains(t, raw.Meta, message.MetaMD5A)
|
||||
assert.Equal(t, []byte("plain"), raw.Body)
|
||||
}
|
||||
|
||||
// A record as an earlier release wrote it, appended directly, still reads.
|
||||
func TestReceiveReadsEarlierPlainRecords(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
enq := strconv.FormatInt(fixedNow.UnixMilli(), 10)
|
||||
_, err := e.store.Append(context.Background(), q.Namespace(), message.AppendInput{Body: []byte("from before"), Meta: map[string]string{
|
||||
message.MetaV: "1", message.MetaID: "11111111-1111-4111-8111-111111111111", message.MetaEnq: enq, message.MetaVis: enq,
|
||||
message.MetaSender: sender, message.MetaMD5: md5Hex([]byte("from before")), message.MetaAcct: acct,
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := e.receiver(nil).Receive(context.Background(), recv("orders", 1))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, []byte("from before"), got[0].Body)
|
||||
assert.Zero(t, got[0].Attributes.Len())
|
||||
assert.Empty(t, got[0].MD5OfMessageAttributes)
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
package jetstream_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var claimNS = message.Namespace{MessageShard: 1, StateShard: 1, Queue: "claimq"}
|
||||
|
||||
const claimRetention = 4 * 24 * time.Hour
|
||||
|
||||
// appendAt stores a message enqueued at enq and visible from vis.
|
||||
func appendAt(t *testing.T, s message.Store, enq, vis time.Time) message.Position {
|
||||
t.Helper()
|
||||
res, err := s.Append(context.Background(), claimNS, message.AppendInput{Body: []byte("hello"), Meta: map[string]string{
|
||||
message.MetaV: "1", message.MetaID: "id-" + strconv.FormatInt(enq.UnixNano(), 10),
|
||||
message.MetaEnq: strconv.FormatInt(enq.UnixMilli(), 10), message.MetaVis: strconv.FormatInt(vis.UnixMilli(), 10),
|
||||
message.MetaSender: sender, message.MetaMD5: "5d41402abc4b2a76b9719d911017c592",
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
return res.Position
|
||||
}
|
||||
|
||||
// claimStore is a single-node dedicated store, as the first profile runs.
|
||||
func claimStore(t *testing.T) message.Store {
|
||||
t.Helper()
|
||||
clustersize.DeclareForTest(t, 1)
|
||||
return openStore(t, storeMaxPayload)
|
||||
}
|
||||
|
||||
func claimAt(s message.Store, now time.Time) *message.Claimer {
|
||||
return &message.Claimer{Store: s, Now: func() time.Time { return now }}
|
||||
}
|
||||
|
||||
func claimPos(pos message.Position, vt time.Duration) message.ClaimInput {
|
||||
return message.ClaimInput{Namespace: claimNS, Position: pos, Visibility: vt, Retention: claimRetention}
|
||||
}
|
||||
|
||||
func TestClaimLifecycleOnJetStream(t *testing.T) {
|
||||
s := claimStore(t)
|
||||
ctx := context.Background()
|
||||
pos := appendAt(t, s, fixedNow, fixedNow.Add(10*time.Second))
|
||||
|
||||
_, err := claimAt(s, fixedNow).Claim(ctx, claimPos(pos, 30*time.Second))
|
||||
require.ErrorIs(t, err, message.ErrNotVisible, "delayed")
|
||||
_, err = s.GetState(ctx, claimNS, pos)
|
||||
require.ErrorIs(t, err, message.ErrNotFound, "a delayed message has no state record")
|
||||
|
||||
t1 := fixedNow.Add(10 * time.Second)
|
||||
first, err := claimAt(s, t1).Claim(ctx, claimPos(pos, 30*time.Second))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(1), first.Generation)
|
||||
assert.Equal(t, []byte("hello"), first.Message.Body)
|
||||
|
||||
_, err = claimAt(s, t1.Add(29*time.Second)).Claim(ctx, claimPos(pos, 30*time.Second))
|
||||
require.ErrorIs(t, err, message.ErrNotVisible, "in flight")
|
||||
|
||||
t2 := t1.Add(30 * time.Second)
|
||||
second, err := claimAt(s, t2).Claim(ctx, claimPos(pos, 5*time.Second))
|
||||
require.NoError(t, err, "visible again at the deadline")
|
||||
assert.Equal(t, uint64(2), second.ReceiveCount)
|
||||
assert.Equal(t, uint64(2), second.Generation)
|
||||
assert.Equal(t, t1.UnixMilli(), second.FirstReceive.UnixMilli())
|
||||
assert.Equal(t, t2.Add(5*time.Second).UnixMilli(), second.Deadline.UnixMilli())
|
||||
|
||||
st, err := s.GetState(ctx, claimNS, pos)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, second.Revision, st.Revision, "the claim reports the acknowledged revision")
|
||||
}
|
||||
|
||||
// Competing receivers on one message: exactly one claim is acknowledged.
|
||||
func TestClaimCompetingReceiversOnJetStream(t *testing.T) {
|
||||
s := claimStore(t)
|
||||
ctx := context.Background()
|
||||
for _, prior := range []bool{false, true} {
|
||||
pos := appendAt(t, s, fixedNow, fixedNow)
|
||||
now := fixedNow.Add(time.Minute)
|
||||
if prior {
|
||||
_, err := claimAt(s, fixedNow.Add(time.Second)).Claim(ctx, claimPos(pos, time.Second))
|
||||
require.NoError(t, err, "an expired earlier claim makes the race a revision race")
|
||||
}
|
||||
const receivers = 16
|
||||
var wg sync.WaitGroup
|
||||
results := make([]error, receivers)
|
||||
claims := make([]message.Claim, receivers)
|
||||
for i := range receivers {
|
||||
wg.Go(func() {
|
||||
claims[i], results[i] = claimAt(s, now).Claim(ctx, claimPos(pos, time.Minute))
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
won := 0
|
||||
for i, err := range results {
|
||||
switch {
|
||||
case err == nil:
|
||||
won++
|
||||
assert.Equal(t, uint64(1+boolInt(prior)), claims[i].Generation)
|
||||
case errors.Is(err, message.ErrClaimLost), errors.Is(err, message.ErrNotVisible):
|
||||
default:
|
||||
t.Fatalf("receiver %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 1, won, "prior claim %v", prior)
|
||||
}
|
||||
}
|
||||
|
||||
func boolInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// lostClaimReply applies the claim, then reports a lost acknowledgement.
|
||||
type lostClaimReply struct {
|
||||
message.Store
|
||||
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
}
|
||||
|
||||
func (l *lostClaimReply) PutState(ctx context.Context, ns message.Namespace, pos message.Position, v []byte, exp message.Revision) (message.Revision, error) {
|
||||
l.mu.Lock()
|
||||
l.calls++
|
||||
l.mu.Unlock()
|
||||
if _, err := l.Store.PutState(ctx, ns, pos, v, exp); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return 0, errors.New("nats: timeout")
|
||||
}
|
||||
|
||||
func TestClaimAppliedButLostReplyOnJetStream(t *testing.T) {
|
||||
s := claimStore(t)
|
||||
ctx := context.Background()
|
||||
pos := appendAt(t, s, fixedNow, fixedNow)
|
||||
lossy := &lostClaimReply{Store: s}
|
||||
|
||||
_, err := claimAt(lossy, fixedNow).Claim(ctx, claimPos(pos, 30*time.Second))
|
||||
require.ErrorIs(t, err, message.ErrUnavailable)
|
||||
assert.Equal(t, 1, lossy.calls, "the claim is not resent")
|
||||
|
||||
st, err := s.GetState(ctx, claimNS, pos)
|
||||
require.NoError(t, err)
|
||||
var rec map[string]any
|
||||
require.NoError(t, json.Unmarshal(st.Value, &rec))
|
||||
assert.Equal(t, 1.0, rec["c"], "the unanswered claim still counts as a receive")
|
||||
|
||||
_, err = claimAt(s, fixedNow.Add(29*time.Second)).Claim(ctx, claimPos(pos, 30*time.Second))
|
||||
require.ErrorIs(t, err, message.ErrNotVisible, "no other receiver gets it before its deadline")
|
||||
again, err := claimAt(s, fixedNow.Add(30*time.Second)).Claim(ctx, claimPos(pos, 30*time.Second))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(2), again.ReceiveCount)
|
||||
assert.Equal(t, uint64(2), again.Generation, "the lost generation is never reissued")
|
||||
}
|
||||
|
||||
// hookedStore runs a hook after the n-th body read, and before each state
|
||||
// read and after each state write, so a test can interleave other writers or
|
||||
// move the clock at exact points of one claim.
|
||||
type hookedStore struct {
|
||||
message.Store
|
||||
|
||||
afterGet func(n int)
|
||||
beforeState func()
|
||||
beforePut func()
|
||||
afterPut func()
|
||||
gets, states int
|
||||
}
|
||||
|
||||
func (h *hookedStore) Get(ctx context.Context, ns message.Namespace, pos message.Position) (message.Message, error) {
|
||||
m, err := h.Store.Get(ctx, ns, pos)
|
||||
h.gets++
|
||||
if h.afterGet != nil {
|
||||
h.afterGet(h.gets)
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
func (h *hookedStore) GetState(ctx context.Context, ns message.Namespace, pos message.Position) (message.State, error) {
|
||||
h.states++
|
||||
if h.beforeState != nil {
|
||||
h.beforeState()
|
||||
}
|
||||
return h.Store.GetState(ctx, ns, pos)
|
||||
}
|
||||
|
||||
func (h *hookedStore) PutState(ctx context.Context, ns message.Namespace, pos message.Position, v []byte, rev message.Revision) (message.Revision, error) {
|
||||
if h.beforePut != nil {
|
||||
h.beforePut()
|
||||
}
|
||||
r, err := h.Store.PutState(ctx, ns, pos, v, rev)
|
||||
if h.afterPut != nil {
|
||||
h.afterPut()
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
// A deletion that tombstones the record, deletes the body and removes the
|
||||
// tombstone between the claim's body read and its state read leaves the
|
||||
// record absent again; the claim must not resurrect the deleted message.
|
||||
func TestClaimDoesNotResurrectRemovedMessage(t *testing.T) {
|
||||
s := claimStore(t)
|
||||
ctx := context.Background()
|
||||
pos := appendAt(t, s, fixedNow, fixedNow)
|
||||
held, err := claimAt(s, fixedNow).Claim(ctx, claimPos(pos, time.Minute))
|
||||
require.NoError(t, err)
|
||||
|
||||
removed := false
|
||||
h := &hookedStore{Store: s, afterGet: func(n int) {
|
||||
if n != 1 {
|
||||
return
|
||||
}
|
||||
tomb, err := json.Marshal(map[string]any{"v": 1, "s": message.StatusDeleted})
|
||||
require.NoError(t, err)
|
||||
tombRev, err := s.PutState(ctx, claimNS, pos, tomb, held.Revision)
|
||||
require.NoError(t, err, "DeleteMessage tombstones the record")
|
||||
require.NoError(t, s.Delete(ctx, claimNS, pos), "the sweeper deletes the body")
|
||||
require.NoError(t, s.DeleteState(ctx, claimNS, pos, tombRev), "then the tombstone")
|
||||
removed = true
|
||||
}}
|
||||
|
||||
later := fixedNow.Add(2 * time.Minute)
|
||||
got, err := claimAt(h, later).Claim(ctx, claimPos(pos, time.Minute))
|
||||
require.True(t, removed, "the deletion ran inside the claim")
|
||||
require.ErrorIs(t, err, message.ErrGone)
|
||||
assert.Nil(t, got.Message.Body, "no deleted body is returned")
|
||||
_, err = s.GetState(ctx, claimNS, pos)
|
||||
assert.ErrorIs(t, err, message.ErrNotFound, "the record the claim created is withdrawn")
|
||||
}
|
||||
|
||||
// steppingClock is a test clock a hook can move forward.
|
||||
type steppingClock struct{ at time.Time }
|
||||
|
||||
func (c *steppingClock) now() time.Time {
|
||||
return c.at
|
||||
}
|
||||
|
||||
func (c *steppingClock) step(d time.Duration) {
|
||||
c.at = c.at.Add(d)
|
||||
}
|
||||
|
||||
func (c *steppingClock) claimer(s message.Store) *message.Claimer {
|
||||
return &message.Claimer{Store: s, Now: c.now}
|
||||
}
|
||||
|
||||
// A slow body or state read does not spend the visibility: the deadline is
|
||||
// measured from the claim attempt, after both reads.
|
||||
func TestClaimDeadlineFromAttemptNotRequestStart(t *testing.T) {
|
||||
s := claimStore(t)
|
||||
ctx := context.Background()
|
||||
pos := appendAt(t, s, fixedNow, fixedNow)
|
||||
clock := &steppingClock{at: fixedNow}
|
||||
h := &hookedStore{Store: s,
|
||||
afterGet: func(n int) {
|
||||
if n == 1 {
|
||||
clock.step(40 * time.Second)
|
||||
}
|
||||
},
|
||||
beforeState: func() { clock.step(40 * time.Second) },
|
||||
}
|
||||
|
||||
got, err := clock.claimer(h).Claim(ctx, claimPos(pos, 30*time.Second))
|
||||
require.NoError(t, err)
|
||||
attempt := fixedNow.Add(80 * time.Second)
|
||||
assert.Equal(t, attempt.UnixMilli(), got.ReceivedAt.UnixMilli())
|
||||
assert.Equal(t, attempt.Add(30*time.Second).UnixMilli(), got.Deadline.UnixMilli())
|
||||
assert.Equal(t, attempt.UnixMilli(), got.FirstReceive.UnixMilli())
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package jetstream_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func (e receiveEnv) deleter(store message.Store) *message.Deleter {
|
||||
if store == nil {
|
||||
store = e.store
|
||||
}
|
||||
return &message.Deleter{Store: store, Queues: e.queues, Keys: receiveKeys}
|
||||
}
|
||||
|
||||
func del(name, handle string) message.DeleteInput {
|
||||
return message.DeleteInput{Account: acct, Region: region, Queue: name, ReceiptHandle: handle}
|
||||
}
|
||||
|
||||
// receiveOne sends a message and receives it, returning the received copy.
|
||||
func (e receiveEnv) receiveOne(t *testing.T, name string) message.Received {
|
||||
t.Helper()
|
||||
e.send(t, name, "hello", nil)
|
||||
got, err := e.receiver(nil).Receive(context.Background(), recv(name, 1))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
return got[0]
|
||||
}
|
||||
|
||||
func (e receiveEnv) state(t *testing.T, q queue.Record, h string) (map[string]any, message.Revision) {
|
||||
t.Helper()
|
||||
rc, err := receiveKeys.Decode(h, q.Namespace().Queue)
|
||||
require.NoError(t, err)
|
||||
st, err := e.store.GetState(context.Background(), q.Namespace(), rc.Position)
|
||||
require.NoError(t, err)
|
||||
var m map[string]any
|
||||
require.NoError(t, json.Unmarshal(st.Value, &m))
|
||||
return m, st.Revision
|
||||
}
|
||||
|
||||
func TestDeleteCurrentHandleWritesTombstone(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
|
||||
require.NoError(t, e.deleter(nil).Delete(context.Background(), del("orders", m.ReceiptHandle)))
|
||||
st, _ := e.state(t, q, m.ReceiptHandle)
|
||||
assert.Equal(t, message.StatusDeleted, st["s"])
|
||||
assert.EqualValues(t, 1, st["g"], "the tombstone keeps the generation")
|
||||
|
||||
e.clock.advance(time.Hour)
|
||||
got, err := e.receiver(nil).Receive(context.Background(), recv("orders", 10))
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got, "a deleted message is never received again")
|
||||
|
||||
rc, err := receiveKeys.Decode(m.ReceiptHandle, q.Namespace().Queue)
|
||||
require.NoError(t, err)
|
||||
body, err := e.store.Get(context.Background(), q.Namespace(), rc.Position)
|
||||
require.NoError(t, err, "the body is left for the sweeper")
|
||||
assert.Equal(t, []byte("hello"), body.Body)
|
||||
}
|
||||
|
||||
// An older handle succeeds without deleting the newer receive.
|
||||
func TestDeleteOlderHandleKeepsNewerReceive(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
first := e.receiveOne(t, "orders")
|
||||
e.clock.advance(time.Hour)
|
||||
again, err := e.receiver(nil).Receive(context.Background(), recv("orders", 1))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, again, 1)
|
||||
_, before := e.state(t, q, first.ReceiptHandle)
|
||||
|
||||
require.NoError(t, e.deleter(nil).Delete(context.Background(), del("orders", first.ReceiptHandle)))
|
||||
st, after := e.state(t, q, first.ReceiptHandle)
|
||||
assert.Equal(t, message.StatusInflight, st["s"])
|
||||
assert.EqualValues(t, 2, st["g"])
|
||||
assert.Equal(t, before, after, "nothing was written")
|
||||
|
||||
require.NoError(t, e.deleter(nil).Delete(context.Background(), del("orders", again[0].ReceiptHandle)))
|
||||
st, _ = e.state(t, q, first.ReceiptHandle)
|
||||
assert.Equal(t, message.StatusDeleted, st["s"], "the newest handle deletes")
|
||||
}
|
||||
|
||||
func TestDeleteIsIdempotent(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
d := e.deleter(nil)
|
||||
require.NoError(t, d.Delete(context.Background(), del("orders", m.ReceiptHandle)))
|
||||
_, rev := e.state(t, q, m.ReceiptHandle)
|
||||
require.NoError(t, d.Delete(context.Background(), del("orders", m.ReceiptHandle)))
|
||||
_, again := e.state(t, q, m.ReceiptHandle)
|
||||
assert.Equal(t, rev, again, "a second delete writes nothing")
|
||||
|
||||
rc, err := receiveKeys.Decode(m.ReceiptHandle, q.Namespace().Queue)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, e.store.DeleteState(context.Background(), q.Namespace(), rc.Position, rev))
|
||||
require.NoError(t, d.Delete(context.Background(), del("orders", m.ReceiptHandle)), "a record already gone succeeds")
|
||||
}
|
||||
|
||||
// A delete racing a re-receive either tombstones before it, so the receive
|
||||
// gets nothing, or loses to it and leaves the newer receive in flight.
|
||||
func TestDeleteRacingReReceive(t *testing.T) {
|
||||
for range 10 {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
e.clock.advance(time.Hour)
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
got []message.Received
|
||||
rerr error
|
||||
)
|
||||
wg.Go(func() { got, rerr = e.receiver(nil).Receive(context.Background(), recv("orders", 1)) })
|
||||
derr := e.deleter(nil).Delete(context.Background(), del("orders", m.ReceiptHandle))
|
||||
wg.Wait()
|
||||
require.NoError(t, derr)
|
||||
require.NoError(t, rerr)
|
||||
st, _ := e.state(t, q, m.ReceiptHandle)
|
||||
if len(got) == 1 {
|
||||
assert.Equal(t, message.StatusInflight, st["s"], "the newer receive was not deleted")
|
||||
assert.EqualValues(t, 2, st["g"])
|
||||
} else {
|
||||
assert.Equal(t, message.StatusDeleted, st["s"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRefusesInvalidHandles(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
e.create(t, acct, "audit", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
forger, err := message.NewReceiptKeys(9, map[uint32][]byte{9: bytes.Repeat([]byte{3}, message.MinReceiptKeyLen)})
|
||||
require.NoError(t, err)
|
||||
rc, err := receiveKeys.Decode(m.ReceiptHandle, q.Namespace().Queue)
|
||||
require.NoError(t, err)
|
||||
forged, err := forger.Encode(rc)
|
||||
require.NoError(t, err)
|
||||
_, before := e.state(t, q, m.ReceiptHandle)
|
||||
|
||||
for name, in := range map[string]message.DeleteInput{
|
||||
"malformed": del("orders", "not-a-handle"),
|
||||
"empty": del("orders", ""),
|
||||
"tampered": del("orders", m.ReceiptHandle[:len(m.ReceiptHandle)-2]+"AA"),
|
||||
"unknown key": del("orders", forged),
|
||||
"another queue": del("audit", m.ReceiptHandle),
|
||||
"another account's": {Account: other, Region: region, Queue: "orders", ReceiptHandle: m.ReceiptHandle},
|
||||
} {
|
||||
err := e.deleter(nil).Delete(context.Background(), in)
|
||||
require.Error(t, err, name)
|
||||
if name != "another account's" {
|
||||
require.ErrorIs(t, err, message.ErrReceiptInvalid, name)
|
||||
}
|
||||
}
|
||||
_, after := e.state(t, q, m.ReceiptHandle)
|
||||
assert.Equal(t, before, after, "no refused handle wrote anything")
|
||||
}
|
||||
|
||||
func TestDeleteUnsettledOutcomes(t *testing.T) {
|
||||
t.Run("lost reply, tombstone applied", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
require.NoError(t, e.deleter(&receiveFaults{Store: e.store, putFails: 1}).Delete(context.Background(), del("orders", m.ReceiptHandle)),
|
||||
"a tombstone in place settles the lost reply")
|
||||
st, _ := e.state(t, q, m.ReceiptHandle)
|
||||
assert.Equal(t, message.StatusDeleted, st["s"])
|
||||
})
|
||||
t.Run("lost reply, nothing applied", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
_, before := e.state(t, q, m.ReceiptHandle)
|
||||
err := e.deleter(&lostPut{Store: e.store}).Delete(context.Background(), del("orders", m.ReceiptHandle))
|
||||
require.ErrorIs(t, err, message.ErrUnavailable, "the write may still land")
|
||||
_, after := e.state(t, q, m.ReceiptHandle)
|
||||
assert.Equal(t, before, after)
|
||||
})
|
||||
t.Run("state read fails", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
err := e.deleter(&failGetState{Store: e.store}).Delete(context.Background(), del("orders", m.ReceiptHandle))
|
||||
require.ErrorIs(t, err, message.ErrUnavailable)
|
||||
})
|
||||
for name, value := range map[string]string{
|
||||
"unknown state": `{"v":1,"s":"archived","g":1}`,
|
||||
"newer version": `{"v":2,"s":"inflight","c":1,"g":1,"d":1,"r":1,"f":1}`,
|
||||
"moving": `{"v":1,"s":"moving","c":1,"g":1}`,
|
||||
"older than key": `{"v":1,"s":"inflight","c":1,"g":0,"d":1,"r":1,"f":1}`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
rc, err := receiveKeys.Decode(m.ReceiptHandle, q.Namespace().Queue)
|
||||
require.NoError(t, err)
|
||||
_, rev := e.state(t, q, m.ReceiptHandle)
|
||||
rev, err = e.store.PutState(context.Background(), q.Namespace(), rc.Position, []byte(value), rev)
|
||||
require.NoError(t, err)
|
||||
err = e.deleter(nil).Delete(context.Background(), del("orders", m.ReceiptHandle))
|
||||
require.Error(t, err, "never reported as deleted")
|
||||
st, err := e.store.GetState(context.Background(), q.Namespace(), rc.Position)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, st.Revision, "nothing was written")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Neither handles nor bodies appear in errors or logs on any refusal.
|
||||
func TestDeleteKeepsHandlesOutOfErrorsAndLogs(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
prev := slog.Default()
|
||||
slog.SetDefault(slog.New(slog.NewJSONHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
|
||||
t.Cleanup(func() { slog.SetDefault(prev) })
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
e.create(t, acct, "audit", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
var errs []string
|
||||
for _, in := range []message.DeleteInput{
|
||||
del("audit", m.ReceiptHandle),
|
||||
del("orders", m.ReceiptHandle[:len(m.ReceiptHandle)-2]+"AA"),
|
||||
} {
|
||||
if err := e.deleter(&lostPut{Store: e.store}).Delete(context.Background(), in); err != nil {
|
||||
errs = append(errs, err.Error())
|
||||
}
|
||||
}
|
||||
errs = append(errs, e.deleter(&lostPut{Store: e.store}).Delete(context.Background(), del("orders", m.ReceiptHandle)).Error())
|
||||
all := strings.Join(errs, "\n") + logs.String()
|
||||
assert.NotContains(t, all, m.ReceiptHandle[:20])
|
||||
assert.NotContains(t, all, "hello")
|
||||
}
|
||||
|
||||
// lostPut drops every state write and reports a lost acknowledgement.
|
||||
type lostPut struct{ message.Store }
|
||||
|
||||
func (lostPut) PutState(context.Context, message.Namespace, message.Position, []byte, message.Revision) (message.Revision, error) {
|
||||
return 0, errors.New("acknowledgement lost")
|
||||
}
|
||||
|
||||
type failGetState struct{ message.Store }
|
||||
|
||||
func (failGetState) GetState(context.Context, message.Namespace, message.Position) (message.State, error) {
|
||||
return message.State{}, errors.New("timeout")
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package jetstream_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/nats-io/nkeys"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// guestLib is the store guest's shell library, from the repository root.
|
||||
func guestLib(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, here, _, ok := runtime.Caller(0)
|
||||
require.True(t, ok)
|
||||
p := filepath.Join(filepath.Dir(here), "..", "..", "..", "..", "..", "build", "microvm", "sqs-store", "sqs-store-lib.sh")
|
||||
_, err := os.Stat(p)
|
||||
require.NoError(t, err)
|
||||
return p
|
||||
}
|
||||
|
||||
// guestStore starts a server from the config the store guest itself writes,
|
||||
// changing only the port, and connects with the client NKey it admits.
|
||||
func guestStore(t *testing.T) (message.Store, *server.Options) {
|
||||
t.Helper()
|
||||
kp, err := nkeys.CreateUser()
|
||||
require.NoError(t, err)
|
||||
pub, err := kp.PublicKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
dir := t.TempDir()
|
||||
conf := filepath.Join(dir, "nats.conf")
|
||||
cmd := exec.Command("bash", "-c", `. "$0" && sqs_write_config 127.0.0.1 "$1" "$2"`, guestLib(t), pub, conf)
|
||||
cmd.Env = append(os.Environ(), "SQS_DATA="+filepath.Join(dir, "data"), "SQS_PORT=4222")
|
||||
out, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, "%s", out)
|
||||
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
require.NoError(t, err)
|
||||
opts.Port = -1
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
ns, err := server.NewServer(opts)
|
||||
require.NoError(t, err)
|
||||
go ns.Start()
|
||||
require.True(t, ns.ReadyForConnections(10*time.Second))
|
||||
t.Cleanup(ns.Shutdown)
|
||||
|
||||
nc, err := nats.Connect(ns.ClientURL(), nats.Nkey(pub, kp.Sign))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
require.NoError(t, err)
|
||||
ms, err := sqsjs.New(context.Background(), js, sqsjs.Config{MessageStreams: 3, StateBuckets: 2, DuplicateWindow: time.Minute})
|
||||
require.NoError(t, err)
|
||||
return ms, opts
|
||||
}
|
||||
|
||||
// ADR-0011:S4: the guest's own config admits a full 1 MiB message with the
|
||||
// metadata a send writes, while the application limit stays at 1 MiB.
|
||||
func TestS4_GuestConfigAdmitsAFullMessage(t *testing.T) {
|
||||
e := newSendEnv(t)
|
||||
store, opts := guestStore(t)
|
||||
assert.Equal(t, int32(2<<20), opts.MaxPayload)
|
||||
assert.True(t, opts.SyncAlways, "the guest syncs every write")
|
||||
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
in := input("orders", "")
|
||||
in.Body = bytes.Repeat([]byte("a"), message.MaxMessageSize)
|
||||
res, err := e.sender(store).Send(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := store.Get(context.Background(), q.Namespace(), res.Position)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, in.Body, got.Body)
|
||||
assert.Equal(t, res.MessageID, got.Meta[message.MetaID])
|
||||
assert.Equal(t, res.MD5OfBody, got.Meta[message.MetaMD5])
|
||||
|
||||
in.Body = append(in.Body, 'a')
|
||||
_, err = e.sender(store).Send(context.Background(), in)
|
||||
require.ErrorIs(t, err, message.ErrInvalidParameter)
|
||||
u, err := store.Count(context.Background(), q.Namespace())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(1), u.Messages)
|
||||
}
|
||||
|
||||
// ADR-0011:S4: a full 1 MiB message whose size is shared between ten
|
||||
// attributes and the body still fits the guest's own payload limit.
|
||||
func TestS4_GuestConfigAdmitsAFullMessageWithAttributes(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
store, _ := guestStore(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
|
||||
attrs := []message.Attribute{binAttr("blob", "Binary.raw", bytes.Repeat([]byte{0xff}, 256<<10))}
|
||||
for i := range 9 {
|
||||
attrs = append(attrs, strAttr("s"+string(rune('0'+i)), "String.label", string(bytes.Repeat([]byte("é"), 1000))))
|
||||
}
|
||||
set, err := message.NewAttributes(attrs)
|
||||
require.NoError(t, err)
|
||||
in := input("orders", "")
|
||||
in.Body = bytes.Repeat([]byte("a"), message.MaxMessageSize-set.Size())
|
||||
in.MessageAttributes = attrs
|
||||
res, err := e.sender(store).Send(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := e.receiver(store).Receive(context.Background(), recv("orders", 1))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, in.Body, got[0].Body)
|
||||
assert.Equal(t, set.All(), got[0].Attributes.All())
|
||||
assert.Equal(t, res.MD5OfMessageAttributes, got[0].MD5OfMessageAttributes)
|
||||
|
||||
in.Body = append(in.Body, 'a')
|
||||
_, err = e.sender(store).Send(context.Background(), in)
|
||||
require.ErrorIs(t, err, message.ErrInvalidParameter)
|
||||
u, err := store.Count(context.Background(), q.Namespace())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(1), u.Messages)
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
package jetstream_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// longPollEnv runs on the wall clock: a long poll waits in real time.
|
||||
type longPollEnv struct {
|
||||
queues *queue.Store
|
||||
store message.Store
|
||||
js jetstream.JetStream
|
||||
}
|
||||
|
||||
// Long enough that only the wake feed or a known visibility can end a
|
||||
// test's wait in time.
|
||||
const noRecheck = time.Minute
|
||||
|
||||
func newLongPollEnv(t *testing.T) longPollEnv {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
qs, err := queue.Open(context.Background(), js, 3, 2)
|
||||
require.NoError(t, err)
|
||||
ms, mjs := openStoreJS(t, storeMaxPayload)
|
||||
return longPollEnv{queues: qs, store: ms, js: mjs}
|
||||
}
|
||||
|
||||
func (e longPollEnv) create(t *testing.T, name string, attrs map[string]string) {
|
||||
t.Helper()
|
||||
_, err := e.queues.Create(context.Background(), queue.CreateRequest{Account: acct, Region: region, Name: name, Attributes: attrs})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func (e longPollEnv) receiver(store message.Store, recheck time.Duration) *message.Receiver {
|
||||
if store == nil {
|
||||
store = e.store
|
||||
}
|
||||
return &message.Receiver{Store: store, Queues: e.queues, Keys: receiveKeys, Recheck: recheck}
|
||||
}
|
||||
|
||||
func (e longPollEnv) send(t *testing.T, name, body string, delay *int) message.SendResult {
|
||||
t.Helper()
|
||||
in := input(name, body)
|
||||
in.DelaySeconds = delay
|
||||
res, err := (&message.Sender{Store: e.store, Queues: e.queues}).Send(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
return res
|
||||
}
|
||||
|
||||
func wait(name string, secs int) message.ReceiveInput {
|
||||
in := recv(name, 1)
|
||||
in.WaitTimeSeconds = ptr(secs)
|
||||
return in
|
||||
}
|
||||
|
||||
// noConsumers asserts every wake feed was released on every message shard.
|
||||
func (e longPollEnv) noConsumers(t *testing.T) {
|
||||
t.Helper()
|
||||
for i := range 3 {
|
||||
assert.Zero(t, consumerCount(t, e.js, sqsjs.StreamName(message.Shard(i))), "shard %d", i)
|
||||
}
|
||||
}
|
||||
|
||||
type pollResult struct {
|
||||
got []message.Received
|
||||
err error
|
||||
took time.Duration
|
||||
}
|
||||
|
||||
func goReceive(r *message.Receiver, ctx context.Context, in message.ReceiveInput) <-chan pollResult {
|
||||
out := make(chan pollResult, 1)
|
||||
go func() {
|
||||
start := time.Now()
|
||||
got, err := r.Receive(ctx, in)
|
||||
out <- pollResult{got, err, time.Since(start)}
|
||||
}()
|
||||
return out
|
||||
}
|
||||
|
||||
// A send from a separate sender wakes a waiting receiver without a recheck.
|
||||
func TestLongPollWakesOnSend(t *testing.T) {
|
||||
e := newLongPollEnv(t)
|
||||
e.create(t, "orders", nil)
|
||||
res := goReceive(e.receiver(nil, noRecheck), context.Background(), wait("orders", 10))
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
sent := e.send(t, "orders", "hello", nil)
|
||||
|
||||
r := <-res
|
||||
require.NoError(t, r.err)
|
||||
require.Len(t, r.got, 1)
|
||||
assert.Equal(t, sent.MessageID, r.got[0].MessageID)
|
||||
assert.Less(t, r.took, 3*time.Second, "woken by the append, not the wait's end")
|
||||
e.noConsumers(t)
|
||||
}
|
||||
|
||||
// An append to another queue on the same shard wakes nothing it can take,
|
||||
// so the waiter keeps waiting and times out empty.
|
||||
func TestLongPollIgnoresOtherQueues(t *testing.T) {
|
||||
e := newLongPollEnv(t)
|
||||
e.create(t, "orders", nil)
|
||||
e.create(t, "other", nil)
|
||||
res := goReceive(e.receiver(nil, noRecheck), context.Background(), wait("orders", 1))
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
e.send(t, "other", "not yours", nil)
|
||||
|
||||
r := <-res
|
||||
require.NoError(t, r.err)
|
||||
assert.Empty(t, r.got)
|
||||
assert.GreaterOrEqual(t, r.took, time.Second)
|
||||
}
|
||||
|
||||
func TestLongPollEmptyTimesOut(t *testing.T) {
|
||||
e := newLongPollEnv(t)
|
||||
e.create(t, "orders", nil)
|
||||
start := time.Now()
|
||||
got, err := e.receiver(nil, 0).Receive(context.Background(), wait("orders", 2))
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got)
|
||||
took := time.Since(start)
|
||||
assert.GreaterOrEqual(t, took, 2*time.Second)
|
||||
assert.Less(t, took, 4*time.Second, "the wait is bounded")
|
||||
e.noConsumers(t)
|
||||
}
|
||||
|
||||
// Two waiting receivers on separate instances: one send is taken once.
|
||||
func TestLongPollCompetingReceivers(t *testing.T) {
|
||||
e := newLongPollEnv(t)
|
||||
e.create(t, "orders", nil)
|
||||
a := goReceive(e.receiver(nil, noRecheck), context.Background(), wait("orders", 3))
|
||||
b := goReceive(e.receiver(nil, noRecheck), context.Background(), wait("orders", 3))
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
sent := e.send(t, "orders", "one", nil)
|
||||
|
||||
ra, rb := <-a, <-b
|
||||
require.NoError(t, ra.err)
|
||||
require.NoError(t, rb.err)
|
||||
got := append(ra.got, rb.got...)
|
||||
require.Len(t, got, 1, "exactly one receiver takes the message")
|
||||
assert.Equal(t, sent.MessageID, got[0].MessageID)
|
||||
assert.Less(t, min(ra.took, rb.took), 2*time.Second, "the taker was woken")
|
||||
e.noConsumers(t)
|
||||
}
|
||||
|
||||
func TestLongPollStopsOnCancel(t *testing.T) {
|
||||
e := newLongPollEnv(t)
|
||||
e.create(t, "orders", nil)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
res := goReceive(e.receiver(nil, noRecheck), ctx, wait("orders", 20))
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
cancel()
|
||||
|
||||
r := <-res
|
||||
require.ErrorIs(t, r.err, context.Canceled)
|
||||
assert.Empty(t, r.got)
|
||||
assert.Less(t, r.took, 2*time.Second, "stops promptly")
|
||||
e.noConsumers(t)
|
||||
}
|
||||
|
||||
// lostFeed opens a real wake feed, then fails Next once lose is closed.
|
||||
type lostFeed struct {
|
||||
message.Store
|
||||
|
||||
lose chan struct{}
|
||||
}
|
||||
|
||||
func (s lostFeed) Feed(ctx context.Context, shard message.Shard, after message.Cursor) (message.Feed, error) {
|
||||
f, err := s.Store.Feed(ctx, shard, after)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return failingFeed{Feed: f, lose: s.lose}, nil
|
||||
}
|
||||
|
||||
type failingFeed struct {
|
||||
message.Feed
|
||||
|
||||
lose chan struct{}
|
||||
}
|
||||
|
||||
func (f failingFeed) Next(ctx context.Context) (message.Change, error) {
|
||||
select {
|
||||
case <-f.lose:
|
||||
return message.Change{}, errors.New("feed lost")
|
||||
case <-ctx.Done():
|
||||
return message.Change{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// A wake feed that fails after opening ends the wait as unavailable, not
|
||||
// as a successful empty receive.
|
||||
func TestLongPollFeedFailureAfterOpen(t *testing.T) {
|
||||
e := newLongPollEnv(t)
|
||||
e.create(t, "orders", nil)
|
||||
lose := make(chan struct{})
|
||||
res := goReceive(e.receiver(lostFeed{Store: e.store, lose: lose}, noRecheck), context.Background(), wait("orders", 20))
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
close(lose)
|
||||
|
||||
r := <-res
|
||||
require.ErrorIs(t, r.err, message.ErrUnavailable)
|
||||
assert.Empty(t, r.got)
|
||||
assert.Less(t, r.took, 2*time.Second)
|
||||
e.noConsumers(t)
|
||||
}
|
||||
|
||||
// heldStore holds Feed or Scan until its context ends; scans hold only
|
||||
// after the first heldAfter complete.
|
||||
type heldStore struct {
|
||||
message.Store
|
||||
|
||||
feed bool
|
||||
heldAfter int
|
||||
// heldErr, when set, replaces the context error a held scan returns.
|
||||
heldErr error
|
||||
mu sync.Mutex
|
||||
scans int
|
||||
}
|
||||
|
||||
func (s *heldStore) Feed(ctx context.Context, shard message.Shard, after message.Cursor) (message.Feed, error) {
|
||||
if s.feed {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return s.Store.Feed(ctx, shard, after)
|
||||
}
|
||||
|
||||
func (s *heldStore) Scan(ctx context.Context, shard message.Shard, from message.ScanCursor, limit int) (message.ScanPage, error) {
|
||||
s.mu.Lock()
|
||||
s.scans++
|
||||
n := s.scans
|
||||
s.mu.Unlock()
|
||||
if s.heldAfter >= 0 && n > s.heldAfter {
|
||||
<-ctx.Done()
|
||||
if s.heldErr != nil {
|
||||
return message.ScanPage{}, s.heldErr
|
||||
}
|
||||
return message.ScanPage{}, ctx.Err()
|
||||
}
|
||||
return s.Store.Scan(ctx, shard, from, limit)
|
||||
}
|
||||
|
||||
// A scan held past the wait ends it on time: empty after a completed poll,
|
||||
// unavailable when no poll ever completed. No wake consumer is left.
|
||||
func TestLongPollHeldScan(t *testing.T) {
|
||||
e := newLongPollEnv(t)
|
||||
e.create(t, "orders", nil)
|
||||
|
||||
after := &heldStore{Store: e.store, heldAfter: 1}
|
||||
r := <-goReceive(e.receiver(after, 100*time.Millisecond), context.Background(), wait("orders", 1))
|
||||
require.NoError(t, r.err)
|
||||
assert.Empty(t, r.got)
|
||||
assert.Less(t, r.took, 2*time.Second, "bounded by the wait")
|
||||
|
||||
never := &heldStore{Store: e.store, heldAfter: 0}
|
||||
r = <-goReceive(e.receiver(never, noRecheck), context.Background(), wait("orders", 1))
|
||||
require.ErrorIs(t, r.err, message.ErrUnavailable)
|
||||
assert.Empty(t, r.got)
|
||||
assert.Less(t, r.took, 2*time.Second, "bounded by the wait")
|
||||
e.noConsumers(t)
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
package jetstream_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var receiveKeys = func() *message.ReceiptKeys {
|
||||
k, err := message.NewReceiptKeys(1, map[uint32][]byte{1: bytes.Repeat([]byte{7}, message.MinReceiptKeyLen)})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return k
|
||||
}()
|
||||
|
||||
// clock is a shared test clock for a sender and its receivers.
|
||||
type clock struct {
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func (c *clock) now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.at
|
||||
}
|
||||
|
||||
func (c *clock) advance(d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.at = c.at.Add(d)
|
||||
}
|
||||
|
||||
type receiveEnv struct {
|
||||
sendEnv
|
||||
|
||||
clock *clock
|
||||
}
|
||||
|
||||
func newReceiveEnv(t *testing.T) receiveEnv {
|
||||
t.Helper()
|
||||
return receiveEnv{sendEnv: newSendEnv(t), clock: &clock{at: fixedNow}}
|
||||
}
|
||||
|
||||
func (e receiveEnv) receiver(store message.Store) *message.Receiver {
|
||||
if store == nil {
|
||||
store = e.store
|
||||
}
|
||||
return &message.Receiver{Store: store, Queues: e.queues, Keys: receiveKeys, Now: e.clock.now}
|
||||
}
|
||||
|
||||
func (e receiveEnv) send(t *testing.T, name, body string, delay *int) message.SendResult {
|
||||
t.Helper()
|
||||
in := input(name, body)
|
||||
in.DelaySeconds = delay
|
||||
s := &message.Sender{Store: e.store, Queues: e.queues, Now: e.clock.now}
|
||||
res, err := s.Send(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
return res
|
||||
}
|
||||
|
||||
func recv(name string, n int) message.ReceiveInput {
|
||||
return message.ReceiveInput{Account: acct, Region: region, Queue: name, MaxNumberOfMessages: &n}
|
||||
}
|
||||
|
||||
func TestReceiveReturnsSentMessageWithHandle(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", map[string]string{"VisibilityTimeout": "30"})
|
||||
sent := e.send(t, "orders", "hello", nil)
|
||||
r := e.receiver(nil)
|
||||
|
||||
got, err := r.Receive(context.Background(), recv("orders", 1))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
m := got[0]
|
||||
assert.Equal(t, sent.MessageID, m.MessageID)
|
||||
assert.Equal(t, []byte("hello"), m.Body)
|
||||
assert.Equal(t, sent.MD5OfBody, m.MD5OfBody)
|
||||
assert.Equal(t, sender, m.SenderID)
|
||||
assert.Equal(t, sent.EnqueuedAt, m.SentAt)
|
||||
assert.Equal(t, uint64(1), m.ReceiveCount)
|
||||
assert.Equal(t, fixedNow.Add(30*time.Second).UnixMilli(), m.Deadline.UnixMilli())
|
||||
|
||||
rc, err := receiveKeys.Decode(m.ReceiptHandle, q.Namespace().Queue)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, message.Receipt{Queue: q.Namespace().Queue, Position: sent.Position, Generation: 1}, rc)
|
||||
|
||||
again, err := r.Receive(context.Background(), recv("orders", 10))
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, again, "an in-flight message is not received twice")
|
||||
|
||||
e.clock.advance(30 * time.Second)
|
||||
back, err := r.Receive(context.Background(), recv("orders", 10))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, back, 1, "visible again at its deadline")
|
||||
assert.Equal(t, uint64(2), back[0].ReceiveCount)
|
||||
assert.Equal(t, fixedNow.UnixMilli(), back[0].FirstReceive.UnixMilli())
|
||||
assert.NotEqual(t, m.ReceiptHandle, back[0].ReceiptHandle)
|
||||
rc2, err := receiveKeys.Decode(back[0].ReceiptHandle, q.Namespace().Queue)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(2), rc2.Generation)
|
||||
}
|
||||
|
||||
func TestReceiveMaxNumberOfMessages(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
for i := range 15 {
|
||||
e.send(t, "orders", "m"+strconv.Itoa(i), nil)
|
||||
}
|
||||
r := e.receiver(nil)
|
||||
|
||||
one, err := r.Receive(context.Background(), message.ReceiveInput{Account: acct, Region: region, Queue: "orders"})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, one, 1, "default 1")
|
||||
ten, err := r.Receive(context.Background(), recv("orders", 10))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, ten, 10)
|
||||
rest, err := r.Receive(context.Background(), recv("orders", 10))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, rest, 4)
|
||||
|
||||
seen := map[string]bool{}
|
||||
for _, m := range append(append(one, ten...), rest...) {
|
||||
assert.False(t, seen[m.MessageID], "no message is returned twice")
|
||||
seen[m.MessageID] = true
|
||||
}
|
||||
assert.Len(t, seen, 15)
|
||||
|
||||
for _, n := range []int{0, -1, 11} {
|
||||
_, err := r.Receive(context.Background(), recv("orders", n))
|
||||
require.ErrorIs(t, err, message.ErrInvalidParameter, "max %d", n)
|
||||
}
|
||||
for _, v := range []int{-1, message.MaxVisibilitySeconds + 1} {
|
||||
in := recv("orders", 1)
|
||||
in.VisibilityTimeout = ptr(v)
|
||||
_, err := r.Receive(context.Background(), in)
|
||||
require.ErrorIs(t, err, message.ErrInvalidParameter, "visibility %d", v)
|
||||
}
|
||||
for _, w := range []int{-1, message.MaxWaitSeconds + 1} {
|
||||
in := recv("orders", 1)
|
||||
in.WaitTimeSeconds = ptr(w)
|
||||
_, err = r.Receive(context.Background(), in)
|
||||
require.ErrorIs(t, err, message.ErrInvalidParameter, "wait %d", w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReceiveIsolatesQueues(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
e.create(t, acct, "audit", nil)
|
||||
e.create(t, other, "orders", nil)
|
||||
e.send(t, "audit", "audit-only", nil)
|
||||
in := input("orders", "other-account")
|
||||
in.Account = other
|
||||
_, err := (&message.Sender{Store: e.store, Queues: e.queues, Now: e.clock.now}).Send(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := e.receiver(nil).Receive(context.Background(), recv("orders", 10))
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got, "neither a sibling queue's nor another account's messages")
|
||||
|
||||
_, err = e.receiver(nil).Receive(context.Background(), recv("missing", 1))
|
||||
require.ErrorIs(t, err, message.ErrQueueDoesNotExist)
|
||||
}
|
||||
|
||||
// A queue whose first visible message lies beyond one receive's scan budget
|
||||
// is reached by the next receive, which resumes the pass.
|
||||
func TestReceiveResumesBeyondScanBudget(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
budget := message.ReceiveScanPage * message.ReceiveScanPages
|
||||
for i := range budget + 20 {
|
||||
e.send(t, "orders", "d"+strconv.Itoa(i), ptr(message.MaxDelaySeconds))
|
||||
}
|
||||
last := e.send(t, "orders", "ready", nil)
|
||||
counted := &scanCounter{Store: e.store}
|
||||
r := e.receiver(counted)
|
||||
|
||||
got, err := r.Receive(context.Background(), recv("orders", 1))
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got, "the first receive's budget ends before the message")
|
||||
assert.LessOrEqual(t, counted.calls, message.ReceiveScanPages, "scan work per receive is bounded")
|
||||
|
||||
counted.calls = 0
|
||||
got, err = r.Receive(context.Background(), recv("orders", 1))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1, "the next receive resumes the pass")
|
||||
assert.Equal(t, last.MessageID, got[0].MessageID)
|
||||
assert.LessOrEqual(t, counted.calls, message.ReceiveScanPages)
|
||||
}
|
||||
|
||||
func TestReceiveCompetingConsumers(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
for i := range 20 {
|
||||
e.send(t, "orders", "m"+strconv.Itoa(i), nil)
|
||||
}
|
||||
var (
|
||||
mu sync.Mutex
|
||||
seen = map[string]int{}
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
for range 4 {
|
||||
r := e.receiver(nil)
|
||||
wg.Go(func() {
|
||||
for range 5 {
|
||||
got, err := r.Receive(context.Background(), recv("orders", 10))
|
||||
assert.NoError(t, err)
|
||||
mu.Lock()
|
||||
for _, m := range got {
|
||||
seen[m.MessageID]++
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
assert.Len(t, seen, 20, "every message is received")
|
||||
for id, n := range seen {
|
||||
assert.Equal(t, 1, n, "message %s received once within its visibility", id)
|
||||
}
|
||||
}
|
||||
|
||||
// A scan failure or an unsettled claim is never an empty queue; a receive
|
||||
// that claimed something still returns it.
|
||||
func TestReceiveReportsUnsettledOutcomes(t *testing.T) {
|
||||
t.Run("scan fails", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
e.send(t, "orders", "hello", nil)
|
||||
r := e.receiver(&receiveFaults{Store: e.store, scanErr: errors.New("timeout")})
|
||||
_, err := r.Receive(context.Background(), recv("orders", 1))
|
||||
require.ErrorIs(t, err, message.ErrUnavailable)
|
||||
})
|
||||
t.Run("only claim is ambiguous", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
e.send(t, "orders", "hello", nil)
|
||||
r := e.receiver(&receiveFaults{Store: e.store, putFails: 1})
|
||||
got, err := r.Receive(context.Background(), recv("orders", 1))
|
||||
require.ErrorIs(t, err, message.ErrUnavailable)
|
||||
assert.Empty(t, got)
|
||||
})
|
||||
t.Run("another claim succeeds", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
e.send(t, "orders", "first", nil)
|
||||
second := e.send(t, "orders", "second", nil)
|
||||
r := e.receiver(&receiveFaults{Store: e.store, putFails: 1})
|
||||
got, err := r.Receive(context.Background(), recv("orders", 10))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, second.MessageID, got[0].MessageID)
|
||||
})
|
||||
t.Run("newer record", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
sent := e.send(t, "orders", "hello", nil)
|
||||
_, err := e.store.PutState(context.Background(), q.Namespace(), sent.Position, []byte(`{"v":99}`), 0)
|
||||
require.NoError(t, err)
|
||||
_, err = e.receiver(nil).Receive(context.Background(), recv("orders", 1))
|
||||
require.Error(t, err, "a record this release cannot act on is not an empty queue")
|
||||
})
|
||||
}
|
||||
|
||||
type scanCounter struct {
|
||||
message.Store
|
||||
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *scanCounter) Scan(ctx context.Context, shard message.Shard, from message.ScanCursor, limit int) (message.ScanPage, error) {
|
||||
s.calls++
|
||||
return s.Store.Scan(ctx, shard, from, limit)
|
||||
}
|
||||
|
||||
// receiveFaults fails every scan with scanErr, or the first putFails state
|
||||
// writes after they apply, as a lost acknowledgement would.
|
||||
type receiveFaults struct {
|
||||
message.Store
|
||||
|
||||
scanErr error
|
||||
putFails int
|
||||
}
|
||||
|
||||
func (f *receiveFaults) Scan(ctx context.Context, shard message.Shard, from message.ScanCursor, limit int) (message.ScanPage, error) {
|
||||
if f.scanErr != nil {
|
||||
return message.ScanPage{}, f.scanErr
|
||||
}
|
||||
return f.Store.Scan(ctx, shard, from, limit)
|
||||
}
|
||||
|
||||
func (f *receiveFaults) PutState(ctx context.Context, ns message.Namespace, pos message.Position, v []byte, rev message.Revision) (message.Revision, error) {
|
||||
r, err := f.Store.PutState(ctx, ns, pos, v, rev)
|
||||
if err == nil && f.putFails > 0 {
|
||||
f.putFails--
|
||||
return 0, errors.New("acknowledgement lost")
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package jetstream_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
acct = "000000000001"
|
||||
other = "000000000002"
|
||||
region = "us-east-1"
|
||||
sender = "AIDAEXAMPLESENDER"
|
||||
)
|
||||
|
||||
var fixedNow = time.Date(2026, 10, 30, 12, 0, 0, 123_456_789, time.UTC)
|
||||
|
||||
type sendEnv struct {
|
||||
queues *queue.Store
|
||||
store message.Store
|
||||
}
|
||||
|
||||
func newSendEnv(t *testing.T) sendEnv {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ctx := context.Background()
|
||||
qs, err := queue.Open(ctx, js, 3, 2)
|
||||
require.NoError(t, err)
|
||||
return sendEnv{queues: qs, store: openStore(t, storeMaxPayload)}
|
||||
}
|
||||
|
||||
// The dedicated store is its own server; ADR-0011:S4 requires its
|
||||
// max_payload to admit a full message, which the spike showed at 2MB.
|
||||
const storeMaxPayload = 2 << 20
|
||||
|
||||
func openStore(t *testing.T, maxPayload int32) message.Store {
|
||||
t.Helper()
|
||||
ms, _ := openStoreJS(t, maxPayload)
|
||||
return ms
|
||||
}
|
||||
|
||||
func openStoreJS(t *testing.T, maxPayload int32) (message.Store, jetstream.JetStream) {
|
||||
t.Helper()
|
||||
ns, err := server.NewServer(&server.Options{
|
||||
Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(),
|
||||
NoLog: true, NoSigs: true, MaxPayload: maxPayload,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
go ns.Start()
|
||||
require.True(t, ns.ReadyForConnections(10*time.Second))
|
||||
t.Cleanup(ns.Shutdown)
|
||||
nc, err := nats.Connect(ns.ClientURL())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
require.NoError(t, err)
|
||||
ms, err := sqsjs.New(context.Background(), js, sqsjs.Config{MessageStreams: 3, StateBuckets: 2, DuplicateWindow: time.Minute})
|
||||
require.NoError(t, err)
|
||||
return ms, js
|
||||
}
|
||||
|
||||
func (e sendEnv) sender(store message.Store) *message.Sender {
|
||||
if store == nil {
|
||||
store = e.store
|
||||
}
|
||||
return &message.Sender{Store: store, Queues: e.queues, Now: func() time.Time { return fixedNow }}
|
||||
}
|
||||
|
||||
func (e sendEnv) create(t *testing.T, account, name string, attrs map[string]string) queue.Record {
|
||||
t.Helper()
|
||||
r, err := e.queues.Create(context.Background(), queue.CreateRequest{Account: account, Region: region, Name: name, Attributes: attrs})
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}
|
||||
|
||||
func (e sendEnv) count(t *testing.T, r queue.Record) uint64 {
|
||||
t.Helper()
|
||||
u, err := e.store.Count(context.Background(), r.Namespace())
|
||||
require.NoError(t, err)
|
||||
return u.Messages
|
||||
}
|
||||
|
||||
func input(name, body string) message.SendInput {
|
||||
return message.SendInput{Account: acct, Region: region, Queue: name, Sender: sender, Body: []byte(body)}
|
||||
}
|
||||
|
||||
func ptr(n int) *int { return &n }
|
||||
|
||||
func TestSendAppendsBodyAndMetadata(t *testing.T) {
|
||||
e := newSendEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
|
||||
res, err := e.sender(nil).Send(context.Background(), input("orders", "hello"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "5d41402abc4b2a76b9719d911017c592", res.MD5OfBody)
|
||||
assert.Regexp(t, uuidV4, res.MessageID)
|
||||
enq := fixedNow.Truncate(time.Millisecond)
|
||||
assert.Equal(t, enq, res.EnqueuedAt)
|
||||
assert.Equal(t, enq, res.VisibleAfter, "the queue's default delay is 0")
|
||||
|
||||
got, err := e.store.Get(context.Background(), q.Namespace(), res.Position)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []byte("hello"), got.Body)
|
||||
ms := strconv.FormatInt(enq.UnixMilli(), 10)
|
||||
assert.Equal(t, map[string]string{
|
||||
message.MetaV: "1", message.MetaID: res.MessageID, message.MetaEnq: ms, message.MetaVis: ms,
|
||||
message.MetaSender: sender, message.MetaMD5: res.MD5OfBody, message.MetaAcct: acct,
|
||||
}, got.Meta)
|
||||
}
|
||||
|
||||
var uuidV4 = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
|
||||
|
||||
// faultStore wraps a real store's Append.
|
||||
type faultStore struct {
|
||||
message.Store
|
||||
|
||||
append func(context.Context, message.Namespace, message.AppendInput) (message.AppendResult, error)
|
||||
}
|
||||
|
||||
func (f faultStore) Append(ctx context.Context, ns message.Namespace, in message.AppendInput) (message.AppendResult, error) {
|
||||
return f.append(ctx, ns, in)
|
||||
}
|
||||
|
||||
func TestSendLostAppendReplyIsNeverSuccess(t *testing.T) {
|
||||
e := newSendEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
errLost := errors.New("reply lost")
|
||||
|
||||
t.Run("applied but the reply was lost", func(t *testing.T) {
|
||||
fs := faultStore{Store: e.store, append: func(ctx context.Context, ns message.Namespace, in message.AppendInput) (message.AppendResult, error) {
|
||||
_, err := e.store.Append(ctx, ns, in)
|
||||
require.NoError(t, err)
|
||||
return message.AppendResult{}, errLost
|
||||
}}
|
||||
res, err := e.sender(fs).Send(context.Background(), input("orders", "x"))
|
||||
require.ErrorIs(t, err, message.ErrUnavailable)
|
||||
assert.Zero(t, res)
|
||||
assert.Equal(t, uint64(1), e.count(t, q), "the stored copy is a duplicate if the client retries")
|
||||
})
|
||||
}
|
||||
|
||||
// Neither logs nor errors carry a body, whatever the outcome.
|
||||
func TestSendNeverExposesTheBody(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
prev := slog.Default()
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
|
||||
t.Cleanup(func() { slog.SetDefault(prev) })
|
||||
|
||||
e := newSendEnv(t)
|
||||
e.create(t, acct, "small", map[string]string{"MaximumMessageSize": "1024"})
|
||||
const secret = "SECRET-BODY-7f3a"
|
||||
failing := faultStore{Store: e.store, append: func(context.Context, message.Namespace, message.AppendInput) (message.AppendResult, error) {
|
||||
return message.AppendResult{}, errors.New("boom")
|
||||
}}
|
||||
var errs []error
|
||||
for _, tc := range []struct {
|
||||
store message.Store
|
||||
body string
|
||||
}{
|
||||
{nil, secret},
|
||||
{nil, secret + "\x00"},
|
||||
{nil, secret + strings.Repeat("a", 2000)},
|
||||
{failing, secret},
|
||||
} {
|
||||
_, err := e.sender(tc.store).Send(context.Background(), input("small", tc.body))
|
||||
errs = append(errs, err)
|
||||
}
|
||||
require.NoError(t, errs[0])
|
||||
for _, err := range errs[1:] {
|
||||
require.Error(t, err)
|
||||
assert.NotContains(t, err.Error(), secret)
|
||||
}
|
||||
assert.NotContains(t, logs.String(), secret)
|
||||
}
|
||||
@@ -276,9 +276,16 @@ func (s *Store) stream(ctx context.Context, ns message.Namespace) (jsapi.Stream,
|
||||
return stream, nil
|
||||
}
|
||||
|
||||
// owned reads the message at pos if it belongs to the namespace's subject;
|
||||
// a position on a neighbour's subject is reported as ErrNotFound.
|
||||
// errDirectGet refuses a read the client would serve by direct get, which
|
||||
// any replica may answer with a stale copy.
|
||||
var errDirectGet = errors.New("stream allows direct get; leader read cannot be assured")
|
||||
|
||||
// owned reads the message at pos from the stream leader if it belongs to the
|
||||
// namespace's subject; a neighbour's position is reported as ErrNotFound.
|
||||
func (s *Store) owned(ctx context.Context, stream jsapi.Stream, ns message.Namespace, pos message.Position) (*jsapi.RawStreamMsg, error) {
|
||||
if info := stream.CachedInfo(); info == nil || info.Config.AllowDirect || info.Config.MirrorDirect {
|
||||
return nil, fmt.Errorf("get message %s/%d: %w", ns.Queue, pos, errDirectGet)
|
||||
}
|
||||
raw, err := stream.GetMsg(ctx, uint64(pos))
|
||||
switch {
|
||||
case errors.Is(err, jsapi.ErrMsgNotFound):
|
||||
@@ -413,12 +420,13 @@ func (s *Store) Feed(ctx context.Context, messageShard message.Shard, after mess
|
||||
f := &feed{shard: messageShard, stream: stream, cursor: uint64(after), idleCheck: feedIdleCheck, closeWait: feedCloseWait}
|
||||
f.closing, f.closeNow = context.WithCancel(context.Background())
|
||||
if after == math.MaxUint64 {
|
||||
// No position follows the maximum cursor; only later appends can.
|
||||
info, err := stream.Info(ctx)
|
||||
// No position follows the maximum cursor; only later appends can. A
|
||||
// fresh handle: refreshing the shared one would race its readers.
|
||||
fresh, err := s.js.Stream(ctx, StreamName(messageShard))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open feed on %s: %w", StreamName(messageShard), err)
|
||||
}
|
||||
f.cursor = info.State.LastSeq
|
||||
f.cursor = fresh.CachedInfo().State.LastSeq
|
||||
}
|
||||
if err := f.start(ctx); err != nil {
|
||||
f.closeNow()
|
||||
@@ -1068,19 +1076,19 @@ func (s *Store) Scan(ctx context.Context, messageShard message.Shard, from messa
|
||||
if err := message.ValidateScan(from, limit); err != nil {
|
||||
return message.ScanPage{}, err
|
||||
}
|
||||
stream, err := s.stream(ctx, message.Namespace{MessageShard: messageShard, Queue: "scan"})
|
||||
if err != nil {
|
||||
if _, err := s.stream(ctx, message.Namespace{MessageShard: messageShard, Queue: "scan"}); err != nil {
|
||||
return message.ScanPage{}, err
|
||||
}
|
||||
if from.Bound == 0 {
|
||||
info, err := stream.Info(ctx)
|
||||
// A fresh handle: refreshing the shared one would race its readers.
|
||||
fresh, err := s.js.Stream(ctx, StreamName(messageShard))
|
||||
if err != nil {
|
||||
return message.ScanPage{}, fmt.Errorf("scan %s: %w", StreamName(messageShard), err)
|
||||
}
|
||||
from.Bound = message.Cursor(info.State.LastSeq)
|
||||
from.Bound = message.Cursor(fresh.CachedInfo().State.LastSeq)
|
||||
}
|
||||
page := message.ScanPage{Next: from}
|
||||
last, done, err := readRange(ctx, s.js, stream.CachedInfo().Config.Name, shardFilter(messageShard), uint64(from.After), uint64(from.Bound), func(msg *jsapi.RawStreamMsg) (bool, error) {
|
||||
last, done, err := readRange(ctx, s.js, StreamName(messageShard), shardFilter(messageShard), uint64(from.After), uint64(from.Bound), func(msg *jsapi.RawStreamMsg) (bool, error) {
|
||||
seq := msg.Sequence
|
||||
ns, err := decodeNamespace(messageShard, msg.Subject, msg.Header)
|
||||
if err != nil {
|
||||
|
||||
@@ -954,3 +954,27 @@ func TestNamespacesLeavesNoConsumer(t *testing.T) {
|
||||
}
|
||||
assert.Equal(t, 0, consumerCount(t, js, sqsjs.StreamName(1)))
|
||||
}
|
||||
|
||||
// Get must be a leader read: a stream that allows direct get, as an earlier
|
||||
// configuration's shard might, would let any replica answer with a stale
|
||||
// body. The store refuses the read rather than serve one.
|
||||
func TestGetRefusesDirectGetStream(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
_, js := startServer(t, t.TempDir())
|
||||
s, err := sqsjs.New(ctx, js, testConfig)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, testConfig.MessageStreamConfig(0, 1).AllowDirect, "the store never provisions direct get")
|
||||
|
||||
earlier := message.Shard(testConfig.MessageStreams)
|
||||
cfg := testConfig.MessageStreamConfig(earlier, 1)
|
||||
cfg.AllowDirect = true
|
||||
_, err = js.CreateStream(ctx, cfg)
|
||||
require.NoError(t, err)
|
||||
n := ns(earlier, 0, "directq")
|
||||
ack, err := js.Publish(ctx, sqsjs.Subject(n), []byte("b"))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = s.Get(ctx, n, message.Position(ack.Sequence))
|
||||
require.Error(t, err)
|
||||
assert.NotErrorIs(t, err, message.ErrNotFound, "a refused read is never taken as removed")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
package jetstream_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func (e receiveEnv) changer(store message.Store) *message.VisibilityChanger {
|
||||
if store == nil {
|
||||
store = e.store
|
||||
}
|
||||
return &message.VisibilityChanger{Store: store, Queues: e.queues, Keys: receiveKeys, Now: e.clock.now}
|
||||
}
|
||||
|
||||
func vis(name, handle string, seconds int) message.ChangeVisibilityInput {
|
||||
return message.ChangeVisibilityInput{Account: acct, Region: region, Queue: name, ReceiptHandle: handle, VisibilityTimeout: seconds}
|
||||
}
|
||||
|
||||
func TestChangeVisibilityExtendsDeadline(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", map[string]string{"VisibilityTimeout": "30"})
|
||||
m := e.receiveOne(t, "orders")
|
||||
before, _ := e.state(t, q, m.ReceiptHandle)
|
||||
|
||||
e.clock.advance(10 * time.Second)
|
||||
require.NoError(t, e.changer(nil).ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 120)))
|
||||
st, _ := e.state(t, q, m.ReceiptHandle)
|
||||
assert.EqualValues(t, fixedNow.Add(130*time.Second).UnixMilli(), st["d"], "deadline is now plus the timeout")
|
||||
assert.Equal(t, before["g"], st["g"], "the generation is unchanged")
|
||||
assert.Equal(t, before["r"], st["r"])
|
||||
assert.Equal(t, message.StatusInflight, st["s"])
|
||||
|
||||
e.clock.advance(119 * time.Second)
|
||||
got, err := e.receiver(nil).Receive(context.Background(), recv("orders", 10))
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got, "still hidden past the queue's 30 seconds")
|
||||
|
||||
e.clock.advance(time.Second)
|
||||
got, err = e.receiver(nil).Receive(context.Background(), recv("orders", 10))
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, got, 1, "visible at the new deadline")
|
||||
}
|
||||
|
||||
// The cap is the issuing receive's time plus 12 hours, inclusive, and does
|
||||
// not reset when the deadline is extended.
|
||||
func TestChangeVisibilityTwelveHourCap(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", map[string]string{"VisibilityTimeout": "30"})
|
||||
m := e.receiveOne(t, "orders")
|
||||
c := e.changer(nil)
|
||||
|
||||
require.NoError(t, c.ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 43200)), "exactly the cap")
|
||||
st, _ := e.state(t, q, m.ReceiptHandle)
|
||||
assert.EqualValues(t, fixedNow.Add(12*time.Hour).UnixMilli(), st["d"])
|
||||
|
||||
e.clock.advance(time.Second)
|
||||
_, before := e.state(t, q, m.ReceiptHandle)
|
||||
require.ErrorIs(t, c.ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 43200)), message.ErrVisibilityCap)
|
||||
_, after := e.state(t, q, m.ReceiptHandle)
|
||||
assert.Equal(t, before, after, "a refused change writes nothing")
|
||||
|
||||
require.NoError(t, c.ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 43199)), "the remaining time is allowed")
|
||||
|
||||
e.clock.advance(6 * time.Hour)
|
||||
require.ErrorIs(t, c.ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 6*3600+1)), message.ErrVisibilityCap,
|
||||
"an earlier extension does not reset the cap")
|
||||
require.NoError(t, c.ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 6*3600-1)))
|
||||
}
|
||||
|
||||
func TestChangeVisibilityRefusesInvalidHandles(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
e.create(t, acct, "audit", nil)
|
||||
e.create(t, other, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
_, before := e.state(t, q, m.ReceiptHandle)
|
||||
for name, in := range map[string]message.ChangeVisibilityInput{
|
||||
"malformed": vis("orders", "not-a-handle", 60),
|
||||
"tampered": vis("orders", m.ReceiptHandle[:len(m.ReceiptHandle)-2]+"AA", 60),
|
||||
"another queue": vis("audit", m.ReceiptHandle, 60),
|
||||
"another account's": {Account: other, Region: region, Queue: "orders", ReceiptHandle: m.ReceiptHandle, VisibilityTimeout: 60},
|
||||
} {
|
||||
require.ErrorIs(t, e.changer(nil).ChangeVisibility(context.Background(), in), message.ErrReceiptInvalid, name)
|
||||
}
|
||||
_, after := e.state(t, q, m.ReceiptHandle)
|
||||
assert.Equal(t, before, after)
|
||||
}
|
||||
|
||||
// A delete landing between the read and the write wins; the change retries,
|
||||
// finds the tombstone and reports not in flight.
|
||||
func TestChangeVisibilityRacingDelete(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
racer := &deleteFirst{Store: e.store, d: e.deleter(nil), in: del("orders", m.ReceiptHandle)}
|
||||
require.ErrorIs(t, e.changer(racer).ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 60)), message.ErrNotInflight)
|
||||
require.True(t, racer.done)
|
||||
st, _ := e.state(t, q, m.ReceiptHandle)
|
||||
assert.Equal(t, message.StatusDeleted, st["s"], "the change never overwrote the tombstone")
|
||||
}
|
||||
|
||||
// Concurrent changes of one handle each apply, or fail closed once their
|
||||
// bounded retries are spent; the record ends at one of the requested
|
||||
// deadlines with its generation intact.
|
||||
func TestChangeVisibilityConcurrent(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
ok int
|
||||
)
|
||||
for i := range 6 {
|
||||
c := e.changer(nil)
|
||||
wg.Go(func() {
|
||||
err := c.ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 60+i))
|
||||
if err != nil {
|
||||
assert.ErrorIs(t, err, message.ErrUnavailable, "a lost race is never reported as settled")
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
ok++
|
||||
mu.Unlock()
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
assert.Positive(t, ok, "at least one change applies")
|
||||
st, _ := e.state(t, q, m.ReceiptHandle)
|
||||
assert.EqualValues(t, 1, st["g"])
|
||||
d := int64(st["d"].(float64))
|
||||
assert.GreaterOrEqual(t, d, fixedNow.Add(60*time.Second).UnixMilli())
|
||||
assert.LessOrEqual(t, d, fixedNow.Add(65*time.Second).UnixMilli())
|
||||
}
|
||||
|
||||
func TestChangeVisibilityUnsettledOutcomes(t *testing.T) {
|
||||
t.Run("lost reply, applied", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
f := &receiveFaults{Store: e.store, putFails: 1}
|
||||
require.ErrorIs(t, e.changer(f).ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 60)), message.ErrUnavailable)
|
||||
})
|
||||
t.Run("lost reply, not applied, not resent", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
_, before := e.state(t, q, m.ReceiptHandle)
|
||||
lost := &countLostPut{Store: e.store}
|
||||
require.ErrorIs(t, e.changer(lost).ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 60)), message.ErrUnavailable)
|
||||
assert.Equal(t, 1, lost.puts, "an unknown outcome is never resent")
|
||||
_, after := e.state(t, q, m.ReceiptHandle)
|
||||
assert.Equal(t, before, after)
|
||||
})
|
||||
t.Run("state read fails", func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
require.ErrorIs(t, e.changer(&failGetState{Store: e.store}).ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 60)), message.ErrUnavailable)
|
||||
})
|
||||
for name, value := range map[string]string{
|
||||
"unknown state": `{"v":1,"s":"archived","g":1}`,
|
||||
"newer version": `{"v":2,"s":"inflight","c":1,"g":1,"d":1,"r":1,"f":1}`,
|
||||
"moving": `{"v":1,"s":"moving","c":1,"g":1}`,
|
||||
"older than key": `{"v":1,"s":"inflight","c":1,"g":0,"d":1,"r":1,"f":1}`,
|
||||
"incomplete": `{"v":1,"s":"inflight","c":1,"g":1}`,
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
e := newReceiveEnv(t)
|
||||
q := e.create(t, acct, "orders", nil)
|
||||
m := e.receiveOne(t, "orders")
|
||||
rc, err := receiveKeys.Decode(m.ReceiptHandle, q.Namespace().Queue)
|
||||
require.NoError(t, err)
|
||||
_, rev := e.state(t, q, m.ReceiptHandle)
|
||||
rev, err = e.store.PutState(context.Background(), q.Namespace(), rc.Position, []byte(value), rev)
|
||||
require.NoError(t, err)
|
||||
err = e.changer(nil).ChangeVisibility(context.Background(), vis("orders", m.ReceiptHandle, 60))
|
||||
require.Error(t, err)
|
||||
assert.NotErrorIs(t, err, message.ErrNotInflight, "a record not ours is never read as settled")
|
||||
st, err := e.store.GetState(context.Background(), q.Namespace(), rc.Position)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rev, st.Revision, "nothing was written")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// deleteFirst runs a delete just before the first state write it sees.
|
||||
type deleteFirst struct {
|
||||
message.Store
|
||||
|
||||
d *message.Deleter
|
||||
in message.DeleteInput
|
||||
done bool
|
||||
}
|
||||
|
||||
func (s *deleteFirst) PutState(ctx context.Context, ns message.Namespace, pos message.Position, v []byte, rev message.Revision) (message.Revision, error) {
|
||||
if !s.done {
|
||||
s.done = true
|
||||
if err := s.d.Delete(ctx, s.in); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return s.Store.PutState(ctx, ns, pos, v, rev)
|
||||
}
|
||||
|
||||
// countLostPut drops every state write, counting attempts.
|
||||
type countLostPut struct {
|
||||
message.Store
|
||||
|
||||
puts int
|
||||
}
|
||||
|
||||
func (s *countLostPut) PutState(context.Context, message.Namespace, message.Position, []byte, message.Revision) (message.Revision, error) {
|
||||
s.puts++
|
||||
return 0, errLostAck
|
||||
}
|
||||
|
||||
var errLostAck = errors.New("acknowledgement lost")
|
||||
@@ -0,0 +1,167 @@
|
||||
package message_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
testutil "github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
lcAccount = "000000000001"
|
||||
lcRegion = "ap-southeast-2"
|
||||
)
|
||||
|
||||
// owners is the queue store and the four message owners a poller drives,
|
||||
// over a real JetStream message store.
|
||||
type owners struct {
|
||||
queues *queue.Store
|
||||
send *message.Sender
|
||||
recv *message.Receiver
|
||||
del *message.Deleter
|
||||
vis *message.VisibilityChanger
|
||||
}
|
||||
|
||||
func newOwners(t *testing.T) owners {
|
||||
t.Helper()
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ctx := context.Background()
|
||||
qs, err := queue.Open(ctx, js, 3, 2)
|
||||
require.NoError(t, err)
|
||||
ms, err := sqsjs.New(ctx, js, sqsjs.Config{MessageStreams: 3, StateBuckets: 2, DuplicateWindow: time.Minute})
|
||||
require.NoError(t, err)
|
||||
keys, err := message.NewReceiptKeys(1, map[uint32][]byte{1: bytes.Repeat([]byte{9}, message.MinReceiptKeyLen)})
|
||||
require.NoError(t, err)
|
||||
return owners{
|
||||
queues: qs,
|
||||
send: &message.Sender{Store: ms, Queues: qs},
|
||||
recv: &message.Receiver{Store: ms, Queues: qs, Keys: keys},
|
||||
del: &message.Deleter{Store: ms, Queues: qs, Keys: keys},
|
||||
vis: &message.VisibilityChanger{Store: ms, Queues: qs, Keys: keys},
|
||||
}
|
||||
}
|
||||
|
||||
func (o owners) create(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
_, err := o.queues.Create(context.Background(), queue.CreateRequest{Account: lcAccount, Region: lcRegion, Name: name,
|
||||
Attributes: map[string]string{"VisibilityTimeout": "30"}})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func (o owners) put(t *testing.T, name, body string) {
|
||||
t.Helper()
|
||||
_, err := o.send.Send(context.Background(), message.SendInput{Account: lcAccount, Region: lcRegion, Queue: name,
|
||||
Sender: "AIDAPOLLER", Body: []byte(body)})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func (o owners) receive(t *testing.T, name string, n, wait int) []message.Received {
|
||||
t.Helper()
|
||||
got, err := o.recv.Receive(context.Background(), message.ReceiveInput{Account: lcAccount, Region: lcRegion, Queue: name,
|
||||
MaxNumberOfMessages: &n, WaitTimeSeconds: &wait})
|
||||
require.NoError(t, err)
|
||||
return got
|
||||
}
|
||||
|
||||
func (o owners) changeVisibility(name, handle string, timeout int) error {
|
||||
return o.vis.ChangeVisibility(context.Background(), message.ChangeVisibilityInput{Account: lcAccount, Region: lcRegion,
|
||||
Queue: name, ReceiptHandle: handle, VisibilityTimeout: timeout})
|
||||
}
|
||||
|
||||
func (o owners) delete(name, handle string) error {
|
||||
return o.del.Delete(context.Background(), message.DeleteInput{Account: lcAccount, Region: lcRegion, Queue: name, ReceiptHandle: handle})
|
||||
}
|
||||
|
||||
// A poller receives a batch, returns one record for retry and deletes the
|
||||
// rest; the retried record comes back with a new handle and a higher count.
|
||||
func TestPollerLifecycle(t *testing.T) {
|
||||
o := newOwners(t)
|
||||
o.create(t, "jobs")
|
||||
o.put(t, "jobs", "a")
|
||||
o.put(t, "jobs", "b")
|
||||
|
||||
first := o.receive(t, "jobs", 10, 0)
|
||||
require.Len(t, first, 2)
|
||||
for _, m := range first {
|
||||
assert.Equal(t, uint64(1), m.ReceiveCount)
|
||||
assert.Equal(t, "AIDAPOLLER", m.SenderID)
|
||||
assert.False(t, m.FirstReceive.IsZero())
|
||||
}
|
||||
assert.Empty(t, o.receive(t, "jobs", 10, 0), "both are in flight")
|
||||
|
||||
retry, done := first[0], first[1]
|
||||
require.NoError(t, o.changeVisibility("jobs", retry.ReceiptHandle, 0))
|
||||
require.NoError(t, o.delete("jobs", done.ReceiptHandle))
|
||||
|
||||
again := o.receive(t, "jobs", 10, 0)
|
||||
require.Len(t, again, 1, "only the released record is redelivered")
|
||||
assert.Equal(t, retry.MessageID, again[0].MessageID)
|
||||
assert.Equal(t, retry.Body, again[0].Body)
|
||||
assert.Equal(t, uint64(2), again[0].ReceiveCount)
|
||||
assert.Equal(t, retry.FirstReceive, again[0].FirstReceive, "first receive is kept across redelivery")
|
||||
assert.NotEqual(t, retry.ReceiptHandle, again[0].ReceiptHandle)
|
||||
|
||||
assert.Error(t, o.changeVisibility("jobs", retry.ReceiptHandle, 0), "the superseded handle cannot release the new receive")
|
||||
|
||||
require.NoError(t, o.delete("jobs", again[0].ReceiptHandle))
|
||||
require.NoError(t, o.delete("jobs", again[0].ReceiptHandle), "repeating a delete succeeds")
|
||||
assert.Error(t, o.changeVisibility("jobs", again[0].ReceiptHandle, 10), "a deleted message is no longer in flight")
|
||||
assert.Empty(t, o.receive(t, "jobs", 10, 0))
|
||||
}
|
||||
|
||||
// Extending visibility keeps a slow record hidden from other pollers.
|
||||
func TestExtendVisibilityKeepsMessageHidden(t *testing.T) {
|
||||
o := newOwners(t)
|
||||
o.create(t, "slow")
|
||||
o.put(t, "slow", "work")
|
||||
got := o.receive(t, "slow", 1, 0)
|
||||
require.Len(t, got, 1)
|
||||
|
||||
require.NoError(t, o.changeVisibility("slow", got[0].ReceiptHandle, 600))
|
||||
assert.Empty(t, o.receive(t, "slow", 10, 0))
|
||||
assert.ErrorIs(t, o.changeVisibility("slow", got[0].ReceiptHandle, 43201), message.ErrInvalidParameter)
|
||||
}
|
||||
|
||||
// A handle only acts on the queue that issued it.
|
||||
func TestHandleIsBoundToItsQueue(t *testing.T) {
|
||||
o := newOwners(t)
|
||||
o.create(t, "a")
|
||||
o.create(t, "b")
|
||||
o.put(t, "a", "x")
|
||||
o.put(t, "b", "y")
|
||||
fromA := o.receive(t, "a", 1, 0)
|
||||
require.Len(t, fromA, 1)
|
||||
|
||||
assert.ErrorIs(t, o.delete("b", fromA[0].ReceiptHandle), message.ErrReceiptInvalid)
|
||||
assert.ErrorIs(t, o.changeVisibility("b", fromA[0].ReceiptHandle, 0), message.ErrReceiptInvalid)
|
||||
assert.ErrorIs(t, o.delete("a", "not-a-handle"), message.ErrReceiptInvalid)
|
||||
got := o.receive(t, "b", 1, 0)
|
||||
require.Len(t, got, 1, "queue b's message was untouched")
|
||||
assert.Equal(t, []byte("y"), got[0].Body)
|
||||
}
|
||||
|
||||
// A long poll returns waiting messages at once and an empty poll waits.
|
||||
func TestLongPoll(t *testing.T) {
|
||||
o := newOwners(t)
|
||||
o.create(t, "events")
|
||||
|
||||
start := time.Now()
|
||||
assert.Empty(t, o.receive(t, "events", 10, 1))
|
||||
assert.GreaterOrEqual(t, time.Since(start), time.Second)
|
||||
|
||||
o.put(t, "events", "ready")
|
||||
start = time.Now()
|
||||
got := o.receive(t, "events", 10, 20)
|
||||
require.Len(t, got, 1)
|
||||
assert.Less(t, time.Since(start), 10*time.Second, "a waiting message ends the poll early")
|
||||
|
||||
_, err := o.recv.Receive(context.Background(), message.ReceiveInput{Account: lcAccount, Region: lcRegion, Queue: "missing"})
|
||||
assert.ErrorIs(t, err, message.ErrQueueDoesNotExist)
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package message
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
)
|
||||
|
||||
// ErrReceiptInvalid is a receipt handle that is malformed, unauthenticated,
|
||||
// signed by an unknown key or issued for another queue. It never carries
|
||||
// the handle.
|
||||
var ErrReceiptInvalid = errors.New("message: receipt handle is invalid")
|
||||
|
||||
const (
|
||||
// ReceiptVersion is the handle layout this release signs and accepts.
|
||||
ReceiptVersion byte = 1
|
||||
// MinReceiptKeyLen is the shortest HMAC-SHA256 key a ring accepts.
|
||||
MinReceiptKeyLen = 32
|
||||
// MaxReceiptHandleLen bounds an encoded handle before it is decoded.
|
||||
MaxReceiptHandleLen = 1024
|
||||
|
||||
receiptMACLen = sha256.Size
|
||||
// version, key ID, queue ID length, position, generation.
|
||||
receiptFixedLen = 1 + 4 + 1 + 8 + 8
|
||||
)
|
||||
|
||||
var receiptEncoding = base64.RawURLEncoding.Strict()
|
||||
|
||||
// Receipt is what a handle binds: the queue, the message and the receive
|
||||
// attempt that issued it (ADR-0012:S4).
|
||||
type Receipt struct {
|
||||
Queue QueueID
|
||||
Position Position
|
||||
Generation uint64
|
||||
}
|
||||
|
||||
// ReceiptKeys signs handles with one key and verifies them with any key it
|
||||
// holds, so a rotation accepts the previous key. Key material is injected;
|
||||
// it never appears in its formatted or logged value.
|
||||
type ReceiptKeys struct {
|
||||
signing uint32
|
||||
keys map[uint32][]byte
|
||||
}
|
||||
|
||||
// NewReceiptKeys builds a ring that signs with the key named signing and
|
||||
// verifies with every key given. Each key is at least MinReceiptKeyLen.
|
||||
func NewReceiptKeys(signing uint32, keys map[uint32][]byte) (*ReceiptKeys, error) {
|
||||
if _, ok := keys[signing]; !ok {
|
||||
return nil, fmt.Errorf("%w: signing key %d not in ring", ErrInvalidArgument, signing)
|
||||
}
|
||||
held := make(map[uint32][]byte, len(keys))
|
||||
for id, k := range keys {
|
||||
if len(k) < MinReceiptKeyLen {
|
||||
return nil, fmt.Errorf("%w: receipt key %d shorter than %d bytes", ErrInvalidArgument, id, MinReceiptKeyLen)
|
||||
}
|
||||
held[id] = append([]byte(nil), k...)
|
||||
}
|
||||
return &ReceiptKeys{signing: signing, keys: held}, nil
|
||||
}
|
||||
|
||||
// Check reports whether the ring holds a usable signing key; a zero-value
|
||||
// or nil ring has none and must not sign.
|
||||
func (r *ReceiptKeys) Check() error {
|
||||
if r == nil || len(r.keys[r.signing]) < MinReceiptKeyLen {
|
||||
return fmt.Errorf("%w: receipt ring has no usable signing key", ErrInvalidArgument)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// String names the signing key ID and the key count, never key bytes.
|
||||
func (r *ReceiptKeys) String() string {
|
||||
return fmt.Sprintf("ReceiptKeys{signing: %d, keys: %d}", r.signing, len(r.keys))
|
||||
}
|
||||
|
||||
// GoString is String, so %#v cannot print key bytes.
|
||||
func (r *ReceiptKeys) GoString() string { return r.String() }
|
||||
|
||||
// LogValue logs the signing key ID and key count only.
|
||||
func (r *ReceiptKeys) LogValue() slog.Value {
|
||||
return slog.GroupValue(slog.Any("signing", r.signing), slog.Int("keys", len(r.keys)))
|
||||
}
|
||||
|
||||
// Encode issues the handle for a receive. The queue must be a valid ID and
|
||||
// the position and generation non-zero.
|
||||
func (r *ReceiptKeys) Encode(rc Receipt) (string, error) {
|
||||
if err := r.Check(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := (Namespace{Queue: rc.Queue}).Validate(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if rc.Position == 0 || rc.Generation == 0 {
|
||||
return "", fmt.Errorf("%w: receipt needs a position and a generation", ErrInvalidArgument)
|
||||
}
|
||||
qlen := len(rc.Queue)
|
||||
if qlen > math.MaxUint8 {
|
||||
return "", fmt.Errorf("%w: queue ID too long for a receipt", ErrInvalidArgument)
|
||||
}
|
||||
body := make([]byte, 0, receiptFixedLen+qlen+receiptMACLen)
|
||||
body = append(body, ReceiptVersion)
|
||||
body = binary.BigEndian.AppendUint32(body, r.signing)
|
||||
body = append(body, byte(qlen))
|
||||
body = append(body, rc.Queue...)
|
||||
body = binary.BigEndian.AppendUint64(body, uint64(rc.Position))
|
||||
body = binary.BigEndian.AppendUint64(body, rc.Generation)
|
||||
body = append(body, receiptMAC(r.keys[r.signing], body)...)
|
||||
return receiptEncoding.EncodeToString(body), nil
|
||||
}
|
||||
|
||||
// Decode authenticates a handle and returns what it binds, refusing one
|
||||
// issued for any queue but queue, including an earlier queue of the same
|
||||
// name. Every refusal is ErrReceiptInvalid.
|
||||
func (r *ReceiptKeys) Decode(handle string, queue QueueID) (Receipt, error) {
|
||||
if len(handle) > MaxReceiptHandleLen {
|
||||
return Receipt{}, ErrReceiptInvalid
|
||||
}
|
||||
b, err := receiptEncoding.DecodeString(handle)
|
||||
if err != nil || len(b) < receiptFixedLen+1+receiptMACLen || b[0] != ReceiptVersion {
|
||||
return Receipt{}, ErrReceiptInvalid
|
||||
}
|
||||
key, ok := r.keys[binary.BigEndian.Uint32(b[1:5])]
|
||||
qlen := int(b[5])
|
||||
if !ok || len(b) != receiptFixedLen+qlen+receiptMACLen {
|
||||
return Receipt{}, ErrReceiptInvalid
|
||||
}
|
||||
signed, mac := b[:len(b)-receiptMACLen], b[len(b)-receiptMACLen:]
|
||||
if !hmac.Equal(mac, receiptMAC(key, signed)) {
|
||||
return Receipt{}, ErrReceiptInvalid
|
||||
}
|
||||
rc := Receipt{
|
||||
Queue: QueueID(b[6 : 6+qlen]),
|
||||
Position: Position(binary.BigEndian.Uint64(b[6+qlen:])),
|
||||
Generation: binary.BigEndian.Uint64(b[14+qlen:]),
|
||||
}
|
||||
if rc.Queue != queue || rc.Position == 0 || rc.Generation == 0 {
|
||||
return Receipt{}, ErrReceiptInvalid
|
||||
}
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
// receiptMAC signs the version, queue ID, position and generation; the key
|
||||
// ID only selects the key.
|
||||
func receiptMAC(key, signed []byte) []byte {
|
||||
m := hmac.New(sha256.New, key)
|
||||
m.Write(signed[:1])
|
||||
m.Write(signed[5:])
|
||||
return m.Sum(nil)
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package message_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
keyA = bytes.Repeat([]byte{0xA1}, message.MinReceiptKeyLen)
|
||||
keyB = bytes.Repeat([]byte{0xB2}, message.MinReceiptKeyLen)
|
||||
)
|
||||
|
||||
func ring(t *testing.T, signing uint32, keys map[uint32][]byte) *message.ReceiptKeys {
|
||||
t.Helper()
|
||||
r, err := message.NewReceiptKeys(signing, keys)
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}
|
||||
|
||||
var receiptA = message.Receipt{Queue: "QUEUEAAAA", Position: 42, Generation: 3}
|
||||
|
||||
func TestReceiptRoundTrip(t *testing.T) {
|
||||
r := ring(t, 1, map[uint32][]byte{1: keyA})
|
||||
for _, rc := range []message.Receipt{
|
||||
receiptA,
|
||||
{Queue: "q", Position: 1, Generation: 1},
|
||||
{Queue: message.QueueID(strings.Repeat("Z", message.MaxQueueIDLen)), Position: ^message.Position(0), Generation: ^uint64(0)},
|
||||
} {
|
||||
h, err := r.Encode(rc)
|
||||
require.NoError(t, err)
|
||||
assert.LessOrEqual(t, len(h), message.MaxReceiptHandleLen)
|
||||
got, err := r.Decode(h, rc.Queue)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, rc, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Changing any byte, including the version and key ID, leaves a handle
|
||||
// that no key in the ring accepts.
|
||||
func TestReceiptRejectsEveryTamperedByte(t *testing.T) {
|
||||
r := ring(t, 1, map[uint32][]byte{1: keyA, 2: keyB})
|
||||
h, err := r.Encode(receiptA)
|
||||
require.NoError(t, err)
|
||||
raw, err := base64.RawURLEncoding.DecodeString(h)
|
||||
require.NoError(t, err)
|
||||
for i := range raw {
|
||||
for _, flip := range []byte{0x01, 0x80} {
|
||||
b := bytes.Clone(raw)
|
||||
b[i] ^= flip
|
||||
_, err := r.Decode(base64.RawURLEncoding.EncodeToString(b), receiptA.Queue)
|
||||
require.ErrorIs(t, err, message.ErrReceiptInvalid, "byte %d ^ %#x", i, flip)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A forger who knows the layout but not the key cannot mint a handle, even
|
||||
// for an existing message and the current generation.
|
||||
func TestReceiptRejectsForgery(t *testing.T) {
|
||||
r := ring(t, 1, map[uint32][]byte{1: keyA})
|
||||
attacker := ring(t, 1, map[uint32][]byte{1: keyB})
|
||||
h, err := attacker.Encode(receiptA)
|
||||
require.NoError(t, err)
|
||||
_, err = r.Decode(h, receiptA.Queue)
|
||||
require.ErrorIs(t, err, message.ErrReceiptInvalid)
|
||||
}
|
||||
|
||||
func TestReceiptRejectsMalformed(t *testing.T) {
|
||||
r := ring(t, 1, map[uint32][]byte{1: keyA})
|
||||
h, err := r.Encode(receiptA)
|
||||
require.NoError(t, err)
|
||||
raw, err := base64.RawURLEncoding.DecodeString(h)
|
||||
require.NoError(t, err)
|
||||
lenAt := func(n byte) string {
|
||||
b := bytes.Clone(raw)
|
||||
b[5] = n
|
||||
return base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
for name, bad := range map[string]string{
|
||||
"empty": "",
|
||||
"not base64": "!!!!",
|
||||
"padded": h + "==",
|
||||
"whitespace": h[:8] + " " + h[8:],
|
||||
"truncated": h[:len(h)-4],
|
||||
"extended": h + "AAAA",
|
||||
"too long": strings.Repeat("A", message.MaxReceiptHandleLen+1),
|
||||
"zero queue len": lenAt(0),
|
||||
"queue len short": lenAt(byte(len(receiptA.Queue) - 1)),
|
||||
"queue len long": lenAt(byte(len(receiptA.Queue) + 1)),
|
||||
"only fixed bytes": base64.RawURLEncoding.EncodeToString(raw[:6]),
|
||||
} {
|
||||
_, err := r.Decode(bad, receiptA.Queue)
|
||||
require.ErrorIs(t, err, message.ErrReceiptInvalid, name)
|
||||
}
|
||||
}
|
||||
|
||||
// forge builds a handle with a valid MAC for any version, so a test can
|
||||
// separate the version check from the MAC check.
|
||||
func forge(key []byte, v byte, keyID uint32, rc message.Receipt) string {
|
||||
b := []byte{v}
|
||||
b = binary.BigEndian.AppendUint32(b, keyID)
|
||||
b = append(b, byte(len(rc.Queue)))
|
||||
b = append(b, rc.Queue...)
|
||||
b = binary.BigEndian.AppendUint64(b, uint64(rc.Position))
|
||||
b = binary.BigEndian.AppendUint64(b, rc.Generation)
|
||||
mac := hmacSHA256(key, append([]byte{v}, b[5:]...))
|
||||
return base64.RawURLEncoding.EncodeToString(append(b, mac...))
|
||||
}
|
||||
|
||||
// Rotation: the new ring signs with key 2 and still accepts key 1's handles
|
||||
// until key 1 is dropped; a key the ring never held is refused.
|
||||
func TestReceiptKeyRotation(t *testing.T) {
|
||||
old := ring(t, 1, map[uint32][]byte{1: keyA})
|
||||
issued, err := old.Encode(receiptA)
|
||||
require.NoError(t, err)
|
||||
|
||||
rotated := ring(t, 2, map[uint32][]byte{1: keyA, 2: keyB})
|
||||
got, err := rotated.Decode(issued, receiptA.Queue)
|
||||
require.NoError(t, err, "the previous key is accepted")
|
||||
assert.Equal(t, receiptA, got)
|
||||
fresh, err := rotated.Encode(receiptA)
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, issued, fresh)
|
||||
|
||||
_, err = old.Decode(fresh, receiptA.Queue)
|
||||
require.ErrorIs(t, err, message.ErrReceiptInvalid, "a ring without key 2 refuses its handles")
|
||||
|
||||
retired := ring(t, 2, map[uint32][]byte{2: keyB})
|
||||
_, err = retired.Decode(issued, receiptA.Queue)
|
||||
require.ErrorIs(t, err, message.ErrReceiptInvalid, "a dropped key is refused")
|
||||
|
||||
// Same bytes under another key ID: the key ID selects the key, so it
|
||||
// cannot redirect verification to a key the handle was not signed with.
|
||||
_, err = rotated.Decode(forge(keyA, message.ReceiptVersion, 2, receiptA), receiptA.Queue)
|
||||
require.ErrorIs(t, err, message.ErrReceiptInvalid)
|
||||
}
|
||||
|
||||
// A queue deleted and recreated under its name gets a new queue ID, so the
|
||||
// predecessor's handles are refused; so is any other queue's handle.
|
||||
func TestReceiptRejectsAnotherQueue(t *testing.T) {
|
||||
r := ring(t, 1, map[uint32][]byte{1: keyA})
|
||||
h, err := r.Encode(receiptA)
|
||||
require.NoError(t, err)
|
||||
for _, q := range []message.QueueID{"QUEUEBBBB", "QUEUEAAAAA", "QUEUEAAA", "queueaaaa", ""} {
|
||||
_, err := r.Decode(h, q)
|
||||
require.ErrorIs(t, err, message.ErrReceiptInvalid, "queue %q", q)
|
||||
}
|
||||
}
|
||||
|
||||
// Neither keys nor handles reach formatted output, logs or errors.
|
||||
func TestReceiptSecretsStayOutOfOutput(t *testing.T) {
|
||||
r := ring(t, 1, map[uint32][]byte{1: keyA, 2: keyB})
|
||||
h, err := r.Encode(receiptA)
|
||||
require.NoError(t, err)
|
||||
|
||||
var logs bytes.Buffer
|
||||
slog.New(slog.NewJSONHandler(&logs, nil)).Info("ring", "keys", r)
|
||||
_, decErr := r.Decode(h+"x", receiptA.Queue)
|
||||
require.Error(t, decErr)
|
||||
|
||||
out := strings.Join([]string{
|
||||
fmt.Sprint(r), fmt.Sprintf("%v %+v %#v %s", r, r, r, r), logs.String(), decErr.Error(),
|
||||
}, "\n")
|
||||
for _, k := range [][]byte{keyA, keyB} {
|
||||
for _, enc := range []string{string(k), hex.EncodeToString(k), fmt.Sprintf("%d", k), fmt.Sprintf("%#v", k), base64.StdEncoding.EncodeToString(k)} {
|
||||
assert.NotContains(t, out, enc)
|
||||
}
|
||||
}
|
||||
assert.NotContains(t, out, "0xa1, 0xa1")
|
||||
assert.NotContains(t, out, "0xb2, 0xb2")
|
||||
assert.NotContains(t, out, h)
|
||||
assert.Contains(t, logs.String(), `"signing":1`)
|
||||
}
|
||||
|
||||
func hmacSHA256(key, data []byte) []byte {
|
||||
m := hmac.New(sha256.New, key)
|
||||
m.Write(data)
|
||||
return m.Sum(nil)
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
package message
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Receive limits. MaxVisibilitySeconds is the queue and per-call ceiling.
|
||||
const (
|
||||
MaxReceiveMessages = 10
|
||||
MaxVisibilitySeconds = 43200
|
||||
// ReceiveScanPage and ReceiveScanPages bound one receive's scan work:
|
||||
// at most ReceiveScanPages pages of ReceiveScanPage positions.
|
||||
ReceiveScanPage = 100
|
||||
ReceiveScanPages = 4
|
||||
// MaxWaitSeconds is the longest long poll. LongPollRecheck bounds how
|
||||
// long a long poll waits between polls without an append to wake it.
|
||||
MaxWaitSeconds = 20
|
||||
LongPollRecheck = time.Second
|
||||
)
|
||||
|
||||
// errScanCut marks a poll whose scan did not complete.
|
||||
var errScanCut = errors.New("scan queue")
|
||||
|
||||
// ReceiveTarget is an active queue resolved for a receive.
|
||||
type ReceiveTarget struct {
|
||||
Namespace Namespace
|
||||
Account string
|
||||
// VisibilityTimeout, MessageRetentionPeriod and ReceiveWaitTime are the
|
||||
// queue's current values in seconds.
|
||||
VisibilityTimeout int
|
||||
MessageRetentionPeriod int
|
||||
ReceiveWaitTime int
|
||||
}
|
||||
|
||||
// ReceiveQueues resolves an active queue for a receive; it is the queue
|
||||
// owner's capability, injected so this package never reads queue records.
|
||||
type ReceiveQueues interface {
|
||||
ReceiveTarget(ctx context.Context, account, region, name string) (ReceiveTarget, error)
|
||||
}
|
||||
|
||||
// ReceiveInput is one ReceiveMessage request after wire decoding.
|
||||
type ReceiveInput struct {
|
||||
Account, Region, Queue string
|
||||
// MaxNumberOfMessages is 1 to MaxReceiveMessages, default 1.
|
||||
MaxNumberOfMessages *int
|
||||
// VisibilityTimeout overrides the queue's for this receive when set.
|
||||
VisibilityTimeout *int
|
||||
// WaitTimeSeconds overrides the queue's ReceiveMessageWaitTimeSeconds; an
|
||||
// effective wait above 0 is a long poll of that many seconds.
|
||||
WaitTimeSeconds *int
|
||||
}
|
||||
|
||||
// Received is one message taken in flight by a receive, with its handle.
|
||||
type Received struct {
|
||||
MessageID string
|
||||
ReceiptHandle string
|
||||
Body []byte
|
||||
MD5OfBody string
|
||||
// Attributes and MD5OfMessageAttributes are empty for a message sent
|
||||
// without attributes.
|
||||
Attributes Attributes
|
||||
MD5OfMessageAttributes string
|
||||
SenderID string
|
||||
SentAt time.Time
|
||||
ReceiveCount uint64
|
||||
FirstReceive time.Time
|
||||
Deadline time.Time
|
||||
}
|
||||
|
||||
// Receiver is the ReceiveMessage owner core, short and long poll.
|
||||
type Receiver struct {
|
||||
Store Store
|
||||
Queues ReceiveQueues
|
||||
Keys *ReceiptKeys
|
||||
// Now defaults to the wall clock.
|
||||
Now func() time.Time
|
||||
// Recheck is the longest a long poll waits between polls without an
|
||||
// append to wake it; zero means LongPollRecheck.
|
||||
Recheck time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
// passes resumes each queue's scan pass across receives, so repeated
|
||||
// polls cover the shard beyond the first page budget. It is a hint
|
||||
// only: a restart begins a new pass and every claim is still checked.
|
||||
passes map[QueueID]ScanCursor
|
||||
}
|
||||
|
||||
// Receive returns up to MaxNumberOfMessages messages taken in flight, waiting
|
||||
// up to the effective wait for one. It claims only once queue and signer are
|
||||
// known; an unsettled scan or claim with nothing claimed is ErrUnavailable.
|
||||
func (r *Receiver) Receive(ctx context.Context, in ReceiveInput) ([]Received, error) {
|
||||
if r == nil || r.Store == nil || r.Queues == nil {
|
||||
return nil, fmt.Errorf("%w: receiver is not configured", ErrUnavailable)
|
||||
}
|
||||
if err := r.Keys.Check(); err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
want, visOverride, err := validateReceive(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t, err := r.Queues.ReceiveTarget(ctx, in.Account, in.Region, in.Queue)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vis, retention, wait, err := receiveTiming(in.Account, t, visOverride, in.WaitTimeSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if wait == 0 {
|
||||
out, _, err := r.poll(ctx, t.Namespace, want, vis, retention)
|
||||
return out, err
|
||||
}
|
||||
return r.longPoll(ctx, t.Namespace, want, vis, retention, wait)
|
||||
}
|
||||
|
||||
// longPoll repeats the bounded poll until a message is claimed, the wait
|
||||
// ends or ctx is done. The wake feed is open before the first poll, so an
|
||||
// append landing between a poll and the wait still wakes it. All waiting
|
||||
// work is bounded by the wait; empty is returned only after a full poll.
|
||||
func (r *Receiver) longPoll(ctx context.Context, ns Namespace, want int, vis, retention, wait time.Duration) ([]Received, error) {
|
||||
wctx, cancel := context.WithTimeout(ctx, wait)
|
||||
defer cancel()
|
||||
feed, err := r.Store.Feed(wctx, ns.MessageShard, math.MaxUint64)
|
||||
if err != nil {
|
||||
if cerr := ctx.Err(); cerr != nil {
|
||||
return nil, cerr
|
||||
}
|
||||
return nil, fmt.Errorf("%w: open wake feed: %w", ErrUnavailable, err)
|
||||
}
|
||||
var (
|
||||
wake = make(chan struct{}, 1)
|
||||
done = make(chan struct{})
|
||||
closing atomic.Bool
|
||||
feedErr error
|
||||
)
|
||||
go func() {
|
||||
defer close(done)
|
||||
for {
|
||||
ch, err := feed.Next(wctx)
|
||||
if err != nil {
|
||||
if wctx.Err() == nil && !closing.Load() {
|
||||
feedErr = err
|
||||
}
|
||||
return
|
||||
}
|
||||
if ch.Namespace == ns {
|
||||
select {
|
||||
case wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
var once sync.Once
|
||||
// stop releases the feed and reports a failure it saw before release.
|
||||
stop := func() error {
|
||||
once.Do(func() {
|
||||
closing.Store(true)
|
||||
if err := feed.Close(); err != nil {
|
||||
slog.WarnContext(ctx, "SQS receive wake feed close failed", "err", err)
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
})
|
||||
if feedErr != nil {
|
||||
return fmt.Errorf("%w: wake feed: %w", ErrUnavailable, feedErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
defer func() { _ = stop() }()
|
||||
checked := false
|
||||
for {
|
||||
out, next, err := r.poll(wctx, ns, want, vis, retention)
|
||||
if len(out) > 0 {
|
||||
return out, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, r.endErr(ctx, wctx, err, checked, stop)
|
||||
}
|
||||
checked = true
|
||||
nap := r.recheck()
|
||||
// A delayed or in-flight message becomes visible without an append,
|
||||
// so the wait is cut to the next known visibility or the recheck.
|
||||
if !next.IsZero() {
|
||||
nap = min(nap, max(next.Sub(r.now()), 0))
|
||||
}
|
||||
timer := time.NewTimer(nap)
|
||||
select {
|
||||
case <-wctx.Done():
|
||||
case <-done:
|
||||
case <-wake:
|
||||
timer.Stop()
|
||||
continue
|
||||
case <-timer.C:
|
||||
continue
|
||||
}
|
||||
timer.Stop()
|
||||
if cerr := ctx.Err(); cerr != nil {
|
||||
return nil, cerr
|
||||
}
|
||||
return nil, stop()
|
||||
}
|
||||
}
|
||||
|
||||
// endErr settles a poll that failed with nothing claimed. Caller
|
||||
// cancellation wins; only a scan ended by the wait's deadline, after a full
|
||||
// poll and with a healthy feed, is an empty result; anything else is
|
||||
// unavailable.
|
||||
func (r *Receiver) endErr(ctx, wctx context.Context, err error, checked bool, stop func() error) error {
|
||||
if cerr := ctx.Err(); cerr != nil {
|
||||
return cerr
|
||||
}
|
||||
if wctx.Err() != nil && checked && errors.Is(err, errScanCut) && errors.Is(err, context.DeadlineExceeded) {
|
||||
return stop()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// poll is one bounded scan-and-claim pass. It returns every message it
|
||||
// claimed and the earliest delay end it saw; a scan or claim it could not
|
||||
// settle is ErrUnavailable only when it claimed nothing.
|
||||
func (r *Receiver) poll(ctx context.Context, ns Namespace, want int, vis, retention time.Duration) ([]Received, time.Time, error) {
|
||||
claimer := &Claimer{Store: r.Store, Now: r.Now}
|
||||
var (
|
||||
out []Received
|
||||
unsettled error
|
||||
next time.Time
|
||||
)
|
||||
cur := r.resume(ns.Queue)
|
||||
// fresh is true once a pass begun in this poll is under way; a second
|
||||
// completed pass would only repeat it, so the poll stops there.
|
||||
fresh := cur == ScanCursor{}
|
||||
for pages := 0; pages < ReceiveScanPages && len(out) < want; pages++ {
|
||||
page, err := r.Store.Scan(ctx, ns.MessageShard, cur, ReceiveScanPage)
|
||||
if err != nil {
|
||||
if unsettled == nil {
|
||||
unsettled = fmt.Errorf("%w: %w: %w", ErrUnavailable, errScanCut, err)
|
||||
}
|
||||
break
|
||||
}
|
||||
full := false
|
||||
for _, ch := range page.Changes {
|
||||
if len(out) == want {
|
||||
// Resume the pass at the first change not yet tried.
|
||||
cur, full = ScanCursor{After: ch.Cursor - 1, Bound: page.Next.Bound}, true
|
||||
break
|
||||
}
|
||||
if ch.Namespace != ns {
|
||||
continue
|
||||
}
|
||||
if at, ok := r.delayedUntil(ch); ok {
|
||||
if next.IsZero() || at.Before(next) {
|
||||
next = at
|
||||
}
|
||||
continue
|
||||
}
|
||||
got, err := claimer.Claim(ctx, ClaimInput{Namespace: ns, Position: ch.Position, Visibility: vis, Retention: retention})
|
||||
switch {
|
||||
case err == nil:
|
||||
rc, err := r.received(ns.Queue, got)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
out = append(out, rc)
|
||||
case errors.Is(err, ErrNotVisible), errors.Is(err, ErrGone), errors.Is(err, ErrClaimLost), errors.Is(err, ErrLeaseExpired):
|
||||
default:
|
||||
if unsettled == nil {
|
||||
unsettled = err
|
||||
}
|
||||
}
|
||||
}
|
||||
if full {
|
||||
break
|
||||
}
|
||||
cur = page.Next
|
||||
if page.Done {
|
||||
cur = ScanCursor{}
|
||||
if fresh {
|
||||
break
|
||||
}
|
||||
fresh = true
|
||||
}
|
||||
}
|
||||
r.save(ns.Queue, cur)
|
||||
if len(out) == 0 && unsettled != nil {
|
||||
return nil, time.Time{}, unsettled
|
||||
}
|
||||
return out, next, nil
|
||||
}
|
||||
|
||||
// ValidateReceive checks the request alone, so an adapter can refuse it
|
||||
// before authorizing; the queue's wait default is checked at Receive.
|
||||
func ValidateReceive(in ReceiveInput) error {
|
||||
_, _, err := validateReceive(in)
|
||||
return err
|
||||
}
|
||||
|
||||
func validateReceive(in ReceiveInput) (want int, vis *int, err error) {
|
||||
want = 1
|
||||
if in.MaxNumberOfMessages != nil {
|
||||
want = *in.MaxNumberOfMessages
|
||||
if want < 1 || want > MaxReceiveMessages {
|
||||
return 0, nil, fmt.Errorf("%w: MaxNumberOfMessages must be between 1 and %d", ErrInvalidParameter, MaxReceiveMessages)
|
||||
}
|
||||
}
|
||||
if in.VisibilityTimeout != nil && (*in.VisibilityTimeout < 0 || *in.VisibilityTimeout > MaxVisibilitySeconds) {
|
||||
return 0, nil, fmt.Errorf("%w: VisibilityTimeout must be between 0 and %d", ErrInvalidParameter, MaxVisibilitySeconds)
|
||||
}
|
||||
if in.WaitTimeSeconds != nil {
|
||||
if w := *in.WaitTimeSeconds; w < 0 || w > MaxWaitSeconds {
|
||||
return 0, nil, fmt.Errorf("%w: WaitTimeSeconds must be between 0 and %d", ErrInvalidParameter, MaxWaitSeconds)
|
||||
}
|
||||
}
|
||||
return want, in.VisibilityTimeout, nil
|
||||
}
|
||||
|
||||
// receiveTiming checks the resolved queue before any claim: a target for
|
||||
// another account or with unreadable timing is unavailable, not defaulted.
|
||||
// The wait is the request's WaitTimeSeconds, else the queue's default.
|
||||
func receiveTiming(account string, t ReceiveTarget, override, wait *int) (vis, retention, waitFor time.Duration, err error) {
|
||||
switch {
|
||||
case t.Account != account:
|
||||
return 0, 0, 0, fmt.Errorf("%w: resolved queue belongs to another account", ErrUnavailable)
|
||||
case t.VisibilityTimeout < 0 || t.VisibilityTimeout > MaxVisibilitySeconds:
|
||||
return 0, 0, 0, fmt.Errorf("%w: queue visibility timeout %d outside 0..%d", ErrUnavailable, t.VisibilityTimeout, MaxVisibilitySeconds)
|
||||
case t.MessageRetentionPeriod < 60 || t.MessageRetentionPeriod > 1209600:
|
||||
return 0, 0, 0, fmt.Errorf("%w: queue retention %d outside 60..1209600", ErrUnavailable, t.MessageRetentionPeriod)
|
||||
case t.ReceiveWaitTime < 0 || t.ReceiveWaitTime > MaxWaitSeconds:
|
||||
return 0, 0, 0, fmt.Errorf("%w: queue receive wait %d outside 0..%d", ErrUnavailable, t.ReceiveWaitTime, MaxWaitSeconds)
|
||||
}
|
||||
if err := t.Namespace.Validate(); err != nil {
|
||||
return 0, 0, 0, fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
secs, w := t.VisibilityTimeout, t.ReceiveWaitTime
|
||||
if override != nil {
|
||||
secs = *override
|
||||
}
|
||||
if wait != nil {
|
||||
w = *wait
|
||||
}
|
||||
return time.Duration(secs) * time.Second, time.Duration(t.MessageRetentionPeriod) * time.Second, time.Duration(w) * time.Second, nil
|
||||
}
|
||||
|
||||
// delayedUntil reports a still-delayed message's visibility, skipped
|
||||
// without a state read, only when all its metadata is valid; anything else
|
||||
// goes to the claim, which fails closed.
|
||||
func (r *Receiver) delayedUntil(ch Change) (time.Time, bool) {
|
||||
h, err := readHeaders(ch.Meta)
|
||||
if err != nil || h.vis <= r.now().UnixMilli() {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.UnixMilli(h.vis), true
|
||||
}
|
||||
|
||||
func (r *Receiver) received(q QueueID, c Claim) (Received, error) {
|
||||
h, err := r.Keys.Encode(Receipt{Queue: q, Position: c.Message.Position, Generation: c.Generation})
|
||||
if err != nil {
|
||||
return Received{}, fmt.Errorf("%w: sign receipt: %w", ErrUnavailable, err)
|
||||
}
|
||||
enq, _ := strconv.ParseInt(c.Message.Meta[MetaEnq], 10, 64)
|
||||
return Received{
|
||||
MessageID: c.MessageID, ReceiptHandle: h, Body: c.Message.Body,
|
||||
MD5OfBody: c.Message.Meta[MetaMD5], SenderID: c.Message.Meta[MetaSender],
|
||||
Attributes: c.Attributes, MD5OfMessageAttributes: c.Attributes.MD5(),
|
||||
SentAt: time.UnixMilli(enq).UTC(), ReceiveCount: c.ReceiveCount,
|
||||
FirstReceive: c.FirstReceive, Deadline: c.Deadline,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *Receiver) recheck() time.Duration {
|
||||
if r.Recheck > 0 {
|
||||
return r.Recheck
|
||||
}
|
||||
return LongPollRecheck
|
||||
}
|
||||
|
||||
func (r *Receiver) now() time.Time {
|
||||
if r.Now != nil {
|
||||
return r.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (r *Receiver) resume(q QueueID) ScanCursor {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.passes[q]
|
||||
}
|
||||
|
||||
func (r *Receiver) save(q QueueID, c ScanCursor) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if c == (ScanCursor{}) {
|
||||
delete(r.passes, q)
|
||||
return
|
||||
}
|
||||
if r.passes == nil {
|
||||
r.passes = make(map[QueueID]ScanCursor)
|
||||
}
|
||||
r.passes[q] = c
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
package message
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5" //nolint:gosec // MD5OfMessageBody is an AWS wire checksum, not a security control.
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// MetaVersion is the newest message schema this release reads. A send
|
||||
// writes MetaVersionPlain unless the message has attributes, so a reader that
|
||||
// predates attributes refuses such a message instead of misreading its body.
|
||||
const (
|
||||
MetaVersion = 2
|
||||
MetaVersionPlain = 1
|
||||
)
|
||||
|
||||
// MaxDelaySeconds is the largest per-message or queue DelaySeconds.
|
||||
const MaxDelaySeconds = 900
|
||||
|
||||
// MinMessageSize and MaxMessageSize bound a queue's MaximumMessageSize.
|
||||
const (
|
||||
MinMessageSize = 1024
|
||||
MaxMessageSize = 1 << 20
|
||||
)
|
||||
|
||||
// Metadata keys a send writes beside the body. They are the lifecycle
|
||||
// contract's Sqs-V, Sqs-Id, Sqs-Enq, Sqs-Vis, Sqs-Sender, Sqs-Md5, Sqs-Md5A
|
||||
// and Sqs-Acct.
|
||||
const (
|
||||
MetaV = "V"
|
||||
MetaID = "Id"
|
||||
MetaEnq = "Enq"
|
||||
MetaVis = "Vis"
|
||||
MetaSender = "Sender"
|
||||
MetaMD5 = "Md5"
|
||||
MetaMD5A = "Md5A"
|
||||
MetaAcct = "Acct"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrQueueDoesNotExist reports that the target is not an active queue.
|
||||
ErrQueueDoesNotExist = errors.New("message: queue does not exist")
|
||||
// ErrBodyRequired reports an absent or empty message body.
|
||||
ErrBodyRequired = errors.New("message: body is required")
|
||||
// ErrInvalidContents reports a body with characters SQS does not allow.
|
||||
ErrInvalidContents = errors.New("message: invalid message contents")
|
||||
// ErrInvalidParameter reports an out-of-range size or delay.
|
||||
ErrInvalidParameter = errors.New("message: invalid parameter value")
|
||||
// ErrUnsupported reports a send feature this release does not implement.
|
||||
ErrUnsupported = errors.New("message: unsupported")
|
||||
// ErrUnavailable reports that the append was not acknowledged; it may
|
||||
// still have been stored, and a retry can duplicate it.
|
||||
ErrUnavailable = errors.New("message: unavailable")
|
||||
)
|
||||
|
||||
// Target is an active queue resolved for a send.
|
||||
type Target struct {
|
||||
Namespace Namespace
|
||||
Account string
|
||||
// DelaySeconds and MaximumMessageSize are the queue's current values.
|
||||
DelaySeconds int
|
||||
MaximumMessageSize int
|
||||
}
|
||||
|
||||
// Queues resolves an active queue for a send; it is the queue owner's
|
||||
// capability, injected so this package never reads queue records itself.
|
||||
type Queues interface {
|
||||
SendTarget(ctx context.Context, account, region, name string) (Target, error)
|
||||
}
|
||||
|
||||
// SendInput is one SendMessage request after wire decoding.
|
||||
type SendInput struct {
|
||||
Account, Region, Queue string
|
||||
// Sender is the SenderId recorded with the message.
|
||||
Sender string
|
||||
Body []byte
|
||||
// DelaySeconds overrides the queue's delay when set.
|
||||
DelaySeconds *int
|
||||
// QueueID, when set, pins the queue: if the name now resolves to any
|
||||
// other queue the send is ErrQueueDoesNotExist and nothing is appended.
|
||||
QueueID QueueID
|
||||
|
||||
// MessageAttributes is a list so a repeated name reaches validation.
|
||||
MessageAttributes []Attribute
|
||||
MessageSystemAttributes map[string]AttributeValue
|
||||
// Presence alone is refused: an empty string is still a supplied member.
|
||||
MessageGroupID *string
|
||||
MessageDeduplicationID *string
|
||||
}
|
||||
|
||||
// SendResult is returned only after the append is acknowledged as durable.
|
||||
type SendResult struct {
|
||||
MessageID string
|
||||
MD5OfBody string
|
||||
// MD5OfMessageAttributes is empty when the message has no attributes.
|
||||
MD5OfMessageAttributes string
|
||||
Position Position
|
||||
EnqueuedAt time.Time
|
||||
VisibleAfter time.Time
|
||||
}
|
||||
|
||||
// Sender is the SendMessage owner core.
|
||||
type Sender struct {
|
||||
Store Store
|
||||
Queues Queues
|
||||
// Now and NewID default to the wall clock and a random UUID.
|
||||
Now func() time.Time
|
||||
NewID func() (string, error)
|
||||
}
|
||||
|
||||
// Send validates a message, resolves its queue and appends it durably. It
|
||||
// answers only after the store acknowledges the append; any append failure
|
||||
// is ErrUnavailable, never success.
|
||||
func (s *Sender) Send(ctx context.Context, in SendInput) (SendResult, error) {
|
||||
if s == nil || s.Store == nil || s.Queues == nil {
|
||||
return SendResult{}, fmt.Errorf("%w: sender is not configured", ErrUnavailable)
|
||||
}
|
||||
if err := validateRequest(in); err != nil {
|
||||
return SendResult{}, err
|
||||
}
|
||||
t, err := s.Queues.SendTarget(ctx, in.Account, in.Region, in.Queue)
|
||||
if err != nil {
|
||||
return SendResult{}, err
|
||||
}
|
||||
if t.Account != in.Account {
|
||||
return SendResult{}, fmt.Errorf("%w: resolved queue belongs to another account", ErrUnavailable)
|
||||
}
|
||||
if in.QueueID != "" && t.Namespace.Queue != in.QueueID {
|
||||
return SendResult{}, fmt.Errorf("%w: the queue name now resolves to another queue", ErrQueueDoesNotExist)
|
||||
}
|
||||
if t.MaximumMessageSize < MinMessageSize || t.MaximumMessageSize > MaxMessageSize {
|
||||
return SendResult{}, fmt.Errorf("%w: queue size limit %d outside %d..%d", ErrUnavailable, t.MaximumMessageSize, MinMessageSize, MaxMessageSize)
|
||||
}
|
||||
if t.DelaySeconds < 0 || t.DelaySeconds > MaxDelaySeconds {
|
||||
return SendResult{}, fmt.Errorf("%w: queue delay %d outside 0..%d", ErrUnavailable, t.DelaySeconds, MaxDelaySeconds)
|
||||
}
|
||||
if err := t.Namespace.Validate(); err != nil {
|
||||
return SendResult{}, fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
attrs, err := NewAttributes(in.MessageAttributes)
|
||||
if err != nil {
|
||||
return SendResult{}, err
|
||||
}
|
||||
if len(in.Body)+attrs.Size() > t.MaximumMessageSize {
|
||||
return SendResult{}, fmt.Errorf("%w: message body and attributes must be shorter than %d bytes", ErrInvalidParameter, t.MaximumMessageSize+1)
|
||||
}
|
||||
delay := t.DelaySeconds
|
||||
if in.DelaySeconds != nil {
|
||||
delay = *in.DelaySeconds
|
||||
}
|
||||
id, err := s.newID()
|
||||
if err != nil {
|
||||
return SendResult{}, fmt.Errorf("%w: message ID: %w", ErrUnavailable, err)
|
||||
}
|
||||
enq := s.now().UTC().Truncate(time.Millisecond)
|
||||
vis := enq.Add(time.Duration(delay) * time.Second)
|
||||
sum := md5.Sum(in.Body) //nolint:gosec // AWS wire checksum.
|
||||
digest := hex.EncodeToString(sum[:])
|
||||
meta := map[string]string{
|
||||
MetaV: strconv.Itoa(MetaVersionPlain),
|
||||
MetaID: id,
|
||||
MetaEnq: strconv.FormatInt(enq.UnixMilli(), 10),
|
||||
MetaVis: strconv.FormatInt(vis.UnixMilli(), 10),
|
||||
MetaSender: in.Sender,
|
||||
MetaMD5: digest,
|
||||
MetaAcct: t.Account,
|
||||
}
|
||||
payload := in.Body
|
||||
if attrs.Len() > 0 {
|
||||
meta[MetaV] = strconv.Itoa(MetaVersion)
|
||||
meta[MetaMD5A] = attrs.MD5()
|
||||
payload = encodePayload(attrs, in.Body)
|
||||
}
|
||||
res, err := s.Store.Append(ctx, t.Namespace, AppendInput{Body: payload, Meta: meta})
|
||||
if err != nil {
|
||||
return SendResult{}, fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
if res.Position == 0 || res.Duplicate {
|
||||
return SendResult{}, fmt.Errorf("%w: append returned no new position", ErrUnavailable)
|
||||
}
|
||||
return SendResult{
|
||||
MessageID: id, MD5OfBody: digest, MD5OfMessageAttributes: attrs.MD5(),
|
||||
Position: res.Position, EnqueuedAt: enq, VisibleAfter: vis,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// validateRequest checks what needs no queue: the request and its sender.
|
||||
func validateRequest(in SendInput) error {
|
||||
if err := ValidateMessage(in); err != nil {
|
||||
return err
|
||||
}
|
||||
if in.Sender == "" || strings.ContainsAny(in.Sender, "\r\n") {
|
||||
return fmt.Errorf("%w: sender ID is not recordable", ErrUnavailable)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateMessage checks what a send needs neither queue nor sender for:
|
||||
// supported features, attributes, the body's characters and the delay.
|
||||
func ValidateMessage(in SendInput) error {
|
||||
switch {
|
||||
case len(in.MessageSystemAttributes) > 0:
|
||||
return fmt.Errorf("%w: message system attributes", ErrUnsupported)
|
||||
case in.MessageGroupID != nil:
|
||||
return fmt.Errorf("%w: MessageGroupId on a standard queue", ErrUnsupported)
|
||||
case in.MessageDeduplicationID != nil:
|
||||
return fmt.Errorf("%w: MessageDeduplicationId on a standard queue", ErrUnsupported)
|
||||
}
|
||||
if len(in.Body) == 0 {
|
||||
return ErrBodyRequired
|
||||
}
|
||||
if err := validateContents(in.Body); err != nil {
|
||||
return err
|
||||
}
|
||||
if d := in.DelaySeconds; d != nil && (*d < 0 || *d > MaxDelaySeconds) {
|
||||
return fmt.Errorf("%w: DelaySeconds must be an integer from 0 to %d", ErrInvalidParameter, MaxDelaySeconds)
|
||||
}
|
||||
attrs, err := NewAttributes(in.MessageAttributes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Whether AWS digests and returns a Number as sent or with its zeros
|
||||
// stripped is not established, so Number is refused until it is.
|
||||
for _, a := range attrs.All() {
|
||||
if base, _, _ := strings.Cut(a.Value.DataType, "."); base == "Number" {
|
||||
return fmt.Errorf("%w: message attribute %q of type Number", ErrUnsupported, a.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateContents allows only the characters AWS documents for a body:
|
||||
// #x9, #xA, #xD, #x20 to #xD7FF, #xE000 to #xFFFD and #x10000 to #x10FFFF.
|
||||
// The error names the offset, never the body.
|
||||
func validateContents(body []byte) error {
|
||||
for i := 0; i < len(body); {
|
||||
r, n := utf8.DecodeRune(body[i:])
|
||||
if r == utf8.RuneError && n <= 1 {
|
||||
return fmt.Errorf("%w: invalid UTF-8 at byte %d", ErrInvalidContents, i)
|
||||
}
|
||||
if !allowedRune(r) {
|
||||
return fmt.Errorf("%w: disallowed character U+%04X at byte %d", ErrInvalidContents, r, i)
|
||||
}
|
||||
i += n
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func allowedRune(r rune) bool {
|
||||
return r == 0x9 || r == 0xA || r == 0xD || r >= 0x20 && r <= 0xD7FF ||
|
||||
r >= 0xE000 && r <= 0xFFFD || r >= 0x10000 && r <= 0x10FFFF
|
||||
}
|
||||
|
||||
func (s *Sender) now() time.Time {
|
||||
if s.Now != nil {
|
||||
return s.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (s *Sender) newID() (string, error) {
|
||||
if s.NewID != nil {
|
||||
return s.NewID()
|
||||
}
|
||||
return NewMessageID()
|
||||
}
|
||||
|
||||
// NewMessageID mints a random (version 4) UUID.
|
||||
func NewMessageID() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
b[6] = b[6]&0x0f | 0x40
|
||||
b[8] = b[8]&0x3f | 0x80
|
||||
h := hex.EncodeToString(b[:])
|
||||
return h[0:8] + "-" + h[8:12] + "-" + h[12:16] + "-" + h[16:20] + "-" + h[20:], nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package message_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const sendAcct = "000000000001"
|
||||
|
||||
var sendNS = message.Namespace{MessageShard: 0, StateShard: 0, Queue: "q1"}
|
||||
|
||||
// recordingStore answers Append with a fixed result and counts the calls;
|
||||
// every other Store method is unused by Send.
|
||||
type recordingStore struct {
|
||||
message.Store
|
||||
|
||||
appends int
|
||||
res message.AppendResult
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *recordingStore) Append(context.Context, message.Namespace, message.AppendInput) (message.AppendResult, error) {
|
||||
r.appends++
|
||||
return r.res, r.err
|
||||
}
|
||||
|
||||
type fixedTarget struct {
|
||||
t message.Target
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fixedTarget) SendTarget(context.Context, string, string, string) (message.Target, error) {
|
||||
return f.t, f.err
|
||||
}
|
||||
|
||||
func goodTarget() message.Target {
|
||||
return message.Target{Namespace: sendNS, Account: sendAcct, MaximumMessageSize: 1024}
|
||||
}
|
||||
|
||||
func sendInput(body string) message.SendInput {
|
||||
return message.SendInput{Account: sendAcct, Region: "us-east-1", Queue: "orders", Sender: "AIDAEXAMPLE", Body: []byte(body)}
|
||||
}
|
||||
|
||||
func send(store *recordingStore, tgt message.Target, in message.SendInput) error {
|
||||
s := &message.Sender{Store: store, Queues: fixedTarget{t: tgt}}
|
||||
_, err := s.Send(context.Background(), in)
|
||||
return err
|
||||
}
|
||||
|
||||
func TestSendRejectsUnimplementedFeatures(t *testing.T) {
|
||||
one := "1"
|
||||
attr := map[string]message.AttributeValue{"a": {DataType: "String", StringValue: &one}}
|
||||
for name, mutate := range map[string]func(*message.SendInput){
|
||||
"Number message attributes": func(in *message.SendInput) {
|
||||
in.MessageAttributes = []message.Attribute{{Name: "a", Value: message.AttributeValue{DataType: "Number.int", StringValue: &one}}}
|
||||
},
|
||||
"message system attributes": func(in *message.SendInput) { in.MessageSystemAttributes = attr },
|
||||
"MessageGroupId": func(in *message.SendInput) { g := ""; in.MessageGroupID = &g },
|
||||
"MessageDeduplicationId": func(in *message.SendInput) { d := ""; in.MessageDeduplicationID = &d },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
st := &recordingStore{res: message.AppendResult{Position: 1}}
|
||||
in := sendInput("x")
|
||||
mutate(&in)
|
||||
require.ErrorIs(t, send(st, goodTarget(), in), message.ErrUnsupported)
|
||||
assert.Zero(t, st.appends)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendRejectsDisallowedContents(t *testing.T) {
|
||||
for name, body := range map[string][]byte{
|
||||
"NUL": {'a', 0x00},
|
||||
"control 0x08": {0x08},
|
||||
"U+FFFE": []byte(""),
|
||||
"U+FFFF": []byte(""),
|
||||
"invalid UTF-8": {0xff, 'a'},
|
||||
"surrogate": {0xed, 0xa0, 0x80},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
st := &recordingStore{res: message.AppendResult{Position: 1}}
|
||||
in := sendInput("")
|
||||
in.Body = body
|
||||
require.ErrorIs(t, send(st, goodTarget(), in), message.ErrInvalidContents)
|
||||
assert.Zero(t, st.appends)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendAppendResultsThatAreNotSuccess(t *testing.T) {
|
||||
for name, st := range map[string]*recordingStore{
|
||||
"refused append": {err: errors.New("refused")},
|
||||
"no position": {},
|
||||
"duplicate": {res: message.AppendResult{Position: 7, Duplicate: true}},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
s := &message.Sender{Store: st, Queues: fixedTarget{t: goodTarget()}}
|
||||
res, err := s.Send(context.Background(), sendInput("x"))
|
||||
require.ErrorIs(t, err, message.ErrUnavailable)
|
||||
assert.Zero(t, res)
|
||||
assert.Equal(t, 1, st.appends, "exactly one append, never a resend")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -149,7 +149,8 @@ type Store interface {
|
||||
// Append durably stores a message and returns only after the write is
|
||||
// acknowledged as durable (ADR-0011:S3).
|
||||
Append(ctx context.Context, ns Namespace, in AppendInput) (AppendResult, error)
|
||||
// Get reads a message by position; ErrNotFound once deleted.
|
||||
// Get reads a message by position authoritatively: once deleted it
|
||||
// returns ErrNotFound or fails, never the deleted body.
|
||||
Get(ctx context.Context, ns Namespace, pos Position) (Message, error)
|
||||
// Delete removes one message. Deleting an absent message succeeds, so
|
||||
// a replay after a crash is safe.
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package message
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// visibilityAttempts bounds retries after a definite revision conflict.
|
||||
const visibilityAttempts = 5
|
||||
|
||||
var (
|
||||
// ErrNotInflight reports a handle whose message is not in flight under
|
||||
// that handle: its deadline passed, it was received again, or it is gone.
|
||||
ErrNotInflight = errors.New("message: not in flight")
|
||||
// ErrVisibilityCap reports a timeout that would end more than 12 hours
|
||||
// after the receive that issued the handle.
|
||||
ErrVisibilityCap = errors.New("message: visibility beyond the 12-hour limit")
|
||||
)
|
||||
|
||||
// ChangeVisibilityInput is one ChangeMessageVisibility request after wire
|
||||
// decoding. VisibilityTimeout is whole seconds from now.
|
||||
type ChangeVisibilityInput struct {
|
||||
Account, Region, Queue string
|
||||
ReceiptHandle string
|
||||
VisibilityTimeout int
|
||||
}
|
||||
|
||||
// VisibilityChanger is the ChangeMessageVisibility owner core.
|
||||
type VisibilityChanger struct {
|
||||
Store Store
|
||||
Queues ReceiveQueues
|
||||
Keys *ReceiptKeys
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// ChangeVisibility moves the deadline of the message a current, in-flight
|
||||
// handle names to now plus the timeout; zero makes it visible at once. The
|
||||
// deadline may not pass the issuing receive's time plus 12 hours.
|
||||
func (v *VisibilityChanger) ChangeVisibility(ctx context.Context, in ChangeVisibilityInput) error {
|
||||
if v == nil || v.Store == nil || v.Queues == nil {
|
||||
return fmt.Errorf("%w: visibility changer is not configured", ErrUnavailable)
|
||||
}
|
||||
if err := v.Keys.Check(); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
if in.VisibilityTimeout < 0 || in.VisibilityTimeout > MaxVisibilitySeconds {
|
||||
return fmt.Errorf("%w: VisibilityTimeout must be between 0 and %d", ErrInvalidParameter, MaxVisibilitySeconds)
|
||||
}
|
||||
t, err := v.Queues.ReceiveTarget(ctx, in.Account, in.Region, in.Queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if t.Account != in.Account {
|
||||
return fmt.Errorf("%w: resolved queue belongs to another account", ErrUnavailable)
|
||||
}
|
||||
if err := t.Namespace.Validate(); err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
rc, err := v.Keys.Decode(in.ReceiptHandle, t.Namespace.Queue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for range visibilityAttempts {
|
||||
cur, err := v.Store.GetState(ctx, t.Namespace, rc.Position)
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
return ErrNotInflight
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: read state: %w", ErrUnavailable, err)
|
||||
}
|
||||
rec, err := decodeState(cur.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case rec.s == StatusDeleted || rec.s == StatusExpired || rec.s == StatusMoved:
|
||||
return ErrNotInflight
|
||||
case rec.s == StatusMoving:
|
||||
return fmt.Errorf("%w: message is being moved", ErrUnavailable)
|
||||
case rec.g > rc.Generation:
|
||||
return ErrNotInflight
|
||||
case rec.g < rc.Generation:
|
||||
return fmt.Errorf("%w: state is older than the handle", ErrUnavailable)
|
||||
}
|
||||
now := v.now().UnixMilli()
|
||||
if rec.d <= now {
|
||||
return ErrNotInflight
|
||||
}
|
||||
d := now + int64(in.VisibilityTimeout)*1000
|
||||
if d > rec.r+MaxVisibilityTimeout.Milliseconds() {
|
||||
return ErrVisibilityCap
|
||||
}
|
||||
rec.d = d
|
||||
value, err := rec.encode()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: encode state: %w", ErrUnavailable, err)
|
||||
}
|
||||
_, err = v.Store.PutState(ctx, t.Namespace, rc.Position, value, cur.Revision)
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case errors.Is(err, ErrRevisionMismatch):
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("%w: visibility change outcome unknown: %w", ErrUnavailable, err)
|
||||
}
|
||||
return fmt.Errorf("%w: visibility change kept conflicting", ErrUnavailable)
|
||||
}
|
||||
|
||||
func (v *VisibilityChanger) now() time.Time {
|
||||
if v.Now != nil {
|
||||
return v.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
// Package queue owns SQS queue identity, attributes and records
|
||||
// (ADR-0012:S1, S3).
|
||||
package queue
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Attribute names a queue record can carry in this profile.
|
||||
const (
|
||||
AttrVisibilityTimeout = "VisibilityTimeout"
|
||||
AttrMessageRetentionPeriod = "MessageRetentionPeriod"
|
||||
AttrMaximumMessageSize = "MaximumMessageSize"
|
||||
AttrDelaySeconds = "DelaySeconds"
|
||||
AttrReceiveMessageWaitTimeSeconds = "ReceiveMessageWaitTimeSeconds"
|
||||
AttrSqsManagedSseEnabled = "SqsManagedSseEnabled"
|
||||
AttrFifoQueue = "FifoQueue"
|
||||
)
|
||||
|
||||
// Errors a request can fail validation with. The API adapter maps each to
|
||||
// its AWS error code; the codes for ErrUnsupportedAttribute and
|
||||
// ErrInvalidQueueName are provisional until characterised against AWS.
|
||||
var (
|
||||
ErrInvalidAttributeName = errors.New("invalid attribute name")
|
||||
ErrInvalidAttributeValue = errors.New("invalid attribute value")
|
||||
ErrUnsupportedAttribute = errors.New("attribute not supported")
|
||||
ErrInvalidQueueName = errors.New("invalid queue name")
|
||||
)
|
||||
|
||||
type intRange struct {
|
||||
min, max, def int
|
||||
}
|
||||
|
||||
// intAttributes are the settable integer attributes with AWS ranges and
|
||||
// defaults (ADR-0012 rows A5, A6, A8, A9, A10).
|
||||
var intAttributes = map[string]intRange{
|
||||
AttrVisibilityTimeout: {0, 43200, 30},
|
||||
AttrMessageRetentionPeriod: {60, 1209600, 345600},
|
||||
AttrMaximumMessageSize: {1024, 1048576, 1048576},
|
||||
AttrDelaySeconds: {0, 900, 0},
|
||||
AttrReceiveMessageWaitTimeSeconds: {0, 20, 0},
|
||||
}
|
||||
|
||||
// unsupported are attributes AWS accepts that this profile rejects
|
||||
// explicitly rather than ignoring.
|
||||
var unsupported = map[string]bool{
|
||||
"ContentBasedDeduplication": true,
|
||||
"DeduplicationScope": true,
|
||||
"FifoThroughputLimit": true,
|
||||
"KmsMasterKeyId": true,
|
||||
"KmsDataKeyReusePeriodSeconds": true,
|
||||
"Policy": true,
|
||||
"RedrivePolicy": true,
|
||||
"RedriveAllowPolicy": true,
|
||||
}
|
||||
|
||||
// Attributes is a queue's attribute set keyed by AWS attribute name, with
|
||||
// canonical values. Names this release does not know are kept as written.
|
||||
type Attributes map[string]string
|
||||
|
||||
// Defaults is the effective attribute set of a queue created with none.
|
||||
func Defaults() Attributes {
|
||||
a := Attributes{AttrSqsManagedSseEnabled: "false"}
|
||||
for name, r := range intAttributes {
|
||||
a[name] = strconv.Itoa(r.def)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// ValidateCreate checks CreateQueue attributes and returns them in canonical
|
||||
// form. FifoQueue=false is accepted and dropped: every queue is standard.
|
||||
func ValidateCreate(in map[string]string) (Attributes, error) {
|
||||
return validate(in, true)
|
||||
}
|
||||
|
||||
// ValidateSet checks SetQueueAttributes attributes and returns them in
|
||||
// canonical form; FifoQueue is fixed at creation and rejected here.
|
||||
func ValidateSet(in map[string]string) (Attributes, error) {
|
||||
return validate(in, false)
|
||||
}
|
||||
|
||||
func validate(in map[string]string, create bool) (Attributes, error) {
|
||||
out := make(Attributes, len(in))
|
||||
for _, name := range sortedKeys(in) {
|
||||
value := in[name]
|
||||
if r, ok := intAttributes[name]; ok {
|
||||
n, err := parseInt(value)
|
||||
if err != nil || n < r.min || n > r.max {
|
||||
return nil, fmt.Errorf("%w: %s must be an integer from %d to %d", ErrInvalidAttributeValue, name, r.min, r.max)
|
||||
}
|
||||
out[name] = strconv.Itoa(n)
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case name == AttrSqsManagedSseEnabled:
|
||||
b, err := parseBool(name, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b {
|
||||
return nil, fmt.Errorf("%w: %s=true; server-side encryption is not available", ErrUnsupportedAttribute, name)
|
||||
}
|
||||
out[name] = "false"
|
||||
case name == AttrFifoQueue && create:
|
||||
b, err := parseBool(name, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b {
|
||||
return nil, fmt.Errorf("%w: FIFO queues are not available", ErrUnsupportedAttribute)
|
||||
}
|
||||
case name == AttrFifoQueue:
|
||||
return nil, fmt.Errorf("%w: %s can only be set when the queue is created", ErrInvalidAttributeName, name)
|
||||
case unsupported[name]:
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedAttribute, name)
|
||||
default:
|
||||
return nil, fmt.Errorf("%w: %s", ErrInvalidAttributeName, name)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// WithDefaults returns a copy of a with every unset attribute at its default.
|
||||
func (a Attributes) WithDefaults() Attributes {
|
||||
out := Defaults()
|
||||
maps.Copy(out, a)
|
||||
return out
|
||||
}
|
||||
|
||||
// Matches reports whether every supplied attribute equals a's value for it.
|
||||
// Both must be canonical. Provisional: which attributes a repeat must match
|
||||
// is not yet characterised against AWS.
|
||||
func (a Attributes) Matches(supplied Attributes) bool {
|
||||
for name, v := range supplied {
|
||||
if a[name] != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// parseInt accepts decimal digits only, so "+30", "-0" and " 30" fail.
|
||||
func parseInt(s string) (int, error) {
|
||||
if s == "" || len(s) > 10 {
|
||||
return 0, strconv.ErrSyntax
|
||||
}
|
||||
for _, r := range s {
|
||||
if r < '0' || r > '9' {
|
||||
return 0, strconv.ErrSyntax
|
||||
}
|
||||
}
|
||||
return strconv.Atoi(s)
|
||||
}
|
||||
|
||||
// parseBool is case-insensitive; provisional until AWS's casing rule is
|
||||
// characterised.
|
||||
func parseBool(name, s string) (bool, error) {
|
||||
switch {
|
||||
case strings.EqualFold(s, "true"):
|
||||
return true, nil
|
||||
case strings.EqualFold(s, "false"):
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("%w: %s must be true or false", ErrInvalidAttributeValue, name)
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]string) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The range cases are the scen_p create and set boundaries; each limit is
|
||||
// accepted and one past it rejected (ADR-0012 rows A5, A6, A8, A9, A10).
|
||||
func TestValidateIntegerBoundaries(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, value string
|
||||
ok bool
|
||||
}{
|
||||
{AttrVisibilityTimeout, "0", true},
|
||||
{AttrVisibilityTimeout, "43200", true},
|
||||
{AttrVisibilityTimeout, "43201", false},
|
||||
{AttrMessageRetentionPeriod, "59", false},
|
||||
{AttrMessageRetentionPeriod, "60", true},
|
||||
{AttrMessageRetentionPeriod, "1209600", true},
|
||||
{AttrMessageRetentionPeriod, "1209601", false},
|
||||
{AttrMaximumMessageSize, "1023", false},
|
||||
{AttrMaximumMessageSize, "1024", true},
|
||||
{AttrMaximumMessageSize, "1048576", true},
|
||||
{AttrMaximumMessageSize, "1048577", false},
|
||||
{AttrDelaySeconds, "900", true},
|
||||
{AttrDelaySeconds, "901", false},
|
||||
{AttrReceiveMessageWaitTimeSeconds, "20", true},
|
||||
{AttrReceiveMessageWaitTimeSeconds, "21", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
for _, validate := range []func(map[string]string) (Attributes, error){ValidateCreate, ValidateSet} {
|
||||
got, err := validate(map[string]string{tc.name: tc.value})
|
||||
if tc.ok {
|
||||
require.NoError(t, err, "%s=%s", tc.name, tc.value)
|
||||
assert.Equal(t, tc.value, got[tc.name])
|
||||
continue
|
||||
}
|
||||
assert.ErrorIs(t, err, ErrInvalidAttributeValue, "%s=%s", tc.name, tc.value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUnknownNameIsInvalidAttributeName(t *testing.T) {
|
||||
for _, name := range []string{"Bogus", "visibilitytimeout", "QueueArn", "ApproximateNumberOfMessages", "All"} {
|
||||
_, err := ValidateSet(map[string]string{name: "1"})
|
||||
assert.ErrorIs(t, err, ErrInvalidAttributeName, name)
|
||||
}
|
||||
}
|
||||
|
||||
// Attributes AWS accepts but this profile does not serve are refused, never
|
||||
// silently dropped, so a client cannot believe it has a FIFO, encrypted or
|
||||
// redriven queue.
|
||||
func TestValidateRejectsUnsupportedAttributes(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
AttrFifoQueue: "true",
|
||||
AttrSqsManagedSseEnabled: "true",
|
||||
"ContentBasedDeduplication": "false",
|
||||
"DeduplicationScope": "queue",
|
||||
"FifoThroughputLimit": "perQueue",
|
||||
"KmsMasterKeyId": "alias/aws/sqs",
|
||||
"KmsDataKeyReusePeriodSeconds": "300",
|
||||
"Policy": "{}",
|
||||
"RedrivePolicy": "{}",
|
||||
"RedriveAllowPolicy": "{}",
|
||||
}
|
||||
for name, value := range cases {
|
||||
_, err := ValidateCreate(map[string]string{name: value})
|
||||
assert.ErrorIs(t, err, ErrUnsupportedAttribute, "%s=%s", name, value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFifoQueueFalseAcceptedOnlyAtCreate(t *testing.T) {
|
||||
got, err := ValidateCreate(map[string]string{AttrFifoQueue: "false"})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got)
|
||||
|
||||
_, err = ValidateSet(map[string]string{AttrFifoQueue: "false"})
|
||||
assert.ErrorIs(t, err, ErrInvalidAttributeName)
|
||||
|
||||
_, err = ValidateCreate(map[string]string{AttrFifoQueue: "yes"})
|
||||
assert.ErrorIs(t, err, ErrInvalidAttributeValue)
|
||||
}
|
||||
|
||||
func TestDefaults(t *testing.T) {
|
||||
assert.Equal(t, Attributes{
|
||||
AttrVisibilityTimeout: "30",
|
||||
AttrMessageRetentionPeriod: "345600",
|
||||
AttrMaximumMessageSize: "1048576",
|
||||
AttrDelaySeconds: "0",
|
||||
AttrReceiveMessageWaitTimeSeconds: "0",
|
||||
AttrSqsManagedSseEnabled: "false",
|
||||
}, Defaults())
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// MarkDeleting records DeleteQueue's durable intent: one compare-and-swap of
|
||||
// the pinned active queue to deleting with requestedAt and generation + 1.
|
||||
// It removes nothing; reservation, purge and removal follow separately.
|
||||
func (s *Store) MarkDeleting(ctx context.Context, account, region, name string) (Record, error) {
|
||||
var pinned string
|
||||
for range updateAttempts {
|
||||
r, rev, err := s.resolvePinned(ctx, account, region, name, pinned)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
pinned = r.QueueID
|
||||
if err := settable(&r); err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
|
||||
next := r
|
||||
next.State = StateDeleting
|
||||
next.Generation = r.Generation + 1
|
||||
d := Deletion{}
|
||||
if r.Deletion != nil {
|
||||
d = *r.Deletion
|
||||
}
|
||||
d.RequestedAt = s.now().UTC()
|
||||
d.Generation = next.Generation
|
||||
next.Deletion = &d
|
||||
enc, err := next.Encode()
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
_, err = s.kv.update(ctx, recordKey(account, r.QueueID), enc, rev)
|
||||
switch {
|
||||
case err == nil:
|
||||
return next, nil
|
||||
case isConflict(err):
|
||||
continue
|
||||
}
|
||||
|
||||
done, retry, uerr := s.settleDelete(ctx, account, region, name, r.QueueID, rev, err)
|
||||
if done != nil {
|
||||
return *done, nil
|
||||
}
|
||||
if !retry {
|
||||
return Record{}, uerr
|
||||
}
|
||||
}
|
||||
return Record{}, fmt.Errorf("%w: DeleteQueue %s lost %d races", ErrUnavailable, name, updateAttempts)
|
||||
}
|
||||
|
||||
// settleDelete decides a deletion whose reply was lost. A record now deleting
|
||||
// satisfies the request, whichever writer moved it; an unmoved revision may
|
||||
// still take the write, so it fails closed; any other move is retried.
|
||||
func (s *Store) settleDelete(ctx context.Context, account, region, name, queueID string, rev uint64, cause error) (*Record, bool, error) {
|
||||
r, grev, err := s.resolvePinned(ctx, account, region, name, queueID)
|
||||
switch {
|
||||
case errors.Is(err, ErrQueueDoesNotExist):
|
||||
return nil, false, err
|
||||
case err != nil, grev == rev:
|
||||
return nil, false, fmt.Errorf("%w: DeleteQueue %s outcome unknown: %w", ErrUnavailable, name, cause)
|
||||
case r.State == StateDeleting:
|
||||
return &r, false, nil
|
||||
}
|
||||
return nil, true, nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func markDeleting(f *fixture, name string) (Record, error) {
|
||||
return f.store.MarkDeleting(context.Background(), acct, region, name)
|
||||
}
|
||||
|
||||
func TestMarkDeleting(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
ctx := context.Background()
|
||||
q, err := f.store.Create(ctx, CreateRequest{Account: acct, Region: region, Name: "orders",
|
||||
Attributes: map[string]string{AttrDelaySeconds: "5"}, Tags: map[string]string{"env": "prod"}})
|
||||
require.NoError(t, err)
|
||||
be, err := f.kv.Get(ctx, bindingKey(acct, region, "orders"))
|
||||
require.NoError(t, err)
|
||||
f.now = f.now.Add(time.Minute)
|
||||
|
||||
r, err := markDeleting(f, "orders")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, StateDeleting, r.State)
|
||||
|
||||
stored, _ := f.record(t, q.QueueID)
|
||||
assert.Equal(t, StateDeleting, stored.State)
|
||||
assert.Equal(t, uint64(2), stored.Generation)
|
||||
require.NotNil(t, stored.Deletion)
|
||||
assert.True(t, stored.Deletion.RequestedAt.Equal(f.now))
|
||||
assert.Equal(t, uint64(2), stored.Deletion.Generation)
|
||||
assert.Equal(t, q.Attributes, stored.Attributes)
|
||||
assert.Equal(t, q.Tags, stored.Tags)
|
||||
assert.True(t, stored.AttributesModifiedAt.Equal(q.AttributesModifiedAt))
|
||||
|
||||
ae, err := f.kv.Get(ctx, bindingKey(acct, region, "orders"))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, be.Revision(), ae.Revision(), "the binding is not changed by the intent")
|
||||
|
||||
_, err = f.store.Resolve(ctx, acct, region, "orders")
|
||||
assert.ErrorIs(t, err, ErrQueueDoesNotExist, "publicly absent from the intent write")
|
||||
l, err := list(f, ListRequest{})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, l.Queues)
|
||||
_, err = set(f, "orders", map[string]string{AttrDelaySeconds: "1"})
|
||||
assert.ErrorIs(t, err, ErrQueueDoesNotExist)
|
||||
}
|
||||
|
||||
func TestMarkDeletingNotActive(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
_, err := markDeleting(f, "missing")
|
||||
assert.ErrorIs(t, err, ErrQueueDoesNotExist)
|
||||
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
_, err = markDeleting(f, "orders")
|
||||
require.NoError(t, err)
|
||||
_, before := f.record(t, q.QueueID)
|
||||
_, err = markDeleting(f, "orders")
|
||||
assert.ErrorIs(t, err, ErrQueueDoesNotExist, "a deleting queue is already absent")
|
||||
_, after := f.record(t, q.QueueID)
|
||||
assert.Equal(t, before, after)
|
||||
|
||||
for name, change := range map[string]func(*Record){
|
||||
"unknown state": func(r *Record) { r.State = "migrating" },
|
||||
"missing attributes": func(r *Record) { r.Attributes = nil },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
f.rewrite(t, q.QueueID, change)
|
||||
_, before := f.record(t, q.QueueID)
|
||||
_, err = markDeleting(f, "orders")
|
||||
assert.ErrorIs(t, err, ErrUnavailable)
|
||||
_, after := f.record(t, q.QueueID)
|
||||
assert.Equal(t, before, after)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Of concurrent deletes exactly one writes the intent; the rest find the
|
||||
// queue already absent.
|
||||
func TestMarkDeletingConcurrent(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
const n = 5
|
||||
errs := make([]error, n)
|
||||
var wg sync.WaitGroup
|
||||
for i := range n {
|
||||
wg.Go(func() { _, errs[i] = markDeleting(f, "orders") })
|
||||
}
|
||||
wg.Wait()
|
||||
ok := 0
|
||||
for _, err := range errs {
|
||||
if err == nil {
|
||||
ok++
|
||||
continue
|
||||
}
|
||||
assert.ErrorIs(t, err, ErrQueueDoesNotExist)
|
||||
}
|
||||
assert.Equal(t, 1, ok)
|
||||
stored, _ := f.record(t, q.QueueID)
|
||||
assert.Equal(t, uint64(2), stored.Generation)
|
||||
assert.Equal(t, uint64(2), stored.Deletion.Generation)
|
||||
}
|
||||
|
||||
func TestMarkDeletingAppliedReplyLost(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
f.fault.onUpdate = func(key string, value []byte, rev uint64) (uint64, error, bool) {
|
||||
f.fault.onUpdate = nil
|
||||
_, err := f.fault.real.update(context.Background(), key, value, rev)
|
||||
require.NoError(t, err)
|
||||
return 0, errLostReply, true
|
||||
}
|
||||
r, err := markDeleting(f, "orders")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, StateDeleting, r.State)
|
||||
stored, _ := f.record(t, q.QueueID)
|
||||
assert.Equal(t, uint64(2), stored.Generation)
|
||||
}
|
||||
|
||||
func TestMarkDeletingUnknownOutcomeFailsClosed(t *testing.T) {
|
||||
cases := map[string]func(f *fixture){
|
||||
"record unmoved": func(*fixture) {},
|
||||
"re-read fails": func(f *fixture) {
|
||||
f.fault.onGet = func(string) ([]byte, uint64, error, bool) {
|
||||
f.fault.onGet = nil
|
||||
return nil, 0, errLostReply, true
|
||||
}
|
||||
},
|
||||
}
|
||||
for name, after := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
sends := 0
|
||||
f.fault.onUpdate = func(string, []byte, uint64) (uint64, error, bool) {
|
||||
sends++
|
||||
after(f)
|
||||
return 0, errLostReply, true
|
||||
}
|
||||
_, err = markDeleting(f, "orders")
|
||||
assert.ErrorIs(t, err, ErrUnavailable)
|
||||
assert.Equal(t, 1, sends)
|
||||
stored, _ := f.record(t, q.QueueID)
|
||||
assert.Equal(t, StateActive, stored.State)
|
||||
|
||||
f.fault.onUpdate = nil
|
||||
_, err = markDeleting(f, "orders")
|
||||
require.NoError(t, err)
|
||||
stored, _ = f.record(t, q.QueueID)
|
||||
assert.Equal(t, uint64(2), stored.Generation)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// ListQueues bounds. Without MaxResults a call returns at most
|
||||
// defaultListLimit queues and no token; both are provisional until
|
||||
// characterised against AWS.
|
||||
const (
|
||||
maxListResults = 1000
|
||||
defaultListLimit = 1000
|
||||
maxPrefixLen = 80
|
||||
listTokenVersion = 1
|
||||
)
|
||||
|
||||
// ErrInvalidListRequest rejects a bad MaxResults, prefix or NextToken; its
|
||||
// wire code is provisional.
|
||||
var ErrInvalidListRequest = errors.New("invalid ListQueues request")
|
||||
|
||||
// ListRequest is a ListQueues call for one account and Region.
|
||||
type ListRequest struct {
|
||||
Account, Region string
|
||||
Prefix string
|
||||
// MaxResults is nil when the caller supplied none.
|
||||
MaxResults *int
|
||||
NextToken string
|
||||
}
|
||||
|
||||
// ListResult is one page; NextToken is empty on the last page.
|
||||
type ListResult struct {
|
||||
Queues []Record
|
||||
NextToken string
|
||||
}
|
||||
|
||||
// listToken is the opaque pagination cursor. It is bound to the caller's
|
||||
// account, Region and prefix, so it cannot page another account's names.
|
||||
type listToken struct {
|
||||
V int `json:"v"`
|
||||
Acct string `json:"a"`
|
||||
Region string `json:"r"`
|
||||
Prefix string `json:"p"`
|
||||
After string `json:"after"`
|
||||
}
|
||||
|
||||
// List returns active queues in name order whose binding and record agree.
|
||||
// Names come from one key listing that may lag recent writes; each is then
|
||||
// read from the leader. One traversal never repeats a name.
|
||||
func (s *Store) List(ctx context.Context, req ListRequest) (ListResult, error) {
|
||||
if err := validIdentity(req.Account, req.Region); err != nil {
|
||||
return ListResult{}, err
|
||||
}
|
||||
limit, after, err := req.validate()
|
||||
if err != nil {
|
||||
return ListResult{}, err
|
||||
}
|
||||
base := "n." + req.Account + "." + req.Region + "."
|
||||
keys, err := s.kv.keys(ctx, base+">")
|
||||
if err != nil {
|
||||
return ListResult{}, fmt.Errorf("%w: list queue names: %w", ErrUnavailable, err)
|
||||
}
|
||||
names := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
name := strings.TrimPrefix(k, base)
|
||||
if ValidateName(name) != nil {
|
||||
return ListResult{}, fmt.Errorf("%w: malformed binding key %s", ErrUnavailable, k)
|
||||
}
|
||||
if strings.HasPrefix(name, req.Prefix) && name > after {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
slices.Sort(names)
|
||||
names = slices.Compact(names)
|
||||
|
||||
var out ListResult
|
||||
for i, name := range names {
|
||||
if len(out.Queues) == limit {
|
||||
if req.MaxResults != nil {
|
||||
if out.NextToken, err = req.token(names[i-1]); err != nil {
|
||||
return ListResult{}, err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
r, ok, err := s.listed(ctx, req.Account, req.Region, name)
|
||||
if err != nil {
|
||||
return ListResult{}, err
|
||||
}
|
||||
if ok {
|
||||
out.Queues = append(out.Queues, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// listed reads one name as Resolve would: absent or non-active queues are
|
||||
// skipped, and any read or decode failure fails the call rather than
|
||||
// silently omitting a queue that may exist.
|
||||
func (s *Store) listed(ctx context.Context, account, region, name string) (Record, bool, error) {
|
||||
r, _, err := s.resolve(ctx, account, region, name)
|
||||
switch {
|
||||
case errors.Is(err, ErrQueueDoesNotExist):
|
||||
return Record{}, false, nil
|
||||
case err != nil:
|
||||
return Record{}, false, err
|
||||
case r.State != StateActive:
|
||||
return Record{}, false, nil
|
||||
}
|
||||
return r, true, nil
|
||||
}
|
||||
|
||||
// Validate rejects a bad MaxResults, prefix or NextToken without reading the
|
||||
// store, so an adapter can refuse the request before authorizing it.
|
||||
func (req *ListRequest) Validate() error {
|
||||
_, _, err := req.validate()
|
||||
return err
|
||||
}
|
||||
|
||||
func (req *ListRequest) validate() (limit int, after string, err error) {
|
||||
switch {
|
||||
case req.MaxResults != nil && (*req.MaxResults < 1 || *req.MaxResults > maxListResults):
|
||||
return 0, "", fmt.Errorf("%w: MaxResults must be from 1 to %d", ErrInvalidListRequest, maxListResults)
|
||||
case len(req.Prefix) > maxPrefixLen || (req.Prefix != "" && ValidateName(req.Prefix) != nil):
|
||||
return 0, "", fmt.Errorf("%w: invalid QueueNamePrefix", ErrInvalidListRequest)
|
||||
}
|
||||
limit = defaultListLimit
|
||||
if req.MaxResults != nil {
|
||||
limit = *req.MaxResults
|
||||
}
|
||||
if req.NextToken == "" {
|
||||
return limit, "", nil
|
||||
}
|
||||
b, err := base64.RawURLEncoding.DecodeString(req.NextToken)
|
||||
if err != nil {
|
||||
return 0, "", fmt.Errorf("%w: invalid NextToken", ErrInvalidListRequest)
|
||||
}
|
||||
var t listToken
|
||||
if err := json.Unmarshal(b, &t); err != nil ||
|
||||
t.V != listTokenVersion || t.Acct != req.Account || t.Region != req.Region ||
|
||||
t.Prefix != req.Prefix || ValidateName(t.After) != nil || !strings.HasPrefix(t.After, req.Prefix) {
|
||||
return 0, "", fmt.Errorf("%w: invalid NextToken", ErrInvalidListRequest)
|
||||
}
|
||||
return limit, t.After, nil
|
||||
}
|
||||
|
||||
func (req *ListRequest) token(after string) (string, error) {
|
||||
b, err := json.Marshal(listToken{V: listTokenVersion, Acct: req.Account, Region: req.Region, Prefix: req.Prefix, After: after})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// errListIncomplete reports a key listing that ended before the server
|
||||
// signalled it had delivered every current key.
|
||||
var errListIncomplete = errors.New("key listing ended early")
|
||||
|
||||
// keys lists keys matching filter through an ordered consumer, which may lag
|
||||
// the leader. It fails unless the watcher reaches its end-of-initial-values
|
||||
// marker, so a cut-short listing is never mistaken for a complete one.
|
||||
func (k jsKV) keys(ctx context.Context, filter string) ([]string, error) {
|
||||
w, err := k.kv.WatchFiltered(ctx, []string{filter}, jetstream.IgnoreDeletes(), jetstream.MetaOnly())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
return drainKeys(ctx, w.Updates())
|
||||
}
|
||||
|
||||
// drainKeys collects keys until the nil end-of-initial-values marker; a
|
||||
// channel closed before it, or a cancelled context, is an incomplete listing.
|
||||
func drainKeys(ctx context.Context, updates <-chan jetstream.KeyValueEntry) ([]string, error) {
|
||||
var out []string
|
||||
for {
|
||||
select {
|
||||
case e, ok := <-updates:
|
||||
switch {
|
||||
case !ok:
|
||||
return nil, errListIncomplete
|
||||
case e == nil:
|
||||
return out, nil
|
||||
}
|
||||
out = append(out, e.Key())
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const otherAcct = "222233334444"
|
||||
|
||||
func list(f *fixture, req ListRequest) (ListResult, error) {
|
||||
if req.Account == "" {
|
||||
req.Account, req.Region = acct, region
|
||||
}
|
||||
return f.store.List(context.Background(), req)
|
||||
}
|
||||
|
||||
func ptr(n int) *int { return &n }
|
||||
|
||||
func queueNames(r ListResult) []string {
|
||||
out := make([]string, 0, len(r.Queues))
|
||||
for _, q := range r.Queues {
|
||||
out = append(out, q.Name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestListScopedToAccountAndRegion(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
ctx := context.Background()
|
||||
for _, n := range []string{"b", "a", "c"} {
|
||||
_, err := create(f, n, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
_, err := f.store.Create(ctx, CreateRequest{Account: otherAcct, Region: region, Name: "a-other"})
|
||||
require.NoError(t, err)
|
||||
_, err = f.store.Create(ctx, CreateRequest{Account: acct, Region: "ap-south-1", Name: "a-region"})
|
||||
require.NoError(t, err)
|
||||
|
||||
r, err := list(f, ListRequest{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"a", "b", "c"}, queueNames(r))
|
||||
assert.Empty(t, r.NextToken, "without MaxResults no token is returned")
|
||||
|
||||
r, err = list(f, ListRequest{Account: otherAcct, Region: region})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"a-other"}, queueNames(r))
|
||||
}
|
||||
|
||||
func TestListPrefix(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
for _, n := range []string{"orders", "orders-dlq", "Orders", "ord", "payments"} {
|
||||
_, err := create(f, n, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
r, err := list(f, ListRequest{Prefix: "orders"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"orders", "orders-dlq"}, queueNames(r), "prefix match is case-sensitive")
|
||||
|
||||
r, err = list(f, ListRequest{Prefix: "zzz"})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, r.Queues)
|
||||
}
|
||||
|
||||
func TestListPagination(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
var want []string
|
||||
for i := range 7 {
|
||||
n := fmt.Sprintf("q%d", i)
|
||||
want = append(want, n)
|
||||
_, err := create(f, n, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
_, err := create(f, "other", nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
var got []string
|
||||
var pages int
|
||||
req := ListRequest{Prefix: "q", MaxResults: ptr(3)}
|
||||
for {
|
||||
r, err := list(f, req)
|
||||
require.NoError(t, err)
|
||||
pages++
|
||||
assert.LessOrEqual(t, len(r.Queues), 3)
|
||||
got = append(got, queueNames(r)...)
|
||||
if r.NextToken == "" {
|
||||
break
|
||||
}
|
||||
req.NextToken = r.NextToken
|
||||
}
|
||||
assert.Equal(t, want, got)
|
||||
assert.Equal(t, 3, pages)
|
||||
|
||||
r, err := list(f, ListRequest{Prefix: "q", MaxResults: ptr(7)})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, r.Queues, 7)
|
||||
assert.Empty(t, r.NextToken, "a page that ends the listing carries no token")
|
||||
}
|
||||
|
||||
func TestListInvalidRequests(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
for i := range 3 {
|
||||
_, err := create(f, fmt.Sprintf("q%d", i), nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
r, err := list(f, ListRequest{MaxResults: ptr(1)})
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, r.NextToken)
|
||||
good := r.NextToken
|
||||
|
||||
forge := func(tok listToken) string {
|
||||
b, err := json.Marshal(tok)
|
||||
require.NoError(t, err)
|
||||
return base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
bad := map[string]ListRequest{
|
||||
"negative max": {MaxResults: ptr(-1)},
|
||||
"supplied zero max": {MaxResults: ptr(0)},
|
||||
"max above limit": {MaxResults: ptr(maxListResults + 1)},
|
||||
"prefix with dot": {Prefix: "a.b"},
|
||||
"prefix too long": {Prefix: strings.Repeat("a", maxPrefixLen+1)},
|
||||
"garbage token": {MaxResults: ptr(1), NextToken: "!!"},
|
||||
"non-json token": {MaxResults: ptr(1), NextToken: base64.RawURLEncoding.EncodeToString([]byte("x"))},
|
||||
"other account token": {Account: otherAcct, Region: region, MaxResults: ptr(1), NextToken: good},
|
||||
"other region token": {Account: acct, Region: "ap-south-1", MaxResults: ptr(1), NextToken: good},
|
||||
"other prefix token": {Prefix: "q", MaxResults: ptr(1), NextToken: good},
|
||||
"future token version": {MaxResults: ptr(1), NextToken: forge(listToken{V: 2, Acct: acct, Region: region, After: "q"})},
|
||||
"cursor outside prefix": {Prefix: "x", MaxResults: ptr(1),
|
||||
NextToken: forge(listToken{V: 1, Acct: acct, Region: region, Prefix: "x", After: "q"})},
|
||||
"cursor not a name": {MaxResults: ptr(1), NextToken: forge(listToken{V: 1, Acct: acct, Region: region, After: "a.b"})},
|
||||
}
|
||||
for name, req := range bad {
|
||||
_, err := list(f, req)
|
||||
assert.ErrorIs(t, err, ErrInvalidListRequest, name)
|
||||
}
|
||||
}
|
||||
|
||||
// A queue deleted between the name listing and its read is omitted; one
|
||||
// created after the listing is not on this page.
|
||||
func TestListConcurrentCreateAndDelete(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
ctx := context.Background()
|
||||
for _, n := range []string{"a", "b"} {
|
||||
_, err := create(f, n, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
f.fault.onKeys = func(filter string) ([]string, error, bool) {
|
||||
f.fault.onKeys = nil
|
||||
keys, err := f.fault.real.keys(ctx, filter)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.kv.Delete(ctx, bindingKey(acct, region, "b")))
|
||||
_, err = create(f, "c", nil)
|
||||
require.NoError(t, err)
|
||||
return keys, nil, true
|
||||
}
|
||||
r, err := list(f, ListRequest{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"a"}, queueNames(r))
|
||||
|
||||
r, err = list(f, ListRequest{})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"a", "c"}, queueNames(r))
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base32"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
)
|
||||
|
||||
// RecordVersion is the schema version this release writes.
|
||||
const RecordVersion = 1
|
||||
|
||||
// MaxNameLen is AWS's queue-name limit (ADR-0012 row A15).
|
||||
const MaxNameLen = 80
|
||||
|
||||
// Queue and binding states.
|
||||
const (
|
||||
StateActive = "active"
|
||||
StateDeleting = "deleting"
|
||||
|
||||
BindingBound = "bound"
|
||||
BindingReserved = "reserved"
|
||||
|
||||
TypeStandard = "standard"
|
||||
)
|
||||
|
||||
// Record is the authoritative queue record, keyed q/<account>/<queueId>.
|
||||
// Fields a newer release adds are kept on decode and written back unchanged.
|
||||
type Record struct {
|
||||
V int `json:"v"`
|
||||
QueueID string `json:"queueId"`
|
||||
MessageShard message.Shard `json:"messageShard"`
|
||||
StateShard message.Shard `json:"stateShard"`
|
||||
Account string `json:"account"`
|
||||
Region string `json:"region"`
|
||||
Name string `json:"name"`
|
||||
ARN string `json:"arn"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
Type string `json:"type"`
|
||||
State string `json:"state"`
|
||||
Generation uint64 `json:"generation"`
|
||||
Attributes Attributes `json:"attributes"`
|
||||
AttributesModifiedAt time.Time `json:"attributesModifiedAt"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
Deletion *Deletion `json:"deletion,omitempty"`
|
||||
|
||||
unknown map[string]json.RawMessage
|
||||
}
|
||||
|
||||
// Deletion is the durable deletion intent of a deleting queue.
|
||||
type Deletion struct {
|
||||
RequestedAt time.Time `json:"requestedAt"`
|
||||
Generation uint64 `json:"generation"`
|
||||
RetainedReason string `json:"retainedReason,omitempty"`
|
||||
LastCheck time.Time `json:"lastCheck,omitzero"`
|
||||
|
||||
unknown map[string]json.RawMessage
|
||||
}
|
||||
|
||||
type deletionFields Deletion
|
||||
|
||||
// MarshalJSON writes the known fields and every unknown field read earlier.
|
||||
func (d *Deletion) MarshalJSON() ([]byte, error) {
|
||||
return marshalPreserving(deletionFields(*d), d.unknown)
|
||||
}
|
||||
|
||||
// UnmarshalJSON reads the known fields and keeps the rest.
|
||||
func (d *Deletion) UnmarshalJSON(b []byte) error {
|
||||
var f deletionFields
|
||||
unknown, err := unmarshalPreserving(b, &f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*d = Deletion(f)
|
||||
d.unknown = unknown
|
||||
return nil
|
||||
}
|
||||
|
||||
// Binding is a name's resolution, keyed n/<account>/<region>/<name>.
|
||||
type Binding struct {
|
||||
V int `json:"v"`
|
||||
QueueID string `json:"queueId"`
|
||||
State string `json:"state"`
|
||||
ReuseAfter time.Time `json:"reuseAfter,omitzero"`
|
||||
|
||||
unknown map[string]json.RawMessage
|
||||
}
|
||||
|
||||
// Namespace is the queue's store namespace; it needs no store call to exist.
|
||||
func (r *Record) Namespace() message.Namespace {
|
||||
return message.Namespace{MessageShard: r.MessageShard, StateShard: r.StateShard, Queue: message.QueueID(r.QueueID)}
|
||||
}
|
||||
|
||||
type recordFields Record
|
||||
type bindingFields Binding
|
||||
|
||||
// Encode writes the known fields and every unknown field read by Decode.
|
||||
func (r *Record) Encode() ([]byte, error) {
|
||||
return marshalPreserving(recordFields(*r), r.unknown)
|
||||
}
|
||||
|
||||
// DecodeRecord reads the known fields and keeps the rest for Encode.
|
||||
func DecodeRecord(b []byte) (Record, error) {
|
||||
var f recordFields
|
||||
unknown, err := unmarshalPreserving(b, &f)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
r := Record(f)
|
||||
r.unknown = unknown
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Encode writes the known fields and every unknown field read by Decode.
|
||||
func (b *Binding) Encode() ([]byte, error) {
|
||||
return marshalPreserving(bindingFields(*b), b.unknown)
|
||||
}
|
||||
|
||||
// DecodeBinding reads the known fields and keeps the rest for Encode.
|
||||
func DecodeBinding(data []byte) (Binding, error) {
|
||||
var f bindingFields
|
||||
unknown, err := unmarshalPreserving(data, &f)
|
||||
if err != nil {
|
||||
return Binding{}, err
|
||||
}
|
||||
b := Binding(f)
|
||||
b.unknown = unknown
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func marshalPreserving(known any, unknown map[string]json.RawMessage) ([]byte, error) {
|
||||
b, err := json.Marshal(known)
|
||||
if err != nil || len(unknown) == 0 {
|
||||
return b, err
|
||||
}
|
||||
var merged map[string]json.RawMessage
|
||||
if err := json.Unmarshal(b, &merged); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range unknown {
|
||||
if _, ok := merged[k]; !ok {
|
||||
merged[k] = v
|
||||
}
|
||||
}
|
||||
return json.Marshal(merged)
|
||||
}
|
||||
|
||||
func unmarshalPreserving(b []byte, known any) (map[string]json.RawMessage, error) {
|
||||
if err := json.Unmarshal(b, known); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var all map[string]json.RawMessage
|
||||
if err := json.Unmarshal(b, &all); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := jsonNames(reflect.TypeOf(known).Elem())
|
||||
for k := range all {
|
||||
if names[k] {
|
||||
delete(all, k)
|
||||
}
|
||||
}
|
||||
if len(all) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
func jsonNames(t reflect.Type) map[string]bool {
|
||||
names := make(map[string]bool, t.NumField())
|
||||
for f := range t.Fields() {
|
||||
if tag, ok := f.Tag.Lookup("json"); ok {
|
||||
names[strings.Split(tag, ",")[0]] = true
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
var queueIDEncoding = base32.StdEncoding.WithPadding(base32.NoPadding)
|
||||
|
||||
// NewQueueID mints an unguessable per-create identifier: 128 random bits,
|
||||
// base32, never derived from the name or account (ADR-0011:S9).
|
||||
func NewQueueID() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", fmt.Errorf("mint queue ID: %w", err)
|
||||
}
|
||||
return queueIDEncoding.EncodeToString(b[:]), nil
|
||||
}
|
||||
|
||||
// Allocate picks a new queue's message and state shards from the counts
|
||||
// configured now (ADR-0011:S10). The result is recorded and never recomputed.
|
||||
func Allocate(queueID string, messageStreams, stateBuckets int) (message.Shard, message.Shard, error) {
|
||||
if messageStreams < 1 || stateBuckets < 1 {
|
||||
return 0, 0, fmt.Errorf("allocate shards: counts %d and %d must be positive", messageStreams, stateBuckets)
|
||||
}
|
||||
if messageStreams > math.MaxUint32 || stateBuckets > math.MaxUint32 {
|
||||
return 0, 0, fmt.Errorf("allocate shards: counts %d and %d exceed the shard range", messageStreams, stateBuckets)
|
||||
}
|
||||
sum := sha256.Sum256([]byte(queueID))
|
||||
m := binary.BigEndian.Uint32(sum[0:4]) % uint32(messageStreams)
|
||||
s := binary.BigEndian.Uint32(sum[4:8]) % uint32(stateBuckets)
|
||||
return message.Shard(m), message.Shard(s), nil
|
||||
}
|
||||
|
||||
// ValidateName enforces AWS's standard-queue name rule: 1 to 80 characters
|
||||
// of letters, digits, hyphen and underscore, case-sensitive (row A15).
|
||||
func ValidateName(name string) error {
|
||||
if name == "" || len(name) > MaxNameLen {
|
||||
return fmt.Errorf("%w: length %d outside 1..%d", ErrInvalidQueueName, len(name), MaxNameLen)
|
||||
}
|
||||
for _, r := range name {
|
||||
if !isNameRune(r) {
|
||||
return fmt.Errorf("%w: only letters, digits, hyphens and underscores are allowed", ErrInvalidQueueName)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isNameRune(r rune) bool {
|
||||
return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_'
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// An N-1 reader must write back fields an N writer added, byte for byte,
|
||||
// including inside the attribute set (ADR-0006:S4).
|
||||
func TestRecordPreservesUnknownFields(t *testing.T) {
|
||||
in := `{"v":2,"queueId":"Q1","messageShard":3,"stateShard":5,"account":"111122223333",` +
|
||||
`"region":"ap-southeast-2","name":"orders","arn":"arn:aws:sqs:ap-southeast-2:111122223333:orders",` +
|
||||
`"createdAt":"2026-10-30T00:00:00Z","type":"standard","state":"active","generation":4,` +
|
||||
`"attributes":{"DelaySeconds":"5","FutureAttribute":"on"},"attributesModifiedAt":"2026-10-30T00:00:01Z",` +
|
||||
`"fifoThroughputLimit":null,"futureBlock":{"a":[1,2,{"b":"c"}]}}`
|
||||
|
||||
r, err := DecodeRecord([]byte(in))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, message.Shard(3), r.MessageShard)
|
||||
assert.Equal(t, "on", r.Attributes["FutureAttribute"])
|
||||
|
||||
r.Generation++
|
||||
out, err := r.Encode()
|
||||
require.NoError(t, err)
|
||||
|
||||
var got map[string]json.RawMessage
|
||||
require.NoError(t, json.Unmarshal(out, &got))
|
||||
assert.JSONEq(t, `{"a":[1,2,{"b":"c"}]}`, string(got["futureBlock"]))
|
||||
assert.Equal(t, "null", string(got["fifoThroughputLimit"]))
|
||||
assert.Equal(t, "5", string(got["generation"]))
|
||||
assert.Equal(t, "2", string(got["v"]))
|
||||
}
|
||||
|
||||
func TestNewQueueIDIsStoreSafeAndUnique(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for range 1000 {
|
||||
id, err := NewQueueID()
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, id, 26)
|
||||
require.NoError(t, message.Namespace{Queue: message.QueueID(id)}.Validate())
|
||||
assert.False(t, seen[id], "duplicate queue ID %s", id)
|
||||
seen[id] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllocateIsStableAndInRange(t *testing.T) {
|
||||
counts := map[message.Shard]int{}
|
||||
for range 2000 {
|
||||
id, err := NewQueueID()
|
||||
require.NoError(t, err)
|
||||
m, s, err := Allocate(id, 16, 8)
|
||||
require.NoError(t, err)
|
||||
require.Less(t, uint32(m), uint32(16))
|
||||
require.Less(t, uint32(s), uint32(8))
|
||||
m2, s2, _ := Allocate(id, 16, 8)
|
||||
assert.Equal(t, m, m2)
|
||||
assert.Equal(t, s, s2)
|
||||
counts[m]++
|
||||
}
|
||||
assert.Len(t, counts, 16, "every message shard should receive queues")
|
||||
|
||||
_, _, err := Allocate("x", 0, 8)
|
||||
assert.Error(t, err)
|
||||
_, _, err = Allocate("x", 8, 0)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestValidateName(t *testing.T) {
|
||||
for _, ok := range []string{"a", "Orders_2026-q1", strings.Repeat("x", 80)} {
|
||||
assert.NoError(t, ValidateName(ok), ok)
|
||||
}
|
||||
for _, bad := range []string{"", strings.Repeat("x", 81), "orders.fifo", "has space", "slash/name", "ünicode"} {
|
||||
assert.ErrorIs(t, ValidateName(bad), ErrInvalidQueueName, bad)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// ReservationPeriod is how long a deleted queue's name stays reserved after
|
||||
// its deletion was requested (lifecycle 4.6 step 2).
|
||||
const ReservationPeriod = 60 * time.Second
|
||||
|
||||
// ownedBy refuses a record whose identity does not match the account it was
|
||||
// read under or cannot name a binding, so it never selects another's key.
|
||||
func ownedBy(r *Record, account string) error {
|
||||
if r.Account != account || validIdentity(r.Account, r.Region) != nil || ValidateName(r.Name) != nil {
|
||||
return fmt.Errorf("%w: queue %s record identity is inconsistent", ErrUnavailable, r.QueueID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reservedUntil is when a reserved name may be reused: the later of the
|
||||
// binding's deadline and its deleting queue's intent, if the record remains.
|
||||
// A remaining record that is inconsistent or not deleting fails closed.
|
||||
func (s *Store) reservedUntil(ctx context.Context, account, region, name string, b Binding) (time.Time, error) {
|
||||
r, _, err := s.readRecord(ctx, account, b.QueueID)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
return b.ReuseAfter, nil
|
||||
case err != nil:
|
||||
return time.Time{}, err
|
||||
case !r.names(account, region, name):
|
||||
return time.Time{}, fmt.Errorf("%w: reservation %s names queue %s of another name", ErrUnavailable, name, b.QueueID)
|
||||
}
|
||||
until, err := r.reuseAfter()
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if b.ReuseAfter.After(until) {
|
||||
return b.ReuseAfter, nil
|
||||
}
|
||||
return until, nil
|
||||
}
|
||||
|
||||
// errReusable reports a name bound to a deleting queue whose reservation has
|
||||
// lapsed, so CreateQueue may rebind it.
|
||||
var errReusable = errors.New("name reusable")
|
||||
|
||||
// reuseAfter derives when a deleting queue's name may be reused from its
|
||||
// durable deletion intent, refusing an intent it cannot trust.
|
||||
func (r *Record) reuseAfter() (time.Time, error) {
|
||||
d := r.Deletion
|
||||
if r.State != StateDeleting || d == nil || d.RequestedAt.IsZero() || d.Generation < 1 || d.Generation > r.Generation {
|
||||
return time.Time{}, fmt.Errorf("%w: queue %s has no valid deletion intent", ErrUnavailable, r.QueueID)
|
||||
}
|
||||
return d.RequestedAt.Add(ReservationPeriod), nil
|
||||
}
|
||||
|
||||
// Reserve reserves deleting queue queueID's name until its intent's deadline
|
||||
// by compare-and-swap of a binding still naming it, extending an earlier one.
|
||||
// Other bindings, or none, are left alone; the record is never written.
|
||||
func (s *Store) Reserve(ctx context.Context, account, queueID string) error {
|
||||
r, _, err := s.readRecord(ctx, account, queueID)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
return ErrQueueDoesNotExist
|
||||
case err != nil:
|
||||
return err
|
||||
}
|
||||
if err := ownedBy(&r, account); err != nil {
|
||||
return err
|
||||
}
|
||||
until, err := r.reuseAfter()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := bindingKey(r.Account, r.Region, r.Name)
|
||||
for range updateAttempts {
|
||||
b, brev, err := s.readBinding(ctx, key)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
return nil
|
||||
case err != nil:
|
||||
return err
|
||||
case b.QueueID != queueID:
|
||||
return nil
|
||||
case b.State == BindingReserved && !b.ReuseAfter.Before(until):
|
||||
return nil
|
||||
case b.State != BindingBound && b.State != BindingReserved:
|
||||
return fmt.Errorf("%w: binding %s in state %q", ErrUnavailable, key, b.State)
|
||||
case b.V > RecordVersion:
|
||||
return fmt.Errorf("%w: binding %s has schema version %d, newer than %d", ErrUnavailable, key, b.V, RecordVersion)
|
||||
}
|
||||
|
||||
nb := b
|
||||
nb.State, nb.ReuseAfter = BindingReserved, until
|
||||
enc, err := nb.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.kv.update(ctx, key, enc, brev)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if isConflict(err) {
|
||||
continue
|
||||
}
|
||||
// A lost reply is settled by the next pass: an applied write shows the
|
||||
// reservation; an unmoved or unreadable binding may still take it.
|
||||
_, grev, rerr := s.readBinding(ctx, key)
|
||||
if (rerr != nil && !errors.Is(rerr, jetstream.ErrKeyNotFound)) || (rerr == nil && grev == brev) {
|
||||
return fmt.Errorf("%w: reserve %s outcome unknown: %w", ErrUnavailable, key, err)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("%w: reserve %s lost %d races", ErrUnavailable, key, updateAttempts)
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func reserveName(f *fixture, queueID string) error {
|
||||
return f.store.Reserve(context.Background(), acct, queueID)
|
||||
}
|
||||
|
||||
func (f *fixture) binding(t *testing.T, name string) (Binding, uint64) {
|
||||
t.Helper()
|
||||
e, err := f.kv.Get(context.Background(), bindingKey(acct, region, name))
|
||||
require.NoError(t, err)
|
||||
b, err := DecodeBinding(e.Value())
|
||||
require.NoError(t, err)
|
||||
return b, e.Revision()
|
||||
}
|
||||
|
||||
// deleted creates a queue and records its deletion intent.
|
||||
func (f *fixture) deleted(t *testing.T, name string) Record {
|
||||
t.Helper()
|
||||
_, err := create(f, name, nil)
|
||||
require.NoError(t, err)
|
||||
r, err := markDeleting(f, name)
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}
|
||||
|
||||
func TestReserve(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
a := f.deleted(t, "orders")
|
||||
_, rrev := f.record(t, a.QueueID)
|
||||
|
||||
require.NoError(t, reserveName(f, a.QueueID))
|
||||
b, brev := f.binding(t, "orders")
|
||||
assert.Equal(t, BindingReserved, b.State)
|
||||
assert.Equal(t, a.QueueID, b.QueueID)
|
||||
assert.True(t, b.ReuseAfter.Equal(a.Deletion.RequestedAt.Add(ReservationPeriod)))
|
||||
_, after := f.record(t, a.QueueID)
|
||||
assert.Equal(t, rrev, after, "the queue record is not written")
|
||||
|
||||
require.NoError(t, reserveName(f, a.QueueID), "reserving again is a no-op")
|
||||
_, again := f.binding(t, "orders")
|
||||
assert.Equal(t, brev, again)
|
||||
}
|
||||
|
||||
func TestReservedNameReuse(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
a := f.deleted(t, "orders")
|
||||
require.NoError(t, reserveName(f, a.QueueID))
|
||||
|
||||
f.now = a.Deletion.RequestedAt.Add(ReservationPeriod - 1)
|
||||
_, err := create(f, "orders", nil)
|
||||
assert.ErrorIs(t, err, ErrQueueDeletedRecently)
|
||||
|
||||
f.now = a.Deletion.RequestedAt.Add(ReservationPeriod)
|
||||
b, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
assert.NotEqual(t, a.QueueID, b.QueueID)
|
||||
stored, _ := f.record(t, a.QueueID)
|
||||
assert.Equal(t, StateDeleting, stored.State, "the deleted queue's record remains")
|
||||
}
|
||||
|
||||
// Reserve and CreateQueue race on the same binding revision: one wins, and
|
||||
// the name is never bound to a replacement before the deadline.
|
||||
func TestReserveRacesCreate(t *testing.T) {
|
||||
for _, lapsed := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("lapsed=%v", lapsed), func(t *testing.T) {
|
||||
for range 10 {
|
||||
f := newFixture(t)
|
||||
a := f.deleted(t, "orders")
|
||||
if lapsed {
|
||||
f.now = a.Deletion.RequestedAt.Add(ReservationPeriod)
|
||||
}
|
||||
var rerr, cerr error
|
||||
var b Record
|
||||
var wg sync.WaitGroup
|
||||
wg.Go(func() { rerr = reserveName(f, a.QueueID) })
|
||||
wg.Go(func() { b, cerr = create(f, "orders", nil) })
|
||||
wg.Wait()
|
||||
require.NoError(t, rerr)
|
||||
got, _ := f.binding(t, "orders")
|
||||
if !lapsed {
|
||||
assert.ErrorIs(t, cerr, ErrQueueDeletedRecently)
|
||||
assert.Equal(t, a.QueueID, got.QueueID)
|
||||
assert.Equal(t, BindingReserved, got.State)
|
||||
continue
|
||||
}
|
||||
require.NoError(t, cerr)
|
||||
assert.Equal(t, b.QueueID, got.QueueID)
|
||||
assert.Equal(t, BindingBound, got.State)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReserveLostReply(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
after func(t *testing.T, f *fixture, key string, value []byte, rev uint64)
|
||||
wantErr error
|
||||
}{
|
||||
"applied": {
|
||||
after: func(t *testing.T, f *fixture, key string, value []byte, rev uint64) {
|
||||
_, err := f.fault.real.update(context.Background(), key, value, rev)
|
||||
require.NoError(t, err)
|
||||
},
|
||||
},
|
||||
"unmoved": {after: func(*testing.T, *fixture, string, []byte, uint64) {}, wantErr: ErrUnavailable},
|
||||
"re-read fails": {
|
||||
after: func(_ *testing.T, f *fixture, _ string, _ []byte, _ uint64) {
|
||||
f.fault.onGet = func(string) ([]byte, uint64, error, bool) {
|
||||
f.fault.onGet = nil
|
||||
return nil, 0, errLostReply, true
|
||||
}
|
||||
},
|
||||
wantErr: ErrUnavailable,
|
||||
},
|
||||
}
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
a := f.deleted(t, "orders")
|
||||
sends := 0
|
||||
f.fault.onUpdate = func(key string, value []byte, rev uint64) (uint64, error, bool) {
|
||||
sends++
|
||||
tc.after(t, f, key, value, rev)
|
||||
return 0, errLostReply, true
|
||||
}
|
||||
err := reserveName(f, a.QueueID)
|
||||
f.fault.onUpdate = nil
|
||||
assert.Equal(t, 1, sends)
|
||||
if tc.wantErr != nil {
|
||||
assert.ErrorIs(t, err, tc.wantErr)
|
||||
b, _ := f.binding(t, "orders")
|
||||
assert.Equal(t, BindingBound, b.State)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
b, _ := f.binding(t, "orders")
|
||||
assert.Equal(t, BindingReserved, b.State)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
|
||||
)
|
||||
|
||||
var _ message.Queues = (*Store)(nil)
|
||||
|
||||
// SendTarget resolves the active queue a message is sent to, with the delay
|
||||
// and size limit in effect now. A record whose values cannot be read is
|
||||
// unavailable rather than defaulted.
|
||||
func (s *Store) SendTarget(ctx context.Context, account, region, name string) (message.Target, error) {
|
||||
r, err := s.Resolve(ctx, account, region, name)
|
||||
if errors.Is(err, ErrQueueDoesNotExist) {
|
||||
return message.Target{}, fmt.Errorf("%w: %w", message.ErrQueueDoesNotExist, err)
|
||||
}
|
||||
if err != nil {
|
||||
return message.Target{}, fmt.Errorf("%w: %w", message.ErrUnavailable, err)
|
||||
}
|
||||
delay, err := intAttribute(r, AttrDelaySeconds)
|
||||
if err != nil {
|
||||
return message.Target{}, err
|
||||
}
|
||||
size, err := intAttribute(r, AttrMaximumMessageSize)
|
||||
if err != nil {
|
||||
return message.Target{}, err
|
||||
}
|
||||
return message.Target{Namespace: r.Namespace(), Account: r.Account, DelaySeconds: delay, MaximumMessageSize: size}, nil
|
||||
}
|
||||
|
||||
func intAttribute(r Record, name string) (int, error) {
|
||||
v, ok := r.Attributes[name]
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("%w: queue %s has no %s", message.ErrUnavailable, r.QueueID, name)
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%w: queue %s %s is not an integer", message.ErrUnavailable, r.QueueID, name)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
var _ message.ReceiveQueues = (*Store)(nil)
|
||||
|
||||
// ReceiveTarget resolves the active queue a receive reads, with the visibility
|
||||
// timeout, retention and receive wait in effect now. A record whose values
|
||||
// cannot be read is unavailable rather than defaulted.
|
||||
func (s *Store) ReceiveTarget(ctx context.Context, account, region, name string) (message.ReceiveTarget, error) {
|
||||
r, err := s.Resolve(ctx, account, region, name)
|
||||
if errors.Is(err, ErrQueueDoesNotExist) {
|
||||
return message.ReceiveTarget{}, fmt.Errorf("%w: %w", message.ErrQueueDoesNotExist, err)
|
||||
}
|
||||
if err != nil {
|
||||
return message.ReceiveTarget{}, fmt.Errorf("%w: %w", message.ErrUnavailable, err)
|
||||
}
|
||||
vis, err := intAttribute(r, AttrVisibilityTimeout)
|
||||
if err != nil {
|
||||
return message.ReceiveTarget{}, err
|
||||
}
|
||||
retention, err := intAttribute(r, AttrMessageRetentionPeriod)
|
||||
if err != nil {
|
||||
return message.ReceiveTarget{}, err
|
||||
}
|
||||
wait, err := intAttribute(r, AttrReceiveMessageWaitTimeSeconds)
|
||||
if err != nil {
|
||||
return message.ReceiveTarget{}, err
|
||||
}
|
||||
return message.ReceiveTarget{
|
||||
Namespace: r.Namespace(), Account: r.Account,
|
||||
VisibilityTimeout: vis, MessageRetentionPeriod: retention, ReceiveWaitTime: wait,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"time"
|
||||
)
|
||||
|
||||
// updateAttempts bounds how often an update retries after losing a
|
||||
// compare-and-swap to another writer.
|
||||
const updateAttempts = 5
|
||||
|
||||
// change describes one desired-state update: apply produces the record to
|
||||
// write, or an error that rejects the whole request.
|
||||
type change struct {
|
||||
op string
|
||||
apply func(r *Record) (Record, error)
|
||||
}
|
||||
|
||||
// SetAttributes applies attrs to the named queue all or nothing, advancing its
|
||||
// generation on every accepted call, same values included (ADR-0012:S3).
|
||||
func (s *Store) SetAttributes(ctx context.Context, account, region, name string, attrs map[string]string) (Record, error) {
|
||||
supplied, err := ValidateSet(attrs)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
return s.update(ctx, account, region, name, change{
|
||||
op: "SetQueueAttributes",
|
||||
apply: func(r *Record) (Record, error) {
|
||||
next := *r
|
||||
next.Attributes = maps.Clone(r.Attributes)
|
||||
maps.Copy(next.Attributes, supplied)
|
||||
next.AttributesModifiedAt = s.now().UTC().Truncate(time.Second)
|
||||
return next, nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// update applies c to the named active queue by compare-and-swap, advancing
|
||||
// its generation, and retries only a definite conflict. It is pinned to the
|
||||
// queue it first resolves, so a replacement under the name reports absent.
|
||||
func (s *Store) update(ctx context.Context, account, region, name string, c change) (Record, error) {
|
||||
var pinned string
|
||||
for range updateAttempts {
|
||||
r, rev, err := s.resolvePinned(ctx, account, region, name, pinned)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
pinned = r.QueueID
|
||||
if err := settable(&r); err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
next, err := c.apply(&r)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
next.Generation = r.Generation + 1
|
||||
enc, err := next.Encode()
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
_, err = s.kv.update(ctx, recordKey(account, r.QueueID), enc, rev)
|
||||
switch {
|
||||
case err == nil:
|
||||
return next, nil
|
||||
case isConflict(err):
|
||||
continue
|
||||
}
|
||||
// Another writer leaving the same values cannot prove this write
|
||||
// applied, so a lost reply is never settled by re-reading.
|
||||
return Record{}, fmt.Errorf("%w: %s %s outcome unknown: %w", ErrUnavailable, c.op, name, err)
|
||||
}
|
||||
return Record{}, fmt.Errorf("%w: %s %s lost %d races", ErrUnavailable, c.op, name, updateAttempts)
|
||||
}
|
||||
|
||||
// resolvePinned resolves the name and, once pinned, refuses any queue other
|
||||
// than the pinned one: the original was deleted and the name reused.
|
||||
func (s *Store) resolvePinned(ctx context.Context, account, region, name, pinned string) (Record, uint64, error) {
|
||||
r, rev, err := s.resolve(ctx, account, region, name)
|
||||
if err != nil {
|
||||
return Record{}, 0, err
|
||||
}
|
||||
if pinned != "" && r.QueueID != pinned {
|
||||
return Record{}, 0, ErrQueueDoesNotExist
|
||||
}
|
||||
return r, rev, nil
|
||||
}
|
||||
|
||||
// settable refuses a queue that is not active: a deleting queue is already
|
||||
// absent to the tenant, and a state this release does not know is not its to
|
||||
// change.
|
||||
func settable(r *Record) error {
|
||||
switch r.State {
|
||||
case StateActive:
|
||||
if r.Attributes == nil {
|
||||
return fmt.Errorf("%w: queue %s has no attributes", ErrUnavailable, r.QueueID)
|
||||
}
|
||||
return nil
|
||||
case StateDeleting:
|
||||
return ErrQueueDoesNotExist
|
||||
}
|
||||
return fmt.Errorf("%w: queue %s in state %q", ErrUnavailable, r.QueueID, r.State)
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func set(f *fixture, name string, attrs map[string]string) (Record, error) {
|
||||
return f.store.SetAttributes(context.Background(), acct, region, name, attrs)
|
||||
}
|
||||
|
||||
func (f *fixture) record(t *testing.T, queueID string) (Record, uint64) {
|
||||
t.Helper()
|
||||
e, err := f.kv.Get(context.Background(), recordKey(acct, queueID))
|
||||
require.NoError(t, err)
|
||||
r, err := DecodeRecord(e.Value())
|
||||
require.NoError(t, err)
|
||||
return r, e.Revision()
|
||||
}
|
||||
|
||||
func (f *fixture) rewrite(t *testing.T, queueID string, change func(*Record)) {
|
||||
t.Helper()
|
||||
r, rev := f.record(t, queueID)
|
||||
change(&r)
|
||||
enc, err := r.Encode()
|
||||
require.NoError(t, err)
|
||||
_, err = f.kv.Update(context.Background(), recordKey(acct, queueID), enc, rev)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestSetAttributes(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", map[string]string{AttrDelaySeconds: "5"})
|
||||
require.NoError(t, err)
|
||||
f.now = f.now.Add(time.Hour)
|
||||
|
||||
r, err := set(f, "orders", map[string]string{AttrVisibilityTimeout: "120", AttrMessageRetentionPeriod: "60"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(2), r.Generation)
|
||||
assert.True(t, r.CreatedAt.Equal(q.CreatedAt))
|
||||
assert.True(t, r.AttributesModifiedAt.Equal(f.now))
|
||||
|
||||
stored, _ := f.record(t, q.QueueID)
|
||||
assert.Equal(t, "120", stored.Attributes[AttrVisibilityTimeout])
|
||||
assert.Equal(t, "60", stored.Attributes[AttrMessageRetentionPeriod])
|
||||
assert.Equal(t, "5", stored.Attributes[AttrDelaySeconds], "unnamed attributes keep their values")
|
||||
assert.Equal(t, uint64(2), stored.Generation)
|
||||
}
|
||||
|
||||
func TestSetAttributesAllOrNothing(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
_, before := f.record(t, q.QueueID)
|
||||
|
||||
for _, attrs := range []map[string]string{
|
||||
{AttrDelaySeconds: "5", AttrVisibilityTimeout: "43201"},
|
||||
{AttrDelaySeconds: "5", "Bogus": "1"},
|
||||
{AttrDelaySeconds: "5", "RedrivePolicy": "{}"},
|
||||
{AttrDelaySeconds: "5", AttrFifoQueue: "false"},
|
||||
} {
|
||||
_, err := set(f, "orders", attrs)
|
||||
assert.Error(t, err, "%v", attrs)
|
||||
}
|
||||
stored, after := f.record(t, q.QueueID)
|
||||
assert.Equal(t, before, after, "a rejected request writes nothing")
|
||||
assert.Equal(t, "0", stored.Attributes[AttrDelaySeconds])
|
||||
}
|
||||
|
||||
// Concurrent updates of different attributes each apply exactly once.
|
||||
func TestSetAttributesConcurrent(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
updates := []map[string]string{
|
||||
{AttrDelaySeconds: "9"},
|
||||
{AttrVisibilityTimeout: "99"},
|
||||
{AttrMessageRetentionPeriod: "999"},
|
||||
{AttrMaximumMessageSize: "9999"},
|
||||
{AttrReceiveMessageWaitTimeSeconds: "9"},
|
||||
}
|
||||
errs := make([]error, len(updates))
|
||||
var wg sync.WaitGroup
|
||||
for i, u := range updates {
|
||||
wg.Go(func() { _, errs[i] = set(f, "orders", u) })
|
||||
}
|
||||
wg.Wait()
|
||||
for _, err := range errs {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
r, _ := f.record(t, q.QueueID)
|
||||
for _, u := range updates {
|
||||
for k, v := range u {
|
||||
assert.Equal(t, v, r.Attributes[k], k)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, uint64(1+len(updates)), r.Generation)
|
||||
}
|
||||
|
||||
// A DeleteQueue that commits between the read and the update wins: the
|
||||
// update's compare-and-swap is refused and the queue is reported absent.
|
||||
func TestSetAttributesLosesToDeletion(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
f.fault.onUpdate = func(key string, value []byte, rev uint64) (uint64, error, bool) {
|
||||
if !strings.HasPrefix(key, "q.") {
|
||||
return 0, nil, false
|
||||
}
|
||||
f.fault.onUpdate = nil
|
||||
f.rewrite(t, q.QueueID, func(r *Record) {
|
||||
r.State = StateDeleting
|
||||
r.Generation++
|
||||
r.Deletion = &Deletion{RequestedAt: f.now, Generation: r.Generation}
|
||||
})
|
||||
n, err := f.fault.real.update(context.Background(), key, value, rev)
|
||||
return n, err, true
|
||||
}
|
||||
_, err = set(f, "orders", map[string]string{AttrDelaySeconds: "5"})
|
||||
assert.ErrorIs(t, err, ErrQueueDoesNotExist)
|
||||
r, _ := f.record(t, q.QueueID)
|
||||
assert.Equal(t, StateDeleting, r.State)
|
||||
assert.Equal(t, "0", r.Attributes[AttrDelaySeconds])
|
||||
assert.Equal(t, uint64(2), r.Generation)
|
||||
}
|
||||
|
||||
// A lost reply is not success even when the write applied: the request
|
||||
// cannot prove its own outcome, so it fails closed and is not resent.
|
||||
func TestSetAttributesAppliedReplyLost(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
sends := 0
|
||||
f.fault.onUpdate = func(key string, value []byte, rev uint64) (uint64, error, bool) {
|
||||
sends++
|
||||
_, err := f.fault.real.update(context.Background(), key, value, rev)
|
||||
require.NoError(t, err)
|
||||
return 0, errLostReply, true
|
||||
}
|
||||
_, err = set(f, "orders", map[string]string{AttrDelaySeconds: "5"})
|
||||
assert.ErrorIs(t, err, ErrUnavailable)
|
||||
assert.Equal(t, 1, sends)
|
||||
stored, _ := f.record(t, q.QueueID)
|
||||
assert.Equal(t, uint64(2), stored.Generation, "the update applied once")
|
||||
}
|
||||
|
||||
// An update with no reply against a record that has not moved may still
|
||||
// apply, so the request fails closed; a retry applies it once.
|
||||
func TestSetAttributesUnknownOutcomeFailsClosed(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
q, err := create(f, "orders", nil)
|
||||
require.NoError(t, err)
|
||||
sends := 0
|
||||
f.fault.onUpdate = func(string, []byte, uint64) (uint64, error, bool) {
|
||||
sends++
|
||||
return 0, errLostReply, true
|
||||
}
|
||||
_, err = set(f, "orders", map[string]string{AttrDelaySeconds: "5"})
|
||||
assert.ErrorIs(t, err, ErrUnavailable)
|
||||
assert.Equal(t, 1, sends, "an unknown outcome is not resent in the same request")
|
||||
stored, _ := f.record(t, q.QueueID)
|
||||
assert.Equal(t, uint64(1), stored.Generation)
|
||||
|
||||
f.fault.onUpdate = nil
|
||||
_, err = set(f, "orders", map[string]string{AttrDelaySeconds: "5"})
|
||||
require.NoError(t, err)
|
||||
stored, _ = f.record(t, q.QueueID)
|
||||
assert.Equal(t, uint64(2), stored.Generation)
|
||||
assert.Equal(t, "5", stored.Attributes[AttrDelaySeconds])
|
||||
}
|
||||
@@ -0,0 +1,463 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// Bucket is the control-plane KV bucket holding queue records and name
|
||||
// bindings.
|
||||
const Bucket = "spinifex-sqs-queues"
|
||||
|
||||
// createAttempts bounds how often CreateQueue retries after losing a race.
|
||||
const createAttempts = 3
|
||||
|
||||
// Errors the queue owner returns for a resolvable request.
|
||||
var (
|
||||
ErrQueueDoesNotExist = errors.New("queue does not exist")
|
||||
ErrQueueNameExists = errors.New("queue name exists with different attributes")
|
||||
ErrQueueDeletedRecently = errors.New("queue name was deleted recently")
|
||||
ErrInvalidIdentity = errors.New("invalid account or region")
|
||||
// ErrUnavailable means the outcome could not be decided; nothing was
|
||||
// granted and the request may be retried.
|
||||
ErrUnavailable = errors.New("queue store unavailable")
|
||||
)
|
||||
|
||||
var (
|
||||
accountPattern = regexp.MustCompile(`^[0-9]{12}$`)
|
||||
regionPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
|
||||
)
|
||||
|
||||
// kvOps is the bucket access the owner needs. Gets go to the stream leader,
|
||||
// key listings may lag it, and writes are create-if-absent or revision CAS.
|
||||
type kvOps interface {
|
||||
get(ctx context.Context, key string) ([]byte, uint64, error)
|
||||
create(ctx context.Context, key string, value []byte) (uint64, error)
|
||||
update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error)
|
||||
delete(ctx context.Context, key string, rev uint64) error
|
||||
keys(ctx context.Context, filter string) ([]string, error)
|
||||
}
|
||||
|
||||
type jsKV struct {
|
||||
js jetstream.JetStream
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (k jsKV) get(ctx context.Context, key string) ([]byte, uint64, error) {
|
||||
return sqsjs.LeaderKVGet(ctx, k.js, Bucket, key)
|
||||
}
|
||||
|
||||
func (k jsKV) create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
return k.kv.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
func (k jsKV) update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
|
||||
return k.kv.Update(ctx, key, value, rev)
|
||||
}
|
||||
|
||||
func (k jsKV) delete(ctx context.Context, key string, rev uint64) error {
|
||||
return k.kv.Delete(ctx, key, jetstream.LastRevision(rev))
|
||||
}
|
||||
|
||||
// Store is the queue owner (ADR-0012:S1, S3). Every decision reads the
|
||||
// stream leader and writes by compare-and-swap at the revision read, so a
|
||||
// stale read or an ambiguous write grants nothing.
|
||||
type Store struct {
|
||||
kv kvOps
|
||||
// Now is the clock; nil means time.Now.
|
||||
Now func() time.Time
|
||||
// MessageStreams and StateBuckets are the shard counts new queues are
|
||||
// allocated from.
|
||||
MessageStreams int
|
||||
StateBuckets int
|
||||
}
|
||||
|
||||
// Open opens or creates the bucket, refusing one that could lose records.
|
||||
func Open(ctx context.Context, js jetstream.JetStream, messageStreams, stateBuckets int) (*Store, error) {
|
||||
kv, err := sqsjs.OpenDurableKV(ctx, js, Bucket, "SQS queue records and name bindings")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Store{kv: jsKV{js: js, kv: kv}, MessageStreams: messageStreams, StateBuckets: stateBuckets}, nil
|
||||
}
|
||||
|
||||
func recordKey(account, queueID string) string { return "q." + account + "." + queueID }
|
||||
|
||||
func bindingKey(account, region, name string) string {
|
||||
return "n." + account + "." + region + "." + name
|
||||
}
|
||||
|
||||
func (s *Store) now() time.Time {
|
||||
if s.Now != nil {
|
||||
return s.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// CreateRequest is a validated-on-entry CreateQueue.
|
||||
type CreateRequest struct {
|
||||
Account string
|
||||
Region string
|
||||
Name string
|
||||
Attributes map[string]string
|
||||
// Tags apply only when the queue is new; a repeat CreateQueue ignores
|
||||
// them, provisionally until characterised against AWS.
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
// Create creates the queue, or returns the existing one when every supplied
|
||||
// attribute matches its effective value (row A15). The record is written
|
||||
// before the binding, so a crash between them leaves a queue nothing resolves.
|
||||
func (s *Store) Create(ctx context.Context, req CreateRequest) (Record, error) {
|
||||
if err := validIdentity(req.Account, req.Region); err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
if err := ValidateName(req.Name); err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
supplied, err := ValidateCreate(req.Attributes)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
if err := ValidateTags(req.Tags, true); err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
bkey := bindingKey(req.Account, req.Region, req.Name)
|
||||
for range createAttempts {
|
||||
b, brev, err := s.readBinding(ctx, bkey)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
b, brev = Binding{}, 0
|
||||
case err != nil:
|
||||
return Record{}, err
|
||||
case b.State == BindingBound:
|
||||
r, err := s.existing(ctx, req, b, supplied)
|
||||
if !errors.Is(err, errReusable) {
|
||||
return r, err
|
||||
}
|
||||
if b.V > RecordVersion {
|
||||
return Record{}, fmt.Errorf("%w: binding %s has schema version %d, newer than %d", ErrUnavailable, bkey, b.V, RecordVersion)
|
||||
}
|
||||
case b.State == BindingReserved:
|
||||
until, err := s.reservedUntil(ctx, req.Account, req.Region, req.Name, b)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
if s.now().Before(until) {
|
||||
return Record{}, ErrQueueDeletedRecently
|
||||
}
|
||||
if b.V > RecordVersion {
|
||||
return Record{}, fmt.Errorf("%w: binding %s has schema version %d, newer than %d", ErrUnavailable, bkey, b.V, RecordVersion)
|
||||
}
|
||||
default:
|
||||
return Record{}, fmt.Errorf("%w: binding %s in state %q", ErrUnavailable, bkey, b.State)
|
||||
}
|
||||
|
||||
r, rrev, err := s.writeRecord(ctx, req, supplied)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
won, retry, err := s.bind(ctx, bkey, b, brev, r)
|
||||
switch {
|
||||
case won:
|
||||
return r, nil
|
||||
case retry:
|
||||
s.discardUnbound(ctx, r, rrev)
|
||||
continue
|
||||
default:
|
||||
return Record{}, err
|
||||
}
|
||||
}
|
||||
return Record{}, fmt.Errorf("%w: CreateQueue %s lost %d races", ErrUnavailable, req.Name, createAttempts)
|
||||
}
|
||||
|
||||
// existing answers a CreateQueue whose name is bound. A binding naming no
|
||||
// readable active record fails closed: a reconciler repairs it, not a create.
|
||||
func (s *Store) existing(ctx context.Context, req CreateRequest, b Binding, supplied Attributes) (Record, error) {
|
||||
r, _, err := s.readRecord(ctx, req.Account, b.QueueID)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
return Record{}, fmt.Errorf("%w: %s is bound to missing queue %s", ErrUnavailable, req.Name, b.QueueID)
|
||||
case err != nil:
|
||||
return Record{}, err
|
||||
case !r.names(req.Account, req.Region, req.Name):
|
||||
return Record{}, fmt.Errorf("%w: binding %s names queue %s of another name", ErrUnavailable, req.Name, b.QueueID)
|
||||
case r.State == StateDeleting:
|
||||
until, err := r.reuseAfter()
|
||||
switch {
|
||||
case err != nil:
|
||||
return Record{}, err
|
||||
case s.now().Before(until):
|
||||
return Record{}, ErrQueueDeletedRecently
|
||||
}
|
||||
return Record{}, errReusable
|
||||
case r.State != StateActive:
|
||||
return Record{}, fmt.Errorf("%w: queue %s in state %q", ErrUnavailable, r.QueueID, r.State)
|
||||
case !r.Attributes.Matches(supplied):
|
||||
return Record{}, ErrQueueNameExists
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// writeRecord creates a new queue record under a fresh queue ID. A failed or
|
||||
// unanswered write leaves a record no name resolves to yet; its binding write
|
||||
// may still land, so age alone never makes it safe to remove.
|
||||
func (s *Store) writeRecord(ctx context.Context, req CreateRequest, supplied Attributes) (Record, uint64, error) {
|
||||
id, err := NewQueueID()
|
||||
if err != nil {
|
||||
return Record{}, 0, fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
mshard, sshard, err := Allocate(id, s.MessageStreams, s.StateBuckets)
|
||||
if err != nil {
|
||||
return Record{}, 0, fmt.Errorf("%w: %w", ErrUnavailable, err)
|
||||
}
|
||||
at := s.now().UTC().Truncate(time.Second)
|
||||
r := Record{
|
||||
V: RecordVersion,
|
||||
QueueID: id,
|
||||
MessageShard: mshard,
|
||||
StateShard: sshard,
|
||||
Account: req.Account,
|
||||
Region: req.Region,
|
||||
Name: req.Name,
|
||||
ARN: arn.FormatSQSQueue(req.Region, req.Account, req.Name),
|
||||
CreatedAt: at,
|
||||
Type: TypeStandard,
|
||||
State: StateActive,
|
||||
Generation: 1,
|
||||
Attributes: supplied.WithDefaults(),
|
||||
AttributesModifiedAt: at,
|
||||
}
|
||||
if len(req.Tags) > 0 {
|
||||
r.Tags = maps.Clone(req.Tags)
|
||||
}
|
||||
enc, err := r.Encode()
|
||||
if err != nil {
|
||||
return Record{}, 0, err
|
||||
}
|
||||
rev, err := s.kv.create(ctx, recordKey(req.Account, id), enc)
|
||||
if err != nil {
|
||||
return Record{}, 0, fmt.Errorf("%w: write queue record: %w", ErrUnavailable, err)
|
||||
}
|
||||
return r, rev, nil
|
||||
}
|
||||
|
||||
// bind points the name at r by compare-and-swap at brev. It reports won when
|
||||
// the binding names r, and retry when the write provably cannot apply; any
|
||||
// other outcome fails closed and keeps r, which may yet be bound.
|
||||
func (s *Store) bind(ctx context.Context, key string, prev Binding, brev uint64, r Record) (won, retry bool, err error) {
|
||||
nb := prev
|
||||
nb.V, nb.QueueID, nb.State, nb.ReuseAfter = RecordVersion, r.QueueID, BindingBound, time.Time{}
|
||||
enc, err := nb.Encode()
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if brev == 0 {
|
||||
_, err = s.kv.create(ctx, key, enc)
|
||||
} else {
|
||||
_, err = s.kv.update(ctx, key, enc, brev)
|
||||
}
|
||||
if err == nil {
|
||||
return true, false, nil
|
||||
}
|
||||
if isConflict(err) {
|
||||
return false, true, nil
|
||||
}
|
||||
|
||||
got, grev, rerr := s.readBinding(ctx, key)
|
||||
switch {
|
||||
case rerr == nil && got.QueueID == r.QueueID:
|
||||
if got.State == BindingBound {
|
||||
return true, false, nil
|
||||
}
|
||||
case rerr == nil && grev != brev:
|
||||
return false, true, nil
|
||||
case errors.Is(rerr, jetstream.ErrKeyNotFound) && brev != 0:
|
||||
return false, true, nil
|
||||
}
|
||||
return false, false, fmt.Errorf("%w: binding %s outcome unknown: %w", ErrUnavailable, key, err)
|
||||
}
|
||||
|
||||
// discardUnbound removes a record whose binding write cannot apply. It is
|
||||
// conditional on the record's own revision; a failure leaves it for the
|
||||
// reconciler, which removes unbound records.
|
||||
func (s *Store) discardUnbound(ctx context.Context, r Record, rev uint64) {
|
||||
_ = s.kv.delete(ctx, recordKey(r.Account, r.QueueID), rev)
|
||||
}
|
||||
|
||||
// Resolve returns the active queue a name resolves to: its binding must be
|
||||
// bound and name an active record of that name (lifecycle contract 4.2).
|
||||
func (s *Store) Resolve(ctx context.Context, account, region, name string) (Record, error) {
|
||||
r, _, err := s.resolve(ctx, account, region, name)
|
||||
if err != nil {
|
||||
return Record{}, err
|
||||
}
|
||||
if r.State != StateActive {
|
||||
return Record{}, ErrQueueDoesNotExist
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// resolve returns the record a bound name names, in any state, with its
|
||||
// revision.
|
||||
func (s *Store) resolve(ctx context.Context, account, region, name string) (Record, uint64, error) {
|
||||
if err := validIdentity(account, region); err != nil {
|
||||
return Record{}, 0, err
|
||||
}
|
||||
if ValidateName(name) != nil {
|
||||
return Record{}, 0, ErrQueueDoesNotExist
|
||||
}
|
||||
b, _, err := s.readBinding(ctx, bindingKey(account, region, name))
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
return Record{}, 0, ErrQueueDoesNotExist
|
||||
case err != nil:
|
||||
return Record{}, 0, err
|
||||
case b.State != BindingBound:
|
||||
return Record{}, 0, ErrQueueDoesNotExist
|
||||
}
|
||||
r, rev, err := s.readRecord(ctx, account, b.QueueID)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
return Record{}, 0, ErrQueueDoesNotExist
|
||||
case err != nil:
|
||||
return Record{}, 0, err
|
||||
case !r.names(account, region, name):
|
||||
return Record{}, 0, ErrQueueDoesNotExist
|
||||
}
|
||||
return r, rev, nil
|
||||
}
|
||||
|
||||
// readableAttributes are every name GetQueueAttributes accepts.
|
||||
var readableAttributes = []string{
|
||||
"All", "Policy", AttrVisibilityTimeout, AttrMaximumMessageSize, AttrMessageRetentionPeriod,
|
||||
"ApproximateNumberOfMessages", "ApproximateNumberOfMessagesNotVisible", "CreatedTimestamp",
|
||||
"LastModifiedTimestamp", "QueueArn", "ApproximateNumberOfMessagesDelayed", AttrDelaySeconds,
|
||||
AttrReceiveMessageWaitTimeSeconds, "RedrivePolicy", AttrFifoQueue, "ContentBasedDeduplication",
|
||||
"KmsMasterKeyId", "KmsDataKeyReusePeriodSeconds", "DeduplicationScope", "FifoThroughputLimit",
|
||||
"RedriveAllowPolicy", AttrSqsManagedSseEnabled,
|
||||
}
|
||||
|
||||
// unservedAttributes are valid names whose values are not available yet. A
|
||||
// request needing one, including All, is refused rather than answered without
|
||||
// it.
|
||||
var unservedAttributes = []string{
|
||||
"All", "ApproximateNumberOfMessages", "ApproximateNumberOfMessagesNotVisible", "ApproximateNumberOfMessagesDelayed",
|
||||
}
|
||||
|
||||
// AttributesFor returns the named attributes r carries; names valid in AWS
|
||||
// that r does not carry, such as an unset Policy, are omitted.
|
||||
func (r *Record) AttributesFor(names []string) (map[string]string, error) {
|
||||
if err := ValidateAttributeNames(names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all := r.visibleAttributes()
|
||||
out := map[string]string{}
|
||||
for _, name := range names {
|
||||
if v, ok := all[name]; ok {
|
||||
out[name] = v
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ValidateAttributeNames rejects a name GetQueueAttributes does not accept,
|
||||
// and one whose value cannot be served yet.
|
||||
func ValidateAttributeNames(names []string) error {
|
||||
for _, name := range names {
|
||||
if !slices.Contains(readableAttributes, name) {
|
||||
return fmt.Errorf("%w: %s", ErrInvalidAttributeName, name)
|
||||
}
|
||||
if slices.Contains(unservedAttributes, name) {
|
||||
return fmt.Errorf("%w: %s cannot be read yet", ErrUnsupportedAttribute, name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Record) visibleAttributes() map[string]string {
|
||||
all := map[string]string{
|
||||
"QueueArn": r.ARN,
|
||||
"CreatedTimestamp": strconv.FormatInt(r.CreatedAt.Unix(), 10),
|
||||
"LastModifiedTimestamp": strconv.FormatInt(r.AttributesModifiedAt.Unix(), 10),
|
||||
}
|
||||
for name, v := range r.Attributes {
|
||||
if slices.Contains(readableAttributes, name) {
|
||||
all[name] = v
|
||||
}
|
||||
}
|
||||
return all
|
||||
}
|
||||
|
||||
func (r *Record) names(account, region, name string) bool {
|
||||
return r.Account == account && r.Region == region && r.Name == name
|
||||
}
|
||||
|
||||
func (s *Store) readBinding(ctx context.Context, key string) (Binding, uint64, error) {
|
||||
v, rev, err := s.kv.get(ctx, key)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return Binding{}, 0, err
|
||||
}
|
||||
return Binding{}, 0, fmt.Errorf("%w: read %s: %w", ErrUnavailable, key, err)
|
||||
}
|
||||
b, err := DecodeBinding(v)
|
||||
if err != nil {
|
||||
return Binding{}, 0, fmt.Errorf("%w: decode %s: %w", ErrUnavailable, key, err)
|
||||
}
|
||||
if !b.wellFormed() {
|
||||
return Binding{}, 0, fmt.Errorf("%w: binding %s is malformed", ErrUnavailable, key)
|
||||
}
|
||||
return b, rev, nil
|
||||
}
|
||||
|
||||
// wellFormed refuses a binding that cannot be acted on safely: a reservation
|
||||
// without a deadline must never authorise reuse of its name.
|
||||
func (b *Binding) wellFormed() bool {
|
||||
switch {
|
||||
case b.V < 1 || b.QueueID == "":
|
||||
return false
|
||||
case b.State == BindingReserved:
|
||||
return !b.ReuseAfter.IsZero()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Store) readRecord(ctx context.Context, account, queueID string) (Record, uint64, error) {
|
||||
key := recordKey(account, queueID)
|
||||
v, rev, err := s.kv.get(ctx, key)
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return Record{}, 0, err
|
||||
}
|
||||
return Record{}, 0, fmt.Errorf("%w: read %s: %w", ErrUnavailable, key, err)
|
||||
}
|
||||
r, err := DecodeRecord(v)
|
||||
if err != nil || r.QueueID != queueID {
|
||||
return Record{}, 0, fmt.Errorf("%w: decode %s", ErrUnavailable, key)
|
||||
}
|
||||
return r, rev, nil
|
||||
}
|
||||
|
||||
func validIdentity(account, region string) error {
|
||||
if !accountPattern.MatchString(account) || !regionPattern.MatchString(region) {
|
||||
return ErrInvalidIdentity
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isConflict reports a compare-and-swap the server refused: the write did
|
||||
// not apply.
|
||||
func isConflict(err error) bool {
|
||||
return errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user