gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-21 19:56:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into account-quota

Browse Source
This commit is contained in:
Ben Duncan
2026-08-20 04:01:14 +10:00
parent cfb0abc8a8 fcde6b8de3
commit ae2a1830ef
135 changed files with 2768 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e-nightly.yml
+8
View File
@@ -342,6 +342,14 @@ jobs:
| tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
# ovn-controller and ovs-vswitchd log to files, never to journald:
# without these a cross-chassis black-hole cannot be attributed to
# IPsec versus SB port binding.
mkdir -p "${ARTIFACT_DIR}/ovs-logs-node${idx}"
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo tar -C /var/log -cf - ovn openvswitch 2>/dev/null || true' \
| tar -C "${ARTIFACT_DIR}/ovs-logs-node${idx}" -xf - 2>/dev/null
done
cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
.golangci.yml
+4 -1
View File
@@ -26,7 +26,7 @@ linters:
- errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
- exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
- fatcontext # detects nested contexts in loops
- forbidigo # blocks the legacy JetStream/KV symbols
- forbidigo # blocks forbidden identifiers
- gocheckcompilerdirectives # validates go compiler directive comments (//go:)
- godot # checks that declaration comments end in a period
- goprintffuncname # checks that printf-like functions are named with f at the end
@@ -92,6 +92,8 @@ linters:
forbidigo:
analyze-types: true
forbid:
- pattern: '^panic$'
msg: panic terminates the process; return an error or explicitly justify the panic.
- pattern: '\.JetStream$'
pkg: '^github\.com/nats-io/nats\.go$'
msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
@@ -166,6 +168,7 @@ linters:
- bodyclose
- errcheck
- errchkjson
- forbidigo
- gosec
# spinifex-specific: gosec exclusions
build/systemd/spinifex-predastore.service
+4 -4
View File
@@ -1,4 +1,4 @@
# spinifex-unit-version: 2
# spinifex-unit-version: 3
[Unit]
Description=Spinifex Predastore (S3-compatible Object Storage)
PartOf=spinifex.target
@@ -7,9 +7,9 @@ Wants=spinifex-nats.service
[Service]
EnvironmentFile=-/etc/spinifex/telemetry.env
# Guest block I/O arrives with no caller trace; sample those roots at 5% so
# they don't drown request traces. Request-linked spans are always kept.
Environment=MULGA_ROOT_TRACE_RATIO=0.05
# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O
# instead, and it can see whether a trace errored. A head ratio decides before
# the error exists, so it discarded failures at the same rate as successes.
Type=simple
Slice=spinifex-system.slice
User=spinifex-storage
cmd/rds-agent/agent_test.go
+9
View File
@@ -207,6 +207,8 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
MasterUsername: "master",
MasterUserPassword: password,
Port: 5432,
ListenAddress: testListenAddress,
ClientCIDR: testClientCIDR,
DataVolumeID: "vol-data-01",
DataVolumeSerial: "voldata01",
VMGeneration: 1,
@@ -217,6 +219,13 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
const testPayloadID = "bp-0123456789abcdef"
// The customer ENI address the engine binds, and the VPC range its client
// authentication admits. writeHandoff refuses a config without either.
const (
testListenAddress = "10.0.1.20"
testClientCIDR = "10.0.0.0/16"
)
// Writes the completion receipt rds-init leaves behind, so the acknowledgement
// step finds what a finished initialization would have produced.
func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {
cmd/rds-agent/bootstrap.go
+11
View File
@@ -96,6 +96,15 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
if cfg.MasterUsername == "" {
return fmt.Errorf("bootstrap config carries no master username")
}
// Half a handoff would leave rds-init deciding what to bind and who to admit,
// and the only answers it could reach on its own are the wildcards this
// contract exists to remove.
if cfg.ListenAddress == "" {
return fmt.Errorf("bootstrap config carries no listen address for the engine to bind")
}
if cfg.ClientCIDR == "" {
return fmt.Errorf("bootstrap config carries no client CIDR to scope client authentication to")
}
if err := os.MkdirAll(dir, handoffDirMode); err != nil {
return fmt.Errorf("create handoff dir %s: %w", dir, err)
}
@@ -134,6 +143,8 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
writeEnvLine(&b, "RDS_MODE", cfg.Mode)
writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
writeEnvLine(&b, "RDS_LISTEN_ADDRESS", cfg.ListenAddress)
writeEnvLine(&b, "RDS_CLIENT_CIDR", cfg.ClientCIDR)
writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))
cmd/rds-agent/bootstrap_test.go
+10
View File
@@ -18,6 +18,8 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
Mode: handlers_rds.BootstrapModeInitialize,
Engine: "postgres",
MasterUsername: "master",
ListenAddress: "10.0.1.20",
ClientCIDR: "10.0.0.0/16",
MasterUserPassword: &password,
DBName: "appdb",
Port: 5433,
@@ -85,6 +87,8 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
Mode: handlers_rds.BootstrapModeAttach,
Engine: "postgres",
MasterUsername: "master",
ListenAddress: "10.0.1.20",
ClientCIDR: "10.0.0.0/16",
Port: 5432,
}); err != nil {
t.Fatalf("writeHandoff: %v", err)
@@ -116,6 +120,8 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
Mode: handlers_rds.BootstrapModeInitialize,
Engine: "postgres",
MasterUsername: "master",
ListenAddress: "10.0.1.20",
ClientCIDR: "10.0.0.0/16",
MasterUserPassword: &password,
Port: 5432,
}); err != nil {
@@ -137,6 +143,8 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
Mode: handlers_rds.BootstrapModeAttach,
Engine: "postgres",
MasterUsername: "master",
ListenAddress: "10.0.1.20",
ClientCIDR: "10.0.0.0/16",
Port: 5432,
ServingCertificate: "CERT",
}); err != nil {
@@ -177,6 +185,8 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
Mode: handlers_rds.BootstrapModeAttach,
Engine: "postgres",
MasterUsername: "master",
ListenAddress: "10.0.1.20",
ClientCIDR: "10.0.0.0/16",
Port: 5432,
}); err != nil {
t.Fatalf("writeHandoff: %v", err)
cmd/rds-agent/config.go
-6
View File
@@ -13,7 +13,6 @@ const (
defaultEnvFile = "/etc/spinifex-rds/agent.env"
defaultGatewayCA = "/etc/spinifex-rds/gateway-ca.pem"
defaultHandoffDir = "/run/spinifex-rds"
defaultEngineHost = "127.0.0.1"
// Where setup.sh stamps the engine the image bakes. The agent builds its
// engine implementation from this file rather than from anything delivered.
defaultEngineFile = "/etc/spinifex-rds/engine"
@@ -38,7 +37,6 @@ type config struct {
// version alone rather than clearing it.
EngineVersion string
HandoffDir string
EngineHost string
EnginePort int
PollWait time.Duration
@@ -89,7 +87,6 @@ func loadConfig(envFile string) config {
EngineFile: get("RDS_ENGINE_FILE"),
EngineVersion: get("RDS_ENGINE_VERSION"),
HandoffDir: get("RDS_HANDOFF_DIR"),
EngineHost: get("RDS_ENGINE_HOST"),
EngineBinDir: get("RDS_ENGINE_BIN"),
EngineDataDir: get("RDS_ENGINE_DATA"),
SocketDir: get("RDS_SOCKET_DIR"),
@@ -108,9 +105,6 @@ func loadConfig(envFile string) config {
if cfg.HandoffDir == "" {
cfg.HandoffDir = defaultHandoffDir
}
if cfg.EngineHost == "" {
cfg.EngineHost = defaultEngineHost
}
if cfg.EngineFile == "" {
cfg.EngineFile = defaultEngineFile
}
cmd/rds-agent/config_test.go
+2 -2
View File
@@ -26,8 +26,8 @@ func TestLoadConfig_Defaults(t *testing.T) {
if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
}
if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
if cfg.PollWait != defaultPollWait {
t.Errorf("pollWait = %v, want the built-in default", cfg.PollWait)
}
if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)
cmd/rds-agent/engine.go
+1 -3
View File
@@ -108,9 +108,7 @@ var engineLayouts = map[string]engineLayout{
dataMount: "/var/lib/postgresql",
port: 5432,
newProbe: newPostgresProbe,
newEngine: func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
return newPostgresEngine(cfg, run, startSess, probe), nil
},
newEngine: newPostgresEngineFromCatalog,
},
engineMariaDB: {
binDir: "/usr/bin",
cmd/rds-agent/engine_mariadb_test.go
+3 -2
View File
@@ -129,10 +129,11 @@ func newScriptedTestMariaDBEngine(t *testing.T, run commandRunner) *mariadbEngin
// that is registered: what a spec says about a parameter is what the guest must
// see, and an unlisted name must read as static rather than as dynamic.
func TestControlPlaneRulesFrom_TakesTheClassificationFromTheCatalog(t *testing.T) {
rules := controlPlaneRulesFrom(postgresEngineMeta)
meta := testPostgresEngineMeta(t)
rules := controlPlaneRulesFrom(meta)
for name, want := range map[string]bool{"shared_buffers": true, "work_mem": false} {
spec, ok := postgresEngineMeta.LookupParameter(name)
spec, ok := meta.LookupParameter(name)
if !ok {
t.Fatalf("the catalog no longer carries %s", name)
}
cmd/rds-agent/engine_postgres.go
+17 -16
View File
@@ -41,8 +41,13 @@ var _ engine = (*postgresEngine)(nil)
// is not there. Resolved on PATH, where the client package puts it.
const postgresProbeBinary = "pg_isready"
// The socket directory rather than loopback: the engine binds the customer ENI
// and nothing else, so there is no TCP path for the probe to take. It also keeps
// liveness independent of the network path, of the pg_hba scope and of the TLS
// enforcement rule — libpq reads a host beginning with / as a socket directory,
// and the generated `local ... peer` line covers it.
func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.EngineHost, run))
return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.SocketDir, run))
}
func postgresProbeState(host string, run probeRunner) probeStateFn {
@@ -65,19 +70,6 @@ func postgresProbeState(host string, run probeRunner) probeStateFn {
}
}
// Resolved once, under the name the control plane knows this implementation by.
// A lookup failure is a mismatch between the agent and the control plane it was
// built against, so it fails at startup rather than at the first rotation.
var postgresEngineMeta = mustLookupEngine(enginePostgres)
func mustLookupEngine(name string) handlers_rds.Engine {
engine, err := handlers_rds.LookupEngine(name)
if err != nil {
panic("rds-agent: " + err.Error())
}
return engine
}
// The include the resolved set is rendered to, and the copy of the last one the
// engine accepted. Both live beside the data rather than in /etc, matching
// rds-init: a class change boots a fresh root volume and would revert them.
@@ -109,9 +101,18 @@ hostnossl all all ::/0 reject
postgresForceSSLRuleCount = 2
)
func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
// The layout's factory resolves the control plane metadata during startup.
func newPostgresEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
meta, err := handlers_rds.LookupEngine(enginePostgres)
if err != nil {
return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", enginePostgres, err)
}
return newPostgresEngine(cfg, meta, run, startSess, probe), nil
}
func newPostgresEngine(cfg config, meta handlers_rds.Engine, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
return &postgresEngine{
meta: postgresEngineMeta,
meta: meta,
run: run,
startSess: startSess,
psql: filepath.Join(cfg.EngineBinDir, "psql"),
cmd/rds-agent/engine_test.go
+23 -13
View File
@@ -140,10 +140,19 @@ func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
return r.out, r.err
}
func testPostgresEngineMeta(t *testing.T) handlers_rds.Engine {
t.Helper()
meta, err := handlers_rds.LookupEngine(enginePostgres)
if err != nil {
t.Fatalf("LookupEngine(%s): %v", enginePostgres, err)
}
return meta
}
func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
t.Helper()
cfg := testEngineConfig(t)
return newPostgresEngine(cfg, withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
return newPostgresEngine(cfg, testPostgresEngineMeta(t), withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
}
// The same engine with the read-backs left to the runner, for the cases that are
@@ -152,7 +161,7 @@ func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
func newScriptedTestEngine(t *testing.T, run commandRunner) *postgresEngine {
t.Helper()
cfg := testEngineConfig(t)
return newPostgresEngine(cfg, run, nil, newPostgresProbe(cfg, staticProbe(0)))
return newPostgresEngine(cfg, testPostgresEngineMeta(t), run, nil, newPostgresProbe(cfg, staticProbe(0)))
}
func testEngineConfig(t *testing.T) config {
@@ -479,10 +488,11 @@ func (r *tlsReadBackRunner) run(ctx context.Context, c command) (string, error)
return r.recordingRunner.run(ctx, c)
}
func forceSSLParameter(value string) []handlers_rds.Parameter {
func forceSSLParameter(t *testing.T, value string) []handlers_rds.Parameter {
t.Helper()
return []handlers_rds.Parameter{
{Name: "work_mem", Value: "4096"},
{Name: postgresEngineMeta.TLSEnforcementParameter(), Value: value},
{Name: testPostgresEngineMeta(t).TLSEnforcementParameter(), Value: value},
}
}
@@ -492,7 +502,7 @@ func TestPostgresEngine_ApplyParametersWritesTheEnforcementRule(t *testing.T) {
runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
engine := newScriptedTestEngine(t, runner.run)
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
t.Fatalf("ApplyParameters: %v", err)
}
rule, err := os.ReadFile(engine.forceSSLRulePath())
@@ -516,7 +526,7 @@ func TestPostgresEngine_ApplyParametersRemovesAStaleEnforcementRule(t *testing.T
t.Fatalf("write the rule the restored volume carried: %v", err)
}
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err != nil {
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
t.Fatalf("ApplyParameters: %v", err)
}
if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
@@ -547,7 +557,7 @@ func TestPostgresEngine_ApplyParametersRefusesAnUnparsableEnforcementValue(t *te
runner := &tlsReadBackRunner{ssl: "on"}
engine := newScriptedTestEngine(t, runner.run)
_, err := engine.ApplyParameters(t.Context(), forceSSLParameter("yes"))
_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "yes"))
if err == nil || !strings.Contains(err.Error(), "neither 1 nor 0") {
t.Fatalf("ApplyParameters error = %v, want a refusal naming the unreadable value", err)
}
@@ -562,7 +572,7 @@ func TestPostgresEngine_ApplyParametersRefusesToEnforceWithoutTLS(t *testing.T)
runner := &tlsReadBackRunner{ssl: "off"}
engine := newScriptedTestEngine(t, runner.run)
_, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1"))
_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1"))
if err == nil || !strings.Contains(err.Error(), "serving without TLS") {
t.Fatalf("ApplyParameters error = %v, want a refusal naming the engine's own TLS state", err)
}
@@ -592,7 +602,7 @@ func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing
runner := &tlsReadBackRunner{ssl: "on", ruleRows: tc.ruleRows, brokenRows: tc.brokenRows}
engine := newScriptedTestEngine(t, runner.run)
_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(tc.value))
_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, tc.value))
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("ApplyParameters error = %v, want %q", err, tc.want)
}
@@ -606,13 +616,13 @@ func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing
func TestPostgresEngine_ApplyParametersWithdrawsBothFilesTogether(t *testing.T) {
runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
engine := newScriptedTestEngine(t, runner.run)
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
t.Fatalf("first ApplyParameters: %v", err)
}
// Turning enforcement off, verified against a pg_hba that still carries the
// rule — so the apply fails after both files have already been replaced.
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err == nil {
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err == nil {
t.Fatal("ApplyParameters succeeded against an unverifiable enforcement")
}
installed, err := os.ReadFile(engine.params.installedPath())
@@ -639,14 +649,14 @@ func TestPostgresEngine_RestoreLastKnownGoodReDerivesEnforcement(t *testing.T) {
runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
engine := newScriptedTestEngine(t, runner.run)
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
t.Fatalf("ApplyParameters(enforcing): %v", err)
}
if err := engine.RecordServingParameters(t.Context()); err != nil {
t.Fatalf("RecordServingParameters: %v", err)
}
runner.ruleRows = 0
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err != nil {
if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
t.Fatalf("ApplyParameters(not enforcing): %v", err)
}
if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
cmd/rds-agent/health_test.go
+33 -2
View File
@@ -110,6 +110,37 @@ func TestExecProbeRunner_CapturesTheClientsStderr(t *testing.T) {
}
}
func testProbeConfig() config {
return config{EngineHost: defaultEngineHost, EnginePort: engineLayouts[enginePostgres].port}
// The engine binds the customer ENI and nothing else, so a loopback probe would
// report a serving engine as absent. libpq reads a host beginning with / as a
// socket directory, which is also what keeps liveness off the network path.
func TestPostgresProbe_ProbesTheUnixSocket(t *testing.T) {
cfg := testProbeConfig()
var probed []string
probe := newPostgresProbe(cfg, func(_ context.Context, name string, args ...string) (int, string, error) {
probed = append([]string{name}, args...)
return 0, "", nil
})
if state, _ := probe.state(context.Background()); state != engineServing {
t.Fatalf("probe state = %v, want engineServing", state)
}
host := ""
for i, arg := range probed {
if arg == "-h" && i+1 < len(probed) {
host = probed[i+1]
}
}
if host != cfg.SocketDir {
t.Errorf("probe host = %q, want the socket directory %q", host, cfg.SocketDir)
}
if !strings.HasPrefix(host, "/") {
t.Errorf("probe host = %q, which libpq reads as a TCP host rather than a socket directory", host)
}
}
func testProbeConfig() config {
return config{
SocketDir: engineLayouts[enginePostgres].socketDir,
EnginePort: engineLayouts[enginePostgres].port,
}
}
cmd/spinifex/cmd/admin.go
+2 -6
View File
@@ -415,15 +415,11 @@ func init() {
imagesRemoveCmd.Flags().String("image-id", "", "AMI ID to remove (required)")
imagesRemoveCmd.Flags().Bool("force", false, "Bypass dependency, ownership and config-corrupt checks (salvage mode)")
imagesRemoveCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
if err := imagesRemoveCmd.MarkFlagRequired("image-id"); err != nil {
panic(err)
}
_ = imagesRemoveCmd.MarkFlagRequired("image-id")
imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
if err := imagesPromoteCmd.MarkFlagRequired("image-id"); err != nil {
panic(err)
}
_ = imagesPromoteCmd.MarkFlagRequired("image-id")
}
const bytesPerGiB = 1024 * 1024 * 1024
cmd/spinifex/cmd/admin_acm.go
+2 -6
View File
@@ -60,16 +60,12 @@ func init() {
certCreateTenantCACmd.Flags().StringSlice("domain", nil, "Domain the tenant CA is permitted to sign for (repeatable)")
certCreateTenantCACmd.Flags().Bool("regenerate", false, "Replace an existing tenant CA with a fresh root (invalidates every device's trust store — requires confirmation)")
certCreateTenantCACmd.Flags().Bool("yes", false, "Skip the --regenerate confirmation prompt")
if err := certCreateTenantCACmd.MarkFlagRequired("domain"); err != nil {
panic(err)
}
_ = certCreateTenantCACmd.MarkFlagRequired("domain")
certCmd.AddCommand(certForceRenewCmd)
certForceRenewCmd.Flags().String("arn", "", "Certificate ARN to force-renew (required)")
certForceRenewCmd.Flags().String("account", "", "Account ID that owns the certificate; defaults to the bootstrap account")
if err := certForceRenewCmd.MarkFlagRequired("arn"); err != nil {
panic(err)
}
_ = certForceRenewCmd.MarkFlagRequired("arn")
}
func runCertForceRenew(cmd *cobra.Command, _ []string) {
cmd/spinifex/cmd/admin_ochre.go
+2 -6
View File
@@ -1138,15 +1138,11 @@ func init() {
c.Flags().String("account-id", "", "12-digit account ID to change (required)")
c.Flags().String("model-id", "", "Model ID to change (e.g. meta.llama3-2-1b-instruct-v1:0)")
c.Flags().Bool("all-models", false, "Apply to every model in the platform catalog")
if err := c.MarkFlagRequired("account-id"); err != nil {
panic(err)
}
_ = c.MarkFlagRequired("account-id")
}
adminOchreAccessListCmd.Flags().String("account-id", "", "12-digit account ID to inspect (required)")
if err := adminOchreAccessListCmd.MarkFlagRequired("account-id"); err != nil {
panic(err)
}
_ = adminOchreAccessListCmd.MarkFlagRequired("account-id")
}
// ochreAccessStore connects to the cluster and returns the grant store along
docs/COMMANDS.md
+2
View File
@@ -904,6 +904,8 @@ Spinifex both stores externally-issued certificates (`import-certificate`) and i
Each DB instance is one dedicated system-owned VM running the engine directly, launched from the `spinifex-rds-postgres` or `spinifex-rds-mariadb` AMI, tagged `spinifex:managed-by=rds` and therefore hidden from the customer's EC2 API. The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so **the endpoint is private — reachable from inside the VPC only**, from any subnet of it rather than the endpoint ENI's own. `Endpoint.Address` is `{db-instance-identifier}.{account-id}.{region}.rds.{base-domain}` where northstar is configured, and the endpoint ENI's private IP where it is not; the IP is stable across VM replacement either way.
**The endpoint is the only path to the engine.** The DB VM has two other NICs that no customer security group governs — a system NIC in a shared RDS system VPC, which is how the in-guest agent reaches the control plane, and a management NIC on the host's management bridge — and the engine binds neither. It binds the endpoint ENI's address alone, so the port is not open on them at all rather than open and gated. The system NIC additionally carries a dedicated region-wide security group that authorizes no ingress; agent traffic is outbound and its replies are matched by connection state. PostgreSQL's generated `pg_hba.conf` admits the customer VPC's address range and nothing wider, so a client outside the VPC is refused authentication even if it somehow reached the socket. MariaDB has no equivalent file: its analogue is the host pattern on a grant, which lives in the data volume and would follow a restore into another VPC, so for that engine the bind is the whole boundary.
Everything below applies to both engines unless it names one. `Engine` is fixed at create — there is no in-place engine change, no cross-engine snapshot restore and no migration between the two.
| | `postgres` | `mariadb` |
docs/README.md
+16
View File
@@ -3,3 +3,19 @@
This directory contains the Spinifex documentation. This is available in individual markdown files, and a generated website to aid navigation and browsing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/)
The [AWS model operation coverage](aws-model-operation-coverage.md) report is generated from Spinifex's dispatch tables and the pinned AWS API models. Run `make aws-model-coverage` to regenerate and print it.
## Frontmatter
Each published page lives at `<category>/<slug>/README.md` and carries YAML frontmatter that the docs site turns into page metadata. Three fields have length rules, because search engines flag a title tag or meta description that falls outside them:
| Field | Renders as | Length |
| ------------- | ----------------------------------------- | ------------------ |
| `title` | Page heading, sidebar label, landing card | Short — it is UI text |
| `seoTitle` | The browser and search-result title | 50-60 characters |
| `description` | The search-result snippet, not shown on the page | 150-160 characters |
`seoTitle` is used exactly as written and must end with ` — Spinifex Docs`. Leave it out and the title falls back to `title` plus that suffix, which is normally too short to satisfy the rule.
Keep the two titles distinct. `title` is navigation text in a narrow sidebar, so stretching it to reach 50 characters damages the layout to satisfy a rule that only applies to the title tag. Put the search-facing wording in `seoTitle`.
A new page is only published once its path is added to `scripts/docs-config.json` in the `mulga-docs` repo. That repo's `scripts/build-docs.js` warns about any field outside its range.
docs/admin/setting-up-your-cluster/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Setting Up Your Cluster"
description: "Import an AMI, create a VPC with a public subnet, and launch your first EC2 instance on Spinifex."
seoTitle: "Set Up Your First Spinifex Cluster — Spinifex Docs"
description: "Import an AMI, create an SSH key pair and a VPC with a public subnet, then launch your first EC2 instance and connect to it on a fresh Spinifex cluster."
category: "Admin"
tags:
- setup
docs/admin/spinifex-admin-cli/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Spinifex Admin CLI"
description: "Complete reference for the Spinifex administration CLI. Manage accounts, nodes, VMs, and services."
seoTitle: "spx: The Spinifex Administration CLI — Spinifex Docs"
description: "Complete reference for spx, the Spinifex admin CLI: initialise a cluster, manage accounts and nodes, drive the VM lifecycle, and start or stop services."
category: "Admin"
tags:
- cli
docs/admin/update/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Updating Spinifex"
description: "Upgrade an existing Spinifex installation to the latest release."
seoTitle: "Update Spinifex to the Latest Release — Spinifex Docs"
description: "Upgrade an existing Spinifex install with the same installer used to deploy it, or take the manual path to review configuration migrations before applying them."
category: "Admin"
tags:
- update
docs/compute/gpu-passthrough/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "GPU Passthrough"
description: "Configure VFIO GPU passthrough on a Spinifex node to expose NVIDIA or AMD GPUs to EC2 instances."
seoTitle: "VFIO GPU Passthrough for EC2 Instances — Spinifex Docs"
description: "Configure VFIO GPU passthrough on a Spinifex node to bind NVIDIA or AMD GPUs to guest VMs and expose GPU-enabled EC2 instance types to your workloads."
category: "Compute"
tags:
- gpu
docs/compute/launching-instances/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Launching Instances"
description: "Launch, manage, and connect to EC2-compatible virtual machines on Spinifex."
seoTitle: "Launching AWS EC2 Instances on Spinifex — Spinifex Docs"
description: "Launch, manage, and connect to EC2-compatible virtual machines on Spinifex, with cloud-init, SSH key injection, VPC networking, and AWS lifecycle operations."
category: "Compute"
tags:
- ec2
docs/compute/placement-groups/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Placement Groups"
description: "Create and manage spread and cluster placement groups for hardware-level instance placement control."
seoTitle: "AWS EC2 Placement Groups on Spinifex — Spinifex Docs"
description: "Create and manage spread and cluster placement groups to control how Spinifex places EC2 instances across physical hosts for fault isolation or low latency."
category: "Compute"
tags:
- ec2
docs/compute/vpc-networking/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "VPC Networking"
description: "How Spinifex implements AWS-compatible VPC networking with public and private subnets, security groups, and Elastic IPs using OVN."
seoTitle: "AWS-Compatible VPC Networking with OVN — Spinifex Docs"
description: "How Spinifex implements AWS-compatible VPC networking on bare metal with OVN: public and private subnets, security groups, route tables, and Elastic IPs."
category: "Compute"
tags:
- vpc
docs/containers/ecr/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "ECR (Container Registry)"
description: "Store and serve container images from Spinifex's AWS-compatible Elastic Container Registry — create a repository, authenticate Docker, push and pull images, and let your EKS workers pull from it, using the AWS CLI, the Spinifex console, or Terraform."
seoTitle: "AWS ECR Container Registry on Spinifex — Spinifex Docs"
description: "Store and serve container images from Spinifex's AWS-compatible ECR: create a repository, authenticate Docker, push and pull images, and let EKS workers pull."
category: "Containers"
tags:
- ecr
docs/containers/ecs/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "ECS (Elastic Container Service)"
description: "Run AWS-compatible ECS workloads on Spinifex — create a cluster, register a task definition, boot container instances from the ECS node image, run tasks, and front a long-running service with an Application Load Balancer, all via the AWS CLI, the Spinifex console, or Terraform."
seoTitle: "AWS ECS Elastic Container Service Guide — Spinifex Docs"
description: "Create an ECS cluster on Spinifex, register a task definition, boot container instances, run tasks, and front a service with an Application Load Balancer."
category: "Containers"
tags:
- ecs
docs/containers/eks/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "EKS (Managed Kubernetes)"
description: "Run a managed, AWS-compatible Kubernetes cluster on Spinifex — provision the control plane and worker node group, wire up the VPC, IAM, and security groups it needs, and deploy your first workload via the AWS CLI, the Spinifex console, or Terraform."
seoTitle: "AWS EKS Managed Kubernetes on Spinifex — Spinifex Docs"
description: "Provision an AWS-compatible EKS control plane and managed node group on Spinifex, wire up the VPC, IAM, and security groups, then deploy your first workload."
category: "Containers"
tags:
- eks
docs/install/install-airgapped/README.md
+1
View File
@@ -1,5 +1,6 @@
---
title: "Air-Gapped Install"
seoTitle: "Install Spinifex in an Air-Gapped Network — Spinifex Docs"
description: "Deploy Spinifex in environments without internet connectivity. Covers using a pre-built release tarball, mirrored APT packages, and locally-staged cloud images."
category: "Install"
tags:
docs/install/install-multi-node/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Multi-Node Install"
description: "Deploy Spinifex across multiple servers to create an availability zone with high availability, data durability, and fault tolerance."
seoTitle: "Multi-Node Spinifex Cluster Install — Spinifex Docs"
description: "Deploy Spinifex across several servers to form an availability zone with automatic cluster formation, high availability, data durability, and fault tolerance."
category: "Install"
tags:
- install
docs/install/install-source/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Source Install"
description: "Build Spinifex from source for development, custom builds, or contributing."
seoTitle: "Build and Install Spinifex from Source — Spinifex Docs"
description: "Build Spinifex from source on Ubuntu or Debian for development, custom builds, or contributing changes, then install and run the resulting binaries locally."
category: "Install"
tags:
- install
docs/install/install-usb/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Bootable USB Install"
description: "Install Spinifex on bare-metal hardware by flashing the Spinifex ISO to a USB drive and booting the target device from it."
seoTitle: "Install Spinifex from a Bootable USB — Spinifex Docs"
description: "Install Spinifex on bare-metal x86 hardware by flashing the Spinifex ISO to a USB drive, booting the target server from it, and wiping the disk you select."
category: "Install"
sections:
- overview
docs/install/install/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Single-Node Install"
description: "Install Spinifex on a single server using the binary installer."
seoTitle: "Install Spinifex on a Single Server — Spinifex Docs"
description: "Install Spinifex on one Ubuntu or Debian server with the binary installer and get an AWS-compatible EC2, S3, and VPC stack running on your own hardware."
category: "Install"
tags:
- install
docs/migration/hybrid-sync/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Hybrid Sync"
description: "Synchronize data between Spinifex and AWS when network connectivity is available."
seoTitle: "Hybrid Sync Between Spinifex and AWS — Spinifex Docs"
description: "Synchronise data bidirectionally between Spinifex and AWS when connectivity allows, so local infrastructure stays usable at intermittently connected sites."
category: "Migration"
tags:
- hybrid
docs/migration/moving-aws-workload/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Moving an AWS Workload to Mulga"
description: "Migrate existing AWS workloads to Spinifex using compatible APIs, SDKs, and Terraform."
seoTitle: "Migrate an AWS Workload to Spinifex — Spinifex Docs"
description: "Move existing AWS workloads onto Spinifex using compatible APIs, SDKs, and Terraform across EC2, VPC, EBS, S3, IAM, STS, ELBv2, ACM, ECR, ECS, and EKS."
category: "Migration"
tags:
- migration
docs/reference-architectures/OnLogic-HX401-single-node/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Single-Node Spinifex on OnLogic HX401"
description: "Run a complete EC2, EBS and S3 stack on one fanless OnLogic edge node, and measure what the platform costs against bare metal."
seoTitle: "Single-Node Spinifex on OnLogic HX401 — Spinifex Docs"
description: "Run a complete EC2, EBS, and S3 stack on one fanless OnLogic HX401 edge node, and measure what the platform layer costs compared with bare-metal baselines."
category: "Reference Architectures"
tags:
- onlogic
docs/reference-architectures/RTX-Pro-6000-EKS/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Spinifex EKS AI Platform on Dual RTX Pro 6000 Baremetal"
description: "Deploy a GPU-accelerated AI inference platform — an OpenAI-compatible LLM API and a real-time CV stream — in Kubernetes on bare-metal hardware, managed entirely with standard AWS tooling."
seoTitle: "EKS AI Platform on Dual RTX Pro 6000 — Spinifex Docs"
description: "Deploy a GPU-accelerated AI inference platform — an OpenAI-compatible LLM API and a real-time CV stream — on Kubernetes on bare metal with standard AWS tooling."
category: "Reference Architectures"
tags:
- nvidia
docs/reference-architectures/SMCI-H14-orchestration/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Multi-Tenant AI on Supermicro H14 with AMD MI350X"
description: "Provision isolated GPU VMs for simultaneous AI workloads on a Supermicro H14 bare-metal node using Spinifex's EC2-compatible API."
seoTitle: "Multi-Tenant AI on Supermicro H14 MI350X — Spinifex Docs"
description: "Provision isolated GPU VMs for simultaneous AI workloads on a Supermicro H14 bare-metal node with AMD MI350X GPUs, using Spinifex's EC2-compatible API."
category: "Reference Architectures"
tags:
- supermicro
docs/reference-architectures/SMCI-MIG-orchestration/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Mixed AI Workloads on a Single H200 Chassis: Guest-Managed MIG"
description: "Install Spinifex from source, configure host-local networking, attach Predastore storage, and run four concurrent AI workloads across MIG partitions managed inside guest VMs."
seoTitle: "Guest-Managed MIG on an H200 Chassis — Spinifex Docs"
description: "Install Spinifex from source, configure host-local networking, attach Predastore storage, and run four concurrent AI workloads across guest-managed MIG slices."
category: "Reference Architectures"
tags:
- nvidia
docs/reference-architectures/arcee-trinity-mi350x/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Arcee Trinity 400B FP8 on AMD MI350X with Spinifex"
description: "Provision, load, and benchmark arcee-ai/Trinity-Large-Preview-FP8 across two AMD MI350X GPUs on a Supermicro H14 using Spinifex's EC2-compatible API."
seoTitle: "Arcee Trinity 400B FP8 on AMD MI350X — Spinifex Docs"
description: "Provision, load, and benchmark arcee-ai/Trinity-Large-Preview-FP8 across two AMD MI350X GPUs on a Supermicro H14 node using Spinifex's EC2-compatible API."
category: "Reference Architectures"
tags:
- supermicro
docs/reference-architectures/cisco-ucs-llm-serving/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "vLLM Serving on Cisco UCS: Intel AMX vs NVIDIA L4"
description: "Qwen2.5-7B-Instruct served with vLLM on the same Cisco UCS Spinifex cluster, comparing Intel AMX-accelerated CPU serving against NVIDIA L4 GPU serving across input length and concurrency."
seoTitle: "vLLM on Cisco UCS: Intel AMX vs NVIDIA L4 — Spinifex Docs"
description: "Qwen2.5-7B-Instruct served with vLLM on a Cisco UCS Spinifex cluster, comparing Intel AMX-accelerated CPU serving with NVIDIA L4 GPU serving under concurrency."
category: "Reference Architectures"
tags:
- cisco
docs/reference-architectures/cisco-ucs-platform-benchmark/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Spinifex Platform Benchmark on Cisco UCS (3-Node)"
description: "Measured CPU, disk, network, and S3 performance of Spinifex on a 3-node Cisco Unified Edge cluster — host versus guest, Intel AMX confirmed executing at the ISA level, and EC2 instance worker concurrency limits."
seoTitle: "Spinifex Benchmark on 3-Node Cisco UCS — Spinifex Docs"
description: "Measured CPU, disk, network, and S3 performance of Spinifex on a 3-node Cisco Unified Edge cluster, comparing host against guest with Intel AMX confirmed."
category: "Reference Architectures"
tags:
- cisco
docs/reference-architectures/cisco-ucs-vision-pipeline/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Spinifex Vision Pipeline on Cisco UCS"
description: "A YOLO11m object-detection + Qwen2-VL scene-captioning pipeline on Spinifex, streaming from a shared Predastore bucket across two independent EC2-compatible instances, with Intel AMX and NVIDIA L4 acceleration compared head-to-head."
seoTitle: "Spinifex Vision Pipeline on Cisco UCS — Spinifex Docs"
description: "A YOLO11m detection and Qwen2-VL captioning pipeline streaming from a shared Predastore bucket across two EC2 instances, with Intel AMX and NVIDIA L4 compared."
category: "Reference Architectures"
tags:
- cisco
docs/security/flaw-remediation-policy/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Flaw Remediation Policy"
description: "CVSS-tiered SLAs for identifying, reporting, and correcting software flaws in Spinifex"
seoTitle: "Spinifex Flaw Remediation Policy (CVSS) — Spinifex Docs"
description: "CVSS-tiered SLAs for identifying, reporting, and correcting software flaws in Spinifex and its direct dependencies, for maintainers and CMMC Level 1 operators."
category: "Security"
sections:
- overview
docs/security/malware-protection/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Malware Protection"
description: "Operator guide for host-based malware protection and file integrity monitoring on Spinifex nodes"
seoTitle: "Malware Protection for Spinifex Nodes — Spinifex Docs"
description: "Operator guide to host-based malware protection and file integrity monitoring on the Linux hosts running Spinifex services, aligned to CMMC Level 1 needs."
category: "Security"
sections:
- overview
docs/security/media-sanitization/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Media Sanitization and Disposal"
description: "Operator guide for sanitizing and disposing of storage media used by Spinifex nodes"
seoTitle: "Media Sanitization and Disposal Guide — Spinifex Docs"
description: "Operator guide to sanitizing and disposing of storage media used by Spinifex nodes, covering system disks, Viperblock and Predastore volumes, and key tokens."
category: "Security"
sections:
- overview
docs/security/network-connections/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "External Connection Inventory"
description: "Operator inventory of inbound listeners and outbound connections on Spinifex nodes"
seoTitle: "Spinifex External Connection Inventory — Spinifex Docs"
description: "Operator inventory of every inbound listener and outbound connection on Spinifex nodes, with ports, protocols, and purpose, for documented CMMC Level 1 sites."
category: "Security"
sections:
- overview
docs/security/physical-security-guide/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Physical Security Operator Guide"
description: "Operator guide for physical access controls, visitor handling, access logging, and access-device management at sites hosting Spinifex nodes"
seoTitle: "Physical Security Guide for Operators — Spinifex Docs"
description: "Operator guide to physical access controls, visitor handling, access logging, and access-device management at sites hosting Spinifex nodes and network gear."
category: "Security"
sections:
- overview
docs/terraform-workbooks/bastion-private-subnet/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Bastion with Private Subnet"
description: "Deploy a bastion host to access an isolated private compute instance with no internet connectivity."
seoTitle: "Terraform Bastion Host and Private Subnet — Spinifex Docs"
description: "Deploy a VPC with public and private subnets on Spinifex, then use a bastion host as the only route to an isolated EC2 instance that has no internet access."
category: "Terraform Workbooks"
tags:
- terraform
docs/terraform-workbooks/ecs-quickstart/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "ECS Quickstart"
description: "Stand up a complete AWS-compatible ECS stack on Spinifex with Terraform — a VPC, IAM roles, a cluster, a task definition with a task role, container instances launched from the ECS node image, and an awsvpc service fronted by an Application Load Balancer."
seoTitle: "Terraform ECS Quickstart on Spinifex — Spinifex Docs"
description: "Stand up a full AWS-compatible ECS stack with Terraform: a VPC, IAM roles, a cluster, a task definition, container instances, and a load-balanced service."
category: "Terraform Workbooks"
tags:
- terraform
docs/terraform-workbooks/eks-gitops-argocd/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "GitOps on EKS (Argo CD + EBS-CSI)"
description: "Deliver a Spinifex-themed app to EKS with GitOps — the Argo CD addon syncs it from a git repo, an EBS-CSI (Viperblock) PersistentVolume holds its state, and it is served over HTTPS through the AWS Load Balancer Controller + ACM, using Terraform on Spinifex."
seoTitle: "GitOps on EKS with Argo CD and EBS-CSI — Spinifex Docs"
description: "Deliver an app to EKS with GitOps: the Argo CD addon syncs it from git, an EBS-CSI PersistentVolume holds its state, and HTTPS is served via LBC and ACM."
category: "Terraform Workbooks"
tags:
- terraform
docs/terraform-workbooks/eks-https-ingress/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "EKS HTTPS Ingress (LBC + ACM)"
description: "Serve a Spinifex-themed demo app over HTTPS on EKS using the AWS Load Balancer Controller addon and an ACM certificate — an internet-facing ALB provisioned from a Kubernetes Ingress, using Terraform on Spinifex."
seoTitle: "EKS HTTPS Ingress with LBC and ACM — Spinifex Docs"
description: "Serve a demo app over HTTPS on EKS using the AWS Load Balancer Controller addon and an ACM certificate, with an internet-facing ALB built from an Ingress."
category: "Terraform Workbooks"
tags:
- terraform
docs/terraform-workbooks/eks-quickstart/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "EKS Quickstart"
description: "Stand up a minimal managed-Kubernetes cluster and a browsable Spinifex-themed demo app — VPC, IAM roles, an EKS cluster, a one- or three-worker node group, an ECR repository, and a load-balanced web page — using Terraform on Spinifex."
seoTitle: "Terraform EKS Quickstart on Spinifex — Spinifex Docs"
description: "Stand up a minimal managed Kubernetes cluster with Terraform: a VPC, IAM roles, an EKS cluster, a worker node group, an ECR repository, and a demo web app."
category: "Terraform Workbooks"
tags:
- terraform
docs/terraform-workbooks/nginx-alb/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Nginx Web Server (Load Balanced)"
description: "Deploy a VPC with two private EC2 instances running Nginx, fronted by an internet-facing ALB using Terraform on Spinifex."
seoTitle: "Load-Balanced Nginx Behind an AWS ALB — Spinifex Docs"
description: "Deploy a VPC with two private EC2 instances running Nginx behind an internet-facing Application Load Balancer on Spinifex, using Terraform or OpenTofu."
category: "Terraform Workbooks"
tags:
- terraform
docs/terraform-workbooks/nginx-webserver/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Nginx Web Server"
description: "Deploy a VPC with a public subnet and an EC2 instance running Nginx using Terraform on Spinifex."
seoTitle: "Terraform Nginx Web Server on Spinifex — Spinifex Docs"
description: "Provision a VPC, public subnet, internet gateway, route table, security group, and an EC2 instance that installs and starts Nginx from cloud-init user-data."
category: "Terraform Workbooks"
tags:
- terraform
docs/terraform-workbooks/rds-quickstart/README.md
+4 -1
View File
@@ -1,6 +1,7 @@
---
title: "RDS Quickstart (PostgreSQL)"
description: "Stand up a managed PostgreSQL database on Spinifex with Terraform — a VPC, a DB subnet group, a parameter group, an aws_db_instance, and a client VM inside the VPC that connects to it with psql."
seoTitle: "Terraform RDS PostgreSQL Quickstart — Spinifex Docs"
description: "Stand up a managed PostgreSQL database on Spinifex with Terraform: a VPC, DB subnet group, parameter group, aws_db_instance, and a client VM that runs psql."
category: "Terraform Workbooks"
tags:
- terraform
@@ -53,6 +54,8 @@ The client subnet is public in that it has an internet-gateway route, which the
The database security group admits `5432` from the client security group and nothing else, which compiles to an ACL on the endpoint ENI itself — the port is deny-by-default for everything else in the VPC.
That holds for the instance, not just for the endpoint ENI. The DB VM has two further NICs the platform uses to manage it, and no customer security group governs either; the engine binds the endpoint ENI's address alone, so it is not listening on them. PostgreSQL's client authentication rules are scoped to this VPC's own range as well, so the endpoint really is the whole of the reachable surface.
## Prerequisites
- **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)) and OpenTofu (or Terraform) installed.
docs/terraform-workbooks/s3-webapp/README.md
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "S3-Backed Web App"
description: "Deploy a Flask file-sharing web application backed by S3 (Predastore) using Terraform on Spinifex."
seoTitle: "Terraform S3-Backed Flask Web App on EC2 — Spinifex Docs"
description: "Deploy a Flask file-sharing app on EC2 backed by S3 (Predastore) with Terraform, using an IAM instance profile and short-lived STS credentials from IMDS."
category: "Terraform Workbooks"
tags:
- terraform
internal/fipsboot/fipsboot.go
+1
View File
@@ -9,6 +9,7 @@ import (
func init() {
if !fips140.Enabled() {
//nolint:forbidigo // Refuse to start when the required FIPS mode is disabled.
panic("fipsboot: FIPS 140-3 mode is not enabled at runtime — refusing to start. " +
"Build with GOFIPS140=v1.0.0 and do not set GODEBUG=fips140=off.")
}
scripts/images/rds-mariadb/rds-agent.initd
+1 -1
View File
@@ -35,7 +35,7 @@ start_pre() {
# shellcheck disable=SC1090
. "${AGENT_ENV}"
export RDS_GATEWAY_URL RDS_GATEWAY_CA RDS_REGION RDS_DB_INSTANCE_IDENTIFIER \
RDS_ENGINE RDS_ENGINE_VERSION RDS_HANDOFF_DIR RDS_ENGINE_HOST RDS_ENGINE_PORT \
RDS_ENGINE RDS_ENGINE_VERSION RDS_HANDOFF_DIR RDS_ENGINE_PORT \
RDS_POLL_WAIT RDS_DATA_MOUNT
fi
}
scripts/images/rds-mariadb/rds-init
+70 -2
View File
@@ -14,6 +14,7 @@
# RDS_MASTER_PASSWORD=<present only when RDS_MODE=initialize>
# RDS_DB_NAME=<initial database; optional, as on AWS>
# RDS_PORT=<listen port; defaults to 3306>
# RDS_LISTEN_ADDRESS=<the customer ENI address the engine binds>
# RDS_DATA_VOLUME_ID / RDS_DATA_VOLUME_SERIAL=<expected volume identity>
# RDS_VM_GENERATION / RDS_FORMAT_AUTHORIZED=<create-only format grant>
# RDS_BOOTSTRAP_PENDING / RDS_PAYLOAD_ID=<the control plane is still
@@ -120,6 +121,11 @@ SECURE_FILE_DIR="${RDS_SECURE_FILE_DIR:-/var/lib/mysql-files}"
MOUNTS_FILE="${RDS_MOUNTS_FILE:-/proc/mounts}"
# How long the bind address may take to appear on a local interface. The lease
# is taken in the boot runlevel on a 600s budget and this script runs in the
# default one, so the wait is a guard rather than an expected delay.
LISTEN_ADDRESS_TIMEOUT="${RDS_LISTEN_ADDRESS_TIMEOUT:-120}"
# How long the private bootstrap server gets to open its socket, as probes one
# BOOTSTRAP_POLL apart. InnoDB creates its redo log and system tablespace on this
# first start, so it is not instant.
@@ -138,6 +144,50 @@ CREATE_DATABASE_PROCEDURE="create_database"
log() { echo "[rds-init] $*"; }
fail() { echo "[rds-init] ERROR: $*" >&2; exit 1; }
# A dotted-quad IPv4 literal. bind_address would also take a hostname, but the
# control plane sends an address, and accepting a name would reintroduce the
# resolution step the bind exists to remove.
is_ipv4() {
case "$1" in
'' | *[!0-9.]*) return 1 ;;
esac
_rest="$1"
_octets=0
while [ -n "${_rest}" ]; do
_octet="${_rest%%.*}"
case "${_rest}" in
*.*) _rest="${_rest#*.}" ;;
*) _rest="" ;;
esac
# A leading zero reads as octal to some parsers and decimal to others,
# so the two would disagree about which address was bound.
case "${_octet}" in
0 | [1-9] | [1-9][0-9] | [1-9][0-9][0-9]) ;;
*) return 1 ;;
esac
[ "${_octet}" -le 255 ] || return 1
_octets=$((_octets + 1))
done
[ "${_octets}" = 4 ]
}
# The engine binds one address and refuses to start if it is not there yet, and
# an engine that will not start for an unexplained reason is the worst way to
# learn that DHCP lost its race. Bounded, then fatal with the address named.
wait_for_listen_address() {
_deadline=$(($(date +%s) + LISTEN_ADDRESS_TIMEOUT))
while :; do
if ip -4 -o addr show 2>/dev/null |
awk -v want="${RDS_LISTEN_ADDRESS}" \
'{ split($4, addr, "/"); if (addr[1] == want) found = 1 } END { exit !found }'; then
return 0
fi
[ "$(date +%s)" -lt "${_deadline}" ] || break
sleep 1
done
fail "the endpoint address ${RDS_LISTEN_ADDRESS} did not appear on any interface within ${LISTEN_ADDRESS_TIMEOUT}s; the engine binds it and nothing else, so refusing to start"
}
# Between these two, the datadir carries a mysql system database but no usable
# master user. _server_pid is the private bootstrap server, if one is running.
_created_datadir=0
@@ -199,6 +249,7 @@ load_handoff() {
RDS_MASTER_PASSWORD="${RDS_MASTER_PASSWORD:-}"
RDS_DB_NAME="${RDS_DB_NAME:-}"
RDS_PORT="${RDS_PORT:-3306}"
RDS_LISTEN_ADDRESS="${RDS_LISTEN_ADDRESS:-}"
RDS_VM_GENERATION="${RDS_VM_GENERATION:-}"
# Absent from an agent that predates the receipt protocol, which reads as
# "nothing is pending" and leaves every path below unchanged.
@@ -252,6 +303,14 @@ load_handoff() {
'' | *[!0-9]*) fail "bootstrap config has a non-numeric RDS_PORT: ${RDS_PORT}" ;;
esac
# No fallback to a wildcard. The engine is reachable over the customer ENI
# and nowhere else, and a handoff that cannot say which address that is has
# no safe reading — the DB VM's other two NICs carry no security group the
# customer can see, let alone set.
[ -n "${RDS_LISTEN_ADDRESS}" ] || fail "bootstrap config has no RDS_LISTEN_ADDRESS; refusing to start an engine that would answer on every interface"
is_ipv4 "${RDS_LISTEN_ADDRESS}" ||
fail "bootstrap config has a non-IPv4 RDS_LISTEN_ADDRESS: ${RDS_LISTEN_ADDRESS}"
# redact_stream is line-oriented, so a password spanning a line boundary
# would match neither half and reach /dev/console in the clear. The control
# plane rejects one; refusing here too keeps that an asserted property of
@@ -537,7 +596,8 @@ EOF
# The 90- prefix sorts last within the include directory and MariaDB takes the
# last occurrence of a setting, so a resolved parameter can never move the
# endpoint's port, redirect the datadir, turn off TLS or lower the protocol floor.
# endpoint's port, widen the bind, redirect the datadir, turn off TLS or lower
# the protocol floor.
#
# innodb_buffer_pool_chunk_size and innodb_buffer_pool_instances are deliberately
# absent although they are platform-owned: 11.8 removed the second outright, so
@@ -554,7 +614,14 @@ write_platform_config() {
port = ${RDS_PORT}
datadir = ${DATADIR}
socket = ${SOCKET}
bind_address = 0.0.0.0
# The customer ENI and nothing else. The DB VM's other two NICs sit on the
# shared RDS system VPC and the host's management bridge, neither of which any
# customer security group governs, so a socket bound there is reachable across
# accounts. MariaDB's analogue of a scoped pg_hba is the host pattern on a
# grant, which lives in mysql.user on the data volume and would survive a
# restore into another VPC — so for this engine the bind is the boundary.
bind_address = ${RDS_LISTEN_ADDRESS}
# InnoDB is the only storage engine the snapshot guarantee holds for: it keeps a
# redo log and recovers from a crash-consistent snapshot, and Aria and MyISAM do
@@ -963,6 +1030,7 @@ consume_secrets() {
}
load_handoff
wait_for_listen_address
require_data_volume
require_engine_stamp
scripts/images/rds-mariadb/rds-init_test.sh
+66 -3
View File
@@ -201,6 +201,18 @@ EOF
# an installed receipt and an armed sweep coexist. Matched on the receipt
# directory by name: the engine stamp syncs its directory too, long before the
# traps are armed, and killing there would leave the window untested.
# ip stub: stands in for `ip -4 -o addr show`, printing one line per address in
# IP_ADDR_LIST in the format the bind-address wait parses.
cat > "${STUBBIN}/ip" <<'EOF'
#!/bin/sh
n=1
for cidr in ${IP_ADDR_LIST:-}; do
echo "${n}: eth$((n - 1)) inet ${cidr} scope global eth$((n - 1))"
n=$((n + 1))
done
exit 0
EOF
cat > "${STUBBIN}/sync" <<'EOF'
#!/bin/sh
if [ "${SYNC_KILL_PARENT:-0}" = "1" ]; then
@@ -236,6 +248,9 @@ SECURE_FILE_DIR="${WORK}/mysql-files"
LOG_DIR="${WORK}/log"
ENGINE_LOG="${LOG_DIR}/error.log"
MOUNTS="${WORK}/mounts"
# The customer ENI address the engine binds, as the control plane delivers it.
LISTEN_ADDRESS_DEFAULT="10.30.1.15"
KILL_PID_FILE="${WORK}/rds-init.pid"
MARIADBD_TEST_PID_FILE="${WORK}/mariadbd.pid"
WORKDIR="${WORK}"
@@ -266,6 +281,13 @@ write_handoff() {
[ -n "$3" ] && echo "RDS_DB_NAME='$3'"
[ -n "${PENDING:-}" ] && echo "RDS_BOOTSTRAP_PENDING=${PENDING}"
[ -n "${PAYLOAD_ID:-}" ] && echo "RDS_PAYLOAD_ID=${PAYLOAD_ID}"
# Written unset rather than defaulted when the case is about a handoff
# that omits it, which rds-init has to refuse rather than fill in.
[ "${OMIT_LISTEN_ADDRESS:-0}" = "1" ] ||
echo "RDS_LISTEN_ADDRESS=${LISTEN_ADDRESS:-${LISTEN_ADDRESS_DEFAULT}}"
# Delivered but unread here: MariaDB's analogue of a scoped pg_hba is a
# grant host pattern on the data volume, which a restore would carry.
echo "RDS_CLIENT_CIDR=10.30.0.0/16"
# Last, and unconditional: an optional line failing its test would make
# the whole group's status non-zero and `set -e` would end the run here.
echo "RDS_PORT=6543"
@@ -327,6 +349,10 @@ reset_state() {
unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE CLIENT_STATUS_CORRUPT STAMP_OVERRIDE || true
unset MARIADBD_START_FAIL MARIADBD_KILL_PARENT MARIADBD_IGNORE_TERM CLIENT_KILL_PARENT || true
unset ADMIN_SHUTDOWN_FAIL SYNC_KILL_PARENT || true
unset LISTEN_ADDRESS OMIT_LISTEN_ADDRESS || true
# The customer ENI carries its address by the time rds-init runs, which is
# the ordinary case; the cases about a late lease drop it themselves.
IP_ADDR_LIST="${LISTEN_ADDRESS_DEFAULT}/24 10.99.0.7/24"
rm -f "${MARIADBD_TEST_PID_FILE}"
write_tls
}
@@ -346,6 +372,8 @@ run_env() {
RDS_RECEIPT_DIR="${RECEIPT_DIR_OVERRIDE:-${RECEIPT_DIR}}" \
RDS_ENGINE_STAMP="${STAMP_OVERRIDE:-${STAMP}}" \
RDS_ALLOW_LOCAL_DATADIR="${RDS_ALLOW_LOCAL_DATADIR:-0}" \
RDS_LISTEN_ADDRESS_TIMEOUT="${LISTEN_TIMEOUT:-0}" \
IP_ADDR_LIST="${IP_ADDR_LIST:-}" \
RDS_BOOTSTRAP_POLL=0.05 \
RDS_BOOTSTRAP_PROBES=200 \
RDS_BOOTSTRAP_STOP_GRACE=0.1 \
@@ -493,9 +521,12 @@ if run_ok "initialize"; then
&& pass "initialize: delivered port applied" || fail "initialize: port not applied"
grep -q "^datadir = ${DATADIR}" "${PLATFORM_FILE}" \
&& pass "initialize: datadir pinned by the platform" || fail "initialize: no datadir"
grep -q "^bind_address = 0.0.0.0" "${PLATFORM_FILE}" \
&& pass "initialize: the engine listens for the customer ENI" \
|| fail "initialize: no bind_address"
grep -q "^bind_address = ${LISTEN_ADDRESS_DEFAULT}$" "${PLATFORM_FILE}" \
&& pass "initialize: the engine binds the customer ENI" \
|| fail "initialize: the engine does not bind the delivered address"
grep -qE '^bind_address = (0\.0\.0\.0|\*|::)$' "${PLATFORM_FILE}" \
&& fail "initialize: the engine still answers on every interface" \
|| pass "initialize: no wildcard bind"
grep -q "^default_storage_engine = InnoDB" "${PLATFORM_FILE}" \
&& pass "initialize: tables land in the engine snapshots recover" \
|| fail "initialize: default_storage_engine not pinned"
@@ -1233,6 +1264,38 @@ grep -q 'engine stamp' "${WORK}/out" \
&& pass "stamp-unwritable: refusal names the stamp" || fail "stamp-unwritable: no refusal message"
unset STAMP_OVERRIDE
# --- Case 13: a handoff that cannot say what to bind is refused ---
# The refusal is before the one-shot password is spent, so the create is still
# retryable. The only reading rds-init could reach on its own is the wildcard
# the bind exists to remove.
for _case in omit-listen bad-listen leading-zero short-quad; do
reset_state
case "${_case}" in
omit-listen) OMIT_LISTEN_ADDRESS=1 ;;
bad-listen) LISTEN_ADDRESS="db.internal" ;;
leading-zero) LISTEN_ADDRESS="010.30.1.15" ;;
short-quad) LISTEN_ADDRESS="10.30.1" ;;
esac
write_handoff initialize 's3cr3t' appdb
run_fails "${_case}"
[ -s "${INSTALLDB_CALLS}" ] \
&& fail "${_case}: initialised on a handoff that could not be read" \
|| pass "${_case}: refused before the datadir was created"
done
# --- Case 13a: the bind address must be on an interface before the engine starts ---
# An engine asked to bind an address that is not there refuses to start, which is
# the worst way to learn that DHCP lost its race.
reset_state
IP_ADDR_LIST="10.99.0.7/24"
write_handoff initialize 's3cr3t' appdb
run_fails "bind-wait"
grep -q "${LISTEN_ADDRESS_DEFAULT}" "${WORK}/out" \
&& pass "bind-wait: the refusal names the address" || fail "bind-wait: the refusal does not name the address"
[ -s "${INSTALLDB_CALLS}" ] \
&& fail "bind-wait: initialised before the address existed" \
|| pass "bind-wait: refused before the datadir was created"
if [ "${FAILS}" -eq 0 ]; then
echo "PASS: all rds-init cases"
exit 0
scripts/images/rds-postgres/rds-agent.initd
+1 -1
View File
@@ -35,7 +35,7 @@ start_pre() {
# shellcheck disable=SC1090
. "${AGENT_ENV}"
export RDS_GATEWAY_URL RDS_GATEWAY_CA RDS_REGION RDS_DB_INSTANCE_IDENTIFIER \
RDS_ENGINE RDS_ENGINE_VERSION RDS_HANDOFF_DIR RDS_ENGINE_HOST RDS_ENGINE_PORT \
RDS_ENGINE RDS_ENGINE_VERSION RDS_HANDOFF_DIR RDS_ENGINE_PORT \
RDS_POLL_WAIT RDS_DATA_MOUNT
fi
}
scripts/images/rds-postgres/rds-init
+93 -8
View File
@@ -14,6 +14,8 @@
# RDS_MASTER_PASSWORD=<present only when RDS_MODE=initialize>
# RDS_DB_NAME=<initial database; optional, as on AWS>
# RDS_PORT=<listen port; defaults to 5432>
# RDS_LISTEN_ADDRESS=<the customer ENI address the engine binds>
# RDS_CLIENT_CIDR=<the customer VPC range client authentication admits>
# RDS_DATA_VOLUME_ID / RDS_DATA_VOLUME_SERIAL=<expected volume identity>
# RDS_VM_GENERATION / RDS_FORMAT_AUTHORIZED=<create-only format grant>
# RDS_BOOTSTRAP_PENDING / RDS_PAYLOAD_ID=<the control plane is still
@@ -93,6 +95,11 @@ BOOTSTRAP_DIR="${RDS_BOOTSTRAP_DIR:-/run/rds-init}"
MOUNTS_FILE="${RDS_MOUNTS_FILE:-/proc/mounts}"
# How long the bind address may take to appear on a local interface. The lease
# is taken in the boot runlevel on a 600s budget and this script runs in the
# default one, so the wait is a guard rather than an expected delay.
LISTEN_ADDRESS_TIMEOUT="${RDS_LISTEN_ADDRESS_TIMEOUT:-120}"
# The group role the master's administrative privileges hang off. Named as on
# AWS, so a customer's `GRANT rds_superuser TO ...` works here too.
MASTER_GROUP_ROLE="rds_superuser"
@@ -134,6 +141,63 @@ clear_unfinished_datadir() {
# The server refuses to run as root.
as_pg() { su "${PG_USER}" -c "$1"; }
# A dotted-quad IPv4 literal. postgresql.conf would also take a hostname here,
# but the control plane sends an address, and accepting a name would reintroduce
# the resolution step the bind exists to remove.
is_ipv4() {
case "$1" in
'' | *[!0-9.]*) return 1 ;;
esac
_rest="$1"
_octets=0
while [ -n "${_rest}" ]; do
_octet="${_rest%%.*}"
case "${_rest}" in
*.*) _rest="${_rest#*.}" ;;
*) _rest="" ;;
esac
# A leading zero reads as octal to some parsers and decimal to others,
# so the two would disagree about which address was bound.
case "${_octet}" in
0 | [1-9] | [1-9][0-9] | [1-9][0-9][0-9]) ;;
*) return 1 ;;
esac
[ "${_octet}" -le 255 ] || return 1
_octets=$((_octets + 1))
done
[ "${_octets}" = 4 ]
}
is_ipv4_cidr() {
case "$1" in
*/*) ;;
*) return 1 ;;
esac
_prefix="${1#*/}"
case "${_prefix}" in
'' | *[!0-9]*) return 1 ;;
esac
[ "${#_prefix}" -le 2 ] && [ "${_prefix}" -le 32 ] || return 1
is_ipv4 "${1%%/*}"
}
# The engine binds one address and fails to start if it is not there yet, and an
# engine that will not start for an unexplained reason is the worst way to learn
# that DHCP lost its race. Bounded, then fatal with the address in the message.
wait_for_listen_address() {
_deadline=$(($(date +%s) + LISTEN_ADDRESS_TIMEOUT))
while :; do
if ip -4 -o addr show 2>/dev/null |
awk -v want="${RDS_LISTEN_ADDRESS}" \
'{ split($4, addr, "/"); if (addr[1] == want) found = 1 } END { exit !found }'; then
return 0
fi
[ "$(date +%s)" -lt "${_deadline}" ] || break
sleep 1
done
fail "the endpoint address ${RDS_LISTEN_ADDRESS} did not appear on any interface within ${LISTEN_ADDRESS_TIMEOUT}s; the engine binds it and nothing else, so refusing to start"
}
load_handoff() {
[ -f "${ENV_FILE}" ] || fail "no bootstrap handoff at ${ENV_FILE} — rds-agent has not delivered the bootstrap config"
@@ -146,6 +210,8 @@ load_handoff() {
RDS_MASTER_PASSWORD="${RDS_MASTER_PASSWORD:-}"
RDS_DB_NAME="${RDS_DB_NAME:-}"
RDS_PORT="${RDS_PORT:-5432}"
RDS_LISTEN_ADDRESS="${RDS_LISTEN_ADDRESS:-}"
RDS_CLIENT_CIDR="${RDS_CLIENT_CIDR:-}"
RDS_VM_GENERATION="${RDS_VM_GENERATION:-}"
# Absent from an agent that predates the receipt protocol, which reads as
# "nothing is pending" and leaves every path below unchanged.
@@ -192,6 +258,17 @@ load_handoff() {
'' | *[!0-9]*) fail "bootstrap config has a non-numeric RDS_PORT: ${RDS_PORT}" ;;
esac
# Neither falls back to a wildcard. The engine is reachable over the customer
# ENI and nowhere else, and a handoff that cannot say which address that is
# has no safe reading — the DB VM's other two NICs carry no security group
# the customer can see, let alone set.
[ -n "${RDS_LISTEN_ADDRESS}" ] || fail "bootstrap config has no RDS_LISTEN_ADDRESS; refusing to start an engine that would answer on every interface"
is_ipv4 "${RDS_LISTEN_ADDRESS}" ||
fail "bootstrap config has a non-IPv4 RDS_LISTEN_ADDRESS: ${RDS_LISTEN_ADDRESS}"
[ -n "${RDS_CLIENT_CIDR}" ] || fail "bootstrap config has no RDS_CLIENT_CIDR; refusing to start an engine that would accept clients from anywhere"
is_ipv4_cidr "${RDS_CLIENT_CIDR}" ||
fail "bootstrap config has a non-IPv4 RDS_CLIENT_CIDR: ${RDS_CLIENT_CIDR}"
# redact_stream is line-oriented, so a password spanning a line boundary
# would match neither half and reach /dev/console in the clear. The control
# plane rejects one; refusing here too keeps that an asserted property of
@@ -443,13 +520,15 @@ write_pg_hba() {
# Managed by rds-init — regenerated on every boot. Edits here are lost.
#
# Clients reach the instance over its customer ENI, gated by the DB security
# group as on AWS RDS, and authenticate with scram-sha-256. The catch-alls cover
# loopback too, so initdb's own 127.0.0.1 and ::1 lines are not carried over.
# The local socket is how rds-init and rds-agent run privileged SQL.
# group as on AWS RDS, and authenticate with scram-sha-256. The host rule is
# scoped to the customer VPC rather than the DB subnet: an app subnet reaching a
# DB subnet arrives carrying its own address, with no NAT in between. The engine
# does not bind loopback, so initdb's own 127.0.0.1 and ::1 lines govern nothing
# and are not carried over. The local socket is how rds-init and rds-agent run
# privileged SQL.
local all all peer
include_if_exists "${HBA_FORCE_SSL_INCLUDE}"
host all all 0.0.0.0/0 scram-sha-256
host all all ::/0 scram-sha-256
host all all ${RDS_CLIENT_CIDR} scram-sha-256
EOF
chown "${PG_USER}:${PG_USER}" "${_tmp}"
chmod 0600 "${_tmp}"
@@ -466,8 +545,13 @@ EOF
}
# The 90- prefix sorts last within include_dir and the last setting wins, so a
# resolved parameter can never move the endpoint's port, turn off TLS or lower
# the protocol floor.
# resolved parameter can never move the endpoint's port, widen the bind, turn off
# TLS or lower the protocol floor.
#
# listen_addresses is the customer ENI and nothing else — not the wildcard, and
# not loopback either. The DB VM's other two NICs sit on the shared RDS system
# VPC and the host's management bridge, neither of which any customer security
# group governs, so a socket bound there is reachable across accounts.
#
# The floor is written whether or not a cert was delivered: it costs nothing with
# ssl off, and making it conditional would mean a boot without a cert leaves the
@@ -478,7 +562,7 @@ write_rds_conf() {
cat > "${_tmp}" <<EOF
# Managed by rds-init — regenerated on every boot from the bootstrap config.
# Edits here are lost; instance parameters belong in the parameter group.
listen_addresses = '*'
listen_addresses = '${RDS_LISTEN_ADDRESS}'
port = ${RDS_PORT}
unix_socket_directories = '${SOCKET_DIR}'
ssl = ${_ssl}
@@ -794,6 +878,7 @@ consume_secrets() {
}
load_handoff
wait_for_listen_address
require_data_volume
require_engine_stamp
scripts/images/rds-postgres/rds-init_test.sh
+113 -5
View File
@@ -65,6 +65,18 @@ cat > "${STUBBIN}/ls" <<'EOF'
exec "${REAL_LS}" "$@"
EOF
# ip stub: stands in for `ip -4 -o addr show`, printing one line per address in
# IP_ADDR_LIST in the format the bind-address wait parses.
cat > "${STUBBIN}/ip" <<'EOF'
#!/bin/sh
n=1
for cidr in ${IP_ADDR_LIST:-}; do
echo "${n}: eth$((n - 1)) inet ${cidr} scope global eth$((n - 1))"
n=$((n + 1))
done
exit 0
EOF
# su stub: `su postgres -c "cmd"` runs cmd in the harness user's shell.
cat > "${STUBBIN}/su" <<'EOF'
#!/bin/sh
@@ -205,6 +217,11 @@ RECEIPT="${RECEIPT_DIR}/receipt.env"
STAMP="${DATA_MOUNT}/.spinifex-rds/engine"
HANDOFF="${WORK}/run/spinifex-rds"
MOUNTS="${WORK}/mounts"
# The customer ENI address the engine binds and the VPC range its client
# authentication admits, as the control plane delivers them.
LISTEN_ADDRESS_DEFAULT="10.30.1.15"
CLIENT_CIDR_DEFAULT="10.30.0.0/16"
KILL_PID_FILE="${WORK}/rds-init.pid"
export KILL_PID_FILE
@@ -234,6 +251,12 @@ write_handoff() {
[ -n "$3" ] && echo "RDS_DB_NAME='$3'"
[ -n "${PENDING:-}" ] && echo "RDS_BOOTSTRAP_PENDING=${PENDING}"
[ -n "${PAYLOAD_ID:-}" ] && echo "RDS_PAYLOAD_ID=${PAYLOAD_ID}"
# Written unset rather than defaulted when the case is about a handoff
# that omits one, which rds-init has to refuse rather than fill in.
[ "${OMIT_LISTEN_ADDRESS:-0}" = "1" ] ||
echo "RDS_LISTEN_ADDRESS=${LISTEN_ADDRESS:-${LISTEN_ADDRESS_DEFAULT}}"
[ "${OMIT_CLIENT_CIDR:-0}" = "1" ] ||
echo "RDS_CLIENT_CIDR=${CLIENT_CIDR:-${CLIENT_CIDR_DEFAULT}}"
# Last, and unconditional: an optional line failing its test would make
# the whole group's status non-zero and `set -e` would end the run here.
echo "RDS_PORT=6543"
@@ -270,6 +293,10 @@ reset_state() {
: > "${PSQL_CALLS}"
unset INITDB_FAIL PG_HBA_AS_DIR PSQL_FAIL LS_FAIL RDS_ALLOW_LOCAL_DATADIR MASTER_USER || true
unset PENDING PAYLOAD_ID DB_ID RECEIPT_DIR_OVERRIDE PSQL_STATUS_CORRUPT STAMP_OVERRIDE || true
unset LISTEN_ADDRESS CLIENT_CIDR OMIT_LISTEN_ADDRESS OMIT_CLIENT_CIDR || true
# The customer ENI carries its address by the time rds-init runs, which is
# the ordinary case; the cases about a late lease drop it themselves.
IP_ADDR_LIST="${LISTEN_ADDRESS_DEFAULT}/24 10.99.0.7/24"
write_tls
}
@@ -293,6 +320,8 @@ run_env() {
RDS_RECEIPT_DIR="${RECEIPT_DIR_OVERRIDE:-${RECEIPT_DIR}}" \
RDS_ENGINE_STAMP="${STAMP_OVERRIDE:-${STAMP}}" \
RDS_ALLOW_LOCAL_DATADIR="${RDS_ALLOW_LOCAL_DATADIR:-0}" \
RDS_LISTEN_ADDRESS_TIMEOUT="${LISTEN_TIMEOUT:-0}" \
IP_ADDR_LIST="${IP_ADDR_LIST:-}" \
INITDB_FAIL="${INITDB_FAIL:-0}" \
PG_HBA_AS_DIR="${PG_HBA_AS_DIR:-0}" \
PSQL_FAIL="${PSQL_FAIL:-0}" \
@@ -328,14 +357,24 @@ if run_ok "initialize"; then
grep -q 'initdb' "${INITDB_CALLS}" && pass "initialize: initdb ran" || fail "initialize: no initdb"
grep -q -- '--data-checksums' "${INITDB_CALLS}" \
&& pass "initialize: data checksums on" || fail "initialize: no --data-checksums"
grep -q 'host all all 0.0.0.0/0 scram-sha-256' "${PGDATA}/pg_hba.conf" \
&& pass "initialize: remote scram hba rule written" || fail "initialize: no hba rule"
grep -q "host all all ${CLIENT_CIDR_DEFAULT} scram-sha-256" "${PGDATA}/pg_hba.conf" \
&& pass "initialize: the scram hba rule is scoped to the customer VPC" || fail "initialize: no hba rule"
# The scope is the whole boundary this file carries: a catch-all here admits
# the system VPC and the management bridge, which no customer security group
# governs. IPv6 never appears — the platform's VPCs are IPv4-only.
grep -qE 'host .*(0\.0\.0\.0/0|::/0)' "${PGDATA}/pg_hba.conf" \
&& fail "initialize: a catch-all host rule survived the VPC scoping" \
|| pass "initialize: no catch-all host rule"
[ "$(grep -c '^host ' "${PGDATA}/pg_hba.conf")" = 1 ] \
&& pass "initialize: exactly one host rule" \
|| fail "initialize: more than one host rule, so the scope is not the boundary"
# pg_hba is first-match-wins, so initdb's loopback rules sorting above the
# catch-all would shadow anything the platform appends below them. rds-init
# owns the whole file; the catch-alls cover loopback themselves.
# platform's would shadow it. rds-init owns the whole file, and the engine
# does not bind loopback at all.
grep -qE '127\.0\.0\.1/32|::1/128' "${PGDATA}/pg_hba.conf" \
&& fail "initialize: initdb's loopback TCP rules survived above the catch-all" \
&& fail "initialize: initdb's loopback TCP rules survived" \
|| pass "initialize: initdb's loopback TCP rules are gone"
grep -q 'replication' "${PGDATA}/pg_hba.conf" \
&& fail "initialize: initdb's replication rules survived" \
@@ -358,6 +397,12 @@ if run_ok "initialize"; then
&& pass "initialize: include_dir hooked" || fail "initialize: no include_dir"
grep -q '^port = 6543' "${PGDATA}/conf.d/90-rds-init.conf" \
&& pass "initialize: delivered port applied" || fail "initialize: port not applied"
grep -q "^listen_addresses = '${LISTEN_ADDRESS_DEFAULT}'$" "${PGDATA}/conf.d/90-rds-init.conf" \
&& pass "initialize: the engine binds the customer ENI" \
|| fail "initialize: the engine does not bind the delivered address"
grep -q "^listen_addresses = '\*'" "${PGDATA}/conf.d/90-rds-init.conf" \
&& fail "initialize: the engine still answers on every interface" \
|| pass "initialize: no wildcard bind"
grep -q '^ssl = on' "${PGDATA}/conf.d/90-rds-init.conf" \
&& pass "initialize: ssl enabled" || fail "initialize: ssl not enabled"
grep -q "^ssl_min_protocol_version = 'TLSv1.3'$" "${PGDATA}/conf.d/90-rds-init.conf" \
@@ -1159,6 +1204,69 @@ grep -q 'engine stamp' "${WORK}/out" \
&& pass "stamp-unwritable: refusal names the stamp" || fail "stamp-unwritable: no refusal message"
unset STAMP_OVERRIDE
# --- Case 12: a handoff that cannot say what to bind is refused before initdb ---
# The refusal is before the one-shot password is spent, so the create is still
# retryable. The only readings rds-init could reach on its own are the wildcards
# the bind exists to remove.
for _case in omit-listen bad-listen omit-cidr bad-cidr; do
reset_state
case "${_case}" in
omit-listen) OMIT_LISTEN_ADDRESS=1 ;;
bad-listen) LISTEN_ADDRESS="db.internal" ;;
omit-cidr) OMIT_CLIENT_CIDR=1 ;;
bad-cidr) CLIENT_CIDR="10.30.0.0/33" ;;
esac
write_handoff initialize 's3cr3t' appdb
run_fails "${_case}"
[ -s "${INITDB_CALLS}" ] \
&& fail "${_case}: initdb ran on a handoff that could not be read" \
|| pass "${_case}: refused before initdb"
done
# --- Case 12a: an octet-shaped address that is not one is refused ---
# 10.30.1.256 and 010.30.1.15 both parse as addresses to a permissive check, and
# the second binds a different host to anything reading the leading zero as octal.
for _addr in 10.30.1.256 010.30.1.15 10.30.1 10.30.1.15.1; do
reset_state
LISTEN_ADDRESS="${_addr}"
write_handoff initialize 's3cr3t' appdb
run_fails "listen-${_addr}"
done
# --- Case 13: the bind address must be on an interface before the engine starts ---
# An engine asked to bind an address that is not there exits with `could not
# bind`, which is the worst way to learn that DHCP lost its race.
reset_state
IP_ADDR_LIST="10.99.0.7/24"
write_handoff initialize 's3cr3t' appdb
run_fails "bind-wait"
grep -q "${LISTEN_ADDRESS_DEFAULT}" "${WORK}/out" \
&& pass "bind-wait: the refusal names the address" || fail "bind-wait: the refusal does not name the address"
[ -s "${INITDB_CALLS}" ] \
&& fail "bind-wait: initdb ran before the address existed" \
|| pass "bind-wait: refused before initdb"
# --- Case 13a: a restored volume takes the destination VPC's scope ---
# A restore attaches a data volume carrying the source instance's pg_hba into a
# possibly different VPC. Regenerating the whole file every boot is what stops
# the restored instance locking out its own VPC's clients.
reset_state
write_handoff initialize 's3cr3t' appdb
write_parameters
if run_ok "restore-scope-setup"; then
CLIENT_CIDR="172.31.0.0/16"
write_handoff attach '' appdb
write_parameters
if run_ok "restore-scope"; then
grep -q 'host all all 172.31.0.0/16 scram-sha-256' "${PGDATA}/pg_hba.conf" \
&& pass "restore-scope: the destination VPC's range is admitted" \
|| fail "restore-scope: the restored instance did not take its own VPC's scope"
grep -q "${CLIENT_CIDR_DEFAULT}" "${PGDATA}/pg_hba.conf" \
&& fail "restore-scope: the source VPC's range survived the restore" \
|| pass "restore-scope: the source VPC's range is gone"
fi
fi
if [ "${FAILS}" -eq 0 ]; then
echo "PASS: all rds-init cases"
exit 0
spinifex/daemon/daemon.go
+17 -2
View File
@@ -1702,8 +1702,20 @@ func (d *Daemon) startCluster() error {
slog.Info("EIP service disabled — no external IPAM; serving empty/unsupported responses")
}
d.instanceService.SetTerminationDeps(d.volumeService, d.vpcService, d.externalIPAM, d.tagsService)
d.instanceService.SetRunInstancesDeps(d.imageService, d.keyService, &daemonENICreator{d: d}, d.externalIPAM)
// A nil *ExternalIPAM stored in an interface is not a nil interface, so hand
// the concrete value over only once init succeeded — otherwise the service's
// own nil checks pass and the allocate path derefs a nil receiver.
var (
ipAllocator handlers_ec2_instance.PublicIPAllocator
ipReleaser handlers_ec2_instance.PublicIPReleaser
)
if d.externalIPAM != nil {
ipAllocator = d.externalIPAM
ipReleaser = d.externalIPAM
}
d.instanceService.SetTerminationDeps(d.volumeService, d.vpcService, ipReleaser, d.tagsService)
d.instanceService.SetRunInstancesDeps(d.imageService, d.keyService, &daemonENICreator{d: d}, ipAllocator)
if d.gpuManager != nil {
d.instanceService.SetGPUClaimer(&daemonGPUClaimer{d: d})
@@ -1806,6 +1818,9 @@ func (d *Daemon) startCluster() error {
// whichever node the queue group picks, so a node without it would make the
// first boot of a DB instance fail intermittently rather than not at all.
d.rdsService, err = initServiceWithRetry("RDS service", func() (*handlers_rds.Service, error) {
if registryErr := handlers_rds.ValidateEngineRegistry(); registryErr != nil {
return nil, registryErr
}
deps, depsErr := d.buildRDSDeps()
if depsErr != nil {
return nil, depsErr
spinifex/daemon/daemon_system_instance.go
+21 -2
View File
@@ -52,6 +52,25 @@ func (d *Daemon) recordENIInstanceOwner(eniAccountID, eniID, instanceOwnerID str
}
}
// attachExtraENI attaches one pre-created extra ENI and, when the caller asked
// for an explicit DeleteOnTermination, persists it onto the record. An ENI that
// has to outlive its VM is only safe once that flag is stored, so a failed stamp
// fails the attach rather than leaving a disposable NIC behind.
func (d *Daemon) attachExtraENI(eniAccountID string, extra sysinstance.ExtraENIInput, instanceID string, deviceIndex int64) error {
if _, err := d.vpcService.AttachENI(eniAccountID, extra.ENIID, instanceID, deviceIndex); err != nil {
return err
}
if extra.DeleteOnTermination == nil {
return nil
}
if err := d.vpcService.UpdateENI(eniAccountID, extra.ENIID, func(r *handlers_ec2_vpc.ENIRecord) {
r.DeleteOnTermination = extra.DeleteOnTermination
}); err != nil {
return fmt.Errorf("set DeleteOnTermination on ENI %s: %w", extra.ENIID, err)
}
return nil
}
// LaunchSystemInstance creates and starts a system-managed VM (ELBv2 LB).
// The VM is owned by the system account (GlobalAccountID), is not visible to
// customer DescribeInstances calls, and always boots via direct kernel boot
@@ -160,7 +179,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
// customer-VPC ENI) lives in its own account; the ENI record is
// account-keyed, so attach under extra.AccountID when set.
extraAccount := resolveENIAccount(extra.AccountID, eniAccountID)
if _, attachErr := d.vpcService.AttachENI(extraAccount, extra.ENIID, instance.ID, int64(idx+1)); attachErr != nil {
if attachErr := d.attachExtraENI(extraAccount, extra, instance.ID, int64(idx+1)); attachErr != nil {
slog.Error("LaunchSystemInstance: failed to attach extra ENI", "eniId", extra.ENIID, "instanceId", instance.ID, "err", attachErr)
d.cleanupFailedSystemInstance(instance, instanceType)
return nil, fmt.Errorf("attach extra ENI %s: %w", extra.ENIID, attachErr)
@@ -488,7 +507,7 @@ func (d *Daemon) launchAMISystemInstance(input *sysinstance.SystemInstanceInput)
for idx, extra := range input.ExtraENIs {
if d.vpcService != nil {
extraAccount := resolveENIAccount(extra.AccountID, input.AccountID)
if _, attachErr := d.vpcService.AttachENI(extraAccount, extra.ENIID, inst.ID, int64(idx+1)); attachErr != nil {
if attachErr := d.attachExtraENI(extraAccount, extra, inst.ID, int64(idx+1)); attachErr != nil {
slog.Error("launchAMISystemInstance: failed to attach extra ENI", "eniId", extra.ENIID, "instanceId", inst.ID, "err", attachErr)
d.vmMgr.MarkFailed(context.Background(), inst, "extra_eni_attach_failed")
return nil, fmt.Errorf("attach extra ENI %s: %w", extra.ENIID, attachErr)
spinifex/daemon/daemon_system_instance_test.go
+49
View File
@@ -12,10 +12,12 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/spinifex/awserrors"
handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip"
handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/mulgadc/spinifex/spinifex/handlers/sysinstance"
"github.com/mulgadc/spinifex/spinifex/network/external"
"github.com/mulgadc/spinifex/spinifex/tags"
"github.com/mulgadc/spinifex/spinifex/testutil"
@@ -877,3 +879,50 @@ func TestReleaseSystemInstanceEIP_ReleasesEipServiceAllocation(t *testing.T) {
assert.Empty(t, inst.PublicIPAllocID, "instance alloc ID must be cleared so externalIPAM teardown does not double-release")
assert.Empty(t, inst.PublicIPAssocID)
}
// --- attachExtraENI: DeleteOnTermination ---
// TestAttachExtraENI_DeleteOnTerminationFalseSurvivesTerminate covers the RDS
// replace: the customer/endpoint ENI is attached as an extra NIC and has to
// outlive the VM, because a replacement launch resolves the endpoint by that
// ENI ID. Without the explicit flag the terminate sweep deletes it and the
// replacement fails with InvalidNetworkInterfaceID.NotFound.
func TestAttachExtraENI_DeleteOnTerminationFalseSurvivesTerminate(t *testing.T) {
f := newENIHotPlugFixture(t)
f.vmInst.AccountID = testAccountID
require.NoError(t, f.daemon.attachExtraENI(testAccountID, sysinstance.ExtraENIInput{
ENIID: f.eniID,
DeleteOnTermination: aws.Bool(false),
}, f.vmInst.ID, 1))
rec, err := f.daemon.vpcService.GetENIRecord(testAccountID, f.eniID)
require.NoError(t, err)
require.NotNil(t, rec.DeleteOnTermination, "the attach must stamp the flag, not leave the attach-time default")
assert.False(t, *rec.DeleteOnTermination)
require.NoError(t, newInstanceCleanerAdapter(f.daemon).DetachAndDeleteENI(f.vmInst))
rec, err = f.daemon.vpcService.GetENIRecord(testAccountID, f.eniID)
require.NoError(t, err, "the endpoint ENI must survive the terminate half of a replace")
assert.Equal(t, "available", rec.Status, "it must be detached, so the replacement VM can re-attach it")
assert.Empty(t, rec.InstanceId)
}
// TestAttachExtraENI_NilDeleteOnTerminationStaysDisposable holds the ELBv2
// multi-subnet ALB and EKS cross-account NLB extras at their current behaviour:
// they carry no flag and are deleted with the VM.
func TestAttachExtraENI_NilDeleteOnTerminationStaysDisposable(t *testing.T) {
f := newENIHotPlugFixture(t)
f.vmInst.AccountID = testAccountID
require.NoError(t, f.daemon.attachExtraENI(testAccountID, sysinstance.ExtraENIInput{
ENIID: f.eniID,
}, f.vmInst.ID, 1))
require.NoError(t, newInstanceCleanerAdapter(f.daemon).DetachAndDeleteENI(f.vmInst))
_, err := f.daemon.vpcService.GetENIRecord(testAccountID, f.eniID)
require.Error(t, err, "an extra ENI with no explicit flag stays disposable")
assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidNetworkInterfaceIDNotFound))
}
spinifex/gateway/auth_test.go
+1 -1
View File
@@ -1971,7 +1971,7 @@ func setupPolicyResourceTestHandler(gw *GatewayConfig, service, action, resource
r := chi.NewRouter()
r.Use(gw.SigV4AuthMiddleware())
r.HandleFunc("/*", func(w http.ResponseWriter, req *http.Request) {
if err := gw.checkPolicyResource(req, service, action, resource); err != nil {
if err := gw.checkPolicyResources(req, service, action, []string{resource}); err != nil {
gw.ErrorHandler(w, req, err)
return
}
spinifex/gateway/ec2.go
+4 -4
View File
@@ -126,7 +126,7 @@ var ec2Actions = map[string]EC2Handler{
}),
"RunInstances": ec2HandlerWithReq(func(ctx context.Context, input *ec2.RunInstancesInput, gw *GatewayConfig, accountID string, r *http.Request) (any, error) {
passRoleCheck := func(roleARN string) error {
return gw.checkPolicyResource(r, "iam", "PassRole", roleARN)
return gw.checkPolicyResources(r, "iam", "PassRole", []string{roleARN})
}
launchQuotaCheck := func() error {
return gw.Quota.EnforceLaunch(ctx, accountID, aws.StringValue(input.InstanceType), int(aws.Int64Value(input.MaxCount)))
@@ -144,7 +144,7 @@ var ec2Actions = map[string]EC2Handler{
}),
"AssociateIamInstanceProfile": ec2HandlerWithReq(func(ctx context.Context, input *ec2.AssociateIamInstanceProfileInput, gw *GatewayConfig, accountID string, r *http.Request) (any, error) {
passRoleCheck := func(roleARN string) error {
return gw.checkPolicyResource(r, "iam", "PassRole", roleARN)
return gw.checkPolicyResources(r, "iam", "PassRole", []string{roleARN})
}
return gateway_ec2_instance.AssociateIamInstanceProfile(ctx, input, gw.NATSConn, gw.IAMService, accountID, passRoleCheck)
}),
@@ -153,7 +153,7 @@ var ec2Actions = map[string]EC2Handler{
}),
"ReplaceIamInstanceProfileAssociation": ec2HandlerWithReq(func(ctx context.Context, input *ec2.ReplaceIamInstanceProfileAssociationInput, gw *GatewayConfig, accountID string, r *http.Request) (any, error) {
passRoleCheck := func(roleARN string) error {
return gw.checkPolicyResource(r, "iam", "PassRole", roleARN)
return gw.checkPolicyResources(r, "iam", "PassRole", []string{roleARN})
}
return gateway_ec2_instance.ReplaceIamInstanceProfileAssociation(ctx, input, gw.NATSConn, gw.IAMService, gw.DiscoverActiveNodes(ctx), accountID, passRoleCheck)
}),
@@ -399,7 +399,7 @@ var ec2Actions = map[string]EC2Handler{
}),
"RequestSpotInstances": ec2HandlerWithReq(func(ctx context.Context, input *ec2.RequestSpotInstancesInput, gw *GatewayConfig, accountID string, r *http.Request) (any, error) {
passRoleCheck := func(roleARN string) error {
return gw.checkPolicyResource(r, "iam", "PassRole", roleARN)
return gw.checkPolicyResources(r, "iam", "PassRole", []string{roleARN})
}
return gateway_ec2_spotinstance.RequestSpotInstances(ctx, input, gw.NATSConn, gw.IAMService, accountID, gw.AZ, passRoleCheck, gw.Quota, gw.ExpectedNodes)
}),
spinifex/gateway/ecr_authorization.go
+3 -1
View File
@@ -53,7 +53,9 @@ func (gw *GatewayConfig) ecrOperationAuthorization(next http.Handler) http.Handl
gateway_ecr.WriteError(w, http.StatusServiceUnavailable, "UNKNOWN", "authorization unavailable")
return
}
if err := gw.evaluatePrincipalPolicy(principal, policy.IAMAction("ecr", req.Action), resource); err != nil {
if err := gw.evaluatePrincipalPolicyResources(
principal, policy.IAMAction("ecr", req.Action), []string{resource},
); err != nil {
if err.Error() == awserrors.ErrorInternalError {
// IAM/STS/NATS dependency failure: fail closed, never dispatch.
gateway_ecr.WriteError(w, http.StatusServiceUnavailable, "UNKNOWN", "authorization unavailable")
spinifex/gateway/ecs.go
+1 -1
View File
@@ -56,7 +56,7 @@ func (gw *GatewayConfig) ECS_Request(w http.ResponseWriter, r *http.Request) err
// enforce iam:PassRole against the caller's identity on this request, for
// whichever role ARN the action later resolves.
passRoleCheck := func(roleARN string) error {
return gw.checkPolicyResource(r, "iam", "PassRole", roleARN)
return gw.checkPolicyResources(r, "iam", "PassRole", []string{roleARN})
}
output, err := handler(r.Context(), gw.NATSConn, accountID, body, passRoleCheck)
spinifex/gateway/gateway.go
+17 -22
View File
@@ -515,21 +515,15 @@ func isNATSTransient(err error) bool {
}
// checkPolicy evaluates IAM policies against resource "*".
// Shorthand for checkPolicyResource(r, service, action, "*").
func (gw *GatewayConfig) checkPolicy(r *http.Request, service, action string) error {
return gw.checkPolicyResource(r, service, action, "*")
return gw.checkPolicyResources(r, service, action, []string{"*"})
}
// checkPolicyResource evaluates IAM policies against a specific resource ARN.
// Root users bypass evaluation. A nil IAMService is a server fault and an
// unauthenticated request is denied — neither can reach a gated handler through
// the route tree. Used by EC2 paths that enforce iam:PassRole before attaching
// an instance profile.
func (gw *GatewayConfig) checkPolicyResource(r *http.Request, service, action, resource string) error {
// checkPolicyResources evaluates every resource against one resolved policy
// snapshot. Used when one API request authorizes multiple resource ARNs.
func (gw *GatewayConfig) checkPolicyResources(r *http.Request, service, action string, resources []string) error {
// Every dispatcher — query-protocol and REST-JSON alike — reaches this
// point with its resolved action, so telemetry enrichment lives here
// rather than duplicated per REST-JSON handler. Runs before the IAM
// checks below so it still fires when IAM is unconfigured.
// point with its resolved action, so telemetry enrichment lives here.
recordResolvedAction(r.Context(), service, action)
if gw.IAMService == nil {
@@ -566,7 +560,7 @@ func (gw *GatewayConfig) checkPolicyResource(r *http.Request, service, action, r
assumedRoleID: mustCtxString(r, ctxAssumedRoleID),
underlyingRoleARN: mustCtxString(r, ctxUnderlyingRoleARN),
}
return gw.evaluatePrincipalPolicy(principal, policy.IAMAction(service, action), resource)
return gw.evaluatePrincipalPolicyResources(principal, policy.IAMAction(service, action), resources)
}
// mustCtxString reads a string context value, defaulting to "" for an absent
@@ -576,12 +570,11 @@ func mustCtxString(r *http.Request, key contextKey) string {
return v
}
// evaluatePrincipalPolicy is the request-shape-independent core of policy
// enforcement: given an already-resolved principal (from SigV4 or, for /v2/*
// ECR requests, a freshly rehydrated token identity), it resolves the
// principal's current policies and evaluates iamAction against resource.
// A nil IAMService fails closed here, matching checkPolicyResource.
func (gw *GatewayConfig) evaluatePrincipalPolicy(principal principalContext, iamAction, resource string) error {
// evaluatePrincipalPolicyResources resolves policies once and evaluates every
// resource in the request against that same snapshot.
func (gw *GatewayConfig) evaluatePrincipalPolicyResources(
principal principalContext, iamAction string, resources []string,
) error {
if gw.IAMService == nil {
slog.Error("evaluatePrincipalPolicy: IAM service not available", "action", iamAction)
return errors.New(awserrors.ErrorInternalError)
@@ -644,11 +637,13 @@ func (gw *GatewayConfig) evaluatePrincipalPolicy(principal principalContext, iam
return errors.New(awserrors.ErrorInternalError)
}
if iampolicy.Evaluate(iamAction, resource, policies) == iampolicy.Deny {
slog.Info("evaluatePrincipalPolicy: access denied", "identity", logIdentity, "action", iamAction, "resource", resource)
return errors.New(awserrors.ErrorAccessDenied)
for _, resource := range resources {
if iampolicy.Evaluate(iamAction, resource, policies) == iampolicy.Deny {
slog.Info("evaluatePrincipalPolicy: access denied",
"identity", logIdentity, "action", iamAction, "resource", resource)
return errors.New(awserrors.ErrorAccessDenied)
}
}
return nil
}
spinifex/gateway/gateway_test.go
+2 -2
View File
@@ -1122,8 +1122,8 @@ func TestCheckPolicy_PassRoleFailsClosed(t *testing.T) {
gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService()}
req := httptest.NewRequest(http.MethodPost, "/", nil)
err := gw.checkPolicyResource(req, "iam", "PassRole",
"arn:aws:iam::123456789012:role/app")
err := gw.checkPolicyResources(req, "iam", "PassRole",
[]string{"arn:aws:iam::123456789012:role/app"})
require.Error(t, err)
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
}
spinifex/gateway/rds.go
+2 -2
View File
@@ -43,11 +43,11 @@ func (gw *GatewayConfig) RDS_Request(w http.ResponseWriter, r *http.Request) err
return err
}
resource, err := gateway_rds.ResourceARN(action, gw.Region, caller.AccountID, queryArgs)
resources, err := gateway_rds.ResourceARN(action, gw.Region, caller.AccountID, queryArgs)
if err != nil {
return err
}
if err := gw.checkPolicyResource(r, "rds", action, resource); err != nil {
if err := gw.checkPolicyResources(r, "rds", action, resources); err != nil {
return err
}
spinifex/gateway/rds/authz.go
+27 -22
View File
@@ -15,9 +15,8 @@ import (
// address.
const anyResource = "*"
// Which query parameter names the single resource an action acts on, and the ARN
// type it resolves to. An action naming no resource, or more than one, carries no
// scope at all.
// Which query parameter names a resource an action acts on, and the ARN type it
// resolves to. Actions carry one scope for each resource IAM evaluates.
type resourceScope struct {
param string
// Empty when param already carries a full ARN — the tag actions — which is
@@ -62,33 +61,39 @@ func requireAgentPrincipal(ctx context.Context, caller Caller) error {
return nil
}
// ResourceARN builds the resource the action's policy check evaluates against. A
// ResourceARN builds every resource the action's policy checks evaluate. A
// policy written for arn:aws:rds:*:*:db:prod-* but checked against "*" would
// permit every instance in the account, which is worse than no policy at all.
func ResourceARN(action, region, accountID string, q map[string]string) (string, error) {
func ResourceARN(action, region, accountID string, q map[string]string) ([]string, error) {
def, ok := actions[action]
// Rejected here too, not just by the dispatcher: a caller that skipped the
// action check must not get a resource back for rds:<garbage>.
if !ok {
return "", errors.New(awserrors.ErrorInvalidAction)
return nil, errors.New(awserrors.ErrorInvalidAction)
}
if def.scope == nil {
return anyResource, nil
if len(def.scopes) == 0 {
return []string{anyResource}, nil
}
identifier := q[def.scope.param]
// An absent identifier is a validation fault the handler reports precisely;
// answering it with a denial here would hide the real reason.
if identifier == "" {
return anyResource, nil
}
if def.scope.kind == "" {
// Parsed rather than passed through, so a foreign account or region is an
// InvalidParameterValue before the evaluator sees it: cross-account sharing
// is out of v1, and a permissive parser is how it arrives anyway.
if _, err := handlers_rds.ParseARN(identifier, region, accountID); err != nil {
return "", err
resources := make([]string, 0, len(def.scopes))
for _, scope := range def.scopes {
identifier := q[scope.param]
// A missing member contributes "*" independently. It remains the handler's
// validation fault rather than becoming an ARN resolution failure here.
if identifier == "" {
resources = append(resources, anyResource)
continue
}
return identifier, nil
if scope.kind == "" {
// Parsed rather than passed through, so a foreign account or region is an
// InvalidParameterValue before the evaluator sees it.
if _, err := handlers_rds.ParseARN(identifier, region, accountID); err != nil {
return nil, err
}
resources = append(resources, identifier)
continue
}
resources = append(resources, handlers_rds.FormatARN(scope.kind, region, accountID, identifier))
}
return handlers_rds.FormatARN(def.scope.kind, region, accountID, identifier), nil
return resources, nil
}
spinifex/gateway/rds/authz_test.go
+60 -21
View File
@@ -108,13 +108,6 @@ func TestResourceARN_ScopesSingleResourceActions(t *testing.T) {
"arn:aws:rds:ap-southeast-2:123456789012:db:orders-db"},
{"DeleteDBSnapshot", map[string]string{"DBSnapshotIdentifier": "orders-db-pre-upgrade"},
"arn:aws:rds:ap-southeast-2:123456789012:snapshot:orders-db-pre-upgrade"},
// Both name a source and a target; each is scoped to its source.
{"CreateDBSnapshot", map[string]string{
"DBInstanceIdentifier": "orders-db", "DBSnapshotIdentifier": "orders-db-pre-upgrade"},
"arn:aws:rds:ap-southeast-2:123456789012:db:orders-db"},
{"RestoreDBInstanceFromDBSnapshot", map[string]string{
"DBInstanceIdentifier": "orders-db-restored", "DBSnapshotIdentifier": "orders-db-pre-upgrade"},
"arn:aws:rds:ap-southeast-2:123456789012:snapshot:orders-db-pre-upgrade"},
{"DeleteDBSubnetGroup", map[string]string{"DBSubnetGroupName": "db-private"},
"arn:aws:rds:ap-southeast-2:123456789012:subgrp:db-private"},
{"ModifyDBParameterGroup", map[string]string{"DBParameterGroupName": "pg16-tuned"},
@@ -128,6 +121,32 @@ func TestResourceARN_ScopesSingleResourceActions(t *testing.T) {
t.Run(tt.action, func(t *testing.T) {
got, err := ResourceARN(tt.action, testRegion, testAccountID, tt.params)
require.NoError(t, err)
assert.Equal(t, []string{tt.want}, got)
})
}
}
func TestResourceARN_ScopesSnapshotActionsToSourceAndTarget(t *testing.T) {
tests := []struct {
action string
want []string
}{
{"CreateDBSnapshot", []string{
"arn:aws:rds:ap-southeast-2:123456789012:db:orders-db",
"arn:aws:rds:ap-southeast-2:123456789012:snapshot:orders-db-pre-upgrade",
}},
{"RestoreDBInstanceFromDBSnapshot", []string{
"arn:aws:rds:ap-southeast-2:123456789012:snapshot:orders-db-pre-upgrade",
"arn:aws:rds:ap-southeast-2:123456789012:db:orders-db",
}},
}
params := map[string]string{
"DBInstanceIdentifier": "orders-db", "DBSnapshotIdentifier": "orders-db-pre-upgrade",
}
for _, tt := range tests {
t.Run(tt.action, func(t *testing.T) {
got, err := ResourceARN(tt.action, testRegion, testAccountID, params)
require.NoError(t, err)
assert.Equal(t, tt.want, got)
})
}
@@ -153,7 +172,7 @@ func TestResourceARN_UnscopedActions(t *testing.T) {
"DBSnapshotIdentifier": "orders-db-pre-upgrade",
})
require.NoError(t, err)
assert.Equal(t, anyResource, got)
assert.Equal(t, []string{anyResource}, got)
})
}
}
@@ -196,9 +215,15 @@ func TestResourceARN_ResourceScopedDenyReachesTheSnapshotActions(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resource, err := ResourceARN(tt.action, testRegion, testAccountID, tt.params)
resources, err := ResourceARN(tt.action, testRegion, testAccountID, tt.params)
require.NoError(t, err)
assert.Equal(t, tt.want, iampolicy.Evaluate(policy.IAMAction("rds", tt.action), resource, policies))
got := iampolicy.Allow
for _, resource := range resources {
if iampolicy.Evaluate(policy.IAMAction("rds", tt.action), resource, policies) == iampolicy.Deny {
got = iampolicy.Deny
}
}
assert.Equal(t, tt.want, got)
})
}
}
@@ -211,7 +236,7 @@ func TestResourceARN_TagActionsUseTheSuppliedARN(t *testing.T) {
t.Run(action, func(t *testing.T) {
got, err := ResourceARN(action, testRegion, testAccountID, map[string]string{"ResourceName": arn})
require.NoError(t, err)
assert.Equal(t, arn, got)
assert.Equal(t, []string{arn}, got)
})
}
}
@@ -237,16 +262,30 @@ func TestResourceARN_RejectsUnusableARNs(t *testing.T) {
}
}
// A missing identifier is the handler's validation fault to report. Answering it
// with a denial here would tell the caller the wrong thing about their policy.
func TestResourceARN_MissingIdentifierFallsBackToAnyResource(t *testing.T) {
for _, action := range []string{
"DeleteDBInstance", "DescribeDBParameters", "ListTagsForResource",
"CreateDBSnapshot", "RestoreDBInstanceFromDBSnapshot",
} {
got, err := ResourceARN(action, testRegion, testAccountID, map[string]string{})
require.NoError(t, err, "action %q", action)
assert.Equal(t, anyResource, got, "action %q", action)
// A missing identifier is the handler's validation fault to report. Each absent
// member contributes "*" instead of failing ARN resolution.
func TestResourceARN_MissingIdentifierFallsBackToAnyResourcePerMember(t *testing.T) {
tests := []struct {
action string
params map[string]string
want []string
}{
{"DeleteDBInstance", nil, []string{anyResource}},
{"DescribeDBParameters", nil, []string{anyResource}},
{"ListTagsForResource", nil, []string{anyResource}},
{"CreateDBSnapshot", map[string]string{"DBInstanceIdentifier": "orders-db"}, []string{
"arn:aws:rds:ap-southeast-2:123456789012:db:orders-db", anyResource,
}},
{"RestoreDBInstanceFromDBSnapshot", map[string]string{"DBSnapshotIdentifier": "orders-db-snapshot"}, []string{
"arn:aws:rds:ap-southeast-2:123456789012:snapshot:orders-db-snapshot", anyResource,
}},
}
for _, tt := range tests {
t.Run(tt.action, func(t *testing.T) {
got, err := ResourceARN(tt.action, testRegion, testAccountID, tt.params)
require.NoError(t, err)
assert.Equal(t, tt.want, got)
})
}
}
spinifex/gateway/rds/handler.go
+26 -22
View File
@@ -80,8 +80,8 @@ func typed[In any](handler func(context.Context, *In, *nats.Conn, Caller) (any,
}
// One entry per action: what serves it, whether it is agent-only, and which
// resource its policy check evaluates against. Both authorization facts live on
// the table so a new action cannot be added without deciding either.
// resources its policy checks evaluate against. Both authorization facts live
// here so a new action cannot be added without deciding either.
type actionDef struct {
handler Handler
// Agent-only: refused to every customer principal by class, before any
@@ -92,7 +92,7 @@ type actionDef struct {
unsupported bool
// nil evaluates against "*", which is right for creates and for describes
// that filter rather than address one resource.
scope *resourceScope
scopes []*resourceScope
}
// The whole namespace is registered from day one, so an action outside v1 stays
@@ -101,19 +101,20 @@ var actions = map[string]actionDef{
// Instance lifecycle.
"CreateDBInstance": {handler: typedEnv(CreateDBInstance)},
"DescribeDBInstances": {handler: typed(DescribeDBInstances)},
"ModifyDBInstance": {handler: typed(ModifyDBInstance), scope: dbInstanceScope},
"DeleteDBInstance": {handler: typed(DeleteDBInstance), scope: dbInstanceScope},
"RebootDBInstance": {handler: typed(RebootDBInstance), scope: dbInstanceScope},
"StartDBInstance": {handler: typed(StartDBInstance), scope: dbInstanceScope},
"StopDBInstance": {handler: typed(StopDBInstance), scope: dbInstanceScope},
"ModifyDBInstance": {handler: typed(ModifyDBInstance), scopes: []*resourceScope{dbInstanceScope}},
"DeleteDBInstance": {handler: typed(DeleteDBInstance), scopes: []*resourceScope{dbInstanceScope}},
"RebootDBInstance": {handler: typed(RebootDBInstance), scopes: []*resourceScope{dbInstanceScope}},
"StartDBInstance": {handler: typed(StartDBInstance), scopes: []*resourceScope{dbInstanceScope}},
"StopDBInstance": {handler: typed(StopDBInstance), scopes: []*resourceScope{dbInstanceScope}},
// Snapshots. A create and a restore each name two resources, and both are
// scoped to the source: a deny written on an instance has to stop it being
// snapshotted, and a deny on a snapshot has to stop it being restored.
"CreateDBSnapshot": {handler: typed(CreateDBSnapshot), scope: dbInstanceScope},
"DescribeDBSnapshots": {handler: typed(DescribeDBSnapshots)},
"DeleteDBSnapshot": {handler: typed(DeleteDBSnapshot), scope: dbSnapshotScope},
"RestoreDBInstanceFromDBSnapshot": {handler: typed(RestoreDBInstanceFromDBSnapshot), scope: dbSnapshotScope},
// Snapshot creates and restores evaluate both the source and target. A deny
// on either resource must block the operation.
"CreateDBSnapshot": {handler: typed(CreateDBSnapshot),
scopes: []*resourceScope{dbInstanceScope, dbSnapshotScope}},
"DescribeDBSnapshots": {handler: typed(DescribeDBSnapshots)},
"DeleteDBSnapshot": {handler: typed(DeleteDBSnapshot), scopes: []*resourceScope{dbSnapshotScope}},
"RestoreDBInstanceFromDBSnapshot": {handler: typed(RestoreDBInstanceFromDBSnapshot),
scopes: []*resourceScope{dbSnapshotScope, dbInstanceScope}},
// Automated backups.
"DescribeDBInstanceAutomatedBackups": {handler: typed(DescribeDBInstanceAutomatedBackups)},
@@ -121,21 +122,24 @@ var actions = map[string]actionDef{
// Subnet groups.
"CreateDBSubnetGroup": {handler: typed(CreateDBSubnetGroup)},
"DescribeDBSubnetGroups": {handler: typed(DescribeDBSubnetGroups)},
"DeleteDBSubnetGroup": {handler: typed(DeleteDBSubnetGroup), scope: dbSubnetGroupScope},
"DeleteDBSubnetGroup": {handler: typed(DeleteDBSubnetGroup), scopes: []*resourceScope{dbSubnetGroupScope}},
// Parameter groups. DescribeDBParameters is scoped despite being a describe:
// its parameter group is required and singular, so it addresses one resource.
"CreateDBParameterGroup": {handler: typed(CreateDBParameterGroup)},
"DescribeDBParameterGroups": {handler: typed(DescribeDBParameterGroups)},
"ModifyDBParameterGroup": {handler: typed(ModifyDBParameterGroup), scope: dbParameterGroupScope},
"DescribeDBParameters": {handler: typed(DescribeDBParameters), scope: dbParameterGroupScope},
"DeleteDBParameterGroup": {handler: typed(DeleteDBParameterGroup), scope: dbParameterGroupScope},
"ModifyDBParameterGroup": {handler: typed(ModifyDBParameterGroup),
scopes: []*resourceScope{dbParameterGroupScope}},
"DescribeDBParameters": {handler: typed(DescribeDBParameters),
scopes: []*resourceScope{dbParameterGroupScope}},
"DeleteDBParameterGroup": {handler: typed(DeleteDBParameterGroup),
scopes: []*resourceScope{dbParameterGroupScope}},
// Tags. The request names its resource by ARN, so the scope validates one
// rather than building it.
"AddTagsToResource": {handler: typed(AddTagsToResource), scope: taggedResourceScope},
"RemoveTagsFromResource": {handler: typed(RemoveTagsFromResource), scope: taggedResourceScope},
"ListTagsForResource": {handler: typed(ListTagsForResource), scope: taggedResourceScope},
"AddTagsToResource": {handler: typed(AddTagsToResource), scopes: []*resourceScope{taggedResourceScope}},
"RemoveTagsFromResource": {handler: typed(RemoveTagsFromResource), scopes: []*resourceScope{taggedResourceScope}},
"ListTagsForResource": {handler: typed(ListTagsForResource), scopes: []*resourceScope{taggedResourceScope}},
// Events. The ring is per-account and a filter names no single resource.
"DescribeEvents": {handler: typed(DescribeEvents)},
spinifex/gateway/rds_test.go
+104
View File
@@ -192,6 +192,110 @@ func TestRDSRequest_ResourceScopedPolicyRestrictsByIdentifier(t *testing.T) {
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
}
func TestRDSRequest_SnapshotActionsAuthorizeSourceAndTarget(t *testing.T) {
tests := []struct {
name string
action string
query string
resources handlers_iam.StringOrArr
}{
{
name: "create snapshot",
action: "rds:CreateDBSnapshot",
query: "Action=CreateDBSnapshot&DBInstanceIdentifier=orders-db" +
"&DBSnapshotIdentifier=orders-db-nightly",
resources: handlers_iam.StringOrArr{
handlers_rds.DBInstanceARN("ap-southeast-2", rdsTestAccountID, "orders-db"),
handlers_rds.DBSnapshotARN("ap-southeast-2", rdsTestAccountID, "orders-db-nightly"),
},
},
{
name: "restore snapshot",
action: "rds:RestoreDBInstanceFromDBSnapshot",
query: "Action=RestoreDBInstanceFromDBSnapshot&DBSnapshotIdentifier=orders-db-nightly" +
"&DBInstanceIdentifier=orders-db-restored",
resources: handlers_iam.StringOrArr{
handlers_rds.DBSnapshotARN("ap-southeast-2", rdsTestAccountID, "orders-db-nightly"),
handlers_rds.DBInstanceARN("ap-southeast-2", rdsTestAccountID, "orders-db-restored"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
policies := []handlers_iam.PolicyDocument{{
Version: "2012-10-17",
Statement: []handlers_iam.Statement{{
Effect: "Allow", Action: handlers_iam.StringOrArr{tt.action}, Resource: tt.resources,
}},
}}
gw, probe := newRDSPolicyGateway(policies)
err := gw.RDS_Request(httptest.NewRecorder(), setupRDSUserRequest(tt.query, rdsTestAccountID))
require.Error(t, err)
assert.Equal(t, 1, probe.consulted)
assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
})
}
}
func TestRDSRequest_SnapshotActionsUseOnePolicySnapshot(t *testing.T) {
source := handlers_rds.DBInstanceARN("ap-southeast-2", rdsTestAccountID, "orders-db")
target := handlers_rds.DBSnapshotARN("ap-southeast-2", rdsTestAccountID, "orders-db-nightly")
resolution := 0
gw, probe := newRDSGatewayWithResolver(func(_, _ string) ([]handlers_iam.PolicyDocument, error) {
resolution++
if resolution == 1 {
return allowPolicyResource("rds:CreateDBSnapshot", source), nil
}
return allowPolicyResource("rds:CreateDBSnapshot", target), nil
})
err := gw.RDS_Request(httptest.NewRecorder(), setupRDSUserRequest(
"Action=CreateDBSnapshot&DBInstanceIdentifier=orders-db&DBSnapshotIdentifier=orders-db-nightly",
rdsTestAccountID,
))
require.Error(t, err)
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
assert.Equal(t, 1, probe.consulted)
}
func TestRDSRequest_SnapshotActionsHonorTargetDeny(t *testing.T) {
tests := []struct {
name string
action string
query string
target string
}{
{
name: "create snapshot", action: "rds:CreateDBSnapshot",
query: "Action=CreateDBSnapshot&DBInstanceIdentifier=orders-db" +
"&DBSnapshotIdentifier=orders-db-nightly",
target: handlers_rds.DBSnapshotARN("ap-southeast-2", rdsTestAccountID, "orders-db-nightly"),
},
{
name: "restore snapshot", action: "rds:RestoreDBInstanceFromDBSnapshot",
query: "Action=RestoreDBInstanceFromDBSnapshot&DBSnapshotIdentifier=orders-db-nightly" +
"&DBInstanceIdentifier=orders-db-restored",
target: handlers_rds.DBInstanceARN("ap-southeast-2", rdsTestAccountID, "orders-db-restored"),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
policies := []handlers_iam.PolicyDocument{{
Version: "2012-10-17",
Statement: []handlers_iam.Statement{
{Effect: "Allow", Action: handlers_iam.StringOrArr{tt.action}, Resource: handlers_iam.StringOrArr{"*"}},
{Effect: "Deny", Action: handlers_iam.StringOrArr{tt.action}, Resource: handlers_iam.StringOrArr{tt.target}},
},
}}
gw, probe := newRDSPolicyGateway(policies)
err := gw.RDS_Request(httptest.NewRecorder(), setupRDSUserRequest(tt.query, rdsTestAccountID))
require.Error(t, err)
assert.Equal(t, 1, probe.consulted)
assert.Equal(t, awserrors.ErrorAccessDenied, err.Error())
})
}
}
// Creates and plural describes name no one resource, so a policy scoped to a
// single instance must not be read as permitting them.
func TestRDSRequest_UnscopedActionsEvaluateAgainstAnyResource(t *testing.T) {
spinifex/handlers/ec2/instance/service_impl.go
+77 -66
View File
@@ -716,74 +716,69 @@ func (s *InstanceServiceImpl) PrepareRunInstances(ctx context.Context, input *ec
"mac", instance.ENIMac,
)
if s.ipAllocator != nil {
subnet, subErr := s.eniCreator.GetSubnet(ctx, accountID, *input.SubnetId)
wantPublic := subErr == nil && subnet != nil && subnet.MapPublicIpOnLaunch
if len(input.NetworkInterfaces) > 0 && input.NetworkInterfaces[0] != nil && input.NetworkInterfaces[0].AssociatePublicIpAddress != nil {
wantPublic = *input.NetworkInterfaces[0].AssociatePublicIpAddress
}
if wantPublic {
region := s.config.Region
az := s.config.AZ
publicIP, poolName, allocErr := s.ipAllocator.AllocateIP(ctx, region, az, handlers_ec2_vpc.PurposeENIPublic, "", *eni.NetworkInterfaceId, instance.ID)
if allocErr != nil {
// Fail rather than boot an unreachable instance;
// detach before delete since in-use ENIs reject deletion.
slog.ErrorContext(ctx, "PrepareRunInstances: public IP allocation failed — aborting launch",
"instanceId", instance.ID, "eniId", *eni.NetworkInterfaceId, "err", allocErr)
if detErr := s.eniCreator.DetachENI(ctx, accountID, *eni.NetworkInterfaceId); detErr != nil {
slog.WarnContext(ctx, "PrepareRunInstances: failed to detach ENI after public-IP allocation failure",
"eniId", *eni.NetworkInterfaceId, "err", detErr)
}
if _, delErr := s.eniDeleter.DeleteNetworkInterface(ctx, &ec2.DeleteNetworkInterfaceInput{
NetworkInterfaceId: eni.NetworkInterfaceId,
}, accountID); delErr != nil {
slog.WarnContext(ctx, "PrepareRunInstances: failed to delete ENI after public-IP allocation failure",
"eniId", *eni.NetworkInterfaceId, "err", delErr)
}
// Carry the allocator's own error rather than a flat
// capacity code: the gateway resolves the AWS code out
// of the wrap chain, so a genuinely exhausted pool still
// surfaces InsufficientAddressCapacity while an upstream
// DHCP or IPAM fault reports as itself instead of
// wearing a capacity code it did not earn.
lastRunErr = fmt.Errorf("allocate public IP for %s: %w", instance.ID, allocErr)
if reservationID == "" {
s.resourceMgr.Deallocate(instanceType)
} else {
s.resourceMgr.ReleaseToReservation(reservationID, instanceType)
}
continue
} else {
if updateErr := s.eniCreator.UpdateENIPublicIP(ctx, accountID, *eni.NetworkInterfaceId, publicIP, poolName); updateErr != nil {
slog.WarnContext(ctx, "PrepareRunInstances: failed to update ENI with public IP", "eniId", *eni.NetworkInterfaceId, "err", updateErr)
}
portName := topology.Port(*eni.NetworkInterfaceId)
if natErr := utils.AddNAT(s.natsConn, *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress); natErr != nil {
slog.ErrorContext(ctx, "PrepareRunInstances: vpc.add-nat failed — rolling back public IP to avoid surfacing an unreachable address",
"instanceId", instance.ID, "publicIp", publicIP, "pool", poolName, "err", natErr)
// Neutralise before releasing in case timeout committed the rule.
utils.PublishNATEvent(s.natsConn, "vpc.delete-nat", *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress)
s.rollbackAutoAssignedPublicIP(ctx, accountID, instance.ID, *eni.NetworkInterfaceId, publicIP, poolName)
lastRunErr = natErr
if reservationID == "" {
s.resourceMgr.Deallocate(instanceType)
} else {
s.resourceMgr.ReleaseToReservation(reservationID, instanceType)
}
continue
}
ec2Instance.PublicIpAddress = aws.String(publicIP)
instance.PublicIP = publicIP
instance.PublicIPPool = poolName
slog.InfoContext(ctx, "PrepareRunInstances: auto-assigned public IP",
"instanceId", instance.ID,
"publicIp", publicIP,
"privateIp", *eni.PrivateIpAddress,
"pool", poolName,
)
subnet, subErr := s.eniCreator.GetSubnet(ctx, accountID, *input.SubnetId)
wantPublic := subErr == nil && subnet != nil && subnet.MapPublicIpOnLaunch
if len(input.NetworkInterfaces) > 0 && input.NetworkInterfaces[0] != nil && input.NetworkInterfaces[0].AssociatePublicIpAddress != nil {
wantPublic = *input.NetworkInterfaces[0].AssociatePublicIpAddress
}
if wantPublic {
publicIP, poolName, allocErr := s.allocatePublicIP(ctx, *eni.NetworkInterfaceId, instance.ID)
if allocErr != nil {
// Fail rather than boot an unreachable instance;
// detach before delete since in-use ENIs reject deletion.
slog.ErrorContext(ctx, "PrepareRunInstances: public IP allocation failed — aborting launch",
"instanceId", instance.ID, "eniId", *eni.NetworkInterfaceId, "err", allocErr)
if detErr := s.eniCreator.DetachENI(ctx, accountID, *eni.NetworkInterfaceId); detErr != nil {
slog.WarnContext(ctx, "PrepareRunInstances: failed to detach ENI after public-IP allocation failure",
"eniId", *eni.NetworkInterfaceId, "err", detErr)
}
if _, delErr := s.eniDeleter.DeleteNetworkInterface(ctx, &ec2.DeleteNetworkInterfaceInput{
NetworkInterfaceId: eni.NetworkInterfaceId,
}, accountID); delErr != nil {
slog.WarnContext(ctx, "PrepareRunInstances: failed to delete ENI after public-IP allocation failure",
"eniId", *eni.NetworkInterfaceId, "err", delErr)
}
// Carry the allocator's own error rather than a flat
// capacity code: the gateway resolves the AWS code out
// of the wrap chain, so a genuinely exhausted pool still
// surfaces InsufficientAddressCapacity while an upstream
// DHCP or IPAM fault reports as itself instead of
// wearing a capacity code it did not earn.
lastRunErr = fmt.Errorf("allocate public IP for %s: %w", instance.ID, allocErr)
if reservationID == "" {
s.resourceMgr.Deallocate(instanceType)
} else {
s.resourceMgr.ReleaseToReservation(reservationID, instanceType)
}
continue
}
if updateErr := s.eniCreator.UpdateENIPublicIP(ctx, accountID, *eni.NetworkInterfaceId, publicIP, poolName); updateErr != nil {
slog.WarnContext(ctx, "PrepareRunInstances: failed to update ENI with public IP", "eniId", *eni.NetworkInterfaceId, "err", updateErr)
}
portName := topology.Port(*eni.NetworkInterfaceId)
if natErr := utils.AddNAT(s.natsConn, *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress); natErr != nil {
slog.ErrorContext(ctx, "PrepareRunInstances: vpc.add-nat failed — rolling back public IP to avoid surfacing an unreachable address",
"instanceId", instance.ID, "publicIp", publicIP, "pool", poolName, "err", natErr)
// Neutralise before releasing in case timeout committed the rule.
utils.PublishNATEvent(s.natsConn, "vpc.delete-nat", *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress)
s.rollbackAutoAssignedPublicIP(ctx, accountID, instance.ID, *eni.NetworkInterfaceId, publicIP, poolName)
lastRunErr = natErr
if reservationID == "" {
s.resourceMgr.Deallocate(instanceType)
} else {
s.resourceMgr.ReleaseToReservation(reservationID, instanceType)
}
continue
}
ec2Instance.PublicIpAddress = aws.String(publicIP)
instance.PublicIP = publicIP
instance.PublicIPPool = poolName
slog.InfoContext(ctx, "PrepareRunInstances: auto-assigned public IP",
"instanceId", instance.ID,
"publicIp", publicIP,
"privateIp", *eni.PrivateIpAddress,
"pool", poolName,
)
}
}
@@ -2508,6 +2503,22 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
_ = instanceID
}
// allocatePublicIP allocates an auto-assigned public IP for a new ENI. A node
// whose external IPAM never initialised has no allocator wired: report no
// capacity so the caller aborts rather than booting an unreachable instance.
func (s *InstanceServiceImpl) allocatePublicIP(ctx context.Context, eniID, instanceID string) (string, string, error) {
if s.ipAllocator == nil {
return "", "", fmt.Errorf("no public IP allocator on this node: %w", errors.New(awserrors.ErrorInsufficientAddressCapacity))
}
region := ""
az := ""
if s.config != nil {
region = s.config.Region
az = s.config.AZ
}
return s.ipAllocator.AllocateIP(ctx, region, az, handlers_ec2_vpc.PurposeENIPublic, "", eniID, instanceID)
}
// rollbackAutoAssignedPublicIP unwinds a failed auto-assign: clears the ENI
// public IP record, releases the IPAM lease, then detaches and deletes the ENI.
func (s *InstanceServiceImpl) rollbackAutoAssignedPublicIP(ctx context.Context, accountID, instanceID, eniID, publicIP, poolName string) {
spinifex/handlers/ec2/instance/service_impl_test.go
+50
View File
@@ -16,6 +16,7 @@ import (
"github.com/mulgadc/spinifex/spinifex/config"
"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
"github.com/mulgadc/spinifex/spinifex/gpu"
handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/tags"
"github.com/mulgadc/spinifex/spinifex/testutil"
spxtypes "github.com/mulgadc/spinifex/spinifex/types"
@@ -3165,6 +3166,55 @@ func TestPrepareRunInstances_PublicIPAutoAssigned(t *testing.T) {
}
}
// TestPrepareRunInstances_PublicIPWithoutAllocator covers a node whose external
// IPAM never initialised. A nil *ExternalIPAM in the interface reads as non-nil,
// so the launch used to deref a nil receiver and take the daemon down with the
// ENI already persisted; both wirings must now fail closed and unwind the ENI.
func TestPrepareRunInstances_PublicIPWithoutAllocator(t *testing.T) {
cases := []struct {
name string
ipam PublicIPAllocator
}{
{name: "typed_nil_external_ipam", ipam: (*handlers_ec2_vpc.ExternalIPAM)(nil)},
{name: "no_allocator_wired", ipam: nil},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
eni := &fakeENICreator{
subnet: &SubnetInfo{SubnetID: "subnet-1", VpcID: "vpc-1", MapPublicIpOnLaunch: true},
createOut: &ec2.CreateNetworkInterfaceOutput{
NetworkInterface: &ec2.NetworkInterface{
NetworkInterfaceId: aws.String("eni-no-ipam"),
MacAddress: aws.String("aa:bb:cc:dd:ee:11"),
PrivateIpAddress: aws.String("10.0.0.40"),
VpcId: aws.String("vpc-1"),
},
},
}
deleter := &fakeENIDeleter{}
svc, prov := prepareSvcWithENI(t, eni, nil)
svc.eniDeleter = deleter
svc.SetRunInstancesDeps(svc.amiLoader, nil, eni, tc.ipam)
_, instances, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{
InstanceType: aws.String("t3.micro"),
ImageId: aws.String("ami-1"),
SubnetId: aws.String("subnet-1"),
MinCount: aws.Int64(1),
MaxCount: aws.Int64(1),
}, "acc", "")
require.Error(t, err, "a public-IP launch with no allocator must fail, not boot an instance with no public address")
assert.Equal(t, awserrors.ErrorInsufficientAddressCapacity, err.Error())
assert.Empty(t, instances)
assert.Equal(t, 1, eni.detachCalls, "ENI must be detached before delete")
assert.Equal(t, []string{"eni-no-ipam"}, deleter.calls,
"the auto-created ENI must be deleted, otherwise it strands and blocks security group deletion")
assert.Len(t, prov.deallocated, 1, "capacity must be returned when the launch aborts")
})
}
}
// TestPrepareRunInstances_NATFailureRollsBackPublicIP verifies that a vpc.add-nat
// failure drops the instance, clears the ENI public IP, releases the IPAM
// lease, and deallocates capacity.
spinifex/handlers/ec2/vpc/external_ipam.go
+9
View File
@@ -79,7 +79,12 @@ func (m *ExternalIPAM) EnableDHCP(client *dhcp.NATSClient) error {
// AllocateIP allocates the next available external IP from the best pool
// matching the given region/AZ. Returns the allocated IP and pool name.
// Nil-receiver tolerant: a node whose IPAM never initialised reports no
// capacity rather than panicking through a typed-nil interface.
func (m *ExternalIPAM) AllocateIP(ctx context.Context, region, az, purpose, allocID, eniID, instanceID string) (string, string, error) {
if m == nil {
return "", "", fmt.Errorf("external IPAM unavailable on this node: %w", errors.New(awserrors.ErrorInsufficientAddressCapacity))
}
pool := m.findPool(region, az)
if pool == nil {
return "", "", fmt.Errorf("no external pool available for region=%q az=%q: %w", region, az, errors.New(awserrors.ErrorInsufficientAddressCapacity))
@@ -113,7 +118,11 @@ func (m *ExternalIPAM) AllocateFromPool(ctx context.Context, poolName, purpose,
// ReleaseIP releases a previously allocated external IP back to its pool.
// ownerENIID, when non-empty, scopes the release to the ENI that currently owns
// the lease so a stale or duplicated teardown for a recycled IP is a no-op.
// Nil-receiver tolerant for the same reason as AllocateIP.
func (m *ExternalIPAM) ReleaseIP(ctx context.Context, poolName, ip, ownerENIID string) error {
if m == nil {
return errors.New("external IPAM unavailable on this node")
}
addr, err := netip.ParseAddr(ip)
if err != nil {
return fmt.Errorf("parse release IP %q: %w", ip, err)
spinifex/handlers/eks/addon_catalog.go
+28 -10
View File
@@ -1,6 +1,9 @@
package handlers_eks
import "slices"
import (
"fmt"
"slices"
)
// nvidiaDevicePluginAddonName is auto-staged on GPU nodegroup create (see
// stageGPUDeviceAddon in nodegroup.go), never user-requested directly.
@@ -57,18 +60,18 @@ func hiddenAddonSpec(s AddonSpec) AddonSpec {
return s
}
// newAddonSpec builds a spec with DefaultVersion = versions[0]. Panics on empty versions.
// newAddonSpec builds a spec with the first version as its default.
func newAddonSpec(name string, requiresIRSA bool, description string, versions ...string) AddonSpec {
if len(versions) == 0 {
panic("eks: addon spec " + name + " has no versions")
spec := AddonSpec{
Name: name,
Versions: versions,
RequiresIRSA: requiresIRSA,
Description: description,
}
return AddonSpec{
Name: name,
Versions: versions,
DefaultVersion: versions[0],
RequiresIRSA: requiresIRSA,
Description: description,
if len(versions) > 0 {
spec.DefaultVersion = versions[0]
}
return spec
}
// buildAddonCatalog indexes the specs by name.
@@ -80,6 +83,21 @@ func buildAddonCatalog(specs ...AddonSpec) map[string]AddonSpec {
return out
}
func validateAddonCatalog(catalog map[string]AddonSpec) error {
for name, spec := range catalog {
if spec.Name != name {
return fmt.Errorf("add-on catalog key %q does not match spec name %q", name, spec.Name)
}
if len(spec.Versions) == 0 {
return fmt.Errorf("add-on %q has no versions", name)
}
if !spec.supportsVersion(spec.DefaultVersion) {
return fmt.Errorf("add-on %q default version %q is not supported", name, spec.DefaultVersion)
}
}
return nil
}
// lookupAddon returns the spec for name and whether it is in the catalog.
func lookupAddon(name string) (AddonSpec, bool) {
spec, ok := addonCatalog[name]
spinifex/handlers/eks/addon_catalog_test.go
+55 -4
View File
@@ -40,8 +40,59 @@ func TestCatalogSpecs_SortedByName(t *testing.T) {
}
}
func TestNewAddonSpec_PanicsWithoutVersions(t *testing.T) {
assert.Panics(t, func() {
newAddonSpec("broken", false, "no versions")
})
func TestValidateAddonCatalog(t *testing.T) {
tests := []struct {
name string
catalog map[string]AddonSpec
wantErr string
}{
{
name: "valid catalog",
catalog: buildAddonCatalog(newAddonSpec("valid", false, "valid add-on", "1.0.0")),
},
{
name: "no versions",
catalog: buildAddonCatalog(newAddonSpec("broken", false, "no versions")),
wantErr: `add-on "broken" has no versions`,
},
{
name: "mismatched name",
catalog: map[string]AddonSpec{
"catalog-name": newAddonSpec("spec-name", false, "mismatched name", "1.0.0"),
},
wantErr: `add-on catalog key "catalog-name" does not match spec name "spec-name"`,
},
{
name: "unsupported default",
catalog: map[string]AddonSpec{
"broken": {
Name: "broken",
Versions: []string{"1.0.0"},
DefaultVersion: "2.0.0",
},
},
wantErr: `add-on "broken" default version "2.0.0" is not supported`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateAddonCatalog(tt.catalog)
if tt.wantErr != "" {
require.EqualError(t, err, tt.wantErr)
return
}
require.NoError(t, err)
})
}
}
func TestNewEKSServiceImpl_RejectsInvalidAddonCatalog(t *testing.T) {
previousCatalog := addonCatalog
addonCatalog = buildAddonCatalog(newAddonSpec("broken", false, "no versions"))
t.Cleanup(func() { addonCatalog = previousCatalog })
svc, err := NewEKSServiceImpl(EKSServiceDeps{})
require.EqualError(t, err, `eks: validate add-on catalog: add-on "broken" has no versions`)
assert.Nil(t, svc)
}
spinifex/handlers/eks/service_impl.go
+3
View File
@@ -289,6 +289,9 @@ const defaultK8sVersion = "1.32"
// NewEKSServiceImpl initialises EKSServiceImpl, wiring the leader KV and reconciler registry.
func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
if err := validateAddonCatalog(addonCatalog); err != nil {
return nil, fmt.Errorf("eks: validate add-on catalog: %w", err)
}
if deps.NATSConn == nil {
return nil, errors.New("eks: NewEKSServiceImpl nil NATSConn")
}
spinifex/handlers/iam/managed_policies.go
+9 -7
View File
@@ -282,18 +282,20 @@ var awsManagedPolicyDocs = map[string]string{
}
// builtinManagedPolicyParsed is the parsed form of awsManagedPolicyDocs, built
// once at init. A malformed builtin document is a programming error and panics.
var builtinManagedPolicyParsed = func() map[string]PolicyDocument {
parsed := make(map[string]PolicyDocument, len(awsManagedPolicyDocs))
for arn, raw := range awsManagedPolicyDocs {
// once at init. NewIAMServiceImpl propagates any parsing failure.
var builtinManagedPolicyParsed, builtinManagedPolicyParseErr = parseBuiltinManagedPolicies(awsManagedPolicyDocs)
func parseBuiltinManagedPolicies(rawPolicies map[string]string) (map[string]PolicyDocument, error) {
parsed := make(map[string]PolicyDocument, len(rawPolicies))
for arn, raw := range rawPolicies {
var doc PolicyDocument
if err := json.Unmarshal([]byte(raw), &doc); err != nil {
panic(fmt.Sprintf("iam: malformed builtin managed policy %s: %v", arn, err))
return nil, fmt.Errorf("parse managed policy %s: %w", arn, err)
}
parsed[arn] = doc
}
return parsed
}()
return parsed, nil
}
// builtinManagedPolicyDoc returns the modeled grant document for an AWS-managed
// policy ARN. ok is false for managed ARNs Spinifex does not model.
spinifex/handlers/iam/managed_policies_test.go
+47
View File
@@ -1,6 +1,7 @@
package handlers_iam
import (
"errors"
"testing"
"github.com/mulgadc/bluebottle/pkg/iampolicy"
@@ -22,6 +23,52 @@ var rdsInternalActions = []string{
"rds:GetDBBootstrapConfig",
}
func TestParseBuiltinManagedPolicies(t *testing.T) {
tests := []struct {
name string
raw map[string]string
wantErr string
}{
{
name: "valid document",
raw: map[string]string{
"arn:valid": `{"Version":"2012-10-17","Statement":[]}`,
},
},
{
name: "malformed document",
raw: map[string]string{
"arn:malformed": `{`,
},
wantErr: "parse managed policy arn:malformed",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
parsed, err := parseBuiltinManagedPolicies(tt.raw)
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr)
assert.Nil(t, parsed)
return
}
require.NoError(t, err)
assert.Len(t, parsed, len(tt.raw))
})
}
}
func TestNewIAMServiceImpl_PropagatesBuiltinManagedPolicyParseError(t *testing.T) {
parseErr := errors.New("malformed builtin policy")
previousErr := builtinManagedPolicyParseErr
builtinManagedPolicyParseErr = parseErr
t.Cleanup(func() { builtinManagedPolicyParseErr = previousErr })
_, err := NewIAMServiceImpl(t.Context(), nil, make([]byte, 32), 1)
require.ErrorIs(t, err, parseErr)
assert.ErrorContains(t, err, "init builtin managed policies")
}
// A role whose permissions come entirely from an attached AWS-managed ARN has to
// resolve to a grant, the way the stock EKS roles already do, rather than fail
// closed on a document Spinifex never stores.
spinifex/handlers/iam/mock/mock.go
+43 -5
View File
@@ -4,6 +4,7 @@ package mock
import (
"errors"
"sync"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iam"
@@ -15,8 +16,13 @@ import (
// handlers_iam.SystemInstanceRoleEnsurer. Roles and instance profiles are
// stored keyed by name; Get* miss with NoSuchEntity until the matching
// Create* call, and a repeat Create* reports EntityAlreadyExists, mirroring
// the live IAM API.
// the live IAM API. Every method is safe for concurrent use, so callers that
// converge on one profile from several goroutines can be exercised under the
// race detector; exported fields must only be touched while no call is in
// flight.
type SystemInstanceRoleEnsurer struct {
mu sync.Mutex
Roles map[string]*iam.Role
Profiles map[string]*iam.InstanceProfile
@@ -64,15 +70,35 @@ func New() *SystemInstanceRoleEnsurer {
}
}
// copyRole and copyProfile hand callers a snapshot rather than a pointer into
// the mock's state, so a concurrent AddRoleToInstanceProfile cannot mutate a
// row another goroutine is still reading.
func copyRole(r *iam.Role) *iam.Role {
c := *r
return &c
}
func copyProfile(p *iam.InstanceProfile) *iam.InstanceProfile {
c := *p
c.Roles = append([]*iam.Role(nil), p.Roles...)
return &c
}
func (e *SystemInstanceRoleEnsurer) GetRole(_ string, in *iam.GetRoleInput) (*iam.GetRoleOutput, error) {
e.mu.Lock()
defer e.mu.Unlock()
r, ok := e.Roles[aws.StringValue(in.RoleName)]
if !ok {
return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
}
return &iam.GetRoleOutput{Role: r}, nil
return &iam.GetRoleOutput{Role: copyRole(r)}, nil
}
func (e *SystemInstanceRoleEnsurer) CreateRole(accountID string, in *iam.CreateRoleInput) (*iam.CreateRoleOutput, error) {
e.mu.Lock()
defer e.mu.Unlock()
e.CreateRoleCalls++
e.LastTrustDoc = aws.StringValue(in.AssumeRolePolicyDocument)
e.LastRoleAcct = accountID
@@ -85,10 +111,13 @@ func (e *SystemInstanceRoleEnsurer) CreateRole(accountID string, in *iam.CreateR
Arn: aws.String("arn:aws:iam::" + accountID + ":role/" + name),
}
e.Roles[name] = r
return &iam.CreateRoleOutput{Role: r}, nil
return &iam.CreateRoleOutput{Role: copyRole(r)}, nil
}
func (e *SystemInstanceRoleEnsurer) PutRolePolicy(_ string, in *iam.PutRolePolicyInput) (*iam.PutRolePolicyOutput, error) {
e.mu.Lock()
defer e.mu.Unlock()
e.PolicyCalls = append(e.PolicyCalls, *in)
e.RolePolicies[aws.StringValue(in.RoleName)] = aws.StringValue(in.PolicyDocument)
if e.PutRolePolicyErr != nil {
@@ -98,14 +127,20 @@ func (e *SystemInstanceRoleEnsurer) PutRolePolicy(_ string, in *iam.PutRolePolic
}
func (e *SystemInstanceRoleEnsurer) GetInstanceProfile(_ string, in *iam.GetInstanceProfileInput) (*iam.GetInstanceProfileOutput, error) {
e.mu.Lock()
defer e.mu.Unlock()
p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
if !ok {
return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
}
return &iam.GetInstanceProfileOutput{InstanceProfile: p}, nil
return &iam.GetInstanceProfileOutput{InstanceProfile: copyProfile(p)}, nil
}
func (e *SystemInstanceRoleEnsurer) CreateInstanceProfile(accountID string, in *iam.CreateInstanceProfileInput) (*iam.CreateInstanceProfileOutput, error) {
e.mu.Lock()
defer e.mu.Unlock()
e.CreateInstanceProfileCalls++
e.LastProfileAcct = accountID
if e.CreateInstanceProfileErr != nil {
@@ -124,10 +159,13 @@ func (e *SystemInstanceRoleEnsurer) CreateInstanceProfile(accountID string, in *
Arn: aws.String(arn),
}
e.Profiles[name] = p
return &iam.CreateInstanceProfileOutput{InstanceProfile: p}, nil
return &iam.CreateInstanceProfileOutput{InstanceProfile: copyProfile(p)}, nil
}
func (e *SystemInstanceRoleEnsurer) AddRoleToInstanceProfile(_ string, in *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) {
e.mu.Lock()
defer e.mu.Unlock()
e.AddRoleToInstanceProfileCalls++
if e.AddRoleToInstanceProfileErr != nil {
return nil, e.AddRoleToInstanceProfileErr
spinifex/handlers/iam/service_impl.go
+3
View File
@@ -112,6 +112,9 @@ var _ IAMService = (*IAMServiceImpl)(nil)
// clusterSize sets the replication factor; pass 1 for single-node or test setups.
// The context bounds bucket creation and the schema migrations only.
func NewIAMServiceImpl(ctx context.Context, natsConn *nats.Conn, masterKey []byte, clusterSize int) (*IAMServiceImpl, error) {
if builtinManagedPolicyParseErr != nil {
return nil, fmt.Errorf("init builtin managed policies: %w", builtinManagedPolicyParseErr)
}
if len(masterKey) != 32 {
return nil, fmt.Errorf("master key must be 32 bytes, got %d", len(masterKey))
}
spinifex/handlers/rds/agent.go
+53
View File
@@ -78,6 +78,13 @@ type GetDBBootstrapConfigOutput struct {
MasterUserPassword *string `json:"masterUserPassword,omitempty" locationName:"MasterUserPassword"`
Port int64 `json:"port" locationName:"Port"`
// The customer ENI address the engine binds, and the range its client
// authentication rules admit. Together they make the endpoint the only path
// to the engine: the management NICs no customer security group governs are
// not bound at all.
ListenAddress string `json:"listenAddress" locationName:"ListenAddress"`
ClientCIDR string `json:"clientCidr" locationName:"ClientCIDR"`
DataVolumeID string `json:"dataVolumeId,omitempty" locationName:"DataVolumeId" xml:"DataVolumeId"`
DataVolumeSerial string `json:"dataVolumeSerial,omitempty" locationName:"DataVolumeSerial" xml:"DataVolumeSerial"`
VMGeneration int64 `json:"vmGeneration,omitempty" locationName:"VMGeneration" xml:"VMGeneration"`
@@ -348,6 +355,18 @@ func (s *Service) GetDBBootstrapConfig(ctx context.Context, input *GetDBBootstra
return nil, errors.New(awserrors.ErrorAccessDenied)
}
// Both fail the fetch rather than falling back to a wildcard. The agent
// retries on a backoff, so an unresolvable one presents as a stuck bootstrap
// with the reason on the guest console rather than as an engine that came up
// reachable from every network it is attached to.
if rec.ENIPrivateIP == "" {
return nil, errors.New("rds bootstrap: the DB instance record carries no endpoint ENI address for the engine to bind")
}
clientCIDR, err := s.resolveClientCIDR(ctx, kv, &rec)
if err != nil {
return nil, err
}
out := &GetDBBootstrapConfigOutput{
Mode: BootstrapModeAttach,
DBInstanceIdentifier: rec.DBInstanceIdentifier,
@@ -356,6 +375,8 @@ func (s *Service) GetDBBootstrapConfig(ctx context.Context, input *GetDBBootstra
DBName: rec.DBName,
MasterUsername: rec.MasterUsername,
Port: rec.Port,
ListenAddress: rec.ENIPrivateIP,
ClientCIDR: clientCIDR,
DataVolumeID: rec.DataVolumeID,
DataVolumeSerial: serial,
VMGeneration: rec.VMGeneration,
@@ -380,6 +401,37 @@ func (s *Service) GetDBBootstrapConfig(ctx context.Context, input *GetDBBootstra
return out, nil
}
// The range the guest's client authentication rules admit. A record predating
// the field is backfilled from its VPC and persisted, so the describe is paid
// once rather than on every boot of every instance.
//
// A failure to persist the backfill is not a failure to serve it: the value is
// re-derivable and the next fetch records it, whereas failing here would hold a
// bootable instance down for a bookkeeping write.
func (s *Service) resolveClientCIDR(ctx context.Context, kv jetstream.KeyValue, rec *DBInstanceRecord) (string, error) {
if rec.VpcCIDR != "" {
return rec.VpcCIDR, nil
}
if rec.VpcID == "" {
return "", fmt.Errorf("rds bootstrap: DB instance %s records no VPC to scope its client authentication to",
rec.DBInstanceIdentifier)
}
if s.deps.Network == nil {
return "", errors.New("rds bootstrap: RDS networking is not wired on this node")
}
cidr, err := s.vpcCIDR(ctx, rec.AccountID, rec.VpcID)
if err != nil {
return "", err
}
if err := s.updateInstance(ctx, kv, rec.DBInstanceIdentifier, func(stored *DBInstanceRecord) {
stored.VpcCIDR = cidr
}); err != nil {
slog.ErrorContext(ctx, "rds bootstrap: the VPC address range could not be recorded; serving it regardless",
"dbInstance", rec.DBInstanceIdentifier, "vpcId", rec.VpcID, "err", err)
}
return cidr, nil
}
// Fills in the initialize half of the response when a payload is staged for this
// generation, and leaves the attach response untouched otherwise.
//
@@ -556,6 +608,7 @@ func (s *Service) mintServingCert(rec *DBInstanceRecord) (*bootstrapCert, error)
DBInstanceIdentifier: rec.DBInstanceIdentifier,
PrivateIP: rec.ENIPrivateIP,
DNSName: rec.DNSName,
KeyBits: s.deps.ServingCertKeyBits,
})
if err != nil {
return nil, err
spinifex/handlers/rds/agent_test.go
+82 -4
View File
@@ -22,9 +22,16 @@ import (
"github.com/stretchr/testify/require"
)
// Every test service mints its serving certs at 1024 bits. The path under test
// is the same one production runs; only the keygen, which dominated it under
// GOFIPS140, gets cheaper.
const testServingCertKeyBits = 1024
const (
testDBID = "orders-db"
testInstance = "i-0abc123"
testVpcID = "vpc-0aaa111"
testVpcCIDR = "10.20.0.0/16"
)
// Minted once per package rather than per harness. An RSA-2048 keygen costs
@@ -66,7 +73,9 @@ func newTestCA(t *testing.T) CALoader {
func newTestService(t *testing.T) *Service {
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
return NewService(nc, testRegion).WithDeps(Deps{LoadCA: newTestCA(t), MasterKey: testMasterKey})
return NewService(nc, testRegion).WithDeps(Deps{
LoadCA: newTestCA(t), MasterKey: testMasterKey, ServingCertKeyBits: testServingCertKeyBits,
})
}
// seedInstance writes a DB instance record without the payload a pending
@@ -93,6 +102,8 @@ func defaultRecord() DBInstanceRecord {
InstanceID: testInstance,
VMGeneration: firstVMGeneration,
ENIPrivateIP: "10.20.30.40",
VpcID: testVpcID,
VpcCIDR: testVpcCIDR,
DNSName: "orders-db.123456789012.ap-southeast-2.rds.example.internal",
Bootstrap: BootstrapState{
State: BootstrapStatePending,
@@ -271,7 +282,11 @@ func TestGetDBBootstrapConfig_NoCAStillServesConfig(t *testing.T) {
assert.Equal(t, int64(5432), out.Port)
}
func TestGetDBBootstrapConfig_ConfiguredCARequiresENIPrivateIP(t *testing.T) {
// The address the engine binds, so a record without one fails the fetch on
// every deployment rather than only where TLS needs it as a SAN. The agent
// retries, so this is a stuck bootstrap with a reason rather than an engine
// that came up on the wildcard.
func TestGetDBBootstrapConfig_RequiresENIPrivateIP(t *testing.T) {
t.Parallel()
svc := newTestService(t)
rec := defaultRecord()
@@ -280,7 +295,69 @@ func TestGetDBBootstrapConfig_ConfiguredCARequiresENIPrivateIP(t *testing.T) {
_, err := svc.GetDBBootstrapConfig(t.Context(), bootstrapInput(), testAccountID)
require.Error(t, err)
assert.Contains(t, err.Error(), "configured TLS requires an ENI private IP")
assert.Contains(t, err.Error(), "no endpoint ENI address")
envelope, _ := storedPayload(t, svc, testDBID)
assert.NotNil(t, envelope, "a failed fetch must leave the payload replayable")
}
// The endpoint address and the VPC range reach the guest, which is what makes
// the engine bind one NIC and admit one range.
func TestGetDBBootstrapConfig_CarriesListenAddressAndClientCIDR(t *testing.T) {
t.Parallel()
svc := newTestService(t)
rec := defaultRecord()
seedInstance(t, svc, rec)
out, err := svc.GetDBBootstrapConfig(t.Context(),
&GetDBBootstrapConfigInput{DBInstanceIdentifier: testDBID, InstanceID: testInstance}, testAccountID)
require.NoError(t, err)
assert.Equal(t, rec.ENIPrivateIP, out.ListenAddress)
assert.Equal(t, testVpcCIDR, out.ClientCIDR)
}
// A record predating the field resolves its VPC once and keeps the answer, so
// the describe is not paid on every boot of every instance.
func TestGetDBBootstrapConfig_BackfillsVpcCIDR(t *testing.T) {
t.Parallel()
_, nc, _ := testutil.StartTestJetStream(t)
network := newFakeNetwork()
svc := NewService(nc, testRegion).WithDeps(Deps{
LoadCA: newTestCA(t), MasterKey: testMasterKey, Network: network,
ServingCertKeyBits: testServingCertKeyBits,
})
rec := defaultRecord()
rec.VpcID = testDefaultVPC
rec.VpcCIDR = ""
seedInstance(t, svc, rec)
in := &GetDBBootstrapConfigInput{DBInstanceIdentifier: testDBID, InstanceID: testInstance}
out, err := svc.GetDBBootstrapConfig(t.Context(), in, testAccountID)
require.NoError(t, err)
assert.Equal(t, testDefaultVPCCIDR, out.ClientCIDR)
stored, _ := readRecord(t, svc)
assert.Equal(t, testDefaultVPCCIDR, stored.VpcCIDR, "the backfill must be persisted")
describes := len(network.accts)
_, err = svc.GetDBBootstrapConfig(t.Context(), in, testAccountID)
require.NoError(t, err)
assert.Len(t, network.accts, describes, "a backfilled record must not describe its VPC again")
}
// Without a VPC on the record there is nothing to scope client authentication
// to, and the permissive reading is the one this plan exists to remove.
func TestGetDBBootstrapConfig_UnresolvableClientCIDRFailsClosed(t *testing.T) {
t.Parallel()
svc := newTestService(t)
rec := defaultRecord()
rec.VpcID = ""
rec.VpcCIDR = ""
seedPending(t, svc, rec)
_, err := svc.GetDBBootstrapConfig(t.Context(), bootstrapInput(), testAccountID)
require.Error(t, err)
assert.Contains(t, err.Error(), "records no VPC")
envelope, _ := storedPayload(t, svc, testDBID)
assert.NotNil(t, envelope, "a failed fetch must leave the payload replayable")
@@ -312,7 +389,8 @@ func TestGetDBBootstrapConfig_UnloadableCALeavesPasswordRecoverable(t *testing.T
loadErr := errors.New("read CA key /etc/spinifex/ca.key: permission denied")
broken := true
svc := NewService(nc, testRegion).WithDeps(Deps{
MasterKey: testMasterKey,
MasterKey: testMasterKey,
ServingCertKeyBits: testServingCertKeyBits,
LoadCA: func() (*x509.Certificate, *rsa.PrivateKey, error) {
if broken {
return nil, nil, loadErr
spinifex/handlers/rds/backup.go
+10 -14
View File
@@ -31,12 +31,6 @@ const (
// would pay the whole storage cost above for a fraction of the coverage.
defaultBackupRetentionDays = 7
// The UTC blocks an unnamed window is assigned inside. Deliberately disjoint:
// the two derived windows then cannot overlap, which is the pair AWS refuses
// to accept and the pair that would quiesce an engine mid-maintenance.
defaultBackupWindowBlock = "03:00-11:00" //nolint:gosec // a UTC window block, not a credential
defaultMaintenanceWindowBlock = "11:00-19:00"
// How many automated snapshots one retention pass may delete. A pass that
// under-collects is corrected by the next one two minutes later; a pass that
// walks an unbounded number of snapshots is not.
@@ -60,6 +54,13 @@ const (
automatedBackupRetryShiftCap = 3
)
// The UTC blocks an unnamed window is assigned inside. Deliberately disjoint,
// so derived backup and maintenance windows cannot overlap.
var (
defaultBackupWindowBlock = dailyWindow{start: 3 * time.Hour, end: 11 * time.Hour}
defaultMaintenanceWindowBlock = dailyWindow{start: 11 * time.Hour, end: 19 * time.Hour}
)
// The operator-tunable backup settings: bounds and defaults, never a per-instance
// setting — those live on the DB instance record.
type BackupPolicy struct {
@@ -114,20 +115,15 @@ func (s *Service) maintenanceWindowBlock() dailyWindow {
return s.windowBlock(s.deps.Backup.MaintenanceWindowBlock, defaultMaintenanceWindowBlock, "maintenance")
}
func (s *Service) windowBlock(configured, fallback, kind string) dailyWindow {
func (s *Service) windowBlock(configured string, fallback dailyWindow, kind string) dailyWindow {
if configured != "" {
if block, err := parseDailyWindow("block", configured); err == nil {
return block
}
slog.Warn("rds: the configured "+kind+" window block is malformed; using the built-in block",
"block", configured, "fallback", fallback)
"block", configured, "fallback", fallback.String())
}
block, err := parseDailyWindow("block", fallback)
if err != nil {
// Unreachable: the fallbacks are constants this package's tests parse.
panic("rds: built-in " + kind + " window block is malformed: " + err.Error())
}
return block
return fallback
}
// The window in force for this instance. A record that names one uses it; one
spinifex/handlers/rds/backup_sweep_test.go
+7 -6
View File
@@ -246,12 +246,13 @@ func TestSweep_IsBoundedPerPassAndResumes(t *testing.T) {
t.Parallel()
h := newSnapshotHarness(t, false)
h.svc = h.svc.WithDeps(Deps{
LoadCA: newTestCA(t),
MasterKey: testMasterKey,
Launch: h.launch.deps(),
Network: h.network,
Snapshots: h.snaps,
Backup: BackupPolicy{SweepDeleteLimit: 1},
LoadCA: newTestCA(t),
MasterKey: testMasterKey,
Launch: h.launch.deps(),
Network: h.network,
Snapshots: h.snaps,
Backup: BackupPolicy{SweepDeleteLimit: 1},
ServingCertKeyBits: testServingCertKeyBits,
})
seedInstance(t, h.svc, retainingRecord(0))
for _, age := range []time.Duration{oneDay, 2 * oneDay, 3 * oneDay} {
spinifex/handlers/rds/backup_test.go
+2 -7
View File
@@ -321,13 +321,8 @@ func TestWindowBlock_FallsBackToTheBuiltInBlock(t *testing.T) {
MaintenanceWindowBlock: "19:00",
}})
builtIn, err := parseDailyWindow("block", defaultBackupWindowBlock)
require.NoError(t, err)
assert.Equal(t, builtIn, svc.backupWindowBlock())
builtIn, err = parseDailyWindow("block", defaultMaintenanceWindowBlock)
require.NoError(t, err)
assert.Equal(t, builtIn, svc.maintenanceWindowBlock())
assert.Equal(t, defaultBackupWindowBlock, svc.backupWindowBlock())
assert.Equal(t, defaultMaintenanceWindowBlock, svc.maintenanceWindowBlock())
}
func TestValidateRetentionPeriod_BoundsTheRetention(t *testing.T) {
spinifex/handlers/rds/bootstrap_test.go
+13 -6
View File
@@ -304,7 +304,10 @@ func TestAcknowledgeDBBootstrap_NeverDecrypts(t *testing.T) {
svc := newTestService(t)
payloadID := seedPending(t, svc, defaultRecord())
rotated := svc.WithDeps(Deps{LoadCA: newTestCA(t), MasterKey: []byte(testOtherMasterKeyS)})
rotated := svc.WithDeps(Deps{
LoadCA: newTestCA(t), MasterKey: []byte(testOtherMasterKeyS),
ServingCertKeyBits: testServingCertKeyBits,
})
_, err := rotated.AcknowledgeDBBootstrap(t.Context(), &AcknowledgeDBBootstrapInput{
DBInstanceIdentifier: testDBID,
InstanceID: testInstance,
@@ -388,7 +391,10 @@ func TestGetDBBootstrapConfig_UnreadablePayloadAttachesAndReportsWhy(t *testing.
svc := newTestService(t)
seedPending(t, svc, defaultRecord())
rotated := svc.WithDeps(Deps{LoadCA: newTestCA(t), MasterKey: []byte(testOtherMasterKeyS)})
rotated := svc.WithDeps(Deps{
LoadCA: newTestCA(t), MasterKey: []byte(testOtherMasterKeyS),
ServingCertKeyBits: testServingCertKeyBits,
})
out, err := rotated.GetDBBootstrapConfig(t.Context(), bootstrapInput(), testAccountID)
require.NoError(t, err)
assert.Equal(t, BootstrapModeAttach, out.Mode)
@@ -433,10 +439,11 @@ func TestCreateDBInstance_WithoutAMasterKeyStagesNothing(t *testing.T) {
t.Parallel()
h := newCreateHarness(t, "")
h.svc = h.svc.WithDeps(Deps{
LoadCA: newTestCA(t),
Launch: h.launch.deps(),
Network: h.network,
IAM: testIAMProvider(h.iam),
LoadCA: newTestCA(t),
Launch: h.launch.deps(),
Network: h.network,
IAM: testIAMProvider(h.iam),
ServingCertKeyBits: testServingCertKeyBits,
})
_, err := h.svc.CreateDBInstance(t.Context(), validCreateInput(), testAccountID)
spinifex/handlers/rds/create.go
+2
View File
@@ -166,6 +166,7 @@ func newDBInstanceRecord(accountID string, req *validatedCreate, placement *endp
Port: req.Port,
SubnetID: placement.SubnetID,
VpcID: placement.VpcID,
VpcCIDR: placement.VpcCIDR,
VpcSecurityGroupIDs: placement.SecurityGroupIDs,
DBSubnetGroupName: req.DBSubnetGroupName,
DBParameterGroupName: req.DBParameterGroupName,
@@ -214,6 +215,7 @@ func (s *Service) recordLaunch(ctx context.Context, kv jetstream.KeyValue, key,
rec.InstanceID = launched.InstanceID
rec.VMGeneration = firstVMGeneration
rec.SystemENIID = launched.SystemENIID
rec.SystemSGID = launched.SystemSGID
rec.ENIID = launched.CustomerENIID
rec.ENIPrivateIP = launched.CustomerENIIP
rec.DataVolumeID = launched.DataVolumeID
spinifex/handlers/rds/create_test.go
+28 -12
View File
@@ -24,10 +24,11 @@ import (
)
const (
testDefaultVPC = "vpc-default01"
testDefaultSG = "sg-default01"
testBaseDomain = "spx3.net"
testDBInstanceID = "orders-db"
testDefaultVPC = "vpc-default01"
testDefaultVPCCIDR = "10.0.0.0/16"
testDefaultSG = "sg-default01"
testBaseDomain = "spx3.net"
testDBInstanceID = "orders-db"
// The one zone this platform exposes, which every subnet reports.
testZone = "spinifexz1"
)
@@ -55,7 +56,11 @@ var _ networkResolver = (*fakeNetwork)(nil)
func newFakeNetwork() *fakeNetwork {
return &fakeNetwork{
vpcs: []*ec2.Vpc{{VpcId: aws.String(testDefaultVPC), IsDefault: aws.Bool(true)}},
vpcs: []*ec2.Vpc{{
VpcId: aws.String(testDefaultVPC),
IsDefault: aws.Bool(true),
CidrBlock: aws.String(testDefaultVPCCIDR),
}},
// Deliberately out of order: the resolver sorts, so the placement is the
// same on every repeat regardless of how the describe happened to order.
subnets: []*ec2.Subnet{
@@ -85,7 +90,17 @@ func (f *fakeNetwork) DescribeVpcs(_ context.Context, input *ec2.DescribeVpcsInp
if f.vpcErr != nil {
return nil, f.vpcErr
}
return &ec2.DescribeVpcsOutput{Vpcs: f.vpcs}, nil
if input == nil || len(input.VpcIds) == 0 {
return &ec2.DescribeVpcsOutput{Vpcs: f.vpcs}, nil
}
wanted := aws.StringValueSlice(input.VpcIds)
var matched []*ec2.Vpc
for _, vpc := range f.vpcs {
if slices.Contains(wanted, aws.StringValue(vpc.VpcId)) {
matched = append(matched, vpc)
}
}
return &ec2.DescribeVpcsOutput{Vpcs: matched}, nil
}
// Honours SubnetIds the way EC2 does — a named subnet that is not in this
@@ -150,12 +165,13 @@ func newCreateHarness(t *testing.T, baseDomain string) *createHarness {
h.stubDNSWriter(t)
h.svc = NewService(nc, testRegion).WithDeps(Deps{
LoadCA: newTestCA(t),
MasterKey: testMasterKey,
Launch: h.launch.deps(),
Network: h.network,
IAM: testIAMProvider(h.iam),
BaseDomain: baseDomain,
LoadCA: newTestCA(t),
MasterKey: testMasterKey,
Launch: h.launch.deps(),
Network: h.network,
IAM: testIAMProvider(h.iam),
BaseDomain: baseDomain,
ServingCertKeyBits: testServingCertKeyBits,
})
return h
}
spinifex/handlers/rds/engine.go
+20 -15
View File
@@ -63,40 +63,45 @@ var engines = map[string]Engine{
// The same registry keyed by parameter-group family, for the callers that hold
// only a family string and have no instance to derive an engine from. Family
// and engine are 1:1 by construction, so one registry serves both.
var enginesByFamily = indexEnginesByFamily()
var enginesByFamily, engineRegistryValidationErr = indexEnginesByFamily(engines)
func indexEnginesByFamily() map[string]Engine {
out := make(map[string]Engine, len(engines))
for _, engine := range engines {
// An engine registered without its own name rule would panic on the first
// create that names a database, rather than here where every build sees it.
// ValidateEngineRegistry reports invalid built-in engine metadata before RDS starts.
func ValidateEngineRegistry() error {
if err := validateParameterCatalogs(); err != nil {
return err
}
return engineRegistryValidationErr
}
func indexEnginesByFamily(registry map[string]Engine) (map[string]Engine, error) {
out := make(map[string]Engine, len(registry))
for _, engine := range registry {
if engine.validateDBName == nil {
panic("rds: engine " + engine.Name + " registers no DBName rule")
return nil, errors.New("rds: engine " + engine.Name + " registers no DBName rule")
}
// An engine without these would tell a customer nothing about what a
// crash-consistent snapshot or an unclean stop of it recovers, on the two
// events that say so.
// Without these, snapshot and unclean-stop events cannot explain what the
// engine will recover.
if engine.crashRecoveryNote == "" {
panic("rds: engine " + engine.Name + " registers no crash-recovery note")
return nil, errors.New("rds: engine " + engine.Name + " registers no crash-recovery note")
}
if engine.uncleanStopNote == "" {
panic("rds: engine " + engine.Name + " registers no unclean-stop note")
return nil, errors.New("rds: engine " + engine.Name + " registers no unclean-stop note")
}
// A name no catalog entry answers to would leave the guest deriving
// enforcement from a key nothing ever writes, which reads as not enforcing.
if name := engine.tlsEnforcementParameter; name != "" {
spec, ok := engine.catalog[name]
if !ok || spec.DataType != ParamTypeBoolean {
panic("rds: engine " + engine.Name + " names " + name + ", which is not a boolean parameter it exposes")
return nil, errors.New("rds: engine " + engine.Name + " names " + name + ", which is not a boolean parameter it exposes")
}
}
family := engine.ParameterGroupFamily()
if _, exists := out[family]; exists {
panic("rds: parameter group family " + family + " is claimed by two engines")
return nil, errors.New("rds: parameter group family " + family + " is claimed by two engines")
}
out[family] = engine
}
return out
return out, nil
}
// The pinned version an AMI lookup resolves against, which is the major alone:
spinifex/handlers/rds/engine_mariadb.go
+1 -1
View File
@@ -49,7 +49,7 @@ var engineMariaDB = Engine{
// Platform-owned settings are absent, except the ones AWS exposes as modifiable
// and this platform pins: those are present and unmodifiable, so a refusal reads
// as policy rather than as a missing feature.
var mariadbParameterCatalog = buildParameterCatalog(
var mariadbParameterCatalog, mariadbParameterCatalogErr = buildParameterCatalog(
// Connections and threads. max_connections is what a size-derived default
// matters most for: RDS's own formula is {DBInstanceClassMemory/12582880}.
ParameterSpec{
spinifex/handlers/rds/engine_postgres.go
+1 -1
View File
@@ -43,7 +43,7 @@ var enginePostgres = Engine{
// Platform-owned settings are absent, except the ones AWS exposes as modifiable
// and this platform pins: those are present and unmodifiable, so a refusal reads
// as policy rather than as a missing feature.
var postgresParameterCatalog = buildParameterCatalog(
var postgresParameterCatalog, postgresParameterCatalogErr = buildParameterCatalog(
// Connections. max_connections is what a size-derived default matters most
// for: RDS's own formula is LEAST({DBInstanceClassMemory/9531392}, 5000).
ParameterSpec{
spinifex/handlers/rds/enginefamily_test.go
+68 -6
View File
@@ -34,18 +34,18 @@ var engineUnderTest = Engine{
reservedUsernamePrefixes: []string{"testdb_"},
maxUsernameLen: 80,
validateDBName: dbNameRule(64),
catalog: buildParameterCatalog(
ParameterSpec{
catalog: map[string]ParameterSpec{
"buffer_size": {
Name: "buffer_size", DataType: ParamTypeInteger, ApplyType: ApplyTypeStatic,
IsModifiable: true, Min: 1, Max: 1024, Default: "64", Unit: "MB",
Description: "Buffer the server allocates at startup, in MB.",
},
ParameterSpec{
"connection_limit": {
Name: "connection_limit", DataType: ParamTypeInteger, ApplyType: ApplyTypeDynamic,
IsModifiable: true, Min: 1, Max: 1000, Default: "100",
Description: "Maximum concurrent connections to the server.",
},
),
},
validateCombinations: func([]Parameter) error { return nil },
crashRecoveryNote: "It will recover when it is restored.",
uncleanStopNote: "It will recover on the next start.",
@@ -57,13 +57,75 @@ func registerTestEngine(t *testing.T) {
t.Helper()
require.NotContains(t, engines, engineUnderTest.Name)
engines[engineUnderTest.Name] = engineUnderTest
enginesByFamily = indexEnginesByFamily()
indexed, err := indexEnginesByFamily(engines)
require.NoError(t, err)
enginesByFamily = indexed
t.Cleanup(func() {
delete(engines, engineUnderTest.Name)
enginesByFamily = indexEnginesByFamily()
indexed, err := indexEnginesByFamily(engines)
require.NoError(t, err)
enginesByFamily = indexed
})
}
func TestValidateEngineRegistry(t *testing.T) {
require.NoError(t, ValidateEngineRegistry())
}
func TestIndexEnginesByFamily_RejectsInvalidMetadata(t *testing.T) {
noDBNameRule := engineUnderTest
noDBNameRule.validateDBName = nil
noCrashRecoveryNote := engineUnderTest
noCrashRecoveryNote.crashRecoveryNote = ""
noUncleanStopNote := engineUnderTest
noUncleanStopNote.uncleanStopNote = ""
invalidTLSParameter := engineUnderTest
invalidTLSParameter.tlsEnforcementParameter = "not-a-boolean-parameter"
tests := []struct {
name string
registry map[string]Engine
wantErr string
}{
{
name: "missing DB name rule",
registry: map[string]Engine{"test": noDBNameRule},
wantErr: "registers no DBName rule",
},
{
name: "missing crash recovery note",
registry: map[string]Engine{"test": noCrashRecoveryNote},
wantErr: "registers no crash-recovery note",
},
{
name: "missing unclean stop note",
registry: map[string]Engine{"test": noUncleanStopNote},
wantErr: "registers no unclean-stop note",
},
{
name: "invalid TLS enforcement parameter",
registry: map[string]Engine{"test": invalidTLSParameter},
wantErr: "is not a boolean parameter it exposes",
},
{
name: "duplicate parameter group family",
registry: map[string]Engine{
"first": engineUnderTest,
"second": engineUnderTest,
},
wantErr: "is claimed by two engines",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
indexed, err := indexEnginesByFamily(tt.registry)
require.ErrorContains(t, err, tt.wantErr)
assert.Nil(t, indexed)
})
}
}
func testEngineGroupInput(name string) *rds.CreateDBParameterGroupInput {
return &rds.CreateDBParameterGroupInput{
DBParameterGroupName: aws.String(name),

Some files were not shown because too many files have changed in this diff Show More