gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/iso-installer

Browse Source
This commit is contained in:
Julian Sommer
2026-04-15 15:50:37 +10:00
committed by GitHub
parent 054485e930 c8429f98ae
commit 32effaa16e
19 changed files with 571 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e.yml
+27 -27
View File
@@ -61,16 +61,16 @@ jobs:
run: |
export TF_VAR_git_branch="${GIT_BRANCH}"
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="192.168.1.230-192.168.1.240" \
./bootstrap-install.sh env3 \
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
@@ -89,9 +89,9 @@ jobs:
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
e2e_pseudo_multi:
name: Pseudo Multi-Node E2E
@@ -126,17 +126,17 @@ jobs:
run: |
export TF_VAR_git_branch="${GIT_BRANCH}"
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
- name: Bootstrap Cluster
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_PREBUILT=true \
SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
./bootstrap.sh env4 >> "$TOFU_LOG" 2>&1
./bootstrap.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
- name: Run E2E Tests
working-directory: mulga/scripts/tofu-cluster
@@ -146,15 +146,15 @@ jobs:
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
'cd ~/Development/mulga/spinifex && SPX_EXTERNAL_POOL="192.168.1.220-192.168.1.229" ./tests/e2e/run-pseudo-multinode-e2e.sh'
"cd ~/Development/mulga/spinifex && SPX_EXTERNAL_POOL=\"${SPINIFEX_CI_POOL}\" ./tests/e2e/run-pseudo-multinode-e2e.sh"
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
tofu workspace select -or-create env4 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env4.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh env4 >> "$TOFU_LOG" 2>&1 || true
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
e2e_real_multi:
name: Real Multi-Node E2E
@@ -193,16 +193,16 @@ jobs:
run: |
export TF_VAR_git_branch="${GIT_BRANCH}"
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="192.168.1.240-192.168.1.250" \
./bootstrap-install.sh env2 \
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
@@ -231,9 +231,9 @@ jobs:
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
tofu workspace select -or-create env2 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env2.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh env2 >> "$TOFU_LOG" 2>&1 || true
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
e2e_status:
name: Report E2E Status
.github/workflows/release.yml
+9 -9
View File
@@ -128,7 +128,7 @@ jobs:
- uses: actions/create-github-app-token@v3
id: app-token
with:
app-id: ${{ secrets.APP_ID }}
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
@@ -149,17 +149,17 @@ jobs:
working-directory: mulga/scripts/tofu-cluster
run: |
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SETUP_SH=$(find "${GITHUB_WORKSPACE}/release-assets" -name "setup.sh" | head -1)
TARBALL=$(find "${GITHUB_WORKSPACE}/release-assets" -name "*-linux-amd64.tar.gz" | head -1)
./bootstrap-install.sh env3 "$SETUP_SH" "$TARBALL"
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" "$SETUP_SH" "$TARBALL"
- name: Promote Release
if: success()
@@ -189,6 +189,6 @@ jobs:
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
tofu workspace select -or-create env3 >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/env3.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh env3 >> "$TOFU_LOG" 2>&1 || true
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
cmd/spinifex/cmd/templates/awsgw.toml
+23 -1
View File
@@ -1,6 +1,28 @@
# AWS Gateway configuration file in TOML format.
version = "1.0"
version = "2"
region = "{{.Region}}"
# Enable verbose debugging
debug = false
# API request throttling (per-account, per-action token bucket)
[ratelimit]
enabled = false
rate = 20 # Default sustained requests per second per account per action
burst = 100 # Default maximum burst capacity
[ratelimit.action.RunInstances]
rate = 2
burst = 40
[ratelimit.action.TerminateInstances]
rate = 2
burst = 40
[ratelimit.action.StartInstances]
rate = 2
burst = 40
[ratelimit.action.StopInstances]
rate = 2
burst = 40
cmd/spinifex/cmd/templates/predastore-multinode.toml
+15 -1
View File
@@ -1,6 +1,6 @@
# Predastore S3-compatible storage configuration file in TOML format.
# Generated for multi-node cluster
version = "1.0"
version = "2"
region = "{{.Region}}"
host = "0.0.0.0"
@@ -47,6 +47,20 @@ policy = [
{ bucket = "predastore", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
]
# API request throttling (per-account, per-action token bucket)
[ratelimit]
enabled = false
rate = 100 # Default sustained requests per second per account per action
burst = 500 # Default maximum burst capacity
[ratelimit.action."s3:PutObject"]
rate = 50
burst = 200
[ratelimit.action."s3:DeleteObject"]
rate = 50
burst = 200
# IAM authentication via NATS KV (enables multi-account S3 access)
[iam]
nats_url = "nats://{{.BindIP}}:4222"
cmd/spinifex/cmd/templates/predastore.toml
+15 -1
View File
@@ -1,6 +1,6 @@
# Predastore S3-compatible storage configuration file in TOML format.
# MULTI NODE used for testing
version = "1.0"
version = "2"
region = "{{.Region}}"
# Optionally configure port and host, or specify via daemon CLI options
@@ -81,6 +81,20 @@ policy = [
{ bucket = "predastore", actions = ["s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListAllMyBuckets"] },
]
# API request throttling (per-account, per-action token bucket)
[ratelimit]
enabled = false
rate = 100 # Default sustained requests per second per account per action
burst = 500 # Default maximum burst capacity
[ratelimit.action."s3:PutObject"]
rate = 50
burst = 200
[ratelimit.action."s3:DeleteObject"]
rate = 50
burst = 200
# IAM authentication via NATS KV (enables multi-account S3 access)
[iam]
nats_url = "nats://{{if eq .BindIP "0.0.0.0"}}localhost{{else}}{{.BindIP}}{{end}}:4222"
docs/terraform/nginx-alb/README.md
+1 -1
View File
@@ -120,7 +120,7 @@ export TF_VAR_instance_type="t3.small"
Or alternatively, using the AWS CLI tool query your instance for available types (e.g Intel, AMD, ARM) that support 2 vCPUs and 1 GB RAM.
```bash
TF_VAR_instance_type=$(aws ec2 describe-instance-types \
export TF_VAR_instance_type=$(aws ec2 describe-instance-types \
--query "sort_by(InstanceTypes[?VCpuInfo.DefaultVCpus==\`2\` && MemoryInfo.SizeInMiB>=\`1024\`], &MemoryInfo.SizeInMiB)[0].InstanceType" \
--output text)
```
go.mod
+1 -1
View File
@@ -11,7 +11,7 @@ require (
github.com/google/uuid v1.6.0
github.com/kdomanski/iso9660 v0.4.0
github.com/klauspost/cpuid/v2 v2.3.0
github.com/mulgadc/predastore v1.1.5
github.com/mulgadc/predastore v1.1.6
github.com/mulgadc/viperblock v1.1.6
github.com/nats-io/nats-server/v2 v2.12.6
github.com/nats-io/nats.go v1.50.0
go.sum
+2 -6
View File
@@ -291,12 +291,8 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/mulgadc/predastore v1.1.4 h1:czeGQWWqmmZuGnUuvyrORavySN/SGPTwjUxBd/9swuU=
github.com/mulgadc/predastore v1.1.4/go.mod h1:YP4TaathWczm9weoI8YomZ1/IgnFB9WJowkHPidh6rw=
github.com/mulgadc/predastore v1.1.5 h1:BjTEP2xbzIh0FswuCvnVcq9uy6l235lcIOwkGG/Z+f4=
github.com/mulgadc/predastore v1.1.5/go.mod h1:la4jzK8lSLfUyUR5wNdVrkXXq8JQMGiKFvp/eXT0wjM=
github.com/mulgadc/viperblock v1.1.5 h1:ojmoJQrY7VkDGzdQj07w2F7ICRJX9RlGAKyn87Wa55o=
github.com/mulgadc/viperblock v1.1.5/go.mod h1:utoIQLwCmPujOBaT0mL5NhW3m2fwjQPQiQRJH6+Wtk8=
github.com/mulgadc/predastore v1.1.6 h1:67rF2wa5BXges5cVJlFqYurYXnlkYleF3XJ1RyB+ZXk=
github.com/mulgadc/predastore v1.1.6/go.mod h1:WO6lNV8oxsYZGN2BzHROyX42+raOLflvX7iiirvpteA=
github.com/mulgadc/viperblock v1.1.6 h1:SzvpZIFWZVpWWdUSPD9k7fgmVAOsyXj9Ra686MyFsRc=
github.com/mulgadc/viperblock v1.1.6/go.mod h1:KoJH5WFaZdIEYQ/uxugFKhQRLcMa9gmNOtyZjRHf/wI=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
scripts/setup.sh
+8
View File
@@ -321,6 +321,10 @@ create_directories() {
$SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore
$SUDO chmod 0750 /etc/spinifex/predastore
$SUDO mkdir -p /etc/spinifex/awsgw
$SUDO chown "spinifex-gw:$SPINIFEX_GROUP" /etc/spinifex/awsgw
$SUDO chmod 0750 /etc/spinifex/awsgw
# Per-service data directories
$SUDO mkdir -p /var/lib/spinifex/nats
$SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /var/lib/spinifex/nats
@@ -399,6 +403,10 @@ fix_file_ownership() {
$SUDO chown -R "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore \
|| fatal "Failed to set ownership on /etc/spinifex/predastore"
fi
if [ -d /etc/spinifex/awsgw ]; then
$SUDO chown -R "spinifex-gw:$SPINIFEX_GROUP" /etc/spinifex/awsgw \
|| fatal "Failed to set ownership on /etc/spinifex/awsgw"
fi
# Shared config files — root:spinifex with per-file modes
for f in spinifex.toml master.key server.key; do
spinifex/awserrors/awserrors.go
+2
View File
@@ -379,6 +379,7 @@ var (
ErrorSubnetLimitExceeded = "SubnetLimitExceeded"
ErrorTagLimitExceeded = "TagLimitExceeded"
ErrorTagPolicyViolation = "TagPolicyViolation"
ErrorThrottling = "Throttling"
ErrorTargetCapacityLimitExceededException = "TargetCapacityLimitExceededException"
ErrorTrafficMirrorFilterInUse = "TrafficMirrorFilterInUse"
ErrorTrafficMirrorFilterLimitExceeded = "TrafficMirrorFilterLimitExceeded"
@@ -838,6 +839,7 @@ var ErrorLookup = map[string]ErrorMessage{
ErrorSubnetLimitExceeded: {HTTPCode: 400, Message: "You've reached the limit on the number of subnets that you can create for the specified VPC. For more information, see Amazon VPC quotas."},
ErrorTagLimitExceeded: {HTTPCode: 400, Message: "You've reached the limit on the number of tags that you can assign to the specified resource. For more information, see Tag restrictions."},
ErrorTagPolicyViolation: {HTTPCode: 400, Message: "You attempted to create or update a resource with tags that are not compliant with the tag policy requirements for this account. For more information, see Grant permission to tag resources during creation."},
ErrorThrottling: {HTTPCode: 400, Message: "Rate exceeded."},
ErrorTargetCapacityLimitExceededException: {HTTPCode: 400, Message: "The value for targetCapacity exceeds your limit on the amount of Spot placement target capacity you can explore. Reduce the targetCapacity value, and try again. For more information, see Spot placement score."},
ErrorTrafficMirrorFilterInUse: {HTTPCode: 400, Message: "The Traffic Mirror filter cannot be deleted because a Traffic Mirror session is currently using it."},
ErrorTrafficMirrorFilterLimitExceeded: {HTTPCode: 400, Message: "The maximum number of Traffic Mirror filters has been exceeded."},
spinifex/awserrors/awserrors_test.go
+1
View File
@@ -382,6 +382,7 @@ func TestErrorLookup(t *testing.T) {
{code: "TagLimitExceeded", http: 400, message: "You've reached the limit on the number of tags that you can assign to the specified resource. For more information, see Tag restrictions."},
{code: "TagPolicyViolation", http: 400, message: "You attempted to create or update a resource with tags that are not compliant with the tag policy requirements for this account. For more information, see Grant permission to tag resources during creation."},
{code: "TargetCapacityLimitExceededException", http: 400, message: "The value for targetCapacity exceeds your limit on the amount of Spot placement target capacity you can explore. Reduce the targetCapacity value, and try again. For more information, see Spot placement score."},
{code: "Throttling", http: 400, message: "Rate exceeded."},
{code: "TrafficMirrorFilterInUse", http: 400, message: "The Traffic Mirror filter cannot be deleted because a Traffic Mirror session is currently using it."},
{code: "TrafficMirrorFilterLimitExceeded", http: 400, message: "The maximum number of Traffic Mirror filters has been exceeded."},
{code: "TrafficMirrorFilterRuleAlreadyExists", http: 409, message: "The Traffic Mirror filter rule already exists."},
spinifex/gateway/auth.go
+8
View File
@@ -188,6 +188,14 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
ctx = context.WithValue(ctx, ctxRegion, region)
ctx = context.WithValue(ctx, ctxAccessKey, accessKey)
// Extract the Action parameter from the already-buffered body
// so downstream middleware (throttle) can read it from context
// instead of re-reading/re-parsing the body.
args := ParseAWSQueryArgs(string(body))
if action := args["Action"]; action != "" {
ctx = context.WithValue(ctx, ctxAction, action)
}
slog.Debug("SigV4 authentication successful", "accessKey", accessKey, "identity", ak.UserName)
gw.RateLimiter.RecordSuccess(clientIP)
next.ServeHTTP(w, r.WithContext(ctx))
spinifex/gateway/gateway.go
+59 -3
View File
@@ -14,6 +14,7 @@ import (
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/mulgadc/predastore/ratelimit"
"github.com/mulgadc/spinifex/spinifex/awsec2query"
"github.com/mulgadc/spinifex/spinifex/awserrors"
"github.com/mulgadc/spinifex/spinifex/gateway/policy"
@@ -32,6 +33,7 @@ const (
ctxService contextKey = "sigv4.service"
ctxRegion contextKey = "sigv4.region"
ctxAccessKey contextKey = "sigv4.accessKey"
ctxAction contextKey = "sigv4.action"
)
type GatewayConfig struct {
@@ -43,9 +45,10 @@ type GatewayConfig struct {
Region string // Region this gateway is running in
AZ string // Availability zone this gateway is running in
IAMService handlers_iam.IAMService
RateLimiter *AuthRateLimiter // Per-IP auth failure rate limiter
Version string // Build-time version string (set from cmd.Version)
Commit string // Build-time commit hash (set from cmd.Commit)
RateLimiter *AuthRateLimiter // Per-IP auth failure rate limiter
Throttler *ratelimit.Throttler // Per-account+action API request throttler
Version string // Build-time version string (set from cmd.Version)
Commit string // Build-time commit hash (set from cmd.Commit)
}
var supportedServices = map[string]bool{
@@ -106,12 +109,65 @@ func (gw *GatewayConfig) SetupRoutes() http.Handler {
// AWS SigV4 authentication middleware
r.Use(gw.SigV4AuthMiddleware())
// API request throttling (post-auth, per-account+action token bucket)
if gw.Throttler != nil {
r.Use(gw.Throttler.Middleware(
gw.throttleKeyFuncs(),
gw.writeThrottleError,
))
}
// Catch-all routes
r.HandleFunc("/*", gw.Request)
return r
}
// throttleKeyFuncs returns the KeyFunc slice for the API throttle middleware.
// The first func extracts the account-id (set by SigV4 auth), the second
// extracts the action from context (set by SigV4 auth from the request body).
func (gw *GatewayConfig) throttleKeyFuncs() []ratelimit.KeyFunc {
return []ratelimit.KeyFunc{
func(r *http.Request) (string, error) {
acct, ok := r.Context().Value(ctxAccountID).(string)
if !ok || acct == "" {
return "", fmt.Errorf("account-id missing from request context")
}
return acct, nil
},
func(r *http.Request) (string, error) {
action, _ := r.Context().Value(ctxAction).(string)
if action != "" {
return action, nil
}
return "unknown", nil
},
}
}
// writeThrottleError writes the service-appropriate throttle rejection response.
func (gw *GatewayConfig) writeThrottleError(w http.ResponseWriter, r *http.Request) {
requestID := uuid.NewString()
svc, _ := r.Context().Value(ctxService).(string)
var xmlErr []byte
var statusCode int
switch svc {
case "iam":
xmlErr = GenerateIAMErrorResponse(awserrors.ErrorThrottling, "Rate exceeded", requestID)
statusCode = 400
default: // ec2, elasticloadbalancing, account, spinifex
xmlErr = GenerateEC2ErrorResponse(awserrors.ErrorRequestLimitExceeded, "Request limit exceeded.", requestID)
statusCode = 503
}
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(statusCode)
if _, err := w.Write(xmlErr); err != nil {
slog.Error("Failed to write throttle error response", "err", err)
}
}
// Note, custom endpoints can be configured via ENV vars to the AWS SDK/CLI tool, with individual endpoints depending the service
// AWS_ENDPOINT_URL_EC2=https://localhost:9999/ aws --no-verify-ssl ec2 describe-instances
// aws --endpoint-url https://localhost:9999/ --no-verify-ssl eks list-clusters
spinifex/gateway/gateway_test.go
+188
View File
@@ -12,6 +12,7 @@ import (
"strings"
"testing"
"github.com/mulgadc/predastore/ratelimit"
"github.com/mulgadc/spinifex/spinifex/awserrors"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/mulgadc/spinifex/spinifex/testutil"
@@ -1032,3 +1033,190 @@ func TestParseArgsToStruct(t *testing.T) {
assert.Equal(t, "InvalidParameter", err.Error())
})
}
// --- Throttle middleware integration tests ---
func TestThrottleKeyFuncs_ExtractsAccountAndAction(t *testing.T) {
gw := &GatewayConfig{DisableLogging: true}
keyFuncs := gw.throttleKeyFuncs()
require.Len(t, keyFuncs, 2)
req := httptest.NewRequest(http.MethodPost, "/", nil)
ctx := context.WithValue(req.Context(), ctxAccountID, "123456789012")
ctx = context.WithValue(ctx, ctxAction, "DescribeInstances")
req = req.WithContext(ctx)
acct, err := keyFuncs[0](req)
require.NoError(t, err)
assert.Equal(t, "123456789012", acct)
action, err := keyFuncs[1](req)
require.NoError(t, err)
assert.Equal(t, "DescribeInstances", action)
}
func TestThrottleKeyFuncs_UnknownAction(t *testing.T) {
gw := &GatewayConfig{DisableLogging: true}
keyFuncs := gw.throttleKeyFuncs()
// No ctxAction in context — should return "unknown".
req := httptest.NewRequest(http.MethodPost, "/", nil)
ctx := context.WithValue(req.Context(), ctxAccountID, "123")
req = req.WithContext(ctx)
action, err := keyFuncs[1](req)
require.NoError(t, err)
assert.Equal(t, "unknown", action)
}
func TestThrottleKeyFuncs_MissingAccountID(t *testing.T) {
gw := &GatewayConfig{DisableLogging: true}
keyFuncs := gw.throttleKeyFuncs()
// No ctxAccountID in context — should return an error.
req := httptest.NewRequest(http.MethodPost, "/", nil)
_, err := keyFuncs[0](req)
require.Error(t, err)
assert.Contains(t, err.Error(), "account-id missing")
}
func TestWriteThrottleError_EC2(t *testing.T) {
gw := &GatewayConfig{DisableLogging: true}
req := httptest.NewRequest(http.MethodPost, "/", nil)
ctx := context.WithValue(req.Context(), ctxService, "ec2")
req = req.WithContext(ctx)
w := httptest.NewRecorder()
gw.writeThrottleError(w, req)
resp := w.Result()
body, _ := io.ReadAll(resp.Body)
assert.Equal(t, 503, resp.StatusCode)
assert.Equal(t, "application/xml", resp.Header.Get("Content-Type"))
assert.Contains(t, string(body), "<Code>RequestLimitExceeded</Code>")
assert.Contains(t, string(body), "<Response>")
}
func TestWriteThrottleError_IAM(t *testing.T) {
gw := &GatewayConfig{DisableLogging: true}
req := httptest.NewRequest(http.MethodPost, "/", nil)
ctx := context.WithValue(req.Context(), ctxService, "iam")
req = req.WithContext(ctx)
w := httptest.NewRecorder()
gw.writeThrottleError(w, req)
resp := w.Result()
body, _ := io.ReadAll(resp.Body)
assert.Equal(t, 400, resp.StatusCode)
assert.Contains(t, string(body), "<Code>Throttling</Code>")
assert.Contains(t, string(body), "<ErrorResponse>")
}
func TestThrottleMiddleware_Integration(t *testing.T) {
cfg := ratelimit.Config{
Enabled: true,
Rate: 1,
Burst: 2,
}
throttler := ratelimit.New(cfg)
defer throttler.Stop()
gw := &GatewayConfig{
DisableLogging: true,
Throttler: throttler,
}
// Build a minimal handler that the throttle middleware wraps.
inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
})
mw := throttler.Middleware(gw.throttleKeyFuncs(), gw.writeThrottleError)
handler := mw(inner)
makeReq := func() *http.Response {
req := httptest.NewRequest(http.MethodPost, "/", nil)
ctx := context.WithValue(req.Context(), ctxAccountID, "acct1")
ctx = context.WithValue(ctx, ctxService, "ec2")
ctx = context.WithValue(ctx, ctxAction, "DescribeInstances")
req = req.WithContext(ctx)
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
return w.Result()
}
// First two requests should succeed (burst=2).
resp1 := makeReq()
assert.Equal(t, 200, resp1.StatusCode)
resp2 := makeReq()
assert.Equal(t, 200, resp2.StatusCode)
// Third request should be throttled.
resp3 := makeReq()
assert.Equal(t, 503, resp3.StatusCode)
assert.NotEmpty(t, resp3.Header.Get("Retry-After"))
body3, _ := io.ReadAll(resp3.Body)
assert.Contains(t, string(body3), "RequestLimitExceeded")
}
func TestThrottleMiddleware_DisabledConfig(t *testing.T) {
// When Throttler is nil, SetupRoutes skips middleware entirely.
gw := &GatewayConfig{DisableLogging: true, Throttler: nil}
handler := gw.SetupRoutes()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=DescribeInstances"))
ctx := context.WithValue(req.Context(), ctxAccountID, "acct1")
ctx = context.WithValue(ctx, ctxService, "ec2")
req = req.WithContext(ctx)
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
// Without auth it'll fail on SigV4, not on throttling — that's expected.
// The key assertion: no panic from nil Throttler.
resp := w.Result()
assert.NotEqual(t, 503, resp.StatusCode)
}
func TestThrottleMiddleware_PerActionIsolation(t *testing.T) {
cfg := ratelimit.Config{
Enabled: true,
Rate: 1,
Burst: 1,
}
throttler := ratelimit.New(cfg)
defer throttler.Stop()
gw := &GatewayConfig{DisableLogging: true}
inner := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
})
mw := throttler.Middleware(gw.throttleKeyFuncs(), gw.writeThrottleError)
handler := mw(inner)
makeReq := func(action string) *http.Response {
req := httptest.NewRequest(http.MethodPost, "/", nil)
ctx := context.WithValue(req.Context(), ctxAccountID, "acct1")
ctx = context.WithValue(ctx, ctxService, "ec2")
ctx = context.WithValue(ctx, ctxAction, action)
req = req.WithContext(ctx)
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
return w.Result()
}
// Exhaust DescribeInstances.
resp := makeReq("DescribeInstances")
assert.Equal(t, 200, resp.StatusCode)
resp = makeReq("DescribeInstances")
assert.Equal(t, 503, resp.StatusCode)
// RunInstances should be independent.
resp = makeReq("RunInstances")
assert.Equal(t, 200, resp.StatusCode)
}
spinifex/migrate/002_ratelimit.go
+100
View File
@@ -0,0 +1,100 @@
package migrate
import (
"os"
"path/filepath"
"strings"
)
const awsgwRatelimitSection = `
# API request throttling (per-account, per-action token bucket)
[ratelimit]
enabled = false
rate = 20 # Default sustained requests per second per account per action
burst = 100 # Default maximum burst capacity
[ratelimit.action.RunInstances]
rate = 2
burst = 40
[ratelimit.action.TerminateInstances]
rate = 2
burst = 40
[ratelimit.action.StartInstances]
rate = 2
burst = 40
[ratelimit.action.StopInstances]
rate = 2
burst = 40
`
const predastoreRatelimitSection = `
# API request throttling (per-account, per-action token bucket)
[ratelimit]
enabled = false
rate = 100 # Default sustained requests per second per account per action
burst = 500 # Default maximum burst capacity
[ratelimit.action."s3:PutObject"]
rate = 50
burst = 200
[ratelimit.action."s3:DeleteObject"]
rate = 50
burst = 200
`
func init() {
// awsgw.toml: version 1 → 2 — append [ratelimit] section.
DefaultRegistry.RegisterConfigTarget("awsgw.toml", "awsgw/awsgw.toml", &TOMLVersionReader{})
DefaultRegistry.RegisterConfig("awsgw.toml", ConfigMigration{
FromVersion: 1,
ToVersion: 2,
Description: "Add [ratelimit] section for API request throttling",
Run: func(ctx ConfigContext) error {
return appendRatelimitSection(
filepath.Join(ctx.ConfigDir, "awsgw", "awsgw.toml"),
awsgwRatelimitSection,
)
},
})
// predastore.toml: version 1 → 2 — append [ratelimit] section.
DefaultRegistry.RegisterConfigTarget("predastore.toml", "predastore/predastore.toml", &TOMLVersionReader{})
DefaultRegistry.RegisterConfig("predastore.toml", ConfigMigration{
FromVersion: 1,
ToVersion: 2,
Description: "Add [ratelimit] section for API request throttling",
Run: func(ctx ConfigContext) error {
return appendRatelimitSection(
filepath.Join(ctx.ConfigDir, "predastore", "predastore.toml"),
predastoreRatelimitSection,
)
},
})
}
// appendRatelimitSection appends the ratelimit TOML section to a config file.
// Idempotent — skips if [ratelimit] is already present.
func appendRatelimitSection(path, section string) error {
data, err := os.ReadFile(path)
if err != nil {
return err
}
if strings.Contains(string(data), "[ratelimit]") {
return nil // already present
}
info, err := os.Stat(path)
if err != nil {
return err
}
updated := append(data, []byte(section)...)
return os.WriteFile(path, updated, info.Mode())
}
spinifex/migrate/migrate_test.go
+1 -1
View File
@@ -263,7 +263,7 @@ port = 8080
data, err := os.ReadFile(path)
require.NoError(t, err)
assert.Contains(t, string(data), "version = 2")
assert.Contains(t, string(data), `version = "2"`)
assert.NotContains(t, string(data), `"1.0"`)
assert.Contains(t, string(data), "port = 8080") // rest preserved
}
spinifex/migrate/version_readers.go
+2 -2
View File
@@ -16,7 +16,7 @@ type ConfigVersionReader interface {
// TOMLVersionReader reads/writes the version field in TOML config files.
// Handles both legacy string values ("1.0" → 1) and integer values (2 → 2).
// New writes always use integers (version = 2).
// New writes always use quoted strings (version = "2").
type TOMLVersionReader struct{}
var tomlVersionRe = regexp.MustCompile(`(?m)^version\s*=\s*(?:"([^"]+)"|(\d+))\s*$`)
@@ -57,7 +57,7 @@ func (r *TOMLVersionReader) WriteVersion(path string, version int) error {
return err
}
replacement := fmt.Sprintf("version = %d", version)
replacement := fmt.Sprintf("version = \"%d\"", version)
if tomlVersionRe.Match(data) {
data = tomlVersionRe.ReplaceAll(data, []byte(replacement))
spinifex/services/awsgw/awsgw.go
+33
View File
@@ -10,11 +10,13 @@ import (
"strings"
"time"
"github.com/mulgadc/predastore/ratelimit"
"github.com/mulgadc/spinifex/spinifex/config"
"github.com/mulgadc/spinifex/spinifex/gateway"
handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
"github.com/mulgadc/spinifex/spinifex/utils"
"github.com/nats-io/nats.go"
toml "github.com/pelletier/go-toml/v2"
)
var serviceName = "awsgw"
@@ -76,6 +78,25 @@ func (svc *Service) Reload() (err error) {
return nil
}
// awsgwTOML is the top-level structure of awsgw.toml used to extract the
// ratelimit section. Other fields are parsed elsewhere (e.g. region, debug).
type awsgwTOML struct {
Ratelimit ratelimit.Config `toml:"ratelimit"`
}
// loadThrottleConfig parses the [ratelimit] section from the awsgw TOML config.
func loadThrottleConfig(path string) (ratelimit.Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return ratelimit.Config{}, fmt.Errorf("read awsgw config %s: %w", path, err)
}
var cfg awsgwTOML
if err := toml.Unmarshal(data, &cfg); err != nil {
return ratelimit.Config{}, fmt.Errorf("parse awsgw config %s: %w", path, err)
}
return cfg.Ratelimit, nil
}
func launchService(config *config.ClusterConfig) error {
nodeConfig := config.Nodes[config.Node]
@@ -129,6 +150,13 @@ func launchService(config *config.ClusterConfig) error {
return fmt.Errorf("load bootstrap from %s: %w", bootstrapPath, err)
}
// Load API throttle config from awsgw.toml [ratelimit] section.
awsgwTomlPath := filepath.Join(nodeConfig.BaseDir, "config", "awsgw", "awsgw.toml")
throttleCfg, err := loadThrottleConfig(awsgwTomlPath)
if err != nil {
slog.Warn("Failed to load throttle config, throttling disabled", "err", err)
}
// Create gateway with NATS connection
gw := gateway.GatewayConfig{
Debug: nodeConfig.AWSGW.Debug,
@@ -143,6 +171,11 @@ func launchService(config *config.ClusterConfig) error {
Commit: commit,
}
if throttleCfg.Enabled {
gw.Throttler = ratelimit.New(throttleCfg)
defer gw.Throttler.Stop()
}
handler := gw.SetupRoutes()
// Load TLS certificate
spinifex/services/awsgw/awsgw_test.go
+76
View File
@@ -0,0 +1,76 @@
package awsgw
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLoadThrottleConfig_Enabled(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "awsgw.toml")
content := `
version = 2
region = "us-east-1"
[ratelimit]
enabled = true
rate = 20
burst = 100
[ratelimit.action.RunInstances]
rate = 2
burst = 40
`
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
cfg, err := loadThrottleConfig(path)
require.NoError(t, err)
assert.True(t, cfg.Enabled)
assert.Equal(t, 20, cfg.Rate)
assert.Equal(t, 100, cfg.Burst)
assert.Equal(t, 2, cfg.Action["RunInstances"].Rate)
assert.Equal(t, 40, cfg.Action["RunInstances"].Burst)
}
func TestLoadThrottleConfig_Disabled(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "awsgw.toml")
content := `
version = 2
[ratelimit]
enabled = false
rate = 20
burst = 100
`
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
cfg, err := loadThrottleConfig(path)
require.NoError(t, err)
assert.False(t, cfg.Enabled)
}
func TestLoadThrottleConfig_NoSection(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "awsgw.toml")
content := `
version = "1.0"
region = "us-east-1"
debug = false
`
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
cfg, err := loadThrottleConfig(path)
require.NoError(t, err)
// Missing section → zero-value config (disabled, rate=0, burst=0).
assert.False(t, cfg.Enabled)
assert.Equal(t, 0, cfg.Rate)
}
func TestLoadThrottleConfig_MissingFile(t *testing.T) {
_, err := loadThrottleConfig("/nonexistent/awsgw.toml")
assert.Error(t, err)
}