mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-11-02 09:13:33 +00:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 305315bf8b | |||
| 79e83aa0b7 | |||
| 213ba6d129 | |||
| 4535b6e5aa |
@@ -1203,16 +1203,6 @@ jobs:
|
||||
sleep 2
|
||||
rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log
|
||||
|
||||
# The ISO build runs as root inside the workspace, and the next job on
|
||||
# this runner, from any workflow, cannot remove what it leaves there.
|
||||
- name: Release the workspace
|
||||
if: always()
|
||||
run: |
|
||||
sudo umount -R -l "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache/rootfs 2>/dev/null || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache \
|
||||
"$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/dist
|
||||
sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
|
||||
|
||||
iso-multi:
|
||||
name: cell-22 (iso/dhcp/three-node)
|
||||
# After cell-21 rather than beside it: same concurrency group, and running
|
||||
@@ -1364,16 +1354,6 @@ jobs:
|
||||
rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
|
||||
"$WORK_DIR"
|
||||
|
||||
# The ISO build runs as root inside the workspace, and the next job on
|
||||
# this runner, from any workflow, cannot remove what it leaves there.
|
||||
- name: Release the workspace
|
||||
if: always()
|
||||
run: |
|
||||
sudo umount -R -l "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache/rootfs 2>/dev/null || true
|
||||
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/.cache \
|
||||
"$GITHUB_WORKSPACE"/mulga/scripts/iso-builder/dist
|
||||
sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
|
||||
|
||||
# Predastore throughput ("cell 23"). Predastore has no runners of its own and
|
||||
# spinifex depends on it, so the benchmark lives here and checks predastore
|
||||
# out as a sibling.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
ARG VERSION=dev
|
||||
|
||||
FROM golang:1.27.3-bookworm@sha256:d9b6a1beb9474adc2a1ce842573639b44b2a0dd49b6669f51d3821baf9ec37c9 AS builder
|
||||
FROM golang:1.27.3-bookworm@sha256:972b4110ba0f47539a975c31e52c64765e1ff19f2d74469ae4a595c6676e80a5 AS builder
|
||||
|
||||
ARG VERSION
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ quota state.
|
||||
a request/reply route on which a gateway, in the `awsgw-quota-seal` queue
|
||||
group, irreversibly seals one account's quota override record.
|
||||
`domains/admission/quota` owns it: `awsgw` serves it.
|
||||
`operator/accountlifecycle` is the consumer: it calls this subject to fence every
|
||||
`accountteardown` is the consumer: it calls this subject to fence every
|
||||
override writer at the quota owner before the account record is removed.
|
||||
|
||||
The request names the account in its body (`account_id`) and may carry an
|
||||
@@ -31,20 +31,10 @@ Any current gateway may answer; the leader-accepted revision is the evidence
|
||||
whichever one does, and the override bucket is opened at seal time even on a
|
||||
gateway whose quotas were disabled at startup.
|
||||
|
||||
## Seal-quota-override capability
|
||||
|
||||
`AccountQuotaOverrideSealCapabilitySubject(node)` (`admission.v1.AccountQuotaOverrideSealCapability.<node>`) is a per-node, read-only request/reply route on which `operator/accountlifecycle` asks one configured gateway whether it supports the quota override seal, immediately before every seal attempt.
|
||||
The quota owner (`domains/admission/quota`) owns it: `awsgw` serves it with no queue group, so a request addressed to one node reaches only subscribers configured as that node, and two gateways sharing a node ID both answer.
|
||||
The consumer's quota seal reaper probes every configured node that runs `awsgw` and sends no seal request unless every one answers compatible.
|
||||
|
||||
`AccountQuotaOverrideSealCapabilityVersion` is a claim about the whole binary's writers, not about the responder: version 1 means every quota override writer in the binary is revision-guarded and refuses a sealed record, and the binary serves the seal subject.
|
||||
A gateway binary without this responder has no subscriber, so the probe for that node reads as incomplete (`no-reply`), the same as a stopped or partitioned node.
|
||||
The reply proves nothing beyond the moment it was sent: a gateway downgraded or replaced after a successful probe is outside the guarantee.
|
||||
|
||||
### Compatibility boundary
|
||||
|
||||
The subject name carries the contract major; `account_quota_seal_test.go` and `account_quota_seal_capability_test.go`
|
||||
preserve the version-1 JSON shapes and the subject builder.
|
||||
The subject name carries the contract major; `account_quota_seal_test.go`
|
||||
preserves the version-1 JSON shape.
|
||||
Producer and consumer are in the coordinated Spinifex release, so the window
|
||||
is N and N-1.
|
||||
A gateway that predates the subject has no subscription, so the request gets
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package admissionv1
|
||||
|
||||
import "strings"
|
||||
|
||||
// AccountQuotaOverrideSealCapabilityVersion is the version this binary reports.
|
||||
// Version 1: every quota override writer in the binary is revision-guarded and
|
||||
// refuses a sealed record, and the seal subject is served.
|
||||
const AccountQuotaOverrideSealCapabilityVersion = 1
|
||||
|
||||
// AccountQuotaOverrideSealCapabilitySubject is one gateway's read-only
|
||||
// seal-capability route, addressed per node so a probe reaches exactly the
|
||||
// node it asks about.
|
||||
func AccountQuotaOverrideSealCapabilitySubject(node string) string {
|
||||
return "admission.v1.AccountQuotaOverrideSealCapability." + node
|
||||
}
|
||||
|
||||
// ValidSealCapabilityNode reports whether node is exactly one NATS subject
|
||||
// token. Responder and probe both check it before building a subject, so
|
||||
// neither can address a wildcard or another node's route.
|
||||
func ValidSealCapabilityNode(node string) bool {
|
||||
return node != "" && !strings.ContainsAny(node, ".*>\t\n\r ")
|
||||
}
|
||||
|
||||
// AccountQuotaOverrideSealCapabilityRequest asks node whether it supports the
|
||||
// quota override seal. CorrelationRef is an opaque caller reference echoed for
|
||||
// logs; it grants nothing and is never a fence or authority to write.
|
||||
type AccountQuotaOverrideSealCapabilityRequest struct {
|
||||
Node string `json:"node"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
|
||||
// AccountQuotaOverrideSealCapabilityReply reports the responding node's
|
||||
// capability version. It carries no account data and proves nothing beyond
|
||||
// the moment it was sent.
|
||||
type AccountQuotaOverrideSealCapabilityReply struct {
|
||||
Node string `json:"node"`
|
||||
CapabilityVersion int `json:"capability_version"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package admissionv1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAccountQuotaOverrideSealCapabilitySubject(t *testing.T) {
|
||||
require.Equal(t, "admission.v1.AccountQuotaOverrideSealCapability.node-1", AccountQuotaOverrideSealCapabilitySubject("node-1"))
|
||||
require.Equal(t, "admission.v1.AccountQuotaOverrideSealCapability.node-2", AccountQuotaOverrideSealCapabilitySubject("node-2"))
|
||||
}
|
||||
|
||||
func TestAccountQuotaOverrideSealCapabilityRequestJSONContract(t *testing.T) {
|
||||
data, err := json.Marshal(AccountQuotaOverrideSealCapabilityRequest{Node: "node-1", CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1","correlation_ref":"req-1"}`, string(data))
|
||||
|
||||
data, err = json.Marshal(AccountQuotaOverrideSealCapabilityRequest{Node: "node-1"})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1"}`, string(data))
|
||||
}
|
||||
|
||||
func TestAccountQuotaOverrideSealCapabilityReplyJSONContract(t *testing.T) {
|
||||
reply := AccountQuotaOverrideSealCapabilityReply{Node: "node-1", CapabilityVersion: 1, CorrelationRef: "req-1"}
|
||||
data, err := json.Marshal(reply)
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1","capability_version":1,"correlation_ref":"req-1"}`, string(data))
|
||||
|
||||
data, err = json.Marshal(AccountQuotaOverrideSealCapabilityReply{Node: "node-1", CapabilityVersion: 1})
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{"node":"node-1","capability_version":1}`, string(data))
|
||||
}
|
||||
|
||||
func TestValidSealCapabilityNode(t *testing.T) {
|
||||
for _, node := range []string{"node-1", "node_2", "n3"} {
|
||||
require.True(t, ValidSealCapabilityNode(node), node)
|
||||
}
|
||||
for _, node := range []string{"", "a.b", "*", ">", "node-*", "node >", "node\t1", "node\n1"} {
|
||||
require.False(t, ValidSealCapabilityNode(node), "%q", node)
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -155,7 +155,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
|
||||
| `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
|
||||
| `spx admin account quota get <account-id>` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Reads the limits in force for one account through `POST /admin/GetAccountQuota` over SigV4, and reports for each dimension whether it came from the `[quota]` block in `awsgw.toml` (`config`) or from an override stored against the account (`override`). Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:GetAccountQuota`. |
|
||||
| `spx admin account quota set <account-id>` | `--vcpus`, `--vpcs`, `--subnets`, `--eips`, `--volumes`, `--volumes-gib`, `--rds-instances`, `--load-balancers`, `--clear`, plus `--endpoint`, `--region`, `--ca-bundle` | Overrides individual limits for one account through `POST /admin/PutAccountQuota`, leaving every dimension not named inheriting the configured baseline. `-1` uncaps a dimension; `0` is a real limit that denies every request on it. `--clear` removes the whole override record and cannot be combined with a limit flag. The account must already exist. Prints the resulting limits, so a caller never has to guess what it applied. The caller must hold `spinifex:PutAccountQuota`. |
|
||||
| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record) | Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
|
||||
| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record) | Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
|
||||
| `spx admin account delete --force` | as above | Escalates only for resources the ordinary API refuses: it clears a volume attachment in the control plane without the guest's cooperation (`ec2.ForceDetachVolume`), hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never widens scope beyond the account. Use it once the guest is being destroyed — this is the escape from a volume stuck attached to an instance that will not terminate, where neither can otherwise be deleted. |
|
||||
| `spx admin account delete --remote` | as above, plus `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Same teardown driven through `POST /admin/DeleteAccount` over SigV4 instead of NATS, then follows `DescribeAccountDeletion` to completion, printing each stage as it lands. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. |
|
||||
| `spx admin principal create <name>` | `--grant` (repeatable or comma-separated; default: every admin method — `CreateAccount`, `DeleteAccount`, `DescribeAccountDeletion`, `GetAccountQuota`, `ListAccounts`, `PutAccountQuota`) | Creates an IAM user in the super-admin account with an inline `spx-admin-methods` policy allowing the requested methods, revokes any access key left from a previous run, and mints one new key. Each method is named rather than granted as `spinifex:*`, so a later addition to the admin surface is not authorised by an existing key. The principal's grants are exactly that one policy: re-running with fewer `--grant` values narrows it, and any other inline policy on the user is removed, so a rename cannot leave a wider grant behind. The secret is printed once; store it in a secret manager or an AWS profile, never in a config file. `admin` is refused as a name — it is the account's own administrator and rotating its key would revoke the bootstrap credential. |
|
||||
|
||||
@@ -371,7 +371,6 @@ while the branch is in flight; they do not describe the umbrella repository's
|
||||
| `spinifex#1378` | `spinifex/gateway` `oidc_discovery.go` and `registry.go` (the `/oidc/eks` routes and the `/v2` mount), the `DiscoverActiveNodes` method and its cache, and the `GatewayConfig` fields `Config`, `RootCAs`, `InternalSuffix`, `Version`, `Commit`, `IAMService`, `STSService`, `NATSConn` and `ExpectedNodes` | `spinifex/runtime/roles/awsgw` (`PublicRoutes`, `NodeDiscovery.ActiveNodes`, `composeGateway`); the gateway keeps one `PublicRoutes` mount hook and the `Admin` registration name supplied by the role |
|
||||
| `spinifex#1384` | `spinifex/gateway` (`GatewayConfig`, the SigV4 and policy middleware, `Request`, the admin dispatch, `AWSOperationInventory`) | `spinifex/ingress/aws/pipeline` (`Config`) and `spinifex/ingress/aws/dispatch` (`AWSOperationInventory`); the gateway layering rows are retired and R2 binds the pipeline |
|
||||
| `spinifex#1334` | `spinifex/gateway/oidc_discovery.go` (`eksAccountKV`, the discovery-document and JWKS read and construction) | `spinifex/domains/eks/cluster` (`OIDCPublisher`, `OIDCDiscoveryDocument`), composed by `spinifex/runtime/roles/awsgw` | The EKS domain now reads its own account bucket and builds the per-cluster discovery document and JWKS responses (ADR-0005:S4), and the gateway only maps the unauthenticated `/oidc/eks/...` routes onto an `OIDCPublisher` interface it is handed. Responses are byte-identical, pinned by `oidc_discovery_current_behaviour_test.go`, whose assertions are unchanged. The gateway no longer imports `domains/eks/cluster` in production code. `STS` keeps its own `ClusterJWKSResolver` (`OIDCJWKSReader`), untouched. Recorded debt, not repaired here: a stored JWKS is served verbatim with 200 even when it is not valid JSON. |
|
||||
| `spinifex#1245`, `spinifex#1390` | — | `contracts/admission/v1` | The quota override seal route (`admission.v1.SealAccountQuotaOverride`, served by `awsgw`) and the per-gateway read-only capability probe (`admission.v1.AccountQuotaOverrideSealCapability.<node>`) are an explicit versioned cross-process contract between the quota owner and `operator/accountlifecycle`. The consumer wiring is new: `NewClusterEngine` takes a `ClusterNodes` value with separate daemon and gateway node sets, `ClusterConfig.NodesRunning` is the single definition of which nodes run a service, and the seal reaper runs in `StageAccount` after the EC2 settings seal. A termination whose gateway probe is incomplete seals nothing and leaves the account `TERMINATING`. Contract, responder, probe and reaper tests pin the subjects, JSON shapes and fail-closed cases; `TestAccountTermination_QuotaOverrideEndsSealed` replaces the former characterization that overrides survive teardown. |
|
||||
The EC2 contract row is intentionally different from the directory moves: it creates
|
||||
a compatibility boundary. `ec2.cmd.*` retains its deployed one-token NATS
|
||||
subscription and permission shape, while callers use
|
||||
|
||||
@@ -4,13 +4,13 @@
|
||||
# distroless base (no shell, no package manager) for a small, production-shaped
|
||||
# image suitable for pushing to ECR and pulling onto EKS workers.
|
||||
|
||||
FROM golang:1.27.3-alpine@sha256:d200d2dd76b03a99b52651fd1325aff7dcd35422c35c01bac56571b0507f3deb AS build
|
||||
FROM golang:1.27.3-alpine@sha256:f2c872bcee949b7be9c490efd02b48a8a2fbf947cdf533f1c58525ec9d8170fd AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod ./
|
||||
COPY main.go ./
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/spinifex-demo .
|
||||
|
||||
FROM gcr.io/distroless/static:nonroot@sha256:25ac8f0c3b2334a9eb05452266bd4697864a8c8ef0ef71e5e6480d7f92868329
|
||||
FROM gcr.io/distroless/static:nonroot@sha256:18f4cc26c9ffa7f20a359298be7deadac5c607c8e03a02b8119f1a5fc2f82c99
|
||||
COPY --from=build /out/spinifex-demo /spinifex-demo
|
||||
EXPOSE 8080
|
||||
USER nonroot:nonroot
|
||||
|
||||
@@ -4,15 +4,15 @@ go 1.27.3
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.12
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.12
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5
|
||||
github.com/aws/aws-sdk-go-v2/service/lambda v1.113.4
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.7
|
||||
github.com/aws/smithy-go v1.28.7
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.4
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3
|
||||
github.com/aws/aws-sdk-go-v2/service/lambda v1.113.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.6
|
||||
github.com/aws/smithy-go v1.28.6
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
github.com/charmbracelet/bubbles v1.0.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
@@ -31,8 +31,8 @@ require (
|
||||
github.com/mulgadc/northstar v1.23.0
|
||||
github.com/mulgadc/predastore v1.23.0
|
||||
github.com/mulgadc/viperblock v1.23.0
|
||||
github.com/nats-io/nats-server/v2 v2.15.2
|
||||
github.com/nats-io/nats.go v1.55.0
|
||||
github.com/nats-io/nats-server/v2 v2.15.1
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
github.com/nats-io/nkeys v0.4.16
|
||||
github.com/opencontainers/runtime-spec v1.3.0
|
||||
github.com/oracle/oci-go-sdk/v65 v65.128.0
|
||||
@@ -43,9 +43,9 @@ require (
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/spf13/viper v1.21.0
|
||||
github.com/stretchr/testify v1.12.1
|
||||
go.etcd.io/etcd/api/v3 v3.7.3
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.3
|
||||
go.etcd.io/etcd/client/v3 v3.7.3
|
||||
go.etcd.io/etcd/api/v3 v3.5.35
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.35
|
||||
go.etcd.io/etcd/client/v3 v3.5.35
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
|
||||
go.opentelemetry.io/otel v1.47.0
|
||||
go.opentelemetry.io/otel/metric v1.47.0
|
||||
@@ -53,14 +53,14 @@ require (
|
||||
go.opentelemetry.io/otel/sdk/metric v1.47.0
|
||||
go.opentelemetry.io/otel/trace v1.47.0
|
||||
go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b
|
||||
go.yaml.in/yaml/v3 v3.0.6
|
||||
go.yaml.in/yaml/v3 v3.0.5
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/sync v0.24.0
|
||||
golang.org/x/sys v0.49.0
|
||||
golang.org/x/term v0.47.0
|
||||
golang.org/x/time v0.16.0
|
||||
golang.org/x/tools v0.52.0
|
||||
google.golang.org/grpc v1.85.0
|
||||
google.golang.org/grpc v1.83.2
|
||||
gopkg.in/ini.v1 v1.67.3
|
||||
)
|
||||
|
||||
@@ -72,17 +72,17 @@ require (
|
||||
github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
@@ -111,7 +111,7 @@ require (
|
||||
github.com/containerd/plugin v1.1.0 // indirect
|
||||
github.com/containerd/ttrpc v1.2.9 // indirect
|
||||
github.com/containerd/typeurl/v2 v2.3.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
|
||||
github.com/dgraph-io/badger/v4 v4.9.6 // indirect
|
||||
github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect
|
||||
@@ -128,6 +128,7 @@ require (
|
||||
github.com/go-playground/validator/v10 v10.30.5 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/gofrs/flock v0.10.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/google/flatbuffers v25.12.19+incompatible // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
@@ -174,6 +175,7 @@ require (
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.18 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_golang v1.24.1 // indirect
|
||||
github.com/prometheus/client_model v0.6.3 // indirect
|
||||
github.com/prometheus/common v0.71.0 // indirect
|
||||
@@ -201,9 +203,10 @@ require (
|
||||
go.opentelemetry.io/otel/log v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/log v1.47.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
go.uber.org/zap v1.17.0 // indirect
|
||||
golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 // indirect
|
||||
golang.org/x/mod v0.42.0 // indirect
|
||||
golang.org/x/net v0.61.0 // indirect
|
||||
|
||||
@@ -22,46 +22,46 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8=
|
||||
github.com/aws/aws-sdk-go-v2/service/lambda v1.113.4 h1:wvAdq1sNfwuedO1BX8GjY+oZWzoyeD4hhdjJI+4EPnI=
|
||||
github.com/aws/aws-sdk-go-v2/service/lambda v1.113.4/go.mod h1:HgkYN4d72Fawg3MAJQepmNnROFoN11XYu14hgFfChTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.7 h1:Eip+mZ0wFZY/10e5w89lWhF09Pa8tbbGwxePP3T2iMc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.7/go.mod h1:bXp38eBNyvSevFIao60mYcC3pQIAkPYWOddhFVv9z8E=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
|
||||
github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE=
|
||||
github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.4 h1:bTvk6UdSSSvaQsfLVj1JRDEEqj/PDRwGf1KanaRzau0=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.4/go.mod h1:XkrjAZvr705Nytv6PkPUhoSB2W9+vVHBp5oAWpkh8Ak=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 h1:t7FxqJ5KHDeKceyD4HtGaG6z7hMSeVGjs5/4c+L5ArM=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24/go.mod h1:5XK6NRVlK8s8yj4zKcM43HLluGA/rAz+azLLvWDMRr0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7 h1:fSNAt6hs4qLjb9tMZc6vj3PHlOCYGXK2JjJZ7AW2tf4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7/go.mod h1:TMav+EuXQu5kNpZZz/6rIC8xx4mCsW+6AKHrjvOSAsE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7 h1:7NS6kqBjkWIN8P/AEdmtyq6eQGAhN1oHTZdzXclpH6Y=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7/go.mod h1:1PFqtAUsIgF1nEVojzRJSf/v88zjzqrGW+w+UAtkXyQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
|
||||
github.com/aws/aws-sdk-go-v2/service/lambda v1.113.3 h1:oSHldse7GDHpdq81XeAJnhIbsdDKP9bMIt00G8W6uXI=
|
||||
github.com/aws/aws-sdk-go-v2/service/lambda v1.113.3/go.mod h1:Y//K046w4uhiR3o77beo/y52Jv2uEY13iJ+3+jicLO8=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4 h1:MrkWRB58Qw5TEkb51QVooVLjlM3IevO2iVXEz74B8pg=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4/go.mod h1:yfIxaKRHasvfy6tNEm3/LpqKsupMzQjMaWj9Y0jMVbo=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.6 h1:Suq1LNhMBpsmShEv3vO+J3H8HEeK+GwYMJDYLE4IJSs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.52.6/go.mod h1:rHGuzFg4AfVnWZa3/533tn5GQd1LzVkTXI9WMT8LqAc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM=
|
||||
github.com/aws/smithy-go v1.28.6 h1:Pj+/sUviNYjow8ELRtNwKHZ9ECeF+7pATX3+shjLIrw=
|
||||
github.com/aws/smithy-go v1.28.6/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
@@ -128,8 +128,8 @@ github.com/containerd/ttrpc v1.2.9 h1:ha0ak962T0s3CA/RoZ6S6xiWZQF24GrBaEpiGX1uih
|
||||
github.com/containerd/ttrpc v1.2.9/go.mod h1:jjtQRwXm4DL3KsHKW8vDiUOV6wO0hi6IPhmJhxU7aEs=
|
||||
github.com/containerd/typeurl/v2 v2.3.0 h1:HZHPhRWo5XMy3QGQoPrUzbW/2ckwjfweHmOwlkIrPAQ=
|
||||
github.com/containerd/typeurl/v2 v2.3.0/go.mod h1:Qk+PAdUYArVj41TnGi6rJ+48RF0PkcTc4i/taoBcK0w=
|
||||
github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
|
||||
github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
|
||||
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
|
||||
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
@@ -181,6 +181,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/gofrs/flock v0.10.0 h1:SHMXenfaB03KbroETaCMtbBg3Yn29v4w1r+tgy4ff4k=
|
||||
github.com/gofrs/flock v0.10.0/go.mod h1:FirDy1Ing0mI2+kB6wk+vyyAH+e6xiE+EYA0jnzV9jc=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
@@ -247,6 +249,8 @@ github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtL
|
||||
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
|
||||
github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc=
|
||||
github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
|
||||
github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/klauspost/cpuid/v2 v2.4.1 h1:I9KxadAkvo+QjtcEE2f2tyfEvmIla/T0m+14PrZNl0U=
|
||||
@@ -313,10 +317,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
|
||||
github.com/nats-io/jwt/v2 v2.8.2/go.mod h1:Ag/56sq9OblL4JgdYufDd16Egb17Kr/8WwwuO/forVc=
|
||||
github.com/nats-io/nats-server/v2 v2.15.2 h1:yLsgt2kslgymvxvjFq1WGWck5hkat2YYfbXvQ145tzU=
|
||||
github.com/nats-io/nats-server/v2 v2.15.2/go.mod h1:VXHCZgDglV5wO2B8ZE58yG2HzqxQxYUbQqibUh5Xcr8=
|
||||
github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3M=
|
||||
github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nats-server/v2 v2.15.1 h1:fK3AeYFIji1Smng60jB0hVoAPZezA/UzsaI3BqXFNEI=
|
||||
github.com/nats-io/nats-server/v2 v2.15.1/go.mod h1:yeuYRtKVKIFtmMz2/YPk+6hvmNBnk+WwTUzErBdH6IE=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
@@ -336,6 +340,7 @@ github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8
|
||||
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
@@ -404,15 +409,17 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU=
|
||||
go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk=
|
||||
go.etcd.io/etcd/api/v3 v3.7.3 h1:2tumepXe+wi7Wab64uUO71K7ISQyUWLGC1DJbbc59sY=
|
||||
go.etcd.io/etcd/api/v3 v3.7.3/go.mod h1:HOR9Qn4gb/1/hVAmYprmlLpvTe6ZgfgNS9Z/+Ty65po=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.3 h1:5zMwNu3ycrzCeA5NwOfQbxFRhqq974chq1DrYMua4QI=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.3/go.mod h1:/OhzqYrF0U7d1NOk55VMyUQGbYD1WoIe4bJwxb47wOk=
|
||||
go.etcd.io/etcd/client/v3 v3.7.3 h1:UrXcetgkuSEocaBlQlulQRJlXLEnUAHuBRbHh0lue1Q=
|
||||
go.etcd.io/etcd/client/v3 v3.7.3/go.mod h1:DcPxpqcpV8WIaRi/2waPCx0/Uq8rQUweDJaDYZk1gEE=
|
||||
go.etcd.io/etcd/api/v3 v3.5.35 h1:lB//Av9mFrkljv94XDNAY+qiyQA9W6S/TTA2Uwx2jsI=
|
||||
go.etcd.io/etcd/api/v3 v3.5.35/go.mod h1:z1FGQolu/tpmQ2MPFwQNt7G3cXBiSkTevZBhfOtEArQ=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.35 h1:nUZutZyJVvsKddK6zddXMBxIhmVi0SzPMBRS0FnNM3o=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.35/go.mod h1:0MNjMMqajrOI0/l8OLFkJBhE5Vm6BAi7cerKlrO38hw=
|
||||
go.etcd.io/etcd/client/v3 v3.5.35 h1:dk0aSLjY5D1eOR0SNnPXWOWCVA9yIzJoB8a912qxmoI=
|
||||
go.etcd.io/etcd/client/v3 v3.5.35/go.mod h1:LjtCl6G5li7PgsGjoQIpYXz/OvSbI1pCxpSp1kqo9P4=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNRaZy4v4YgVenGd52dRBI=
|
||||
@@ -449,22 +456,26 @@ go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrF
|
||||
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b h1:FHYZT12VzXnb6P399kd1i9Ai7aLF+CFSUsOFylDt7Lk=
|
||||
go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
|
||||
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a h1:CsRd824LAzMmlpu9nYv2PHOqPQYv4Zjd4kGM+g0ahS0=
|
||||
go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a/go.mod h1:i9Hi8GeZFLBtUCXP73ZNSHKZ4K2JtxT0pLmo3sbJzlo=
|
||||
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
|
||||
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
|
||||
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
|
||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
|
||||
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
@@ -472,24 +483,33 @@ golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+h
|
||||
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
|
||||
golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 h1:GTD/WuaexTazIG/SxLOz4rEKZPDVilmVVC2nz4xhwfE=
|
||||
golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976/go.mod h1:PqrXSW65cXDZH0k4IeUbhmg/bcAZDbzNz3byBpKCsXo=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
|
||||
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
|
||||
golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -522,6 +542,8 @@ golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
|
||||
golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
|
||||
@@ -533,14 +555,17 @@ golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGN
|
||||
golang.org/x/vuln v1.6.0 h1:FeMO9Rm/HwyduOztbvKcOw+zvDEPr4I4aQNSfevFcKY=
|
||||
golang.org/x/vuln v1.6.0/go.mod h1:bWlG2493/sjR7ksvicBgMrznH3eYQEyK8ifUYBrqUbg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc=
|
||||
google.golang.org/grpc v1.85.0 h1:BPBCfuVRQ1qvBhKDOq9uvQllMvO59rKsYhXpp7Dd8vw=
|
||||
google.golang.org/grpc v1.85.0/go.mod h1:0rQrOD58yjal/0EaVbGBANindixsCZNQPDV3VKtaKuQ=
|
||||
google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
|
||||
google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
@@ -550,7 +575,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc=
|
||||
pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk=
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
// Package deadpid gives tests a PID that no process can hold.
|
||||
package deadpid
|
||||
|
||||
// PID is above the kernel's PID_MAX_LIMIT (4194304), so kill(2) on it always
|
||||
// fails with ESRCH. A plausible-looking literal such as 99999 is a real PID on
|
||||
// a busy host, and signalling it terminates whatever process holds it.
|
||||
const PID = 1 << 30
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
@@ -69,7 +70,7 @@ func Start(t *testing.T, selfID uint64, members []Member) *Server {
|
||||
})
|
||||
s := &Server{Endpoint: "https://" + lis.Addr().String(), CertDir: dir, members: members, selfID: selfID}
|
||||
gs := grpc.NewServer(grpc.Creds(creds))
|
||||
pb.RegisterClusterServer(gs, &cluster{s: s})
|
||||
pb.RegisterClusterServer(gs, &cluster{s})
|
||||
pb.RegisterMaintenanceServer(gs, &maintenance{s: s})
|
||||
go func() { _ = gs.Serve(lis) }()
|
||||
t.Cleanup(gs.Stop)
|
||||
@@ -90,11 +91,7 @@ func (s *Server) FailWith(err error) {
|
||||
s.failAll = err
|
||||
}
|
||||
|
||||
type cluster struct {
|
||||
pb.UnimplementedClusterServer
|
||||
|
||||
s *Server
|
||||
}
|
||||
type cluster struct{ s *Server }
|
||||
|
||||
func (c *cluster) MemberList(context.Context, *pb.MemberListRequest) (*pb.MemberListResponse, error) {
|
||||
c.s.mu.Lock()
|
||||
@@ -119,6 +116,18 @@ func (c *cluster) MemberRemove(_ context.Context, req *pb.MemberRemoveRequest) (
|
||||
return &pb.MemberRemoveResponse{Header: &pb.ResponseHeader{MemberId: c.s.selfID}}, nil
|
||||
}
|
||||
|
||||
func (c *cluster) MemberAdd(context.Context, *pb.MemberAddRequest) (*pb.MemberAddResponse, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (c *cluster) MemberUpdate(context.Context, *pb.MemberUpdateRequest) (*pb.MemberUpdateResponse, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (c *cluster) MemberPromote(context.Context, *pb.MemberPromoteRequest) (*pb.MemberPromoteResponse, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
type maintenance struct {
|
||||
pb.UnimplementedMaintenanceServer
|
||||
|
||||
|
||||
@@ -7,14 +7,17 @@ package pagedstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
)
|
||||
|
||||
// Store wraps a MemoryObjectStore whose ListObjectsV2 pages like S3, capping
|
||||
// every page at PageSize entries so small fixtures span several pages.
|
||||
// Store wraps a MemoryObjectStore and splits ListObjectsV2's answer into
|
||||
// pages of at most PageSize objects, sorted by key so pagination is
|
||||
// deterministic across runs (MemoryObjectStore itself iterates a map).
|
||||
type Store struct {
|
||||
*objectstore.MemoryObjectStore
|
||||
|
||||
@@ -25,20 +28,51 @@ type Store struct {
|
||||
Calls int
|
||||
}
|
||||
|
||||
// New returns an empty Store that serves at most pageSize entries per page.
|
||||
// New returns a paging store with the given per-page object limit.
|
||||
func New(pageSize int) *Store {
|
||||
return &Store{MemoryObjectStore: objectstore.NewMemoryObjectStore(), PageSize: pageSize}
|
||||
}
|
||||
|
||||
var _ objectstore.ObjectStore = (*Store)(nil)
|
||||
|
||||
// ListObjectsV2 lowers the caller's MaxKeys to PageSize and lets the memory
|
||||
// store page, so tokens, ordering and common prefixes follow S3's rules.
|
||||
// ListObjectsV2 delegates to MemoryObjectStore for prefix/delimiter matching,
|
||||
// then slices the sorted result into one page per call, resuming from the
|
||||
// offset encoded in ContinuationToken.
|
||||
func (s *Store) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
s.Calls++
|
||||
page := *input
|
||||
if page.MaxKeys == nil || *page.MaxKeys > int64(s.PageSize) {
|
||||
page.MaxKeys = aws.Int64(int64(s.PageSize))
|
||||
|
||||
full, err := s.MemoryObjectStore.ListObjectsV2(ctx, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.MemoryObjectStore.ListObjectsV2(ctx, &page)
|
||||
|
||||
sort.Slice(full.Contents, func(i, j int) bool {
|
||||
return aws.StringValue(full.Contents[i].Key) < aws.StringValue(full.Contents[j].Key)
|
||||
})
|
||||
|
||||
start := 0
|
||||
if tok := aws.StringValue(input.ContinuationToken); tok != "" {
|
||||
start, err = strconv.Atoi(tok)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
end := min(start+s.PageSize, len(full.Contents))
|
||||
if start > end {
|
||||
start = end
|
||||
}
|
||||
|
||||
page := &s3.ListObjectsV2Output{
|
||||
Contents: full.Contents[start:end],
|
||||
CommonPrefixes: full.CommonPrefixes,
|
||||
Name: input.Bucket,
|
||||
KeyCount: aws.Int64(int64(end - start)),
|
||||
}
|
||||
if end < len(full.Contents) {
|
||||
page.IsTruncated = aws.Bool(true)
|
||||
page.NextContinuationToken = aws.String(strconv.Itoa(end))
|
||||
} else {
|
||||
page.IsTruncated = aws.Bool(false)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
# Debian is the build host regardless of what is being built — mkosi bootstraps
|
||||
# the target distribution from its own archive, so an Ubuntu image builds fine
|
||||
# from here (see ubuntu-keyring below).
|
||||
FROM debian:trixie-slim@sha256:f6b08b09a05725f9431502d3b5714f5c4576e82c1a84fa1debe0c24a7e0c3ec4
|
||||
FROM debian:trixie-slim@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad
|
||||
|
||||
# The runner user is uid 1000; matching it keeps bind-mounted build output
|
||||
# owned by the invoking user instead of root.
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
------------------------------------------------- MODULE Capabilities -------------------------------------------------
|
||||
(*********************************************************************************************************************)
|
||||
(* The optional behaviour that an implementation advertises. *)
|
||||
(*********************************************************************************************************************)
|
||||
EXTENDS EBSProvider
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Interface methods
|
||||
|
||||
GetCapabilitiesRequests == {<<>>}
|
||||
|
||||
GetCapabilities(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* The implementation reports which optional behaviour it supports. *)
|
||||
(*******************************************************************************************************************)
|
||||
\* The provider failed to answer.
|
||||
\/ /\ val = None
|
||||
/\ err \in ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\* Every answer is the same, so an earlier one may be cached.
|
||||
\/ /\ val = Capabilities
|
||||
/\ err = None
|
||||
/\ UNCHANGED vars
|
||||
|
||||
=======================================================================================================================
|
||||
@@ -1,163 +0,0 @@
|
||||
------------------------------------------------- MODULE EBSProvider --------------------------------------------------
|
||||
(*********************************************************************************************************************)
|
||||
(* The state that an implementation exposes, and the types and constants that every other module shares. *)
|
||||
(*********************************************************************************************************************)
|
||||
EXTENDS
|
||||
Naturals,
|
||||
Sequences,
|
||||
FiniteSets,
|
||||
Functions,
|
||||
TLC
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Constants
|
||||
|
||||
CONSTANTS
|
||||
MaxBytes,
|
||||
VolumeIDs, VolumeHandles,
|
||||
NodeIDs, NBDURIs, DevicePaths,
|
||||
SnapshotIDs, SnapshotHandles,
|
||||
AvailabilityZones, Timestamps,
|
||||
Parameters, ProviderData,
|
||||
ListTokens, MaxListResults,
|
||||
\* The optional behaviour that the implementation advertises, which never changes.
|
||||
Capabilities,
|
||||
\* Whether the provider may fail to answer at any point in a behaviour.
|
||||
ProviderMayFail,
|
||||
\* Stands in for an optional field that a request or response omitted.
|
||||
None
|
||||
|
||||
ASSUME MaxBytes \in Nat \ {0}
|
||||
ASSUME VolumeIDs /= {} /\ VolumeHandles /= {}
|
||||
ASSUME NodeIDs /= {} => NBDURIs /= {} \/ DevicePaths /= {}
|
||||
ASSUME SnapshotIDs /= {} => SnapshotHandles /= {} /\ Timestamps /= {}
|
||||
ASSUME ListTokens /= {}
|
||||
ASSUME MaxListResults \in Nat \ {0}
|
||||
ASSUME None \notin UNION {VolumeIDs, VolumeHandles, NodeIDs, SnapshotIDs, SnapshotHandles,
|
||||
AvailabilityZones, Timestamps, Parameters, ProviderData, ListTokens}
|
||||
ASSUME Capabilities \in [online_expansion : BOOLEAN,
|
||||
sparse_extent_reporting : BOOLEAN,
|
||||
crash_consistent_snapshot : BOOLEAN,
|
||||
volume_seeding : BOOLEAN,
|
||||
read_only_publish : BOOLEAN,
|
||||
volume_enumeration : BOOLEAN,
|
||||
snapshot_enumeration : BOOLEAN,
|
||||
owner_routing : BOOLEAN,
|
||||
exclusion : [scope : {"none", "node", "cluster"},
|
||||
claim_ttl_seconds : Nat,
|
||||
fences_lost_claim : BOOLEAN]]
|
||||
ASSUME ProviderMayFail \in BOOLEAN
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Variables
|
||||
|
||||
VARIABLES
|
||||
\* Every volume that the implementation exposes, keyed by volume ID.
|
||||
volumes,
|
||||
\* Every publication that the implementation exposes, keyed by the volume and the node it is published to.
|
||||
publications,
|
||||
\* Every snapshot that the implementation exposes, keyed by snapshot ID.
|
||||
snapshots
|
||||
|
||||
vars == <<volumes, publications, snapshots>>
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Miscellaneous
|
||||
|
||||
IsPartialFn(f, D, R) == DOMAIN f \subseteq D /\ \A x \in DOMAIN f : f[x] \in R
|
||||
|
||||
Optional(S) == S \union {None}
|
||||
|
||||
InsertKey(f, x, y) == f @@ (x :> y)
|
||||
|
||||
RemoveKey(f, x) == RestrictDomain(f, LAMBDA x_2 : x_2 /= x)
|
||||
|
||||
KeyExists(f, x) == x \in DOMAIN f
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Types
|
||||
|
||||
Bytes == 0..MaxBytes
|
||||
|
||||
\* The errors that an implementation may return when its provider fails to answer.
|
||||
ProviderErrors == IF ProviderMayFail
|
||||
THEN {"internal", "unavailable"}
|
||||
\* Otherwise an implementation could conform by refusing to answer.
|
||||
ELSE {}
|
||||
|
||||
CapacityRanges == [required_bytes : Bytes, limit_bytes : Bytes]
|
||||
|
||||
VolumeStates == {"available", "in-use"}
|
||||
|
||||
Volumes == [capacity_bytes : 1..MaxBytes,
|
||||
state : VolumeStates,
|
||||
handle : VolumeHandles,
|
||||
availability_zone : Optional(AvailabilityZones),
|
||||
|
||||
\* Never returned, but kept to tell a retry from a conflicting request.
|
||||
source_snapshot_id : Optional(SnapshotIDs),
|
||||
parameters : Optional(Parameters),
|
||||
provider_data : Optional(ProviderData)]
|
||||
|
||||
\* How a published volume is reached on its node, either over NBD or as a host block device.
|
||||
Exports == [kind : {"nbd"},
|
||||
nbd_uri : NBDURIs]
|
||||
\union
|
||||
[kind : {"block"},
|
||||
device_path : DevicePaths,
|
||||
size_bytes : 1..MaxBytes]
|
||||
|
||||
\* A volume published to a node, and whether it was published read-only.
|
||||
Publications == [export : Exports,
|
||||
read_only : BOOLEAN]
|
||||
|
||||
SnapshotStates == {"pending", "completed", "error"}
|
||||
|
||||
Snapshots == [size_bytes : 1..MaxBytes,
|
||||
created_at : Timestamps,
|
||||
state : SnapshotStates,
|
||||
handle : SnapshotHandles,
|
||||
source_volume_id : VolumeIDs]
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* State helpers
|
||||
|
||||
\* The part of a volume that the implementation returns.
|
||||
VolumeView(volID) ==
|
||||
("id" :> volID) @@ Restrict(volumes[volID], {"capacity_bytes",
|
||||
"state",
|
||||
"handle",
|
||||
"availability_zone",
|
||||
"provider_data"})
|
||||
|
||||
\* The nodes that a volume is published to.
|
||||
PublishedNodes(volID) == {n \in NodeIDs : KeyExists(publications, <<volID, n>>)}
|
||||
|
||||
\* The part of a publication that the implementation returns.
|
||||
PublicationView(volID, nodeID) == [volume_id |-> volID,
|
||||
node_id |-> nodeID,
|
||||
export |-> publications[volID, nodeID].export]
|
||||
|
||||
\* The part of a snapshot that the implementation returns.
|
||||
SnapshotView(snapID) == ("id" :> snapID) @@ snapshots[snapID]
|
||||
|
||||
\* The number of refs that a page may hold, with an omitted or oversized request clamped to the maximum.
|
||||
PageSize(m) == IF m <= 0 \/ m > MaxListResults THEN MaxListResults ELSE m
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Invariants
|
||||
|
||||
TypeOK ==
|
||||
/\ IsPartialFn(volumes, VolumeIDs, Volumes)
|
||||
/\ IsPartialFn(publications, VolumeIDs \X NodeIDs, Publications)
|
||||
/\ IsPartialFn(snapshots, SnapshotIDs, Snapshots)
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Initial state
|
||||
|
||||
Init ==
|
||||
/\ volumes = <<>>
|
||||
/\ publications = <<>>
|
||||
/\ snapshots = <<>>
|
||||
|
||||
=======================================================================================================================
|
||||
@@ -1,140 +0,0 @@
|
||||
# EBSProvider specification
|
||||
|
||||
The contract that every implementation of the `EBSProvider` interface must meet.
|
||||
|
||||
## Terminology
|
||||
|
||||
- **Provider:** the service that backs an implementation's operations, such as Viperblock.
|
||||
The memory implementation has none.
|
||||
- **Implementation:** a Go implementation of the `EBSProvider` interface, which adapts between the shapes of the contract and the shapes that its provider expects.
|
||||
- **Contract:** the behaviour that this specification requires of every implementation, as observed through the interface.
|
||||
|
||||
## Open questions
|
||||
|
||||
### EBSProvider
|
||||
|
||||
- What may a failed call change?
|
||||
A failed call leaves the state unchanged, but a timeout can arrive after the provider acted, and a crash can leave work half done.
|
||||
- Is the schema version part of the contract?
|
||||
Every method refuses an unsupported version with `unsupported_version`, and the conformance suite tests it, but the specification does not model versions.
|
||||
- Are `internal` and `unavailable` the only errors that a failed provider may cause?
|
||||
- Must a present but malformed ID be refused with `invalid_argument`?
|
||||
IDs are opaque here, so only an omitted one is refused.
|
||||
|
||||
### Capabilities
|
||||
|
||||
#### GetCapabilities
|
||||
|
||||
- Is it a lasting promise that capabilities never change, even when an implementation or its provider is upgraded or reconfigured after an earlier answer was cached?
|
||||
|
||||
### Volumes
|
||||
|
||||
- Who decides whether a volume is `in-use`?
|
||||
An implementation may track attachments itself, or derive the state from whether a publication claim is held.
|
||||
|
||||
#### CreateVolume
|
||||
|
||||
- May the capacity be anywhere in the requested range, as CSI allows?
|
||||
An implementation that rounds up would then fail the comparison that tells a retry from a conflicting request.
|
||||
- May seed data be combined with a source snapshot?
|
||||
The Viperblock implementation refuses, the memory implementation accepts.
|
||||
- Must the source snapshot be completed?
|
||||
AWS refuses one that is not.
|
||||
- Must an implementation store the availability zone, parameters and source snapshot, so that it can detect a conflicting retry?
|
||||
The Viperblock implementation does not.
|
||||
- Should a retry be compared against the original request rather than the current volume?
|
||||
A retry after ExpandVolume conflicts on capacity.
|
||||
- Should a retry with a different source snapshot volume ID conflict?
|
||||
It is not compared.
|
||||
- Is byte equality the meaning of equal parameters, or may an implementation compare them semantically?
|
||||
- Should `MaxSeedBytes` be modelled?
|
||||
Seed data larger than it is refused with `invalid_argument`.
|
||||
|
||||
#### GetVolume
|
||||
|
||||
- Must an implementation compare the handle?
|
||||
The Viperblock implementation ignores it.
|
||||
- Is `not_found` the right error for a volume that was deleted and recreated under the same ID?
|
||||
|
||||
#### ListVolumes
|
||||
|
||||
- Must following next tokens list every volume that exists throughout the walk exactly once?
|
||||
The conformance suite tests it, but the specification describes each call alone, so it does not say how one page relates to the next.
|
||||
- What should a starting token that the implementation never issued do?
|
||||
Every implementation treats any token as a position to resume after, while CSI refuses an invalid one with `ABORTED`.
|
||||
- May a page hold fewer refs than requested yet carry a next token?
|
||||
AWS and CSI allow it, but here a next token comes only with a full page, so that a walk cannot stall on empty pages.
|
||||
|
||||
#### ExpandVolume
|
||||
|
||||
- Should a request below the current capacity succeed without change, as CSI recommends, rather than be refused with `invalid_argument`?
|
||||
It is refused here, as AWS refuses `ModifyVolume` to a smaller size and the conformance suite tests, while CSI says the plugin should reply OK when the volume is already at least the requested size.
|
||||
- Must a block export's size follow an online expansion?
|
||||
A published volume keeps its export's `size_bytes`, so after it grows the export reports the old size.
|
||||
|
||||
#### PublishVolume
|
||||
|
||||
- Should `volume_in_use` depend on the exclusion scope?
|
||||
An implementation whose scope is `none` lets two writers in, and one whose scope is `node` cannot see another node, yet both are refused here.
|
||||
- Must a republish to the same node with a different `read_only` be refused?
|
||||
The Viperblock implementation refuses it with `volume_in_use`, while the memory implementation ignores the flag.
|
||||
- May two volumes on one node share an NBD URI or a device path?
|
||||
- Must a block export's size equal the volume's capacity?
|
||||
|
||||
#### UnpublishVolume
|
||||
|
||||
- Must an implementation compare the handle?
|
||||
Both the Viperblock and memory implementations ignore it.
|
||||
- Is a provider that fails to seal the volume a failed provider?
|
||||
The Viperblock implementation answers `internal` and leaves the volume published, which the specification allows only when the provider may fail.
|
||||
- Should a claim that lapses through its TTL be modelled, as a step that happens without a call?
|
||||
- Should a volume that does not exist be refused with `not_found`?
|
||||
It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DetachVolume` on a missing volume with `InvalidVolume.NotFound`.
|
||||
|
||||
#### DeleteVolume
|
||||
|
||||
- Must an implementation compare the handle before refusing a published volume?
|
||||
The memory implementation does, and succeeds without deleting it, while the Viperblock implementation ignores the handle and refuses with `volume_in_use`.
|
||||
- Should a volume that does not exist be refused with `not_found`?
|
||||
It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DeleteVolume` on a missing volume with `InvalidVolume.NotFound`.
|
||||
|
||||
### Snapshots
|
||||
|
||||
- Should a pending snapshot's move to `completed` or `error` be modelled, as a step that happens without a call?
|
||||
Without it a pending snapshot never completes, and no snapshot is ever in `error`.
|
||||
|
||||
#### CreateSnapshot
|
||||
|
||||
- May success return a `pending` snapshot?
|
||||
AWS starts a snapshot as `pending`, while the memory implementation returns `completed` and the NATS implementation waits for completion and refuses anything else.
|
||||
- Must a retry compare the volume handle?
|
||||
It is ignored, as in the memory implementation, so a volume deleted and recreated under the same ID returns the old snapshot.
|
||||
- Should a copy's ID count as a retry?
|
||||
A copy keeps its source's volume ID, so creating a snapshot of that volume under the copy's ID returns the copy.
|
||||
- Should `crash_consistent_snapshot` be modelled?
|
||||
The specification does not model volume data, so it cannot say what a snapshot of a published volume holds.
|
||||
|
||||
#### CopySnapshot
|
||||
|
||||
- Should a retried copy return the existing copy, as a retried CreateVolume does, rather than `already_exists`?
|
||||
- May a source that is not completed be copied?
|
||||
AWS refuses until it is.
|
||||
- What state does a copy start in?
|
||||
The memory implementation always returns `completed`, while AWS starts a copy as `pending`.
|
||||
|
||||
#### ListSnapshots
|
||||
|
||||
- The questions for ListVolumes apply here too.
|
||||
- Must a ref carry the source volume ID?
|
||||
The Viperblock implementation leaves it empty, since reading it costs a fetch per snapshot, yet reconciliation needs it to act on a snapshot whose metadata is lost.
|
||||
|
||||
#### DeleteSnapshot
|
||||
|
||||
- Is the handle check part of the contract?
|
||||
The Viperblock implementation ignores the handle and deletes anyway.
|
||||
- May a pending snapshot be deleted?
|
||||
AWS allows it.
|
||||
- What does success promise?
|
||||
The Go docs say the snapshot is gone, but a snapshot whose handle did not match is left in place.
|
||||
- Should a snapshot that does not exist be refused with `not_found`?
|
||||
It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DeleteSnapshot` on a missing snapshot with `InvalidSnapshot.NotFound`.
|
||||
@@ -1,159 +0,0 @@
|
||||
-------------------------------------------------- MODULE Snapshots ---------------------------------------------------
|
||||
(*********************************************************************************************************************)
|
||||
(* The lifecycle of a snapshot, from its creation to its deletion. *)
|
||||
(*********************************************************************************************************************)
|
||||
EXTENDS EBSProvider
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Interface methods
|
||||
|
||||
CreateSnapshotRequests == [snapshot_id : Optional(SnapshotIDs),
|
||||
volume_id : Optional(VolumeIDs),
|
||||
volume_handle : Optional(VolumeHandles)]
|
||||
|
||||
CreateSnapshot(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume's data is frozen as a snapshot, from which new volumes can be restored. A retry of a call that *)
|
||||
(* already succeeded returns the same snapshot. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET snapID == req.snapshot_id
|
||||
volID == req.volume_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> None \in {snapID, volID},
|
||||
already_exists |-> /\ KeyExists(snapshots, snapID)
|
||||
/\ snapshots[snapID].source_volume_id /= volID,
|
||||
\* Only checked for a new snapshot, so a retry outlives the volume.
|
||||
not_found |-> /\ ~KeyExists(snapshots, snapID)
|
||||
/\ \/ ~KeyExists(volumes, volID)
|
||||
\/ req.volume_handle \notin {None, volumes[volID].handle}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* A retry returns the existing snapshot.
|
||||
/\ \/ /\ KeyExists(snapshots, snapID)
|
||||
/\ UNCHANGED snapshots
|
||||
|
||||
\/ /\ ~KeyExists(snapshots, snapID)
|
||||
\* The handle, creation time and state are the implementation's choice.
|
||||
/\ \E h \in SnapshotHandles \ {snapshots[x].handle : x \in DOMAIN snapshots},
|
||||
t \in Timestamps,
|
||||
s \in {"pending", "completed"} :
|
||||
snapshots' = InsertKey(snapshots, snapID, [size_bytes |-> volumes[volID].capacity_bytes,
|
||||
created_at |-> t,
|
||||
state |-> s,
|
||||
handle |-> h,
|
||||
source_volume_id |-> volID])
|
||||
/\ val = SnapshotView(snapID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<volumes, publications>>
|
||||
|
||||
|
||||
CopySnapshotRequests == [source_snapshot_id : Optional(SnapshotIDs),
|
||||
destination_snapshot_id : Optional(SnapshotIDs),
|
||||
volume_id : Optional(VolumeIDs)]
|
||||
|
||||
CopySnapshot(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A snapshot is copied under a new ID, sharing its frozen data, and the copy can be deleted independently *)
|
||||
(* of it. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET srcSnapID == req.source_snapshot_id
|
||||
dstSnapID == req.destination_snapshot_id
|
||||
volID == req.volume_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> \/ None \in {srcSnapID, dstSnapID, volID}
|
||||
\/ srcSnapID = dstSnapID
|
||||
\* Only compared, not looked up, since a snapshot outlives its volume.
|
||||
\/ /\ KeyExists(snapshots, srcSnapID)
|
||||
/\ snapshots[srcSnapID].source_volume_id /= volID,
|
||||
already_exists |-> KeyExists(snapshots, dstSnapID),
|
||||
not_found |-> srcSnapID \notin Optional(DOMAIN snapshots)]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* The handle and creation time are the implementation's choice.
|
||||
/\ \E h \in SnapshotHandles \ {snapshots[x].handle : x \in DOMAIN snapshots},
|
||||
t \in Timestamps :
|
||||
snapshots' = InsertKey(snapshots, dstSnapID, [created_at |-> t,
|
||||
handle |-> h]
|
||||
@@ snapshots[srcSnapID])
|
||||
/\ val = SnapshotView(dstSnapID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<volumes, publications>>
|
||||
|
||||
|
||||
ListSnapshotsRequests == [max_results : -1..(MaxListResults + 1),
|
||||
starting_token : Optional(ListTokens)]
|
||||
|
||||
ListSnapshots(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* Some of the snapshots are listed, a page at a time, and nothing changes. A call with no starting token that *)
|
||||
(* returns no next token lists every snapshot. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET Errors ==
|
||||
LET checks == [unsupported_capability |-> ~Capabilities.snapshot_enumeration]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ LET refs == {[id |-> id,
|
||||
source_volume_id |-> snapshots[id].source_volume_id,
|
||||
handle |-> snapshots[id].handle] : id \in DOMAIN snapshots}
|
||||
\* The page and the next token are the implementation's choice.
|
||||
IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
|
||||
/\ Cardinality(page) <= PageSize(req.max_results)
|
||||
/\ next /= None => Cardinality(page) = PageSize(req.max_results)
|
||||
/\ req.starting_token = None /\ next = None => page = refs
|
||||
/\ val = [snapshots |-> page, next_token |-> next]
|
||||
/\ err = None
|
||||
/\ UNCHANGED vars
|
||||
|
||||
|
||||
DeleteSnapshotRequests == [snapshot_id : Optional(SnapshotIDs),
|
||||
handle : Optional(SnapshotHandles)]
|
||||
|
||||
DeleteSnapshot(req, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A snapshot is deleted, so that no new volume can be restored from it. Volumes already restored from it, *)
|
||||
(* and copies of it, are unaffected. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET snapID == req.snapshot_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> snapID = None]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ snapshots' = IF /\ KeyExists(snapshots, snapID)
|
||||
/\ req.handle \notin {None, snapshots[snapID].handle}
|
||||
THEN snapshots
|
||||
\* Record removal is idempotent so that retries succeed.
|
||||
ELSE RemoveKey(snapshots, snapID)
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<volumes, publications>>
|
||||
|
||||
=======================================================================================================================
|
||||
@@ -1,286 +0,0 @@
|
||||
--------------------------------------------------- MODULE Volumes ----------------------------------------------------
|
||||
(*********************************************************************************************************************)
|
||||
(* The lifecycle of a volume, from its creation to its deletion. *)
|
||||
(*********************************************************************************************************************)
|
||||
EXTENDS EBSProvider
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Interface methods
|
||||
|
||||
CreateVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
capacity_range : CapacityRanges,
|
||||
availability_zone : Optional(AvailabilityZones),
|
||||
source_snapshot_id : Optional(SnapshotIDs),
|
||||
source_snapshot_volume_id : Optional(VolumeIDs),
|
||||
parameters : Optional(Parameters),
|
||||
seed_data : Bytes]
|
||||
|
||||
CreateVolume(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume of the requested capacity is created, either blank or holding a snapshot's data, optionally *)
|
||||
(* with seed data written at its start. A retry of a call that already succeeded returns the same volume. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
snapID == req.source_snapshot_id
|
||||
snapVolID == req.source_snapshot_volume_id
|
||||
capacity == req.capacity_range
|
||||
|
||||
\* Any warranted error may be returned, since the order of checks is not part of the contract.
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> \/ volID = None
|
||||
\/ capacity.required_bytes <= 0
|
||||
\/ /\ capacity.limit_bytes > 0
|
||||
/\ capacity.required_bytes > capacity.limit_bytes
|
||||
\/ req.seed_data > capacity.required_bytes
|
||||
\/ snapID /= None /\ snapVolID = None
|
||||
\* Only checked for a new volume, so a retry outlives the snapshot.
|
||||
\/ /\ ~KeyExists(volumes, volID)
|
||||
/\ KeyExists(snapshots, snapID)
|
||||
/\ \/ capacity.required_bytes < snapshots[snapID].size_bytes
|
||||
\/ snapshots[snapID].source_volume_id /= snapVolID,
|
||||
\* Seed data is not compared, since it only applies on creation.
|
||||
already_exists |-> /\ KeyExists(volumes, volID)
|
||||
/\ \/ volumes[volID].capacity_bytes /= capacity.required_bytes
|
||||
\/ volumes[volID].availability_zone /= req.availability_zone
|
||||
\/ volumes[volID].source_snapshot_id /= snapID
|
||||
\/ volumes[volID].parameters /= req.parameters,
|
||||
not_found |-> /\ ~KeyExists(volumes, volID)
|
||||
/\ snapID \notin Optional(DOMAIN snapshots),
|
||||
unsupported_capability |-> /\ req.seed_data > 0
|
||||
/\ ~Capabilities.volume_seeding]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* A retry returns the existing volume.
|
||||
/\ \/ /\ KeyExists(volumes, volID)
|
||||
/\ UNCHANGED volumes
|
||||
|
||||
\/ /\ ~KeyExists(volumes, volID)
|
||||
\* The handle and provider data are the implementation's choice.
|
||||
/\ \E h \in VolumeHandles \ {volumes[x].handle : x \in DOMAIN volumes},
|
||||
pd \in Optional(ProviderData) :
|
||||
volumes' = InsertKey(volumes, volID, [capacity_bytes |-> capacity.required_bytes,
|
||||
state |-> "available",
|
||||
handle |-> h,
|
||||
availability_zone |-> req.availability_zone,
|
||||
source_snapshot_id |-> snapID,
|
||||
parameters |-> req.parameters,
|
||||
provider_data |-> pd])
|
||||
/\ val = VolumeView(volID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<publications, snapshots>>
|
||||
|
||||
|
||||
GetVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles)]
|
||||
|
||||
GetVolume(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume's current capacity, state and handle are read, and nothing changes. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> volID = None,
|
||||
not_found |-> \/ ~KeyExists(volumes, volID)
|
||||
\/ req.handle \notin {None, volumes[volID].handle}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ val = VolumeView(volID)
|
||||
/\ err = None
|
||||
/\ UNCHANGED vars
|
||||
|
||||
|
||||
ListVolumesRequests == [max_results : -1..(MaxListResults + 1),
|
||||
starting_token : Optional(ListTokens)]
|
||||
|
||||
ListVolumes(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* Some of the volumes are listed, a page at a time, and nothing changes. A call with no starting token that *)
|
||||
(* returns no next token lists every volume. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET Errors ==
|
||||
LET checks == [unsupported_capability |-> ~Capabilities.volume_enumeration]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ LET refs == {[id |-> id, handle |-> volumes[id].handle] : id \in DOMAIN volumes}
|
||||
\* The page and the next token are the implementation's choice.
|
||||
IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
|
||||
/\ Cardinality(page) <= PageSize(req.max_results)
|
||||
/\ next /= None => Cardinality(page) = PageSize(req.max_results)
|
||||
/\ req.starting_token = None /\ next = None => page = refs
|
||||
/\ val = [volumes |-> page, next_token |-> next]
|
||||
/\ err = None
|
||||
/\ UNCHANGED vars
|
||||
|
||||
|
||||
ExpandVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles),
|
||||
capacity_range : CapacityRanges]
|
||||
|
||||
ExpandVolume(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume grows to the requested capacity, keeping its data. A published volume grows only if the *)
|
||||
(* implementation supports online expansion, and a request for the current capacity succeeds without change. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
capacity == req.capacity_range
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> \/ volID = None
|
||||
\/ capacity.required_bytes = 0
|
||||
\/ /\ capacity.limit_bytes > 0
|
||||
/\ capacity.required_bytes > capacity.limit_bytes
|
||||
\/ /\ KeyExists(volumes, volID)
|
||||
/\ capacity.required_bytes < volumes[volID].capacity_bytes,
|
||||
not_found |-> \/ ~KeyExists(volumes, volID)
|
||||
\/ req.handle \notin {None, volumes[volID].handle},
|
||||
volume_in_use |-> /\ PublishedNodes(volID) /= {}
|
||||
/\ ~Capabilities.online_expansion
|
||||
/\ capacity.required_bytes > volumes[volID].capacity_bytes]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
/\ volumes' = [volumes EXCEPT ![volID].capacity_bytes = capacity.required_bytes]
|
||||
/\ val = VolumeView(volID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<publications, snapshots>>
|
||||
|
||||
|
||||
PublishVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles),
|
||||
node_id : Optional(NodeIDs),
|
||||
read_only : BOOLEAN]
|
||||
|
||||
PublishVolume(req, val, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume's data becomes reachable on a node, through an export that a VM there can attach, either an NBD *)
|
||||
(* URI or a host block device. Reads and writes through the export reach the volume, and a read-only export *)
|
||||
(* refuses writes. Only one node at a time may reach a volume. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
nodeID == req.node_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> None \in {volID, nodeID},
|
||||
not_found |-> \/ ~KeyExists(volumes, volID)
|
||||
\/ req.handle \notin {None, volumes[volID].handle},
|
||||
unsupported_capability |-> /\ req.read_only
|
||||
/\ ~Capabilities.read_only_publish,
|
||||
volume_in_use |-> PublishedNodes(volID) \ {nodeID} /= {}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ val = None
|
||||
/\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* A retry returns the existing publication.
|
||||
/\ \/ /\ KeyExists(publications, <<volID, nodeID>>)
|
||||
/\ UNCHANGED publications
|
||||
|
||||
\/ /\ ~KeyExists(publications, <<volID, nodeID>>)
|
||||
\* The type of export is the implementation's choice.
|
||||
/\ \E x \in Exports :
|
||||
publications' = InsertKey(publications, <<volID, nodeID>>, [export |-> x,
|
||||
read_only |-> req.read_only])
|
||||
/\ volumes' = [volumes EXCEPT ![volID].state = "in-use"]
|
||||
/\ val = PublicationView(volID, nodeID)'
|
||||
/\ err = None
|
||||
/\ UNCHANGED snapshots
|
||||
|
||||
|
||||
UnpublishVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles),
|
||||
node_id : Optional(NodeIDs)]
|
||||
|
||||
UnpublishVolume(req, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume's export on a node is torn down, so that its data is no longer reachable there. Once no node *)
|
||||
(* can reach the volume, it is available again. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
nodeID == req.node_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> None \in {volID, nodeID}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* Record removal is idempotent so that retries succeed.
|
||||
/\ publications' = RemoveKey(publications, <<volID, nodeID>>)
|
||||
\* The volume is available again once no node holds it.
|
||||
/\ volumes' = IF \/ ~KeyExists(volumes, volID)
|
||||
\/ PublishedNodes(volID)' /= {}
|
||||
THEN volumes
|
||||
ELSE [volumes EXCEPT ![volID].state = "available"]
|
||||
/\ err = None
|
||||
/\ UNCHANGED snapshots
|
||||
|
||||
|
||||
DeleteVolumeRequests == [volume_id : Optional(VolumeIDs),
|
||||
handle : Optional(VolumeHandles)]
|
||||
|
||||
DeleteVolume(req, err) ==
|
||||
(*******************************************************************************************************************)
|
||||
(* A volume is deleted, so that its data is gone. Snapshots taken from it are unaffected. *)
|
||||
(*******************************************************************************************************************)
|
||||
LET volID == req.volume_id
|
||||
|
||||
Errors ==
|
||||
LET checks == [invalid_argument |-> volID = None,
|
||||
volume_in_use |-> /\ PublishedNodes(volID) /= {}
|
||||
/\ req.handle \in {None, volumes[volID].handle}]
|
||||
IN {e \in DOMAIN checks : checks[e]}
|
||||
|
||||
\* The request was rejected, or the provider failed to answer.
|
||||
IN \/ /\ err \in Errors \union ProviderErrors
|
||||
/\ UNCHANGED vars
|
||||
|
||||
\* The request was accepted.
|
||||
\/ /\ Errors = {}
|
||||
\* Record removal is idempotent so that retries succeed.
|
||||
/\ volumes' = IF /\ KeyExists(volumes, volID)
|
||||
/\ req.handle \notin {None, volumes[volID].handle}
|
||||
THEN volumes
|
||||
ELSE RemoveKey(volumes, volID)
|
||||
/\ err = None
|
||||
/\ UNCHANGED <<publications, snapshots>>
|
||||
|
||||
-----------------------------------------------------------------------------------------------------------------------
|
||||
\* Specification
|
||||
|
||||
=======================================================================================================================
|
||||
@@ -83,10 +83,10 @@ func (a *Agent) reconcile(ctx context.Context) map[string]bool {
|
||||
statuses := make([]ecsv1.ContainerStatus, 0, len(t.running))
|
||||
for _, c := range t.running {
|
||||
name := c.Labels[labelContainerName]
|
||||
go a.waitContainer(ctx, t.as, name, c.ID)
|
||||
statuses = append(statuses, ecsv1.ContainerStatus{Name: name, Status: ecsv1.TaskStatusRunning, ContainerID: c.ID})
|
||||
}
|
||||
a.reportTaskState(t.as, ecsv1.TaskStatusRunning, "", statuses)
|
||||
a.waitContainers(ctx, t.as, statuses)
|
||||
adopted[taskID] = true
|
||||
slog.Info("ecs-agent: re-adopted running task", "task", taskID, "containers", len(t.running))
|
||||
}
|
||||
|
||||
@@ -191,7 +191,6 @@ func (a *Agent) runTask(ctx context.Context, as *ecsv1.Assign) {
|
||||
slog.Error("ecs-agent: pull failed", "task", as.TaskID, "image", c.Image, "err", err)
|
||||
a.teardownTaskNetns(as)
|
||||
a.reportTaskState(as, ecsv1.TaskStatusStopped, "image pull failed: "+err.Error(), statuses)
|
||||
a.waitContainers(ctx, as, statuses)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -224,28 +223,18 @@ func (a *Agent) runTask(ctx context.Context, as *ecsv1.Assign) {
|
||||
slog.Error("ecs-agent: run failed", "task", as.TaskID, "container", c.Name, "err", err)
|
||||
a.teardownTaskNetns(as)
|
||||
a.reportTaskState(as, ecsv1.TaskStatusStopped, "container start failed: "+err.Error(), statuses)
|
||||
a.waitContainers(ctx, as, statuses)
|
||||
return
|
||||
}
|
||||
statuses = append(statuses, ecsv1.ContainerStatus{
|
||||
Name: c.Name, Status: ecsv1.TaskStatusRunning, ContainerID: id, GPUIDs: gpuIDs,
|
||||
})
|
||||
go a.waitContainer(ctx, as, c.Name, id)
|
||||
}
|
||||
|
||||
a.reportTaskState(as, ecsv1.TaskStatusRunning, "", statuses)
|
||||
a.waitContainers(ctx, as, statuses)
|
||||
slog.Info("ecs-agent: task running", "task", as.TaskID, "containers", len(statuses))
|
||||
}
|
||||
|
||||
// waitContainers starts one exit-wait per started container. Callers start
|
||||
// them only after the report that precedes STOPPED is sent, so a container
|
||||
// that exits at once can never have its STOPPED overtaken by that report.
|
||||
func (a *Agent) waitContainers(ctx context.Context, as *ecsv1.Assign, started []ecsv1.ContainerStatus) {
|
||||
for _, c := range started {
|
||||
go a.waitContainer(ctx, as, c.Name, c.ContainerID)
|
||||
}
|
||||
}
|
||||
|
||||
// waitContainer blocks until a container exits, then reports the task STOPPED
|
||||
// with the exit code. v1 stops the whole task when its first container exits.
|
||||
func (a *Agent) waitContainer(ctx context.Context, as *ecsv1.Assign, name, containerID string) {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -824,87 +823,3 @@ func TestPullResolver(t *testing.T) {
|
||||
t.Fatalf("execution role + cred endpoint: want distinct exec-role resolver, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// orderingCP fails the run if a container exit-wait starts before the RUNNING
|
||||
// report is sent: an instant exit's STOPPED could then be overtaken by RUNNING.
|
||||
type orderingCP struct {
|
||||
*fakeCP
|
||||
|
||||
waitStarted chan struct{}
|
||||
early atomic.Bool
|
||||
}
|
||||
|
||||
func newOrderingCP(rt *ctrruntime.FakePuller) *orderingCP {
|
||||
cp := &orderingCP{fakeCP: &fakeCP{}, waitStarted: make(chan struct{}, 8)}
|
||||
rt.OnWait = func(string) { cp.waitStarted <- struct{}{} }
|
||||
return cp
|
||||
}
|
||||
|
||||
func (c *orderingCP) SubmitTaskState(st ecsv1.TaskState) error {
|
||||
if st.LastStatus == ecsv1.TaskStatusRunning {
|
||||
// A wait launched before this report is scheduled well within the window.
|
||||
select {
|
||||
case <-c.waitStarted:
|
||||
c.early.Store(true)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
return c.fakeCP.SubmitTaskState(st)
|
||||
}
|
||||
|
||||
func waitStopped(t *testing.T, cp *orderingCP, taskID string) {
|
||||
t.Helper()
|
||||
deadline := time.After(5 * time.Second)
|
||||
for countStatus(cp.taskStates(), taskID, ecsv1.TaskStatusStopped) == 0 {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("task %s never reported STOPPED; states %+v", taskID, cp.taskStates())
|
||||
case <-time.After(time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertRunningThenStopped(t *testing.T, cp *orderingCP, taskID string) {
|
||||
t.Helper()
|
||||
if cp.early.Load() {
|
||||
t.Fatalf("exit-wait started before the RUNNING report; states %+v", cp.taskStates())
|
||||
}
|
||||
var seq []string
|
||||
for _, st := range cp.taskStates() {
|
||||
if st.TaskID == taskID {
|
||||
seq = append(seq, st.LastStatus)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(seq, []string{ecsv1.TaskStatusRunning, ecsv1.TaskStatusStopped}) {
|
||||
t.Fatalf("task %s reported %v, want [RUNNING STOPPED]", taskID, seq)
|
||||
}
|
||||
}
|
||||
|
||||
// A container that exits the instant it starts must not let RUNNING be
|
||||
// reported after its STOPPED: the gateway would revive a dead task.
|
||||
func TestRunTask_InstantExitStoppedNeverPrecedesRunning(t *testing.T) {
|
||||
rt := &ctrruntime.FakePuller{WaitCode: 7}
|
||||
cp := newOrderingCP(rt)
|
||||
a := newRunAgent(cp, rt)
|
||||
a.runTask(context.Background(), testAssign())
|
||||
|
||||
waitStopped(t, cp, "t-001")
|
||||
assertRunningThenStopped(t, cp, "t-001")
|
||||
}
|
||||
|
||||
// The re-adopt path has the same ordering rule for a container that exits
|
||||
// right after the agent re-attaches to it.
|
||||
func TestReconcile_InstantExitStoppedNeverPrecedesRunning(t *testing.T) {
|
||||
rt := &ctrruntime.FakePuller{
|
||||
Containers: []ctrruntime.Container{
|
||||
adoptedContainer("t-live", "web", "default", "t-live", testTaskRole, "", true),
|
||||
},
|
||||
}
|
||||
cp := newOrderingCP(rt)
|
||||
a := newAgent(Config{}, testIdentity(), cp, rt, nil)
|
||||
a.cred = newCredEndpoint(nil, "us-east-1", "https://gw", "", "127.0.0.1", 0, nil)
|
||||
a.reconcile(context.Background())
|
||||
|
||||
waitStopped(t, cp, "t-live")
|
||||
assertRunningThenStopped(t, cp, "t-live")
|
||||
}
|
||||
|
||||
@@ -26,7 +26,6 @@ type FakePuller struct {
|
||||
Waited []string
|
||||
Removed []string
|
||||
OnRun func(string, RunSpec) (string, error)
|
||||
OnWait func(containerID string) // called, unlocked, before Wait returns
|
||||
|
||||
// Stop bookkeeping: StopTimeouts pairs the container ID with the timeout it
|
||||
// was stopped with, in call order, so a test can assert both the target and
|
||||
@@ -106,11 +105,7 @@ func (f *FakePuller) Run(_ context.Context, id string, spec RunSpec) (string, er
|
||||
func (f *FakePuller) Wait(_ context.Context, containerID string) (RunStatus, error) {
|
||||
f.mu.Lock()
|
||||
f.Waited = append(f.Waited, containerID)
|
||||
onWait := f.OnWait
|
||||
f.mu.Unlock()
|
||||
if onWait != nil {
|
||||
onWait(containerID)
|
||||
}
|
||||
return RunStatus{ExitCode: f.WaitCode}, f.WaitErr
|
||||
}
|
||||
|
||||
|
||||
@@ -624,20 +624,6 @@ func (c Config) HasService(name string) bool {
|
||||
return slices.Contains(services, name)
|
||||
}
|
||||
|
||||
// NodesRunning returns, sorted, the name of every configured node that runs
|
||||
// service. It is the single definition of "which nodes run X" for callers that
|
||||
// must reach every process of one kind.
|
||||
func (cc *ClusterConfig) NodesRunning(service string) []string {
|
||||
var nodes []string
|
||||
for name, node := range cc.Nodes {
|
||||
if node.HasService(service) {
|
||||
nodes = append(nodes, name)
|
||||
}
|
||||
}
|
||||
slices.Sort(nodes)
|
||||
return nodes
|
||||
}
|
||||
|
||||
// GetServices returns the configured service list, defaulting to AllServices.
|
||||
func (c Config) GetServices() []string {
|
||||
if len(c.Services) == 0 {
|
||||
|
||||
@@ -1550,16 +1550,3 @@ func TestLoadConfig_NorthstarNameservers(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesRunning(t *testing.T) {
|
||||
c := ClusterConfig{Nodes: map[string]Config{
|
||||
"b": {Services: []string{"nats", "awsgw"}},
|
||||
"a": {Services: []string{"daemon"}},
|
||||
"c": {},
|
||||
"d": {Services: []string{"awsgw", "daemon"}},
|
||||
}}
|
||||
// An empty service list means every service, so c runs awsgw too.
|
||||
assert.Equal(t, []string{"b", "c", "d"}, c.NodesRunning("awsgw"))
|
||||
assert.Equal(t, []string{"a", "c", "d"}, c.NodesRunning("daemon"))
|
||||
assert.Empty(t, c.NodesRunning("nonesuch"))
|
||||
}
|
||||
|
||||
@@ -68,19 +68,6 @@ func (f *faultyStore) DeleteObject(ctx context.Context, in *s3.DeleteObjectInput
|
||||
return f.MemoryObjectStore.DeleteObject(ctx, in)
|
||||
}
|
||||
|
||||
// ListObjectsV2 returns keys in S3's lexicographic order, so a test can place
|
||||
// one record ahead of another in a scan.
|
||||
func (f *faultyStore) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
out, err := f.MemoryObjectStore.ListObjectsV2(ctx, in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.SortFunc(out.Contents, func(a, b *s3.Object) int {
|
||||
return strings.Compare(aws.StringValue(a.Key), aws.StringValue(b.Key))
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// accountObjects lists every object key under the account's key prefix.
|
||||
func accountObjects(t *testing.T, store objectstore.ObjectStore, account string) []string {
|
||||
t.Helper()
|
||||
@@ -254,22 +241,6 @@ func TestAccountTermination_DeleteKeyPairByNameFailsOnUndecodableMetadata(t *tes
|
||||
assert.Equal(t, sortedStrings(broken, pub), accountObjects(t, store, ctAccount))
|
||||
}
|
||||
|
||||
// An unreadable record listed ahead of the named key pair cannot hide it: the
|
||||
// match proves the name present, so the delete goes ahead.
|
||||
func TestAccountTermination_DeleteKeyPairByNameSucceedsPastUndecodableMetadata(t *testing.T) {
|
||||
store := newFaultyStore()
|
||||
svc := ec2key.NewKeyServiceImplWithStore(store, ctBucket)
|
||||
broken := "keys/" + ctAccount + "/key-000000000000000000.json"
|
||||
putObject(t, store, broken, "{not json")
|
||||
id := aws.StringValue(importKey(t, svc, ctAccount, "deploy").KeyPairId)
|
||||
require.Less(t, broken, "keys/"+ctAccount+"/"+id+".json", "the broken record must be scanned first")
|
||||
|
||||
out, err := svc.DeleteKeyPair(context.Background(), &ec2.DeleteKeyPairInput{KeyName: aws.String("deploy")}, ctAccount)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, id, aws.StringValue(out.KeyPairId))
|
||||
assert.Equal(t, []string{broken}, accountObjects(t, store, ctAccount))
|
||||
}
|
||||
|
||||
func sortedStrings(s ...string) []string {
|
||||
slices.Sort(s)
|
||||
return s
|
||||
|
||||
@@ -482,9 +482,7 @@ func (s *KeyServiceImpl) findKeyPairIdFromKeyName(ctx context.Context, accountID
|
||||
return "", fmt.Errorf("failed to list objects: %w", err)
|
||||
}
|
||||
|
||||
// An unreadable record may be this name's key pair, so it makes absence
|
||||
// unproven, but it cannot hide a match found elsewhere in the listing.
|
||||
var unreadable error
|
||||
// Check each .json metadata file
|
||||
for _, obj := range objects {
|
||||
if obj == nil || obj.Key == nil {
|
||||
continue
|
||||
@@ -505,11 +503,9 @@ func (s *KeyServiceImpl) findKeyPairIdFromKeyName(ctx context.Context, accountID
|
||||
if objectstore.IsNoSuchKeyError(err) {
|
||||
continue
|
||||
}
|
||||
// Unread metadata may be the key pair asked for, so absence is unproven.
|
||||
slog.ErrorContext(ctx, "Failed to get metadata file", "key", *obj.Key, "err", err)
|
||||
if unreadable == nil {
|
||||
unreadable = fmt.Errorf("read metadata %s: %w", *obj.Key, err)
|
||||
}
|
||||
continue
|
||||
return "", fmt.Errorf("read metadata %s: %w", *obj.Key, err)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(getResult.Body)
|
||||
@@ -518,19 +514,14 @@ func (s *KeyServiceImpl) findKeyPairIdFromKeyName(ctx context.Context, accountID
|
||||
}
|
||||
if err != nil {
|
||||
slog.ErrorContext(ctx, "Failed to read metadata body", "key", *obj.Key, "err", err)
|
||||
if unreadable == nil {
|
||||
unreadable = fmt.Errorf("read metadata %s: %w", *obj.Key, err)
|
||||
}
|
||||
continue
|
||||
return "", fmt.Errorf("read metadata %s: %w", *obj.Key, err)
|
||||
}
|
||||
|
||||
metadata, err := decodeKeyPairMetadata(body)
|
||||
if err != nil {
|
||||
// Undecodable metadata may be this name's key pair, so absence is unproven.
|
||||
slog.ErrorContext(ctx, "Undecodable key-pair metadata", "key", *obj.Key, "err", err)
|
||||
if unreadable == nil {
|
||||
unreadable = fmt.Errorf("decode metadata %s: %w", *obj.Key, err)
|
||||
}
|
||||
continue
|
||||
return "", fmt.Errorf("decode metadata %s: %w", *obj.Key, err)
|
||||
}
|
||||
|
||||
// Check if this metadata matches the keyName
|
||||
@@ -540,9 +531,6 @@ func (s *KeyServiceImpl) findKeyPairIdFromKeyName(ctx context.Context, accountID
|
||||
}
|
||||
}
|
||||
}
|
||||
if unreadable != nil {
|
||||
return "", unreadable
|
||||
}
|
||||
|
||||
// Key pair not found
|
||||
return "", errors.New(awserrors.ErrorInvalidKeyPairNotFound)
|
||||
|
||||
@@ -100,9 +100,6 @@ func TestCreateNodegroup_ConcurrentSameNameSingleOwner(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
// The single winning owner launches one worker; its Ready-gate resolves.
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
|
||||
const n = 2
|
||||
errs := make([]error, n)
|
||||
var wg sync.WaitGroup
|
||||
@@ -114,6 +111,8 @@ func TestCreateNodegroup_ConcurrentSameNameSingleOwner(t *testing.T) {
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
// The single winning owner launches one worker; let its Ready-gate resolve.
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
ok, inUse := classifyCreateErrs(t, errs)
|
||||
|
||||
@@ -411,17 +411,9 @@ func TestClusterReconciler_ActiveProbesAndWarnsOnHealthzFail(t *testing.T) {
|
||||
require.NoError(t, SetClusterStatus(t.Context(), acctKV, "alpha", ClusterStatusActive))
|
||||
stub.setResponse(http.StatusServiceUnavailable, nil)
|
||||
|
||||
// Stopped once a pass has recorded the probe, not after a fixed window a
|
||||
// loaded runner can spend before the first pass lands.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ran := make(chan error, 1)
|
||||
go func() { ran <- r.Run(ctx) }()
|
||||
require.Eventually(t, func() bool {
|
||||
meta, err := GetClusterMeta(t.Context(), acctKV, "alpha")
|
||||
return err == nil && meta.HealthIssue != ""
|
||||
}, 10*time.Second, 5*time.Millisecond, "the run loop never recorded the failing probe")
|
||||
cancel()
|
||||
<-ran
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
_ = r.Run(ctx)
|
||||
|
||||
meta, err := GetClusterMeta(t.Context(), acctKV, "alpha")
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -25,9 +25,9 @@ func TestCreateNodegroup_RecreateDoesNotBaselineOnPriorWorkers(t *testing.T) {
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
// First incarnation: two workers register Ready and the nodegroup activates.
|
||||
markWorkersReady(t, f, "c1", "ng", 2)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng", 2), testAccountID)
|
||||
require.NoError(t, err)
|
||||
markWorkersReady(t, f, "c1", "ng", 2)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng")
|
||||
|
||||
@@ -343,8 +343,6 @@ func TestCreateNodegroup_HappyPath(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
// Both workers register Ready → the Ready-gate lets the nodegroup go ACTIVE.
|
||||
markWorkersReady(t, f, "c1", "ng1", 2)
|
||||
out, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 2), testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out.Nodegroup)
|
||||
@@ -358,6 +356,8 @@ func TestCreateNodegroup_HappyPath(t *testing.T) {
|
||||
arn.EKSNodegroupDiscriminator(testAccountID, "c1", "ng1")),
|
||||
aws.StringValue(out.Nodegroup.NodegroupArn))
|
||||
|
||||
// Both workers register Ready → the Ready-gate lets the nodegroup go ACTIVE.
|
||||
markWorkersReady(t, f, "c1", "ng1", 2)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
// The async launch transitions the record to ACTIVE once workers run.
|
||||
@@ -398,7 +398,6 @@ func TestCreateNodegroup_OmittedDiskSizeUsesAWSDefault(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
// createNGInput leaves DiskSize nil, which is the case being covered.
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
@@ -407,6 +406,7 @@ func TestCreateNodegroup_OmittedDiskSizeUsesAWSDefault(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(defaultNodegroupDiskSizeGiB), rec.DiskSize)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
// The default has to reach the launch, not just the record.
|
||||
@@ -427,7 +427,6 @@ func TestCreateNodegroup_GPUInstanceTypeSetsGPUFields(t *testing.T) {
|
||||
in := createNGInput("c1", "ng-gpu", 1)
|
||||
in.InstanceTypes = aws.StringSlice([]string{"g5.xlarge"})
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng-gpu", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -436,6 +435,7 @@ func TestCreateNodegroup_GPUInstanceTypeSetsGPUFields(t *testing.T) {
|
||||
assert.True(t, rec.GPUEnabled)
|
||||
assert.Equal(t, "nvidia", rec.GPUVendor)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng-gpu", 1)
|
||||
f.svc.WaitLaunches()
|
||||
}
|
||||
|
||||
@@ -449,10 +449,10 @@ func TestCreateNodegroup_GPUNodegroupStagesDevicePluginAddon(t *testing.T) {
|
||||
in := createNGInput("c1", "ng-gpu", 1)
|
||||
in.InstanceTypes = aws.StringSlice([]string{"g5.xlarge"})
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng-gpu", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng-gpu", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
rec, err := addon.Get(t.Context(), f.kv, "c1", addon.NvidiaDevicePlugin)
|
||||
@@ -467,10 +467,10 @@ func TestCreateNodegroup_NonGPUNodegroupDoesNotStageDevicePluginAddon(t *testing
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
_, err = addon.Get(t.Context(), f.kv, "c1", addon.NvidiaDevicePlugin)
|
||||
@@ -484,7 +484,6 @@ func TestCreateNodegroup_NonGPUInstanceTypeClearsGPUFields(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -493,6 +492,7 @@ func TestCreateNodegroup_NonGPUInstanceTypeClearsGPUFields(t *testing.T) {
|
||||
assert.False(t, rec.GPUEnabled)
|
||||
assert.Empty(t, rec.GPUVendor)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
}
|
||||
|
||||
@@ -507,7 +507,6 @@ func TestCreateNodegroup_GPUInstanceTypeExposesGPUAmiTypeInDescribe(t *testing.T
|
||||
in := createNGInput("c1", "ng-gpu", 1)
|
||||
in.InstanceTypes = aws.StringSlice([]string{"g5.xlarge"})
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng-gpu", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -518,6 +517,7 @@ func TestCreateNodegroup_GPUInstanceTypeExposesGPUAmiTypeInDescribe(t *testing.T
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, eks.AMITypesAl2X8664Gpu, aws.StringValue(out.Nodegroup.AmiType))
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng-gpu", 1)
|
||||
f.svc.WaitLaunches()
|
||||
}
|
||||
|
||||
@@ -528,7 +528,6 @@ func TestCreateNodegroup_NonGPUInstanceTypeKeepsPlainAmiTypeInDescribe(t *testin
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -539,6 +538,7 @@ func TestCreateNodegroup_NonGPUInstanceTypeKeepsPlainAmiTypeInDescribe(t *testin
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, eks.AMITypesAl2X8664, aws.StringValue(out.Nodegroup.AmiType))
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
}
|
||||
|
||||
@@ -561,10 +561,10 @@ func TestCreateNodegroup_GPUWorkerLaunchUsesGPUAMI(t *testing.T) {
|
||||
|
||||
in := createNGInput("c1", "ng-gpu", 1)
|
||||
in.InstanceTypes = aws.StringSlice([]string{"g5.xlarge"})
|
||||
markWorkersReady(t, f, "c1", "ng-gpu", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng-gpu", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng-gpu")
|
||||
@@ -634,16 +634,15 @@ func TestCreateNodegroup_ReadyGateScopedPerNodegroupNotGlobal(t *testing.T) {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
// Only ng-a's worker registers Ready; ng-b's worker never does. If the
|
||||
// Ready-gate were still scoped globally, the cluster-wide count (1) would
|
||||
// wrongly satisfy ng-b's gate too. Marked before the launch so ng-a's gate
|
||||
// never races the fixture's short Ready timeout.
|
||||
markWorkersReady(t, f, "c1", "ng-a", 1)
|
||||
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng-a", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng-b", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Only ng-a's worker registers Ready; ng-b's worker never does. If the
|
||||
// Ready-gate were still scoped globally, the cluster-wide count (1) would
|
||||
// wrongly satisfy ng-b's gate too.
|
||||
markWorkersReady(t, f, "c1", "ng-a", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
ngA, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng-a")
|
||||
@@ -670,10 +669,10 @@ func TestCreateNodegroup_RelaunchesOnWorkerLaunchFailure(t *testing.T) {
|
||||
f.worker.runErr = errors.New("qmp: timeout waiting for launch")
|
||||
f.worker.failNextN = 1
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
rec, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
|
||||
@@ -712,10 +711,10 @@ func TestCreateNodegroup_DiskSizePropagatesToBlockDeviceMapping(t *testing.T) {
|
||||
|
||||
in := createNGInput("c1", "ng1", 1)
|
||||
in.DiskSize = aws.Int64(30)
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
require.Len(t, f.worker.runCalls, 1)
|
||||
@@ -750,9 +749,9 @@ func TestDescribeAndListNodegroups(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
desc, err := f.svc.DescribeNodegroup(context.Background(), &eks.DescribeNodegroupInput{
|
||||
@@ -778,9 +777,9 @@ func TestUpdateNodegroupConfig_ScaleUpAndDown(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
require.Len(t, f.worker.runCalls, 1)
|
||||
|
||||
@@ -819,9 +818,9 @@ func TestUpdateNodegroupConfig_ConcurrentScaleUpConverges(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
require.Len(t, f.worker.runCalls, 1)
|
||||
|
||||
@@ -865,9 +864,9 @@ func TestUpdateNodegroupConfig_CapacityErrorSurfacesCode(t *testing.T) {
|
||||
scaleUpWithRunErr := func(t *testing.T, runErr error) error {
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
f.worker.runErr = runErr
|
||||
@@ -906,9 +905,9 @@ func TestDeleteNodegroup_TerminatesAndIsIdempotent(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
markWorkersReady(t, f, "c1", "ng1", 2)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 2), testAccountID)
|
||||
require.NoError(t, err)
|
||||
markWorkersReady(t, f, "c1", "ng1", 2)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
|
||||
@@ -1191,7 +1190,6 @@ func TestCreateNodegroup_CapturesLaunchTemplateCapacityTypeReleaseVersion(t *tes
|
||||
in.CapacityType = aws.String(eks.CapacityTypesSpot)
|
||||
in.ReleaseVersion = aws.String("1.32.0-20240307")
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1203,6 +1201,7 @@ func TestCreateNodegroup_CapturesLaunchTemplateCapacityTypeReleaseVersion(t *tes
|
||||
assert.Equal(t, eks.CapacityTypesSpot, rec.CapacityType)
|
||||
assert.Equal(t, "1.32.0-20240307", rec.ReleaseVersion)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
}
|
||||
|
||||
@@ -1215,7 +1214,6 @@ func TestCreateNodegroup_OmittedCapacityTypeAndReleaseVersionUseDefaults(t *test
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1225,6 +1223,7 @@ func TestCreateNodegroup_OmittedCapacityTypeAndReleaseVersionUseDefaults(t *test
|
||||
assert.Equal(t, defaultNodegroupReleaseVersion, rec.ReleaseVersion)
|
||||
assert.Nil(t, rec.LaunchTemplate)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
}
|
||||
|
||||
@@ -1240,7 +1239,6 @@ func TestCreateNodegroup_CapturesTaints(t *testing.T) {
|
||||
{Key: aws.String("dedicated"), Value: aws.String("gpu"), Effect: aws.String(eks.TaintEffectNoSchedule)},
|
||||
}
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), in, testAccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -1250,6 +1248,7 @@ func TestCreateNodegroup_CapturesTaints(t *testing.T) {
|
||||
assert.Equal(t, "dedicated", rec.Taints[0].Key)
|
||||
assert.Equal(t, eks.TaintEffectNoSchedule, rec.Taints[0].Effect)
|
||||
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
}
|
||||
|
||||
@@ -1291,9 +1290,9 @@ func TestUpdateNodegroupConfig_HonoursTaintsAndUpdateConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
f := newEKSServiceFixture(t)
|
||||
seedActiveClusterWithToken(t, f, "c1")
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
|
||||
require.NoError(t, err)
|
||||
markWorkersReady(t, f, "c1", "ng1", 1)
|
||||
f.svc.WaitLaunches()
|
||||
|
||||
out, err := f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
|
||||
|
||||
@@ -337,13 +337,10 @@ func TestDeleteClusterSGs_FirstErrorSurfacedSweepContinues(t *testing.T) {
|
||||
// cluster SG pins the customer VPC.
|
||||
func TestDeleteClusterSGs_RetriesDependencyViolation(t *testing.T) {
|
||||
origBudget, origInterval := sgDeleteWaitBudget, sgDeleteWaitInterval
|
||||
sgDeleteWaitInterval = time.Millisecond
|
||||
sgDeleteWaitBudget, sgDeleteWaitInterval = 20*time.Millisecond, time.Millisecond
|
||||
defer func() { sgDeleteWaitBudget, sgDeleteWaitInterval = origBudget, origInterval }()
|
||||
|
||||
// The budget is wall-clock, so only the subtest meant to spend it gets a
|
||||
// short one; a slow runner must not run out of time between two retries.
|
||||
t.Run("transient then succeeds", func(t *testing.T) {
|
||||
sgDeleteWaitBudget = origBudget
|
||||
sgp := newFakeSGProvisioner()
|
||||
sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
|
||||
sgp.deleteFailsBefore = map[string]int{"sg-ng": 2} // 2 DepViolations, then ok
|
||||
@@ -354,7 +351,6 @@ func TestDeleteClusterSGs_RetriesDependencyViolation(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("persistent surfaced past budget", func(t *testing.T) {
|
||||
sgDeleteWaitBudget = 20 * time.Millisecond
|
||||
sgp := newFakeSGProvisioner()
|
||||
sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
|
||||
sgp.deleteErr = errors.New("DependencyViolation: still referenced")
|
||||
|
||||
@@ -485,7 +485,6 @@ func (s *IAMServiceImpl) getGroup(ctx context.Context, accountID, groupName stri
|
||||
// updateGroupCAS applies mutate to a group under optimistic concurrency, like
|
||||
// updateUserCAS.
|
||||
func (s *IAMServiceImpl) updateGroupCAS(ctx context.Context, accountID, groupName string, mutate func(*Group) (bool, error)) error {
|
||||
defer s.recordLocks.lock("group/" + accountID + "." + groupName)()
|
||||
_, err := kvutil.Update(ctx, s.groupsBucket, accountID+"."+groupName, iamRecordCASConfig("group", accountID, groupName), mutate)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -377,7 +377,6 @@ func (s *IAMServiceImpl) getInstanceProfile(ctx context.Context, accountID, prof
|
||||
// re-reading and re-running mutate when a concurrent writer wins the race.
|
||||
// mutate reports whether it changed the record; a false return commits nothing.
|
||||
func (s *IAMServiceImpl) updateInstanceProfileCAS(ctx context.Context, accountID, profileName string, mutate func(*InstanceProfile) (bool, error)) error {
|
||||
defer s.recordLocks.lock("instance-profile/" + accountID + "." + profileName)()
|
||||
_, err := kvutil.Update(ctx, s.instanceProfilesBucket, accountID+"."+profileName, s.instanceProfileCASConfig(accountID, profileName), mutate)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -360,7 +360,6 @@ func (s *IAMServiceImpl) updateOIDCProviderCAS(ctx context.Context, accountID, a
|
||||
}
|
||||
return fmt.Errorf("open IAM account bucket: %w", err)
|
||||
}
|
||||
defer s.recordLocks.lock("oidc-provider/" + accountID + "/" + OIDCProviderKey(issuer))()
|
||||
_, err = kvutil.Update(ctx, kv, OIDCProviderKey(issuer), iamRecordCASConfig("oidc-provider", accountID, arn), mutate)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -93,7 +93,6 @@ func (s *IAMServiceImpl) updatePolicyCAS(ctx context.Context, accountID, policyA
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.recordLocks.lock("policy/" + key)()
|
||||
_, err = kvutil.Update(ctx, s.policiesBucket, key, cfg, func(p *Policy) (bool, error) {
|
||||
// The key omits the path, so an ARN with the wrong path names no policy.
|
||||
if p.ARN != policyARN {
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
package iam
|
||||
|
||||
import "sync"
|
||||
|
||||
// recordLocks queues this process's writers of one IAM record behind each
|
||||
// other. CAS alone keeps a write from being lost, but a caller looping a cheap
|
||||
// self-scoped write can win every revision race and starve an administrator's
|
||||
// DetachUserPolicy until its CAS budget is spent. The zero value is ready to use.
|
||||
type recordLocks struct {
|
||||
mu sync.Mutex
|
||||
locks map[string]*recordLock
|
||||
}
|
||||
|
||||
type recordLock struct {
|
||||
mu sync.Mutex
|
||||
refs int
|
||||
}
|
||||
|
||||
// lock blocks until the caller holds key and returns its release. Entries are
|
||||
// dropped once no writer holds or waits on them, so the map tracks only
|
||||
// records being written right now.
|
||||
func (l *recordLocks) lock(key string) func() {
|
||||
l.mu.Lock()
|
||||
if l.locks == nil {
|
||||
l.locks = make(map[string]*recordLock)
|
||||
}
|
||||
rl := l.locks[key]
|
||||
if rl == nil {
|
||||
rl = &recordLock{}
|
||||
l.locks[key] = rl
|
||||
}
|
||||
rl.refs++
|
||||
l.mu.Unlock()
|
||||
|
||||
rl.mu.Lock()
|
||||
return func() {
|
||||
rl.mu.Unlock()
|
||||
l.mu.Lock()
|
||||
rl.refs--
|
||||
if rl.refs == 0 {
|
||||
delete(l.locks, key)
|
||||
}
|
||||
l.mu.Unlock()
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
@@ -177,7 +176,7 @@ func TestUserWriters_DoNotUndoConcurrentDetach(t *testing.T) {
|
||||
go func() { done <- tc.write(svc, *user.UserName, arg) }()
|
||||
<-gate.reached
|
||||
|
||||
_, err = peerIAMService(t, svc).DetachUserPolicy(testAccountID, &iam.DetachUserPolicyInput{UserName: user.UserName, PolicyArn: policy.Arn})
|
||||
_, err = svc.DetachUserPolicy(testAccountID, &iam.DetachUserPolicyInput{UserName: user.UserName, PolicyArn: policy.Arn})
|
||||
require.NoError(t, err)
|
||||
|
||||
close(gate.release)
|
||||
@@ -293,7 +292,7 @@ func TestGroupWriters_DoNotUndoConcurrentDetach(t *testing.T) {
|
||||
go func() { done <- tc.write(svc, *group.GroupName, *other.Arn) }()
|
||||
<-gate.reached
|
||||
|
||||
_, err = peerIAMService(t, svc).DetachGroupPolicy(testAccountID, &iam.DetachGroupPolicyInput{GroupName: group.GroupName, PolicyArn: policy.Arn})
|
||||
_, err = svc.DetachGroupPolicy(testAccountID, &iam.DetachGroupPolicyInput{GroupName: group.GroupName, PolicyArn: policy.Arn})
|
||||
require.NoError(t, err)
|
||||
|
||||
close(gate.release)
|
||||
@@ -383,78 +382,3 @@ func TestOIDCProviderTagWriters_DoNotRecreateDeletedProvider(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// contendedKV lands a competing write ahead of each of the first budget record
|
||||
// updates it forwards, the order a caller looping TagUser wins every race in.
|
||||
type contendedKV struct {
|
||||
jetstream.KeyValue
|
||||
|
||||
svc *IAMServiceImpl
|
||||
user string
|
||||
budget atomic.Int32
|
||||
armed atomic.Bool
|
||||
pending sync.WaitGroup
|
||||
errs chan error
|
||||
}
|
||||
|
||||
func (c *contendedKV) Update(ctx context.Context, key string, value []byte, last uint64) (uint64, error) {
|
||||
if c.armed.CompareAndSwap(true, false) && c.budget.Add(-1) >= 0 {
|
||||
done := make(chan struct{})
|
||||
c.pending.Go(func() {
|
||||
defer close(done)
|
||||
_, err := c.svc.TagUser(testAccountID, &iam.TagUserInput{
|
||||
UserName: aws.String(c.user),
|
||||
Tags: []*iam.Tag{{Key: aws.String("k"), Value: aws.String(key)}},
|
||||
})
|
||||
c.errs <- err
|
||||
})
|
||||
// Either the competing write lands first or it queues behind this one.
|
||||
for c.svc.recordLocks.holders("user/"+key) < 2 {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Millisecond):
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
c.armed.Store(true)
|
||||
}
|
||||
return c.KeyValue.Update(ctx, key, value, last)
|
||||
}
|
||||
|
||||
func (l *recordLocks) holders(key string) int {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if rl := l.locks[key]; rl != nil {
|
||||
return rl.refs
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// A self-scoped write that wins every revision race on this node must queue
|
||||
// behind an administrator's DetachUserPolicy rather than exhaust its retries.
|
||||
func TestDetachUserPolicy_IsNotStarvedByALoopingSelfWrite(t *testing.T) {
|
||||
t.Parallel()
|
||||
svc := setupTestIAMService(t)
|
||||
user := createTestUser(t, svc, "self-tagger")
|
||||
policy := createTestPolicy(t, svc, "RevokedPolicy")
|
||||
_, err := svc.AttachUserPolicy(testAccountID, &iam.AttachUserPolicyInput{UserName: user.UserName, PolicyArn: policy.Arn})
|
||||
require.NoError(t, err)
|
||||
|
||||
kv := &contendedKV{KeyValue: svc.usersBucket, svc: svc, user: *user.UserName, errs: make(chan error, roleCASMaxRetries)}
|
||||
kv.budget.Store(roleCASMaxRetries)
|
||||
kv.armed.Store(true)
|
||||
svc.usersBucket = kv
|
||||
|
||||
_, err = svc.DetachUserPolicy(testAccountID, &iam.DetachUserPolicyInput{UserName: user.UserName, PolicyArn: policy.Arn})
|
||||
require.NoError(t, err, "a contending self-write starved the detach")
|
||||
kv.pending.Wait()
|
||||
close(kv.errs)
|
||||
for werr := range kv.errs {
|
||||
require.NoError(t, werr)
|
||||
}
|
||||
|
||||
got, err := svc.getUser(t.Context(), testAccountID, *user.UserName)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, got.AttachedPolicies)
|
||||
}
|
||||
|
||||
@@ -619,7 +619,6 @@ func (s *IAMServiceImpl) getRole(ctx context.Context, accountID, roleName string
|
||||
// to one role at once) write the same record concurrently. mutate reports
|
||||
// whether it changed the record; a false return commits nothing.
|
||||
func (s *IAMServiceImpl) updateRoleCAS(ctx context.Context, accountID, roleName string, mutate func(*Role) (bool, error)) error {
|
||||
defer s.recordLocks.lock("role/" + accountID + "." + roleName)()
|
||||
_, err := kvutil.Update(ctx, s.rolesBucket, accountID+"."+roleName, kvutil.CASConfig{
|
||||
Attempts: roleCASMaxRetries,
|
||||
NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
|
||||
|
||||
@@ -32,18 +32,6 @@ func (g *gatedKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry,
|
||||
return entry, err
|
||||
}
|
||||
|
||||
// peerIAMService is another node's IAM service over the same JetStream. It
|
||||
// shares the records but not this service's record locks, so a writer parked
|
||||
// by gatedKV still races it the way a second gateway would.
|
||||
func peerIAMService(t *testing.T, svc *IAMServiceImpl) *IAMServiceImpl {
|
||||
t.Helper()
|
||||
masterKey, err := GenerateMasterKey()
|
||||
require.NoError(t, err)
|
||||
peer, err := NewIAMServiceImpl(t.Context(), svc.natsConn, masterKey)
|
||||
require.NoError(t, err)
|
||||
return peer
|
||||
}
|
||||
|
||||
// A role update that read the record before a DetachRolePolicy and writes after
|
||||
// it must not put the detached policy back.
|
||||
func TestRoleUpdates_DoNotUndoConcurrentDetach(t *testing.T) {
|
||||
@@ -104,7 +92,7 @@ func TestRoleUpdates_DoNotUndoConcurrentDetach(t *testing.T) {
|
||||
go func() { done <- tc.update(svc, *role.RoleName) }()
|
||||
<-gate.reached
|
||||
|
||||
_, err = peerIAMService(t, svc).DetachRolePolicy(testAccountID, &iam.DetachRolePolicyInput{
|
||||
_, err = svc.DetachRolePolicy(testAccountID, &iam.DetachRolePolicyInput{
|
||||
RoleName: role.RoleName,
|
||||
PolicyArn: policy.Arn,
|
||||
})
|
||||
|
||||
@@ -111,7 +111,6 @@ type IAMServiceImpl struct {
|
||||
groupsBucket jetstream.KeyValue
|
||||
key *masterkey.Key
|
||||
sessionRevoker SessionRevoker
|
||||
recordLocks recordLocks
|
||||
}
|
||||
|
||||
var _ IAMService = (*IAMServiceImpl)(nil)
|
||||
@@ -2181,7 +2180,6 @@ func (s *IAMServiceImpl) getUser(ctx context.Context, accountID, userName string
|
||||
// updateRoleCAS. Every writer of a user record must use it: one blind Put can
|
||||
// restore a policy or group membership that a concurrent writer removed.
|
||||
func (s *IAMServiceImpl) updateUserCAS(ctx context.Context, accountID, userName string, mutate func(*User) (bool, error)) error {
|
||||
defer s.recordLocks.lock("user/" + accountID + "." + userName)()
|
||||
_, err := kvutil.Update(ctx, s.usersBucket, accountID+"."+userName, iamRecordCASConfig("user", accountID, userName), mutate)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -37,9 +37,6 @@ type warmupProbe struct {
|
||||
|
||||
mu sync.RWMutex
|
||||
ready bool
|
||||
|
||||
done chan struct{}
|
||||
stopOnce sync.Once
|
||||
}
|
||||
|
||||
// newWarmupProbeWithInterval constructs a warmupProbe for baseURL and starts
|
||||
@@ -48,7 +45,7 @@ type warmupProbe struct {
|
||||
// tests pass a short interval to drive the loop without waiting out the
|
||||
// production cadence.
|
||||
func newWarmupProbeWithInterval(client *http.Client, baseURL string, pollInterval time.Duration) *warmupProbe {
|
||||
p := &warmupProbe{client: client, baseURL: baseURL, done: make(chan struct{})}
|
||||
p := &warmupProbe{client: client, baseURL: baseURL}
|
||||
go p.run(pollInterval)
|
||||
return p
|
||||
}
|
||||
@@ -60,22 +57,11 @@ func (p *warmupProbe) run(pollInterval time.Duration) {
|
||||
p.probe()
|
||||
ticker := time.NewTicker(pollInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-p.done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.probe()
|
||||
}
|
||||
for range ticker.C {
|
||||
p.probe()
|
||||
}
|
||||
}
|
||||
|
||||
// stop ends the poll loop. Production probes live as long as the process; a
|
||||
// test stops its probe so it cannot poll a later test's server on the same port.
|
||||
func (p *warmupProbe) stop() {
|
||||
p.stopOnce.Do(func() { close(p.done) })
|
||||
}
|
||||
|
||||
// probe issues one health check and updates the cached state, logging an
|
||||
// "embedder ready" marker the first time it observes success.
|
||||
func (p *warmupProbe) probe() {
|
||||
|
||||
@@ -29,7 +29,6 @@ func TestWarmupProbe_NotReadyUntilHealthPasses(t *testing.T) {
|
||||
defer ts.Close()
|
||||
|
||||
p := newWarmupProbeWithInterval(ts.Client(), ts.URL, 10*time.Millisecond)
|
||||
t.Cleanup(p.stop)
|
||||
|
||||
// The immediate first probe run issues before the ticker starts lands
|
||||
// against the not-yet-healthy server; give it a moment to complete.
|
||||
@@ -56,7 +55,6 @@ func TestWarmupProbe_ReProbesAfterInterval(t *testing.T) {
|
||||
defer ts.Close()
|
||||
|
||||
p := newWarmupProbeWithInterval(ts.Client(), ts.URL, 10*time.Millisecond)
|
||||
t.Cleanup(p.stop)
|
||||
require.Eventually(t, p.Ready, time.Second, 5*time.Millisecond,
|
||||
"must become ready once the endpoint is healthy")
|
||||
|
||||
@@ -93,7 +91,7 @@ func TestEmbeddingsProvider_Embed_WarmupGateFailsFastUntilHealthy(t *testing.T)
|
||||
p := newEmbeddingsProvider(NewStaticEndpointResolver(map[string]string{modelID: ts.URL}))
|
||||
p.httpClient = ts.Client()
|
||||
p.warmupPollInterval = 10 * time.Millisecond
|
||||
t.Cleanup(p.warmupFor(ts.URL).stop) // mirrors NewEmbedder(resolver, ts.URL)
|
||||
p.warmupFor(ts.URL) // mirrors NewEmbedder(resolver, ts.URL)
|
||||
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
_, err := p.Embed(t.Context(), modelID, []string{"hello"})
|
||||
@@ -107,21 +105,3 @@ func TestEmbeddingsProvider_Embed_WarmupGateFailsFastUntilHealthy(t *testing.T)
|
||||
return err == nil
|
||||
}, time.Second, 10*time.Millisecond, "must succeed once the warm-up probe catches up to the endpoint being healthy")
|
||||
}
|
||||
|
||||
// A probe left polling after its test outlives that test's server, and a later
|
||||
// test's server bound to the recycled port then counts the stray /health hits.
|
||||
func TestWarmupProbe_StopEndsThePollLoop(t *testing.T) {
|
||||
p := &warmupProbe{client: http.DefaultClient, baseURL: "http://127.0.0.1:1", done: make(chan struct{})}
|
||||
exited := make(chan struct{})
|
||||
go func() {
|
||||
p.run(time.Millisecond)
|
||||
close(exited)
|
||||
}()
|
||||
p.stop()
|
||||
p.stop()
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("run kept polling after stop")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -567,7 +567,6 @@ func TestInvokeStreamRouter_GuardrailInputBlock_SkipsBackend(t *testing.T) {
|
||||
|
||||
src, err := rt.InvokeModelWithResponseStream(ctx, grCallerAccount, modelID, []byte(`{"prompt":"this has a badword in it"}`), aws.StringValue(createOut.GuardrailId), guardrailDraftVersion)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = src.Close() }()
|
||||
chunks := drainInvokeStream(t, src)
|
||||
assert.Equal(t, int32(0), hits.Load(), "the backend stream must never be started on an INPUT block")
|
||||
require.Len(t, chunks, 2)
|
||||
@@ -599,7 +598,6 @@ func TestInvokeStreamRouter_GuardrailOutputBlock_Buffered(t *testing.T) {
|
||||
|
||||
src, err := rt.InvokeModelWithResponseStream(ctx, grCallerAccount, modelID, []byte(`{"prompt":"hello"}`), aws.StringValue(createOut.GuardrailId), guardrailDraftVersion)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = src.Close() }()
|
||||
chunks := drainInvokeStream(t, src)
|
||||
require.Len(t, chunks, 2, "the raw deltas must collapse into exactly one guarded delta + final chunk")
|
||||
|
||||
@@ -630,7 +628,6 @@ func TestInvokeStreamRouter_GuardrailOutputAnonymize_Buffered(t *testing.T) {
|
||||
|
||||
src, err := rt.InvokeModelWithResponseStream(ctx, grCallerAccount, modelID, []byte(`{"prompt":"hello"}`), aws.StringValue(createOut.GuardrailId), guardrailDraftVersion)
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = src.Close() }()
|
||||
chunks := drainInvokeStream(t, src)
|
||||
require.Len(t, chunks, 2)
|
||||
|
||||
@@ -657,7 +654,6 @@ func TestInvokeStreamRouter_NoGuardrailHeader_Regression(t *testing.T) {
|
||||
|
||||
src, err := rt.InvokeModelWithResponseStream(context.Background(), "000000000001", modelID, []byte(`{"prompt":"hello"}`), "", "")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = src.Close() }()
|
||||
chunks := drainInvokeStream(t, src)
|
||||
require.Len(t, chunks, 3)
|
||||
|
||||
|
||||
@@ -52,7 +52,6 @@ func TestRouter_ConverseStream_ProvisionedThroughputARN_ResolvesPinnedEndpointFo
|
||||
|
||||
src, err := rt.ConverseStream(context.Background(), ptCallerAccount, arn, converseStreamInput())
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = src.Close() }()
|
||||
events := drainConverseStream(t, src)
|
||||
assert.NotEmpty(t, events)
|
||||
|
||||
@@ -76,7 +75,6 @@ func TestRouter_ConverseStream_BareModelID_StillUsesGlobalShorthand(t *testing.T
|
||||
|
||||
src, err := rt.ConverseStream(context.Background(), ptCallerAccount, selfHostTestModel, converseStreamInput())
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = src.Close() }()
|
||||
drainConverseStream(t, src)
|
||||
|
||||
require.Len(t, spy.endpointCalls, 1)
|
||||
@@ -127,7 +125,6 @@ func TestInvokeStreamRouter_InvokeModelWithResponseStream_ProvisionedThroughputA
|
||||
|
||||
src, err := rt.InvokeModelWithResponseStream(context.Background(), ptCallerAccount, arn, []byte(`{"prompt":"hello"}`), "", "")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = src.Close() }()
|
||||
chunks := drainInvokeStream(t, src)
|
||||
assert.NotEmpty(t, chunks)
|
||||
|
||||
@@ -149,7 +146,6 @@ func TestInvokeStreamRouter_InvokeModelWithResponseStream_BareModelID_StillUsesG
|
||||
|
||||
src, err := rt.InvokeModelWithResponseStream(context.Background(), ptCallerAccount, selfHostTestModel, []byte(`{"prompt":"hello"}`), "", "")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = src.Close() }()
|
||||
drainInvokeStream(t, src)
|
||||
|
||||
require.Len(t, spy.endpointCalls, 1)
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestModifyDBInstance_HoldsTheLeaseSoAReconcilePassCannotReEnterTheChange(t
|
||||
// change as outstanding. Once, or a re-entered pass would recurse here.
|
||||
var sweep sync.Once
|
||||
var swept error
|
||||
h.storage.onModify = func(context.Context) {
|
||||
h.storage.onModify = func() {
|
||||
sweep.Do(func() { swept = onePass(t, h.rec) })
|
||||
}
|
||||
|
||||
@@ -185,17 +185,11 @@ func TestModifyDBInstance_LeaseTakeoverLeavesTheTransitionRetryable(t *testing.T
|
||||
seedInstance(t, h.svc, modifiableRecord())
|
||||
kv := h.kv(t)
|
||||
|
||||
// The takeover lands mid-grow; the step after it must not run until the
|
||||
// renewer has seen the takeover and cancelled the apply.
|
||||
h.storage.onModify = func(ctx context.Context) {
|
||||
h.storage.onModify = func() {
|
||||
require.NoError(t, h.svc.updateInstance(t.Context(), kv, testDBID, func(stored *DBInstanceRecord) {
|
||||
stored.ModifyLease = heldLease(time.Second)
|
||||
}))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Error("the apply context was not cancelled after the lease takeover")
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
in := modifyInput()
|
||||
in.AllocatedStorage, in.ApplyImmediately = aws.Int64(50), aws.Bool(true)
|
||||
|
||||
@@ -185,7 +185,7 @@ func TestReplaceInstanceVM_GrowsTheVolumeWhileNoVMHoldsIt(t *testing.T) {
|
||||
|
||||
var terminatedAtGrow int
|
||||
var launchedAtGrow bool
|
||||
h.storage.onModify = func(context.Context) {
|
||||
h.storage.onModify = func() {
|
||||
terminatedAtGrow = len(h.launch.launcher.terminated)
|
||||
launchedAtGrow = h.launch.launcher.input != nil
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ type fakeVolumeResizer struct {
|
||||
|
||||
// onModify runs inside the resize, so a test can observe what else has and
|
||||
// has not happened by the time the volume is taken.
|
||||
onModify func(context.Context)
|
||||
onModify func()
|
||||
}
|
||||
|
||||
var _ volumeResizer = (*fakeVolumeResizer)(nil)
|
||||
@@ -55,9 +55,9 @@ func (f *fakeVolumeResizer) DescribeVolumes(_ context.Context, in *ec2.DescribeV
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeVolumeResizer) ModifyVolume(ctx context.Context, in *ec2.ModifyVolumeInput, _ string) (*ec2.ModifyVolumeOutput, error) {
|
||||
func (f *fakeVolumeResizer) ModifyVolume(_ context.Context, in *ec2.ModifyVolumeInput, _ string) (*ec2.ModifyVolumeOutput, error) {
|
||||
if f.onModify != nil {
|
||||
f.onModify(ctx)
|
||||
f.onModify()
|
||||
}
|
||||
if f.modifyErr != nil {
|
||||
return nil, f.modifyErr
|
||||
@@ -136,7 +136,7 @@ func TestGrowInstanceStorage_StopsTheEngineAndTheVMBeforeGrowing(t *testing.T) {
|
||||
rec := modifiableRecord()
|
||||
|
||||
var vmStoppedAtGrow bool
|
||||
h.storage.onModify = func(context.Context) {
|
||||
h.storage.onModify = func() {
|
||||
vmStoppedAtGrow = assert.ObjectsAreEqual([]string{"stop:" + testInstance}, h.cmdr.calls)
|
||||
}
|
||||
|
||||
@@ -161,7 +161,7 @@ func TestGrowInstanceStorage_WaitsForTheDetachBeforeGrowing(t *testing.T) {
|
||||
rec := modifiableRecord()
|
||||
|
||||
var vmDrainingAtGrow bool
|
||||
h.storage.onModify = func(context.Context) { vmDrainingAtGrow = h.vmState.detachReads > 0 }
|
||||
h.storage.onModify = func() { vmDrainingAtGrow = h.vmState.detachReads > 0 }
|
||||
|
||||
require.NoError(t, h.svc.growInstanceStorage(t.Context(), testAccountID, &rec, 50))
|
||||
|
||||
@@ -201,7 +201,7 @@ func TestGrowInstanceStorage_ReportsTheGrowAndRestartFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
h := newModifyHarness(t)
|
||||
h.storage.modifyErr = errors.New("the volume store is unavailable")
|
||||
h.storage.onModify = func(context.Context) { h.cmdr.err = errors.New("the node did not answer") }
|
||||
h.storage.onModify = func() { h.cmdr.err = errors.New("the node did not answer") }
|
||||
rec := modifiableRecord()
|
||||
|
||||
err := h.svc.growInstanceStorage(t.Context(), testAccountID, &rec, 50)
|
||||
@@ -219,7 +219,7 @@ func TestGrowInstanceStorage_DoesNotRestartAfterLosingTheModifyLease(t *testing.
|
||||
h := newModifyHarness(t)
|
||||
ctx, cancel := context.WithCancelCause(t.Context())
|
||||
h.storage.modifyErr = errors.New("the volume store is unavailable")
|
||||
h.storage.onModify = func(context.Context) { cancel(errModifyLeaseLost) }
|
||||
h.storage.onModify = func() { cancel(errModifyLeaseLost) }
|
||||
rec := modifiableRecord()
|
||||
|
||||
err := h.svc.growInstanceStorage(ctx, testAccountID, &rec, 50)
|
||||
|
||||
@@ -19,19 +19,17 @@ import (
|
||||
)
|
||||
|
||||
// delayProxy forwards TCP to target, holding each chunk for the current
|
||||
// delay in both directions while preserving order. A connection accepted
|
||||
// before setTarget waits for it.
|
||||
// delay in both directions while preserving order.
|
||||
type delayProxy struct {
|
||||
l net.Listener
|
||||
target string
|
||||
ready chan struct{}
|
||||
delay atomic.Int64
|
||||
}
|
||||
|
||||
func newDelayProxy(t *testing.T) *delayProxy {
|
||||
func newDelayProxy(t *testing.T, target string) *delayProxy {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
p := &delayProxy{l: l, ready: make(chan struct{})}
|
||||
p := &delayProxy{l: l, target: target}
|
||||
t.Cleanup(func() { _ = l.Close() })
|
||||
go func() {
|
||||
for {
|
||||
@@ -39,29 +37,18 @@ func newDelayProxy(t *testing.T) *delayProxy {
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go p.forward(c)
|
||||
u, err := net.Dial("tcp", p.target)
|
||||
if err != nil {
|
||||
_ = c.Close()
|
||||
continue
|
||||
}
|
||||
go p.pipe(c, u)
|
||||
go p.pipe(u, c)
|
||||
}
|
||||
}()
|
||||
return p
|
||||
}
|
||||
|
||||
// setTarget is called once, with the address the target server bound.
|
||||
func (p *delayProxy) setTarget(addr string) {
|
||||
p.target = addr
|
||||
close(p.ready)
|
||||
}
|
||||
|
||||
func (p *delayProxy) forward(c net.Conn) {
|
||||
<-p.ready
|
||||
u, err := net.Dial("tcp", p.target)
|
||||
if err != nil {
|
||||
_ = c.Close()
|
||||
return
|
||||
}
|
||||
go p.pipe(c, u)
|
||||
go p.pipe(u, c)
|
||||
}
|
||||
|
||||
type delayedChunk struct {
|
||||
due time.Time
|
||||
b []byte
|
||||
@@ -101,11 +88,12 @@ type lagCluster struct {
|
||||
|
||||
func startLagCluster(t *testing.T) *lagCluster {
|
||||
clustersize.RedeclareForTest(t, 3)
|
||||
ports := routePorts(t)
|
||||
c := &lagCluster{}
|
||||
for i := range 3 {
|
||||
for j := range 3 {
|
||||
if i != j {
|
||||
c.proxy[i][j] = newDelayProxy(t)
|
||||
c.proxy[i][j] = newDelayProxy(t, fmt.Sprintf("127.0.0.1:%d", ports[j]))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -121,7 +109,7 @@ func startLagCluster(t *testing.T) *lagCluster {
|
||||
JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
|
||||
// An unreachable advertised address stops gossip-learned routes that
|
||||
// would bypass the proxies; the configured routes are a full mesh.
|
||||
Cluster: server.ClusterOpts{Name: "lag", Host: "127.0.0.1", Port: server.RANDOM_PORT, PoolSize: -1, NoAdvertise: true, Advertise: "127.0.0.1:1"},
|
||||
Cluster: server.ClusterOpts{Name: "lag", Host: "127.0.0.1", Port: ports[i], PoolSize: -1, NoAdvertise: true, Advertise: "127.0.0.1:1"},
|
||||
Routes: routes,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
@@ -132,14 +120,6 @@ func startLagCluster(t *testing.T) *lagCluster {
|
||||
for i, srv := range c.servers {
|
||||
require.True(t, srv.ReadyForConnections(10*time.Second), "node %d", i)
|
||||
}
|
||||
// Each server bound its own route port, so none was free to be taken.
|
||||
for i := range 3 {
|
||||
for j := range 3 {
|
||||
if i != j {
|
||||
c.proxy[i][j].setTarget(c.servers[j].ClusterAddr().String())
|
||||
}
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -2,13 +2,12 @@ package jetstream_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -24,16 +23,31 @@ import (
|
||||
)
|
||||
|
||||
// cluster is a real three-node JetStream cluster in this process: separate
|
||||
// stores, routes and Raft groups, restartable over the same data.
|
||||
// stores, routes and Raft groups, restartable over the same data and ports.
|
||||
type cluster struct {
|
||||
t *testing.T
|
||||
dirs [3]string
|
||||
client [3]int
|
||||
route [3]int
|
||||
logDir string
|
||||
mu sync.Mutex // guards servers against a reconnecting client
|
||||
servers [3]*server.Server
|
||||
js jsapi.JetStream
|
||||
}
|
||||
|
||||
// routePorts reserves three distinct ports at once. Route ports must stay
|
||||
// fixed across restarts; client ports are chosen by each server's first start.
|
||||
func routePorts(t *testing.T) [3]int {
|
||||
t.Helper()
|
||||
var ports [3]int
|
||||
for i := range ports {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer l.Close()
|
||||
ports[i] = l.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
return ports
|
||||
}
|
||||
|
||||
func startCluster(t *testing.T) *cluster {
|
||||
t.Helper()
|
||||
return startClusterLogged(t, "")
|
||||
@@ -46,26 +60,29 @@ func startClusterLogged(t *testing.T, logDir string) *cluster {
|
||||
for i := range 3 {
|
||||
c.dirs[i] = t.TempDir()
|
||||
}
|
||||
c.route = routePorts(t)
|
||||
clustersize.RedeclareForTest(t, 3)
|
||||
c.start()
|
||||
t.Cleanup(c.stop)
|
||||
return c
|
||||
}
|
||||
|
||||
// start binds every listener on a port the server picks itself, so no port
|
||||
// is ever free to be taken. Nodes 1 and 2 route to node 0 and learn each
|
||||
// other from it.
|
||||
func (c *cluster) start() {
|
||||
t := c.t
|
||||
// JetStream needs a configured route, so node 0, which cannot name the
|
||||
// others yet, gets port 1, where nothing listens.
|
||||
seed := []*url.URL{{Scheme: "nats", Host: "127.0.0.1:1"}}
|
||||
routes := make([]*url.URL, 3)
|
||||
for i := range 3 {
|
||||
routes[i] = &url.URL{Scheme: "nats", Host: fmt.Sprintf("127.0.0.1:%d", c.route[i])}
|
||||
}
|
||||
for i := range 3 {
|
||||
port := c.client[i]
|
||||
if port == 0 {
|
||||
port = server.RANDOM_PORT
|
||||
}
|
||||
opts := &server.Options{
|
||||
ServerName: fmt.Sprintf("sqs-%d", i), Host: "127.0.0.1", Port: server.RANDOM_PORT,
|
||||
ServerName: fmt.Sprintf("sqs-%d", i), Host: "127.0.0.1", Port: port,
|
||||
JetStream: true, StoreDir: c.dirs[i], NoLog: true, NoSigs: true,
|
||||
Cluster: server.ClusterOpts{Name: "sqs-r3", Host: "127.0.0.1", Port: server.RANDOM_PORT},
|
||||
Routes: seed,
|
||||
Cluster: server.ClusterOpts{Name: "sqs-r3", Host: "127.0.0.1", Port: c.route[i]},
|
||||
Routes: routes,
|
||||
}
|
||||
if c.logDir != "" {
|
||||
opts.NoLog, opts.LogFile = false, filepath.Join(c.logDir, fmt.Sprintf("sqs-%d.log", i))
|
||||
@@ -77,15 +94,16 @@ func (c *cluster) start() {
|
||||
srv.ConfigureLogger()
|
||||
}
|
||||
go srv.Start()
|
||||
require.True(t, srv.ReadyForConnections(10*time.Second), "node %d not ready", i)
|
||||
if i == 0 {
|
||||
seed = []*url.URL{{Scheme: "nats", Host: srv.ClusterAddr().String()}}
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.servers[i] = srv
|
||||
c.mu.Unlock()
|
||||
}
|
||||
nc, err := nats.Connect("", nats.InProcessServer(c), nats.MaxReconnects(-1), nats.ReconnectWait(100*time.Millisecond))
|
||||
urls := make([]string, 3)
|
||||
for i, srv := range c.servers {
|
||||
require.True(t, srv.ReadyForConnections(10*time.Second), "node %d not ready", i)
|
||||
// Keep the client port across restarts so open connections reconnect.
|
||||
c.client[i] = srv.Addr().(*net.TCPAddr).Port
|
||||
urls[i] = srv.ClientURL()
|
||||
}
|
||||
nc, err := nats.Connect(strings.Join(urls, ","), nats.MaxReconnects(-1), nats.ReconnectWait(100*time.Millisecond))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jsapi.New(nc)
|
||||
@@ -102,19 +120,6 @@ func (c *cluster) start() {
|
||||
c.js = js
|
||||
}
|
||||
|
||||
// InProcessConn connects a client to a running node. A client opened before
|
||||
// a restart reconnects through it to the restarted nodes.
|
||||
func (c *cluster) InProcessConn() (net.Conn, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for _, srv := range c.servers {
|
||||
if srv != nil && srv.Running() {
|
||||
return srv.InProcessConn()
|
||||
}
|
||||
}
|
||||
return nil, errors.New("no cluster node is running")
|
||||
}
|
||||
|
||||
func allStreamsLed(js jsapi.JetStream) bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -879,19 +878,15 @@ func TestGather_StopOnFirstSkipsErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, nc := testutil.StartTestNATS(t)
|
||||
|
||||
var errorsSent sync.WaitGroup
|
||||
errorsSent.Add(2)
|
||||
for range 2 {
|
||||
_, err := nc.Subscribe("test.gather.first", func(msg *nats.Msg) {
|
||||
_ = msg.Respond(awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound))
|
||||
errorsSent.Done()
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
// One connection publishes in order, so waiting here puts both error
|
||||
// frames ahead of the success in the inbox.
|
||||
// Delayed so the two error frames are processed (and skipped) first.
|
||||
_, err := nc.Subscribe("test.gather.first", func(msg *nats.Msg) {
|
||||
errorsSent.Wait()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
_ = msg.Respond([]byte(`{"value":"found"}`))
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -3,7 +3,8 @@ package kvstore_test
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -17,42 +18,36 @@ import (
|
||||
)
|
||||
|
||||
// startJetStreamCluster starts n routed JetStream servers and returns a handle
|
||||
// connected to the first once a create can actually place n replicas.
|
||||
// connected to the first, so a create can actually place n replicas.
|
||||
func startJetStreamCluster(t *testing.T, n int) jetstream.JetStream {
|
||||
t.Helper()
|
||||
// Servers bind their own route ports, so none is ever free to be taken.
|
||||
// JetStream needs a configured route, so the first, which cannot name the
|
||||
// others yet, gets port 1, where nothing listens; the rest route to it.
|
||||
seed := []*url.URL{{Scheme: "nats", Host: "127.0.0.1:1"}}
|
||||
ports := make([]int, n)
|
||||
routes := make([]string, n)
|
||||
for i := range ports {
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
ports[i] = l.Addr().(*net.TCPAddr).Port
|
||||
require.NoError(t, l.Close())
|
||||
routes[i] = fmt.Sprintf("nats://127.0.0.1:%d", ports[i])
|
||||
}
|
||||
servers := make([]*server.Server, n)
|
||||
for i := range servers {
|
||||
ns, err := server.NewServer(&server.Options{
|
||||
ServerName: fmt.Sprintf("s%d", i+1), JetStream: true, StoreDir: t.TempDir(),
|
||||
Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true,
|
||||
Cluster: server.ClusterOpts{Name: "kvstore", Host: "127.0.0.1", Port: -1},
|
||||
Routes: seed,
|
||||
Cluster: server.ClusterOpts{Name: "kvstore", Host: "127.0.0.1", Port: ports[i]},
|
||||
Routes: server.RoutesFromStr(strings.Join(routes, ",")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
ns.Start()
|
||||
servers[i] = ns
|
||||
// Registered after its TempDir, so the server stops before that is removed.
|
||||
t.Cleanup(func() { ns.Shutdown(); ns.WaitForShutdown() })
|
||||
require.True(t, ns.ReadyForConnections(10*time.Second), "server %d not ready", i+1)
|
||||
if i == 0 {
|
||||
seed = []*url.URL{{Scheme: "nats", Host: ns.ClusterAddr().String()}}
|
||||
}
|
||||
}
|
||||
|
||||
// The meta leader places replicas only on peers it has stats for, which
|
||||
// can trail its election, so a create before then finds too few peers.
|
||||
require.Eventually(t, func() bool {
|
||||
t.Cleanup(func() {
|
||||
for _, ns := range servers {
|
||||
if len(ns.JetStreamClusterPeers()) == n {
|
||||
return true
|
||||
}
|
||||
ns.Shutdown()
|
||||
ns.WaitForShutdown()
|
||||
}
|
||||
return false
|
||||
}, 30*time.Second, 50*time.Millisecond, "meta leader never had stats for all %d peers", n)
|
||||
})
|
||||
|
||||
nc, err := nats.Connect("", nats.InProcessServer(servers[0]))
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -12,15 +12,6 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// ClusterNodes is the configured node set a teardown reaches. The two ID lists
|
||||
// are different processes: a daemon on every node, a gateway only on nodes
|
||||
// that run awsgw. They are named fields so one cannot be passed for the other.
|
||||
type ClusterNodes struct {
|
||||
Expected int
|
||||
DaemonIDs []string
|
||||
GatewayIDs []string
|
||||
}
|
||||
|
||||
// NewClusterEngine wires every reaper against a live cluster.
|
||||
//
|
||||
// The CLI and the admin API both build their engine here rather than each
|
||||
@@ -29,7 +20,8 @@ type ClusterNodes struct {
|
||||
func NewClusterEngine(
|
||||
ctx context.Context,
|
||||
nc *nats.Conn,
|
||||
nodes ClusterNodes,
|
||||
expectedNodes int,
|
||||
nodeIDs []string,
|
||||
svc iamdomain.IAMService,
|
||||
buckets BucketStore,
|
||||
) (*Engine, error) {
|
||||
@@ -73,7 +65,7 @@ func NewClusterEngine(
|
||||
// live spot request launches a replacement for anything already terminated.
|
||||
reapers = append(reapers, ELBv2Reapers(nc)...)
|
||||
reapers = append(reapers, EC2ExtraReapers(nc)...)
|
||||
reapers = append(reapers, EC2Reapers(nc, nodes.Expected, nodes.DaemonIDs)...)
|
||||
reapers = append(reapers, EC2Reapers(nc, expectedNodes, nodeIDs)...)
|
||||
|
||||
reapers = append(reapers, S3Reapers(buckets)...)
|
||||
reapers = append(reapers, ACMReapers(nc)...)
|
||||
@@ -82,9 +74,7 @@ func NewClusterEngine(
|
||||
// Before the account counters and reservations: the seal must still be
|
||||
// able to find the account IAM holds as TERMINATING, which the account
|
||||
// record reaper does not remove.
|
||||
reapers = append(reapers, EC2AccountSettingsReapers(nc, nodes.DaemonIDs)...)
|
||||
// Independent of the EC2 seal: either failing keeps the account TERMINATING.
|
||||
reapers = append(reapers, QuotaOverrideSealReapers(nc, nodes.GatewayIDs)...)
|
||||
reapers = append(reapers, EC2AccountSettingsReapers(nc, nodeIDs)...)
|
||||
reapers = append(reapers, AccountReapers(svc, names, usage)...)
|
||||
|
||||
engine := NewEngine(NewIAMAccounts(svc), reapers...)
|
||||
|
||||
@@ -12,10 +12,6 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func testClusterNodes() ClusterNodes {
|
||||
return ClusterNodes{Expected: 1, DaemonIDs: []string{"node-1"}, GatewayIDs: []string{"node-1"}}
|
||||
}
|
||||
|
||||
// The CLI and the admin API share this wiring precisely so neither can be
|
||||
// missing a reaper. A stage with none registered would silently skip whatever
|
||||
// it owns, leaving resources behind that nothing else deletes.
|
||||
@@ -27,13 +23,13 @@ func TestNewClusterEngineCoversEveryStage(t *testing.T) {
|
||||
svc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
engine, err := NewClusterEngine(t.Context(), nc, testClusterNodes(), svc, newFakeBuckets())
|
||||
engine, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, engine.Accounts)
|
||||
|
||||
// A teardown with no object store would report every stage drained and
|
||||
// still delete the account record, leaving the tenant's objects behind.
|
||||
_, err = NewClusterEngine(t.Context(), nc, testClusterNodes(), svc, nil)
|
||||
_, err = NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, nil)
|
||||
require.Error(t, err)
|
||||
|
||||
for _, stage := range Stages() {
|
||||
@@ -56,7 +52,7 @@ func TestNewClusterEngineRegistersTheEC2SettingsSealReaper(t *testing.T) {
|
||||
svc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
engine, err := NewClusterEngine(t.Context(), nc, testClusterNodes(), svc, newFakeBuckets())
|
||||
engine, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
|
||||
require.NoError(t, err)
|
||||
|
||||
var kinds []string
|
||||
@@ -66,37 +62,6 @@ func TestNewClusterEngineRegistersTheEC2SettingsSealReaper(t *testing.T) {
|
||||
assert.Contains(t, kinds, "ec2-account-settings")
|
||||
}
|
||||
|
||||
// The quota seal reaper must be registered in StageAccount on every surface,
|
||||
// and must probe the gateway nodes, not the daemon nodes: a swapped set would
|
||||
// compile and probe processes that never write an override.
|
||||
func TestNewClusterEngineRegistersTheQuotaOverrideSealReaper(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
masterKey, err := iamdomain.GenerateMasterKey()
|
||||
require.NoError(t, err)
|
||||
svc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
nodes := ClusterNodes{Expected: 3, DaemonIDs: []string{"d-1", "d-2", "d-3"}, GatewayIDs: []string{"gw-1"}}
|
||||
engine, err := NewClusterEngine(t.Context(), nc, nodes, svc, newFakeBuckets())
|
||||
require.NoError(t, err)
|
||||
|
||||
var quotaReaper *quotaOverrideSealReaper
|
||||
var ec2Reaper *accountSettingsSealReaper
|
||||
for _, reaper := range engine.reapersFor(StageAccount) {
|
||||
switch r := reaper.(type) {
|
||||
case *quotaOverrideSealReaper:
|
||||
quotaReaper = r
|
||||
case *accountSettingsSealReaper:
|
||||
ec2Reaper = r
|
||||
}
|
||||
}
|
||||
require.NotNil(t, quotaReaper, "quota-override reaper not registered in StageAccount")
|
||||
require.NotNil(t, ec2Reaper)
|
||||
assert.Equal(t, []string{"gw-1"}, quotaReaper.capability.(*sealCapabilityClient).nodeIDs)
|
||||
assert.Equal(t, []string{"d-1", "d-2", "d-3"}, ec2Reaper.capability.(*sealCapabilityClient).nodeIDs)
|
||||
}
|
||||
|
||||
// A teardown must refuse the accounts no credential may delete before it reads
|
||||
// anything, and refuse a name confirmation that does not match the record.
|
||||
func TestClusterEnginePrechecks(t *testing.T) {
|
||||
@@ -107,7 +72,7 @@ func TestClusterEnginePrechecks(t *testing.T) {
|
||||
svc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
engine, err := NewClusterEngine(t.Context(), nc, testClusterNodes(), svc, newFakeBuckets())
|
||||
engine, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
|
||||
require.NoError(t, err)
|
||||
|
||||
// Account ids are sequential and the first is the protected super admin.
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
admissionv1 "github.com/mulgadc/spinifex/contracts/admission/v1"
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
@@ -67,40 +66,12 @@ type sealCapabilityClient struct {
|
||||
nodeIDs []string
|
||||
wait time.Duration // 3 s, shortened by the caller's deadline
|
||||
grace time.Duration // 150 ms ResponderGrace
|
||||
subject func(node string) string
|
||||
valid func(node string) bool
|
||||
}
|
||||
|
||||
// sealCapabilityRequest and sealCapabilityReply are the wire shape every seal
|
||||
// capability contract shares: the node asked about, the version it reports and
|
||||
// an opaque correlation reference.
|
||||
type sealCapabilityRequest struct {
|
||||
Node string `json:"node"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
|
||||
type sealCapabilityReply struct {
|
||||
Node string `json:"node"`
|
||||
CapabilityVersion int `json:"capability_version"`
|
||||
CorrelationRef string `json:"correlation_ref,omitempty"`
|
||||
}
|
||||
|
||||
var _ sealCapabilityChecker = (*sealCapabilityClient)(nil)
|
||||
|
||||
func newSealCapabilityClient(nc *nats.Conn, nodeIDs []string) *sealCapabilityClient {
|
||||
return &sealCapabilityClient{
|
||||
nc: nc, nodeIDs: nodeIDs, wait: sealCapabilityWait, grace: sealCapabilityGrace,
|
||||
subject: ec2v1.AccountSettingsSealCapabilitySubject, valid: ec2v1.ValidSealCapabilityNode,
|
||||
}
|
||||
}
|
||||
|
||||
// newQuotaSealCapabilityClient probes the quota override seal capability of
|
||||
// every node in nodeIDs, which are the configured nodes that run awsgw.
|
||||
func newQuotaSealCapabilityClient(nc *nats.Conn, nodeIDs []string) *sealCapabilityClient {
|
||||
return &sealCapabilityClient{
|
||||
nc: nc, nodeIDs: nodeIDs, wait: sealCapabilityWait, grace: sealCapabilityGrace,
|
||||
subject: admissionv1.AccountQuotaOverrideSealCapabilitySubject, valid: admissionv1.ValidSealCapabilityNode,
|
||||
}
|
||||
return &sealCapabilityClient{nc: nc, nodeIDs: nodeIDs, wait: sealCapabilityWait, grace: sealCapabilityGrace}
|
||||
}
|
||||
|
||||
// CheckSealCapability probes every configured node in parallel, bounded by
|
||||
@@ -151,16 +122,16 @@ func (c *sealCapabilityClient) CheckSealCapability(ctx context.Context, correlat
|
||||
// only when exactly one identified, decodable, matching, version >= 1 frame
|
||||
// arrived, with no duplicate, conflict or cap hit.
|
||||
func (c *sealCapabilityClient) checkNode(ctx context.Context, node, correlationRef string, wait time.Duration) nodeSealCapability {
|
||||
if !c.valid(node) {
|
||||
if !ec2v1.ValidSealCapabilityNode(node) {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonInvalidNodeID}
|
||||
}
|
||||
|
||||
data, err := json.Marshal(sealCapabilityRequest{Node: node, CorrelationRef: correlationRef})
|
||||
data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: node, CorrelationRef: correlationRef})
|
||||
if err != nil {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonMalformed}
|
||||
}
|
||||
|
||||
frames, sum, err := natsmsg.Gather(ctx, c.nc, c.subject(node), data, natsmsg.GatherOpts{
|
||||
frames, sum, err := natsmsg.Gather(ctx, c.nc, ec2v1.AccountSettingsSealCapabilitySubject(node), data, natsmsg.GatherOpts{
|
||||
Timeout: wait,
|
||||
ExpectedResponders: 1,
|
||||
Mode: natsmsg.CollectUntilDeadline,
|
||||
@@ -186,7 +157,7 @@ func (c *sealCapabilityClient) checkNode(ctx context.Context, node, correlationR
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
|
||||
}
|
||||
|
||||
var reply sealCapabilityReply
|
||||
var reply ec2v1.AccountSettingsSealCapabilityReply
|
||||
if json.Unmarshal(frame.Data, &reply) != nil {
|
||||
return nodeSealCapability{Node: node, Reason: sealCapabilityReasonMalformed}
|
||||
}
|
||||
@@ -202,15 +173,6 @@ func (c *sealCapabilityClient) checkNode(ctx context.Context, node, correlationR
|
||||
// errSealCapabilityIncomplete builds the retryable refusal from report. It
|
||||
// names node IDs and reason categories only, never wire error text.
|
||||
func errSealCapabilityIncomplete(report sealCapabilityReport) error {
|
||||
return errCapabilityIncomplete("EC2 account-settings seal capability", report)
|
||||
}
|
||||
|
||||
// errQuotaSealCapabilityIncomplete is the quota override seal's refusal.
|
||||
func errQuotaSealCapabilityIncomplete(report sealCapabilityReport) error {
|
||||
return errCapabilityIncomplete("quota override seal capability", report)
|
||||
}
|
||||
|
||||
func errCapabilityIncomplete(what string, report sealCapabilityReport) error {
|
||||
detail := sealCapabilityReasonSetUnknown
|
||||
var parts []string
|
||||
for _, n := range report.Nodes {
|
||||
@@ -222,7 +184,7 @@ func errCapabilityIncomplete(what string, report sealCapabilityReport) error {
|
||||
detail = strings.Join(parts, ", ")
|
||||
}
|
||||
return awserrors.WithRetryAfter(
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable, "%s incomplete: %s", what, detail),
|
||||
awserrors.Errorf(awserrors.ErrorServiceUnavailable, "EC2 account-settings seal capability incomplete: %s", detail),
|
||||
5*time.Second,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -169,7 +169,7 @@ func TestOnlyTheExplicitDryRunCallsTheEC2PlatformInventory(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
svc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, masterKey)
|
||||
require.NoError(t, err)
|
||||
cluster, err := NewClusterEngine(t.Context(), nc, testClusterNodes(), svc, newFakeBuckets())
|
||||
cluster, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
|
||||
require.NoError(t, err)
|
||||
assert.IsType(t, &ec2PlatformInventoryClient{}, cluster.ec2Inventory)
|
||||
|
||||
|
||||
@@ -187,15 +187,6 @@ func (e *Engine) runStage(ctx context.Context, stage Stage, req Request) (StageR
|
||||
return result, nil
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
// remaining was listed before this pass's deletes, so list again:
|
||||
// a resource deleted on the last pass is gone, not stuck.
|
||||
if remaining, err = e.list(ctx, reapers, req); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if len(remaining) == 0 {
|
||||
result.Elapsed = time.Since(started).String()
|
||||
return result, nil
|
||||
}
|
||||
for _, resource := range remaining {
|
||||
reason := reasons[resource.Kind+"/"+resource.ID]
|
||||
if reason == "" {
|
||||
@@ -217,19 +208,6 @@ func (e *Engine) runStage(ctx context.Context, stage Stage, req Request) (StageR
|
||||
}
|
||||
}
|
||||
|
||||
// list returns what every reaper in the stage still lists, deleting nothing.
|
||||
func (e *Engine) list(ctx context.Context, reapers []Reaper, req Request) ([]Resource, error) {
|
||||
var remaining []Resource
|
||||
for _, reaper := range reapers {
|
||||
found, err := reaper.List(ctx, req.AccountID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list %s: %w", reaper.Kind(), err)
|
||||
}
|
||||
remaining = append(remaining, found...)
|
||||
}
|
||||
return remaining, nil
|
||||
}
|
||||
|
||||
// sweep lists every reaper in the stage and issues a delete for whatever it
|
||||
// finds, returning what was still listed at the start of this pass.
|
||||
func (e *Engine) sweep(
|
||||
|
||||
@@ -296,26 +296,6 @@ func TestStuckResourceIsReportedAndTheAccountSurvives(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A pass that runs past the stage deadline can still delete everything it
|
||||
// listed. Those resources are gone, so they must not be reported stuck.
|
||||
func TestResourceDeletedOnThePassThatCrossesTheDeadlineIsNotStuck(t *testing.T) {
|
||||
accounts := newFakeAccounts()
|
||||
volumes := &fakeReaper{
|
||||
kind: "volume", stage: StageStorage,
|
||||
resources: []Resource{{Kind: "volume", ID: "vol-1"}},
|
||||
}
|
||||
engine := quietEngine(accounts, volumes)
|
||||
engine.Timeouts.StageDrain = time.Nanosecond
|
||||
|
||||
result, err := engine.Teardown(context.Background(), request())
|
||||
if err != nil {
|
||||
t.Fatalf("teardown: %v (stuck %d)", err, result.StuckCount())
|
||||
}
|
||||
if !accounts.deleted || volumes.deletesAttempted != 1 {
|
||||
t.Fatalf("want one delete and the account gone, got deletes=%d deleted=%v", volumes.deletesAttempted, accounts.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// The reason a resource would not go is the useful half of the report.
|
||||
func TestStuckResourceCarriesTheDeleteFailureReason(t *testing.T) {
|
||||
volumes := &fakeReaper{
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
package accountlifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
admissionv1 "github.com/mulgadc/spinifex/contracts/admission/v1"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// quotaOverrideSealWait bounds one seal request; an earlier caller deadline wins.
|
||||
const quotaOverrideSealWait = 30 * time.Second
|
||||
|
||||
// quotaOverrideSealer is the quota owner's seal of one account's override
|
||||
// record.
|
||||
type quotaOverrideSealer interface {
|
||||
SealAccountQuotaOverride(ctx context.Context, accountID string) (revision uint64, err error)
|
||||
}
|
||||
|
||||
// quotaOverrideSealClient reaches the quota override seal contract over NATS.
|
||||
// Only a reply for this account, disposition sealed and a non-zero revision is
|
||||
// a seal; anything else, including no responder, is an error.
|
||||
type quotaOverrideSealClient struct {
|
||||
nc *nats.Conn
|
||||
wait time.Duration
|
||||
}
|
||||
|
||||
var _ quotaOverrideSealer = (*quotaOverrideSealClient)(nil)
|
||||
|
||||
func newQuotaOverrideSealClient(nc *nats.Conn) *quotaOverrideSealClient {
|
||||
return "aOverrideSealClient{nc: nc, wait: quotaOverrideSealWait}
|
||||
}
|
||||
|
||||
func (c *quotaOverrideSealClient) SealAccountQuotaOverride(ctx context.Context, accountID string) (uint64, error) {
|
||||
wait := c.wait
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
wait = min(wait, time.Until(deadline))
|
||||
}
|
||||
if ctx.Err() != nil || wait <= 0 {
|
||||
return 0, fmt.Errorf("seal quota override: %w", context.DeadlineExceeded)
|
||||
}
|
||||
|
||||
reply, err := natsmsg.NATSRequest[admissionv1.SealAccountQuotaOverrideReply](ctx, c.nc, admissionv1.SealAccountQuotaOverrideSubject,
|
||||
&admissionv1.SealAccountQuotaOverrideRequest{AccountID: accountID}, wait, accountID)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("seal quota override: %w", err)
|
||||
}
|
||||
if reply.AccountID != accountID {
|
||||
return 0, fmt.Errorf("seal quota override: reply named account %q, not %q", reply.AccountID, accountID)
|
||||
}
|
||||
if reply.Disposition != admissionv1.AccountQuotaOverrideDispositionSealed {
|
||||
return 0, fmt.Errorf("seal quota override: unexpected disposition %q", reply.Disposition)
|
||||
}
|
||||
if reply.Revision == 0 {
|
||||
return 0, errors.New("seal quota override: reply carried no revision")
|
||||
}
|
||||
return reply.Revision, nil
|
||||
}
|
||||
|
||||
// quotaOverrideSealReaper seals the quota owner's override record in
|
||||
// StageAccount. List does no I/O and stays pending until this instance has
|
||||
// sealed, so DryRun never seals; state is per engine, so a fresh run seals again.
|
||||
type quotaOverrideSealReaper struct {
|
||||
sealer quotaOverrideSealer
|
||||
capability sealCapabilityChecker
|
||||
|
||||
mu sync.Mutex
|
||||
sealed map[string]uint64 // account ID -> leader revision, this run only
|
||||
}
|
||||
|
||||
func (r *quotaOverrideSealReaper) Kind() string { return "quota-override" }
|
||||
func (r *quotaOverrideSealReaper) Stage() Stage { return StageAccount }
|
||||
|
||||
func (r *quotaOverrideSealReaper) alreadySealed(accountID string) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
_, ok := r.sealed[accountID]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (r *quotaOverrideSealReaper) List(_ context.Context, accountID string) ([]Resource, error) {
|
||||
if r.alreadySealed(accountID) {
|
||||
return nil, nil
|
||||
}
|
||||
return []Resource{{Kind: r.Kind(), ID: accountID, Detail: "seal"}}, nil
|
||||
}
|
||||
|
||||
// Delete probes every configured gateway immediately before each seal attempt,
|
||||
// never reusing an earlier probe. Incomplete capability sends no seal and is
|
||||
// retryable; force is ignored, since nothing may force past the probe.
|
||||
func (r *quotaOverrideSealReaper) Delete(ctx context.Context, accountID string, _ Resource, _ bool) error {
|
||||
report := r.capability.CheckSealCapability(ctx, accountID)
|
||||
if !report.Complete {
|
||||
return errQuotaSealCapabilityIncomplete(report)
|
||||
}
|
||||
|
||||
revision, err := r.sealer.SealAccountQuotaOverride(ctx, accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
if r.sealed == nil {
|
||||
r.sealed = map[string]uint64{}
|
||||
}
|
||||
r.sealed[accountID] = revision
|
||||
r.mu.Unlock()
|
||||
slog.InfoContext(ctx, "Account teardown sealed quota override", "accountID", accountID, "revision", revision)
|
||||
return nil
|
||||
}
|
||||
|
||||
// QuotaOverrideSealReapers returns the StageAccount reaper that seals the
|
||||
// account's quota override record. gatewayNodeIDs is the configured set of
|
||||
// nodes that run awsgw, the only processes that write overrides; an empty set
|
||||
// is unknown, never complete.
|
||||
func QuotaOverrideSealReapers(nc *nats.Conn, gatewayNodeIDs []string) []Reaper {
|
||||
return []Reaper{"aOverrideSealReaper{
|
||||
sealer: newQuotaOverrideSealClient(nc),
|
||||
capability: newQuotaSealCapabilityClient(nc, gatewayNodeIDs),
|
||||
}}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package accountlifecycle
|
||||
|
||||
//test:in-package — the probe client and its reasons are unexported; this pins that the
|
||||
// quota probe addresses the admission subjects and reuses the shared classification.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
admissionv1 "github.com/mulgadc/spinifex/contracts/admission/v1"
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func testQuotaCapabilityClient(nc *nats.Conn, nodeIDs []string) *sealCapabilityClient {
|
||||
c := newQuotaSealCapabilityClient(nc, nodeIDs)
|
||||
c.wait = 200 * time.Millisecond
|
||||
c.grace = 30 * time.Millisecond
|
||||
return c
|
||||
}
|
||||
|
||||
func quotaCapabilityReply(replyNode, headerNode string, version int) nats.MsgHandler {
|
||||
return func(msg *nats.Msg) {
|
||||
data, _ := json.Marshal(admissionv1.AccountQuotaOverrideSealCapabilityReply{Node: replyNode, CapabilityVersion: version})
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = data
|
||||
if headerNode != "" {
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, headerNode)
|
||||
}
|
||||
_ = msg.RespondMsg(reply)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuotaSealCapabilityComplete(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
for _, node := range []string{"gw-1", "gw-2"} {
|
||||
serveCapability(t, nc, admissionv1.AccountQuotaOverrideSealCapabilitySubject(node), quotaCapabilityReply(node, node, 1))
|
||||
}
|
||||
report := testQuotaCapabilityClient(nc, []string{"gw-2", "gw-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
assert.True(t, report.Complete)
|
||||
assert.Equal(t, 1, nodeResult(t, report, "gw-1").Version)
|
||||
assert.Equal(t, 1, nodeResult(t, report, "gw-2").Version)
|
||||
}
|
||||
|
||||
func TestQuotaSealCapabilityIncomplete(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
serve func(t *testing.T, nc *nats.Conn)
|
||||
nodes []string
|
||||
reason string
|
||||
}{
|
||||
{"missing", func(*testing.T, *nats.Conn) {}, []string{"gw-1"}, sealCapabilityReasonNoReply},
|
||||
{"duplicate", func(t *testing.T, nc *nats.Conn) {
|
||||
subject := admissionv1.AccountQuotaOverrideSealCapabilitySubject("gw-1")
|
||||
serveCapability(t, nc, subject, quotaCapabilityReply("gw-1", "gw-1", 1))
|
||||
serveCapability(t, nc, subject, quotaCapabilityReply("gw-1", "gw-1", 1))
|
||||
}, []string{"gw-1"}, sealCapabilityReasonDuplicate},
|
||||
{"no node header", func(t *testing.T, nc *nats.Conn) {
|
||||
serveCapability(t, nc, admissionv1.AccountQuotaOverrideSealCapabilitySubject("gw-1"), quotaCapabilityReply("gw-1", "", 1))
|
||||
}, []string{"gw-1"}, sealCapabilityReasonWrongIdentity},
|
||||
{"body names another node", func(t *testing.T, nc *nats.Conn) {
|
||||
serveCapability(t, nc, admissionv1.AccountQuotaOverrideSealCapabilitySubject("gw-1"), quotaCapabilityReply("gw-2", "gw-1", 1))
|
||||
}, []string{"gw-1"}, sealCapabilityReasonWrongIdentity},
|
||||
{"version zero", func(t *testing.T, nc *nats.Conn) {
|
||||
serveCapability(t, nc, admissionv1.AccountQuotaOverrideSealCapabilitySubject("gw-1"), quotaCapabilityReply("gw-1", "gw-1", 0))
|
||||
}, []string{"gw-1"}, sealCapabilityReasonOldVersion},
|
||||
{"invalid node id", func(*testing.T, *nats.Conn) {}, []string{"gw.1"}, sealCapabilityReasonInvalidNodeID},
|
||||
// The EC2 responder on a daemon is not the gateway's quota capability.
|
||||
{"only the ec2 subject answers", func(t *testing.T, nc *nats.Conn) {
|
||||
serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("gw-1"), capabilityReply("gw-1", "gw-1", 1))
|
||||
}, []string{"gw-1"}, sealCapabilityReasonNoReply},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
tc.serve(t, nc)
|
||||
report := testQuotaCapabilityClient(nc, tc.nodes).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Equal(t, tc.reason, nodeResult(t, report, tc.nodes[0]).Reason)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuotaSealCapabilityEmptySetIsIncomplete(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
report := testQuotaCapabilityClient(nc, nil).CheckSealCapability(testCtx(t), "req")
|
||||
assert.False(t, report.Complete)
|
||||
assert.Contains(t, errQuotaSealCapabilityIncomplete(report).Error(), "quota override seal capability incomplete: "+sealCapabilityReasonSetUnknown)
|
||||
}
|
||||
|
||||
func TestQuotaSealCapabilityIncompleteErrorNamesNodeAndReason(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
report := testQuotaCapabilityClient(nc, []string{"gw-1"}).CheckSealCapability(testCtx(t), "req")
|
||||
err := errQuotaSealCapabilityIncomplete(report)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "quota override seal capability incomplete: gw-1="+sealCapabilityReasonNoReply)
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
package accountlifecycle
|
||||
|
||||
//test:in-package — the reaper, sealer and capability checker are unexported;
|
||||
// no seal without a complete probe, completion only after a sealed reply and
|
||||
// idempotent fresh runs are only observable from inside the package.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
admissionv1 "github.com/mulgadc/spinifex/contracts/admission/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// fakeQuotaSealer is a recording quotaOverrideSealer. Each call beyond failFor
|
||||
// succeeds with a fresh, strictly increasing revision, unless err is set, in
|
||||
// which case every call after failFor fails with it.
|
||||
type fakeQuotaSealer struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
failFor int
|
||||
err error
|
||||
revision uint64
|
||||
}
|
||||
|
||||
func (f *fakeQuotaSealer) SealAccountQuotaOverride(_ context.Context, accountID string) (uint64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, accountID)
|
||||
if len(f.calls) <= f.failFor {
|
||||
return 0, errors.New("quota owner temporarily unavailable")
|
||||
}
|
||||
if f.err != nil {
|
||||
return 0, f.err
|
||||
}
|
||||
f.revision++
|
||||
return f.revision, nil
|
||||
}
|
||||
|
||||
func incompleteReport(node string) sealCapabilityReport {
|
||||
return sealCapabilityReport{Nodes: []nodeSealCapability{{Node: node, Reason: sealCapabilityReasonNoReply}}}
|
||||
}
|
||||
|
||||
// No subscriber on either subject makes the capability probe incomplete, so no
|
||||
// seal request is sent and the account stays TERMINATING with one stuck
|
||||
// quota-override item.
|
||||
func TestQuotaSealReaperNoResponderKeepsAccountTerminating(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
|
||||
capability := newQuotaSealCapabilityClient(nc, []string{"gw-1"})
|
||||
capability.wait = 50 * time.Millisecond
|
||||
capability.grace = 10 * time.Millisecond
|
||||
sealer := newQuotaOverrideSealClient(nc)
|
||||
sealer.wait = 50 * time.Millisecond
|
||||
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, "aOverrideSealReaper{sealer: sealer, capability: capability})
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.ErrorIs(t, err, ErrResourcesStuck)
|
||||
assert.False(t, result.AccountDeleted)
|
||||
assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
|
||||
|
||||
var stuck []Stuck
|
||||
for _, stage := range result.Stages {
|
||||
stuck = append(stuck, stage.Stuck...)
|
||||
}
|
||||
require.Len(t, stuck, 1)
|
||||
assert.Equal(t, "quota-override", stuck[0].Resource.Kind)
|
||||
}
|
||||
|
||||
// An incomplete probe sends no seal at all.
|
||||
func TestQuotaSealReaperIncompleteCapabilitySendsNoSeal(t *testing.T) {
|
||||
sealer := &fakeQuotaSealer{}
|
||||
reaper := "aOverrideSealReaper{sealer: sealer, capability: &fakeSealCapability{report: incompleteReport("gw-1")}}
|
||||
|
||||
err := reaper.Delete(testCtx(t), testAccountID, Resource{}, true)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, awserrors.ErrorServiceUnavailable, awserrors.ValidErrorCodeFromError(err))
|
||||
assert.Contains(t, err.Error(), "quota override seal capability incomplete: gw-1="+sealCapabilityReasonNoReply)
|
||||
assert.Empty(t, sealer.calls)
|
||||
}
|
||||
|
||||
// Whatever the owner refuses with, the account stays TERMINATING: an owner
|
||||
// refusal, a timeout, and each reply shape the client rejects.
|
||||
func TestQuotaSealReaperOwnerFailureKeepsAccountTerminating(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{"incorrect-state", errors.New(awserrors.ErrorIncorrectState)},
|
||||
{"internal-error", errors.New(awserrors.ErrorInternalError)},
|
||||
{"timeout", context.DeadlineExceeded},
|
||||
{"wrong-account", errors.New(`seal quota override: reply named account "999999999999", not "000000000042"`)},
|
||||
{"wrong-disposition", errors.New(`seal quota override: unexpected disposition "pending"`)},
|
||||
{"zero-revision", errors.New("seal quota override: reply carried no revision")},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
sealer := &fakeQuotaSealer{err: tc.err}
|
||||
reaper := "aOverrideSealReaper{sealer: sealer, capability: &fakeSealCapability{report: completeReport("gw-1")}}
|
||||
accounts := newFakeAccounts()
|
||||
engine := quietEngine(accounts, reaper)
|
||||
|
||||
result, err := engine.Teardown(testCtx(t), request())
|
||||
require.ErrorIs(t, err, ErrResourcesStuck)
|
||||
assert.False(t, result.AccountDeleted)
|
||||
assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
|
||||
assert.NotEmpty(t, sealer.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The client rejects every reply that is not a seal of this account.
|
||||
func TestQuotaSealClientRejectsMalformedReplies(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
reply admissionv1.SealAccountQuotaOverrideReply
|
||||
want string
|
||||
}{
|
||||
{"wrong account", admissionv1.SealAccountQuotaOverrideReply{AccountID: "999999999999", Disposition: admissionv1.AccountQuotaOverrideDispositionSealed, Revision: 3}, "reply named account"},
|
||||
{"wrong disposition", admissionv1.SealAccountQuotaOverrideReply{AccountID: testAccountID, Disposition: "pending", Revision: 3}, "unexpected disposition"},
|
||||
{"zero revision", admissionv1.SealAccountQuotaOverrideReply{AccountID: testAccountID, Disposition: admissionv1.AccountQuotaOverrideDispositionSealed}, "no revision"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, nc, _ := testutil.StartTestJetStream(t)
|
||||
sub, err := nc.Subscribe(admissionv1.SealAccountQuotaOverrideSubject, func(msg *nats.Msg) {
|
||||
data, _ := json.Marshal(tc.reply)
|
||||
_ = msg.Respond(data)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
_, err = newQuotaOverrideSealClient(nc).SealAccountQuotaOverride(testCtx(t), testAccountID)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A transient owner failure is retried by the engine's next pass and then seals.
|
||||
func TestQuotaSealReaperRetriesThenSeals(t *testing.T) {
|
||||
sealer := &fakeQuotaSealer{failFor: 1}
|
||||
capability := &fakeSealCapability{report: completeReport("gw-1")}
|
||||
reaper := "aOverrideSealReaper{sealer: sealer, capability: capability}
|
||||
|
||||
require.Error(t, reaper.Delete(testCtx(t), testAccountID, Resource{}, false))
|
||||
listed, err := reaper.List(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, listed, 1, "a failed seal leaves the item pending")
|
||||
|
||||
require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{}, false))
|
||||
listed, err = reaper.List(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, listed)
|
||||
}
|
||||
|
||||
// The probe runs immediately before every seal attempt and is never reused.
|
||||
func TestQuotaSealReaperProbedBeforeEverySeal(t *testing.T) {
|
||||
sealer := &fakeQuotaSealer{failFor: 2}
|
||||
capability := &fakeSealCapability{report: completeReport("gw-1")}
|
||||
reaper := "aOverrideSealReaper{sealer: sealer, capability: capability}
|
||||
|
||||
for range 3 {
|
||||
_ = reaper.Delete(testCtx(t), testAccountID, Resource{}, false)
|
||||
}
|
||||
assert.Equal(t, 3, capability.calls)
|
||||
assert.Len(t, sealer.calls, 3)
|
||||
}
|
||||
|
||||
// Inventory and DryRun list the item without sealing or probing anything.
|
||||
func TestQuotaSealReaperListDoesNoIO(t *testing.T) {
|
||||
sealer := &fakeQuotaSealer{}
|
||||
capability := &fakeSealCapability{report: completeReport("gw-1")}
|
||||
reaper := "aOverrideSealReaper{sealer: sealer, capability: capability}
|
||||
|
||||
listed, err := reaper.List(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, listed, 1)
|
||||
assert.Equal(t, "quota-override", listed[0].Kind)
|
||||
assert.Equal(t, testAccountID, listed[0].ID)
|
||||
assert.Empty(t, sealer.calls)
|
||||
assert.Zero(t, capability.calls)
|
||||
}
|
||||
|
||||
// State is per engine: a fresh reaper lists and seals again.
|
||||
func TestQuotaSealReaperFreshRunSealsAgain(t *testing.T) {
|
||||
sealer := &fakeQuotaSealer{}
|
||||
capability := &fakeSealCapability{report: completeReport("gw-1")}
|
||||
|
||||
first := "aOverrideSealReaper{sealer: sealer, capability: capability}
|
||||
require.NoError(t, first.Delete(testCtx(t), testAccountID, Resource{}, false))
|
||||
|
||||
second := "aOverrideSealReaper{sealer: sealer, capability: capability}
|
||||
listed, err := second.List(testCtx(t), testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, listed, 1)
|
||||
require.NoError(t, second.Delete(testCtx(t), testAccountID, Resource{}, false))
|
||||
assert.Equal(t, []string{testAccountID, testAccountID}, sealer.calls)
|
||||
}
|
||||
+16
-30
@@ -1,9 +1,8 @@
|
||||
package accountlifecycle
|
||||
|
||||
//test:in-package — drives the real final-stage reapers and the seal reaper against the real quota service to pin that teardown seals the override record.
|
||||
//test:in-package — drives the real final-stage reapers against the real quota service to pin that teardown leaves the override bucket alone.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
@@ -15,23 +14,12 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// serviceSealer seals through the real quota service, standing in for the
|
||||
// awsgw responder that owns the override record.
|
||||
type serviceSealer struct{ svc *quota.Service }
|
||||
|
||||
func (s serviceSealer) SealAccountQuotaOverride(ctx context.Context, accountID string) (uint64, error) {
|
||||
seal, err := s.svc.SealAccountOverrides(ctx, accountID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return seal.Revision, nil
|
||||
}
|
||||
|
||||
// Running the real engine with the seal reaper and the real StageAccount
|
||||
// reapers against an account with a raised vCPU override: the usage counter is
|
||||
// removed as before, and the override record outlives the account only as a
|
||||
// sealed tombstone that no writer can change or recreate.
|
||||
func TestAccountTermination_QuotaOverrideEndsSealed(t *testing.T) {
|
||||
// No reaper in this package references the quota override bucket or calls an
|
||||
// override delete. Running the real engine and the real StageAccount reapers
|
||||
// against an account with a raised vCPU override, with usage wired, shows the
|
||||
// consequence: the override outlives the account record it belongs to, while
|
||||
// the usage counter the reapers do own is removed.
|
||||
func TestCurrentBehaviour_AccountTermination_QuotaOverridesSurviveTeardown(t *testing.T) {
|
||||
_, _, js := testutil.StartTestJetStream(t)
|
||||
ctx := testCtx(t)
|
||||
|
||||
@@ -44,17 +32,12 @@ func TestAccountTermination_QuotaOverrideEndsSealed(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
svc := quota.New(quota.Limits{Enabled: true, VCPUs: 16}, usage)
|
||||
svc.SetOverrides(overridesKV)
|
||||
svc.SetAccountTerminationChecker(alwaysTerminatingChecker{})
|
||||
unlimited := quota.Unlimited
|
||||
require.NoError(t, svc.PutAccountQuota(ctx, testAccountID, quota.Overrides{VCPUs: &unlimited}, "operator"))
|
||||
|
||||
accounts := newFakeAccounts()
|
||||
iam := &accountIAM{account: &iamdomain.Account{AccountID: testAccountID, AccountName: testAccountName}}
|
||||
reapers := append([]Reaper{"aOverrideSealReaper{
|
||||
sealer: serviceSealer{svc: svc},
|
||||
capability: &fakeSealCapability{report: completeReport("gw-1")},
|
||||
}}, AccountReapers(iam, nil, usage)...)
|
||||
engine := quietEngine(accounts, reapers...)
|
||||
engine := quietEngine(accounts, AccountReapers(iam, nil, usage)...)
|
||||
|
||||
result, err := engine.Teardown(ctx, request())
|
||||
require.NoError(t, err)
|
||||
@@ -62,14 +45,17 @@ func TestAccountTermination_QuotaOverrideEndsSealed(t *testing.T) {
|
||||
assert.True(t, accounts.deleted)
|
||||
assert.Zero(t, result.StuckCount())
|
||||
|
||||
// The reaper this package does own removed the usage counter, which shows
|
||||
// which store teardown actually clears.
|
||||
_, err = usage.Get(ctx, testAccountID)
|
||||
assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "teardown's quota-usage reaper must remove the vCPU counter")
|
||||
|
||||
_, err = overridesKV.Get(ctx, testAccountID)
|
||||
require.NoError(t, err, "the sealed record stays as the tombstone")
|
||||
require.NoError(t, err, "the override key must still be present after the account is torn down")
|
||||
|
||||
err = svc.PutAccountQuota(ctx, testAccountID, quota.Overrides{VCPUs: &unlimited}, "operator")
|
||||
require.Error(t, err, "a sealed record must refuse a later override write")
|
||||
_, _, err = svc.GetAccountQuota(ctx, testAccountID)
|
||||
require.Error(t, err, "a sealed record must not yield the old override")
|
||||
over, effective, err := svc.GetAccountQuota(ctx, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, over.VCPUs)
|
||||
assert.Equal(t, quota.Unlimited, *over.VCPUs)
|
||||
assert.Equal(t, quota.Unlimited, effective.VCPUs)
|
||||
}
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
package accountlifecycle
|
||||
|
||||
//test:in-package — the probe's local wire shape is unexported; this pins that it
|
||||
// still decodes both contracts, so a renamed contract tag cannot silently
|
||||
// disable a seal.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
admissionv1 "github.com/mulgadc/spinifex/contracts/admission/v1"
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// The local shape must read exactly what each contract's responder writes.
|
||||
func TestSealCapabilityLocalReplyDecodesBothContracts(t *testing.T) {
|
||||
want := sealCapabilityReply{Node: "node-1", CapabilityVersion: 7, CorrelationRef: "req-1"}
|
||||
|
||||
ec2Data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 7, CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
admissionData, err := json.Marshal(admissionv1.AccountQuotaOverrideSealCapabilityReply{Node: "node-1", CapabilityVersion: 7, CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
for name, data := range map[string][]byte{"ec2v1": ec2Data, "admissionv1": admissionData} {
|
||||
var got sealCapabilityReply
|
||||
require.NoError(t, json.Unmarshal(data, &got), name)
|
||||
assert.Equal(t, want, got, name)
|
||||
}
|
||||
}
|
||||
|
||||
// Each contract's responder must read exactly what the local shape sends.
|
||||
func TestSealCapabilityLocalRequestDecodesInBothContracts(t *testing.T) {
|
||||
data, err := json.Marshal(sealCapabilityRequest{Node: "node-1", CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
var ec2Got ec2v1.AccountSettingsSealCapabilityRequest
|
||||
require.NoError(t, json.Unmarshal(data, &ec2Got))
|
||||
assert.Equal(t, ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-1", CorrelationRef: "req-1"}, ec2Got)
|
||||
|
||||
var admissionGot admissionv1.AccountQuotaOverrideSealCapabilityRequest
|
||||
require.NoError(t, json.Unmarshal(data, &admissionGot))
|
||||
assert.Equal(t, admissionv1.AccountQuotaOverrideSealCapabilityRequest{Node: "node-1", CorrelationRef: "req-1"}, admissionGot)
|
||||
}
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -30,17 +29,6 @@ var forbiddenSettingsImports = []string{
|
||||
// contracts/ec2/v1 and NATS.
|
||||
var forbiddenSettingsIdentifiers = []string{"spinifex-ec2-account-settings", "KVBucketAccountSettings"}
|
||||
|
||||
// quotaPackage is the quota owner. The legacy engine may name it only for the
|
||||
// usage bucket it creates; the override record is reached over
|
||||
// contracts/admission/v1 and NATS alone.
|
||||
const quotaPackage = "github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
|
||||
|
||||
// forbiddenQuotaSelectors would let this package open, read, write or seal
|
||||
// the override record, or build the quota service, instead of asking the owner.
|
||||
var forbiddenQuotaSelectors = []string{"KVBucketAccountQuota", "SetOverrides", "PutAccountQuota", "GetAccountQuota", "SealAccountOverrides", "New"}
|
||||
|
||||
const quotaOverrideBucket = "spinifex-account-quota"
|
||||
|
||||
// Teardown reaches EC2 settings only over contracts/ec2/v1: no import of the
|
||||
// settings owner or bootstrap/config (nodes arrive via NewClusterEngine), and
|
||||
// no literal naming the settings bucket or its constant.
|
||||
@@ -73,27 +61,6 @@ func TestAccountTeardownHasNoSettingsKVAccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
quotaName := ""
|
||||
for _, imp := range f.Imports {
|
||||
if p, _ := strconv.Unquote(imp.Path.Value); p == quotaPackage {
|
||||
quotaName = "quota"
|
||||
if imp.Name != nil {
|
||||
quotaName = imp.Name.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
sel, ok := n.(*ast.SelectorExpr)
|
||||
if !ok || quotaName == "" {
|
||||
return true
|
||||
}
|
||||
if x, isIdent := sel.X.(*ast.Ident); isIdent && x.Name == quotaName && slices.Contains(forbiddenQuotaSelectors, sel.Sel.Name) {
|
||||
t.Errorf("%s: uses quota.%s; the override record belongs to the quota owner and is reached only through contracts/admission/v1",
|
||||
fset.Position(sel.Pos()), sel.Sel.Name)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
lit, ok := n.(*ast.BasicLit)
|
||||
if !ok || lit.Kind != token.STRING {
|
||||
@@ -103,10 +70,6 @@ func TestAccountTeardownHasNoSettingsKVAccess(t *testing.T) {
|
||||
if uerr != nil {
|
||||
return true
|
||||
}
|
||||
if value == quotaOverrideBucket {
|
||||
t.Errorf("%s: names %q; the override bucket is the quota owner's own, never this package's",
|
||||
fset.Position(lit.Pos()), value)
|
||||
}
|
||||
for _, bad := range forbiddenSettingsIdentifiers {
|
||||
if value == bad {
|
||||
t.Errorf("%s: names %q; the settings bucket and its key are the owner's own, never this package's",
|
||||
|
||||
@@ -91,7 +91,7 @@ func (d Deps) deleteAccount(ctx context.Context, body []byte) (any, error) {
|
||||
return nil, errors.New(awserrors.ErrorInternalError)
|
||||
}
|
||||
|
||||
engine, err := accountteardown.NewClusterEngine(ctx, d.NATS, accountteardown.ClusterNodes{Expected: d.ExpectedNodes, DaemonIDs: d.NodeIDs, GatewayIDs: d.GatewayNodeIDs}, d.IAM, d.BucketStore)
|
||||
engine, err := accountteardown.NewClusterEngine(ctx, d.NATS, d.ExpectedNodes, d.NodeIDs, d.IAM, d.BucketStore)
|
||||
if err != nil {
|
||||
slog.Error("DeleteAccount: failed to build teardown engine", "err", err)
|
||||
return nil, errors.New(awserrors.ErrorServiceUnavailable)
|
||||
|
||||
@@ -59,7 +59,7 @@ func TestAdminDeleteAccountDryRunLogsTheShadowUnderTheRequestID(t *testing.T) {
|
||||
gotJSON, err := json.Marshal(got)
|
||||
require.NoError(t, err)
|
||||
|
||||
engine, err := accountteardown.NewClusterEngine(t.Context(), gw.NATS, accountteardown.ClusterNodes{Expected: gw.ExpectedNodes, DaemonIDs: gw.NodeIDs, GatewayIDs: gw.GatewayNodeIDs}, gw.IAM, gw.BucketStore)
|
||||
engine, err := accountteardown.NewClusterEngine(t.Context(), gw.NATS, gw.ExpectedNodes, gw.NodeIDs, gw.IAM, gw.BucketStore)
|
||||
require.NoError(t, err)
|
||||
inventory, err := engine.Inventory(t.Context(), account.AccountID)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
admissionv1 "github.com/mulgadc/spinifex/contracts/admission/v1"
|
||||
ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
|
||||
@@ -99,39 +98,14 @@ func newDeleteAccountGateway(t *testing.T) Deps {
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = capSub.Unsubscribe() })
|
||||
}
|
||||
|
||||
quotaSealSub, err := nc.Subscribe(admissionv1.SealAccountQuotaOverrideSubject, func(msg *nats.Msg) {
|
||||
var req admissionv1.SealAccountQuotaOverrideRequest
|
||||
if json.Unmarshal(msg.Data, &req) != nil {
|
||||
return
|
||||
}
|
||||
data, _ := json.Marshal(admissionv1.SealAccountQuotaOverrideReply{
|
||||
AccountID: req.AccountID, CorrelationRef: req.CorrelationRef,
|
||||
Disposition: admissionv1.AccountQuotaOverrideDispositionSealed, Revision: 1,
|
||||
})
|
||||
_ = msg.Respond(data)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = quotaSealSub.Unsubscribe() })
|
||||
|
||||
quotaCapSub, err := nc.Subscribe(admissionv1.AccountQuotaOverrideSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
|
||||
data, _ := json.Marshal(admissionv1.AccountQuotaOverrideSealCapabilityReply{Node: "node-1", CapabilityVersion: admissionv1.AccountQuotaOverrideSealCapabilityVersion})
|
||||
reply := nats.NewMsg(msg.Reply)
|
||||
reply.Data = data
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
|
||||
_ = msg.RespondMsg(reply)
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = quotaCapSub.Unsubscribe() })
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
return Deps{
|
||||
IAM: svc,
|
||||
NATS: nc,
|
||||
ExpectedNodes: 1,
|
||||
NodeIDs: []string{"node-1"},
|
||||
GatewayNodeIDs: []string{"node-1"},
|
||||
BucketStore: emptyBucketStore{},
|
||||
IAM: svc,
|
||||
NATS: nc,
|
||||
ExpectedNodes: 1,
|
||||
NodeIDs: []string{"node-1"},
|
||||
BucketStore: emptyBucketStore{},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -55,7 +55,6 @@ type Deps struct {
|
||||
Quota *quota.Service
|
||||
ExpectedNodes int
|
||||
NodeIDs []string
|
||||
GatewayNodeIDs []string
|
||||
BucketStore accountteardown.BucketStore
|
||||
SignupMaxAccounts int
|
||||
}
|
||||
|
||||
@@ -95,12 +95,8 @@ func runAccountDelete(cmd *cobra.Command, args []string) error {
|
||||
|
||||
// The configured node set, in the namespace daemons reply under, is what
|
||||
// the instance inventory must hear from in full before it reads as empty.
|
||||
nodes := accountteardown.ClusterNodes{
|
||||
Expected: len(cfg.Nodes),
|
||||
DaemonIDs: slices.Collect(maps.Keys(cfg.Nodes)),
|
||||
GatewayIDs: cfg.NodesRunning("awsgw"),
|
||||
}
|
||||
engine, err := accountteardown.NewClusterEngine(ctx, nc, nodes, svc, buckets)
|
||||
nodeIDs := slices.Collect(maps.Keys(cfg.Nodes))
|
||||
engine, err := accountteardown.NewClusterEngine(ctx, nc, len(cfg.Nodes), nodeIDs, svc, buckets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -86,10 +86,6 @@ func followAccountDeletion(ctx context.Context, target adminTarget, accountID st
|
||||
for {
|
||||
job, err := describeAccountDeletionRemote(ctx, target, accountID)
|
||||
if err != nil {
|
||||
// The deadline often lands mid-poll; the caller still needs the account.
|
||||
if ctx.Err() != nil {
|
||||
return stoppedFollowing(accountID, ctx.Err())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -108,16 +104,12 @@ func followAccountDeletion(ctx context.Context, target adminTarget, accountID st
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return stoppedFollowing(accountID, ctx.Err())
|
||||
return fmt.Errorf("stopped following teardown of %s: %w", accountID, ctx.Err())
|
||||
case <-time.After(accountDeletePollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func stoppedFollowing(accountID string, cause error) error {
|
||||
return fmt.Errorf("stopped following teardown of %s: %w", accountID, cause)
|
||||
}
|
||||
|
||||
func deleteAccountRemote(
|
||||
ctx context.Context,
|
||||
target adminTarget,
|
||||
|
||||
@@ -181,32 +181,6 @@ func TestFollowAccountDeletionStopsWhenTheContextEnds(t *testing.T) {
|
||||
assert.Contains(t, err.Error(), "000000000042")
|
||||
}
|
||||
|
||||
// The deadline usually expires while a poll is in flight, not between polls,
|
||||
// and that path must name the account too.
|
||||
func TestFollowAccountDeletionStopsWhenTheContextEndsMidPoll(t *testing.T) {
|
||||
adminTestCredentials(t)
|
||||
|
||||
// Held until cleanup: the client may cancel before the server notices.
|
||||
release := make(chan struct{})
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
<-release
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
t.Cleanup(func() { close(release) })
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
go func() {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
err := cli.FollowAccountDeletion(ctx,
|
||||
cli.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
|
||||
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Contains(t, err.Error(), "000000000042")
|
||||
}
|
||||
|
||||
// An unreadable job is not a finished one. Reporting it as complete would hide
|
||||
// a teardown still holding an account's resources.
|
||||
func TestFollowAccountDeletionFailsOnAnUnreadableJob(t *testing.T) {
|
||||
|
||||
@@ -65,16 +65,10 @@ func dnsClusterToml(t *testing.T, nodes int, extra string) string {
|
||||
func runNameservers(t *testing.T, path string) (string, error) {
|
||||
t.Helper()
|
||||
resetGlobalViper(t)
|
||||
// LoadConfig declares the process-wide cluster size; clear it for this
|
||||
// load and put the package's count back afterwards.
|
||||
was := clustersize.Nodes()
|
||||
// LoadConfig declares the process-wide cluster size; undo it so later
|
||||
// tests in this package can declare their own.
|
||||
clustersize.ResetForTest()
|
||||
t.Cleanup(func() {
|
||||
clustersize.ResetForTest()
|
||||
if was > 0 {
|
||||
_ = clustersize.Declare(was)
|
||||
}
|
||||
})
|
||||
t.Cleanup(clustersize.ResetForTest)
|
||||
viper.Set("config", path)
|
||||
var out bytes.Buffer
|
||||
cmd := &cobra.Command{}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -169,15 +169,25 @@ func TestKVDigestStreamFilterAndDeletedSeq(t *testing.T) {
|
||||
assert.Equal(t, 1, d[0].Replicas)
|
||||
}
|
||||
|
||||
func freePort(t *testing.T) int {
|
||||
t.Helper()
|
||||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
defer l.Close()
|
||||
return l.Addr().(*net.TCPAddr).Port
|
||||
}
|
||||
|
||||
// A replicated stream's meta.inf carries num_replicas > 1, which a standalone
|
||||
// server refuses to load; this is the case every production node presents.
|
||||
func TestKVDigestClusteredReplicas(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const n = 3
|
||||
// Servers bind their own route ports, so none is ever free to be taken.
|
||||
// JetStream needs a configured route, so the first, which cannot name the
|
||||
// others yet, gets port 1, where nothing listens; the rest route to it.
|
||||
seed := []*url.URL{{Scheme: "nats", Host: "127.0.0.1:1"}}
|
||||
ports := make([]int, n)
|
||||
routes := make([]string, n)
|
||||
for i := range ports {
|
||||
ports[i] = freePort(t)
|
||||
routes[i] = fmt.Sprintf("nats://127.0.0.1:%d", ports[i])
|
||||
}
|
||||
stores := make([]string, n)
|
||||
servers := make([]*server.Server, n)
|
||||
for i := range servers {
|
||||
@@ -185,19 +195,19 @@ func TestKVDigestClusteredReplicas(t *testing.T) {
|
||||
ns, err := server.NewServer(&server.Options{
|
||||
ServerName: fmt.Sprintf("s%d", i+1), JetStream: true, StoreDir: stores[i],
|
||||
Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true,
|
||||
Cluster: server.ClusterOpts{Name: "digest", Host: "127.0.0.1", Port: -1},
|
||||
Routes: seed,
|
||||
Cluster: server.ClusterOpts{Name: "digest", Host: "127.0.0.1", Port: ports[i]},
|
||||
Routes: server.RoutesFromStr(strings.Join(routes, ",")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
ns.Start()
|
||||
servers[i] = ns
|
||||
// Registered after its TempDir, so the server stops before that is removed.
|
||||
t.Cleanup(func() { ns.Shutdown(); ns.WaitForShutdown() })
|
||||
require.True(t, ns.ReadyForConnections(10*time.Second), "server %d not ready", i+1)
|
||||
if i == 0 {
|
||||
seed = []*url.URL{{Scheme: "nats", Host: ns.ClusterAddr().String()}}
|
||||
}
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, ns := range servers {
|
||||
ns.Shutdown()
|
||||
ns.WaitForShutdown()
|
||||
}
|
||||
})
|
||||
|
||||
nc, err := nats.Connect("", nats.InProcessServer(servers[0]))
|
||||
require.NoError(t, err)
|
||||
@@ -218,18 +228,22 @@ func TestKVDigestClusteredReplicas(t *testing.T) {
|
||||
_, err := kv.PutString(ctx, fmt.Sprintf("k%d", i%2), fmt.Sprintf("v%d", i))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
// A replica the leader reports current holds the writes in its Raft log,
|
||||
// not yet in its store, so ask each server for its own store's state.
|
||||
s, err := js.Stream(ctx, "KV_rep")
|
||||
require.NoError(t, err)
|
||||
require.Eventually(t, func() bool {
|
||||
for _, ns := range servers {
|
||||
if localLastSeq(ns, "KV_rep") != 5 {
|
||||
info, err := s.Info(ctx)
|
||||
if err != nil || info.Cluster == nil || len(info.Cluster.Replicas) != n-1 {
|
||||
return false
|
||||
}
|
||||
for _, r := range info.Cluster.Replicas {
|
||||
if !r.Current || r.Lag != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}, 30*time.Second, 50*time.Millisecond)
|
||||
}, 30*time.Second, 250*time.Millisecond)
|
||||
|
||||
// Applied is not flushed; shutdown flushes every block to disk.
|
||||
// Current means applied, not flushed; shutdown flushes every block to disk.
|
||||
nc.Close()
|
||||
for _, ns := range servers {
|
||||
ns.Shutdown()
|
||||
@@ -256,22 +270,6 @@ func TestKVDigestClusteredReplicas(t *testing.T) {
|
||||
assert.False(t, res.divergent(), "%+v", res)
|
||||
}
|
||||
|
||||
// localLastSeq is the last sequence stream holds in srv's own replica.
|
||||
func localLastSeq(srv *server.Server, stream string) uint64 {
|
||||
info, err := srv.Jsz(&server.JSzOptions{Accounts: true, Streams: true})
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
for _, a := range info.AccountDetails {
|
||||
for _, sd := range a.Streams {
|
||||
if sd.Name == stream {
|
||||
return sd.State.LastSeq
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// Recovery applies MaxAge, so without the rewrite an expired bucket copy
|
||||
// would read as empty and mask whatever the replica holds.
|
||||
func TestKVDigestIgnoresExpiry(t *testing.T) {
|
||||
|
||||
@@ -103,7 +103,7 @@ func TestRunOchreApplianceTeardown_MissingConfirmExits1(t *testing.T) {
|
||||
return nil
|
||||
})
|
||||
|
||||
cmd := copyCmd(t, ochreApplianceTeardownCmd)
|
||||
cmd := *ochreApplianceTeardownCmd
|
||||
code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
|
||||
require.Equal(t, 1, code)
|
||||
}
|
||||
@@ -111,7 +111,7 @@ func TestRunOchreApplianceTeardown_MissingConfirmExits1(t *testing.T) {
|
||||
func TestRunOchreApplianceTeardown_ConfirmedPrintsSuccess(t *testing.T) {
|
||||
withApplianceTeardown(t, func(context.Context, bool) error { return nil })
|
||||
|
||||
cmd := copyCmd(t, ochreApplianceTeardownCmd)
|
||||
cmd := *ochreApplianceTeardownCmd
|
||||
require.NoError(t, cmd.Flags().Set("confirm", "true"))
|
||||
|
||||
var out string
|
||||
@@ -127,7 +127,7 @@ func TestRunOchreApplianceTeardown_TeardownErrorExits1(t *testing.T) {
|
||||
return errors.New("ochrevector: platform appliance is not enabled or not up on this node")
|
||||
})
|
||||
|
||||
cmd := copyCmd(t, ochreApplianceTeardownCmd)
|
||||
cmd := *ochreApplianceTeardownCmd
|
||||
require.NoError(t, cmd.Flags().Set("confirm", "true"))
|
||||
|
||||
code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
|
||||
@@ -143,7 +143,7 @@ func TestRunOchreApplianceTeardown_PurgeMetadataFlagReachesTeardownFn(t *testing
|
||||
return nil
|
||||
})
|
||||
|
||||
cmd := copyCmd(t, ochreApplianceTeardownCmd)
|
||||
cmd := *ochreApplianceTeardownCmd
|
||||
require.NoError(t, cmd.Flags().Set("confirm", "true"))
|
||||
require.NoError(t, cmd.Flags().Set("purge-metadata", "true"))
|
||||
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/domains/ochre"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -273,29 +271,6 @@ func TestFormatEndpointRecord_AbsentIsMinimal(t *testing.T) {
|
||||
require.Equal(t, 2, strings.Count(strings.TrimSpace(out), "\n")+1)
|
||||
}
|
||||
|
||||
// copyCmd returns a copy of a package-level command for one test. The copy
|
||||
// still shares the original's flag set, so every flag is put back on cleanup;
|
||||
// otherwise a flag one test sets leaks into the next test, or the next -count.
|
||||
func copyCmd(t *testing.T, src *cobra.Command) cobra.Command {
|
||||
t.Helper()
|
||||
type saved struct {
|
||||
value string
|
||||
changed bool
|
||||
}
|
||||
before := map[string]saved{}
|
||||
src.Flags().VisitAll(func(f *pflag.Flag) { before[f.Name] = saved{f.Value.String(), f.Changed} })
|
||||
t.Cleanup(func() {
|
||||
src.Flags().VisitAll(func(f *pflag.Flag) {
|
||||
b := before[f.Name]
|
||||
if f.Value.String() != b.value {
|
||||
require.NoError(t, f.Value.Set(b.value), "restore flag %s", f.Name)
|
||||
}
|
||||
f.Changed = b.changed
|
||||
})
|
||||
})
|
||||
return *src
|
||||
}
|
||||
|
||||
// withEndpointService swaps endpointServiceFn for one returning svc, so a Run
|
||||
// wrapper's real connect/validate/exit control flow runs without a daemon.
|
||||
func withEndpointService(t *testing.T, svc ochre.EndpointService, connErr error) {
|
||||
@@ -315,7 +290,7 @@ func TestRunOchreEndpointEnsure_PrintsRecord(t *testing.T) {
|
||||
ensure: ochre.EndpointRecord{ModelID: testModelID, State: ochre.StateStarting},
|
||||
}, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointEnsureCmd)
|
||||
cmd := *ochreEndpointEnsureCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
|
||||
var out string
|
||||
@@ -329,7 +304,7 @@ func TestRunOchreEndpointEnsure_PrintsRecord(t *testing.T) {
|
||||
func TestRunOchreEndpointEnsure_ConnectFailureExits1(t *testing.T) {
|
||||
withEndpointService(t, nil, errors.New("dial nats: connection refused"))
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointEnsureCmd)
|
||||
cmd := *ochreEndpointEnsureCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
|
||||
code := withOchreExitCapture(t, func() { runOchreEndpointEnsure(&cmd, nil) })
|
||||
@@ -339,7 +314,7 @@ func TestRunOchreEndpointEnsure_ConnectFailureExits1(t *testing.T) {
|
||||
func TestRunOchreEndpointEnsure_ServiceErrorExits1(t *testing.T) {
|
||||
withEndpointService(t, &fakeEndpointService{ensureErr: errors.New("ResourceNotFoundException")}, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointEnsureCmd)
|
||||
cmd := *ochreEndpointEnsureCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", "no.such.model"))
|
||||
|
||||
code := withOchreExitCapture(t, func() { runOchreEndpointEnsure(&cmd, nil) })
|
||||
@@ -351,7 +326,7 @@ func TestRunOchreEndpointDescribe_PrintsRecord(t *testing.T) {
|
||||
{ModelID: testModelID, State: ochre.StateReady, InstanceID: "i-abc"},
|
||||
}}, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointDescribeCmd)
|
||||
cmd := *ochreEndpointDescribeCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
|
||||
var out string
|
||||
@@ -372,7 +347,7 @@ func TestRunOchreEndpointDescribe_DefaultAccountIsEmpty(t *testing.T) {
|
||||
}}
|
||||
withEndpointService(t, svc, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointDescribeCmd)
|
||||
cmd := *ochreEndpointDescribeCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
|
||||
withOchreExitCapture(t, func() {
|
||||
@@ -392,7 +367,7 @@ func TestRunOchreEndpointDescribe_AccountFlagScopesLookup(t *testing.T) {
|
||||
}}
|
||||
withEndpointService(t, svc, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointDescribeCmd)
|
||||
cmd := *ochreEndpointDescribeCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
require.NoError(t, cmd.Flags().Set("account", testAccountID))
|
||||
|
||||
@@ -409,7 +384,7 @@ func TestRunOchreEndpointDescribe_AccountFlagScopesLookup(t *testing.T) {
|
||||
func TestRunOchreEndpointDescribe_ErrorExits1(t *testing.T) {
|
||||
withEndpointService(t, &fakeEndpointService{describeErr: errors.New("nats: timeout")}, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointDescribeCmd)
|
||||
cmd := *ochreEndpointDescribeCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
|
||||
code := withOchreExitCapture(t, func() { runOchreEndpointDescribe(&cmd, nil) })
|
||||
@@ -440,7 +415,7 @@ func TestRunOchreEndpointDelete_ReportsTeardown(t *testing.T) {
|
||||
svc := &fakeEndpointService{deleteRemoved: true}
|
||||
withEndpointService(t, svc, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointDeleteCmd)
|
||||
cmd := *ochreEndpointDeleteCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
|
||||
var out string
|
||||
@@ -460,7 +435,7 @@ func TestRunOchreEndpointDelete_NoRecordDoesNotClaimTeardown(t *testing.T) {
|
||||
svc := &fakeEndpointService{deleteRemoved: false}
|
||||
withEndpointService(t, svc, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointDeleteCmd)
|
||||
cmd := *ochreEndpointDeleteCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
|
||||
var out string
|
||||
@@ -479,7 +454,7 @@ func TestRunOchreEndpointDelete_AccountFlagReachesInput(t *testing.T) {
|
||||
svc := &fakeEndpointService{deleteRemoved: true}
|
||||
withEndpointService(t, svc, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointDeleteCmd)
|
||||
cmd := *ochreEndpointDeleteCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
require.NoError(t, cmd.Flags().Set("account", testAccountID))
|
||||
|
||||
@@ -494,7 +469,7 @@ func TestRunOchreEndpointDelete_AccountFlagReachesInput(t *testing.T) {
|
||||
func TestRunOchreEndpointDelete_ErrorExits1(t *testing.T) {
|
||||
withEndpointService(t, &fakeEndpointService{deleteErr: errors.New("ModelNotReadyException")}, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreEndpointDeleteCmd)
|
||||
cmd := *ochreEndpointDeleteCmd
|
||||
require.NoError(t, cmd.Flags().Set("model-id", testModelID))
|
||||
|
||||
code := withOchreExitCapture(t, func() { runOchreEndpointDelete(&cmd, nil) })
|
||||
|
||||
@@ -289,7 +289,7 @@ func TestRunOchreVectorIndexCreate_PrintsRecord(t *testing.T) {
|
||||
Index: ochrevector.Record{ID: testIndexID, State: ochrevector.StateReady},
|
||||
}}, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreVectorIndexCreateCmd)
|
||||
cmd := *ochreVectorIndexCreateCmd
|
||||
require.NoError(t, cmd.Flags().Set("name", "kb1"))
|
||||
require.NoError(t, cmd.Flags().Set("dimension", "768"))
|
||||
|
||||
@@ -304,7 +304,7 @@ func TestRunOchreVectorIndexCreate_PrintsRecord(t *testing.T) {
|
||||
func TestRunOchreVectorIndexCreate_ConnectFailureExits1(t *testing.T) {
|
||||
withVectorService(t, nil, errors.New("dial nats: connection refused"))
|
||||
|
||||
cmd := copyCmd(t, ochreVectorIndexCreateCmd)
|
||||
cmd := *ochreVectorIndexCreateCmd
|
||||
require.NoError(t, cmd.Flags().Set("name", "kb1"))
|
||||
require.NoError(t, cmd.Flags().Set("dimension", "768"))
|
||||
|
||||
@@ -343,7 +343,7 @@ func TestRunOchreVectorIngest_PrintsJobID(t *testing.T) {
|
||||
Job: ochrevector.JobRecord{ID: "job-abc", IndexID: testIndexID, State: ochrevector.JobStatePending},
|
||||
}}, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreVectorIngestCmd)
|
||||
cmd := *ochreVectorIngestCmd
|
||||
require.NoError(t, cmd.Flags().Set("index", testIndexID))
|
||||
require.NoError(t, cmd.Flags().Set("bucket", "docs"))
|
||||
|
||||
@@ -373,7 +373,7 @@ func TestRunOchreVectorQuery_PrintsTable(t *testing.T) {
|
||||
{Chunk: "hello world", SourceKey: "docs/a.txt", Score: 0.9},
|
||||
}}, nil)
|
||||
|
||||
cmd := copyCmd(t, ochreVectorQueryCmd)
|
||||
cmd := *ochreVectorQueryCmd
|
||||
require.NoError(t, cmd.Flags().Set("index", testIndexID))
|
||||
require.NoError(t, cmd.Flags().Set("text", "hello"))
|
||||
|
||||
@@ -388,7 +388,7 @@ func TestRunOchreVectorQuery_PrintsTable(t *testing.T) {
|
||||
func TestRunOchreVectorQuery_ConnectFailureExits1(t *testing.T) {
|
||||
withVectorService(t, nil, errors.New("dial nats: connection refused"))
|
||||
|
||||
cmd := copyCmd(t, ochreVectorQueryCmd)
|
||||
cmd := *ochreVectorQueryCmd
|
||||
require.NoError(t, cmd.Flags().Set("index", testIndexID))
|
||||
require.NoError(t, cmd.Flags().Set("text", "hello"))
|
||||
|
||||
@@ -465,7 +465,7 @@ func TestRunOchreVectorBackup_PrintsSuccess(t *testing.T) {
|
||||
return &ochrevector.BackupAccountResponse{ObjectKey: testBackupAccountID + "/backup.sql.gz", SizeBytes: 1024}, nil
|
||||
})
|
||||
|
||||
cmd := copyCmd(t, ochreVectorBackupCmd)
|
||||
cmd := *ochreVectorBackupCmd
|
||||
require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
|
||||
|
||||
var out string
|
||||
@@ -481,7 +481,7 @@ func TestRunOchreVectorBackup_ErrorExits1(t *testing.T) {
|
||||
return nil, errors.New("ochrevector: appliance not available")
|
||||
})
|
||||
|
||||
cmd := copyCmd(t, ochreVectorBackupCmd)
|
||||
cmd := *ochreVectorBackupCmd
|
||||
require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
|
||||
|
||||
code := withOchreExitCapture(t, func() { runOchreVectorBackup(&cmd, nil) })
|
||||
@@ -491,7 +491,7 @@ func TestRunOchreVectorBackup_ErrorExits1(t *testing.T) {
|
||||
func TestRunOchreVectorRestore_PrintsSuccess(t *testing.T) {
|
||||
withRestoreAccount(t, func(context.Context, string, string) error { return nil })
|
||||
|
||||
cmd := copyCmd(t, ochreVectorRestoreCmd)
|
||||
cmd := *ochreVectorRestoreCmd
|
||||
require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
|
||||
require.NoError(t, cmd.Flags().Set("from", testBackupAccountID+"/backup.sql.gz"))
|
||||
|
||||
@@ -508,7 +508,7 @@ func TestRunOchreVectorRestore_ErrorExits1(t *testing.T) {
|
||||
return errors.New("ochrevector: restore requires an object key")
|
||||
})
|
||||
|
||||
cmd := copyCmd(t, ochreVectorRestoreCmd)
|
||||
cmd := *ochreVectorRestoreCmd
|
||||
require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
|
||||
require.NoError(t, cmd.Flags().Set("from", testBackupAccountID+"/backup.sql.gz"))
|
||||
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
|
||||
)
|
||||
|
||||
// Declaring is write-once per process, so whichever test loaded a config first
|
||||
// would otherwise fix the count for the rest: a multi-node config loaded early
|
||||
// leaves every one-node embedded JetStream fixture unable to declare.
|
||||
func TestMain(m *testing.M) {
|
||||
if err := clustersize.Declare(1); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to declare cluster size: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
@@ -326,14 +326,10 @@ func TestVerifyImageChecksum(t *testing.T) {
|
||||
|
||||
t.Run("context deadline exceeded wraps fetch failure", func(t *testing.T) {
|
||||
img := writeTempImage(t, debianName, debianBytes)
|
||||
// Hold until cleanup, not until the client gives up: returning on
|
||||
// r.Context() sends an empty 200 that can win the client's deadline race.
|
||||
release := make(chan struct{})
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
<-release
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
t.Cleanup(func() { close(release) })
|
||||
trustTestServer(t, srv)
|
||||
|
||||
prev := checksumFetchTimeout
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package viperblock
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
@@ -29,9 +27,7 @@ func TestProviderHandlers_CreateVolume_LeavesNoLocalState(t *testing.T) {
|
||||
cfg.SecretKey = fixture.SecretKey
|
||||
nc := startProviderSubjects(t, cfg, natsURL)
|
||||
|
||||
// The predastore fixture outlives the test, so a fixed name would make a
|
||||
// repeat run (-count) take the already-exists path instead of creating.
|
||||
volumeName := "vol-createlocalstate-" + strings.ToLower(rand.Text())
|
||||
const volumeName = "vol-createlocalstate01"
|
||||
body := marshalRequest(t, map[string]any{
|
||||
"schema_version": ebsprovider.SchemaVersion,
|
||||
"volume_id": volumeName,
|
||||
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
|
||||
awssdk "github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/deadpid"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/mulgadc/viperblock/viperblock"
|
||||
@@ -777,7 +776,7 @@ func TestProviderHandlers_DeleteVolume_MountedVolumeRefused(t *testing.T) {
|
||||
Name: volumeName, VB: vb, Socket: socketPath,
|
||||
// A PID that (almost certainly) does not exist: KillProcess must fail
|
||||
// gracefully (logged, not fatal) rather than block or panic.
|
||||
PID: deadpid.PID,
|
||||
PID: 999999,
|
||||
})
|
||||
|
||||
body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion, "volume_id": volumeName})
|
||||
@@ -977,7 +976,7 @@ func TestProviderHandlers_UnpublishVolume_RemovesMountedVolume(t *testing.T) {
|
||||
const volumeName = "vol-unpubremoved01"
|
||||
cfg.MountedVolumes = append(cfg.MountedVolumes, MountedVolume{
|
||||
Name: volumeName,
|
||||
PID: deadpid.PID, // Fake PID that doesn't exist; KillProcess must fail gracefully.
|
||||
PID: 999999, // Fake PID that doesn't exist; KillProcess must fail gracefully.
|
||||
})
|
||||
|
||||
unpublishSubject, err := ebsprovider.UnpublishSubject(cfg.NodeName)
|
||||
@@ -1125,7 +1124,7 @@ func TestProviderHandlers_OwnerSubscription_UnmountRemovesItAndFallsBack(t *test
|
||||
Name: volumeName, VB: vb, OwnerSubs: ownerSubs,
|
||||
// A PID that (almost certainly) does not exist: KillProcess must fail
|
||||
// gracefully during unmountVolume rather than block or panic.
|
||||
PID: deadpid.PID,
|
||||
PID: 999999,
|
||||
})
|
||||
|
||||
ownerSubject, err := ebsprovider.GetVolumeOwnerSubject(volumeName)
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/deadpid"
|
||||
"github.com/mulgadc/viperblock/viperblock"
|
||||
"github.com/mulgadc/viperblock/viperblock/backends/file"
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -67,7 +66,7 @@ func TestIntegration_EBSDeleteMountedVolume(t *testing.T) {
|
||||
Name: "vol-del-test",
|
||||
Port: 10809,
|
||||
Socket: socketPath,
|
||||
PID: deadpid.PID,
|
||||
PID: 99999,
|
||||
ConfigSub: configSub,
|
||||
},
|
||||
}
|
||||
@@ -410,7 +409,7 @@ func TestIntegration_EBSUnmountRemovesSocket(t *testing.T) {
|
||||
{
|
||||
Name: "vol-unmount-socket",
|
||||
Socket: socketPath,
|
||||
PID: deadpid.PID,
|
||||
PID: 99999,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -448,7 +447,7 @@ func TestIntegration_EBSDeleteRemovesSocket(t *testing.T) {
|
||||
require.NoError(t, os.WriteFile(socketPath, []byte("fake"), 0600))
|
||||
cfg := setupTestConfig(t, natsURL)
|
||||
cfg.MountedVolumes = []MountedVolume{
|
||||
{Name: "vol-del-socket", Socket: socketPath, PID: deadpid.PID},
|
||||
{Name: "vol-del-socket", Socket: socketPath, PID: 99999},
|
||||
}
|
||||
|
||||
startTestService(t, cfg)
|
||||
@@ -558,7 +557,7 @@ func TestIntegration_EBSDeleteWithVBInstance(t *testing.T) {
|
||||
{
|
||||
Name: "vol-del-vb",
|
||||
Socket: socketPath,
|
||||
PID: deadpid.PID,
|
||||
PID: 99999,
|
||||
VB: vb,
|
||||
ConfigSub: configSub,
|
||||
},
|
||||
@@ -609,7 +608,7 @@ func TestIntegration_EBSUnmountWithVBInstance(t *testing.T) {
|
||||
{
|
||||
Name: "vol-unmount-vb",
|
||||
Socket: socketPath,
|
||||
PID: deadpid.PID,
|
||||
PID: 99999,
|
||||
VB: vb,
|
||||
ConfigSub: configSub,
|
||||
},
|
||||
@@ -645,7 +644,7 @@ func TestIntegration_EBSUnmountDualPublish(t *testing.T) {
|
||||
|
||||
cfg := setupTestConfig(t, natsURL)
|
||||
cfg.MountedVolumes = []MountedVolume{
|
||||
{Name: "vol-dual-pub", PID: deadpid.PID},
|
||||
{Name: "vol-dual-pub", PID: 99999},
|
||||
}
|
||||
|
||||
startTestService(t, cfg)
|
||||
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
|
||||
viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/deadpid"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -274,7 +273,7 @@ func TestIntegration_EBSUnmountRequest(t *testing.T) {
|
||||
Name: "vol-test-unmount",
|
||||
Port: 10809,
|
||||
Socket: "/tmp/vol-test-unmount.sock",
|
||||
PID: deadpid.PID, // Fake PID that doesn't exist
|
||||
PID: 99999, // Fake PID that doesn't exist
|
||||
},
|
||||
}
|
||||
|
||||
@@ -384,7 +383,7 @@ func TestIntegration_EBSUnmountRetryAfterCompletedSealReportsNotFound(t *testing
|
||||
|
||||
cfg := setupTestConfig(t, natsURL)
|
||||
cfg.MountedVolumes = []MountedVolume{
|
||||
{Name: "vol-retry-unmount", PID: deadpid.PID},
|
||||
{Name: "vol-retry-unmount", PID: 99999},
|
||||
}
|
||||
|
||||
startTestService(t, cfg)
|
||||
@@ -442,7 +441,7 @@ func TestIntegration_EBSUnmountSealFailureKeepsVolumeMounted(t *testing.T) {
|
||||
return fmt.Errorf("seal %s: injected backend failure", volumeName)
|
||||
}
|
||||
cfg.MountedVolumes = []MountedVolume{
|
||||
{Name: "vol-seal-fail", PID: deadpid.PID},
|
||||
{Name: "vol-seal-fail", PID: 99999},
|
||||
}
|
||||
|
||||
startTestService(t, cfg)
|
||||
@@ -494,7 +493,7 @@ func TestIntegration_EBSUnmountReceiptSkipsFallbackSeal(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
cfg.MountedVolumes = []MountedVolume{
|
||||
{Name: "vol-clean-receipt", PID: deadpid.PID},
|
||||
{Name: "vol-clean-receipt", PID: 99999},
|
||||
}
|
||||
|
||||
startTestService(t, cfg)
|
||||
@@ -545,7 +544,7 @@ func TestIntegration_EBSUnmountStateDirSealsRegardlessOfReceipt(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
cfg.MountedVolumes = []MountedVolume{
|
||||
{Name: "vol-state-and-receipt", PID: deadpid.PID},
|
||||
{Name: "vol-state-and-receipt", PID: 99999},
|
||||
}
|
||||
|
||||
startTestService(t, cfg)
|
||||
@@ -587,7 +586,7 @@ func TestIntegration_EBSUnmountAuxVolumeNeverSeals(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
cfg.MountedVolumes = []MountedVolume{
|
||||
{Name: "vol-aux-efi", PID: deadpid.PID},
|
||||
{Name: "vol-aux-efi", PID: 99999},
|
||||
}
|
||||
|
||||
startTestService(t, cfg)
|
||||
@@ -629,7 +628,7 @@ func TestIntegration_EBSUnmountNoStateNoReceiptWarns(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
cfg.MountedVolumes = []MountedVolume{
|
||||
{Name: "vol-no-state-no-receipt", PID: deadpid.PID},
|
||||
{Name: "vol-no-state-no-receipt", PID: 99999},
|
||||
}
|
||||
|
||||
startTestService(t, cfg)
|
||||
@@ -813,7 +812,7 @@ func TestIntegration_ServiceGracefulShutdown(t *testing.T) {
|
||||
{
|
||||
Name: "vol-shutdown-test",
|
||||
Port: 10809,
|
||||
PID: deadpid.PID, // Fake PID
|
||||
PID: 99999, // Fake PID
|
||||
},
|
||||
}
|
||||
|
||||
@@ -852,7 +851,7 @@ func TestIntegration_GenericTopicRouting(t *testing.T) {
|
||||
cfg := setupTestConfig(t, natsURL)
|
||||
cfg.NodeName = "" // Single-node / generic mode
|
||||
cfg.MountedVolumes = []MountedVolume{
|
||||
{Name: "vol-generic", PID: deadpid.PID},
|
||||
{Name: "vol-generic", PID: 99999},
|
||||
}
|
||||
|
||||
startTestService(t, cfg)
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
package objectstore_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func putKeys(t *testing.T, store *objectstore.MemoryObjectStore, bucket string, keys ...string) {
|
||||
t.Helper()
|
||||
for _, k := range keys {
|
||||
_, err := store.PutObject(t.Context(), &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(k), Body: strings.NewReader(k)})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func listedKeys(out *s3.ListObjectsV2Output) []string {
|
||||
var keys []string
|
||||
for _, o := range out.Contents {
|
||||
keys = append(keys, aws.StringValue(o.Key))
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func listedPrefixes(out *s3.ListObjectsV2Output) []string {
|
||||
var ps []string
|
||||
for _, p := range out.CommonPrefixes {
|
||||
ps = append(ps, aws.StringValue(p.Prefix))
|
||||
}
|
||||
return ps
|
||||
}
|
||||
|
||||
// S3 returns keys in UTF-8 binary order, so uppercase sorts before lowercase
|
||||
// and a multi-byte rune after every ASCII byte. A map-ordered double let tests
|
||||
// pass or fail depending on iteration order.
|
||||
func TestMemoryListObjectsV2_KeysInUTF8BinaryOrder(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
putKeys(t, store, "b", "k/é", "k/b", "k/B", "k/a10", "k/a2", "k/~", "other/x")
|
||||
putKeys(t, store, "b2", "k/0")
|
||||
|
||||
want := []string{"k/B", "k/a10", "k/a2", "k/b", "k/~", "k/é"}
|
||||
for range 20 {
|
||||
out, err := store.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{Bucket: aws.String("b"), Prefix: aws.String("k/")})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, want, listedKeys(out))
|
||||
assert.False(t, aws.BoolValue(out.IsTruncated))
|
||||
assert.EqualValues(t, len(want), aws.Int64Value(out.KeyCount))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryListObjectsV2_DelimiterRollsUpCommonPrefixesInOrder(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
putKeys(t, store, "b", "p/z/1", "p/a/2", "p/a/1", "p/m", "p/b/1", "p/0")
|
||||
|
||||
out, err := store.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String("b"), Prefix: aws.String("p/"), Delimiter: aws.String("/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"p/0", "p/m"}, listedKeys(out))
|
||||
assert.Equal(t, []string{"p/a/", "p/b/", "p/z/"}, listedPrefixes(out))
|
||||
assert.EqualValues(t, 5, aws.Int64Value(out.KeyCount), "KeyCount counts keys and common prefixes")
|
||||
}
|
||||
|
||||
// Common prefixes count toward MaxKeys, and a continuation never repeats a
|
||||
// prefix or a key from an earlier page.
|
||||
func TestMemoryListObjectsV2_PagesByMaxKeysAndContinuationToken(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
putKeys(t, store, "b", "a/1", "a/2", "a/3", "b", "c/1", "d", "e/1", "e/2")
|
||||
|
||||
var keys, prefixes []string
|
||||
var token *string
|
||||
pages := 0
|
||||
for {
|
||||
out, err := store.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String("b"), Delimiter: aws.String("/"), MaxKeys: aws.Int64(2), ContinuationToken: token,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
pages++
|
||||
require.LessOrEqual(t, aws.Int64Value(out.KeyCount), int64(2))
|
||||
keys = append(keys, listedKeys(out)...)
|
||||
prefixes = append(prefixes, listedPrefixes(out)...)
|
||||
if !aws.BoolValue(out.IsTruncated) {
|
||||
break
|
||||
}
|
||||
require.NotEmpty(t, aws.StringValue(out.NextContinuationToken))
|
||||
token = out.NextContinuationToken
|
||||
}
|
||||
assert.Equal(t, []string{"b", "d"}, keys)
|
||||
assert.Equal(t, []string{"a/", "c/", "e/"}, prefixes)
|
||||
assert.Equal(t, 3, pages)
|
||||
}
|
||||
|
||||
func TestMemoryListObjectsV2_DefaultPageIs1000AndListAllDrainsIt(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
for i := range 1001 {
|
||||
putKeys(t, store, "b", fmt.Sprintf("k/%04d", i))
|
||||
}
|
||||
|
||||
out, err := store.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{Bucket: aws.String("b")})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, out.Contents, 1000)
|
||||
assert.True(t, aws.BoolValue(out.IsTruncated))
|
||||
|
||||
all, _, err := objectstore.ListAll(t.Context(), store, &s3.ListObjectsV2Input{Bucket: aws.String("b")})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, all, 1001)
|
||||
assert.Equal(t, "k/1000", aws.StringValue(all[1000].Key))
|
||||
}
|
||||
|
||||
func TestMemoryListObjectsV2_StartAfterIsExclusiveAndKeepsLaterPrefixGroups(t *testing.T) {
|
||||
store := objectstore.NewMemoryObjectStore()
|
||||
putKeys(t, store, "b", "a/1", "a/9", "b", "c")
|
||||
|
||||
out, err := store.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{Bucket: aws.String("b"), StartAfter: aws.String("b")})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"c"}, listedKeys(out))
|
||||
|
||||
out, err = store.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String("b"), Delimiter: aws.String("/"), StartAfter: aws.String("a/5"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"a/"}, listedPrefixes(out), "a/9 sorts after StartAfter, so its group is listed")
|
||||
assert.Equal(t, []string{"b", "c"}, listedKeys(out))
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -286,83 +285,61 @@ func (m *MemoryObjectStore) DeleteObject(_ context.Context, input *s3.DeleteObje
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
|
||||
// ListObjectsV2 pages like S3: keys and common prefixes in UTF-8 binary order,
|
||||
// at most MaxKeys (default 1000) entries per page, resumed by StartAfter or
|
||||
// the returned NextContinuationToken.
|
||||
func (m *MemoryObjectStore) ListObjectsV2(_ context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
bucket := *input.Bucket
|
||||
prefix := aws.StringValue(input.Prefix)
|
||||
delimiter := aws.StringValue(input.Delimiter)
|
||||
startAfter := aws.StringValue(input.StartAfter)
|
||||
token := aws.StringValue(input.ContinuationToken)
|
||||
maxKeys := int64(memoryListMaxKeys)
|
||||
if input.MaxKeys != nil && *input.MaxKeys < maxKeys {
|
||||
maxKeys = max(*input.MaxKeys, 0)
|
||||
prefix := ""
|
||||
if input.Prefix != nil {
|
||||
prefix = *input.Prefix
|
||||
}
|
||||
delimiter := ""
|
||||
if input.Delimiter != nil {
|
||||
delimiter = *input.Delimiter
|
||||
}
|
||||
|
||||
// S3 lists in UTF-8 binary key order; Go string comparison is bytewise.
|
||||
var keys []string
|
||||
for key := range m.objects {
|
||||
if objectKey, ok := strings.CutPrefix(key, bucket+"/"); ok && strings.HasPrefix(objectKey, prefix) && objectKey > startAfter {
|
||||
keys = append(keys, objectKey)
|
||||
var contents []*s3.Object
|
||||
prefixes := make(map[string]bool)
|
||||
|
||||
for key, data := range m.objects {
|
||||
if !strings.HasPrefix(key, bucket+"/") {
|
||||
continue
|
||||
}
|
||||
objectKey := key[len(bucket)+1:]
|
||||
if prefix != "" && !strings.HasPrefix(objectKey, prefix) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
slices.Sort(keys)
|
||||
|
||||
out := &s3.ListObjectsV2Output{
|
||||
Name: input.Bucket,
|
||||
Prefix: input.Prefix,
|
||||
Delimiter: input.Delimiter,
|
||||
StartAfter: input.StartAfter,
|
||||
ContinuationToken: input.ContinuationToken,
|
||||
MaxKeys: aws.Int64(maxKeys),
|
||||
IsTruncated: aws.Bool(false),
|
||||
}
|
||||
// The token is the last key or common prefix returned. Every key under a
|
||||
// returned common prefix rolls up to that same entry, so "entry <= token"
|
||||
// skips the whole group on the next page.
|
||||
var count int64
|
||||
last := ""
|
||||
for _, objectKey := range keys {
|
||||
entry, isPrefix := objectKey, false
|
||||
if delimiter != "" {
|
||||
if idx := strings.Index(objectKey[len(prefix):], delimiter); idx >= 0 {
|
||||
entry, isPrefix = objectKey[:len(prefix)+idx+len(delimiter)], true
|
||||
afterPrefix := objectKey[len(prefix):]
|
||||
if idx := strings.Index(afterPrefix, delimiter); idx >= 0 {
|
||||
commonPrefix := objectKey[:len(prefix)+idx+len(delimiter)]
|
||||
prefixes[commonPrefix] = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
if (token != "" && entry <= token) || (isPrefix && entry == last) {
|
||||
continue
|
||||
}
|
||||
if maxKeys == 0 {
|
||||
break
|
||||
}
|
||||
if count == maxKeys {
|
||||
out.IsTruncated = aws.Bool(true)
|
||||
out.NextContinuationToken = aws.String(last)
|
||||
break
|
||||
}
|
||||
last = entry
|
||||
count++
|
||||
if isPrefix {
|
||||
out.CommonPrefixes = append(out.CommonPrefixes, &s3.CommonPrefix{Prefix: aws.String(entry)})
|
||||
continue
|
||||
}
|
||||
full := bucket + "/" + objectKey
|
||||
out.Contents = append(out.Contents, &s3.Object{
|
||||
|
||||
contents = append(contents, &s3.Object{
|
||||
Key: aws.String(objectKey),
|
||||
Size: aws.Int64(int64(len(m.objects[full]))),
|
||||
LastModified: aws.Time(m.modified[full]),
|
||||
Size: aws.Int64(int64(len(data))),
|
||||
LastModified: aws.Time(m.modified[key]),
|
||||
})
|
||||
}
|
||||
out.KeyCount = aws.Int64(count)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// memoryListMaxKeys is S3's per-page cap on ListObjectsV2 results.
|
||||
const memoryListMaxKeys = 1000
|
||||
var commonPrefixes []*s3.CommonPrefix
|
||||
for p := range prefixes {
|
||||
commonPrefixes = append(commonPrefixes, &s3.CommonPrefix{
|
||||
Prefix: aws.String(p),
|
||||
})
|
||||
}
|
||||
|
||||
return &s3.ListObjectsV2Output{
|
||||
Contents: contents,
|
||||
CommonPrefixes: commonPrefixes,
|
||||
Name: input.Bucket,
|
||||
KeyCount: aws.Int64(int64(len(contents))),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// EnsureBucket is a no-op: the memory store has no bucket namespace, keys are
|
||||
// prefixed with the bucket name on write.
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit/deadpid"
|
||||
hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -60,7 +59,7 @@ func TestOrphanQEMUReaper(t *testing.T) {
|
||||
t.Run("stale PID file (dead process) is cleaned without counting as a reap", func(t *testing.T) {
|
||||
reaper, store := newOrphanReaperManager(t)
|
||||
const id = "i-term-stale"
|
||||
require.NoError(t, hostprocess.WritePidFile(id, deadpid.PID))
|
||||
require.NoError(t, hostprocess.WritePidFile(id, 999999)) // dead pid
|
||||
store.terminated[id] = &VM{ID: id, Status: StateTerminated}
|
||||
|
||||
reaped, err := reaper.Sweep(context.Background())
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/mulgadc/spinifex/internal/testkit/deadpid"
|
||||
"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
|
||||
hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -1399,7 +1398,7 @@ func TestDetachVolume_ConfirmedDeadQEMU_SkipsQMPAndSeals(t *testing.T) {
|
||||
defer cancel()
|
||||
|
||||
const id = "i-1"
|
||||
require.NoError(t, hostprocess.WritePidFile(id, deadpid.PID)) // PID file present but process dead
|
||||
require.NoError(t, hostprocess.WritePidFile(id, 999999)) // PID file present but process dead
|
||||
|
||||
mounter := &fakeVolumeMounter{}
|
||||
updater := &fakeVolumeStateUpdater{}
|
||||
|
||||
@@ -41,28 +41,27 @@ func beat(t *testing.T, jsm *JetStreamManager, node string) {
|
||||
|
||||
func TestKVIPSecBarrier_ReportsEveryLiveNodesState(t *testing.T) {
|
||||
barrier, jsm := newTestIPSecBarrier(t)
|
||||
a, b := uniqueAdapterID("ipsec-a"), uniqueAdapterID("ipsec-b")
|
||||
nodes := []string{a, b}
|
||||
nodes := []string{"ipsec-a", "ipsec-b"}
|
||||
for _, n := range nodes {
|
||||
beat(t, jsm, n)
|
||||
}
|
||||
|
||||
require.NoError(t, barrier.Publish(t.Context(), a, host.IPSecNodeStatus{Ready: true, NBReachable: true}))
|
||||
require.NoError(t, barrier.Publish(t.Context(), "ipsec-a", host.IPSecNodeStatus{Ready: true, NBReachable: true}))
|
||||
|
||||
cluster, err := barrier.Cluster(t.Context(), nodes)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]host.IPSecNodeStatus{
|
||||
a: {Ready: true, NBReachable: true},
|
||||
b: {},
|
||||
"ipsec-a": {Ready: true, NBReachable: true},
|
||||
"ipsec-b": {},
|
||||
}, cluster)
|
||||
|
||||
require.NoError(t, barrier.Publish(t.Context(), b, host.IPSecNodeStatus{Ready: true}))
|
||||
require.NoError(t, barrier.Publish(t.Context(), "ipsec-b", host.IPSecNodeStatus{Ready: true}))
|
||||
|
||||
cluster, err = barrier.Cluster(t.Context(), nodes)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]host.IPSecNodeStatus{
|
||||
a: {Ready: true, NBReachable: true},
|
||||
b: {Ready: true},
|
||||
"ipsec-a": {Ready: true, NBReachable: true},
|
||||
"ipsec-b": {Ready: true},
|
||||
}, cluster)
|
||||
}
|
||||
|
||||
@@ -70,14 +69,13 @@ func TestKVIPSecBarrier_ReportsEveryLiveNodesState(t *testing.T) {
|
||||
// is exactly the case the flag must not be asserted over.
|
||||
func TestKVIPSecBarrier_ExplicitlyUnreadyNodeIsReported(t *testing.T) {
|
||||
barrier, jsm := newTestIPSecBarrier(t)
|
||||
node := uniqueAdapterID("ipsec-unready")
|
||||
beat(t, jsm, node)
|
||||
beat(t, jsm, "ipsec-unready")
|
||||
|
||||
require.NoError(t, barrier.Publish(t.Context(), node, host.IPSecNodeStatus{}))
|
||||
require.NoError(t, barrier.Publish(t.Context(), "ipsec-unready", host.IPSecNodeStatus{}))
|
||||
|
||||
cluster, err := barrier.Cluster(t.Context(), []string{node})
|
||||
cluster, err := barrier.Cluster(t.Context(), []string{"ipsec-unready"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]host.IPSecNodeStatus{node: {}}, cluster)
|
||||
assert.Equal(t, map[string]host.IPSecNodeStatus{"ipsec-unready": {}}, cluster)
|
||||
}
|
||||
|
||||
// A node taken out of service has no chassis registered, so there is no tunnel
|
||||
@@ -85,15 +83,14 @@ func TestKVIPSecBarrier_ExplicitlyUnreadyNodeIsReported(t *testing.T) {
|
||||
// the peers that are still talking to each other.
|
||||
func TestKVIPSecBarrier_NodeThatStoppedHeartbeatingDropsOutOfTheSet(t *testing.T) {
|
||||
barrier, jsm := newTestIPSecBarrier(t)
|
||||
node := uniqueAdapterID("ipsec-gone")
|
||||
beat(t, jsm, node)
|
||||
beat(t, jsm, "ipsec-gone")
|
||||
|
||||
require.NoError(t, barrier.Publish(t.Context(), node, host.IPSecNodeStatus{Ready: true}))
|
||||
require.NoError(t, barrier.Publish(t.Context(), "ipsec-gone", host.IPSecNodeStatus{Ready: true}))
|
||||
|
||||
// Far enough ahead that both the status record and the heartbeat are stale.
|
||||
barrier.now = func() time.Time { return time.Now().Add(ipsecStatusFreshness + time.Hour) }
|
||||
|
||||
cluster, err := barrier.Cluster(t.Context(), []string{node})
|
||||
cluster, err := barrier.Cluster(t.Context(), []string{"ipsec-gone"})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, cluster)
|
||||
}
|
||||
@@ -102,30 +99,28 @@ func TestKVIPSecBarrier_NodeThatStoppedHeartbeatingDropsOutOfTheSet(t *testing.T
|
||||
// longer vouch for its own configuration, so it counts as not ready.
|
||||
func TestKVIPSecBarrier_LiveNodeWithAStaleStatusIsNotReady(t *testing.T) {
|
||||
barrier, jsm := newTestIPSecBarrier(t)
|
||||
node := uniqueAdapterID("ipsec-stale")
|
||||
|
||||
require.NoError(t, barrier.Publish(t.Context(), node, host.IPSecNodeStatus{Ready: true}))
|
||||
require.NoError(t, barrier.Publish(t.Context(), "ipsec-stale", host.IPSecNodeStatus{Ready: true}))
|
||||
|
||||
// The status is stale, but a heartbeat lands at the shifted clock.
|
||||
shifted := time.Now().Add(ipsecStatusFreshness + time.Minute)
|
||||
barrier.now = func() time.Time { return shifted }
|
||||
require.NoError(t, jsm.WriteHeartbeat(&Heartbeat{
|
||||
Node: node,
|
||||
Node: "ipsec-stale",
|
||||
Timestamp: shifted.UTC().Format(time.RFC3339),
|
||||
}))
|
||||
|
||||
cluster, err := barrier.Cluster(t.Context(), []string{node})
|
||||
cluster, err := barrier.Cluster(t.Context(), []string{"ipsec-stale"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]host.IPSecNodeStatus{node: {}}, cluster)
|
||||
assert.Equal(t, map[string]host.IPSecNodeStatus{"ipsec-stale": {}}, cluster)
|
||||
}
|
||||
|
||||
// A node that has never published anything and never heartbeated is not part of
|
||||
// the running cluster yet.
|
||||
func TestKVIPSecBarrier_UnknownNodeIsAbsent(t *testing.T) {
|
||||
barrier, _ := newTestIPSecBarrier(t)
|
||||
node := uniqueAdapterID("ipsec-never-seen")
|
||||
|
||||
cluster, err := barrier.Cluster(t.Context(), []string{node})
|
||||
cluster, err := barrier.Cluster(t.Context(), []string{"ipsec-never-seen"})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, cluster)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -174,14 +175,13 @@ func TestEC2CommandRecordsUnderTheCommandNotTheInstance(t *testing.T) {
|
||||
data, err := json.Marshal(commandFor(instanceID,
|
||||
func(a *ec2v1.EC2CommandAttributes) { a.StopInstance = true }))
|
||||
require.NoError(t, err)
|
||||
before := requestCountFor(t, ec2CmdAction("StopInstance"), outcomeClientError)
|
||||
_, err = daemon.natsConn.Request(subject, data, 5*time.Second)
|
||||
require.NoError(t, err)
|
||||
|
||||
// The handler records after it replies, so poll rather than assume the point
|
||||
// has landed by the time the reply arrives.
|
||||
require.Eventually(t, func() bool {
|
||||
return requestCountFor(t, ec2CmdAction("StopInstance"), outcomeClientError) > before
|
||||
return slices.Contains(requestOutcomesFor(t, ec2CmdAction("StopInstance")), outcomeClientError)
|
||||
}, 5*time.Second, 20*time.Millisecond,
|
||||
"a stop naming an instance this node does not hold is the caller's error, and must be counted")
|
||||
assert.Empty(t, requestOutcomesFor(t, subject),
|
||||
@@ -199,7 +199,7 @@ func TestDrainVolumeRecordsItsPointAfterTheWork(t *testing.T) {
|
||||
testutil.StartDrainSocket(t, daemon.config.DataDir, volumeID, "OK\n")
|
||||
subscribeEC2Commands(t, daemon, instanceID)
|
||||
|
||||
before := requestCountFor(t, ec2CmdAction("DrainVolume"), outcomeSuccess)
|
||||
before := len(requestOutcomesFor(t, ec2CmdAction("DrainVolume")))
|
||||
reply := drainRequest(t, daemon, instanceID, volumeID)
|
||||
|
||||
var ack ec2v1.DrainVolumeResponse
|
||||
@@ -209,6 +209,8 @@ func TestDrainVolumeRecordsItsPointAfterTheWork(t *testing.T) {
|
||||
// The goroutine records after it replies, so poll rather than assume the
|
||||
// point has landed by the time the reply arrives.
|
||||
require.Eventually(t, func() bool {
|
||||
return requestCountFor(t, ec2CmdAction("DrainVolume"), outcomeSuccess) > before
|
||||
}, 5*time.Second, 20*time.Millisecond, "the drain goroutine must record its own success point")
|
||||
return len(requestOutcomesFor(t, ec2CmdAction("DrainVolume"))) > before
|
||||
}, 5*time.Second, 20*time.Millisecond, "the drain goroutine must record its own point")
|
||||
|
||||
assert.Contains(t, requestOutcomesFor(t, ec2CmdAction("DrainVolume")), outcomeSuccess)
|
||||
}
|
||||
|
||||
@@ -73,35 +73,6 @@ func requestOutcomesFor(t *testing.T, action string) []string {
|
||||
return outcomes
|
||||
}
|
||||
|
||||
// requestCountFor sums the mulga.requests points recorded under action with
|
||||
// outcome. Points are cumulative for the life of the binary, so a test that
|
||||
// can repeat (-count) must compare counts, not whether a point exists.
|
||||
func requestCountFor(t *testing.T, action, outcome string) int64 {
|
||||
t.Helper()
|
||||
installOutcomeReader(t)
|
||||
|
||||
var rm metricdata.ResourceMetrics
|
||||
require.NoError(t, outcomeReader.Collect(context.Background(), &rm))
|
||||
|
||||
var n int64
|
||||
for _, sm := range rm.ScopeMetrics {
|
||||
for _, m := range sm.Metrics {
|
||||
sum, ok := m.Data.(metricdata.Sum[int64])
|
||||
if m.Name != "mulga.requests" || !ok {
|
||||
continue
|
||||
}
|
||||
for _, dp := range sum.DataPoints {
|
||||
a, _ := dp.Attributes.Value(attribute.Key("rpc.method"))
|
||||
o, _ := dp.Attributes.Value(attribute.Key("outcome"))
|
||||
if a.AsString() == action && o.AsString() == outcome {
|
||||
n += dp.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// The wrapper must record whatever the handler reports. Before the handler
|
||||
// type carried an outcome there was nothing to record and every daemon NATS
|
||||
// point arrived with the attribute unset.
|
||||
|
||||
@@ -1,31 +1,20 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Measures a child the test owns: the test binary's own RSS grows with every
|
||||
// test that ran before it, so it cannot bound anything.
|
||||
func TestMeasureQEMURSSMiB_Self(t *testing.T) {
|
||||
cmd := exec.Command("sleep", "60")
|
||||
require.NoError(t, cmd.Start())
|
||||
t.Cleanup(func() {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
})
|
||||
|
||||
rss, err := measureQEMURSSMiB(cmd.Process.Pid)
|
||||
rss, err := measureQEMURSSMiB(os.Getpid())
|
||||
if err != nil {
|
||||
t.Fatalf("measureQEMURSSMiB child: %v", err)
|
||||
t.Fatalf("measureQEMURSSMiB self: %v", err)
|
||||
}
|
||||
if rss <= 0 {
|
||||
t.Errorf("expected RSS > 0, got %f", rss)
|
||||
}
|
||||
if rss > 64 {
|
||||
t.Errorf("RSS %f MiB seems unreasonably high for sleep(1)", rss)
|
||||
if rss > 1024 {
|
||||
t.Errorf("RSS %f MiB seems unreasonably high for a test process", rss)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mulgadc/spinifex/internal/testkit/deadpid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -209,7 +208,7 @@ func TestKillProcess(t *testing.T) {
|
||||
wg.Wait()
|
||||
|
||||
// Test killing non-existent process
|
||||
err = KillProcess(deadpid.PID)
|
||||
err = KillProcess(999999)
|
||||
assert.Error(t, err, "Should error when killing non-existent process")
|
||||
}
|
||||
|
||||
@@ -224,7 +223,7 @@ func TestProcessAlive(t *testing.T) {
|
||||
assert.True(t, ProcessAlive(pid), "a running process must report alive")
|
||||
assert.False(t, ProcessAlive(0), "pid 0 must report not alive")
|
||||
assert.False(t, ProcessAlive(-1), "negative pid must report not alive")
|
||||
assert.False(t, ProcessAlive(deadpid.PID), "a non-existent pid must report not alive")
|
||||
assert.False(t, ProcessAlive(999999), "a non-existent pid must report not alive")
|
||||
|
||||
require.NoError(t, cmd.Process.Kill())
|
||||
wg.Wait()
|
||||
|
||||
@@ -592,20 +592,6 @@ func launchService(config *config.ClusterConfig) error {
|
||||
}
|
||||
}()
|
||||
|
||||
// Served whatever the gateway's [quota] setting: the claim is about this
|
||||
// binary's override writers, not about whether quotas are enabled.
|
||||
quotaSealCapSub, err := registerQuotaOverrideSealCapability(natsConn, config.Node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("subscribe quota override seal capability: %w", err)
|
||||
}
|
||||
if quotaSealCapSub != nil {
|
||||
defer func() {
|
||||
if err := quotaSealCapSub.Unsubscribe(); err != nil {
|
||||
slog.Warn("Quota override seal capability: unsubscribe failed", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Bedrock token quota reads the stream-fed usage counters built above,
|
||||
// independent of whether the standing-infra dimensions are enabled.
|
||||
quotaSvc.SetBedrockUsage(bedrockUsage)
|
||||
@@ -658,7 +644,7 @@ func launchService(config *config.ClusterConfig) error {
|
||||
})
|
||||
stsDeps := stsawsapi.Deps{STS: stsService, IAM: iamService}
|
||||
stsControlPlane := stsawsapi.NewRegistration(stsDeps)
|
||||
operatorControlPlane := operatoradmin.NewRegistration(withClusterNodes(operatoradmin.Deps{
|
||||
operatorControlPlane := operatoradmin.NewRegistration(operatoradmin.Deps{
|
||||
Version: version,
|
||||
Commit: commit,
|
||||
NATS: natsConn,
|
||||
@@ -670,9 +656,11 @@ func launchService(config *config.ClusterConfig) error {
|
||||
|
||||
IAM: iamService,
|
||||
Quota: quotaSvc,
|
||||
ExpectedNodes: len(config.Nodes),
|
||||
NodeIDs: nodeIDs,
|
||||
BucketStore: objStore,
|
||||
SignupMaxAccounts: signupMaxAccounts,
|
||||
}, config))
|
||||
})
|
||||
// The SQS worker's internal surface binds its buckets per request and never
|
||||
// creates them, so it refuses every caller until a worker launch has.
|
||||
sqsWorkerControlPlane := sqsworkerapi.NewRegistration(sqsworkerapi.JetStreamDeps(js))
|
||||
@@ -874,13 +862,3 @@ func parseBedrockEndpoints(raw string) map[string]string {
|
||||
}
|
||||
return endpoints
|
||||
}
|
||||
|
||||
// withClusterNodes sets the configured node sets teardown reaches: the
|
||||
// daemon on every node, and the gateway only on nodes that run awsgw. An
|
||||
// empty gateway set would fail every account termination closed.
|
||||
func withClusterNodes(d operatoradmin.Deps, config *config.ClusterConfig) operatoradmin.Deps {
|
||||
d.ExpectedNodes = len(config.Nodes)
|
||||
d.NodeIDs = slices.Sorted(maps.Keys(config.Nodes))
|
||||
d.GatewayNodeIDs = config.NodesRunning("awsgw")
|
||||
return d
|
||||
}
|
||||
|
||||
@@ -3,9 +3,6 @@ package awsgw
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
|
||||
operatoradmin "github.com/mulgadc/spinifex/spinifex/operator/admin"
|
||||
|
||||
awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
|
||||
stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
|
||||
"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
|
||||
@@ -36,20 +33,3 @@ func TestComposeGateway_SetsEveryProductionHook(t *testing.T) {
|
||||
assert.NotNil(t, gw.Admission, "Admission")
|
||||
assert.NotNil(t, gw.Readiness, "Readiness")
|
||||
}
|
||||
|
||||
// Account termination probes the gateway node set before sealing, so awsgw
|
||||
// must hand the operator surface both node sets from the cluster config.
|
||||
func TestWithClusterNodes_SetsDaemonAndGatewaySets(t *testing.T) {
|
||||
cfg := &config.ClusterConfig{Nodes: map[string]config.Config{
|
||||
"n1": {Services: []string{"nats", "awsgw", "daemon"}},
|
||||
"n2": {Services: []string{"daemon"}},
|
||||
"n3": {},
|
||||
}}
|
||||
|
||||
got := withClusterNodes(operatoradmin.Deps{}, cfg)
|
||||
|
||||
assert.Equal(t, 3, got.ExpectedNodes)
|
||||
assert.Equal(t, []string{"n1", "n2", "n3"}, got.NodeIDs)
|
||||
// n3 has no service list, which means every service, awsgw included.
|
||||
assert.Equal(t, []string{"n1", "n3"}, got.GatewayNodeIDs)
|
||||
}
|
||||
|
||||
@@ -2,13 +2,9 @@ package awsgw
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
admissionv1 "github.com/mulgadc/spinifex/contracts/admission/v1"
|
||||
admissionquota "github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
@@ -41,51 +37,3 @@ func handleSealAccountQuotaOverride(quota *admissionquota.Service) func(context.
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// registerQuotaOverrideSealCapability serves this gateway's read-only seal
|
||||
// capability on its own node's subject, with no queue group so a probe reaches
|
||||
// every gateway configured as that node. An invalid node ID serves nothing and
|
||||
// returns a nil subscription, so the probe reads this gateway as incompatible.
|
||||
func registerQuotaOverrideSealCapability(nc *nats.Conn, node string) (*nats.Subscription, error) {
|
||||
if !admissionv1.ValidSealCapabilityNode(node) {
|
||||
slog.Warn("Quota override seal: node ID is not a valid subject token, capability not served", "node", node)
|
||||
return nil, nil
|
||||
}
|
||||
return nc.Subscribe(admissionv1.AccountQuotaOverrideSealCapabilitySubject(node), func(msg *nats.Msg) {
|
||||
respondWithNodeID(node, msg, quotaOverrideSealCapabilityReply(node, msg.Data))
|
||||
})
|
||||
}
|
||||
|
||||
// quotaOverrideSealCapabilityReply answers one capability request: its own
|
||||
// node with version 1, or an error payload for a malformed request or a
|
||||
// request naming another node. It reads no KV and takes no account.
|
||||
func quotaOverrideSealCapabilityReply(node string, data []byte) []byte {
|
||||
var req admissionv1.AccountQuotaOverrideSealCapabilityRequest
|
||||
if errResp := awserrors.UnmarshalJsonPayload(&req, data); errResp != nil {
|
||||
return errResp
|
||||
}
|
||||
if req.Node != node {
|
||||
return awserrors.GenerateErrorPayloadWithMessage(awserrors.ErrorInvalidParameterValue,
|
||||
fmt.Sprintf("seal capability request for node %q answered by node %q", req.Node, node))
|
||||
}
|
||||
out, err := json.Marshal(admissionv1.AccountQuotaOverrideSealCapabilityReply{
|
||||
Node: node,
|
||||
CapabilityVersion: admissionv1.AccountQuotaOverrideSealCapabilityVersion,
|
||||
CorrelationRef: req.CorrelationRef,
|
||||
})
|
||||
if err != nil {
|
||||
return awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// respondWithNodeID replies with the X-Node-ID header the probe uses to tell
|
||||
// which gateway answered; natsmsg.ServeNATSRequestCtx sets none.
|
||||
func respondWithNodeID(node string, msg *nats.Msg, data []byte) {
|
||||
reply := nats.NewMsg("")
|
||||
reply.Data = data
|
||||
reply.Header.Set(natsmsg.NodeIDHeader, node)
|
||||
if err := msg.RespondMsg(reply); err != nil {
|
||||
slog.Error("Quota override seal: failed to respond to capability request", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,16 +2,13 @@ package awsgw
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
admissionv1 "github.com/mulgadc/spinifex/contracts/admission/v1"
|
||||
"github.com/mulgadc/spinifex/internal/testkit"
|
||||
admissionquota "github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
|
||||
awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
|
||||
natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -85,90 +82,3 @@ func TestQuotaOverrideSeal_NoResponderWithoutGateway(t *testing.T) {
|
||||
nc, admissionv1.SealAccountQuotaOverrideSubject, req, 500*time.Millisecond, "")
|
||||
require.Error(t, err, "no subscriber must surface as a no-responders error")
|
||||
}
|
||||
|
||||
func newQuotaSealCapabilityConns(t *testing.T) (*nats.Conn, *nats.Conn) {
|
||||
t.Helper()
|
||||
ns, nc := testutil.StartTestNATS(t)
|
||||
caller, err := nats.Connect(ns.ClientURL())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(caller.Close)
|
||||
return nc, caller
|
||||
}
|
||||
|
||||
func quotaSealCapabilityRequest(t *testing.T, caller *nats.Conn, subjectNode, bodyNode string) *nats.Msg {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(admissionv1.AccountQuotaOverrideSealCapabilityRequest{Node: bodyNode, CorrelationRef: "req-1"})
|
||||
require.NoError(t, err)
|
||||
reply, err := caller.Request(admissionv1.AccountQuotaOverrideSealCapabilitySubject(subjectNode), data, 2*time.Second)
|
||||
require.NoError(t, err)
|
||||
return reply
|
||||
}
|
||||
|
||||
func TestQuotaOverrideSealCapability_AnswersOwnNode(t *testing.T) {
|
||||
nc, caller := newQuotaSealCapabilityConns(t)
|
||||
sub, err := registerQuotaOverrideSealCapability(nc, "node-1")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
raw := quotaSealCapabilityRequest(t, caller, "node-1", "node-1")
|
||||
assert.Equal(t, "node-1", raw.Header.Get(natsmsg.NodeIDHeader))
|
||||
|
||||
var reply admissionv1.AccountQuotaOverrideSealCapabilityReply
|
||||
require.NoError(t, json.Unmarshal(raw.Data, &reply))
|
||||
assert.Equal(t, "node-1", reply.Node)
|
||||
assert.Equal(t, admissionv1.AccountQuotaOverrideSealCapabilityVersion, reply.CapabilityVersion)
|
||||
assert.Equal(t, "req-1", reply.CorrelationRef)
|
||||
}
|
||||
|
||||
func TestQuotaOverrideSealCapability_RefusesOtherNode(t *testing.T) {
|
||||
nc, caller := newQuotaSealCapabilityConns(t)
|
||||
sub, err := registerQuotaOverrideSealCapability(nc, "node-1")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
raw := quotaSealCapabilityRequest(t, caller, "node-1", "node-2")
|
||||
var re struct {
|
||||
Code string `json:"Code"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(raw.Data, &re))
|
||||
assert.Equal(t, awserrors.ErrorInvalidParameterValue, re.Code)
|
||||
}
|
||||
|
||||
// Two gateways configured with one node ID both answer: the subject has no
|
||||
// queue group, which is how the probe catches a duplicate.
|
||||
func TestQuotaOverrideSealCapability_HasNoQueueGroup(t *testing.T) {
|
||||
nc, caller := newQuotaSealCapabilityConns(t)
|
||||
for range 2 {
|
||||
sub, err := registerQuotaOverrideSealCapability(nc, "node-1")
|
||||
require.NoError(t, err)
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
}
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
inbox := caller.NewRespInbox()
|
||||
replies, err := caller.SubscribeSync(inbox)
|
||||
require.NoError(t, err)
|
||||
data, err := json.Marshal(admissionv1.AccountQuotaOverrideSealCapabilityRequest{Node: "node-1"})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, caller.PublishRequest(admissionv1.AccountQuotaOverrideSealCapabilitySubject("node-1"), inbox, data))
|
||||
|
||||
for range 2 {
|
||||
_, err := replies.NextMsg(2 * time.Second)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuotaOverrideSealCapability_InvalidNodeServesNothing(t *testing.T) {
|
||||
nc, caller := newQuotaSealCapabilityConns(t)
|
||||
for _, node := range []string{"", "a.b", "node >"} {
|
||||
sub, err := registerQuotaOverrideSealCapability(nc, node)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, sub, "%q", node)
|
||||
}
|
||||
require.NoError(t, nc.Flush())
|
||||
|
||||
_, err := caller.Request(admissionv1.AccountQuotaOverrideSealCapabilitySubject("a"), []byte(`{"node":"a"}`), 200*time.Millisecond)
|
||||
assert.ErrorIs(t, err, nats.ErrNoResponders)
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,2 +1,2 @@
|
||||
const __vite__mapDeps=(i,m=__vite__mapDeps,d=(m.f||(m.f=["assets/_-CY111NVs.js","assets/schemas-BGv4hnp9.js","assets/rolldown-runtime-XKyaHtl9.js","assets/object-browser-DuO9QBxg.js","assets/QueryClientProvider-CTm_NoFP.js","assets/queryOptions-twzzLLsx.js","assets/useQuery-CZvSQig-.js","assets/lazyRouteComponent-BnnVgQVQ.js","assets/matchContext-DBnoSZp3.js","assets/link-D5tcioHY.js","assets/useSelector-Cvbg8b7C.js","assets/data-table-CqrUk0mI.js","assets/visuallyHidden-D4_nmHIB.js","assets/check-B7aLYzUb.js","assets/useRegisterFieldControl-2x4pEgaC.js","assets/useAriaLabelledBy-BMp0PcVw.js","assets/useControlled-BP9HwLGp.js","assets/table-BEe-3So1.js","assets/error-banner-DYkSEx5d.js","assets/chevron-right-DH7mP24a.js","assets/upload-C5uIcpy7.js","assets/sliders-horizontal-DZ-kpCNX.js","assets/detail-panel-BBYLO5I7.js","assets/x-8n_Gyi12.js","assets/chevron-down-By1UhB9P.js","assets/copy-BpriwVcd.js","assets/loader-circle-CiggOyRj.js","assets/useScrollLock-FIlKWKju.js","assets/useValueAsRef-BoFfo9R-.js","assets/ToolbarRootContext-B6mBfOj0.js","assets/isElementDisabled-CuK_3Aso.js","assets/store-BTg3TzdU.js","assets/collapsible-C7X1O8X9.js","assets/tabs-DFnnXw6G.js","assets/use-copy-to-clipboard-CAJ8gQCx.js","assets/input-group-C8b5mBfA.js","assets/field-B2ItYtQp.js","assets/textarea-CUIsQWob.js","assets/trash-REoRRV_c.js","assets/uploads-4BEEb2qM.js","assets/awsClient-pD_9VmT7.js","assets/auth-DA1pq184.js","assets/index.browser-BEm-79SP.js","assets/s3-C1ORaLWG.js","assets/useMutation-CutM3PJn.js","assets/s3-BU8yVfSp.js","assets/page-heading-75JYUGDy.js","assets/resource-view-BXETZc8v.js","assets/selection-bar-MuN8z7Qc.js","assets/use-row-selection-CZtmDppg.js","assets/bulk-delete-B7E-9nKZ.js","assets/circle-x-DHQkYhPI.js","assets/alert-dialog-DKUcMJKb.js","assets/DialogTitle-BC_80-Qv.js","assets/confirm-phrase-Bvuzco3W.js","assets/resource-delete-dialog-DgcEsN9Y.js","assets/delete-confirmation-dialog-DRnnnG6z.js"])))=>i.map(i=>d[i]);
|
||||
import{K as e,s as t}from"./schemas-BGv4hnp9.js";import{n,t as r}from"./lazyRouteComponent-BnnVgQVQ.js";import{t as i}from"./searchMiddleware-VYy8PwPp.js";import{h as a,m as o}from"./detail-panel-BBYLO5I7.js";import{Mo as s}from"./awsClient-pD_9VmT7.js";import{t as c}from"./s3-BU8yVfSp.js";var l=n(`/_auth/s3/ls/$bucket/$`)({validateSearch:t(a),search:{middlewares:[i(o)]},loader:async({context:t,params:n})=>{await t.queryClient.query({...c(n.bucket,e(n._splat??``)),staleTime:`static`})},head:({params:t})=>({meta:[{title:`${t.bucket}/${e(t._splat??``)} | Object Storage | Mulga`}]}),component:r(()=>s(()=>import(`./_-CY111NVs.js`),__vite__mapDeps([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56])),`component`)});export{l as t};
|
||||
const __vite__mapDeps=(i,m=__vite__mapDeps,d=(m.f||(m.f=["assets/_-DvgDl9x5.js","assets/schemas-BGv4hnp9.js","assets/rolldown-runtime-XKyaHtl9.js","assets/object-browser-osJBAXQ0.js","assets/QueryClientProvider-CTm_NoFP.js","assets/queryOptions-twzzLLsx.js","assets/useQuery-CZvSQig-.js","assets/lazyRouteComponent-BnnVgQVQ.js","assets/matchContext-DBnoSZp3.js","assets/link-D5tcioHY.js","assets/useSelector-Cvbg8b7C.js","assets/data-table-CqrUk0mI.js","assets/visuallyHidden-D4_nmHIB.js","assets/check-B7aLYzUb.js","assets/useRegisterFieldControl-2x4pEgaC.js","assets/useAriaLabelledBy-BMp0PcVw.js","assets/useControlled-BP9HwLGp.js","assets/table-BEe-3So1.js","assets/error-banner-DYkSEx5d.js","assets/chevron-right-DH7mP24a.js","assets/upload-C5uIcpy7.js","assets/sliders-horizontal-DZ-kpCNX.js","assets/detail-panel-BBYLO5I7.js","assets/x-8n_Gyi12.js","assets/chevron-down-By1UhB9P.js","assets/copy-BpriwVcd.js","assets/loader-circle-CiggOyRj.js","assets/useScrollLock-FIlKWKju.js","assets/useValueAsRef-BoFfo9R-.js","assets/ToolbarRootContext-B6mBfOj0.js","assets/isElementDisabled-CuK_3Aso.js","assets/store-BTg3TzdU.js","assets/collapsible-C7X1O8X9.js","assets/tabs-DFnnXw6G.js","assets/use-copy-to-clipboard-CAJ8gQCx.js","assets/input-group-C8b5mBfA.js","assets/field-B2ItYtQp.js","assets/textarea-CUIsQWob.js","assets/trash-REoRRV_c.js","assets/uploads-CQA8Vkth.js","assets/awsClient-lo5RjjXh.js","assets/auth-DA1pq184.js","assets/index.browser-BEm-79SP.js","assets/s3-DVw6bxYM.js","assets/useMutation-CutM3PJn.js","assets/s3-C23jzLs5.js","assets/page-heading-75JYUGDy.js","assets/resource-view-BXETZc8v.js","assets/selection-bar-MuN8z7Qc.js","assets/use-row-selection-CZtmDppg.js","assets/bulk-delete-B7E-9nKZ.js","assets/circle-x-DHQkYhPI.js","assets/alert-dialog-DKUcMJKb.js","assets/DialogTitle-BC_80-Qv.js","assets/confirm-phrase-Bvuzco3W.js","assets/resource-delete-dialog-DgcEsN9Y.js","assets/delete-confirmation-dialog-DRnnnG6z.js"])))=>i.map(i=>d[i]);
|
||||
import{K as e,s as t}from"./schemas-BGv4hnp9.js";import{n,t as r}from"./lazyRouteComponent-BnnVgQVQ.js";import{t as i}from"./searchMiddleware-VYy8PwPp.js";import{h as a,m as o}from"./detail-panel-BBYLO5I7.js";import{Mo as s}from"./awsClient-lo5RjjXh.js";import{t as c}from"./s3-C23jzLs5.js";var l=n(`/_auth/s3/ls/$bucket/$`)({validateSearch:t(a),search:{middlewares:[i(o)]},loader:async({context:t,params:n})=>{await t.queryClient.query({...c(n.bucket,e(n._splat??``)),staleTime:`static`})},head:({params:t})=>({meta:[{title:`${t.bucket}/${e(t._splat??``)} | Object Storage | Mulga`}]}),component:r(()=>s(()=>import(`./_-DvgDl9x5.js`),__vite__mapDeps([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56])),`component`)});export{l as t};
|
||||
+1
-1
@@ -1 +1 @@
|
||||
import{Ht as e,K as t,Rt as n}from"./schemas-BGv4hnp9.js";import{t as r}from"./object-browser-DuO9QBxg.js";import{t as i}from"./_-JEkaz5i0.js";var a=n(),o=e();function s(){let e=(0,a.c)(10),{bucket:n,_splat:s}=i.useParams(),c=s??``,l;e[0]===c?l=e[1]:(l=t(c),e[0]=c,e[1]=l);let u=l,d=i.useSearch(),f=i.useNavigate(),p;e[2]===f?p=e[3]:(p=function(e){f({search:t=>({...t,...e}),replace:!(`selected`in e)})},e[2]=f,e[3]=p);let m=p,h=`${n}/${u}`,g;return e[4]!==n||e[5]!==u||e[6]!==m||e[7]!==d||e[8]!==h?(g=(0,o.jsx)(r,{bucket:n,folder:u,onSearchChange:m,search:d},h),e[4]=n,e[5]=u,e[6]=m,e[7]=d,e[8]=h,e[9]=g):g=e[9],g}export{s as component};
|
||||
import{Ht as e,K as t,Rt as n}from"./schemas-BGv4hnp9.js";import{t as r}from"./object-browser-osJBAXQ0.js";import{t as i}from"./_-Dt-n8jD3.js";var a=n(),o=e();function s(){let e=(0,a.c)(10),{bucket:n,_splat:s}=i.useParams(),c=s??``,l;e[0]===c?l=e[1]:(l=t(c),e[0]=c,e[1]=l);let u=l,d=i.useSearch(),f=i.useNavigate(),p;e[2]===f?p=e[3]:(p=function(e){f({search:t=>({...t,...e}),replace:!(`selected`in e)})},e[2]=f,e[3]=p);let m=p,h=`${n}/${u}`,g;return e[4]!==n||e[5]!==u||e[6]!==m||e[7]!==d||e[8]!==h?(g=(0,o.jsx)(r,{bucket:n,folder:u,onSearchChange:m,search:d},h),e[4]=n,e[5]=u,e[6]=m,e[7]=d,e[8]=h,e[9]=g):g=e[9],g}export{s as component};
|
||||
+1
-1
File diff suppressed because one or more lines are too long
+2
-2
@@ -1,2 +1,2 @@
|
||||
const __vite__mapDeps=(i,m=__vite__mapDeps,d=(m.f||(m.f=["assets/_bucket-Du31dpcE.js","assets/schemas-BGv4hnp9.js","assets/rolldown-runtime-XKyaHtl9.js","assets/object-browser-DuO9QBxg.js","assets/QueryClientProvider-CTm_NoFP.js","assets/queryOptions-twzzLLsx.js","assets/useQuery-CZvSQig-.js","assets/lazyRouteComponent-BnnVgQVQ.js","assets/matchContext-DBnoSZp3.js","assets/link-D5tcioHY.js","assets/useSelector-Cvbg8b7C.js","assets/data-table-CqrUk0mI.js","assets/visuallyHidden-D4_nmHIB.js","assets/check-B7aLYzUb.js","assets/useRegisterFieldControl-2x4pEgaC.js","assets/useAriaLabelledBy-BMp0PcVw.js","assets/useControlled-BP9HwLGp.js","assets/table-BEe-3So1.js","assets/error-banner-DYkSEx5d.js","assets/chevron-right-DH7mP24a.js","assets/upload-C5uIcpy7.js","assets/sliders-horizontal-DZ-kpCNX.js","assets/detail-panel-BBYLO5I7.js","assets/x-8n_Gyi12.js","assets/chevron-down-By1UhB9P.js","assets/copy-BpriwVcd.js","assets/loader-circle-CiggOyRj.js","assets/useScrollLock-FIlKWKju.js","assets/useValueAsRef-BoFfo9R-.js","assets/ToolbarRootContext-B6mBfOj0.js","assets/isElementDisabled-CuK_3Aso.js","assets/store-BTg3TzdU.js","assets/collapsible-C7X1O8X9.js","assets/tabs-DFnnXw6G.js","assets/use-copy-to-clipboard-CAJ8gQCx.js","assets/input-group-C8b5mBfA.js","assets/field-B2ItYtQp.js","assets/textarea-CUIsQWob.js","assets/trash-REoRRV_c.js","assets/uploads-4BEEb2qM.js","assets/awsClient-pD_9VmT7.js","assets/auth-DA1pq184.js","assets/index.browser-BEm-79SP.js","assets/s3-C1ORaLWG.js","assets/useMutation-CutM3PJn.js","assets/s3-BU8yVfSp.js","assets/page-heading-75JYUGDy.js","assets/resource-view-BXETZc8v.js","assets/selection-bar-MuN8z7Qc.js","assets/use-row-selection-CZtmDppg.js","assets/bulk-delete-B7E-9nKZ.js","assets/circle-x-DHQkYhPI.js","assets/alert-dialog-DKUcMJKb.js","assets/DialogTitle-BC_80-Qv.js","assets/confirm-phrase-Bvuzco3W.js","assets/resource-delete-dialog-DgcEsN9Y.js","assets/delete-confirmation-dialog-DRnnnG6z.js"])))=>i.map(i=>d[i]);
|
||||
import{s as e}from"./schemas-BGv4hnp9.js";import{n as t,t as n}from"./lazyRouteComponent-BnnVgQVQ.js";import{t as r}from"./searchMiddleware-VYy8PwPp.js";import{h as i,m as a}from"./detail-panel-BBYLO5I7.js";import{Mo as o}from"./awsClient-pD_9VmT7.js";import{t as s}from"./s3-BU8yVfSp.js";var c=t(`/_auth/s3/ls/$bucket/`)({validateSearch:e(i),search:{middlewares:[r(a)]},loader:async({context:e,params:t})=>{await e.queryClient.query({...s(t.bucket,``),staleTime:`static`})},head:({params:e})=>({meta:[{title:`${e.bucket} | Object Storage | Mulga`}]}),component:n(()=>o(()=>import(`./_bucket-Du31dpcE.js`),__vite__mapDeps([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56])),`component`)});export{c as t};
|
||||
const __vite__mapDeps=(i,m=__vite__mapDeps,d=(m.f||(m.f=["assets/_bucket-BVDoXi_y.js","assets/schemas-BGv4hnp9.js","assets/rolldown-runtime-XKyaHtl9.js","assets/object-browser-osJBAXQ0.js","assets/QueryClientProvider-CTm_NoFP.js","assets/queryOptions-twzzLLsx.js","assets/useQuery-CZvSQig-.js","assets/lazyRouteComponent-BnnVgQVQ.js","assets/matchContext-DBnoSZp3.js","assets/link-D5tcioHY.js","assets/useSelector-Cvbg8b7C.js","assets/data-table-CqrUk0mI.js","assets/visuallyHidden-D4_nmHIB.js","assets/check-B7aLYzUb.js","assets/useRegisterFieldControl-2x4pEgaC.js","assets/useAriaLabelledBy-BMp0PcVw.js","assets/useControlled-BP9HwLGp.js","assets/table-BEe-3So1.js","assets/error-banner-DYkSEx5d.js","assets/chevron-right-DH7mP24a.js","assets/upload-C5uIcpy7.js","assets/sliders-horizontal-DZ-kpCNX.js","assets/detail-panel-BBYLO5I7.js","assets/x-8n_Gyi12.js","assets/chevron-down-By1UhB9P.js","assets/copy-BpriwVcd.js","assets/loader-circle-CiggOyRj.js","assets/useScrollLock-FIlKWKju.js","assets/useValueAsRef-BoFfo9R-.js","assets/ToolbarRootContext-B6mBfOj0.js","assets/isElementDisabled-CuK_3Aso.js","assets/store-BTg3TzdU.js","assets/collapsible-C7X1O8X9.js","assets/tabs-DFnnXw6G.js","assets/use-copy-to-clipboard-CAJ8gQCx.js","assets/input-group-C8b5mBfA.js","assets/field-B2ItYtQp.js","assets/textarea-CUIsQWob.js","assets/trash-REoRRV_c.js","assets/uploads-CQA8Vkth.js","assets/awsClient-lo5RjjXh.js","assets/auth-DA1pq184.js","assets/index.browser-BEm-79SP.js","assets/s3-DVw6bxYM.js","assets/useMutation-CutM3PJn.js","assets/s3-C23jzLs5.js","assets/page-heading-75JYUGDy.js","assets/resource-view-BXETZc8v.js","assets/selection-bar-MuN8z7Qc.js","assets/use-row-selection-CZtmDppg.js","assets/bulk-delete-B7E-9nKZ.js","assets/circle-x-DHQkYhPI.js","assets/alert-dialog-DKUcMJKb.js","assets/DialogTitle-BC_80-Qv.js","assets/confirm-phrase-Bvuzco3W.js","assets/resource-delete-dialog-DgcEsN9Y.js","assets/delete-confirmation-dialog-DRnnnG6z.js"])))=>i.map(i=>d[i]);
|
||||
import{s as e}from"./schemas-BGv4hnp9.js";import{n as t,t as n}from"./lazyRouteComponent-BnnVgQVQ.js";import{t as r}from"./searchMiddleware-VYy8PwPp.js";import{h as i,m as a}from"./detail-panel-BBYLO5I7.js";import{Mo as o}from"./awsClient-lo5RjjXh.js";import{t as s}from"./s3-C23jzLs5.js";var c=t(`/_auth/s3/ls/$bucket/`)({validateSearch:e(i),search:{middlewares:[r(a)]},loader:async({context:e,params:t})=>{await e.queryClient.query({...s(t.bucket,``),staleTime:`static`})},head:({params:e})=>({meta:[{title:`${e.bucket} | Object Storage | Mulga`}]}),component:n(()=>o(()=>import(`./_bucket-BVDoXi_y.js`),__vite__mapDeps([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56])),`component`)});export{c as t};
|
||||
+1
-1
@@ -1 +1 @@
|
||||
import{Ht as e,Rt as t}from"./schemas-BGv4hnp9.js";import{t as n}from"./object-browser-DuO9QBxg.js";import{t as r}from"./_bucket-BIwJQdV1.js";var i=t(),a=e();function o(){let e=(0,i.c)(6),{bucket:t}=r.useParams(),o=r.useSearch(),s=r.useNavigate(),c;e[0]===s?c=e[1]:(c=function(e){s({search:t=>({...t,...e}),replace:!(`selected`in e)})},e[0]=s,e[1]=c);let l=c,u;return e[2]!==t||e[3]!==l||e[4]!==o?(u=(0,a.jsx)(n,{bucket:t,folder:``,onSearchChange:l,search:o},t),e[2]=t,e[3]=l,e[4]=o,e[5]=u):u=e[5],u}export{o as component};
|
||||
import{Ht as e,Rt as t}from"./schemas-BGv4hnp9.js";import{t as n}from"./object-browser-osJBAXQ0.js";import{t as r}from"./_bucket-BK_jd9s3.js";var i=t(),a=e();function o(){let e=(0,i.c)(6),{bucket:t}=r.useParams(),o=r.useSearch(),s=r.useNavigate(),c;e[0]===s?c=e[1]:(c=function(e){s({search:t=>({...t,...e}),replace:!(`selected`in e)})},e[0]=s,e[1]=c);let l=c,u;return e[2]!==t||e[3]!==l||e[4]!==o?(u=(0,a.jsx)(n,{bucket:t,folder:``,onSearchChange:l,search:o},t),e[2]=t,e[3]=l,e[4]=o,e[5]=u):u=e[5],u}export{o as component};
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user