mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
Josh Siviour
62e7f983ee
* test: assert the gateway->vpcd->OVN crossing in the integration tier
The tier already ran the real VPCServiceImpl for the AWS-facing half, but
nothing consumed the vpc.* events it published: the SG topics were answered
by a canned {"success":true} and the fire-and-forget topology topics were
dropped. Both sides of that wire were tested only against stubs, so the join
was asserted nowhere below the live e2e tier.
Wire a real subscribers.Subscriber over a real in-process OVN Northbound DB
(ovntest.StartNB), opt-in per test so only tests that assert OVN state pay
the startup cost. StartDaemonLite gains WithRealVPCD to suppress its stubs,
since a stub and the subscriber both answering one request-reply would race.
Scope is the crossing, not NB row shapes — those stay with the reconcile
scenario tests. Everything host-dependent (ovn-controller, OVS, taps, netns,
nftables, packet paths) remains the live tier's claim.
* fix: make the integration tier re-runnable and quiet the ensure probe
Two defects the vpcd/OVN work surfaced.
The tier was green at -count=1 and red package-wide at -count=2. Test
accounts were keyed by t.Name(), which is unique among tests but repeats
across passes in one TestMain, so a second pass inherited the first pass's
JetStream namespace — including an ECR signing key sealed with a master key
StartGateway regenerates per call. Key each account off a counter instead.
TestRunInstances_ClientTokenIdempotency then collided with itself for the
same reason: its dedup store outlives a pass, so its fixed ClientToken made
the second pass's opening launch look like a replay. Suffix the tokens per
invocation.
ensureNamedRowOps' wait-op is built to fail fast when the row it guards
already exists, which is the ordinary idempotent path, but transactOps
reported it as "OVSDB operation failed" at ERROR. Every re-ensure emitted
one, drowning real faults in the log sink. Classify that one result as the
probe outcome it is and log it at Debug.
* test: cover transactOps' failed-result paths
The classifier split one log line into two branches and neither was
exercised: transactOps sat at 31% because EnsureLogicalRouter's cache
lookup absorbs a re-ensure before it ever transacts, so no unit test
reached a failed operation result.
Drive both kinds directly against a real NB — the guard probe finding an
existing row, and two same-named inserts violating the schema's name
index — since only the second is a fault and they now log at different
levels. transactOps goes to 94%; the remainder is the transport-error
return, which needs a broken connection to reach.
106 lines
4.3 KiB
Go
106 lines
4.3 KiB
Go
//go:build integration
|
|
|
|
package integration
|
|
|
|
import (
|
|
"fmt"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/service/ec2"
|
|
"github.com/mulgadc/spinifex/spinifex/types"
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// clientTokenRun numbers each invocation of the test below so its ClientToken
|
|
// values are fresh on every pass; see the test's doc comment.
|
|
var clientTokenRun atomic.Int64
|
|
|
|
// TestRunInstances_ClientTokenIdempotency proves ClientToken dedup
|
|
// (gateway/ec2/instance/RunInstances.go's ClientTokenStore wrapping) runs for
|
|
// real through the full gateway: a replayed token must not reach the daemon a
|
|
// second time, and a different token must launch again.
|
|
//
|
|
// getClientTokenStore (clienttoken.go) binds its JetStream KV bucket via a
|
|
// process-wide sync.Once, so the very first test in this package's binary
|
|
// that supplies a ClientToken permanently decides which test's NATS account
|
|
// backs the store for the rest of the run. No other test can collide with
|
|
// this one — every key is namespaced by accountID+token, and these token
|
|
// values appear nowhere else in the suite — but THIS test collides with
|
|
// itself under -count=N: the store outlives each pass, so a fixed token would
|
|
// make the second pass's opening launch look like a replay of the first
|
|
// pass's. Hence the per-invocation suffix.
|
|
func TestRunInstances_ClientTokenIdempotency(t *testing.T) {
|
|
gw := StartGateway(t)
|
|
|
|
run := clientTokenRun.Add(1)
|
|
|
|
const (
|
|
instanceType = "t3.micro"
|
|
nodeID = "ct-node"
|
|
)
|
|
|
|
statusResp := mustMarshal(t, &types.NodeStatusResponse{
|
|
Node: nodeID,
|
|
InstanceTypes: []types.InstanceTypeCap{
|
|
{Name: instanceType, Available: 10},
|
|
},
|
|
})
|
|
gw.StubSubject(t, "spinifex.node.status", statusResp)
|
|
|
|
// dispatchCount records how many times the daemon-side per-node launch
|
|
// subject actually fired, so a replayed ClientToken that (incorrectly)
|
|
// dispatched again is caught even though the SDK call still succeeds.
|
|
var dispatchCount atomic.Int64
|
|
launchSubject := fmt.Sprintf("ec2.RunInstances.%s.%s", instanceType, nodeID)
|
|
sub, err := gw.NATSConn.Subscribe(launchSubject, func(msg *nats.Msg) {
|
|
n := dispatchCount.Add(1)
|
|
_ = msg.Respond(mustMarshal(t, &ec2.Reservation{
|
|
ReservationId: aws.String(fmt.Sprintf("r-ct-%d", n)),
|
|
Instances: []*ec2.Instance{
|
|
{InstanceId: aws.String(fmt.Sprintf("i-ct-%d", n))},
|
|
},
|
|
}))
|
|
})
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
|
|
|
baseInput := func(token string) *ec2.RunInstancesInput {
|
|
return &ec2.RunInstancesInput{
|
|
ImageId: aws.String("ami-0123456789abcdef0"),
|
|
InstanceType: aws.String(instanceType),
|
|
MinCount: aws.Int64(1),
|
|
MaxCount: aws.Int64(1),
|
|
ClientToken: aws.String(token),
|
|
}
|
|
}
|
|
|
|
tokenA := fmt.Sprintf("integration-test-clienttoken-a-%d", run)
|
|
res1, err := gw.EC2Client(t).RunInstances(baseInput(tokenA))
|
|
require.NoError(t, err, "first launch with token A")
|
|
require.Len(t, res1.Instances, 1)
|
|
require.Equal(t, int64(1), dispatchCount.Load(), "first call must dispatch to the daemon once")
|
|
firstInstanceID := aws.StringValue(res1.Instances[0].InstanceId)
|
|
firstReservationID := aws.StringValue(res1.ReservationId)
|
|
|
|
// Replay: same token, same params. Must return the identical reservation
|
|
// without a second daemon dispatch.
|
|
res2, err := gw.EC2Client(t).RunInstances(baseInput(tokenA))
|
|
require.NoError(t, err, "replay with token A")
|
|
require.Equal(t, int64(1), dispatchCount.Load(), "replaying the same token must not dispatch again")
|
|
require.Equal(t, firstReservationID, aws.StringValue(res2.ReservationId), "replay must return the original reservation")
|
|
require.Len(t, res2.Instances, 1)
|
|
require.Equal(t, firstInstanceID, aws.StringValue(res2.Instances[0].InstanceId), "replay must return the original instance")
|
|
|
|
// A different token for the same account must launch again, independently.
|
|
tokenB := fmt.Sprintf("integration-test-clienttoken-b-%d", run)
|
|
res3, err := gw.EC2Client(t).RunInstances(baseInput(tokenB))
|
|
require.NoError(t, err, "launch with token B")
|
|
require.Equal(t, int64(2), dispatchCount.Load(), "a different token must dispatch a fresh launch")
|
|
require.Len(t, res3.Instances, 1)
|
|
require.NotEqual(t, firstInstanceID, aws.StringValue(res3.Instances[0].InstanceId),
|
|
"a different token must not replay token A's instance")
|
|
}
|