Merge pull request #2231 from rcourtman/maintainer/20260924T081300Z

Verify provider network isolation before Pulse releases

.github/workflows/build-and-test.yml

@@ -116,6 +116,50 @@ jobs:
           fi
           echo "frontend_deps=$frontend_deps" >> "$GITHUB_OUTPUT"
 
+  provider-pair-docker:
+    name: Provider pair Docker acceptance
+    needs: changes
+    # Security backports need a live, exact-branch result before the steward
+    # freezes a release packet. Candidate qualification repeats the same case
+    # against the verified release image after freeze.
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release/') && needs.changes.outputs.code == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout exact release-line source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+
+      - name: Verify exact release-line source
+        env:
+          EXPECTED_SOURCE_SHA: ${{ github.sha }}
+        run: test "$(git rev-parse HEAD)" = "$EXPECTED_SOURCE_SHA"
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Preload pinned Docker helper image
+        run: docker pull 'alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6'
+
+      - name: Prove provider pair provisioning and cleanup
+        env:
+          PULSE_RUN_PROVIDER_PAIR_DOCKER_INTEGRATION: '1'
+          PULSE_DOCKER_INTEGRATION_IMAGE: alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6
+        run: |
+          set -euo pipefail
+          go test -count=1 -timeout 3m ./internal/cloudcp/docker \
+            -run '^TestIntegrationProviderPairNetworkIsolation$' -v \
+            | tee "$RUNNER_TEMP/provider-pair-network-test.log"
+          grep -q '^--- PASS: TestIntegrationProviderPairNetworkIsolation ' \
+            "$RUNNER_TEMP/provider-pair-network-test.log"
+
   frontend:
     name: Frontend
     needs: changes

.github/workflows/qualify-release-containers.yml

@@ -21,7 +21,7 @@ jobs:
     # Every channel qualifies on a fresh VM so previous source execution cannot
     # influence the Docker daemon, smoke results or filesystem state.
     runs-on: ubuntu-24.04
-    timeout-minutes: 15
+    timeout-minutes: 20
     env:
       VERSION: ${{ inputs.version }}
     steps:
@@ -111,6 +111,26 @@ jobs:
             done
           '
 
+      - name: Set up Go for provider-network acceptance
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Verify two-provider network isolation on live Docker
+        env:
+          PULSE_RUN_PROVIDER_PAIR_DOCKER_INTEGRATION: '1'
+          PULSE_DOCKER_INTEGRATION_IMAGE: pulse-control-plane-candidate:${{ inputs.version }}
+        run: |
+          set -euo pipefail
+          go test -count=1 -timeout 3m ./internal/cloudcp/docker \
+            -run '^TestIntegrationProviderPairNetworkIsolation$' -v \
+            | tee "$RUNNER_TEMP/provider-pair-network-test.log"
+          # A test with its opt-in gate unset exits successfully as SKIP.
+          # Require the exact live test to have run and passed.
+          grep -q '^--- PASS: TestIntegrationProviderPairNetworkIsolation ' \
+            "$RUNNER_TEMP/provider-pair-network-test.log"
+
       - name: Set up Helm
         uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
         with:

docs/release-control/v6/internal/subsystems/deployment-installability.md

@@ -1267,6 +1267,21 @@ artifact-selection behaviour.
    candidate workflow; publishing releases invoke it as a sibling of inert
    draft staging so qualification and upload overlap without weakening the
    activation join.
+   A provider-isolation security backport also requires a live two-installation
+   Docker result before the release steward freezes a packet. On protected
+   `release/v*` pushes, Build and Test checks out the event SHA, preloads a
+   digest-pinned helper image, and runs the opt-in provider-pair case against a
+   fresh daemon; a skipped test is not a pass. The case provisions the same
+   tenant ID in two independently named provider networks, verifies each
+   provider's support containers attach only to its own tenant network,
+   refuses a same-name unowned network, and checks that cleanup of one
+   provider neither detaches nor removes the other's resources. This is
+   source-line acceptance, not candidate-image or installed-service proof.
+   After packet freeze, the reusable container qualifier repeats the case
+   against its already assembled and digest-verified exact-candidate control-
+   plane image before Helm smoke. It requires the explicit live-test PASS and
+   retains the caller SHA and immutable payload binding; the earlier branch
+   result cannot substitute for this candidate qualification or vice versa.
    The server executable and its detached Minisign and SSH signatures are one
    architecture-bound payload unit; cross-archive deduplication must allow
    those three files to differ while continuing to reject drift in every

internal/cloudcp/docker/manager_integration_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/containerd/errdefs"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/network"
 	"github.com/moby/moby/client"
@@ -183,6 +184,155 @@ func TestIntegrationTenantNetworksAreNotMutuallyReachable(t *testing.T) {
 	}
 }
 
+// This exercises exact checked-out manager source against a fresh Docker daemon.
+// The release-line push check preloads a pinned helper image; final candidate
+// qualification uses its verified control-plane image instead. Unit fixtures
+// cover the API requests; this checks their effects with two installations.
+func TestIntegrationProviderPairNetworkIsolation(t *testing.T) {
+	if os.Getenv("PULSE_RUN_PROVIDER_PAIR_DOCKER_INTEGRATION") != "1" {
+		t.Skip("set PULSE_RUN_PROVIDER_PAIR_DOCKER_INTEGRATION=1 for live two-provider Docker proof")
+	}
+	image := strings.TrimSpace(os.Getenv("PULSE_DOCKER_INTEGRATION_IMAGE"))
+	if image == "" {
+		t.Fatal("PULSE_DOCKER_INTEGRATION_IMAGE must name a preloaded test image")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+
+	suffix := dockerIntegrationSuffix(t)
+	tenantID := "t-shared-" + suffix
+	managerFor := func(provider string) *Manager {
+		t.Helper()
+		mgr, err := NewManager(ManagerConfig{
+			Image:                 image,
+			Network:               "pulse-it-provider-" + provider + "-" + suffix,
+			IsolateTenantNetworks: true,
+		})
+		if err != nil {
+			t.Fatalf("create provider %s manager: %v", provider, err)
+		}
+		t.Cleanup(func() { _ = mgr.Close() })
+		if _, err := mgr.cli.ImageInspect(ctx, image); err != nil {
+			t.Fatalf("inspect preloaded test image %q: %v", image, err)
+		}
+		if _, err := mgr.cli.NetworkCreate(ctx, mgr.cfg.Network, client.NetworkCreateOptions{
+			Driver: "bridge",
+			Labels: map[string]string{"pulse.integration": "provider-pair-network-proof"},
+		}); err != nil {
+			t.Fatalf("create provider %s ingress network: %v", provider, err)
+		}
+		t.Cleanup(func() {
+			_, _ = mgr.cli.NetworkRemove(context.Background(), mgr.cfg.Network, client.NetworkRemoveOptions{})
+		})
+		return mgr
+	}
+	a, b := managerFor("a"), managerFor("b")
+	aSupport := []string{
+		dockerIntegrationCreateSupportContainer(t, ctx, a, a.cfg.Network, providerSupportTraefikLabel),
+		dockerIntegrationCreateSupportContainer(t, ctx, a, a.cfg.Network, providerSupportControlPlaneLabel),
+	}
+	bSupport := []string{
+		dockerIntegrationCreateSupportContainer(t, ctx, b, b.cfg.Network, providerSupportTraefikLabel),
+		dockerIntegrationCreateSupportContainer(t, ctx, b, b.cfg.Network, providerSupportControlPlaneLabel),
+	}
+
+	aTenant, err := a.ensureTenantNetwork(ctx, tenantID)
+	if err != nil {
+		t.Fatalf("provision A tenant network: %v", err)
+	}
+	t.Cleanup(func() { _ = a.removeTenantNetwork(context.Background(), aTenant) })
+	bTenant, err := b.ensureTenantNetwork(ctx, tenantID)
+	if err != nil {
+		t.Fatalf("provision B tenant network: %v", err)
+	}
+	t.Cleanup(func() { _ = b.removeTenantNetwork(context.Background(), bTenant) })
+	if err := a.connectSupportContainersToTenantNetwork(ctx, aTenant); err != nil {
+		t.Fatalf("attach A support: %v", err)
+	}
+	if err := b.connectSupportContainersToTenantNetwork(ctx, bTenant); err != nil {
+		t.Fatalf("attach B support: %v", err)
+	}
+	createTenantContainer := func(mgr *Manager, provider, networkName string) string {
+		t.Helper()
+		resp, err := mgr.cli.ContainerCreate(ctx, client.ContainerCreateOptions{
+			Config: &container.Config{
+				Image:      image,
+				Entrypoint: []string{"sleep"},
+				Cmd:        []string{"300"},
+				Labels: map[string]string{
+					"pulse.managed":   "true",
+					"pulse.tenant.id": tenantID,
+				},
+			},
+			NetworkingConfig: &network.NetworkingConfig{
+				EndpointsConfig: map[string]*network.EndpointSettings{networkName: {}},
+			},
+			Name: "pulse-it-client-" + provider + "-" + suffix,
+		})
+		if err != nil {
+			t.Fatalf("create provider %s tenant container: %v", provider, err)
+		}
+		t.Cleanup(func() {
+			_, _ = mgr.cli.ContainerRemove(context.Background(), resp.ID, client.ContainerRemoveOptions{Force: true})
+		})
+		if _, err := mgr.cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
+			t.Fatalf("start provider %s tenant container: %v", provider, err)
+		}
+		return resp.ID
+	}
+	aClient := createTenantContainer(a, "a", aTenant)
+	bClient := createTenantContainer(b, "b", bTenant)
+	for _, id := range aSupport {
+		dockerIntegrationAssertAttached(t, ctx, a, id, aTenant)
+		dockerIntegrationAssertNotAttached(t, ctx, a, id, bTenant)
+	}
+	for _, id := range bSupport {
+		dockerIntegrationAssertAttached(t, ctx, b, id, bTenant)
+		dockerIntegrationAssertNotAttached(t, ctx, b, id, aTenant)
+	}
+
+	// An unrelated unlabelled network with our derived name must not be adopted.
+	collision := a.tenantNetworkName("t-collision-" + suffix)
+	if _, err := a.cli.NetworkCreate(ctx, collision, client.NetworkCreateOptions{Driver: "bridge"}); err != nil {
+		t.Fatalf("create unowned collision network: %v", err)
+	}
+	t.Cleanup(func() { _, _ = a.cli.NetworkRemove(context.Background(), collision, client.NetworkRemoveOptions{}) })
+	if _, err := a.ensureTenantNetwork(ctx, "t-collision-"+suffix); err == nil {
+		t.Fatal("provider A adopted an unowned same-name network")
+	}
+	if _, err := a.cli.NetworkInspect(ctx, collision, client.NetworkInspectOptions{}); err != nil {
+		t.Fatalf("unowned collision network was changed: %v", err)
+	}
+
+	if err := a.Remove(ctx, aClient); err != nil {
+		t.Fatalf("remove A tenant container: %v", err)
+	}
+	if _, err := a.cli.NetworkInspect(ctx, aTenant, client.NetworkInspectOptions{}); !errdefs.IsNotFound(err) {
+		t.Fatalf("A tenant network remains or lookup failed unexpectedly: %v", err)
+	}
+	for _, id := range aSupport {
+		dockerIntegrationAssertNotAttached(t, ctx, a, id, aTenant)
+		dockerIntegrationAssertAttached(t, ctx, a, id, a.cfg.Network)
+	}
+	if _, err := b.cli.NetworkInspect(ctx, bTenant, client.NetworkInspectOptions{}); err != nil {
+		t.Fatalf("A cleanup removed B tenant network: %v", err)
+	}
+	for _, id := range bSupport {
+		dockerIntegrationAssertAttached(t, ctx, b, id, bTenant)
+		dockerIntegrationAssertAttached(t, ctx, b, id, b.cfg.Network)
+	}
+	if err := b.Remove(ctx, bClient); err != nil {
+		t.Fatalf("remove B tenant container: %v", err)
+	}
+	if _, err := b.cli.NetworkInspect(ctx, bTenant, client.NetworkInspectOptions{}); !errdefs.IsNotFound(err) {
+		t.Fatalf("B tenant network remains or lookup failed unexpectedly: %v", err)
+	}
+	for _, id := range bSupport {
+		dockerIntegrationAssertNotAttached(t, ctx, b, id, bTenant)
+		dockerIntegrationAssertAttached(t, ctx, b, id, b.cfg.Network)
+	}
+}
+
 func dockerIntegrationSuffix(t *testing.T) string {
 	t.Helper()
 	var raw [4]byte
@@ -437,3 +587,14 @@ func dockerIntegrationAssertNotAttached(t *testing.T, ctx context.Context, mgr *
 		t.Fatalf("container %s is unexpectedly attached to provider network %s", containerID[:12], networkName)
 	}
 }
+
+func dockerIntegrationAssertAttached(t *testing.T, ctx context.Context, mgr *Manager, containerID, networkName string) {
+	t.Helper()
+	inspect, err := mgr.cli.ContainerInspect(ctx, containerID, client.ContainerInspectOptions{})
+	if err != nil {
+		t.Fatalf("inspect container %s: %v", containerID, err)
+	}
+	if inspect.Container.NetworkSettings == nil || inspect.Container.NetworkSettings.Networks[networkName] == nil {
+		t.Fatalf("container %s is not attached to network %s", containerID[:12], networkName)
+	}
+}

scripts/installtests/build_release_assets_test.go

@@ -3701,6 +3701,73 @@ func TestReleaseTrainCITriggersIncludeBuildAndE2E(t *testing.T) {
 	}
 }
 
+func TestProviderPairDockerProofRunsBeforeFreezeAndOnExactCandidate(t *testing.T) {
+	const testName = "TestIntegrationProviderPairNetworkIsolation"
+	const helperImage = "alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6"
+	const liveFlag = "PULSE_RUN_PROVIDER_PAIR_DOCKER_INTEGRATION: '1'"
+	const requiredPass = "grep -q '^--- PASS: " + testName + " '"
+
+	ciBytes, err := os.ReadFile(repoFile(".github", "workflows", "build-and-test.yml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	branchJob := workflowJobBlock(t, string(ciBytes), "provider-pair-docker")
+	for _, required := range []string{
+		"github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release/')",
+		"needs.changes.outputs.code == 'true'",
+		"runs-on: ubuntu-24.04",
+		"ref: ${{ github.sha }}",
+		"persist-credentials: false",
+		"EXPECTED_SOURCE_SHA: ${{ github.sha }}",
+		`test "$(git rev-parse HEAD)" = "$EXPECTED_SOURCE_SHA"`,
+		"docker pull '" + helperImage + "'",
+		liveFlag,
+		"PULSE_DOCKER_INTEGRATION_IMAGE: " + helperImage,
+		"-run '^" + testName + "$' -v",
+		requiredPass,
+	} {
+		if !strings.Contains(branchJob, required) {
+			t.Fatalf("release-line provider pair job missing %q", required)
+		}
+	}
+	if strings.Index(branchJob, "Verify exact release-line source") > strings.Index(branchJob, "Prove provider pair provisioning and cleanup") {
+		t.Fatal("provider pair check ran before exact release-line source verification")
+	}
+
+	qualifiedBytes, err := os.ReadFile(repoFile(".github", "workflows", "qualify-release-containers.yml"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	qualifiedJob := workflowJobBlock(t, string(qualifiedBytes), "qualify")
+	for _, required := range []string{
+		"ref: ${{ github.sha }}",
+		"persist-credentials: false",
+		"--source-sha \"${{ github.sha }}\"",
+		"Verify container binaries match immutable candidate",
+		liveFlag,
+		"PULSE_DOCKER_INTEGRATION_IMAGE: pulse-control-plane-candidate:${{ inputs.version }}",
+		"-run '^" + testName + "$' -v",
+		requiredPass,
+	} {
+		if !strings.Contains(qualifiedJob, required) {
+			t.Fatalf("exact-candidate provider pair job missing %q", required)
+		}
+	}
+	verify := strings.Index(qualifiedJob, "Verify container binaries match immutable candidate")
+	pair := strings.Index(qualifiedJob, "Verify two-provider network isolation on live Docker")
+	helm := strings.Index(qualifiedJob, "Helm smoke test with local release-line image")
+	if verify < 0 || pair <= verify || helm <= pair {
+		t.Fatal("live provider pair check must follow payload digest verification and precede Helm smoke")
+	}
+
+	assertFileContainsAll(t, repoFile("internal", "cloudcp", "docker", "manager_integration_test.go"),
+		"func "+testName+"(t *testing.T)",
+		`os.Getenv("PULSE_RUN_PROVIDER_PAIR_DOCKER_INTEGRATION") != "1"`,
+		"provider A adopted an unowned same-name network",
+		"A cleanup removed B tenant network",
+	)
+}
+
 func TestBenchmarkQualificationRetainsProvenance(t *testing.T) {
 	content, err := os.ReadFile(repoFile(".github", "workflows", "build-and-test.yml"))
 	if err != nil {