From f338bd47655586d68150f603fb0aff4f8d465cdf Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 23 Sep 2026 22:46:47 +0100 Subject: [PATCH] Judge client health from Docker when status runs out of band provider-msp status, which upgrade.sh and the install proof gate on, runs in a one-off `docker compose run` container attached only to the provider ingress network. Its health check dialled each client's container on the client's isolated network, which that container cannot reach, and an unreachable client counts as unhealthy. So on any install with a real client, status failed with "unhealthy active workspaces" and ./upgrade.sh --dry-run exited 1: the backup-gated upgrade path refused to run as soon as a provider onboarded anyone. CI proofs never saw it because they only ever run against an empty install. Reproduced on a fresh v6.4.1 provider bundle with one client whose container Docker reported healthy. The runtime image declares a HEALTHCHECK that probes the same /api/health inside the container, and Docker records the result where any caller with the socket can read it. Use that when it exists and keep the HTTP probe for images without one. With the change the same install reports healthy_tenants=1 and upgrade.sh --dry-run plans cleanly. --- .../v6/internal/subsystems/cloud-paid.md | 13 +++++++++ .../subsystems/deployment-installability.md | 12 ++++++++ internal/cloudcp/docker/manager.go | 27 ++++++++++++++++++ internal/cloudcp/docker/manager_test.go | 28 +++++++++++++++++++ 4 files changed, 80 insertions(+) diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index 1c13c62786..3a45d6124e 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -3431,6 +3431,19 @@ Pulse-service-backed capabilities. `providerChained` retains its narrower meaning: a Pulse-signed licence is available to embed so release builds can verify the lease. +### Provider MSP status reads client health that any caller can observe + +`provider-msp status`, which `upgrade.sh` gates on before every provider +upgrade, decides client-workspace health from the runtime image's recorded +Docker HEALTHCHECK when the container declares one, and falls back to the +direct `/api/health` HTTP probe only for images without it. The command runs +in a one-off container on the provider ingress network, which cannot reach a +client's isolated tenant network, so the HTTP probe alone reported every real +client unhealthy and blocked upgrades on any install with a client. The +long-lived control plane's health monitor uses the same rule. Regression +coverage: `TestDockerRecordedHealthDecidesWhenTheImageDeclaresOne` in +`internal/cloudcp/docker/manager_test.go`. + ### Runtime display load remains outside commercial bootstrap ownership `frontend-modern/src/useAppRuntimeState.ts` now loads presentation defaults diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index eaa6eb2501..a6f98f12ea 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -2416,6 +2416,18 @@ artifact-selection behaviour. ## Current State +### Provider MSP upgrades work once a client exists + +`deploy/provider-msp/upgrade.sh` and `run-install-proof.sh` run +`provider-msp status` through `docker compose run --no-deps`, a container +attached only to the provider ingress network. Client health is now read from +the runtime image's recorded Docker HEALTHCHECK rather than an HTTP dial into +each client's isolated network, so `upgrade.sh --dry-run` plans cleanly on an +install with real clients instead of failing with `unhealthy active +workspaces`. Install proofs that only ever ran against an empty install could +not see this; verification on 2026-09-23 used a v6.4.1 bundle with one client +whose container Docker reported healthy. + ### Reviewed frontend dependency security floors The 2026-09-21 npm-minor-patch group raises the reviewed floors of the diff --git a/internal/cloudcp/docker/manager.go b/internal/cloudcp/docker/manager.go index 133a3b0f43..d579159919 100644 --- a/internal/cloudcp/docker/manager.go +++ b/internal/cloudcp/docker/manager.go @@ -1031,6 +1031,10 @@ func (m *Manager) HealthCheck(ctx context.Context, containerID string) (bool, er return false, nil } + if healthy, recorded := dockerRecordedHealth(inspect); recorded { + return healthy, nil + } + networkNames := m.healthCheckNetworkCandidates(inspect) var netSettings *network.EndpointSettings for _, networkName := range networkNames { @@ -1055,6 +1059,29 @@ func (m *Manager) HealthCheck(ctx context.Context, containerID string) (bool, er return resp.StatusCode == http.StatusOK, nil } +// dockerRecordedHealth returns the result of the runtime image's own Docker +// HEALTHCHECK, which probes /api/health inside the container, when the +// container declares one. Docker records it on the container, so any caller +// with the socket reads the same answer. The HTTP fallback below only works +// from a process attached to the workspace's isolated network: the long-lived +// control plane is, but `provider-msp status` runs in a one-off container on +// the ingress network alone, so every real client read as unhealthy there and +// upgrade.sh and the install proof refused to run on any install with a +// client. A container still in its start period is not yet healthy. +func dockerRecordedHealth(inspect container.InspectResponse) (healthy bool, recorded bool) { + if inspect.State == nil || inspect.State.Health == nil { + return false, false + } + switch inspect.State.Health.Status { + case container.Healthy: + return true, true + case container.Unhealthy, container.Starting: + return false, true + default: + return false, false + } +} + func (m *Manager) healthCheckNetworkCandidates(inspect container.InspectResponse) []string { var names []string seen := make(map[string]struct{}) diff --git a/internal/cloudcp/docker/manager_test.go b/internal/cloudcp/docker/manager_test.go index e214390758..40d4726c2b 100644 --- a/internal/cloudcp/docker/manager_test.go +++ b/internal/cloudcp/docker/manager_test.go @@ -575,3 +575,31 @@ func TestCheckRuntimePrerequisitesReportsMissingDockerDaemon(t *testing.T) { t.Fatalf("failures = %q, want docker daemon ping failure", got) } } + +// provider-msp status and upgrade.sh probe tenant health from a one-off +// container that is not attached to any isolated tenant network. The image's +// own Docker HEALTHCHECK result must decide health whenever it exists, so a +// healthy client does not read as unhealthy just because the caller cannot +// route to it; only a container with no HEALTHCHECK falls back to HTTP. +func TestDockerRecordedHealthDecidesWhenTheImageDeclaresOne(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + health *container.Health + wantHealthy bool + wantRecorded bool + }{ + {name: "healthy", health: &container.Health{Status: container.Healthy}, wantHealthy: true, wantRecorded: true}, + {name: "unhealthy", health: &container.Health{Status: container.Unhealthy}, wantHealthy: false, wantRecorded: true}, + {name: "starting", health: &container.Health{Status: container.Starting}, wantHealthy: false, wantRecorded: true}, + {name: "no healthcheck", health: &container.Health{Status: container.NoHealthcheck}, wantRecorded: false}, + {name: "health absent", health: nil, wantRecorded: false}, + } { + inspect := container.InspectResponse{State: &container.State{Running: true, Health: tc.health}} + healthy, recorded := dockerRecordedHealth(inspect) + if healthy != tc.wantHealthy || recorded != tc.wantRecorded { + t.Errorf("%s: dockerRecordedHealth = (%v, %v), want (%v, %v)", tc.name, healthy, recorded, tc.wantHealthy, tc.wantRecorded) + } + } +}