Bound legacy demo history startup

This commit is contained in:
Richard Courtman
2026-08-29 03:46:20 +01:00
parent 2bf37a58c7
commit db0145b2b9
7 changed files with 33 additions and 9 deletions
+9 -3
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env bash
set -euo pipefail
if [ "$#" -ne 15 ]; then
echo "usage: recover-demo-runtime.sh EXPECTED_HOSTNAME LOCAL_BASE_URL EXPECTED_VERSION MOCK_NODES MOCK_VMS_PER_NODE MOCK_LXCS_PER_NODE MOCK_DOCKER_HOSTS MOCK_DOCKER_CONTAINERS MOCK_GENERIC_HOSTS MOCK_K8S_CLUSTERS MOCK_K8S_NODES MOCK_K8S_PODS MOCK_K8S_DEPLOYMENTS MOCK_SEED_DURATION MOCK_UPDATE_INTERVAL" >&2
if [ "$#" -ne 16 ]; then
echo "usage: recover-demo-runtime.sh EXPECTED_HOSTNAME LOCAL_BASE_URL EXPECTED_VERSION MOCK_NODES MOCK_VMS_PER_NODE MOCK_LXCS_PER_NODE MOCK_DOCKER_HOSTS MOCK_DOCKER_CONTAINERS MOCK_GENERIC_HOSTS MOCK_K8S_CLUSTERS MOCK_K8S_NODES MOCK_K8S_PODS MOCK_K8S_DEPLOYMENTS MOCK_SEED_DURATION MOCK_SAMPLE_INTERVAL MOCK_UPDATE_INTERVAL" >&2
exit 2
fi
@@ -20,7 +20,8 @@ MOCK_K8S_NODES="${11}"
MOCK_K8S_PODS="${12}"
MOCK_K8S_DEPLOYMENTS="${13}"
MOCK_SEED_DURATION="${14}"
MOCK_UPDATE_INTERVAL="${15}"
MOCK_SAMPLE_INTERVAL="${15}"
MOCK_UPDATE_INTERVAL="${16}"
SERVICE_NAME="pulse"
RELAY_SERVICE_NAME="pulse-relay"
EXPECTED_BINARY="/opt/pulse/bin/pulse"
@@ -42,6 +43,7 @@ for fixture_count in \
[[ "$fixture_count" =~ ^[1-9][0-9]*$ ]]
done
[[ "$MOCK_SEED_DURATION" =~ ^[1-9][0-9]*[smhd]$ ]]
[[ "$MOCK_SAMPLE_INTERVAL" =~ ^[1-9][0-9]*[smh]$ ]]
[[ "$MOCK_UPDATE_INTERVAL" =~ ^[1-9][0-9]*[smh]$ ]]
log() {
@@ -103,6 +105,7 @@ runtime_profile_matches() {
sudo grep -Fxq "PULSE_MOCK_K8S_DEPLOYMENTS=${MOCK_K8S_DEPLOYMENTS}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_SEED_METRICS_STORE=false" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_TRENDS_SEED_DURATION=${MOCK_SEED_DURATION}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_TRENDS_SAMPLE_INTERVAL=${MOCK_SAMPLE_INTERVAL}" /etc/pulse/.env &&
sudo grep -Fxq "PULSE_MOCK_UPDATE_INTERVAL=${MOCK_UPDATE_INTERVAL}" /etc/pulse/.env
}
@@ -178,6 +181,7 @@ emit_evidence() {
--arg mock_k8s_pods "$MOCK_K8S_PODS" \
--arg mock_k8s_deployments "$MOCK_K8S_DEPLOYMENTS" \
--arg mock_seed_duration "$MOCK_SEED_DURATION" \
--arg mock_sample_interval "$MOCK_SAMPLE_INTERVAL" \
--arg mock_update_interval "$MOCK_UPDATE_INTERVAL" \
'{schema_version: 1, status: $status, mutated: $mutated,
hostname: $hostname, service: $service,
@@ -207,6 +211,7 @@ emit_evidence() {
mock_k8s_pods: $mock_k8s_pods,
mock_k8s_deployments: $mock_k8s_deployments,
mock_seed_duration: $mock_seed_duration,
mock_sample_interval: $mock_sample_interval,
mock_update_interval: $mock_update_interval}}'
}
@@ -293,6 +298,7 @@ set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS"
# opt-in here can turn startup into an unbounded SQLite backfill.
set_env_value PULSE_MOCK_SEED_METRICS_STORE false
set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"
set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"
set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"
sudo chown pulse:pulse /etc/pulse/.env
sudo chmod 600 /etc/pulse/.env
@@ -32,6 +32,7 @@ if git grep -q 'mockEagerHistoryPVEGuestLimit' "${TAG_REF}" -- internal/monitori
MOCK_K8S_PODS=40
MOCK_K8S_DEPLOYMENTS=14
MOCK_SEED_DURATION=48h
MOCK_SAMPLE_INTERVAL=1m
MOCK_UPDATE_INTERVAL=2s
else
PROFILE="legacy-bounded"
@@ -50,6 +51,10 @@ else
MOCK_K8S_PODS=12
MOCK_K8S_DEPLOYMENTS=4
MOCK_SEED_DURATION=2h
# The compatibility runtime seeds history synchronously before binding the
# HTTP listener. Bound the point density as well as the duration so a small
# demo host can reach health without changing the installed release bits.
MOCK_SAMPLE_INTERVAL=5m
MOCK_UPDATE_INTERVAL=15s
fi
@@ -65,4 +70,5 @@ printf 'mock_k8s_nodes=%s\n' "$MOCK_K8S_NODES"
printf 'mock_k8s_pods=%s\n' "$MOCK_K8S_PODS"
printf 'mock_k8s_deployments=%s\n' "$MOCK_K8S_DEPLOYMENTS"
printf 'mock_seed_duration=%s\n' "$MOCK_SEED_DURATION"
printf 'mock_sample_interval=%s\n' "$MOCK_SAMPLE_INTERVAL"
printf 'mock_update_interval=%s\n' "$MOCK_UPDATE_INTERVAL"
+2 -1
View File
@@ -105,6 +105,7 @@ jobs:
MOCK_K8S_PODS: ${{ steps.runtime_profile.outputs.mock_k8s_pods }}
MOCK_K8S_DEPLOYMENTS: ${{ steps.runtime_profile.outputs.mock_k8s_deployments }}
MOCK_SEED_DURATION: ${{ steps.runtime_profile.outputs.mock_seed_duration }}
MOCK_SAMPLE_INTERVAL: ${{ steps.runtime_profile.outputs.mock_sample_interval }}
MOCK_UPDATE_INTERVAL: ${{ steps.runtime_profile.outputs.mock_update_interval }}
run: |
set -euo pipefail
@@ -113,7 +114,7 @@ jobs:
-o StrictHostKeyChecking=yes \
-o UserKnownHostsFile=~/.ssh/known_hosts \
"$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
"bash -s -- $(printf '%q ' "$DEMO_EXPECTED_HOSTNAME" "$DEMO_LOCAL_BASE_URL" "$EXPECTED_VERSION" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_UPDATE_INTERVAL")" \
"bash -s -- $(printf '%q ' "$DEMO_EXPECTED_HOSTNAME" "$DEMO_LOCAL_BASE_URL" "$EXPECTED_VERSION" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_SAMPLE_INTERVAL" "$MOCK_UPDATE_INTERVAL")" \
< .github/scripts/recover-demo-runtime.sh \
> demo-runtime-recovery.json
jq -e '.schema_version == 1 and (.status == "recovered" or .status == "healthy_noop")' \
+6 -2
View File
@@ -60,6 +60,7 @@ jobs:
mock_k8s_pods: ${{ steps.runtime_profile.outputs.mock_k8s_pods }}
mock_k8s_deployments: ${{ steps.runtime_profile.outputs.mock_k8s_deployments }}
mock_seed_duration: ${{ steps.runtime_profile.outputs.mock_seed_duration }}
mock_sample_interval: ${{ steps.runtime_profile.outputs.mock_sample_interval }}
mock_update_interval: ${{ steps.runtime_profile.outputs.mock_update_interval }}
steps:
@@ -564,6 +565,7 @@ jobs:
MOCK_K8S_PODS="${{ needs.resolve.outputs.mock_k8s_pods }}"
MOCK_K8S_DEPLOYMENTS="${{ needs.resolve.outputs.mock_k8s_deployments }}"
MOCK_SEED_DURATION="${{ needs.resolve.outputs.mock_seed_duration }}"
MOCK_SAMPLE_INTERVAL="${{ needs.resolve.outputs.mock_sample_interval }}"
MOCK_UPDATE_INTERVAL="${{ needs.resolve.outputs.mock_update_interval }}"
REMOTE_SCRIPT=$(cat <<'EOF'
set -euo pipefail
@@ -580,7 +582,8 @@ jobs:
MOCK_K8S_PODS="${11}"
MOCK_K8S_DEPLOYMENTS="${12}"
MOCK_SEED_DURATION="${13}"
MOCK_UPDATE_INTERVAL="${14}"
MOCK_SAMPLE_INTERVAL="${14}"
MOCK_UPDATE_INTERVAL="${15}"
CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}"
SERVICE_USER="$(systemctl show "$SERVICE_NAME" --property=User --value 2>/dev/null || true)"
SERVICE_GROUP="$(systemctl show "$SERVICE_NAME" --property=Group --value 2>/dev/null || true)"
@@ -678,6 +681,7 @@ jobs:
# persistent public demo data directory.
set_env_value PULSE_MOCK_SEED_METRICS_STORE false
set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"
set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"
set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"
ensure_demo_fixture_entitlement
@@ -700,7 +704,7 @@ jobs:
EOF
)
ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \
"bash -s -- $(printf '%q ' "$SERVICE_NAME" "$DEMO_LOCAL_BASE_URL" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_UPDATE_INTERVAL")" <<<"$REMOTE_SCRIPT"
"bash -s -- $(printf '%q ' "$SERVICE_NAME" "$DEMO_LOCAL_BASE_URL" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_SAMPLE_INTERVAL" "$MOCK_UPDATE_INTERVAL")" <<<"$REMOTE_SCRIPT"
- name: Verify demo runtime
env:
@@ -3079,8 +3079,9 @@ bounded compatibility profile across every fixture family: 8 Proxmox nodes
with 6 VMs and 4 LXCs per node, 2 Docker hosts with 8 containers each, 2
generic hosts, and 1 Kubernetes cluster with 3 nodes, 12 pods, and 4
deployments. That profile retains two hours of eager history and uses a
15-second full-estate update cadence, preventing newer workflow defaults from
imposing an unqualified bootstrap, memory, and CPU load on an older binary.
five-minute history sample interval plus a 15-second full-estate update cadence,
preventing newer workflow defaults from imposing an unqualified synchronous
bootstrap, memory, and CPU load on an older binary before it binds health.
The manually dispatched stable-demo recovery path consumes that same resolver
and may mutate only the resolver-owned bounded mock-profile values before
restarting the already-installed `pulse` service. It must retain the current stable binary,
@@ -1845,6 +1845,7 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) {
`MOCK_VMS_PER_NODE=10`,
`MOCK_K8S_PODS=40`,
`MOCK_SEED_DURATION=48h`,
`MOCK_SAMPLE_INTERVAL=1m`,
`MOCK_UPDATE_INTERVAL=2s`,
`PROFILE="legacy-bounded"`,
`MOCK_NODES=8`,
@@ -1858,6 +1859,7 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) {
`MOCK_K8S_PODS=12`,
`MOCK_K8S_DEPLOYMENTS=4`,
`MOCK_SEED_DURATION=2h`,
`MOCK_SAMPLE_INTERVAL=5m`,
`MOCK_UPDATE_INTERVAL=15s`,
`resolve_config_dir`,
`set_env_value DEMO_MODE true`,
@@ -1869,6 +1871,7 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) {
`set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"`,
`set_env_value PULSE_MOCK_SEED_METRICS_STORE false`,
`set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"`,
`set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"`,
`set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"`,
`ensure_demo_fixture_entitlement`,
`"demo_fixtures"`,
@@ -1956,6 +1956,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
self.assertIn("MOCK_K8S_PODS=12", demo_profile)
self.assertIn("MOCK_K8S_DEPLOYMENTS=4", demo_profile)
self.assertIn("MOCK_SEED_DURATION=2h", demo_profile)
self.assertIn("MOCK_SAMPLE_INTERVAL=5m", demo_profile)
self.assertIn("MOCK_UPDATE_INTERVAL=15s", demo_profile)
self.assertIn("resolve_config_dir", demo)
self.assertIn("set_env_value DEMO_MODE true", demo)
@@ -1963,6 +1964,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
self.assertIn('set_env_value PULSE_MOCK_NODES "$MOCK_NODES"', demo)
self.assertIn("set_env_value PULSE_MOCK_SEED_METRICS_STORE false", demo)
self.assertIn('set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"', demo)
self.assertIn('set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"', demo)
self.assertIn('set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"', demo)
self.assertIn("ensure_demo_fixture_entitlement", demo)
self.assertIn('"demo_fixtures"', demo)
@@ -2100,7 +2102,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
self.assertNotIn("install.sh", workflow)
self.assertNotIn("scp ", workflow)
self.assertIn('if [ "$#" -ne 15 ]', recovery)
self.assertIn('if [ "$#" -ne 16 ]', recovery)
self.assertIn('SERVICE_NAME="pulse"', recovery)
self.assertIn('RELAY_SERVICE_NAME="pulse-relay"', recovery)
self.assertIn('EXPECTED_BINARY="/opt/pulse/bin/pulse"', recovery)
@@ -2118,6 +2120,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase):
self.assertIn("set_env_value PULSE_MOCK_SEED_METRICS_STORE false", recovery)
self.assertIn('grep -Fxq "PULSE_MOCK_SEED_METRICS_STORE=false"', recovery)
self.assertIn('set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"', recovery)
self.assertIn('set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"', recovery)
self.assertIn('set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"', recovery)
self.assertIn('sudo systemctl restart "$SERVICE_NAME"', recovery)
self.assertIn('sudo systemctl stop "$SERVICE_NAME"', recovery)