Add provider MSP backup command

This commit is contained in:
rcourtman
2026-06-02 14:03:19 +01:00
parent e46f4d648e
commit c9f84c5192
11 changed files with 1224 additions and 11 deletions
+1
View File
@@ -13,6 +13,7 @@ func newProviderMSPCmd() *cobra.Command {
Short: "Operate a provider-hosted MSP control plane",
}
cmd.AddCommand(newProviderMSPBootstrapCmd())
cmd.AddCommand(newProviderMSPBackupCmd())
cmd.AddCommand(newProviderMSPPreflightCmd())
cmd.AddCommand(newProviderMSPProofCmd())
cmd.AddCommand(newProviderMSPStatusCmd())
@@ -0,0 +1,125 @@
package main
import (
"fmt"
"strings"
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
"github.com/spf13/cobra"
)
func newProviderMSPBackupCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "backup",
Short: "Create and verify provider-hosted MSP recovery archives",
}
cmd.AddCommand(newProviderMSPBackupCreateCmd())
cmd.AddCommand(newProviderMSPBackupVerifyCmd())
return cmd
}
func newProviderMSPBackupCreateCmd() *cobra.Command {
var outputPath string
cmd := &cobra.Command{
Use: "create",
Short: "Create a provider-hosted MSP recovery archive",
RunE: func(cmd *cobra.Command, args []string) error {
cfg, err := cloudcp.LoadConfig()
if err != nil {
return fmt.Errorf("load control plane config: %w", err)
}
result, err := cloudcp.CreateProviderMSPBackup(cmd.Context(), cfg, outputPath)
if err != nil {
return err
}
printProviderMSPBackupCreateResult(result)
return nil
},
}
cmd.Flags().StringVar(&outputPath, "output", "", "Backup archive path (default: <CP_DATA_DIR>/backups/provider-msp/provider-msp-backup-<timestamp>.tar.gz)")
return cmd
}
func newProviderMSPBackupVerifyCmd() *cobra.Command {
var archivePath string
cmd := &cobra.Command{
Use: "verify [archive]",
Short: "Verify a provider-hosted MSP recovery archive",
Args: cobra.MaximumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 1 {
if strings.TrimSpace(archivePath) != "" && archivePath != args[0] {
return fmt.Errorf("archive path provided both as --archive and positional argument")
}
archivePath = args[0]
}
result, err := cloudcp.VerifyProviderMSPBackup(cmd.Context(), archivePath)
if err != nil {
return err
}
printProviderMSPBackupVerifyResult(result)
return nil
},
}
cmd.Flags().StringVar(&archivePath, "archive", "", "Backup archive path to verify")
return cmd
}
func printProviderMSPBackupCreateResult(result *cloudcp.ProviderMSPBackupCreateResult) {
if result == nil {
fmt.Println("provider_msp_backup_created=false")
return
}
fmt.Println("provider_msp_backup_created=true")
fmt.Printf("archive_path=%s\n", result.ArchivePath)
fmt.Printf("archive_bytes=%d\n", result.BytesWritten)
printProviderMSPBackupManifest(result.Manifest)
fmt.Printf("control_plane_entries=%d\n", result.ControlPlaneEntries)
fmt.Printf("tenant_entries=%d\n", result.TenantEntries)
fmt.Printf("license_entries=%d\n", result.LicenseEntries)
}
func printProviderMSPBackupVerifyResult(result *cloudcp.ProviderMSPBackupVerifyResult) {
if result == nil {
fmt.Println("provider_msp_backup_verified=false")
return
}
fmt.Println("provider_msp_backup_verified=true")
fmt.Printf("archive_path=%s\n", result.ArchivePath)
fmt.Printf("archive_bytes=%d\n", result.VerifiedArchiveBytes)
printProviderMSPBackupManifest(result.Manifest)
fmt.Printf("control_plane_entries=%d\n", result.ControlPlaneEntries)
fmt.Printf("tenant_entries=%d\n", result.TenantEntries)
fmt.Printf("license_entries=%d\n", result.LicenseEntries)
fmt.Printf("tenant_registry_db_present=%t\n", result.HasTenantRegistryDB)
fmt.Printf("license_file_present=%t\n", result.HasLicenseFile)
for _, dbFile := range result.ControlPlaneDBFiles {
fmt.Printf("control_plane_db_backup=%s\n", dbFile)
}
for _, tenantID := range result.RuntimeTenantDirs {
fmt.Printf("runtime_tenant_dir=%s\n", tenantID)
}
}
func printProviderMSPBackupManifest(manifest cloudcp.ProviderMSPBackupManifest) {
fmt.Printf("manifest_version=%s\n", manifest.Version)
fmt.Printf("created_at=%s\n", manifest.CreatedAt.Format("2006-01-02T15:04:05Z07:00"))
fmt.Printf("control_plane_mode=%s\n", manifest.ControlPlaneMode)
fmt.Printf("environment=%s\n", manifest.Environment)
fmt.Printf("base_url=%s\n", manifest.BaseURL)
fmt.Printf("plan_version=%s\n", manifest.PlanVersion)
fmt.Printf("plan_source=%s\n", manifest.PlanSource)
fmt.Printf("license_id=%s\n", manifest.LicenseID)
fmt.Printf("license_email=%s\n", manifest.LicenseEmail)
fmt.Printf("license_included=%t\n", manifest.LicenseIncluded)
fmt.Printf("workspace_limit=%d\n", manifest.WorkspaceLimit)
fmt.Printf("registry_account_count=%d\n", manifest.RegistryAccountCount)
fmt.Printf("registry_tenant_count=%d\n", manifest.RegistryTenantCount)
fmt.Printf("runtime_tenant_count=%d\n", manifest.RuntimeTenantCount)
for _, tenantID := range manifest.RuntimeTenantIDs {
fmt.Printf("runtime_tenant_id=%s\n", tenantID)
}
for _, dbFile := range manifest.ControlPlaneDBBackups {
fmt.Printf("manifest_db_backup=%s\n", dbFile)
}
}
@@ -0,0 +1,29 @@
package main
import "testing"
func TestProviderMSPCommandExposesBackup(t *testing.T) {
cmd := newProviderMSPCmd()
for _, child := range cmd.Commands() {
if child.Name() == "backup" {
foundCreate := false
foundVerify := false
for _, backupChild := range child.Commands() {
switch backupChild.Name() {
case "create":
foundCreate = true
case "verify":
foundVerify = true
}
}
if !foundCreate {
t.Fatal("provider-msp backup create command is not registered")
}
if !foundVerify {
t.Fatal("provider-msp backup verify command is not registered")
}
return
}
}
t.Fatal("provider-msp backup command is not registered")
}
+7
View File
@@ -43,6 +43,13 @@ PULSE_EMAIL_REPLY_TO=support@example.com
# Non-mutating operational status:
# docker compose run --rm control-plane provider-msp status
#
# Recovery backup before upgrades or recovery drills:
# docker compose run --rm control-plane provider-msp backup create
#
# Verify a recovery archive before relying on it:
# docker compose run --rm control-plane provider-msp backup verify \
# /data/backups/provider-msp/provider-msp-backup-YYYYMMDDTHHMMSSZ.tar.gz
#
# End-to-end provider proof. This creates two proof client workspaces, generates
# client-bound agent install tokens, verifies tenant-local agent report ingest,
# and exercises the portal handoff path. Keep --cleanup once you have captured
@@ -29,6 +29,10 @@ Provider-hosted MSP status uses the same tenant registry, health summary, and
stuck-provisioning threshold as the control-plane cleanup loop, so operator
readiness reports and automated failure handling cannot drift into separate
definitions of a failed provider workspace.
Provider-hosted MSP backup and verification are part of the cloud-paid operator
contract because the recovery artifact must preserve the provider's
license-backed plan identity, control-plane account/workspace registry, and
tenant-local runtime state without depending on Stripe billing surfaces.
## Canonical Files
@@ -112,6 +116,7 @@ definitions of a failed provider workspace.
88. `internal/hosted/hosted_metrics.go`, `internal/hosted/reaper.go`
89. `internal/cloudcp/public_msp_signup_handlers.go`
90. `internal/cloudcp/provider_msp_bootstrap.go`
91. `internal/cloudcp/provider_msp_backup.go`
## Shared Boundaries
@@ -182,7 +187,12 @@ definitions of a failed provider workspace.
Pulse Pro operations bundle on a recurring systemd timer, write durable
status/log output, and emit Prometheus textfile metrics so a clean GA
baseline is continuously monitored rather than manually rediscovered.
10. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
10. `internal/cloudcp/provider_msp_backup.go` shared with `deployment-installability`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary.
License-backed provider MSP backups must include the signed MSP license
file as a recovery artifact while exposing only license metadata in command
output, and they must keep the archive Stripe-free so provider-hosted MSP
recovery does not inherit Pulse-hosted SaaS billing assumptions.
11. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
Hosted tenant runtime reconciliation must treat a registered tenant with
preserved tenant data but no live Docker runtime as a recoverable managed
state, not as a terminal skip. The control-plane-owned reconcile path must
@@ -27,12 +27,14 @@ surfaces.
4. `cmd/pulse-control-plane/main.go`
5. `cmd/pulse-control-plane/mobile_proof_cmd.go`
6. `cmd/pulse-control-plane/provider_msp.go`
7. `cmd/pulse-control-plane/provider_msp_preflight.go`
8. `cmd/pulse-control-plane/provider_msp_proof.go`
9. `cmd/pulse-control-plane/provider_msp_status.go`
10. `internal/cloudcp/docker/manager.go`
11. `internal/cloudcp/docker/labels.go`
12. `internal/cloudcp/tenant_runtime_rollout.go`
7. `cmd/pulse-control-plane/provider_msp_backup.go`
8. `cmd/pulse-control-plane/provider_msp_preflight.go`
9. `cmd/pulse-control-plane/provider_msp_proof.go`
10. `cmd/pulse-control-plane/provider_msp_status.go`
11. `internal/cloudcp/provider_msp_backup.go`
12. `internal/cloudcp/docker/manager.go`
13. `internal/cloudcp/docker/labels.go`
14. `internal/cloudcp/tenant_runtime_rollout.go`
13. `.github/workflows/create-release.yml`
14. `.github/workflows/deploy-demo-server.yml`
15. `.github/workflows/helm-pages.yml`
@@ -117,7 +119,7 @@ surfaces.
2. `internal/api/updates.go` shared with `api-contracts`: update handlers are both a deployment-installability control surface and a canonical API payload contract boundary.
3. `internal/cloudcp/docker/labels.go` shared with `cloud-paid`: hosted tenant Docker labels are both a Pulse Cloud runtime contract boundary and a deployment-installability rollout boundary.
4. `internal/cloudcp/docker/manager.go` shared with `cloud-paid`: hosted tenant container management is both a Pulse Cloud runtime contract boundary and a deployment-installability rollout boundary.
Tenant runtime containers must be created with bounded Docker `json-file`
Tenant runtime containers must use bounded Docker `json-file`
logging so rollout and canary fleets cannot consume unbounded production
host storage while they remain running.
Provider-hosted MSP workspace creation and preflight must prepare or report
@@ -137,13 +139,22 @@ surfaces.
state/health counts, stuck provisioning workspaces, Docker runtime
prerequisites, storage guardrails, and the same license-backed plan identity
without pulling tenant images unless the operator asks for it.
5. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
6. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
5. `internal/cloudcp/provider_msp_backup.go` shared with `cloud-paid`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary.
`pulse-control-plane provider-msp backup create` and `backup verify` must
create a Stripe-free recovery archive outside the live
control-plane/tenant source trees, snapshot SQLite control-plane databases
through an online backup path, include tenant runtime directories for all
non-deleted registry workspaces, include the signed MSP license file when
the plan source is license-backed, and verify the manifest, tenant registry
snapshot, license artifact, and tenant runtime directories before the
archive is treated as usable for upgrades or recovery drills.
6. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
7. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
It must expose a non-mutating preflight for the exact Windows agent
architecture before Administrator-only install changes, accept token-file
enrollment input, and avoid interactive download-failure prompts when
launched by generated non-interactive onboarding commands.
7. `scripts/install.sh` shared with `agent-lifecycle`: the shell installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
8. `scripts/install.sh` shared with `agent-lifecycle`: the shell installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
## Extension Points
@@ -634,6 +634,14 @@
"deployment-installability"
]
},
{
"path": "internal/cloudcp/provider_msp_backup.go",
"rationale": "provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary",
"subsystems": [
"cloud-paid",
"deployment-installability"
]
},
{
"path": "internal/cloudcp/tenant_runtime_rollout.go",
"rationale": "hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary",
@@ -2624,6 +2632,7 @@
".github/workflows/validate-release-assets.yml",
"cmd/pulse-control-plane/main.go",
"cmd/pulse-control-plane/provider_msp.go",
"cmd/pulse-control-plane/provider_msp_backup.go",
"cmd/pulse-control-plane/provider_msp_preflight.go",
"cmd/pulse-control-plane/provider_msp_proof.go",
"cmd/pulse-control-plane/provider_msp_status.go",
@@ -2644,6 +2653,7 @@
"internal/api/updates.go",
"internal/cloudcp/docker/labels.go",
"internal/cloudcp/docker/manager.go",
"internal/cloudcp/provider_msp_backup.go",
"internal/cloudcp/tenant_runtime_rollout.go",
"Makefile",
"package-lock.json",
@@ -2893,20 +2903,24 @@
"match_files": [
"cmd/pulse-control-plane/main.go",
"cmd/pulse-control-plane/provider_msp.go",
"cmd/pulse-control-plane/provider_msp_backup.go",
"cmd/pulse-control-plane/provider_msp_preflight.go",
"cmd/pulse-control-plane/provider_msp_proof.go",
"cmd/pulse-control-plane/provider_msp_status.go",
"internal/cloudcp/docker/labels.go",
"internal/cloudcp/docker/manager.go",
"internal/cloudcp/provider_msp_backup.go",
"internal/cloudcp/tenant_runtime_rollout.go"
],
"allow_same_subsystem_tests": false,
"test_prefixes": [],
"exact_files": [
"cmd/pulse-control-plane/provider_msp_backup_test.go",
"cmd/pulse-control-plane/provider_msp_preflight_test.go",
"cmd/pulse-control-plane/provider_msp_proof_test.go",
"cmd/pulse-control-plane/provider_msp_status_test.go",
"internal/cloudcp/docker/manager_test.go",
"internal/cloudcp/provider_msp_backup_test.go",
"internal/cloudcp/tenant_runtime_rollout_test.go",
"scripts/installtests/provider_msp_deploy_test.go"
]
+754
View File
@@ -0,0 +1,754 @@
package cloudcp
import (
"archive/tar"
"compress/gzip"
"context"
"database/sql"
"encoding/json"
"fmt"
"io"
"net/url"
"os"
"path"
"path/filepath"
"sort"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
_ "modernc.org/sqlite"
)
const (
ProviderMSPBackupManifestVersion = "provider-msp-backup/v1"
providerMSPBackupManifestName = "manifest.json"
providerMSPBackupControlPlaneDir = "control-plane"
providerMSPBackupTenantsDir = "tenants"
providerMSPBackupLicenseDir = "license"
providerMSPBackupLicenseName = "provider-msp-license.jwt"
maxProviderMSPBackupManifestSize = 1024 * 1024
)
// ProviderMSPBackupManifest is the recovery contract stored inside every
// provider-hosted MSP backup archive.
type ProviderMSPBackupManifest struct {
Version string `json:"version"`
CreatedAt time.Time `json:"created_at"`
ControlPlaneMode string `json:"control_plane_mode"`
Environment string `json:"environment"`
BaseURL string `json:"base_url"`
PlanVersion string `json:"plan_version"`
PlanSource string `json:"plan_source"`
LicenseID string `json:"license_id,omitempty"`
LicenseEmail string `json:"license_email,omitempty"`
LicenseIncluded bool `json:"license_included"`
WorkspaceLimit int `json:"workspace_limit"`
ControlPlaneDir string `json:"control_plane_dir"`
TenantsDir string `json:"tenants_dir"`
RegistryAccountCount int `json:"registry_account_count"`
RegistryTenantCount int `json:"registry_tenant_count"`
RuntimeTenantCount int `json:"runtime_tenant_count"`
RuntimeTenantIDs []string `json:"runtime_tenant_ids"`
RegistryTenantIDs []string `json:"registry_tenant_ids"`
RegistryStateCounts map[string]int `json:"registry_state_counts"`
ControlPlaneDBBackups []string `json:"control_plane_db_backups"`
}
// ProviderMSPBackupCreateResult describes a newly written provider MSP backup.
type ProviderMSPBackupCreateResult struct {
ArchivePath string
Manifest ProviderMSPBackupManifest
ControlPlaneEntries int
TenantEntries int
LicenseEntries int
BytesWritten int64
}
// ProviderMSPBackupVerifyResult describes a verified provider MSP backup.
type ProviderMSPBackupVerifyResult struct {
ArchivePath string
Manifest ProviderMSPBackupManifest
ControlPlaneEntries int
TenantEntries int
LicenseEntries int
HasTenantRegistryDB bool
HasLicenseFile bool
RuntimeTenantDirs []string
ControlPlaneDBFiles []string
VerifiedArchiveBytes int64
}
// DefaultProviderMSPBackupPath returns the compose-friendly default backup
// location under CP_DATA_DIR without placing the archive inside the source trees.
func DefaultProviderMSPBackupPath(cfg *CPConfig, now time.Time) string {
if now.IsZero() {
now = time.Now().UTC()
}
name := "provider-msp-backup-" + now.UTC().Format("20060102T150405Z") + ".tar.gz"
if cfg == nil || strings.TrimSpace(cfg.DataDir) == "" {
return name
}
return filepath.Join(cfg.DataDir, "backups", "provider-msp", name)
}
// CreateProviderMSPBackup creates a recovery archive for a provider-hosted MSP
// control plane. It backs up the control-plane registry through SQLite's online
// VACUUM INTO path instead of copying live WAL files directly.
func CreateProviderMSPBackup(ctx context.Context, cfg *CPConfig, outputPath string) (*ProviderMSPBackupCreateResult, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if cfg == nil {
return nil, fmt.Errorf("control plane config is required")
}
if !cfg.IsProviderHostedMSP() {
return nil, fmt.Errorf("provider MSP backup requires CP_CONTROL_PLANE_MODE=%s", ControlPlaneModeProviderHostedMSP)
}
if cfg.UsesStripeBilling() {
return nil, fmt.Errorf("provider MSP backup is unavailable for Stripe-backed control planes")
}
if strings.TrimSpace(outputPath) == "" {
outputPath = DefaultProviderMSPBackupPath(cfg, time.Now().UTC())
}
archivePath, err := filepath.Abs(filepath.Clean(outputPath))
if err != nil {
return nil, fmt.Errorf("resolve backup archive path: %w", err)
}
controlPlaneDir, err := filepath.Abs(filepath.Clean(cfg.ControlPlaneDir()))
if err != nil {
return nil, fmt.Errorf("resolve control-plane dir: %w", err)
}
tenantsDir, err := filepath.Abs(filepath.Clean(cfg.TenantsDir()))
if err != nil {
return nil, fmt.Errorf("resolve tenants dir: %w", err)
}
if pathIsInside(archivePath, controlPlaneDir) || pathIsInside(archivePath, tenantsDir) {
return nil, fmt.Errorf("backup archive must not be written inside %s or %s", cfg.ControlPlaneDir(), cfg.TenantsDir())
}
if err := requireDirectory(controlPlaneDir, "control-plane dir"); err != nil {
return nil, err
}
registryDB := filepath.Join(controlPlaneDir, "tenants.db")
if err := requireRegularFile(registryDB, "tenant registry database"); err != nil {
return nil, err
}
reg, err := registry.NewTenantRegistry(controlPlaneDir)
if err != nil {
return nil, fmt.Errorf("open tenant registry: %w", err)
}
tenants, listErr := reg.List()
accounts, accountsErr := reg.ListAccounts()
closeErr := reg.Close()
if listErr != nil {
return nil, fmt.Errorf("list tenant registry rows: %w", listErr)
}
if accountsErr != nil {
return nil, fmt.Errorf("list registry accounts: %w", accountsErr)
}
if closeErr != nil {
return nil, fmt.Errorf("close tenant registry before backup: %w", closeErr)
}
manifest, err := buildProviderMSPBackupManifest(cfg, tenants, len(accounts))
if err != nil {
return nil, err
}
if err := requireProviderMSPRuntimeTenantDirs(tenantsDir, manifest.RuntimeTenantIDs); err != nil {
return nil, err
}
licensePath, licenseIncluded, err := resolveProviderMSPBackupLicensePath(cfg)
if err != nil {
return nil, err
}
manifest.LicenseIncluded = licenseIncluded
if err := os.MkdirAll(filepath.Dir(archivePath), 0o755); err != nil {
return nil, fmt.Errorf("create backup archive parent: %w", err)
}
tempDir, err := os.MkdirTemp(filepath.Dir(archivePath), ".provider-msp-backup-*")
if err != nil {
return nil, fmt.Errorf("create backup staging dir: %w", err)
}
defer os.RemoveAll(tempDir)
tmpArchive := archivePath + ".tmp"
_ = os.Remove(tmpArchive)
file, err := os.OpenFile(tmpArchive, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
return nil, fmt.Errorf("create backup archive: %w", err)
}
result, err := writeProviderMSPBackupArchive(ctx, file, tempDir, controlPlaneDir, tenantsDir, licensePath, manifest)
closeArchiveErr := file.Close()
if err != nil {
_ = os.Remove(tmpArchive)
return nil, err
}
if closeArchiveErr != nil {
_ = os.Remove(tmpArchive)
return nil, fmt.Errorf("close backup archive: %w", closeArchiveErr)
}
if err := os.Rename(tmpArchive, archivePath); err != nil {
_ = os.Remove(tmpArchive)
return nil, fmt.Errorf("publish backup archive: %w", err)
}
info, err := os.Stat(archivePath)
if err != nil {
return nil, fmt.Errorf("stat backup archive: %w", err)
}
result.ArchivePath = archivePath
result.BytesWritten = info.Size()
return result, nil
}
// VerifyProviderMSPBackup validates that a provider MSP backup contains the
// manifest, tenant registry snapshot, required tenant directories, and license
// artifact declared by the manifest.
func VerifyProviderMSPBackup(ctx context.Context, archivePath string) (*ProviderMSPBackupVerifyResult, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
archivePath = strings.TrimSpace(archivePath)
if archivePath == "" {
return nil, fmt.Errorf("backup archive path is required")
}
resolvedArchivePath, err := filepath.Abs(filepath.Clean(archivePath))
if err != nil {
return nil, fmt.Errorf("resolve backup archive path: %w", err)
}
file, err := os.Open(resolvedArchivePath)
if err != nil {
return nil, fmt.Errorf("open backup archive: %w", err)
}
defer file.Close()
info, err := file.Stat()
if err != nil {
return nil, fmt.Errorf("stat backup archive: %w", err)
}
gz, err := gzip.NewReader(file)
if err != nil {
return nil, fmt.Errorf("open backup gzip stream: %w", err)
}
defer gz.Close()
result := &ProviderMSPBackupVerifyResult{
ArchivePath: resolvedArchivePath,
VerifiedArchiveBytes: info.Size(),
}
tenantDirs := map[string]bool{}
dbFiles := map[string]bool{}
var manifestBytes []byte
tr := tar.NewReader(gz)
for {
if err := ctx.Err(); err != nil {
return nil, err
}
header, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("read backup archive: %w", err)
}
name, err := cleanProviderMSPArchiveName(header.Name)
if err != nil {
return nil, err
}
switch {
case name == providerMSPBackupManifestName:
manifestBytes, err = readProviderMSPBackupManifestBytes(tr, header.Size)
if err != nil {
return nil, err
}
case name == providerMSPBackupControlPlaneDir || strings.HasPrefix(name, providerMSPBackupControlPlaneDir+"/"):
result.ControlPlaneEntries++
if strings.HasSuffix(name, ".db") {
dbFiles[name] = true
}
if name == providerMSPBackupControlPlaneDir+"/tenants.db" {
result.HasTenantRegistryDB = true
}
case name == providerMSPBackupTenantsDir || strings.HasPrefix(name, providerMSPBackupTenantsDir+"/"):
result.TenantEntries++
if tenantID := archiveTenantID(name); tenantID != "" {
tenantDirs[tenantID] = true
}
case name == providerMSPBackupLicenseDir || strings.HasPrefix(name, providerMSPBackupLicenseDir+"/"):
result.LicenseEntries++
if name == providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName {
result.HasLicenseFile = true
}
default:
return nil, fmt.Errorf("unexpected top-level backup entry %q", name)
}
}
if len(manifestBytes) == 0 {
return nil, fmt.Errorf("backup manifest is missing")
}
if err := json.Unmarshal(manifestBytes, &result.Manifest); err != nil {
return nil, fmt.Errorf("parse backup manifest: %w", err)
}
if err := validateProviderMSPBackupManifest(result.Manifest); err != nil {
return nil, err
}
if !result.HasTenantRegistryDB {
return nil, fmt.Errorf("backup is missing control-plane tenant registry snapshot")
}
if result.ControlPlaneEntries == 0 {
return nil, fmt.Errorf("backup is missing control-plane data")
}
if result.TenantEntries == 0 {
return nil, fmt.Errorf("backup is missing tenants directory")
}
if result.Manifest.LicenseIncluded && !result.HasLicenseFile {
return nil, fmt.Errorf("backup manifest declares included MSP license but license file is missing")
}
for _, dbFile := range result.Manifest.ControlPlaneDBBackups {
if !dbFiles[dbFile] {
return nil, fmt.Errorf("backup manifest declares control-plane DB snapshot %q but it is missing", dbFile)
}
}
for _, tenantID := range result.Manifest.RuntimeTenantIDs {
if !tenantDirs[tenantID] {
return nil, fmt.Errorf("backup is missing runtime tenant directory %q", tenantID)
}
}
for tenantID := range tenantDirs {
result.RuntimeTenantDirs = append(result.RuntimeTenantDirs, tenantID)
}
sort.Strings(result.RuntimeTenantDirs)
for name := range dbFiles {
result.ControlPlaneDBFiles = append(result.ControlPlaneDBFiles, name)
}
sort.Strings(result.ControlPlaneDBFiles)
return result, nil
}
func buildProviderMSPBackupManifest(cfg *CPConfig, tenants []*registry.Tenant, accountCount int) (ProviderMSPBackupManifest, error) {
workspaceLimit, known := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion)
if !known {
return ProviderMSPBackupManifest{}, fmt.Errorf("provider MSP plan %q has no known workspace limit", cfg.ProviderMSPPlanVersion)
}
manifest := ProviderMSPBackupManifest{
Version: ProviderMSPBackupManifestVersion,
CreatedAt: time.Now().UTC(),
ControlPlaneMode: string(cfg.ControlPlaneMode),
Environment: strings.TrimSpace(cfg.Environment),
BaseURL: strings.TrimSpace(cfg.BaseURL),
PlanVersion: strings.TrimSpace(cfg.ProviderMSPPlanVersion),
PlanSource: providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource),
LicenseID: strings.TrimSpace(cfg.ProviderMSPLicenseID),
LicenseEmail: strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)),
WorkspaceLimit: workspaceLimit,
ControlPlaneDir: providerMSPBackupControlPlaneDir,
TenantsDir: providerMSPBackupTenantsDir,
RegistryAccountCount: accountCount,
RegistryStateCounts: map[string]int{},
ControlPlaneDBBackups: []string{},
}
for _, tenant := range tenants {
if tenant == nil {
continue
}
tenantID := strings.TrimSpace(tenant.ID)
if tenantID == "" {
continue
}
manifest.RegistryTenantIDs = append(manifest.RegistryTenantIDs, tenantID)
manifest.RegistryStateCounts[string(tenant.State)]++
if providerMSPBackupRequiresRuntimeTenantDir(tenant.State) {
manifest.RuntimeTenantIDs = append(manifest.RuntimeTenantIDs, tenantID)
}
}
sort.Strings(manifest.RegistryTenantIDs)
sort.Strings(manifest.RuntimeTenantIDs)
manifest.RegistryTenantCount = len(manifest.RegistryTenantIDs)
manifest.RuntimeTenantCount = len(manifest.RuntimeTenantIDs)
return manifest, nil
}
func providerMSPBackupRequiresRuntimeTenantDir(state registry.TenantState) bool {
switch state {
case registry.TenantStateCanceled, registry.TenantStateDeleting, registry.TenantStateDeleted:
return false
default:
return true
}
}
func resolveProviderMSPBackupLicensePath(cfg *CPConfig) (string, bool, error) {
if providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource) != ProviderMSPPlanSourceLicenseFile {
return "", false, nil
}
licensePath := strings.TrimSpace(cfg.ProviderMSPLicenseFile)
if licensePath == "" {
return "", false, fmt.Errorf("provider MSP backup requires CP_PROVIDER_MSP_LICENSE_FILE when plan_source=%s", ProviderMSPPlanSourceLicenseFile)
}
if err := requireRegularFile(licensePath, "provider MSP license file"); err != nil {
return "", false, err
}
return licensePath, true, nil
}
func requireProviderMSPRuntimeTenantDirs(tenantsDir string, tenantIDs []string) error {
if len(tenantIDs) == 0 {
return nil
}
if err := requireDirectory(tenantsDir, "tenants dir"); err != nil {
return err
}
for _, tenantID := range tenantIDs {
dir := filepath.Join(tenantsDir, tenantID)
if err := requireDirectory(dir, "runtime tenant dir "+tenantID); err != nil {
return err
}
}
return nil
}
func writeProviderMSPBackupArchive(ctx context.Context, output io.Writer, tempDir, controlPlaneDir, tenantsDir, licensePath string, manifest ProviderMSPBackupManifest) (*ProviderMSPBackupCreateResult, error) {
gz := gzip.NewWriter(output)
tw := tar.NewWriter(gz)
result := &ProviderMSPBackupCreateResult{
Manifest: manifest,
}
controlPlaneEntries, dbBackups, err := addProviderMSPBackupDir(ctx, tw, tempDir, controlPlaneDir, providerMSPBackupControlPlaneDir, true, manifest.CreatedAt)
if err != nil {
return nil, err
}
result.ControlPlaneEntries = controlPlaneEntries
manifest.ControlPlaneDBBackups = dbBackups
tenantEntries, _, err := addProviderMSPBackupDir(ctx, tw, tempDir, tenantsDir, providerMSPBackupTenantsDir, false, manifest.CreatedAt)
if err != nil {
return nil, err
}
result.TenantEntries = tenantEntries
if strings.TrimSpace(licensePath) != "" {
if err := writeProviderMSPBackupRootDir(tw, providerMSPBackupLicenseDir, manifest.CreatedAt); err != nil {
return nil, err
}
result.LicenseEntries++
if err := addProviderMSPBackupFile(tw, licensePath, providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName, 0); err != nil {
return nil, err
}
result.LicenseEntries++
}
result.Manifest = manifest
manifestBytes, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return nil, fmt.Errorf("encode backup manifest: %w", err)
}
manifestBytes = append(manifestBytes, '\n')
if err := writeProviderMSPBackupBytes(tw, providerMSPBackupManifestName, manifestBytes, 0o600, manifest.CreatedAt); err != nil {
return nil, err
}
if err := tw.Close(); err != nil {
return nil, fmt.Errorf("close backup tar stream: %w", err)
}
if err := gz.Close(); err != nil {
return nil, fmt.Errorf("close backup gzip stream: %w", err)
}
return result, nil
}
func addProviderMSPBackupDir(ctx context.Context, tw *tar.Writer, tempDir, sourceDir, archiveRoot string, snapshotSQLite bool, createdAt time.Time) (int, []string, error) {
if strings.TrimSpace(sourceDir) == "" {
return 0, nil, fmt.Errorf("source dir is required for %s", archiveRoot)
}
entries := 0
dbBackups := []string{}
if err := writeProviderMSPBackupRootDir(tw, archiveRoot, createdAt); err != nil {
return 0, nil, err
}
entries++
if _, err := os.Stat(sourceDir); os.IsNotExist(err) {
return entries, dbBackups, nil
} else if err != nil {
return 0, nil, fmt.Errorf("stat %s: %w", sourceDir, err)
}
err := filepath.WalkDir(sourceDir, func(filePath string, entry os.DirEntry, walkErr error) error {
if err := ctx.Err(); err != nil {
return err
}
if walkErr != nil {
return fmt.Errorf("walk %s: %w", filePath, walkErr)
}
rel, err := filepath.Rel(sourceDir, filePath)
if err != nil {
return fmt.Errorf("resolve backup relative path: %w", err)
}
if rel == "." {
return nil
}
archiveName := filepath.ToSlash(filepath.Join(archiveRoot, rel))
info, err := entry.Info()
if err != nil {
return fmt.Errorf("stat backup source %s: %w", filePath, err)
}
if entry.Type()&os.ModeSymlink != 0 {
if err := addProviderMSPBackupSymlink(tw, filePath, archiveName, info); err != nil {
return err
}
entries++
return nil
}
if entry.IsDir() {
if err := addProviderMSPBackupHeader(tw, info, archiveName, ""); err != nil {
return err
}
entries++
return nil
}
if !entry.Type().IsRegular() {
return fmt.Errorf("unsupported backup source file type at %s", filePath)
}
if snapshotSQLite && isProviderMSPSQLiteSidecar(rel) {
return nil
}
if snapshotSQLite && strings.EqualFold(filepath.Ext(filePath), ".db") {
snapshotPath := filepath.Join(tempDir, archiveRoot, rel)
if err := snapshotSQLiteDatabase(filePath, snapshotPath); err != nil {
return err
}
if err := addProviderMSPBackupFile(tw, snapshotPath, archiveName, info.Mode().Perm()); err != nil {
return err
}
dbBackups = append(dbBackups, archiveName)
entries++
return nil
}
if err := addProviderMSPBackupFile(tw, filePath, archiveName, 0); err != nil {
return err
}
entries++
return nil
})
sort.Strings(dbBackups)
if err != nil {
return 0, nil, err
}
return entries, dbBackups, nil
}
func snapshotSQLiteDatabase(sourcePath, destPath string) error {
if err := requireRegularFile(sourcePath, "sqlite database"); err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
return fmt.Errorf("create sqlite snapshot parent: %w", err)
}
_ = os.Remove(destPath)
dsn := sourcePath + "?" + url.Values{
"_pragma": []string{
"busy_timeout(30000)",
"foreign_keys(ON)",
},
}.Encode()
db, err := sql.Open("sqlite", dsn)
if err != nil {
return fmt.Errorf("open sqlite database for backup: %w", err)
}
defer db.Close()
if _, err := db.Exec("VACUUM INTO ?", destPath); err != nil {
return fmt.Errorf("snapshot sqlite database %s: %w", filepath.Base(sourcePath), err)
}
return nil
}
func addProviderMSPBackupFile(tw *tar.Writer, sourcePath, archiveName string, modeOverride os.FileMode) error {
info, err := os.Lstat(sourcePath)
if err != nil {
return fmt.Errorf("stat backup file %s: %w", sourcePath, err)
}
if modeOverride != 0 {
info = providerMSPFileInfoWithMode{FileInfo: info, mode: modeOverride}
}
file, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("open backup file %s: %w", sourcePath, err)
}
defer file.Close()
if err := addProviderMSPBackupHeader(tw, info, archiveName, ""); err != nil {
return err
}
if _, err := io.Copy(tw, file); err != nil {
return fmt.Errorf("write backup file %s: %w", archiveName, err)
}
return nil
}
func addProviderMSPBackupSymlink(tw *tar.Writer, sourcePath, archiveName string, info os.FileInfo) error {
target, err := os.Readlink(sourcePath)
if err != nil {
return fmt.Errorf("read backup symlink %s: %w", sourcePath, err)
}
return addProviderMSPBackupHeader(tw, info, archiveName, target)
}
func addProviderMSPBackupHeader(tw *tar.Writer, info os.FileInfo, archiveName, linkTarget string) error {
header, err := tar.FileInfoHeader(info, linkTarget)
if err != nil {
return fmt.Errorf("create backup tar header %s: %w", archiveName, err)
}
header.Name = archiveName
header.Uid = 0
header.Gid = 0
header.Uname = ""
header.Gname = ""
if info.IsDir() && !strings.HasSuffix(header.Name, "/") {
header.Name += "/"
}
if err := tw.WriteHeader(header); err != nil {
return fmt.Errorf("write backup tar header %s: %w", archiveName, err)
}
return nil
}
func writeProviderMSPBackupRootDir(tw *tar.Writer, archiveName string, modTime time.Time) error {
name := strings.TrimSuffix(archiveName, "/") + "/"
header := &tar.Header{
Name: name,
Mode: 0o755,
Typeflag: tar.TypeDir,
ModTime: modTime,
}
if err := tw.WriteHeader(header); err != nil {
return fmt.Errorf("write backup tar header %s: %w", archiveName, err)
}
return nil
}
func writeProviderMSPBackupBytes(tw *tar.Writer, archiveName string, content []byte, mode int64, modTime time.Time) error {
header := &tar.Header{
Name: archiveName,
Mode: mode,
Size: int64(len(content)),
Typeflag: tar.TypeReg,
ModTime: modTime,
}
if err := tw.WriteHeader(header); err != nil {
return fmt.Errorf("write backup tar header %s: %w", archiveName, err)
}
if _, err := tw.Write(content); err != nil {
return fmt.Errorf("write backup entry %s: %w", archiveName, err)
}
return nil
}
func validateProviderMSPBackupManifest(manifest ProviderMSPBackupManifest) error {
if manifest.Version != ProviderMSPBackupManifestVersion {
return fmt.Errorf("unsupported provider MSP backup manifest version %q", manifest.Version)
}
if manifest.ControlPlaneMode != string(ControlPlaneModeProviderHostedMSP) {
return fmt.Errorf("backup control_plane_mode = %q, want %q", manifest.ControlPlaneMode, ControlPlaneModeProviderHostedMSP)
}
if strings.TrimSpace(manifest.PlanVersion) == "" {
return fmt.Errorf("backup manifest is missing plan_version")
}
if strings.TrimSpace(manifest.PlanSource) == "" {
return fmt.Errorf("backup manifest is missing plan_source")
}
if manifest.ControlPlaneDir != providerMSPBackupControlPlaneDir {
return fmt.Errorf("backup control_plane_dir = %q, want %q", manifest.ControlPlaneDir, providerMSPBackupControlPlaneDir)
}
if manifest.TenantsDir != providerMSPBackupTenantsDir {
return fmt.Errorf("backup tenants_dir = %q, want %q", manifest.TenantsDir, providerMSPBackupTenantsDir)
}
if manifest.RegistryTenantCount != len(manifest.RegistryTenantIDs) {
return fmt.Errorf("backup registry_tenant_count = %d, want %d ids", manifest.RegistryTenantCount, len(manifest.RegistryTenantIDs))
}
if manifest.RuntimeTenantCount != len(manifest.RuntimeTenantIDs) {
return fmt.Errorf("backup runtime_tenant_count = %d, want %d ids", manifest.RuntimeTenantCount, len(manifest.RuntimeTenantIDs))
}
return nil
}
func readProviderMSPBackupManifestBytes(reader io.Reader, size int64) ([]byte, error) {
if size < 0 || size > maxProviderMSPBackupManifestSize {
return nil, fmt.Errorf("backup manifest size %d is outside allowed range", size)
}
limited := io.LimitReader(reader, maxProviderMSPBackupManifestSize+1)
content, err := io.ReadAll(limited)
if err != nil {
return nil, fmt.Errorf("read backup manifest: %w", err)
}
if len(content) > maxProviderMSPBackupManifestSize {
return nil, fmt.Errorf("backup manifest exceeds %d bytes", maxProviderMSPBackupManifestSize)
}
return content, nil
}
func cleanProviderMSPArchiveName(raw string) (string, error) {
name := strings.TrimSpace(strings.ReplaceAll(raw, "\\", "/"))
if name == "" {
return "", fmt.Errorf("backup archive contains empty entry name")
}
name = strings.TrimPrefix(name, "./")
cleaned := path.Clean(name)
if cleaned == "." || strings.HasPrefix(cleaned, "../") || path.IsAbs(cleaned) {
return "", fmt.Errorf("backup archive contains unsafe entry name %q", raw)
}
return cleaned, nil
}
func archiveTenantID(name string) string {
if name == providerMSPBackupTenantsDir || !strings.HasPrefix(name, providerMSPBackupTenantsDir+"/") {
return ""
}
rest := strings.TrimPrefix(name, providerMSPBackupTenantsDir+"/")
parts := strings.Split(rest, "/")
if len(parts) == 0 {
return ""
}
return strings.TrimSpace(parts[0])
}
func isProviderMSPSQLiteSidecar(rel string) bool {
name := strings.ToLower(filepath.Base(rel))
return strings.HasSuffix(name, ".db-wal") || strings.HasSuffix(name, ".db-shm")
}
func pathIsInside(candidate, parent string) bool {
candidate = filepath.Clean(candidate)
parent = filepath.Clean(parent)
rel, err := filepath.Rel(parent, candidate)
if err != nil {
return false
}
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))
}
func requireDirectory(dir, label string) error {
info, err := os.Stat(dir)
if err != nil {
return fmt.Errorf("%s unavailable: %w", label, err)
}
if !info.IsDir() {
return fmt.Errorf("%s %q is not a directory", label, dir)
}
return nil
}
func requireRegularFile(filePath, label string) error {
info, err := os.Stat(filePath)
if err != nil {
return fmt.Errorf("%s unavailable: %w", label, err)
}
if !info.Mode().IsRegular() {
return fmt.Errorf("%s %q is not a regular file", label, filePath)
}
return nil
}
type providerMSPFileInfoWithMode struct {
os.FileInfo
mode os.FileMode
}
func (i providerMSPFileInfoWithMode) Mode() os.FileMode {
return i.mode
}
@@ -0,0 +1,258 @@
package cloudcp
import (
"archive/tar"
"compress/gzip"
"context"
"database/sql"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
cpauth "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/auth"
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
)
func TestProviderMSPBackupCreateVerifyIncludesRecoveryContract(t *testing.T) {
cfg := testProviderMSPBackupConfig(t)
reg := seedProviderMSPBackupRegistry(t, cfg)
if err := reg.Close(); err != nil {
t.Fatalf("close registry: %v", err)
}
magicLinks, err := cpauth.NewService(cfg.ControlPlaneDir())
if err != nil {
t.Fatalf("NewService: %v", err)
}
magicLinks.Close()
if err := os.MkdirAll(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001"), 0o755); err != nil {
t.Fatalf("create tenant dir: %v", err)
}
if err := os.WriteFile(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001", "runtime.json"), []byte(`{"ok":true}`), 0o600); err != nil {
t.Fatalf("write tenant runtime file: %v", err)
}
if err := os.WriteFile(filepath.Join(cfg.ControlPlaneDir(), "operator-note.txt"), []byte("keep me"), 0o600); err != nil {
t.Fatalf("write control-plane note: %v", err)
}
if err := os.WriteFile(filepath.Join(cfg.ControlPlaneDir(), "tenants.db-wal"), []byte("stale wal sidecar"), 0o600); err != nil {
t.Fatalf("write wal sidecar: %v", err)
}
archivePath := filepath.Join(t.TempDir(), "provider-msp-backup.tar.gz")
result, err := CreateProviderMSPBackup(context.Background(), cfg, archivePath)
if err != nil {
t.Fatalf("CreateProviderMSPBackup: %v", err)
}
if result.Manifest.Version != ProviderMSPBackupManifestVersion {
t.Fatalf("manifest version = %q", result.Manifest.Version)
}
if !result.Manifest.LicenseIncluded {
t.Fatal("expected provider MSP license to be included in recovery archive")
}
if result.Manifest.RegistryAccountCount != 1 || result.Manifest.RegistryTenantCount != 2 || result.Manifest.RuntimeTenantCount != 1 {
t.Fatalf("manifest counts = accounts %d registry tenants %d runtime tenants %d", result.Manifest.RegistryAccountCount, result.Manifest.RegistryTenantCount, result.Manifest.RuntimeTenantCount)
}
if !providerMSPBackupTestContainsString(result.Manifest.ControlPlaneDBBackups, "control-plane/tenants.db") {
t.Fatalf("manifest db backups = %#v, want tenant registry", result.Manifest.ControlPlaneDBBackups)
}
if !providerMSPBackupTestContainsString(result.Manifest.ControlPlaneDBBackups, "control-plane/cp_magic_links.db") {
t.Fatalf("manifest db backups = %#v, want magic-link db", result.Manifest.ControlPlaneDBBackups)
}
verify, err := VerifyProviderMSPBackup(context.Background(), archivePath)
if err != nil {
t.Fatalf("VerifyProviderMSPBackup: %v", err)
}
if !verify.HasTenantRegistryDB || !verify.HasLicenseFile {
t.Fatalf("verify flags = registry %t license %t", verify.HasTenantRegistryDB, verify.HasLicenseFile)
}
if len(verify.RuntimeTenantDirs) != 1 || verify.RuntimeTenantDirs[0] != "t-ACTIVE001" {
t.Fatalf("runtime tenant dirs = %#v", verify.RuntimeTenantDirs)
}
entries := readProviderMSPBackupEntries(t, archivePath)
for _, want := range []string{
"manifest.json",
"control-plane/tenants.db",
"control-plane/cp_magic_links.db",
"control-plane/operator-note.txt",
"tenants/t-ACTIVE001/runtime.json",
"license/provider-msp-license.jwt",
} {
if _, ok := entries[want]; !ok {
t.Fatalf("backup missing %q; entries=%#v", want, sortedEntryNames(entries))
}
}
if _, ok := entries["control-plane/tenants.db-wal"]; ok {
t.Fatalf("backup included raw sqlite WAL sidecar")
}
assertProviderMSPBackupRegistrySnapshotCount(t, entries["control-plane/tenants.db"], 2)
}
func TestProviderMSPBackupRejectsArchiveInsideSourceTrees(t *testing.T) {
cfg := testProviderMSPBackupConfig(t)
reg := seedProviderMSPBackupRegistry(t, cfg)
if err := reg.Close(); err != nil {
t.Fatalf("close registry: %v", err)
}
if err := os.MkdirAll(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001"), 0o755); err != nil {
t.Fatalf("create tenant dir: %v", err)
}
_, err := CreateProviderMSPBackup(context.Background(), cfg, filepath.Join(cfg.ControlPlaneDir(), "backup.tar.gz"))
if err == nil {
t.Fatal("expected archive inside control-plane dir to be rejected")
}
if !strings.Contains(err.Error(), "must not be written inside") {
t.Fatalf("error = %v", err)
}
}
func TestProviderMSPBackupRequiresRuntimeTenantDirs(t *testing.T) {
cfg := testProviderMSPBackupConfig(t)
reg := seedProviderMSPBackupRegistry(t, cfg)
if err := reg.Close(); err != nil {
t.Fatalf("close registry: %v", err)
}
if err := os.MkdirAll(cfg.TenantsDir(), 0o755); err != nil {
t.Fatalf("create tenants dir: %v", err)
}
_, err := CreateProviderMSPBackup(context.Background(), cfg, filepath.Join(t.TempDir(), "backup.tar.gz"))
if err == nil {
t.Fatal("expected missing active tenant runtime dir to fail")
}
if !strings.Contains(err.Error(), "runtime tenant dir t-ACTIVE001") {
t.Fatalf("error = %v", err)
}
}
func testProviderMSPBackupConfig(t *testing.T) *CPConfig {
t.Helper()
root := t.TempDir()
licenseFile := filepath.Join(root, "provider-msp-license.jwt")
if err := os.WriteFile(licenseFile, []byte("signed-provider-msp-license"), 0o600); err != nil {
t.Fatalf("write license: %v", err)
}
return &CPConfig{
DataDir: filepath.Join(root, "data"),
Environment: "production",
ControlPlaneMode: ControlPlaneModeProviderHostedMSP,
BaseURL: "https://msp.example.com",
ProviderMSPPlanVersion: "msp_growth",
ProviderMSPPlanSource: ProviderMSPPlanSourceLicenseFile,
ProviderMSPLicenseFile: licenseFile,
ProviderMSPLicenseID: "lic_provider_msp_backup",
ProviderMSPLicenseEmail: "provider@example.com",
}
}
func seedProviderMSPBackupRegistry(t *testing.T, cfg *CPConfig) *registry.TenantRegistry {
t.Helper()
reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())
if err != nil {
t.Fatalf("NewTenantRegistry: %v", err)
}
if err := reg.CreateAccount(&registry.Account{
ID: "acct_backup",
Kind: registry.AccountKindMSP,
DisplayName: "Backup MSP",
}); err != nil {
t.Fatalf("CreateAccount: %v", err)
}
if err := reg.Create(&registry.Tenant{
ID: "t-ACTIVE001",
AccountID: "acct_backup",
Email: "client-a@example.com",
DisplayName: "Client A",
State: registry.TenantStateActive,
}); err != nil {
t.Fatalf("Create active tenant: %v", err)
}
if err := reg.Create(&registry.Tenant{
ID: "t-DELETED001",
AccountID: "acct_backup",
Email: "client-old@example.com",
DisplayName: "Deleted Client",
State: registry.TenantStateDeleted,
}); err != nil {
t.Fatalf("Create deleted tenant: %v", err)
}
return reg
}
func readProviderMSPBackupEntries(t *testing.T, archivePath string) map[string][]byte {
t.Helper()
file, err := os.Open(archivePath)
if err != nil {
t.Fatalf("open archive: %v", err)
}
defer file.Close()
gz, err := gzip.NewReader(file)
if err != nil {
t.Fatalf("open gzip: %v", err)
}
defer gz.Close()
entries := map[string][]byte{}
tr := tar.NewReader(gz)
for {
header, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("read tar: %v", err)
}
name := strings.TrimSuffix(header.Name, "/")
if header.Typeflag == tar.TypeReg {
content, err := io.ReadAll(tr)
if err != nil {
t.Fatalf("read %s: %v", header.Name, err)
}
entries[name] = content
} else {
entries[name] = nil
}
}
return entries
}
func assertProviderMSPBackupRegistrySnapshotCount(t *testing.T, dbBytes []byte, want int) {
t.Helper()
dbPath := filepath.Join(t.TempDir(), "tenants.db")
if err := os.WriteFile(dbPath, dbBytes, 0o600); err != nil {
t.Fatalf("write snapshot db: %v", err)
}
db, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatalf("open snapshot db: %v", err)
}
defer db.Close()
var got int
if err := db.QueryRow(`SELECT COUNT(*) FROM tenants`).Scan(&got); err != nil {
t.Fatalf("query snapshot db: %v", err)
}
if got != want {
t.Fatalf("snapshot tenant count = %d, want %d", got, want)
}
}
func sortedEntryNames(entries map[string][]byte) []string {
names := make([]string, 0, len(entries))
for name := range entries {
names = append(names, name)
}
sort.Strings(names)
return names
}
func providerMSPBackupTestContainsString(values []string, want string) bool {
for _, value := range values {
if value == want {
return true
}
}
return false
}
@@ -49,6 +49,8 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) {
"docker compose run --rm control-plane provider-msp bootstrap",
"docker compose run --rm control-plane provider-msp preflight",
"docker compose run --rm control-plane provider-msp status",
"docker compose run --rm control-plane provider-msp backup create",
"docker compose run --rm control-plane provider-msp backup verify",
"docker compose run --rm control-plane provider-msp proof",
"--account-name",
"--owner-email",
@@ -3405,10 +3405,12 @@ class SubsystemLookupTest(unittest.TestCase):
self.assertEqual(
match["verification_requirement"]["exact_files"],
[
"cmd/pulse-control-plane/provider_msp_backup_test.go",
"cmd/pulse-control-plane/provider_msp_preflight_test.go",
"cmd/pulse-control-plane/provider_msp_proof_test.go",
"cmd/pulse-control-plane/provider_msp_status_test.go",
"internal/cloudcp/docker/manager_test.go",
"internal/cloudcp/provider_msp_backup_test.go",
"internal/cloudcp/tenant_runtime_rollout_test.go",
"scripts/installtests/provider_msp_deploy_test.go",
],