mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 14:00:29 +00:00
Add provider MSP backup command
This commit is contained in:
@@ -13,6 +13,7 @@ func newProviderMSPCmd() *cobra.Command {
|
||||
Short: "Operate a provider-hosted MSP control plane",
|
||||
}
|
||||
cmd.AddCommand(newProviderMSPBootstrapCmd())
|
||||
cmd.AddCommand(newProviderMSPBackupCmd())
|
||||
cmd.AddCommand(newProviderMSPPreflightCmd())
|
||||
cmd.AddCommand(newProviderMSPProofCmd())
|
||||
cmd.AddCommand(newProviderMSPStatusCmd())
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newProviderMSPBackupCmd() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "backup",
|
||||
Short: "Create and verify provider-hosted MSP recovery archives",
|
||||
}
|
||||
cmd.AddCommand(newProviderMSPBackupCreateCmd())
|
||||
cmd.AddCommand(newProviderMSPBackupVerifyCmd())
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newProviderMSPBackupCreateCmd() *cobra.Command {
|
||||
var outputPath string
|
||||
cmd := &cobra.Command{
|
||||
Use: "create",
|
||||
Short: "Create a provider-hosted MSP recovery archive",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := cloudcp.LoadConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load control plane config: %w", err)
|
||||
}
|
||||
result, err := cloudcp.CreateProviderMSPBackup(cmd.Context(), cfg, outputPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printProviderMSPBackupCreateResult(result)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&outputPath, "output", "", "Backup archive path (default: <CP_DATA_DIR>/backups/provider-msp/provider-msp-backup-<timestamp>.tar.gz)")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newProviderMSPBackupVerifyCmd() *cobra.Command {
|
||||
var archivePath string
|
||||
cmd := &cobra.Command{
|
||||
Use: "verify [archive]",
|
||||
Short: "Verify a provider-hosted MSP recovery archive",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) == 1 {
|
||||
if strings.TrimSpace(archivePath) != "" && archivePath != args[0] {
|
||||
return fmt.Errorf("archive path provided both as --archive and positional argument")
|
||||
}
|
||||
archivePath = args[0]
|
||||
}
|
||||
result, err := cloudcp.VerifyProviderMSPBackup(cmd.Context(), archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printProviderMSPBackupVerifyResult(result)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&archivePath, "archive", "", "Backup archive path to verify")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func printProviderMSPBackupCreateResult(result *cloudcp.ProviderMSPBackupCreateResult) {
|
||||
if result == nil {
|
||||
fmt.Println("provider_msp_backup_created=false")
|
||||
return
|
||||
}
|
||||
fmt.Println("provider_msp_backup_created=true")
|
||||
fmt.Printf("archive_path=%s\n", result.ArchivePath)
|
||||
fmt.Printf("archive_bytes=%d\n", result.BytesWritten)
|
||||
printProviderMSPBackupManifest(result.Manifest)
|
||||
fmt.Printf("control_plane_entries=%d\n", result.ControlPlaneEntries)
|
||||
fmt.Printf("tenant_entries=%d\n", result.TenantEntries)
|
||||
fmt.Printf("license_entries=%d\n", result.LicenseEntries)
|
||||
}
|
||||
|
||||
func printProviderMSPBackupVerifyResult(result *cloudcp.ProviderMSPBackupVerifyResult) {
|
||||
if result == nil {
|
||||
fmt.Println("provider_msp_backup_verified=false")
|
||||
return
|
||||
}
|
||||
fmt.Println("provider_msp_backup_verified=true")
|
||||
fmt.Printf("archive_path=%s\n", result.ArchivePath)
|
||||
fmt.Printf("archive_bytes=%d\n", result.VerifiedArchiveBytes)
|
||||
printProviderMSPBackupManifest(result.Manifest)
|
||||
fmt.Printf("control_plane_entries=%d\n", result.ControlPlaneEntries)
|
||||
fmt.Printf("tenant_entries=%d\n", result.TenantEntries)
|
||||
fmt.Printf("license_entries=%d\n", result.LicenseEntries)
|
||||
fmt.Printf("tenant_registry_db_present=%t\n", result.HasTenantRegistryDB)
|
||||
fmt.Printf("license_file_present=%t\n", result.HasLicenseFile)
|
||||
for _, dbFile := range result.ControlPlaneDBFiles {
|
||||
fmt.Printf("control_plane_db_backup=%s\n", dbFile)
|
||||
}
|
||||
for _, tenantID := range result.RuntimeTenantDirs {
|
||||
fmt.Printf("runtime_tenant_dir=%s\n", tenantID)
|
||||
}
|
||||
}
|
||||
|
||||
func printProviderMSPBackupManifest(manifest cloudcp.ProviderMSPBackupManifest) {
|
||||
fmt.Printf("manifest_version=%s\n", manifest.Version)
|
||||
fmt.Printf("created_at=%s\n", manifest.CreatedAt.Format("2006-01-02T15:04:05Z07:00"))
|
||||
fmt.Printf("control_plane_mode=%s\n", manifest.ControlPlaneMode)
|
||||
fmt.Printf("environment=%s\n", manifest.Environment)
|
||||
fmt.Printf("base_url=%s\n", manifest.BaseURL)
|
||||
fmt.Printf("plan_version=%s\n", manifest.PlanVersion)
|
||||
fmt.Printf("plan_source=%s\n", manifest.PlanSource)
|
||||
fmt.Printf("license_id=%s\n", manifest.LicenseID)
|
||||
fmt.Printf("license_email=%s\n", manifest.LicenseEmail)
|
||||
fmt.Printf("license_included=%t\n", manifest.LicenseIncluded)
|
||||
fmt.Printf("workspace_limit=%d\n", manifest.WorkspaceLimit)
|
||||
fmt.Printf("registry_account_count=%d\n", manifest.RegistryAccountCount)
|
||||
fmt.Printf("registry_tenant_count=%d\n", manifest.RegistryTenantCount)
|
||||
fmt.Printf("runtime_tenant_count=%d\n", manifest.RuntimeTenantCount)
|
||||
for _, tenantID := range manifest.RuntimeTenantIDs {
|
||||
fmt.Printf("runtime_tenant_id=%s\n", tenantID)
|
||||
}
|
||||
for _, dbFile := range manifest.ControlPlaneDBBackups {
|
||||
fmt.Printf("manifest_db_backup=%s\n", dbFile)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestProviderMSPCommandExposesBackup(t *testing.T) {
|
||||
cmd := newProviderMSPCmd()
|
||||
for _, child := range cmd.Commands() {
|
||||
if child.Name() == "backup" {
|
||||
foundCreate := false
|
||||
foundVerify := false
|
||||
for _, backupChild := range child.Commands() {
|
||||
switch backupChild.Name() {
|
||||
case "create":
|
||||
foundCreate = true
|
||||
case "verify":
|
||||
foundVerify = true
|
||||
}
|
||||
}
|
||||
if !foundCreate {
|
||||
t.Fatal("provider-msp backup create command is not registered")
|
||||
}
|
||||
if !foundVerify {
|
||||
t.Fatal("provider-msp backup verify command is not registered")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("provider-msp backup command is not registered")
|
||||
}
|
||||
@@ -43,6 +43,13 @@ PULSE_EMAIL_REPLY_TO=support@example.com
|
||||
# Non-mutating operational status:
|
||||
# docker compose run --rm control-plane provider-msp status
|
||||
#
|
||||
# Recovery backup before upgrades or recovery drills:
|
||||
# docker compose run --rm control-plane provider-msp backup create
|
||||
#
|
||||
# Verify a recovery archive before relying on it:
|
||||
# docker compose run --rm control-plane provider-msp backup verify \
|
||||
# /data/backups/provider-msp/provider-msp-backup-YYYYMMDDTHHMMSSZ.tar.gz
|
||||
#
|
||||
# End-to-end provider proof. This creates two proof client workspaces, generates
|
||||
# client-bound agent install tokens, verifies tenant-local agent report ingest,
|
||||
# and exercises the portal handoff path. Keep --cleanup once you have captured
|
||||
|
||||
@@ -29,6 +29,10 @@ Provider-hosted MSP status uses the same tenant registry, health summary, and
|
||||
stuck-provisioning threshold as the control-plane cleanup loop, so operator
|
||||
readiness reports and automated failure handling cannot drift into separate
|
||||
definitions of a failed provider workspace.
|
||||
Provider-hosted MSP backup and verification are part of the cloud-paid operator
|
||||
contract because the recovery artifact must preserve the provider's
|
||||
license-backed plan identity, control-plane account/workspace registry, and
|
||||
tenant-local runtime state without depending on Stripe billing surfaces.
|
||||
|
||||
## Canonical Files
|
||||
|
||||
@@ -112,6 +116,7 @@ definitions of a failed provider workspace.
|
||||
88. `internal/hosted/hosted_metrics.go`, `internal/hosted/reaper.go`
|
||||
89. `internal/cloudcp/public_msp_signup_handlers.go`
|
||||
90. `internal/cloudcp/provider_msp_bootstrap.go`
|
||||
91. `internal/cloudcp/provider_msp_backup.go`
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
@@ -182,7 +187,12 @@ definitions of a failed provider workspace.
|
||||
Pulse Pro operations bundle on a recurring systemd timer, write durable
|
||||
status/log output, and emit Prometheus textfile metrics so a clean GA
|
||||
baseline is continuously monitored rather than manually rediscovered.
|
||||
10. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
|
||||
10. `internal/cloudcp/provider_msp_backup.go` shared with `deployment-installability`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary.
|
||||
License-backed provider MSP backups must include the signed MSP license
|
||||
file as a recovery artifact while exposing only license metadata in command
|
||||
output, and they must keep the archive Stripe-free so provider-hosted MSP
|
||||
recovery does not inherit Pulse-hosted SaaS billing assumptions.
|
||||
11. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
|
||||
Hosted tenant runtime reconciliation must treat a registered tenant with
|
||||
preserved tenant data but no live Docker runtime as a recoverable managed
|
||||
state, not as a terminal skip. The control-plane-owned reconcile path must
|
||||
|
||||
@@ -27,12 +27,14 @@ surfaces.
|
||||
4. `cmd/pulse-control-plane/main.go`
|
||||
5. `cmd/pulse-control-plane/mobile_proof_cmd.go`
|
||||
6. `cmd/pulse-control-plane/provider_msp.go`
|
||||
7. `cmd/pulse-control-plane/provider_msp_preflight.go`
|
||||
8. `cmd/pulse-control-plane/provider_msp_proof.go`
|
||||
9. `cmd/pulse-control-plane/provider_msp_status.go`
|
||||
10. `internal/cloudcp/docker/manager.go`
|
||||
11. `internal/cloudcp/docker/labels.go`
|
||||
12. `internal/cloudcp/tenant_runtime_rollout.go`
|
||||
7. `cmd/pulse-control-plane/provider_msp_backup.go`
|
||||
8. `cmd/pulse-control-plane/provider_msp_preflight.go`
|
||||
9. `cmd/pulse-control-plane/provider_msp_proof.go`
|
||||
10. `cmd/pulse-control-plane/provider_msp_status.go`
|
||||
11. `internal/cloudcp/provider_msp_backup.go`
|
||||
12. `internal/cloudcp/docker/manager.go`
|
||||
13. `internal/cloudcp/docker/labels.go`
|
||||
14. `internal/cloudcp/tenant_runtime_rollout.go`
|
||||
13. `.github/workflows/create-release.yml`
|
||||
14. `.github/workflows/deploy-demo-server.yml`
|
||||
15. `.github/workflows/helm-pages.yml`
|
||||
@@ -117,7 +119,7 @@ surfaces.
|
||||
2. `internal/api/updates.go` shared with `api-contracts`: update handlers are both a deployment-installability control surface and a canonical API payload contract boundary.
|
||||
3. `internal/cloudcp/docker/labels.go` shared with `cloud-paid`: hosted tenant Docker labels are both a Pulse Cloud runtime contract boundary and a deployment-installability rollout boundary.
|
||||
4. `internal/cloudcp/docker/manager.go` shared with `cloud-paid`: hosted tenant container management is both a Pulse Cloud runtime contract boundary and a deployment-installability rollout boundary.
|
||||
Tenant runtime containers must be created with bounded Docker `json-file`
|
||||
Tenant runtime containers must use bounded Docker `json-file`
|
||||
logging so rollout and canary fleets cannot consume unbounded production
|
||||
host storage while they remain running.
|
||||
Provider-hosted MSP workspace creation and preflight must prepare or report
|
||||
@@ -137,13 +139,22 @@ surfaces.
|
||||
state/health counts, stuck provisioning workspaces, Docker runtime
|
||||
prerequisites, storage guardrails, and the same license-backed plan identity
|
||||
without pulling tenant images unless the operator asks for it.
|
||||
5. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
|
||||
6. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
|
||||
5. `internal/cloudcp/provider_msp_backup.go` shared with `cloud-paid`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary.
|
||||
`pulse-control-plane provider-msp backup create` and `backup verify` must
|
||||
create a Stripe-free recovery archive outside the live
|
||||
control-plane/tenant source trees, snapshot SQLite control-plane databases
|
||||
through an online backup path, include tenant runtime directories for all
|
||||
non-deleted registry workspaces, include the signed MSP license file when
|
||||
the plan source is license-backed, and verify the manifest, tenant registry
|
||||
snapshot, license artifact, and tenant runtime directories before the
|
||||
archive is treated as usable for upgrades or recovery drills.
|
||||
6. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
|
||||
7. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
|
||||
It must expose a non-mutating preflight for the exact Windows agent
|
||||
architecture before Administrator-only install changes, accept token-file
|
||||
enrollment input, and avoid interactive download-failure prompts when
|
||||
launched by generated non-interactive onboarding commands.
|
||||
7. `scripts/install.sh` shared with `agent-lifecycle`: the shell installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
|
||||
8. `scripts/install.sh` shared with `agent-lifecycle`: the shell installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
|
||||
|
||||
## Extension Points
|
||||
|
||||
|
||||
@@ -634,6 +634,14 @@
|
||||
"deployment-installability"
|
||||
]
|
||||
},
|
||||
{
|
||||
"path": "internal/cloudcp/provider_msp_backup.go",
|
||||
"rationale": "provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary",
|
||||
"subsystems": [
|
||||
"cloud-paid",
|
||||
"deployment-installability"
|
||||
]
|
||||
},
|
||||
{
|
||||
"path": "internal/cloudcp/tenant_runtime_rollout.go",
|
||||
"rationale": "hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary",
|
||||
@@ -2624,6 +2632,7 @@
|
||||
".github/workflows/validate-release-assets.yml",
|
||||
"cmd/pulse-control-plane/main.go",
|
||||
"cmd/pulse-control-plane/provider_msp.go",
|
||||
"cmd/pulse-control-plane/provider_msp_backup.go",
|
||||
"cmd/pulse-control-plane/provider_msp_preflight.go",
|
||||
"cmd/pulse-control-plane/provider_msp_proof.go",
|
||||
"cmd/pulse-control-plane/provider_msp_status.go",
|
||||
@@ -2644,6 +2653,7 @@
|
||||
"internal/api/updates.go",
|
||||
"internal/cloudcp/docker/labels.go",
|
||||
"internal/cloudcp/docker/manager.go",
|
||||
"internal/cloudcp/provider_msp_backup.go",
|
||||
"internal/cloudcp/tenant_runtime_rollout.go",
|
||||
"Makefile",
|
||||
"package-lock.json",
|
||||
@@ -2893,20 +2903,24 @@
|
||||
"match_files": [
|
||||
"cmd/pulse-control-plane/main.go",
|
||||
"cmd/pulse-control-plane/provider_msp.go",
|
||||
"cmd/pulse-control-plane/provider_msp_backup.go",
|
||||
"cmd/pulse-control-plane/provider_msp_preflight.go",
|
||||
"cmd/pulse-control-plane/provider_msp_proof.go",
|
||||
"cmd/pulse-control-plane/provider_msp_status.go",
|
||||
"internal/cloudcp/docker/labels.go",
|
||||
"internal/cloudcp/docker/manager.go",
|
||||
"internal/cloudcp/provider_msp_backup.go",
|
||||
"internal/cloudcp/tenant_runtime_rollout.go"
|
||||
],
|
||||
"allow_same_subsystem_tests": false,
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"cmd/pulse-control-plane/provider_msp_backup_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_preflight_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_proof_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_status_test.go",
|
||||
"internal/cloudcp/docker/manager_test.go",
|
||||
"internal/cloudcp/provider_msp_backup_test.go",
|
||||
"internal/cloudcp/tenant_runtime_rollout_test.go",
|
||||
"scripts/installtests/provider_msp_deploy_test.go"
|
||||
]
|
||||
|
||||
@@ -0,0 +1,754 @@
|
||||
package cloudcp
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
|
||||
pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const (
|
||||
ProviderMSPBackupManifestVersion = "provider-msp-backup/v1"
|
||||
providerMSPBackupManifestName = "manifest.json"
|
||||
providerMSPBackupControlPlaneDir = "control-plane"
|
||||
providerMSPBackupTenantsDir = "tenants"
|
||||
providerMSPBackupLicenseDir = "license"
|
||||
providerMSPBackupLicenseName = "provider-msp-license.jwt"
|
||||
maxProviderMSPBackupManifestSize = 1024 * 1024
|
||||
)
|
||||
|
||||
// ProviderMSPBackupManifest is the recovery contract stored inside every
|
||||
// provider-hosted MSP backup archive.
|
||||
type ProviderMSPBackupManifest struct {
|
||||
Version string `json:"version"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ControlPlaneMode string `json:"control_plane_mode"`
|
||||
Environment string `json:"environment"`
|
||||
BaseURL string `json:"base_url"`
|
||||
PlanVersion string `json:"plan_version"`
|
||||
PlanSource string `json:"plan_source"`
|
||||
LicenseID string `json:"license_id,omitempty"`
|
||||
LicenseEmail string `json:"license_email,omitempty"`
|
||||
LicenseIncluded bool `json:"license_included"`
|
||||
WorkspaceLimit int `json:"workspace_limit"`
|
||||
ControlPlaneDir string `json:"control_plane_dir"`
|
||||
TenantsDir string `json:"tenants_dir"`
|
||||
RegistryAccountCount int `json:"registry_account_count"`
|
||||
RegistryTenantCount int `json:"registry_tenant_count"`
|
||||
RuntimeTenantCount int `json:"runtime_tenant_count"`
|
||||
RuntimeTenantIDs []string `json:"runtime_tenant_ids"`
|
||||
RegistryTenantIDs []string `json:"registry_tenant_ids"`
|
||||
RegistryStateCounts map[string]int `json:"registry_state_counts"`
|
||||
ControlPlaneDBBackups []string `json:"control_plane_db_backups"`
|
||||
}
|
||||
|
||||
// ProviderMSPBackupCreateResult describes a newly written provider MSP backup.
|
||||
type ProviderMSPBackupCreateResult struct {
|
||||
ArchivePath string
|
||||
Manifest ProviderMSPBackupManifest
|
||||
ControlPlaneEntries int
|
||||
TenantEntries int
|
||||
LicenseEntries int
|
||||
BytesWritten int64
|
||||
}
|
||||
|
||||
// ProviderMSPBackupVerifyResult describes a verified provider MSP backup.
|
||||
type ProviderMSPBackupVerifyResult struct {
|
||||
ArchivePath string
|
||||
Manifest ProviderMSPBackupManifest
|
||||
ControlPlaneEntries int
|
||||
TenantEntries int
|
||||
LicenseEntries int
|
||||
HasTenantRegistryDB bool
|
||||
HasLicenseFile bool
|
||||
RuntimeTenantDirs []string
|
||||
ControlPlaneDBFiles []string
|
||||
VerifiedArchiveBytes int64
|
||||
}
|
||||
|
||||
// DefaultProviderMSPBackupPath returns the compose-friendly default backup
|
||||
// location under CP_DATA_DIR without placing the archive inside the source trees.
|
||||
func DefaultProviderMSPBackupPath(cfg *CPConfig, now time.Time) string {
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
name := "provider-msp-backup-" + now.UTC().Format("20060102T150405Z") + ".tar.gz"
|
||||
if cfg == nil || strings.TrimSpace(cfg.DataDir) == "" {
|
||||
return name
|
||||
}
|
||||
return filepath.Join(cfg.DataDir, "backups", "provider-msp", name)
|
||||
}
|
||||
|
||||
// CreateProviderMSPBackup creates a recovery archive for a provider-hosted MSP
|
||||
// control plane. It backs up the control-plane registry through SQLite's online
|
||||
// VACUUM INTO path instead of copying live WAL files directly.
|
||||
func CreateProviderMSPBackup(ctx context.Context, cfg *CPConfig, outputPath string) (*ProviderMSPBackupCreateResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("control plane config is required")
|
||||
}
|
||||
if !cfg.IsProviderHostedMSP() {
|
||||
return nil, fmt.Errorf("provider MSP backup requires CP_CONTROL_PLANE_MODE=%s", ControlPlaneModeProviderHostedMSP)
|
||||
}
|
||||
if cfg.UsesStripeBilling() {
|
||||
return nil, fmt.Errorf("provider MSP backup is unavailable for Stripe-backed control planes")
|
||||
}
|
||||
if strings.TrimSpace(outputPath) == "" {
|
||||
outputPath = DefaultProviderMSPBackupPath(cfg, time.Now().UTC())
|
||||
}
|
||||
|
||||
archivePath, err := filepath.Abs(filepath.Clean(outputPath))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve backup archive path: %w", err)
|
||||
}
|
||||
controlPlaneDir, err := filepath.Abs(filepath.Clean(cfg.ControlPlaneDir()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve control-plane dir: %w", err)
|
||||
}
|
||||
tenantsDir, err := filepath.Abs(filepath.Clean(cfg.TenantsDir()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve tenants dir: %w", err)
|
||||
}
|
||||
if pathIsInside(archivePath, controlPlaneDir) || pathIsInside(archivePath, tenantsDir) {
|
||||
return nil, fmt.Errorf("backup archive must not be written inside %s or %s", cfg.ControlPlaneDir(), cfg.TenantsDir())
|
||||
}
|
||||
|
||||
if err := requireDirectory(controlPlaneDir, "control-plane dir"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
registryDB := filepath.Join(controlPlaneDir, "tenants.db")
|
||||
if err := requireRegularFile(registryDB, "tenant registry database"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reg, err := registry.NewTenantRegistry(controlPlaneDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open tenant registry: %w", err)
|
||||
}
|
||||
tenants, listErr := reg.List()
|
||||
accounts, accountsErr := reg.ListAccounts()
|
||||
closeErr := reg.Close()
|
||||
if listErr != nil {
|
||||
return nil, fmt.Errorf("list tenant registry rows: %w", listErr)
|
||||
}
|
||||
if accountsErr != nil {
|
||||
return nil, fmt.Errorf("list registry accounts: %w", accountsErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, fmt.Errorf("close tenant registry before backup: %w", closeErr)
|
||||
}
|
||||
|
||||
manifest, err := buildProviderMSPBackupManifest(cfg, tenants, len(accounts))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := requireProviderMSPRuntimeTenantDirs(tenantsDir, manifest.RuntimeTenantIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
licensePath, licenseIncluded, err := resolveProviderMSPBackupLicensePath(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manifest.LicenseIncluded = licenseIncluded
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(archivePath), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create backup archive parent: %w", err)
|
||||
}
|
||||
tempDir, err := os.MkdirTemp(filepath.Dir(archivePath), ".provider-msp-backup-*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create backup staging dir: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
tmpArchive := archivePath + ".tmp"
|
||||
_ = os.Remove(tmpArchive)
|
||||
file, err := os.OpenFile(tmpArchive, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create backup archive: %w", err)
|
||||
}
|
||||
result, err := writeProviderMSPBackupArchive(ctx, file, tempDir, controlPlaneDir, tenantsDir, licensePath, manifest)
|
||||
closeArchiveErr := file.Close()
|
||||
if err != nil {
|
||||
_ = os.Remove(tmpArchive)
|
||||
return nil, err
|
||||
}
|
||||
if closeArchiveErr != nil {
|
||||
_ = os.Remove(tmpArchive)
|
||||
return nil, fmt.Errorf("close backup archive: %w", closeArchiveErr)
|
||||
}
|
||||
if err := os.Rename(tmpArchive, archivePath); err != nil {
|
||||
_ = os.Remove(tmpArchive)
|
||||
return nil, fmt.Errorf("publish backup archive: %w", err)
|
||||
}
|
||||
info, err := os.Stat(archivePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat backup archive: %w", err)
|
||||
}
|
||||
result.ArchivePath = archivePath
|
||||
result.BytesWritten = info.Size()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// VerifyProviderMSPBackup validates that a provider MSP backup contains the
|
||||
// manifest, tenant registry snapshot, required tenant directories, and license
|
||||
// artifact declared by the manifest.
|
||||
func VerifyProviderMSPBackup(ctx context.Context, archivePath string) (*ProviderMSPBackupVerifyResult, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
archivePath = strings.TrimSpace(archivePath)
|
||||
if archivePath == "" {
|
||||
return nil, fmt.Errorf("backup archive path is required")
|
||||
}
|
||||
resolvedArchivePath, err := filepath.Abs(filepath.Clean(archivePath))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve backup archive path: %w", err)
|
||||
}
|
||||
file, err := os.Open(resolvedArchivePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open backup archive: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat backup archive: %w", err)
|
||||
}
|
||||
gz, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open backup gzip stream: %w", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
result := &ProviderMSPBackupVerifyResult{
|
||||
ArchivePath: resolvedArchivePath,
|
||||
VerifiedArchiveBytes: info.Size(),
|
||||
}
|
||||
tenantDirs := map[string]bool{}
|
||||
dbFiles := map[string]bool{}
|
||||
var manifestBytes []byte
|
||||
tr := tar.NewReader(gz)
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
header, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read backup archive: %w", err)
|
||||
}
|
||||
name, err := cleanProviderMSPArchiveName(header.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case name == providerMSPBackupManifestName:
|
||||
manifestBytes, err = readProviderMSPBackupManifestBytes(tr, header.Size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case name == providerMSPBackupControlPlaneDir || strings.HasPrefix(name, providerMSPBackupControlPlaneDir+"/"):
|
||||
result.ControlPlaneEntries++
|
||||
if strings.HasSuffix(name, ".db") {
|
||||
dbFiles[name] = true
|
||||
}
|
||||
if name == providerMSPBackupControlPlaneDir+"/tenants.db" {
|
||||
result.HasTenantRegistryDB = true
|
||||
}
|
||||
case name == providerMSPBackupTenantsDir || strings.HasPrefix(name, providerMSPBackupTenantsDir+"/"):
|
||||
result.TenantEntries++
|
||||
if tenantID := archiveTenantID(name); tenantID != "" {
|
||||
tenantDirs[tenantID] = true
|
||||
}
|
||||
case name == providerMSPBackupLicenseDir || strings.HasPrefix(name, providerMSPBackupLicenseDir+"/"):
|
||||
result.LicenseEntries++
|
||||
if name == providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName {
|
||||
result.HasLicenseFile = true
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected top-level backup entry %q", name)
|
||||
}
|
||||
}
|
||||
if len(manifestBytes) == 0 {
|
||||
return nil, fmt.Errorf("backup manifest is missing")
|
||||
}
|
||||
if err := json.Unmarshal(manifestBytes, &result.Manifest); err != nil {
|
||||
return nil, fmt.Errorf("parse backup manifest: %w", err)
|
||||
}
|
||||
if err := validateProviderMSPBackupManifest(result.Manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !result.HasTenantRegistryDB {
|
||||
return nil, fmt.Errorf("backup is missing control-plane tenant registry snapshot")
|
||||
}
|
||||
if result.ControlPlaneEntries == 0 {
|
||||
return nil, fmt.Errorf("backup is missing control-plane data")
|
||||
}
|
||||
if result.TenantEntries == 0 {
|
||||
return nil, fmt.Errorf("backup is missing tenants directory")
|
||||
}
|
||||
if result.Manifest.LicenseIncluded && !result.HasLicenseFile {
|
||||
return nil, fmt.Errorf("backup manifest declares included MSP license but license file is missing")
|
||||
}
|
||||
for _, dbFile := range result.Manifest.ControlPlaneDBBackups {
|
||||
if !dbFiles[dbFile] {
|
||||
return nil, fmt.Errorf("backup manifest declares control-plane DB snapshot %q but it is missing", dbFile)
|
||||
}
|
||||
}
|
||||
for _, tenantID := range result.Manifest.RuntimeTenantIDs {
|
||||
if !tenantDirs[tenantID] {
|
||||
return nil, fmt.Errorf("backup is missing runtime tenant directory %q", tenantID)
|
||||
}
|
||||
}
|
||||
for tenantID := range tenantDirs {
|
||||
result.RuntimeTenantDirs = append(result.RuntimeTenantDirs, tenantID)
|
||||
}
|
||||
sort.Strings(result.RuntimeTenantDirs)
|
||||
for name := range dbFiles {
|
||||
result.ControlPlaneDBFiles = append(result.ControlPlaneDBFiles, name)
|
||||
}
|
||||
sort.Strings(result.ControlPlaneDBFiles)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func buildProviderMSPBackupManifest(cfg *CPConfig, tenants []*registry.Tenant, accountCount int) (ProviderMSPBackupManifest, error) {
|
||||
workspaceLimit, known := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion)
|
||||
if !known {
|
||||
return ProviderMSPBackupManifest{}, fmt.Errorf("provider MSP plan %q has no known workspace limit", cfg.ProviderMSPPlanVersion)
|
||||
}
|
||||
manifest := ProviderMSPBackupManifest{
|
||||
Version: ProviderMSPBackupManifestVersion,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ControlPlaneMode: string(cfg.ControlPlaneMode),
|
||||
Environment: strings.TrimSpace(cfg.Environment),
|
||||
BaseURL: strings.TrimSpace(cfg.BaseURL),
|
||||
PlanVersion: strings.TrimSpace(cfg.ProviderMSPPlanVersion),
|
||||
PlanSource: providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource),
|
||||
LicenseID: strings.TrimSpace(cfg.ProviderMSPLicenseID),
|
||||
LicenseEmail: strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)),
|
||||
WorkspaceLimit: workspaceLimit,
|
||||
ControlPlaneDir: providerMSPBackupControlPlaneDir,
|
||||
TenantsDir: providerMSPBackupTenantsDir,
|
||||
RegistryAccountCount: accountCount,
|
||||
RegistryStateCounts: map[string]int{},
|
||||
ControlPlaneDBBackups: []string{},
|
||||
}
|
||||
for _, tenant := range tenants {
|
||||
if tenant == nil {
|
||||
continue
|
||||
}
|
||||
tenantID := strings.TrimSpace(tenant.ID)
|
||||
if tenantID == "" {
|
||||
continue
|
||||
}
|
||||
manifest.RegistryTenantIDs = append(manifest.RegistryTenantIDs, tenantID)
|
||||
manifest.RegistryStateCounts[string(tenant.State)]++
|
||||
if providerMSPBackupRequiresRuntimeTenantDir(tenant.State) {
|
||||
manifest.RuntimeTenantIDs = append(manifest.RuntimeTenantIDs, tenantID)
|
||||
}
|
||||
}
|
||||
sort.Strings(manifest.RegistryTenantIDs)
|
||||
sort.Strings(manifest.RuntimeTenantIDs)
|
||||
manifest.RegistryTenantCount = len(manifest.RegistryTenantIDs)
|
||||
manifest.RuntimeTenantCount = len(manifest.RuntimeTenantIDs)
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func providerMSPBackupRequiresRuntimeTenantDir(state registry.TenantState) bool {
|
||||
switch state {
|
||||
case registry.TenantStateCanceled, registry.TenantStateDeleting, registry.TenantStateDeleted:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func resolveProviderMSPBackupLicensePath(cfg *CPConfig) (string, bool, error) {
|
||||
if providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource) != ProviderMSPPlanSourceLicenseFile {
|
||||
return "", false, nil
|
||||
}
|
||||
licensePath := strings.TrimSpace(cfg.ProviderMSPLicenseFile)
|
||||
if licensePath == "" {
|
||||
return "", false, fmt.Errorf("provider MSP backup requires CP_PROVIDER_MSP_LICENSE_FILE when plan_source=%s", ProviderMSPPlanSourceLicenseFile)
|
||||
}
|
||||
if err := requireRegularFile(licensePath, "provider MSP license file"); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return licensePath, true, nil
|
||||
}
|
||||
|
||||
func requireProviderMSPRuntimeTenantDirs(tenantsDir string, tenantIDs []string) error {
|
||||
if len(tenantIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := requireDirectory(tenantsDir, "tenants dir"); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, tenantID := range tenantIDs {
|
||||
dir := filepath.Join(tenantsDir, tenantID)
|
||||
if err := requireDirectory(dir, "runtime tenant dir "+tenantID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeProviderMSPBackupArchive(ctx context.Context, output io.Writer, tempDir, controlPlaneDir, tenantsDir, licensePath string, manifest ProviderMSPBackupManifest) (*ProviderMSPBackupCreateResult, error) {
|
||||
gz := gzip.NewWriter(output)
|
||||
tw := tar.NewWriter(gz)
|
||||
result := &ProviderMSPBackupCreateResult{
|
||||
Manifest: manifest,
|
||||
}
|
||||
controlPlaneEntries, dbBackups, err := addProviderMSPBackupDir(ctx, tw, tempDir, controlPlaneDir, providerMSPBackupControlPlaneDir, true, manifest.CreatedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.ControlPlaneEntries = controlPlaneEntries
|
||||
manifest.ControlPlaneDBBackups = dbBackups
|
||||
tenantEntries, _, err := addProviderMSPBackupDir(ctx, tw, tempDir, tenantsDir, providerMSPBackupTenantsDir, false, manifest.CreatedAt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.TenantEntries = tenantEntries
|
||||
if strings.TrimSpace(licensePath) != "" {
|
||||
if err := writeProviderMSPBackupRootDir(tw, providerMSPBackupLicenseDir, manifest.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.LicenseEntries++
|
||||
if err := addProviderMSPBackupFile(tw, licensePath, providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName, 0); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.LicenseEntries++
|
||||
}
|
||||
result.Manifest = manifest
|
||||
manifestBytes, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode backup manifest: %w", err)
|
||||
}
|
||||
manifestBytes = append(manifestBytes, '\n')
|
||||
if err := writeProviderMSPBackupBytes(tw, providerMSPBackupManifestName, manifestBytes, 0o600, manifest.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return nil, fmt.Errorf("close backup tar stream: %w", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
return nil, fmt.Errorf("close backup gzip stream: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func addProviderMSPBackupDir(ctx context.Context, tw *tar.Writer, tempDir, sourceDir, archiveRoot string, snapshotSQLite bool, createdAt time.Time) (int, []string, error) {
|
||||
if strings.TrimSpace(sourceDir) == "" {
|
||||
return 0, nil, fmt.Errorf("source dir is required for %s", archiveRoot)
|
||||
}
|
||||
entries := 0
|
||||
dbBackups := []string{}
|
||||
if err := writeProviderMSPBackupRootDir(tw, archiveRoot, createdAt); err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
entries++
|
||||
if _, err := os.Stat(sourceDir); os.IsNotExist(err) {
|
||||
return entries, dbBackups, nil
|
||||
} else if err != nil {
|
||||
return 0, nil, fmt.Errorf("stat %s: %w", sourceDir, err)
|
||||
}
|
||||
err := filepath.WalkDir(sourceDir, func(filePath string, entry os.DirEntry, walkErr error) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if walkErr != nil {
|
||||
return fmt.Errorf("walk %s: %w", filePath, walkErr)
|
||||
}
|
||||
rel, err := filepath.Rel(sourceDir, filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve backup relative path: %w", err)
|
||||
}
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
archiveName := filepath.ToSlash(filepath.Join(archiveRoot, rel))
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat backup source %s: %w", filePath, err)
|
||||
}
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
if err := addProviderMSPBackupSymlink(tw, filePath, archiveName, info); err != nil {
|
||||
return err
|
||||
}
|
||||
entries++
|
||||
return nil
|
||||
}
|
||||
if entry.IsDir() {
|
||||
if err := addProviderMSPBackupHeader(tw, info, archiveName, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
entries++
|
||||
return nil
|
||||
}
|
||||
if !entry.Type().IsRegular() {
|
||||
return fmt.Errorf("unsupported backup source file type at %s", filePath)
|
||||
}
|
||||
if snapshotSQLite && isProviderMSPSQLiteSidecar(rel) {
|
||||
return nil
|
||||
}
|
||||
if snapshotSQLite && strings.EqualFold(filepath.Ext(filePath), ".db") {
|
||||
snapshotPath := filepath.Join(tempDir, archiveRoot, rel)
|
||||
if err := snapshotSQLiteDatabase(filePath, snapshotPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := addProviderMSPBackupFile(tw, snapshotPath, archiveName, info.Mode().Perm()); err != nil {
|
||||
return err
|
||||
}
|
||||
dbBackups = append(dbBackups, archiveName)
|
||||
entries++
|
||||
return nil
|
||||
}
|
||||
if err := addProviderMSPBackupFile(tw, filePath, archiveName, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
entries++
|
||||
return nil
|
||||
})
|
||||
sort.Strings(dbBackups)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return entries, dbBackups, nil
|
||||
}
|
||||
|
||||
func snapshotSQLiteDatabase(sourcePath, destPath string) error {
|
||||
if err := requireRegularFile(sourcePath, "sqlite database"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
|
||||
return fmt.Errorf("create sqlite snapshot parent: %w", err)
|
||||
}
|
||||
_ = os.Remove(destPath)
|
||||
dsn := sourcePath + "?" + url.Values{
|
||||
"_pragma": []string{
|
||||
"busy_timeout(30000)",
|
||||
"foreign_keys(ON)",
|
||||
},
|
||||
}.Encode()
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open sqlite database for backup: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.Exec("VACUUM INTO ?", destPath); err != nil {
|
||||
return fmt.Errorf("snapshot sqlite database %s: %w", filepath.Base(sourcePath), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func addProviderMSPBackupFile(tw *tar.Writer, sourcePath, archiveName string, modeOverride os.FileMode) error {
|
||||
info, err := os.Lstat(sourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat backup file %s: %w", sourcePath, err)
|
||||
}
|
||||
if modeOverride != 0 {
|
||||
info = providerMSPFileInfoWithMode{FileInfo: info, mode: modeOverride}
|
||||
}
|
||||
file, err := os.Open(sourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open backup file %s: %w", sourcePath, err)
|
||||
}
|
||||
defer file.Close()
|
||||
if err := addProviderMSPBackupHeader(tw, info, archiveName, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(tw, file); err != nil {
|
||||
return fmt.Errorf("write backup file %s: %w", archiveName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func addProviderMSPBackupSymlink(tw *tar.Writer, sourcePath, archiveName string, info os.FileInfo) error {
|
||||
target, err := os.Readlink(sourcePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read backup symlink %s: %w", sourcePath, err)
|
||||
}
|
||||
return addProviderMSPBackupHeader(tw, info, archiveName, target)
|
||||
}
|
||||
|
||||
func addProviderMSPBackupHeader(tw *tar.Writer, info os.FileInfo, archiveName, linkTarget string) error {
|
||||
header, err := tar.FileInfoHeader(info, linkTarget)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create backup tar header %s: %w", archiveName, err)
|
||||
}
|
||||
header.Name = archiveName
|
||||
header.Uid = 0
|
||||
header.Gid = 0
|
||||
header.Uname = ""
|
||||
header.Gname = ""
|
||||
if info.IsDir() && !strings.HasSuffix(header.Name, "/") {
|
||||
header.Name += "/"
|
||||
}
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return fmt.Errorf("write backup tar header %s: %w", archiveName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeProviderMSPBackupRootDir(tw *tar.Writer, archiveName string, modTime time.Time) error {
|
||||
name := strings.TrimSuffix(archiveName, "/") + "/"
|
||||
header := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o755,
|
||||
Typeflag: tar.TypeDir,
|
||||
ModTime: modTime,
|
||||
}
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return fmt.Errorf("write backup tar header %s: %w", archiveName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeProviderMSPBackupBytes(tw *tar.Writer, archiveName string, content []byte, mode int64, modTime time.Time) error {
|
||||
header := &tar.Header{
|
||||
Name: archiveName,
|
||||
Mode: mode,
|
||||
Size: int64(len(content)),
|
||||
Typeflag: tar.TypeReg,
|
||||
ModTime: modTime,
|
||||
}
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return fmt.Errorf("write backup tar header %s: %w", archiveName, err)
|
||||
}
|
||||
if _, err := tw.Write(content); err != nil {
|
||||
return fmt.Errorf("write backup entry %s: %w", archiveName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateProviderMSPBackupManifest(manifest ProviderMSPBackupManifest) error {
|
||||
if manifest.Version != ProviderMSPBackupManifestVersion {
|
||||
return fmt.Errorf("unsupported provider MSP backup manifest version %q", manifest.Version)
|
||||
}
|
||||
if manifest.ControlPlaneMode != string(ControlPlaneModeProviderHostedMSP) {
|
||||
return fmt.Errorf("backup control_plane_mode = %q, want %q", manifest.ControlPlaneMode, ControlPlaneModeProviderHostedMSP)
|
||||
}
|
||||
if strings.TrimSpace(manifest.PlanVersion) == "" {
|
||||
return fmt.Errorf("backup manifest is missing plan_version")
|
||||
}
|
||||
if strings.TrimSpace(manifest.PlanSource) == "" {
|
||||
return fmt.Errorf("backup manifest is missing plan_source")
|
||||
}
|
||||
if manifest.ControlPlaneDir != providerMSPBackupControlPlaneDir {
|
||||
return fmt.Errorf("backup control_plane_dir = %q, want %q", manifest.ControlPlaneDir, providerMSPBackupControlPlaneDir)
|
||||
}
|
||||
if manifest.TenantsDir != providerMSPBackupTenantsDir {
|
||||
return fmt.Errorf("backup tenants_dir = %q, want %q", manifest.TenantsDir, providerMSPBackupTenantsDir)
|
||||
}
|
||||
if manifest.RegistryTenantCount != len(manifest.RegistryTenantIDs) {
|
||||
return fmt.Errorf("backup registry_tenant_count = %d, want %d ids", manifest.RegistryTenantCount, len(manifest.RegistryTenantIDs))
|
||||
}
|
||||
if manifest.RuntimeTenantCount != len(manifest.RuntimeTenantIDs) {
|
||||
return fmt.Errorf("backup runtime_tenant_count = %d, want %d ids", manifest.RuntimeTenantCount, len(manifest.RuntimeTenantIDs))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readProviderMSPBackupManifestBytes(reader io.Reader, size int64) ([]byte, error) {
|
||||
if size < 0 || size > maxProviderMSPBackupManifestSize {
|
||||
return nil, fmt.Errorf("backup manifest size %d is outside allowed range", size)
|
||||
}
|
||||
limited := io.LimitReader(reader, maxProviderMSPBackupManifestSize+1)
|
||||
content, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read backup manifest: %w", err)
|
||||
}
|
||||
if len(content) > maxProviderMSPBackupManifestSize {
|
||||
return nil, fmt.Errorf("backup manifest exceeds %d bytes", maxProviderMSPBackupManifestSize)
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
func cleanProviderMSPArchiveName(raw string) (string, error) {
|
||||
name := strings.TrimSpace(strings.ReplaceAll(raw, "\\", "/"))
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("backup archive contains empty entry name")
|
||||
}
|
||||
name = strings.TrimPrefix(name, "./")
|
||||
cleaned := path.Clean(name)
|
||||
if cleaned == "." || strings.HasPrefix(cleaned, "../") || path.IsAbs(cleaned) {
|
||||
return "", fmt.Errorf("backup archive contains unsafe entry name %q", raw)
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
func archiveTenantID(name string) string {
|
||||
if name == providerMSPBackupTenantsDir || !strings.HasPrefix(name, providerMSPBackupTenantsDir+"/") {
|
||||
return ""
|
||||
}
|
||||
rest := strings.TrimPrefix(name, providerMSPBackupTenantsDir+"/")
|
||||
parts := strings.Split(rest, "/")
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(parts[0])
|
||||
}
|
||||
|
||||
func isProviderMSPSQLiteSidecar(rel string) bool {
|
||||
name := strings.ToLower(filepath.Base(rel))
|
||||
return strings.HasSuffix(name, ".db-wal") || strings.HasSuffix(name, ".db-shm")
|
||||
}
|
||||
|
||||
func pathIsInside(candidate, parent string) bool {
|
||||
candidate = filepath.Clean(candidate)
|
||||
parent = filepath.Clean(parent)
|
||||
rel, err := filepath.Rel(parent, candidate)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))
|
||||
}
|
||||
|
||||
func requireDirectory(dir, label string) error {
|
||||
info, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s unavailable: %w", label, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("%s %q is not a directory", label, dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func requireRegularFile(filePath, label string) error {
|
||||
info, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s unavailable: %w", label, err)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s %q is not a regular file", label, filePath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type providerMSPFileInfoWithMode struct {
|
||||
os.FileInfo
|
||||
mode os.FileMode
|
||||
}
|
||||
|
||||
func (i providerMSPFileInfoWithMode) Mode() os.FileMode {
|
||||
return i.mode
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package cloudcp
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
cpauth "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/auth"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
|
||||
)
|
||||
|
||||
func TestProviderMSPBackupCreateVerifyIncludesRecoveryContract(t *testing.T) {
|
||||
cfg := testProviderMSPBackupConfig(t)
|
||||
reg := seedProviderMSPBackupRegistry(t, cfg)
|
||||
if err := reg.Close(); err != nil {
|
||||
t.Fatalf("close registry: %v", err)
|
||||
}
|
||||
magicLinks, err := cpauth.NewService(cfg.ControlPlaneDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewService: %v", err)
|
||||
}
|
||||
magicLinks.Close()
|
||||
if err := os.MkdirAll(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001"), 0o755); err != nil {
|
||||
t.Fatalf("create tenant dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001", "runtime.json"), []byte(`{"ok":true}`), 0o600); err != nil {
|
||||
t.Fatalf("write tenant runtime file: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(cfg.ControlPlaneDir(), "operator-note.txt"), []byte("keep me"), 0o600); err != nil {
|
||||
t.Fatalf("write control-plane note: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(cfg.ControlPlaneDir(), "tenants.db-wal"), []byte("stale wal sidecar"), 0o600); err != nil {
|
||||
t.Fatalf("write wal sidecar: %v", err)
|
||||
}
|
||||
|
||||
archivePath := filepath.Join(t.TempDir(), "provider-msp-backup.tar.gz")
|
||||
result, err := CreateProviderMSPBackup(context.Background(), cfg, archivePath)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateProviderMSPBackup: %v", err)
|
||||
}
|
||||
if result.Manifest.Version != ProviderMSPBackupManifestVersion {
|
||||
t.Fatalf("manifest version = %q", result.Manifest.Version)
|
||||
}
|
||||
if !result.Manifest.LicenseIncluded {
|
||||
t.Fatal("expected provider MSP license to be included in recovery archive")
|
||||
}
|
||||
if result.Manifest.RegistryAccountCount != 1 || result.Manifest.RegistryTenantCount != 2 || result.Manifest.RuntimeTenantCount != 1 {
|
||||
t.Fatalf("manifest counts = accounts %d registry tenants %d runtime tenants %d", result.Manifest.RegistryAccountCount, result.Manifest.RegistryTenantCount, result.Manifest.RuntimeTenantCount)
|
||||
}
|
||||
if !providerMSPBackupTestContainsString(result.Manifest.ControlPlaneDBBackups, "control-plane/tenants.db") {
|
||||
t.Fatalf("manifest db backups = %#v, want tenant registry", result.Manifest.ControlPlaneDBBackups)
|
||||
}
|
||||
if !providerMSPBackupTestContainsString(result.Manifest.ControlPlaneDBBackups, "control-plane/cp_magic_links.db") {
|
||||
t.Fatalf("manifest db backups = %#v, want magic-link db", result.Manifest.ControlPlaneDBBackups)
|
||||
}
|
||||
|
||||
verify, err := VerifyProviderMSPBackup(context.Background(), archivePath)
|
||||
if err != nil {
|
||||
t.Fatalf("VerifyProviderMSPBackup: %v", err)
|
||||
}
|
||||
if !verify.HasTenantRegistryDB || !verify.HasLicenseFile {
|
||||
t.Fatalf("verify flags = registry %t license %t", verify.HasTenantRegistryDB, verify.HasLicenseFile)
|
||||
}
|
||||
if len(verify.RuntimeTenantDirs) != 1 || verify.RuntimeTenantDirs[0] != "t-ACTIVE001" {
|
||||
t.Fatalf("runtime tenant dirs = %#v", verify.RuntimeTenantDirs)
|
||||
}
|
||||
|
||||
entries := readProviderMSPBackupEntries(t, archivePath)
|
||||
for _, want := range []string{
|
||||
"manifest.json",
|
||||
"control-plane/tenants.db",
|
||||
"control-plane/cp_magic_links.db",
|
||||
"control-plane/operator-note.txt",
|
||||
"tenants/t-ACTIVE001/runtime.json",
|
||||
"license/provider-msp-license.jwt",
|
||||
} {
|
||||
if _, ok := entries[want]; !ok {
|
||||
t.Fatalf("backup missing %q; entries=%#v", want, sortedEntryNames(entries))
|
||||
}
|
||||
}
|
||||
if _, ok := entries["control-plane/tenants.db-wal"]; ok {
|
||||
t.Fatalf("backup included raw sqlite WAL sidecar")
|
||||
}
|
||||
assertProviderMSPBackupRegistrySnapshotCount(t, entries["control-plane/tenants.db"], 2)
|
||||
}
|
||||
|
||||
func TestProviderMSPBackupRejectsArchiveInsideSourceTrees(t *testing.T) {
|
||||
cfg := testProviderMSPBackupConfig(t)
|
||||
reg := seedProviderMSPBackupRegistry(t, cfg)
|
||||
if err := reg.Close(); err != nil {
|
||||
t.Fatalf("close registry: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001"), 0o755); err != nil {
|
||||
t.Fatalf("create tenant dir: %v", err)
|
||||
}
|
||||
|
||||
_, err := CreateProviderMSPBackup(context.Background(), cfg, filepath.Join(cfg.ControlPlaneDir(), "backup.tar.gz"))
|
||||
if err == nil {
|
||||
t.Fatal("expected archive inside control-plane dir to be rejected")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "must not be written inside") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderMSPBackupRequiresRuntimeTenantDirs(t *testing.T) {
|
||||
cfg := testProviderMSPBackupConfig(t)
|
||||
reg := seedProviderMSPBackupRegistry(t, cfg)
|
||||
if err := reg.Close(); err != nil {
|
||||
t.Fatalf("close registry: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(cfg.TenantsDir(), 0o755); err != nil {
|
||||
t.Fatalf("create tenants dir: %v", err)
|
||||
}
|
||||
|
||||
_, err := CreateProviderMSPBackup(context.Background(), cfg, filepath.Join(t.TempDir(), "backup.tar.gz"))
|
||||
if err == nil {
|
||||
t.Fatal("expected missing active tenant runtime dir to fail")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "runtime tenant dir t-ACTIVE001") {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func testProviderMSPBackupConfig(t *testing.T) *CPConfig {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
licenseFile := filepath.Join(root, "provider-msp-license.jwt")
|
||||
if err := os.WriteFile(licenseFile, []byte("signed-provider-msp-license"), 0o600); err != nil {
|
||||
t.Fatalf("write license: %v", err)
|
||||
}
|
||||
return &CPConfig{
|
||||
DataDir: filepath.Join(root, "data"),
|
||||
Environment: "production",
|
||||
ControlPlaneMode: ControlPlaneModeProviderHostedMSP,
|
||||
BaseURL: "https://msp.example.com",
|
||||
ProviderMSPPlanVersion: "msp_growth",
|
||||
ProviderMSPPlanSource: ProviderMSPPlanSourceLicenseFile,
|
||||
ProviderMSPLicenseFile: licenseFile,
|
||||
ProviderMSPLicenseID: "lic_provider_msp_backup",
|
||||
ProviderMSPLicenseEmail: "provider@example.com",
|
||||
}
|
||||
}
|
||||
|
||||
func seedProviderMSPBackupRegistry(t *testing.T, cfg *CPConfig) *registry.TenantRegistry {
|
||||
t.Helper()
|
||||
reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewTenantRegistry: %v", err)
|
||||
}
|
||||
if err := reg.CreateAccount(®istry.Account{
|
||||
ID: "acct_backup",
|
||||
Kind: registry.AccountKindMSP,
|
||||
DisplayName: "Backup MSP",
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateAccount: %v", err)
|
||||
}
|
||||
if err := reg.Create(®istry.Tenant{
|
||||
ID: "t-ACTIVE001",
|
||||
AccountID: "acct_backup",
|
||||
Email: "client-a@example.com",
|
||||
DisplayName: "Client A",
|
||||
State: registry.TenantStateActive,
|
||||
}); err != nil {
|
||||
t.Fatalf("Create active tenant: %v", err)
|
||||
}
|
||||
if err := reg.Create(®istry.Tenant{
|
||||
ID: "t-DELETED001",
|
||||
AccountID: "acct_backup",
|
||||
Email: "client-old@example.com",
|
||||
DisplayName: "Deleted Client",
|
||||
State: registry.TenantStateDeleted,
|
||||
}); err != nil {
|
||||
t.Fatalf("Create deleted tenant: %v", err)
|
||||
}
|
||||
return reg
|
||||
}
|
||||
|
||||
func readProviderMSPBackupEntries(t *testing.T, archivePath string) map[string][]byte {
|
||||
t.Helper()
|
||||
file, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
t.Fatalf("open archive: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
gz, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
t.Fatalf("open gzip: %v", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
entries := map[string][]byte{}
|
||||
tr := tar.NewReader(gz)
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("read tar: %v", err)
|
||||
}
|
||||
name := strings.TrimSuffix(header.Name, "/")
|
||||
if header.Typeflag == tar.TypeReg {
|
||||
content, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", header.Name, err)
|
||||
}
|
||||
entries[name] = content
|
||||
} else {
|
||||
entries[name] = nil
|
||||
}
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func assertProviderMSPBackupRegistrySnapshotCount(t *testing.T, dbBytes []byte, want int) {
|
||||
t.Helper()
|
||||
dbPath := filepath.Join(t.TempDir(), "tenants.db")
|
||||
if err := os.WriteFile(dbPath, dbBytes, 0o600); err != nil {
|
||||
t.Fatalf("write snapshot db: %v", err)
|
||||
}
|
||||
db, err := sql.Open("sqlite", dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open snapshot db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
var got int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM tenants`).Scan(&got); err != nil {
|
||||
t.Fatalf("query snapshot db: %v", err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("snapshot tenant count = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func sortedEntryNames(entries map[string][]byte) []string {
|
||||
names := make([]string, 0, len(entries))
|
||||
for name := range entries {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
func providerMSPBackupTestContainsString(values []string, want string) bool {
|
||||
for _, value := range values {
|
||||
if value == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -49,6 +49,8 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) {
|
||||
"docker compose run --rm control-plane provider-msp bootstrap",
|
||||
"docker compose run --rm control-plane provider-msp preflight",
|
||||
"docker compose run --rm control-plane provider-msp status",
|
||||
"docker compose run --rm control-plane provider-msp backup create",
|
||||
"docker compose run --rm control-plane provider-msp backup verify",
|
||||
"docker compose run --rm control-plane provider-msp proof",
|
||||
"--account-name",
|
||||
"--owner-email",
|
||||
|
||||
@@ -3405,10 +3405,12 @@ class SubsystemLookupTest(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
match["verification_requirement"]["exact_files"],
|
||||
[
|
||||
"cmd/pulse-control-plane/provider_msp_backup_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_preflight_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_proof_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_status_test.go",
|
||||
"internal/cloudcp/docker/manager_test.go",
|
||||
"internal/cloudcp/provider_msp_backup_test.go",
|
||||
"internal/cloudcp/tenant_runtime_rollout_test.go",
|
||||
"scripts/installtests/provider_msp_deploy_test.go",
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user