PSProxmoxVE/tests/PSProxmoxVE.Tests/Integration/ZZ.ClusterConfig.Integration.Tests.ps1

#Requires -Module Pester
<#
.SYNOPSIS
    Pester 5 integration tests for cluster configuration and HA lifecycle.

    These tests exercise the full cluster lifecycle: creating a cluster on
    node A, joining node B, managing HA groups, and removing node B.

    Nodes are provisioned STANDALONE (not clustered). The tests create and
    destroy the cluster during the run.

    Required environment variables:
        PVETEST_HOST       - Node A hostname or IP
        PVETEST_PORT       - API port (default 8006)
        PVETEST_APITOKEN   - Node A API token
        PVETEST_NODE       - Node A node name

    Optional (multi-node tests):
        PVETEST_HOST_B     - Node B hostname or IP
        PVETEST_APITOKEN_B - Node B API token
        PVETEST_PASSWORD   - Root password for nested PVE instances

    Optional:
        PVETEST_PVE_VERSION - PVE major version (8 or 9)

    WARNING: These tests CREATE and DESTROY a cluster on the target nodes.
    Never run against a production cluster.
#>

BeforeAll {
    . $PSScriptRoot/../_TestHelper.ps1

    # --- Required env vars ---
    $requiredVars = @('PVETEST_HOST', 'PVETEST_PORT', 'PVETEST_APITOKEN', 'PVETEST_NODE')
    $script:SkipReason = $null

    foreach ($var in $requiredVars) {
        if (-not [System.Environment]::GetEnvironmentVariable($var)) {
            $script:SkipReason = "No live Proxmox VE target configured. Set: $($requiredVars -join ', ')"
            break
        }
    }

    # --- Convenience accessors ---
    $script:Host_       = [System.Environment]::GetEnvironmentVariable('PVETEST_HOST')
    $portEnv            = [System.Environment]::GetEnvironmentVariable('PVETEST_PORT')
    $script:Port        = [int]$(if ($portEnv) { $portEnv } else { '8006' })
    $script:ApiToken    = [System.Environment]::GetEnvironmentVariable('PVETEST_APITOKEN')
    $script:Node        = [System.Environment]::GetEnvironmentVariable('PVETEST_NODE')

    # --- Optional multi-node vars ---
    $script:HostB       = [System.Environment]::GetEnvironmentVariable('PVETEST_HOST_B')
    $script:ApiTokenB   = [System.Environment]::GetEnvironmentVariable('PVETEST_APITOKEN_B')
    $script:Password    = [System.Environment]::GetEnvironmentVariable('PVETEST_PASSWORD')
    $script:PveVersion  = [System.Environment]::GetEnvironmentVariable('PVETEST_PVE_VERSION')

    # --- State tracking ---
    $script:ClusterCreated = $false
    $script:JoinInfo       = $null
    $script:NodeBName      = $null
    $script:HaRuleCreated  = $false

    # --- Skip helpers ---
    function script:Skip-IfNoTarget {
        if ($script:SkipReason) {
            Set-ItResult -Skipped -Because $script:SkipReason
            return $true
        }
        return $false
    }

    function script:Skip-IfNoNodeB {
        if (Skip-IfNoTarget) { return $true }
        if (-not $script:HostB -or -not $script:ApiTokenB -or -not $script:Password) {
            Set-ItResult -Skipped -Because 'Multi-node env vars not set (PVETEST_HOST_B, PVETEST_APITOKEN_B, PVETEST_PASSWORD)'
            return $true
        }
        return $false
    }

    function script:Skip-IfNoCluster {
        if (Skip-IfNoTarget) { return $true }
        if (-not $script:ClusterCreated) {
            Set-ItResult -Skipped -Because 'Cluster was not created (multi-node env vars may not be set)'
            return $true
        }
        return $false
    }

    function script:Skip-IfNoPve9 {
        if (Skip-IfNoCluster) { return $true }
        if (-not $script:PveVersion -or [int]$script:PveVersion -lt 9) {
            Set-ItResult -Skipped -Because 'HA rules require PVE 9.0+'
            return $true
        }
        return $false
    }

    function script:Skip-IfPve9HaGroups {
        # PVE 9.0+ migrated HA groups to rules — groups API returns 500
        if (Skip-IfNoCluster) { return $true }
        if ($script:PveVersion -and [int]$script:PveVersion -ge 9) {
            Set-ItResult -Skipped -Because 'PVE 9.0+ migrated HA groups to rules — use Get/New/Set/Remove-PveHaRule instead'
            return $true
        }
        return $false
    }
}

AfterAll {
    # Best-effort cleanup of HA rules created during testing
    if ($script:HaRuleCreated) {
        try { Remove-PveHaRule -Rule 'pester-rule-1' -Confirm:$false -ErrorAction SilentlyContinue } catch { }
    }

    # Node removal from a 2-node cluster is not possible via API (quorum loss).
    # Test infrastructure handles cleanup via reprovisioning.
    try { Disconnect-PveServer -ErrorAction SilentlyContinue } catch { }
}

Describe 'Cluster Config & HA Lifecycle — Integration' -Tag 'Integration' {

    # -------------------------------------------------------------------
    Context 'Connection' {
        It 'Should connect to PVE node A' {
            if (Skip-IfNoTarget) { return }

            $session = Connect-PveServer `
                -Server $script:Host_ `
                -Port $script:Port `
                -ApiToken $script:ApiToken `
                -SkipCertificateCheck `
                -PassThru

            $session | Should -Not -BeNullOrEmpty
        }
    }

    # -------------------------------------------------------------------
    Context 'Pre-cluster reads on standalone node' {
        It 'Get-PveClusterStatus returns node info' {
            if (Skip-IfNoTarget) { return }

            $status = Get-PveClusterStatus -ErrorAction Stop
            $status | Should -Not -BeNullOrEmpty
            @($status).Count | Should -BeGreaterOrEqual 1
        }

        It 'Get-PveClusterNextId returns valid VMID' {
            if (Skip-IfNoTarget) { return }

            $nextId = Get-PveClusterNextId -ErrorAction Stop
            $nextId | Should -Not -BeNullOrEmpty
            [int]$nextId | Should -BeGreaterOrEqual 100
        }

        It 'Get-PveClusterOption returns options' {
            if (Skip-IfNoTarget) { return }

            $options = Get-PveClusterOption -ErrorAction Stop
            $options | Should -Not -BeNullOrEmpty
        }

        It 'Get-PveClusterConfig returns config' {
            if (Skip-IfNoTarget) { return }

            # /cluster/config returns a directory listing (array) on standalone nodes
            # and may return an object on clustered nodes — either is valid
            { Get-PveClusterConfig -ErrorAction Stop } | Should -Not -Throw
        }

        It 'Get-PveHaStatus returns status without throwing' {
            if (Skip-IfNoTarget) { return }

            # On a standalone node this may return an empty list — that is fine
            { Get-PveHaStatus -ErrorAction Stop } | Should -Not -Throw
        }
    }

    # -------------------------------------------------------------------
    Context 'Create cluster on node A' {
        It 'New-PveCluster creates a cluster named pester-cluster' {
            if (Skip-IfNoNodeB) { return }

            # Cluster creation requires root@pam ticket auth, not API token
            $secPw = ConvertTo-SecureString $script:Password -AsPlainText -Force
            $cred = New-Object System.Management.Automation.PSCredential('root@pam', $secPw)
            Connect-PveServer `
                -Server $script:Host_ `
                -Port $script:Port `
                -Credential $cred `
                -SkipCertificateCheck

            $result = New-PveCluster -ClusterName 'pester-cluster' -Wait -Confirm:$false -ErrorAction Stop
            $result | Should -Not -BeNullOrEmpty
            $script:ClusterCreated = $true

            # Allow corosync to fully stabilize after cluster creation
            Start-Sleep -Seconds 5
        }

        It 'Get-PveClusterStatus shows cluster formed' {
            if (Skip-IfNoCluster) { return }

            $status = Get-PveClusterStatus -ErrorAction Stop
            # On a newly created single-node cluster, we should see at least
            # a node entry for this node. The "cluster" type entry may take
            # a moment to appear depending on corosync state.
            @($status).Count | Should -BeGreaterOrEqual 1
        }

        It 'Get-PveClusterConfigNode lists node A' {
            if (Skip-IfNoCluster) { return }

            $nodes = Get-PveClusterConfigNode -ErrorAction Stop
            $nodes | Should -Not -BeNullOrEmpty
            @($nodes).Count | Should -BeGreaterOrEqual 1
        }
    }

    # -------------------------------------------------------------------
    Context 'Join node B to cluster' {
        It 'Get-PveClusterJoinInfo returns join token from node A' {
            if (Skip-IfNoNodeB) { return }
            if (Skip-IfNoCluster) { return }

            $script:JoinInfo = Get-PveClusterJoinInfo -ErrorAction Stop
            $script:JoinInfo | Should -Not -BeNullOrEmpty

            # Nodelist is a Newtonsoft JArray — convert to native array for PowerShell compatibility
            $nodelistJson = $script:JoinInfo.Nodelist.ToString()
            $nodelist = $nodelistJson | ConvertFrom-Json
            $nodelist | Should -Not -BeNullOrEmpty
            $script:Fingerprint = $nodelist[0].pve_fp
            $script:Fingerprint | Should -Not -BeNullOrEmpty
        }

        It 'Add-PveClusterMember joins node B to cluster' {
            if (Skip-IfNoNodeB) { return }
            if (Skip-IfNoCluster) { return }
            if (-not $script:Fingerprint) {
                Set-ItResult -Skipped -Because 'Fingerprint was not extracted from join info'
                return
            }

            # Connect to node B with root@pam (join requires ticket auth, not API token)
            $secPw = ConvertTo-SecureString $script:Password -AsPlainText -Force
            $credB = New-Object System.Management.Automation.PSCredential('root@pam', $secPw)
            Connect-PveServer `
                -Server $script:HostB `
                -Port $script:Port `
                -Credential $credB `
                -SkipCertificateCheck

            $fingerprint = $script:Fingerprint

            try {
                $result = Add-PveClusterMember `
                    -Hostname $script:Host_ `
                    -Fingerprint $fingerprint `
                    -Password $secPw `
                    -Wait `
                    -Confirm:$false `
                    -ErrorAction Stop

                $result | Should -Not -BeNullOrEmpty

                # Brief pause for pmxcfs to sync after task completion
                Start-Sleep -Seconds 5
            }
            finally {
                # Always reconnect to node A — whether join succeeded or failed,
                # the current session points at node B which may be invalid.
                $secPwA = ConvertTo-SecureString $script:Password -AsPlainText -Force
                $credA = New-Object System.Management.Automation.PSCredential('root@pam', $secPwA)
                Connect-PveServer `
                    -Server $script:Host_ `
                    -Port $script:Port `
                    -Credential $credA `
                    -SkipCertificateCheck
            }
        }

        It 'Get-PveClusterConfigNode shows both nodes' {
            if (Skip-IfNoNodeB) { return }
            if (Skip-IfNoCluster) { return }

            $nodes = Get-PveClusterConfigNode -ErrorAction Stop
            @($nodes).Count | Should -BeGreaterOrEqual 2

            # Discover and store node B's name for cleanup
            $nodeNames = @($nodes) | ForEach-Object { $_.Name }
            $script:NodeBName = $nodeNames | Where-Object { $_ -ne $script:Node } | Select-Object -First 1
            $script:NodeBName | Should -Not -BeNullOrEmpty
        }

        It 'Get-PveClusterStatus shows 2 nodes online' {
            if (Skip-IfNoNodeB) { return }
            if (Skip-IfNoCluster) { return }

            $status = Get-PveClusterStatus -ErrorAction Stop
            $nodeEntries = @($status) | Where-Object { $_.Type -eq 'node' }
            @($nodeEntries).Count | Should -BeGreaterOrEqual 2

            $onlineNodes = @($nodeEntries) | Where-Object { $_.Online -eq 1 }
            @($onlineNodes).Count | Should -BeGreaterOrEqual 2
        }
    }

    # -------------------------------------------------------------------
    Context 'Cluster options' {
        It 'Set-PveClusterOption sets keyboard to en-us' {
            if (Skip-IfNoCluster) { return }

            # Already connected as root@pam from cluster creation context
            # Save original keyboard setting
            $options = Get-PveClusterOption -ErrorAction Stop
            $script:OriginalKeyboard = $options.Keyboard

            Set-PveClusterOption -Keyboard 'en-us' -Confirm:$false -ErrorAction Stop
        }

        It 'Get-PveClusterOption reflects the keyboard change' {
            if (Skip-IfNoCluster) { return }

            $options = Get-PveClusterOption -ErrorAction Stop
            $options.Keyboard | Should -Be 'en-us'
        }

        It 'Set-PveClusterOption restores original keyboard' {
            if (Skip-IfNoCluster) { return }

            if ($script:OriginalKeyboard) {
                Set-PveClusterOption -Keyboard $script:OriginalKeyboard -Confirm:$false -ErrorAction Stop
            }
            else {
                # Original was unset — delete the key to restore default
                Set-PveClusterOption -Delete 'keyboard' -Confirm:$false -ErrorAction Stop
            }

            $options = Get-PveClusterOption -ErrorAction Stop
            $options.Keyboard | Should -Be $script:OriginalKeyboard
        }
    }

    # -------------------------------------------------------------------
    Context 'HA group management' {
        It 'New-PveHaGroup creates test group' {
            if (Skip-IfPve9HaGroups) { return }

            $nodeList = "$($script:Node):1"
            if ($script:NodeBName) {
                $nodeList += ",$($script:NodeBName):2"
            }

            { New-PveHaGroup -Group 'pester-group' -Nodes $nodeList -Confirm:$false -ErrorAction Stop } |
                Should -Not -Throw
        }

        It 'Get-PveHaGroup lists groups including pester-group' {
            if (Skip-IfPve9HaGroups) { return }

            $groups = @(Get-PveHaGroup -ErrorAction Stop)
            $match = $groups | Where-Object { $_.Group -eq 'pester-group' }
            $match | Should -Not -BeNullOrEmpty
        }

        It 'Get-PveHaGroup returns specific group' {
            if (Skip-IfPve9HaGroups) { return }

            $group = Get-PveHaGroup -Group 'pester-group' -ErrorAction Stop
            $group | Should -Not -BeNullOrEmpty
            $group.Group | Should -Be 'pester-group'
        }

        It 'Set-PveHaGroup updates comment' {
            if (Skip-IfPve9HaGroups) { return }

            { Set-PveHaGroup -Group 'pester-group' -Comment 'pester test' -Confirm:$false -ErrorAction Stop } |
                Should -Not -Throw

            $group = Get-PveHaGroup -Group 'pester-group' -ErrorAction Stop
            $group.Comment | Should -Be 'pester test'
        }

        It 'Remove-PveHaGroup deletes test group' {
            if (Skip-IfPve9HaGroups) { return }

            { Remove-PveHaGroup -Group 'pester-group' -Confirm:$false -ErrorAction Stop } |
                Should -Not -Throw

            # Verify the group is gone
            $groups = @(Get-PveHaGroup -ErrorAction Stop)
            $match = $groups | Where-Object { $_.Group -eq 'pester-group' }
            $match | Should -BeNullOrEmpty
        }
    }

    # -------------------------------------------------------------------
    Context 'HA resource management' {
        It 'Skipped — HA resources require a test VM' {
            Set-ItResult -Skipped -Because 'HA resource tests require a running VM, which is not provisioned in this test file'
        }
    }

    # -------------------------------------------------------------------
    Context 'HA rules CRUD (PVE 9.0+)' {
        It 'Get-PveHaRule returns rules list without throwing' {
            if (Skip-IfNoPve9) { return }

            # May return an empty list on a fresh cluster — that is fine
            { Get-PveHaRule -ErrorAction Stop } | Should -Not -Throw
        }

        It 'New-PveHaRule creates a node-affinity rule' {
            if (Skip-IfNoPve9) { return }
            if (Skip-IfNoCluster) { return }

            # Create a node-affinity rule that pins a fake resource to node A
            { New-PveHaRule -Type 'node-affinity' `
                -Properties @{
                    rule      = 'pester-rule-1'
                    resources = 'vm:99999'
                    nodes     = $script:Node
                    affinity  = 'positive'
                } `
                -Comment 'Pester test rule' `
                -Confirm:$false `
                -ErrorAction Stop } | Should -Not -Throw

            $script:HaRuleCreated = $true
        }

        It 'Get-PveHaRule lists the created rule' {
            if (Skip-IfNoPve9) { return }
            if (-not $script:HaRuleCreated) {
                Set-ItResult -Skipped -Because 'HA rule was not created'
                return
            }

            $rules = Get-PveHaRule -ErrorAction Stop
            $rules | Should -Not -BeNullOrEmpty
            $match = @($rules) | Where-Object { $_.Rule -eq 'pester-rule-1' }
            $match | Should -Not -BeNullOrEmpty
        }

        It 'Get-PveHaRule returns specific rule by ID' {
            if (Skip-IfNoPve9) { return }
            if (-not $script:HaRuleCreated) {
                Set-ItResult -Skipped -Because 'HA rule was not created'
                return
            }

            $rule = Get-PveHaRule -Rule 'pester-rule-1' -ErrorAction Stop
            $rule | Should -Not -BeNullOrEmpty
            $rule.Rule | Should -Be 'pester-rule-1'
            $rule.Comment | Should -Be 'Pester test rule'
        }

        It 'Set-PveHaRule updates the comment' {
            if (Skip-IfNoPve9) { return }
            if (-not $script:HaRuleCreated) {
                Set-ItResult -Skipped -Because 'HA rule was not created'
                return
            }

            { Set-PveHaRule -Rule 'pester-rule-1' `
                -Comment 'Updated by Pester' `
                -Confirm:$false `
                -ErrorAction Stop } | Should -Not -Throw

            $rule = Get-PveHaRule -Rule 'pester-rule-1' -ErrorAction Stop
            $rule.Comment | Should -Be 'Updated by Pester'
        }

        It 'Remove-PveHaRule deletes the rule' {
            if (Skip-IfNoPve9) { return }
            if (-not $script:HaRuleCreated) {
                Set-ItResult -Skipped -Because 'HA rule was not created'
                return
            }

            { Remove-PveHaRule -Rule 'pester-rule-1' -Confirm:$false -ErrorAction Stop } |
                Should -Not -Throw

            $script:HaRuleCreated = $false

            # Verify it's gone
            $rules = Get-PveHaRule -ErrorAction Stop
            $match = @($rules) | Where-Object { $_.Rule -eq 'pester-rule-1' }
            $match | Should -BeNullOrEmpty
        }
    }

    # -------------------------------------------------------------------
    # NOTE: Removing a node from a 2-node cluster via API is not supported
    # because the remaining node loses quorum mid-operation. PVE requires
    # stopping corosync on the departing node first (pvecm expected 1),
    # which is not available via the REST API. Test infrastructure handles
    # cleanup via reprovisioning.
    Context 'Verify cluster state at end of test' {
        It 'Get-PveClusterConfigNode shows both nodes still present' {
            if (Skip-IfNoNodeB) { return }
            if (Skip-IfNoCluster) { return }

            $nodes = Get-PveClusterConfigNode -ErrorAction Stop
            @($nodes).Count | Should -BeGreaterOrEqual 2
        }
    }
}