mirror of
https://github.com/Katakate/k7.git
synced 2026-09-23 10:24:35 +00:00
G
76a8139774
HTTPS-by-default for k7-api, cluster-wide Cilium isolation, first-class --docker on Kata and k7d, and RuntimeClass k7-fc. Playbook pins k7d 0.6.0.
340 lines
12 KiB
Python
340 lines
12 KiB
Python
"""Integration tests for ``k7 restore``, multi-node cluster.
|
|
|
|
Coverage:
|
|
|
|
- Restore after the source sandbox has been deleted entirely (the
|
|
motivating use case).
|
|
- ``--image`` override against a legacy snapshot that lacks
|
|
``k7.io/source-image`` annotations.
|
|
- ``--no-keep-snapshot`` deletes the source snapshot after success.
|
|
- Restoring into an existing sandbox name is rejected with 409 semantics.
|
|
- Restoring from a snapshot whose ``readyToUse`` is false fails fast.
|
|
|
|
These exercise ``K7Core.restore_sandbox`` directly (async tests) so we
|
|
share an event loop with the kubernetes_asyncio clients; the
|
|
``dev.sh``-based ``test_pause_resume.py`` covers the CLI surface.
|
|
"""
|
|
|
|
import asyncio
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
|
|
from k7.core.core import K7Core
|
|
from k7.core.models import SandboxConfig, SandboxConfigOverrides
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
_K3S = "/usr/local/bin/k3s"
|
|
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
def _run_kubectl(*args: str) -> subprocess.CompletedProcess:
|
|
return subprocess.run([_K3S, "kubectl", *args], capture_output=True, text=True)
|
|
|
|
|
|
def _wait_pod_ready(name: str, namespace: str, timeout: int = 240) -> None:
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
out = _run_kubectl(
|
|
"get",
|
|
"pods",
|
|
"-n",
|
|
namespace,
|
|
"-l",
|
|
f"app={name}",
|
|
"-o",
|
|
"jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}",
|
|
)
|
|
if out.returncode == 0 and out.stdout.strip() == "True":
|
|
return
|
|
time.sleep(2)
|
|
raise TimeoutError(f"pod for sandbox {name} not Ready within {timeout}s")
|
|
|
|
|
|
def _snapshot_exists(name: str, namespace: str) -> bool:
|
|
out = _run_kubectl("get", "volumesnapshot", name, "-n", namespace, "-o", "name")
|
|
return out.returncode == 0 and out.stdout.strip() != ""
|
|
|
|
|
|
def _force_delete_snapshot(name: str, namespace: str) -> None:
|
|
_run_kubectl(
|
|
"patch",
|
|
"volumesnapshot",
|
|
name,
|
|
"-n",
|
|
namespace,
|
|
"--type=merge",
|
|
"-p",
|
|
'{"metadata":{"finalizers":[]}}',
|
|
)
|
|
_run_kubectl("delete", "volumesnapshot", name, "-n", namespace, "--ignore-not-found", "--wait=false")
|
|
|
|
|
|
@pytest.fixture()
|
|
def restore_sandbox_cleanup(test_namespace: str):
|
|
"""Track sandboxes and snapshots for teardown — restore tests are messy."""
|
|
sandboxes: list[str] = []
|
|
snapshots: list[str] = []
|
|
|
|
def _register_sandbox(name: str) -> None:
|
|
sandboxes.append(name)
|
|
|
|
def _register_snapshot(name: str) -> None:
|
|
snapshots.append(name)
|
|
|
|
yield _register_sandbox, _register_snapshot
|
|
|
|
for snap in snapshots:
|
|
_force_delete_snapshot(snap, test_namespace)
|
|
for name in sandboxes:
|
|
try:
|
|
asyncio.run(K7Core().delete_sandbox(name, namespace=test_namespace))
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _ql_config(name: str, namespace: str) -> SandboxConfig:
|
|
return SandboxConfig(
|
|
name=name,
|
|
image="alpine:3.20",
|
|
namespace=namespace,
|
|
backend="kata-qemu-longhorn",
|
|
)
|
|
|
|
|
|
class TestRestoreAfterSourceDeleted:
|
|
"""The motivating use case: pause, snapshot, delete source, restore."""
|
|
|
|
async def test_restore_after_pause_then_delete_source(
|
|
self,
|
|
test_namespace: str,
|
|
restore_sandbox_cleanup,
|
|
):
|
|
register_sandbox, register_snapshot = restore_sandbox_cleanup
|
|
source = f"restore-src-{uuid4().hex[:6]}"
|
|
snap = f"{source}-paused-keep"
|
|
restored = f"{source}-r"
|
|
register_sandbox(source)
|
|
register_sandbox(restored)
|
|
register_snapshot(snap)
|
|
|
|
k7 = K7Core()
|
|
|
|
# 1. Create + write a marker into the source's persistent state.
|
|
create = await k7.create_sandbox(_ql_config(source, test_namespace))
|
|
assert create.success, f"create failed: {create.error}"
|
|
_wait_pod_ready(source, test_namespace)
|
|
marker = await k7.exec_command(
|
|
source,
|
|
"echo restore-marker > /mnt/state/marker && cat /mnt/state/marker",
|
|
namespace=test_namespace,
|
|
)
|
|
assert marker.exit_code == 0, f"marker write failed: {marker.stderr}"
|
|
|
|
# 2. Pause + snapshot (kind=pause, persistent).
|
|
pause = await k7.pause_sandbox(source, namespace=test_namespace, snapshot_name=snap)
|
|
assert pause.success, pause.error
|
|
# Wait for the snapshot to become readyToUse before we delete the source.
|
|
deadline = time.time() + 180
|
|
ready = False
|
|
while time.time() < deadline:
|
|
info = await k7.get_snapshot(snap, namespace=test_namespace)
|
|
if info is not None and info.ready_to_use:
|
|
ready = True
|
|
break
|
|
await asyncio.sleep(2)
|
|
assert ready, f"snapshot {snap} never reached readyToUse=True"
|
|
|
|
# 3. Delete the source Deployment + PVC entirely. The snapshot is the
|
|
# only trace of the original sandbox now.
|
|
delete = await k7.delete_sandbox(source, namespace=test_namespace)
|
|
assert delete.success, delete.error
|
|
# Confirm the source's root PVC is gone.
|
|
deadline = time.time() + 60
|
|
while time.time() < deadline:
|
|
out = _run_kubectl("get", "pvc", f"{source}-root-lh", "-n", test_namespace, "-o", "name")
|
|
if out.returncode != 0 or out.stdout.strip() == "":
|
|
break
|
|
time.sleep(2)
|
|
|
|
# 4. Restore from the snapshot.
|
|
restore_k7 = K7Core() # fresh K7Core (separate event loop is fine here)
|
|
restore_result = await restore_k7.restore_sandbox(
|
|
snapshot_name=snap,
|
|
new_sandbox_name=restored,
|
|
namespace=test_namespace,
|
|
)
|
|
assert restore_result.success, f"restore failed: {restore_result.error}"
|
|
_wait_pod_ready(restored, test_namespace)
|
|
|
|
# 5. The marker survives.
|
|
check = await restore_k7.exec_command(restored, "cat /mnt/state/marker", namespace=test_namespace)
|
|
assert check.exit_code == 0
|
|
assert "restore-marker" in check.stdout
|
|
|
|
# 6. Default keep_snapshot=True → snapshot still there.
|
|
assert _snapshot_exists(snap, test_namespace)
|
|
|
|
|
|
class TestRestoreOverrides:
|
|
"""``--image`` override against an annotation-less snapshot."""
|
|
|
|
async def test_image_override_when_annotation_missing(
|
|
self,
|
|
test_namespace: str,
|
|
restore_sandbox_cleanup,
|
|
):
|
|
register_sandbox, register_snapshot = restore_sandbox_cleanup
|
|
source = f"restore-ovr-{uuid4().hex[:6]}"
|
|
snap = f"{source}-legacy"
|
|
restored = f"{source}-r"
|
|
register_sandbox(source)
|
|
register_sandbox(restored)
|
|
register_snapshot(snap)
|
|
|
|
k7 = K7Core()
|
|
create = await k7.create_sandbox(_ql_config(source, test_namespace))
|
|
assert create.success, create.error
|
|
_wait_pod_ready(source, test_namespace)
|
|
|
|
# Create a "legacy" snapshot manually: no ``k7.io/source-*`` annotations.
|
|
custom = await k7._get_custom_objects_client()
|
|
await custom.create_namespaced_custom_object(
|
|
group="snapshot.storage.k8s.io",
|
|
version="v1",
|
|
namespace=test_namespace,
|
|
plural="volumesnapshots",
|
|
body={
|
|
"apiVersion": "snapshot.storage.k8s.io/v1",
|
|
"kind": "VolumeSnapshot",
|
|
"metadata": {"name": snap, "namespace": test_namespace},
|
|
"spec": {
|
|
"volumeSnapshotClassName": "longhorn",
|
|
"source": {"persistentVolumeClaimName": f"{source}-root-lh"},
|
|
},
|
|
},
|
|
)
|
|
# Wait for ready.
|
|
deadline = time.time() + 180
|
|
ready = False
|
|
while time.time() < deadline:
|
|
info = await k7.get_snapshot(snap, namespace=test_namespace)
|
|
if info is not None and info.ready_to_use:
|
|
ready = True
|
|
break
|
|
await asyncio.sleep(2)
|
|
assert ready, "legacy snapshot never became ready"
|
|
|
|
# Restore *without* --image must fail loudly.
|
|
restore_k7 = K7Core()
|
|
bad = await restore_k7.restore_sandbox(
|
|
snapshot_name=snap,
|
|
new_sandbox_name=restored,
|
|
namespace=test_namespace,
|
|
)
|
|
assert not bad.success
|
|
assert "k7.io/source-image" in bad.error
|
|
assert "--image" in bad.error
|
|
|
|
# Restore *with* --image succeeds.
|
|
good = await restore_k7.restore_sandbox(
|
|
snapshot_name=snap,
|
|
new_sandbox_name=restored,
|
|
namespace=test_namespace,
|
|
overrides=SandboxConfigOverrides(image="alpine:3.20"),
|
|
)
|
|
assert good.success, good.error
|
|
_wait_pod_ready(restored, test_namespace)
|
|
|
|
|
|
class TestRestoreNoKeepSnapshot:
|
|
async def test_no_keep_snapshot_deletes_snapshot(
|
|
self,
|
|
test_namespace: str,
|
|
restore_sandbox_cleanup,
|
|
):
|
|
register_sandbox, register_snapshot = restore_sandbox_cleanup
|
|
source = f"restore-nk-{uuid4().hex[:6]}"
|
|
snap = f"{source}-disposable"
|
|
restored = f"{source}-r"
|
|
register_sandbox(source)
|
|
register_sandbox(restored)
|
|
register_snapshot(snap)
|
|
|
|
k7 = K7Core()
|
|
await k7.create_sandbox(_ql_config(source, test_namespace))
|
|
_wait_pod_ready(source, test_namespace)
|
|
await k7.pause_sandbox(source, namespace=test_namespace, snapshot_name=snap)
|
|
|
|
# Wait for ready.
|
|
deadline = time.time() + 180
|
|
while time.time() < deadline:
|
|
info = await k7.get_snapshot(snap, namespace=test_namespace)
|
|
if info is not None and info.ready_to_use:
|
|
break
|
|
await asyncio.sleep(2)
|
|
|
|
restore_k7 = K7Core()
|
|
result = await restore_k7.restore_sandbox(
|
|
snapshot_name=snap,
|
|
new_sandbox_name=restored,
|
|
namespace=test_namespace,
|
|
keep_snapshot=False,
|
|
)
|
|
assert result.success, result.error
|
|
_wait_pod_ready(restored, test_namespace)
|
|
|
|
# Snapshot should be deleted (give Longhorn a few seconds to settle).
|
|
deadline = time.time() + 30
|
|
while time.time() < deadline and _snapshot_exists(snap, test_namespace):
|
|
time.sleep(1)
|
|
assert not _snapshot_exists(snap, test_namespace)
|
|
|
|
|
|
class TestRestoreErrors:
|
|
async def test_restore_into_existing_name_fails(
|
|
self,
|
|
test_namespace: str,
|
|
restore_sandbox_cleanup,
|
|
):
|
|
register_sandbox, register_snapshot = restore_sandbox_cleanup
|
|
source = f"restore-conf-{uuid4().hex[:6]}"
|
|
snap = f"{source}-snap"
|
|
register_sandbox(source)
|
|
register_snapshot(snap)
|
|
|
|
k7 = K7Core()
|
|
await k7.create_sandbox(_ql_config(source, test_namespace))
|
|
_wait_pod_ready(source, test_namespace)
|
|
await k7.pause_sandbox(source, namespace=test_namespace, snapshot_name=snap)
|
|
deadline = time.time() + 180
|
|
while time.time() < deadline:
|
|
info = await k7.get_snapshot(snap, namespace=test_namespace)
|
|
if info is not None and info.ready_to_use:
|
|
break
|
|
await asyncio.sleep(2)
|
|
|
|
# Restore into the source's *own* name → must fail.
|
|
restore_k7 = K7Core()
|
|
result = await restore_k7.restore_sandbox(
|
|
snapshot_name=snap,
|
|
new_sandbox_name=source, # already exists
|
|
namespace=test_namespace,
|
|
)
|
|
assert not result.success
|
|
assert "already exists" in result.error
|
|
|
|
async def test_restore_missing_snapshot_returns_helpful_error(self, test_namespace: str):
|
|
k7 = K7Core()
|
|
result = await k7.restore_sandbox(
|
|
snapshot_name="does-not-exist",
|
|
new_sandbox_name="any",
|
|
namespace=test_namespace,
|
|
)
|
|
assert not result.success
|
|
assert "not found" in result.error.lower()
|