diff --git a/.devin/wiki.json b/.devin/wiki.json deleted file mode 100644 index ca161ee..0000000 --- a/.devin/wiki.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "repo_notes": [ - { - "content": "Start the wiki with an Architecture section that opens with diagrams explaining how Kubernetes (single-node K3s), Kata, and Firecracker fit together. Do NOT show multiple nodes yet — only a single K3s node is supported currently.", - "author": "Editors" - }, - { - "content": "Katakate (k7) provides self-hosted secure VM sandboxes on Kubernetes using Kata + Firecracker. Code lives under src/k7 (CLI, API, core) and src/katakate (Python SDK). Use this as the source of truth for deep architecture and behavior; the Mintlify docs under docs/ are user-facing.", - "author": "Maintainers" - }, - { - "content": "Security is central: VM isolation via Kata/Firecracker + Jailer, default capability drop, non-root options, Seccomp RuntimeDefault, deny-all ingress, optional egress whitelist with DNS allowance. Keep this model explicit and front-and-center.", - "author": "Maintainers" - }, - { - "content": "Core flows to document deeply: sandbox lifecycle (create/list/delete), before_script execution and readiness probe, egress lockdown policy generation, metrics fetching via metrics.k8s.io. Implemented in src/k7/core/core.py.", - "author": "Maintainers" - }, - { - "content": "API key management (generation, storage, expiry, last_used) lives in src/k7/api/main.py and CLI commands. Keys are stored at /etc/k7/api_keys.json (0600).", - "author": "Maintainers" - }, - { - "content": "The API is deployed with Docker Compose using embedded compose/Dockerfile resolved at runtime by K7Core._get_embedded_docker_compose(). Explain kubeconfig override behavior and Cloudflared tunnel URL discovery.", - "author": "Maintainers" - }, - { - "content": "Packaging: PyPI ships only src/katakate; CLI/API are packaged as .deb under dist/ via src/k7/cli/build.sh. Do not conflate these paths in installation instructions.", - "author": "Maintainers" - }, - { - "content": "Examples and templates live in examples/ (sandbox YAMLs) and tutorials/ (LangChain agent). Reuse when explaining quickstarts.", - "author": "Maintainers" - }, - { - "content": "Known issue: Jailer may be ignored by Kata despite configuration; see README Known issues. Mention as caveat.", - "author": "Maintainers" - } - ], - "pages": [ - { - "title": "Architecture", - "purpose": "Kubernetes + K3s, Kata, Firecracker, Devmapper thin-pool; how components interact (diagrams first)", - "parent": null, - "page_notes": [ - { - "content": "Begin the page with a large, single-node diagram: one K3s node on the host OS. Inside it, depict a Kubernetes Pod configured with runtimeClass 'kata' that launches a Kata microVM (Firecracker). Inside the microVM, show the kata-agent, the guest rootfs, and the container root filesystem. Clearly label boundaries: Host OS, Kubernetes, VM boundary (Firecracker), and Container." - }, - { - "content": "Show the storage path: container image layers resolved by containerd's devmapper snapshotter into a thin pool of logical volumes (LVs). Each sandbox gets an LV snapshot (thin-provisioned). Explain copy-on-write behavior at the disk block level (blocks are shared until written); memory is not shared across microVMs." - }, - { - "content": "Illustrate how the snapshot LV is attached to the Firecracker microVM as a block device and becomes the container rootfs inside the guest. Call out where 'before_script' writes go (into the snapshot)." - }, - { - "content": "Add a second, focused diagram for storage only: Image layers -> devmapper thin pool -> per-sandbox snapshot LV (CoW) -> Firecracker drive -> guest mount -> container rootfs." - }, - { - "content": "Important: depict only a single node (single K3s). Avoid multi-node cluster visuals for now. Optionally add a small 'Coming soon' note about multi-node." - }, - { - "content": "Make it clear the single node can run many sandbox pods concurrently (dozens per node) without drawing them all: add a small annotation/arrow like '... more kata pods' with a brief capacity note." - } - ] - }, - { - "title": "Katakate Overview", - "purpose": "What K7 is, capabilities, core value, links to docs and repo structure", - "parent": null - }, - { - "title": "Installation & Node Setup", - "purpose": "Node requirements, APT install, Ansible-driven installer flow and progress events", - "parent": null - }, - { - "title": "CLI Usage", - "purpose": "How to manage sandboxes from nodes; commands: install, create, list, delete, delete-all, logs, shell, top", - "parent": null, - "page_notes": [ - { "content": "Reference src/k7/cli/k7.py for exact options and behaviors." } - ] - }, - { - "title": "Sandbox Configuration (k7.yaml)", - "purpose": "Explain YAML fields (name, image, namespace, limits, env_file, before_script, egress_whitelist, security flags cap_add/cap_drop, non-root)", - "parent": "CLI Usage" - }, - { - "title": "API Overview", - "purpose": "FastAPI service, auth via API keys, error schema, health", - "parent": null, - "page_notes": [ - { "content": "Entry point: src/k7/api/main.py; app title/version from k7.__version__." } - ] - }, - { - "title": "Authentication & API Keys", - "purpose": "Key creation/list/revoke, storage, expiry, last_used update, headers (X-API-Key or Bearer)", - "parent": "API Overview" - }, - { - "title": "API Endpoints", - "purpose": "REST endpoints for sandboxes, exec, metrics, health with request/response shapes", - "parent": "API Overview" - }, - { - "title": "API: Sandboxes", - "purpose": "POST /api/v1/sandboxes, GET list/get, DELETE single/all, Location header behavior", - "parent": "API Endpoints" - }, - { - "title": "API: Exec", - "purpose": "POST /api/v1/sandboxes/{name}/exec to run commands; response fields", - "parent": "API Endpoints" - }, - { - "title": "API: Metrics", - "purpose": "GET /api/v1/sandboxes/metrics; source: metrics.k8s.io; units parsing", - "parent": "API Endpoints" - }, - { - "title": "API: Health", - "purpose": "GET /health and root", - "parent": "API Endpoints" - }, - { - "title": "Python SDK", - "purpose": "Using katakate Client/AsyncClient to create/list/exec/delete sandboxes; install via pip", - "parent": null, - "page_notes": [ - { "content": "Point to src/katakate/client.py; mirror README examples and types." } - ] - }, - { - "title": "Security Model", - "purpose": "Explain VM isolation, seccomp, capabilities, non-root modes at pod/container level, network isolation strategy", - "parent": null - }, - { - "title": "Network Policies", - "purpose": "Egress whitelist generation + kube-dns allow; deny-all ingress policy created for sandbox label selector", - "parent": "Security Model" - }, - { - "title": "Before Script Lifecycle", - "purpose": "How before_script runs inside main container; readiness gating file; log streaming behavior in CLI", - "parent": "Sandbox Configuration (k7.yaml)" - }, - { - "title": "Metrics and Monitoring", - "purpose": "How top command parses CPU n/u/m units and memory Ki/Mi/Gi; limitations", - "parent": null - }, - { - "title": "Packaging & Releases", - "purpose": "Distribution strategy: PyPI for SDK, Debian for CLI/API; build and install flow", - "parent": null - }, - { - "title": "Tutorials", - "purpose": "Walk through LangChain ReAct agent with K7 sandbox tool", - "parent": null - }, - { - "title": "Development", - "purpose": "Build from source, API container build/run, repo layout, contribution pointers", - "parent": null - }, - { - "title": "Known Issues & Caveats", - "purpose": "Document current limitations (Jailer ignore), roadmap items", - "parent": null - } - ] -} - - diff --git a/.gitignore b/.gitignore index 57f82b8..2a76fdc 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,10 @@ coverage.xml # IDE/editor .vscode/ .idea/ +editor-config/plans/ + +# Temporary files +tmp/ # Work directories (ignore any 'work' dir at any depth) **/work/** @@ -44,3 +48,14 @@ debian/prebuilt/ # macOS metadata **/.DS_Store + +# Kata osbuilder (external dependency, cloned by build scripts) +osbuilder/ + +# Rootfs build artifacts +kata-rootfs/ +rootfs-qemu-lh.img* + +# SSH keys (temporary baked key, will move to K8s Secrets) +k7-vm-ssh-key +k7-vm-ssh-key.pub diff --git a/CHALLENGES.md b/CHALLENGES.md new file mode 100644 index 0000000..4dee48b --- /dev/null +++ b/CHALLENGES.md @@ -0,0 +1,299 @@ +# Challenges & solutions log + +Tracking non-obvious bugs in k7 (the sandbox management layer) and how they +were solved. Same format as k7d-dev's `CHALLENGES.md`. + +## 1. `k7 install --backend` extra-var silently clobbered inventory `k7_backends` (spec 18e) + +**Symptom:** A 3-node HA install with an inventory declaring +`k7_backends=kfd,kql,k7d` per host would have provisioned only the two Kata +backends — the `k7d` backend would silently disappear from every node. + +**Root cause:** `k7 install` always forwarded `k7_backends` as an Ansible +**extra-var** (built from the `--backend` option's *default* value even when +the user never passed `--backend`). Extra-vars have the highest precedence in +Ansible, so the CLI default overrode the per-host inventory var. + +**Fix:** `install()` now checks `ctx.get_parameter_source("backend")`; when +the user supplied `-i ` without an explicit `--backend`, the +`k7_backends` extra-var is dropped so the inventory wins. An explicit +`--backend` alongside `-i` prints a warning that it overrides the inventory. + +**Reference:** none (Ansible variable-precedence rules). + +**Time lost:** caught in pre-flight review (spec 18e Phase 0), ~1 hour of +code reading. Would have cost a full reset+reinstall cycle if it had shipped. + +--- + +## 2. Longhorn StorageClass `numberOfReplicas` parameter makes `default-replica-count` a no-op (spec 18e) + +**Symptom:** `bench_docker_perf.py`'s r1/r2 legs patched Longhorn's +`default-replica-count` setting — but sandbox volumes kept the replica count +baked into the `longhorn` StorageClass (`numberOfReplicas: "3"` on the HA +cluster). The r1/r2/r3 benches would all have silently measured the same +replica count. + +**Root cause:** Longhorn only consults the `default-replica-count` setting +when the StorageClass has **no** `numberOfReplicas` parameter. k7's install +playbook pins the parameter in the SC (topology-aware SC from the +`longhorn-storageclass` ConfigMap), so the setting never applies to k7 +volumes. + +**Fix:** the bench now patches `spec.numberOfReplicas` on the sandbox's +Longhorn **Volume CRs** after creation (`_set_sandbox_volume_replicas`), +waits until exactly N running replicas exist, and records the replica → node +placement in the log header as proof. + +**Reference:** Longhorn docs (volume-level replica count update). + +**Time lost:** ~1 hour. The old benches "passed" — the mismeasurement was +invisible without checking actual replica CRs. + +--- + +## 3. k7d VM operations are node-local; multi-node scheduler breaks pause/fork tests (spec 18e) + +**Symptom:** On the 3-node cluster, `test_k7d.py` pause/fork tests failed +with `sandbox X runs on node k7-node-02, but this k7 process runs on +k7-node-01; k7d VM operations must run on the pod's node`. + +**Root cause:** k7d pause/resume/fork go through the **node-local** +`/run/k7d/k7d.sock`; cross-node VM ops are explicitly out of scope (k7d spec +9a M12) and core fails loudly on the mismatch. On a single-node cluster the +tests never noticed; with 3 schedulable nodes the sandbox lands anywhere. + +**Fix:** tests that exercise VM ops pin their sandboxes with +`SandboxConfig(node_name=os.uname().nodename)` — the field that exists +precisely for host-side-inspection tests. The centralized-API implication +(k7-api pod can only pause/fork k7d sandboxes co-located on the first +master) is recorded as a release-readiness limitation. + +**Reference:** k7d spec 9a M12 (cross-node fork out of scope). + +**Time lost:** ~30 min (the loud error message made it easy). + +--- + +## 4. kql fork loses guest writes made just before the fork (crash-consistency) + +**Symptom:** `test_api.py::test_sdk_pause_resume_fork_round_trip` flaked on +the HA cluster: a file written via exec seconds before `fork()` did not +exist in the fork (`cat: can't open '/mnt/state/marker'`). The near-identical +`test_qemu.py::test_fork_clones_data` data check passed in the same run — +pure timing luck. + +**Root cause:** the kql fork path cuts a **block-level** Longhorn +VolumeSnapshot of the source's root PVC. That is only crash-consistent: guest +writes still sitting in the VM's page cache are not on the block device yet +and are missing from the clone. + +**Fix:** `fork_sandbox` now execs `sync` in the source sandbox before +creating the snapshot (only when the deployment has ready replicas — a +paused source has no writers), failing loudly if the flush fails. + +**Reference:** none (standard crash-vs-application consistency). + +**Time lost:** ~1 hour including re-runs. Note: *named* snapshots of live +sandboxes (`k7 snapshot`) remain crash-consistent by design — documented +behavior, unchanged. + +--- + +## 5. NVMe enumeration swaps across reboots — hardcoded `k7_devmapper_disk` hit the OS disk + +**Symptom:** The second full reset+reinstall loop of spec 18e failed on +k7-node-03: `Device '/dev/nvme1n1' has partitions; wipe it with +utils/wipe-disk.sh or choose another disk`. The identical inventory had just +worked on the first loop. + +**Root cause:** Linux NVMe controller enumeration (`nvme0n1` vs `nvme1n1`) +is not stable across reboots. After the second reset, node-03 booted with +its OS on the disk now enumerated `nvme1n1`, and the raw spare as +`nvme0n1` — the inventory's hardcoded `k7_devmapper_disk=/dev/nvme1n1` +pointed at the OS disk. The playbook's safety checks caught it (fail-loud +worked as designed). + +**Fix:** omit `k7_devmapper_disk` on identical dual-NVMe boxes — the +playbook's auto-detect ("first empty, non-removable, non-root whole disk") +is enumeration-proof. `inventory.ini.example` now documents this. + +**Reference:** none (kernel device-naming behavior). + +**Time lost:** ~30 min (one wasted install attempt + one extra +reset+reinstall loop of all three nodes). + +--- + +## 6. Orphaned Firecracker microVMs leak on pod deletion and each burns a full CPU core + +**Symptom:** During spec 18e Phase 3, the `k7-ql-r2` bench leg started +failing mid-run with `Pod is not running (status: Pending)` and the +`k7-ql-r3` leg failed entirely; longhorn-manager / cilium-envoy / +coredns readiness probes were flapping cluster-wide. `k7-node-03` had a +load average of ~21. + +**Root cause:** 14 orphaned `/firecracker` processes (2 on node-01, 2 on +node-02, 10 on node-03) whose pods had been deleted hours earlier — +zero live Kata pods existed cluster-wide. Each orphan spun at ~97% CPU +(TIME ≈ ETIME in `ps`), starving Longhorn/Cilium/CoreDNS and the bench +sandbox itself. The kata-fc shim intermittently fails to kill the +microVM on pod deletion under parallel pod churn (~14 leaks over ~30 +kfd pod deletions that day). Evidence: +`/tmp/leaked-firecracker-vms.txt` (agent run artifact). + +**Fix (remediation):** verified no live Kata pods, then `pkill -9 +firecracker` on all three nodes; loads recovered and the r2/r3 bench +legs were re-run green. **Root-cause fix still open** — tracked as a +release blocker in spec 18f-release-blockers (investigate +containerd-shim-kata-v2 / jailer cleanup path; add a leak-detection +integration test that asserts zero firecracker processes after suite +teardown). + +**Reference:** none yet (kata-containers shim lifecycle). + +**Time lost:** ~1.5 hours (failed bench legs + diagnosis + re-run). + +--- + +## 7. Remote test loop tied to the SSH session died mid-run (Broken pipe) + +**Symptom:** A multi-suite pytest loop launched over plain `ssh host 'for +f in ...; do pytest ...; done'` died silently when the SSH connection +dropped (`client_loop: send disconnect: Broken pipe`) — the remote shell got +SIGHUP'd between suites. + +**Root cause:** the remote loop was a child of the SSH session; NAT idle +timeouts kill long-lived connections even with keepalives. + +**Fix:** write the loop to a script on the node and launch it with +`setsid nohup ... < /dev/null &`, then poll a progress file. (Same class of +issue `utils/run-integration-tests.sh` already documents for its keepalive +settings.) + +**Reference:** none. + +**Time lost:** ~20 min (one interrupted suite sequence, `test_restore` had +finished right before the drop). + +--- + +## 8. Firecracker leak root cause: `jailer --daemonize` makes the kata shim signal a dead PID (spec 18f) + +**Symptom:** Follow-up to #6. Reproduced at will on the 18f run: create a +naked kfd pod (`sleep` workload), delete it — the pod terminates cleanly +but its `/firecracker` process survives with PPID 1 and climbs to ~100% +CPU. Two out of two attempts leaked. Shim logs at 18e leak time showed +`Agent did not stop sandbox: Dead agent` + `failed to ping agent: +CheckRequest timed out`. + +**Root cause:** kata 3.24.0 `virtcontainers/fc.go`. When jailed (spec 8a +enabled the jailer), `fcInit` launches `jailer --daemonize`, which +double-forks — firecracker reparents to init immediately, and +`fc.info.PID = cmd.Process.Pid` records the **jailer's** PID, which is +already dead. `fcEnd()` then calls `WaitLocalProcess(pid, …, SIGTERM)` on +that stale PID: a no-op. The VMM normally exits because the in-guest agent +shuts the VM down; whenever that graceful path fails (dead/hung agent under +churn, wedged guest IO), nothing ever kills the firecracker process. The +`getting vm status failed … firecracker.socket: no such file or directory` +error seen at every kfd VM boot is a side effect of the same daemonize +handling (the shim polls the jailed API socket path before it exists) and +is harmless noise. + +**Fix:** upstream fix belongs in kata (record the real VMM PID when +jailed). In k7: (a) `k7 install` now deploys a per-node systemd timer +`k7-vmm-reaper.timer` (1 min cadence) that SIGKILLs firecracker processes +whose 32-hex `--id` matches no live `containerd-shim-kata-v2 … -id` +(a live jailed firecracker always has PPID 1, so parentage cannot be used) +and qemu processes reparented to init; (b) +`tests/integration/test_zz_leaks.py` runs last in the suite and asserts +every node's VMM process count equals its live Kata pod count via hostPID +scan pods. + +**Reference:** kata-containers `src/runtime/virtcontainers/fc.go` +(`fcInit`/`fcEnd`), firecracker jailer docs (`--daemonize`). + +**Time lost:** ~1.5 h (live repro + kata source dive), on top of the ~1.5 h +in #6. + +--- + +## 9. Cilium `matchPattern` `*` never crosses label boundaries — `*.docker.com` silently misses CDN blob hosts (spec 18f) + +**Symptom:** `docker pull` inside a sandbox with +`--egress '*.docker.io' --egress '*.docker.com' --egress docker.io +--egress '*.cloudfront.net'` fetches the manifest fine but times out +downloading blobs (`dial tcp 108.156.22.x:443: i/o timeout`), even though +`cilium fqdn cache list` shows `production.cloudfront.docker.com` being +learned. Hubble showed the SYNs `Policy denied DROPPED` with the CloudFront +IPs still carrying identity `world`; `cilium ip list` had `fqdn:*.docker.io` +entries (single-label subdomain `registry-1`) but nothing for the blob host. + +**Root cause:** in Cilium's FQDN `matchPattern` grammar +(`pkg/fqdn/matchpattern`), `*` expands to `[-a-zA-Z0-9_]*` — DNS characters +within a **single label**. `production.cloudfront.docker.com` therefore +does not match `*.docker.com` (and it is not under `cloudfront.net` at +all, so that entry never helped). The multi-label subdomain wildcard is the +non-obvious `**.` prefix form. Not a Cilium bug — a semantics trap between +k7's documented "wildcards like *.huggingface.co" UX and Cilium's grammar. + +**Fix:** `K7Core._apply_cilium_egress_policy` now translates a leading `*.` +into `**.` (explicit `**.` and mid-label wildcards pass through). Verified +live: the same pull that timed out for 2m40s completes in ~8s. Also set +Cilium `dnsProxy.minTtl=3600` at install: CDN DNS TTLs are 30–60s while +dockerd's blob downloader keeps dialing its cached IP for minutes, so with +`minTtl=0` the learned FQDN→identity mapping can expire mid-download. +Integration coverage: `test_docker_pull_through_fqdn_whitelist`. + +**Reference:** cilium `pkg/fqdn/matchpattern/matchpattern.go` +(`escapeRegexpCharacters`), Cilium docs "DNS based" policies. + +**Time lost:** ~2 h (repro, hubble/ipcache/fqdn-cache spelunking, a wrong +first hypothesis on TTL expiry that the live test disproved). + +## 10. kql-r3 dind IO wedge: single-threaded virtiofsd starves the kata-agent health ping (spec 18g) + +**Symptom:** A kql (kata-qemu-longhorn) sandbox with the docker sidecar, +running the spec-10b `run_io` workload (2k files + 512 MB `dd conv=fsync`) +on an r=3 Longhorn volume, would intermittently (~50% per rep) "wedge": +exec 500s, both containers restarted, `Pod sandbox changed, it will be +killed and re-created`. First seen 2/2 in the spec-18e bench (`run_read` +unmeasurable on r3). + +**Root cause:** Not the guest, not dockerd, not Longhorn. A `dmesg -c` + +`/proc/meminfo` stream running inside the guest right through the death +showed a healthy VM (load 0.4, 1.5 GB free, zero dirty/writeback, no OOM, +no hung tasks) — the last log lines were normal container veth setup. The +containerd log on the node had the smoking gun: floods of +`ttrpc: received message on inactive stream`, then +`failed to ping agent: CheckRequest timed out` → `Dead agent` → +`sandbox stopped unexpectedly` — the **kata shim killed a healthy VM**. +Kata's default virtiofsd runs `--thread-pool-size=1`, so ALL virtio-fs IO +(container rootfs + the Longhorn-PVC `/var/lib/docker`) serializes through +one thread. `docker run` on a vfs-driver dind copies the whole ~790 MB +image rootfs and then fsyncs 512 MB through that single thread against an +r=3 volume (60–80 s saturated). Agent RPCs that touch virtio-fs queue +behind the convoy; the shim's health ping starves and it declares the +agent dead. r≥2 matters only because Longhorn write amplification makes +the convoy long enough to exceed the ping deadline. + +**Misdiagnoses ruled out on the way:** guest memory sizing (MemAvailable +1.8 GB throughout), vCPU count (4-vCPU guest wedged *faster*), Longhorn +backpressure/faults (volume `attached healthy`, no rebuilds), kubelet +exec-probe pressure (relaxing `docker info`/`true` probe timeouts from the +kubelet default 1 s reduced cancelled-ttrpc noise but did NOT stop the +kill). + +**Fix:** playbook now sets +`virtio_fs_extra_args = ["--thread-pool-size=16", "--announce-submounts"]` +in `configuration-qemu.toml` (kata reads it per sandbox start, no restart +needed). Verified on the live 3-node HA cluster: 6/6 run_io reps + +run_read (~45 s) with zero VM restarts on the same r=3 volume. The probe +relaxations in `core.py` were kept as well (less cancelled-exec churn on +the shim↔agent ttrpc channel). + +**Reference:** kata-containers virtiofsd integration (default +`--thread-pool-size=1`), virtiofsd docs on request queueing. + +**Time lost:** ~3 h (bench-faithful repro, guest-side dmesg/meminfo +streaming, two disproven hypotheses, virtiofsd A/B). diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a85ad76 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,44 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.2.0] — 2026-08-11 + +First public release. Ships the CLI/API deb and PyPI `k7-sdk`. +The API image is built on the node by the install playbook; a prebuilt GHCR +image and the apt/PPA story are fast follow-ups. + +### Added + +- **Multiple sandbox backends** on one install / cluster — pick per sandbox + or specialize nodes: + - `kata-firecracker-devmapper` (`kfd`) — Firecracker + jailer + LVM + thin-pool + - `kata-qemu-longhorn` (`kql`) — **QEMU** via Kata + Longhorn PVC root + (named snapshots, restore, disk-only fork) + - `k7d` — Katakate Rust VMM / `runtimeClassName: k7` (warm CoW fork; + install via artifact URL / `--k7d-artifact` until the public + `Katakate/k7d` release is live) +- Multi-node / HA install (Ansible inventory, Longhorn topology) +- Cilium CNI with FQDN egress (`CiliumNetworkPolicy`) +- API + SDK parity for pause / resume / fork +- Snapshot lifecycle + GC CronJob; restore from VolumeSnapshot (`kql`) +- CLI talks to the API by default (`k7 api`, `k7 dev api rebuild`) +- Docker-in-VM sidecar + performance bench harness +- Firecracker jailer integration +- Python SDK published as **`k7-sdk`** (`k7_sdk`; `katakate` deprecated) + +### Changed + +- Node-local ops removed from `K7Core` (API/agent split) + +## [0.0.3] + +- Debian package packaging fixes (GHCR image name casing) + +## [0.0.1] + +- Initial tagged release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b4609e9..d1193ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,39 +1,71 @@ # Contributing to K7 -Thanks for your interest in contributing! +Thanks for your interest in contributing. -We chose a lean and minimal approach to make development on this project as simple as possible: -- Infra is handled by a single Ansible playbook -- The CLI and API are implemented respectively with Typer and FastAPI in Python, for simplicity. They both wrap over the same `core` module. +K7 stays lean on purpose: -## Project Direction +- Infra is a single Ansible playbook (`src/k7/deploy/`) +- CLI (Typer) and API (FastAPI) both wrap the same `core` module +- Tooling: **uv**, **ruff**, **ty**, **pytest** -Check out the [ROADMAP.md](ROADMAP.md) to see planned features, current priorities, and long-term goals. It's a great starting point if you're looking for areas to contribute! +## Project direction -## Repo Layout -- `src/k7/` CLI, core logic, API server -- `src/katakate/` Python SDK (published to PyPI as katakate) -- `src/k7/deploy/` Ansible playbook to install node -- `utils/` helper scripts +See [`ROADMAP.md`](ROADMAP.md) for priorities. Near-term focus is the +public release pipeline (PPA / GHCR / PyPI) after `Katakate/k7d` is +published. + +## Repo layout + +- `src/k7/` — CLI, core, API, Ansible playbook +- `src/k7_sdk/` — Python SDK (PyPI: **`k7-sdk`**) +- `src/katakate/` — deprecated import shim → `k7_sdk` +- `tests/` — unit + integration +- `utils/` — helper scripts +- `docs/BACKENDS.md` — backend comparison (full docs: https://docs.katakate.org) ## Packaging -- The root Python packaging (`setup.py` and `MANIFEST.in`) builds the `katakate` SDK for PyPI only. -- Assets under `src/k7/` (including `src/k7/deploy/*`) are not included in the PyPI package; they are used by the Debian/CLI packaging flow. -## Code Style -- Python: PEP8, explicit types for public APIs, early returns, no inline comments -- Lint/format with Ruff: - - Install: `pip install ruff` - - Check: `ruff check src` - - Format: `ruff format src` +- Root packaging (`setup.py`) builds the **`k7-sdk`** wheel for PyPI. +- CLI / playbook assets ship via the Debian package / install path, not + the PyPI SDK package. -## Building -- CLI deb helpers live in `src/k7/cli/` scripts -- Make targets may be available: `make` to list +## Code style + +- Python: PEP 8, explicit types on public APIs, early returns +- Lint / format with Ruff via uv: + +```bash +uv run ruff check src/ tests/ +uv run ruff format src/ tests/ +# or: make lint +``` + +Typecheck: `make typecheck`. Unit tests: `make test`. + +## Fast CLI iteration (`dev.sh`) + +Avoid `make build && make install` while hacking the CLI: + +```bash +./src/k7/cli/dev.sh --help +./src/k7/cli/dev.sh list +./src/k7/cli/dev.sh create --name test --image alpine:latest +``` + +Same flags as the installed `k7` binary. For playbook / core / API +changes, escalate: `dev.sh` → `make test-integration-remote` → +`make build && make install` on a Linux node. Stack targets Linux x86; +do not deploy or run the full stack on macOS ARM. ## Releases -- Bump versions in `src/k7/__init__.py` and `src/katakate/__init__.py` -- Tag `vX.Y.Z` to build artifacts (CI may publish .deb and wheels) -## Reporting Issues -- Include steps, expected vs actual, logs, and environment (arch/OS/hardware) \ No newline at end of file +- Keep versions aligned in `src/k7/__init__.py`, `src/k7_sdk/__init__.py`, + `setup.py`, `pyproject.toml`, and `debian/changelog` +- Tag `vX.Y.Z` once the public release pipeline is live +- See [`CHANGELOG.md`](CHANGELOG.md) + +## Reporting issues + +Include steps, expected vs actual, logs, and environment (OS, arch, +backend: `kfd` / `kql` / `k7d`, single- vs multi-node). Security reports: +see [`SECURITY.md`](SECURITY.md). diff --git a/LICENSE b/LICENSE index dafd5b5..b8afd51 100644 --- a/LICENSE +++ b/LICENSE @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [2025] [katakate.org] + Copyright 2026 Gary Becigneul Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/MANIFEST.in b/MANIFEST.in index 4e533bb..7240921 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include LICENSE include README.md +recursive-include src/k7/deploy/manifests *.yaml # Exclude common junk global-exclude __pycache__ *.py[cod] *.so *.dylib *.dll .DS_Store .idea .vscode \ No newline at end of file diff --git a/Makefile b/Makefile index 06af5d6..f85c9db 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,13 @@ SUDO := $(shell command -v sudo >/dev/null 2>&1 && [ "$$(id -u)" -ne 0 ] && echo BUILD_SCRIPT := src/k7/cli/build.sh INSTALL_SCRIPT := src/k7/cli/install.sh -.PHONY: help build install uninstall api-build-local api-run-local +# All shell scripts for linting +SH_FILES := $(shell find src/ utils/ rootfs-build/ -name '*.sh' 2>/dev/null) +# Ansible playbooks +ANSIBLE_PLAYBOOKS := src/k7/deploy/k7-install-node.yaml + +.PHONY: help build install uninstall api-build-local \ + lint lint-shell lint-ansible typecheck test test-integration test-integration-remote rsync-all check help: ## Show this help message @echo "Available targets:" @@ -32,7 +38,45 @@ api-build-local: ## Build the API container locally (dev tag) @echo "Building local API image: k7-api:dev" docker build -f src/k7/api/Dockerfile.api -t k7-api:dev . -api-run-local: ## Run API using the local image (no pull) - @echo "Starting API with local image (k7-api:dev)" - docker pull cloudflare/cloudflared:latest || true - K7_API_IMAGE=k7-api K7_API_TAG=dev k7 start-api --yes \ No newline at end of file +# ── Lint ────────────────────────────────────────────────────────── +lint: ## Lint & format-check Python code (ruff) + uv run ruff check src/ tests/ + uv run ruff format --check src/ tests/ + +lint-shell: ## Lint shell scripts (bash -n + shellcheck) + @echo "==> bash -n syntax check" + @for f in $(SH_FILES); do bash -n "$$f" || exit 1; done + @echo "==> shellcheck" + shellcheck -S warning $(SH_FILES) + +lint-ansible: ## Lint Ansible playbooks (ansible-lint) + uv run ansible-lint --profile basic $(ANSIBLE_PLAYBOOKS) + +# ── Typecheck ───────────────────────────────────────────────────── +typecheck: ## Type-check Python code (ty) + uv run ty check src/k7 + +# ── Test ────────────────────────────────────────────────────────── +test: ## Run unit tests (pytest, excludes integration) + uv run pytest + +test-integration: ## Run integration tests (requires live k7 node) + uv run pytest -m integration + +test-integration-remote: ## Run integration tests on remote k7 node via SSH (set K7_NODE_IP) + @echo "Run integration tests on the node itself: rsync this repo there, then 'make test-integration'." >&2; exit 1 + +rsync-all: ## Rsync repo to all nodes (K7_NODE_IPS=ip1,ip2,ip3) + @IFS=',' read -ra IPS <<< "$${K7_NODE_IPS:-$${K7_NODE_IP:?set K7_NODE_IP to your node IP}}"; \ + for ip in "$${IPS[@]}"; do \ + echo "==> Syncing to $${K7_NODE_USER:-root}@$$ip"; \ + rsync -az --delete \ + --exclude .venv --exclude .git --exclude __pycache__ \ + --exclude '*.pyc' --exclude .ruff_cache --exclude .pytest_cache \ + --exclude .mypy_cache --exclude .coverage --exclude uv.lock \ + -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -i $${SSH_PRIVKEY:-$$HOME/.ssh/id_ed25519}" \ + ./ "$${K7_NODE_USER:-root}@$$ip:/root/k7/"; \ + done + +# ── Combined ────────────────────────────────────────────────────── +check: lint lint-shell lint-ansible typecheck test ## Run all lints + typecheck + unit tests \ No newline at end of file diff --git a/PERFORMANCE.md b/PERFORMANCE.md new file mode 100644 index 0000000..52aca40 --- /dev/null +++ b/PERFORMANCE.md @@ -0,0 +1,250 @@ +# Performance + +## Backend lifecycle: kql vs k7d — 2026-08-10 (spec 9a M11) + +Hetzner AX41 dedicated node (Ryzen 5 3600, 64 GiB, NVMe), Ubuntu 24.04, +kernel 6.8.0-137, k3s v1.36.3, flannel CNI, Longhorn 1.10 (r=1), k7d 0.1.0. +Single node, interleaved runs, `alpine:3.20` sandboxes; median of 3 +(sidecar legs n=1). Raw samples in the JSON the bench writes. Reproduce +on a node with both backends installed: + +```bash +K7_BENCH_BACKENDS=kata-qemu-longhorn,k7d K7_BENCH_REPS=3 \ + uv run pytest -m bench tests/integration/bench_backend_lifecycle.py -v -s +``` + +| Operation | kql (kata-qemu-longhorn) | k7d | notes | +|-----------|--------------------------|-----|-------| +| create → pod Ready | 17.11 s (15.05–17.13) | **2.13 s** (2.13–2.16) | cold boot; kql pays Longhorn PVC provision + QEMU boot | +| exec round-trip | **0.04 s** (0.03–0.04) | 0.12 s (0.12–0.13) | k7d exec bridges through vsock | +| named snapshot ready | 6.51 s (2.72–6.52) | n/a | Longhorn-only by design; k7d rejects `k7 snapshot` loudly | +| fork (API call) | 7.78 s (7.77–7.80) | 2.15 s (2.14–2.16) | kql: Longhorn snapshot + PVC clone; k7d: CoW disk+memory `fork_vm` | +| **fork → forked pod Ready + exec** | 46.66 s (46.45–46.67) | **2.37 s** (2.37–2.38) | **~20×**: kql cold-boots a VM on the cloned disk; the k7d fork *inherits the source's live memory* (tmpfs, processes, page cache) | +| pause effective | 1.31 s (1.29–1.90) | **0.20 s** (0.20) | kql: scale-to-0, pods terminated; k7d: vCPUs frozen in place | +| resume → exec answers | 4.11 s (2.10–9.19) | **0.34 s** (0.33–0.38) | kql: reschedules pod + VM boot; k7d: restart vCPU loop, memory intact | +| delete | 0.05 s | 0.04 s | | +| docker sidecar: create → `docker info` | 21.11 s | **8.60 s** | dind in the same VM (k7d) / pod (kql) | +| docker sidecar: `docker pull alpine:3.21` | **2.35 s** | 8.21 s | **kql wins**: its dind unpacks onto the Longhorn ext4 volume, k7d's dind data dir sits on guest tmpfs behind virtio + NAT — pull streaming is the k7d sidecar's slow path today | +| docker sidecar: `docker run --rm alpine echo` | 0.91 s | **0.33 s** | | + +Honest summary: k7d dominates every lifecycle operation (create ~8×, +fork-to-usable ~20×, pause ~6×, resume ~12×) and is the only backend whose +fork carries **memory state** — the forked sandbox resumes mid-thought +instead of cold-booting. kql keeps two real advantages: named snapshots +that persist after the sandbox dies (restore later, GC, cross-pod +persistence) and faster registry pulls inside the docker sidecar. Plain +exec is also ~3× faster on kql (80 ms absolute difference; both are +interactive-fast). + +## Kata-qemu-longhorn backend baseline (2026-04-10) + +Observed latencies on a single Hetzner dedicated node (3x NVMe, Ubuntu 24.04, Longhorn replicas=1). + +| Operation | Latency | Limit | +|-----------|---------|-------| +| Cold create to pod ready | 15.20s | — | +| Snapshot ready | 6.98s | < 60s | +| Pause (scale to 0) | 0.13s | — | +| Resume (scale to 1 + pod ready) | 4.54s | < 60s | +| **Fork (total)** | **44.85s** | < 120s | +| — snapshot | 2.97s | | +| — clone PVC bound | 2.71s | | +| — deployment ready (pod + VM boot) | ~39.17s | | + +Fork is ~3x slower than a cold create. The snapshot and clone PVC steps add ~6s, but the main cost is the forked deployment's VM boot (~39s vs ~15s for a fresh PVC) — likely due to Longhorn replaying cloned data on first attach. + +Measured via integration tests (`tests/integration/test_qemu.py`) on 2026-04-10. + +## Docker workloads inside sandboxes — 2026-05-29 (spec 10b) + +Hetzner AX52 dedicated node (Ryzen 7 7700, 64 GiB, NVMe), Ubuntu 24.04, +kernel 6.8.0-100, k3s v1.35.5, docker host 29.5.2 / sandbox 27.5.1, +Longhorn 1.10. 3 runs per cell after one warm-up; median (range in parens). +⚠ marks cells whose (max−min)/median > 0.30 (range noise gate). + +Workload is `bench/docker-perf/bench.Dockerfile`: pull `debian:12-slim`, a +no-cache build that does `apt install build-essential python3 git ca-certificates`, +`pip install numpy pandas requests pytest httpx pydantic`, a 256 MB +`dd ... conv=fsync`, and `pip check`. Run phase exercises a 10-s CPU loop, +a 2 000-small-files + 512 MB-fsync IO workload, and a cat-the-venv-tree +read workload. Reproduce with: + +```bash +K7_BENCH_ENVS=host,k7-fd,k7-ql-r1,k7-ql-r2 K7_BENCH_REPS=3 \ + K7_BENCH_OUT=/tmp/bench-out \ + uv run pytest -m bench tests/integration/bench_docker_perf.py +uv run python bench/docker-perf/render.py -i /tmp/bench-out/bench-results-*.csv \ + --title "Docker workloads ..." --hardware "" +``` + +| Operation | host | k7-fd | k7-ql-r1 | k7-ql-r2 | +|---------------------------|------|-------|----------|----------| +| pull debian:12-slim | 3.06 s (3.00 s–3.06 s) | 2.78 s (2.77 s–2.83 s) | 3.66 s (3.62 s–3.67 s) | 3.64 s (3.62 s–3.81 s) | +| build (no-cache) | 53.2 s (53.2 s–53.8 s) | 26.0 s (25.0 s–30.2 s) | 72.3 s (72.0 s–72.8 s) | 68.7 s (68.6 s–71.6 s) | +| build (cached) | ⚠ 714 ms (710 ms–936 ms) | 250 ms (250 ms–291 ms) | ⚠ 690 ms (469 ms–4.49 s) | ⚠ 448 ms (436 ms–663 ms) | +| run cpu (10s budget) | 10.6 s (10.5 s–10.6 s) | 10.2 s (10.2 s–10.2 s) | 27.6 s (26.0 s–29.7 s) | 27.3 s (25.9 s–29.1 s) | +| run io (2k small + 512 MB)| ⚠ 1.14 s (1.08 s–1.68 s) | 485 ms (475 ms–593 ms) | 37.3 s (37.2 s–37.7 s) | 37.4 s (36.9 s–37.6 s) | +| run read (venv tree cat) | 992 ms (989 ms–1.02 s) | 225 ms (213 ms–235 ms) | 16.4 s (16.3 s–16.9 s) | 16.4 s (16.4 s–16.9 s) | + +Ratios to host (the number that matters for the sandbox tax story): + +| Cell | k7-fd / host | k7-ql-r1 / host | k7-ql-r2 / host | +|---------------------------|--------------|-----------------|-----------------| +| build (no-cache) ratio | 0.49× | 1.36× | 1.29× | +| run io ratio | 0.42× | 32.6× | 32.7× | +| run cpu ratio | 0.96× | 2.62× | 2.59× | + +### What this says + +**k7-fd (Firecracker, docker daemon on emptyDir) is *faster* than the +host** on every operation that touches disk — `build (no-cache)` 0.49×, +`run io` 0.42×, `run read` 0.23×. CPU is host-parity (0.96×). + +Why faster, when the FD path strictly has more layers (Firecracker VM + +kata virtio-fs sharing the emptyDir from `/var/lib/kubelet/.../sidecar-data/` +into the guest, vs the host docker daemon going straight to ext4 on +`/dev/md2`)? Four contributing factors, in roughly decreasing order of +size: + +1. **Storage driver delta.** The host docker daemon picks `overlayfs` + (the legacy single-layer overlay implementation that ships with the + host kernel). The FD sandbox's docker daemon picks `overlay2`. On + metadata-heavy workloads — apt extracts ~5 600 files, pip+venv + extracts another few thousand — overlay2 is significantly faster + than overlayfs (different layer-handling code path, better + d_type/userxattr behaviour). The whole "build (no-cache)" column is + apt+pip metadata churn, which is exactly where this difference bites. +2. **Clean cache state.** The FD sandbox starts with an empty + `/var/lib/docker`. The host docker daemon has accumulated state from + prior `k7 install` builds, k3s-imported images, and the bench's own + warm-up — manifest lookups, dangling layer GC, layer dedup all run + on a populated tree. +3. **virtio-fs writeback caching.** Kata-fc shares the emptyDir into + the guest via virtio-fs (the default for shared filesystem mounts; + the alternative virtio-blk would require an explicit pod annotation + for "direct block device" which we don't set). virtio-fs runs a + writeback cache in virtiofsd on the host. Inside the guest VM, + `dd if=/dev/zero of=/tmp/x bs=1M count=256 conv=fsync` measured + 1.3 GB/s — the fsync completes when virtiofsd acks, not necessarily + when the data is durable on NVMe. This is a weaker fsync than the + host gets directly on ext4. It's the same trade-off `cache=writeback` + gets you in qemu/9pfs setups: faster, less crash-safe. +4. **Devmapper snapshotter on the VM rootfs.** The Firecracker VM's + rootfs is a thin-provisioned LVM volume (containerd devmapper + snapshotter). The `dd` and IO ops the bench runs go through the + sandbox container's rootfs (devmapper) when writing inside the + docker daemon's overlay2 upperdir, which itself is on the emptyDir + (virtio-fs). LVM thin pools are read-cached aggressively at the + page-cache layer — read-heavy ops like `run read` (cat the venv + tree) benefit from page cache hits on the *host* even when the + guest thinks it's doing fresh reads. + +**Net of all of this:** k7-fd's 0.4–0.5× ratios *do* reflect a real +performance win for build/dev workloads on this hardware, but it would +be wrong to attribute the win to "VM is faster than bare metal". +Honest framing: *host docker is using a slow storage driver against a +populated daemon, and the FD sandbox is using a fast storage driver +against a clean daemon, with virtio-fs writeback caching softening the +guest's fsync semantics*. A cleaner future bench would (a) run host +docker with `--storage-driver=overlay2` and a fresh `/var/lib/docker`, +(b) note the virtio-fs cache mode explicitly, and (c) report the +device-level bandwidth so the absolute numbers are anchorable. + +**k7-ql (qemu, docker daemon on a Longhorn PVC sub_path) pays a steep +storage tax** — `build (no-cache)` 1.3×, `run cpu` 2.6×, `run io` ~33×, +`run read` ~16×. Two things stacked here: (1) docker auto-picks the **vfs** +storage driver inside the qemu VM because Longhorn's iSCSI-attached block +device doesn't expose the filesystem features overlay2 wants; vfs copies +entire layer trees on every operation, so cached-build and image-layer +work get hit hard, and (2) reads/writes go disk → iSCSI → qemu virtio → +guest, where the host went directly to the page cache. The `run io` +column (~37 s for 2 000 small files plus a 512 MB fsync) is the worst-case +shape — pure disk-bound work with no compute. + +**r=2 vs r=1 is statistically indistinguishable** (build 68.7 s vs 72.3 s, +io 37.4 s vs 37.3 s). The second Longhorn replica adds one cross-node +sync but the dominant cost on this hardware is vfs inside the guest, not +the replica copy on the wire. Useful negative result — picking r=2 for +durability does not double the cost on these workloads. + +### Known confounds + +- **Storage driver mismatch (the big one).** Host uses overlayfs; k7-fd + uses overlay2 on a virtio-fs-shared emptyDir; k7-ql uses vfs. Every + cross-env comparison is also a "different docker storage driver" + comparison. We cannot cleanly isolate "VM tax" from "driver tax" + without rebuilding the docker daemon image inside the guest with a + matching driver, which the spec deliberately doesn't attempt. +- **Asymmetric daemon state.** Host docker has accumulated images, + layers, and dangling refs from prior `k7 install` runs. Each sandbox + starts with a fresh daemon. We're partly measuring "warm vs cold + daemon". +- **virtio-fs writeback caching changes fsync semantics.** Guest fsync + acks when virtiofsd has the data, not when NVMe has it. Compare the + k7-fd dd fsync (1.3 GB/s) with the host's bare dd fsync (not + benchmarked here — would need a follow-up rep). +- **Build noise.** `build (cached)` is sub-second everywhere and lands + in the noise floor; the ⚠ flags on host and k7-ql-r1 are real + variance (4.5 s outlier on k7-ql-r1 rep 3) but not a signal about the + backend — they're "this op is too fast to time meaningfully with 3 reps." +- **Same physical disk for everything.** k7-fd's `/var/lib/docker` + emptyDir (which lives as a regular directory under + `/var/lib/kubelet/pods/.../volumes/kubernetes.io~empty-dir/` on the + host ext4 root), k7-ql's Longhorn PV, and the host's docker root all + live on the same `/dev/md2` (RAID1 NVMe pair). We're not measuring + cross-disk effects. Note: emptyDir without `medium: Memory` is *not* + tmpfs — it's a plain directory on the kubelet root FS. +- **Single-node sample.** Bench was driven on `k7-node-01` only. r=2's + cross-node sync went to `k7-node-02` but the workload pod stayed on + the primary. A future spec could pin pods to different nodes to also + exercise the read-from-remote-replica path. + +Raw per-leg logs and the aggregated CSV are under `bench/docker-perf/results/` +(gitignored — keep them in agent or local scratch space, paste into the +table above when adding a new run). + +## Docker-in-VM under real Longhorn replica counts (2026-08, spec 18e/18h) + +The "r=2 vs r=1 indistinguishable" result above is **invalid**: the old +bench patched a Longhorn *setting* that only affects newly-created volumes, +so both legs actually ran r=1 (CHALLENGES.md #2). `bench_docker_perf.py` now +sets real per-volume replica counts (`k7-ql-r3` leg). Host / kfd / r1 / r2 +medians from the 18e HA run (5 reps); **k7-ql-r3 column re-filled in +spec 18h** after the virtiofsd wedge fix (5 reps, zero VM restarts): + +| op | host | k7-fd | k7-ql-r1 | k7-ql-r2 | k7-ql-r3 | +|---|---|---|---|---|---| +| build no-cache | 76.4s | 52.5s | 108.2s | 270.9s | 292.5s | +| run io (2k files + 512MB fsync) | 1.43s | 1.02s | 41.4s | 68.4s | 88.3s | +| run read (venv tree cat) | — | — | — | — | 47.7s | +| run cpu (10s budget) | 10.5s | 10.4s | 41.5s | 47.8s | 56.4s | + +Two takeaways (spec 18f issue 8 / 18h): + +- **The r1→r2 jump dominates the redundancy cost** (build 108→271s; r2→r3 + adds only ~8% on build): the second replica forces synchronous + cross-node writes, the third mostly parallelizes with them. Post-wedge + r3 `run io` (88.3s) is higher than the old single-rep 64.5s sample — + that sample was the lucky survivor of a ~50% kill rate, not a median. +- **kql-r3 IO wedge — root-caused and FIXED (spec 18g):** the "VM exec + path dies after a run_io rep" wedge was not guest memory, not dockerd, + and not Longhorn faulting — the guest was healthy (load 0.4, 1.5 GB + free, zero dirty pages, clean dmesg) at the moment of death. The killer + was the **kata shim**: kata's default virtiofsd runs with + `--thread-pool-size=1`, so all virtio-fs IO (container rootfs + the + Longhorn-PVC-backed `/var/lib/docker`) serializes through one thread. + A `docker run` of the ~790 MB bench image makes vfs copy the whole + rootfs and then fsync 512 MB through that single thread against an + r=3 volume (~60–80 s saturated); any agent RPC touching virtio-fs + blocks behind it, the shim's agent health ping (`CheckRequest`) times + out, and the shim declares "Dead agent" and kills the healthy VM + (`sandbox stopped unexpectedly`, pod sandbox recreated). Repro rate was + ~50% per run_io rep. Fix: `k7 install` now sets + `virtio_fs_extra_args = ["--thread-pool-size=16", ...]` in the + kata-qemu config — 18h re-ran 5/5 `run_io` + 5/5 `run_read` with zero + VM restarts on the same r=3 volume (`run_read` median 47.7s). + Exec-probe pressure was reduced too (kubelet's default 1 s exec-probe + timeout sprayed cancelled ttrpc execs — `docker info` legitimately + takes >1 s while dockerd copies vfs layers), which cuts the + `ttrpc: received message on inactive stream` noise but was NOT + sufficient on its own. diff --git a/README.md b/README.md index ae32012..a83a1a8 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,7 @@ +

k7

+

- - KATAKATE - -

- - -

- Self-hosted secure VM sandboxes for AI compute at scale + Self-hosted secure VM sandboxes for AI compute at scale

@@ -21,16 +16,13 @@

- Show HN #1 + Show HN #1 📸 - Featured on Console.dev + Featured on Console.dev 📸 - - Featured on Changelog - GitHub Trending (Oct 23, 2025) @@ -48,7 +40,7 @@ - +

@@ -61,12 +53,12 @@ -Katakate aims to make it easy to create, manage and orchestrate lightweight safe VM sandboxes for executing untrusted code, at scale. It is built on battle-tested VM isolation with Kata, Firecracker and Kubernetes. It is orignally motivated by AI agents that need to run arbitrary code at scale but it is also great for: +Katakate aims to make it easy to create, manage and orchestrate lightweight safe VM sandboxes for executing untrusted code, at scale. It is built on battle-tested VM isolation with Kata, Firecracker, QEMU, Longhorn, and Kubernetes — plus Katakate's own k7d runtime. It is orignally motivated by AI agents that need to run arbitrary code at scale but it is also great for: - Custom serverless (like AWS Fargate, but yours) - Hardened CI/CD runners (no Docker-in-Docker risks) - Blockchain execution layers for AI dApps -> 100% open‑source (Apache‑2.0). For technical support, write us at: hi@katakate.org +> 100% open‑source (Apache‑2.0). For technical support, write us at: hi@katakate.org

The Tech Stack @@ -75,21 +67,47 @@ The Tech Stack Katakate is built on: - Kubernetes for orchestration, with K3s which is prod-ready and a great choice for edge nodes, - Kata to encapsulate containers into light-weight virtual-machines, -- Firecracker as the chosen VM, for super-fast boots, light footprints and minimal attack surface, -- Devmapper Snapshotter with thin-pool provisioning of logical volumes for efficient use of disk space shared by dozens of VMs per node. - +- Firecracker (`kfd`) for super-fast boots, light footprints and minimal attack surface (with the jailer), +- Devmapper Snapshotter with thin-pool provisioning of logical volumes for efficient disk use across many Firecracker VMs per node, +- QEMU (`kql`) via Kata when you want a fuller VMM and durable sandbox disks, +- Longhorn for replicated PVC-backed root disks on the QEMU path — named snapshots, restore, disk-only fork, and cross-node mobility, +- k7d — Katakate's own microVM runtime daemon (katakate/k7d) with VM-level warm fork (CoW disk+memory) and in-place pause/resume.

-Coming Soon +Sandbox backends

+`k7 install --backend ` provisions one or more backends per node; `k7 create --backend …` picks one per sandbox. See [docs/BACKENDS.md](docs/BACKENDS.md) for the architecture and [PERFORMANCE.md](PERFORMANCE.md) for the full measurements (Hetzner AX41 node, medians). -- 🛠️ Docker build / run / compose support inside the VM sandbox -- 🌐 Multi-node cluster capabilities for distributed workloads -- 🔍 Cilium FQDN-based DNS resolution to safely whitelist domains, not just IP blocks -- ⚙️ Support other VMM such as Qemu for GPU workloads +| | `kfd` (kata-firecracker-devmapper) | `kql` (kata-qemu-longhorn) | `k7d` | +|---|---|---|---| +| VMM | Firecracker (Kata) | QEMU (Kata) | k7d (custom KVM VMM) | +| RuntimeClass | `kata` | `kata-qemu` | `k7` | +| Sandbox storage | devmapper thin-pool (needs a spare raw disk) | Longhorn PVC (replicated, persistent) | erofs images + reflink XFS + guest tmpfs | +| Create → Ready* | not re-measured† | 17.1s | **2.1s** | +| Named snapshot* | — | 6.5s (Longhorn, disk-only) | — (VM snapshot trees via the k7d API) | +| Fork → usable* | — | 46.7s (disk clone + cold boot) | **~5 ms VM CoW fork**; **~2.4 s** end-to-end via k7/k8s (pod Ready + exec) | +| Pause / resume* | scale to 0 / 1 | 1.3s / 4.1s (disk survives) | **0.2s / 0.3s (VM frozen in place, memory survives)** | +| Docker-in-VM sidecar | ✅ (ephemeral docker data) | ✅ (persistent docker data; fastest `docker pull`) | ✅ (VM-lifetime docker data) | +| Cross-pod persistence | ✗ | ✅ snapshots/restore | ✗ (fork carries state instead) | -📋 **See [ROADMAP.md](ROADMAP.md) for the complete feature roadmap and project priorities.** +\* medians of 3 on one Hetzner AX41 node — methodology, ranges, and the docker-sidecar +numbers are in [PERFORMANCE.md](PERFORMANCE.md). +† kfd needs a spare raw disk the benchmark node didn't have; its docker-workload numbers +are in the [PERFORMANCE.md](PERFORMANCE.md) spec-10b section. + +

+Also available today +

+ +- 🛠️ Docker build / run inside VM sandboxes (docker sidecar on kfd, kql, and k7d; see [PERFORMANCE.md](PERFORMANCE.md)) +- ⚡ Warm VM fork on the k7d backend: k7 fork CoW-clones a running sandbox's disk and memory in ~5 ms at the VMM; end-to-end through k7/Kubernetes is ~2 s to a Ready pod +- 🌐 Multi-node clusters (Ansible + Longhorn) +- 🔍 Cilium CNI with FQDN egress policies +- 📸 Pause / resume / fork / restore and k7 snapshot lifecycle +- 🐍 Python SDK: pip install k7-sdk (katakate package deprecated) + +📋 **See [ROADMAP.md](ROADMAP.md) for upcoming work (GPU passthrough, …).**

@@ -106,14 +124,17 @@ For usage you need: We provide a: - **CLI**: to use on the node(s) directly --> `apt install k7` -- **API**: deployed on the (master) node(s) --> `k7 start-api` -- **Python SDK**: Python client sync/async talking to API --> `pip install katakate` +- **API**: deployed automatically by `k7 install` (toggle with `k7 api enable` / `k7 api disable`) +- **Python SDK**: HTTP client sync/async --> `pip install k7-sdk` ## Current requirements ### For the node(s) - Ubuntu (amd64 or arm64) host. + - **`k7d` backend is amd64 / x86_64 only** (same ISA; Debian calls it `amd64`, + the release tarball is `*-x86_64-linux.tar.gz`). `kfd` and `kql` support + amd64 and arm64. - Hardware virtualization (KVM) available and accessible - Check: `ls /dev/kvm` should exist. - This is typically available on your own Linux machine. @@ -138,11 +159,22 @@ We provide a: ``` Already tested setups: - - Hetzner Robot instance with Ubuntu 24.04, x86_64 or ARM64 arch, booked with 1 extra empty disk `nvme2n1` for the thin-pool provisioning. See the setup guide (PDF): [tutorials/k7_hetzner_node_setup.pdf](tutorials/k7_hetzner_node_setup.pdf). + - Hetzner Robot dedicated with Ubuntu 24.04 and a **spare raw NVMe** for the `kfd` thin-pool. Dual-NVMe boxes (no third drive): install the OS on one disk only — see [tutorials/k7_hetzner_node_setup.md](tutorials/k7_hetzner_node_setup.md). (Older PDF that assumed an add-on third NVMe: [tutorials/k7_hetzner_node_setup.pdf](tutorials/k7_hetzner_node_setup.pdf).) ### For the client -Just recent Python. +Recent Python, or the **`k7`** CLI / **`k7-sdk`** from a Linux node or your laptop (API URL + key). + +#### Development on macOS + +The **`.deb` / PPA package is Linux-only** (amd64/arm64). On a MacBook: + +- **CLI from source:** `./src/k7/cli/dev.sh` (same commands as `k7`; uses `uv` + `PYTHONPATH=src`) +- **API client from laptop:** set `K7_API_URL` and `K7_API_KEY`, then `dev.sh create` / `dev.sh list` (no `--core`) +- **`k7 install`** targets Linux servers with KVM — run on the node or via SSH, not on macOS locally +- **`pip install k7-sdk`** for Python scripts only + +Do not install the Ubuntu `.deb` on macOS. ## Quick Start @@ -168,20 +200,20 @@ Current task: Reminder about logging out and back in for group changes Optionally pass `-v` for a verbose output. -> It will also tell you which raw disk was auto-selected for the LVM thin-pool. If you prefer, specify the disk explicitly: +> It will also tell you which raw disk was auto-selected for the LVM thin-pool. If you prefer, specify the disk explicitly (on a dual-NVMe Hetzner box this is usually the spare, e.g. `/dev/nvme1n1`): > ```bash -> k7 install --disk /dev/nvme2n1 +> k7 install --disk /dev/nvme1n1 > ``` -This will install and most importantly connect together the following components: +This will install and most importantly connect together the following components (depending on `--backend`): - Kubernetes (K3s prod-ready distribution) - Kata (for container virtualization) -- Firecracker (as Virtual Machine Manager) -- Jailer (to secure Firecracker VMs further into a chroot) -- devmapper snapshotter with thin-pool provisioning of logical volumes for VM efficient disk memory usage +- Firecracker + Jailer + devmapper thin-pool (`kfd`) +- QEMU via Kata + Longhorn PVC-backed roots (`kql`) +- k7d daemon + `containerd-shim-k7-v1` + RuntimeClass `k7` (`k7d`) -Careful design: config updates will not touch your existing Docker or containerd setups. We chose to use K3s' own containerd for minimal disruption. Installation may however overwrite existing installations of K3s, Kata, Firecracker, Jailer. +Careful design: config updates will not touch your existing Docker or containerd setups. We chose to use K3s' own containerd for minimal disruption. Installation may however overwrite existing installations of K3s, Kata, Firecracker, Jailer, QEMU/Kata config, or Longhorn. ### CLI Usage @@ -220,6 +252,9 @@ env_file: path/to/your/secrets/.env # Create a sandbox (uses k7.yaml in the current directory by default, but you can also pass: -f myfile.yaml) k7 create +# Or pick a backend explicitly (kfd | kql | k7d — aliases for the full names) +k7 create -f k7.yaml --backend k7d + # List sandboxes k7 list @@ -230,21 +265,59 @@ k7 delete my-sandbox-123 k7 delete-all ``` +#### Fork / pause / snapshot + +```bash +# Warm CoW fork (disk + memory) — source must be a k7d sandbox +k7 create -f k7.yaml --backend k7d # name from yaml, e.g. my-sandbox-123 +k7 exec my-sandbox-123 sh -c 'echo hi > /tmp/state.txt' +k7 fork my-sandbox-123 branch-a +k7 exec branch-a cat /tmp/state.txt # inherited memory + disk + +# Disk-only fork (cold boot from cloned PVC) — kql / kata-qemu-longhorn +k7 create -f k7.yaml --backend kql +k7 fork my-sandbox-123 branch-b +# optional: pin the Longhorn VolumeSnapshot name used for the clone +k7 fork my-sandbox-123 branch-c --snapshot my-snap + +# Parallel branches from one base +for i in $(seq 0 7); do k7 fork my-sandbox-123 exp-$i & done; wait + +# Pause / resume (kql keeps the PVC; k7d freezes the live VM) +k7 pause my-sandbox-123 +k7 resume my-sandbox-123 + +# Named disk snapshot without pausing (kql) +k7 snapshot create my-sandbox-123 my-named-snap +``` + +On **k7d**, the VMM fork itself is ~5 ms; end-to-end through Kubernetes +to a Ready pod is ~2 s. On **kql**, fork is a Longhorn snapshot + PVC +clone + cold boot (~45 s). See [PERFORMANCE.md](PERFORMANCE.md) and +[docs/BACKENDS.md](docs/BACKENDS.md). + ### API usage -If you'd like to manage workloads remotely, just use the API: +The K7 API is deployed automatically by `k7 install` as the `k7-api` +Deployment in `kube-system`. K3s keeps it running on its own; there's no +separate "start" step. ```shell -# Start API server (containerized and SSL support with Cloudflared) -k7 start-api +# Check status + endpoint +k7 api status +k7 api endpoint # Generate API key k7 generate-api-key my-key1 + +# Temporarily disable / re-enable +k7 api disable +k7 api enable ``` -Make sure your user is in the `Docker` group to be allowed to start or stop the API. +Generating / listing / revoking keys talks to `/etc/k7/api_keys.json`, so +those subcommands need to run on the node (typically `sudo` or `root`). -As for generating / listing / revoking keys, you might need `sudo` or `root`. ### Python SDK Usage @@ -252,43 +325,51 @@ After your k7 API is up, usage is very simple. Install the Python SDK via: ```shell -pip install katakate +pip install k7-sdk ``` Or if you want async support: ```shell -pip install "katakate[async-sdk]" +pip install "k7-sdk[async]" ``` +The legacy `katakate` PyPI name remains as a one-release shim that re-exports `k7_sdk` with a deprecation warning. + Then use with: ```python -from katakate import Client +from k7_sdk import Client k7 = Client( endpoint='https://', api_key='your-key') -# Create sandbox +# Create sandbox (pick backend: kata-firecracker-devmapper | kata-qemu-longhorn | k7d) sb = k7.create({ - "name": "my-sandbox", - "image": "alpine:latest" + "name": "base", + "image": "alpine:latest", + "backend": "k7d", }) # Execute code -result = sb.exec('echo "Hello World"') +result = sb.exec('echo "Hello World" > /tmp/hi.txt && cat /tmp/hi.txt') print(result['stdout']) -# List all sandboxes -sandboxes = k7.list() +# Fork: k7d = warm CoW (disk + memory); kql = disk clone + cold boot +branch = sb.fork("branch-a") +print(branch.exec("cat /tmp/hi.txt")["stdout"]) # still there on k7d -# Delete sandbox +# Parallel exploration +forks = [sb.fork(f"exp-{i}") for i in range(4)] + +# List / delete +sandboxes = k7.list() sb.delete() ``` #### Async variant ```python import asyncio -from katakate import AsyncClient +from k7_sdk import AsyncClient async def main(): k7 = AsyncClient( @@ -338,26 +419,6 @@ sudo make uninstall Note: we recommend running `make uninstall` before reinstalling if it is not your first install, to avoid stale copies of cached files in the .deb package. -### Fast development workflow - -For faster development iterations when working on the CLI, you can use `dev.sh` to run `k7` commands directly without rebuilding the binary: - -```shell -# Basic commands -./src/k7/cli/dev.sh install -./src/k7/cli/dev.sh list -./src/k7/cli/dev.sh create - -# Install with options -./src/k7/cli/dev.sh install -v -./src/k7/cli/dev.sh install --disk /dev/nvme2n1 - - -# Create sandbox with options -./src/k7/cli/dev.sh create --name test --image alpine:latest -``` - -This script uses `uv run` to execute the CLI with all dependencies, so you can test changes immediately without running `make build` every time. This is especially useful when iterating on CLI code changes. ### Build and run the API container @@ -371,7 +432,7 @@ make api-run-local ``` -### Build the katakate Python SDK from source +### Build the k7-sdk Python SDK from source Preferred (uv): @@ -389,9 +450,10 @@ uv pip install -e . K7 sandboxes are hardened by default with multiple layers of security: -- **VM isolation**: Kata Containers provide hardware-level isolation via lightweight VMs with Firecracker - - VMs are further restricted into a chroot using Jailer - - Kata's Seccomp restrictions are enabled +- **VM isolation**: Kata Containers (Firecracker or QEMU) or the k7d RuntimeClass provide hardware-level isolation via lightweight VMs + - On `kfd`, Firecracker processes are further restricted into a chroot using the Jailer + - Kata's Seccomp restrictions are enabled on the Kata backends + - `kql` uses QEMU + Longhorn for durable, cross-node-mobile disks; `k7d` has its own CoW-fork isolation trade-offs (see k7d `SECURITY.md`) - **Linux capabilities**: All capabilities are dropped by default (`drop: ALL`) for defense-in-depth - Only explicitly add back capabilities you need via `cap_add` parameter @@ -409,26 +471,18 @@ K7 sandboxes are hardened by default with multiple layers of security: - **Network policies**: Complete network isolation for VM sandboxes - **Ingress isolation**: All inter-VM communication is blocked by default to prevent sandbox-to-sandbox access - - **Egress lockdown**: Control outbound traffic with CIDR-based restrictions using Kubernetes NetworkPolicies - - **DNS is blocked** when egress is locked down; only IPs/CIDRs in `egress_whitelist` are reachable + - **Egress lockdown**: per-sandbox allowlists — CIDRs via Kubernetes NetworkPolicy, or **FQDN / domain** allowlists via Cilium (`CiliumNetworkPolicy`; default CNI) + - **DNS is blocked** when egress is locked down; only entries in `egress_whitelist` (CIDR or domain) are reachable - Administrative access via `kubectl exec` and `k7 shell` is preserved (uses Kubernetes API, not pod networking) - - Soon to come: Cilium integration for domain name whitelisting -More security features are currently on the roadmap, including integrating AppArmor. +More security features are on the roadmap (e.g. AppArmor). ## Packaging & Releases - Layout uses `src/`: - CLI, API, core live under `src/k7/` - - SDK under `src/katakate/` -- Root packaging targets the `katakate` SDK only; assets under `src/k7/` are not part of the PyPI distribution. -- `MANIFEST.in` (for the `katakate` SDK) should include essentials like `LICENSE` and `README.md` only; deploy assets from `src/k7/deploy/*` belong to the Debian/CLI packaging flow, not to the PyPI package. -- `setup.py` for `katakate` lives at repo root; packages from `src/`. + - SDK under `src/k7_sdk/` (PyPI package `k7-sdk`; `src/katakate/` is a deprecation shim) +- Root `setup.py` publishes the SDK; assets under `src/k7/` belong to the Debian CLI / API image, not the PyPI wheel. +- User docs: `~/docs/k7/` (Mintlify). See `docs/README.md` in this repo. - The CLI Debian package is built via `src/k7/cli/build.sh` and produces `dist/k7__amd64.deb` and `dist/k7__arm64.deb`. -- CI (tags `v*`) can publish the PyPI SDK and upload the `.deb` artifact. - - - -## Known issues - -- Jailer seems to be currently ignored by Kata despite being passed correctly into its configuration, and despite the Jailer process being started. The use of Kubernetes secrets could be a reason of incompatibility. This is under investigation. \ No newline at end of file +- CI (tags `v*`) can publish the PyPI SDK and upload the `.deb` artifact. \ No newline at end of file diff --git a/ROADMAP.md b/ROADMAP.md index 1f57bc3..ed847c1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,64 +1,57 @@ -# 🗺️ Project Roadmap +# Project Roadmap -This document outlines the upcoming milestones, goals, and long-term vision for **K7**. -It helps contributors and users understand where the project is heading. +Where **K7** is headed — for contributors and operators. --- -## 🚀 Current Focus - -Core stability and foundational runtime improvements. - -- [x] Add `--disk` argument to `k7 install` to specify external disk path explicitly for thin pool provisioning, and test it (merged PR #5) -- [x] Test if removing DNS resolution completely doesn't break functionality (to protect against DNS exfiltration) (merged PR #6) -- [ ] Add pause/resume/fork/clone support for sandboxes -- [ ] Fix jailer functionality (known issue) -- [ ] Add multi-node support (currently single K3s node supported) - +## Current focus +Release engineering: apt/PPA, GHCR `k7-api`, and PyPI `k7-sdk`, after the +sibling [`Katakate/k7d`](https://github.com/Katakate/k7d) v0.1.0 artifact +exists (default install URL depends on it). --- -## 🧩 Next Goals - -Broader compatibility and container integration. - -- [x] Add ARM support for Linux Debian (merged PR #4, big thanks to @spullara) -- [ ] Add Docker build / run / compose capabilities in VM sandboxes (major feature!) -- [ ] Integrate Cilium networking -- [ ] Implement Docker pull deny/whitelist +## Recently shipped +- [x] Multi-node Ansible, Longhorn topology, HA / cross-node tests +- [x] Cilium CNI + FQDN egress +- [x] API + SDK parity: pause / resume / fork +- [x] Snapshot lifecycle + GC; restore from VolumeSnapshot +- [x] CLI → API by default; `k7 api` / `k7 dev api rebuild` +- [x] Docker-in-VM sidecar + bench harness +- [x] Firecracker jailer +- [x] Python SDK as **`k7-sdk`** / `k7_sdk` (`katakate` deprecated) +- [x] `k7d` backend install path (artifact URL / local override) --- -## ⚙️ Future Work +## Next goals -Cross-platform support and continuous delivery. - -- [ ] Add QEMU support (macOS ARM, GPU support) -- [ ] Cross-node mobility of snapshots (dependent on multi-node + sandbox snapshot/resume/fork features) -- [ ] Add AppArmor integration -- [ ] Add CI/CD and deployment tests +- [ ] PPA (`apt install k7`), GHCR `k7-api`, PyPI `k7-sdk` +- [ ] Default `k7d` install from public `Katakate/k7d` GitHub Releases +- [ ] Optional macOS CLI artifacts (tarball / Homebrew) — after the above --- -## 🔐 Advanced Features +## Future work -Security, customization, and extended runtime capabilities. - -- [ ] Add TEE (Trusted Execution Environment) support -- [ ] Add custom rootfs support (lighter, alternative images) +- [ ] GPU passthrough support +- [ ] Cross-node mobility of snapshots / forks for the `k7d` backend + (`kql` already moves state across nodes via Longhorn) +- [ ] AppArmor integration +- [ ] CI/CD deployment tests on every public tag +- [ ] TEE support; custom rootfs; persistent in-API interpreter --- -## 💬 How to Contribute +## How to contribute -We welcome ideas and feedback! -If you'd like to suggest a feature or help with one listed above: -1. Open a [Discussion](https://github.com/katakate/k7/discussions) or [Issue](https://github.com/katakate/k7/issues) -2. Reference the relevant roadmap item -3. Let's collaborate on the design or implementation +1. Open a [Discussion](https://github.com/Katakate/k7/discussions) or + [Issue](https://github.com/Katakate/k7/issues) +2. Reference the roadmap item +3. See [`CONTRIBUTING.md`](CONTRIBUTING.md) --- -📅 *Last updated: October 2025* +*Last updated: August 2026* diff --git a/SECURITY.md b/SECURITY.md index de96555..3302bf0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,39 +2,69 @@ ## Supported Versions -This project is pre-1.0 (currently 0.0.1) and under active development and security hardening. Breaking changes may occur between minor versions until 1.0.0. +This project is pre-1.0 (targeting **0.1.0** for the public cut; working +tree may still show `0.0.4-dev`) and under active development. Breaking +changes may occur until 1.0.0. Security fixes land on the latest release +line only. ## Reporting a Vulnerability If you believe you have found a security vulnerability, please email: -- security@katakate.org (preferred) -- Or open a private security advisory via GitHub (Security → Advisories → Report a vulnerability) +- **security@katakate.org** (preferred) +- Or open a private security advisory via GitHub + (Security → Advisories → Report a vulnerability) Please include: + - A detailed description of the issue and potential impact - Steps to reproduce or proof-of-concept -- Affected versions/commit SHAs and environment details +- Affected versions / commit SHAs and environment details -We aim to acknowledge reports within 72 hours and provide a remediation plan or mitigation timeline when applicable. +We aim to acknowledge reports within 72 hours and provide a remediation +plan or mitigation timeline when applicable. -## Scope and Current Model +Do **not** open a public issue for security-sensitive reports. -- Nodes run K3s + Kata + Firecracker; containers run as non-root with restricted capabilities. -- API uses API keys with hashed storage and expiry; file-backed by default. -- Egress network restrictions via Kubernetes NetworkPolicies (IP-based whitelists). When egress lockdown is enabled, DNS resolution is blocked by default. -- All ingress network blocked by default to avoid default K8s pod to pod communications; this doesn't affect kubectl exec / k7 shell into sandboxes which are based on the k8s API. +## Scope and current model -Known limitations (pre-0.1.0): +- Nodes run **K3s**. A cluster (or a single node) can install **multiple + sandbox backends**; each sandbox picks one: + - **`kfd`** (`kata-firecracker-devmapper`) — Firecracker via Kata. The + Firecracker process runs inside the **jailer** (chroot + dropped + capabilities + seccomp). An integration test asserts the jailer is + active after install. + - **`kql`** (`kata-qemu-longhorn`) — **QEMU** via Kata with a Longhorn + PVC root (durable disk, named snapshots / restore / disk-only fork). + - **`k7d`** — [Katakate/k7d](https://github.com/Katakate/k7d) + (`runtimeClassName: k7`); CoW sibling-fork isolation differs — see + k7d's `SECURITY.md`. +- Sandbox containers run as non-root with restricted capabilities on top + of the VM boundary. +- The control plane API uses API keys with hashed storage and expiry + (file-backed by default at `/etc/k7/api_keys.json` — rotate and protect + that file). +- **Ingress** to sandboxes is denied by default (NetworkPolicy). + **Egress** is per-sandbox: open, blocked, CIDR allowlist, or **FQDN** + allowlist when Cilium is the CNI (default). DNS is blocked by default + when egress is locked down. +- **Multi-node** clusters are supported (Ansible inventory; Longhorn for + the QEMU/`kql` path). Cilium FQDN egress applies cluster-wide. -- No rate limiting or abuse protection at API layer yet. -- API key storage is local file; rotate and protect `/etc/k7/api_keys.json`. -- No domain-based egress control (planned via Cilium/FQDN policies). -- Jailer currently ignored by Kata -- Only single-node supported right now, multi-node support high on the roadmap -- We might want to get rid of the compose setup for the API and instead directly deploy the API on the K3s cluster by writing a few manifests. -- If keeping API out-of-cluster we should rather pass to the API a dedicated RBAC restricted Kube config instead of the admin config. +See also the docs: security model, networking, and backends comparison. + +### Known limitations (pre-1.0) + +- No rate limiting or abuse protection at the API layer yet. +- API key storage is local file-backed; treat the API host as trusted. +- Young project; no independent security audit yet. +- The `k7d` backend has a different isolation trade-off for CoW sibling + forks — see k7d's `SECURITY.md`. +- Prefer a dedicated RBAC-restricted kubeconfig for the API rather than + cluster-admin credentials in production. ## Responsible Disclosure -Do not publicly disclose vulnerabilities before we have had a reasonable time to investigate and release fixes. We appreciate coordinated disclosure and will credit reporters unless anonymity is requested. \ No newline at end of file +Do not publicly disclose vulnerabilities before we have had a reasonable +time to investigate and release fixes. We appreciate coordinated +disclosure and will credit reporters unless anonymity is requested. diff --git a/bench/docker-perf/README.md b/bench/docker-perf/README.md new file mode 100644 index 0000000..bbb98a1 --- /dev/null +++ b/bench/docker-perf/README.md @@ -0,0 +1,145 @@ +# Docker workload benchmark — Spec 10b + +Quantifies the **storage tax** of running Docker inside a k7 sandbox vs +natively on the host. Output lands in [`PERFORMANCE.md`](../../PERFORMANCE.md) +under a dated section, formatted so the median + ratio numbers can be +cited verbatim from any blog post. + +## What gets benched + +| Label | What it is | +|-------------|------------| +| `host` | Native Docker on the Hetzner node (no k7 involved) | +| `k7-fd` | k7 sandbox, `kata-firecracker-devmapper` (kfd) backend, `--sidecar docker` (docker daemon's `/var/lib/docker` is an emptyDir) | +| `k7-ql-r1` | k7 sandbox, `kata-qemu-longhorn` (kql) backend, `--sidecar docker`, Longhorn `replicas=1` | +| `k7-ql-r2` | Same as `k7-ql-r1` but Longhorn `replicas=2` (requires ≥ 2-node cluster) | + +The four environments stack the storage path cleanly: `host` → no VM, +no Longhorn. `k7-fd` → VM but no Longhorn. `k7-ql-r1` → VM + one local +Longhorn replica. `k7-ql-r2` → VM + one local + one cross-node Longhorn +replica. Subtracting `k7-fd / host` from `k7-ql-r1 / host` isolates the +**Longhorn tax** from the **kata-qemu tax**. + +## Workloads (5 runs each, after a discarded warm-up) + +| Operation | Isolates | +|-----------|----------| +| `pull debian:12-slim` | Network + extract + write of an external image | +| `build (no-cache)` | The headline number: apt + pip + git + 256 MB fsync | +| `build (cached)` | Sanity — should be sub-second; non-zero on `k7-ql-*` would mean the bind mount of `/var/lib/docker` is broken | +| `run cpu (10s budget)` | Kata-qemu CPU overhead (Python busy-loop) | +| `run io (2k small + 512 MB fsync)` | Write path through the storage stack | +| `run read (venv tree cat)` | Read path (mostly page-cache hits after the first call) | + +The Dockerfile that drives `build (no-cache)` is pinned at +[`bench/docker-perf/bench.Dockerfile`](bench.Dockerfile) — apt + pip +generate thousands of small files (metadata pressure), git clone is +inode-heavy, and `dd … conv=fsync` measures sync-write throughput. + +## How to run it + +The bench is a **pytest module** (not a shell script) — it reuses the +exact same `k7_core` / `test_namespace` fixtures that +`tests/integration/test_sidecar_docker.py` already uses to spin up a +`docker:27.5-cli` sandbox with `--sidecar docker`. The harness times +`k7_core.exec_command(...)` calls instead of doing anything new. + +Run all three envs: + +```bash +# On the cluster node (or any host that already runs the integration suite): +K7_BENCH_ENVS=host,k7-ql-r1,k7-ql-r2 \ +uv run pytest -m bench tests/integration/bench_docker_perf.py -v -s +``` + +Pick a subset (useful for iterating on tooling): + +```bash +K7_BENCH_ENVS=host uv run pytest -m bench -v -s tests/integration/bench_docker_perf.py +K7_BENCH_ENVS=k7-ql-r1 uv run pytest -m bench -v -s tests/integration/bench_docker_perf.py +``` + +Switching `r1 → r2` is done **in-cluster** by patching Longhorn's +`default-replica-count` setting via `kubectl` — no `k7 install` reinstall, +no cluster-wide downtime, and the value is restored at session teardown. + +### Environment variables + +| Var | Default | Purpose | +|-----|---------|---------| +| `K7_BENCH_ENVS` | `host,k7-ql-r1,k7-ql-r2` | Comma-separated subset of envs to run | +| `K7_BENCH_REPS` | `5` | Reps per cell | +| `K7_BENCH_WARMUP` | `1` | 0 to disable the warm-up | +| `K7_BENCH_OUT` | `/tmp` | Where logs + the aggregated CSV land | + +### Output + +Each leg writes a `bench-

-

{title}

-

... snippet content ...

-
-); -``` - - - MDX does not compile inside the body of an arrow function. Stick to HTML - syntax when you can or use a default export if you need to use MDX. - - -2. Import the snippet into your destination file and pass in the props - -```mdx destination-file.mdx ---- -title: My title -description: My Description ---- - -import { MyComponent } from '/snippets/custom-component.mdx'; - -Lorem ipsum dolor sit amet. - - -``` diff --git a/docs/essentials/settings.mdx b/docs/essentials/settings.mdx deleted file mode 100644 index 884de13..0000000 --- a/docs/essentials/settings.mdx +++ /dev/null @@ -1,318 +0,0 @@ ---- -title: 'Global Settings' -description: 'Mintlify gives you complete control over the look and feel of your documentation using the docs.json file' -icon: 'gear' ---- - -Every Mintlify site needs a `docs.json` file with the core configuration settings. Learn more about the [properties](#properties) below. - -## Properties - - -Name of your project. Used for the global title. - -Example: `mintlify` - - - - - An array of groups with all the pages within that group - - - The name of the group. - - Example: `Settings` - - - - The relative paths to the markdown files that will serve as pages. - - Example: `["customization", "page"]` - - - - - - - - Path to logo image or object with path to "light" and "dark" mode logo images - - - Path to the logo in light mode - - - Path to the logo in dark mode - - - Where clicking on the logo links you to - - - - - - Path to the favicon image - - - - Hex color codes for your global theme - - - The primary color. Used for most often for highlighted content, section - headers, accents, in light mode - - - The primary color for dark mode. Used for most often for highlighted - content, section headers, accents, in dark mode - - - The primary color for important buttons - - - The color of the background in both light and dark mode - - - The hex color code of the background in light mode - - - The hex color code of the background in dark mode - - - - - - - - Array of `name`s and `url`s of links you want to include in the topbar - - - The name of the button. - - Example: `Contact us` - - - The url once you click on the button. Example: `https://mintlify.com/docs` - - - - - - - - - Link shows a button. GitHub shows the repo information at the url provided including the number of GitHub stars. - - - If `link`: What the button links to. - - If `github`: Link to the repository to load GitHub information from. - - - Text inside the button. Only required if `type` is a `link`. - - - - - - - Array of version names. Only use this if you want to show different versions - of docs with a dropdown in the navigation bar. - - - - An array of the anchors, includes the `icon`, `color`, and `url`. - - - The [Font Awesome](https://fontawesome.com/search?q=heart) icon used to feature the anchor. - - Example: `comments` - - - The name of the anchor label. - - Example: `Community` - - - The start of the URL that marks what pages go in the anchor. Generally, this is the name of the folder you put your pages in. - - - The hex color of the anchor icon background. Can also be a gradient if you pass an object with the properties `from` and `to` that are each a hex color. - - - Used if you want to hide an anchor until the correct docs version is selected. - - - Pass `true` if you want to hide the anchor until you directly link someone to docs inside it. - - - One of: "brands", "duotone", "light", "sharp-solid", "solid", or "thin" - - - - - - - Override the default configurations for the top-most anchor. - - - The name of the top-most anchor - - - Font Awesome icon. - - - One of: "brands", "duotone", "light", "sharp-solid", "solid", or "thin" - - - - - - An array of navigational tabs. - - - The name of the tab label. - - - The start of the URL that marks what pages go in the tab. Generally, this - is the name of the folder you put your pages in. - - - - - - Configuration for API settings. Learn more about API pages at [API Components](/api-playground/demo). - - - The base url for all API endpoints. If `baseUrl` is an array, it will enable for multiple base url - options that the user can toggle. - - - - - - The authentication strategy used for all API endpoints. - - - The name of the authentication parameter used in the API playground. - - If method is `basic`, the format should be `[usernameName]:[passwordName]` - - - The default value that's designed to be a prefix for the authentication input field. - - E.g. If an `inputPrefix` of `AuthKey` would inherit the default input result of the authentication field as `AuthKey`. - - - - - - Configurations for the API playground - - - - Whether the playground is showing, hidden, or only displaying the endpoint with no added user interactivity `simple` - - Learn more at the [playground guides](/api-playground/demo) - - - - - - Enabling this flag ensures that key ordering in OpenAPI pages matches the key ordering defined in the OpenAPI file. - - This behavior will soon be enabled by default, at which point this field will be deprecated. - - - - - - - A string or an array of strings of URL(s) or relative path(s) pointing to your - OpenAPI file. - - Examples: - - ```json Absolute - "openapi": "https://example.com/openapi.json" - ``` - ```json Relative - "openapi": "/openapi.json" - ``` - ```json Multiple - "openapi": ["https://example.com/openapi1.json", "/openapi2.json", "/openapi3.json"] - ``` - - - - - - An object of social media accounts where the key:property pair represents the social media platform and the account url. - - Example: - ```json - { - "x": "https://x.com/mintlify", - "website": "https://mintlify.com" - } - ``` - - - One of the following values `website`, `facebook`, `x`, `discord`, `slack`, `github`, `linkedin`, `instagram`, `hacker-news` - - Example: `x` - - - The URL to the social platform. - - Example: `https://x.com/mintlify` - - - - - - Configurations to enable feedback buttons - - - - Enables a button to allow users to suggest edits via pull requests - - - Enables a button to allow users to raise an issue about the documentation - - - - - - Customize the dark mode toggle. - - - Set if you always want to show light or dark mode for new users. When not - set, we default to the same mode as the user's operating system. - - - Set to true to hide the dark/light mode toggle. You can combine `isHidden` with `default` to force your docs to only use light or dark mode. For example: - - - ```json Only Dark Mode - "modeToggle": { - "default": "dark", - "isHidden": true - } - ``` - - ```json Only Light Mode - "modeToggle": { - "default": "light", - "isHidden": true - } - ``` - - - - - - - - - A background image to be displayed behind every page. See example with - [Infisical](https://infisical.com/docs) and [FRPC](https://frpc.io). - diff --git a/docs/favicon.png b/docs/favicon.png deleted file mode 100644 index 3585b5c..0000000 Binary files a/docs/favicon.png and /dev/null differ diff --git a/docs/getting-started/installation.mdx b/docs/getting-started/installation.mdx deleted file mode 100644 index 86fa9cb..0000000 --- a/docs/getting-started/installation.mdx +++ /dev/null @@ -1,220 +0,0 @@ ---- -title: "Quickstart" -description: "Install k7, prepare your node, start the API, and run your first sandbox" ---- - -Katakate (k7) lets you run secure, lightweight VM sandboxes backed by Kata Containers and Firecracker, orchestrated with Kubernetes. This Quickstart gets you from zero to a working sandbox via CLI and Python SDK. - - -If you already installed k7 previously, consider running `make uninstall` before reinstalling to avoid stale cached files in a previous `.deb`. - - -## Requirements - -- Linux (amd64) host with hardware virtualization (KVM) - - Check: `ls /dev/kvm` should exist - - Cloud guidance: AWS `.metal`, GCP (enable nested virtualization), Azure D/Ev series; typical VPS often lack KVM -- One raw, unformatted disk for thin‑pool provisioning (recommended for many sandboxes) -- Docker with Compose plugin (for the API) - - Install Docker: `curl -fsSL https://get.docker.com | sh` -- Ansible for the installer (Ubuntu): - -```bash -sudo add-apt-repository universe -y -sudo apt update -sudo apt install -y ansible -``` - -- Python 3.10+ on the client for the SDK - - -Tested setup example: Hetzner Robot instance, Ubuntu 24.04 (x86_64), with an extra empty NVMe disk (for the thin‑pool). See the detailed setup guide (PDF): k7_hetzner_node_setup.pdf. - - -## Install the CLI (APT) - -Install the `k7` CLI on the node(s) that will host the VM sandboxes: - -```bash -sudo add-apt-repository ppa:katakate.org/k7 -sudo apt update -sudo apt install k7 -``` - -## Install K7 on your node(s) - -This installs and wires up Kubernetes (K3s), Kata, Firecracker, Jailer, and the devmapper snapshotter with thin‑pool provisioning: - -```bash -k7 install -``` - -![Example output: k7 install](/images/ex-install.png) - - -You should see "Installation completed successfully!" when done. Add `-v` for verbose output. - - -## Start the API and manage keys - -### Start the API - -```bash -k7 start-api -``` - -![Example: k7 start-api](/images/ex-start-api.png) - -### Check API status - -```bash -k7 api-status -``` - -![Example: k7 api-status](/images/ex-api-status.png) - -### Get the public endpoint - -```bash -k7 get-api-endpoint -``` - -![Example: k7 get-api-endpoint](/images/ex-get-api-endpoint.png) - -### Generate an API key - -```bash -k7 generate-api-key mykey -``` - -![Example: k7 generate-api-key](/images/ex-generate-api-key.png) - -### Stop the API - -```bash -k7 stop-api -``` - -![Example: k7 stop-api](/images/ex-stop-api.png) - - -- Ensure your user is in the `docker` group to manage the API containers. -- API keys are stored at `/etc/k7/api_keys.json` by default. Authentication accepts `X-API-Key` header or `Authorization: Bearer `. - - -## Create your first sandbox via CLI - -Example `k7.yaml`: - -```yaml -name: demo -image: alpine:3.20 -namespace: default -env_file: /root/secrets.env -limits: - cpu: "100m" - memory: "128Mi" -before_script: | - # Installing curl. Egress open during before_script, then restricted (empty whitelist) afterwards - apk add curl - echo $ENV_VAR_1 -egress_whitelist: [] -``` - -### Create a sandbox - -```bash -# Uses k7.yaml in the current directory by default -k7 create -``` - -![Example: k7 create](/images/ex-create.png) - -### Shell into your sandbox - -```bash -k7 shell demo -``` - -![Example: k7 shell](/images/ex-shell.png) - -### List sandboxes - -```bash -k7 list -``` - -![Example: k7 list](/images/ex-list.png) - -### Delete a sandbox - -```bash -k7 delete my-sandbox-123 -``` - -### Delete all sandboxes - -```bash -k7 delete-all -``` - -### Prerequisites for the SDK - -```bash -# Ensure the API is running and you have an endpoint and API key -k7 start-api -k7 get-api-endpoint -k7 generate-api-key my-key -``` - -## Create your first sandbox via Python SDK - -Install the SDK on your client machine: - -```bash -pip install katakate -``` - -Use the synchronous client: - -```python -from katakate import Client - -k7 = Client(endpoint="https://", api_key="") - -# Create sandbox -sb = k7.create({ - "name": "my-sandbox", - "image": "alpine:latest" -}) - -# Execute code -result = sb.exec('echo "Hello World"') -print(result["stdout"]) # or just print(sb.exec("echo hi")) - -# List and cleanup -print(k7.list()) -sb.delete() -``` - -Async variant: - -```python -import asyncio -from katakate import AsyncClient - -async def main(): - k7 = AsyncClient(endpoint="https://", api_key="") - print(await k7.list()) - await k7.aclose() - -asyncio.run(main()) -``` - -## Next steps - -- Explore the CLI guide: `/guides/cli` -- Explore the Python SDK guide: `/guides/python-sdk` -- Integrate with the REST API: `/api/introduction` - - diff --git a/docs/guides/cli.mdx b/docs/guides/cli.mdx deleted file mode 100644 index 89f10fd..0000000 --- a/docs/guides/cli.mdx +++ /dev/null @@ -1,190 +0,0 @@ ---- -title: "CLI reference" -description: "All k7 commands with options and examples" ---- - -Use `k7 -h` for built-in help. Below are the primary commands. - -## install - -Install K7 components on host node(s). - -```bash -k7 install [-v] -``` - -- **-v**: verbose output - -## version - -Check version of installed K7 .deb package - -```bash -k7 -V -``` - -## create - -Create a sandbox from a YAML file or flags. - -```bash -k7 create -f k7.yaml -# or -k7 create --name my-sb --image alpine:latest \ - --cpu 1 --memory 1Gi --storage 2Gi \ - --env-file .env --egress 10.0.0.5/32 \ - --before-script "apk add curl" -``` - -### YAML configuration reference - -All fields map to the server-side `SandboxConfig`: - -- **name** (string, required): unique sandbox name. -- **image** (string, required): container image, e.g. `alpine:latest`. -- **namespace** (string, default `default`): Kubernetes namespace. -- **env_file** (string, optional): (absolute) path to an env file on the host node. -- **egress_whitelist** (array of CIDR strings, optional): allowed egress IPs, e.g. `"1.1.1.1/32"` for single hosts or `"10.0.0.0/8"` for ranges. -- **limits** (object, optional): resource limits: - - **cpu** (string): cores or millicores, e.g. `"1"` or `"500m"`. - - **memory** (string): e.g. `"1Gi"`, `"512Mi"`. - - **ephemeral-storage** (string): e.g. `"2Gi"`. -- **before_script** (string, optional): shell script run once at container start. - - Runs with open egress; readiness waits for completion when set. -- **pod_non_root** (boolean, optional): run Pod as non-root (UID/GID/FSGroup 65532). -- **container_non_root** (boolean, optional): run container as non-root (UID 65532), no privilege escalation. -- **cap_add** (string[], optional): add back Linux capabilities (default policy drops ALL). -- **cap_drop** (string[], optional): override drop policy. If omitted, `ALL` is dropped by default. - -Example `k7.yaml`: - -```yaml -name: project-build -image: alpine:latest -namespace: default -egress_whitelist: - - "10.0.0.5/32" # Private egress proxy/gateway -limits: - cpu: "1" - memory: "1Gi" - ephemeral-storage: "2Gi" -before_script: | - # Non-root friendly example: create a working dir and print versions - mkdir -p "$HOME/work" && cd "$HOME/work" - echo "PATH=$PATH" - echo "whoami: $(whoami)" -pod_non_root: false -container_non_root: false -cap_add: - - CHOWN -``` - - -Do not whitelist public DNS resolvers (e.g., 1.1.1.1, 8.8.8.8). Doing so re-enables DNS exfiltration (UDP/TCP 53 and DoH over 443). Prefer whitelisting only your own egress proxy IP and enforce DNS/DoH policies at the proxy. - -If using package managers that require root (e.g., `apk add`, `apt-get install`) in `before_script` make sure you didn't add security policies that prevent it such as running the pod or container as non-root. Check Security & Networking section in the API reference for more. - - - -## list - -```bash -k7 list [-n NAMESPACE] -``` -Lists sandboxes with status, readiness, restarts, age, and image. - -## delete - -```bash -k7 delete NAME [-n NAMESPACE] -``` -Deletes one sandbox. - -## delete-all - -```bash -k7 delete-all [-n NAMESPACE] -``` -Deletes all sandboxes in a namespace (with confirmation). - -## shell - -```bash -k7 shell NAME [-n NAMESPACE] -``` -Opens an interactive shell in the sandbox pod. - -## logs - -```bash -k7 logs NAME [-n NAMESPACE] [--tail 200] [-f] -``` -Shows container logs (before script and main container). - -## top - -```bash -k7 top [-n NAMESPACE] [--refresh-interval 1] -``` -Top-like view of CPU and memory usage. - -## start-api - -```bash -k7 start-api -``` -Starts the API and Cloudflared tunnel via Docker Compose. - -## api-status - -```bash -k7 api-status -``` -Shows API running state and public URL. - -## get-api-endpoint - -```bash -k7 get-api-endpoint -``` -Prints the public URL if available. - -## stop-api - -```bash -k7 stop-api -``` -Stops API and Cloudflared containers. - -## API keys - -```bash -k7 generate-api-key NAME [--expires-days 365] -k7 list-api-keys -k7 revoke-api-key NAME -``` - -Keys are stored at `/etc/k7/api_keys.json`. Use with `X-API-Key` or `Authorization: Bearer`. - -### Flag reference (create) - -- **-n, --namespace**: Kubernetes namespace (default `default`). -- **-f, --file**: YAML config file (defaults to `k7.yaml` when using `k7 create`). -- **--name**: Sandbox name (when not using YAML). -- **--image**: Container image (when not using YAML). -- **--cpu**: CPU limit (e.g., `1`, `500m`). -- **--memory**: Memory limit (e.g., `1Gi`, `512Mi`). -- **--storage**: Ephemeral storage limit (e.g., `2Gi`). -- **--env-file**: Path to env file on the host node injected as a Secret. -- **--egress CIDR**: Repeatable; whitelist CIDR blocks for egress (omit to keep open; use none for full block). -- **--before-script**: Shell script to run once at start; runs with open egress before lockdown. -- **--pod-non-root / --no-pod-non-root**: Pod-level non-root defaults. -- **--container-non-root / --no-container-non-root**: Container runs as UID 65532, no privilege escalation. -- **--cap-add CAP**: Repeatable; add back Linux capabilities (default drop ALL). -- **--cap-drop CAP**: Repeatable; override default drop policy. - - -Package installs like `apk add` require root inside the container. Either leave `container_non_root` disabled for setup or prebuild an image. See Security & networking: `/api/security`. - - - diff --git a/docs/guides/langchain-agent.mdx b/docs/guides/langchain-agent.mdx deleted file mode 100644 index 7dbb9a7..0000000 --- a/docs/guides/langchain-agent.mdx +++ /dev/null @@ -1,119 +0,0 @@ ---- -title: "LangChain agent tutorial" -description: "Build a ReAct agent that executes inside a K7 sandbox" ---- - -This tutorial walks you through wiring a LangChain ReAct agent with a tool that executes shell commands in a K7 sandbox. - -## Prerequisites - -- K7 API running (`k7 start-api`) and reachable -- API key generated: `k7 generate-api-key ` -- Python 3.10+ - -## Setup - -Create a `.env` file with your credentials and defaults: - -```env -K7_ENDPOINT=https://your-k7-endpoint -K7_API_KEY=your-api-key -K7_SANDBOX_NAME=lc-agent -K7_SANDBOX_IMAGE=alpine:latest -K7_NAMESPACE=default -OPENAI_API_KEY=sk-your-openai-key -OPENAI_MODEL=gpt-4o-mini -``` - -Install dependencies: - -```bash -pip install langchain langchain-openai python-dotenv katakate -``` - -## Agent code - -```python -import os, time -from pathlib import Path -from typing import Optional -from dotenv import load_dotenv -from langchain.agents import initialize_agent, AgentType -from langchain.memory import ConversationBufferMemory -from langchain.tools import Tool -from langchain_openai import ChatOpenAI -from katakate import Client, SandboxProxy - -load_dotenv() - -K7_ENDPOINT = os.getenv("K7_ENDPOINT") -K7_API_KEY = os.getenv("K7_API_KEY") -SANDBOX_NAME = os.getenv("K7_SANDBOX_NAME", "lc-agent") -SANDBOX_IMAGE = os.getenv("K7_SANDBOX_IMAGE", "alpine:latest") -SANDBOX_NAMESPACE = os.getenv("K7_NAMESPACE", "default") - -k7 = Client(endpoint=K7_ENDPOINT, api_key=K7_API_KEY) -_sb: Optional[SandboxProxy] = None - -def ensure_sandbox_ready(timeout_seconds: int = 60) -> SandboxProxy: - try: - sb = k7.create({ - "name": SANDBOX_NAME, - "image": SANDBOX_IMAGE, - "namespace": SANDBOX_NAMESPACE, - }) - except Exception: - sb = SandboxProxy(SANDBOX_NAME, SANDBOX_NAMESPACE, k7) - - deadline = time.time() + timeout_seconds - while time.time() < deadline: - for info in k7.list(namespace=SANDBOX_NAMESPACE): - if info.get("name") == SANDBOX_NAME and info.get("status") == "Running": - return sb - time.sleep(2) - raise RuntimeError("Sandbox did not become Running in time") - -def run_code_in_sandbox(code: str) -> str: - global _sb - if _sb is None: - _sb = ensure_sandbox_ready() - result = _sb.exec(code) - if result.get("exit_code", 1) != 0: - return f"[stderr]\n{result.get('stderr','')}\n[stdout]\n{result.get('stdout','')}" - return result.get("stdout", "") - -tool = Tool( - name="sandbox_exec", - description="Execute a shell command inside an isolated K7 sandbox. Input should be a shell command string.", - func=run_code_in_sandbox, -) - -llm = ChatOpenAI(model=os.getenv("OPENAI_MODEL", "gpt-4o-mini"), temperature=0) -memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True) - -agent = initialize_agent( - tools=[tool], - llm=llm, - agent=AgentType.CONVERSATIONAL_REACT_DESCRIPTION, - memory=memory, - verbose=True, - handle_parsing_errors=True, -) - -print("Ask me to run a command in a sandbox, e.g.: 'List files in /'\n") -while True: - try: - user = input("You: ") - except (EOFError, KeyboardInterrupt): - break - if not user.strip(): - continue - resp = agent.invoke({"input": user}) - print("Agent:", resp.get("output", str(resp))) -``` - - -You can shell into the same sandbox in parallel: `k7 shell lc-agent`. - - - diff --git a/docs/guides/python-sdk.mdx b/docs/guides/python-sdk.mdx deleted file mode 100644 index 767bb36..0000000 --- a/docs/guides/python-sdk.mdx +++ /dev/null @@ -1,225 +0,0 @@ ---- -title: "Python SDK" -description: "Use the katakate client to manage sandboxes" ---- - -Install the SDK: - -```bash -pip install katakate -``` - -## Synchronous client - -```python -from katakate import Client - -k7 = Client(endpoint="https://", api_key="") - -# Create a sandbox -sb = k7.create({ - "name": "my-sandbox", - "image": "alpine:latest", - # optional: "namespace": "default", - # optional: "env_file": ".env", - # optional: "egress_whitelist": ["10.0.0.5/32"], # private egress proxy - # optional: "limits": {"cpu": "1", "memory": "1Gi", "ephemeral-storage": "2Gi"}, - # optional: "before_script": "apk add curl" -}) - -# Execute a command -result = sb.exec('echo "Hello World"') -print(result["stdout"]) # Also includes stderr and exit_code - -# List sandboxes -print(k7.list()) - -# Delete sandbox -sb.delete() -``` - -### Client configuration - -- `endpoint`: Base URL of your API, e.g. `https://`. -- `api_key`: Your API key. The SDK sends it via `X-API-Key` automatically. - - -Get your endpoint and API key using the CLI: `k7 api-status`, `k7 get-api-endpoint`, `k7 generate-api-key `. See the CLI guide: `/guides/cli`. - - -### Create with non-root, capabilities, egress controls, limits - -By default, all Linux capabilities are dropped. You can add back minimal ones if needed. - -```python -sb = k7.create({ - "name": "secure-sb", - "image": "alpine:latest", - "namespace": "default", - - # Non-root execution - "pod_non_root": True, # Pod UID/GID/FSGroup 65532 - "container_non_root": True, # Container UID 65532, no privilege escalation - - # Capabilities: drop ALL by default, add minimal ones back - "cap_add": ["CHOWN"], - "cap_drop": ["NET_RAW"], - - # Network egress control - # - Omit key to keep egress open - # - [] blocks all egress (DNS blocked) - # - [CIDRs] allows only those CIDRs (DNS still blocked) - "egress_whitelist": [ - "10.0.0.5/32", # private egress proxy - "203.0.113.0/24" - ], - - # Resource limits/requests (same values used for both) - "limits": {"cpu": "500m", "memory": "512Mi", "ephemeral-storage": "2Gi"}, - - # Optional setup commands run before Ready (executed with open egress) - "before_script": "apk add --no-cache curl git" -}) -``` - - -`env_file` points to a file on the API host filesystem (server-side), not the client machine. If you need environment variables and you’re calling a remote API, pass values directly for now. - - -### Wait until sandbox is Ready - -```python -import time - -def wait_until_ready(name: str, namespace: str = "default", timeout_seconds: int = 120) -> None: - deadline = time.time() + timeout_seconds - while time.time() < deadline: - for info in k7.list(namespace=namespace): - if info.get("name") == name and info.get("status") == "Running" and info.get("ready") == "True": - return - time.sleep(2) - raise TimeoutError("Sandbox did not become Ready in time") - -wait_until_ready("secure-sb") -``` - -### Execute commands and handle errors - -```python -res = sb.exec("echo hello && uname -a") -print(res["stdout"]) # command output -print(res["stderr"]) # error stream (if any) -print(res["exit_code"]) # 0 on success - -# Example of a failing command -bad = sb.exec("sh -lc 'exit 2'") -if bad["exit_code"] != 0: - print("Command failed:") - print("stderr:", bad.get("stderr", "")) -``` - -### List, filter by namespace - -```python -print(k7.list()) # all namespaces -print(k7.list(namespace="dev")) # only dev -``` - -### Delete and delete all - -```python -k7.delete("secure-sb") -k7.delete_all(namespace="default") -``` - - -## Async client - -```python -import asyncio -from katakate import AsyncClient - -async def main(): - k7 = AsyncClient(endpoint="https://", api_key="") - sandboxes = await k7.list() - print(sandboxes) - await k7.aclose() - -asyncio.run(main()) -``` - -### Async examples - -Create, wait, exec, delete: - -```python -import os -import asyncio -from katakate import AsyncClient - -K7_ENDPOINT = os.getenv("K7_ENDPOINT") -K7_API_KEY = os.getenv("K7_API_KEY") - -async def main(): - - try: - k7 = AsyncClient(endpoint=K7_ENDPOINT, api_key=K7_API_KEY) - - cfg = { - "name": "async-sb", - "image": "alpine:latest", - "pod_non_root": True, - "container_non_root": True, - "cap_add": ["CHOWN"], - # "before_script": "apk add --no-cache curl" # This is commented out here as it would fail, because 'apk add' needs root access, which we removed with pod_non_root and container_non_root set to True - "egress_whitelist": [], # full network lockdown after the before_script - } - - print("Creating sandbox...") - await k7.create(cfg) - print("Sandbox created.") - - # (Optional) Simple readiness wait (poll list). This can be removed, it is just here to illustrate. - for _ in range(60): - sbs = await k7.list() - if any(s.get("name") == "async-sb" and s.get("status") == "Running" and s.get("ready") == "True" for s in sbs): - break - await asyncio.sleep(2) - - out = await k7.exec("async-sb", "echo from async") - print("Output of execution:", out) - - except Exception as e: - raise e - - # Include a finally block to clean resources even if code fails - finally: - print("Deleting sandbox 'async-sb'...") - try: - await k7.delete("async-sb") - print("Sandbox 'async-sb' deleted.") - except: - raise Exception("Failed to delete async-sb, you might need to clean resources manually.") - - print("Closing the client's httpx connection...") - try: - await k7.aclose() - print("Connection closed.) - except: - raise Exception("Failed to close the K7 client's httpx connection, you might need to clean resources manually.) - -asyncio.run(main()) -``` - - -## Errors and responses - -- Successful responses are wrapped as `{ "data": ... }` by the API; the SDK unwraps them. -- Errors are returned as `{ "error": { "code": string, "message": string } }` with appropriate HTTP status codes. - -## Tips - -- Provide a `namespace` explicitly if you use non-default namespaces. -- Keep API keys secret; rotate via `k7 revoke-api-key` and `k7 generate-api-key`. - - diff --git a/docs/guides/releasing.mdx b/docs/guides/releasing.mdx deleted file mode 100644 index af91f61..0000000 --- a/docs/guides/releasing.mdx +++ /dev/null @@ -1,82 +0,0 @@ ---- -title: Releasing (internal) -hidden: true -noindex: true ---- - -## Releasing (Deb/PPA + PyPI) - -### Prereqs -- Docker on your host (for Docker-based deb builds) -- Ubuntu 24.04 shell/container for Debian tooling: - - apt-get install -y build-essential devscripts debhelper dh-python fakeroot lintian docker.io dput gnupg - -### Build CLI .deb locally (no host pollution) -1) Start a clean builder that talks to host Docker: -```bash -docker run --rm -it \ - -v "$PWD":/src -w /src \ - -v /var/run/docker.sock:/var/run/docker.sock \ - ubuntu:24.04 bash -``` -2) Inside the container: -```bash -apt-get update -apt-get install -y build-essential devscripts debhelper dh-python fakeroot lintian docker.io -dpkg-buildpackage -b -d -ls -la ../k7_*_amd64.deb -``` -3) Test install: -```bash -dpkg -i ../k7_*_amd64.deb || apt-get -y -f install -k7 -V -``` - -Notes: -- debian/rules uses Docker to compile the Nuitka onefile and packages only /usr/bin/k7. -- We disable strip/dwz so the onefile payload remains intact. - -### Prepare and upload source to Launchpad PPA -You can smoke-test locally without signing: -```bash -dpkg-buildpackage -S -sa -d -lintian -i ../k7_*_source.changes -``` - -Signed upload (requires your GPG key registered on Launchpad): -```bash -gpg --batch --import /path/to/your-private-key.asc -KEYID=$(gpg --list-keys --with-colons | awk -F: '/^pub/ {print $5; exit}') -dpkg-buildpackage -S -sa -k"$KEYID" -dput ppa:katakate.org/k7 ../k7_*_source.changes -``` - -Helper script: -```bash -scripts/test-launchpad-build.sh # unsigned -scripts/test-launchpad-build.sh -s KEYID # signed -``` - -Versioning: -- Update `src/k7/__init__.py` before tagging. -- For native format (3.0 native), `debian/changelog` versions like `0.0.1` (no `-1`). - -### GitHub CI (tags vX.Y.Z) -- PyPI publish: builds sdist/wheel and uploads with `PYPI_API_TOKEN`. -- Deb artifact: builds .deb via Docker (make build), uploads artifact. -- Launchpad upload: builds signed source with `dpkg-buildpackage -S -sa -d` and `PPA_GPG_PRIVATE_KEY`. - -### Publish katakate (PyPI SDK) locally -1) Bump version in `src/katakate/__init__.py`. -2) Build and upload: -```bash -python -m pip install --upgrade pip build twine -python -m build -twine upload dist/* -``` - -Notes: -- Only `src/katakate` is packaged for PyPI; assets in `src/k7/*` are not part of the SDK. -- Ensure `~/.pypirc` or `TWINE_USERNAME=__token__` and `TWINE_PASSWORD=` are set. - - diff --git a/docs/guides/utilities.mdx b/docs/guides/utilities.mdx deleted file mode 100644 index ebf6705..0000000 --- a/docs/guides/utilities.mdx +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: "Utilities" -description: "Helper scripts: disk wipe for thin‑pool prep and high‑density stress testing" ---- - -This guide explains how to use the helper scripts under `utils/`. - -## Wipe a disk for thin‑pool provisioning - -Script: `utils/wipe-disk.sh` - - -Destructive operation. This irreversibly erases all partitions, RAID metadata, filesystems, and attempts discards on the target device. Double‑check the device path. - - -### Usage - -```bash -sudo ./utils/wipe-disk.sh /dev/nvme2n1 -``` - -You will be prompted to type `YES` to proceed. The script will: - -- Remove filesystem signatures (`wipefs -a`) -- Zap the partition table (`sgdisk --zap-all`) -- Zero the beginning and end of the disk (`dd`) -- Attempt block discard (`blkdiscard`) if supported - -List disks to find the correct device: - -```bash -lsblk -o NAME,SIZE,TYPE,MOUNTPOINT -``` - -Requirements: Linux with `wipefs`, `sgdisk`, and `blkdiscard`; run as root or via `sudo`. - -## High‑density CPU/memory stress test - -Script: `utils/stress_test.sh` - -This script launches many sandboxes to validate CPU limit enforcement and observe resource behavior. - -### What it does - -- Creates namespace `stress-test` -- Generates `k7-stress-*.yaml` files, each with: - - `before_script` that installs `stress-ng` and `htop` via `apk` - - CPU and memory limits per sandbox -- Launches sandboxes in batches (default 50 total, batches of 10) -- Sets up a cleanup trap on Ctrl+C to delete resources and namespace - -Default parameters (edit inside the script if desired): - -- `COUNT=50` -- `NAMESPACE="stress-test"` -- `CPU_LIMIT="300m"` -- `MEM_LIMIT="2Gi"` -- `STRESS_MEM="1500M"` - -### Run - -```bash -bash utils/stress_test.sh -``` - -Monitor during the test: - -```bash -k7 top -n stress-test -watch 'k3s kubectl top pods -n stress-test --sort-by=cpu' -``` - -Cleanup when done (also done automatically on Ctrl+C): - -```bash -k7 delete-all -n stress-test -y -rm k7-stress-*.yaml -k3s kubectl delete namespace stress-test -``` - -Notes: - -- The generated YAML uses Alpine and installs packages in `before_script`. Ensure the container can run `apk` (i.e., not forced non‑root during setup). If you enforce strict non‑root, consider prebuilding an image with dependencies. -- Ensure your node(s) have sufficient CPU/RAM to handle the configured load. - - diff --git a/docs/images/ex-api-status.png b/docs/images/ex-api-status.png deleted file mode 100644 index e13412c..0000000 Binary files a/docs/images/ex-api-status.png and /dev/null differ diff --git a/docs/images/ex-create.png b/docs/images/ex-create.png deleted file mode 100644 index 67dd2af..0000000 Binary files a/docs/images/ex-create.png and /dev/null differ diff --git a/docs/images/ex-generate-api-key.png b/docs/images/ex-generate-api-key.png deleted file mode 100644 index a4847f7..0000000 Binary files a/docs/images/ex-generate-api-key.png and /dev/null differ diff --git a/docs/images/ex-get-api-endpoint.png b/docs/images/ex-get-api-endpoint.png deleted file mode 100644 index a41e40c..0000000 Binary files a/docs/images/ex-get-api-endpoint.png and /dev/null differ diff --git a/docs/images/ex-install.png b/docs/images/ex-install.png deleted file mode 100644 index 6821b82..0000000 Binary files a/docs/images/ex-install.png and /dev/null differ diff --git a/docs/images/ex-list-api-keys.png b/docs/images/ex-list-api-keys.png deleted file mode 100644 index 37a2446..0000000 Binary files a/docs/images/ex-list-api-keys.png and /dev/null differ diff --git a/docs/images/ex-list.png b/docs/images/ex-list.png deleted file mode 100644 index ef87db6..0000000 Binary files a/docs/images/ex-list.png and /dev/null differ diff --git a/docs/images/ex-revoke-api-key.png b/docs/images/ex-revoke-api-key.png deleted file mode 100644 index fbe8bbc..0000000 Binary files a/docs/images/ex-revoke-api-key.png and /dev/null differ diff --git a/docs/images/ex-shell.png b/docs/images/ex-shell.png deleted file mode 100644 index 6aa512b..0000000 Binary files a/docs/images/ex-shell.png and /dev/null differ diff --git a/docs/images/ex-start-api.png b/docs/images/ex-start-api.png deleted file mode 100644 index 6f14886..0000000 Binary files a/docs/images/ex-start-api.png and /dev/null differ diff --git a/docs/images/ex-stop-api.png b/docs/images/ex-stop-api.png deleted file mode 100644 index 0155861..0000000 Binary files a/docs/images/ex-stop-api.png and /dev/null differ diff --git a/docs/images/ex-top.png b/docs/images/ex-top.png deleted file mode 100644 index b711749..0000000 Binary files a/docs/images/ex-top.png and /dev/null differ diff --git a/docs/images/k7-cover-upgrade.png b/docs/images/k7-cover-upgrade.png deleted file mode 100644 index faa44c0..0000000 Binary files a/docs/images/k7-cover-upgrade.png and /dev/null differ diff --git a/docs/images/k7-logo.png b/docs/images/k7-logo.png deleted file mode 100644 index 94f2bd2..0000000 Binary files a/docs/images/k7-logo.png and /dev/null differ diff --git a/docs/index.mdx b/docs/index.mdx deleted file mode 100644 index b703bd0..0000000 --- a/docs/index.mdx +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: "Katakate" -description: "Secure sandboxed compute for AI agents and workloads" ---- - - -Katakate logo - - -Katakate (k7) gives you production-grade, isolated sandboxes backed by Kata Containers and Firecracker, orchestrated with Kubernetes. Use the `k7` CLI to provision sandboxes, the REST API to manage them remotely, and the Python SDK to integrate into apps and agents. - - - -Get running in minutes: install, start API, create your first sandbox. - - -All `k7` commands with examples. - - -Sync and async clients with complete examples. - - -Endpoint overview, request/response formats, and errors. - - -Build a ReAct agent that executes inside a sandbox. - - diff --git a/docs/snippets/snippet-intro.mdx b/docs/snippets/snippet-intro.mdx deleted file mode 100644 index e20fbb6..0000000 --- a/docs/snippets/snippet-intro.mdx +++ /dev/null @@ -1,4 +0,0 @@ -One of the core principles of software development is DRY (Don't Repeat -Yourself). This is a principle that applies to documentation as -well. If you find yourself repeating the same content in multiple places, you -should consider creating a custom snippet to keep your content in sync. diff --git a/docs/tutorials/k7_hetzner_node_setup.pdf b/docs/tutorials/k7_hetzner_node_setup.pdf deleted file mode 100644 index d76d34f..0000000 Binary files a/docs/tutorials/k7_hetzner_node_setup.pdf and /dev/null differ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6d18510 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,85 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "k7" +version = "0.2.0" +description = "Self-hosted VM sandboxes for untrusted and AI code (CLI, API, SDK)" +readme = "README.md" +requires-python = ">=3.10.11" +dependencies = [ + "fastapi>=0.135.3", + "httpx>=0.28.1", + "kubernetes-asyncio>=31.1.0", + "pydantic>=2.12.5", + "python-dotenv>=1.2.2", + "python-multipart>=0.0.26", + "pyyaml>=6.0.3", + "requests>=2.32.3", + "rich>=14.3.3", + "typer>=0.24.1", + "uvicorn[standard]>=0.44.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/k7", "src/k7_sdk", "src/katakate"] + +[dependency-groups] +dev = [ + "ansible-lint>=26.4.0", + "pytest>=9.0.3", + "pytest-asyncio>=0.25.0", + "pytest-cov>=7.1.0", + "ruff>=0.15.10", + "ty>=0.0.29", +] + +# ---------- ruff ---------- +[tool.ruff] +src = ["src", "tests"] +target-version = "py310" +line-length = 120 + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B", "SIM"] +ignore = [ + "E501", # line length handled by formatter + "B008", # function call in defaults — standard typer/fastapi pattern + "B904", # raise-without-from in except (many existing patterns) + "SIM105", # contextlib.suppress — existing try/except/pass patterns are intentional + "SIM108", # ternary — readability preference +] + +[tool.ruff.lint.isort] +known-first-party = ["k7", "k7_sdk", "katakate"] + +# ---------- pytest ---------- +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "integration: tests requiring a live k7 node", + "firecracker: tests specific to kata-firecracker-devmapper backend", + "qemu: tests specific to kata-qemu-longhorn backend", + "k7d: tests specific to the k7d backend", + "multinode: tests requiring a multi-node k7 cluster (>=2 Ready nodes)", + "bench: spec-10b benchmark module; opt-in only (-m bench), not run by default", +] +addopts = [ + "--strict-markers", + "-m", "not integration and not bench", + "--cov=src/k7", + "--cov-report=term-missing", +] +asyncio_mode = "auto" + +# ---------- coverage ---------- +[tool.coverage.run] +source = ["src/k7"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if __name__ == .__main__.", + "pass", +] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..70d4373 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +ruff +ty +pytest +pytest-cov +ansible-lint +httpx diff --git a/setup.py b/setup.py index 32742d7..2324c07 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,27 @@ -from setuptools import setup, find_packages +from pathlib import Path + +from setuptools import find_packages, setup setup( - name="katakate", - version="0.0.4-dev", - description="Katakate Sandbox Management Python SDK", - packages=find_packages(where="src", include=["katakate", "katakate.*"]), + name="k7-sdk", + version="0.2.0", + description="K7 sandbox management Python SDK (HTTP client for the k7 API)", + long_description=Path(__file__).with_name("README.md").read_text(), + long_description_content_type="text/markdown", + url="https://github.com/Katakate/k7", + license="Apache-2.0", + packages=find_packages( + where="src", + include=["k7_sdk", "k7_sdk.*", "katakate", "katakate.*"], + ), package_dir={"": "src"}, include_package_data=True, install_requires=[ "requests>=2.31.0", ], extras_require={ - "sdk-async": ["httpx>=0.27.0"], + "async": ["httpx>=0.27.0"], + "sdk-async": ["httpx>=0.27.0"], # back-compat extra name }, python_requires=">=3.8", ) diff --git a/src/README.md b/src/README.md index f5e9dcb..eba47bb 100644 --- a/src/README.md +++ b/src/README.md @@ -1,2 +1,2 @@ -- `k7` is the .deb package containing the CLI and API, aimed to be used on the node(s), installable with `apt get`. -- `katakate` is the PyPI package containing the Python SDK, aimed to be used remotely e.g. from local, installable with `pip install katakate`. The SDK client talks to the API deployed on the node. \ No newline at end of file +- `k7` is the Debian package containing the CLI (and embedded installer playbook), used on Linux nodes: `apt install k7`. +- `k7-sdk` is the PyPI package (`import k7_sdk`) for the HTTP API client from your laptop or apps. The legacy `katakate` name is a one-release deprecation shim. diff --git a/src/k7/__init__.py b/src/k7/__init__.py index d88cf82..c5cc4ac 100644 --- a/src/k7/__init__.py +++ b/src/k7/__init__.py @@ -1,3 +1,3 @@ """K7 Sandbox Management System""" -__version__ = "0.0.4-dev" +__version__ = "0.2.0" diff --git a/src/k7/api/Dockerfile.api b/src/k7/api/Dockerfile.api index 8730b83..f55d576 100644 --- a/src/k7/api/Dockerfile.api +++ b/src/k7/api/Dockerfile.api @@ -9,18 +9,35 @@ RUN uv venv /app/.venv && \ . /app/.venv/bin/activate && \ uv pip install --no-cache -r requirements.txt +# crictl (pinned): k7d VM ops (pause/resume/fork) resolve a pod to its CRI +# sandbox id via `crictl pods` against the node's containerd socket, which +# the k7-api deployment hostPath-mounts (spec 18f issue 2). The node's own +# /usr/local/bin/crictl is a k3s symlink and can't be mounted usefully. +ARG CRICTL_VERSION=v1.31.1 +ARG CRICTL_SHA256=0a03ba6b1e4c253d63627f8d210b2ea07675a8712587e697657b236d06d7d231 +RUN curl -fsSL -o /tmp/crictl.tar.gz \ + "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" && \ + echo "${CRICTL_SHA256} /tmp/crictl.tar.gz" | sha256sum -c - && \ + tar -C /usr/local/bin -xzf /tmp/crictl.tar.gz crictl && \ + rm /tmp/crictl.tar.gz + FROM python:3.12-slim AS runtime ENV VIRTUAL_ENV=/app/.venv ENV PATH="/app/.venv/bin:$PATH" ENV PYTHONPATH="/app" WORKDIR /app +# lvm2: the k7-agent DaemonSet (same image, `k7.api.agent:app`) reports the +# kfd thin-pool utilization via `lvs` (spec 18g) — needs the privileged +# agent container with /dev hostPath-mounted. RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ + lvm2 \ && rm -rf /var/lib/apt/lists/* COPY --from=build /app/.venv /app/.venv COPY --from=build /app/k7 /app/k7 +COPY --from=build /usr/local/bin/crictl /usr/local/bin/crictl RUN useradd -m -u 1000 k7user && chown -R k7user:k7user /app USER k7user diff --git a/src/k7/api/agent.py b/src/k7/api/agent.py new file mode 100644 index 0000000..5210d3f --- /dev/null +++ b/src/k7/api/agent.py @@ -0,0 +1,173 @@ +"""Per-node k7 agent (spec 18g). + +Runs as a DaemonSet (``k7-agent``, kube-system) on every node, reusing the +``k7-api:local`` image with an overridden command +(``uvicorn k7.api.agent:app``). It exposes ONLY the node-local operations +that the centralized k7-api pod cannot perform for sandboxes on other +nodes: + +- ``POST /agent/v1/vm/{pause,resume,fork,lookup}`` — thin wrappers over the + ``K7Core`` k7d helpers (the k7d daemon socket, containerd socket, and + crictl are all node-local). +- ``GET /agent/v1/storage`` — node storage-pool utilization (kfd LVM + thin-pool via ``lvs``, k7d disks pool via ``df``). + +Auth: every request must carry the shared agent token (header +``X-K7-Agent-Token``) generated by the install playbook and distributed to +``/etc/k7/agent_token`` on every node. A CiliumNetworkPolicy additionally +restricts pod-originated ingress to the k7-api pod. Requests without a +valid token are rejected — never served. +""" + +import json +import os +import secrets +import subprocess + +from fastapi import Depends, FastAPI, Header, HTTPException, Request, status +from fastapi.responses import JSONResponse + +from .. import __version__ +from ..core.core import K7Core + +app = FastAPI(title="K7 Node Agent", version=__version__) + +AGENT_TOKEN_FILE = os.getenv("K7_AGENT_TOKEN_FILE", "/etc/k7/agent_token") +KATA_VG = os.getenv("K7_KATA_VG", "kata-vg") +K7D_DISKS_DIR = os.getenv("K7D_DISKS_DIR", "/var/lib/k7d/disks") + + +def _load_agent_token() -> str: + """Read the shared agent token — unreadable/empty is a deployment bug + and must fail loudly (a silent 401 would be misdiagnosed as a bad + caller token).""" + try: + with open(AGENT_TOKEN_FILE) as f: + token = f.read().strip() + except OSError as e: + raise HTTPException( + status_code=500, + detail=( + f"agent token {AGENT_TOKEN_FILE} is unreadable ({e}) — the install playbook " + "provisions it on every node; re-run `k7 install`" + ), + ) + if not token: + raise HTTPException(status_code=500, detail=f"agent token {AGENT_TOKEN_FILE} is empty — re-run `k7 install`") + return token + + +async def verify_agent_token(x_k7_agent_token: str | None = Header(None)): + if not x_k7_agent_token or not secrets.compare_digest(x_k7_agent_token.strip(), _load_agent_token()): + raise HTTPException(status_code=401, detail="Invalid or missing agent token") + + +@app.exception_handler(Exception) +async def unhandled_exception_handler(request: Request, exc: Exception): # type: ignore[override] + # Fail loud WITH the message — the k7-api forwarder surfaces it verbatim. + return JSONResponse( + content={"error": {"code": "InternalServerError", "message": str(exc)}}, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + +@app.get("/health") +async def health(): + return {"status": "healthy", "node": os.environ.get("K7_NODE_NAME", "")} + + +def _required_name(body: dict | None) -> tuple[str, str]: + body = body or {} + name = body.get("name") + if not name or not isinstance(name, str): + raise HTTPException(status_code=400, detail="name is required") + return name, body.get("namespace", "default") + + +@app.post("/agent/v1/vm/pause", dependencies=[Depends(verify_agent_token)]) +async def vm_pause(body: dict | None = None): + name, namespace = _required_name(body) + result = await K7Core().pause_sandbox(name=name, namespace=namespace, snapshot_name=(body or {}).get("snapshot")) + return result.to_dict() + + +@app.post("/agent/v1/vm/resume", dependencies=[Depends(verify_agent_token)]) +async def vm_resume(body: dict | None = None): + name, namespace = _required_name(body) + result = await K7Core().resume_sandbox(name=name, namespace=namespace) + return result.to_dict() + + +@app.post("/agent/v1/vm/fork", dependencies=[Depends(verify_agent_token)]) +async def vm_fork(body: dict | None = None): + name, namespace = _required_name(body) + new_name = (body or {}).get("new_name") + if not new_name or not isinstance(new_name, str): + raise HTTPException(status_code=400, detail="new_name is required") + result = await K7Core().fork_sandbox( + source_name=name, + new_name=new_name, + namespace=namespace, + snapshot_name=(body or {}).get("snapshot"), + ) + return result.to_dict() + + +@app.post("/agent/v1/vm/lookup", dependencies=[Depends(verify_agent_token)]) +async def vm_lookup(body: dict | None = None): + name, namespace = _required_name(body) + return await K7Core().lookup_k7d_vm(name, namespace) + + +# --------------------------------------------------------------------------- +# Node storage-pool utilization (spec 18g part 2 / 18f issue 5 leftover). +# --------------------------------------------------------------------------- + + +def _run(cmd: list[str]) -> str: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30, check=False) + if result.returncode != 0: + raise HTTPException( + status_code=500, + detail=f"{' '.join(cmd)} failed on node {os.environ.get('K7_NODE_NAME', '')}: {result.stderr.strip()}", + ) + return result.stdout + + +def _kata_thinpool() -> dict: + """kfd thin-pool utilization via ``lvs`` (needs a privileged container + with /dev hostPath-mounted — the DaemonSet provides both).""" + out = _run(["lvs", "--reportformat", "json", "--units", "b", "--nosuffix", KATA_VG]) + lvs = json.loads(out)["report"][0]["lv"] + pools = [lv for lv in lvs if lv["lv_attr"].startswith("t")] + if not pools: + raise HTTPException(status_code=500, detail=f"no thin pool LV found in VG {KATA_VG} (lvs returned {lvs})") + lv = pools[0] + return { + "vg": KATA_VG, + "lv": lv["lv_name"], + "size_bytes": int(float(lv["lv_size"])), + "data_percent": float(lv["data_percent"]), + "metadata_percent": float(lv["metadata_percent"]), + } + + +def _k7d_disks() -> dict: + """k7d disks-pool utilization via ``df`` on the hostPath-mounted + /var/lib/k7d/disks XFS loopback mount.""" + if not os.path.isdir(K7D_DISKS_DIR): + raise HTTPException(status_code=500, detail=f"{K7D_DISKS_DIR} not found — is the k7d backend installed?") + out = _run(["df", "--output=size,used,avail", "--block-size=1", K7D_DISKS_DIR]) + size, used, avail = out.splitlines()[1].split() + return { + "path": K7D_DISKS_DIR, + "size_bytes": int(size), + "used_bytes": int(used), + "avail_bytes": int(avail), + "used_percent": round(100.0 * int(used) / int(size), 2) if int(size) else 0.0, + } + + +@app.get("/agent/v1/storage", dependencies=[Depends(verify_agent_token)]) +async def storage(): + return {"kata_thinpool": _kata_thinpool(), "k7d_disks": _k7d_disks()} diff --git a/src/k7/api/docker-compose.yml b/src/k7/api/docker-compose.yml deleted file mode 100644 index 82d3332..0000000 --- a/src/k7/api/docker-compose.yml +++ /dev/null @@ -1,36 +0,0 @@ -services: - cloudflared: - image: cloudflare/cloudflared:latest - container_name: k7-cloudflared - restart: unless-stopped - command: tunnel --no-autoupdate --url http://k7-api:8000 - depends_on: - k7-api: - condition: service_healthy - networks: - - k7-network - - k7-api: - image: ${K7_API_IMAGE:-ghcr.io/katakate/k7-api}:${K7_API_TAG:-latest} - pull_policy: if_not_present - container_name: k7-api - restart: unless-stopped - user: "0:0" - volumes: - - /etc/rancher/k3s/k3s.yaml:/etc/rancher/k3s/k3s.yaml:ro - - /etc/k7:/etc/k7 - environment: - - KUBECONFIG=/etc/rancher/k3s/k3s.yaml - - K7_API_KEYS_FILE=/etc/k7/api_keys.json - healthcheck: - test: ["CMD-SHELL", "python -c 'import sys,urllib.request; sys.exit(0) if urllib.request.urlopen(\"http://127.0.0.1:8000/health\", timeout=2).status==200 else sys.exit(1)' "] - interval: 5s - timeout: 3s - retries: 10 - start_period: 10s - networks: - - k7-network - -networks: - k7-network: - driver: bridge \ No newline at end of file diff --git a/src/k7/api/main.py b/src/k7/api/main.py index a624fc3..4a5148d 100644 --- a/src/k7/api/main.py +++ b/src/k7/api/main.py @@ -1,16 +1,18 @@ -from fastapi import FastAPI, HTTPException, Depends, Header, status, Request -from fastapi.responses import JSONResponse -from typing import Optional, Any, Dict -import os -import json import hashlib +import json +import os import secrets import time +from datetime import timedelta from pathlib import Path +from typing import Any +from fastapi import Depends, FastAPI, Header, HTTPException, Request, status +from fastapi.responses import JSONResponse + +from .. import __version__ from ..core.core import K7Core from ..core.models import SandboxConfig -from .. import __version__ app = FastAPI(title="K7 Sandbox API", version=__version__) @@ -18,14 +20,29 @@ API_KEYS_FILE = Path(os.getenv("K7_API_KEYS_FILE", "/etc/k7/api_keys.json")) def load_api_keys() -> dict: - """Load API keys from file.""" + """Load API keys from file. + + A missing file means "no keys yet" — that is a normal state. An + *unreadable* file is a deployment bug (e.g. the store is not owned by + the API uid) and must fail loudly: silently returning {} would reject + every valid key with a misleading "Invalid API key". + """ if not API_KEYS_FILE.exists(): return {} try: - with open(API_KEYS_FILE, "r") as f: + with open(API_KEYS_FILE) as f: data = json.load(f) - except Exception: - return {} + except (PermissionError, OSError) as e: + raise HTTPException( + status_code=500, + detail=( + f"API key store {API_KEYS_FILE} is unreadable by the API process ({e}). " + "It must be owned by the k7-api container uid — regenerate a key with " + "`k7 generate-api-key` (which fixes ownership) on the node hosting the pod." + ), + ) + except json.JSONDecodeError as e: + raise HTTPException(status_code=500, detail=f"API key store {API_KEYS_FILE} is corrupt: {e}") # Purge expired keys opportunistically now_ts = int(time.time()) changed = False @@ -44,18 +61,21 @@ def save_api_keys(keys: dict): API_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True) with open(API_KEYS_FILE, "w") as f: json.dump(keys, f, indent=2) - os.chmod(API_KEYS_FILE, 0o600) + try: + os.chmod(API_KEYS_FILE, 0o600) + except OSError: + pass async def verify_api_key( - x_api_key: Optional[str] = Header(None), - authorization: Optional[str] = Header(None), + x_api_key: str | None = Header(None), + authorization: str | None = Header(None), ): """Verify API key via X-API-Key or Authorization: Bearer header. Uses timing-attack-resistant comparison and updates last_used on success. """ - token: Optional[str] = None + token: str | None = None if x_api_key and x_api_key.strip(): token = x_api_key.strip() elif authorization and authorization.lower().startswith("bearer "): @@ -90,7 +110,9 @@ async def verify_api_key( return valid_data -def success_response(data: Any, status_code: int = status.HTTP_200_OK, headers: Dict[str, str] | None = None) -> JSONResponse: +def success_response( + data: Any, status_code: int = status.HTTP_200_OK, headers: dict[str, str] | None = None +) -> JSONResponse: return JSONResponse(content={"data": data}, status_code=status_code, headers=headers) @@ -139,7 +161,7 @@ async def create_sandbox(config: dict): try: sandbox_config = SandboxConfig.from_dict(config) core = K7Core() - result = core.create_sandbox(sandbox_config) + result = await core.create_sandbox(sandbox_config) if result.success: resource = { @@ -156,10 +178,10 @@ async def create_sandbox(config: dict): @app.get("/api/v1/sandboxes", dependencies=[Depends(verify_api_key)]) -async def list_sandboxes(namespace: Optional[str] = None): +async def list_sandboxes(namespace: str | None = None): """List all sandboxes.""" core = K7Core() - sandboxes = core.list_sandboxes(namespace) + sandboxes = await core.list_sandboxes(namespace) return success_response([sandbox.to_dict() for sandbox in sandboxes]) @@ -167,7 +189,7 @@ async def list_sandboxes(namespace: Optional[str] = None): async def get_sandbox(name: str, namespace: str = "default"): """Get a single sandbox by name.""" core = K7Core() - items = core.list_sandboxes(namespace) + items = await core.list_sandboxes(namespace) for s in items: if s.name == name: return success_response(s.to_dict()) @@ -178,7 +200,7 @@ async def get_sandbox(name: str, namespace: str = "default"): async def delete_sandbox(name: str, namespace: str = "default"): """Delete a sandbox.""" core = K7Core() - result = core.delete_sandbox(name, namespace) + result = await core.delete_sandbox(name, namespace) if result.success: return success_response({"message": result.message}) @@ -190,7 +212,7 @@ async def delete_sandbox(name: str, namespace: str = "default"): async def delete_all_sandboxes(namespace: str = "default"): """Delete all sandboxes in a namespace.""" core = K7Core() - result = core.delete_all_sandboxes(namespace) + result = await core.delete_all_sandboxes(namespace) if result.success: return success_response({"message": result.message, "results": result.data}) @@ -198,6 +220,100 @@ async def delete_all_sandboxes(namespace: str = "default"): raise HTTPException(status_code=400, detail=result.error) +@app.post("/api/v1/sandboxes/{name}/pause", dependencies=[Depends(verify_api_key)]) +async def pause_sandbox(name: str, body: dict | None = None): + """Pause a sandbox (scale to 0) and optionally take a Longhorn VolumeSnapshot. + + Body keys (all optional): + ``namespace`` (default ``"default"``), + ``snapshot`` (when set, snapshot the sandbox's root PVC under this name). + """ + body = body or {} + namespace = body.get("namespace", "default") + core = K7Core() + result = await core.pause_sandbox( + name=name, + namespace=namespace, + snapshot_name=body.get("snapshot"), + ) + if result.success: + return success_response({"message": result.message}) + raise HTTPException(status_code=400, detail=result.error) + + +@app.post("/api/v1/sandboxes/{name}/resume", dependencies=[Depends(verify_api_key)]) +async def resume_sandbox(name: str, body: dict | None = None): + """Resume a paused sandbox (scale back to 1).""" + body = body or {} + namespace = body.get("namespace", "default") + core = K7Core() + result = await core.resume_sandbox(name=name, namespace=namespace) + if result.success: + return success_response({"message": result.message}) + raise HTTPException(status_code=400, detail=result.error) + + +@app.post("/api/v1/sandboxes/{name}/fork", dependencies=[Depends(verify_api_key)]) +async def fork_sandbox(name: str, body: dict): + """Fork a kata-qemu-longhorn sandbox into a new name with a cloned root disk. + + Required body key: new_name. Optional: namespace, snapshot. + The handler blocks until the cloned PVC is bound (matches CLI behaviour). + """ + new_name = (body or {}).get("new_name") + if not new_name or not isinstance(new_name, str): + raise HTTPException(status_code=400, detail="new_name is required") + namespace = body.get("namespace", "default") + snapshot = body.get("snapshot") + core = K7Core() + result = await core.fork_sandbox( + source_name=name, + new_name=new_name, + namespace=namespace, + snapshot_name=snapshot, + ) + if result.success: + resource = { + "name": new_name, + "namespace": namespace, + "source": name, + "message": result.message, + } + location = f"/api/v1/sandboxes/{new_name}?namespace={namespace}" + return success_response(resource, status_code=status.HTTP_201_CREATED, headers={"Location": location}) + err = (result.error or "").lower() + if "already exists" in err: + raise HTTPException(status_code=409, detail=result.error) + if "not found" in err: + raise HTTPException(status_code=404, detail=result.error) + raise HTTPException(status_code=400, detail=result.error) + + +@app.get("/api/v1/sandboxes/{name}/logs", dependencies=[Depends(verify_api_key)]) +async def get_sandbox_logs( + name: str, + namespace: str = "default", + container: str = "sandbox", + tail: int = 200, + since: int = 0, +): + """Read pod logs (snapshot; no streaming yet — see Spec 10g risks).""" + core = K7Core() + result = await core.get_logs( + sandbox_name=name, + namespace=namespace, + container=container, + tail_lines=tail if tail > 0 else None, + since_seconds=since if since > 0 else None, + ) + if result.success: + return success_response(result.data or {"logs": ""}) + err = (result.error or "").lower() + if "no pod found" in err or "not found" in err: + raise HTTPException(status_code=404, detail=result.error) + raise HTTPException(status_code=400, detail=result.error) + + @app.post("/api/v1/sandboxes/{name}/exec", dependencies=[Depends(verify_api_key)]) async def exec_command(name: str, command_data: dict, namespace: str = "default"): """Execute a command in a sandbox.""" @@ -206,7 +322,7 @@ async def exec_command(name: str, command_data: dict, namespace: str = "default" raise HTTPException(status_code=400, detail="Command is required") core = K7Core() - result = core.exec_command(name, command, namespace) + result = await core.exec_command(name, command, namespace) return success_response(result.to_dict()) @@ -226,9 +342,176 @@ async def install_node(install_data: dict): raise HTTPException(status_code=400, detail=result.error) +@app.get("/api/v1/nodes/storage", dependencies=[Depends(verify_api_key)]) +async def get_nodes_storage(): + """Per-node storage-pool utilization (kfd thin-pool + k7d disks pool), + aggregated from the k7-agent DaemonSet (spec 18g). A node whose agent + is unreachable gets an ``{"error": ...}`` entry — never omitted.""" + core = K7Core() + return success_response(await core.nodes_storage()) + + @app.get("/api/v1/sandboxes/metrics", dependencies=[Depends(verify_api_key)]) -async def get_sandbox_metrics(namespace: Optional[str] = None): +async def get_sandbox_metrics(namespace: str | None = None): """Get resource usage metrics for sandboxes.""" core = K7Core() - metrics = core.get_sandbox_metrics(namespace) + metrics = await core.get_sandbox_metrics(namespace) return success_response(metrics) + + +# --------------------------------------------------------------------------- +# Spec 10e: VolumeSnapshot lifecycle endpoints. +# --------------------------------------------------------------------------- + + +def _parse_keep_fork_for(value: str | None) -> timedelta: + """Accept ``10m`` / ``2h`` / ``3600`` (seconds) — fail loudly on garbage.""" + if value is None or value == "": + return timedelta(minutes=10) + if value.endswith("m"): + return timedelta(minutes=int(value[:-1])) + if value.endswith("h"): + return timedelta(hours=int(value[:-1])) + if value.endswith("s"): + return timedelta(seconds=int(value[:-1])) + return timedelta(seconds=int(value)) + + +@app.get("/api/v1/snapshots", dependencies=[Depends(verify_api_key)]) +async def list_snapshots( + namespace: str | None = "default", + all_namespaces: bool = False, + sandbox: str | None = None, + kind: str | None = None, +): + """List VolumeSnapshots, optionally filtered by namespace / sandbox / kind.""" + core = K7Core() + snaps = await core.list_snapshots( + namespace=namespace, + all_namespaces=all_namespaces, + sandbox=sandbox, + kind=kind, + ) + return success_response([s.to_dict() for s in snaps]) + + +@app.get("/api/v1/snapshots/{name}", dependencies=[Depends(verify_api_key)]) +async def get_snapshot(name: str, namespace: str = "default"): + """Inspect a single VolumeSnapshot by name.""" + core = K7Core() + snap = await core.get_snapshot(name, namespace=namespace) + if snap is None: + raise HTTPException(status_code=404, detail=f"Snapshot {name} not found in namespace {namespace}") + return success_response(snap.to_dict()) + + +@app.post("/api/v1/sandboxes/{name}/snapshot", dependencies=[Depends(verify_api_key)]) +async def create_snapshot(name: str, body: dict): + """Snapshot a running sandbox's root PVC without pausing it (kind=named). + + Body keys: ``snapshot_name`` (required), ``namespace`` (default ``"default"``). + """ + snapshot_name = (body or {}).get("snapshot_name") + if not snapshot_name or not isinstance(snapshot_name, str): + raise HTTPException(status_code=400, detail="snapshot_name is required") + namespace = body.get("namespace", "default") + core = K7Core() + result = await core.create_snapshot(sandbox_name=name, snapshot_name=snapshot_name, namespace=namespace) + if result.success: + resource = {"name": snapshot_name, "namespace": namespace, "source_sandbox": name} + location = f"/api/v1/snapshots/{snapshot_name}?namespace={namespace}" + return success_response(resource, status_code=status.HTTP_201_CREATED, headers={"Location": location}) + err = (result.error or "").lower() + if "already exists" in err: + raise HTTPException(status_code=409, detail=result.error) + raise HTTPException(status_code=400, detail=result.error) + + +@app.delete("/api/v1/snapshots/{name}", dependencies=[Depends(verify_api_key)]) +async def delete_snapshot(name: str, namespace: str = "default"): + """Delete a VolumeSnapshot by name.""" + core = K7Core() + result = await core.delete_snapshot(name, namespace=namespace) + if result.success: + return success_response({"message": result.message}) + if "not found" in (result.error or "").lower(): + raise HTTPException(status_code=404, detail=result.error) + raise HTTPException(status_code=400, detail=result.error) + + +@app.post("/api/v1/snapshots/{name}/restore", dependencies=[Depends(verify_api_key)]) +async def restore_snapshot(name: str, body: dict): + """Boot a brand-new sandbox from a standalone VolumeSnapshot (Spec 10f). + + Body keys: + ``new_sandbox_name`` (required), + ``namespace`` (default ``"default"``), + ``overrides`` (optional dict: image, backend, root_disk_size, sidecar, + limits, entrypoint, cmd, before_script), + ``keep_snapshot`` (default ``true``). + """ + body = body or {} + new_name = body.get("new_sandbox_name") + if not new_name or not isinstance(new_name, str): + raise HTTPException(status_code=400, detail="new_sandbox_name is required") + namespace = body.get("namespace", "default") + keep_snapshot = bool(body.get("keep_snapshot", True)) + + overrides_dict = body.get("overrides") or {} + if not isinstance(overrides_dict, dict): + raise HTTPException(status_code=400, detail="overrides must be a JSON object") + # Filter to known SandboxConfigOverrides keys; ignore garbage. + allowed = {"image", "backend", "root_disk_size", "sidecar", "limits", "entrypoint", "cmd", "before_script"} + filtered = {k: v for k, v in overrides_dict.items() if k in allowed} + from k7.core.models import SandboxConfigOverrides + + overrides = SandboxConfigOverrides(**filtered) if filtered else None + + core = K7Core() + result = await core.restore_sandbox( + snapshot_name=name, + new_sandbox_name=new_name, + namespace=namespace, + overrides=overrides, + keep_snapshot=keep_snapshot, + ) + if result.success: + resource = { + "name": new_name, + "namespace": namespace, + "source_snapshot": name, + "message": result.message, + } + location = f"/api/v1/sandboxes/{new_name}?namespace={namespace}" + return success_response(resource, status_code=status.HTTP_201_CREATED, headers={"Location": location}) + + err = (result.error or "").lower() + if "not found" in err: + raise HTTPException(status_code=404, detail=result.error) + if "already exists" in err: + raise HTTPException(status_code=409, detail=result.error) + raise HTTPException(status_code=400, detail=result.error) + + +@app.post("/api/v1/snapshots/gc", dependencies=[Depends(verify_api_key)]) +async def gc_snapshots(body: dict | None = None): + """Sweep stale ``kind=fork`` snapshots older than ``keep_fork_for``. + + Body (all optional): + ``namespace`` (default ``"default"``), + ``all_namespaces`` (default ``false``), + ``keep_fork_for`` (default ``"10m"``, also accepts ``2h`` / ``45s`` / plain seconds), + ``dry_run`` (default ``false``). + """ + body = body or {} + keep_for = _parse_keep_fork_for(body.get("keep_fork_for")) + core = K7Core() + result = await core.gc_snapshots( + namespace=body.get("namespace", "default"), + all_namespaces=bool(body.get("all_namespaces", False)), + keep_fork_for=keep_for, + dry_run=bool(body.get("dry_run", False)), + ) + if result.success: + return success_response({"message": result.message, "results": result.data}) + raise HTTPException(status_code=400, detail=result.error) diff --git a/src/k7/api/requirements.txt b/src/k7/api/requirements.txt index 1b58b11..9d6c68f 100644 --- a/src/k7/api/requirements.txt +++ b/src/k7/api/requirements.txt @@ -1,10 +1,10 @@ fastapi==0.104.1 uvicorn[standard]==0.24.0 -kubernetes==28.1.0 +kubernetes-asyncio==31.1.0 pydantic==2.5.0 python-multipart==0.0.6 -requests==2.31.0 +httpx==0.28.1 typer==0.9.0 rich==13.7.0 pyyaml==6.0.1 -python-dotenv==1.0.0 \ No newline at end of file +python-dotenv==1.0.0 diff --git a/src/k7/api/snapshot_gc.py b/src/k7/api/snapshot_gc.py new file mode 100644 index 0000000..cadbf62 --- /dev/null +++ b/src/k7/api/snapshot_gc.py @@ -0,0 +1,67 @@ +"""Snapshot garbage-collection entrypoint (Spec 10e, Option C backstop). + +Run as ``python -m k7.api.snapshot_gc`` inside the ``k7-api`` container. +The accompanying CronJob (``snapshot-gc-cronjob.yaml``) invokes this +every 10 minutes to clean up stale ``kind=fork`` VolumeSnapshots that +the inline cleanup in ``fork_sandbox`` missed (e.g. due to an API pod +crash mid-fork). + +Behaviour: + +- Lists snapshots cluster-wide (``all_namespaces=True``). +- Honours the same ``keep_fork_for`` window as :meth:`K7Core.gc_snapshots`. +- Skips anything that isn't ``kind=fork`` — pause and named snapshots are + never touched. +- Environment overrides: ``K7_GC_KEEP_FORK_FOR_MINUTES`` (default ``10``), + ``K7_GC_DRY_RUN`` (``true``/``false``, default ``false``). +""" + +from __future__ import annotations + +import asyncio +import os +import sys +from datetime import timedelta + +from k7.core.core import K7Core + + +def _env_int(key: str, default: int) -> int: + raw = os.environ.get(key) + if raw is None or raw == "": + return default + try: + return int(raw) + except ValueError: + print(f"⚠️ Invalid integer for {key}={raw!r}; using default {default}", file=sys.stderr) + return default + + +def _env_bool(key: str, default: bool) -> bool: + raw = os.environ.get(key, "").strip().lower() + if raw == "": + return default + return raw in ("1", "true", "yes", "y", "on") + + +async def _main() -> int: + keep_minutes = _env_int("K7_GC_KEEP_FORK_FOR_MINUTES", 10) + dry_run = _env_bool("K7_GC_DRY_RUN", False) + core = K7Core() + result = await core.gc_snapshots( + all_namespaces=True, + keep_fork_for=timedelta(minutes=keep_minutes), + dry_run=dry_run, + ) + if not result.success: + print(f"❌ snapshot-gc failed: {result.error}", file=sys.stderr) + return 1 + print(result.message) + for record in result.data or []: + marker = "would-delete" if dry_run else ("deleted" if record.get("deleted") else "failed") + print(f" [{marker}] {record['namespace']}/{record['name']} (age={record['age']})") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(_main())) diff --git a/src/k7/assets/__init__.py b/src/k7/assets/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/k7/assets/__init__.py @@ -0,0 +1 @@ + diff --git a/src/k7/assets/k7-persist-bind.sh b/src/k7/assets/k7-persist-bind.sh new file mode 100644 index 0000000..c7b4af2 --- /dev/null +++ b/src/k7/assets/k7-persist-bind.sh @@ -0,0 +1,97 @@ +#!/bin/sh +# shellcheck disable=SC3040 # pipefail: guarded probe, not assumed +set -eu +(set -o pipefail) 2>/dev/null && set -o pipefail || true + +SCRIPT_START=$(date +%s) + +STATE_MNT="/mnt/state" +SLOT="${K7_PERSIST_SLOT:?missing K7_PERSIST_SLOT}" + +READY_FILE="${STATE_MNT}/.ready" +BASE="${STATE_MNT}/${SLOT}" +MIGRATION_MARK="${BASE}/.migrated_v1" + +i=0 +while [ ! -f "${READY_FILE}" ]; do + i=$((i+1)) + if [ "$i" -gt 600 ]; then + echo "k7-persist: timeout waiting for ${READY_FILE}" >&2 + exit 1 + fi + sleep 0.1 +done + +mkdir -p "${BASE}" + +is_symlink() { [ -L "$1" ]; } +is_dir() { [ -d "$1" ]; } + +require_tool() { + tool="$1" + if ! command -v "$tool" >/dev/null 2>&1; then + echo "k7-persist: missing required tool '$tool' in image" >&2 + exit 1 + fi +} + +require_tool tar +require_tool mount +if [ ! -x /bin/sh ]; then + echo "k7-persist: missing required shell /bin/sh in image" >&2 + exit 1 +fi + +copy_if_empty() { + src="$1" + dst="$2" + mkdir -p "$dst" + if [ -z "$(ls -A "$dst" 2>/dev/null || true)" ]; then + echo "k7-persist: seeding $dst from $src (one-time)..." + (cd "$src" && tar -cpf - .) | (cd "$dst" && tar -xpf -) + fi +} + +bind_mount_dir() { + target="$1" + name="$2" + if [ ! -e "$target" ]; then + return 0 + fi + if is_symlink "$target"; then + echo "k7-persist: skip symlink $target" + return 0 + fi + if ! is_dir "$target"; then + echo "k7-persist: skip non-dir $target" + return 0 + fi + persist="${BASE}/${name}" + copy_if_empty "$target" "$persist" + mount --make-rprivate / || true + mount --bind "$persist" "$target" + echo "k7-persist: bound $target -> $persist" +} + +bind_mount_dir /etc etc +bind_mount_dir /var var +bind_mount_dir /usr usr +bind_mount_dir /home home +bind_mount_dir /root root +bind_mount_dir /opt opt +bind_mount_dir /bin bin +bind_mount_dir /sbin sbin +bind_mount_dir /lib lib +bind_mount_dir /lib64 lib64 + +SCRIPT_END=$(date +%s) +SCRIPT_ELAPSED=$((SCRIPT_END - SCRIPT_START)) +echo "k7-persist: script completed in ${SCRIPT_ELAPSED}s" >&2 + +date > "${MIGRATION_MARK}" 2>/dev/null || true +if [ "$#" -eq 0 ]; then + echo "k7-persist: no command provided; defaulting to keepalive sleep" >&2 + exec sleep 365d +fi +exec "$@" + diff --git a/src/k7/cli/Dockerfile.cli b/src/k7/cli/Dockerfile.cli index 9d3cdbe..91afe29 100644 --- a/src/k7/cli/Dockerfile.cli +++ b/src/k7/cli/Dockerfile.cli @@ -4,10 +4,13 @@ RUN apt-get update && apt-get install -y python3 python3-pip python3-venv gcc pa WORKDIR /app RUN python3 -m venv /app/venv ENV PATH="/app/venv/bin:$PATH" -RUN pip install --no-cache-dir nuitka typer kubernetes python-dotenv pyyaml rich +RUN pip install --no-cache-dir nuitka typer kubernetes kubernetes_asyncio httpx python-dotenv pyyaml rich requests -# Copy source package into build context (src layout) +# Copy source packages into build context (src layout). The CLI imports +# k7_sdk (its API client since the SDK rename) — without it the onefile +# binary dies at import time with ModuleNotFoundError. COPY src/k7/ /app/k7/ +COPY src/k7_sdk/ /app/k7_sdk/ # Build the binary with deploy assets embedded RUN python3 -m nuitka \ @@ -16,6 +19,10 @@ RUN python3 -m nuitka \ --include-module=rich \ --include-module=typer \ --include-module=kubernetes \ + --include-package=kubernetes_asyncio \ + --include-package=httpx \ --include-module=dotenv \ + --include-package=k7_sdk \ + --include-package=requests \ --include-data-dir=k7=k7 \ k7/cli/k7.py \ No newline at end of file diff --git a/src/k7/cli/_client.py b/src/k7/cli/_client.py new file mode 100644 index 0000000..c689d18 --- /dev/null +++ b/src/k7/cli/_client.py @@ -0,0 +1,202 @@ +"""CLI ↔ API client plumbing (Spec 10g). + +This module owns: + +- ``_resolve_api_url`` / ``_resolve_api_key`` — the four-stage lookup + chain (flag → env → config file → ``/etc/k7/api_*``) shared by every + CLI handler that talks to the API. +- ``CliContext`` — the dataclass carried on ``typer.Context.obj``. Holds + either a configured ``katakate.Client`` (the default) or a flag + signalling the ``--core`` escape hatch (direct ``K7Core`` calls, no + HTTP, used by integration tests and node-side debugging). +- ``ApiUnreachable`` — a small exception type the CLI catches to map + ``requests.ConnectionError`` / ``Timeout`` to a single helpful exit + message ("could not reach API at : ..."). +- ``handle_api_call(fn)`` — decorator-style helper that runs an SDK + call, maps HTTP / connection errors to ``typer.Exit(1)``, and surfaces + the server's ``{"error": {"message": ...}}`` body when present. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import TypeVar + +import requests +import typer + +from k7_sdk.client import Client + +from ._config import get_config_value + +# --------------------------------------------------------------------------- +# Endpoint + key resolution. +# --------------------------------------------------------------------------- + +_ETC_API_ENDPOINT = Path("/etc/k7/api_endpoint") +_ETC_API_KEYS = Path("/etc/k7/api_keys.json") + + +def _resolve_api_url(flag: str | None) -> str | None: + """Look up the API URL through the documented chain. + + Precedence: ``--api-url`` flag > ``K7_API_URL`` env var > config file > + ``/etc/k7/api_endpoint`` (on a cluster node). Returns ``None`` when + nothing is set; the caller decides whether that's an error. + """ + if flag: + return flag + env = os.environ.get("K7_API_URL") + if env: + return env + cfg = get_config_value("api.url") + if cfg: + return cfg + if _ETC_API_ENDPOINT.exists(): + try: + v = _ETC_API_ENDPOINT.read_text().strip() + if v: + return v + except OSError: + pass + return None + + +def _resolve_api_key(flag: str | None) -> str | None: + """Same chain as ``_resolve_api_url`` but for the API key. + + Cluster-node fallback reads ``/etc/k7/api_keys.json`` (the same file + ``generate_api_key`` writes to) and picks the **first** entry's raw + token. Note: that file stores SHA-256 hashes, not raw keys, so the + fallback only works when k7 wrote the raw key alongside (current + layout: each entry has both ``hash`` and ``token`` fields when + created by ``k7 generate-api-key``). If only hashes are present the + fallback fails gracefully and the user must pass ``--api-key`` or + set ``K7_API_KEY``. + """ + if flag: + return flag + env = os.environ.get("K7_API_KEY") + if env: + return env + cfg = get_config_value("api.key") + if cfg: + return cfg + if _ETC_API_KEYS.exists(): + try: + data = json.loads(_ETC_API_KEYS.read_text()) + except (OSError, json.JSONDecodeError): + return None + # File layout: {sha256_hash: {"name": ..., "token": "...", ...}, ...}. + # Newer entries store the raw token under ``token``; pick the first + # one we find (deterministic since dict preserves insertion order). + if isinstance(data, dict): + for entry in data.values(): + if isinstance(entry, dict): + token = entry.get("token") + if isinstance(token, str) and token: + return token + return None + + +# --------------------------------------------------------------------------- +# CliContext + adapters. +# --------------------------------------------------------------------------- + + +@dataclass +class CliContext: + """The ``typer.Context.obj`` populated by ``@app.callback()``. + + The ``katakate.Client`` is built **lazily** by :meth:`client`, so + handlers that don't talk to the API (``install``, ``start-api``, + ``config set`` …) never trigger the missing-URL / missing-key + error paths. + + ``use_core=True`` flips the CLI to direct ``K7Core`` calls (the + legacy in-process path) — useful for integration tests on the node + and for debugging when the API itself is misbehaving. + """ + + use_core: bool = False + api_url: str | None = None + api_key: str | None = None + _client: Client | None = None + + def client(self) -> Client: + """Resolve and cache the SDK client; exits 1 when the URL / key are unset.""" + if self._client is None: + self._client = resolve_client(self.api_url, self.api_key) + return self._client + + +class ApiUnreachable(RuntimeError): + """Raised when the SDK can't reach the API at all (connection / timeout).""" + + +def resolve_client(api_url: str | None, api_key: str | None) -> Client: + """Build a ``katakate.Client`` from the resolved URL + key. + + Raises ``typer.Exit(1)`` with a pointed message when either is + missing — the CLI should never silently fall through to a broken + Client constructor. + """ + url = _resolve_api_url(api_url) + if not url: + typer.echo( + "❌ No API URL configured. Pass --api-url, set K7_API_URL, " + "or run `k7 config set api.url https://:`.", + err=True, + ) + raise typer.Exit(1) + key = _resolve_api_key(api_key) + if not key: + typer.echo( + "❌ No API key configured. Pass --api-key, set K7_API_KEY, " + "or run `k7 config set api.key ` " + "(generate one with `k7 generate-api-key ` on the node).", + err=True, + ) + raise typer.Exit(1) + # NodePort exposes plain HTTP; off-cluster setups should put the API + # behind ingress + TLS, in which case verify_ssl=True (the default) + # does the right thing. For local/demo HTTP, the SDK skips verify. + return Client(endpoint=url, api_key=key, verify_ssl=url.startswith("https://")) + + +T = TypeVar("T") + + +def handle_api_call(fn: Callable[[], T]) -> T: + """Run an SDK call and exit-1 on the documented failure shapes. + + Translates ``requests.ConnectionError`` / ``Timeout`` into a single + "could not reach API" message and maps ``HTTPError`` to the server's + ``{"error": {"message": ...}}`` body when present (otherwise the + raw ``status_code``). + """ + try: + return fn() + except requests.ConnectionError as e: + typer.echo(f"❌ Could not reach the K7 API: {e}", err=True) + raise typer.Exit(1) from None + except requests.Timeout as e: + typer.echo(f"❌ Timed out talking to the K7 API: {e}", err=True) + raise typer.Exit(1) from None + except requests.HTTPError as e: + message: str | None = None + if e.response is not None: + try: + body = e.response.json() + if isinstance(body, dict) and isinstance(body.get("error"), dict): + message = body["error"].get("message") + except ValueError: + pass + if not message: + message = f"HTTP {e.response.status_code}: {e.response.text[:300]}" + typer.echo(f"❌ {message or e}", err=True) + raise typer.Exit(1) from None diff --git a/src/k7/cli/_config.py b/src/k7/cli/_config.py new file mode 100644 index 0000000..84dc9fa --- /dev/null +++ b/src/k7/cli/_config.py @@ -0,0 +1,217 @@ +"""TOML config file CRUD + ``k7 config`` sub-app (Spec 10g). + +Stores per-user CLI config at ``$XDG_CONFIG_HOME/k7/config.toml`` +(falling back to ``~/.config/k7/config.toml``). The single supported +section today is ``[api]`` with ``url`` and ``key`` keys — enough to +power the ``_resolve_api_url`` / ``_resolve_api_key`` chain in +``_client.py`` so users don't have to pass ``--api-url`` / +``--api-key`` on every invocation. + +The file is the same posture as ``~/.docker/config.json`` and +``~/.kube/config`` — plaintext, chmod 0600 on write. A future spec +can layer OS keychain integration on top. +""" + +from __future__ import annotations + +import builtins +import json +import os +import re +import sys +from pathlib import Path + +import typer + +CONFIG_DIR_ENV = "K7_CONFIG_DIR" # tests override this to a tmp path +_SUPPORTED_KEYS: set[str] = {"api.url", "api.key"} + + +def _config_dir() -> Path: + """Return the directory holding ``config.toml``. + + Honours ``K7_CONFIG_DIR`` (tests / explicit overrides), then + ``XDG_CONFIG_HOME``, then ``~/.config/k7``. + """ + explicit = os.environ.get(CONFIG_DIR_ENV) + if explicit: + return Path(explicit) + xdg = os.environ.get("XDG_CONFIG_HOME") + base = Path(xdg) if xdg else Path.home() / ".config" + return base / "k7" + + +def config_file_path() -> Path: + return _config_dir() / "config.toml" + + +_SECTION_RE = re.compile(r"^\s*\[\s*([A-Za-z0-9_.-]+)\s*\]\s*$") +_KEY_RE = re.compile(r'^\s*([A-Za-z0-9_-]+)\s*=\s*"((?:[^"\\]|\\.)*)"\s*$') + + +def _toml_loads_simple(text: str) -> dict: + """Tiny TOML reader for the section-of-string-values shape this file uses. + + Python 3.10 (k7's minimum) lacks ``tomllib`` and we don't want to add + ``tomli`` just for two-key parsing. Supports:: + + [api] + url = "https://10.0.0.1:31000" + key = "k7-..." + + Unknown / malformed lines are ignored (the user's view is "k7 config + wrote the file; k7 config can read it back" — anything outside that + contract is best-effort). + """ + data: dict = {} + current: dict | None = None + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + m = _SECTION_RE.match(line) + if m: + current = data.setdefault(m.group(1), {}) + continue + m = _KEY_RE.match(line) + if not m or current is None: + continue + value = m.group(2).encode("utf-8").decode("unicode_escape") + current[m.group(1)] = value + return data + + +def read_config() -> dict: + """Return the parsed config dict; ``{}`` when the file is absent / unreadable.""" + p = config_file_path() + if not p.exists(): + return {} + try: + return _toml_loads_simple(p.read_text()) + except Exception: + # A malformed config file shouldn't crash the CLI — fall back to no config + # and let the next resolution layer (env vars / on-node fallback) take over. + return {} + + +def _get_dotted(d: dict, key: str) -> str | None: + """Look up ``"api.url"`` against ``{"api": {"url": "..."}}``.""" + section, _, leaf = key.partition(".") + if not leaf: + return None + sub = d.get(section) + if not isinstance(sub, dict): + return None + val = sub.get(leaf) + return str(val) if val is not None else None + + +def _set_dotted(d: dict, key: str, value: str) -> None: + section, _, leaf = key.partition(".") + if not leaf: + raise ValueError(f"Invalid config key {key!r}; expected 'section.field' form") + sub = d.setdefault(section, {}) + if not isinstance(sub, dict): # pragma: no cover - defensive against malformed user files + sub = {} + d[section] = sub + sub[leaf] = value + + +def _toml_dumps(d: dict) -> str: + """Serialise a simple ``{section: {key: str}}`` dict to TOML. + + We don't pull in ``tomli_w`` for this — k7 only ever writes flat + string fields under a single ``[api]`` section. Keep the surface + tiny on purpose. + """ + lines: builtins.list[str] = [] + for section in sorted(d.keys()): + body = d[section] + if not isinstance(body, dict): + continue + lines.append(f"[{section}]") + for key in sorted(body.keys()): + value = body[key] + if value is None: + continue + lines.append(f"{key} = {json.dumps(str(value))}") + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def write_config(data: dict) -> Path: + """Persist ``data`` to disk, creating the directory if needed (mode 0600).""" + p = config_file_path() + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(_toml_dumps(data)) + try: + os.chmod(p, 0o600) + except OSError: + pass + return p + + +def get_config_value(key: str) -> str | None: + return _get_dotted(read_config(), key) + + +def set_config_value(key: str, value: str) -> Path: + if key not in _SUPPORTED_KEYS: + raise ValueError(f"Unknown config key {key!r}. Supported keys: {', '.join(sorted(_SUPPORTED_KEYS))}") + data = read_config() + _set_dotted(data, key, value) + return write_config(data) + + +# --------------------------------------------------------------------------- +# ``k7 config`` Typer sub-app. +# --------------------------------------------------------------------------- + + +config_app = typer.Typer( + help="Read / write per-user CLI configuration.", + context_settings={"help_option_names": ["-h", "--help"]}, +) + + +@config_app.command("set") +def config_set( + key: str = typer.Argument(..., help="Config key in 'section.field' form (e.g. api.url)"), + value: str = typer.Argument(..., help="Value to store"), +): + """Set a config value (persists to ~/.config/k7/config.toml).""" + try: + path = set_config_value(key, value) + except ValueError as e: + typer.echo(f"❌ {e}", err=True) + raise typer.Exit(1) from None + redacted = "" if key.endswith(".key") else value + typer.echo(f"Wrote {key} = {redacted} to {path}") + + +@config_app.command("get") +def config_get( + key: str = typer.Argument(..., help="Config key in 'section.field' form"), +): + """Print one config value. Exits 1 when the key is unset.""" + value = get_config_value(key) + if value is None: + typer.echo("", err=False) + raise typer.Exit(1) + typer.echo(value) + + +@config_app.command("show") +def config_show(): + """Print the full config (API keys are redacted).""" + data = read_config() + if not data: + typer.echo(f"(empty — config file: {config_file_path()})") + return + # Redact secrets for terminal display. + redacted_view: dict = {} + for section, body in data.items(): + if not isinstance(body, dict): + continue + redacted_view[section] = {k: ("" if k == "key" else v) for k, v in body.items()} + sys.stdout.write(_toml_dumps(redacted_view)) diff --git a/src/k7/cli/build.sh b/src/k7/cli/build.sh index 921d0cc..e43b086 100755 --- a/src/k7/cli/build.sh +++ b/src/k7/cli/build.sh @@ -14,11 +14,9 @@ ARCH=$(uname -m) case "$ARCH" in x86_64) DEB_ARCH="amd64" - DOCKER_PLATFORM="linux/amd64" ;; aarch64|arm64) DEB_ARCH="arm64" - DOCKER_PLATFORM="linux/arm64" ;; *) echo "Unsupported architecture: $ARCH" diff --git a/src/k7/cli/k7.py b/src/k7/cli/k7.py index c89c59c..527462b 100644 --- a/src/k7/cli/k7.py +++ b/src/k7/cli/k7.py @@ -1,37 +1,43 @@ #!/usr/bin/env python3 -import os -import subprocess -import time -import threading -import typer -import json +import asyncio +import builtins import hashlib +import json +import os import secrets -from pathlib import Path -from typing import Optional, List +import shutil +import subprocess +import sys +import threading +import time from datetime import datetime, timedelta +from pathlib import Path + +import typer +from click.core import ParameterSource +from rich.console import Console, Group from rich.live import Live -from rich.table import Table from rich.progress import ( + BarColumn, Progress, SpinnerColumn, - BarColumn, - TextColumn, TaskProgressColumn, + TextColumn, TimeElapsedColumn, ) -from rich.console import Console, Group +from rich.table import Table from rich.text import Text -import shutil -import socket from k7 import __version__ as K7_VERSION - +from k7.cli._client import CliContext, handle_api_call +from k7.cli._config import config_app from k7.core.core import K7Core -from k7.core.models import SandboxConfig +from k7.core.models import SandboxConfig, SandboxConfigOverrides +from k7.core.sidecar import SIDECAR_REGISTRY app = typer.Typer(context_settings={"help_option_names": ["-h", "--help"]}) +app.add_typer(config_app, name="config") @app.callback(invoke_without_command=True) @@ -45,10 +51,38 @@ def _main( is_eager=True, is_flag=True, ), + api_url: str | None = typer.Option( + None, + "--api-url", + envvar="K7_API_URL", + help="K7 API endpoint (overrides env / config file). e.g. https://10.0.0.1:31000", + ), + api_key: str | None = typer.Option( + None, + "--api-key", + envvar="K7_API_KEY", + help="API key for the K7 API (overrides env / config file).", + ), + use_core: bool = typer.Option( + False, + "--core", + hidden=True, + help="Bypass the API; call K7Core directly in-process (debugging / on-node tests).", + ), ): + """K7 — sandbox management CLI. + + By default, sandbox-management commands talk to the K7 API. Set the URL + and key via ``--api-url`` / ``--api-key``, ``K7_API_URL`` / ``K7_API_KEY``, + or ``k7 config set api.url ...``. On a cluster node, the CLI falls back to + ``/etc/k7/api_endpoint`` and ``/etc/k7/api_keys.json`` automatically. + """ if version: typer.echo(K7_VERSION) raise typer.Exit() + # CliContext is built lazily — handlers that never call ``ctx.obj.client()`` + # (install / start-api / config / ...) don't trigger the missing-URL error. + ctx.obj = CliContext(use_core=use_core, api_url=api_url, api_key=api_key) if ctx.invoked_subcommand is None: # Show top-level help when no command is provided try: @@ -59,163 +93,330 @@ def _main( API_KEYS_FILE = Path(os.getenv("K7_API_KEYS_FILE", "/etc/k7/api_keys.json")) -def _detect_host_ip_for_kubeapi() -> Optional[str]: - """Detect the host IP address to reach the kube-apiserver from a container. +# The k7-api container runs as this fixed non-root uid (`useradd -u 1000 +# k7user` in src/k7/api/Dockerfile.api) and reads the key store through a +# hostPath mount. The store holds sha256 *hashes* (never raw keys), but we +# still keep it 0600 — which means it must be owned by the API uid or the +# pod cannot read it and every key is rejected. +K7_API_UID = 1000 - Uses a UDP connect trick to a well-known internet IP to determine the - primary outbound interface IP. + +def _write_api_keys(api_keys: dict) -> None: + """Persist the key store with permissions the k7-api pod can use. + + Mode 0600 owned by the API container uid. The chown needs root (the + normal case on a node); when unavailable we warn loudly instead of + leaving the API silently locked out. """ + API_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(API_KEYS_FILE, "w") as f: + json.dump(api_keys, f, indent=2) + os.chmod(API_KEYS_FILE, 0o600) try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.settimeout(1) - s.connect(("1.1.1.1", 80)) - host_ip = s.getsockname()[0] - s.close() - return host_ip - except Exception: + os.chown(API_KEYS_FILE, K7_API_UID, K7_API_UID) + except (PermissionError, OSError) as e: + typer.echo( + f"⚠️ Could not chown {API_KEYS_FILE} to uid {K7_API_UID} ({e}). " + "The k7-api pod runs as that uid and will NOT be able to read the " + "key store — re-run as root on the node hosting the k7-api pod.", + err=True, + ) + + +BACKEND_ALLOWED = {"kata-firecracker-devmapper", "kata-qemu-longhorn", "k7d"} +BACKEND_ALIASES = { + "kfd": "kata-firecracker-devmapper", + "kql": "kata-qemu-longhorn", + "k7": "k7d", +} +# Deprecated short + pre-rename full names. Still accepted; emit a warning. +BACKEND_DEPRECATED_ALIASES = { + "fd": "kata-firecracker-devmapper", + "ql": "kata-qemu-longhorn", + "firecracker-devmapper": "kata-firecracker-devmapper", + "qemu-longhorn": "kata-qemu-longhorn", +} + + +def _normalize_backend(backend: str | None) -> str | None: + if backend is None: return None - - -def _prepare_container_kubeconfig_and_override(compose_path: str) -> Optional[str]: - """Create a container-friendly kubeconfig and a compose override. - - - Copies host kubeconfig (env KUBECONFIG or /etc/rancher/k3s/k3s.yaml) - - Rewrites server URL from 127.0.0.1 to host primary IP (if needed) - - Writes to /etc/k7/k3s.docker.yaml (fallbacks to user data dir if needed) - - Generates a small compose override that mounts the rewritten kubeconfig - to /etc/rancher/k3s/k3s.yaml inside the container. - - Returns the override file path to pass as an extra -f to docker compose. - """ - host_kube = os.getenv("KUBECONFIG", "/etc/rancher/k3s/k3s.yaml") - try: - content = Path(host_kube).read_text() - except Exception: - return None - - # Only rewrite if server is pointing to 127.0.0.1 - if "https://127.0.0.1:6443" in content: - host_ip = _detect_host_ip_for_kubeapi() - if not host_ip: - return None - content = content.replace("https://127.0.0.1:6443", f"https://{host_ip}:6443") - - # Always use root-owned secure path only - d = Path("/etc/k7") - try: - d.mkdir(parents=True, exist_ok=True) - try: - os.chmod(d, 0o700) - except Exception: - pass - kube_out = d / "k3s.docker.yaml" - kube_out.write_text(content) - try: - os.chmod(kube_out, 0o600) - except Exception: - pass - # Verify effective permissions - try: - dir_mode = d.stat().st_mode & 0o777 - file_mode = kube_out.stat().st_mode & 0o777 - if (dir_mode & 0o077) != 0 or (file_mode & 0o077) != 0: - try: - kube_out.unlink(missing_ok=True) - except Exception: - pass - return None - except Exception: - return None - except Exception: - return None - - # Build a minimal override compose that remaps the kubeconfig source - override_content = ( - "services:\n" - " k7-api:\n" - f" volumes:\n - {str(kube_out)}:/etc/rancher/k3s/k3s.yaml:ro\n" + value = backend.strip().lower() + if not value: + raise typer.BadParameter("Backend value cannot be empty.") + if value in BACKEND_ALIASES: + return BACKEND_ALIASES[value] + if value in BACKEND_DEPRECATED_ALIASES: + canonical = BACKEND_DEPRECATED_ALIASES[value] + short = {"kata-firecracker-devmapper": "kfd", "kata-qemu-longhorn": "kql"}[canonical] + typer.echo( + f"⚠️ Backend '{value}' is deprecated. Use '{short}' or '{canonical}'.", + err=True, + ) + return canonical + if value in BACKEND_ALLOWED: + return value + allowed = ", ".join(sorted(BACKEND_ALLOWED)) + raise typer.BadParameter( + f"Unsupported backend '{backend}'. Use {allowed} (aliases: kfd, kql, k7; deprecated: fd, ql)." ) - override_path = d / "k7-compose.override.yml" - try: - override_path.write_text(override_content) - except Exception: - return None - return str(override_path) +def _parse_backends(value: str) -> list[str]: + """Parse a comma-separated list of backend names, normalize and de-dupe. - -def _resolve_compose_path_or_fail(user_compose_file: Optional[str]) -> (str, str): - """Resolve the docker-compose.yml path and its working directory. - - Preference order: - 1) Explicit --compose-file (must exist) - 2) Packaged compose inside the installed/bundled package (works with Nuitka) - - Fails with exit code 1 if none are available. - Returns (compose_path, workdir) + Used by `k7 install` where multiple backends can be installed on a node. + Preserves first-seen order; raises typer.BadParameter on unknown entries. """ - if user_compose_file: - p = Path(user_compose_file) - if not p.exists(): - typer.echo(f"❌ Provided compose file not found: {user_compose_file}", err=True) - raise typer.Exit(1) - return str(p), str(p.parent) + if value is None: + raise typer.BadParameter("Backend value cannot be empty.") + raw = [p for p in (s.strip() for s in value.split(",")) if p] + if not raw: + raise typer.BadParameter("Backend value cannot be empty.") + seen: builtins.list[str] = [] + for entry in raw: + normalized = _normalize_backend(entry) + if normalized and normalized not in seen: + seen.append(normalized) + return seen - # Use the embedded paths exposed by core helpers to ensure consistency - core = K7Core() - compose_path = core._get_embedded_docker_compose() - dockerfile_path = core._get_embedded_dockerfile_api() - if compose_path and dockerfile_path: - return compose_path, str(Path(compose_path).parent) - typer.echo("❌ docker-compose.yml not found. Ensure it is available in the package or specify --compose-file.", err=True) +def _kubectl_cmd() -> list[str]: + """Return the kubectl command prefix (k3s kubectl or kubectl).""" + return ["k3s", "kubectl"] if shutil.which("k3s") else ["kubectl"] + + +def _build_default_inventory( + hosts: list[str] | None, + role: str, + backends: list[str], + disk: str | None, + longhorn_extra_disk: str | None, +) -> str: + """Build a minimal multi-group inventory when the user did not provide one. + + Single-host shortcut: no `hosts` argument → localhost server with the + user-supplied backends. `backends` may contain one or both of + `kata-firecracker-devmapper` and `kata-qemu-longhorn`; the playbook will install + each backend's prerequisites only when listed. + """ + server_lines: builtins.list[str] = [] + agent_lines: builtins.list[str] = [] + backends_csv = ",".join(backends) + has_devmapper = "kata-firecracker-devmapper" in backends + has_longhorn = "kata-qemu-longhorn" in backends + + def _host_vars(extra_host: str | None = None) -> str: + parts = [f"k7_backends={backends_csv}"] + if has_devmapper and disk: + parts.append(f"k7_devmapper_disk={disk}") + if has_longhorn and longhorn_extra_disk: + parts.append(f"longhorn_extra_disk={longhorn_extra_disk}") + if extra_host: + parts.insert(0, extra_host) + return " ".join(parts) + + if hosts: + for host in hosts: + line = f"{host} ansible_user=root ansible_ssh_private_key_file=~/.ssh/id_rsa {_host_vars()}" + (server_lines if role == "server" else agent_lines).append(line) + else: + server_lines.append(f"localhost ansible_connection=local ansible_user=root {_host_vars()}") + + lines = [ + "[k7_servers]", + *server_lines, + "", + "[k7_agents]", + *agent_lines, + "", + "[k7_cluster:children]", + "k7_servers", + "k7_agents", + ] + return "\n".join(lines) + + +def _read_api_endpoint(kubectl: list[str]) -> str | None: + """Read the K7 API NodePort endpoint from the K3s service.""" + port_result = subprocess.run( + kubectl + ["get", "svc", "k7-api", "-n", "kube-system", "-o", "jsonpath={.spec.ports[0].nodePort}"], + capture_output=True, + text=True, + ) + if port_result.returncode != 0 or not port_result.stdout.strip(): + return None + node_port = port_result.stdout.strip() + + # Get node addresses as JSON and pick the first IPv4 InternalIP + node_result = subprocess.run( + kubectl + ["get", "nodes", "-o", "jsonpath={.items[0].status.addresses}"], + capture_output=True, + text=True, + ) + node_ip = "localhost" + if node_result.returncode == 0 and node_result.stdout.strip(): + try: + addrs = json.loads(node_result.stdout.strip()) + for addr in addrs: + if addr.get("type") == "InternalIP" and ":" not in addr.get("address", ":"): + node_ip = addr["address"] + break + except (json.JSONDecodeError, TypeError, KeyError): + pass + + return f"http://{node_ip}:{node_port}" + + +def _get_api_manifests_dir() -> str: + """Locate the embedded K3s API manifests directory.""" + from importlib import resources as _resources + + try: + manifest_dir = _resources.files("k7.deploy").joinpath("manifests/k7-api") + with _resources.as_file(manifest_dir) as p: + if p.is_dir(): + return str(p) + except Exception: + pass + # Fallback: relative to this file (source tree layout) + src_path = Path(__file__).resolve().parent.parent / "deploy" / "manifests" / "k7-api" + if src_path.is_dir(): + return str(src_path) + typer.echo("❌ K3s API manifests not found in package", err=True) raise typer.Exit(1) @app.command() def install( - hosts: Optional[List[str]] = typer.Argument( - None, help="Optional target hosts; defaults to localhost" - ), - playbook: Optional[str] = typer.Option( - None, "-p", "--playbook", help="Path to custom Ansible playbook" - ), - inventory: Optional[str] = typer.Option( - None, "-i", "--inventory", help="Path to custom Ansible inventory" - ), - disk: Optional[str] = typer.Option( + ctx: typer.Context, + hosts: list[str] | None = typer.Argument(None, help="Optional target hosts; defaults to localhost"), + playbook: str | None = typer.Option(None, "-p", "--playbook", help="Path to custom Ansible playbook"), + inventory: str | None = typer.Option(None, "-i", "--inventory", help="Path to custom Ansible inventory"), + disk: str | None = typer.Option( None, "--disk", help="Block device to use for LVM thin-pool (e.g., /dev/nvme2n1)", ), - verbose: bool = typer.Option( - False, "-v", "--verbose", help="Enable verbose output" + backend: str = typer.Option( + "kata-firecracker-devmapper,kata-qemu-longhorn", + "--backend", + "-b", + help=( + "Comma-separated sandbox backends to install on this node. " + "Choices: kata-firecracker-devmapper (kfd), kata-qemu-longhorn (kql), k7d. " + "Default installs the two Kata backends; add k7d explicitly for the " + "warm-fork microVM runtime." + ), + show_default=True, ), + k7d_version: str | None = typer.Option( + None, + "--k7d-version", + help="k7d release version to install (k7d backend only; default from playbook: 0.1.0)", + ), + k7d_artifact: str | None = typer.Option( + None, + "--k7d-artifact", + help=( + "Path to a local k7d release tarball (k7d-v-x86_64-linux.tar.gz) to install " + "instead of downloading from GitHub (k7d backend only)." + ), + ), + role: str = typer.Option( + "server", + "--role", + help="Role for ad-hoc hosts when no inventory is given: server (master) or agent (worker)", + show_default=True, + ), + join: str | None = typer.Option( + None, + "--join", + help="Join an existing cluster: master URL like https://:6443 (requires --role agent or server)", + ), + join_token: str | None = typer.Option( + None, + "--join-token", + help="K3s join token for --join (read from :/var/lib/rancher/k3s/server/node-token)", + ), + ha: bool = typer.Option( + False, + "--ha", + help="Enable HA: configure multiple masters with embedded etcd (requires inventory with 3+ servers)", + ), + replicas: int | None = typer.Option( + None, + "--replicas", + help="Longhorn replica count (kata-qemu-longhorn only). Defaults to min(3, node count).", + ), + longhorn_data_path: str | None = typer.Option( + None, + "--longhorn-data-path", + help="Longhorn default data path (default: /var/lib/longhorn)", + ), + longhorn_extra_disk: str | None = typer.Option( + None, + "--longhorn-extra-disk", + help="Extra disk/partition to register as a Longhorn storage node disk", + ), + cni: str = typer.Option( + "cilium", + "--cni", + help="CNI plugin: cilium (default, enables FQDN egress) or flannel (CIDR-only)", + show_default=True, + ), + no_api: bool = typer.Option( + False, + "--no-api", + help="Skip deploying the K7 API into K3s (CLI-only install). Re-run `k7 install` later to add it.", + ), + verbose: bool = typer.Option(False, "-v", "--verbose", help="Enable verbose output"), ): - """Install K7 on target hosts using Ansible.""" + """Install K7 on target hosts using Ansible. + + Single-node (default): `k7 install` provisions localhost with **both** + backends (kata-firecracker-devmapper + kata-qemu-longhorn). Pass `--backend kfd` or + `--backend kql` to install only one. + + Multi-node: `k7 install -i inventory.ini` reads roles, backends, and disks + from the Ansible inventory. See `inventory.ini.example` for the layout. + """ + if role not in ("server", "agent"): + raise typer.BadParameter(f"--role must be 'server' or 'agent', got '{role}'") + + cni_value = (cni or "").strip().lower() + if cni_value not in ("cilium", "flannel"): + raise typer.BadParameter(f"--cni must be 'cilium' or 'flannel', got '{cni}'") + + if join and not join_token: + raise typer.BadParameter("--join requires --join-token") + + if ha and not inventory and (not hosts or len(hosts) < 3): + typer.echo( + "⚠️ --ha is intended for inventory-based multi-master clusters with 3+ servers; " + "ignoring (single-node install).", + err=True, + ) playbook_content = None if playbook and os.path.exists(playbook): - with open(playbook, "r") as f: + with open(playbook) as f: playbook_content = f.read() + backends_list = _parse_backends(backend) + inventory_content = None if inventory and os.path.exists(inventory): - with open(inventory, "r") as f: + with open(inventory) as f: inventory_content = f.read() else: - inventory_lines = ["[k7_nodes]"] - if hosts: - for host in hosts: - inventory_lines.append( - f"{host} ansible_user=root ansible_ssh_private_key_file=~/.ssh/id_rsa" - ) - else: - inventory_lines.append( - "localhost ansible_connection=local ansible_user=root" - ) - inventory_content = "\n".join(inventory_lines) + inventory_content = _build_default_inventory( + hosts=hosts, + role=role, + backends=backends_list, + disk=disk, + longhorn_extra_disk=longhorn_extra_disk, + ) core = K7Core() @@ -273,8 +474,69 @@ def install( except Exception: pass - # Kick off install and update UI as lines arrive - extra_vars = {"k7_disk": disk} if disk else None + # Kick off install and update UI as lines arrive. + # When --replicas is unset and we built the inventory ourselves, default + # to min(3, node count). When the user supplied an inventory we can't + # know the node count up-front, so we let the playbook compute its own + # default (min(3, len(k7_cluster))) and only forward an explicit value. + inventory_user_supplied = bool(inventory) + node_count = len(hosts) if hosts else 1 + if replicas is not None: + effective_replicas: int | None = replicas + if not inventory_user_supplied and replicas > node_count: + typer.echo( + f"⚠️ --replicas={replicas} exceeds node count ({node_count}); " + "Longhorn cannot satisfy replica count > node count", + err=True, + ) + elif inventory_user_supplied: + effective_replicas = None + else: + effective_replicas = min(3, node_count) + # The playbook builds the k7-api Docker image on the first master via + # `delegate_to`, with `chdir` pointing to the source tree. Pass the + # controller's cwd as `k7_repo_root`; the playbook validates that the + # path exists on the first master and fails loudly otherwise. The user + # is expected to invoke `k7 install` from a checkout of the repo (or + # via SSH on a node that has the source tree at the same path). + repo_root = os.getcwd() + extra_vars = { + # `k7_disk` is the legacy name; the playbook prefers `k7_devmapper_disk` + # but still falls back to `k7_disk` for backward compat. + "k7_disk": disk, + # `k7_backends` (plural, comma-separated) is authoritative; the + # playbook accepts a single `k7_backend` only as a back-compat + # fallback when neither inventory `k7_backends` nor extra-var + # `k7_backends` is set. + "k7_backends": ",".join(backends_list), + "longhorn_replicas": effective_replicas, + "longhorn_data_path": longhorn_data_path, + "longhorn_extra_disk": longhorn_extra_disk, + "k7_cni": cni_value, + "k7_repo_root": repo_root, + # Spec 10d: ``--no-api`` flips the playbook's existing + # ``k7_api_enabled`` gate (default true) so the API + # manifests aren't applied. + "k7_api_enabled": "false" if no_api else "true", + # k7d backend artifact source overrides (spec 9a M11). + "k7d_version": k7d_version, + "k7d_artifact_local_path": k7d_artifact, + } + # Drop None to avoid leaking unused vars + extra_vars = {k: v for k, v in extra_vars.items() if v is not None} + # Ansible extra-vars have the highest precedence and would clobber + # per-host `k7_backends` declared in a user-supplied inventory. Only + # forward the CLI value when the user explicitly passed --backend; + # otherwise the inventory is authoritative (spec 18e). + backend_explicit = ctx.get_parameter_source("backend") == ParameterSource.COMMANDLINE + if inventory_user_supplied and not backend_explicit: + extra_vars.pop("k7_backends", None) + elif inventory_user_supplied and backend_explicit: + typer.echo( + "⚠️ --backend overrides any per-host k7_backends in the inventory " + "(extra-vars precedence). Omit --backend to let the inventory win.", + err=True, + ) result = core.install_node( playbook_content, inventory_content, @@ -302,58 +564,98 @@ def install( @app.command() def create( - name: Optional[str] = None, - image: Optional[str] = None, - config: Optional[str] = typer.Option( - None, "-f", "--file", help="Path to k7.yaml config file" + ctx: typer.Context, + name: str | None = typer.Argument(None, help="Sandbox name (auto-generated if omitted)"), + image: str | None = typer.Argument(None, help="Container image (e.g., 'ubuntu:24.04')"), + config: str | None = typer.Option(None, "-f", "--file", help="Path to k7.yaml config file"), + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), + cpu_limit: str | None = typer.Option(None, "--cpu", help="CPU limit (e.g., '1', '500m')"), + memory_limit: str | None = typer.Option(None, "--memory", help="Memory limit (e.g., '1Gi', '512Mi')"), + storage_limit: str | None = typer.Option( + None, "--storage", help="Kubernetes ephemeral storage limit (tmpfs, emptyDir). Not the root disk." ), - namespace: str = typer.Option( - "default", "-n", "--namespace", help="Kubernetes namespace" - ), - cpu_limit: Optional[str] = typer.Option( - None, "--cpu", help="CPU limit (e.g., '1', '500m')" - ), - memory_limit: Optional[str] = typer.Option( - None, "--memory", help="Memory limit (e.g., '1Gi', '512Mi')" - ), - storage_limit: Optional[str] = typer.Option( - None, "--storage", help="Ephemeral storage limit (e.g., '2Gi', '1Gi')" - ), - env_file: Optional[str] = typer.Option( - None, "--env-file", help="Path to environment file containing secrets" - ), - egress_whitelist: Optional[List[str]] = typer.Option( + env_file: str | None = typer.Option(None, "--env-file", help="Path to environment file containing secrets"), + egress_whitelist: list[str] | None = typer.Option( None, "--egress", - help="CIDR blocks for egress whitelist (can be used multiple times)", + help=( + "Egress whitelist entry (CIDR like 10.0.0.0/8 or domain like api.openai.com; " + "wildcards like *.huggingface.co supported with Cilium CNI). Repeatable. " + "Default (no --egress/--egress-open): block all egress." + ), ), - before_script: Optional[str] = typer.Option( + egress_open: bool = typer.Option( + False, + "--egress-open", + help=( + "Allow ALL egress (no network policy). Mutually exclusive with --egress. " + "Without this flag and without --egress, all egress is blocked." + ), + ), + before_script: str | None = typer.Option( None, "--before-script", help="Script to run before starting the main container process", ), - pod_non_root: Optional[bool] = typer.Option( + pod_non_root: bool | None = typer.Option( None, "--pod-non-root/--no-pod-non-root", help="Run pod with non-root defaults (user/group/fsGroup 65532)", ), - container_non_root: Optional[bool] = typer.Option( + container_non_root: bool | None = typer.Option( None, "--container-non-root/--no-container-non-root", help="Run main container as non-root (uid 65532)", ), - cap_add: Optional[List[str]] = typer.Option( + cap_add: list[str] | None = typer.Option( None, "--cap-add", help="Linux capabilities to add back (can be used multiple times)", ), - cap_drop: Optional[List[str]] = typer.Option( + cap_drop: list[str] | None = typer.Option( None, "--cap-drop", help="Linux capabilities to drop (can be used multiple times)", ), + runtime_class: str | None = typer.Option( + None, + "--runtime-class", + help="Override runtime class (e.g., kata or kata-qemu)", + ), + entrypoint: list[str] | None = typer.Option( + None, + "--entrypoint", + help="Override image ENTRYPOINT (repeatable, ordered)", + ), + cmd: list[str] | None = typer.Option( + None, + "--cmd", + help="Override image CMD (repeatable, ordered)", + ), + root_disk_size: str | None = typer.Option( + "10Gi", + "--root-disk-size", + help="Longhorn root disk size, kata-qemu-longhorn backend only (e.g., 10Gi, 20Gi)", + ), + backend: str | None = typer.Option( + None, + "--backend", + "-b", + help="Backend: kata-firecracker-devmapper (kfd) or kata-qemu-longhorn (kql) (auto-detected if not specified)", + ), + sidecar: str | None = typer.Option( + None, + "--sidecar", + help="Sidecar daemon type to inject (e.g. 'docker'). See SIDECAR_REGISTRY for available types.", + ), ): """Create a new sandbox from YAML config or CLI arguments.""" + # Spec 18f issue 4: three explicit egress modes. --egress-open → open + # (no policy, egress_whitelist=None); --egress ... → whitelist; neither + # → block-all ([]). Combining both is ambiguous — fail loudly. + if egress_open and egress_whitelist: + raise typer.BadParameter("--egress-open cannot be combined with --egress (pick one egress mode)") + # Auto-detect default config file in current directory when not provided if not config: for candidate in ("k7.yaml", "k7.yml"): @@ -375,7 +677,9 @@ def create( sandbox_config.namespace = namespace if env_file: sandbox_config.env_file = env_file - if egress_whitelist: + if egress_open: + sandbox_config.egress_whitelist = None + elif egress_whitelist: sandbox_config.egress_whitelist = egress_whitelist if before_script: sandbox_config.before_script = before_script @@ -388,6 +692,21 @@ def create( sandbox_config.cap_add = cap_add if cap_drop is not None: sandbox_config.cap_drop = cap_drop + if runtime_class: + sandbox_config.runtime_class_name = runtime_class + if entrypoint is not None: + sandbox_config.entrypoint = entrypoint + if cmd is not None: + sandbox_config.cmd = cmd + if root_disk_size: + sandbox_config.root_disk_size = root_disk_size + if backend: + sandbox_config.backend = _normalize_backend(backend) + if sidecar is not None: + sandbox_config.sidecar = sidecar + + if not sandbox_config.image: + raise typer.BadParameter("image is required") if cpu_limit or memory_limit or storage_limit: if not sandbox_config.limits: @@ -399,10 +718,11 @@ def create( if storage_limit: sandbox_config.limits["ephemeral-storage"] = storage_limit else: - if not name or not image: - raise typer.BadParameter( - "Name and image must be provided via CLI or k7.yaml" - ) + if not name: + raise typer.BadParameter("Name must be provided via CLI or k7.yaml") + + if not image: + raise typer.BadParameter("image is required") limits = {} if cpu_limit: @@ -417,15 +737,29 @@ def create( image=image, namespace=namespace, env_file=env_file, - egress_whitelist=egress_whitelist or [], + egress_whitelist=None if egress_open else (egress_whitelist or []), limits=limits if limits else None, before_script=before_script or "", + entrypoint=entrypoint, + cmd=cmd, + sidecar=sidecar, pod_non_root=pod_non_root if pod_non_root is not None else False, container_non_root=container_non_root if container_non_root is not None else False, cap_add=cap_add, cap_drop=cap_drop, + runtime_class_name=runtime_class, + root_disk_size=root_disk_size, + backend=_normalize_backend(backend) if backend else None, ) + resolved_backend = _normalize_backend(sandbox_config.backend) if sandbox_config.backend else None + if root_disk_size and resolved_backend == "kata-firecracker-devmapper": + typer.echo("⚠️ --root-disk-size has no effect with the kata-firecracker-devmapper backend", err=True) + + if sandbox_config.sidecar is not None and sandbox_config.sidecar not in SIDECAR_REGISTRY: + available = ", ".join(sorted(SIDECAR_REGISTRY.keys())) + raise typer.BadParameter(f"Unknown sidecar type '{sandbox_config.sidecar}'. Available: {available}") + core = K7Core() progress = Progress( SpinnerColumn(), @@ -453,18 +787,12 @@ def create( elif status == "done": status_text.plain = "Provisioned" status_text.stylize("green") - progress.update( - stage_task, description="[green]Provisioned[/green]", total=None - ) + progress.update(stage_task, description="[green]Provisioned[/green]", total=None) elif stage == "before_script": if status == "waiting": script = event.get("script", "") first_line = script.strip().split("\n")[0] - shown = ( - first_line - if len(first_line) <= 200 - else first_line[:200] + "..." - ) + shown = first_line if len(first_line) <= 200 else first_line[:200] + "..." status_text.plain = "Running before script" status_text.stylize("yellow") details_text.plain = shown @@ -522,16 +850,12 @@ def create( status_text.plain = msg status_text.stylize("green") details_text.plain = "" - progress.update( - stage_task, description=f"[green]{msg}[/green]", total=None - ) + progress.update(stage_task, description=f"[green]{msg}[/green]", total=None) elif stage == "error": err = event.get("error", "") status_text.plain = f"Error: {err}" status_text.stylize("red") - progress.update( - stage_task, description=f"[red]Error: {err}[/red]", total=None - ) + progress.update(stage_task, description=f"[red]Error: {err}[/red]", total=None) except Exception: pass @@ -541,7 +865,9 @@ def create( # Show key YAML parameters up-front for clarity try: egress_mode = ( - "open" if sandbox_config.egress_whitelist is None else ( + "open" + if sandbox_config.egress_whitelist is None + else ( "block_all" if sandbox_config.egress_whitelist == [] else f"whitelist={sandbox_config.egress_whitelist}" ) ) @@ -580,12 +906,12 @@ def create( # Prepare before_script log streaming (started when core signals 'waiting') kubectl_cmd = ["k3s", "kubectl"] if shutil.which("k3s") else ["kubectl"] stop_log_event: threading.Event = threading.Event() - log_thread: Optional[threading.Thread] = None + log_thread: threading.Thread | None = None log_started_event: threading.Event = threading.Event() log_ended_event: threading.Event = threading.Event() - before_log_lines: list[str] = [] + before_log_lines: builtins.list[str] = [] - resolved_pod_name: Optional[str] = None + resolved_pod_name: str | None = None def _stream_before_script_logs(): pod_name = None @@ -624,6 +950,7 @@ def create( # Announce start try: from rich.text import Text as RichText + live.console.print(RichText(f"===== START before_script ({sandbox_config.name}) =====", style="bold")) log_started_event.set() except Exception: @@ -681,13 +1008,13 @@ def create( pass try: from rich.text import Text as RichText + live.console.print(RichText(f"===== END before_script ({sandbox_config.name}) =====", style="bold")) log_ended_event.set() except Exception: pass - - def on_progress(event: dict): + def on_progress(event: dict): # noqa: F811 try: stage = event.get("stage") status = event.get("status") @@ -703,18 +1030,12 @@ def create( elif status == "done": status_text.plain = "Provisioned" status_text.stylize("green") - progress.update( - stage_task, description="[green]Provisioned[/green]", total=None - ) + progress.update(stage_task, description="[green]Provisioned[/green]", total=None) elif stage == "before_script": if status == "waiting": script = event.get("script", "") first_line = script.strip().split("\n")[0] - shown = ( - first_line - if len(first_line) <= 200 - else first_line[:200] + "..." - ) + shown = first_line if len(first_line) <= 200 else first_line[:200] + "..." status_text.plain = "Running before script" status_text.stylize("yellow") details_text.plain = shown @@ -736,7 +1057,7 @@ def create( if before_log_lines: try: tail = "".join(before_log_lines[-50:]).rstrip() - details_text.plain = ("Before script log tail (last 50 lines):\n" + tail) + details_text.plain = "Before script log tail (last 50 lines):\n" + tail details_text.stylize("dim") except Exception: details_text.plain = "" @@ -791,61 +1112,98 @@ def create( if log_started_event.is_set() and not log_ended_event.is_set(): try: from rich.text import Text as RichText - live.console.print(RichText(f"===== END before_script ({sandbox_config.name}) =====", style="bold")) + + live.console.print( + RichText(f"===== END before_script ({sandbox_config.name}) =====", style="bold") + ) except Exception: pass # Keep the log tail summary visible after completion - progress.update( - stage_task, description=f"[green]{msg}[/green]", total=None - ) + progress.update(stage_task, description=f"[green]{msg}[/green]", total=None) stop_log_event.set() # Do not attempt any file-based fallbacks; only stream container logs elif stage == "error": err = event.get("error", "") status_text.plain = f"Error: {err}" status_text.stylize("red") - progress.update( - stage_task, description=f"[red]Error: {err}[/red]", total=None - ) + progress.update(stage_task, description=f"[red]Error: {err}[/red]", total=None) stop_log_event.set() except Exception: pass - group = RichGroup(status_text, details_text, progress) - with RichLive(group, refresh_per_second=8, transient=False) as live: - result = core.create_sandbox(sandbox_config, progress_callback=on_progress) - - if not result.success: - typer.echo(f"❌ Failed to create sandbox: {result.error}", err=True) - raise typer.Exit(1) - typer.echo( - f"👉 Shell in: k7 shell {sandbox_config.name}" - + ( - f" -n {sandbox_config.namespace}" - if sandbox_config.namespace != "default" - else "" + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + group = RichGroup(status_text, details_text, progress) + with RichLive(group, refresh_per_second=8, transient=False) as live: + result = asyncio.run(core.create_sandbox(sandbox_config, progress_callback=on_progress)) + if not result.success: + typer.echo(f"❌ Failed to create sandbox: {result.error}", err=True) + raise typer.Exit(1) + sandboxes_for_ready = asyncio.run(core.list_sandboxes(sandbox_config.namespace)) + target_ready = any(s.name == sandbox_config.name and s.ready == "True" for s in sandboxes_for_ready) + else: + # The API doesn't stream progress yet (a separate spec). Print a + # one-shot status line, then call the SDK. The server-side handler + # waits for the pod to become Ready before responding, so by the + # time we get a result back the sandbox is up. + typer.echo(f"Creating sandbox {sandbox_config.name} via API ...") + handle_api_call(lambda: cli_ctx.client().create(sandbox_config.to_dict())) + sandboxes = handle_api_call(lambda: cli_ctx.client().list(namespace=sandbox_config.namespace)) + target_ready = any(s.get("name") == sandbox_config.name and s.get("ready") == "True" for s in sandboxes) + if target_ready: + typer.echo(f"✅ Sandbox {sandbox_config.name} created.") + typer.echo( + f"👉 Shell in: k7 shell {sandbox_config.name}" + + (f" -n {sandbox_config.namespace}" if sandbox_config.namespace != "default" else "") + ) + else: + typer.echo( + f"✅ Sandbox {sandbox_config.name} created. Check status with:" + + (f" k7 list -n {sandbox_config.namespace}" if sandbox_config.namespace != "default" else " k7 list") + ) + typer.echo( + f"👉 When Ready, run: k7 shell {sandbox_config.name}" + + (f" -n {sandbox_config.namespace}" if sandbox_config.namespace != "default" else "") ) - ) @app.command() def list( - namespace: Optional[str] = typer.Option( + ctx: typer.Context, + namespace: str | None = typer.Option( None, "-n", "--namespace", help="Filter sandboxes by namespace. If not provided, shows sandboxes from all namespaces.", ), + name: str | None = typer.Option( + None, + "--name", + help="Filter sandboxes by exact name.", + ), ): """List all running sandboxes.""" - core = K7Core() - sandboxes = core.list_sandboxes(namespace) + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + sandboxes = [s.to_dict() for s in asyncio.run(core.list_sandboxes(namespace))] + else: + sandboxes = handle_api_call(lambda: cli_ctx.client().list(namespace=namespace)) + + if name: + sandboxes = [s for s in sandboxes if s.get("name") == name] if not sandboxes: if namespace: - typer.echo(f"No sandboxes found in namespace '{namespace}'.") + if name: + typer.echo(f"No sandbox named '{name}' found in namespace '{namespace}'.") + else: + typer.echo(f"No sandboxes found in namespace '{namespace}'.") else: - typer.echo("No sandboxes found.") + if name: + typer.echo(f"No sandbox named '{name}' found.") + else: + typer.echo("No sandboxes found.") return console = Console() @@ -857,33 +1215,35 @@ def list( table.add_column("Restarts", justify="center") table.add_column("Age") table.add_column("Image", style="green") + table.add_column("Backend", style="magenta") + table.add_column("Node", style="yellow") table.add_column("Error", style="red") for sandbox in sandboxes: - if sandbox.status == "Running": - status_display = f"[green]{sandbox.status}[/green]" - elif sandbox.status == "Pending": - status_display = f"[yellow]{sandbox.status}[/yellow]" - elif sandbox.status == "Failed": - status_display = f"[red]{sandbox.status}[/red]" + status_raw = sandbox.get("status", "") + ready_raw = sandbox.get("ready", "") + if status_raw == "Running": + status_display = f"[green]{status_raw}[/green]" + elif status_raw == "Pending": + status_display = f"[yellow]{status_raw}[/yellow]" + elif status_raw == "Failed": + status_display = f"[red]{status_raw}[/red]" else: - status_display = sandbox.status + status_display = status_raw - ready_display = ( - f"[green]{sandbox.ready}[/green]" - if sandbox.ready == "True" - else f"[red]{sandbox.ready}[/red]" - ) + ready_display = f"[green]{ready_raw}[/green]" if ready_raw == "True" else f"[red]{ready_raw}[/red]" table.add_row( - sandbox.name, - sandbox.namespace, + sandbox.get("name", ""), + sandbox.get("namespace", ""), status_display, ready_display, - str(sandbox.restarts), - sandbox.age, - sandbox.image, - sandbox.error_message, + str(sandbox.get("restarts", 0)), + sandbox.get("age", ""), + sandbox.get("image", ""), + sandbox.get("backend", ""), + sandbox.get("node", ""), + sandbox.get("error_message", ""), ) console.print(table) @@ -891,6 +1251,7 @@ def list( @app.command() def delete( + ctx: typer.Context, name: str, namespace: str = typer.Option( "default", @@ -900,18 +1261,22 @@ def delete( ), ): """Delete a sandbox and all its associated resources.""" - core = K7Core() - result = core.delete_sandbox(name, namespace) - - if result.success: + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + result = asyncio.run(core.delete_sandbox(name, namespace)) + if not result.success: + typer.echo(f"❌ Failed to delete sandbox: {result.error}", err=True) + raise typer.Exit(1) typer.echo(f"✅ {result.message}") - else: - typer.echo(f"❌ Failed to delete sandbox: {result.error}", err=True) - raise typer.Exit(1) + return + data = handle_api_call(lambda: cli_ctx.client().delete(name, namespace=namespace)) + typer.echo(f"✅ {data.get('message', f'Sandbox {name} deleted')}") @app.command() def delete_all( + ctx: typer.Context, namespace: str = typer.Option( "default", "-n", @@ -920,34 +1285,42 @@ def delete_all( ), ): """Delete all sandboxes in a namespace.""" - core = K7Core() - - sandboxes = core.list_sandboxes(namespace) + cli_ctx: CliContext = ctx.obj + # Always show the user the list + a confirm prompt before destroying. List + # goes through the same routing as `k7 list` so off-cluster runs work. + if cli_ctx.use_core: + core = K7Core() + sandboxes = [s.to_dict() for s in asyncio.run(core.list_sandboxes(namespace))] + else: + sandboxes = handle_api_call(lambda: cli_ctx.client().list(namespace=namespace)) if not sandboxes: typer.echo(f"No sandboxes found in namespace {namespace}") return - sandbox_names = [s.name for s in sandboxes] + sandbox_names = [s.get("name", "") for s in sandboxes] typer.echo(f"Found {len(sandbox_names)} sandbox(es) in namespace {namespace}:") - for name in sandbox_names: - typer.echo(f" - {name}") + for n in sandbox_names: + typer.echo(f" - {n}") if not typer.confirm("Are you sure you want to delete all these sandboxes?"): typer.echo("Deletion cancelled") return - result = core.delete_all_sandboxes(namespace) - - if result.success: - typer.echo(f"✅ {result.message}") - else: + if cli_ctx.use_core: + core = K7Core() + result = asyncio.run(core.delete_all_sandboxes(namespace)) + if result.success: + typer.echo(f"✅ {result.message}") + return typer.echo(f"❌ Failed to delete all sandboxes: {result.error}", err=True) if result.data: for item in result.data: if not item["success"]: typer.echo(f" - {item['name']}: {item['error']}") raise typer.Exit(1) + data = handle_api_call(lambda: cli_ctx.client().delete_all(namespace=namespace)) + typer.echo(f"✅ {data.get('message', 'Deleted all sandboxes')}") @app.command() @@ -961,14 +1334,16 @@ def shell( ), ): """Shell into sandbox (bypasses network policy).""" - kubectl_cmd = ["k3s", "kubectl"] if shutil.which("k3s") else ["kubectl"] - subprocess.run( - kubectl_cmd + ["exec", "-it", f"deploy/{name}", "-n", namespace, "--", "sh"] - ) + core = K7Core() + result = core.shell_into_sandbox(name, namespace) + if not result.success: + typer.echo(f"Error: {result.error}", err=True) + raise typer.Exit(1) @app.command() def logs( + ctx: typer.Context, name: str, namespace: str = typer.Option( "default", @@ -976,58 +1351,90 @@ def logs( "--namespace", help="Kubernetes namespace containing the sandbox.", ), - follow: bool = typer.Option(False, "-f", "--follow", help="Follow logs output"), - tail: int = typer.Option( - 200, "--tail", help="Number of lines to show from the end of the logs" - ), + follow: bool = typer.Option(False, "-f", "--follow", help="Follow logs output (--core only)"), + tail: int = typer.Option(200, "--tail", help="Number of lines to show from the end of the logs"), ): - """Show sandbox pod logs (before script and main container).""" - kubectl_cmd = ["k3s", "kubectl"] if shutil.which("k3s") else ["kubectl"] + """Show sandbox pod logs (before script and main container). - # Resolve pod name - try: - pod_name_proc = subprocess.run( - kubectl_cmd - + [ - "get", - "pods", - "-n", - namespace, - "-l", - f"app={name}", - "-o", - "jsonpath={.items[0].metadata.name}", - ], - capture_output=True, - text=True, - check=True, - ) - pod_name = pod_name_proc.stdout.strip() - except subprocess.CalledProcessError as e: - typer.echo( - f"❌ Failed to resolve pod for sandbox '{name}': {e.stderr.strip()}", - err=True, - ) - raise typer.Exit(1) + Default route is the K7 API (a snapshot, no streaming). For live follow, + pass ``--core`` to run against ``kubectl logs -f`` directly on the node. + """ + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + kubectl_cmd = ["k3s", "kubectl"] if shutil.which("k3s") else ["kubectl"] + try: + pod_name_proc = subprocess.run( + kubectl_cmd + + [ + "get", + "pods", + "-n", + namespace, + "-l", + f"app={name}", + "-o", + "jsonpath={.items[0].metadata.name}", + ], + capture_output=True, + text=True, + check=True, + ) + pod_name = pod_name_proc.stdout.strip() + except subprocess.CalledProcessError as e: + typer.echo(f"❌ Failed to resolve pod for sandbox '{name}': {e.stderr.strip()}", err=True) + raise typer.Exit(1) from None + if not pod_name: + typer.echo(f"❌ No pod found for sandbox '{name}' in namespace '{namespace}'.", err=True) + raise typer.Exit(1) + args = ["logs", pod_name, "-n", namespace, "--tail", str(tail)] + if follow: + args.append("-f") + subprocess.run(kubectl_cmd + args) + return - if not pod_name: - typer.echo( - f"❌ No pod found for sandbox '{name}' in namespace '{namespace}'.", - err=True, - ) - raise typer.Exit(1) - - # Show logs; since before_script runs in main container, one container is enough - args = ["logs", pod_name, "-n", namespace, "--tail", str(tail)] if follow: - args.append("-f") - subprocess.run(kubectl_cmd + args) + typer.echo( + "⚠️ --follow is not supported via the API yet; printing a snapshot instead. " + "Use `k7 --core logs --follow ...` on the node for live tailing.", + err=True, + ) + text = handle_api_call(lambda: cli_ctx.client().logs(name, namespace=namespace, tail=tail)) + if text: + typer.echo(text, nl=False) + + +@app.command() +def exec( + ctx: typer.Context, + name: str = typer.Argument(..., help="Sandbox name"), + command: builtins.list[str] = typer.Argument(..., help="Shell command to run (joined with spaces)"), + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), +): + """Run a shell command in a sandbox; prints stdout, exits with the command's exit code. + + Buffered (non-streaming) — large outputs are capped server-side. For + interactive shells use ``k7 shell`` (currently --core only; WebSocket + proxy is a follow-up). + """ + joined = " ".join(command) + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + result = asyncio.run(core.exec_command(name, joined, namespace=namespace)) + data = result.to_dict() + else: + data = handle_api_call(lambda: cli_ctx.client().exec(name, joined, namespace=namespace)) + if data.get("stdout"): + typer.echo(data["stdout"], nl=False) + if data.get("stderr"): + typer.echo(data["stderr"], err=True, nl=False) + raise typer.Exit(int(data.get("exit_code", 0) or 0)) @app.command() def top( refresh_interval: int = 1, - namespace: Optional[str] = typer.Option( + namespace: str | None = typer.Option( None, "-n", "--namespace", @@ -1044,8 +1451,7 @@ def top( table.add_column("CPU Usage (cores)") table.add_column("Memory Usage (MiB)") - # Get metrics from core - metrics_list = core.get_sandbox_metrics(namespace) + metrics_list = asyncio.run(core.get_sandbox_metrics(namespace)) for metric in metrics_list: sb_name = metric["name"] @@ -1113,16 +1519,14 @@ def top( @app.command() -def generate_api_key( - name: str, expires_days: int = typer.Option(365, help="API key expiration in days") -): +def generate_api_key(name: str, expires_days: int = typer.Option(365, help="API key expiration in days")): """Generate a new API key.""" api_key = secrets.token_urlsafe(32) key_hash = hashlib.sha256(api_key.encode()).hexdigest() api_keys = {} if API_KEYS_FILE.exists(): - with open(API_KEYS_FILE, "r") as f: + with open(API_KEYS_FILE) as f: api_keys = json.load(f) expiry_timestamp = int((datetime.now() + timedelta(days=expires_days)).timestamp()) @@ -1133,10 +1537,7 @@ def generate_api_key( "last_used": None, } - API_KEYS_FILE.parent.mkdir(parents=True, exist_ok=True) - with open(API_KEYS_FILE, "w") as f: - json.dump(api_keys, f, indent=2) - os.chmod(API_KEYS_FILE, 0o600) + _write_api_keys(api_keys) typer.echo(f"Generated API key for '{name}':") typer.echo(f"API Key: {api_key}") @@ -1151,7 +1552,7 @@ def list_api_keys(): typer.echo("No API keys found.") return - with open(API_KEYS_FILE, "r") as f: + with open(API_KEYS_FILE) as f: api_keys = json.load(f) console = Console() @@ -1161,14 +1562,12 @@ def list_api_keys(): table.add_column("Expires", style="yellow") table.add_column("Last Used", style="green") - for key_hash, key_data in api_keys.items(): + for _key_hash, key_data in api_keys.items(): created = datetime.fromtimestamp(key_data["created"]).strftime("%Y-%m-%d %H:%M") expires = datetime.fromtimestamp(key_data["expires"]).strftime("%Y-%m-%d %H:%M") last_used = "Never" if key_data["last_used"]: - last_used = datetime.fromtimestamp(key_data["last_used"]).strftime( - "%Y-%m-%d %H:%M" - ) + last_used = datetime.fromtimestamp(key_data["last_used"]).strftime("%Y-%m-%d %H:%M") table.add_row(key_data["name"], created, expires, last_used) @@ -1182,7 +1581,7 @@ def revoke_api_key(name: str): typer.echo("No API keys found.") return - with open(API_KEYS_FILE, "r") as f: + with open(API_KEYS_FILE) as f: api_keys = json.load(f) key_to_remove = None @@ -1193,280 +1592,813 @@ def revoke_api_key(name: str): if key_to_remove: del api_keys[key_to_remove] - with open(API_KEYS_FILE, "w") as f: - json.dump(api_keys, f, indent=2) + _write_api_keys(api_keys) typer.echo(f"API key '{name}' revoked successfully.") else: typer.echo(f"API key '{name}' not found.") +_PAUSE_SNAPSHOT_AUTO = "__auto__" + + +def _rewrite_bare_snapshot_flag(argv: builtins.list[str]) -> builtins.list[str]: + """Rewrite bare ``--snapshot`` on the ``pause`` subcommand to ``--snapshot=``. + + Typer no longer supports the Click ``is_flag=False, flag_value=...`` pattern, + so we emulate "flag-or-value" by preprocessing ``sys.argv``: ``--snapshot`` + not followed by a value (next token absent or another flag) becomes + ``--snapshot=__auto__``. ``--snapshot=NAME`` and ``--snapshot NAME`` are + untouched. Other subcommands are not affected. + """ + if "pause" not in argv: + return argv + out: builtins.list[str] = [] + i = 0 + n = len(argv) + while i < n: + tok = argv[i] + if tok == "--snapshot": + nxt = argv[i + 1] if i + 1 < n else None + if nxt is None or nxt.startswith("-"): + out.append(f"--snapshot={_PAUSE_SNAPSHOT_AUTO}") + i += 1 + continue + out.append(tok) + i += 1 + return out + + @app.command() -def start_api( - port: int = typer.Option(8000, help="Port to run API on"), - host: str = typer.Option("0.0.0.0", help="Host to bind to"), - compose_file: Optional[str] = typer.Option( +def pause( + ctx: typer.Context, + name: str, + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), + snapshot: str | None = typer.Option( None, - "--compose-file", - help="Path to docker-compose.yml to use when --docker is set", - ), - yes: bool = typer.Option( - False, - "--yes", - "-y", - help="Non-interactive; accept using local image overrides if set", + "--snapshot", + help=( + "Take a crash-consistent VolumeSnapshot of the sandbox PVC. " + "Pass --snapshot for an auto-named snapshot, or --snapshot=NAME for a custom name." + ), ), ): - """Start the K7 API server.""" - typer.echo("Starting K7 API with Docker Compose...") - cmd = ["docker", "compose"] - workdir = None - compose_path: Optional[str] = None - compose_path, workdir = _resolve_compose_path_or_fail(compose_file) + """Pause a sandbox (scale to 0) and optionally take a crash-consistent disk snapshot.""" + snapshot_name: str | None = None + if snapshot is not None: + snapshot_name = f"{name}-paused-{int(time.time())}" if snapshot == _PAUSE_SNAPSHOT_AUTO else snapshot + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + result = asyncio.run(core.pause_sandbox(name=name, namespace=namespace, snapshot_name=snapshot_name)) + if not result.success: + typer.echo(f"❌ {result.error}", err=True) + raise typer.Exit(1) + typer.echo(result.message) + return + data = handle_api_call(lambda: cli_ctx.client().pause(name, namespace=namespace, snapshot=snapshot_name)) + typer.echo(data.get("message", f"Sandbox {name} paused")) - cmd += ["-f", compose_path] - # Add kubeconfig override if needed so container can reach apiserver - kube_override = _prepare_container_kubeconfig_and_override(compose_path) - if kube_override: - cmd += ["-f", kube_override] - # Safety: if user overrides image/tag via env, confirm unless --yes provided - use_local_image = bool(os.getenv("K7_API_IMAGE")) or bool(os.getenv("K7_API_TAG")) - auto_yes = yes - if use_local_image and not auto_yes: - typer.echo("Detected local override via K7_API_IMAGE/K7_API_TAG.") - typer.echo("Use this local image? [y/N] (unset env vars to use remote)") - try: - choice = input().strip().lower() - except EOFError: - choice = "n" - if choice != "y": - # Unset envs for this process so compose pulls remote - os.environ.pop("K7_API_IMAGE", None) - os.environ.pop("K7_API_TAG", None) - # Ensure relative paths in compose resolve properly - workdir = str(Path(workdir)) - # No build step: image is either remote (default) or local (override) - up_args = cmd + ["up", "-d"] - if use_local_image: - # Avoid pulling when a local override is requested - up_args += ["--pull", "never"] - up = subprocess.run(up_args, cwd=workdir) - if up.returncode != 0: - typer.echo("❌ Failed to start API via Docker Compose", err=True) + +@app.command() +def resume( + ctx: typer.Context, + name: str, + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), +): + """Resume a paused sandbox (scale to 1).""" + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + result = asyncio.run(core.resume_sandbox(name=name, namespace=namespace)) + if not result.success: + typer.echo(f"❌ {result.error}", err=True) + raise typer.Exit(1) + typer.echo(result.message) + return + data = handle_api_call(lambda: cli_ctx.client().resume(name, namespace=namespace)) + typer.echo(data.get("message", f"Sandbox {name} resumed")) + + +@app.command() +def restore( + ctx: typer.Context, + snapshot_name: str = typer.Argument(..., help="Existing VolumeSnapshot name"), + new_sandbox_name: str = typer.Argument(..., help="Name for the restored sandbox"), + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), + image: str | None = typer.Option( + None, + "--image", + help="Override the source image; required if the snapshot lacks the k7.io/source-image annotation", + ), + root_disk_size: str | None = typer.Option( + None, + "--root-disk-size", + help="Override root disk size (must be >= the snapshot's restoreSize); defaults to source size", + ), + cpu: str | None = typer.Option(None, "--cpu", help="Override CPU limit (e.g. 1, 500m)"), + memory: str | None = typer.Option(None, "--memory", help="Override memory limit (e.g. 1Gi)"), + storage: str | None = typer.Option(None, "--storage", help="Override ephemeral-storage limit"), + sidecar: str | None = typer.Option( + None, + "--sidecar", + help="Override sidecar plugin (e.g. docker); pass empty string to disable", + ), + cmd: builtins.list[str] | None = typer.Option(None, "--cmd", help="Override container CMD (repeatable)"), + entrypoint: builtins.list[str] | None = typer.Option( + None, "--entrypoint", help="Override container ENTRYPOINT (repeatable)" + ), + keep_snapshot: bool = typer.Option( + True, + "--keep-snapshot/--no-keep-snapshot", + help="Keep the source snapshot after a successful restore (default: keep)", + ), +): + """Restore a brand-new sandbox from a standalone VolumeSnapshot. + + Unlike ``k7 fork``, this does not require the original sandbox's Deployment + to still exist — it only needs the snapshot. The new sandbox boots from a + PVC cloned from the snapshot's data; image / backend / sidecar / limits / + root-disk-size default to the source sandbox's values via the + ``k7.io/source-*`` annotations stamped on the snapshot at creation time. + Use the flags above to override individual fields, or to supply them when + the snapshot was created before those annotations existed. + """ + limits: dict[str, str] | None = None + if cpu or memory or storage: + limits = {} + if cpu: + limits["cpu"] = cpu + if memory: + limits["memory"] = memory + if storage: + limits["ephemeral-storage"] = storage + overrides = SandboxConfigOverrides( + image=image, + root_disk_size=root_disk_size, + sidecar=sidecar, + limits=limits, + entrypoint=builtins.list(entrypoint) if entrypoint else None, + cmd=builtins.list(cmd) if cmd else None, + ) + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + result = asyncio.run( + core.restore_sandbox( + snapshot_name=snapshot_name, + new_sandbox_name=new_sandbox_name, + namespace=namespace, + overrides=overrides, + keep_snapshot=keep_snapshot, + ) + ) + if not result.success: + typer.echo(f"❌ {result.error}", err=True) + raise typer.Exit(1) + typer.echo(result.message) + return + handle_api_call( + lambda: cli_ctx.client().restore( + snapshot_name=snapshot_name, + new_sandbox_name=new_sandbox_name, + namespace=namespace, + overrides=overrides.to_dict() or None, + keep_snapshot=keep_snapshot, + ) + ) + typer.echo(f"Sandbox {new_sandbox_name} restored from snapshot {snapshot_name}") + + +@app.command() +def fork( + ctx: typer.Context, + source: str = typer.Argument(..., help="Existing sandbox name to fork"), + new_name: str = typer.Argument(..., help="Name of the new sandbox"), + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), + snapshot: str | None = typer.Option( + None, + "--snapshot", + help="Optional snapshot name to use for the forked disk (kata-qemu-longhorn)", + ), +): + """Clone a sandbox and its root disk (kata-qemu-longhorn) to a new name.""" + cli_ctx: CliContext = ctx.obj + if not cli_ctx.use_core: + handle_api_call(lambda: cli_ctx.client().fork(source, new_name, namespace=namespace, snapshot=snapshot)) + typer.echo(f"Forked {source} -> {new_name}") + return + core = K7Core() + result = asyncio.run( + core.fork_sandbox( + source_name=source, + new_name=new_name, + namespace=namespace, + snapshot_name=snapshot, + ) + ) + if result.success: + typer.echo(result.message) + else: + typer.echo(f"❌ {result.error}", err=True) raise typer.Exit(1) - # Try to display endpoint if available (read docker logs directly) + +def _build_and_import_api_image() -> bool: + """Build k7-api:local with Docker and import into k3s containerd. + + Returns True on success, False if Docker is unavailable or the build fails. + """ + if not shutil.which("docker"): + typer.echo("❌ Docker is required to build the API image. Install Docker first.", err=True) + return False + + # Find the Dockerfile relative to the package + from importlib import resources as _resources + + dockerfile = None try: - # Try docker logs first - logs = subprocess.run( - ["docker", "logs", "--since", "15m", "--tail", "10000", "k7-cloudflared"], + ref = _resources.files("k7.api").joinpath("Dockerfile.api") + with _resources.as_file(ref) as p: + if p.exists(): + dockerfile = str(p) + except Exception: + pass + if not dockerfile: + src_path = Path(__file__).resolve().parent.parent / "api" / "Dockerfile.api" + if src_path.exists(): + dockerfile = str(src_path) + if not dockerfile: + typer.echo("⚠️ Dockerfile.api not found; skipping local image build", err=True) + return False + + # Dockerfile is at src/k7/api/Dockerfile.api; context must be repo root + # (the dir containing src/) so COPY src/k7/... paths resolve correctly + dockerfile_path = Path(dockerfile).resolve() + context = str(dockerfile_path.parent.parent.parent.parent) + + typer.echo("Building k7-api:local image...") + build = subprocess.run( + # ``--network=host`` works around a Hetzner-node DNS flake where the + # default Docker bridge fails to resolve pypi.org mid-build; matches + # the Ansible playbook's invocation. + ["docker", "build", "--network=host", "-f", dockerfile, "-t", "k7-api:local", context], + ) + if build.returncode != 0: + typer.echo("❌ Docker build failed", err=True) + return False + + typer.echo("Importing image into k3s containerd...") + save = subprocess.run( + ["docker", "save", "k7-api:local"], + capture_output=True, + ) + if save.returncode != 0: + typer.echo("❌ docker save failed", err=True) + return False + + ctr_import = subprocess.run( + ["k3s", "ctr", "images", "import", "-"], + input=save.stdout, + ) + if ctr_import.returncode != 0: + typer.echo("❌ k3s ctr images import failed", err=True) + return False + + return True + + +# --------------------------------------------------------------------------- +# Spec 10d: ``k7 api ...`` sub-app — feature-toggle framing for the API server. +# The API is deployed by ``k7 install`` (gated on ``k7_api_enabled``) and lives +# on as a normal Kubernetes Deployment from then on. ``api enable/disable`` +# scale that Deployment up/down for temporary off/on; ``status``/``endpoint`` +# are read-only diagnostics. The dev-only "rebuild + roll" path moved under +# the hidden ``k7 dev api`` group so it stops cluttering ``--help`` for users +# who never set foot in the source tree. +# --------------------------------------------------------------------------- + + +api_app = typer.Typer( + help="API server feature toggle and diagnostics (status / endpoint / enable / disable).", + context_settings={"help_option_names": ["-h", "--help"]}, +) +app.add_typer(api_app, name="api") + +# Hidden ``k7 dev …`` group for developer-only utilities (mostly local-build +# flows that only make sense when you're iterating on the source tree). +dev_app = typer.Typer( + help="Developer-only commands (not for normal users).", + hidden=True, + context_settings={"help_option_names": ["-h", "--help"]}, +) +app.add_typer(dev_app, name="dev") +dev_api_app = typer.Typer( + help="Developer-only API commands (local Docker build / re-import / roll).", + context_settings={"help_option_names": ["-h", "--help"]}, +) +dev_app.add_typer(dev_api_app, name="api") + + +def _api_scale_deployment(replicas: int) -> None: + """Scale ``deployment/k7-api`` in ``kube-system`` to ``replicas``. + + Errors are loud — they exit the CLI with a non-zero status — because + "I asked to disable but nothing happened" is exactly the silent + failure mode 10d is trying to eliminate. + """ + kubectl = _kubectl_cmd() + result = subprocess.run( + kubectl + ["scale", "deployment", "k7-api", "-n", "kube-system", f"--replicas={replicas}"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + typer.echo(f"❌ Failed to scale k7-api to {replicas}: {result.stderr.strip()}", err=True) + raise typer.Exit(1) + + +@api_app.command("status") +def api_status_cmd(): + """Show API server readiness, endpoint, and key-management hints.""" + kubectl = _kubectl_cmd() + result = subprocess.run( + kubectl + + [ + "get", + "deployment", + "k7-api", + "-n", + "kube-system", + "-o", + "jsonpath={.status.readyReplicas}/{.spec.replicas}", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + typer.echo("❌ K7 API deployment not found in K3s") + typer.echo("Run `k7 install` to deploy it, or `k7 install --no-api` if that's deliberate.") + return + + ready_info = result.stdout.strip() + parts = ready_info.split("/") + ready_count = int(parts[0]) if parts[0] else 0 + desired = int(parts[1]) if len(parts) > 1 and parts[1] else 0 + + if ready_count > 0: + typer.echo(f"✅ K7 API is running in K3s ({ready_count}/{desired} ready)") + elif desired == 0: + typer.echo("⚠️ K7 API deployment exists but is scaled to 0 (disabled). Run `k7 api enable`.") + else: + typer.echo(f"⚠️ K7 API deployment exists but no pods ready ({ready_count}/{desired})") + + pod_result = subprocess.run( + kubectl + ["get", "pods", "-n", "kube-system", "-l", "app=k7-api", "-o", "wide", "--no-headers"], + capture_output=True, + text=True, + ) + if pod_result.returncode == 0 and pod_result.stdout.strip(): + typer.echo(f"\n📦 Pod(s):\n{pod_result.stdout.strip()}") + + endpoint = _read_api_endpoint(kubectl) + if endpoint: + typer.echo(f"\n🌐 API endpoint: {endpoint}") + + typer.echo("\n📝 SDK usage example:") + typer.echo(" from k7_sdk import Client") + typer.echo(" k7 = Client(endpoint='', api_key='')") + typer.echo(" sb = k7.create({'name': 'test', 'image': 'alpine:3.21'})") + typer.echo("\n🔐 Manage API keys:") + typer.echo(" k7 generate-api-key ") + typer.echo(" k7 list-api-keys") + typer.echo(" k7 revoke-api-key ") + + +@api_app.command("endpoint") +def api_endpoint_cmd(): + """Print the K7 API NodePort URL (machine-readable, single line).""" + kubectl = _kubectl_cmd() + endpoint = _read_api_endpoint(kubectl) + if endpoint: + typer.echo(endpoint) + else: + typer.echo("❌ K7 API service not found", err=True) + raise typer.Exit(1) + + +@api_app.command("enable") +def api_enable_cmd(): + """Scale the existing k7-api Deployment to 1 replica.""" + _api_scale_deployment(1) + typer.echo("✅ K7 API enabled (replicas=1).") + + +@api_app.command("disable") +def api_disable_cmd(): + """Scale the existing k7-api Deployment to 0 replicas (temporary off).""" + _api_scale_deployment(0) + typer.echo("✅ K7 API disabled (replicas=0). Run `k7 api enable` to bring it back.") + + +@dev_api_app.command("rebuild") +def dev_api_rebuild_cmd( + api_port: int = typer.Option(31007, "--api-port", help="NodePort for the API service (default 31007)"), + skip_build: bool = typer.Option(False, "--skip-build", help="Skip building the API image (assume it exists)"), +): + """Rebuild the local k7-api image, re-import into k3s, re-apply manifests. + + Developer iteration loop — only makes sense when the controller IS a + cluster node (single-node dev). Refuses to run otherwise (kubectl must be + reachable and the k7-api manifests must be present). + """ + kubectl = _kubectl_cmd() + manifests = _get_api_manifests_dir() + + if not skip_build and not _build_and_import_api_image(): + typer.echo("⚠️ Image build skipped or failed; deployment may fail if image is not present", err=True) + + # Patch NodePort in the service manifest if non-default. + service_file = Path(manifests) / "service.yaml" + if api_port != 31007 and service_file.exists(): + content = service_file.read_text() + content = content.replace("nodePort: 31007", f"nodePort: {api_port}") + service_file.write_text(content) + + typer.echo("Re-applying K7 API manifests...") + apply = subprocess.run(kubectl + ["apply", "-f", manifests], capture_output=True, text=True) + if apply.returncode != 0: + typer.echo(f"❌ Failed to apply API manifests: {apply.stderr.strip()}", err=True) + raise typer.Exit(1) + + typer.echo("Rolling k7-api deployment to pick up new image...") + subprocess.run( + kubectl + ["rollout", "restart", "deployment/k7-api", "-n", "kube-system"], + capture_output=True, + text=True, + ) + rollout = subprocess.run( + kubectl + ["rollout", "status", "deployment/k7-api", "-n", "kube-system", "--timeout=180s"], + capture_output=True, + text=True, + ) + if rollout.returncode != 0: + typer.echo(f"⚠️ Deployment not ready yet: {rollout.stderr.strip()}", err=True) + typer.echo("Check status with: k7 api status") + else: + typer.echo("✅ K7 API rolled out with the new image.") + + endpoint = _read_api_endpoint(kubectl) + if endpoint: + typer.echo(f"🌐 API endpoint: {endpoint}") + + +# Deprecated top-level commands. ``hidden=True`` keeps them off the main +# ``--help`` output while leaving the names callable so existing user +# scripts don't break. They emit a one-line deprecation warning to stderr +# and forward to the new home. Drop these in the next release. + + +@app.command(name="start-api", hidden=True) +def _deprecated_start_api( + api_port: int = typer.Option(31007, "--api-port", help="NodePort for the API service"), + skip_build: bool = typer.Option(False, "--skip-build", help="Skip building the API image"), +): + """Deprecated alias for ``k7 dev api rebuild``.""" + typer.echo( + "⚠️ `k7 start-api` is deprecated. The API is deployed by `k7 install`. " + "For dev iteration use `k7 dev api rebuild`.", + err=True, + ) + dev_api_rebuild_cmd(api_port=api_port, skip_build=skip_build) + + +@app.command(name="stop-api", hidden=True) +def _deprecated_stop_api( + delete: bool = typer.Option(False, "--delete", help="Delete API resources instead of scaling to 0"), +): + """Deprecated alias for ``k7 api disable``.""" + if delete: + typer.echo( + "⚠️ `k7 stop-api --delete` is deprecated. Remove the manifests with " + "`kubectl delete -f /etc/k7/manifests/k7-api/` if you really mean to uninstall.", + err=True, + ) + kubectl = _kubectl_cmd() + manifests = _get_api_manifests_dir() + subprocess.run( + kubectl + ["delete", "-f", manifests, "--ignore-not-found"], capture_output=True, text=True, ) - public_url = None - if logs.returncode == 0 and logs.stdout: - for line in logs.stdout.splitlines(): - if "trycloudflare.com" in line: - parts = [ - tok - for tok in line.split() - if tok.startswith("https://") or tok.startswith("http://") - ] - if parts: - public_url = parts[0] - break - # Fallback to compose logs if not found - if not public_url: - compose_logs = subprocess.run(cmd + ["logs", "cloudflared"], capture_output=True, text=True) - if compose_logs.returncode == 0 and compose_logs.stdout: - for line in compose_logs.stdout.splitlines(): - if "trycloudflare.com" in line: - parts = [ - tok - for tok in line.split() - if tok.startswith("https://") or tok.startswith("http://") - ] - if parts: - public_url = parts[0] - break - if public_url: - typer.echo(f"API started. Public endpoint: {public_url}") - else: - typer.echo("API started.") - typer.echo("Next steps:") - typer.echo("- Run: k7 api-status") - typer.echo("- Run: k7 get-api-endpoint") - typer.echo("- Generate API key: k7 generate-api-key ") - typer.echo("- List API keys: k7 list-api-keys") - typer.echo("- Stop when done: k7 stop-api") - except Exception: - typer.echo("API started.") - typer.echo("Next steps:") - typer.echo("- Run: k7 api-status") - typer.echo("- Run: k7 get-api-endpoint") - typer.echo("- Generate API key: k7 generate-api-key ") - typer.echo("- List API keys: k7 list-api-keys") - typer.echo("- Stop when done: k7 stop-api") + typer.echo("K7 API resources deleted") + return + typer.echo( + "⚠️ `k7 stop-api` is deprecated. Use `k7 api disable` (scale to 0).", + err=True, + ) + api_disable_cmd() -@app.command() -def api_status( - compose_file: Optional[str] = typer.Option( - None, "--compose-file", help="Path to docker-compose.yml used to start the API" - ), -): - """Show API server status and connection info.""" - try: - cmd = ["docker", "compose"] - compose_path, _workdir = _resolve_compose_path_or_fail(compose_file) - cmd += ["-f", compose_path] - - # Prefer raw docker inspect to avoid compose context mismatches - inspect_api = subprocess.run(["docker", "inspect", "-f", "{{.State.Running}}", "k7-api"], capture_output=True, text=True) - inspect_tun = subprocess.run(["docker", "inspect", "-f", "{{.State.Running}}", "k7-cloudflared"], capture_output=True, text=True) - - if inspect_api.stdout.strip() == "true" and inspect_tun.stdout.strip() == "true": - typer.echo("✅ K7 API is running via Docker Compose") - # Try to extract Cloudflared public URL from logs - logs = subprocess.run(["docker", "logs", "--since", "15m", "--tail", "10000", "k7-cloudflared"], capture_output=True, text=True) - public_url = None - for line in logs.stdout.splitlines(): - if "trycloudflare.com" in line: - # simple parse to extract URL - parts = [ - tok - for tok in line.split() - if tok.startswith("https://") or tok.startswith("http://") - ] - if parts: - public_url = parts[0] - break - if public_url: - typer.echo(f"🌐 Public URL: {public_url}") - else: - # Fallback to compose logs if docker logs missed it - compose_logs = subprocess.run(cmd + ["logs", "cloudflared"], capture_output=True, text=True) - if compose_logs.returncode == 0: - for line in compose_logs.stdout.splitlines(): - if "trycloudflare.com" in line: - parts = [ - tok - for tok in line.split() - if tok.startswith("https://") or tok.startswith("http://") - ] - if parts: - public_url = parts[0] - break - if public_url: - typer.echo(f"🌐 Public URL: {public_url}") - else: - typer.echo("🌐 Public URL: not detected yet; try 'k7 get-api-endpoint'") - - typer.echo("\n📝 SDK Usage Example:") - typer.echo("from katakate import Client") - typer.echo("k7 = Client(endpoint='https://', api_key='')") - typer.echo("sb = k7.create({'name': 'test', 'image': 'alpine:latest'})") - typer.echo("print(k7.list())") - typer.echo("print(sb.exec('echo Hello')) # returns dict with stdout/stderr") - typer.echo("\n🔐 Manage API keys:") - typer.echo("k7 generate-api-key ") - typer.echo("k7 list-api-keys") - typer.echo("k7 revoke-api-key ") - else: - typer.echo("❌ K7 API is not running") - typer.echo("Start with: k7 start-api") - - except FileNotFoundError: - typer.echo("❌ Docker not found or compose plugin missing") - typer.echo("Install Docker and compose plugin.") +@app.command(name="api-status", hidden=True) +def _deprecated_api_status(): + """Deprecated alias for ``k7 api status``.""" + typer.echo("⚠️ `k7 api-status` is deprecated. Use `k7 api status`.", err=True) + api_status_cmd() -@app.command() -def stop_api( - compose_file: Optional[str] = typer.Option( - None, "--compose-file", help="Path to docker-compose.yml used to start the API" - ), - remove_volumes: bool = typer.Option( - False, "--prune", help="Remove named volumes as part of shutdown" - ), -): - """Stop the K7 API server and Cloudflared tunnel.""" - cmd = ["docker", "compose"] - compose_path, _workdir = _resolve_compose_path_or_fail(compose_file) - cmd += ["-f", compose_path] - down_cmd = cmd + ["down"] + (["-v"] if remove_volumes else []) - try: - subprocess.run(down_cmd, check=False) - finally: - # Ensure containers are gone even if compose file moved - subprocess.run( - ["docker", "rm", "-f", "k7-api", "k7-cloudflared"], - check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, +@app.command(name="get-api-endpoint", hidden=True) +def _deprecated_get_api_endpoint(): + """Deprecated alias for ``k7 api endpoint``.""" + typer.echo("⚠️ `k7 get-api-endpoint` is deprecated. Use `k7 api endpoint`.", err=True) + api_endpoint_cmd() + + +# --------------------------------------------------------------------------- +# Spec 18h: ``k7 nodes …`` sub-app for cluster-node queries. +# --------------------------------------------------------------------------- + + +nodes_app = typer.Typer( + help="Query cluster nodes (storage pools, …).", + context_settings={"help_option_names": ["-h", "--help"]}, +) +app.add_typer(nodes_app, name="nodes") + + +def _print_nodes_storage_table(data: dict) -> None: + """Render ``nodes_storage()`` output as a per-node Rich table.""" + if not data: + typer.echo("No nodes found.") + return + table = Table(title="K7 Node Storage") + table.add_column("NODE", style="cyan") + table.add_column("THINPOOL", justify="right") + table.add_column("DATA%", justify="right") + table.add_column("META%", justify="right") + table.add_column("K7D POOL", justify="right") + table.add_column("K7D USED%", justify="right") + table.add_column("ERROR", style="red") + for node in sorted(data): + entry = data[node] + if "error" in entry: + table.add_row(node, "-", "-", "-", "-", "-", entry["error"]) + continue + tp = entry.get("kata_thinpool") or {} + kd = entry.get("k7d_disks") or {} + table.add_row( + node, + _format_size_bytes(int(tp.get("size_bytes", 0) or 0)), + f"{float(tp.get('data_percent', 0)):.2f}", + f"{float(tp.get('metadata_percent', 0)):.2f}", + _format_size_bytes(int(kd.get("size_bytes", 0) or 0)), + f"{float(kd.get('used_percent', 0)):.2f}", + "", ) - # Clean extracted build context for a fresh next start - try: - shutil.rmtree("/tmp/k7-api-build") - except Exception: - pass - typer.echo("K7 API stopped") + Console().print(table) -@app.command() -def get_api_endpoint( - compose_file: Optional[str] = typer.Option( - None, "--compose-file", help="Path to docker-compose.yml used to start the API" - ), +@nodes_app.command("storage") +def nodes_storage( + ctx: typer.Context, + as_json: bool = typer.Option(False, "--json", help="Print raw JSON instead of a table"), ): - """Print the current Cloudflared public URL for the API, if available.""" - cmd = ["docker", "compose"] - compose_path, _workdir = _resolve_compose_path_or_fail(compose_file) - cmd += ["-f", compose_path] - - # Check if service is up via raw docker inspect - inspect_api = subprocess.run(["docker", "inspect", "-f", "{{.State.Running}}", "k7-api"], capture_output=True, text=True) - inspect_tun = subprocess.run(["docker", "inspect", "-f", "{{.State.Running}}", "k7-cloudflared"], capture_output=True, text=True) - if inspect_api.stdout.strip() != "true" or inspect_tun.stdout.strip() != "true": - raise typer.Exit(1) - - # Get URL from docker logs (same approach as api-status for consistency) - url = None - logs = subprocess.run(["docker", "logs", "--since", "15m", "--tail", "10000", "k7-cloudflared"], capture_output=True, text=True) - if logs.returncode == 0 and logs.stdout: - for line in logs.stdout.splitlines(): - if "trycloudflare.com" in line: - parts = [ - tok - for tok in line.split() - if tok.startswith("https://") or tok.startswith("http://") - ] - if parts: - url = parts[0] - break - if not url: - # Fallback to compose logs using resolved compose file (no /tmp fallback) - cmd = ["docker", "compose", "-f", compose_path] - for _ in range(5): - clogs = subprocess.run(cmd + ["logs", "cloudflared"], capture_output=True, text=True) - if clogs.returncode == 0 and clogs.stdout: - for line in clogs.stdout.splitlines(): - if "trycloudflare.com" in line: - parts = [ - tok - for tok in line.split() - if tok.startswith("https://") or tok.startswith("http://") - ] - if parts: - url = parts[0] - break - if url: - break - time.sleep(1) - if url: - typer.echo(url) + """Show per-node kfd thin-pool and k7d disks-pool utilization.""" + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + data = asyncio.run(K7Core().nodes_storage()) else: + data = handle_api_call(lambda: cli_ctx.client().nodes_storage()) + if as_json: + typer.echo(json.dumps(data, indent=2, default=str)) + return + _print_nodes_storage_table(data) + + +# Spec 10e: ``k7 snapshot …`` sub-app for VolumeSnapshot CRUD + GC. +# --------------------------------------------------------------------------- + + +snapshot_app = typer.Typer( + help="Manage VolumeSnapshots (pause / fork / named).", + context_settings={"help_option_names": ["-h", "--help"]}, +) +app.add_typer(snapshot_app, name="snapshot") + + +def _format_size_bytes(size: int) -> str: + """Render a byte count as Gi/Mi/Ki (matches ``kubectl get pvc`` style).""" + if size <= 0: + return "-" + for unit, factor in (("Gi", 1 << 30), ("Mi", 1 << 20), ("Ki", 1 << 10)): + if size >= factor: + return f"{size / factor:.1f}{unit}" + return f"{size}B" + + +def _format_age(age: str) -> str: + """Trim a ``str(timedelta)`` to a kubectl-style short form.""" + if not age or age == "Unknown": + return age or "-" + # ``str(datetime.now(...) - created)`` looks like ``0:05:42.123456`` or + # ``2 days, 0:05:42``. Strip the microseconds and the "0 days," prefix. + out = age.split(".", 1)[0] + return out.replace("0:", "0h", 1) if out.startswith("0:") else out + + +def _print_snapshot_table(snaps: builtins.list[dict], all_namespaces: bool) -> None: + """Render a list of snapshot dicts (from K7Core or the SDK) as a Rich table.""" + if not snaps: + typer.echo("No snapshots found.") + return + table = Table(box=None) + table.add_column("NAME") + if all_namespaces: + table.add_column("NAMESPACE") + table.add_column("SANDBOX") + table.add_column("SIZE") + table.add_column("AGE") + table.add_column("READY") + table.add_column("KIND") + for s in snaps: + row = [s.get("name", "")] + if all_namespaces: + row.append(s.get("namespace", "")) + row.extend( + [ + s.get("source_sandbox") or "-", + _format_size_bytes(int(s.get("size_bytes", 0) or 0)), + _format_age(s.get("age", "")), + "True" if s.get("ready_to_use") else "False", + s.get("kind", ""), + ] + ) + table.add_row(*row) + Console().print(table) + + +@snapshot_app.command("list") +def snapshot_list( + ctx: typer.Context, + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), + all_namespaces: bool = typer.Option(False, "-A", "--all-namespaces", help="List across all namespaces"), + sandbox: str | None = typer.Option(None, "--sandbox", help="Filter to snapshots tied to a sandbox"), + kind: str | None = typer.Option(None, "--kind", help="Filter by kind: pause / fork / named"), +): + """List VolumeSnapshots managed by k7.""" + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + snap_objs = asyncio.run( + core.list_snapshots( + namespace=namespace, + all_namespaces=all_namespaces, + sandbox=sandbox, + kind=kind, + ) + ) + snaps = [s.to_dict() for s in snap_objs] + else: + snaps = handle_api_call( + lambda: cli_ctx.client().list_snapshots( + namespace=namespace, + all_namespaces=all_namespaces, + sandbox=sandbox, + kind=kind, + ) + ) + _print_snapshot_table(snaps, all_namespaces) + + +@snapshot_app.command("inspect") +def snapshot_inspect( + ctx: typer.Context, + name: str, + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), +): + """Show full details for a single snapshot.""" + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + snap = asyncio.run(core.get_snapshot(name, namespace=namespace)) + if snap is None: + typer.echo(f"❌ Snapshot {name} not found in namespace {namespace}", err=True) + raise typer.Exit(1) + typer.echo(json.dumps(snap.to_dict(), indent=2, default=str)) + return + data = handle_api_call(lambda: cli_ctx.client().get_snapshot(name, namespace=namespace)) + if data is None: + typer.echo(f"❌ Snapshot {name} not found in namespace {namespace}", err=True) raise typer.Exit(1) + typer.echo(json.dumps(data, indent=2, default=str)) + + +@snapshot_app.command("create") +def snapshot_create( + ctx: typer.Context, + sandbox_name: str = typer.Argument(..., help="Sandbox whose root PVC to snapshot"), + snapshot_name: str = typer.Argument(..., help="Name to give the new VolumeSnapshot"), + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), +): + """Snapshot a running sandbox's root PVC without pausing it (kind=named).""" + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + result = asyncio.run(core.create_snapshot(sandbox_name, snapshot_name, namespace=namespace)) + if not result.success: + typer.echo(f"❌ {result.error}", err=True) + raise typer.Exit(1) + typer.echo(result.message) + return + handle_api_call(lambda: cli_ctx.client().create_snapshot(sandbox_name, snapshot_name, namespace=namespace)) + typer.echo(f"Snapshot {snapshot_name} created for sandbox {sandbox_name}") + + +@snapshot_app.command("delete") +def snapshot_delete( + ctx: typer.Context, + name: str, + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt"), +): + """Delete a snapshot by name.""" + if not yes: + if not sys.stdin.isatty(): + typer.echo("❌ Refusing to read confirmation from a non-TTY; pass --yes to proceed.", err=True) + raise typer.Exit(1) + confirmed = typer.confirm(f"Delete VolumeSnapshot {name} in namespace {namespace}?") + if not confirmed: + typer.echo("Aborted.") + raise typer.Exit(1) + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + core = K7Core() + result = asyncio.run(core.delete_snapshot(name, namespace=namespace)) + if not result.success: + typer.echo(f"❌ {result.error}", err=True) + raise typer.Exit(1) + typer.echo(result.message) + return + data = handle_api_call(lambda: cli_ctx.client().delete_snapshot(name, namespace=namespace)) + typer.echo(data.get("message", f"Snapshot {name} deleted")) + + +@snapshot_app.command("gc") +def snapshot_gc( + ctx: typer.Context, + namespace: str = typer.Option("default", "-n", "--namespace", help="Kubernetes namespace"), + all_namespaces: bool = typer.Option(False, "-A", "--all-namespaces", help="GC across all namespaces"), + keep_fork_for: str = typer.Option( + "10m", + "--keep-fork-for", + help="Keep fork snapshots younger than this duration (e.g. 10m, 2h, 45s)", + ), + dry_run: bool = typer.Option(False, "--dry-run", help="List what would be deleted without deleting"), +): + """Delete stale ``kind=fork`` snapshots older than ``--keep-fork-for``. + + Pause snapshots and named snapshots are never touched. + """ + cli_ctx: CliContext = ctx.obj + if cli_ctx.use_core: + if keep_fork_for.endswith("m"): + td = timedelta(minutes=int(keep_fork_for[:-1])) + elif keep_fork_for.endswith("h"): + td = timedelta(hours=int(keep_fork_for[:-1])) + elif keep_fork_for.endswith("s"): + td = timedelta(seconds=int(keep_fork_for[:-1])) + else: + td = timedelta(seconds=int(keep_fork_for)) + core = K7Core() + result = asyncio.run( + core.gc_snapshots( + namespace=namespace, + all_namespaces=all_namespaces, + keep_fork_for=td, + dry_run=dry_run, + ) + ) + if not result.success: + typer.echo(f"❌ {result.error}", err=True) + raise typer.Exit(1) + typer.echo(result.message) + for record in result.data or []: + marker = "would-delete" if dry_run else ("deleted" if record.get("deleted") else "failed") + typer.echo(f" [{marker}] {record['namespace']}/{record['name']} (age={record['age']})") + return + data = handle_api_call( + lambda: cli_ctx.client().gc_snapshots( + namespace=namespace, + all_namespaces=all_namespaces, + keep_fork_for=keep_fork_for, + dry_run=dry_run, + ) + ) + typer.echo(data.get("message", "gc complete")) + for record in data.get("results", []) or []: + marker = "would-delete" if dry_run else ("deleted" if record.get("deleted") else "failed") + typer.echo(f" [{marker}] {record['namespace']}/{record['name']} (age={record['age']})") if __name__ == "__main__": + sys.argv = _rewrite_bare_snapshot_flag(sys.argv) app() diff --git a/src/k7/core/__init__.py b/src/k7/core/__init__.py index 3370f95..b09c600 100644 --- a/src/k7/core/__init__.py +++ b/src/k7/core/__init__.py @@ -1,4 +1,4 @@ from .core import K7Core -from .models import SandboxConfig, SandboxInfo, ExecResult +from .models import ExecResult, SandboxConfig, SandboxInfo __all__ = ["K7Core", "SandboxConfig", "SandboxInfo", "ExecResult"] diff --git a/src/k7/core/core.py b/src/k7/core/core.py index 8f586bf..297f84b 100644 --- a/src/k7/core/core.py +++ b/src/k7/core/core.py @@ -1,44 +1,120 @@ +import asyncio +import copy +import ipaddress +import json +import math import os -import tempfile -import subprocess -import time import re -import sys -from typing import Optional, List, Dict, Callable -from pathlib import Path import shutil -import yaml -from datetime import datetime -from kubernetes import client, config -from kubernetes.client.exceptions import ApiException -from kubernetes.stream import stream -from .models import SandboxConfig, SandboxInfo, ExecResult, OperationResult +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable +from datetime import datetime, timedelta, timezone from importlib import resources +from typing import Any + +import httpx +import yaml +from kubernetes_asyncio import client, config +from kubernetes_asyncio.client.exceptions import ApiException +from kubernetes_asyncio.stream import WsApiClient + +from .models import ( + SNAPSHOT_KIND_FORK, + SNAPSHOT_KIND_NAMED, + SNAPSHOT_KIND_PAUSE, + ExecResult, + OperationResult, + SandboxConfig, + SandboxConfigOverrides, + SandboxInfo, + SnapshotInfo, +) +from .sidecar import SIDECAR_REGISTRY + +# k7d backend (spec 9a M11): pod-level annotations the k7d containerd shim +# understands. `fork-source-*` boot a pod as a warm (CoW disk+memory) fork +# of a live source sandbox VM (k7d spec 17d). +K7D_ANN_FORK_SOURCE_CLUSTER = "k7d.katakate.org/fork-source-cluster" +K7D_ANN_FORK_SOURCE_VM = "k7d.katakate.org/fork-source-vm" +# Deployment-level marker k7 stamps so `k7 list` / resume can tell a +# VM-frozen (paused) k7d sandbox from a running one. +K7D_ANN_PAUSED = "k7.katakate.org/k7d-paused" +# Message pointing users at k7d's native VM snapshot trees for the verbs +# that intentionally do NOT map onto the k7d backend. +K7D_SNAPSHOT_UNSUPPORTED = ( + "named disk snapshots/restore are a Longhorn feature of the kata-qemu-longhorn (kql) " + "backend. k7d sandboxes support live warm fork (`k7 fork`) and VM pause/resume instead; " + "VM-level snapshot trees (fork/rollback/suspend of whole VM states) are available through " + "the k7d daemon API — see https://github.com/katakate/k7d" +) + + +def _classify_egress_entries(entries: list[str]) -> tuple[list[str], list[str]]: + """Split egress values into (CIDRs, FQDNs). + + An entry is treated as a CIDR when it is a valid IPv4/IPv6 network; bare + IPs are normalised to their /32 (or /128) form. Anything else is treated + as an FQDN (supports exact names and simple `*.example.com` wildcards). + """ + cidrs: list[str] = [] + fqdns: list[str] = [] + for raw in entries: + value = (raw or "").strip() + if not value: + continue + try: + if "/" in value: + net = ipaddress.ip_network(value, strict=False) + cidrs.append(str(net)) + else: + ip = ipaddress.ip_address(value) + cidrs.append(f"{ip}/{'32' if ip.version == 4 else '128'}") + except ValueError: + fqdns.append(value) + return cidrs, fqdns class K7Core: """Core business logic for sandbox management""" - def __init__(self, kubeconfig_path: Optional[str] = None): + def __init__(self, kubeconfig_path: str | None = None): self.kubeconfig_path = kubeconfig_path self._apps_v1_client = None self._core_v1_client = None self._networking_v1_client = None self._metrics_client = None + self._custom_objects_client = None self._config_loaded = False + self._api_client = None - def _load_k3s_config(self): - """Load k3s kubeconfig with fallback to standard locations.""" + async def _load_k3s_config(self): + """Load k3s kubeconfig with fallback to standard locations. + + When running inside a pod (KUBERNETES_SERVICE_HOST is set), prefer + in-cluster config via the ServiceAccount token. This avoids needing + a hostPath kubeconfig mount and gives explicit RBAC scoping. + """ if self._config_loaded: return + if os.environ.get("KUBERNETES_SERVICE_HOST"): + try: + config.load_incluster_config() + self._config_loaded = True + return + except config.ConfigException: + pass + k3s_config_path = self.kubeconfig_path or "/etc/rancher/k3s/k3s.yaml" try: if os.path.exists(k3s_config_path): - config.load_kube_config(config_file=k3s_config_path) + await config.load_kube_config(config_file=k3s_config_path) else: - config.load_kube_config() + await config.load_kube_config() except config.ConfigException: try: config.load_incluster_config() @@ -47,126 +123,593 @@ class K7Core: self._config_loaded = True - def _get_apps_v1_client(self): + async def _get_apps_v1_client(self): """Get or create AppsV1Api client instance.""" if self._apps_v1_client is None: - self._load_k3s_config() + await self._load_k3s_config() self._apps_v1_client = client.AppsV1Api() return self._apps_v1_client - def _get_core_v1_client(self): + async def _get_core_v1_client(self): """Get or create CoreV1Api client instance.""" if self._core_v1_client is None: - self._load_k3s_config() + await self._load_k3s_config() self._core_v1_client = client.CoreV1Api() return self._core_v1_client - def _get_networking_v1_client(self): + async def _get_networking_v1_client(self): """Get or create NetworkingV1Api client instance.""" if self._networking_v1_client is None: - self._load_k3s_config() + await self._load_k3s_config() self._networking_v1_client = client.NetworkingV1Api() return self._networking_v1_client - def _get_metrics_client(self): - """Get or create CustomObjectsApi client instance.""" + async def _get_metrics_client(self): + """Get or create CustomObjectsApi client for metrics.""" if self._metrics_client is None: - self._load_k3s_config() + await self._load_k3s_config() self._metrics_client = client.CustomObjectsApi() return self._metrics_client + async def _get_custom_objects_client(self): + """Get or create CustomObjectsApi client for VolumeSnapshots.""" + if self._custom_objects_client is None: + await self._load_k3s_config() + self._custom_objects_client = client.CustomObjectsApi() + return self._custom_objects_client + + def _load_persist_bind_script(self) -> str: + """Load the bind-mount wrapper script from package assets.""" + try: + return resources.files("k7").joinpath("assets/k7-persist-bind.sh").read_text() + except Exception as e: + raise Exception(f"Failed to load k7-persist-bind.sh: {e}") from e + + def _normalize_image_argv(self, value: object) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v) for v in value if v is not None] + if isinstance(value, str): + return [value] if value else [] + return [] + + def _extract_entrypoint_cmd(self, data: dict) -> tuple[list, list]: + info = data.get("info") if isinstance(data, dict) else None + if isinstance(info, dict) and isinstance(info.get("config"), dict): + ep = self._normalize_image_argv(info["config"].get("Entrypoint")) + cmd = self._normalize_image_argv(info["config"].get("Cmd")) + return ep, cmd + config_data = data.get("config") if isinstance(data, dict) else None + if isinstance(config_data, dict): + ep = self._normalize_image_argv(config_data.get("Entrypoint")) + cmd = self._normalize_image_argv(config_data.get("Cmd")) + return ep, cmd + return [], [] + + def _parse_image_reference(self, image: str) -> tuple[str, str, str]: + """Parse a container image reference into (registry, repository, tag_or_digest). + + Handles Docker Hub short names (alpine:3.21 → registry-1.docker.io/library/alpine, 3.21). + """ + ref = image + tag = "latest" + if "@" in ref: + ref, tag = ref.rsplit("@", 1) + elif ":" in ref: + parts = ref.rsplit(":", 1) + if "/" in parts[1] or parts[1].isdigit(): + pass + else: + ref, tag = parts[0], parts[1] + + if "/" not in ref: + return "registry-1.docker.io", f"library/{ref}", tag + first_part = ref.split("/")[0] + if "." in first_part or ":" in first_part or first_part == "localhost": + registry = first_part + repo = ref[len(first_part) + 1 :] + else: + registry = "registry-1.docker.io" + repo = ref + return registry, repo, tag + + async def _get_registry_image_config(self, image: str) -> dict: + """Fetch the OCI image config from a container registry. + + Supports Docker Hub (with token auth), ghcr.io, quay.io, and any + OCI-compliant registry that supports anonymous pulls. + """ + registry, repo, tag = self._parse_image_reference(image) + scheme = "http" if registry == "localhost" or registry.startswith("localhost:") else "https" + base = f"{scheme}://{registry}" + + headers: dict[str, str] = {} + + async with httpx.AsyncClient() as http_client: + if "docker.io" in registry: + try: + token_resp = await http_client.get( + f"https://auth.docker.io/token?service=registry.docker.io&scope=repository:{repo}:pull", + timeout=10, + ) + token_resp.raise_for_status() + token = token_resp.json()["token"] + headers["Authorization"] = f"Bearer {token}" + except Exception: + pass + + accept = ", ".join( + [ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + ] + ) + headers["Accept"] = accept + + manifest_url = f"{base}/v2/{repo}/manifests/{tag}" + manifest_resp = await http_client.get(manifest_url, headers=headers, timeout=15) + manifest_resp.raise_for_status() + manifest = manifest_resp.json() + + media_type = manifest.get("mediaType", "") + if "manifest.list" in media_type or "image.index" in media_type: + for m in manifest.get("manifests", []): + platform = m.get("platform", {}) + if platform.get("os") == "linux" and platform.get("architecture") == "amd64": + digest = m["digest"] + headers["Accept"] = ( + "application/vnd.oci.image.manifest.v1+json, " + "application/vnd.docker.distribution.manifest.v2+json" + ) + inner_resp = await http_client.get( + f"{base}/v2/{repo}/manifests/{digest}", headers=headers, timeout=15 + ) + inner_resp.raise_for_status() + manifest = inner_resp.json() + break + else: + raise ValueError(f"No linux/amd64 manifest found in index for {image}") + + config_digest = manifest.get("config", {}).get("digest") + if not config_digest: + raise ValueError(f"No config digest in manifest for {image}") + + config_resp = await http_client.get(f"{base}/v2/{repo}/blobs/{config_digest}", headers=headers, timeout=15) + config_resp.raise_for_status() + return config_resp.json() + + async def _get_image_entrypoint_cmd(self, image: str) -> tuple[list, list]: + """Extract Entrypoint and Cmd from an image via OCI registry inspection.""" + if not image: + return [], [] + try: + image_config = await self._get_registry_image_config(image) + container_config = image_config.get("config", {}) + ep = self._normalize_image_argv(container_config.get("Entrypoint")) + cmd = self._normalize_image_argv(container_config.get("Cmd")) + return ep, cmd + except Exception: + return [], [] + + def _compute_image_argv( + self, + image_entrypoint: list, + image_cmd: list, + override_entrypoint: list | None, + override_cmd: list | None, + ) -> list[str]: + if override_entrypoint is not None: + ep = list(override_entrypoint) + cmd = list(override_cmd) if override_cmd is not None else list(image_cmd) + return ep + cmd + ep = list(image_entrypoint) + cmd = list(override_cmd) if override_cmd is not None else list(image_cmd) + return ep + cmd + + async def _list_backends_per_node(self) -> dict[str, list[str]]: + """Return {node_name: [supported backends]} from k7 backend labels.""" + v1 = await self._get_core_v1_client() + nodes = await v1.list_node() + out: dict[str, list[str]] = {} + for n in nodes.items: + labels = n.metadata.labels or {} + backends = sorted( + { + self._canonicalize_backend(k.removeprefix("k7.katakate.org/backend-")) or "" + for k, v in labels.items() + if k.startswith("k7.katakate.org/backend-") and v == "true" + } + - {""} + ) + out[n.metadata.name] = backends + return out + + async def _check_scheduling( + self, + sandbox_name: str, + namespace: str, + backend: str, + timeout_seconds: int = 30, + ) -> OperationResult: + """Poll the sandbox pod for FailedScheduling events tied to the backend selector. + + Returns success when no scheduling failure is detected within `timeout_seconds`, + or when the pod is already running. Returns an explanatory error when the + node selector does not match any node. + """ + v1 = await self._get_core_v1_client() + selector_label = f"k7.katakate.org/backend-{backend}" + start = time.time() + while time.time() - start < timeout_seconds: + pods = await v1.list_namespaced_pod(namespace=namespace, label_selector=f"app={sandbox_name}") + if pods.items: + pod = pods.items[0] + phase = (pod.status.phase if pod.status else None) or "Pending" + if phase != "Pending": + return OperationResult(success=True) + try: + events = await v1.list_namespaced_event( + namespace=namespace, + field_selector=f"involvedObject.name={pod.metadata.name}", + ) + except ApiException: + events = None + if events: + for ev in events.items: + if ( + ev.reason == "FailedScheduling" + and ev.message + and ("node affinity/selector" in ev.message or selector_label in ev.message) + ): + backends_per_node = await self._list_backends_per_node() + available: dict[str, list[str]] = {} + for node, backends in backends_per_node.items(): + for b in backends: + available.setdefault(b, []).append(node) + summary = ( + ", ".join(f"{b} (nodes: {', '.join(sorted(nodes))})" for b, nodes in available.items()) + or "" + ) + return OperationResult( + success=False, + error=( + f"No node available supporting backend '{backend}'. " + "This can happen if all nodes supporting this backend are cordoned, " + "tainted, or have insufficient resources.\n" + f"Available backends across cluster: {summary}" + ), + ) + await asyncio.sleep(2) + return OperationResult(success=True) + + async def _wait_for_pod_container_started( + self, + sandbox_name: str, + namespace: str, + timeout_seconds: int = 300, + wait_all_containers: bool = False, + ) -> str | None: + v1 = await self._get_core_v1_client() + start_time = time.time() + while time.time() - start_time < timeout_seconds: + pods = await v1.list_namespaced_pod(namespace=namespace, label_selector=f"app={sandbox_name}") + if not pods.items: + await asyncio.sleep(2) + continue + pod = pods.items[0] + if pod.status and pod.status.phase != "Running": + await asyncio.sleep(2) + continue + statuses = pod.status.container_statuses or [] + if wait_all_containers and statuses: + if all(s.state and s.state.running for s in statuses): + return pod.metadata.name + else: + for status in statuses: + if status.name == "sandbox" and status.state and status.state.running: + return pod.metadata.name + await asyncio.sleep(2) + return None + + @staticmethod + def _canonicalize_backend(backend: str | None) -> str | None: + """Map deprecated backend names to canonical ones; pass through unknown values.""" + if backend is None: + return None + legacy = { + "firecracker-devmapper": "kata-firecracker-devmapper", + "qemu-longhorn": "kata-qemu-longhorn", + "fd": "kata-firecracker-devmapper", + "ql": "kata-qemu-longhorn", + "kfd": "kata-firecracker-devmapper", + "kql": "kata-qemu-longhorn", + "k7": "k7d", + } + return legacy.get(backend, backend) + + async def _detect_backend(self, sandbox_name: str | None = None, namespace: str = "default") -> str: + """Detect backend with priority: deployment annotation > /etc/k7/backend > default.""" + allowed = ("kata-firecracker-devmapper", "kata-qemu-longhorn", "k7d") + if sandbox_name: + try: + apps_v1 = await self._get_apps_v1_client() + deployment = await apps_v1.read_namespaced_deployment(sandbox_name, namespace) + backend = self._canonicalize_backend(deployment.metadata.annotations.get("k7.katakate.org/backend")) + if backend in allowed: + return backend + except Exception: + pass + + try: + if os.path.exists("/etc/k7/backend"): + with open("/etc/k7/backend") as f: + backend = self._canonicalize_backend(f.read().strip()) + if backend in allowed: + return backend + except Exception: + pass + + return "kata-firecracker-devmapper" + + async def _update_deployment_annotation(self, name: str, namespace: str, key: str, value: str): + """Update a single deployment annotation.""" + apps_v1 = await self._get_apps_v1_client() + body = {"metadata": {"annotations": {key: value}}} + await apps_v1.patch_namespaced_deployment(name=name, namespace=namespace, body=body) + + # ------------------------------------------------------------------- + # k7d backend (spec 9a M11): daemon control-socket plumbing. + # + # The k7d daemon owns every `runtimeClassName: k7` VM on the node and + # exposes VM-level operations (fork / pause / resume / lookup) over a + # newline-delimited-JSON Unix socket. These helpers run on the node + # that hosts the sandbox pod — a remote pod is a loud error, never a + # silent no-op. + # ------------------------------------------------------------------- + + @staticmethod + def _k7d_socket_path() -> str: + return os.environ.get("K7D_SOCKET", "/run/k7d/k7d.sock") + + async def _k7d_request(self, payload: dict) -> dict: + """Send one request to the local k7d daemon and return its response. + + Raises ``RuntimeError`` on transport errors or an ``error`` response — + callers surface the message verbatim (fail loud, no fallbacks). + """ + socket_path = self._k7d_socket_path() + + def _call() -> dict: + import socket as _socket + + if not os.path.exists(socket_path): + raise RuntimeError( + f"k7d control socket {socket_path} not found — is the k7d daemon " + "running on this node? (k7 install --backend k7d sets it up)" + ) + with _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) as sock: + sock.settimeout(120) + sock.connect(socket_path) + sock.sendall((json.dumps(payload) + "\n").encode()) + buf = b"" + while not buf.endswith(b"\n"): + chunk = sock.recv(65536) + if not chunk: + break + buf += chunk + if not buf: + raise RuntimeError("k7d daemon closed the control connection without a response") + return json.loads(buf.decode()) + + resp = await asyncio.to_thread(_call) + if resp.get("status") == "error": + raise RuntimeError(f"k7d daemon error for op={payload.get('op')}: {resp.get('message')}") + return resp + + async def _k7d_running_pod(self, sandbox_name: str, namespace: str): + """Return the Running pod of a k7d sandbox, failing loud when the pod + is missing or scheduled on a different node than this process.""" + v1 = await self._get_core_v1_client() + pods = await v1.list_namespaced_pod(namespace=namespace, label_selector=f"app={sandbox_name}") + running = [ + p for p in pods.items if p.status and p.status.phase == "Running" and not p.metadata.deletion_timestamp + ] + if not running: + raise RuntimeError(f"no Running pod found for sandbox {sandbox_name} in {namespace}") + pod = running[0] + # Inside the k7-api pod, os.uname().nodename is the POD name, not the + # Kubernetes node — the deployment injects K7_NODE_NAME via the + # downward API so co-located k7d VM ops still work through the API. + local_hostname = os.environ.get("K7_NODE_NAME") or os.uname().nodename + if pod.spec.node_name and pod.spec.node_name != local_hostname: + raise RuntimeError( + f"sandbox {sandbox_name} runs on node {pod.spec.node_name}, but this k7 process " + f"runs on {local_hostname}; k7d VM operations must run on the pod's node" + ) + return pod + + async def _k7d_cri_sandbox_id(self, pod_name: str, namespace: str) -> str: + """Resolve a pod to its CRI (containerd) sandbox id via crictl.""" + crictl = shutil.which("crictl") or "/usr/local/bin/crictl" + cmd = [ + crictl, + "--runtime-endpoint", + "unix:///run/k3s/containerd/containerd.sock", + "pods", + "--name", + pod_name, + "--namespace", + namespace, + "--state", + "ready", + "-q", + ] + + def _run() -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, capture_output=True, text=True, timeout=30, check=False) + + result = await asyncio.to_thread(_run) + if result.returncode != 0: + raise RuntimeError(f"crictl pods failed for {pod_name}: {result.stderr.strip()}") + sandbox_ids = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if not sandbox_ids: + raise RuntimeError(f"no ready CRI sandbox found for pod {pod_name} in {namespace}") + return sandbox_ids[0] + + async def _k7d_vm_for_sandbox(self, sandbox_name: str, namespace: str) -> dict: + """Resolve a k7d sandbox to its daemon VM: returns the daemon's + ``sandbox_found`` response plus the CRI ``sandbox_id`` used.""" + pod = await self._k7d_running_pod(sandbox_name, namespace) + cri_sandbox_id = await self._k7d_cri_sandbox_id(pod.metadata.name, namespace) + resp = await self._k7d_request({"op": "lookup_sandbox", "sandbox_id": cri_sandbox_id}) + if resp.get("status") != "sandbox_found": + raise RuntimeError(f"unexpected k7d lookup_sandbox response: {resp}") + resp["sandbox_id"] = cri_sandbox_id + return resp + + # ------------------------------------------------------------------- + # Spec 18g: per-node k7 agent. When a k7d sandbox lives on a different + # node than this process, VM ops are forwarded over HTTP to the + # k7-agent DaemonSet pod on that node (which runs this same code with + # the node-local sockets). No agent there / no token → loud error. + # ------------------------------------------------------------------- + + K7_AGENT_PORT = 8000 + + @staticmethod + def _k7d_local_node() -> str: + # Inside a pod, os.uname().nodename is the POD name — deployments + # inject the real node via the downward API (K7_NODE_NAME). + return os.environ.get("K7_NODE_NAME") or os.uname().nodename + + @staticmethod + def _k7d_agent_token() -> str: + path = os.environ.get("K7_AGENT_TOKEN_FILE", "/etc/k7/agent_token") + try: + with open(path) as f: + token = f.read().strip() + except OSError as e: + raise RuntimeError( + f"k7 agent token {path} is unreadable ({e}) — cannot talk to the per-node " + "k7-agent. The install playbook provisions it on every node; re-run `k7 install`." + ) + if not token: + raise RuntimeError(f"k7 agent token {path} is empty — re-run `k7 install`") + return token + + async def _k7d_sandbox_node(self, sandbox_name: str, namespace: str) -> str: + """Node hosting the sandbox's Running pod (loud when there is none).""" + v1 = await self._get_core_v1_client() + pods = await v1.list_namespaced_pod(namespace=namespace, label_selector=f"app={sandbox_name}") + running = [ + p for p in pods.items if p.status and p.status.phase == "Running" and not p.metadata.deletion_timestamp + ] + if not running: + raise RuntimeError(f"no Running pod found for sandbox {sandbox_name} in {namespace}") + return running[0].spec.node_name + + async def _k7d_agent_base_url(self, node_name: str) -> str: + """Resolve the Ready k7-agent pod on ``node_name`` to its pod IP.""" + v1 = await self._get_core_v1_client() + pods = await v1.list_namespaced_pod( + namespace="kube-system", + label_selector="app=k7-agent", + field_selector=f"spec.nodeName={node_name}", + ) + for p in pods.items: + ready = any(c.type == "Ready" and c.status == "True" for c in (p.status.conditions or [])) + if p.status.phase == "Running" and ready and p.status.pod_ip: + return f"http://{p.status.pod_ip}:{self.K7_AGENT_PORT}" + raise RuntimeError( + f"no Ready k7-agent pod on node {node_name} — k7d VM operations for sandboxes on " + "another node need the k7-agent DaemonSet (deployed by `k7 install`); check " + "`kubectl -n kube-system get pods -l app=k7-agent -o wide`" + ) + + async def _k7d_forward_vm_op(self, node_name: str, op: str, body: dict, timeout: float) -> OperationResult: + if os.environ.get("K7_AGENT") == "1": + raise RuntimeError( + f"k7-agent on {self._k7d_local_node()} received op={op} for a sandbox on node " + f"{node_name} — refusing to re-forward (the caller resolved the wrong agent)" + ) + base = await self._k7d_agent_base_url(node_name) + token = self._k7d_agent_token() + async with httpx.AsyncClient(timeout=timeout) as http: + resp = await http.post(f"{base}/agent/v1/vm/{op}", json=body, headers={"X-K7-Agent-Token": token}) + if resp.status_code != 200: + raise RuntimeError(f"k7-agent on {node_name} failed op={op}: HTTP {resp.status_code}: {resp.text}") + data = resp.json() + return OperationResult( + success=bool(data.get("success")), + message=data.get("message") or "", + error=data.get("error") or "", + data=data.get("data"), + ) + + async def lookup_k7d_vm(self, sandbox_name: str, namespace: str = "default") -> dict: + """Resolve a k7d sandbox to its daemon VM record, forwarding to the + per-node agent when the sandbox lives on another node.""" + node = await self._k7d_sandbox_node(sandbox_name, namespace) + if node and node != self._k7d_local_node(): + base = await self._k7d_agent_base_url(node) + token = self._k7d_agent_token() + async with httpx.AsyncClient(timeout=60) as http: + resp = await http.post( + f"{base}/agent/v1/vm/lookup", + json={"name": sandbox_name, "namespace": namespace}, + headers={"X-K7-Agent-Token": token}, + ) + if resp.status_code != 200: + raise RuntimeError(f"k7-agent on {node} failed op=lookup: HTTP {resp.status_code}: {resp.text}") + return resp.json() + return await self._k7d_vm_for_sandbox(sandbox_name, namespace) + + async def nodes_storage(self) -> dict: + """Per-node storage-pool utilization, aggregated from every node's + k7-agent (spec 18g part 2). Nodes whose agent is unreachable get a + loud ``{"error": ...}`` entry — never silently omitted.""" + v1 = await self._get_core_v1_client() + nodes = await v1.list_node() + token = self._k7d_agent_token() + out: dict[str, dict] = {} + for node in nodes.items: + name = node.metadata.name + try: + base = await self._k7d_agent_base_url(name) + async with httpx.AsyncClient(timeout=30) as http: + resp = await http.get(f"{base}/agent/v1/storage", headers={"X-K7-Agent-Token": token}) + if resp.status_code != 200: + raise RuntimeError(f"k7-agent storage query failed: HTTP {resp.status_code}: {resp.text}") + out[name] = resp.json() + except Exception as e: + out[name] = {"error": str(e)} + return out + def _get_embedded_playbook(self) -> str: """Get embedded Ansible playbook content.""" try: - return ( - resources.files("k7.deploy") - .joinpath("k7-install-node.yaml") - .read_text() - ) + return resources.files("k7.deploy").joinpath("k7-install-node.yaml").read_text() except Exception: return "" - def _materialize_embedded_package_root(self) -> Path: - """Ensure the embedded k7 package exists at a stable on-disk path. - - Returns the directory path containing the `k7/` package tree that Docker builds can use - as context (without relying on ephemeral temp dirs). - """ - # Allow override via env (user-writable dir), default to FHS path - base_dir = Path(os.getenv("K7_EMBEDDED_ROOT", "/var/lib/k7/embedded")) - pkg_dir = base_dir / "k7" - try: - base_dir.mkdir(parents=True, exist_ok=True) - # Extract current installed/bundled k7 to the stable path - k7_root = resources.files("k7") - with resources.as_file(k7_root) as src_root: - src_root_path = Path(str(src_root)) - if src_root_path.exists(): - # Copy tree (refresh) to ensure it matches the bundled content - if pkg_dir.exists(): - # Keep existing but refresh files (dirs_exist_ok requires Python 3.8+) - shutil.rmtree(pkg_dir) - shutil.copytree(src_root_path, pkg_dir, dirs_exist_ok=False) - except Exception: - # Best effort; if something fails, continue with whatever exists - pass - return pkg_dir - - def _get_embedded_docker_compose(self) -> str: - """Get absolute path to embedded docker-compose.yml, or empty string if missing.""" - try: - base_dir = Path(os.getenv("K7_EMBEDDED_ROOT", "/var/lib/k7/embedded")) - pkg_dir = self._materialize_embedded_package_root() - compose_path = pkg_dir / "api" / "docker-compose.yml" - dockerfile_path = pkg_dir / "api" / "Dockerfile.api" - if compose_path.exists() and dockerfile_path.exists(): - # Ensure build context points at the embedded base dir (not repo-relative) - try: - txt = compose_path.read_text() - desired = f"context: {base_dir}" - if "context: ../.." in txt and desired not in txt: - txt = txt.replace("context: ../..", desired) - # Ensure runtime sees the embedded code under /app/k7 (read-only) - embed_mount = f" - {base_dir}/k7:/app/k7:ro" - if embed_mount not in txt: - # Insert after the /etc/k7 mount if present, else at the start of volumes - if " - /etc/k7:/etc/k7\n" in txt: - txt = txt.replace( - " - /etc/k7:/etc/k7\n", - " - /etc/k7:/etc/k7\n" + embed_mount + "\n", - ) - elif " volumes:\n" in txt: - txt = txt.replace( - " volumes:\n", - " volumes:\n" + embed_mount + "\n", - ) - compose_path.write_text(txt) - except Exception: - pass - return str(compose_path) - except Exception: - pass - return "" - - def _get_embedded_dockerfile_api(self) -> str: - """Get absolute path to embedded Dockerfile.api, or empty string if missing.""" - try: - pkg_dir = self._materialize_embedded_package_root() - dockerfile_path = pkg_dir / "api" / "Dockerfile.api" - if dockerfile_path.exists(): - return str(dockerfile_path) - except Exception: - pass - return "" - - def _get_embedded_inventory(self, hosts: List[str]) -> str: - """Generate Ansible inventory from host list.""" - inventory_lines = ["[k7_nodes]"] - for host in hosts: - inventory_lines.append( - f"{host} ansible_user=root ansible_ssh_private_key_file=~/.ssh/id_rsa" - ) - return "\n".join(inventory_lines) + def _get_embedded_inventory(self, hosts: list[str]) -> str: + """Generate Ansible inventory from host list (treats every host as a server).""" + server_lines = [f"{host} ansible_user=root ansible_ssh_private_key_file=~/.ssh/id_rsa" for host in hosts] + return "\n".join( + [ + "[k7_servers]", + *server_lines, + "", + "[k7_agents]", + "", + "[k7_cluster:children]", + "k7_servers", + "k7_agents", + ] + ) def _parse_resource_value(self, value: str) -> int: """Parse Kubernetes resource value to numeric form.""" @@ -188,17 +731,39 @@ class K7Core: except ValueError: return 0 - def _validate_limits(self, limits: Dict[str, str]) -> bool: + def _validate_limits(self, limits: dict[str, str]) -> bool: """Validate resource limits.""" if not limits: return True for key, value in limits.items(): - if key in ["cpu", "memory", "ephemeral-storage"]: - if self._parse_resource_value(value) <= 0: - return False + if key in ["cpu", "memory", "ephemeral-storage"] and self._parse_resource_value(value) <= 0: + return False return True + def _memory_limit_to_mib(self, value: str) -> int: + """Convert a Kubernetes-style memory string to MiB (Ki/Mi/Gi/Ti only).""" + if not isinstance(value, str) or not value.strip(): + raise ValueError("Memory limit must be a non-empty string") + + raw = value.strip() + match = re.match(r"^(\d+(?:\.\d+)?)(Ki|Mi|Gi|Ti)$", raw) + if not match: + raise ValueError(f"Unsupported memory format '{value}'. Use Ki, Mi, Gi, or Ti.") + + amount = float(match.group(1)) + unit = match.group(2) + if unit == "Ki": + mib = amount / 1024.0 + elif unit == "Mi": + mib = amount + elif unit == "Gi": + mib = amount * 1024.0 + else: # Ti + mib = amount * 1024.0 * 1024.0 + + return int(math.ceil(mib)) + def _count_playbook_tasks(self, playbook_content: str) -> int: """Count tasks in Ansible playbook.""" try: @@ -209,104 +774,710 @@ class K7Core: pass return 0 - def _get_kata_sandboxes(self, namespace: Optional[str] = None) -> List: + async def _collect_source_annotations( + self, + source_sandbox: str, + namespace: str, + ) -> dict[str, str]: + """Read a source sandbox's Deployment + container spec into ``k7.io/source-*`` + annotations that ``restore_sandbox`` can later use to rehydrate a + :class:`SandboxConfig` from a standalone snapshot (Spec 10f). + + Best-effort: any lookup failure returns an empty dict (the snapshot + still gets ``k7.io/kind`` and ``k7.io/source-sandbox`` — restore will + require the user to pass the missing fields explicitly). + """ + out: dict[str, str] = {} + try: + apps_v1 = await self._get_apps_v1_client() + deployment = await apps_v1.read_namespaced_deployment(name=source_sandbox, namespace=namespace) + except Exception: + return out + dep_annotations = (deployment.metadata.annotations or {}) if deployment.metadata else {} + backend = dep_annotations.get("k7.katakate.org/backend") + if backend: + out["k7.io/source-backend"] = backend + sidecar = dep_annotations.get("k7.katakate.org/sidecar") + if sidecar: + out["k7.io/source-sidecar"] = sidecar + containers = [] + try: + containers = deployment.spec.template.spec.containers or [] # type: ignore[union-attr] + except Exception: + containers = [] + # First non-sidecar container is the sandbox; sidecars are named + # ``docker-sidecar`` etc. via the SIDECAR_REGISTRY. + sandbox_container = next((c for c in containers if c.name == "sandbox"), containers[0] if containers else None) + if sandbox_container is not None: + if sandbox_container.image: + out["k7.io/source-image"] = sandbox_container.image + limits: dict[str, str] = {} + try: + raw_limits = (sandbox_container.resources.limits or {}) if sandbox_container.resources else {} + for k, v in raw_limits.items(): + if k in ("cpu", "memory", "ephemeral-storage"): + limits[k] = str(v) + except Exception: + pass + if limits: + out["k7.io/source-limits"] = json.dumps(limits, sort_keys=True) + # Root PVC size: read from the PVC the sandbox is bound to. Falls back + # to the snapshot's restoreSize at restore time when unset. + pvc_name = dep_annotations.get("k7.katakate.org/root-pvc-name") or self._root_pvc_name(source_sandbox) + try: + v1 = await self._get_core_v1_client() + pvc = await v1.read_namespaced_persistent_volume_claim(name=pvc_name, namespace=namespace) + size = None + if pvc.spec and pvc.spec.resources and pvc.spec.resources.requests: + size = pvc.spec.resources.requests.get("storage") + if size: + out["k7.io/source-root-disk-size"] = str(size) + except Exception: + pass + return out + + async def _create_volume_snapshot( + self, + pvc_name: str, + snapshot_name: str, + snapshot_class: str = "longhorn", + namespace: str = "default", + kind: str = SNAPSHOT_KIND_NAMED, + source_sandbox: str | None = None, + ) -> OperationResult: + """Create a crash-consistent VolumeSnapshot for a PVC via the K8s API. + + ``kind`` and ``source_sandbox`` are stamped onto the resulting + ``VolumeSnapshot`` via ``k7.io/`` annotations so ``list_snapshots`` / + ``gc_snapshots`` can classify it reliably (no name-pattern guessing). + When ``source_sandbox`` is set we also stamp ``k7.io/source-*`` + annotations describing the source Deployment's image / backend / + sidecar / limits / root-disk-size (Spec 10f), so a future + :meth:`restore_sandbox` can rehydrate a config without those + being passed on the command line. + """ + custom = await self._get_custom_objects_client() + annotations: dict[str, str] = { + "k7.io/kind": kind, + "k7.io/created-by": "k7-core", + } + if source_sandbox: + annotations["k7.io/source-sandbox"] = source_sandbox + # Only worth the lookup for snapshots a user might later restore. + # ``fork`` auto-snapshots get cleaned up immediately, so we skip + # the extra API calls for them. + if kind != SNAPSHOT_KIND_FORK: + annotations.update(await self._collect_source_annotations(source_sandbox, namespace)) + snapshot_body = { + "apiVersion": "snapshot.storage.k8s.io/v1", + "kind": "VolumeSnapshot", + "metadata": { + "name": snapshot_name, + "namespace": namespace, + "annotations": annotations, + }, + "spec": { + "volumeSnapshotClassName": snapshot_class, + "source": { + "persistentVolumeClaimName": pvc_name, + }, + }, + } + try: + await custom.create_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace, + plural="volumesnapshots", + body=snapshot_body, + ) + except ApiException as e: + return OperationResult( + success=False, + error=f"Snapshot creation failed: {e.reason or e.body}", + ) + return OperationResult(success=True, message=f"Snapshot {snapshot_name} created for PVC {pvc_name}") + + async def _wait_for_snapshot_ready( + self, snapshot_name: str, namespace: str = "default", timeout: int = 120 + ) -> OperationResult: + """Wait for a VolumeSnapshot to become readyToUse.""" + custom = await self._get_custom_objects_client() + start = time.time() + poll_interval = 0.5 + while time.time() - start < timeout: + try: + snap = await custom.get_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace, + plural="volumesnapshots", + name=snapshot_name, + ) + ready = (snap.get("status") or {}).get("readyToUse") + if ready is True: + elapsed = time.time() - start + print(f"⏱️ Snapshot {snapshot_name} ready in {elapsed:.2f}s", file=sys.stderr) + return OperationResult(success=True) + except ApiException: + pass + await asyncio.sleep(poll_interval) + poll_interval = min(poll_interval * 1.2, 2.0) + return OperationResult(success=False, error=f"VolumeSnapshot {snapshot_name} not ready within {timeout}s") + + def _infer_snapshot_kind(self, name: str, annotations: dict | None) -> str: + """Classify a VolumeSnapshot for ``k7 snapshot list`` / ``gc``. + + Prefer the ``k7.io/kind`` annotation we stamp at creation time. Fall back + to a name-pattern heuristic for snapshots created before Spec 10e or by + external tools — pause snapshots end in ``-paused-``, fork + snapshots end in ``-fork-``; anything else is "named". + """ + if annotations and annotations.get("k7.io/kind") in ( + SNAPSHOT_KIND_PAUSE, + SNAPSHOT_KIND_FORK, + SNAPSHOT_KIND_NAMED, + ): + return annotations["k7.io/kind"] + if re.search(r"-paused-\d+$", name): + return SNAPSHOT_KIND_PAUSE + if re.search(r"-fork-\d+$", name): + return SNAPSHOT_KIND_FORK + return SNAPSHOT_KIND_NAMED + + def _snapshot_info_from_object(self, snap: dict) -> SnapshotInfo: + """Convert a raw ``VolumeSnapshot`` dict to a typed :class:`SnapshotInfo`.""" + metadata = snap.get("metadata") or {} + spec = snap.get("spec") or {} + status = snap.get("status") or {} + annotations = metadata.get("annotations") or {} + source_pvc = ((spec.get("source") or {}).get("persistentVolumeClaimName")) or "" + # ``status.restoreSize`` is "Xi" style (e.g. "10Gi") once readyToUse is True. + size_bytes = self._parse_resource_value(str(status.get("restoreSize") or "0")) + # Convert "10Gi" → 10*1024*1024*1024 if _parse_resource_value returned Mi units; + # however our existing helper already returns Mi for "Gi"-suffixed inputs. For + # display we just keep the raw byte estimate; consumers care about presence. + ready = bool(status.get("readyToUse")) + ctime = metadata.get("creationTimestamp") or "" + age = "Unknown" + try: + if ctime: + created = datetime.fromisoformat(ctime.replace("Z", "+00:00")) + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + age = str(datetime.now(timezone.utc) - created) + except Exception: + age = "Unknown" + name = metadata.get("name") or "" + return SnapshotInfo( + name=name, + namespace=metadata.get("namespace") or "default", + source_pvc=source_pvc, + source_sandbox=annotations.get("k7.io/source-sandbox", ""), + kind=self._infer_snapshot_kind(name, annotations), + ready_to_use=ready, + creation_timestamp=ctime, + age=age, + size_bytes=size_bytes, + snapshot_class=spec.get("volumeSnapshotClassName") or "longhorn", + ) + + async def _create_pvc_from_snapshot( + self, + target_pvc_name: str, + namespace: str, + snapshot_name: str, + storage_size: str, + source_pvc_spec: Any = None, + ) -> OperationResult: + """Create a PVC cloned from a VolumeSnapshot's data (Spec 10e/10f shared helper). + + ``source_pvc_spec`` is the ``spec`` of an existing PVC to copy + ``access_modes`` / ``volume_mode`` / ``storage_class_name`` from + (fork's case, where the source sandbox still exists). When ``None`` + (restore's case), defaults to ``ReadWriteOnce`` / ``Filesystem`` / + ``longhorn`` — the only combination k7's kata-qemu-longhorn backend uses + today. + """ + v1 = await self._get_core_v1_client() + if source_pvc_spec is not None: + access_modes = source_pvc_spec.access_modes or ["ReadWriteOnce"] + volume_mode = source_pvc_spec.volume_mode or "Filesystem" + storage_class = source_pvc_spec.storage_class_name or "longhorn" + else: + access_modes = ["ReadWriteOnce"] + volume_mode = "Filesystem" + storage_class = "longhorn" + pvc_body = client.V1PersistentVolumeClaim( + metadata=client.V1ObjectMeta(name=target_pvc_name, namespace=namespace), + spec=client.V1PersistentVolumeClaimSpec( + access_modes=access_modes, + volume_mode=volume_mode, + storage_class_name=storage_class, + resources=client.V1VolumeResourceRequirements(requests={"storage": storage_size}), + data_source=client.V1TypedLocalObjectReference( + api_group="snapshot.storage.k8s.io", + kind="VolumeSnapshot", + name=snapshot_name, + ), + ), + ) + try: + await v1.create_namespaced_persistent_volume_claim(namespace=namespace, body=pvc_body) + except ApiException as e: + if e.status == 409: + return OperationResult( + success=False, + error=f"PVC {target_pvc_name} already exists in namespace {namespace}", + ) + return OperationResult( + success=False, + error=f"Failed to create cloned PVC {target_pvc_name}: {e}", + ) + return OperationResult(success=True, message=f"PVC {target_pvc_name} created from snapshot {snapshot_name}") + + async def _wait_for_pvc_bound( + self, pvc_name: str, namespace: str = "default", timeout: int = 180 + ) -> OperationResult: + """Wait for PVC to reach Bound.""" + v1 = await self._get_core_v1_client() + start = time.time() + poll_interval = 0.5 + while time.time() - start < timeout: + try: + pvc = await v1.read_namespaced_persistent_volume_claim(name=pvc_name, namespace=namespace) + phase = getattr(pvc.status, "phase", None) + if phase == "Bound": + elapsed = time.time() - start + print(f"⏱️ PVC {pvc_name} bound in {elapsed:.2f}s", file=sys.stderr) + return OperationResult(success=True) + except ApiException as e: + if e.status == 404: + pass + else: + return OperationResult(success=False, error=f"PVC check error: {e}") + await asyncio.sleep(poll_interval) + poll_interval = min(poll_interval * 1.2, 2.0) + return OperationResult(success=False, error=f"PVC {pvc_name} not Bound within {timeout}s") + + def _root_pvc_name(self, sandbox_name: str) -> str: + """Derive the canonical root PVC name for a sandbox.""" + return f"{sandbox_name}-root-lh" + + async def _wait_for_job(self, job_name: str, namespace: str = "default", timeout: int = 600) -> OperationResult: + """Wait for a Kubernetes Job to complete.""" + await self._load_k3s_config() + start = time.time() + while time.time() - start < timeout: + try: + batch = client.BatchV1Api() + job = await batch.read_namespaced_job(name=job_name, namespace=namespace) + status = job.status + if status.succeeded and status.succeeded >= 1: + return OperationResult(success=True, message=f"Job {job_name} completed") + if status.failed and status.failed > 0: + return OperationResult(success=False, error=f"Job {job_name} failed") + except ApiException as e: + if e.status == 404: + pass + else: + return OperationResult(success=False, error=f"Job wait error: {e}") + await asyncio.sleep(3) + return OperationResult(success=False, error=f"Job {job_name} did not complete within {timeout}s") + + async def _ensure_root_pvc( + self, + sandbox_name: str, + namespace: str, + root_disk_size: str, + ) -> OperationResult: + """Ensure the Longhorn filesystem PVC exists for the sandbox.""" + v1 = await self._get_core_v1_client() + pvc_name = self._root_pvc_name(sandbox_name) + try: + existing_pvc = await v1.read_namespaced_persistent_volume_claim(name=pvc_name, namespace=namespace) + existing_mode = (existing_pvc.spec.volume_mode or "").lower() if existing_pvc and existing_pvc.spec else "" + if existing_mode != "filesystem": + return OperationResult( + success=False, + error=( + f"PVC {pvc_name} has volumeMode '{existing_pvc.spec.volume_mode}', " + "but kata-qemu-longhorn requires Filesystem. Delete/recreate the PVC." + ), + ) + return OperationResult( + success=True, + message=f"PVC {pvc_name} already exists", + data={"created": False, "pvc_name": pvc_name}, + ) + except ApiException as e: + if e.status != 404: + return OperationResult(success=False, error=f"PVC lookup error: {e}") + pvc_body = client.V1PersistentVolumeClaim( + metadata=client.V1ObjectMeta(name=pvc_name, namespace=namespace), + spec=client.V1PersistentVolumeClaimSpec( + access_modes=["ReadWriteOnce"], + volume_mode="Filesystem", + storage_class_name="longhorn", + resources=client.V1VolumeResourceRequirements(requests={"storage": root_disk_size}), + ), + ) + try: + await v1.create_namespaced_persistent_volume_claim(namespace=namespace, body=pvc_body) + except ApiException as ce: + return OperationResult(success=False, error=f"Failed to create PVC {pvc_name}: {ce}") + return OperationResult( + success=True, + message=f"Created PVC {pvc_name}", + data={"created": True, "pvc_name": pvc_name}, + ) + + async def _ensure_persist_wrapper_configmap( + self, + namespace: str, + name: str, + script_text: str, + ) -> OperationResult: + """Ensure the persistence wrapper ConfigMap exists.""" + v1 = await self._get_core_v1_client() + cm_name = f"{name}-persist-wrapper" + cm_body = client.V1ConfigMap( + metadata=client.V1ObjectMeta(name=cm_name, namespace=namespace), + data={"k7-persist-bind.sh": script_text}, + ) + try: + await v1.read_namespaced_config_map(name=cm_name, namespace=namespace) + return OperationResult( + success=True, + message=f"ConfigMap {cm_name} already exists", + data={"created": False, "cm_name": cm_name}, + ) + except ApiException as e: + if e.status != 404: + return OperationResult(success=False, error=f"ConfigMap lookup error: {e}") + try: + await v1.create_namespaced_config_map(namespace=namespace, body=cm_body) + except ApiException as ce: + return OperationResult(success=False, error=f"Failed to create ConfigMap {cm_name}: {ce}") + return OperationResult( + success=True, + message=f"Created ConfigMap {cm_name}", + data={"created": True, "cm_name": cm_name}, + ) + + async def _cilium_available(self) -> bool: + """Return True when the CiliumNetworkPolicy CRD is registered on the cluster.""" + await self._load_k3s_config() + api_ext = client.ApiextensionsV1Api() + try: + await api_ext.read_custom_resource_definition(name="ciliumnetworkpolicies.cilium.io") + return True + except ApiException as e: + if e.status == 404: + return False + raise + + async def _apply_cilium_egress_policy( + self, + sandbox_name: str, + namespace: str, + cidrs: list[str], + fqdns: list[str], + ) -> OperationResult: + """Create (or replace) a CiliumNetworkPolicy that allows FQDN + optional CIDR egress. + + Cilium's FQDN matching requires DNS lookups to pass through Cilium's DNS + proxy, so we always whitelist cluster DNS (kube-dns) on UDP/TCP 53 when + FQDNs are present. + """ + custom = await self._get_custom_objects_client() + + egress_rules: list[dict] = [] + + # DNS egress + FQDN allowlist (paired so the FQDN proxy can observe lookups). + dns_rule: dict = { + "toEndpoints": [ + { + "matchLabels": { + "k8s:io.kubernetes.pod.namespace": "kube-system", + "k8s:k8s-app": "kube-dns", + } + } + ], + "toPorts": [ + { + "ports": [ + {"port": "53", "protocol": "UDP"}, + {"port": "53", "protocol": "TCP"}, + ], + "rules": {"dns": [{"matchPattern": "*"}]}, + } + ], + } + egress_rules.append(dns_rule) + + fqdn_matchers: list[dict] = [] + for fqdn in fqdns: + if "*" in fqdn: + # Cilium matchPattern semantics trap (spec 18f issue 3): `*` + # matches DNS characters within a SINGLE label — it never + # crosses dots. `*.docker.com` therefore does NOT match + # `production.cloudfront.docker.com`, which silently breaks + # CDN-backed registries (Docker Hub blobs). k7 promises + # "any subdomain", so translate a leading `*.` into Cilium's + # multi-label subdomain wildcard `**.` (one or more labels). + if fqdn.startswith("*.") and not fqdn.startswith("**"): + fqdn = "*" + fqdn + fqdn_matchers.append({"matchPattern": fqdn}) + else: + fqdn_matchers.append({"matchName": fqdn}) + if fqdn_matchers: + egress_rules.append({"toFQDNs": fqdn_matchers}) + + if cidrs: + egress_rules.append({"toCIDR": cidrs}) + + body = { + "apiVersion": "cilium.io/v2", + "kind": "CiliumNetworkPolicy", + "metadata": { + "name": f"{sandbox_name}-egress", + "namespace": namespace, + }, + "spec": { + "endpointSelector": {"matchLabels": {"katakate.org/sandbox": sandbox_name}}, + "egress": egress_rules, + }, + } + + try: + await custom.create_namespaced_custom_object( + group="cilium.io", + version="v2", + namespace=namespace, + plural="ciliumnetworkpolicies", + body=body, + ) + except ApiException as e: + if e.status == 409: + try: + await custom.replace_namespaced_custom_object( + group="cilium.io", + version="v2", + namespace=namespace, + plural="ciliumnetworkpolicies", + name=f"{sandbox_name}-egress", + body=body, + ) + except ApiException as replace_err: + return OperationResult( + success=False, + error=f"Failed to update CiliumNetworkPolicy: {replace_err}", + ) + else: + return OperationResult( + success=False, + error=f"Failed to create CiliumNetworkPolicy: {e}", + ) + return OperationResult(success=True) + + async def _get_kata_sandboxes(self, namespace: str | None = None) -> list: """Get all Kata sandboxes (deployments with kata runtime).""" - apps_v1 = self._get_apps_v1_client() + apps_v1 = await self._get_apps_v1_client() if namespace: - deployments = apps_v1.list_namespaced_deployment(namespace=namespace) + deployments = await apps_v1.list_namespaced_deployment(namespace=namespace) else: - deployments = apps_v1.list_deployment_for_all_namespaces() + deployments = await apps_v1.list_deployment_for_all_namespaces() kata_deployments = [] for deployment in deployments.items: if deployment.spec.template.spec.runtime_class_name == "kata" or ( - deployment.metadata.labels - and deployment.metadata.labels.get("runtime") == "kata" + deployment.metadata.labels and deployment.metadata.labels.get("runtime") == "kata" ): kata_deployments.append(deployment) return kata_deployments - def _delete_sandbox_resources(self, name: str, namespace: str) -> OperationResult: + async def _delete_sandbox_resources(self, name: str, namespace: str) -> OperationResult: """Delete all resources associated with a sandbox.""" - apps_v1 = self._get_apps_v1_client() - v1 = self._get_core_v1_client() - networking_v1 = self._get_networking_v1_client() + apps_v1 = await self._get_apps_v1_client() + v1 = await self._get_core_v1_client() + networking_v1 = await self._get_networking_v1_client() errors = [] try: - apps_v1.delete_namespaced_deployment(name=name, namespace=namespace) + await apps_v1.delete_namespaced_deployment(name=name, namespace=namespace) except ApiException as e: if e.status != 404: errors.append(f"deployment: {e}") try: - v1.delete_namespaced_secret(name=f"{name}-env", namespace=namespace) + await v1.delete_namespaced_secret(name=f"{name}-env", namespace=namespace) except ApiException as e: if e.status != 404: errors.append(f"secret: {e}") try: - networking_v1.delete_namespaced_network_policy( - name=f"{name}-netpol", namespace=namespace + await v1.delete_namespaced_config_map(name=f"{name}-persist-wrapper", namespace=namespace) + except ApiException as e: + if e.status != 404: + errors.append(f"configmap: {e}") + + # Spec 10e/10f contract: pause and named snapshots persist until the + # user explicitly deletes them — that's what makes ``k7 restore`` work + # after the source sandbox is gone. Only sweep ``kind=fork`` stragglers + # here (the inline-delete in ``fork_sandbox`` already cleans the happy + # path; this is a defence-in-depth safety net alongside the + # snapshot-gc CronJob). + # + # Also count surviving pause/named snapshots that target this sandbox's + # root PVC: if any exist, we must KEEP the PVC. Longhorn snapshots + # reference their source volume's data, so deleting the PVC (and the + # underlying Longhorn volume via the StorageClass's Delete reclaim + # policy) makes the snapshot un-restorable later. The PVC is annotated + # ``k7.io/orphaned-by`` so users can find and clean up by hand when + # they delete the last referencing snapshot. + pvc_name = f"{name}-root-lh" + persistent_snapshots_present = False + try: + custom = await self._get_custom_objects_client() + snap_list = await custom.list_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace, + plural="volumesnapshots", ) + for snap in snap_list.get("items", []): + metadata = snap.get("metadata") or {} + snap_name = metadata.get("name", "") + annotations = metadata.get("annotations") or {} + source = annotations.get("k7.io/source-sandbox") + snap_source_pvc = ((snap.get("spec") or {}).get("source") or {}).get("persistentVolumeClaimName") + kind = self._infer_snapshot_kind(snap_name, annotations) + if kind == SNAPSHOT_KIND_FORK: + if source and source != name: + continue + if not source and not snap_name.startswith(f"{name}-fork-"): + continue + try: + await custom.delete_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace, + plural="volumesnapshots", + name=snap_name, + ) + except Exception: + pass + else: + # pause/named: track whether any reference our PVC. + if snap_source_pvc == pvc_name or source == name: + persistent_snapshots_present = True + except Exception: + pass + + if persistent_snapshots_present: + # Mark the PVC orphaned so the user can find it; do NOT delete it + # — Longhorn would tear down the underlying volume and break + # ``k7 restore`` from any pause/named snapshot that points here. + try: + await v1.patch_namespaced_persistent_volume_claim( + name=pvc_name, + namespace=namespace, + body={ + "metadata": { + "annotations": { + "k7.io/orphaned-by": f"sandbox-deleted:{name}", + } + } + }, + ) + except ApiException: + pass + else: + try: + await v1.delete_namespaced_persistent_volume_claim(name=pvc_name, namespace=namespace) + except ApiException as e: + if e.status != 404: + errors.append(f"pvc: {e}") + try: + await v1.patch_namespaced_persistent_volume_claim( + name=pvc_name, + namespace=namespace, + body={"metadata": {"finalizers": []}}, + ) + except ApiException: + pass + + try: + await networking_v1.delete_namespaced_network_policy(name=f"{name}-netpol", namespace=namespace) except ApiException as e: if e.status != 404: errors.append(f"network policy: {e}") try: - networking_v1.delete_namespaced_network_policy( - name=f"{name}-deny-ingress", namespace=namespace - ) + await networking_v1.delete_namespaced_network_policy(name=f"{name}-deny-ingress", namespace=namespace) except ApiException as e: if e.status != 404: errors.append(f"network policy deny-ingress: {e}") + # Delete CiliumNetworkPolicy (FQDN egress) if present; ignore if CRD absent. + try: + custom = await self._get_custom_objects_client() + await custom.delete_namespaced_custom_object( + group="cilium.io", + version="v2", + namespace=namespace, + plural="ciliumnetworkpolicies", + name=f"{name}-egress", + ) + except ApiException as e: + if e.status not in (404, 405): + errors.append(f"cilium network policy: {e}") + except Exception: + pass + if errors: return OperationResult(success=False, error="; ".join(errors)) - return OperationResult( - success=True, message=f"Sandbox {name} deleted successfully" - ) + return OperationResult(success=True, message=f"Sandbox {name} deleted successfully") def install_node( self, - playbook_content: Optional[str] = None, - inventory_content: Optional[str] = None, + playbook_content: str | None = None, + inventory_content: str | None = None, verbose: bool = False, - progress_callback: Optional[Callable[[Dict], None]] = None, + progress_callback: Callable[[dict], None] | None = None, stream_output: bool = False, - extra_vars: Optional[Dict] = None, + extra_vars: dict | None = None, ) -> OperationResult: - """Install K7 on target nodes using Ansible.""" + """Install K7 on target nodes using Ansible. Stays sync (interactive/CLI-only).""" try: if not playbook_content: playbook_content = self._get_embedded_playbook() if not playbook_content: - return OperationResult( - success=False, error="No playbook content available" - ) + return OperationResult(success=False, error="No playbook content available") - with tempfile.NamedTemporaryFile( - mode="w", suffix=".yaml", delete=False - ) as playbook_file: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as playbook_file: playbook_file.write(playbook_content) playbook_path = playbook_file.name - with tempfile.NamedTemporaryFile( - mode="w", suffix=".ini", delete=False - ) as inventory_file: + with tempfile.NamedTemporaryFile(mode="w", suffix=".ini", delete=False) as inventory_file: inventory_file.write( inventory_content - or "[k7_nodes]\nlocalhost ansible_connection=local ansible_user=root" + or ( + "[k7_servers]\n" + "localhost ansible_connection=local ansible_user=root\n" + "\n[k7_agents]\n" + "\n[k7_cluster:children]\nk7_servers\nk7_agents\n" + ) ) inventory_path = inventory_file.name - # If any Ansible extra-vars are provided, write them to a temp file and pass via -e @file - extra_vars_path: Optional[str] = None + extra_vars_path: str | None = None if extra_vars: try: with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as evf: @@ -328,7 +1499,6 @@ class K7Core: if verbose: cmd.append("-v") - # Stream output to allow progress parsing process = subprocess.Popen( cmd, stdout=subprocess.PIPE, @@ -348,12 +1518,10 @@ class K7Core: combined_output.append(line) if stream_output: try: - # Print the line as-is to stdout in verbose mode sys.stdout.write(line) sys.stdout.flush() except Exception: pass - # Strip common ANSI escape sequences before matching clean_line = ansi_escape.sub("", line) match = task_pattern.search(clean_line) if match: @@ -387,11 +1555,8 @@ class K7Core: pass if process.returncode == 0: - return OperationResult( - success=True, message="Installation completed successfully" - ) + return OperationResult(success=True, message="Installation completed successfully") else: - # Include tail of output for easier debugging tail = "".join(combined_output[-50:]) if combined_output else "" return OperationResult( success=False, @@ -401,36 +1566,35 @@ class K7Core: except Exception as e: return OperationResult(success=False, error=str(e)) - def create_sandbox( + async def create_sandbox( self, config: SandboxConfig, - progress_callback: Optional[Callable[[Dict], None]] = None, + progress_callback: Callable[[dict], None] | None = None, ) -> OperationResult: """Create a new sandbox with the given configuration.""" try: - def _emit(event: Dict): + def _emit(event: dict): if progress_callback: try: progress_callback(event) except Exception: pass - if not self._validate_limits(config.limits): + if config.limits and not self._validate_limits(config.limits): return OperationResult(success=False, error="Invalid resource limits") - apps_v1 = self._get_apps_v1_client() - v1 = self._get_core_v1_client() - networking_v1 = self._get_networking_v1_client() + apps_v1 = await self._get_apps_v1_client() + v1 = await self._get_core_v1_client() + networking_v1 = await self._get_networking_v1_client() _emit({"stage": "provisioning", "status": "start"}) if config.env_file and os.path.exists(config.env_file): - with open(config.env_file, "r") as f: + with open(config.env_file) as f: env_content = f.read() - # Parse env file lines into individual key/value string_data entries - env_vars: Dict[str, str] = {} + env_vars: dict[str, str] = {} for line in env_content.splitlines(): stripped = line.strip() if not stripped or stripped.startswith("#") or "=" not in stripped: @@ -448,41 +1612,24 @@ class K7Core: ) secret = client.V1Secret( - metadata=client.V1ObjectMeta( - name=f"{config.name}-env", namespace=config.namespace - ), + metadata=client.V1ObjectMeta(name=f"{config.name}-env", namespace=config.namespace), string_data=env_vars, ) try: - v1.create_namespaced_secret(namespace=config.namespace, body=secret) + await v1.create_namespaced_secret(namespace=config.namespace, body=secret) except ApiException as e: if e.status != 409: - return OperationResult( - success=False, error=f"Failed to create secret: {e}" - ) + return OperationResult(success=False, error=f"Failed to create secret: {e}") - # Build main container command with optional before_script that runs inside the main container - # Use sandbox name to avoid race conditions when multiple sandboxes run simultaneously + # Per-sandbox marker file: /tmp is shared with anything else running + # in the same container, so a generic name races with a previous + # `before_script` run (or a parallel one in the same image's + # container scope). before_done_file = f"/tmp/k7_before_done_{config.name}" - if config.before_script: - # Ensure failures halt startup; mark completion to drive readiness - # Since -o pipefail is not supported in all shells, we use a fallback approach - script_block = config.before_script.strip() - main_cmd = ( - f"(set -o pipefail) 2>/dev/null && set -o pipefail; " - f"set -eu; " - f"rm -f {before_done_file}; " - f"{script_block}; " - f"touch {before_done_file}; exec sleep 365d" - ) - else: - main_cmd = "sleep 365d" - # Build container security context based on config - # Default capability policy: drop ALL, optionally add back caps via cap_add - drop_caps: Optional[List[str]] - add_caps: Optional[List[str]] + drop_caps: list[str] | None + add_caps: list[str] | None if getattr(config, "cap_drop", None) is None: drop_caps = ["ALL"] else: @@ -500,10 +1647,26 @@ class K7Core: ), ) + backend = self._canonicalize_backend(config.backend) or await self._detect_backend() + if backend not in ("kata-firecracker-devmapper", "kata-qemu-longhorn", "k7d"): + return OperationResult( + success=False, + error=( + f"Unsupported backend '{backend}'. " + "Use kata-firecracker-devmapper (kfd), kata-qemu-longhorn (kql), or k7d." + ), + ) + + container_command = None + container_args = None + if backend != "kata-qemu-longhorn": + container_command = ["/bin/sh", "-c", "sleep 365d"] + container = client.V1Container( name="sandbox", - image=config.image, - command=["/bin/sh", "-c", main_cmd], + image=config.image or "busybox", + command=container_command, + args=container_args, resources=client.V1ResourceRequirements( limits=config.limits if config.limits else None, requests=config.limits if config.limits else None, @@ -513,33 +1676,96 @@ class K7Core: if config.env_file: container.env_from = [ - client.V1EnvFromSource( - secret_ref=client.V1SecretEnvSource(name=f"{config.name}-env") - ) + client.V1EnvFromSource(secret_ref=client.V1SecretEnvSource(name=f"{config.name}-env")) ] - # Harden container: handled above via container_sec_ctx (configurable) - - # Readiness probe flips to Ready only after before_script completes if config.before_script: container.readiness_probe = client.V1Probe( - _exec=client.V1ExecAction( - command=["/bin/sh", "-c", f"test -f {before_done_file}"] - ), + _exec=client.V1ExecAction(command=["/bin/sh", "-c", f"test -f {before_done_file}"]), initial_delay_seconds=1, - period_seconds=2, - timeout_seconds=2, + period_seconds=5, + timeout_seconds=5, failure_threshold=30, ) else: - # Immediately Ready when no before_script + # Exec probes on kata go shim→ttrpc→agent→vsock; kubelet's + # default timeoutSeconds=1 cancels them whenever the guest is + # busy, and a flood of cancelled execs corrupts the shim↔agent + # ttrpc connection until the shim declares "Dead agent" and + # kills a healthy VM (spec 18g, the kql-r3 dind IO wedge). + # Generous timeout + long period keep probe pressure low. container.readiness_probe = client.V1Probe( _exec=client.V1ExecAction(command=["/bin/sh", "-c", "true"]), initial_delay_seconds=0, - period_seconds=2, + period_seconds=10, + timeout_seconds=5, + failure_threshold=6, ) - runtime_class = getattr(config, "runtime_class_name", None) or "kata" + base_annotations = { + "k7.katakate.org/backend": backend, + } + + memory_annotation_value: str | None = None + # The Kata hypervisor memory annotation is meaningless for k7d — + # the k7d shim sizes the VM straight from the pod's CRI + # cpu/memory limits (spec 10c/17a). + if config.limits and "memory" in config.limits and backend != "k7d": + try: + memory_mib = self._memory_limit_to_mib(config.limits["memory"]) + except ValueError as e: + return OperationResult(success=False, error=str(e)) + base_annotations["io.katacontainers.config.hypervisor.default_memory"] = str(memory_mib) + memory_annotation_value = str(memory_mib) + + if backend == "kata-qemu-longhorn": + default_runtime_class = "kata-qemu" + elif backend == "k7d": + default_runtime_class = "k7" + else: + default_runtime_class = "kata" + + runtime_class = getattr(config, "runtime_class_name", None) or default_runtime_class + + root_pvc_name = None + pvc_created = False + wrapper_created = False + wrapper_cm_name = None + if backend == "kata-qemu-longhorn" and runtime_class == "kata-qemu": + if not config.image: + return OperationResult( + success=False, + error="Image is required for kata-qemu-longhorn backend", + ) + root_pvc_name = self._root_pvc_name(config.name) + base_annotations["k7.katakate.org/root-pvc-name"] = root_pvc_name + ensure_res = await self._ensure_root_pvc( + sandbox_name=config.name, + namespace=config.namespace, + root_disk_size=config.root_disk_size or "10Gi", + ) + if not ensure_res.success: + return ensure_res + pvc_created = bool(ensure_res.data and ensure_res.data.get("created")) + wrapper_script = self._load_persist_bind_script() + wrapper_res = await self._ensure_persist_wrapper_configmap( + namespace=config.namespace, + name=config.name, + script_text=wrapper_script, + ) + if not wrapper_res.success: + return wrapper_res + wrapper_cm_name = wrapper_res.data["cm_name"] + wrapper_created = bool(wrapper_res.data and wrapper_res.data.get("created")) + else: + # kata-firecracker-devmapper and k7d: no PVC, no ConfigMap + # wrapper, no persist-bind script (k7d persistence rides the + # VM's own state; spec 9a M11). + if not config.image: + return OperationResult( + success=False, + error=f"Image is required for {backend} backend", + ) pod_sec_ctx = None if getattr(config, "pod_non_root", False): @@ -555,39 +1781,243 @@ class K7Core: runtime_class_name=runtime_class, restart_policy="Always", security_context=pod_sec_ctx, + node_selector={f"k7.katakate.org/backend-{backend}": "true"}, + node_name=getattr(config, "node_name", None) or None, + ) + + if root_pvc_name and wrapper_cm_name: + pod_spec.volumes = (pod_spec.volumes or []) + [ + client.V1Volume( + name="state-disk", + persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(claim_name=root_pvc_name), + ), + client.V1Volume( + name="persist-wrapper", + config_map=client.V1ConfigMapVolumeSource( + name=wrapper_cm_name, + default_mode=0o755, + ), + ), + ] + init_cmd = "set -eu; mkdir -p /mnt/state/pods" + if config.sidecar and config.sidecar in SIDECAR_REGISTRY: + init_cmd += f"; mkdir -p /mnt/state/{SIDECAR_REGISTRY[config.sidecar].pvc_subdir}" + init_cmd += "; echo ok > /mnt/state/.ready" + pod_spec.init_containers = [ + client.V1Container( + name="init-state", + image="busybox:1.36", + command=["/bin/sh", "-c", init_cmd], + volume_mounts=[client.V1VolumeMount(name="state-disk", mount_path="/mnt/state")], + ) + ] + container.volume_mounts = (container.volume_mounts or []) + [ + client.V1VolumeMount(name="state-disk", mount_path="/mnt/state"), + client.V1VolumeMount(name="persist-wrapper", mount_path="/opt/k7/bin"), + ] + container.env = (container.env or []) + [ + client.V1EnvVar(name="K7_PERSIST_SLOT", value="main"), + ] + container.command = ["/opt/k7/bin/k7-persist-bind.sh"] + image_entrypoint, image_cmd = await self._get_image_entrypoint_cmd(config.image) + orig_argv = self._compute_image_argv( + image_entrypoint=image_entrypoint, + image_cmd=image_cmd, + override_entrypoint=config.entrypoint, + override_cmd=config.cmd, + ) + container.args = orig_argv + if container.security_context is None: + container.security_context = client.V1SecurityContext() + container.security_context.privileged = True + container.security_context.allow_privilege_escalation = True + + # --- Generic sidecar injection (driven entirely by SIDECAR_REGISTRY) --- + if config.sidecar: + spec = SIDECAR_REGISTRY[config.sidecar] + + socket_vol = client.V1Volume( + name="sidecar-socket", + empty_dir=client.V1EmptyDirVolumeSource(), + ) + pod_spec.volumes = (pod_spec.volumes or []) + [socket_vol] + + if backend == "kata-qemu-longhorn": + data_mount = client.V1VolumeMount( + name="state-disk", + mount_path=spec.data_path, + sub_path=spec.pvc_subdir, + ) + else: + data_vol = client.V1Volume( + name="sidecar-data", + empty_dir=client.V1EmptyDirVolumeSource(), + ) + pod_spec.volumes = (pod_spec.volumes or []) + [data_vol] + data_mount = client.V1VolumeMount( + name="sidecar-data", + mount_path=spec.data_path, + ) + + sidecar_container = client.V1Container( + name="sidecar", + image=spec.image, + args=spec.args or None, + security_context=client.V1SecurityContext(privileged=spec.privileged), + env=[client.V1EnvVar(name=k, value=v) for k, v in spec.env.items()], + volume_mounts=[ + client.V1VolumeMount(name="sidecar-socket", mount_path=spec.socket_mount), + data_mount, + ], + # `docker info` legitimately takes >1s while dockerd is + # busy (vfs copies a whole image rootfs per `docker run`, + # amplified by Longhorn r≥2). With kubelet's default 1s + # timeout this probe was cancelled dozens of times per + # workload, poisoning the kata shim↔agent ttrpc channel + # ("received message on inactive stream") until the shim + # killed the healthy VM (spec 18g wedge root cause). + readiness_probe=client.V1Probe( + _exec=client.V1ExecAction(command=spec.readiness_cmd), + initial_delay_seconds=3, + period_seconds=15, + timeout_seconds=12, + failure_threshold=4, + ), + ) + pod_spec.containers.append(sidecar_container) + + container.volume_mounts = (container.volume_mounts or []) + [ + client.V1VolumeMount( + name="sidecar-socket", + mount_path=spec.socket_mount, + read_only=False, + ) + ] + + base_annotations["k7.katakate.org/sidecar"] = config.sidecar + + pod_metadata = client.V1ObjectMeta( + labels={ + "app": config.name, + "katakate.org/sandbox": config.name, + }, + annotations=base_annotations, ) deployment = client.V1Deployment( metadata=client.V1ObjectMeta( name=config.name, namespace=config.namespace, - labels={"app": config.name, "runtime": "kata", "katakate.org/sandbox": config.name}, + labels={ + "app": config.name, + "runtime": "kata", + "katakate.org/sandbox": config.name, + }, + annotations=base_annotations, ), spec=client.V1DeploymentSpec( replicas=1, selector=client.V1LabelSelector(match_labels={"app": config.name}), template=client.V1PodTemplateSpec( - metadata=client.V1ObjectMeta(labels={"app": config.name, "katakate.org/sandbox": config.name}), + metadata=pod_metadata, spec=pod_spec, ), ), ) + deployment_created = False + create_error: ApiException | None = None + deployment_create_start = time.time() try: - apps_v1.create_namespaced_deployment( - namespace=config.namespace, body=deployment + await apps_v1.create_namespaced_deployment(namespace=config.namespace, body=deployment) + deployment_created = True + print( + f"⏱️ Deployment created in {time.time() - deployment_create_start:.2f}s", + file=sys.stderr, ) except ApiException as e: - if e.status == 409: - return OperationResult( - success=False, error=f"Sandbox {config.name} already exists" - ) + create_error = e + finally: + if not deployment_created: + if wrapper_created and wrapper_cm_name: + try: + await v1.delete_namespaced_config_map(name=wrapper_cm_name, namespace=config.namespace) + except Exception: + pass + if pvc_created and root_pvc_name: + try: + await v1.delete_namespaced_persistent_volume_claim( + name=root_pvc_name, namespace=config.namespace + ) + except Exception: + pass + if not deployment_created: + if create_error and create_error.status == 409: + return OperationResult(success=False, error=f"Sandbox {config.name} already exists") return OperationResult( - success=False, error=f"Failed to create deployment: {e}" + success=False, + error=f"Failed to create deployment: {create_error}", ) + + sched_check = await self._check_scheduling( + sandbox_name=config.name, + namespace=config.namespace, + backend=backend, + ) + if not sched_check.success: + # Scheduling failed (no node matches the backend selector). + # Roll back the deployment + dependent resources before returning. + try: + await apps_v1.delete_namespaced_deployment(name=config.name, namespace=config.namespace) + except Exception: + pass + if wrapper_created and wrapper_cm_name: + try: + await v1.delete_namespaced_config_map(name=wrapper_cm_name, namespace=config.namespace) + except Exception: + pass + if pvc_created and root_pvc_name: + try: + await v1.delete_namespaced_persistent_volume_claim( + name=root_pvc_name, namespace=config.namespace + ) + except Exception: + pass + _emit({"stage": "error", "error": sched_check.error}) + return sched_check + + async def _fail_and_rollback(error: str) -> OperationResult: + """A failed create must not leave a half-provisioned sandbox + behind (deployment/PVC/netpols). delete_sandbox already + tears down every resource and tolerates missing ones.""" + try: + await self.delete_sandbox(config.name, namespace=config.namespace) + except Exception as cleanup_err: + error = f"{error} (rollback also failed: {cleanup_err})" + _emit({"stage": "error", "error": error}) + return OperationResult(success=False, error=error) + + if memory_annotation_value: + try: + await apps_v1.patch_namespaced_deployment( + name=config.name, + namespace=config.namespace, + body={ + "spec": { + "template": { + "metadata": { + "annotations": { + "io.katacontainers.config.hypervisor.default_memory": memory_annotation_value + } + } + } + } + }, + ) + except ApiException as e: + return await _fail_and_rollback(f"Failed to patch pod template annotations: {e}") _emit({"stage": "provisioning", "status": "done"}) - # Always emit before_script lifecycle if a script is present if config.before_script: _emit( { @@ -597,73 +2027,98 @@ class K7Core: } ) try: - timeout_seconds = 300 - start_time = time.time() - while time.time() - start_time < timeout_seconds: - pods = v1.list_namespaced_pod( - namespace=config.namespace, - label_selector=f"app={config.name}", + pod_wait_start = time.time() + pod_name = await self._wait_for_pod_container_started( + sandbox_name=config.name, + namespace=config.namespace, + timeout_seconds=300, + wait_all_containers=bool(config.sidecar), + ) + if not pod_name: + return await _fail_and_rollback("Timed out waiting for sandbox container to start") + print( + f"⏱️ Pod container started in {time.time() - pod_wait_start:.2f}s", + file=sys.stderr, + ) + script = config.before_script.strip() + # `pipefail` isn't a POSIX sh builtin (the `sleep 365d` pid 1 + # may be busybox sh, dash, ...). Try to enable it but fall + # back gracefully so the script still runs under stricter + # shells; `set -eu` is universal. + exec_cmd = [ + "/bin/sh", + "-c", + ( + "(set -o pipefail) 2>/dev/null && set -o pipefail; " + f"set -eu; {script}; touch {before_done_file}" + ), + ] + async with WsApiClient() as ws_api: + v1_ws = client.CoreV1Api(api_client=ws_api) + await v1_ws.connect_get_namespaced_pod_exec( + pod_name, + config.namespace, + container="sandbox", + command=exec_cmd, # ty: ignore[invalid-argument-type] + stderr=True, + stdin=False, + stdout=True, + tty=False, ) - if pods.items: - pod = pods.items[0] - conds = pod.status.conditions or [] - if any( - getattr(c, "type", None) == "Ready" - and getattr(c, "status", None) == "True" - for c in conds - ): - break - time.sleep(2) - except Exception: - pass + except Exception as e: + return await _fail_and_rollback(f"before_script failed: {e}") _emit({"stage": "before_script", "status": "done"}) else: _emit({"stage": "before_script", "status": "skipped"}) - # Apply egress policy after before_script completes if whitelist specified if config.egress_whitelist is not None: _emit({"stage": "network_lockdown", "status": "applying"}) - egress_rules = [] - for cidr in config.egress_whitelist: - egress_rules.append( + cidrs, fqdns = _classify_egress_entries(config.egress_whitelist) + + if fqdns: + cilium_available = await self._cilium_available() + if not cilium_available: + return await _fail_and_rollback( + "Domain-based egress " + f"({', '.join(fqdns)}) requires the Cilium CNI. " + "Re-install the cluster with: k7 install --cni cilium" + ) + cnp_result = await self._apply_cilium_egress_policy( + sandbox_name=config.name, + namespace=config.namespace, + cidrs=cidrs, + fqdns=fqdns, + ) + if not cnp_result.success: + return await _fail_and_rollback(cnp_result.error or "Cilium egress policy failed") + else: + egress_rules = [ client.V1NetworkPolicyEgressRule( - to=[ - client.V1NetworkPolicyPeer( - ip_block=client.V1IPBlock(cidr=cidr) - ) - ] + to=[client.V1NetworkPolicyPeer(ip_block=client.V1IPBlock(cidr=cidr))] ) - ) + for cidr in cidrs + ] - - - network_policy = client.V1NetworkPolicy( - metadata=client.V1ObjectMeta( - name=f"{config.name}-netpol", namespace=config.namespace - ), - spec=client.V1NetworkPolicySpec( - pod_selector=client.V1LabelSelector( - match_labels={"katakate.org/sandbox": config.name} + network_policy = client.V1NetworkPolicy( + metadata=client.V1ObjectMeta(name=f"{config.name}-netpol", namespace=config.namespace), + spec=client.V1NetworkPolicySpec( + pod_selector=client.V1LabelSelector(match_labels={"katakate.org/sandbox": config.name}), + policy_types=["Egress"], + egress=egress_rules, ), - policy_types=["Egress"], - egress=egress_rules, - ), - ) - - try: - networking_v1.create_namespaced_network_policy( - namespace=config.namespace, body=network_policy ) - except ApiException as e: - if e.status != 409: - return OperationResult( - success=False, error=f"Failed to create network policy: {e}" + + try: + await networking_v1.create_namespaced_network_policy( + namespace=config.namespace, body=network_policy ) + except ApiException as e: + if e.status != 409: + return await _fail_and_rollback(f"Failed to create network policy: {e}") _emit({"stage": "network_lockdown", "status": "done"}) else: _emit({"stage": "network_lockdown", "status": "skipped"}) - # Hardcoded deny-all ingress to block inter-VM communication try: ingress_np = client.V1NetworkPolicy( metadata=client.V1ObjectMeta( @@ -671,16 +2126,12 @@ class K7Core: namespace=config.namespace, ), spec=client.V1NetworkPolicySpec( - pod_selector=client.V1LabelSelector( - match_labels={"katakate.org/sandbox": config.name} - ), + pod_selector=client.V1LabelSelector(match_labels={"katakate.org/sandbox": config.name}), policy_types=["Ingress"], ingress=[], ), ) - networking_v1.create_namespaced_network_policy( - namespace=config.namespace, body=ingress_np - ) + await networking_v1.create_namespaced_network_policy(namespace=config.namespace, body=ingress_np) except ApiException as e: status = getattr(e, "status", None) if status == 409: @@ -694,7 +2145,6 @@ class K7Core: ) except Exception: pass - # idempotent success else: try: _emit( @@ -706,10 +2156,7 @@ class K7Core: ) except Exception: pass - return OperationResult( - success=False, - error=f"Failed to create ingress deny policy: {e}", - ) + return await _fail_and_rollback(f"Failed to create ingress deny policy: {e}") _emit( { @@ -718,9 +2165,7 @@ class K7Core: "message": f"Sandbox {config.name} created successfully", } ) - return OperationResult( - success=True, message=f"Sandbox {config.name} created successfully" - ) + return OperationResult(success=True, message=f"Sandbox {config.name} created successfully") except Exception as e: try: @@ -729,58 +2174,59 @@ class K7Core: pass return OperationResult(success=False, error=str(e)) - def list_sandboxes(self, namespace: Optional[str] = None) -> List[SandboxInfo]: + async def list_sandboxes(self, namespace: str | None = None) -> list[SandboxInfo]: """List all sandboxes.""" try: - v1 = self._get_core_v1_client() - sandboxes = self._get_kata_sandboxes(namespace) + v1 = await self._get_core_v1_client() + sandboxes = await self._get_kata_sandboxes(namespace) sandbox_list = [] for deployment in sandboxes: name = deployment.metadata.name ns = deployment.metadata.namespace + annotations = deployment.metadata.annotations or {} + backend = annotations.get("k7.katakate.org/backend", "unknown") + try: - pods = v1.list_namespaced_pod( - namespace=ns, label_selector=f"app={name}" - ) - if pods.items: - pod = pods.items[0] + pods = await v1.list_namespaced_pod(namespace=ns, label_selector=f"app={name}") + # Ignore Terminating pods: right after a resume the old + # replica can linger with a stale Ready=True condition, + # making clients (and tests) exec into a dying VM. + live = [p for p in pods.items if not p.metadata.deletion_timestamp] + if live: + pod = live[0] status = pod.status.phase or "Unknown" ready = ( "True" if pod.status.conditions - and any( - c.type == "Ready" and c.status == "True" - for c in pod.status.conditions - ) + and any(c.type == "Ready" and c.status == "True" for c in pod.status.conditions) else "False" ) - restarts = sum( - cs.restart_count - for cs in pod.status.container_statuses or [] - ) - age = str( - datetime.now() - - pod.metadata.creation_timestamp.replace(tzinfo=None) - ) - image = ( - pod.spec.containers[0].image - if pod.spec.containers - else "Unknown" - ) + restarts = sum(cs.restart_count for cs in pod.status.container_statuses or []) + created_at = pod.metadata.creation_timestamp + if created_at is None: + age = "Unknown" + else: + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + age = str(datetime.now(timezone.utc) - created_at) + image = pod.spec.containers[0].image if pod.spec.containers else "Unknown" + node = pod.spec.node_name or "" else: status = "No Pods" ready = "False" restarts = 0 age = "Unknown" image = "Unknown" + node = "" except Exception: status = "Error" ready = "False" restarts = 0 age = "Unknown" image = "Unknown" + node = "" sandbox_list.append( SandboxInfo( @@ -791,6 +2237,8 @@ class K7Core: restarts=restarts, age=age, image=image, + backend=backend, + node=node, ) ) @@ -799,20 +2247,843 @@ class K7Core: except Exception: return [] - def delete_sandbox(self, name: str, namespace: str = "default") -> OperationResult: - """Delete a sandbox.""" - return self._delete_sandbox_resources(name, namespace) + async def pause_sandbox( + self, + name: str, + namespace: str = "default", + pvc_name: str | None = None, + snapshot_name: str | None = None, + snapshot_class: str = "longhorn", + ) -> OperationResult: + """Scale down the sandbox and optionally take a crash-consistent snapshot of its PVC. - def delete_all_sandboxes(self, namespace: str = "default") -> OperationResult: + Snapshot is taken whenever ``snapshot_name`` is set. ``pvc_name`` defaults + to ``_root_pvc_name(name)`` (kata-qemu-longhorn convention) when not supplied. + + k7d backend: the sandbox's microVM is paused **in place** through the + k7d daemon (vCPUs stopped, guest memory and vsock identity retained) — + the pod stays scheduled, and ``resume_sandbox`` restarts the vCPUs on + the exact same state. Longhorn-style ``snapshot_name`` does not apply. + """ + try: + backend = await self._detect_backend(name, namespace) + if backend == "k7d": + if snapshot_name: + return OperationResult( + success=False, + error=f"--snapshot is not supported when pausing a k7d sandbox: {K7D_SNAPSHOT_UNSUPPORTED}", + ) + node = await self._k7d_sandbox_node(name, namespace) + if node and node != self._k7d_local_node(): + return await self._k7d_forward_vm_op( + node, "pause", {"name": name, "namespace": namespace}, timeout=120 + ) + vm = await self._k7d_vm_for_sandbox(name, namespace) + await self._k7d_request({"op": "pause_vm", "vm_id": vm["vm_id"]}) + await self._update_deployment_annotation(name, namespace, K7D_ANN_PAUSED, "true") + return OperationResult( + success=True, + message=( + f"Sandbox {name} paused (k7d VM {vm['vm_id']} frozen in place; memory retained, vCPUs stopped)." + ), + ) + + apps_v1 = await self._get_apps_v1_client() + scale_body = {"spec": {"replicas": 0}} + await apps_v1.patch_namespaced_deployment_scale(name=name, namespace=namespace, body=scale_body) + snap_result = None + if snapshot_name: + effective_pvc = pvc_name or self._root_pvc_name(name) + snap_result = await self._create_volume_snapshot( + pvc_name=effective_pvc, + snapshot_name=snapshot_name, + snapshot_class=snapshot_class, + namespace=namespace, + kind=SNAPSHOT_KIND_PAUSE, + source_sandbox=name, + ) + if not snap_result.success: + return snap_result + msg = f"Sandbox {name} paused (replicas=0)." + if snap_result and snap_result.success: + msg += f" Snapshot {snapshot_name} created." + return OperationResult(success=True, message=msg) + except ApiException as e: + return OperationResult(success=False, error=f"Kubernetes error while pausing: {e}") + except Exception as e: + return OperationResult(success=False, error=str(e)) + + async def resume_sandbox(self, name: str, namespace: str = "default") -> OperationResult: + """Scale sandbox back to 1 replica (k7d: unfreeze the VM in place).""" + try: + backend = await self._detect_backend(name, namespace) + if backend == "k7d": + node = await self._k7d_sandbox_node(name, namespace) + if node and node != self._k7d_local_node(): + return await self._k7d_forward_vm_op( + node, "resume", {"name": name, "namespace": namespace}, timeout=120 + ) + vm = await self._k7d_vm_for_sandbox(name, namespace) + await self._k7d_request({"op": "resume_vm", "vm_id": vm["vm_id"]}) + await self._update_deployment_annotation(name, namespace, K7D_ANN_PAUSED, "false") + return OperationResult( + success=True, + message=f"Sandbox {name} resumed (k7d VM {vm['vm_id']} vCPUs restarted).", + ) + + apps_v1 = await self._get_apps_v1_client() + scale_body = {"spec": {"replicas": 1}} + await apps_v1.patch_namespaced_deployment_scale(name=name, namespace=namespace, body=scale_body) + return OperationResult(success=True, message=f"Sandbox {name} resumed (replicas=1).") + except ApiException as e: + return OperationResult(success=False, error=f"Kubernetes error while resuming: {e}") + except Exception as e: + return OperationResult(success=False, error=str(e)) + + def shell_into_sandbox( + self, + sandbox_name: str, + namespace: str = "default", + ) -> OperationResult: + """Shell into a sandbox pod via kubectl exec. Stays sync (interactive CLI-only).""" + v1 = asyncio.run(self._get_core_v1_client()) + pods = asyncio.run(v1.list_namespaced_pod(namespace=namespace, label_selector=f"app={sandbox_name}")) + if not pods.items: + return OperationResult(success=False, error="Pod not found") + + pod_name = pods.items[0].metadata.name + kubectl_cmd = ["k3s", "kubectl"] if shutil.which("k3s") else ["kubectl"] + exec_command = kubectl_cmd + ["exec", "-it", pod_name, "-n", namespace, "-c", "sandbox", "--", "/bin/sh"] + + try: + result = subprocess.run(exec_command, check=False) + except Exception as e: + return OperationResult(success=False, error=f"Failed to exec into pod: {e}") + + if result.returncode != 0: + return OperationResult( + success=False, + error=f"Shell exited with code {result.returncode}", + ) + return OperationResult(success=True) + + async def fork_sandbox( + self, + source_name: str, + new_name: str, + namespace: str = "default", + snapshot_name: str | None = None, + ) -> OperationResult: + """Clone a sandbox Deployment definition to a new name, cloning its root PVC. + + k7d backend: instead of a Longhorn disk clone, the new pod carries the + ``k7d.katakate.org/fork-source-*`` annotations, so its containerd shim + boots it as a **warm fork** (CoW disk+memory) of the source's live VM + through the k7d daemon. + """ + fork_start = time.time() + try: + backend = await self._detect_backend(source_name, namespace) + if backend == "k7d": + return await self._fork_sandbox_k7d( + source_name=source_name, + new_name=new_name, + namespace=namespace, + snapshot_name=snapshot_name, + fork_start=fork_start, + ) + + apps_v1 = await self._get_apps_v1_client() + v1 = await self._get_core_v1_client() + src = await apps_v1.read_namespaced_deployment(name=source_name, namespace=namespace) + + source_pvc_name = self._root_pvc_name(source_name) + target_pvc_name = self._root_pvc_name(new_name) + + try: + source_pvc = await v1.read_namespaced_persistent_volume_claim(name=source_pvc_name, namespace=namespace) + except ApiException as e: + if e.status == 404: + return OperationResult( + success=False, + error=f"Source root PVC {source_pvc_name} not found; cannot fork storage", + ) + return OperationResult(success=False, error=f"PVC lookup error: {e}") + + # Spec 10e: classify the snapshot so list/gc can find it. A user-supplied + # ``snapshot_name`` is treated as a "named" snapshot (persists across the + # fork). The auto-named case uses the canonical ``-fork-`` + # form and is marked ``kind=fork`` so it gets cleaned up below (and by + # the GC backstop if the inline delete fails for any reason). + if snapshot_name: + snap_name = snapshot_name + snap_kind = SNAPSHOT_KIND_NAMED + auto_temp_snapshot = False + else: + snap_name = f"{source_name}-fork-{int(time.time())}" + snap_kind = SNAPSHOT_KIND_FORK + auto_temp_snapshot = True + # A Longhorn VolumeSnapshot is block-level and only + # crash-consistent: guest writes still sitting in the VM's page + # cache would be missing from the fork's clone. Flush them first + # when the source is running (fail loudly if the flush fails); + # a scaled-to-0 (paused) source has no writers and needs no flush. + if (src.status.ready_replicas or 0) > 0: + sync_res = await self.exec_command(source_name, "sync", namespace=namespace) + if sync_res.exit_code != 0: + return OperationResult( + success=False, + error=f"pre-fork sync in source sandbox failed: {sync_res.stderr}", + ) + snap_result = await self._create_volume_snapshot( + pvc_name=source_pvc_name, + snapshot_name=snap_name, + snapshot_class="longhorn", + namespace=namespace, + kind=snap_kind, + source_sandbox=source_name, + ) + if not snap_result.success: + return snap_result + + ready = await self._wait_for_snapshot_ready(snap_name, namespace=namespace) + if not ready.success: + return ready + + # Derive the cloned PVC's size from the source PVC's request. + source_size = "10Gi" + try: + source_size = str(source_pvc.spec.resources.requests.get("storage") or source_size) + except Exception: + pass + pvc_clone = await self._create_pvc_from_snapshot( + target_pvc_name=target_pvc_name, + namespace=namespace, + snapshot_name=snap_name, + storage_size=source_size, + source_pvc_spec=source_pvc.spec, + ) + if not pvc_clone.success: + return pvc_clone + + new_dep = copy.deepcopy(src) + new_dep.metadata.name = new_name + new_dep.metadata.resource_version = None + new_dep.metadata.uid = None + if hasattr(new_dep.metadata, "managed_fields"): + new_dep.metadata.managed_fields = None + if hasattr(new_dep.metadata, "creation_timestamp"): + new_dep.metadata.creation_timestamp = None + if hasattr(new_dep.metadata, "generation"): + new_dep.metadata.generation = None + if new_dep.metadata.labels: + new_dep.metadata.labels["app"] = new_name + new_dep.metadata.labels["katakate.org/sandbox"] = new_name + if new_dep.spec and new_dep.spec.selector and new_dep.spec.selector.match_labels: + new_dep.spec.selector.match_labels["app"] = new_name + if new_dep.spec and new_dep.spec.template and new_dep.spec.template.metadata: + if new_dep.spec.template.metadata.labels is None: + new_dep.spec.template.metadata.labels = {} + if hasattr(new_dep.spec.template.metadata, "managed_fields"): + new_dep.spec.template.metadata.managed_fields = None + if hasattr(new_dep.spec.template.metadata, "creation_timestamp"): + new_dep.spec.template.metadata.creation_timestamp = None + new_dep.spec.template.metadata.labels["app"] = new_name + new_dep.spec.template.metadata.labels["katakate.org/sandbox"] = new_name + new_dep.spec.replicas = 1 + + if new_dep.spec and new_dep.spec.template and new_dep.spec.template.spec: + vols = new_dep.spec.template.spec.volumes or [] + for v in vols: + pvc_ref = getattr(v, "persistent_volume_claim", None) + if pvc_ref and pvc_ref.claim_name == source_pvc_name: + pvc_ref.claim_name = target_pvc_name + + try: + await apps_v1.create_namespaced_deployment(namespace=namespace, body=new_dep) + except ApiException as e: + if e.status == 409: + return OperationResult(success=False, error=f"Sandbox {new_name} already exists") + raise + + # With WaitForFirstConsumer the cloned PVC stays Pending until the pod is + # scheduled, so wait_for_pvc_bound MUST run after the Deployment exists. + bound = await self._wait_for_pvc_bound(target_pvc_name, namespace=namespace) + if not bound.success: + try: + await apps_v1.delete_namespaced_deployment(name=new_name, namespace=namespace) + except Exception: + pass + try: + await v1.delete_namespaced_persistent_volume_claim(name=target_pvc_name, namespace=namespace) + except Exception: + pass + return bound + + # Spec 10e Option A: an auto-named fork snapshot has done its job once + # the cloned PVC is Bound (Longhorn has finished provisioning the new + # volume from the snapshot's data). Delete it inline so namespaces + # don't accumulate stale ``-fork-`` snapshots. Best-effort: failures + # are swallowed because the fork itself succeeded; the GC backstop + # (Option C, snapshot-gc CronJob) will eventually catch any leak. + if auto_temp_snapshot: + try: + await self.delete_snapshot(snap_name, namespace=namespace) + except Exception as e: + print( + f"⚠️ Failed to inline-delete fork snapshot {snap_name}: {e}", + file=sys.stderr, + ) + + fork_elapsed = time.time() - fork_start + return OperationResult( + success=True, + message=( + f"Forked sandbox {source_name} -> {new_name} with cloned disk " + f"(PVC {target_pvc_name} from snapshot {snap_name}) in {fork_elapsed:.2f}s" + ), + ) + except ApiException as e: + return OperationResult(success=False, error=f"Kubernetes error while forking: {e}") + except Exception as e: + return OperationResult(success=False, error=str(e)) + + async def _fork_sandbox_k7d( + self, + source_name: str, + new_name: str, + namespace: str, + snapshot_name: str | None, + fork_start: float, + ) -> OperationResult: + """k7d warm fork: create a copy of the source Deployment whose pod + carries the ``k7d.katakate.org/fork-source-*`` annotations. The k7d + shim resolves the source VM through the daemon and boots the new pod + as a CoW disk+memory fork of it (k7d spec 17d). + + Limitations (fail loud, documented): single-workload sandboxes only + (no sidecar), and the fork pod re-forks from the live source if its + pod is ever restarted. Whole-cluster fork (forking a multi-VM inner + k8s cluster as one unit) is a k7d-native feature, not a k7 verb — + drive it through the k7d daemon API (https://github.com/katakate/k7d). + """ + if snapshot_name: + return OperationResult( + success=False, + error=f"--snapshot is not supported when forking a k7d sandbox: {K7D_SNAPSHOT_UNSUPPORTED}", + ) + apps_v1 = await self._get_apps_v1_client() + src = await apps_v1.read_namespaced_deployment(name=source_name, namespace=namespace) + + sidecar_ann = (src.metadata.annotations or {}).get("k7.katakate.org/sidecar") + if sidecar_ann: + return OperationResult( + success=False, + error=( + f"forking a k7d sandbox with a '{sidecar_ann}' sidecar is not supported: " + "the fork adopts the single workload container running inside the forked VM " + "(k7d spec 17d). Fork the sandbox without a sidecar, or use the kql backend " + "for sidecar forks." + ), + ) + + # Spec 18g: the fork must run on the source VM's node (the daemon + # socket is node-local; the fork pod is pinned there too). Forward + # the WHOLE fork to that node's k7-agent when the source is remote. + node = await self._k7d_sandbox_node(source_name, namespace) + if node and node != self._k7d_local_node(): + return await self._k7d_forward_vm_op( + node, + "fork", + {"name": source_name, "new_name": new_name, "namespace": namespace, "snapshot": snapshot_name}, + timeout=600, + ) + + # Resolve the live source VM up-front so a dead/unknown source fails + # here with a clear message instead of leaving a CrashLooping fork pod. + vm = await self._k7d_vm_for_sandbox(source_name, namespace) + fork_annotations = { + K7D_ANN_FORK_SOURCE_CLUSTER: vm.get("cluster_id") or vm["sandbox_id"], + K7D_ANN_FORK_SOURCE_VM: vm["sandbox_id"], + } + + new_dep = copy.deepcopy(src) + new_dep.metadata.name = new_name + new_dep.metadata.resource_version = None + new_dep.metadata.uid = None + for attr in ("managed_fields", "creation_timestamp", "generation"): + if hasattr(new_dep.metadata, attr): + setattr(new_dep.metadata, attr, None) + if new_dep.metadata.labels: + new_dep.metadata.labels["app"] = new_name + new_dep.metadata.labels["katakate.org/sandbox"] = new_name + if new_dep.spec and new_dep.spec.selector and new_dep.spec.selector.match_labels: + new_dep.spec.selector.match_labels["app"] = new_name + template = new_dep.spec.template if new_dep.spec else None + if template and template.metadata: + for attr in ("managed_fields", "creation_timestamp"): + if hasattr(template.metadata, attr): + setattr(template.metadata, attr, None) + if template.metadata.labels is None: + template.metadata.labels = {} + template.metadata.labels["app"] = new_name + template.metadata.labels["katakate.org/sandbox"] = new_name + template.metadata.annotations = { + **(template.metadata.annotations or {}), + **fork_annotations, + } + new_dep.spec.replicas = 1 + # The fork must land on the source VM's node — the k7d daemon socket + # is node-local (cross-node fork is k7d spec 9a M12, not built yet). + source_pod = await self._k7d_running_pod(source_name, namespace) + new_dep.spec.template.spec.node_name = source_pod.spec.node_name + + try: + await apps_v1.create_namespaced_deployment(namespace=namespace, body=new_dep) + except ApiException as e: + if e.status == 409: + return OperationResult(success=False, error=f"Sandbox {new_name} already exists") + raise + + pod_name = await self._wait_for_pod_container_started( + sandbox_name=new_name, + namespace=namespace, + timeout_seconds=120, + ) + if not pod_name: + try: + await apps_v1.delete_namespaced_deployment(name=new_name, namespace=namespace) + except Exception: + pass + return OperationResult( + success=False, + error=( + f"forked sandbox {new_name} never reached Running — the k7d fork failed " + "(check `kubectl describe pod` events and the k7d daemon logs)" + ), + ) + + fork_elapsed = time.time() - fork_start + return OperationResult( + success=True, + message=( + f"Forked sandbox {source_name} -> {new_name} as a k7d warm fork " + f"(CoW disk+memory of VM {vm['vm_id']}) in {fork_elapsed:.2f}s" + ), + ) + + # ------------------------------------------------------------------- + # Spec 10e: VolumeSnapshot lifecycle (list / get / create / delete / gc). + # ------------------------------------------------------------------- + + async def list_snapshots( + self, + namespace: str | None = "default", + all_namespaces: bool = False, + sandbox: str | None = None, + kind: str | None = None, + ) -> list[SnapshotInfo]: + """List ``VolumeSnapshot`` objects, classified by k7 kind. + + ``all_namespaces=True`` returns every namespace; otherwise restricts to + ``namespace``. ``sandbox`` filters to snapshots that target a sandbox's + root PVC (by ``k7.io/source-sandbox`` annotation, with a name-pattern + fallback for snapshots created before Spec 10e). + """ + custom = await self._get_custom_objects_client() + try: + if all_namespaces: + resp = await custom.list_cluster_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + plural="volumesnapshots", + ) + else: + resp = await custom.list_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace or "default", + plural="volumesnapshots", + ) + except ApiException as e: + print(f"⚠️ Failed to list VolumeSnapshots: {e}", file=sys.stderr) + return [] + + items = resp.get("items") or [] + out: list[SnapshotInfo] = [] + for item in items: + info = self._snapshot_info_from_object(item) + if kind and info.kind != kind: + continue + if sandbox: + if info.source_sandbox: + if info.source_sandbox != sandbox: + continue + else: + # Fallback: source PVC convention for kata-qemu-longhorn. + if info.source_pvc != self._root_pvc_name(sandbox): + continue + out.append(info) + return out + + async def get_snapshot(self, name: str, namespace: str = "default") -> SnapshotInfo | None: + """Return a single ``SnapshotInfo`` or ``None`` if the object is absent.""" + custom = await self._get_custom_objects_client() + try: + snap = await custom.get_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace, + plural="volumesnapshots", + name=name, + ) + except ApiException as e: + if e.status == 404: + return None + raise + return self._snapshot_info_from_object(snap) + + async def delete_snapshot(self, name: str, namespace: str = "default") -> OperationResult: + """Delete a ``VolumeSnapshot`` by name. + + When the last surviving snapshot for an *orphaned* root PVC (one whose + source sandbox was already deleted — kept around by + :meth:`_delete_sandbox_resources` so Longhorn could still clone from + it) is removed, the PVC is reaped here. That closes the spec-10f + lifecycle: once you delete the last "save point", the disk goes away. + """ + custom = await self._get_custom_objects_client() + # Read the snapshot first so we know its source PVC after the delete. + source_pvc: str | None = None + try: + snap_obj = await custom.get_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace, + plural="volumesnapshots", + name=name, + ) + source_pvc = ((snap_obj.get("spec") or {}).get("source") or {}).get("persistentVolumeClaimName") + except ApiException: + pass + try: + await custom.delete_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace, + plural="volumesnapshots", + name=name, + ) + except ApiException as e: + if e.status == 404: + return OperationResult( + success=False, + error=f"VolumeSnapshot {name} not found in namespace {namespace}", + ) + return OperationResult(success=False, error=f"Snapshot delete failed: {e.reason or e.body}") + + if source_pvc: + await self._maybe_reap_orphaned_pvc(source_pvc, namespace) + return OperationResult(success=True, message=f"Snapshot {name} deleted") + + async def _maybe_reap_orphaned_pvc(self, pvc_name: str, namespace: str) -> None: + """Delete an orphaned root PVC when its last referencing snapshot is gone. + + See :meth:`_delete_sandbox_resources` for context: when a sandbox is + deleted while pause/named snapshots still reference its PVC, the PVC + is kept around with the ``k7.io/orphaned-by`` annotation. When that + last snapshot finally gets deleted, the PVC has no remaining + purpose — clean it up here so we don't leak Longhorn volumes. + """ + v1 = await self._get_core_v1_client() + try: + pvc = await v1.read_namespaced_persistent_volume_claim(name=pvc_name, namespace=namespace) + except ApiException: + return + annotations = (pvc.metadata.annotations or {}) if pvc.metadata else {} + if "k7.io/orphaned-by" not in annotations: + return + # Are there any other snapshots still referencing this PVC? + try: + custom = await self._get_custom_objects_client() + snaps = await custom.list_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace, + plural="volumesnapshots", + ) + for snap in snaps.get("items", []): + src = ((snap.get("spec") or {}).get("source") or {}).get("persistentVolumeClaimName") + if src == pvc_name: + return # someone still holds a reference; keep the PVC. + except Exception: + return + try: + await v1.delete_namespaced_persistent_volume_claim(name=pvc_name, namespace=namespace) + except ApiException: + pass + + async def create_snapshot( + self, + sandbox_name: str, + snapshot_name: str, + namespace: str = "default", + ) -> OperationResult: + """Snapshot a running sandbox's root PVC without pausing it (kind=named).""" + backend = await self._detect_backend(sandbox_name, namespace) + if backend == "k7d": + return OperationResult( + success=False, + error=f"`k7 snapshot create` does not support the k7d backend: {K7D_SNAPSHOT_UNSUPPORTED}", + ) + return await self._create_volume_snapshot( + pvc_name=self._root_pvc_name(sandbox_name), + snapshot_name=snapshot_name, + snapshot_class="longhorn", + namespace=namespace, + kind=SNAPSHOT_KIND_NAMED, + source_sandbox=sandbox_name, + ) + + async def gc_snapshots( + self, + namespace: str = "default", + all_namespaces: bool = False, + keep_fork_for: timedelta = timedelta(minutes=10), + dry_run: bool = False, + ) -> OperationResult: + """Sweep stale ``kind=fork`` snapshots older than ``keep_fork_for``. + + Pause and named snapshots are **never** touched — anything the user + named survives. The data payload of the returned ``OperationResult`` + is a list of ``{name, namespace, age, deleted}`` records describing + what was (or would be) removed. + """ + candidates = await self.list_snapshots( + namespace=namespace, + all_namespaces=all_namespaces, + kind=SNAPSHOT_KIND_FORK, + ) + cutoff = datetime.now(timezone.utc) - keep_fork_for + results: list[dict] = [] + for snap in candidates: + try: + created = datetime.fromisoformat((snap.creation_timestamp or "").replace("Z", "+00:00")) + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + except Exception: + # Without a parseable timestamp we conservatively skip — leave it for the next sweep. + continue + if created > cutoff: + continue + record = { + "name": snap.name, + "namespace": snap.namespace, + "age": snap.age, + "deleted": False, + } + if not dry_run: + deletion = await self.delete_snapshot(snap.name, namespace=snap.namespace) + record["deleted"] = deletion.success + if not deletion.success: + record["error"] = deletion.error + results.append(record) + verb = "would delete" if dry_run else "deleted" + return OperationResult( + success=True, + message=f"GC {verb} {len(results)} fork snapshot(s)", + data=results, + ) + + # ------------------------------------------------------------------- + # Spec 10f: restore a sandbox from a standalone VolumeSnapshot. + # ------------------------------------------------------------------- + + def _rehydrate_config_from_snapshot( + self, + annotations: dict[str, str], + new_name: str, + namespace: str, + overrides: SandboxConfigOverrides | None, + ) -> OperationResult: + """Build a :class:`SandboxConfig` from a snapshot's ``k7.io/source-*`` + annotations, with optional per-call overrides. + + Returns the config in ``data`` on success. Returns success=False with a + helpful error when neither annotation nor override supplies the + required ``image`` field — that's the one value k7 can't safely guess. + """ + ov = overrides or SandboxConfigOverrides() + image = ov.image or annotations.get("k7.io/source-image") + if not image: + return OperationResult( + success=False, + error=( + "Snapshot lacks 'k7.io/source-image' annotation and no override supplied. " + "Pass --image explicitly to restore from this snapshot." + ), + ) + backend = self._canonicalize_backend( + ov.backend or annotations.get("k7.io/source-backend", "kata-qemu-longhorn") + ) + sidecar = ov.sidecar if ov.sidecar is not None else annotations.get("k7.io/source-sidecar") + root_disk_size = ov.root_disk_size or annotations.get("k7.io/source-root-disk-size", "10Gi") + limits = ov.limits + if limits is None: + raw_limits = annotations.get("k7.io/source-limits") + if raw_limits: + try: + parsed = json.loads(raw_limits) + if isinstance(parsed, dict): + limits = {k: str(v) for k, v in parsed.items()} + except Exception: + limits = None + return OperationResult( + success=True, + data=SandboxConfig( + name=new_name, + image=image, + namespace=namespace, + backend=backend, + root_disk_size=root_disk_size, + limits=limits, + sidecar=sidecar or None, + entrypoint=ov.entrypoint, + cmd=ov.cmd, + before_script=ov.before_script or "", + ), + ) + + async def restore_sandbox( + self, + snapshot_name: str, + new_sandbox_name: str, + namespace: str = "default", + overrides: SandboxConfigOverrides | None = None, + keep_snapshot: bool = True, + ) -> OperationResult: + """Boot a brand-new sandbox from a standalone ``VolumeSnapshot``. + + The snapshot's ``k7.io/source-*`` annotations (stamped by + :meth:`_create_volume_snapshot`) supply image / backend / sidecar / + limits / root-disk-size; ``overrides`` (Spec 10f) can override any + field. Restore is **kata-qemu-longhorn only** — there is no PVC to clone + from in the kata-firecracker-devmapper backend. + + The cloned PVC is named ``_root_pvc_name(new_sandbox_name)`` and is + pre-created from the snapshot before ``create_sandbox`` runs; the + idempotent ``_ensure_root_pvc`` in ``create_sandbox`` then detects + the existing PVC and skips its own creation step. + + ``keep_snapshot=False`` deletes the snapshot once the new sandbox + reaches Ready — useful for "thaw and discard" flows. + """ + # Look up the snapshot (typed view + raw annotations). + snap_info = await self.get_snapshot(snapshot_name, namespace=namespace) + if snap_info is None: + return OperationResult( + success=False, + error=f"VolumeSnapshot {snapshot_name} not found in namespace {namespace}", + ) + if not snap_info.ready_to_use: + return OperationResult( + success=False, + error=( + f"VolumeSnapshot {snapshot_name} is not ready_to_use yet. " + f"Wait for the snapshot to settle before restoring." + ), + ) + custom = await self._get_custom_objects_client() + try: + snap_obj = await custom.get_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1", + namespace=namespace, + plural="volumesnapshots", + name=snapshot_name, + ) + except ApiException as e: + return OperationResult(success=False, error=f"Failed to read snapshot {snapshot_name}: {e}") + annotations = ((snap_obj.get("metadata") or {}).get("annotations") or {}) if isinstance(snap_obj, dict) else {} + + config_result = self._rehydrate_config_from_snapshot( + annotations=annotations, + new_name=new_sandbox_name, + namespace=namespace, + overrides=overrides, + ) + if not config_result.success: + return config_result + config: SandboxConfig = config_result.data + + # Restore is kata-qemu-longhorn-only: kata-firecracker-devmapper has no PVC to + # clone, so it has no snapshots either. + if config.backend != "kata-qemu-longhorn": + return OperationResult( + success=False, + error=(f"Restore requires the kata-qemu-longhorn backend; requested backend was '{config.backend}'"), + ) + + # Bail early if the new sandbox name is already taken (so we don't + # half-create a PVC then fail at deployment time). + apps_v1 = await self._get_apps_v1_client() + try: + await apps_v1.read_namespaced_deployment(name=new_sandbox_name, namespace=namespace) + return OperationResult(success=False, error=f"Sandbox {new_sandbox_name} already exists") + except ApiException as e: + if e.status != 404: + return OperationResult(success=False, error=f"Deployment lookup error: {e}") + + target_pvc = self._root_pvc_name(new_sandbox_name) + pvc_clone = await self._create_pvc_from_snapshot( + target_pvc_name=target_pvc, + namespace=namespace, + snapshot_name=snapshot_name, + storage_size=config.root_disk_size or "10Gi", + ) + if not pvc_clone.success: + return pvc_clone + + # ``create_sandbox`` does the rest. ``_ensure_root_pvc`` is idempotent + # and will detect our pre-created PVC; the Deployment, ConfigMap, + # NetworkPolicy, and readiness wait all happen in the normal path. + create_result = await self.create_sandbox(config) + if not create_result.success: + # Roll back the orphaned PVC so we don't leak storage. + try: + v1 = await self._get_core_v1_client() + await v1.delete_namespaced_persistent_volume_claim(name=target_pvc, namespace=namespace) + except Exception: + pass + return create_result + + if not keep_snapshot: + try: + await self.delete_snapshot(snapshot_name, namespace=namespace) + except Exception as e: + print( + f"⚠️ Failed to delete snapshot {snapshot_name} after restore: {e}", + file=sys.stderr, + ) + + return OperationResult( + success=True, + message=f"Sandbox {new_sandbox_name} restored from snapshot {snapshot_name}", + data={"source_snapshot": snapshot_name, "new_sandbox_name": new_sandbox_name}, + ) + + async def delete_sandbox(self, name: str, namespace: str = "default") -> OperationResult: + """Delete a sandbox.""" + return await self._delete_sandbox_resources(name, namespace) + + async def delete_all_sandboxes(self, namespace: str = "default") -> OperationResult: """Delete all sandboxes in a namespace.""" try: - sandboxes = self._get_kata_sandboxes(namespace) + sandboxes = await self._get_kata_sandboxes(namespace) results = [] for deployment in sandboxes: - result = self._delete_sandbox_resources( - deployment.metadata.name, namespace - ) + result = await self._delete_sandbox_resources(deployment.metadata.name, namespace) results.append( { "name": deployment.metadata.name, @@ -829,69 +3100,61 @@ class K7Core: data=results, ) - return OperationResult( - success=True, message=f"Deleted {len(results)} sandboxes", data=results - ) + return OperationResult(success=True, message=f"Deleted {len(results)} sandboxes", data=results) except Exception as e: return OperationResult(success=False, error=str(e)) - def exec_command( - self, sandbox_name: str, command: str, namespace: str = "default" - ) -> ExecResult: + async def exec_command(self, sandbox_name: str, command: str, namespace: str = "default") -> ExecResult: """Execute a command in a sandbox and return the result.""" start_time = time.time() try: - apps_v1 = self._get_apps_v1_client() - v1 = self._get_core_v1_client() + apps_v1 = await self._get_apps_v1_client() + v1 = await self._get_core_v1_client() try: - apps_v1.read_namespaced_deployment( - name=sandbox_name, namespace=namespace - ) + await apps_v1.read_namespaced_deployment(name=sandbox_name, namespace=namespace) except ApiException as e: if e.status == 404: raise Exception(f"Sandbox {sandbox_name} not found") raise Exception(f"Failed to get deployment: {e}") - pods = v1.list_namespaced_pod( - namespace=namespace, label_selector=f"app={sandbox_name}" - ) + pods = await v1.list_namespaced_pod(namespace=namespace, label_selector=f"app={sandbox_name}") - if not pods.items: + # A Terminating pod (e.g. the old replica right after a + # pause→resume cycle) still reports phase=Running and can keep a + # stale Ready condition; exec'ing into it 500s or, worse, writes + # into a dying VM's page cache that never reaches the disk. + # Only pods without a deletionTimestamp are candidates. + live = [p for p in pods.items if not p.metadata.deletion_timestamp] + + if not live: raise Exception(f"No pods found for sandbox {sandbox_name}") - pod = pods.items[0] + pod = live[0] if pod.status.phase != "Running": raise Exception(f"Pod is not running (status: {pod.status.phase})") pod_name = pod.metadata.name - exec_command = ["/bin/sh", "-c", command] - resp = stream( - v1.connect_get_namespaced_pod_exec, - pod_name, - namespace, - command=exec_command, - stderr=True, - stdin=False, - stdout=True, - tty=False, - _preload_content=False, - ) + exec_cmd = ["/bin/sh", "-c", command] + async with WsApiClient() as ws_api: + v1_ws = client.CoreV1Api(api_client=ws_api) + resp = await v1_ws.connect_get_namespaced_pod_exec( + pod_name, + namespace, + container="sandbox", + command=exec_cmd, # ty: ignore[invalid-argument-type] + stderr=True, + stdin=False, + stdout=True, + tty=False, + ) - stdout_data = "" + stdout_data = resp if isinstance(resp, str) else "" stderr_data = "" - - while resp.is_open(): - resp.update(timeout=1) - if resp.peek_stdout(): - stdout_data += resp.read_stdout() - if resp.peek_stderr(): - stderr_data += resp.read_stderr() - - exit_code = 0 if resp.returncode is None else resp.returncode + exit_code = 0 duration_ms = int((time.time() - start_time) * 1000) @@ -904,16 +3167,52 @@ class K7Core: except Exception as e: duration_ms = int((time.time() - start_time) * 1000) - return ExecResult( - exit_code=1, stdout="", stderr=str(e), duration_ms=duration_ms - ) + return ExecResult(exit_code=1, stdout="", stderr=str(e), duration_ms=duration_ms) - def get_sandbox_metrics(self, namespace: Optional[str] = None) -> List[Dict]: + async def get_logs( + self, + sandbox_name: str, + namespace: str = "default", + container: str = "sandbox", + tail_lines: int | None = 200, + since_seconds: int | None = None, + ) -> OperationResult: + """Read pod logs for a sandbox (snapshot — no streaming/follow in V1). + + Resolves the pod by the ``app=`` label, then calls + ``read_namespaced_pod_log``. ``OperationResult.data`` contains a + ``{"logs": "..."}`` payload on success so the API handler can + wrap it in the standard envelope. Follow / tail-and-stream is + deliberately out of scope for Spec 10g; users who need it today + can run ``k7 --core logs --follow``. + """ + try: + v1 = await self._get_core_v1_client() + pods = await v1.list_namespaced_pod(namespace=namespace, label_selector=f"app={sandbox_name}") + if not pods.items: + return OperationResult( + success=False, + error=f"No pod found for sandbox {sandbox_name} in namespace {namespace}", + ) + pod_name = pods.items[0].metadata.name + kwargs: dict = {"name": pod_name, "namespace": namespace, "container": container} + if tail_lines is not None and tail_lines > 0: + kwargs["tail_lines"] = tail_lines + if since_seconds is not None and since_seconds > 0: + kwargs["since_seconds"] = since_seconds + logs_text = await v1.read_namespaced_pod_log(**kwargs) + return OperationResult(success=True, data={"logs": logs_text or ""}) + except ApiException as e: + return OperationResult(success=False, error=f"Kubernetes error while reading logs: {e.reason or e}") + except Exception as e: + return OperationResult(success=False, error=str(e)) + + async def get_sandbox_metrics(self, namespace: str | None = None) -> list[dict]: """Get resource usage metrics for sandboxes.""" try: - metrics_api = self._get_metrics_client() - v1 = self._get_core_v1_client() - sandboxes = self._get_kata_sandboxes(namespace) + metrics_api = await self._get_metrics_client() + v1 = await self._get_core_v1_client() + sandboxes = await self._get_kata_sandboxes(namespace) metrics_list = [] for deployment in sandboxes: @@ -921,9 +3220,7 @@ class K7Core: sb_namespace = deployment.metadata.namespace try: - pods = v1.list_namespaced_pod( - namespace=sb_namespace, label_selector=f"app={sb_name}" - ) + pods = await v1.list_namespaced_pod(namespace=sb_namespace, label_selector=f"app={sb_name}") if not pods.items: continue @@ -933,8 +3230,7 @@ class K7Core: pod_name = pod.metadata.name - # Get metrics using the correct generic method - metrics = metrics_api.get_namespaced_custom_object( + metrics = await metrics_api.get_namespaced_custom_object( group="metrics.k8s.io", version="v1beta1", namespace=sb_namespace, diff --git a/src/k7/core/models.py b/src/k7/core/models.py index 37566de..8d0d34c 100644 --- a/src/k7/core/models.py +++ b/src/k7/core/models.py @@ -1,24 +1,39 @@ -from typing import Optional, List, Dict, Any -from dataclasses import dataclass, asdict, fields +from dataclasses import asdict, dataclass, fields +from typing import Any + import yaml @dataclass class SandboxConfig: - """Data model for sandbox configuration""" + """Data model for sandbox configuration + + For kata-firecracker-devmapper: image is REQUIRED (the container to run) + For kata-qemu-longhorn: image is REQUIRED (no bare VM mode) + For k7d: image is REQUIRED (runs in a k7d microVM, runtimeClassName k7) + """ name: str image: str namespace: str = "default" - env_file: Optional[str] = None - egress_whitelist: Optional[List[str]] = None - limits: Optional[Dict[str, str]] = None + runtime_class_name: str | None = None + root_disk_size: str | None = "10Gi" + backend: str | None = None # "kata-firecracker-devmapper", "kata-qemu-longhorn", or "k7d" + env_file: str | None = None + egress_whitelist: list[str] | None = None + limits: dict[str, str] | None = None before_script: str = "" + entrypoint: list[str] | None = None + cmd: list[str] | None = None + sidecar: str | None = None # key into SIDECAR_REGISTRY, or None # Security toggles (default off) and capabilities configuration pod_non_root: bool = False container_non_root: bool = False - cap_drop: Optional[List[str]] = None # default behavior handled in core: drop ALL - cap_add: Optional[List[str]] = None + cap_drop: list[str] | None = None # default behavior handled in core: drop ALL + cap_add: list[str] | None = None + # Optional explicit node placement (sets pod's node_name). Used by tests + # that need to inspect host-side state for a sandbox they just created. + node_name: str | None = None # Note: ingress isolation is enforced by core with a hardcoded NetworkPolicy def __post_init__(self): @@ -27,7 +42,7 @@ class SandboxConfig: @classmethod def from_yaml(cls, yaml_path: str) -> "SandboxConfig": - with open(yaml_path, "r") as f: + with open(yaml_path) as f: data = yaml.safe_load(f) return cls(**data) @@ -51,6 +66,11 @@ class SandboxInfo: restarts: int age: str image: str + backend: str = "unknown" + # Kubernetes node hosting the sandbox pod ("" while unscheduled). Needed + # by API/SDK clients to reason about k7d VM-op node locality (spec 18f + # issue 6): k7d pause/resume/fork must run on the sandbox's node. + node: str = "" error_message: str = "" def to_dict(self) -> dict: @@ -77,3 +97,57 @@ class OperationResult: def to_dict(self) -> dict: return asdict(self) + + +# Spec 10e: VolumeSnapshot kinds, used both for the ``k7.io/kind`` annotation +# we stamp at creation time and for the heuristic fallback that classifies +# pre-existing snapshots by name pattern. +SNAPSHOT_KIND_PAUSE = "pause" +SNAPSHOT_KIND_FORK = "fork" +SNAPSHOT_KIND_NAMED = "named" + + +@dataclass +class SandboxConfigOverrides: + """Optional per-call overrides for ``K7Core.restore_sandbox`` (Spec 10f). + + Every field is optional. When ``None`` the corresponding value is taken + from the snapshot's ``k7.io/source-*`` annotations (stamped at snapshot + creation time); the user-supplied override wins when set. ``image`` is + the only field that has no safe default — restore fails with a clear + error if neither the annotation nor an override is present. + """ + + image: str | None = None + backend: str | None = None + root_disk_size: str | None = None + sidecar: str | None = None + limits: dict[str, str] | None = None + entrypoint: list[str] | None = None + cmd: list[str] | None = None + before_script: str | None = None + + def to_dict(self) -> dict: + return {k: v for k, v in asdict(self).items() if v is not None} + + +@dataclass +class SnapshotInfo: + """Inspectable view of a Kubernetes ``VolumeSnapshot`` managed by k7. + + Fields mirror what ``k7 snapshot list/inspect`` and the HTTP API surface. + """ + + name: str + namespace: str + source_pvc: str + source_sandbox: str + kind: str # one of SNAPSHOT_KIND_PAUSE / _FORK / _NAMED + ready_to_use: bool + creation_timestamp: str # RFC3339 string (kept verbatim from the API) + age: str # human-readable, computed from creation_timestamp + size_bytes: int = 0 # 0 when restoreSize is unknown / not yet set + snapshot_class: str = "longhorn" + + def to_dict(self) -> dict: + return asdict(self) diff --git a/src/k7/core/sidecar.py b/src/k7/core/sidecar.py new file mode 100644 index 0000000..a3c5227 --- /dev/null +++ b/src/k7/core/sidecar.py @@ -0,0 +1,36 @@ +"""Generic sidecar framework: registry + spec dataclass. + +Adding a new sidecar type requires only a new entry in SIDECAR_REGISTRY. +All injection logic in core.py is driven by SidecarSpec fields — no +type-specific branching. +""" + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class SidecarSpec: + image: str + socket_mount: str # directory containing the socket (shared via emptyDir) + socket_name: str # socket filename within socket_mount + data_path: str # where the daemon stores persistent data + pvc_subdir: str # subdirectory on the Longhorn PVC for ql backend + readiness_cmd: list[str] + privileged: bool + env: dict[str, str] = field(default_factory=dict) + args: list[str] = field(default_factory=list) + + +SIDECAR_REGISTRY: dict[str, SidecarSpec] = { + "docker": SidecarSpec( + image="docker:27.5-dind", + socket_mount="/var/run", + socket_name="docker.sock", + data_path="/var/lib/docker", + pvc_subdir="docker", + readiness_cmd=["docker", "info"], + privileged=True, + env={"DOCKER_TLS_CERTDIR": ""}, + args=["--tls=false"], + ), +} diff --git a/src/k7/deploy/inventory.ini.example b/src/k7/deploy/inventory.ini.example new file mode 100644 index 0000000..4a9a2d4 --- /dev/null +++ b/src/k7/deploy/inventory.ini.example @@ -0,0 +1,71 @@ +; Example multi-node inventory for `k7 install -i inventory.ini`. +; +; Each host declares which backend(s) it supports via `k7_backends` (comma-separated). +; A node may support one or both of: +; - kata-qemu-longhorn (uses Kata QEMU + overlayfs + Longhorn for sandbox storage) +; - kata-firecracker-devmapper (uses Kata Firecracker + LVM thin-pool snapshotter) +; +; Per-host vars: +; ansible_host : SSH target IP +; ansible_user : SSH user (defaults to root) +; k7_backends : comma-separated backend list +; k7_devmapper_disk : block device for FD thin-pool (e.g. /dev/nvme1n1) +; longhorn_extra_disk : extra mount path to register with Longhorn +; kata_thinpool_pv_size : LVM PV size for the kfd thin-pool (default 100G) +; k7d_disks_image_size : sparse XFS image size for k7d volume pool (default 32G) +; +; Group vars (under [k7_cluster:vars]): +; longhorn_replicas : Longhorn replica count (defaults to min(3, node count)) +; longhorn_data_path : Longhorn default data path (defaults to /var/lib/longhorn) + +[k7_servers] +node1 ansible_host=192.0.2.10 k7_backends=kata-qemu-longhorn,kata-firecracker-devmapper k7_devmapper_disk=/dev/nvme1n1 +node2 ansible_host=5.9.18.222 k7_backends=kata-qemu-longhorn +node3 ansible_host=5.9.18.223 k7_backends=kata-qemu-longhorn + +[k7_agents] +node4 ansible_host=5.9.18.224 k7_backends=kata-firecracker-devmapper k7_devmapper_disk=/dev/nvme1n1 + +[k7_cluster:children] +k7_servers +k7_agents + +[k7_cluster:vars] +ansible_user=root +ansible_ssh_private_key_file=~/.ssh/id_ed25519 +longhorn_replicas=2 + +; ────────────────────────────────────────────────────────────────────── +; HA note: K3s embedded etcd needs an odd number of server nodes +; (3, 5, ...) to maintain quorum. With 2 servers etcd has no fault +; tolerance — for a 2-node cluster, put the second node in [k7_agents]. +; The first host listed in [k7_servers] is the cluster-init server. +; +; 3-node HA with ALL backends (kfd + kql + k7d) on every node — spec 18e. +; Run ONE command from a checkout on the first master: +; +; k7 install -i inventory.ini --ha --k7d-artifact /root/k7d-v0.1.0-x86_64-linux.tar.gz +; +; Do NOT pass --backend alongside -i: per-host `k7_backends` in the +; inventory is authoritative (an explicit --backend overrides it). +; +; On dual-NVMe boxes where the OS lives on one disk and the other is a raw +; spare, OMIT k7_devmapper_disk: NVMe enumeration (nvme0n1 vs nvme1n1) is +; NOT stable across reboots, so a hardcoded device can point at the OS disk +; after a reboot. The playbook auto-detects the empty non-root whole disk, +; which is enumeration-proof. Set k7_devmapper_disk only when a node has +; several spare disks and you must pick a specific one. +; +; [k7_servers] +; k7-node-01 ansible_host=192.0.2.11 k7_backends=kfd,kql,k7d +; k7-node-02 ansible_host=192.0.2.12 k7_backends=kfd,kql,k7d +; k7-node-03 ansible_host=192.0.2.13 k7_backends=kfd,kql,k7d +; +; [k7_cluster:children] +; k7_servers +; +; [k7_cluster:vars] +; ansible_user=root +; ansible_ssh_private_key_file=/root/.ssh/id_ed25519 +; longhorn_replicas=3 +; ────────────────────────────────────────────────────────────────────── diff --git a/src/k7/deploy/inventory.local.ini b/src/k7/deploy/inventory.local.ini index 9e85436..b239fd0 100644 --- a/src/k7/deploy/inventory.local.ini +++ b/src/k7/deploy/inventory.local.ini @@ -1,2 +1,9 @@ -[k7_nodes] +; Single-node localhost inventory used by `k7 install` when no inventory is given. +[k7_servers] localhost ansible_connection=local ansible_user=root + +[k7_agents] + +[k7_cluster:children] +k7_servers +k7_agents diff --git a/src/k7/deploy/k7-install-node.yaml b/src/k7/deploy/k7-install-node.yaml index 833cad5..1764592 100644 --- a/src/k7/deploy/k7-install-node.yaml +++ b/src/k7/deploy/k7-install-node.yaml @@ -1,10 +1,36 @@ - -- name: Provision Bare Metal instance for K7 - hosts: k7_nodes - become: yes +- name: Provision Bare Metal instance(s) for K7 + hosts: k7_cluster + become: true vars: - target_user: "{{ ansible_user }}" - helm_version: "v3.15.2" + target_user: "{{ ansible_user | default('root') }}" + k7_backend_default: "kata-firecracker-devmapper" + snapshotter_version: "v8.4.0" + kata_version: "3.24.0" + cni_plugins_version: "v1.9.0" + longhorn_version: "v1.10.1" + firecracker_version: "v1.14.0" + jailer_version: "v1.14.0" + # k7d backend (spec 9a M11): release artifact source. Override + # `k7d_artifact_local_path` (path on the Ansible controller) to install + # from a locally built tarball instead of the GitHub release. + k7d_version: "0.1.0" + k7d_artifact_url: "https://github.com/katakate/k7d/releases/download/v{{ k7d_version }}/k7d-v{{ k7d_version }}-x86_64-linux.tar.gz" + k7d_artifact_local_path: "" + # Sparse XFS (reflink=1) loopback image backing /var/lib/k7d/disks — + # k7d's writable volume images live there so warm forks can CoW-clone + # them with FICLONE reflinks. Per-node override in the inventory: + # `k7d_disks_image_size=64G`. Sizing + failure modes: docs/BACKENDS.md + # "Disk pool sizing" (spec 18f issue 5). + k7d_disks_image_size: "32G" + # Size of the LVM physical volume carved out of the spare disk for the + # kfd kata-vg/thin-pool (the rest of the disk stays unused). Per-node + # override in the inventory: `kata_thinpool_pv_size=400G`. + kata_thinpool_pv_size: "100G" + # CNI: "cilium" (default) enables FQDN egress via CiliumNetworkPolicy. + # "flannel" retains K3s's bundled CNI (CIDR-only egress). + k7_cni_default: "cilium" + cilium_cli_version: "v0.19.2" + cilium_version: "1.19.2" # Map ansible_architecture to Debian/Kata arch format arch_map: x86_64: amd64 @@ -13,23 +39,99 @@ tasks: - name: Gather facts (ensures ansible_distribution_release is available) - setup: + ansible.builtin.setup: + + # ────────────────────────────────────────────────────────────────────── + # Compute per-host backend set + cluster role + # ────────────────────────────────────────────────────────────────────── + - name: Compute per-host backend list (raw) + ansible.builtin.set_fact: + k7_backends_raw: >- + {{ + (k7_backends.split(',') | map('trim') | reject('equalto','') | list) + if (k7_backends is defined and k7_backends | length > 0) + else ([k7_backend] if (k7_backend is defined and k7_backend | length > 0) + else [k7_backend_default]) + }} + + - name: Normalize deprecated backend aliases + ansible.builtin.set_fact: + k7_backends_list: >- + {{ + k7_backends_raw + | map('regex_replace', '^(fd|kfd|firecracker-devmapper)$', 'kata-firecracker-devmapper') + | map('regex_replace', '^(ql|kql|qemu-longhorn)$', 'kata-qemu-longhorn') + | map('regex_replace', '^k7$', 'k7d') + | unique + | list + }} + + - name: Warn on deprecated backend names + ansible.builtin.debug: + msg: >- + Deprecated backend name(s) in inventory were remapped: + {{ k7_backends_raw }} → {{ k7_backends_list }}. + Prefer kata-firecracker-devmapper (kfd) / kata-qemu-longhorn (kql). + when: k7_backends_raw != k7_backends_list + + - name: Validate backends list + ansible.builtin.fail: + msg: "Unknown backend(s) in k7_backends={{ k7_backends_list }}; allowed: kata-firecracker-devmapper, kata-qemu-longhorn, k7d" + when: k7_backends_list | difference(['kata-firecracker-devmapper', 'kata-qemu-longhorn', 'k7d']) | length > 0 + + - name: Normalize CNI selection + ansible.builtin.set_fact: + k7_cni_effective: "{{ (k7_cni | default(k7_cni_default)) | lower }}" + + - name: Validate CNI selection + ansible.builtin.fail: + msg: "Unknown CNI '{{ k7_cni_effective }}'; allowed: cilium, flannel" + when: k7_cni_effective not in ['cilium', 'flannel'] + + # The Kubernetes node object is named after the machine hostname, not + # the Ansible inventory alias — `k7 install` (no hosts) runs against + # `localhost`, which is never a valid node name on a host with a real + # hostname. Resolve the actual node name once and use it for every + # `kubectl get/label node` below. + - name: Resolve Kubernetes node name (hostname when installing via localhost) + ansible.builtin.set_fact: + k7_k8s_node_name: >- + {{ ansible_hostname if inventory_hostname in ['localhost', '127.0.0.1'] else inventory_hostname }} + + - name: Compute cluster role and shortcuts + ansible.builtin.set_fact: + k7_servers_group: "{{ groups['k7_servers'] | default([]) }}" + k7_agents_group: "{{ groups['k7_agents'] | default([]) }}" + k7_node_role: "{{ 'server' if inventory_hostname in (groups['k7_servers'] | default([])) else 'agent' }}" + k7_first_master: "{{ (groups['k7_servers'] | default([inventory_hostname]))[0] }}" + longhorn_replicas_effective: "{{ longhorn_replicas | default([3, ((groups['k7_cluster'] | default([inventory_hostname])) | length)] | min) }}" + k7_has_devmapper: "{{ 'kata-firecracker-devmapper' in k7_backends_list }}" + k7_has_longhorn: "{{ 'kata-qemu-longhorn' in k7_backends_list }}" + k7_has_k7d: "{{ 'k7d' in k7_backends_list }}" + k7_cni_is_cilium: "{{ k7_cni_effective == 'cilium' }}" + + - name: Show node configuration + ansible.builtin.debug: + msg: >- + host={{ inventory_hostname }} role={{ k7_node_role }} + backends={{ k7_backends_list }} first_master={{ k7_first_master }} - name: Update apt cache and upgrade all packages ansible.builtin.apt: - update_cache: yes + update_cache: true upgrade: dist - autoremove: yes + autoremove: true tags: ['system_update'] - # ---- LVM utilities (needed for pvcreate/vgcreate/lvcreate) ---- + # ---- LVM utilities (devmapper-only) ---- - name: Install LVM2 ansible.builtin.apt: name: lvm2 state: present + when: k7_has_devmapper tags: ['lvm'] - # Section: KVM Installation + # Section: KVM Installation (always — both backends use KVM) - name: Install required KVM and support utilities ansible.builtin.apt: name: @@ -50,7 +152,7 @@ ansible.builtin.lineinfile: path: /etc/modules-load.d/k7-kvm.conf line: "{{ item }}" - create: yes + create: true mode: '0644' loop: - kvm @@ -67,121 +169,127 @@ changed_when: false tags: ['kvm'] - - name: Add target user {{ target_user }} to the KVM group + - name: Add target user to the KVM group ansible.builtin.user: name: "{{ target_user }}" groups: kvm - append: yes + append: true tags: ['kvm'] - name: Check /dev/kvm ownership - stat: + ansible.builtin.stat: path: /dev/kvm register: kvm_dev - name: Fail if /dev/kvm is not owned by kvm group - fail: + ansible.builtin.fail: msg: "/dev/kvm must be owned by group 'kvm'. Check system configuration." when: - - kvm_dev.stat.exists - - kvm_dev.stat.grp is defined - - kvm_dev.stat.grp != 'kvm' + - kvm_dev.stat.exists + - kvm_dev.stat.grp is defined + - kvm_dev.stat.grp != 'kvm' # --------------------------------------------------------------------------------------- - # Force iptables/ip6tables legacy backend (slaves follow automatically) (this is for k3s) + # Force iptables/ip6tables legacy backend (this is for k3s) # --------------------------------------------------------------------------------------- - - name: Set iptables master alternative to legacy - command: update-alternatives --set iptables /usr/sbin/iptables-legacy + - name: Set iptables master alternative to legacy + ansible.builtin.command: update-alternatives --set iptables /usr/sbin/iptables-legacy changed_when: "'link group' in result.stdout or result.rc == 0" register: result tags: ['iptables'] - + - name: Set ip6tables master alternative to legacy - command: update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy + ansible.builtin.command: update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy changed_when: "'link group' in result.stdout or result.rc == 0" register: result tags: ['iptables'] - name: Ensure xt_mark module loads at boot and now - lineinfile: + ansible.builtin.lineinfile: path: /etc/modules-load.d/k7.conf line: xt_mark - create: yes + create: true + mode: '0644' notify: Load xt_mark tags: ['iptables'] - name: Load xt_mark module now - command: modprobe xt_mark - changed_when: false # no “changed” output on reruns + ansible.builtin.command: modprobe xt_mark + changed_when: false tags: ['iptables'] - # ------------------------------------------------------------------ - # Ensure the devmapper stanza is absent before first K3s start + # Pre-clean stale containerd config (we always rewrite it below) # ------------------------------------------------------------------ - name: Remove devmapper snapshotter block (if present) early - blockinfile: + ansible.builtin.blockinfile: path: /var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.tmpl marker: "# {mark} ANSIBLE MANAGED DEVMAPPER" state: absent - create: yes + create: true + mode: '0644' tags: ['devmapper'] - # ------------------------------------------------------------------ - # Ensure no stale rendered config.toml forces devmapper on first boot - # ------------------------------------------------------------------ - name: Remove previously rendered containerd config - file: + ansible.builtin.file: path: /var/lib/rancher/k3s/agent/etc/containerd/config.toml state: absent tags: ['k3s'] + - name: Remove stale custom Kata QEMU config (when QEMU not in backends) + ansible.builtin.file: + path: /opt/kata/share/kata-containers/configuration-qemu.toml + state: absent + when: not k7_has_longhorn + tags: ['kata'] + # ------------------------------------------------------------------ - # Minimal CNI so containerd’s CRI plugin can start on first boot + # CNI bootstrap (always) # ------------------------------------------------------------------ - name: Download CNI plugin tarball early - get_url: - url: https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz + ansible.builtin.get_url: + url: https://github.com/containernetworking/plugins/releases/download/{{ cni_plugins_version }}/cni-plugins-linux-amd64-{{ cni_plugins_version }}.tgz dest: /tmp/cni-plugins.tgz mode: '0644' + timeout: 60 + register: cni_dl + retries: 5 + delay: 10 + until: cni_dl is success tags: ['cni'] - name: Ensure /opt/cni/bin exists - file: + ansible.builtin.file: path: /opt/cni/bin state: directory mode: '0755' tags: ['cni'] - name: Extract CNI plugins early - unarchive: + ansible.builtin.unarchive: src: /tmp/cni-plugins.tgz dest: /opt/cni/bin - remote_src: yes + remote_src: true tags: ['cni'] - # ─── CNI bootstrap (single-file loopback) ───────────────────────────── - - name: Wipe CNI conf dirs before first k3s start - file: + ansible.builtin.file: path: "{{ item }}" state: absent loop: - /etc/cni/net.d - - /var/lib/rancher/k3s/agent/etc/cni/net.d # ← just in case - tags: [cni] + - /var/lib/rancher/k3s/agent/etc/cni/net.d + tags: ['cni'] - # Recreate the single dir that containerd will read - name: Re-create /etc/cni/net.d directory - file: + ansible.builtin.file: path: /etc/cni/net.d state: directory mode: '0755' - tags: [cni] + tags: ['cni'] - # Drop exactly one minimal loop-back config - name: Drop loop-back CNI config - copy: + ansible.builtin.copy: dest: /etc/cni/net.d/99-loopback.conf mode: '0644' content: | @@ -190,34 +298,47 @@ "name": "loopback", "type": "loopback" } - tags: [cni] + tags: ['cni'] - # NEW fix - name: Disable nftables FORWARD rules (they conflict with iptables) - shell: | + ansible.builtin.shell: | nft flush ruleset || true systemctl mask nftables || true + changed_when: false tags: ['network'] + - name: Check if Docker is installed + ansible.builtin.command: docker --version # noqa: command-instead-of-module + register: docker_installed + changed_when: false + failed_when: false + tags: ['docker'] + + - name: Install Docker via official convenience script + ansible.builtin.shell: curl -fsSL https://get.docker.com | sh # noqa: command-instead-of-shell command-instead-of-module + when: docker_installed.rc != 0 + changed_when: true + tags: ['docker'] + - name: Restart Docker to recreate iptables chains (after switching to legacy / flushing nft) ansible.builtin.systemd: name: docker state: restarted - tags: ['docker','iptables'] + tags: ['docker', 'iptables'] - name: Detect systemd-resolved resolv.conf - stat: + ansible.builtin.stat: path: /run/systemd/resolve/resolv.conf register: resolved_resolv tags: ['dns'] - name: Set k3s resolv.conf path - set_fact: - k3s_resolv_conf: "{{ resolved_resolv.stat.exists | ternary('/run/systemd/resolve/resolv.conf','/etc/resolv.conf') }}" + ansible.builtin.set_fact: + k3s_resolv_conf: "{{ resolved_resolv.stat.exists | ternary('/run/systemd/resolve/resolv.conf', '/etc/resolv.conf') }}" tags: ['dns'] - name: Persist iptables FORWARD rules for CNI traffic - iptables: + ansible.builtin.iptables: chain: FORWARD in_interface: cni0 jump: ACCEPT @@ -227,7 +348,7 @@ tags: ['network'] - name: Persist iptables FORWARD rules for return CNI traffic - iptables: + ansible.builtin.iptables: chain: FORWARD out_interface: cni0 ctstate: ESTABLISHED,RELATED @@ -237,32 +358,121 @@ state: present tags: ['network'] + # ------------------------------------------------------------------ + # K3s install (role-aware) — server-init / server-join / agent + # ------------------------------------------------------------------ + - name: Compute K3s install env + ansible.builtin.set_fact: + k3s_common_exec: >- + --write-kubeconfig-mode 644 + --cluster-cidr=10.42.0.0/16 + --service-cidr=10.43.0.0/16 + {{ '--flannel-backend=none --disable-network-policy --disable-kube-proxy' if k7_cni_is_cilium else '' }} - - name: Install K3s (but do NOT start it yet) - shell: | - curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true K3S_RESOLV_CONF={{ k3s_resolv_conf }} INSTALL_K3S_EXEC='--disable=traefik --write-kubeconfig-mode 644 --cluster-cidr=10.42.0.0/16 --service-cidr=10.43.0.0/16' sh - + - name: Install K3s on first master (cluster-init) # noqa: command-instead-of-module + ansible.builtin.shell: | + curl -sfL https://get.k3s.io | \ + INSTALL_K3S_SKIP_START=true \ + K3S_RESOLV_CONF={{ k3s_resolv_conf }} \ + INSTALL_K3S_EXEC='server --cluster-init --disable=traefik {{ k3s_common_exec }}' \ + sh - args: creates: /usr/local/bin/k3s + register: k3s_init + retries: 5 + delay: 10 + until: k3s_init is success + when: + - k7_node_role == 'server' + - inventory_hostname == k7_first_master tags: ['k3s'] - # 📦 Section: Firecracker + Jailer Installation - - name: Get latest Firecracker release tag - uri: - url: https://api.github.com/repos/firecracker-microvm/firecracker/releases/latest - return_content: yes - register: firecracker_release_info - tags: ['firecracker'] + # First-master needs to boot once so the join token exists for other masters/agents. + - name: Start K3s on first master to initialise cluster (one-shot) + ansible.builtin.systemd: + name: k3s + state: started + enabled: true + when: + - k7_node_role == 'server' + - inventory_hostname == k7_first_master + tags: ['k3s'] - - name: Set Firecracker version - set_fact: - firecracker_version: "{{ firecracker_release_info.json.tag_name }}" - tags: ['firecracker'] + - name: Wait for K3s server token on first master + ansible.builtin.wait_for: + path: /var/lib/rancher/k3s/server/node-token + timeout: 180 + when: + - k7_node_role == 'server' + - inventory_hostname == k7_first_master + tags: ['k3s'] + - name: Read K3s token on first master + ansible.builtin.slurp: + src: /var/lib/rancher/k3s/server/node-token + register: k3s_token_b64 + when: + - k7_node_role == 'server' + - inventory_hostname == k7_first_master + tags: ['k3s'] + + - name: Distribute K3s token + master URL to all hosts + ansible.builtin.set_fact: + k3s_token: "{{ hostvars[k7_first_master]['k3s_token_b64']['content'] | b64decode | trim }}" + k3s_url: "https://{{ hostvars[k7_first_master]['ansible_host'] | default(k7_first_master) }}:6443" + tags: ['k3s'] + + - name: Install K3s on additional master nodes (server join) # noqa: command-instead-of-module + ansible.builtin.shell: | + curl -sfL https://get.k3s.io | \ + INSTALL_K3S_SKIP_START=true \ + K3S_TOKEN='{{ k3s_token }}' \ + K3S_RESOLV_CONF={{ k3s_resolv_conf }} \ + INSTALL_K3S_EXEC='server --server {{ k3s_url }} --disable=traefik {{ k3s_common_exec }}' \ + sh - + args: + creates: /usr/local/bin/k3s + register: k3s_join + retries: 5 + delay: 10 + until: k3s_join is success + when: + - k7_node_role == 'server' + - inventory_hostname != k7_first_master + tags: ['k3s'] + + - name: Install K3s on agent nodes # noqa: command-instead-of-module + ansible.builtin.shell: | + curl -sfL https://get.k3s.io | \ + INSTALL_K3S_SKIP_START=true \ + K3S_URL='{{ k3s_url }}' \ + K3S_TOKEN='{{ k3s_token }}' \ + K3S_RESOLV_CONF={{ k3s_resolv_conf }} \ + sh - + args: + creates: /usr/local/bin/k3s + register: k3s_agent + retries: 5 + delay: 10 + until: k3s_agent is success + when: k7_node_role == 'agent' + tags: ['k3s'] + + # Section: Firecracker + Jailer Installation (only if FD backend on this host) - name: Download Firecracker & Jailer binaries - get_url: - url: "https://github.com/firecracker-microvm/firecracker/releases/download/{{ firecracker_version }}/firecracker-{{ firecracker_version }}-{{ ansible_architecture }}.tgz" - dest: "/tmp/firecracker-{{ firecracker_version }}-{{ ansible_architecture }}.tgz" + vars: + fc_tgz: "firecracker-{{ firecracker_version }}-{{ ansible_architecture }}.tgz" + fc_base: "https://github.com/firecracker-microvm/firecracker/releases/download" + ansible.builtin.get_url: + url: "{{ fc_base }}/{{ firecracker_version }}/{{ fc_tgz }}" + dest: "/tmp/{{ fc_tgz }}" mode: '0644' + timeout: 60 + register: fc_dl + retries: 5 + delay: 10 + until: fc_dl is success + when: k7_has_devmapper tags: ['firecracker'] - name: Create firecracker bin directory @@ -270,178 +480,325 @@ path: /opt/firecracker state: directory mode: '0755' + when: k7_has_devmapper - name: Extract Firecracker release ansible.builtin.unarchive: src: "/tmp/firecracker-{{ firecracker_version }}-{{ ansible_architecture }}.tgz" dest: "/opt/firecracker/" - remote_src: yes + remote_src: true extra_opts: [--strip-components=1] + when: k7_has_devmapper tags: ['firecracker'] - name: Find firecracker binary - find: + ansible.builtin.find: paths: /opt/firecracker patterns: "firecracker*" - recurse: no + recurse: false register: firecracker_binaries + when: k7_has_devmapper - name: Install `file` utility required for lookup('pipe', 'file ...') ansible.builtin.apt: name: file state: present - tags: ['utils','firecracker'] + when: k7_has_devmapper + tags: ['utils', 'firecracker'] - name: Copy firecracker binary to final path (only if ELF) - copy: + ansible.builtin.copy: src: "{{ item.path }}" dest: "/usr/local/bin/firecracker" - remote_src: yes + remote_src: true mode: '0755' - loop: "{{ firecracker_binaries.files }}" - when: > - 'firecracker' in item.path and - not item.path.endswith('.debug') and - lookup('pipe', 'file ' ~ item.path) is search('ELF .* executable') + loop: "{{ firecracker_binaries.files | default([]) }}" + when: + - k7_has_devmapper + - "'firecracker' in item.path" + - not item.path.endswith('.debug') + - lookup('pipe', 'file ' ~ item.path) is search('ELF .* executable') - name: Find jailer binary - find: + ansible.builtin.find: paths: /opt/firecracker patterns: "jailer*" - recurse: no + recurse: false register: jailer_binaries + when: k7_has_devmapper - name: Copy jailer binary to final path (only if ELF) - copy: + ansible.builtin.copy: src: "{{ item.path }}" dest: "/usr/local/bin/jailer" - remote_src: yes + remote_src: true mode: '0755' - loop: "{{ jailer_binaries.files }}" - when: > - 'jailer' in item.path and - not item.path.endswith('.debug') and - lookup('pipe', 'file ' ~ item.path) is search('ELF .* executable') + loop: "{{ jailer_binaries.files | default([]) }}" + when: + - k7_has_devmapper + - "'jailer' in item.path" + - not item.path.endswith('.debug') + - lookup('pipe', 'file ' ~ item.path) is search('ELF .* executable') - name: Remove bundled Firecracker & jailer - file: + ansible.builtin.file: path: "/opt/kata/bin/{{ item }}" state: absent loop: [firecracker, jailer] - tags: ['kata','cleanup','firecracker','jailer'] + when: k7_has_devmapper + tags: ['kata', 'cleanup', 'firecracker', 'jailer'] - name: Ensure jailer is set‑uid root - file: + ansible.builtin.file: path: /usr/local/bin/jailer mode: '4755' owner: root group: root - tags: ['kata','firecracker','jailer'] + when: k7_has_devmapper + tags: ['kata', 'firecracker', 'jailer'] - name: Check Firecracker version - command: /usr/local/bin/firecracker --version + ansible.builtin.command: /usr/local/bin/firecracker --version changed_when: false + when: k7_has_devmapper tags: ['firecracker'] - name: Check Jailer version - command: /usr/local/bin/jailer --version + ansible.builtin.command: /usr/local/bin/jailer --version changed_when: false - tags: ['firecracker','jailer'] + when: k7_has_devmapper + tags: ['firecracker', 'jailer'] + + # Section: Install Kata Container (always — both runtimes ship in kata-static) + - name: Check whether Kata is already installed (skip re-extract if so) + ansible.builtin.stat: + path: /opt/kata/bin/containerd-shim-kata-v2 + register: kata_shim_present + tags: ['kata'] - # Section: Install Kata Container - name: Download and install Kata Containers - shell: | - KATA_VERSION=$(curl -sSL https://api.github.com/repos/kata-containers/kata-containers/releases/latest | jq -r .tag_name) + register: kata_install + retries: 3 + delay: 15 + until: kata_install is success + ansible.builtin.shell: | + set -eu + KATA_VERSION="{{ kata_version }}" mkdir -p /opt/kata - # Try .tar.zst first (newer format) TARBALL_ZST="kata-static-${KATA_VERSION}-{{ deb_arch }}.tar.zst" URL_ZST="https://github.com/kata-containers/kata-containers/releases/download/${KATA_VERSION}/${TARBALL_ZST}" + TAR_OUT="/tmp/kata-static-${KATA_VERSION}-{{ deb_arch }}.tar" + # Download (curl -f errors on >=400; idempotent re-download is fine) if curl -fsSL "$URL_ZST" -o "/tmp/$TARBALL_ZST" 2>/dev/null; then echo "Downloaded $TARBALL_ZST, extracting with zstd..." - if command -v zstd >/dev/null 2>&1; then - zstd -d "/tmp/$TARBALL_ZST" -o "/tmp/kata-static-${KATA_VERSION}-{{ deb_arch }}.tar" - tar -xvf "/tmp/kata-static-${KATA_VERSION}-{{ deb_arch }}.tar" -C / - else - echo "Installing zstd..." + if ! command -v zstd >/dev/null 2>&1; then apt-get update && apt-get install -y zstd - zstd -d "/tmp/$TARBALL_ZST" -o "/tmp/kata-static-${KATA_VERSION}-{{ deb_arch }}.tar" - tar -xvf "/tmp/kata-static-${KATA_VERSION}-{{ deb_arch }}.tar" -C / fi + # -f: never prompt on existing file; needed for idempotency on re-runs. + zstd -d -f "/tmp/$TARBALL_ZST" -o "$TAR_OUT" + tar -xf "$TAR_OUT" -C / else - # Fallback to .tar.xz (older format) echo "Trying fallback to .tar.xz format..." TARBALL_XZ="kata-static-${KATA_VERSION}-{{ deb_arch }}.tar.xz" URL_XZ="https://github.com/kata-containers/kata-containers/releases/download/${KATA_VERSION}/${TARBALL_XZ}" curl -fsSL "$URL_XZ" -o "/tmp/$TARBALL_XZ" - tar -xvf "/tmp/$TARBALL_XZ" -C / + tar -xf "/tmp/$TARBALL_XZ" -C / fi + # Free space — these are 5GB+ artifacts. Use :- because TARBALL_XZ is + # only set in the .tar.xz fallback branch and `set -u` would error otherwise. + rm -f "/tmp/$TARBALL_ZST" "$TAR_OUT" "/tmp/${TARBALL_XZ:-}" 2>/dev/null || true args: executable: /bin/bash + when: not kata_shim_present.stat.exists + changed_when: true tags: ['kata'] - - name: Point Kata to latest Firecracker / jailer - replace: + - name: Point Kata to Firecracker / jailer + ansible.builtin.replace: path: /opt/kata/share/defaults/kata-containers/configuration-fc.toml regexp: '^({{ item.key }}\s*=\s*).*' replace: '\1"/usr/local/bin/{{ item.name }}"' loop: - - { key: 'path', name: 'firecracker' } - - { key: 'jailer_path', name: 'jailer' } - tags: ['kata','config','firecracker','jailer'] - - + - {key: 'path', name: 'firecracker'} + - {key: 'jailer_path', name: 'jailer'} + when: k7_has_devmapper + tags: ['kata', 'config', 'firecracker', 'jailer'] - name: Set valid_hypervisor_paths - lineinfile: + ansible.builtin.lineinfile: path: /opt/kata/share/defaults/kata-containers/configuration-fc.toml regexp: '^valid_hypervisor_paths' line: 'valid_hypervisor_paths = ["/usr/local/bin/firecracker"]' - tags: ['kata','config','firecracker'] + when: k7_has_devmapper + tags: ['kata', 'config', 'firecracker'] - name: Set valid_jailer_paths - lineinfile: + ansible.builtin.lineinfile: path: /opt/kata/share/defaults/kata-containers/configuration-fc.toml regexp: '^valid_jailer_paths' line: 'valid_jailer_paths = ["/usr/local/bin/jailer"]' - tags: ['kata','config','jailer'] + when: k7_has_devmapper + tags: ['kata', 'config', 'jailer'] + # Spec 18g (kql-r3 dind IO wedge, CHALLENGES.md): kata's default + # virtiofsd runs with --thread-pool-size=1, so ALL virtio-fs IO of a + # kata-qemu sandbox (container rootfs + every Longhorn PVC mount, + # e.g. a dind /var/lib/docker) serializes through ONE thread. Under a + # sustained fsync burst against an r>=2 Longhorn volume that thread + # saturates for minutes; any kata-agent RPC that touches virtio-fs + # blocks behind it, the shim's agent health ping (CheckRequest) times + # out, and the shim kills the (perfectly healthy) VM. A 16-thread pool + # keeps agent-touched IO out of the write burst's convoy. + - name: Widen virtiofsd thread pool for kata-qemu (spec 18g wedge fix) + ansible.builtin.lineinfile: + path: /opt/kata/share/defaults/kata-containers/configuration-qemu.toml + regexp: '^virtio_fs_extra_args\s*=' + line: 'virtio_fs_extra_args = ["--thread-pool-size=16", "--announce-submounts"]' + when: k7_has_longhorn + tags: ['kata', 'config'] + + # Core stamps io.katacontainers.config.hypervisor.default_memory on + # kata sandboxes with a memory limit; kata rejects the whole pod + # ("annotation ... is not enabled") unless the annotation is + # allowlisted here. Found during spec 18g: `k7 create --backend kql + # --memory 2Gi` could not create a pod at all. + - name: Allow hypervisor memory annotation for kata-qemu + ansible.builtin.lineinfile: + path: /opt/kata/share/defaults/kata-containers/configuration-qemu.toml + regexp: '^enable_annotations\s*=' + line: 'enable_annotations = ["enable_iommu", "virtio_fs_extra_args", "kernel_params", "default_memory"]' + when: k7_has_longhorn + tags: ['kata', 'config'] + + # Same stamp/allowlist as kata-qemu above. Without this (and without + # pod_annotations on runtimes.kata), `k7 create --backend kfd --memory` + # silently boots the default 2048 MiB VM — found in spec 18g, fixed 18h. + - name: Allow hypervisor memory annotation for kata-fc + ansible.builtin.lineinfile: + path: /opt/kata/share/defaults/kata-containers/configuration-fc.toml + regexp: '^enable_annotations\s*=' + line: 'enable_annotations = ["enable_iommu", "virtio_fs_extra_args", "kernel_params", "default_memory"]' + when: k7_has_devmapper + tags: ['kata', 'config'] - name: Create containerd shim symlink - file: + ansible.builtin.file: src: "/opt/kata/bin/containerd-shim-kata-v2" dest: "/usr/local/bin/containerd-shim-kata-v2" state: link tags: ['kata', 'containerd'] + # ------------------------------------------------------------------ + # Orphaned VMM reaper (spec 18f issue 1b, CHALLENGES.md #6). + # + # Kata 3.24.0 + jailer: `jailer --daemonize` double-forks, so the shim + # records the (immediately-exiting) jailer PID and its SIGTERM fallback + # in fcEnd() signals a dead PID. Whenever the graceful in-guest + # shutdown fails (dead agent under churn), the firecracker process is + # orphaned and spins at 100% CPU. This watchdog kills VMM processes + # whose kata shim is gone: + # - firecracker: no containerd-shim-kata-v2 with a matching -id + # (the fc --id is the first 32 hex chars of the CRI sandbox id; + # live firecrackers always have PPID 1, so parentage is useless) + # - qemu: reparented to PID 1 (qemu stays a shim child while healthy) + # ------------------------------------------------------------------ + - name: Install orphaned VMM reaper script + ansible.builtin.copy: + dest: /usr/local/bin/k7-vmm-reaper.sh + mode: '0755' + content: | + #!/bin/bash + # Reap orphaned kata VMM processes (spec 18f issue 1b). + set -u + for pid in $(pgrep -x firecracker); do + id=$(tr '\0' '\n' < "/proc/$pid/cmdline" 2>/dev/null | awk 'prev=="--id"{print; exit} {prev=$0}') + [ -n "$id" ] || continue + if ! pgrep -f -- "containerd-shim-kata-v2.*-id ${id}" >/dev/null; then + echo "reaping orphaned firecracker pid=$pid id=$id (no kata shim)" + kill -9 "$pid" || true + fi + done + for pid in $(pgrep -f 'qemu-system'); do + ppid=$(awk '{print $4}' "/proc/$pid/stat" 2>/dev/null || echo "") + if [ "$ppid" = "1" ]; then + echo "reaping orphaned qemu pid=$pid (reparented to init — kata shim gone)" + kill -9 "$pid" || true + fi + done + when: k7_has_devmapper or k7_has_longhorn + tags: ['kata', 'reaper'] + + - name: Install VMM reaper systemd service + ansible.builtin.copy: + dest: /etc/systemd/system/k7-vmm-reaper.service + mode: '0644' + content: | + [Unit] + Description=Reap orphaned kata VMM processes (k7 spec 18f issue 1b) + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/k7-vmm-reaper.sh + when: k7_has_devmapper or k7_has_longhorn + tags: ['kata', 'reaper'] + + - name: Install VMM reaper systemd timer + ansible.builtin.copy: + dest: /etc/systemd/system/k7-vmm-reaper.timer + mode: '0644' + content: | + [Unit] + Description=Run the k7 VMM reaper every minute + + [Timer] + OnBootSec=2min + OnUnitActiveSec=1min + + [Install] + WantedBy=timers.target + when: k7_has_devmapper or k7_has_longhorn + tags: ['kata', 'reaper'] + + - name: Enable and start VMM reaper timer + ansible.builtin.systemd: + name: k7-vmm-reaper.timer + enabled: true + state: started + daemon_reload: true + when: k7_has_devmapper or k7_has_longhorn + tags: ['kata', 'reaper'] + - name: Ensure /opt/kata/share/kata-containers exists - file: + ansible.builtin.file: path: /opt/kata/share/kata-containers state: directory mode: '0755' tags: ['kata'] # ------------------------------------------------------------------ - # Kata kernel (vmlinux.container) + # Kata kernel (vmlinux.container) — only needed for FC backend # ------------------------------------------------------------------ - - name: Check if vmlinux.container already exists (provided by kata-static) - stat: + ansible.builtin.stat: path: /opt/kata/share/kata-containers/vmlinux.container register: kata_kernel + when: k7_has_devmapper tags: ['kata'] - - name: Get latest Kata Containers release metadata (only if kernel missing) - uri: - url: https://api.github.com/repos/kata-containers/kata-containers/releases/latest - return_content: yes + - name: Get Kata Containers release metadata (only if kernel missing) + ansible.builtin.uri: + url: https://api.github.com/repos/kata-containers/kata-containers/releases/tags/{{ kata_version }} + return_content: true register: kata_release_info - when: not kata_kernel.stat.exists + when: + - k7_has_devmapper + - not kata_kernel.stat.exists tags: ['kata'] - name: Set URL for vmlinux.container asset (only if kernel missing) - set_fact: + ansible.builtin.set_fact: kata_kernel_url: >- {{ kata_release_info.json.assets @@ -449,157 +806,167 @@ | map(attribute='browser_download_url') | first | default('') }} - when: not kata_kernel.stat.exists + when: + - k7_has_devmapper + - not kata_kernel.stat.exists tags: ['kata'] - name: Fail when release has no vmlinux.container asset (only if kernel missing) - fail: + ansible.builtin.fail: msg: >- - The latest Kata release ({{ kata_release_info.json.tag_name }}) does not contain - a `vmlinux.container` asset. Either switch to a release that ships the kernel, - or build / copy your own kernel into /opt/kata/share/kata-containers/. + The Kata release ({{ kata_release_info.json.tag_name }}) does not contain + a `vmlinux.container` asset. when: + - k7_has_devmapper - not kata_kernel.stat.exists - kata_kernel_url == '' tags: ['kata'] - name: Download vmlinux.container (only if kernel missing and asset present) - get_url: + ansible.builtin.get_url: url: "{{ kata_kernel_url }}" dest: /opt/kata/share/kata-containers/vmlinux.container mode: '0644' + timeout: 60 + register: kata_kernel_dl + retries: 5 + delay: 10 + until: kata_kernel_dl is success when: + - k7_has_devmapper - not kata_kernel.stat.exists - kata_kernel_url != '' tags: ['kata'] - # - name: Fail if GH_TOKEN is not set - # fail: - # msg: "GH_TOKEN environment variable must be set to access the private GitHub repo." - # when: lookup('env', 'GH_TOKEN') == "" - # tags: ['kata'] - - # - name: Ensure GitHub CLI is installed - # apt: - # name: gh - # state: present - # update_cache: yes - # tags: ['kata'] - - # - name: Download kata-containers.img from GitHub Release - # shell: | - # gh release download v0.0.1 \ - # --repo Katakate/k7 \ - # --pattern "kata-containers.img" \ - # --dir /opt/kata/share/kata-containers \ - # --clobber - # environment: - # GH_TOKEN: "{{ lookup('env', 'GH_TOKEN') }}" - # args: - # creates: /opt/kata/share/kata-containers/kata-containers.img - # tags: ['kata'] - - name: Ensure kernel path is configured in configuration-fc.toml - lineinfile: + ansible.builtin.lineinfile: path: /opt/kata/share/defaults/kata-containers/configuration-fc.toml regexp: '^#?\s*kernel\s*=' line: 'kernel = "/opt/kata/share/kata-containers/vmlinux.container"' - backrefs: yes + backrefs: true + when: k7_has_devmapper tags: ['kata'] - name: Ensure image path is configured in configuration-fc.toml - lineinfile: + ansible.builtin.lineinfile: path: /opt/kata/share/defaults/kata-containers/configuration-fc.toml regexp: '^#?\s*image\s*=' line: 'image = "/opt/kata/share/kata-containers/kata-containers.img"' - backrefs: yes + backrefs: true + when: k7_has_devmapper tags: ['kata'] - - name: Ensure containerd template directory exists - file: + ansible.builtin.file: path: /var/lib/rancher/k3s/agent/etc/containerd state: directory mode: '0755' - tags: ['kata', 'devmapper'] + tags: ['kata', 'devmapper', 'containerd'] - # ──── BEGIN: spare-disk check ──── + # ──── BEGIN: spare-disk check (FD backend only) ──── - name: Detect root disk device - set_fact: + ansible.builtin.set_fact: root_disk: "{{ (ansible_mounts | selectattr('mount', 'equalto', '/') | first).device - | regex_replace('p?[0-9]+$','') }}" + | regex_replace('p?[0-9]+$', '') }}" + when: k7_has_devmapper tags: ['lvm'] - name: Use provided disk override (if any) - set_fact: - requested_disk: "{{ k7_disk | default('') }}" + ansible.builtin.set_fact: + requested_disk: "{{ k7_devmapper_disk | default(k7_disk | default('')) }}" + when: k7_has_devmapper tags: ['lvm'] - name: Normalize requested disk path - set_fact: + ansible.builtin.set_fact: requested_disk_basename: "{{ requested_disk | regex_replace('^/dev/', '') }}" requested_disk_path: "/dev/{{ requested_disk | regex_replace('^/dev/', '') }}" - when: requested_disk != '' + when: + - k7_has_devmapper + - requested_disk != '' tags: ['lvm'] - name: Validate requested disk exists and is a block device - stat: + ansible.builtin.stat: path: "{{ requested_disk_path }}" register: requested_disk_stat - when: requested_disk != '' + when: + - k7_has_devmapper + - requested_disk != '' tags: ['lvm'] - name: Fail if requested disk not present or not a block device - fail: + ansible.builtin.fail: msg: "Specified k7_disk '{{ requested_disk_path }}' does not exist or is not a block device" - when: requested_disk != '' and (not requested_disk_stat.stat.exists or not requested_disk_stat.stat.isblk) + when: + - k7_has_devmapper + - requested_disk != '' + - (not requested_disk_stat.stat.exists or not requested_disk_stat.stat.isblk) tags: ['lvm'] - name: Ensure requested disk is a whole disk (not a partition) - command: lsblk -dn -o TYPE "{{ requested_disk_path }}" + ansible.builtin.command: lsblk -dn -o TYPE "{{ requested_disk_path }}" register: lsblk_type changed_when: false - when: requested_disk != '' + when: + - k7_has_devmapper + - requested_disk != '' tags: ['lvm'] - name: Fail if requested disk is not a whole disk - fail: + ansible.builtin.fail: msg: "Specified k7_disk '{{ requested_disk_path }}' is not a whole disk (TYPE={{ lsblk_type.stdout | trim }})" - when: requested_disk != '' and (lsblk_type.stdout is not defined or (lsblk_type.stdout | trim) != 'disk') + when: + - k7_has_devmapper + - requested_disk != '' + - (lsblk_type.stdout is not defined or (lsblk_type.stdout | trim) != 'disk') tags: ['lvm'] - name: Pull ansible facts for requested disk - set_fact: + ansible.builtin.set_fact: requested_disk_facts: "{{ ansible_devices.get(requested_disk_basename) | default(None) }}" - when: requested_disk != '' + when: + - k7_has_devmapper + - requested_disk != '' tags: ['lvm'] - name: Fail if requested disk not found in ansible_devices - fail: + ansible.builtin.fail: msg: "Device '{{ requested_disk_path }}' not visible in ansible_devices facts" - when: requested_disk != '' and (requested_disk_facts is not defined or requested_disk_facts is none) + when: + - k7_has_devmapper + - requested_disk != '' + - (requested_disk_facts is not defined or requested_disk_facts is none) tags: ['lvm'] - name: Fail if requested disk has partitions - fail: + ansible.builtin.fail: msg: "Device '{{ requested_disk_path }}' has partitions; wipe it with utils/wipe-disk.sh or choose another disk" - when: requested_disk != '' and (requested_disk_facts.partitions is defined and (requested_disk_facts.partitions | length) > 0) + when: + - k7_has_devmapper + - requested_disk != '' + - (requested_disk_facts.partitions is defined and (requested_disk_facts.partitions | length) > 0) tags: ['lvm'] - name: Fail if requested disk appears to be the root disk - fail: + ansible.builtin.fail: msg: "Device '{{ requested_disk_path }}' appears to be the root disk; choose a different disk" - when: requested_disk != '' and (requested_disk_basename == (root_disk | basename)) + when: + - k7_has_devmapper + - requested_disk != '' + - (requested_disk_basename == (root_disk | basename)) tags: ['lvm'] - name: Use requested disk as kata_block - set_fact: + ansible.builtin.set_fact: kata_block: "{{ requested_disk_path }}" - when: requested_disk != '' + when: + - k7_has_devmapper + - requested_disk != '' tags: ['lvm'] - name: Find an empty secondary disk (skip loop, mapper, CD-ROM, zram, etc.) - set_fact: + ansible.builtin.set_fact: kata_block: >- {{ ansible_devices @@ -616,49 +983,63 @@ | map('regex_replace', '^', '/dev/') | first | default('') }} - when: requested_disk == '' + when: + - k7_has_devmapper + - requested_disk == '' tags: ['lvm'] - name: Show selected disk for LVM thin-pool - debug: + ansible.builtin.debug: msg: "Using disk for K7 LVM thin-pool: {{ kata_block }} {{ '(user-specified)' if requested_disk != '' else '(auto-detected)' }}" + when: k7_has_devmapper tags: ['lvm'] - name: Fail if no spare block device is available - fail: + ansible.builtin.fail: msg: | - No empty secondary disk found. + No empty secondary disk found. K7 expects a dedicated data drive for the LVM thin-pool. Attach an extra NVMe/SSD (e.g. /dev/nvme2n1) and re-run the playbook. - when: requested_disk == '' and kata_block == '' + when: + - k7_has_devmapper + - requested_disk == '' + - kata_block == '' tags: ['lvm'] # ──── END spare-disk check ──── - - name: Ensure kata-vg exists - command: vgdisplay kata-vg + ansible.builtin.command: vgdisplay kata-vg register: vgcheck failed_when: false changed_when: false + when: k7_has_devmapper tags: ['lvm'] - - name: pvcreate (with label) - command: pvcreate -y --setphysicalvolumesize 100G --metadatasize 4M --dataalignment 1M {{ kata_block }} - when: vgcheck.rc != 0 + - name: Create physical volume (pvcreate with label) + ansible.builtin.command: > + pvcreate -y --setphysicalvolumesize {{ kata_thinpool_pv_size }} + --metadatasize 4M --dataalignment 1M {{ kata_block }} + when: + - k7_has_devmapper + - vgcheck.rc != 0 tags: ['lvm'] - name: Create VG if missing - command: vgcreate kata-vg {{ kata_block }} - when: vgcheck.rc != 0 + ansible.builtin.command: vgcreate kata-vg {{ kata_block }} + when: + - k7_has_devmapper + - vgcheck.rc != 0 tags: ['lvm'] - name: Add k7 tag to PV - command: pvchange --addtag k7 {{ kata_block }} - when: vgcheck.rc != 0 + ansible.builtin.command: pvchange --addtag k7 {{ kata_block }} + when: + - k7_has_devmapper + - vgcheck.rc != 0 tags: ['lvm'] - name: Ensure thin pool LV exists - command: > + ansible.builtin.command: > lvcreate -T kata-vg/thin-pool -l 95%FREE --poolmetadatasize 1G @@ -666,118 +1047,506 @@ --yes args: creates: /dev/kata-vg/thin-pool + when: k7_has_devmapper tags: ['lvm'] - # – create the VG sub-folder so containerd can write its BoltDB - name: Create devmapper VG subdir - file: + ansible.builtin.file: path: /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.devmapper/kata-vg - #path: /var/lib/containerd/io.containerd.snapshotter.v1.devmapper/kata-vg state: directory mode: '0755' + when: k7_has_devmapper tags: ['devmapper', 'lvm'] - name: Enable LVM thin-pool autoextend service - copy: + ansible.builtin.copy: dest: /etc/lvm/profile/kata-thin.profile + mode: '0644' content: | activation { thin_pool_autoextend_threshold=80 thin_pool_autoextend_percent=20 } + when: k7_has_devmapper tags: ['lvm'] - name: Attach profile to thin pool - command: lvchange --metadataprofile kata-thin kata-vg/thin-pool + ansible.builtin.command: lvchange --metadataprofile kata-thin kata-vg/thin-pool + when: k7_has_devmapper + changed_when: true tags: ['lvm'] + # ------------------------------------------------------------------ + # k7d backend (spec 9a M11): install the k7d daemon, containerd shim, + # and guest artifacts from the release tarball; provision the host + # prerequisites (vhost-vsock, erofs tools, reflink-capable XFS for + # writable volume images) and run k7d as a systemd service. + # ------------------------------------------------------------------ + - name: K7d — check /dev/kvm exists + ansible.builtin.stat: + path: /dev/kvm + register: k7d_kvm_dev + when: k7_has_k7d + tags: ['k7d'] - - name: Ensure skeleton containerd template exists (only once) - copy: - dest: /var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.tmpl - content: | - version = 3 - force: no # do NOT overwrite if it is already there + - name: K7d — fail loudly when /dev/kvm is missing + ansible.builtin.fail: + msg: >- + /dev/kvm not found on {{ inventory_hostname }} — the k7d backend runs hardware- + accelerated microVMs and cannot work without KVM. Enable virtualization (VT-x/AMD-V) + or drop 'k7d' from k7_backends. + when: + - k7_has_k7d + - not k7d_kvm_dev.stat.exists + tags: ['k7d'] + + - name: K7d — install host prerequisites (erofs, xfs, virtiofsd) + ansible.builtin.apt: + name: + - erofs-utils + - xfsprogs + - virtiofsd + state: present + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — verify /usr/libexec/virtiofsd is executable + ansible.builtin.stat: + path: /usr/libexec/virtiofsd + register: k7d_virtiofsd + failed_when: not (k7d_virtiofsd.stat.exists and k7d_virtiofsd.stat.executable) + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — configure vhost-vsock + tun modules to load at boot + ansible.builtin.lineinfile: + path: /etc/modules-load.d/k7d.conf + line: "{{ item }}" + create: true mode: '0644' - tags: ['containerd'] - - - name: Write basic config-v3.toml.tmpl with devmapper only - copy: + loop: + - vhost_vsock + - tun + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — load vhost-vsock + tun modules now + ansible.builtin.command: modprobe {{ item }} + loop: + - vhost_vsock + - tun + when: k7_has_k7d + changed_when: false + tags: ['k7d'] + + - name: K7d — fail loudly when /dev/vhost-vsock is missing + ansible.builtin.stat: + path: /dev/vhost-vsock + register: k7d_vhost_vsock + failed_when: not k7d_vhost_vsock.stat.exists + when: k7_has_k7d + tags: ['k7d'] + + # Writable k7d volume images are raw ext4 files that warm forks clone + # with FICLONE reflinks; the root FS is usually ext4 (no reflink), so + # provision a sparse XFS (reflink=1) image loop-mounted at + # /var/lib/k7d/disks, persisted via /etc/fstab. + - name: K7d — create /var/lib/k7d directory + ansible.builtin.file: + path: /var/lib/k7d + state: directory + mode: '0755' + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — check whether /var/lib/k7d/disks is already mounted + ansible.builtin.command: mountpoint -q /var/lib/k7d/disks + register: k7d_disks_mounted + changed_when: false + failed_when: false + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — create sparse XFS image for /var/lib/k7d/disks + ansible.builtin.command: truncate -s {{ k7d_disks_image_size }} /var/lib/k7d/disks.img + args: + creates: /var/lib/k7d/disks.img + when: + - k7_has_k7d + - k7d_disks_mounted.rc != 0 + tags: ['k7d'] + + - name: K7d — format the disks image as XFS with reflink=1 + ansible.builtin.command: mkfs.xfs -q -m reflink=1 /var/lib/k7d/disks.img + when: + - k7_has_k7d + - k7d_disks_mounted.rc != 0 + register: k7d_mkfs + changed_when: k7d_mkfs.rc == 0 + failed_when: k7d_mkfs.rc != 0 and 'appears to contain an existing filesystem' not in k7d_mkfs.stderr + tags: ['k7d'] + + - name: K7d — persist /var/lib/k7d/disks in /etc/fstab + ansible.builtin.lineinfile: + path: /etc/fstab + line: "/var/lib/k7d/disks.img /var/lib/k7d/disks xfs loop,nofail 0 0" + create: true + mode: '0644' + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — mount /var/lib/k7d/disks + ansible.builtin.shell: | + set -eu + mkdir -p /var/lib/k7d/disks + mountpoint -q /var/lib/k7d/disks || mount /var/lib/k7d/disks + when: k7_has_k7d + changed_when: false + tags: ['k7d'] + + - name: K7d — verify /var/lib/k7d/disks is reflink-capable + ansible.builtin.shell: | + set -eu + echo probe > /var/lib/k7d/disks/.reflink-probe-src + cp --reflink=always /var/lib/k7d/disks/.reflink-probe-src /var/lib/k7d/disks/.reflink-probe-dst + rm -f /var/lib/k7d/disks/.reflink-probe-src /var/lib/k7d/disks/.reflink-probe-dst + when: k7_has_k7d + changed_when: false + tags: ['k7d'] + + # ---- Release artifacts: tarball → /opt/k7d, installed via the + # bundled install.sh (binaries → /usr/local/bin, guest assets → + # /usr/local/share/k7d, k7d.service unit + socket wait). k3s + # registration is NOT delegated to install.sh — the playbook's + # merged containerd template + RuntimeClass tasks own that. + - name: K7d — create /opt/k7d directory + ansible.builtin.file: + path: /opt/k7d + state: directory + mode: '0755' + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — copy release tarball from controller (k7d_artifact_local_path) + ansible.builtin.copy: + src: "{{ k7d_artifact_local_path }}" + dest: /tmp/k7d-release.tar.gz + mode: '0644' + when: + - k7_has_k7d + - k7d_artifact_local_path | length > 0 + tags: ['k7d'] + + - name: K7d — download release tarball ({{ k7d_artifact_url }}) + ansible.builtin.get_url: + url: "{{ k7d_artifact_url }}" + dest: /tmp/k7d-release.tar.gz + mode: '0644' + timeout: 120 + register: k7d_dl + retries: 5 + delay: 10 + until: k7d_dl is success + when: + - k7_has_k7d + - k7d_artifact_local_path | length == 0 + tags: ['k7d'] + + - name: K7d — extract release tarball to /opt/k7d + ansible.builtin.unarchive: + src: /tmp/k7d-release.tar.gz + dest: /opt/k7d + remote_src: true + register: k7d_extracted + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — verify release artifacts are complete + ansible.builtin.stat: + path: "/opt/k7d/k7d-v{{ k7d_version }}-x86_64-linux/{{ item }}" + register: k7d_artifact_stat + failed_when: not k7d_artifact_stat.stat.exists + loop: + - install.sh + - k7d + - containerd-shim-k7-v1 + - vmlinux + - initramfs.cpio.gz + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — check whether the k7d control socket already exists + ansible.builtin.stat: + path: /run/k7d/k7d.sock + register: k7d_socket_stat + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — run the release install.sh (binaries, guest assets, k7d.service) + ansible.builtin.command: ./install.sh + args: + chdir: "/opt/k7d/k7d-v{{ k7d_version }}-x86_64-linux" + when: + - k7_has_k7d + - k7d_extracted.changed or not k7d_socket_stat.stat.exists + changed_when: true + tags: ['k7d'] + + # The static runc rides into every k7d guest on the shim's "tools" + # erofs image; the shim looks for it at /opt/k7d/runc.amd64 (release + # install) before falling back to a host runc. A node with neither + # cannot start containers — fail here, not at first pod. + - name: K7d — check for bundled static runc in the release + ansible.builtin.stat: + path: "/opt/k7d/k7d-v{{ k7d_version }}-x86_64-linux/runc.amd64" + register: k7d_runc_bundled + when: k7_has_k7d + tags: ['k7d'] + + - name: K7d — install bundled static runc to /opt/k7d/runc.amd64 + ansible.builtin.copy: + src: "/opt/k7d/k7d-v{{ k7d_version }}-x86_64-linux/runc.amd64" + dest: /opt/k7d/runc.amd64 + mode: '0755' + remote_src: true + when: + - k7_has_k7d + - k7d_runc_bundled.stat.exists + tags: ['k7d'] + + - name: K7d — verify a static runc is available for the guest tools image + ansible.builtin.shell: | + set -eu + test -x /opt/k7d/runc.amd64 || test -x /usr/bin/runc || test -x /usr/sbin/runc + when: k7_has_k7d + changed_when: false + tags: ['k7d'] + + - name: K7d — wait for the control socket /run/k7d/k7d.sock + ansible.builtin.wait_for: + path: /run/k7d/k7d.sock + state: present + timeout: 30 + when: k7_has_k7d + tags: ['k7d'] + + # ------------------------------------------------------------------ + # Containerd template — merged based on per-host backend set + # ------------------------------------------------------------------ + - name: Render containerd config template (per-host backend mix) + ansible.builtin.copy: dest: /var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.tmpl - mode: "0644" + mode: '0644' owner: root content: | # version = 3 {{ '{{' }} template "base" . {{ '}}' }} - + + {% if k7_has_devmapper %} [plugins."io.containerd.snapshotter.v1.devmapper"] pool_name = "kata--vg-thin--pool" root_path = "/var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.devmapper" base_image_size = "10GB" - - - name: Add Kata runtime configuration to containerd template - blockinfile: - path: /var/lib/rancher/k3s/agent/etc/containerd/config-v3.toml.tmpl - marker: "# {mark} ANSIBLE MANAGED KATA RUNTIME" - block: | + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" privileged_without_host_devices = true snapshotter = "devmapper" + pod_annotations = ["io.katacontainers.*"] [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata.options] BinaryName = "/usr/local/bin/containerd-shim-kata-v2" ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration-fc.toml" - create: no - insertafter: EOF - tags: ['kata', 'containerd'] + {% endif %} + {% if k7_has_longhorn %} + [plugins."io.containerd.snapshotter.v1.overlayfs"] + mount_options = ["nodev", "nosuid"] - # ── restart k3s non-blocking and wait for the API ───────────────────────────── - - name: Restart K3s to apply devmapper changes (async) + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-qemu] + runtime_type = "io.containerd.kata.v2" + privileged_without_host_devices = true + snapshotter = "overlayfs" + pod_annotations = ["io.katacontainers.*"] + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.kata-qemu.options] + BinaryName = "/usr/local/bin/containerd-shim-kata-v2" + ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration-qemu.toml" + {% endif %} + + {% if k7_has_k7d %} + # k7d runtime: containerd resolves the shim binary from runtime_type + # (io.containerd.k7.v1 -> containerd-shim-k7-v1). Do NOT add an + # [options] BinaryName — it leaks into the cgroup path and breaks + # shim startup. pod_annotations forwards the k7d.katakate.org/* + # pod annotations (fork-source, cluster-id, image-slots, ...) into + # the sandbox OCI spec where the shim reads them. + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.k7] + runtime_type = "io.containerd.k7.v1" + pod_annotations = ["k7d.katakate.org/*"] + privileged_without_host_devices = true + snapshotter = "overlayfs" + {% endif %} + tags: ['kata', 'containerd', 'k7d'] + + # ── restart k3s and wait for the API ───────────────────────────── + - name: Restart K3s/k3s-agent to apply config changes (async) ansible.builtin.systemd: - name: k3s + name: "{{ 'k3s' if k7_node_role == 'server' else 'k3s-agent' }}" state: restarted - async: 300 # allow up to 5 min - poll: 0 # fire-and-forget - register: k3s_async - tags: ['devmapper'] + enabled: true + async: 300 + poll: 0 + tags: ['devmapper', 'k3s'] - name: Wait for K3s containerd socket to appear - wait_for: + ansible.builtin.wait_for: path: /run/k3s/containerd/containerd.sock state: present - timeout: 60 + timeout: 120 tags: ['kata'] - - name: Wait for kube-api /readyz - command: k3s kubectl get --raw /readyz + # HA note: joining servers may spend minutes as etcd learners before + # the local apiserver goes Ready (only one learner can join at a time + # when several masters restart together) — hence the generous retries. + - name: Wait for kube-api /readyz (servers only) + ansible.builtin.command: k3s kubectl get --raw /readyz register: readyz - retries: 30 + retries: 60 delay: 5 until: readyz.rc == 0 + changed_when: false + when: k7_node_role == 'server' - - name: Wait for node to be Ready and API stable - shell: | - for i in {1..30}; do - if k3s kubectl get nodes --no-headers | grep -q 'Ready'; then - echo "Node is Ready" - # Test a more complex query to ensure stability - if k3s kubectl get pods -n kube-system >/dev/null 2>&1; then - exit 0 - fi + # ------------------------------------------------------------------ + # Cilium CNI install (cluster-wide, delegated to first master) + # Runs BEFORE node-Ready gate because nodes stay NotReady until a CNI + # is installed when k3s is launched with --flannel-backend=none. + # ------------------------------------------------------------------ + - name: Check if cilium CLI is already installed + ansible.builtin.stat: + path: /usr/local/bin/cilium + register: cilium_cli_stat + when: k7_cni_is_cilium + tags: ['cilium'] + + - name: Download Cilium CLI tarball + ansible.builtin.get_url: + url: "https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version }}/cilium-linux-{{ deb_arch }}.tar.gz" + dest: /tmp/cilium-cli.tar.gz + mode: '0644' + timeout: 60 + register: cilium_cli_dl + retries: 5 + delay: 10 + until: cilium_cli_dl is success + when: + - k7_cni_is_cilium + - not cilium_cli_stat.stat.exists + tags: ['cilium'] + + - name: Extract Cilium CLI + ansible.builtin.unarchive: + src: /tmp/cilium-cli.tar.gz + dest: /usr/local/bin + remote_src: true + extra_opts: ['cilium'] + when: + - k7_cni_is_cilium + - not cilium_cli_stat.stat.exists + tags: ['cilium'] + + - name: Ensure cilium CLI is executable + ansible.builtin.file: + path: /usr/local/bin/cilium + mode: '0755' + when: k7_cni_is_cilium + tags: ['cilium'] + + - name: Check if Cilium is already installed in the cluster + ansible.builtin.command: k3s kubectl -n kube-system get daemonset cilium + register: cilium_installed_check + changed_when: false + failed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + when: k7_cni_is_cilium + tags: ['cilium'] + + # Cilium tuning notes: + # - MTU=1450: VXLAN adds 50B overhead. Cilium's auto-detect can leave + # pods at the underlying NIC MTU (1500), which breaks HTTPS through + # the encap. Pinning pod MTU to 1450 fixes this for both standard + # and Kata pods. + # - socketLB.hostNamespaceOnly=true: required for Kata-Containers + # compatibility (Cilium docs: + # https://docs.cilium.io/en/stable/network/kubernetes/kata/). With + # kubeProxyReplacement=true the socket-LB is normally used for + # ClusterIP routing, but Kata VMs run their sockets in a separate + # netns the host-cgroup BPF can't intercept, so service traffic + # (including kube-dns) stalls. Limiting socket-LB to the host + # namespace forces pod traffic onto the per-packet LB path which + # works for Kata. + # - dnsProxy.minTtl=3600: CDN-backed hosts (CloudFront et al.) return + # DNS TTLs of 30-60s, but clients like dockerd cache the resolved IP + # and keep dialing it for minutes across retries. With minTtl=0 the + # FQDN→identity mapping expires with the upstream TTL and every + # later SYN is policy-dropped (spec 18f issue 3). Keep learned IPs + # allowed for 1h after the lookup. + - name: Install Cilium (kube-proxy replacement, FQDN support) # noqa: no-changed-when + ansible.builtin.shell: | + set -eu + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + /usr/local/bin/cilium install \ + --version {{ cilium_version }} \ + --set kubeProxyReplacement=true \ + --set k8sServiceHost={{ hostvars[k7_first_master]['ansible_host'] | default(k7_first_master) }} \ + --set k8sServicePort=6443 \ + --set ipam.mode=kubernetes \ + --set operator.replicas=1 \ + --set MTU=1450 \ + --set socketLB.hostNamespaceOnly=true \ + --set dnsProxy.minTtl=3600 + args: + executable: /bin/bash + run_once: true + delegate_to: "{{ k7_first_master }}" + when: + - k7_cni_is_cilium + - cilium_installed_check.rc != 0 + tags: ['cilium'] + + - name: Wait for Cilium to report ready # noqa: no-changed-when + ansible.builtin.command: /usr/local/bin/cilium status --wait --wait-duration 5m + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + run_once: true + delegate_to: "{{ k7_first_master }}" + when: k7_cni_is_cilium + tags: ['cilium'] + + - name: Wait for cluster nodes to be Ready (run on first master) + ansible.builtin.shell: | + set -e + expected={{ (groups['k7_cluster'] | default([inventory_hostname])) | length }} + for i in $(seq 1 60); do + ready=$(k3s kubectl get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l) + if [ "$ready" -ge "$expected" ]; then + echo "All $ready/$expected nodes Ready" + exit 0 fi - echo "Waiting for node and API stability... ($i/30)" - sleep 10 + echo "Waiting for nodes Ready: $ready/$expected ($i/60)" + sleep 5 done exit 1 register: stability_check - until: stability_check.rc == 0 - retries: 1 # The loop inside handles retries - delay: 0 + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" tags: ['k3s'] - - - name: Create RuntimeClass 'kata' + # ------------------------------------------------------------------ + # Cluster-wide resources (RuntimeClass, Longhorn, k7-api) + # ------------------------------------------------------------------ + - name: Create RuntimeClass 'kata' (cluster-wide) ansible.builtin.shell: | cat <<'EOF' | k3s kubectl apply -f - apiVersion: node.k8s.io/v1 @@ -792,45 +1561,653 @@ retries: 12 delay: 5 until: "'created' in apply_kata.stdout or 'unchanged' in apply_kata.stdout" - tags: ['kata','k3s'] + changed_when: "'created' in apply_kata.stdout" + run_once: true + delegate_to: "{{ k7_first_master }}" + when: groups['k7_cluster'] | map('extract', hostvars, 'k7_backends_list') | flatten | unique | intersect(['kata-firecracker-devmapper']) | length > 0 + tags: ['kata', 'k3s'] + - name: Create RuntimeClass 'kata-qemu' (cluster-wide) + ansible.builtin.shell: | + cat <<'EOF' | k3s kubectl apply -f - + apiVersion: node.k8s.io/v1 + kind: RuntimeClass + metadata: + name: kata-qemu + handler: kata-qemu + EOF + args: + executable: /bin/bash + register: apply_kata_qemu + retries: 12 + delay: 5 + until: "'created' in apply_kata_qemu.stdout or 'unchanged' in apply_kata_qemu.stdout" + changed_when: "'created' in apply_kata_qemu.stdout" + run_once: true + delegate_to: "{{ k7_first_master }}" + when: groups['k7_cluster'] | map('extract', hostvars, 'k7_backends_list') | flatten | unique | intersect(['kata-qemu-longhorn']) | length > 0 + tags: ['kata', 'k3s'] + + - name: Create RuntimeClass 'k7' (cluster-wide) + ansible.builtin.shell: | + cat <<'EOF' | k3s kubectl apply -f - + apiVersion: node.k8s.io/v1 + kind: RuntimeClass + metadata: + name: k7 + handler: k7 + EOF + args: + executable: /bin/bash + register: apply_k7 + retries: 12 + delay: 5 + until: "'created' in apply_k7.stdout or 'unchanged' in apply_k7.stdout" + changed_when: "'created' in apply_k7.stdout" + run_once: true + delegate_to: "{{ k7_first_master }}" + when: groups['k7_cluster'] | map('extract', hostvars, 'k7_backends_list') | flatten | unique | intersect(['k7d']) | length > 0 + tags: ['k7d', 'k3s'] + + # ------------------------------------------------------------------ + # Per-node backend labels — each node labels itself via first master + # ------------------------------------------------------------------ + - name: Wait for node object to register + ansible.builtin.command: k3s kubectl get node {{ k7_k8s_node_name }} + register: node_check + retries: 30 + delay: 5 + until: node_check.rc == 0 + changed_when: false + delegate_to: "{{ k7_first_master }}" + tags: ['labels'] + + - name: Remove deprecated backend labels + ansible.builtin.command: > + k3s kubectl label node {{ k7_k8s_node_name }} + k7.katakate.org/backend-firecracker-devmapper- + k7.katakate.org/backend-qemu-longhorn- + --overwrite + delegate_to: "{{ k7_first_master }}" + changed_when: true + failed_when: false + tags: ['labels'] + + - name: Label node with supported backends + ansible.builtin.command: > + k3s kubectl label node {{ k7_k8s_node_name }} + k7.katakate.org/backend-{{ item }}=true --overwrite + loop: "{{ k7_backends_list }}" + delegate_to: "{{ k7_first_master }}" + changed_when: true + tags: ['labels'] + + - name: Label first master so the k7-api pod has a stable home + ansible.builtin.command: > + k3s kubectl label node + {{ hostvars[k7_first_master].k7_k8s_node_name | default(k7_k8s_node_name) }} + k7.katakate.org/first-master=true --overwrite + run_once: true + delegate_to: "{{ k7_first_master }}" + changed_when: true + tags: ['labels'] + + - name: Ensure K7 config directory exists + ansible.builtin.file: + path: /etc/k7 + state: directory + mode: '0755' + tags: ['k7', 'config'] + + - name: Store K7 backend configuration (primary backend = first in list) + ansible.builtin.copy: + dest: /etc/k7/backend + content: "{{ k7_backends_list[0] }}" + mode: '0644' + tags: ['k7', 'config'] - name: Show K3s node status - command: k3s kubectl get nodes -o wide + ansible.builtin.command: k3s kubectl get nodes -o wide register: k3s_nodes changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" tags: ['k3s'] - name: Debug K3s node status - debug: + ansible.builtin.debug: msg: "{{ k3s_nodes.stdout_lines }}" + run_once: true tags: ['k3s'] - - # - name: Download Helm installation script - # get_url: - # url: https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 - # dest: /tmp/get_helm_3 - # mode: '0755' - # tags: ['helm'] - # - name: Install Helm using script - # command: /tmp/get_helm_3 --version {{ helm_version }} - # args: - # creates: /usr/local/bin/helm - # environment: - # USE_SUDO: "false" - # HELM_INSTALL_DIR: /usr/local/bin - # changed_when: "'Helm is already installed' not in helm_install_result.stdout" - # register: helm_install_result - # tags: ['helm'] + # ── Longhorn installation (cluster-wide; needs >=1 kata-qemu-longhorn node) ──── + - name: Set fact for any longhorn-capable node in cluster + ansible.builtin.set_fact: + cluster_has_longhorn: >- + {{ + groups['k7_cluster'] + | map('extract', hostvars, 'k7_backends_list') + | flatten | unique + | intersect(['kata-qemu-longhorn']) + | length > 0 + }} + + - name: Download Longhorn manifest + ansible.builtin.get_url: + url: https://raw.githubusercontent.com/longhorn/longhorn/{{ longhorn_version }}/deploy/longhorn.yaml + dest: /tmp/longhorn.yaml + mode: '0644' + timeout: 60 + register: longhorn_dl + retries: 5 + delay: 10 + until: longhorn_dl is success + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn'] + + - name: Patch Longhorn manifest with custom replica count + ansible.builtin.replace: + path: /tmp/longhorn.yaml + regexp: 'numberOfReplicas: "3"' + replace: 'numberOfReplicas: "{{ longhorn_replicas_effective }}"' + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn'] + + - name: Install Longhorn + ansible.builtin.command: k3s kubectl apply -f /tmp/longhorn.yaml + register: longhorn_apply + changed_when: "'created' in longhorn_apply.stdout or 'configured' in longhorn_apply.stdout" + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn'] + + - name: Install CSI VolumeSnapshot CRDs (external-snapshotter) + ansible.builtin.shell: | + set -eu + k3s kubectl apply -k https://github.com/kubernetes-csi/external-snapshotter//client/config/crd?ref={{ snapshotter_version }} + changed_when: true + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn', 'snapshot'] + + - name: Install snapshot-controller (external-snapshotter) + ansible.builtin.shell: | + set -eu + k3s kubectl apply -k https://github.com/kubernetes-csi/external-snapshotter//deploy/kubernetes/snapshot-controller?ref={{ snapshotter_version }} + changed_when: true + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn', 'snapshot'] + + - name: Verify VolumeSnapshot CRDs exist + ansible.builtin.shell: | + set -eu + k3s kubectl get crd volumesnapshots.snapshot.storage.k8s.io + k3s kubectl get crd volumesnapshotcontents.snapshot.storage.k8s.io + k3s kubectl get crd volumesnapshotclasses.snapshot.storage.k8s.io + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn', 'snapshot'] + + - name: Wait for snapshot-controller to be ready + ansible.builtin.shell: | + set -eu + k3s kubectl -n kube-system rollout status deployment/snapshot-controller --timeout=120s + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn', 'snapshot'] + + - name: Ensure Longhorn data path exists on each longhorn-capable host + ansible.builtin.file: + path: "{{ longhorn_data_path | default('/var/lib/longhorn') }}" + state: directory + mode: '0755' + when: k7_has_longhorn + tags: ['longhorn'] + + - name: Ensure Longhorn VolumeSnapshotClass exists + ansible.builtin.shell: | + set -eu + cat <<'EOF' | k3s kubectl apply -f - + apiVersion: snapshot.storage.k8s.io/v1 + kind: VolumeSnapshotClass + metadata: + name: longhorn + driver: driver.longhorn.io + deletionPolicy: Delete + parameters: + type: snap + EOF + changed_when: true + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn', 'snapshot'] + + - name: Wait for Longhorn manager to be ready + ansible.builtin.command: k3s kubectl -n longhorn-system rollout status daemonset/longhorn-manager --timeout=300s + retries: 5 + delay: 15 + register: longhorn_manager_status + until: longhorn_manager_status.rc == 0 + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn'] + + - name: Wait for Longhorn UI deployment to be ready + ansible.builtin.command: k3s kubectl -n longhorn-system rollout status deployment/longhorn-ui --timeout=300s + retries: 5 + delay: 15 + register: longhorn_ui_status + until: longhorn_ui_status.rc == 0 + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn'] + + - name: Wait for Longhorn StorageClass to be created + ansible.builtin.command: k3s kubectl get storageclass longhorn + retries: 20 + delay: 10 + register: longhorn_sc_check + until: longhorn_sc_check.rc == 0 + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn'] + + # Replace the upstream StorageClass with a topology-aware version: + # - WaitForFirstConsumer so PVC binding follows the pod's node + # - dataLocality=best-effort to keep one replica co-located with the pod + # The Longhorn `longhorn-storageclass` ConfigMap is the source of truth — + # longhorn-driver-deployer creates the StorageClass from it at startup. We + # patch the ConfigMap, delete the existing SC, then force the deployer to + # recreate it (volumeBindingMode and parameters are immutable on a SC). + - name: Patch longhorn-storageclass ConfigMap with topology-aware StorageClass # noqa: no-changed-when + ansible.builtin.shell: | + set -eu + cat </dev/null || echo "") + [ "$mode" = "WaitForFirstConsumer" ] + register: sc_topology_check + retries: 30 + delay: 5 + until: sc_topology_check.rc == 0 + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + when: cluster_has_longhorn + tags: ['longhorn', 'topology'] + + - name: Set Longhorn replica auto-balance to best-effort # noqa: no-changed-when + ansible.builtin.shell: | + set -eu + cat < 0 + tags: ['longhorn'] + + - name: Register extra disk with Longhorn (per-host) + ansible.builtin.shell: | + set -eu + cat < 0 + tags: ['longhorn'] + + # ── K7 API deployment into K3s ───────────────────────────────────── + # The CLI passes `k7_repo_root` as an extra-var pointing to the on-controller + # source tree (so the CLI can be invoked from anywhere). Fallback to the + # playbook-relative path for legacy callers that still bundle the playbook + # alongside the source tree. + - name: Resolve k7 repo root for API build + ansible.builtin.set_fact: + k7_repo_root_resolved: "{{ k7_repo_root | default(playbook_dir + '/../../..') }}" + when: k7_api_enabled | default(true) | bool + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Verify k7 repo root contains the API Dockerfile + ansible.builtin.stat: + path: "{{ k7_repo_root_resolved }}/src/k7/api/Dockerfile.api" + register: api_dockerfile_stat + when: k7_api_enabled | default(true) | bool + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Fail when API Dockerfile is missing from the resolved repo root + ansible.builtin.fail: + msg: >- + Resolved k7_repo_root='{{ k7_repo_root_resolved }}' on + {{ k7_first_master }} does not contain src/k7/api/Dockerfile.api. + Pass `k7_repo_root=/abs/path/to/repo` as an extra-var pointing to a + rsynced copy of the k7 source tree on that host. + when: + - k7_api_enabled | default(true) | bool + - not api_dockerfile_stat.stat.exists + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Build k7-api Docker image locally + # ``--network=host`` works around a recurring DNS flake on Hetzner nodes + # where the default Docker bridge can't resolve pypi.org / registry-1 + # mid-build. Host networking shares the node's already-working resolver + # (no functional downside for a build that just fetches Python packages). + ansible.builtin.command: docker build --network=host -f src/k7/api/Dockerfile.api -t k7-api:local . + args: + chdir: "{{ k7_repo_root_resolved }}" + when: k7_api_enabled | default(true) | bool + register: api_build + changed_when: "'Successfully built' in api_build.stdout or 'exporting to image' in api_build.stdout or api_build.rc == 0" + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Export k7-api image to tarball # noqa: no-changed-when command-instead-of-shell + ansible.builtin.command: docker save k7-api:local -o /tmp/k7-api-local.tar + when: k7_api_enabled | default(true) | bool + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Fetch k7-api tarball from first master to controller + ansible.builtin.fetch: + src: /tmp/k7-api-local.tar + dest: /tmp/k7-api-local.tar + flat: true + when: k7_api_enabled | default(true) | bool + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Distribute k7-api tarball to every node (imagePullPolicy=Never) + ansible.builtin.copy: + src: /tmp/k7-api-local.tar + dest: /tmp/k7-api-local.tar + mode: '0644' + when: k7_api_enabled | default(true) | bool + tags: ['k7', 'api'] + + - name: Import k7-api image into k3s containerd on every node # noqa: no-changed-when + ansible.builtin.command: k3s ctr images import /tmp/k7-api-local.tar + when: k7_api_enabled | default(true) | bool + tags: ['k7', 'api'] + + - name: Clean up image tarball on nodes + ansible.builtin.file: + path: /tmp/k7-api-local.tar + state: absent + when: k7_api_enabled | default(true) | bool + tags: ['k7', 'api'] + + # Spec 18g: shared token authenticating k7-api → k7-agent (and root CLI → + # agent) traffic. Generated once on the first master, distributed to + # /etc/k7/agent_token (0600) on every node; both the API deployment and + # the agent DaemonSet read it through the /etc/k7 hostPath mount. + - name: Generate shared k7 agent token on first master # noqa: command-instead-of-shell + ansible.builtin.shell: umask 077 && openssl rand -hex 32 > /etc/k7/agent_token + args: + creates: /etc/k7/agent_token + when: k7_api_enabled | default(true) | bool + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Read agent token from first master + ansible.builtin.slurp: + src: /etc/k7/agent_token + register: k7_agent_token_slurp + no_log: true + when: k7_api_enabled | default(true) | bool + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Distribute agent token to every node + ansible.builtin.copy: + content: "{{ k7_agent_token_slurp.content | b64decode }}" + dest: /etc/k7/agent_token + owner: root + group: root + mode: '0600' + no_log: true + when: k7_api_enabled | default(true) | bool + tags: ['k7', 'api'] + + # The manifests live at src/k7/deploy/manifests/k7-api/. We use remote_src + # because the source tree is already rsynced to the first master at + # k7_repo_root_resolved (the playbook itself runs from a tempfile, so the + # default `playbook_dir`-relative lookup is wrong). + - name: Copy K7 API manifests on first master + ansible.builtin.copy: + src: "{{ k7_repo_root_resolved }}/src/k7/deploy/manifests/k7-api/" + dest: /etc/k7/manifests/k7-api/ + mode: '0644' + remote_src: true + when: k7_api_enabled | default(true) | bool + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Apply K7 API manifests + ansible.builtin.command: k3s kubectl apply -f /etc/k7/manifests/k7-api/ + when: k7_api_enabled | default(true) | bool + register: api_apply + changed_when: "'created' in api_apply.stdout or 'configured' in api_apply.stdout" + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Wait for K7 API deployment to be ready + ansible.builtin.command: k3s kubectl rollout status deployment/k7-api -n kube-system --timeout=120s + when: k7_api_enabled | default(true) | bool + register: api_rollout + retries: 3 + delay: 10 + until: api_rollout.rc == 0 + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Wait for k7-agent DaemonSet rollout (spec 18g) + ansible.builtin.command: k3s kubectl rollout status daemonset/k7-agent -n kube-system --timeout=180s + when: k7_api_enabled | default(true) | bool + register: agent_rollout + retries: 3 + delay: 10 + until: agent_rollout.rc == 0 + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Get K7 API NodePort + ansible.builtin.command: > + k3s kubectl get svc k7-api -n kube-system + -o jsonpath='{.spec.ports[0].nodePort}' + when: k7_api_enabled | default(true) | bool + register: api_nodeport + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Get node IPv4 address for API endpoint + ansible.builtin.shell: > + k3s kubectl get nodes -o json | + python3 -c "import sys,json; + addrs=json.load(sys.stdin)['items'][0]['status']['addresses']; + print(next(a['address'] for a in addrs if a['type']=='InternalIP' and ':' not in a['address']))" + when: k7_api_enabled | default(true) | bool + register: node_ip + changed_when: false + run_once: true + delegate_to: "{{ k7_first_master }}" + tags: ['k7', 'api'] + + - name: Store API endpoint in /etc/k7/api_endpoint on every host + ansible.builtin.copy: + dest: /etc/k7/api_endpoint + content: "http://{{ hostvars[k7_first_master]['node_ip']['stdout'] }}:{{ hostvars[k7_first_master]['api_nodeport']['stdout'] }}" + mode: '0644' + when: k7_api_enabled | default(true) | bool + tags: ['k7', 'api'] - name: Reminder about logging out and back in for group changes - debug: - msg: "IMPORTANT: User '{{ target_user }}' was added to 'docker' and/or 'kvm' groups. You (or the user) will need to log out and log back in on the server for these group changes to take full effect in their shell sessions." + ansible.builtin.debug: + msg: >- + IMPORTANT: User '{{ target_user }}' was added to 'docker' and/or 'kvm' groups. + Log out and back in on the server for group changes to take effect. tags: ['info'] - handlers: - name: Load xt_mark ansible.builtin.command: modprobe xt_mark - + changed_when: false diff --git a/src/k7/deploy/manifests/k7-api/agent-daemonset.yaml b/src/k7/deploy/manifests/k7-api/agent-daemonset.yaml new file mode 100644 index 0000000..a13c658 --- /dev/null +++ b/src/k7/deploy/manifests/k7-api/agent-daemonset.yaml @@ -0,0 +1,114 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: k7-agent + namespace: kube-system + labels: + app: k7-agent +spec: + selector: + matchLabels: + app: k7-agent + template: + metadata: + labels: + app: k7-agent + spec: + # Reuses the k7-api ServiceAccount/RBAC: the agent runs the same + # K7Core code (pod lookup, deployment annotate/create for + # pause/resume/fork). + serviceAccountName: k7-api + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: k7-agent + image: k7-api:local + imagePullPolicy: Never + # Same image as k7-api, different app: only the node-local VM ops + # and the storage endpoint (spec 18g). + command: ["uvicorn", "k7.api.agent:app", "--host", "0.0.0.0", "--port", "8000"] + # Privileged: `lvs` (kfd thin-pool utilization) issues device-mapper + # ioctls and reads raw block devices under the hostPath-mounted + # /dev — the default device cgroup denies that to unprivileged + # containers. Root (the image defaults to uid 1000) is required for + # crictl, the k7d socket, and the 0600 root-owned agent token. + securityContext: + privileged: true + runAsUser: 0 + runAsGroup: 0 + ports: + - containerPort: 8000 + protocol: TCP + env: + - name: K7_AGENT + value: "1" + - name: K7_AGENT_TOKEN_FILE + value: /etc/k7/agent_token + - name: K7_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + # Shared agent token (playbook writes it on every node). + - name: k7-config + mountPath: /etc/k7 + readOnly: true + # k7d daemon control socket (VM pause/resume/fork/lookup). + - name: k7d-run + mountPath: /run/k7d + # k3s containerd socket for crictl pod → CRI sandbox id lookup. + - name: k3s-containerd + mountPath: /run/k3s/containerd + # k7d disks pool (XFS loopback mount) for `df` utilization. + - name: k7d-lib + mountPath: /var/lib/k7d + readOnly: true + # Raw devices for `lvs` (kfd thin-pool utilization). + - name: dev + mountPath: /dev + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: 50m + memory: 96Mi + limits: + cpu: 500m + memory: 256Mi + volumes: + - name: k7-config + hostPath: + path: /etc/k7 + type: DirectoryOrCreate + - name: k7d-run + hostPath: + path: /run/k7d + type: DirectoryOrCreate + - name: k3s-containerd + hostPath: + path: /run/k3s/containerd + type: Directory + - name: k7d-lib + hostPath: + path: /var/lib/k7d + type: DirectoryOrCreate + - name: dev + hostPath: + path: /dev + type: Directory diff --git a/src/k7/deploy/manifests/k7-api/agent-networkpolicy.yaml b/src/k7/deploy/manifests/k7-api/agent-networkpolicy.yaml new file mode 100644 index 0000000..b968ba8 --- /dev/null +++ b/src/k7/deploy/manifests/k7-api/agent-networkpolicy.yaml @@ -0,0 +1,22 @@ +# Spec 18g: restrict pod-originated ingress to the k7-agent DaemonSet to +# the k7-api pod only. `host` / `remote-node` entities stay allowed so +# kubelet health probes and root CLI usage on cluster nodes keep working — +# those callers still need the shared token (/etc/k7/agent_token, root +# 0600), which is the actual authentication. Sandbox pods and any other +# workload pods are denied at the network layer. +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: k7-agent-ingress + namespace: kube-system +spec: + endpointSelector: + matchLabels: + app: k7-agent + ingress: + - fromEndpoints: + - matchLabels: + app: k7-api + - fromEntities: + - host + - remote-node diff --git a/src/k7/deploy/manifests/k7-api/clusterrole.yaml b/src/k7/deploy/manifests/k7-api/clusterrole.yaml new file mode 100644 index 0000000..41804d4 --- /dev/null +++ b/src/k7/deploy/manifests/k7-api/clusterrole.yaml @@ -0,0 +1,39 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: k7-api + labels: + app: k7-api +rules: + - apiGroups: [""] + resources: ["pods", "pods/exec", "pods/log", "secrets", "configmaps", "persistentvolumeclaims", "namespaces", "nodes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["apps"] + resources: ["deployments", "deployments/scale"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # FQDN egress (Spec 4a) is enforced via CiliumNetworkPolicy, so the API + # must be able to manage them to apply/delete sandbox egress rules. + - apiGroups: ["cilium.io"] + resources: ["ciliumnetworkpolicies"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots", "volumesnapshotcontents", "volumesnapshotclasses"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["metrics.k8s.io"] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["get", "list", "watch"] + # FQDN egress support probes for the CiliumNetworkPolicy CRD + # (core._cilium_available) before applying a CNP — without this, sandbox + # creation with domain egress fails only when driven through the API pod. + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get"] diff --git a/src/k7/deploy/manifests/k7-api/clusterrolebinding.yaml b/src/k7/deploy/manifests/k7-api/clusterrolebinding.yaml new file mode 100644 index 0000000..5ae1be4 --- /dev/null +++ b/src/k7/deploy/manifests/k7-api/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: k7-api + labels: + app: k7-api +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k7-api +subjects: + - kind: ServiceAccount + name: k7-api + namespace: kube-system diff --git a/src/k7/deploy/manifests/k7-api/deployment.yaml b/src/k7/deploy/manifests/k7-api/deployment.yaml new file mode 100644 index 0000000..8004c52 --- /dev/null +++ b/src/k7/deploy/manifests/k7-api/deployment.yaml @@ -0,0 +1,105 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k7-api + namespace: kube-system + labels: + app: k7-api +spec: + replicas: 1 + selector: + matchLabels: + app: k7-api + template: + metadata: + labels: + app: k7-api + spec: + serviceAccountName: k7-api + # /etc/k7 is hostPath-mounted (api_keys.json lives there). Pin the API + # pod to the first master so single-node and multi-node clusters both + # have a predictable, stable location for the host-side key store. + # The first master is labelled `k7.katakate.org/first-master=true` by + # the install playbook. + nodeSelector: + k7.katakate.org/first-master: "true" + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: k7-api + image: k7-api:local + imagePullPolicy: Never + # Root is required for the co-located k7d VM-op fallback (spec 18f + # issue 2): `crictl` must reach the k3s containerd socket (0660 + # root:root) to resolve a pod's CRI sandbox id before talking to + # the k7d daemon socket. No added capabilities / not privileged. + # k7d VM ops only work for sandboxes on THIS node (the first + # master) — core fails loudly on a node mismatch; see + # docs/BACKENDS.md. + securityContext: + runAsUser: 0 + runAsGroup: 0 + allowPrivilegeEscalation: false + ports: + - containerPort: 8000 + protocol: TCP + env: + - name: K7_API_KEYS_FILE + value: /etc/k7/api_keys.json + # k7d VM operations (pause/fork) are node-local; core compares + # the sandbox pod's node against this (os.uname() inside a + # container returns the pod name, which never matches). + - name: K7_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: k7-config + mountPath: /etc/k7 + # k7d daemon control socket (VM pause/resume/fork). Present only + # on nodes provisioned with the k7d backend; DirectoryOrCreate + # keeps the pod schedulable on clusters without it (core then + # fails loudly with "socket not found"). + - name: k7d-run + mountPath: /run/k7d + # k3s containerd socket for crictl pod → CRI sandbox id lookup. + - name: k3s-containerd + mountPath: /run/k3s/containerd + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 512Mi + volumes: + - name: k7-config + hostPath: + path: /etc/k7 + type: DirectoryOrCreate + - name: k7d-run + hostPath: + path: /run/k7d + type: DirectoryOrCreate + - name: k3s-containerd + hostPath: + path: /run/k3s/containerd + type: Directory diff --git a/src/k7/deploy/manifests/k7-api/service.yaml b/src/k7/deploy/manifests/k7-api/service.yaml new file mode 100644 index 0000000..0488c5e --- /dev/null +++ b/src/k7/deploy/manifests/k7-api/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: k7-api + namespace: kube-system + labels: + app: k7-api +spec: + type: NodePort + selector: + app: k7-api + ports: + - port: 8000 + targetPort: 8000 + protocol: TCP + nodePort: 31007 diff --git a/src/k7/deploy/manifests/k7-api/serviceaccount.yaml b/src/k7/deploy/manifests/k7-api/serviceaccount.yaml new file mode 100644 index 0000000..c7fcf70 --- /dev/null +++ b/src/k7/deploy/manifests/k7-api/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k7-api + namespace: kube-system + labels: + app: k7-api diff --git a/src/k7/deploy/manifests/k7-api/snapshot-gc-cronjob.yaml b/src/k7/deploy/manifests/k7-api/snapshot-gc-cronjob.yaml new file mode 100644 index 0000000..a6f378e --- /dev/null +++ b/src/k7/deploy/manifests/k7-api/snapshot-gc-cronjob.yaml @@ -0,0 +1,47 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: k7-snapshot-gc + namespace: kube-system + labels: + app: k7-snapshot-gc +spec: + # Sweep every 10 minutes — matches K7Core.gc_snapshots' default keep window. + schedule: "*/10 * * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 2 + ttlSecondsAfterFinished: 600 + template: + metadata: + labels: + app: k7-snapshot-gc + spec: + serviceAccountName: k7-api + restartPolicy: OnFailure + nodeSelector: + k7.katakate.org/first-master: "true" + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: gc + image: k7-api:local + imagePullPolicy: Never + command: ["python", "-m", "k7.api.snapshot_gc"] + env: + - name: K7_GC_KEEP_FORK_FOR_MINUTES + value: "10" + - name: K7_GC_DRY_RUN + value: "false" + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: "500m" + memory: 256Mi diff --git a/src/k7_sdk/__init__.py b/src/k7_sdk/__init__.py new file mode 100644 index 0000000..db7b3eb --- /dev/null +++ b/src/k7_sdk/__init__.py @@ -0,0 +1,11 @@ +"""K7 Python SDK — HTTP client for the k7 API.""" + +from .client import AsyncClient, Client, SandboxProxy + +__all__ = [ + "Client", + "AsyncClient", + "SandboxProxy", +] + +__version__ = "0.2.0" diff --git a/src/k7_sdk/client.py b/src/k7_sdk/client.py new file mode 100644 index 0000000..51ecaab --- /dev/null +++ b/src/k7_sdk/client.py @@ -0,0 +1,550 @@ +from __future__ import annotations + +import requests + +try: + import httpx # optional dependency for async client +except Exception: # pragma: no cover + httpx = None + + +class SandboxProxy: + """Proxy object for sandbox operations.""" + + def __init__(self, name: str, namespace: str, client: Client): + self.name = name + self.namespace = namespace + self._client = client + + def exec(self, code: str) -> dict: + """Execute code in the sandbox.""" + return self._client._exec_command(self.name, code, self.namespace) + + def delete(self) -> dict: + """Delete this sandbox.""" + return self._client.delete(self.name, self.namespace) + + def pause(self, snapshot: str | None = None) -> dict: + """Pause this sandbox (scale to 0), optionally with a VolumeSnapshot. + + Pass ``snapshot="my-snap"`` to create a crash-consistent + ``VolumeSnapshot`` of the sandbox's root PVC before scaling to 0. + """ + return self._client.pause(self.name, namespace=self.namespace, snapshot=snapshot) + + def resume(self) -> dict: + """Resume this sandbox (scale back to 1).""" + return self._client.resume(self.name, namespace=self.namespace) + + def fork(self, new_name: str, snapshot: str | None = None) -> SandboxProxy: + """Fork this sandbox into ``new_name``; returns a proxy for the new sandbox. + + Blocks until the cloned PVC is bound (the server-side `fork_sandbox` + waits for the new pod to schedule before returning). + """ + return self._client.fork(self.name, new_name, namespace=self.namespace, snapshot=snapshot) + + def snapshot(self, snapshot_name: str) -> dict: + """Snapshot this sandbox's root PVC without pausing it (kind=named).""" + return self._client.create_snapshot(self.name, snapshot_name, namespace=self.namespace) + + def logs(self, tail: int = 200, container: str = "sandbox", since: int = 0) -> str: + """Return a snapshot of this sandbox's pod logs.""" + return self._client.logs(self.name, namespace=self.namespace, container=container, tail=tail, since=since) + + +class Client: + """K7 Python SDK Client.""" + + def __init__(self, endpoint: str, api_key: str, verify_ssl: bool = True): + self.base_url = endpoint.rstrip("/") + self.api_key = api_key + self.session = requests.Session() + self.session.headers.update({"X-API-Key": api_key}) + self.session.verify = verify_ssl + + def _unwrap(self, response) -> dict: + data = response.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + def create(self, sandbox_config: dict) -> SandboxProxy: + """Create a new sandbox and return a proxy object.""" + response = self.session.post(f"{self.base_url}/api/v1/sandboxes", json=sandbox_config) + response.raise_for_status() + + name = sandbox_config.get("name") + namespace = sandbox_config.get("namespace", "default") + + return SandboxProxy(name, namespace, self) + + def list(self, namespace: str | None = None) -> list[dict]: + """List all sandboxes.""" + params = {"namespace": namespace} if namespace else {} + response = self.session.get(f"{self.base_url}/api/v1/sandboxes", params=params) + response.raise_for_status() + return self._unwrap(response) + + def delete(self, name: str, namespace: str = "default") -> dict: + """Delete a sandbox.""" + response = self.session.delete(f"{self.base_url}/api/v1/sandboxes/{name}", params={"namespace": namespace}) + response.raise_for_status() + return self._unwrap(response) + + def delete_all(self, namespace: str = "default") -> dict: + """Delete all sandboxes in a namespace.""" + response = self.session.delete(f"{self.base_url}/api/v1/sandboxes", params={"namespace": namespace}) + response.raise_for_status() + return self._unwrap(response) + + def install( + self, + playbook: str | None = None, + inventory: str | None = None, + verbose: bool = False, + ) -> dict: + """Install K7 on target hosts.""" + response = self.session.post( + f"{self.base_url}/api/v1/install", + json={"playbook": playbook, "inventory": inventory, "verbose": verbose}, + ) + response.raise_for_status() + return self._unwrap(response) + + def get_metrics(self, namespace: str | None = None) -> dict: + """Get resource usage metrics for sandboxes.""" + params = {"namespace": namespace} if namespace else {} + response = self.session.get(f"{self.base_url}/api/v1/sandboxes/metrics", params=params) + response.raise_for_status() + return self._unwrap(response) + + def nodes_storage(self) -> dict: + """Per-node storage-pool utilization (kfd thin-pool + k7d disks). + + Returns a map of node name → ``{kata_thinpool, k7d_disks}`` (or + ``{error: ...}`` when that node's agent is unreachable). + """ + response = self.session.get(f"{self.base_url}/api/v1/nodes/storage", timeout=120) + response.raise_for_status() + return self._unwrap(response) + + def pause( + self, + name: str, + namespace: str = "default", + snapshot: str | None = None, + ) -> dict: + """Pause a sandbox (scale to 0), optionally taking a Longhorn VolumeSnapshot. + + ``snapshot``, when set, names a crash-consistent VolumeSnapshot taken + of the sandbox's root PVC. The PVC name and ``VolumeSnapshotClass`` + are derived server-side (kata-qemu-longhorn convention + the playbook's + ``longhorn`` class). + """ + body: dict = {"namespace": namespace} + if snapshot is not None: + body["snapshot"] = snapshot + response = self.session.post(f"{self.base_url}/api/v1/sandboxes/{name}/pause", json=body, timeout=120) + response.raise_for_status() + return self._unwrap(response) + + def resume(self, name: str, namespace: str = "default") -> dict: + """Resume a paused sandbox (scale back to 1).""" + response = self.session.post( + f"{self.base_url}/api/v1/sandboxes/{name}/resume", + json={"namespace": namespace}, + timeout=30, + ) + response.raise_for_status() + return self._unwrap(response) + + def fork( + self, + source: str, + new_name: str, + namespace: str = "default", + snapshot: str | None = None, + ) -> SandboxProxy: + """Fork ``source`` into a new sandbox ``new_name``; returns a proxy for it. + + Blocks until the cloned PVC is bound. Today this takes ~45s for + kata-qemu-longhorn; the HTTP request stays open for the duration. + """ + body: dict = {"new_name": new_name, "namespace": namespace} + if snapshot is not None: + body["snapshot"] = snapshot + response = self.session.post(f"{self.base_url}/api/v1/sandboxes/{source}/fork", json=body, timeout=600) + response.raise_for_status() + self._unwrap(response) + return SandboxProxy(new_name, namespace, self) + + # ------------------------------------------------------------------ + # Spec 10e: VolumeSnapshot CRUD + GC. + # ------------------------------------------------------------------ + + def list_snapshots( + self, + namespace: str = "default", + all_namespaces: bool = False, + sandbox: str | None = None, + kind: str | None = None, + ) -> list[dict]: + params: dict = {"namespace": namespace, "all_namespaces": str(all_namespaces).lower()} + if sandbox is not None: + params["sandbox"] = sandbox + if kind is not None: + params["kind"] = kind + response = self.session.get(f"{self.base_url}/api/v1/snapshots", params=params, timeout=30) + response.raise_for_status() + return self._unwrap(response) + + def get_snapshot(self, name: str, namespace: str = "default") -> dict | None: + response = self.session.get( + f"{self.base_url}/api/v1/snapshots/{name}", params={"namespace": namespace}, timeout=15 + ) + if response.status_code == 404: + return None + response.raise_for_status() + return self._unwrap(response) + + def create_snapshot(self, sandbox: str, snapshot_name: str, namespace: str = "default") -> dict: + response = self.session.post( + f"{self.base_url}/api/v1/sandboxes/{sandbox}/snapshot", + json={"snapshot_name": snapshot_name, "namespace": namespace}, + timeout=120, + ) + response.raise_for_status() + return self._unwrap(response) + + def delete_snapshot(self, name: str, namespace: str = "default") -> dict: + response = self.session.delete( + f"{self.base_url}/api/v1/snapshots/{name}", params={"namespace": namespace}, timeout=60 + ) + response.raise_for_status() + return self._unwrap(response) + + def gc_snapshots( + self, + namespace: str = "default", + all_namespaces: bool = False, + keep_fork_for: str = "10m", + dry_run: bool = False, + ) -> dict: + body: dict = { + "namespace": namespace, + "all_namespaces": all_namespaces, + "keep_fork_for": keep_fork_for, + "dry_run": dry_run, + } + response = self.session.post(f"{self.base_url}/api/v1/snapshots/gc", json=body, timeout=120) + response.raise_for_status() + return self._unwrap(response) + + def restore( + self, + snapshot_name: str, + new_sandbox_name: str, + namespace: str = "default", + overrides: dict | None = None, + keep_snapshot: bool = True, + ) -> SandboxProxy: + """Restore a brand-new sandbox from a standalone VolumeSnapshot (Spec 10f). + + ``overrides`` is a JSON-serialisable dict matching ``SandboxConfigOverrides`` + on the server (keys: ``image``, ``backend``, ``root_disk_size``, ``sidecar``, + ``limits``, ``entrypoint``, ``cmd``, ``before_script``). Pass at minimum + ``{"image": "..."}`` if the snapshot was created before Spec 10f and + therefore lacks the ``k7.io/source-image`` annotation. + + Returns a ``SandboxProxy`` for the new sandbox. The server-side restore + waits for the cloned PVC to be Bound and the Deployment to be Ready + before responding, so the proxy is safe to ``exec`` against immediately. + """ + body: dict = { + "new_sandbox_name": new_sandbox_name, + "namespace": namespace, + "keep_snapshot": keep_snapshot, + } + if overrides: + body["overrides"] = overrides + response = self.session.post( + f"{self.base_url}/api/v1/snapshots/{snapshot_name}/restore", + json=body, + timeout=600, + ) + response.raise_for_status() + self._unwrap(response) + return SandboxProxy(new_sandbox_name, namespace, self) + + def exec(self, name: str, command: str, namespace: str = "default") -> dict: + """Execute a shell command in a sandbox; returns ``{stdout, stderr, exit_code, duration_ms}``.""" + return self._exec_command(name, command, namespace) + + def logs( + self, + name: str, + namespace: str = "default", + container: str = "sandbox", + tail: int = 200, + since: int = 0, + ) -> str: + """Return a snapshot of the sandbox pod's logs (no streaming yet). + + For interactive follow today, use ``k7 --core logs --follow`` on + the node. Streaming support is a separate spec. + """ + params: dict = {"namespace": namespace, "container": container, "tail": tail} + if since > 0: + params["since"] = since + response = self.session.get( + f"{self.base_url}/api/v1/sandboxes/{name}/logs", + params=params, + timeout=60, + ) + response.raise_for_status() + data = self._unwrap(response) + if isinstance(data, dict): + return str(data.get("logs", "")) + return str(data) + + def _exec_command(self, name: str, command: str, namespace: str) -> dict: + """Internal method to execute command in sandbox.""" + response = self.session.post( + f"{self.base_url}/api/v1/sandboxes/{name}/exec", + json={"command": command}, + params={"namespace": namespace}, + ) + response.raise_for_status() + return self._unwrap(response) + + +class AsyncClient: + """K7 Python SDK Async Client.""" + + def __init__( + self, + endpoint: str, + api_key: str, + verify_ssl: bool = True, + timeout: float = 30.0, + ): + if httpx is None: + raise RuntimeError("httpx is required for AsyncClient. Install with `pip install httpx`.") + self.base_url = endpoint.rstrip("/") + self._client = httpx.AsyncClient( + base_url=self.base_url, + headers={"X-API-Key": api_key}, + verify=verify_ssl, + timeout=timeout, + ) + + async def create(self, sandbox_config: dict) -> dict: + r = await self._client.post("/api/v1/sandboxes", json=sandbox_config) + r.raise_for_status() + return r.json() + + async def list(self, namespace: str | None = None) -> list[dict]: + params = {"namespace": namespace} if namespace else {} + r = await self._client.get("/api/v1/sandboxes", params=params) + r.raise_for_status() + data = r.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + async def delete(self, name: str, namespace: str = "default") -> dict: + r = await self._client.delete(f"/api/v1/sandboxes/{name}", params={"namespace": namespace}) + r.raise_for_status() + data = r.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + async def delete_all(self, namespace: str = "default") -> dict: + r = await self._client.delete("/api/v1/sandboxes", params={"namespace": namespace}) + r.raise_for_status() + data = r.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + async def logs( + self, + name: str, + namespace: str = "default", + container: str = "sandbox", + tail: int = 200, + since: int = 0, + ) -> str: + params: dict = {"namespace": namespace, "container": container, "tail": tail} + if since > 0: + params["since"] = since + r = await self._client.get(f"/api/v1/sandboxes/{name}/logs", params=params, timeout=60) + r.raise_for_status() + data = r.json() + unwrapped = data["data"] if isinstance(data, dict) and "data" in data else data + if isinstance(unwrapped, dict): + return str(unwrapped.get("logs", "")) + return str(unwrapped) + + async def exec(self, name: str, command: str, namespace: str = "default") -> dict: + r = await self._client.post( + f"/api/v1/sandboxes/{name}/exec", + json={"command": command}, + params={"namespace": namespace}, + ) + r.raise_for_status() + data = r.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + async def get_metrics(self, namespace: str | None = None) -> dict: + params = {"namespace": namespace} if namespace else {} + r = await self._client.get("/api/v1/sandboxes/metrics", params=params) + r.raise_for_status() + data = r.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + async def nodes_storage(self) -> dict: + """Per-node storage-pool utilization (kfd thin-pool + k7d disks).""" + r = await self._client.get("/api/v1/nodes/storage", timeout=120) + r.raise_for_status() + data = r.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + async def pause( + self, + name: str, + namespace: str = "default", + snapshot: str | None = None, + ) -> dict: + body: dict = {"namespace": namespace} + if snapshot is not None: + body["snapshot"] = snapshot + r = await self._client.post(f"/api/v1/sandboxes/{name}/pause", json=body, timeout=120) + r.raise_for_status() + data = r.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + async def resume(self, name: str, namespace: str = "default") -> dict: + r = await self._client.post( + f"/api/v1/sandboxes/{name}/resume", + json={"namespace": namespace}, + timeout=30, + ) + r.raise_for_status() + data = r.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + async def fork( + self, + source: str, + new_name: str, + namespace: str = "default", + snapshot: str | None = None, + ) -> dict: + body: dict = {"new_name": new_name, "namespace": namespace} + if snapshot is not None: + body["snapshot"] = snapshot + r = await self._client.post(f"/api/v1/sandboxes/{source}/fork", json=body, timeout=600) + r.raise_for_status() + data = r.json() + if isinstance(data, dict) and "data" in data: + return data["data"] + return data + + async def list_snapshots( + self, + namespace: str = "default", + all_namespaces: bool = False, + sandbox: str | None = None, + kind: str | None = None, + ) -> list[dict]: + params: dict = {"namespace": namespace, "all_namespaces": str(all_namespaces).lower()} + if sandbox is not None: + params["sandbox"] = sandbox + if kind is not None: + params["kind"] = kind + r = await self._client.get("/api/v1/snapshots", params=params, timeout=30) + r.raise_for_status() + data = r.json() + return data["data"] if isinstance(data, dict) and "data" in data else data + + async def get_snapshot(self, name: str, namespace: str = "default") -> dict | None: + r = await self._client.get(f"/api/v1/snapshots/{name}", params={"namespace": namespace}, timeout=15) + if r.status_code == 404: + return None + r.raise_for_status() + data = r.json() + return data["data"] if isinstance(data, dict) and "data" in data else data + + async def create_snapshot(self, sandbox: str, snapshot_name: str, namespace: str = "default") -> dict: + r = await self._client.post( + f"/api/v1/sandboxes/{sandbox}/snapshot", + json={"snapshot_name": snapshot_name, "namespace": namespace}, + timeout=120, + ) + r.raise_for_status() + data = r.json() + return data["data"] if isinstance(data, dict) and "data" in data else data + + async def delete_snapshot(self, name: str, namespace: str = "default") -> dict: + r = await self._client.delete(f"/api/v1/snapshots/{name}", params={"namespace": namespace}, timeout=60) + r.raise_for_status() + data = r.json() + return data["data"] if isinstance(data, dict) and "data" in data else data + + async def gc_snapshots( + self, + namespace: str = "default", + all_namespaces: bool = False, + keep_fork_for: str = "10m", + dry_run: bool = False, + ) -> dict: + body: dict = { + "namespace": namespace, + "all_namespaces": all_namespaces, + "keep_fork_for": keep_fork_for, + "dry_run": dry_run, + } + r = await self._client.post("/api/v1/snapshots/gc", json=body, timeout=120) + r.raise_for_status() + data = r.json() + return data["data"] if isinstance(data, dict) and "data" in data else data + + async def restore( + self, + snapshot_name: str, + new_sandbox_name: str, + namespace: str = "default", + overrides: dict | None = None, + keep_snapshot: bool = True, + ) -> dict: + body: dict = { + "new_sandbox_name": new_sandbox_name, + "namespace": namespace, + "keep_snapshot": keep_snapshot, + } + if overrides: + body["overrides"] = overrides + r = await self._client.post( + f"/api/v1/snapshots/{snapshot_name}/restore", + json=body, + timeout=600, + ) + r.raise_for_status() + data = r.json() + return data["data"] if isinstance(data, dict) and "data" in data else data + + async def aclose(self): + await self._client.aclose() diff --git a/src/katakate/__init__.py b/src/katakate/__init__.py index 205ab56..7645c9b 100644 --- a/src/katakate/__init__.py +++ b/src/katakate/__init__.py @@ -1,13 +1,15 @@ -""" -Top-level K7 AI SDK package. -""" +"""Deprecated compatibility shim — use ``k7_sdk`` (``pip install k7-sdk``).""" -from .client import Client, AsyncClient, SandboxProxy +from __future__ import annotations -__all__ = [ - "Client", - "AsyncClient", - "SandboxProxy", -] +import warnings -__version__ = "0.0.4-dev" +from k7_sdk import AsyncClient, Client, SandboxProxy + +warnings.warn( + "The 'katakate' package is deprecated; pip install k7-sdk and use: from k7_sdk import Client", + DeprecationWarning, + stacklevel=2, +) + +__all__ = ["Client", "AsyncClient", "SandboxProxy"] diff --git a/src/katakate/client.py b/src/katakate/client.py deleted file mode 100644 index f7c8240..0000000 --- a/src/katakate/client.py +++ /dev/null @@ -1,190 +0,0 @@ -import requests -from typing import Optional, List - -try: - import httpx # optional dependency for async client -except Exception: # pragma: no cover - httpx = None - - -class SandboxProxy: - """Proxy object for sandbox operations.""" - - def __init__(self, name: str, namespace: str, client: "Client"): - self.name = name - self.namespace = namespace - self._client = client - - def exec(self, code: str) -> dict: - """Execute code in the sandbox.""" - return self._client._exec_command(self.name, code, self.namespace) - - def delete(self) -> dict: - """Delete this sandbox.""" - return self._client.delete(self.name, self.namespace) - - -class Client: - """K7 Python SDK Client.""" - - def __init__(self, endpoint: str, api_key: str, verify_ssl: bool = True): - self.base_url = endpoint.rstrip("/") - self.api_key = api_key - self.session = requests.Session() - self.session.headers.update({"X-API-Key": api_key}) - self.session.verify = verify_ssl - - def _unwrap(self, response) -> dict: - data = response.json() - if isinstance(data, dict) and "data" in data: - return data["data"] - return data - - def create(self, sandbox_config: dict) -> SandboxProxy: - """Create a new sandbox and return a proxy object.""" - response = self.session.post( - f"{self.base_url}/api/v1/sandboxes", json=sandbox_config - ) - response.raise_for_status() - - name = sandbox_config.get("name") - namespace = sandbox_config.get("namespace", "default") - - return SandboxProxy(name, namespace, self) - - def list(self, namespace: Optional[str] = None) -> List[dict]: - """List all sandboxes.""" - params = {"namespace": namespace} if namespace else {} - response = self.session.get(f"{self.base_url}/api/v1/sandboxes", params=params) - response.raise_for_status() - return self._unwrap(response) - - def delete(self, name: str, namespace: str = "default") -> dict: - """Delete a sandbox.""" - response = self.session.delete( - f"{self.base_url}/api/v1/sandboxes/{name}", params={"namespace": namespace} - ) - response.raise_for_status() - return self._unwrap(response) - - def delete_all(self, namespace: str = "default") -> dict: - """Delete all sandboxes in a namespace.""" - response = self.session.delete( - f"{self.base_url}/api/v1/sandboxes", params={"namespace": namespace} - ) - response.raise_for_status() - return self._unwrap(response) - - def install( - self, - playbook: Optional[str] = None, - inventory: Optional[str] = None, - verbose: bool = False, - ) -> dict: - """Install K7 on target hosts.""" - response = self.session.post( - f"{self.base_url}/api/v1/install", - json={"playbook": playbook, "inventory": inventory, "verbose": verbose}, - ) - response.raise_for_status() - return self._unwrap(response) - - def get_metrics(self, namespace: Optional[str] = None) -> dict: - """Get resource usage metrics for sandboxes.""" - params = {"namespace": namespace} if namespace else {} - response = self.session.get( - f"{self.base_url}/api/v1/sandboxes/metrics", params=params - ) - response.raise_for_status() - return self._unwrap(response) - - def _exec_command(self, name: str, command: str, namespace: str) -> dict: - """Internal method to execute command in sandbox.""" - response = self.session.post( - f"{self.base_url}/api/v1/sandboxes/{name}/exec", - json={"command": command}, - params={"namespace": namespace}, - ) - response.raise_for_status() - return self._unwrap(response) - - -class AsyncClient: - """K7 Python SDK Async Client.""" - - def __init__( - self, - endpoint: str, - api_key: str, - verify_ssl: bool = True, - timeout: float = 30.0, - ): - if httpx is None: - raise RuntimeError( - "httpx is required for AsyncClient. Install with `pip install httpx`." - ) - self.base_url = endpoint.rstrip("/") - self._client = httpx.AsyncClient( - base_url=self.base_url, - headers={"X-API-Key": api_key}, - verify=verify_ssl, - timeout=timeout, - ) - - async def create(self, sandbox_config: dict) -> dict: - r = await self._client.post("/api/v1/sandboxes", json=sandbox_config) - r.raise_for_status() - return r.json() - - async def list(self, namespace: Optional[str] = None) -> List[dict]: - params = {"namespace": namespace} if namespace else {} - r = await self._client.get("/api/v1/sandboxes", params=params) - r.raise_for_status() - data = r.json() - if isinstance(data, dict) and "data" in data: - return data["data"] - return data - - async def delete(self, name: str, namespace: str = "default") -> dict: - r = await self._client.delete( - f"/api/v1/sandboxes/{name}", params={"namespace": namespace} - ) - r.raise_for_status() - data = r.json() - if isinstance(data, dict) and "data" in data: - return data["data"] - return data - - async def delete_all(self, namespace: str = "default") -> dict: - r = await self._client.delete( - "/api/v1/sandboxes", params={"namespace": namespace} - ) - r.raise_for_status() - data = r.json() - if isinstance(data, dict) and "data" in data: - return data["data"] - return data - - async def exec(self, name: str, command: str, namespace: str = "default") -> dict: - r = await self._client.post( - f"/api/v1/sandboxes/{name}/exec", - json={"command": command}, - params={"namespace": namespace}, - ) - r.raise_for_status() - data = r.json() - if isinstance(data, dict) and "data" in data: - return data["data"] - return data - - async def get_metrics(self, namespace: Optional[str] = None) -> dict: - params = {"namespace": namespace} if namespace else {} - r = await self._client.get("/api/v1/sandboxes/metrics", params=params) - r.raise_for_status() - data = r.json() - if isinstance(data, dict) and "data" in data: - return data["data"] - return data - - async def aclose(self): - await self._client.aclose() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9af2ecc --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,110 @@ +import shutil +import subprocess + +import pytest + + +def _detect_backends() -> set[str]: + """Return the set of backends supported by the live cluster. + + Sources, in order of preference: + 1. `k7.katakate.org/backend-=true` labels on the cluster's nodes. + 2. `/etc/k7/backend` (legacy, single primary backend) — fallback for + hosts where k3s isn't reachable. + """ + if shutil.which("k3s"): + try: + result = subprocess.run( + [ + "k3s", + "kubectl", + "get", + "nodes", + "-o", + 'jsonpath={range .items[*]}{.metadata.labels}{"\\n"}{end}', + ], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired): + result = None + if result is not None and result.returncode == 0: + legacy = { + "firecracker-devmapper": "kata-firecracker-devmapper", + "qemu-longhorn": "kata-qemu-longhorn", + } + backends: set[str] = set() + for line in result.stdout.splitlines(): + for token in ( + "kata-firecracker-devmapper", + "kata-qemu-longhorn", + "k7d", + "firecracker-devmapper", + "qemu-longhorn", + ): + if f'"k7.katakate.org/backend-{token}":"true"' in line: + backends.add(legacy.get(token, token)) + if backends: + return backends + + try: + with open("/etc/k7/backend") as f: + backend = f.read().strip() + legacy = { + "firecracker-devmapper": "kata-firecracker-devmapper", + "qemu-longhorn": "kata-qemu-longhorn", + } + backend = legacy.get(backend, backend) + if backend in ("kata-firecracker-devmapper", "kata-qemu-longhorn", "k7d"): + return {backend} + except FileNotFoundError: + pass + return {"kata-firecracker-devmapper"} + + +def _detect_node_count() -> int: + """Count Ready k3s nodes; 0 when k3s is unavailable (e.g. local Mac).""" + if not shutil.which("k3s"): + return 0 + try: + result = subprocess.run( + ["k3s", "kubectl", "get", "nodes", "--no-headers"], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired): + return 0 + if result.returncode != 0: + return 0 + return sum(1 for line in result.stdout.splitlines() if " Ready " in f" {line} ") + + +BACKENDS = _detect_backends() +NODE_COUNT = _detect_node_count() + + +@pytest.fixture() +def node_count() -> int: + """Number of Ready nodes in the live k3s cluster.""" + return NODE_COUNT + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + skip_firecracker = pytest.mark.skip(reason="cluster has no kata-firecracker-devmapper-capable node") + skip_qemu = pytest.mark.skip(reason="cluster has no kata-qemu-longhorn-capable node") + skip_k7d = pytest.mark.skip(reason="cluster has no k7d-capable node") + skip_multinode = pytest.mark.skip(reason=f"requires >=2 Ready nodes (have {NODE_COUNT})") + + for item in items: + if "firecracker" in item.keywords and "kata-firecracker-devmapper" not in BACKENDS: + item.add_marker(skip_firecracker) + if "qemu" in item.keywords and "kata-qemu-longhorn" not in BACKENDS: + item.add_marker(skip_qemu) + if "k7d" in item.keywords and "k7d" not in BACKENDS: + item.add_marker(skip_k7d) + if "multinode" in item.keywords and NODE_COUNT < 2: + item.add_marker(skip_multinode) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/bench_backend_lifecycle.py b/tests/integration/bench_backend_lifecycle.py new file mode 100644 index 0000000..a9137ba --- /dev/null +++ b/tests/integration/bench_backend_lifecycle.py @@ -0,0 +1,285 @@ +"""Backend lifecycle benchmark — kql (Longhorn disk snapshots) vs k7d (warm VM fork). + +Measures, per backend, wall-clock time for: + +- ``create`` — ``create_sandbox()`` call + pod Ready +- ``exec`` — ``echo`` round-trip in a Ready sandbox +- ``snapshot`` — named snapshot until ready (kql only; k7d rejects it) +- ``fork_call`` — the ``fork_sandbox()`` call itself +- ``fork_ready`` — fork call + forked pod Ready + exec answering +- ``pause`` — pause until effective (kql: pods gone; k7d: VM frozen) +- ``resume`` — resume until an exec answers again +- ``delete`` — ``delete_sandbox()`` call +- ``sidecar_*`` — docker-in-VM sidecar: create+docker-ready, docker pull, docker run + +Same node, same images, interleaved runs. Invocation (on the k7 node): + + K7_BENCH_BACKENDS=kata-qemu-longhorn,k7d K7_BENCH_REPS=3 \ + uv run pytest -m bench tests/integration/bench_backend_lifecycle.py -v -s + +Results land as a markdown table on stdout and JSON under ``$K7_BENCH_OUT`` +(default ``/tmp``), one file per run: ``bench-backends-.json``. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import statistics +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from k7.core.core import K7Core +from k7.core.models import SandboxConfig + +pytestmark = pytest.mark.bench + +REPS = int(os.environ.get("K7_BENCH_REPS", "3")) +EXEC_REPS = int(os.environ.get("K7_BENCH_EXEC_REPS", "10")) +OUT_DIR = Path(os.environ.get("K7_BENCH_OUT", "/tmp")) +BACKENDS = [b.strip() for b in os.environ.get("K7_BENCH_BACKENDS", "kata-qemu-longhorn,k7d").split(",") if b.strip()] + +SANDBOX_IMAGE = "alpine:3.20" +SIDECAR_IMAGE = "docker:27.5-cli" +DOCKER_PULL_IMAGE = "alpine:3.21" + + +def _pod_ready(sandbox: str, namespace: str) -> bool: + result = subprocess.run( + [ + "k3s", + "kubectl", + "get", + "pods", + "-n", + namespace, + "-l", + f"app={sandbox}", + "-o", + "json", + ], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return False + try: + items = json.loads(result.stdout).get("items", []) + except json.JSONDecodeError: + return False + for pod in items: + statuses = pod.get("status", {}).get("containerStatuses", []) + if statuses and all(s.get("ready") for s in statuses): + return True + return False + + +def _wait(predicate, timeout: float, what: str, interval: float = 0.5) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(interval) + raise TimeoutError(f"timed out waiting for {what} after {timeout}s") + + +def _no_pods(sandbox: str, namespace: str) -> bool: + result = subprocess.run( + ["k3s", "kubectl", "get", "pods", "-n", namespace, "-l", f"app={sandbox}", "--no-headers"], + capture_output=True, + text=True, + check=False, + ) + return result.returncode == 0 and not result.stdout.strip() + + +async def _exec_ok(core: K7Core, sandbox: str, namespace: str, cmd: str = "echo bench-ok") -> bool: + try: + result = await core.exec_command(sandbox, cmd, namespace=namespace) + return result.exit_code == 0 + except Exception: + return False + + +async def _wait_exec(core: K7Core, sandbox: str, namespace: str, timeout: float = 180) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if await _exec_ok(core, sandbox, namespace): + return + await asyncio.sleep(1) + raise TimeoutError(f"exec in {sandbox} never answered within {timeout}s") + + +class Recorder: + def __init__(self) -> None: + self.samples: dict[tuple[str, str], list[float]] = {} + + def add(self, backend: str, metric: str, seconds: float) -> None: + self.samples.setdefault((backend, metric), []).append(seconds) + print(f" [{backend}] {metric}: {seconds:.2f}s", flush=True) + + def table(self) -> str: + metrics: list[str] = [] + for _, metric in self.samples: + if metric not in metrics: + metrics.append(metric) + lines = [ + "| Metric | " + " | ".join(BACKENDS) + " |", + "|--------|" + "|".join(["------"] * len(BACKENDS)) + "|", + ] + for metric in metrics: + row = [metric] + for backend in BACKENDS: + vals = self.samples.get((backend, metric)) + if vals: + med = statistics.median(vals) + row.append(f"{med:.2f}s (n={len(vals)})") + else: + row.append("n/a") + lines.append("| " + " | ".join(row) + " |") + return "\n".join(lines) + + def dump(self) -> Path: + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + out = OUT_DIR / f"bench-backends-{ts}.json" + payload = { + "backends": BACKENDS, + "reps": REPS, + "sandbox_image": SANDBOX_IMAGE, + "samples": {f"{b}/{m}": v for (b, m), v in self.samples.items()}, + } + out.write_text(json.dumps(payload, indent=2)) + return out + + +async def _bench_lifecycle(core: K7Core, recorder: Recorder, backend: str, namespace: str, rep: int) -> None: + name = f"bench-{backend.split('-')[-1][:4]}-{rep}" + fork_name = f"{name}-fork" + + # --- create → Ready --- + start = time.monotonic() + result = await core.create_sandbox( + SandboxConfig(name=name, image=SANDBOX_IMAGE, namespace=namespace, backend=backend) + ) + assert result.success, f"create failed: {result.error}" + _wait(lambda: _pod_ready(name, namespace), 300, f"{name} Ready") + recorder.add(backend, "create_to_ready", time.monotonic() - start) + + try: + # --- exec round-trips --- + exec_times = [] + for _ in range(EXEC_REPS): + t0 = time.monotonic() + assert await _exec_ok(core, name, namespace) + exec_times.append(time.monotonic() - t0) + recorder.add(backend, "exec_median", statistics.median(exec_times)) + + # seed state the fork should inherit + await core.exec_command(name, "echo inherited > /tmp/bench-marker", namespace=namespace) + + # --- named snapshot (kql only — k7d rejects loudly by design) --- + if backend == "kata-qemu-longhorn": + snap_name = f"{name}-snap" + t0 = time.monotonic() + snap = await core.create_snapshot(name, snap_name, namespace=namespace) + assert snap.success, f"snapshot failed: {snap.error}" + ready = await core._wait_for_snapshot_ready(snap_name, namespace=namespace) + assert ready.success, f"snapshot never ready: {ready.error}" + recorder.add(backend, "snapshot_ready", time.monotonic() - t0) + await core.delete_snapshot(snap_name, namespace=namespace) + + # --- fork --- + t0 = time.monotonic() + fork = await core.fork_sandbox(name, fork_name, namespace=namespace) + assert fork.success, f"fork failed: {fork.error}" + fork_call = time.monotonic() - t0 + recorder.add(backend, "fork_call", fork_call) + _wait(lambda: _pod_ready(fork_name, namespace), 300, f"{fork_name} Ready") + await _wait_exec(core, fork_name, namespace) + recorder.add(backend, "fork_to_ready", time.monotonic() - t0) + + if backend == "k7d": + inherited = await core.exec_command(fork_name, "cat /tmp/bench-marker", namespace=namespace) + assert inherited.exit_code == 0 and "inherited" in inherited.stdout, ( + "k7d fork lost the source's in-memory state" + ) + + await core.delete_sandbox(fork_name, namespace=namespace) + + # --- pause / resume --- + t0 = time.monotonic() + pause = await core.pause_sandbox(name, namespace=namespace) + assert pause.success, f"pause failed: {pause.error}" + if backend != "k7d": + _wait(lambda: _no_pods(name, namespace), 180, f"{name} pods gone") + recorder.add(backend, "pause_effective", time.monotonic() - t0) + + t0 = time.monotonic() + resume = await core.resume_sandbox(name, namespace=namespace) + assert resume.success, f"resume failed: {resume.error}" + await _wait_exec(core, name, namespace, timeout=300) + recorder.add(backend, "resume_to_exec", time.monotonic() - t0) + finally: + t0 = time.monotonic() + await core.delete_sandbox(name, namespace=namespace) + recorder.add(backend, "delete", time.monotonic() - t0) + + +async def _bench_sidecar(core: K7Core, recorder: Recorder, backend: str, namespace: str, rep: int) -> None: + name = f"bench-dind-{backend.split('-')[-1][:4]}-{rep}" + + start = time.monotonic() + result = await core.create_sandbox( + SandboxConfig(name=name, image=SIDECAR_IMAGE, namespace=namespace, backend=backend, sidecar="docker") + ) + assert result.success, f"sidecar create failed: {result.error}" + _wait(lambda: _pod_ready(name, namespace), 300, f"{name} Ready") + + async def docker_ready() -> bool: + probe = await core.exec_command(name, "docker info >/dev/null 2>&1 && echo ok", namespace=namespace) + return probe.exit_code == 0 and "ok" in probe.stdout + + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + if await docker_ready(): + break + await asyncio.sleep(2) + else: + raise TimeoutError(f"docker daemon in {name} never became ready") + recorder.add(backend, "sidecar_create_to_docker_ready", time.monotonic() - start) + + try: + t0 = time.monotonic() + pull = await core.exec_command(name, f"docker pull {DOCKER_PULL_IMAGE}", namespace=namespace) + assert pull.exit_code == 0, f"docker pull failed: {pull.stderr}" + recorder.add(backend, "sidecar_docker_pull", time.monotonic() - t0) + + t0 = time.monotonic() + run = await core.exec_command(name, f"docker run --rm {DOCKER_PULL_IMAGE} echo dind-ok", namespace=namespace) + assert run.exit_code == 0 and "dind-ok" in run.stdout, f"docker run failed: {run.stderr}" + recorder.add(backend, "sidecar_docker_run", time.monotonic() - t0) + finally: + await core.delete_sandbox(name, namespace=namespace) + + +async def test_bench_backend_lifecycle(k7_core: K7Core, test_namespace: str): + recorder = Recorder() + print(f"\nBenchmarking backends {BACKENDS} — {REPS} rep(s), image {SANDBOX_IMAGE}", flush=True) + for rep in range(REPS): + for backend in BACKENDS: + print(f" rep {rep + 1}/{REPS} backend={backend}", flush=True) + await _bench_lifecycle(k7_core, recorder, backend, test_namespace, rep) + for backend in BACKENDS: + print(f" sidecar backend={backend}", flush=True) + await _bench_sidecar(k7_core, recorder, backend, test_namespace, 0) + + out = recorder.dump() + print("\n== Backend lifecycle benchmark (median) ==\n", flush=True) + print(recorder.table(), flush=True) + print(f"\nraw samples: {out}", flush=True) diff --git a/tests/integration/bench_docker_perf.py b/tests/integration/bench_docker_perf.py new file mode 100644 index 0000000..2698bea --- /dev/null +++ b/tests/integration/bench_docker_perf.py @@ -0,0 +1,680 @@ +"""Spec 10b/18e: Docker workload benchmark — host vs k7-fd vs k7-ql (r=1..3). + +Why a pytest module rather than a separate bash harness: +``tests/integration/test_sidecar_docker.py`` already pioneered the pattern +of "spin up a ``docker:27.5-cli`` sandbox with ``--sidecar docker`` and run +docker commands via ``k7_core.exec_command``." This benchmark just times +that same pattern — no new infrastructure, no shelling-out to a separate +script, no fiddly Alpine-vs-bash portability problems. + +Why all four environments are valid: +``test_sidecar_docker.py`` has a note claiming Docker-DinD doesn't work on +the Firecracker backend. That note is wrong — confirmed by a manual probe +on the multi-node cluster: ``k7 create --backend kfd --sidecar docker`` + +``docker run hello-world`` exits 0 in ~3s. The sidecar wiring in +``core.py`` (around line 1537) is symmetric across backends; the only +per-backend branch is where the docker daemon's ``/var/lib/docker`` lives +(emptyDir on fd, Longhorn PVC sub_path on ql), which is exactly the +isolation we want from a benchmark: + + - ``k7-fd`` — docker-in-VM without Longhorn in the storage path. + - ``k7-ql-r1`` — adds Longhorn r=1 (one local replica). + - ``k7-ql-r2`` — adds Longhorn r=2 (one local + one cross-node sync). + - ``k7-ql-r3`` — Longhorn r=3 (replicas on all three nodes of an HA + cluster; spec 18e — the full redundancy write-amplification cost). + +Marker / invocation: +This file uses ``@pytest.mark.bench`` so it doesn't run with the rest of +``-m integration``. Drive it explicitly: + + K7_BENCH_ENVS=host,k7-fd,k7-ql-r1,k7-ql-r2,k7-ql-r3 \\ + K7_BENCH_OUT=/tmp/bench-out \\ + uv run pytest -m bench tests/integration/bench_docker_perf.py -v + +Replica counts are applied per sandbox volume by patching the Longhorn +Volume CR's ``spec.numberOfReplicas`` after creation (the ``longhorn`` +StorageClass pins ``numberOfReplicas`` as a provisioning parameter, so +patching Longhorn's ``default-replica-count`` setting has no effect on +these volumes). Each ql log header records the actual replica placement. + +Output: one ``bench-