certctl/deploy/test/loadtest/k6/agent_storm.js

// Phase 8 SCALE-H2 — agent fleet heartbeat storm.
//
// What this measures:
//   5,000 agents heartbeating at 30s intervals = ~167 heartbeats/sec
//   sustained. Each heartbeat is POST /api/v1/agents/{id}/heartbeat
//   with optional metadata. Pre-seeded fleet provided by
//   deploy/test/loadtest/seed/02_agent_fleet.sql.
//
// What this does NOT measure:
//   - The agent work-poll path (GET /api/v1/agents/{id}/work). The
//     heartbeat hot path is the highest-frequency call on a typical
//     fleet (work-poll cadence is 30s default like heartbeat, but
//     work-poll returns the empty set 99% of the time and is cheap;
//     heartbeat does an UPDATE on every call). v2 of the harness
//     could combine them.
//   - The agent CSR-submit path (POST /api/v1/agents/{id}/csr). That
//     fires on per-cert issuance, not per heartbeat, and is exercised
//     by the existing API tier's POST /api/v1/certificates scenario.
//   - Auth-key per-agent rotation. The loadtest stack runs with a
//     single api-key (`load-test-token`); per-agent api-key
//     hashing/rotation isn't a load axis.
//
// Why constant-arrival-rate (not constant-vus):
//   The point is to model what 5K real agents would offer the server
//   at their native cadence. 5K agents * (1 heartbeat / 30s) =
//   166.67 req/s offered. constant-arrival-rate fires at exactly
//   that rate regardless of latency; if the server backpressures,
//   queue builds and p99 shows it. constant-vus would let slow
//   responses block, masking the actual ceiling.
//
// Threshold contract:
//   - p99 < 1s for the heartbeat POST. The handler does an UPDATE on
//     agents.last_heartbeat_at (+ optional metadata columns) and an
//     RBAC check. Even at 200 req/s a tight UPDATE on an indexed
//     primary key should stay sub-second.
//   - p95 < 500ms.
//   - Error rate < 0.1%. The seeded agents are all status='Online'
//     so no 410 Gone (retired-agent) responses; anything 4xx is a
//     bug. 5xx is a server health regression.
//
// Phase 8 reference:
//   - Source finding: SCALE-H2.
//   - Pre-state: heartbeat path not load-tested. The 100-agent demo
//     seed in seed_demo.sql produces ~3 heartbeats/sec, orders of
//     magnitude below fleet scale.

import http from 'k6/http';
import { check } from 'k6';
import { textSummary } from 'https://jslib.k6.io/k6-summary/0.0.2/index.js';

const BASE  = __ENV.CERTCTL_BASE  || 'https://certctl-server:8443';
const TOKEN = __ENV.CERTCTL_TOKEN || 'load-test-token';

// 5000 agents * (1 / 30s) = 166.67 heartbeats/sec. Round to 167.
const TARGET_RATE = parseInt(__ENV.K6_AGENT_RATE || '167', 10);

// Total agents in the fleet seed. The k6 scenario picks an agent at
// random per iteration (deterministic via __ITER) to spread the
// per-row UPDATE pressure across the table.
const FLEET_SIZE = parseInt(__ENV.K6_AGENT_FLEET || '5000', 10);

export const options = {
    scenarios: {
        agent_storm: {
            executor: 'constant-arrival-rate',
            rate: TARGET_RATE,
            timeUnit: '1s',
            duration: '5m',
            preAllocatedVUs: 50,
            maxVUs: 200,
            exec: 'heartbeat',
            tags: { scenario: 'agent_storm' },
        },
    },
    thresholds: {
        'http_req_duration{scenario:agent_storm}': ['p(99)<1000', 'p(95)<500'],
        'http_req_failed{scenario:agent_storm}': ['rate<0.001'],
    },
    summaryTrendStats: ['avg', 'min', 'med', 'p(95)', 'p(99)', 'max'],
    insecureSkipTLSVerify: true,
};

// agentID returns a deterministic agent id from the loadtest fleet
// seed. Spreading round-robin across the fleet means the UPDATE
// pressure hits every row equally rather than the same hot row over
// and over.
function agentID() {
    // __ITER is k6's per-VU iteration counter; combined with __VU
    // (the VU index) we get a unique-per-call number that spans
    // 0..FLEET_SIZE on the modulo.
    const idx = (__VU * 1000 + __ITER) % FLEET_SIZE;
    return 'ag-loadtest-' + String(idx + 1).padStart(5, '0');
}

export function heartbeat() {
    const id = agentID();
    // Optional metadata; the heartbeat handler tolerates an empty body
    // (no metadata) but real agents send their version + hostname on
    // every call so we include them here.
    const payload = JSON.stringify({
        version: '2.1.0',
        hostname: 'loadtest-' + id.slice(-5) + '.fleet.example.test',
        os: 'linux',
        architecture: 'amd64',
    });

    const res = http.post(`${BASE}/api/v1/agents/${id}/heartbeat`, payload, {
        headers: {
            'Content-Type': 'application/json',
            'Authorization': `Bearer ${TOKEN}`,
        },
        tags: { scenario: 'agent_storm' },
    });

    check(res, {
        'heartbeat 2xx': (r) => r.status >= 200 && r.status < 300,
    });
}

export function handleSummary(data) {
    return {
        '/results/summary-agent-storm.json': JSON.stringify(data, null, 2),
        '/results/summary-agent-storm.txt': textSummary(data, { indent: ' ', enableColors: false }),
        stdout: textSummary(data, { indent: ' ', enableColors: true }),
    };
}