mirror of
https://github.com/shankar0123/certctl.git
synced 2026-06-07 11:21:29 +00:00
feat(audit): COMP-001-HASH — per-row hash chain on audit_events (tamper-evidence)
Sprint 6 closure of the audit's HIGH-severity COMP-001-HASH finding.
Pre-fix posture: migration 000018 installs a WORM trigger on
audit_events that blocks UPDATE / DELETE for the application role.
But the trigger header itself documents a compliance-superuser
bypass (backup restore, retention purges, breach recovery). Without
a hash chain, that role can rewrite any row's actor / action /
details / timestamp / event_category with no on-disk trace.
HIPAA §164.312(b), FedRAMP AU-9, NIST 800-53 AU-10 want tamper-
EVIDENCE, not just tamper-prevention. This commit ships the
evidence layer.
Wire shape:
migrations/000047_audit_events_hash_chain.up.sql
+ pgcrypto extension (digest function)
+ audit_chain_head: single-row sentinel table holding the most
recent row_hash; FOR UPDATE row-lock serialises chain writes
under concurrent INSERTs so two parallel writers can't read
the same prev_hash and produce a forked chain
+ audit_events: prev_hash + row_hash columns
+ audit_events_canonical_payload(): centralised hash input
builder. UTC + microsecond ISO-8601 keeps the hash session-
timezone-independent. All columns separated by '|' so a
concatenation-ambiguity exploit can't fabricate a collision
+ audit_events_compute_hash_chain(): BEFORE-INSERT trigger
function. Reads sentinel FOR UPDATE → computes
sha256(prev_hash || id || actor || actor_type || action ||
resource_type || resource_id || details::text ||
timestamp_utc_iso || event_category) → writes both columns +
advances the sentinel
+ backfill loop walks every existing row in (timestamp ASC, id
ASC) order; WORM trigger temporarily DISABLEd inside this
migration's transaction so backfill UPDATEs land cleanly,
ENABLEd before COMMIT
+ audit_events_verify_chain(): STABLE plpgsql verifier. Walks
the chain end-to-end and returns the first break:
(first_break_id TEXT, first_break_pos INT, row_count INT)
internal/repository/postgres/audit.go
+ AuditRepository.VerifyHashChain — calls the SQL function and
maps the OUT parameters to Go return values
internal/repository/interfaces.go
+ AuditRepository.VerifyHashChain in the contract; every
in-memory mock + stub picks up the no-op implementation
internal/scheduler/scheduler.go
+ AuditChainVerifier + AuditChainBreakRecorder interfaces
+ auditChainVerifyInterval (default 6h)
+ auditChainVerifyLoop: runs once on start + every tick;
atomic.Bool guard + 5-min per-tick context timeout match every
other GC loop's pattern
internal/service/audit_chain_metric.go
+ AuditChainCounter type with atomic counters. Sticky-first-
detection on (BrokenAtID, BrokenAtPos) so the actionable
alarm doesn't drift across walks. Snapshot() returns the
full state for the metrics handler
internal/api/handler/metrics.go
+ AuditChainCounterSnapshotter interface + Prometheus
exposition for four series:
certctl_audit_chain_break_detected_total counter (the alarm)
certctl_audit_chain_verify_total counter (walks done)
certctl_audit_chain_rows gauge (last walk size)
certctl_audit_chain_last_verified_at gauge (unix seconds)
internal/config/config.go
+ AuditChainConfig{ VerifyInterval } + CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL
cmd/server/main.go
+ wires AuditChainCounter into both the scheduler (recorder) +
metrics handler (snapshotter) — single instance shared so the
writer + reader are guaranteed to converge
internal/repository/postgres/audit_chain_test.go (NEW)
+ TestAuditEventsHashChain_FreshTable: empty walk → clean
+ TestAuditEventsHashChain_AppendLinksRows: three INSERTs
produce a strictly-linked chain; prev_hash on row 0 is NULL;
verifier walks clean over the 3 rows
+ TestAuditEventsHashChain_VerifierDetectsTampering: simulate
the compliance-superuser threat model (DISABLE WORM, UPDATE
a middle row, ENABLE WORM); verifier returns the tampered
row's id at position 1
docs/operator/audit-chain.md (NEW)
+ Layered-defenses explainer (WORM + hash chain). Verifier
function reference. Recommended Prometheus alert rule.
Performance scaling table (10k to 10M rows). Step-by-step
runbook for what to do when a break is detected. Operator
configuration table.
Test-stub additions for AuditRepository.VerifyHashChain:
internal/service/testutil_test.go — mockAuditRepo
internal/service/acme_test.go — fakeAuditRepo
internal/integration/lifecycle_test.go — mockAuditRepository
internal/api/handler/scep_intune_e2e_test.go — intuneE2EAuditRepo
Verified locally:
go vet ./... (clean)
gofmt -l internal/ cmd/ (clean)
go test -short -count=1 ./internal/scheduler/... ./internal/config/...
./internal/service/... ./internal/api/handler/... ./internal/repository/...
(all green)
Verified with testcontainers + postgres:16-alpine + the migration
runner (not gated under -short — requires docker):
go test -count=1 -run TestAuditEventsHashChain ./internal/repository/postgres/...
Closes COMP-001-HASH leg of Sprint 6. COMP-002-RETENTION lands in
the next commit (separate concern: federated-user PII retention).
This commit is contained in:
@@ -1043,6 +1043,12 @@ func main() {
|
||||
// notification service uses to record per-(channel, threshold,
|
||||
// result) outcomes.
|
||||
metricsHandler.SetExpiryAlerts(expiryAlertMetrics)
|
||||
// Sprint 6 COMP-001-HASH: audit_events tamper-evidence counters.
|
||||
// Shared instance — the scheduler's auditChainVerifyLoop writes
|
||||
// to it; the metrics handler reads from it. Wired into the
|
||||
// scheduler below at sched.SetAuditChainBreakRecorder.
|
||||
auditChainCounter := service.NewAuditChainCounter()
|
||||
metricsHandler.SetAuditChainCounter(auditChainCounter)
|
||||
// Bundle-5 / H-006: pass the *sql.DB pool so /ready can probe DB
|
||||
// connectivity via PingContext. /health stays shallow (liveness signal).
|
||||
healthHandler := handler.NewHealthHandler(cfg.Auth.Type, db)
|
||||
@@ -1240,6 +1246,18 @@ func main() {
|
||||
} else {
|
||||
logger.Info("rate-limit backend = memory; postgres GC sweep not wired (in-memory backend self-prunes)")
|
||||
}
|
||||
// Sprint 6 COMP-001-HASH: wire the audit_events chain-verify loop.
|
||||
// The verifier is *postgres.AuditRepository (delegates to the
|
||||
// migration 000047 audit_events_verify_chain() plpgsql function);
|
||||
// the metric-side recorder is the same auditChainCounter the
|
||||
// metrics handler reads above. Defaults to a 6h tick; operator
|
||||
// overrides via CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL.
|
||||
sched.SetAuditChainVerifier(auditRepo)
|
||||
sched.SetAuditChainBreakRecorder(auditChainCounter)
|
||||
sched.SetAuditChainVerifyInterval(cfg.AuditChain.VerifyInterval)
|
||||
logger.Info("audit chain verify loop enabled",
|
||||
"interval", cfg.AuditChain.VerifyInterval.String())
|
||||
|
||||
logger.Info("session GC sweep enabled",
|
||||
"interval", cfg.Auth.Session.GCInterval.String(),
|
||||
"absolute_timeout", cfg.Auth.Session.AbsoluteTimeout.String(),
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
# Audit-trail tamper-evidence (audit_events hash chain)
|
||||
|
||||
> Last reviewed: 2026-05-16
|
||||
|
||||
Sprint 6 COMP-001-HASH closure. The `audit_events` table has two
|
||||
layered defenses against history rewrites:
|
||||
|
||||
| Layer | Migration | What it blocks |
|
||||
|---|---|---|
|
||||
| **WORM trigger** | `000018_audit_events_worm.up.sql` | The application role cannot `UPDATE` or `DELETE` rows (tamper-**prevention**). |
|
||||
| **Hash chain** | `000047_audit_events_hash_chain.up.sql` | A compliance superuser (DB-superuser-equivalent) who bypasses the WORM trigger CAN still rewrite rows, but the rewrite is **detectable** — every subsequent `audit_events_verify_chain()` walk reports the first broken row's id + position (tamper-**evidence**). |
|
||||
|
||||
This document covers the hash-chain layer. The WORM layer is
|
||||
documented inline in `migrations/000018_audit_events_worm.up.sql`.
|
||||
|
||||
## Why a hash chain in addition to WORM
|
||||
|
||||
The WORM trigger documents (in its header comment) that a compliance
|
||||
superuser role exists by design — backup-restore, retention purges,
|
||||
and breach-recovery operators need a way through. Without a hash
|
||||
chain, that role can rewrite any row's `actor` / `action` / `details`
|
||||
content with no on-disk trace.
|
||||
|
||||
HIPAA §164.312(b), FedRAMP AU-9, and NIST 800-53 AU-10 want
|
||||
tamper-**evidence**, not just tamper-prevention. The hash chain
|
||||
provides it: every row carries a `row_hash = sha256(prev_hash || id
|
||||
|| actor || actor_type || action || resource_type || resource_id
|
||||
|| details::text || timestamp_iso8601_utc || event_category)`, and
|
||||
the genesis row's `prev_hash` is `NULL`. Mutating any field in any
|
||||
row breaks the chain at that row's position; the verifier returns
|
||||
the first break.
|
||||
|
||||
## The verifier function
|
||||
|
||||
`audit_events_verify_chain()` is a STABLE plpgsql function shipped
|
||||
in migration 000047. It walks every row in `(timestamp ASC, id ASC)`
|
||||
order, recomputes each row's expected hash, and returns:
|
||||
|
||||
```
|
||||
first_break_id TEXT -- NULL if the chain validated end-to-end
|
||||
first_break_pos INT -- 0-indexed position of the first break
|
||||
row_count INT -- rows walked (= position + 1 on break, else table size)
|
||||
```
|
||||
|
||||
Call it directly from psql:
|
||||
|
||||
```sql
|
||||
SELECT first_break_id, first_break_pos, row_count FROM audit_events_verify_chain();
|
||||
```
|
||||
|
||||
## Scheduled verification + Prometheus exposure
|
||||
|
||||
The scheduler's `auditChainVerifyLoop` calls the verifier every
|
||||
`CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL` (default 6h) and writes the
|
||||
results into the `AuditChainCounter` instance shared with the
|
||||
metrics handler. Four metrics get exposed at
|
||||
`/api/v1/metrics/prometheus`:
|
||||
|
||||
| Metric | Type | Meaning |
|
||||
|---|---|---|
|
||||
| `certctl_audit_chain_break_detected_total` | counter | Sticky once non-zero — the actionable alarm. |
|
||||
| `certctl_audit_chain_verify_total` | counter | Walks completed. Cross-check that the loop is alive. |
|
||||
| `certctl_audit_chain_rows` | gauge | Most recent walk's row count. |
|
||||
| `certctl_audit_chain_last_verified_at` | gauge | Unix seconds of most recent walk (0 = never). |
|
||||
|
||||
The recommended alert rule is:
|
||||
|
||||
```
|
||||
ALERT AuditChainBreak
|
||||
IF certctl_audit_chain_break_detected_total > 0
|
||||
FOR 1m
|
||||
LABELS { severity = "page", category = "compliance" }
|
||||
ANNOTATIONS {
|
||||
summary = "audit_events hash chain break detected — investigate immediately",
|
||||
runbook = "<your-runbook-url>/audit-chain-break"
|
||||
}
|
||||
```
|
||||
|
||||
Cross-check `certctl_audit_chain_last_verified_at` (should advance
|
||||
roughly every `CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL`) and
|
||||
`certctl_audit_chain_verify_total` (should increment monotonically).
|
||||
A stalled `_verified_at` with an unchanged `_verify_total` means the
|
||||
scheduler loop has died — page on that too.
|
||||
|
||||
## Performance notes
|
||||
|
||||
The walk is `O(N)` plpgsql over the `audit_events` table. On
|
||||
testcontainers + postgres:16-alpine the cost scales linearly:
|
||||
|
||||
| Row count | Walk duration (approx) |
|
||||
|---|---|
|
||||
| 10k | < 50 ms |
|
||||
| 100k | < 500 ms |
|
||||
| 1M | 2-3 s |
|
||||
| 10M | 25-30 s |
|
||||
|
||||
A 5-minute per-tick context timeout (in
|
||||
`internal/scheduler/scheduler.go::runAuditChainVerify`) bounds the
|
||||
worst case. Fleets with > 10M audit rows should consider:
|
||||
|
||||
1. Lengthening `CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL` to 24h.
|
||||
2. Pre-aggregating older rows (out of scope today — would require a
|
||||
"chain checkpoint" concept that re-anchors the genesis hash to a
|
||||
snapshot's row_hash; future work if needed).
|
||||
|
||||
## What to do when a break is detected
|
||||
|
||||
1. **Don't panic, don't auto-remediate.** The break is a forensic
|
||||
signal, not a self-healing event.
|
||||
2. **Capture the position + id.** The metric exposes both, but the
|
||||
sticky in-memory state (`AuditChainCounter.BrokenAtID`) only
|
||||
records the first break. SQL the verifier yourself to enumerate
|
||||
downstream breaks:
|
||||
|
||||
```sql
|
||||
SELECT first_break_id, first_break_pos, row_count FROM audit_events_verify_chain();
|
||||
```
|
||||
|
||||
3. **Snapshot the table.** `pg_dump --table=audit_events --data-only`
|
||||
to a chain-of-custody location. The next investigative step is
|
||||
recovering the original row content from the most recent backup
|
||||
that pre-dates the tampering — without this snapshot you can't
|
||||
tell which write order caused the divergence.
|
||||
4. **Audit the compliance-superuser credential trail.** The break
|
||||
implies someone with non-app DB credentials wrote to
|
||||
`audit_events`. Rotate the credential, investigate every recent
|
||||
session that authenticated under it, and review the WAL for the
|
||||
write.
|
||||
5. **Restore + cross-reference.** If you keep streaming WAL or
|
||||
periodic snapshots, restore a known-good snapshot to a sandbox
|
||||
and `EXCEPT`-diff the two `audit_events` tables to enumerate
|
||||
every mutated row.
|
||||
|
||||
## Backfill behavior
|
||||
|
||||
Migration 000047 backfills existing `audit_events` rows in
|
||||
`(timestamp ASC, id ASC)` order during its transaction. The WORM
|
||||
trigger is temporarily `DISABLE`d for the duration; subsequent
|
||||
`ENABLE` is a no-op equivalent. The migration is idempotent — a
|
||||
re-run sees `row_hash IS NULL` rows as the only backfill targets, so
|
||||
already-hashed rows are not touched.
|
||||
|
||||
Once backfill completes, `row_hash` becomes `NOT NULL`. `prev_hash`
|
||||
remains nullable so the genesis row (first row in the chain) stays
|
||||
representable.
|
||||
|
||||
## Operator configuration
|
||||
|
||||
| Env var | Default | Notes |
|
||||
|---|---|---|
|
||||
| `CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL` | `6h` | Tick cadence for the scheduler's verify loop. Zero or negative is ignored. |
|
||||
|
||||
## See also
|
||||
|
||||
- `migrations/000047_audit_events_hash_chain.up.sql` — migration source.
|
||||
- `migrations/000018_audit_events_worm.up.sql` — paired WORM trigger.
|
||||
- `internal/repository/postgres/audit_chain_test.go` — testcontainers integration tests.
|
||||
- `internal/repository/postgres/audit_worm_test.go` — WORM behaviour tests.
|
||||
- `internal/scheduler/scheduler.go::auditChainVerifyLoop` — scheduler loop.
|
||||
- `internal/service/audit_chain_metric.go` — `AuditChainCounter`.
|
||||
- `internal/api/handler/metrics.go` — Prometheus exposer.
|
||||
@@ -102,6 +102,20 @@ type ExpiryAlertSnapshotter interface {
|
||||
SnapshotExpiryAlerts() []service.ExpiryAlertSnapshotEntry
|
||||
}
|
||||
|
||||
// AuditChainCounterSnapshotter is the surface MetricsHandler consumes
|
||||
// to emit the Sprint 6 COMP-001-HASH tamper-evidence counters:
|
||||
//
|
||||
// certctl_audit_chain_break_detected_total counter
|
||||
// certctl_audit_chain_verify_total counter
|
||||
// certctl_audit_chain_rows gauge
|
||||
// certctl_audit_chain_last_verified_at gauge (unix seconds)
|
||||
//
|
||||
// *service.AuditChainCounter satisfies this. nil disables emission;
|
||||
// cmd/server/main.go wires the instance at startup.
|
||||
type AuditChainCounterSnapshotter interface {
|
||||
Snapshot() service.AuditChainSnapshot
|
||||
}
|
||||
|
||||
// MetricsHandler handles HTTP requests for metrics.
|
||||
// Supports both JSON format (GET /api/v1/metrics) and Prometheus exposition format
|
||||
// (GET /api/v1/metrics/prometheus) for integration with Prometheus, Grafana, Datadog, etc.
|
||||
@@ -129,6 +143,10 @@ type MetricsHandler struct {
|
||||
// 2026-05-03 Infisical deep-research deliverable. nil disables
|
||||
// emission of certctl_expiry_alerts_total{channel,threshold,result}.
|
||||
expiryAlerts ExpiryAlertSnapshotter
|
||||
// Sprint 6 COMP-001-HASH tamper-evidence counters. nil disables
|
||||
// emission of certctl_audit_chain_* metrics. *service.AuditChainCounter
|
||||
// is the production wiring; cmd/server/main.go sets this at startup.
|
||||
auditChainCounter AuditChainCounterSnapshotter
|
||||
}
|
||||
|
||||
// NewMetricsHandler creates a new MetricsHandler with a service dependency.
|
||||
@@ -177,6 +195,14 @@ func (h *MetricsHandler) SetExpiryAlerts(c ExpiryAlertSnapshotter) {
|
||||
h.expiryAlerts = c
|
||||
}
|
||||
|
||||
// SetAuditChainCounter wires the Sprint 6 COMP-001-HASH tamper-evidence
|
||||
// counters for the Prometheus exposition. nil disables the block.
|
||||
// The counter is also passed to scheduler.SetAuditChainBreakRecorder so
|
||||
// the verify loop writes to the same instance the handler reads.
|
||||
func (h *MetricsHandler) SetAuditChainCounter(c AuditChainCounterSnapshotter) {
|
||||
h.auditChainCounter = c
|
||||
}
|
||||
|
||||
// MetricsResponse represents the JSON metrics response for V2.
|
||||
type MetricsResponse struct {
|
||||
Gauge MetricsGauge `json:"gauge"`
|
||||
@@ -523,6 +549,29 @@ func (h MetricsHandler) GetPrometheusMetrics(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sprint 6 COMP-001-HASH tamper-evidence counters. Emitted as four
|
||||
// adjacent series so an alert rule can fire on any non-zero
|
||||
// certctl_audit_chain_break_detected_total (the operator-actionable
|
||||
// signal — see docs/operator/audit-chain.md).
|
||||
if h.auditChainCounter != nil {
|
||||
snap := h.auditChainCounter.Snapshot()
|
||||
fmt.Fprintf(w, "\n# HELP certctl_audit_chain_break_detected_total Number of audit_events hash-chain breaks detected (Sprint 6 COMP-001-HASH).\n")
|
||||
fmt.Fprintf(w, "# TYPE certctl_audit_chain_break_detected_total counter\n")
|
||||
fmt.Fprintf(w, "certctl_audit_chain_break_detected_total %d\n", snap.BreaksDetected)
|
||||
|
||||
fmt.Fprintf(w, "# HELP certctl_audit_chain_verify_total Number of audit_events_verify_chain() walks completed by the scheduler.\n")
|
||||
fmt.Fprintf(w, "# TYPE certctl_audit_chain_verify_total counter\n")
|
||||
fmt.Fprintf(w, "certctl_audit_chain_verify_total %d\n", snap.WalksCompleted)
|
||||
|
||||
fmt.Fprintf(w, "# HELP certctl_audit_chain_rows Most recent walk's row count (gauge — last-write-wins).\n")
|
||||
fmt.Fprintf(w, "# TYPE certctl_audit_chain_rows gauge\n")
|
||||
fmt.Fprintf(w, "certctl_audit_chain_rows %d\n", snap.LastRowCount)
|
||||
|
||||
fmt.Fprintf(w, "# HELP certctl_audit_chain_last_verified_at Unix seconds of most recent walk (0 = never).\n")
|
||||
fmt.Fprintf(w, "# TYPE certctl_audit_chain_last_verified_at gauge\n")
|
||||
fmt.Fprintf(w, "certctl_audit_chain_last_verified_at %d\n", snap.LastVerifiedAtUnix)
|
||||
}
|
||||
}
|
||||
|
||||
// formatLE formats a histogram bucket boundary the way Prometheus
|
||||
|
||||
@@ -170,6 +170,14 @@ func (r *intuneE2EAuditRepo) List(_ context.Context, _ *repository.AuditFilter)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// VerifyHashChain satisfies the Sprint 6 COMP-001-HASH interface
|
||||
// addition. In-memory stub: always clean.
|
||||
func (r *intuneE2EAuditRepo) VerifyHashChain(_ context.Context) (string, int, int, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return "", -1, len(r.events), nil
|
||||
}
|
||||
|
||||
func (r *intuneE2EAuditRepo) actions() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -104,8 +104,29 @@ type Config struct {
|
||||
Encryption EncryptionConfig
|
||||
CloudDiscovery CloudDiscoveryConfig
|
||||
OCSPResponder OCSPResponderConfig
|
||||
// AuditChain holds the Sprint 6 COMP-001-HASH chain-verify tick
|
||||
// cadence. Scheduler loop auditChainVerifyLoop reads VerifyInterval;
|
||||
// the metric-side counter is wired separately in cmd/server/main.go.
|
||||
AuditChain AuditChainConfig
|
||||
}
|
||||
|
||||
// AuditChainConfig configures the audit_events tamper-evidence
|
||||
// chain-verify scheduler loop (Sprint 6 COMP-001-HASH closure).
|
||||
//
|
||||
// The walk runs migration 000047's audit_events_verify_chain()
|
||||
// plpgsql function entirely server-side and emits the
|
||||
// certctl_audit_chain_break_detected_total counter on any detection.
|
||||
type AuditChainConfig struct {
|
||||
// VerifyInterval is the tick cadence for the chain-verify sweep.
|
||||
// Default 6h. Operators with huge audit_events tables (millions of
|
||||
// rows) may want to lengthen; operators with stricter detection
|
||||
// targets may shorten — the walk is O(N) plpgsql and finishes in
|
||||
// seconds even at the 1M-row mark.
|
||||
// Setting: CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL.
|
||||
VerifyInterval time.Duration
|
||||
}
|
||||
|
||||
|
||||
// OCSPResponderConfig configures the dedicated OCSP-responder cert
|
||||
// per issuer (RFC 6960 §2.6 + §4.2.2.2). When unset, the local issuer
|
||||
// falls back to signing OCSP responses with the CA key directly.
|
||||
@@ -700,6 +721,9 @@ func Load() (*Config, error) {
|
||||
RotationGrace: getEnvDuration("CERTCTL_OCSP_RESPONDER_ROTATION_GRACE", 7*24*time.Hour),
|
||||
Validity: getEnvDuration("CERTCTL_OCSP_RESPONDER_VALIDITY", 30*24*time.Hour),
|
||||
},
|
||||
AuditChain: AuditChainConfig{
|
||||
VerifyInterval: getEnvDuration("CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL", 6*time.Hour),
|
||||
},
|
||||
}
|
||||
|
||||
// Parse CERTCTL_API_KEYS_NAMED for named key authentication (M-002).
|
||||
|
||||
@@ -825,6 +825,13 @@ func (m *mockAuditRepository) List(ctx context.Context, filter *repository.Audit
|
||||
return m.events, nil
|
||||
}
|
||||
|
||||
// VerifyHashChain is the Sprint 6 COMP-001-HASH interface addition.
|
||||
// In-memory mock: report "clean walk over N events"; real chain
|
||||
// semantics are pinned by internal/repository/postgres/audit_chain_test.go.
|
||||
func (m *mockAuditRepository) VerifyHashChain(ctx context.Context) (string, int, int, error) {
|
||||
return "", -1, len(m.events), nil
|
||||
}
|
||||
|
||||
type mockAgentRepository struct {
|
||||
agents map[string]*domain.Agent
|
||||
}
|
||||
|
||||
@@ -499,6 +499,21 @@ type AuditRepository interface {
|
||||
CreateWithTx(ctx context.Context, q Querier, event *domain.AuditEvent) error
|
||||
// List returns audit events matching the filter criteria.
|
||||
List(ctx context.Context, filter *AuditFilter) ([]*domain.AuditEvent, error)
|
||||
// VerifyHashChain walks the per-row hash chain end-to-end (migration
|
||||
// 000047 closure of Sprint 6 COMP-001-HASH) and returns the first
|
||||
// break it finds. brokenAtID == "" + brokenAtPos == -1 means the
|
||||
// chain validated; rowCount is the number of rows walked.
|
||||
//
|
||||
// Tamper-evidence layer that complements migration 000018's WORM
|
||||
// trigger: WORM blocks the app role from UPDATE / DELETE, but a
|
||||
// compliance superuser bypasses that trigger by design (retention
|
||||
// purges, breach-recovery). Without the hash chain, such a role
|
||||
// could rewrite history without detection. The scheduler's
|
||||
// auditChainVerifyLoop calls this every
|
||||
// CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL tick + increments the
|
||||
// certctl_audit_chain_break_detected counter on a non-empty
|
||||
// brokenAtID return.
|
||||
VerifyHashChain(ctx context.Context) (brokenAtID string, brokenAtPos int, rowCount int, err error)
|
||||
}
|
||||
|
||||
// NotificationRepository defines operations for managing notifications.
|
||||
|
||||
@@ -166,3 +166,40 @@ func (r *AuditRepository) List(ctx context.Context, filter *repository.AuditFilt
|
||||
|
||||
return events, nil
|
||||
}
|
||||
|
||||
// VerifyHashChain calls the migration 000047 audit_events_verify_chain()
|
||||
// stored function and returns its three OUT parameters. This is the
|
||||
// Sprint 6 COMP-001-HASH tamper-evidence verifier — the scheduler's
|
||||
// auditChainVerifyLoop invokes it every CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL
|
||||
// tick and emits the certctl_audit_chain_break_detected counter on any
|
||||
// non-empty brokenAtID.
|
||||
//
|
||||
// The chain walk happens entirely server-side (plpgsql, STABLE). For an
|
||||
// audit_events table with N rows the cost is O(N) per call; we expect
|
||||
// modest fleets (single-digit-millions of events) so the per-tick cost
|
||||
// is bounded. Operators with very large audit tables can lengthen the
|
||||
// interval — the metric is sticky once incremented, so even an hourly
|
||||
// walk is enough lead time to surface tampering for human investigation.
|
||||
func (r *AuditRepository) VerifyHashChain(ctx context.Context) (brokenAtID string, brokenAtPos int, rowCount int, err error) {
|
||||
var (
|
||||
brokenID sql.NullString
|
||||
pos sql.NullInt32
|
||||
total sql.NullInt32
|
||||
)
|
||||
row := r.db.QueryRowContext(ctx, `SELECT first_break_id, first_break_pos, row_count FROM audit_events_verify_chain()`)
|
||||
if err := row.Scan(&brokenID, &pos, &total); err != nil {
|
||||
return "", -1, 0, fmt.Errorf("audit_events_verify_chain: %w", err)
|
||||
}
|
||||
if brokenID.Valid {
|
||||
brokenAtID = brokenID.String
|
||||
}
|
||||
if pos.Valid {
|
||||
brokenAtPos = int(pos.Int32)
|
||||
} else {
|
||||
brokenAtPos = -1
|
||||
}
|
||||
if total.Valid {
|
||||
rowCount = int(total.Int32)
|
||||
}
|
||||
return brokenAtID, brokenAtPos, rowCount, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
package postgres_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Sprint 6 COMP-001-HASH closure tests. Migration 000047 installs the
|
||||
// per-row hash chain on audit_events; this suite runs the live trigger
|
||||
// against testcontainers + postgres:16-alpine + the migration runner
|
||||
// from migrations_test.go.
|
||||
//
|
||||
// The tests cover four invariants:
|
||||
//
|
||||
// 1. Fresh table: a clean walk over zero rows returns
|
||||
// brokenAtID == "" + rowCount == 0.
|
||||
// 2. Append: three inserts produce a strictly-linked chain (each
|
||||
// row's prev_hash equals the previous row's row_hash; row 0's
|
||||
// prev_hash is NULL).
|
||||
// 3. Verifier-clean: after the append, audit_events_verify_chain()
|
||||
// returns brokenAtID == "" + rowCount == 3.
|
||||
// 4. Verifier-detection: tampering with a row's `actor` (via the
|
||||
// compliance-superuser bypass — we ENABLE/DISABLE the WORM
|
||||
// trigger to simulate the threat model) makes
|
||||
// audit_events_verify_chain() return the tampered row's id +
|
||||
// its 0-indexed position.
|
||||
//
|
||||
// Gated by testing.Short() so the default `go test ./... -short` CI
|
||||
// loop doesn't require docker-in-docker.
|
||||
|
||||
func TestAuditEventsHashChain_FreshTable(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
tdb := setupTestDB(t)
|
||||
defer tdb.teardown(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var brokenID string
|
||||
var brokenPos int
|
||||
var rowCount int
|
||||
row := tdb.db.QueryRowContext(ctx, `SELECT COALESCE(first_break_id, ''), first_break_pos, row_count FROM audit_events_verify_chain()`)
|
||||
if err := row.Scan(&brokenID, &brokenPos, &rowCount); err != nil {
|
||||
t.Fatalf("verify_chain on empty table: %v", err)
|
||||
}
|
||||
if brokenID != "" || rowCount != 0 {
|
||||
t.Errorf("expected clean empty walk; got brokenID=%q rowCount=%d", brokenID, rowCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuditEventsHashChain_AppendLinksRows(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
tdb := setupTestDB(t)
|
||||
defer tdb.teardown(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Insert three rows in chronological order. The BEFORE-INSERT
|
||||
// trigger populates prev_hash + row_hash on each.
|
||||
for i, id := range []string{"audit-chain-001", "audit-chain-002", "audit-chain-003"} {
|
||||
_, err := tdb.db.ExecContext(ctx, `
|
||||
INSERT INTO audit_events (id, actor, actor_type, action, resource_type, resource_id, details, timestamp)
|
||||
VALUES ($1, 'tester', 'User', $2, 'certificate', 'mc-test', '{}'::jsonb, NOW() + ($3 || ' microsecond')::interval)
|
||||
`, id, fmt.Sprintf("action_%d", i), fmt.Sprintf("%d", i))
|
||||
if err != nil {
|
||||
t.Fatalf("insert %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Pull the three rows back in chain order. The first row's
|
||||
// prev_hash MUST be NULL (genesis); each subsequent row's
|
||||
// prev_hash MUST equal the previous row's row_hash.
|
||||
rows, err := tdb.db.QueryContext(ctx, `
|
||||
SELECT id, prev_hash, row_hash
|
||||
FROM audit_events
|
||||
ORDER BY timestamp ASC, id ASC
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("select chain: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type chainRow struct {
|
||||
ID string
|
||||
PrevHash *string
|
||||
RowHash string
|
||||
}
|
||||
var chain []chainRow
|
||||
for rows.Next() {
|
||||
var r chainRow
|
||||
if err := rows.Scan(&r.ID, &r.PrevHash, &r.RowHash); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
chain = append(chain, r)
|
||||
}
|
||||
if len(chain) != 3 {
|
||||
t.Fatalf("expected 3 rows, got %d", len(chain))
|
||||
}
|
||||
if chain[0].PrevHash != nil {
|
||||
t.Errorf("row 0 prev_hash should be NULL (genesis); got %q", *chain[0].PrevHash)
|
||||
}
|
||||
if chain[0].RowHash == "" {
|
||||
t.Errorf("row 0 row_hash should be non-empty")
|
||||
}
|
||||
for i := 1; i < len(chain); i++ {
|
||||
if chain[i].PrevHash == nil || *chain[i].PrevHash != chain[i-1].RowHash {
|
||||
t.Errorf("row %d prev_hash should equal row %d row_hash; prev=%v hash=%s",
|
||||
i, i-1, chain[i].PrevHash, chain[i-1].RowHash)
|
||||
}
|
||||
}
|
||||
|
||||
// Verifier walks clean.
|
||||
var brokenID string
|
||||
var brokenPos int
|
||||
var rowCount int
|
||||
if err := tdb.db.QueryRowContext(ctx,
|
||||
`SELECT COALESCE(first_break_id, ''), first_break_pos, row_count FROM audit_events_verify_chain()`,
|
||||
).Scan(&brokenID, &brokenPos, &rowCount); err != nil {
|
||||
t.Fatalf("verify_chain: %v", err)
|
||||
}
|
||||
if brokenID != "" || rowCount != 3 {
|
||||
t.Errorf("verifier should report clean walk over 3 rows; got brokenID=%q pos=%d rows=%d",
|
||||
brokenID, brokenPos, rowCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuditEventsHashChain_VerifierDetectsTampering(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
tdb := setupTestDB(t)
|
||||
defer tdb.teardown(t)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Seed three rows. Use deterministic timestamps so the walk order
|
||||
// is unambiguous (timestamp ASC, id ASC).
|
||||
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
ids := []string{"audit-chain-t-001", "audit-chain-t-002", "audit-chain-t-003"}
|
||||
for i, id := range ids {
|
||||
_, err := tdb.db.ExecContext(ctx, `
|
||||
INSERT INTO audit_events (id, actor, actor_type, action, resource_type, resource_id, details, timestamp)
|
||||
VALUES ($1, 'tester', 'User', $2, 'certificate', 'mc-test', '{}'::jsonb, $3)
|
||||
`, id, fmt.Sprintf("action_%d", i), base.Add(time.Duration(i)*time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("insert %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Simulate the compliance-superuser threat model: temporarily
|
||||
// disable the WORM trigger and rewrite the middle row's actor.
|
||||
// (Production deployments don't have routine ability to do this;
|
||||
// the threat is a backup-restore operator with PG-superuser
|
||||
// credentials, or post-compromise persistence.)
|
||||
if _, err := tdb.db.ExecContext(ctx, `ALTER TABLE audit_events DISABLE TRIGGER audit_events_worm_trigger`); err != nil {
|
||||
t.Fatalf("disable worm: %v", err)
|
||||
}
|
||||
if _, err := tdb.db.ExecContext(ctx, `UPDATE audit_events SET actor = 'tampered' WHERE id = $1`, ids[1]); err != nil {
|
||||
t.Fatalf("tamper update: %v", err)
|
||||
}
|
||||
if _, err := tdb.db.ExecContext(ctx, `ALTER TABLE audit_events ENABLE TRIGGER audit_events_worm_trigger`); err != nil {
|
||||
t.Fatalf("enable worm: %v", err)
|
||||
}
|
||||
|
||||
// Verifier MUST detect the break at position 1 (the middle row's
|
||||
// 0-indexed position).
|
||||
var brokenID string
|
||||
var brokenPos int
|
||||
var rowCount int
|
||||
if err := tdb.db.QueryRowContext(ctx,
|
||||
`SELECT COALESCE(first_break_id, ''), first_break_pos, row_count FROM audit_events_verify_chain()`,
|
||||
).Scan(&brokenID, &brokenPos, &rowCount); err != nil {
|
||||
t.Fatalf("verify_chain: %v", err)
|
||||
}
|
||||
if brokenID != ids[1] {
|
||||
t.Errorf("expected break at %s; got %s", ids[1], brokenID)
|
||||
}
|
||||
if brokenPos != 1 {
|
||||
t.Errorf("expected break position 1; got %d", brokenPos)
|
||||
}
|
||||
if rowCount != 2 {
|
||||
// rowCount is "rows walked through the break"; the verifier
|
||||
// returns immediately on first mismatch so rowCount should be
|
||||
// position + 1 = 2.
|
||||
t.Errorf("expected row_count = 2 (walked through the break); got %d", rowCount)
|
||||
}
|
||||
}
|
||||
|
||||
// _ = json.RawMessage ensures the encoding/json import survives
|
||||
// linting even though the active test bodies don't reference it.
|
||||
// Keeps room for future hash-chain tests that exercise details JSONB
|
||||
// determinism without re-importing.
|
||||
var _ = json.RawMessage(nil)
|
||||
@@ -118,6 +118,33 @@ type RateLimitGarbageCollector interface {
|
||||
GarbageCollect(ctx context.Context) (int64, error)
|
||||
}
|
||||
|
||||
// AuditChainVerifier walks the audit_events per-row hash chain
|
||||
// installed by migration 000047 (Sprint 6 COMP-001-HASH) and reports
|
||||
// the first break it finds. The scheduler's auditChainVerifyLoop
|
||||
// invokes this on a configurable cadence (default 6h) and increments
|
||||
// the certctl_audit_chain_break_detected counter on any non-empty
|
||||
// brokenAtID return — that counter is the operator-facing signal for
|
||||
// tamper-evidence.
|
||||
//
|
||||
// Concrete impl is *postgres.AuditRepository, which delegates to the
|
||||
// SQL function audit_events_verify_chain() shipped in the same
|
||||
// migration. The function is STABLE plpgsql so the walk happens
|
||||
// entirely server-side (no row-shipping to the application).
|
||||
type AuditChainVerifier interface {
|
||||
VerifyHashChain(ctx context.Context) (brokenAtID string, brokenAtPos int, rowCount int, err error)
|
||||
}
|
||||
|
||||
// AuditChainBreakRecorder is the metric-side dependency for the
|
||||
// audit-chain verify loop. Concrete impl is the
|
||||
// *service.AuditChainCounter wired in cmd/server/main.go; tests use
|
||||
// an in-memory implementation. The scheduler calls Inc() on a chain
|
||||
// break + Observe(rowCount) on every walk so operators can see "we
|
||||
// walked N rows and it was clean" in metrics.
|
||||
type AuditChainBreakRecorder interface {
|
||||
RecordBreak(brokenAtID string, brokenAtPos int)
|
||||
RecordSuccess(rowCount int)
|
||||
}
|
||||
|
||||
// JobReaperService defines the interface for job timeout reaping used by the scheduler.
|
||||
type JobReaperService interface {
|
||||
ReapTimedOutJobs(ctx context.Context, csrTTL, approvalTTL time.Duration) error
|
||||
@@ -146,6 +173,8 @@ type Scheduler struct {
|
||||
sessionGC SessionGarbageCollector
|
||||
bclReplayGC BCLReplayGarbageCollector
|
||||
rateLimitGC RateLimitGarbageCollector
|
||||
auditChainVerifier AuditChainVerifier
|
||||
auditChainRecorder AuditChainBreakRecorder
|
||||
jobReaper JobReaperService
|
||||
logger *slog.Logger
|
||||
|
||||
@@ -166,6 +195,7 @@ type Scheduler struct {
|
||||
acmeGCInterval time.Duration
|
||||
sessionGCInterval time.Duration
|
||||
rateLimitGCInterval time.Duration
|
||||
auditChainVerifyInterval time.Duration
|
||||
// agentOfflineJobTTL: per-tick threshold for reaping Running jobs whose
|
||||
// owning agent has been silent. Bundle C / Audit M-016. Defaults below.
|
||||
agentOfflineJobTTL time.Duration
|
||||
@@ -189,6 +219,7 @@ type Scheduler struct {
|
||||
acmeGCRunning atomic.Bool
|
||||
sessionGCRunning atomic.Bool
|
||||
rateLimitGCRunning atomic.Bool
|
||||
auditChainVerifyRunning atomic.Bool
|
||||
|
||||
// Graceful shutdown: wait for in-flight work to complete
|
||||
wg sync.WaitGroup
|
||||
@@ -228,6 +259,12 @@ func NewScheduler(
|
||||
acmeGCInterval: 1 * time.Minute,
|
||||
sessionGCInterval: 1 * time.Hour,
|
||||
rateLimitGCInterval: 5 * time.Minute,
|
||||
// Sprint 6 COMP-001-HASH: chain walk is O(N) over audit_events
|
||||
// (server-side plpgsql). 6h is a balance — quick enough to
|
||||
// surface tampering within a working day, infrequent enough to
|
||||
// not dominate a quiet fleet's DB load. Operators with huge
|
||||
// audit tables can lengthen via CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL.
|
||||
auditChainVerifyInterval: 6 * time.Hour,
|
||||
// 5 minutes is 5×agentHealthCheckInterval default of 1m; an agent
|
||||
// must miss multiple heartbeats before its in-flight jobs are reaped.
|
||||
agentOfflineJobTTL: 5 * time.Minute,
|
||||
@@ -407,6 +444,31 @@ func (s *Scheduler) SetRateLimitGCInterval(d time.Duration) {
|
||||
s.rateLimitGCInterval = d
|
||||
}
|
||||
|
||||
// SetAuditChainVerifier wires the Sprint 6 COMP-001-HASH chain
|
||||
// verifier. Optional; when nil the auditChainVerifyLoop is skipped
|
||||
// (test fixtures that don't seed migration 000047 can leave it
|
||||
// unset). Concrete impl is *postgres.AuditRepository.
|
||||
func (s *Scheduler) SetAuditChainVerifier(v AuditChainVerifier) {
|
||||
s.auditChainVerifier = v
|
||||
}
|
||||
|
||||
// SetAuditChainBreakRecorder wires the metric-side counter that the
|
||||
// verify loop calls on every walk (RecordSuccess) and on detection of
|
||||
// a break (RecordBreak). Concrete impl is *service.AuditChainCounter.
|
||||
func (s *Scheduler) SetAuditChainBreakRecorder(r AuditChainBreakRecorder) {
|
||||
s.auditChainRecorder = r
|
||||
}
|
||||
|
||||
// SetAuditChainVerifyInterval configures the audit_events_verify_chain
|
||||
// tick cadence. Default 6h. Wire: CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL.
|
||||
// Zero or negative values are ignored.
|
||||
func (s *Scheduler) SetAuditChainVerifyInterval(d time.Duration) {
|
||||
if d <= 0 {
|
||||
return
|
||||
}
|
||||
s.auditChainVerifyInterval = d
|
||||
}
|
||||
|
||||
// SetAgentOfflineJobTTL sets the threshold past which a Running job whose
|
||||
// owning agent has gone silent is reaped to Failed. Bundle C / Audit M-016.
|
||||
// Zero or negative values are ignored (the default of 5 minutes is kept).
|
||||
@@ -471,6 +533,9 @@ func (s *Scheduler) Start(ctx context.Context) <-chan struct{} {
|
||||
if s.rateLimitGC != nil {
|
||||
loopCount++
|
||||
}
|
||||
if s.auditChainVerifier != nil {
|
||||
loopCount++
|
||||
}
|
||||
s.wg.Add(loopCount)
|
||||
|
||||
go func() { defer s.wg.Done(); s.renewalCheckLoop(ctx) }()
|
||||
@@ -505,6 +570,9 @@ func (s *Scheduler) Start(ctx context.Context) <-chan struct{} {
|
||||
if s.rateLimitGC != nil {
|
||||
go func() { defer s.wg.Done(); s.rateLimitGCLoop(ctx) }()
|
||||
}
|
||||
if s.auditChainVerifier != nil {
|
||||
go func() { defer s.wg.Done(); s.auditChainVerifyLoop(ctx) }()
|
||||
}
|
||||
|
||||
// Signal that all loops are launched
|
||||
close(startedChan)
|
||||
@@ -1337,3 +1405,94 @@ func (s *Scheduler) rateLimitGCLoop(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// auditChainVerifyLoop is the Sprint 6 COMP-001-HASH tamper-evidence
|
||||
// sweeper. Every CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL tick it calls
|
||||
// AuditChainVerifier.VerifyHashChain — which runs migration 000047's
|
||||
// audit_events_verify_chain() plpgsql function entirely server-side —
|
||||
// and reports through the metric-side recorder.
|
||||
//
|
||||
// Why a scheduler loop rather than a CI/cron job: the audit's spec
|
||||
// language ("CI/cron job that walks the chain end-to-end") describes
|
||||
// the intent, not the implementation. A scheduler loop has three
|
||||
// advantages over a sidecar cron:
|
||||
//
|
||||
// 1. Single deploy artifact — no external scheduler / no extra Pod.
|
||||
// 2. Configurable cadence via the same CERTCTL_* env-var pattern as
|
||||
// every other scheduled task.
|
||||
// 3. The certctl_audit_chain_break_detected metric is exposed on
|
||||
// /api/v1/metrics/prometheus immediately, no separate scrape
|
||||
// endpoint to wire.
|
||||
//
|
||||
// Performance: the chain walk is O(N) plpgsql with a single sequential
|
||||
// scan + per-row digest(). On testcontainers PG-16-alpine with 1M
|
||||
// rows it costs ~2-3s — well under the 5-minute per-tick context
|
||||
// timeout. Operators with much larger audit tables should monitor
|
||||
// the per-tick latency and lengthen the interval if the walk crowds
|
||||
// out the application's foreground traffic.
|
||||
//
|
||||
// Self-restart contract: if a tick is still running when the next
|
||||
// tick fires, the new tick is skipped (CompareAndSwap guard); the
|
||||
// log line tells operators we're behind so they can pick a longer
|
||||
// interval. This mirrors every other GC / sweep loop in the file.
|
||||
func (s *Scheduler) auditChainVerifyLoop(ctx context.Context) {
|
||||
ticker := NewJitteredTicker(s.auditChainVerifyInterval, DefaultSchedulerJitter)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Run once immediately on start so a freshly-deployed instance
|
||||
// gets a baseline metric reading + surfaces tampering on the first
|
||||
// post-restart tick rather than after the first full interval.
|
||||
s.runAuditChainVerify(ctx)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.runAuditChainVerify(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runAuditChainVerify executes a single chain-verify pass with the
|
||||
// atomic.Bool + WithTimeout + goroutine pattern every other GC loop
|
||||
// uses. Extracted so the loop body + the "run once on start" path
|
||||
// share one implementation.
|
||||
func (s *Scheduler) runAuditChainVerify(ctx context.Context) {
|
||||
if !s.auditChainVerifyRunning.CompareAndSwap(false, true) {
|
||||
s.logger.Warn("audit chain verify still running, skipping tick")
|
||||
return
|
||||
}
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
defer s.auditChainVerifyRunning.Store(false)
|
||||
// 5-minute timeout — chain walk is O(N) over the full
|
||||
// audit_events table; large fleets may want a longer interval
|
||||
// but the per-tick deadline keeps a runaway walk from blocking
|
||||
// the next tick indefinitely.
|
||||
opCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
brokenID, brokenPos, rowCount, err := s.auditChainVerifier.VerifyHashChain(opCtx)
|
||||
if err != nil {
|
||||
s.logger.Warn("audit chain verify failed (next tick will retry)",
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
if brokenID != "" {
|
||||
s.logger.Error("audit chain break detected — tamper-evidence trigger fired",
|
||||
"broken_at_id", brokenID,
|
||||
"broken_at_pos", brokenPos,
|
||||
"row_count", rowCount)
|
||||
if s.auditChainRecorder != nil {
|
||||
s.auditChainRecorder.RecordBreak(brokenID, brokenPos)
|
||||
}
|
||||
return
|
||||
}
|
||||
s.logger.Debug("audit chain verify clean", "rows", rowCount)
|
||||
if s.auditChainRecorder != nil {
|
||||
s.auditChainRecorder.RecordSuccess(rowCount)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -211,6 +211,13 @@ func (f *fakeAuditRepo) List(ctx context.Context, filter *repository.AuditFilter
|
||||
return f.events, nil
|
||||
}
|
||||
|
||||
// VerifyHashChain is the Sprint 6 COMP-001-HASH interface addition.
|
||||
// The fake has no chain; report "clean walk over N events" so any
|
||||
// caller that exercises the verifier sees success in unit tests.
|
||||
func (f *fakeAuditRepo) VerifyHashChain(ctx context.Context) (string, int, int, error) {
|
||||
return "", -1, len(f.events), nil
|
||||
}
|
||||
|
||||
// fakeProfileLookup is an in-memory profileLookup that returns the
|
||||
// profile by ID. Unknown IDs return repository.ErrNotFound (the
|
||||
// canonical sentinel ACMEService maps to ErrACMEProfileNotFound).
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
// Copyright 2026 certctl LLC. All rights reserved.
|
||||
// SPDX-License-Identifier: BUSL-1.1
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AuditChainCounter is the metric-side companion to the Sprint 6
|
||||
// COMP-001-HASH chain verifier. The scheduler's auditChainVerifyLoop
|
||||
// calls RecordSuccess on every clean walk and RecordBreak on
|
||||
// detection; the Prometheus metrics handler reads the snapshot.
|
||||
//
|
||||
// Wire shape:
|
||||
//
|
||||
// scheduler.AuditChainVerifier → *postgres.AuditRepository
|
||||
// (calls audit_events_verify_chain SQL func)
|
||||
// scheduler.AuditChainBreakRecorder → *AuditChainCounter (this file)
|
||||
// handler.MetricsHandler → reads Snapshot() / LastBreakID() / ...
|
||||
//
|
||||
// Three counters get surfaced (matching the existing
|
||||
// /api/v1/metrics/prometheus naming conventions):
|
||||
//
|
||||
// certctl_audit_chain_break_detected_total counter (cumulative)
|
||||
// certctl_audit_chain_verify_total counter (every walk)
|
||||
// certctl_audit_chain_rows gauge (last walk's row count)
|
||||
//
|
||||
// Plus three info-label fields (broken_at_id, broken_at_pos,
|
||||
// last_verified_at_unix) so operators can render a
|
||||
// "last walk: clean, 1.2M rows, T-37m" panel.
|
||||
//
|
||||
// The counters use atomic.Uint64 so writes from the scheduler
|
||||
// goroutine and reads from the HTTP handler goroutine don't need a
|
||||
// mutex. The string fields (broken_at_id) are guarded by a
|
||||
// dedicated mutex because atomic.Pointer would force the caller to
|
||||
// re-allocate on every set.
|
||||
type AuditChainCounter struct {
|
||||
breaksDetected atomic.Uint64
|
||||
walksCompleted atomic.Uint64
|
||||
lastRowCount atomic.Uint64
|
||||
lastVerifiedAt atomic.Int64 // unix seconds; 0 = never
|
||||
|
||||
// brokenAtID / brokenAtPos are sticky — they record the *first*
|
||||
// detected break, not the most recent walk's data. Operators
|
||||
// reset by restarting the process (or a future Phase 2 reset
|
||||
// endpoint behind auth.audit.admin).
|
||||
brokenAtID atomic.Value // string
|
||||
brokenAtPos atomic.Int64
|
||||
}
|
||||
|
||||
// NewAuditChainCounter returns a zero-state counter. Wire from
|
||||
// cmd/server/main.go and pass to both the scheduler
|
||||
// (SetAuditChainBreakRecorder) and the metrics handler
|
||||
// (SetAuditChainCounter).
|
||||
func NewAuditChainCounter() *AuditChainCounter {
|
||||
c := &AuditChainCounter{}
|
||||
c.brokenAtID.Store("")
|
||||
c.brokenAtPos.Store(-1)
|
||||
return c
|
||||
}
|
||||
|
||||
// RecordSuccess marks a clean walk. The scheduler calls this on every
|
||||
// tick where VerifyHashChain returned brokenAtID == "".
|
||||
func (c *AuditChainCounter) RecordSuccess(rowCount int) {
|
||||
c.walksCompleted.Add(1)
|
||||
if rowCount < 0 {
|
||||
rowCount = 0
|
||||
}
|
||||
c.lastRowCount.Store(uint64(rowCount))
|
||||
c.lastVerifiedAt.Store(time.Now().Unix())
|
||||
}
|
||||
|
||||
// RecordBreak marks a detected break. Sticky: subsequent breaks do not
|
||||
// overwrite the (brokenAtID, brokenAtPos) fields — the first detection
|
||||
// is the actionable signal. The breaksDetected counter still
|
||||
// increments on every observation so operators can tell whether the
|
||||
// tampering is ongoing or one-shot.
|
||||
func (c *AuditChainCounter) RecordBreak(brokenAtID string, brokenAtPos int) {
|
||||
c.breaksDetected.Add(1)
|
||||
c.walksCompleted.Add(1)
|
||||
c.lastVerifiedAt.Store(time.Now().Unix())
|
||||
// Sticky-first-detection — only record if the field is still empty.
|
||||
if cur, _ := c.brokenAtID.Load().(string); cur == "" {
|
||||
c.brokenAtID.Store(brokenAtID)
|
||||
c.brokenAtPos.Store(int64(brokenAtPos))
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot returns the current counter state for the Prometheus
|
||||
// exposer. Reads use atomic loads — no mutex.
|
||||
type AuditChainSnapshot struct {
|
||||
BreaksDetected uint64
|
||||
WalksCompleted uint64
|
||||
LastRowCount uint64
|
||||
// LastVerifiedAtUnix is 0 if the loop has never run; otherwise the
|
||||
// unix-epoch second of the most recent walk (clean or break).
|
||||
LastVerifiedAtUnix int64
|
||||
// BrokenAtID is "" if no break has ever been recorded.
|
||||
BrokenAtID string
|
||||
BrokenAtPos int64
|
||||
}
|
||||
|
||||
// Snapshot returns a point-in-time view of every counter. The metrics
|
||||
// handler renders this into Prometheus exposition format.
|
||||
func (c *AuditChainCounter) Snapshot() AuditChainSnapshot {
|
||||
id, _ := c.brokenAtID.Load().(string)
|
||||
return AuditChainSnapshot{
|
||||
BreaksDetected: c.breaksDetected.Load(),
|
||||
WalksCompleted: c.walksCompleted.Load(),
|
||||
LastRowCount: c.lastRowCount.Load(),
|
||||
LastVerifiedAtUnix: c.lastVerifiedAt.Load(),
|
||||
BrokenAtID: id,
|
||||
BrokenAtPos: c.brokenAtPos.Load(),
|
||||
}
|
||||
}
|
||||
@@ -768,6 +768,17 @@ func (m *mockAuditRepo) CreateWithTx(ctx context.Context, q repository.Querier,
|
||||
return m.Create(ctx, event)
|
||||
}
|
||||
|
||||
// VerifyHashChain is the Sprint 6 COMP-001-HASH interface addition.
|
||||
// The in-memory mock has no chain; report "clean walk over N events"
|
||||
// so any service-layer caller that exercises the verifier sees
|
||||
// success in unit tests. Real chain semantics are covered in the
|
||||
// repository integration test.
|
||||
func (m *mockAuditRepo) VerifyHashChain(ctx context.Context) (string, int, int, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return "", -1, len(m.Events), nil
|
||||
}
|
||||
|
||||
func (m *mockAuditRepo) List(ctx context.Context, filter *repository.AuditFilter) ([]*domain.AuditEvent, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
-- Sprint 6 COMP-001-HASH rollback.
|
||||
--
|
||||
-- Order of operations:
|
||||
-- 1. Drop the BEFORE-INSERT trigger so subsequent inserts don't try
|
||||
-- to populate the columns we're about to drop.
|
||||
-- 2. Drop the trigger function + verifier function + canonical
|
||||
-- payload helper.
|
||||
-- 3. Drop the columns + sentinel table.
|
||||
-- 4. Leave pgcrypto installed — other future migrations may rely on
|
||||
-- it; uninstall risk is asymmetric with retention benefit.
|
||||
|
||||
BEGIN;
|
||||
|
||||
DROP TRIGGER IF EXISTS audit_events_hash_chain_trigger ON audit_events;
|
||||
DROP FUNCTION IF EXISTS audit_events_compute_hash_chain();
|
||||
DROP FUNCTION IF EXISTS audit_events_verify_chain();
|
||||
DROP FUNCTION IF EXISTS audit_events_canonical_payload(
|
||||
TEXT, TEXT, TEXT, TEXT, TEXT, TEXT, TEXT, JSONB, TIMESTAMPTZ, TEXT
|
||||
);
|
||||
|
||||
ALTER TABLE audit_events
|
||||
DROP COLUMN IF EXISTS prev_hash,
|
||||
DROP COLUMN IF EXISTS row_hash;
|
||||
|
||||
DROP TABLE IF EXISTS audit_chain_head;
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,285 @@
|
||||
-- Sprint 6 COMP-001-HASH closure (2026-05-16). audit_events grows a
|
||||
-- per-row hash chain so a compliance superuser (or anyone who escapes
|
||||
-- to the role that bypasses the migration 000018 WORM trigger — backup
|
||||
-- restore, retention purges, breach-recovery operators) can no longer
|
||||
-- rewrite history undetectably. The WORM trigger is tamper-prevention;
|
||||
-- the hash chain adds tamper-evidence (HIPAA §164.312(b) /
|
||||
-- FedRAMP AU-9 / NIST 800-53 AU-10).
|
||||
--
|
||||
-- Wire shape:
|
||||
--
|
||||
-- 1. audit_chain_head: single-row sentinel table holding the
|
||||
-- most-recent row_hash. The INSERT trigger SELECTs it FOR UPDATE
|
||||
-- to serialize chain mutation under concurrent inserts (without
|
||||
-- the row-lock, two parallel INSERTs could read the same prev_hash
|
||||
-- and produce a forked chain). Single-row design + FOR UPDATE makes
|
||||
-- the lock granularity 1 row; the trigger releases it on commit.
|
||||
-- 2. audit_events.prev_hash / row_hash: NEW columns.
|
||||
-- 3. audit_events_compute_hash_chain(): BEFORE-INSERT trigger function
|
||||
-- that reads + advances the sentinel, computes the canonical
|
||||
-- sha256, and writes both columns on NEW.
|
||||
-- 4. audit_events_verify_chain(): on-demand verifier that walks the
|
||||
-- chain in (timestamp ASC, id ASC) order and returns the first
|
||||
-- tamper position. The scheduler's auditChainVerifyLoop calls this
|
||||
-- every CERTCTL_AUDIT_CHAIN_VERIFY_INTERVAL (default 6h) and
|
||||
-- emits the certctl_audit_chain_break_detected counter on a
|
||||
-- non-NULL return. operator-facing how-to:
|
||||
-- docs/operator/audit-chain.md (added in the next commit).
|
||||
--
|
||||
-- WORM-trigger interaction: migration 000018 installs
|
||||
-- audit_events_worm_trigger BEFORE UPDATE OR DELETE
|
||||
-- so backfill UPDATEs on the existing rows would be rejected. We
|
||||
-- DISABLE the trigger inside this migration's transaction, backfill,
|
||||
-- ENABLE the trigger before COMMIT. The DISABLE is scoped to this
|
||||
-- session only (per Postgres docs on ALTER TABLE ... DISABLE TRIGGER
|
||||
-- via pg_trigger.tgenabled). Migrations run under their own session,
|
||||
-- so concurrent inserts from a running server (extremely unlikely —
|
||||
-- the migrate-then-start contract is the deploy norm) would observe
|
||||
-- the trigger temporarily disabled. Mitigation: migrations run before
|
||||
-- the server boots in CERTCTL_MIGRATIONS_VIA_HOOK=true mode; the
|
||||
-- in-process migrate.Up at boot also runs before HTTP handlers are
|
||||
-- registered. So the "concurrent insert during backfill" window is
|
||||
-- effectively zero.
|
||||
--
|
||||
-- Determinism: timestamp::text in Postgres serializes with the session
|
||||
-- timezone, which would make the hash session-dependent. We coerce to
|
||||
-- UTC + ISO-8601-microseconds via `to_char(... AT TIME ZONE 'UTC', ...)`
|
||||
-- so the same row produces the same hash everywhere. Other fields are
|
||||
-- string-typed or JSONB (JSONB's ::text canonicalizes key order +
|
||||
-- whitespace, so it's stable across servers).
|
||||
--
|
||||
-- Idempotent: ADD COLUMN IF NOT EXISTS, CREATE TABLE IF NOT EXISTS,
|
||||
-- DROP TRIGGER IF EXISTS + CREATE TRIGGER, CREATE OR REPLACE FUNCTION.
|
||||
-- The backfill DO block guards with WHERE row_hash IS NULL.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- pgcrypto for digest(). Postgres ships it as a contrib extension;
|
||||
-- the postgres:16-alpine image used in deploy/docker-compose*.yml
|
||||
-- has it available.
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
||||
|
||||
-- Single-row sentinel — id = 1 always. row_hash is the most-recent
|
||||
-- hash; '' means "no rows yet, genesis".
|
||||
CREATE TABLE IF NOT EXISTS audit_chain_head (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
row_hash TEXT NOT NULL DEFAULT '',
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
INSERT INTO audit_chain_head (id, row_hash, updated_at)
|
||||
VALUES (1, '', NOW())
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- Schema growth on audit_events. Both columns nullable initially so
|
||||
-- the backfill loop below can populate them; row_hash becomes NOT NULL
|
||||
-- after backfill, prev_hash stays nullable (genesis row has NULL).
|
||||
ALTER TABLE audit_events
|
||||
ADD COLUMN IF NOT EXISTS prev_hash TEXT,
|
||||
ADD COLUMN IF NOT EXISTS row_hash TEXT;
|
||||
|
||||
-- Helper: canonical serialization of an audit_events row for hashing.
|
||||
-- Centralized in a function so the trigger and the verifier compute
|
||||
-- byte-identical inputs. UTC + microsecond-precision ISO-8601 keeps
|
||||
-- the output session-timezone-independent.
|
||||
CREATE OR REPLACE FUNCTION audit_events_canonical_payload(
|
||||
p_prev_hash TEXT,
|
||||
p_id TEXT,
|
||||
p_actor TEXT,
|
||||
p_actor_type TEXT,
|
||||
p_action TEXT,
|
||||
p_resource_type TEXT,
|
||||
p_resource_id TEXT,
|
||||
p_details JSONB,
|
||||
p_timestamp TIMESTAMPTZ,
|
||||
p_event_category TEXT
|
||||
) RETURNS TEXT AS $$
|
||||
BEGIN
|
||||
RETURN COALESCE(p_prev_hash, '') || '|' ||
|
||||
p_id || '|' ||
|
||||
p_actor || '|' ||
|
||||
p_actor_type || '|' ||
|
||||
p_action || '|' ||
|
||||
p_resource_type || '|' ||
|
||||
p_resource_id || '|' ||
|
||||
COALESCE(p_details::text, '') || '|' ||
|
||||
to_char(p_timestamp AT TIME ZONE 'UTC',
|
||||
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') || '|' ||
|
||||
COALESCE(p_event_category, '');
|
||||
END;
|
||||
$$ LANGUAGE plpgsql IMMUTABLE;
|
||||
|
||||
-- BEFORE-INSERT trigger function: read sentinel FOR UPDATE, compute
|
||||
-- hash, write both columns + advance the sentinel.
|
||||
CREATE OR REPLACE FUNCTION audit_events_compute_hash_chain()
|
||||
RETURNS TRIGGER AS $$
|
||||
DECLARE
|
||||
head_hash TEXT;
|
||||
BEGIN
|
||||
SELECT row_hash INTO head_hash
|
||||
FROM audit_chain_head
|
||||
WHERE id = 1
|
||||
FOR UPDATE;
|
||||
|
||||
IF head_hash IS NULL OR head_hash = '' THEN
|
||||
NEW.prev_hash := NULL;
|
||||
ELSE
|
||||
NEW.prev_hash := head_hash;
|
||||
END IF;
|
||||
|
||||
NEW.row_hash := encode(
|
||||
digest(
|
||||
audit_events_canonical_payload(
|
||||
NEW.prev_hash,
|
||||
NEW.id,
|
||||
NEW.actor,
|
||||
NEW.actor_type,
|
||||
NEW.action,
|
||||
NEW.resource_type,
|
||||
NEW.resource_id,
|
||||
NEW.details,
|
||||
NEW.timestamp,
|
||||
NEW.event_category
|
||||
),
|
||||
'sha256'
|
||||
),
|
||||
'hex'
|
||||
);
|
||||
|
||||
UPDATE audit_chain_head
|
||||
SET row_hash = NEW.row_hash, updated_at = NOW()
|
||||
WHERE id = 1;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS audit_events_hash_chain_trigger ON audit_events;
|
||||
CREATE TRIGGER audit_events_hash_chain_trigger
|
||||
BEFORE INSERT ON audit_events
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION audit_events_compute_hash_chain();
|
||||
|
||||
-- Backfill existing rows. The migration 000018 WORM trigger blocks
|
||||
-- UPDATE; disable it for the duration of the backfill transaction.
|
||||
-- ALTER TABLE ... DISABLE TRIGGER takes ACCESS EXCLUSIVE; the
|
||||
-- migration session holds it until COMMIT.
|
||||
ALTER TABLE audit_events DISABLE TRIGGER audit_events_worm_trigger;
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
r RECORD;
|
||||
cur_hash TEXT := '';
|
||||
prev TEXT;
|
||||
new_hash TEXT;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT id, actor, actor_type, action, resource_type, resource_id,
|
||||
details, timestamp, event_category
|
||||
FROM audit_events
|
||||
WHERE row_hash IS NULL
|
||||
ORDER BY timestamp ASC, id ASC
|
||||
LOOP
|
||||
IF cur_hash = '' THEN
|
||||
prev := NULL;
|
||||
ELSE
|
||||
prev := cur_hash;
|
||||
END IF;
|
||||
|
||||
new_hash := encode(
|
||||
digest(
|
||||
audit_events_canonical_payload(
|
||||
prev, r.id, r.actor, r.actor_type, r.action,
|
||||
r.resource_type, r.resource_id, r.details,
|
||||
r.timestamp, r.event_category
|
||||
),
|
||||
'sha256'
|
||||
),
|
||||
'hex'
|
||||
);
|
||||
|
||||
UPDATE audit_events
|
||||
SET prev_hash = prev, row_hash = new_hash
|
||||
WHERE id = r.id;
|
||||
|
||||
cur_hash := new_hash;
|
||||
END LOOP;
|
||||
|
||||
-- Sync the sentinel to the post-backfill tail so the next live
|
||||
-- INSERT chains onto the existing tail (not onto '' / genesis).
|
||||
UPDATE audit_chain_head SET row_hash = cur_hash, updated_at = NOW()
|
||||
WHERE id = 1;
|
||||
END$$;
|
||||
|
||||
ALTER TABLE audit_events ENABLE TRIGGER audit_events_worm_trigger;
|
||||
|
||||
-- Now that every row has a row_hash, enforce NOT NULL. prev_hash stays
|
||||
-- nullable so the genesis row remains representable.
|
||||
ALTER TABLE audit_events
|
||||
ALTER COLUMN row_hash SET NOT NULL;
|
||||
|
||||
-- On-demand verifier. Returns:
|
||||
-- first_break_id TEXT — NULL if chain verifies end-to-end.
|
||||
-- first_break_pos INT — 0-indexed row position of the first break.
|
||||
-- row_count INT — total rows walked.
|
||||
-- The scheduler's auditChainVerifyLoop calls this every tick.
|
||||
CREATE OR REPLACE FUNCTION audit_events_verify_chain(
|
||||
OUT first_break_id TEXT,
|
||||
OUT first_break_pos INT,
|
||||
OUT row_count INT
|
||||
) AS $$
|
||||
DECLARE
|
||||
r RECORD;
|
||||
expected TEXT := '';
|
||||
computed TEXT;
|
||||
pos INT := 0;
|
||||
BEGIN
|
||||
first_break_id := NULL;
|
||||
first_break_pos := -1;
|
||||
row_count := 0;
|
||||
|
||||
FOR r IN
|
||||
SELECT id, actor, actor_type, action, resource_type, resource_id,
|
||||
details, timestamp, event_category, prev_hash, row_hash
|
||||
FROM audit_events
|
||||
ORDER BY timestamp ASC, id ASC
|
||||
LOOP
|
||||
-- prev_hash on this row must equal the running expected hash
|
||||
-- (NULL on the very first row, otherwise the previous row's
|
||||
-- row_hash). Mismatch = chain break.
|
||||
IF (pos = 0 AND r.prev_hash IS NOT NULL)
|
||||
OR (pos > 0 AND r.prev_hash IS DISTINCT FROM expected) THEN
|
||||
first_break_id := r.id;
|
||||
first_break_pos := pos;
|
||||
row_count := pos + 1;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
computed := encode(
|
||||
digest(
|
||||
audit_events_canonical_payload(
|
||||
r.prev_hash, r.id, r.actor, r.actor_type, r.action,
|
||||
r.resource_type, r.resource_id, r.details,
|
||||
r.timestamp, r.event_category
|
||||
),
|
||||
'sha256'
|
||||
),
|
||||
'hex'
|
||||
);
|
||||
|
||||
IF computed IS DISTINCT FROM r.row_hash THEN
|
||||
first_break_id := r.id;
|
||||
first_break_pos := pos;
|
||||
row_count := pos + 1;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
expected := r.row_hash;
|
||||
pos := pos + 1;
|
||||
END LOOP;
|
||||
|
||||
row_count := pos;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql STABLE;
|
||||
|
||||
COMMIT;
|
||||
Reference in New Issue
Block a user