Files
pulse/pkg/metrics/availability_history.go
T
2026-08-30 15:38:34 +01:00

844 lines
31 KiB
Go

package metrics
import (
"database/sql"
"fmt"
"math"
"sort"
"strings"
"time"
pdb "github.com/rcourtman/pulse-go-rewrite/pkg/db"
"github.com/rs/zerolog/log"
)
// AvailabilityOutcome is the closed categorical vocabulary retained by the
// history store. It deliberately does not reuse numeric metric rows: averaging
// these values would erase indeterminate and unknown time.
type AvailabilityOutcome string
const (
AvailabilityReachable AvailabilityOutcome = "reachable"
AvailabilityUnreachable AvailabilityOutcome = "unreachable"
AvailabilityIndeterminate AvailabilityOutcome = "indeterminate"
)
// AvailabilityExecutionSource identifies where an observation executed
// without retaining the assigned agent's identity.
type AvailabilityExecutionSource string
const (
AvailabilitySourceLocal AvailabilityExecutionSource = "local"
AvailabilitySourceAssignedAgent AvailabilityExecutionSource = "assigned_agent"
)
const (
availabilityRawRetention = 48 * time.Hour
availabilityMinuteRetention = 8 * 24 * time.Hour
availabilityHourlyRetention = 92 * 24 * time.Hour
availabilityDailyRetention = 366 * 24 * time.Hour
availabilityMaxValidity = 24 * time.Hour
availabilityMaxBatchTargets = 200
availabilityMaxBuckets = 120
)
// AvailabilityObservation is one accepted scheduled result. TimelineAt is
// server-authored and controls coverage; ObservedAt is evidence metadata only.
type AvailabilityObservation struct {
ObservationID string
TargetID string
ConfigRevision int64
Outcome AvailabilityOutcome
ObservedAt time.Time
TimelineAt time.Time
IngestedAt time.Time
ValidFor time.Duration
ExecutionSource AvailabilityExecutionSource
LatencyMillis *int64
}
// AvailabilityLatencySummary contains reachable-only latency evidence.
type AvailabilityLatencySummary struct {
Average float64 `json:"average"`
Min int64 `json:"min"`
Max int64 `json:"max"`
}
// AvailabilityHistorySummary is derived from state durations over the exact
// requested window. AvailabilityPercent is absent without determinate time.
type AvailabilityHistorySummary struct {
ReachableSeconds float64 `json:"reachableSeconds"`
UnreachableSeconds float64 `json:"unreachableSeconds"`
IndeterminateSeconds float64 `json:"indeterminateSeconds"`
UnknownSeconds float64 `json:"unknownSeconds"`
CoveragePercent float64 `json:"coveragePercent"`
AvailabilityPercent *float64 `json:"availabilityPercent,omitempty"`
ReachableLatencyMillis *AvailabilityLatencySummary `json:"reachableLatencyMillis,omitempty"`
}
// AvailabilityHistoryBucket is one chronological fleet-view bucket.
type AvailabilityHistoryBucket struct {
Start time.Time `json:"start"`
End time.Time `json:"end"`
ReachableSeconds float64 `json:"reachableSeconds"`
UnreachableSeconds float64 `json:"unreachableSeconds"`
IndeterminateSeconds float64 `json:"indeterminateSeconds"`
UnknownSeconds float64 `json:"unknownSeconds"`
LatencyMillis *AvailabilityLatencySummary `json:"latencyMillis,omitempty"`
}
// AvailabilityRevisionBoundary prevents a chart from implying an unchanged
// check across execution-defining configuration edits.
type AvailabilityRevisionBoundary struct {
Revision int64 `json:"revision"`
At time.Time `json:"at"`
}
// AvailabilityHistoryTarget is the source-owned result for one target.
type AvailabilityHistoryTarget struct {
TargetID string `json:"targetId"`
Summary AvailabilityHistorySummary `json:"summary"`
Buckets []AvailabilityHistoryBucket `json:"buckets"`
RevisionBoundaries []AvailabilityRevisionBoundary `json:"revisionBoundaries"`
}
type availabilityAggregate struct {
reachableMillis int64
unreachableMillis int64
indeterminateMillis int64
latencyCount int64
latencySum int64
latencyMin int64
latencyMax int64
}
func (s *Store) initAvailabilityHistorySchema() error {
_, err := s.db.Exec(`
CREATE TABLE IF NOT EXISTS availability_observations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
observation_id TEXT NOT NULL UNIQUE,
target_id TEXT NOT NULL,
config_revision INTEGER NOT NULL,
outcome TEXT NOT NULL CHECK(outcome IN ('reachable', 'unreachable', 'indeterminate')),
observed_at_ns INTEGER NOT NULL,
timeline_at_ns INTEGER NOT NULL,
ingested_at_ns INTEGER NOT NULL,
valid_until_ns INTEGER NOT NULL,
execution_source TEXT NOT NULL CHECK(execution_source IN ('local', 'assigned_agent')),
latency_millis INTEGER
);
CREATE INDEX IF NOT EXISTS idx_availability_observations_target_timeline
ON availability_observations(target_id, timeline_at_ns, id);
CREATE INDEX IF NOT EXISTS idx_availability_observations_retention
ON availability_observations(timeline_at_ns);
CREATE TABLE IF NOT EXISTS availability_history_buckets (
target_id TEXT NOT NULL,
tier TEXT NOT NULL CHECK(tier IN ('minute', 'hourly', 'daily')),
bucket_start INTEGER NOT NULL,
reachable_millis INTEGER NOT NULL DEFAULT 0,
unreachable_millis INTEGER NOT NULL DEFAULT 0,
indeterminate_millis INTEGER NOT NULL DEFAULT 0,
reachable_count INTEGER NOT NULL DEFAULT 0,
unreachable_count INTEGER NOT NULL DEFAULT 0,
indeterminate_count INTEGER NOT NULL DEFAULT 0,
latency_count INTEGER NOT NULL DEFAULT 0,
latency_sum INTEGER NOT NULL DEFAULT 0,
latency_min INTEGER,
latency_max INTEGER,
PRIMARY KEY(target_id, tier, bucket_start)
);
CREATE INDEX IF NOT EXISTS idx_availability_buckets_retention
ON availability_history_buckets(tier, bucket_start);
CREATE TABLE IF NOT EXISTS availability_revision_boundaries (
target_id TEXT NOT NULL,
revision INTEGER NOT NULL,
started_at_ns INTEGER NOT NULL,
PRIMARY KEY(target_id, revision)
);
CREATE INDEX IF NOT EXISTS idx_availability_boundaries_target_time
ON availability_revision_boundaries(target_id, started_at_ns);
`)
if err != nil {
return fmt.Errorf("create availability history schema: %w", err)
}
return nil
}
func normalizeAvailabilityObservation(observation AvailabilityObservation) (AvailabilityObservation, error) {
observation.ObservationID = strings.TrimSpace(observation.ObservationID)
observation.TargetID = strings.TrimSpace(observation.TargetID)
if observation.ObservationID == "" || observation.TargetID == "" {
return AvailabilityObservation{}, fmt.Errorf("availability observation and target ids are required")
}
if observation.ConfigRevision <= 0 {
observation.ConfigRevision = 1
}
switch observation.Outcome {
case AvailabilityReachable, AvailabilityUnreachable, AvailabilityIndeterminate:
default:
return AvailabilityObservation{}, fmt.Errorf("unsupported availability outcome %q", observation.Outcome)
}
switch observation.ExecutionSource {
case AvailabilitySourceLocal, AvailabilitySourceAssignedAgent:
default:
return AvailabilityObservation{}, fmt.Errorf("unsupported availability execution source %q", observation.ExecutionSource)
}
if observation.TimelineAt.IsZero() || observation.IngestedAt.IsZero() {
return AvailabilityObservation{}, fmt.Errorf("availability timeline and ingestion times are required")
}
observation.TimelineAt = observation.TimelineAt.UTC()
observation.IngestedAt = observation.IngestedAt.UTC()
if observation.ObservedAt.IsZero() {
observation.ObservedAt = observation.TimelineAt
} else {
observation.ObservedAt = observation.ObservedAt.UTC()
}
if observation.ValidFor <= 0 {
return AvailabilityObservation{}, fmt.Errorf("availability validity window must be positive")
}
if observation.ValidFor > availabilityMaxValidity {
observation.ValidFor = availabilityMaxValidity
}
if observation.LatencyMillis != nil {
latency := *observation.LatencyMillis
if observation.Outcome != AvailabilityReachable || latency < 0 {
observation.LatencyMillis = nil
} else if latency > int64(availabilityMaxValidity/time.Millisecond) {
latency = int64(availabilityMaxValidity / time.Millisecond)
observation.LatencyMillis = &latency
}
}
return observation, nil
}
// WriteAvailabilityObservationSync is the read-your-writes test and seeding
// path. Live monitoring uses the bounded variant below.
func (s *Store) WriteAvailabilityObservationSync(observation AvailabilityObservation) error {
normalized, err := normalizeAvailabilityObservation(observation)
if err != nil {
return err
}
s.enqueueAndWait(writeRequest{availability: []AvailabilityObservation{normalized}})
return nil
}
// WriteAvailabilityObservationBounded keeps a slow history disk from blocking
// polling. Once queued, the observation remains ordered with numeric writes.
func (s *Store) WriteAvailabilityObservationBounded(observation AvailabilityObservation) error {
normalized, err := normalizeAvailabilityObservation(observation)
if err != nil {
return err
}
s.boundedEnqueueAndWait(writeRequest{availability: []AvailabilityObservation{normalized}})
return nil
}
// DeleteAvailabilityTargetHistory removes raw observations, compact buckets,
// and revision boundaries behind the same writer barrier as ingestion.
func (s *Store) DeleteAvailabilityTargetHistory(targetID string) {
targetID = strings.TrimSpace(targetID)
if s == nil || targetID == "" || s.stopping.Load() {
return
}
s.enqueueAndWait(writeRequest{availabilityDeletes: []string{targetID}})
}
func (s *Store) writeAvailabilityBatch(observations []AvailabilityObservation) {
if len(observations) == 0 {
return
}
tx, err := s.db.Begin()
if err != nil {
log.Error().Err(err).Msg("Failed to begin availability history transaction")
return
}
defer func() { _ = tx.Rollback() }()
seen := make(map[string]struct{}, len(observations))
for _, observation := range observations {
normalized, err := normalizeAvailabilityObservation(observation)
if err != nil {
log.Warn().Err(err).Msg("Dropping invalid availability observation")
continue
}
if _, exists := seen[normalized.ObservationID]; exists {
continue
}
seen[normalized.ObservationID] = struct{}{}
if err := insertAvailabilityObservation(tx, normalized); err != nil {
log.Warn().Err(err).Str("target_id", normalized.TargetID).Msg("Failed to write availability observation")
}
}
if err := tx.Commit(); err != nil {
log.Error().Err(err).Int("batch_size", len(observations)).Msg("Failed to commit availability history batch")
}
}
func insertAvailabilityObservation(tx *pdb.InstrumentedTx, observation AvailabilityObservation) error {
var latency any
if observation.LatencyMillis != nil {
latency = *observation.LatencyMillis
}
result, err := tx.Exec(`
INSERT OR IGNORE INTO availability_observations (
observation_id, target_id, config_revision, outcome, observed_at_ns,
timeline_at_ns, ingested_at_ns, valid_until_ns, execution_source, latency_millis
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`, observation.ObservationID, observation.TargetID, observation.ConfigRevision, string(observation.Outcome),
observation.ObservedAt.UnixNano(), observation.TimelineAt.UnixNano(), observation.IngestedAt.UnixNano(),
observation.TimelineAt.Add(observation.ValidFor).UnixNano(), string(observation.ExecutionSource), latency)
if err != nil {
return err
}
affected, err := result.RowsAffected()
if err != nil || affected == 0 {
return err
}
rowID, err := result.LastInsertId()
if err != nil {
return err
}
if _, err := tx.Exec(`
INSERT OR IGNORE INTO availability_revision_boundaries (target_id, revision, started_at_ns)
VALUES (?, ?, ?)
`, observation.TargetID, observation.ConfigRevision, observation.TimelineAt.UnixNano()); err != nil {
return err
}
if err := addAvailabilityObservationAggregates(tx, observation); err != nil {
return err
}
var previousOutcome string
var previousStart, previousValidUntil int64
err = tx.QueryRow(`
SELECT outcome, timeline_at_ns, valid_until_ns
FROM availability_observations
WHERE target_id = ? AND id < ?
ORDER BY id DESC LIMIT 1
`, observation.TargetID, rowID).Scan(&previousOutcome, &previousStart, &previousValidUntil)
if err == sql.ErrNoRows {
return nil
}
if err != nil {
return err
}
end := observation.TimelineAt.UnixNano()
if previousValidUntil < end {
end = previousValidUntil
}
if end <= previousStart {
return nil
}
return addAvailabilityDurationAggregates(tx, observation.TargetID, AvailabilityOutcome(previousOutcome), previousStart, end)
}
var availabilityStorageTiers = []struct {
tier string
size time.Duration
}{
{string(TierMinute), time.Minute},
{string(TierHourly), time.Hour},
{string(TierDaily), 24 * time.Hour},
}
func addAvailabilityObservationAggregates(tx *pdb.InstrumentedTx, observation AvailabilityObservation) error {
for _, tier := range availabilityStorageTiers {
bucketStart := observation.TimelineAt.Truncate(tier.size).Unix()
reachableCount, unreachableCount, indeterminateCount := int64(0), int64(0), int64(0)
switch observation.Outcome {
case AvailabilityReachable:
reachableCount = 1
case AvailabilityUnreachable:
unreachableCount = 1
case AvailabilityIndeterminate:
indeterminateCount = 1
}
latencyCount, latencySum := int64(0), int64(0)
var latencyMin, latencyMax any
if observation.Outcome == AvailabilityReachable && observation.LatencyMillis != nil {
latencyCount = 1
latencySum = *observation.LatencyMillis
latencyMin = *observation.LatencyMillis
latencyMax = *observation.LatencyMillis
}
_, err := tx.Exec(`
INSERT INTO availability_history_buckets (
target_id, tier, bucket_start, reachable_count, unreachable_count,
indeterminate_count, latency_count, latency_sum, latency_min, latency_max
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(target_id, tier, bucket_start) DO UPDATE SET
reachable_count = reachable_count + excluded.reachable_count,
unreachable_count = unreachable_count + excluded.unreachable_count,
indeterminate_count = indeterminate_count + excluded.indeterminate_count,
latency_count = latency_count + excluded.latency_count,
latency_sum = latency_sum + excluded.latency_sum,
latency_min = CASE WHEN excluded.latency_min IS NULL THEN latency_min
WHEN latency_min IS NULL THEN excluded.latency_min ELSE MIN(latency_min, excluded.latency_min) END,
latency_max = CASE WHEN excluded.latency_max IS NULL THEN latency_max
WHEN latency_max IS NULL THEN excluded.latency_max ELSE MAX(latency_max, excluded.latency_max) END
`, observation.TargetID, tier.tier, bucketStart, reachableCount, unreachableCount,
indeterminateCount, latencyCount, latencySum, latencyMin, latencyMax)
if err != nil {
return err
}
}
return nil
}
func addAvailabilityDurationAggregates(tx *pdb.InstrumentedTx, targetID string, outcome AvailabilityOutcome, startNS, endNS int64) error {
for _, tier := range availabilityStorageTiers {
cursor := time.Unix(0, startNS).UTC()
end := time.Unix(0, endNS).UTC()
for cursor.Before(end) {
bucketStart := cursor.Truncate(tier.size)
segmentEnd := bucketStart.Add(tier.size)
if segmentEnd.After(end) {
segmentEnd = end
}
millis := segmentEnd.Sub(cursor).Milliseconds()
if millis > 0 {
reachable, unreachable, indeterminate := int64(0), int64(0), int64(0)
switch outcome {
case AvailabilityReachable:
reachable = millis
case AvailabilityUnreachable:
unreachable = millis
case AvailabilityIndeterminate:
indeterminate = millis
}
if _, err := tx.Exec(`
INSERT INTO availability_history_buckets (
target_id, tier, bucket_start, reachable_millis, unreachable_millis, indeterminate_millis
) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(target_id, tier, bucket_start) DO UPDATE SET
reachable_millis = reachable_millis + excluded.reachable_millis,
unreachable_millis = unreachable_millis + excluded.unreachable_millis,
indeterminate_millis = indeterminate_millis + excluded.indeterminate_millis
`, targetID, tier.tier, bucketStart.Unix(), reachable, unreachable, indeterminate); err != nil {
return err
}
}
cursor = segmentEnd
}
}
return nil
}
func (s *Store) deleteAvailabilityTargets(targetIDs []string) {
tx, err := s.db.Begin()
if err != nil {
log.Warn().Err(err).Msg("Failed to begin availability history deletion")
return
}
defer func() { _ = tx.Rollback() }()
for _, targetID := range targetIDs {
targetID = strings.TrimSpace(targetID)
if targetID == "" {
continue
}
for _, statement := range []string{
`DELETE FROM availability_observations WHERE target_id = ?`,
`DELETE FROM availability_history_buckets WHERE target_id = ?`,
`DELETE FROM availability_revision_boundaries WHERE target_id = ?`,
} {
if _, err := tx.Exec(statement, targetID); err != nil {
log.Warn().Err(err).Str("target_id", targetID).Msg("Failed to delete availability history")
return
}
}
}
if err := tx.Commit(); err != nil {
log.Warn().Err(err).Msg("Failed to commit availability history deletion")
}
}
func (s *Store) runAvailabilityRetention() {
now := time.Now().UTC()
if _, err := s.db.Exec(`DELETE FROM availability_observations WHERE timeline_at_ns < ?`,
now.Add(-s.effectiveRetention(availabilityRawRetention, now)).UnixNano()); err != nil {
log.Warn().Err(err).Msg("Failed to prune raw availability observations")
}
for _, tier := range []struct {
name string
retention time.Duration
}{
{string(TierMinute), availabilityMinuteRetention},
{string(TierHourly), availabilityHourlyRetention},
{string(TierDaily), availabilityDailyRetention},
} {
cutoff := now.Add(-s.effectiveRetention(tier.retention, now)).Unix()
if _, err := s.db.Exec(`DELETE FROM availability_history_buckets WHERE tier = ? AND bucket_start < ?`, tier.name, cutoff); err != nil {
log.Warn().Err(err).Str("tier", tier.name).Msg("Failed to prune availability history buckets")
}
}
cutoff := now.Add(-s.effectiveRetention(availabilityDailyRetention, now)).UnixNano()
if _, err := s.db.Exec(`DELETE FROM availability_revision_boundaries WHERE started_at_ns < ?`, cutoff); err != nil {
log.Warn().Err(err).Msg("Failed to prune availability revision boundaries")
}
}
// QueryAvailabilityHistory reads all requested targets with three bounded
// batch queries (buckets, live tails, and revision boundaries), never one
// query per target.
func (s *Store) QueryAvailabilityHistory(targetIDs []string, start, end time.Time, maxBuckets int) (map[string]AvailabilityHistoryTarget, error) {
ids := normalizeAvailabilityTargetIDs(targetIDs)
if len(ids) == 0 {
return map[string]AvailabilityHistoryTarget{}, nil
}
if len(ids) > availabilityMaxBatchTargets {
return nil, fmt.Errorf("availability history supports at most %d target ids", availabilityMaxBatchTargets)
}
start, end = start.UTC(), end.UTC()
if !end.After(start) {
return nil, fmt.Errorf("availability history end must be after start")
}
if maxBuckets <= 0 || maxBuckets > availabilityMaxBuckets {
maxBuckets = availabilityMaxBuckets
}
step := availabilityPresentationStep(end.Sub(start), maxBuckets)
results := make(map[string]AvailabilityHistoryTarget, len(ids))
aggregates := make(map[string][]availabilityAggregate, len(ids))
for _, id := range ids {
buckets := makeAvailabilityBuckets(start, end, step)
results[id] = AvailabilityHistoryTarget{TargetID: id, Buckets: buckets, RevisionBoundaries: []AvailabilityRevisionBoundary{}}
aggregates[id] = make([]availabilityAggregate, len(buckets))
}
if err := s.queryAvailabilityBuckets(ids, start, end, step, aggregates); err != nil {
return nil, err
}
if err := s.queryAvailabilityLiveTails(ids, start, end, step, aggregates); err != nil {
return nil, err
}
boundaries, err := s.queryAvailabilityRevisionBoundaries(ids, start, end)
if err != nil {
return nil, err
}
windowMillis := end.Sub(start).Milliseconds()
for _, id := range ids {
target := results[id]
total := availabilityAggregate{}
for index := range target.Buckets {
aggregate := aggregates[id][index]
bucketMillis := target.Buckets[index].End.Sub(target.Buckets[index].Start).Milliseconds()
target.Buckets[index] = finalizeAvailabilityBucket(target.Buckets[index], aggregate, bucketMillis)
mergeAvailabilityAggregate(&total, aggregate)
}
target.Summary = finalizeAvailabilitySummary(total, windowMillis)
target.RevisionBoundaries = boundaries[id]
results[id] = target
}
return results, nil
}
func normalizeAvailabilityTargetIDs(targetIDs []string) []string {
seen := make(map[string]struct{}, len(targetIDs))
ids := make([]string, 0, len(targetIDs))
for _, id := range targetIDs {
id = strings.TrimSpace(id)
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
ids = append(ids, id)
}
return ids
}
func availabilityPresentationStep(window time.Duration, maxBuckets int) time.Duration {
steps := []time.Duration{time.Minute, 2 * time.Minute, 5 * time.Minute, 6 * time.Minute, 12 * time.Minute, 30 * time.Minute, time.Hour, 2 * time.Hour, 6 * time.Hour, 12 * time.Hour, 24 * time.Hour, 4 * 24 * time.Hour}
minimum := time.Duration(math.Ceil(float64(window) / float64(maxBuckets)))
for _, step := range steps {
if step >= minimum {
return step
}
}
return time.Duration(math.Ceil(float64(minimum)/(float64(24*time.Hour)))) * 24 * time.Hour
}
func makeAvailabilityBuckets(start, end time.Time, step time.Duration) []AvailabilityHistoryBucket {
count := int(math.Ceil(float64(end.Sub(start)) / float64(step)))
buckets := make([]AvailabilityHistoryBucket, 0, count)
for cursor := start; cursor.Before(end); cursor = cursor.Add(step) {
bucketEnd := cursor.Add(step)
if bucketEnd.After(end) {
bucketEnd = end
}
buckets = append(buckets, AvailabilityHistoryBucket{Start: cursor, End: bucketEnd})
}
return buckets
}
func availabilityPlaceholders(count int) string {
return strings.TrimSuffix(strings.Repeat("?,", count), ",")
}
func (s *Store) queryAvailabilityBuckets(ids []string, start, end time.Time, step time.Duration, aggregates map[string][]availabilityAggregate) error {
minuteStart := start
if candidate := end.Add(-7 * 24 * time.Hour); candidate.After(minuteStart) {
minuteStart = candidate.Truncate(time.Hour)
}
hourlyStart := start
if candidate := end.Add(-90 * 24 * time.Hour); candidate.After(hourlyStart) {
hourlyStart = candidate.Truncate(24 * time.Hour)
}
args := make([]any, 0, len(ids)+9)
for _, id := range ids {
args = append(args, id)
}
args = append(args,
string(TierDaily), start.Unix(), hourlyStart.Unix(),
string(TierHourly), hourlyStart.Unix(), minuteStart.Unix(),
string(TierMinute), minuteStart.Unix(), end.Unix(),
)
rows, err := s.db.Query(fmt.Sprintf(`
SELECT target_id, bucket_start, reachable_millis, unreachable_millis,
indeterminate_millis, latency_count, latency_sum,
COALESCE(latency_min, 0), COALESCE(latency_max, 0)
FROM availability_history_buckets
WHERE target_id IN (%s) AND (
(tier = ? AND bucket_start >= ? AND bucket_start < ?) OR
(tier = ? AND bucket_start >= ? AND bucket_start < ?) OR
(tier = ? AND bucket_start >= ? AND bucket_start < ?)
)
ORDER BY target_id, bucket_start
`, availabilityPlaceholders(len(ids))), args...)
if err != nil {
return fmt.Errorf("query availability history buckets: %w", err)
}
defer rows.Close()
for rows.Next() {
var targetID string
var bucketStart, reachable, unreachable, indeterminate, latencyCount, latencySum, latencyMin, latencyMax int64
if err := rows.Scan(&targetID, &bucketStart, &reachable, &unreachable, &indeterminate, &latencyCount, &latencySum, &latencyMin, &latencyMax); err != nil {
return err
}
index := int(time.Unix(bucketStart, 0).Sub(start) / step)
if index < 0 || index >= len(aggregates[targetID]) {
continue
}
mergeAvailabilityAggregate(&aggregates[targetID][index], availabilityAggregate{
reachableMillis: reachable, unreachableMillis: unreachable, indeterminateMillis: indeterminate,
latencyCount: latencyCount, latencySum: latencySum, latencyMin: latencyMin, latencyMax: latencyMax,
})
}
return rows.Err()
}
func (s *Store) queryAvailabilityLiveTails(ids []string, start, end time.Time, step time.Duration, aggregates map[string][]availabilityAggregate) error {
args := make([]any, 0, len(ids))
for _, id := range ids {
args = append(args, id)
}
rows, err := s.db.Query(fmt.Sprintf(`
WITH ranked AS (
SELECT target_id, outcome, timeline_at_ns, valid_until_ns,
ROW_NUMBER() OVER (PARTITION BY target_id ORDER BY timeline_at_ns DESC, id DESC) AS rank
FROM availability_observations WHERE target_id IN (%s)
)
SELECT target_id, outcome, timeline_at_ns, valid_until_ns FROM ranked WHERE rank = 1
`, availabilityPlaceholders(len(ids))), args...)
if err != nil {
return fmt.Errorf("query availability live tails: %w", err)
}
defer rows.Close()
for rows.Next() {
var targetID, outcome string
var tailStartNS, validUntilNS int64
if err := rows.Scan(&targetID, &outcome, &tailStartNS, &validUntilNS); err != nil {
return err
}
tailStart := time.Unix(0, tailStartNS).UTC()
if tailStart.Before(start) {
tailStart = start
}
tailEnd := time.Unix(0, validUntilNS).UTC()
if tailEnd.After(end) {
tailEnd = end
}
addAvailabilityRangeToPresentation(aggregates[targetID], start, step, AvailabilityOutcome(outcome), tailStart, tailEnd)
}
return rows.Err()
}
func addAvailabilityRangeToPresentation(buckets []availabilityAggregate, start time.Time, step time.Duration, outcome AvailabilityOutcome, cursor, end time.Time) {
if !end.After(cursor) {
return
}
for cursor.Before(end) {
index := int(cursor.Sub(start) / step)
if index < 0 || index >= len(buckets) {
return
}
segmentEnd := start.Add(time.Duration(index+1) * step)
if segmentEnd.After(end) {
segmentEnd = end
}
millis := segmentEnd.Sub(cursor).Milliseconds()
switch outcome {
case AvailabilityReachable:
buckets[index].reachableMillis += millis
case AvailabilityUnreachable:
buckets[index].unreachableMillis += millis
case AvailabilityIndeterminate:
buckets[index].indeterminateMillis += millis
}
cursor = segmentEnd
}
}
func (s *Store) queryAvailabilityRevisionBoundaries(ids []string, start, end time.Time) (map[string][]AvailabilityRevisionBoundary, error) {
args := make([]any, 0, len(ids)+2)
for _, id := range ids {
args = append(args, id)
}
args = append(args, start.UnixNano(), end.UnixNano())
rows, err := s.db.Query(fmt.Sprintf(`
SELECT target_id, revision, started_at_ns
FROM availability_revision_boundaries
WHERE target_id IN (%s) AND started_at_ns >= ? AND started_at_ns < ?
ORDER BY target_id, started_at_ns
`, availabilityPlaceholders(len(ids))), args...)
if err != nil {
return nil, fmt.Errorf("query availability revision boundaries: %w", err)
}
defer rows.Close()
result := make(map[string][]AvailabilityRevisionBoundary, len(ids))
for rows.Next() {
var targetID string
var revision, at int64
if err := rows.Scan(&targetID, &revision, &at); err != nil {
return nil, err
}
result[targetID] = append(result[targetID], AvailabilityRevisionBoundary{Revision: revision, At: time.Unix(0, at).UTC()})
}
return result, rows.Err()
}
func mergeAvailabilityAggregate(target *availabilityAggregate, source availabilityAggregate) {
target.reachableMillis += source.reachableMillis
target.unreachableMillis += source.unreachableMillis
target.indeterminateMillis += source.indeterminateMillis
if source.latencyCount > 0 {
if target.latencyCount == 0 || source.latencyMin < target.latencyMin {
target.latencyMin = source.latencyMin
}
if target.latencyCount == 0 || source.latencyMax > target.latencyMax {
target.latencyMax = source.latencyMax
}
target.latencyCount += source.latencyCount
target.latencySum += source.latencySum
}
}
func finalizeAvailabilityBucket(bucket AvailabilityHistoryBucket, aggregate availabilityAggregate, bucketMillis int64) AvailabilityHistoryBucket {
known := aggregate.reachableMillis + aggregate.unreachableMillis + aggregate.indeterminateMillis
if known > bucketMillis {
known = bucketMillis
}
bucket.ReachableSeconds = millisecondsToSeconds(aggregate.reachableMillis)
bucket.UnreachableSeconds = millisecondsToSeconds(aggregate.unreachableMillis)
bucket.IndeterminateSeconds = millisecondsToSeconds(aggregate.indeterminateMillis)
bucket.UnknownSeconds = millisecondsToSeconds(bucketMillis - known)
bucket.LatencyMillis = availabilityLatencySummary(aggregate)
return bucket
}
func finalizeAvailabilitySummary(aggregate availabilityAggregate, windowMillis int64) AvailabilityHistorySummary {
known := aggregate.reachableMillis + aggregate.unreachableMillis + aggregate.indeterminateMillis
if known > windowMillis {
known = windowMillis
}
summary := AvailabilityHistorySummary{
ReachableSeconds: millisecondsToSeconds(aggregate.reachableMillis),
UnreachableSeconds: millisecondsToSeconds(aggregate.unreachableMillis),
IndeterminateSeconds: millisecondsToSeconds(aggregate.indeterminateMillis),
UnknownSeconds: millisecondsToSeconds(windowMillis - known),
ReachableLatencyMillis: availabilityLatencySummary(aggregate),
}
if windowMillis > 0 {
summary.CoveragePercent = roundPercent(float64(known) * 100 / float64(windowMillis))
}
determinate := aggregate.reachableMillis + aggregate.unreachableMillis
if determinate > 0 {
value := roundPercent(float64(aggregate.reachableMillis) * 100 / float64(determinate))
summary.AvailabilityPercent = &value
}
return summary
}
func availabilityLatencySummary(aggregate availabilityAggregate) *AvailabilityLatencySummary {
if aggregate.latencyCount <= 0 {
return nil
}
return &AvailabilityLatencySummary{
Average: math.Round(float64(aggregate.latencySum)/float64(aggregate.latencyCount)*100) / 100,
Min: aggregate.latencyMin,
Max: aggregate.latencyMax,
}
}
func millisecondsToSeconds(value int64) float64 {
if value <= 0 {
return 0
}
return math.Round(float64(value)/10) / 100
}
func roundPercent(value float64) float64 {
return math.Round(value*100) / 100
}
// AvailabilityHistoryRowCounts is a narrow diagnostic used by contract tests
// to prove idempotency and full target deletion without exposing stored data.
func (s *Store) AvailabilityHistoryRowCounts(targetID string) (raw, buckets, boundaries int64, err error) {
targetID = strings.TrimSpace(targetID)
queries := []struct {
statement string
value *int64
}{
{`SELECT COUNT(*) FROM availability_observations WHERE target_id = ?`, &raw},
{`SELECT COUNT(*) FROM availability_history_buckets WHERE target_id = ?`, &buckets},
{`SELECT COUNT(*) FROM availability_revision_boundaries WHERE target_id = ?`, &boundaries},
}
for _, query := range queries {
if scanErr := s.db.QueryRow(query.statement, targetID).Scan(query.value); scanErr != nil {
return 0, 0, 0, scanErr
}
}
return raw, buckets, boundaries, nil
}
// SortedAvailabilityHistoryTargets converts the batch map to stable input
// order when callers need a slice response.
func SortedAvailabilityHistoryTargets(results map[string]AvailabilityHistoryTarget, order []string) []AvailabilityHistoryTarget {
items := make([]AvailabilityHistoryTarget, 0, len(results))
seen := make(map[string]struct{}, len(results))
for _, id := range order {
if item, ok := results[id]; ok {
items = append(items, item)
seen[id] = struct{}{}
}
}
remaining := make([]string, 0, len(results)-len(seen))
for id := range results {
if _, ok := seen[id]; !ok {
remaining = append(remaining, id)
}
}
sort.Strings(remaining)
for _, id := range remaining {
items = append(items, results[id])
}
return items
}