pulse/pkg/metrics/store_query_plan_test.go

package metrics

import (
	"database/sql"
	"fmt"
	"net/url"
	"strings"
	"testing"
	"time"

	_ "modernc.org/sqlite"
)

// TestQueryPlansUseIndexes validates that all critical metrics-store SQL queries
// use indexed lookups (SEARCH) rather than full table scans (SCAN TABLE).
// This catches index regressions — if a schema migration drops an index or a
// query change breaks index eligibility, these tests fail immediately.
//
// Each sub-test runs EXPLAIN QUERY PLAN on SQL the store uses at
// runtime. Queries are classified by how they should access the table:
//   - searchRequired: the plan must use SEARCH (B-tree point/range lookup)
//   - allowCoveringIndexScan: the plan may use a covering-index scan, which
//     reads only the index without touching the table — acceptable when no
//     index has the query's filter columns as a prefix
//
// In all cases, a bare "SCAN TABLE metrics" (full table scan) is rejected.
//
// NOTE: Maintenance SQL is duplicated from store.go. Retained reads exercise
// the shared runtime builder directly in TestRetainedQueryPlansUseIndexes.
// If store.go's schema or queries change, these tests must be updated in
// lockstep — a mismatch means they validate stale SQL.
func TestQueryPlansUseIndexes(t *testing.T) {
	db := newPlanTestDB(t)

	// farFuture is an int64 timestamp safe on 32-bit platforms.
	const farFuture int64 = 9999999999

	tests := []struct {
		name  string
		query string
		args  []any
		// wantIndex, if non-empty, asserts this index name appears on the
		// same plan line as a SEARCH on the metrics table.
		wantIndex string
		// allowCoveringIndexScan permits "SCAN metrics USING COVERING INDEX ..."
		// for queries where the planner correctly chooses a covering-index
		// scan over a table scan. When false, only SEARCH is accepted.
		allowCoveringIndexScan bool
		// allowIndexScan permits "SCAN metrics USING INDEX ..." (index-ordered
		// scan) for queries where the planner uses an index to satisfy GROUP BY
		// or ORDER BY without a separate sort, but still reads table rows.
		allowIndexScan bool
	}{
		{
			name:      "retention delete by tier+time",
			query:     `DELETE FROM metrics WHERE tier = ? AND timestamp < ?`,
			args:      []any{"raw", farFuture},
			wantIndex: "idx_metrics_tier_time",
		},
		{
			name: "rollup candidate discovery (legacy per-candidate path)",
			query: `SELECT DISTINCT resource_type, resource_id, metric_type
				FROM metrics
				WHERE tier = ? AND timestamp >= ? AND timestamp < ?`,
			args: []any{"raw", int64(0), farFuture},
			// This query filters on (tier, timestamp) which aren't the leading
			// columns of most indexes. The planner correctly uses a covering-
			// index scan — reading only the index, never the table.
			allowCoveringIndexScan: true,
		},
		{
			name: "rollup aggregation insert (legacy per-candidate path)",
			query: `INSERT OR IGNORE INTO metrics (resource_type, resource_id, metric_type, value, min_value, max_value, timestamp, tier)
				SELECT
					resource_type, resource_id, metric_type,
					AVG(value) as value,
					MIN(COALESCE(min_value, value)) as min_value,
					MAX(COALESCE(max_value, value)) as max_value,
					(timestamp / ?) * ? as bucket_ts,
					?
				FROM metrics
				WHERE resource_type = ? AND resource_id = ? AND metric_type = ?
				AND tier = ? AND timestamp >= ? AND timestamp < ?
				GROUP BY resource_type, resource_id, metric_type, bucket_ts`,
			args:      []any{int64(60), int64(60), "minute", "vm", "vm-1", "cpu", "raw", int64(0), farFuture},
			wantIndex: metricsIdentityIndex,
		},
		{
			name: "batched rollup aggregation insert",
			query: `INSERT OR IGNORE INTO metrics (resource_type, resource_id, metric_type, value, min_value, max_value, timestamp, tier)
				SELECT
					resource_type,
					resource_id,
					metric_type,
					AVG(value) as value,
					MIN(COALESCE(min_value, value)) as min_value,
					MAX(COALESCE(max_value, value)) as max_value,
					(timestamp / ?) * ? as bucket_ts,
					?
				FROM metrics
				WHERE tier = ? AND timestamp >= ? AND timestamp < ?
				GROUP BY resource_type, resource_id, metric_type, bucket_ts`,
			args: []any{int64(60), int64(60), "minute", "raw", int64(0), farFuture},
			// The SELECT filters on (tier, timestamp) without resource/metric
			// columns. SQLite may use an index-ordered scan (SCAN USING INDEX)
			// on the identity index. A TEMP B-TREE may still be used for
			// GROUP BY. The INSERT side uses the same unique index for ON CONFLICT.
			allowIndexScan: true,
		},
		{
			name:      "max timestamp for tier (rollup scheduling)",
			query:     `SELECT MAX(timestamp) FROM metrics WHERE tier = ?`,
			args:      []any{"raw"},
			wantIndex: "idx_metrics_tier_time",
		},
		{
			name:  "tier count stats (GetStats)",
			query: `SELECT tier, COUNT(*) FROM metrics GROUP BY tier`,
			args:  []any{},
			// GROUP BY tier scans all rows — a covering-index scan on
			// idx_metrics_tier_time reads only the index (tier, timestamp),
			// avoiding the full table row fetch. A SEARCH is also acceptable
			// if the planner chooses a different strategy.
			allowCoveringIndexScan: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			plan := explainQueryPlan(t, db, tt.query, tt.args)

			// Reject a bare full table scan — this is always bad.
			if containsFullTableScan(plan) {
				t.Errorf("query uses full table scan on metrics; expected indexed access\nPlan:\n%s", plan)
			}

			if tt.allowIndexScan {
				// Must use SEARCH, covering-index scan, OR index-ordered scan.
				if !containsMetricsSearch(plan) && !containsCoveringIndexScan(plan) && !containsIndexScan(plan) {
					t.Errorf("query plan has neither SEARCH on metrics nor index scan\nPlan:\n%s", plan)
				}
			} else if tt.allowCoveringIndexScan {
				// Must use EITHER a SEARCH on metrics or a covering-index scan on metrics.
				if !containsMetricsSearch(plan) && !containsCoveringIndexScan(plan) {
					t.Errorf("query plan has neither SEARCH on metrics nor covering-index scan\nPlan:\n%s", plan)
				}
			} else {
				// Must contain at least one SEARCH on the metrics table.
				// We scope to "metrics" to avoid false passes from constraint-
				// handling SEARCH nodes in INSERT ... ON CONFLICT plans.
				if !containsMetricsSearch(plan) {
					t.Errorf("query plan does not contain SEARCH on metrics table; may not be using an index\nPlan:\n%s", plan)
				}
			}

			// If a specific index is expected, verify that the same plan line
			// that does a SEARCH on metrics also references that index.
			// This prevents false passes where one node SEARCHes via
			// conflict handling and a different node uses the expected index.
			if tt.wantIndex != "" && !containsMetricsSearchWithIndex(plan, tt.wantIndex) {
				t.Errorf("expected SEARCH on metrics using index %q not found in plan\nPlan:\n%s", tt.wantIndex, plan)
			}
		})
	}
}

// Query, QueryAll and both batch variants share this exact runtime SQL. The
// display step is applied in SQLite after tier reconciliation.
func TestRetainedQueryPlansUseIndexes(t *testing.T) {
	db := newPlanTestDB(t)
	store := &Store{}
	for _, window := range []time.Duration{time.Hour, 24 * time.Hour, 7 * 24 * time.Hour, 30 * 24 * time.Hour} {
		for _, filtered := range []bool{false, true} {
			for _, step := range []int64{0, 60} {
				t.Run(fmt.Sprintf("%s/filtered_%v/step_%d", window, filtered, step), func(t *testing.T) {
					var metrics []string
					if filtered {
						metrics = []string{"cpu", "memory"}
					}
					end := time.Unix(2000000000, 0)
					sql, args := retainedQuerySQL("vm", []string{"vm-1", "vm-2", "vm-3"}, metrics, end.Add(-window), end, step, store.tierFallbacks(window), false)
					plan := explainQueryPlan(t, db, sql, args)
					searches := 0
					for _, line := range strings.Split(plan, "\n") {
						if strings.Contains(line, "SCAN m ") || strings.Contains(line, "SCAN h ") {
							t.Fatalf("unbounded metrics read: %s", plan)
						}
						if strings.Contains(line, "SEARCH m ") || strings.Contains(line, "SEARCH h ") {
							searches++
							for _, constraint := range []string{"resource_type=?", "resource_id=?", "tier=?", "timestamp>?", "timestamp<?"} {
								if !strings.Contains(line, constraint) {
									t.Fatalf("retained search missing indexed %s: %s", constraint, line)
								}
							}
						}
					}
					n := len(store.tierFallbacks(window))
					if searches != n*(n+1)/2 {
						t.Fatalf("expected every tier and overlap probe indexed, got %d: %s", searches, plan)
					}
				})
			}
		}
	}
}

// A single present tier uses direct aggregation. It must retain the same
// bounded identity/tier/time lookup as the reconciled multi-tier query.
func TestRetainedSingleTierQueryPlansUseIndexes(t *testing.T) {
	db := newPlanTestDB(t)
	end := time.Unix(2000000000, 0)
	for _, tier := range []Tier{TierRaw, TierMinute, TierHourly, TierDaily} {
		for _, filtered := range []bool{false, true} {
			t.Run(fmt.Sprintf("%s/filtered_%v", tier, filtered), func(t *testing.T) {
				var metrics []string
				if filtered {
					metrics = []string{"cpu"}
				}
				query, args := retainedQuerySQL("vm", []string{"vm-1"}, metrics, end.Add(-time.Hour), end, 60, []Tier{tier}, false)
				plan := explainQueryPlan(t, db, query, args)
				if strings.Contains(plan, "SCAN m ") || !strings.Contains(plan, "SEARCH m ") {
					t.Fatalf("single tier must use a bounded lookup: %s", plan)
				}
				for _, constraint := range []string{"resource_type=?", "resource_id=?", "tier=?", "timestamp>?", "timestamp<?"} {
					if !strings.Contains(plan, constraint) {
						t.Fatalf("missing indexed %s: %s", constraint, plan)
					}
				}
			})
		}
	}
}

func TestAvailabilityHistoryQueryPlansUseIndexes(t *testing.T) {
	db := newPlanTestDB(t)

	tests := []struct {
		name      string
		query     string
		args      []any
		wantIndex string
	}{
		{
			name: "fleet rollup batch",
			query: `SELECT target_id, bucket_start, reachable_millis, unreachable_millis,
				indeterminate_millis, latency_count, latency_sum,
				COALESCE(latency_min, 0), COALESCE(latency_max, 0)
				FROM availability_history_buckets
				WHERE target_id IN (?, ?, ?) AND tier = ? AND bucket_start >= ? AND bucket_start < ?
				ORDER BY target_id, bucket_start`,
			args:      []any{"target-1", "target-2", "target-3", "minute", int64(0), int64(9999999999)},
			wantIndex: "sqlite_autoindex_availability_history_buckets_1",
		},
		{
			name: "latest live tail batch",
			query: `SELECT target_id, timeline_at_ns, valid_until_ns
				FROM availability_observations
				WHERE target_id IN (?, ?, ?)
				ORDER BY target_id, timeline_at_ns DESC, id DESC`,
			args:      []any{"target-1", "target-2", "target-3"},
			wantIndex: "idx_availability_observations_target_timeline",
		},
		{
			name: "revision boundary batch",
			query: `SELECT target_id, revision, started_at_ns
				FROM availability_revision_boundaries
				WHERE target_id IN (?, ?, ?) AND started_at_ns >= ? AND started_at_ns < ?
				ORDER BY target_id, started_at_ns`,
			args:      []any{"target-1", "target-2", "target-3", int64(0), int64(9999999999)},
			wantIndex: "idx_availability_boundaries_target_time",
		},
		{
			name:      "raw retention",
			query:     `DELETE FROM availability_observations WHERE timeline_at_ns < ?`,
			args:      []any{int64(9999999999)},
			wantIndex: "idx_availability_observations_retention",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			plan := explainQueryPlan(t, db, tt.query, tt.args)
			if !strings.Contains(plan, "SEARCH") || !strings.Contains(plan, tt.wantIndex) {
				t.Fatalf("expected indexed availability-history search using %s\nPlan:\n%s", tt.wantIndex, plan)
			}
		})
	}
}

// lineRefersToMetrics returns true if a plan line references the "metrics"
// table (not metrics_meta). SQLite may emit "SEARCH metrics USING ...",
// "SEARCH TABLE metrics ...", "SCAN metrics ...", or "SCAN TABLE metrics ...".
func lineRefersToMetrics(line string) bool {
	// Match "metrics " (with space after), "metrics(" (function-like),
	// or "metrics" at end of line. Reject "metrics_meta".
	for _, needle := range []string{"metrics ", "metrics("} {
		if strings.Contains(line, needle) && !strings.Contains(line, "metrics_meta") {
			return true
		}
	}
	// Handle "metrics" at end of line (e.g., bare "SCAN metrics").
	if strings.HasSuffix(strings.TrimSpace(line), "metrics") && !strings.Contains(line, "metrics_meta") {
		return true
	}
	return false
}

// containsMetricsSearch returns true if any plan line contains a SEARCH
// specifically on the metrics table (e.g., "SEARCH metrics USING INDEX ..."
// or "SEARCH TABLE metrics USING INDEX ...").
func containsMetricsSearch(plan string) bool {
	for _, line := range strings.Split(plan, "\n") {
		if strings.Contains(line, "SEARCH") && lineRefersToMetrics(line) {
			return true
		}
	}
	return false
}

// containsMetricsSearchWithIndex returns true if a single plan line contains
// a SEARCH on the metrics table AND the given index name. This ensures the
// expected index is used on the metrics-table read path, not on a separate node.
func containsMetricsSearchWithIndex(plan, indexName string) bool {
	for _, line := range strings.Split(plan, "\n") {
		if strings.Contains(line, "SEARCH") && lineRefersToMetrics(line) && strings.Contains(line, indexName) {
			return true
		}
	}
	return false
}

// containsFullTableScan returns true if the plan contains a full table scan
// on the metrics table — "SCAN TABLE metrics" or "SCAN metrics" without a
// "USING ... INDEX" qualifier. A covering-index scan is not a full table scan.
func containsFullTableScan(plan string) bool {
	for _, line := range strings.Split(plan, "\n") {
		if !strings.Contains(line, "SCAN") || !lineRefersToMetrics(line) {
			continue
		}
		// A covering-index scan is acceptable — it reads only the index.
		if strings.Contains(line, "USING") && strings.Contains(line, "INDEX") {
			continue
		}
		// Bare SCAN TABLE metrics or SCAN metrics without index = full scan.
		return true
	}
	return false
}

// containsCoveringIndexScan returns true if any plan line shows a covering-
// index scan on the metrics table (e.g., "SCAN metrics USING COVERING INDEX ...").
func containsCoveringIndexScan(plan string) bool {
	for _, line := range strings.Split(plan, "\n") {
		if strings.Contains(line, "SCAN") && lineRefersToMetrics(line) && strings.Contains(line, "COVERING INDEX") {
			return true
		}
	}
	return false
}

// containsIndexScan returns true if any plan line shows an index-ordered scan
// on the metrics table (e.g., "SCAN metrics USING INDEX idx_metrics_query_all").
// This differs from a covering-index scan in that it reads table rows via the
// index, but still avoids a full table scan.
func containsIndexScan(plan string) bool {
	for _, line := range strings.Split(plan, "\n") {
		if strings.Contains(line, "SCAN") && lineRefersToMetrics(line) && strings.Contains(line, "USING") && strings.Contains(line, "INDEX") {
			return true
		}
	}
	return false
}

// newPlanTestDB creates an in-memory SQLite database with the metrics store
// schema. It uses the same schema and indexes as store.initSchema().
func newPlanTestDB(t *testing.T) *sql.DB {
	t.Helper()

	dsn := ":memory:?" + url.Values{
		"_pragma": []string{
			"busy_timeout(5000)",
			"journal_mode(WAL)",
		},
	}.Encode()

	db, err := sql.Open("sqlite", dsn)
	if err != nil {
		t.Fatalf("open in-memory db: %v", err)
	}
	// Pin to a single connection — a second connection to :memory: would get
	// a different database, causing "no such table" failures.
	db.SetMaxOpenConns(1)
	t.Cleanup(func() { db.Close() })

	schema := `
		CREATE TABLE IF NOT EXISTS metrics (
			id INTEGER PRIMARY KEY AUTOINCREMENT,
			resource_type TEXT NOT NULL,
			resource_id TEXT NOT NULL,
			metric_type TEXT NOT NULL,
			value REAL NOT NULL,
			min_value REAL,
			max_value REAL,
			timestamp INTEGER NOT NULL,
			tier TEXT NOT NULL DEFAULT 'raw'
		);

		CREATE INDEX IF NOT EXISTS idx_metrics_tier_time
		ON metrics(tier, timestamp);

		CREATE UNIQUE INDEX IF NOT EXISTS idx_metrics_query_all
		ON metrics(resource_type, resource_id, tier, timestamp, metric_type);

		CREATE TABLE IF NOT EXISTS metrics_meta (
			key TEXT PRIMARY KEY,
			value TEXT NOT NULL
		);

		CREATE TABLE IF NOT EXISTS availability_observations (
			id INTEGER PRIMARY KEY AUTOINCREMENT,
			observation_id TEXT NOT NULL UNIQUE,
			target_id TEXT NOT NULL,
			config_revision INTEGER NOT NULL,
			outcome TEXT NOT NULL,
			observed_at_ns INTEGER NOT NULL,
			timeline_at_ns INTEGER NOT NULL,
			ingested_at_ns INTEGER NOT NULL,
			valid_until_ns INTEGER NOT NULL,
			execution_source TEXT NOT NULL,
			latency_millis INTEGER
		);
		CREATE INDEX IF NOT EXISTS idx_availability_observations_target_timeline
			ON availability_observations(target_id, timeline_at_ns, id);
		CREATE INDEX IF NOT EXISTS idx_availability_observations_retention
			ON availability_observations(timeline_at_ns);

		CREATE TABLE IF NOT EXISTS availability_history_buckets (
			target_id TEXT NOT NULL,
			tier TEXT NOT NULL,
			bucket_start INTEGER NOT NULL,
			reachable_millis INTEGER NOT NULL DEFAULT 0,
			unreachable_millis INTEGER NOT NULL DEFAULT 0,
			indeterminate_millis INTEGER NOT NULL DEFAULT 0,
			reachable_count INTEGER NOT NULL DEFAULT 0,
			unreachable_count INTEGER NOT NULL DEFAULT 0,
			indeterminate_count INTEGER NOT NULL DEFAULT 0,
			latency_count INTEGER NOT NULL DEFAULT 0,
			latency_sum INTEGER NOT NULL DEFAULT 0,
			latency_min INTEGER,
			latency_max INTEGER,
			PRIMARY KEY(target_id, tier, bucket_start)
		);
		CREATE INDEX IF NOT EXISTS idx_availability_buckets_retention
			ON availability_history_buckets(tier, bucket_start);

		CREATE TABLE IF NOT EXISTS availability_revision_boundaries (
			target_id TEXT NOT NULL,
			revision INTEGER NOT NULL,
			started_at_ns INTEGER NOT NULL,
			PRIMARY KEY(target_id, revision)
		);
		CREATE INDEX IF NOT EXISTS idx_availability_boundaries_target_time
			ON availability_revision_boundaries(target_id, started_at_ns);
	`
	if _, err := db.Exec(schema); err != nil {
		t.Fatalf("create schema: %v", err)
	}

	// Insert a small amount of seed data so the query planner has statistics
	// to reason about. Without data, SQLite may choose different plans.
	for i := 0; i < 100; i++ {
		_, err := db.Exec(
			`INSERT INTO metrics (resource_type, resource_id, metric_type, value, timestamp, tier) VALUES (?, ?, ?, ?, ?, ?)`,
			"vm", fmt.Sprintf("vm-%d", i%10), "cpu", float64(i), int64(1000000+i*5), "raw",
		)
		if err != nil {
			t.Fatalf("seed data: %v", err)
		}

		targetID := fmt.Sprintf("target-%d", i%10)
		_, err = db.Exec(`INSERT INTO availability_observations
			(observation_id, target_id, config_revision, outcome, observed_at_ns, timeline_at_ns,
			 ingested_at_ns, valid_until_ns, execution_source, latency_millis)
			VALUES (?, ?, 1, 'reachable', ?, ?, ?, ?, 'local', 12)`,
			fmt.Sprintf("observation-%d", i), targetID, int64(1000000+i*5), int64(1000000+i*5),
			int64(1000000+i*5), int64(1000060+i*5))
		if err != nil {
			t.Fatalf("seed availability observations: %v", err)
		}
		_, err = db.Exec(`INSERT OR IGNORE INTO availability_history_buckets
			(target_id, tier, bucket_start, reachable_millis, latency_count, latency_sum, latency_min, latency_max)
			VALUES (?, 'minute', ?, 60000, 1, 12, 12, 12)`, targetID, int64(1000000+i*60))
		if err != nil {
			t.Fatalf("seed availability buckets: %v", err)
		}
		_, err = db.Exec(`INSERT OR IGNORE INTO availability_revision_boundaries
			(target_id, revision, started_at_ns) VALUES (?, 1, ?)`, targetID, int64(1000000+i*5))
		if err != nil {
			t.Fatalf("seed availability boundaries: %v", err)
		}
	}

	// Run ANALYZE so the planner has real statistics.
	if _, err := db.Exec("ANALYZE"); err != nil {
		t.Fatalf("analyze: %v", err)
	}

	return db
}

// explainQueryPlan runs EXPLAIN QUERY PLAN on the given query and returns
// the full plan output as a single string.
func explainQueryPlan(t *testing.T, db *sql.DB, query string, args []any) string {
	t.Helper()

	rows, err := db.Query("EXPLAIN QUERY PLAN "+query, args...)
	if err != nil {
		t.Fatalf("EXPLAIN QUERY PLAN: %v\nQuery: %s", err, query)
	}
	defer rows.Close()

	var lines []string
	for rows.Next() {
		var id, parent, notused int
		var detail string
		if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
			t.Fatalf("scan explain row: %v", err)
		}
		lines = append(lines, detail)
	}
	if err := rows.Err(); err != nil {
		t.Fatalf("explain rows: %v", err)
	}

	return strings.Join(lines, "\n")
}

func TestRetainedUnaggregatedQueryAvoidsMetricSort(t *testing.T) {
	db := newPlanTestDB(t)
	end := time.Unix(2000000040, 0)
	query, args := retainedQuerySQL("node", []string{"a", "b"}, nil, end.Add(-time.Hour), end, 0, []Tier{TierRaw, TierMinute, TierHourly}, false)
	for _, line := range strings.Split(explainQueryPlan(t, db, query, args), "\n") {
		if strings.Contains(strings.ToUpper(line), "TEMP B-TREE") {
			t.Fatalf("unaggregated all-metric read sorts despite time-ordered index: %s", line)
		}
	}
}