pulse/pkg/metrics/store.go

// Package metrics provides persistent storage for time-series metrics data
// using SQLite for durability across restarts.
package metrics

import (
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/rs/zerolog/log"
	_ "modernc.org/sqlite"

	"github.com/rcourtman/pulse-go-rewrite/internal/securityutil"
	pdb "github.com/rcourtman/pulse-go-rewrite/pkg/db"
)

const (
	privateDirPerm  = 0o700
	privateFilePerm = 0o600
)

// Tier represents the granularity of stored metrics
type Tier string

const (
	TierRaw    Tier = "raw"    // Raw data, ~5s intervals
	TierMinute Tier = "minute" // 1-minute averages
	TierHourly Tier = "hourly" // 1-hour averages
	TierDaily  Tier = "daily"  // 1-day averages
)

// MetricPoint represents a single metric data point
type MetricPoint struct {
	Timestamp time.Time
	Value     float64
	Min       float64 // For aggregated data
	Max       float64 // For aggregated data
}

// StoreConfig holds configuration for the metrics store
type StoreConfig struct {
	DBPath          string
	WriteBufferSize int           // Number of records to buffer before batch write
	FlushInterval   time.Duration // Max time between flushes
	RollupInterval  time.Duration // How often to aggregate raw samples into coarser tiers
	RetentionRaw    time.Duration // How long to keep raw data
	RetentionMinute time.Duration // How long to keep minute data
	RetentionHourly time.Duration // How long to keep hourly data
	RetentionDaily  time.Duration // How long to keep daily data
}

const (
	minRollupInterval     = 5 * time.Minute
	defaultRollupInterval = 15 * time.Minute
	maxRollupChunkWindow  = 5 * time.Minute
	maxRollupChunksPerRun = 128
)

// DefaultConfig returns sensible defaults for metrics storage
func DefaultConfig(dataDir string) StoreConfig {
	dbPath := filepath.Join(dataDir, "metrics.db")
	if resolvedDBPath, err := resolveStoreDBPath(dbPath); err == nil {
		dbPath = resolvedDBPath
	}

	return StoreConfig{
		DBPath: dbPath,
		// Large installs can enqueue hundreds of metric points per poll cycle.
		// A larger buffer keeps those writes inside a single SQLite transaction
		// more often, which materially reduces WAL churn on SSD-backed setups.
		WriteBufferSize: 500,
		FlushInterval:   5 * time.Second,
		RollupInterval:  defaultRollupInterval,
		RetentionRaw:    2 * time.Hour,
		RetentionMinute: 24 * time.Hour,
		RetentionHourly: 7 * 24 * time.Hour,
		RetentionDaily:  90 * 24 * time.Hour,
	}
}

func resolveStoreDBPath(dbPath string) (string, error) {
	trimmedPath := strings.TrimSpace(dbPath)
	if trimmedPath == "" {
		return "", fmt.Errorf("metrics database path is required")
	}

	cleanedPath := filepath.Clean(trimmedPath)
	dir, err := securityutil.NormalizeStorageDir(filepath.Dir(cleanedPath))
	if err != nil {
		return "", fmt.Errorf("resolve metrics database directory: %w", err)
	}

	resolvedPath, err := securityutil.JoinStorageLeaf(dir, filepath.Base(cleanedPath))
	if err != nil {
		return "", fmt.Errorf("resolve metrics database path: %w", err)
	}

	return resolvedPath, nil
}

func normalizeRollupInterval(interval, rawRetention time.Duration) time.Duration {
	if interval <= 0 {
		interval = defaultRollupInterval
	}
	if interval < minRollupInterval {
		interval = minRollupInterval
	}
	if rawRetention <= 0 {
		return interval
	}

	maxInterval := rawRetention / 2
	if maxInterval < minRollupInterval {
		maxInterval = minRollupInterval
	}
	if interval > maxInterval {
		return maxInterval
	}
	return interval
}

// bufferedMetric holds a metric waiting to be written
type bufferedMetric struct {
	resourceType string
	resourceID   string
	metricType   string
	value        float64
	timestamp    time.Time
	tier         Tier
}

type writeRequest struct {
	metrics []bufferedMetric
	done    chan struct{}
}

type metricBatchKey struct {
	resourceType  string
	resourceID    string
	metricType    string
	timestampUnix int64
	tier          Tier
}

// WriteMetric represents a metric sample to be written synchronously.
type WriteMetric struct {
	ResourceType string
	ResourceID   string
	MetricType   string
	Value        float64
	Timestamp    time.Time
	Tier         Tier
}

// SeriesKey identifies one stored metric series in the normalized form the
// write path persists (lower-cased resource/metric types, trimmed id).
type SeriesKey struct {
	ResourceType string
	ResourceID   string
	MetricType   string
}

// NormalizedSeriesKey builds the SeriesKey the write path would store for the
// given identifiers, so callers can match MaxTimestampsForTier results.
func NormalizedSeriesKey(resourceType, resourceID, metricType string) SeriesKey {
	return SeriesKey{
		ResourceType: normalizeMetricResourceType(resourceType),
		ResourceID:   normalizeMetricIdentifier(resourceID),
		MetricType:   normalizeMetricType(metricType),
	}
}

type maintenanceRequest struct {
	run  func()
	done chan struct{}
}

var (
	startupMaintenanceHook       func()
	metricsIdentityMigrationHook func()
)

// Store provides persistent metrics storage
type Store struct {
	db     *pdb.InstrumentedDB
	config StoreConfig

	// Write buffer
	bufferMu sync.Mutex
	buffer   []bufferedMetric

	// Background workers
	writeCh                    chan writeRequest
	maintenanceCh              chan maintenanceRequest
	stopCh                     chan struct{}
	doneCh                     chan struct{}
	maintenanceDoneCh          chan struct{}
	stopOnce                   sync.Once
	stopping                   atomic.Bool
	identityMigrationPending   atomic.Bool
	commercialRetentionSeconds atomic.Int64
	commercialPurgeEligibleAt  atomic.Int64
}

// SetCommercialHistoryRetention applies a delayed commercial ceiling to the
// configured physical retention. A zero or negative day count clears the
// ceiling. Access control is enforced elsewhere immediately; this method only
// governs when excess history becomes physically purgeable.
func (s *Store) SetCommercialHistoryRetention(days int, purgeEligibleAt time.Time) {
	if s == nil || days <= 0 || purgeEligibleAt.IsZero() {
		if s != nil {
			s.commercialRetentionSeconds.Store(0)
			s.commercialPurgeEligibleAt.Store(0)
		}
		return
	}
	s.commercialRetentionSeconds.Store(int64(time.Duration(days) * 24 * time.Hour / time.Second))
	s.commercialPurgeEligibleAt.Store(purgeEligibleAt.UTC().Unix())
}

func (s *Store) effectiveRetention(base time.Duration, now time.Time) time.Duration {
	seconds := s.commercialRetentionSeconds.Load()
	eligibleAt := s.commercialPurgeEligibleAt.Load()
	if seconds <= 0 || eligibleAt <= 0 || now.UTC().Unix() < eligibleAt {
		return base
	}
	commercial := time.Duration(seconds) * time.Second
	if commercial < base {
		return commercial
	}
	return base
}

// NewStore creates a new metrics store with the given configuration
func NewStore(config StoreConfig) (*Store, error) {
	resolvedDBPath, err := resolveStoreDBPath(config.DBPath)
	if err != nil {
		return nil, err
	}
	config.DBPath = resolvedDBPath
	config.RollupInterval = normalizeRollupInterval(config.RollupInterval, config.RetentionRaw)

	dir := filepath.Dir(config.DBPath)
	if err := ensureOwnerOnlyDir(dir); err != nil {
		return nil, fmt.Errorf("failed to create metrics directory: %w", err)
	}

	if err := rejectSymlinkOrNonRegular(config.DBPath); err != nil && !errors.Is(err, os.ErrNotExist) {
		return nil, err
	}

	// Open database with pragmas in DSN so every pool connection is configured
	dsn := config.DBPath + "?" + url.Values{
		"_pragma": []string{
			"busy_timeout(30000)",
			"journal_mode(WAL)",
			"synchronous(NORMAL)",
			// auto_vacuum is deliberately NOT set here. It is a persistent
			// database property that migrateAutoVacuum establishes once at
			// startup; as a per-connection pragma it replays as a header
			// write on every new pool connection, which blocks connection
			// creation behind the active writer (#1601).
			// Checkpoint less aggressively so high-cardinality installs don't
			// keep rewriting tiny WAL segments back into the main DB file.
			"wal_autocheckpoint(4000)",
		},
	}.Encode()
	rawDB, err := sql.Open("sqlite", dsn)
	if err != nil {
		return nil, fmt.Errorf("failed to open metrics database: %w", err)
	}

	// Configure connection pool. Writes are already funnelled: the flush,
	// rollup, retention, and maintenance paths run on the single background
	// worker goroutine, and the WriteBatchSync poller path serializes on the
	// WAL write lock via busy_timeout. What a pool of one actually did was
	// queue every UI history read behind those commits, which froze charts
	// whenever a commit picked up a WAL checkpoint (#1601). WAL mode reads
	// snapshot-isolated on their own connections, so give readers room.
	rawDB.SetMaxOpenConns(4)
	rawDB.SetMaxIdleConns(4)
	rawDB.SetConnMaxLifetime(0)

	db := pdb.Wrap(rawDB, "metrics")

	store := &Store{
		db:                db,
		config:            config,
		buffer:            make([]bufferedMetric, 0, config.WriteBufferSize),
		writeCh:           make(chan writeRequest, 100), // Buffer for write batches
		maintenanceCh:     make(chan maintenanceRequest, 1),
		stopCh:            make(chan struct{}),
		doneCh:            make(chan struct{}),
		maintenanceDoneCh: make(chan struct{}),
	}

	// Initialize schema
	if err := store.initSchema(); err != nil {
		db.Close()
		return nil, fmt.Errorf("failed to initialize schema: %w", err)
	}
	store.migrateLegacyHostResourceType()

	if err := hardenSQLiteArtifacts(config.DBPath); err != nil {
		db.Close()
		return nil, fmt.Errorf("failed to secure metrics db files: %w", err)
	}

	// Keep metric ingestion independent from rollup, retention, and one-time
	// startup maintenance. SQLite remains the write-serialization boundary,
	// but long maintenance scheduling no longer prevents the write queue from
	// being drained (#1601).
	go store.backgroundWorker()
	go store.maintenanceWorker()
	store.enqueueMaintenance(store.runStartupMaintenance)

	log.Info().
		Str("path", config.DBPath).
		Int("bufferSize", config.WriteBufferSize).
		Dur("rollupInterval", config.RollupInterval).
		Msg("Metrics store initialized")

	return store, nil
}

// initSchema creates the database schema if it doesn't exist
func (s *Store) initSchema() error {
	schema := `
		-- Main metrics table
		CREATE TABLE IF NOT EXISTS metrics (
			id INTEGER PRIMARY KEY AUTOINCREMENT,
			resource_type TEXT NOT NULL,
			resource_id TEXT NOT NULL,
			metric_type TEXT NOT NULL,
			value REAL NOT NULL,
			min_value REAL,
			max_value REAL,
			timestamp INTEGER NOT NULL,
			tier TEXT NOT NULL DEFAULT 'raw'
		);

		-- Index for retention pruning
		CREATE INDEX IF NOT EXISTS idx_metrics_tier_time 
		ON metrics(tier, timestamp);

		-- Covering index for Unified History (QueryAll) performance
		CREATE INDEX IF NOT EXISTS idx_metrics_query_all
		ON metrics(resource_type, resource_id, tier, timestamp, metric_type);

		-- Metadata table for tracking rollup state
		CREATE TABLE IF NOT EXISTS metrics_meta (
			key TEXT PRIMARY KEY,
			value TEXT NOT NULL
		);
	`

	_, err := s.db.Exec(schema)
	if err != nil {
		return fmt.Errorf("failed to create schema: %w", err)
	}

	if err := s.ensureMetricsIdentityIndex(); err != nil {
		return err
	}

	log.Debug().Msg("Metrics schema initialized")
	return nil
}

var metricsIdentityColumns = []string{"resource_type", "resource_id", "metric_type", "tier", "timestamp"}

// ensureMetricsIdentityIndex keeps one B-tree for both metric identity and
// single-series range queries. Older databases have two indexes containing the
// same five columns in different orders: idx_metrics_lookup serves reads while
// idx_metrics_unique enforces identity. Every insert dirties both trees, which
// materially amplifies WAL and checkpoint writes.
func (s *Store) ensureMetricsIdentityIndex() error {
	lookupCurrent, err := s.metricsIndexMatches("idx_metrics_lookup", true, metricsIdentityColumns)
	if err != nil {
		return fmt.Errorf("inspect metrics identity index: %w", err)
	}
	legacyUniqueExists, err := s.metricsIndexExists("idx_metrics_unique")
	if err != nil {
		return fmt.Errorf("inspect legacy metrics unique index: %w", err)
	}

	if legacyUniqueExists {
		// v6.1.1 and earlier already have an authoritative unique index. Keep
		// serving with that crash-safe schema and defer the O(rows) rebuild to
		// the startup-maintenance worker so NewStore latency stays bounded.
		s.identityMigrationPending.Store(true)
		log.Info().Msg("Scheduled metrics identity index consolidation")
		return nil
	}
	if lookupCurrent {
		return nil
	}

	return s.migrateMetricsIdentityIndex()
}

func (s *Store) migrateMetricsIdentityIndex() error {
	if err := s.replaceMetricsIdentityIndex(); err == nil {
		log.Info().Msg("Consolidated metrics lookup and identity indexes")
		return nil
	} else if !isUniqueConstraintError(err) {
		return fmt.Errorf("consolidate metrics identity index: %w", err)
	}

	log.Warn().Msg("Duplicate metrics detected; deduplicating before consolidating identity index")
	if err := s.deduplicateMetrics(); err != nil {
		return err
	}
	if err := s.replaceMetricsIdentityIndex(); err != nil {
		return fmt.Errorf("consolidate metrics identity index after dedupe: %w", err)
	}

	log.Info().Msg("Metrics deduplicated and identity index consolidated")
	return nil
}

func (s *Store) replaceMetricsIdentityIndex() error {
	tx, err := s.db.Begin()
	if err != nil {
		return fmt.Errorf("begin identity index migration: %w", err)
	}
	defer func() { _ = tx.Rollback() }()

	if _, err := tx.Exec(`DROP INDEX IF EXISTS idx_metrics_lookup`); err != nil {
		return fmt.Errorf("drop old lookup index: %w", err)
	}
	if metricsIdentityMigrationHook != nil {
		metricsIdentityMigrationHook()
	}
	if _, err := tx.Exec(`
		CREATE UNIQUE INDEX idx_metrics_lookup
		ON metrics(resource_type, resource_id, metric_type, tier, timestamp)
	`); err != nil {
		return fmt.Errorf("create consolidated lookup index: %w", err)
	}
	if _, err := tx.Exec(`DROP INDEX IF EXISTS idx_metrics_unique`); err != nil {
		return fmt.Errorf("drop old unique index: %w", err)
	}
	if err := tx.Commit(); err != nil {
		return fmt.Errorf("commit identity index migration: %w", err)
	}
	return nil
}

func (s *Store) deduplicateMetrics() error {
	tx, err := s.db.Begin()
	if err != nil {
		return fmt.Errorf("begin metrics dedupe transaction: %w", err)
	}
	defer func() { _ = tx.Rollback() }()

	if _, err := tx.Exec(`
		DELETE FROM metrics
		WHERE rowid NOT IN (
			SELECT MIN(rowid)
			FROM metrics
			GROUP BY resource_type, resource_id, metric_type, tier, timestamp
		)
	`); err != nil {
		return fmt.Errorf("dedupe metrics: %w", err)
	}
	if err := tx.Commit(); err != nil {
		return fmt.Errorf("commit metrics dedupe: %w", err)
	}
	return nil
}

func isUniqueConstraintError(err error) bool {
	if err == nil {
		return false
	}
	lower := strings.ToLower(err.Error())
	return strings.Contains(lower, "unique") ||
		strings.Contains(lower, "constraint") ||
		strings.Contains(lower, "duplicate")
}

func (s *Store) metricsIndexExists(name string) (bool, error) {
	rows, err := s.db.Query(`PRAGMA index_list(metrics)`)
	if err != nil {
		return false, err
	}
	defer rows.Close()

	for rows.Next() {
		var (
			sequence int
			index    string
			unique   int
			origin   string
			partial  int
		)
		if err := rows.Scan(&sequence, &index, &unique, &origin, &partial); err != nil {
			return false, err
		}
		if index == name {
			return true, nil
		}
	}
	return false, rows.Err()
}

func (s *Store) metricsIndexMatches(name string, wantUnique bool, wantColumns []string) (bool, error) {
	rows, err := s.db.Query(`PRAGMA index_list(metrics)`)
	if err != nil {
		return false, err
	}

	found := false
	for rows.Next() {
		var (
			sequence int
			index    string
			unique   int
			origin   string
			partial  int
		)
		if err := rows.Scan(&sequence, &index, &unique, &origin, &partial); err != nil {
			_ = rows.Close()
			return false, err
		}
		if index == name {
			found = (unique == 1) == wantUnique && partial == 0
			break
		}
	}
	if err := rows.Close(); err != nil {
		return false, err
	}
	if !found {
		return false, nil
	}

	columnRows, err := s.db.Query(`
		SELECT name
		FROM pragma_index_info(?)
		ORDER BY seqno
	`, name)
	if err != nil {
		return false, err
	}
	defer columnRows.Close()

	columns := make([]string, 0, len(wantColumns))
	for columnRows.Next() {
		var columnName string
		if err := columnRows.Scan(&columnName); err != nil {
			return false, err
		}
		columns = append(columns, columnName)
	}
	if err := columnRows.Err(); err != nil {
		return false, err
	}
	if len(columns) != len(wantColumns) {
		return false, nil
	}
	for i := range columns {
		if columns[i] != wantColumns[i] {
			return false, nil
		}
	}
	return true, nil
}

// migrateAutoVacuum ensures the database uses incremental auto-vacuum.
// SQLite cannot switch from NONE to INCREMENTAL without a full VACUUM to
// restructure the file, so we detect and convert on first run after upgrade.
func (s *Store) migrateAutoVacuum() {
	var mode int
	if err := s.db.QueryRow("PRAGMA auto_vacuum").Scan(&mode); err != nil {
		log.Debug().Err(err).Msg("Failed to check auto_vacuum mode")
		return
	}
	if mode == 2 { // already INCREMENTAL
		return
	}

	log.Info().Int("current_mode", mode).Msg("Converting metrics database to incremental auto-vacuum (one-time migration)")
	start := time.Now()

	// Set the desired mode then VACUUM to restructure the file.
	if _, err := s.db.Exec("PRAGMA auto_vacuum = INCREMENTAL"); err != nil {
		log.Warn().Err(err).Msg("Failed to set auto_vacuum mode")
		return
	}
	if _, err := s.db.Exec("VACUUM"); err != nil {
		log.Warn().Err(err).Msg("Auto-vacuum migration VACUUM failed (will retry next restart)")
		return
	}

	log.Info().Dur("duration", time.Since(start)).Msg("Metrics database auto-vacuum migration complete")
}

// migrateLegacyHostResourceType rewrites legacy v5 `resource_type=host` rows to
// canonical v6 `resource_type=agent`. This keeps reads/writes agent-only while
// preserving historical data.
func (s *Store) migrateLegacyHostResourceType() {
	tx, err := s.db.Begin()
	if err != nil {
		log.Warn().Err(err).Msg("Failed to start legacy host->agent metrics migration")
		return
	}
	defer func() { _ = tx.Rollback() }()

	// Reinsert legacy rows with canonical type, keeping any existing canonical
	// records when duplicates collide with the unique index.
	if _, err := tx.Exec(`
		INSERT OR IGNORE INTO metrics (
			resource_type, resource_id, metric_type, value, min_value, max_value, timestamp, tier
		)
		SELECT
			'agent', resource_id, metric_type, value, min_value, max_value, timestamp, tier
		FROM metrics
		WHERE resource_type = 'host'
	`); err != nil {
		log.Warn().Err(err).Msg("Failed to copy legacy host metrics rows")
		return
	}

	deleteResult, err := tx.Exec(`DELETE FROM metrics WHERE resource_type = 'host'`)
	if err != nil {
		log.Warn().Err(err).Msg("Failed to delete legacy host metrics rows")
		return
	}

	if err := tx.Commit(); err != nil {
		log.Warn().Err(err).Msg("Failed to commit legacy host->agent metrics migration")
		return
	}

	rowsDeleted, err := deleteResult.RowsAffected()
	if err != nil {
		log.Debug().Err(err).Msg("Failed to read affected rows for legacy host->agent migration")
		return
	}
	if rowsDeleted > 0 {
		log.Info().Int64("rows", rowsDeleted).Msg("Migrated legacy host metrics rows to agent")
	}
}

func normalizeMetricResourceType(resourceType string) string {
	return strings.ToLower(strings.TrimSpace(resourceType))
}

func normalizeMetricType(metricType string) string {
	return strings.ToLower(strings.TrimSpace(metricType))
}

func normalizeMetricIdentifier(value string) string {
	return strings.TrimSpace(value)
}

func isLegacyMetricResourceType(resourceType string) bool {
	return strings.EqualFold(strings.TrimSpace(resourceType), "host")
}

func isSupportedMetricTier(tier Tier) bool {
	switch tier {
	case TierRaw, TierMinute, TierHourly, TierDaily:
		return true
	default:
		return false
	}
}

func validateMetricWrite(resourceType, resourceID, metricType string, tier Tier) (string, string, string, bool, string) {
	normalizedType := normalizeMetricResourceType(resourceType)
	if normalizedType == "" {
		return "", "", "", false, "empty resource type"
	}
	if isLegacyMetricResourceType(normalizedType) {
		return "", "", "", false, `unsupported legacy resource type "host"`
	}

	normalizedID := normalizeMetricIdentifier(resourceID)
	if normalizedID == "" {
		return "", "", "", false, "empty resource id"
	}

	normalizedMetric := normalizeMetricType(metricType)
	if normalizedMetric == "" {
		return "", "", "", false, "empty metric type"
	}

	if !isSupportedMetricTier(tier) {
		return "", "", "", false, fmt.Sprintf("unsupported metric tier %q", tier)
	}

	return normalizedType, normalizedID, normalizedMetric, true, ""
}

// Write adds a metric to the write buffer with the 'raw' tier by default
func (s *Store) Write(resourceType, resourceID, metricType string, value float64, timestamp time.Time) {
	s.WriteWithTier(resourceType, resourceID, metricType, value, timestamp, TierRaw)
}

// WriteWithTier adds a metric to the write buffer with a specific tier
func (s *Store) WriteWithTier(resourceType, resourceID, metricType string, value float64, timestamp time.Time, tier Tier) {
	if s.stopping.Load() {
		return
	}

	normalizedType, normalizedID, normalizedMetric, ok, reason := validateMetricWrite(resourceType, resourceID, metricType, tier)
	if !ok {
		log.Warn().
			Str("resource_type", resourceType).
			Str("resource_id", resourceID).
			Str("metric_type", metricType).
			Str("tier", string(tier)).
			Str("reason", reason).
			Msg("Dropping invalid metrics write")
		return
	}

	s.bufferMu.Lock()

	if s.stopping.Load() {
		s.bufferMu.Unlock()
		return
	}

	s.buffer = append(s.buffer, bufferedMetric{
		resourceType: normalizedType,
		resourceID:   normalizedID,
		metricType:   normalizedMetric,
		value:        value,
		timestamp:    timestamp,
		tier:         tier,
	})

	// Flush if buffer is full
	var toWrite []bufferedMetric
	if len(s.buffer) >= s.config.WriteBufferSize {
		toWrite = s.detachBufferLocked()
	}
	s.bufferMu.Unlock()

	s.enqueueWrite(writeRequest{metrics: toWrite})
}

// WriteBatchSync writes metrics directly to the database without buffering.
func (s *Store) WriteBatchSync(metrics []WriteMetric) {
	if len(metrics) == 0 {
		return
	}

	batch := make([]bufferedMetric, 0, len(metrics))
	droppedInvalid := 0
	for _, metric := range metrics {
		normalizedType, normalizedID, normalizedMetric, ok, reason := validateMetricWrite(
			metric.ResourceType,
			metric.ResourceID,
			metric.MetricType,
			metric.Tier,
		)
		if !ok {
			droppedInvalid++
			log.Warn().
				Str("resource_type", metric.ResourceType).
				Str("resource_id", metric.ResourceID).
				Str("metric_type", metric.MetricType).
				Str("tier", string(metric.Tier)).
				Str("reason", reason).
				Msg("Dropping invalid metrics write from batch")
			continue
		}
		batch = append(batch, bufferedMetric{
			resourceType: normalizedType,
			resourceID:   normalizedID,
			metricType:   normalizedMetric,
			value:        metric.Value,
			timestamp:    metric.Timestamp,
			tier:         metric.Tier,
		})
	}
	if droppedInvalid > 0 {
		log.Warn().
			Int("dropped", droppedInvalid).
			Msg("Dropped invalid metrics writes from batch")
	}
	if len(batch) == 0 {
		return
	}

	s.writeBatch(batch)
}

func (s *Store) enqueueMaintenance(run func()) {
	if run == nil {
		return
	}

	select {
	case s.maintenanceCh <- maintenanceRequest{run: run}:
	default:
		log.Debug().Msg("Metrics maintenance queue full, skipping duplicate request")
	}
}

// WaitForMaintenance blocks until all queued maintenance work has completed.
// Tests and benchmarks use this to measure steady-state hot paths without
// asynchronous startup maintenance distorting the results.
func (s *Store) WaitForMaintenance(timeout time.Duration) error {
	if s == nil {
		return nil
	}
	if s.stopping.Load() {
		return fmt.Errorf("metrics store is stopping")
	}

	done := make(chan struct{})
	barrier := maintenanceRequest{done: done}

	if timeout <= 0 {
		s.maintenanceCh <- barrier
		<-done
		return nil
	}

	queueTimer := time.NewTimer(timeout)
	defer queueTimer.Stop()

	select {
	case s.maintenanceCh <- barrier:
	case <-queueTimer.C:
		return fmt.Errorf("timed out queueing metrics maintenance barrier after %v", timeout)
	}

	waitTimer := time.NewTimer(timeout)
	defer waitTimer.Stop()

	select {
	case <-done:
		return nil
	case <-waitTimer.C:
		return fmt.Errorf("timed out waiting for metrics maintenance after %v", timeout)
	}
}

func (s *Store) runStartupMaintenance() {
	start := time.Now()
	if startupMaintenanceHook != nil {
		startupMaintenanceHook()
	}

	if s.identityMigrationPending.Swap(false) {
		if err := s.migrateMetricsIdentityIndex(); err != nil {
			log.Error().Err(err).Msg("Deferred metrics identity index consolidation failed")
		}
	}

	// Run retention before the deferred auto-vacuum conversion so restart
	// cleanup trims stale rows and redundant-index pages before SQLite
	// potentially rewrites the file.
	s.runRetention()
	s.migrateAutoVacuum()

	log.Info().Dur("duration", time.Since(start)).Msg("Deferred metrics startup maintenance completed")
}

// detachBufferLocked returns the current in-memory buffer and resets it.
// Caller must hold bufferMu.
func (s *Store) detachBufferLocked() []bufferedMetric {
	if len(s.buffer) == 0 {
		return nil
	}

	// Copy buffer for writing
	toWrite := make([]bufferedMetric, len(s.buffer))
	copy(toWrite, s.buffer)
	s.buffer = s.buffer[:0]

	return toWrite
}

// flush writes buffered metrics to the database (caller must hold bufferMu)
func (s *Store) flushLocked() {
	s.enqueueWrite(writeRequest{metrics: s.detachBufferLocked()})
}

func (s *Store) enqueueWrite(req writeRequest) {
	if len(req.metrics) == 0 && req.done == nil {
		return
	}

	select {
	case s.writeCh <- req:
	default:
		log.Warn().
			Str("component", "metrics_store").
			Str("action", "drop_write_batch").
			Int("batch_size", len(req.metrics)).
			Int("write_queue_depth", len(s.writeCh)).
			Int("write_queue_capacity", cap(s.writeCh)).
			Msg("Metrics write channel full, dropping batch")
		if req.done != nil {
			close(req.done)
		}
	}
}

func (s *Store) enqueueAndWait(req writeRequest) {
	if req.done == nil {
		req.done = make(chan struct{})
	}

	s.writeCh <- req
	<-req.done
}

func (s *Store) drainBuffer() []bufferedMetric {
	s.bufferMu.Lock()
	defer s.bufferMu.Unlock()
	return s.detachBufferLocked()
}

func (s *Store) flushBufferedAsync() {
	s.enqueueWrite(writeRequest{metrics: s.drainBuffer()})
}

func (s *Store) processWriteRequests(requests []writeRequest) {
	if len(requests) == 0 {
		return
	}

	combined := make([]bufferedMetric, 0)
	doneChans := make([]chan struct{}, 0)
	for _, req := range requests {
		if len(req.metrics) > 0 {
			combined = append(combined, req.metrics...)
		}
		if req.done != nil {
			doneChans = append(doneChans, req.done)
		}
	}

	if len(combined) > 0 {
		s.writeBatch(combined)
	}
	for _, done := range doneChans {
		close(done)
	}
}

// writeBatch writes a batch of metrics to the database
func (s *Store) writeBatch(metrics []bufferedMetric) {
	if len(metrics) == 0 {
		return
	}

	inputCount := len(metrics)
	metrics = coalesceMetricBatch(metrics)
	if len(metrics) == 0 {
		return
	}
	if len(metrics) < inputCount {
		log.Debug().
			Int("input_count", inputCount).
			Int("count", len(metrics)).
			Msg("Coalesced duplicate metrics before write")
	}

	var tx *pdb.InstrumentedTx
	var err error

	// Retry on SQLITE_BUSY with exponential backoff
	for i := 0; i < 5; i++ {
		tx, err = s.db.Begin()
		if err == nil {
			break
		}
		if i < 4 && (err.Error() == "database is locked" || err.Error() == "sql: database is closed") {
			time.Sleep(time.Duration(100*(i+1)) * time.Millisecond)
			continue
		}
		log.Error().Err(err).
			Str("component", "metrics_store").
			Str("action", "begin_write_tx").
			Int("batch_size", len(metrics)).
			Msg("Failed to begin metrics transaction")
		return
	}

	stmt, err := tx.Prepare(`
		INSERT INTO metrics (resource_type, resource_id, metric_type, value, timestamp, tier)
		VALUES (?, ?, ?, ?, ?, ?)
		ON CONFLICT(resource_type, resource_id, metric_type, tier, timestamp)
		DO UPDATE SET
			value = excluded.value,
			min_value = excluded.min_value,
			max_value = excluded.max_value
	`)
	if err != nil {
		_ = tx.Rollback()
		log.Error().Err(err).
			Str("component", "metrics_store").
			Str("action", "prepare_write_stmt").
			Int("batch_size", len(metrics)).
			Msg("Failed to prepare metrics insert")
		return
	}
	defer stmt.Close()

	for _, m := range metrics {
		_, err := stmt.Exec(m.resourceType, m.resourceID, m.metricType, m.value, m.timestamp.Unix(), string(m.tier))
		if err != nil {
			log.Warn().Err(err).
				Str("component", "metrics_store").
				Str("action", "insert_metric").
				Str("resource_type", m.resourceType).
				Str("resource_id", m.resourceID).
				Str("metric_type", m.metricType).
				Str("tier", string(m.tier)).
				Msg("Failed to insert metric")
		}
	}

	if err := tx.Commit(); err != nil {
		log.Error().Err(err).
			Str("component", "metrics_store").
			Str("action", "commit_write_tx").
			Int("batch_size", len(metrics)).
			Msg("Failed to commit metrics batch")
		return
	}

	log.Debug().Int("input_count", inputCount).Int("count", len(metrics)).Msg("Wrote metrics batch")
}

func coalesceMetricBatch(metrics []bufferedMetric) []bufferedMetric {
	if len(metrics) < 2 {
		return metrics
	}

	indexes := make(map[metricBatchKey]int, len(metrics))
	coalesced := make([]bufferedMetric, 0, len(metrics))
	for _, metric := range metrics {
		key := metricBatchKey{
			resourceType:  metric.resourceType,
			resourceID:    metric.resourceID,
			metricType:    metric.metricType,
			timestampUnix: metric.timestamp.Unix(),
			tier:          metric.tier,
		}
		if index, ok := indexes[key]; ok {
			coalesced[index] = metric
			continue
		}

		indexes[key] = len(coalesced)
		coalesced = append(coalesced, metric)
	}

	return coalesced
}

// coalesceQueuedRequests drains any already-queued write requests so the worker
// can commit pending metrics in a single SQLite transaction while preserving
// Flush completion barriers.
func (s *Store) coalesceQueuedRequests(initial writeRequest) []writeRequest {
	if len(initial.metrics) == 0 && initial.done == nil {
		return nil
	}

	combined := []writeRequest{initial}
	for {
		select {
		case next, ok := <-s.writeCh:
			if !ok {
				return combined
			}
			if len(next.metrics) == 0 && next.done == nil {
				continue
			}
			combined = append(combined, next)
		default:
			return combined
		}
	}
}

// Query retrieves metrics for a resource within a time range, with optional downsampling
func (s *Store) Query(resourceType, resourceID, metricType string, start, end time.Time, stepSecs int64) ([]MetricPoint, error) {
	resourceType = normalizeMetricResourceType(resourceType)
	resourceID = normalizeMetricIdentifier(resourceID)
	metricType = normalizeMetricType(metricType)
	tiers := s.tierFallbacks(end.Sub(start))
	if len(tiers) == 0 {
		return []MetricPoint{}, nil
	}

	for i, tier := range tiers {
		points, err := s.queryWithTier(resourceType, resourceID, metricType, start, end, stepSecs, tier)
		if err != nil {
			return nil, err
		}
		if len(points) > 0 || i == len(tiers)-1 {
			return points, nil
		}

		log.Debug().
			Str("resourceType", resourceType).
			Str("resourceId", resourceID).
			Str("metric", metricType).
			Str("fromTier", string(tier)).
			Str("toTier", string(tiers[i+1])).
			Msg("Metrics query empty; falling back to more detailed tier")
	}

	return []MetricPoint{}, nil
}

func (s *Store) queryWithTier(resourceType, resourceID, metricType string, start, end time.Time, stepSecs int64, tier Tier) ([]MetricPoint, error) {
	var rows *sql.Rows
	var err error

	sqlQuery := `
		SELECT timestamp, value, COALESCE(min_value, value), COALESCE(max_value, value)
		FROM metrics
		WHERE resource_type = ? AND resource_id = ? AND metric_type = ? AND tier = ?
		AND timestamp >= ? AND timestamp <= ?
		ORDER BY timestamp ASC
	`
	queryParams := []interface{}{resourceType, resourceID, metricType, string(tier), start.Unix(), end.Unix()}

	if stepSecs > 1 {
		sqlQuery = `
			SELECT 
				(timestamp / ?) * ? + (? / 2) as bucket_ts, 
				AVG(value), 
				MIN(COALESCE(min_value, value)), 
				MAX(COALESCE(max_value, value))
			FROM metrics
			WHERE resource_type = ? AND resource_id = ? AND metric_type = ? AND tier = ?
			AND timestamp >= ? AND timestamp <= ?
			GROUP BY bucket_ts
			ORDER BY bucket_ts ASC
		`
		queryParams = []interface{}{
			stepSecs, stepSecs, stepSecs,
			resourceType, resourceID, metricType, string(tier), start.Unix(), end.Unix(),
		}
	}

	// Retry on SQLITE_BUSY
	for i := 0; i < 5; i++ {
		rows, err = s.db.Query(sqlQuery, queryParams...)

		if err == nil {
			break
		}
		if i < 4 && (err.Error() == "database is locked" || err.Error() == "sql: database is closed") {
			time.Sleep(time.Duration(100*(i+1)) * time.Millisecond)
			continue
		}
		return nil, fmt.Errorf("failed to query metrics: %w", err)
	}
	defer rows.Close()

	var points []MetricPoint
	for rows.Next() {
		var ts int64
		var p MetricPoint
		if err := rows.Scan(&ts, &p.Value, &p.Min, &p.Max); err != nil {
			log.Warn().Err(err).Msg("Failed to scan metric row")
			continue
		}
		p.Timestamp = time.Unix(ts, 0)
		points = append(points, p)
	}

	return points, rows.Err()
}

// QueryAll retrieves all metric types for a resource within a time range, with optional downsampling
func (s *Store) QueryAll(resourceType, resourceID string, start, end time.Time, stepSecs int64) (map[string][]MetricPoint, error) {
	resourceType = normalizeMetricResourceType(resourceType)
	resourceID = normalizeMetricIdentifier(resourceID)
	tiers := s.tierFallbacks(end.Sub(start))
	if len(tiers) == 0 {
		return map[string][]MetricPoint{}, nil
	}

	result := make(map[string][]MetricPoint)
	for i, tier := range tiers {
		tierResult, err := s.queryAllWithTier(resourceType, resourceID, start, end, stepSecs, tier)
		if err != nil {
			return nil, err
		}
		if len(tierResult) == 0 {
			if i < len(tiers)-1 && len(result) == 0 {
				log.Debug().
					Str("resourceType", resourceType).
					Str("resourceId", resourceID).
					Str("fromTier", string(tier)).
					Str("toTier", string(tiers[i+1])).
					Msg("Metrics query empty; falling back to more detailed tier")
			}
			continue
		}

		// Merge in any metrics missing from higher tier results.
		added := 0
		for metric, points := range tierResult {
			if len(points) == 0 {
				continue
			}
			if existing, ok := result[metric]; !ok || len(existing) == 0 {
				result[metric] = points
				added++
			}
		}

		// If we already have some metrics and this tier didn't add anything new,
		// keep going in case lower tiers have newly introduced metrics.
		if added == 0 && i < len(tiers)-1 && len(result) == 0 {
			log.Debug().
				Str("resourceType", resourceType).
				Str("resourceId", resourceID).
				Str("fromTier", string(tier)).
				Str("toTier", string(tiers[i+1])).
				Msg("Metrics query empty; falling back to more detailed tier")
		}
	}

	return result, nil
}

func (s *Store) queryAllWithTier(resourceType, resourceID string, start, end time.Time, stepSecs int64, tier Tier) (map[string][]MetricPoint, error) {
	var rows *sql.Rows
	var err error

	sqlQuery := `
		SELECT metric_type, timestamp, value, COALESCE(min_value, value), COALESCE(max_value, value)
		FROM metrics
		WHERE resource_type = ? AND resource_id = ? AND tier = ?
		AND timestamp >= ? AND timestamp <= ?
		ORDER BY metric_type, timestamp ASC
	`
	queryParams := []interface{}{resourceType, resourceID, string(tier), start.Unix(), end.Unix()}

	if stepSecs > 1 {
		sqlQuery = `
			SELECT 
				metric_type,
				(timestamp / ?) * ? + (? / 2) as bucket_ts, 
				AVG(value), 
				MIN(COALESCE(min_value, value)), 
				MAX(COALESCE(max_value, value))
			FROM metrics
			WHERE resource_type = ? AND resource_id = ? AND tier = ?
			AND timestamp >= ? AND timestamp <= ?
			GROUP BY metric_type, bucket_ts
			ORDER BY metric_type, bucket_ts ASC
		`
		queryParams = []interface{}{
			stepSecs, stepSecs, stepSecs,
			resourceType, resourceID, string(tier), start.Unix(), end.Unix(),
		}
	}

	// Retry on SQLITE_BUSY
	for i := 0; i < 5; i++ {
		rows, err = s.db.Query(sqlQuery, queryParams...)

		if err == nil {
			break
		}
		if i < 4 && (err.Error() == "database is locked" || err.Error() == "sql: database is closed") {
			time.Sleep(time.Duration(100*(i+1)) * time.Millisecond)
			continue
		}
		return nil, fmt.Errorf("failed to query all metrics: %w", err)
	}
	defer rows.Close()

	result := make(map[string][]MetricPoint)
	for rows.Next() {
		var metricType string
		var ts int64
		var p MetricPoint
		if err := rows.Scan(&metricType, &ts, &p.Value, &p.Min, &p.Max); err != nil {
			log.Warn().Err(err).Msg("Failed to scan metric row")
			continue
		}
		p.Timestamp = time.Unix(ts, 0)
		result[metricType] = append(result[metricType], p)
	}

	return result, rows.Err()
}

// queryAllBatchChunkSize limits the number of resource IDs per SQL IN clause
// to stay well within SQLite's host-parameter ceiling and keep individual
// queries fast.
const queryAllBatchChunkSize = 500

// QueryAllBatch retrieves all metric types for multiple resources of the same
// type in a single query. Returns map[resourceID]map[metricType][]MetricPoint.
// This avoids N+1 query patterns when loading charts for many resources.
// Resource IDs are deduplicated and chunked to stay within SQLite limits.
func (s *Store) QueryAllBatch(resourceType string, resourceIDs []string, start, end time.Time, stepSecs int64) (map[string]map[string][]MetricPoint, error) {
	return s.queryBatch(resourceType, resourceIDs, nil, start, end, stepSecs)
}

// QueryMetricTypesBatch retrieves only the requested metric types for multiple
// resources of the same type in a single query. Passing no metricTypes falls
// back to QueryAllBatch semantics.
func (s *Store) QueryMetricTypesBatch(
	resourceType string,
	resourceIDs []string,
	metricTypes []string,
	start, end time.Time,
	stepSecs int64,
) (map[string]map[string][]MetricPoint, error) {
	return s.queryBatch(resourceType, resourceIDs, metricTypes, start, end, stepSecs)
}

func (s *Store) queryBatch(
	resourceType string,
	resourceIDs []string,
	metricTypes []string,
	start, end time.Time,
	stepSecs int64,
) (map[string]map[string][]MetricPoint, error) {
	resourceType = normalizeMetricResourceType(resourceType)
	// Deduplicate resource IDs.
	seen := make(map[string]struct{}, len(resourceIDs))
	unique := make([]string, 0, len(resourceIDs))
	for _, id := range resourceIDs {
		normalizedID := normalizeMetricIdentifier(id)
		if normalizedID == "" {
			continue
		}
		if _, dup := seen[normalizedID]; !dup {
			seen[normalizedID] = struct{}{}
			unique = append(unique, normalizedID)
		}
	}
	if len(unique) == 0 {
		return map[string]map[string][]MetricPoint{}, nil
	}
	normalizedMetricTypes := normalizeMetricTypes(metricTypes)

	tiers := s.tierFallbacks(end.Sub(start))
	if len(tiers) == 0 {
		return map[string]map[string][]MetricPoint{}, nil
	}

	result := make(map[string]map[string][]MetricPoint, len(unique))
	unresolved := unique

	for _, tier := range tiers {
		if len(unresolved) == 0 {
			break
		}

		nextUnresolved := make([]string, 0, len(unresolved))

		// Process in chunks to stay within SQLite parameter limits.
		for lo := 0; lo < len(unresolved); lo += queryAllBatchChunkSize {
			hi := lo + queryAllBatchChunkSize
			if hi > len(unresolved) {
				hi = len(unresolved)
			}
			chunk := unresolved[lo:hi]
			tierResult, err := s.queryAllBatchWithTier(resourceType, chunk, normalizedMetricTypes, start, end, stepSecs, tier)
			if err != nil {
				return nil, err
			}

			// Match QueryAll semantics per resource: once any tier returns data
			// for a resource, stop falling back for that resource.
			for _, resID := range chunk {
				metricMap, found := tierResult[resID]
				if !found || len(metricMap) == 0 {
					nextUnresolved = append(nextUnresolved, resID)
					continue
				}
				result[resID] = metricMap
			}
		}

		unresolved = nextUnresolved
	}

	return result, nil
}

func normalizeMetricTypes(metricTypes []string) []string {
	if len(metricTypes) == 0 {
		return nil
	}

	seen := make(map[string]struct{}, len(metricTypes))
	normalized := make([]string, 0, len(metricTypes))
	for _, metricType := range metricTypes {
		canonical := normalizeMetricType(metricType)
		if canonical == "" {
			continue
		}
		if _, exists := seen[canonical]; exists {
			continue
		}
		seen[canonical] = struct{}{}
		normalized = append(normalized, canonical)
	}
	if len(normalized) == 0 {
		return nil
	}
	return normalized
}

func (s *Store) queryAllBatchWithTier(resourceType string, resourceIDs []string, metricTypes []string, start, end time.Time, stepSecs int64, tier Tier) (map[string]map[string][]MetricPoint, error) {
	placeholders := make([]string, len(resourceIDs))
	for i := range resourceIDs {
		placeholders[i] = "?"
	}
	inClause := strings.Join(placeholders, ",")
	metricClause := ""
	metricArgs := make([]interface{}, 0, len(metricTypes))
	if len(metricTypes) > 0 {
		metricPlaceholders := make([]string, len(metricTypes))
		for i, metricType := range metricTypes {
			metricPlaceholders[i] = "?"
			metricArgs = append(metricArgs, metricType)
		}
		metricClause = fmt.Sprintf(" AND metric_type IN (%s)", strings.Join(metricPlaceholders, ","))
	}

	var params []interface{}
	var sqlQuery string

	if stepSecs > 1 {
		params = make([]interface{}, 0, len(resourceIDs)+len(metricArgs)+4)
		params = append(params, resourceType)
		for _, id := range resourceIDs {
			params = append(params, id)
		}
		params = append(params, metricArgs...)
		params = append(params, string(tier), start.Unix(), end.Unix())

		sqlQuery = fmt.Sprintf(`
			SELECT
				resource_id,
				metric_type,
				timestamp,
				value,
				COALESCE(min_value, value),
				COALESCE(max_value, value)
			FROM metrics
			WHERE resource_type = ? AND resource_id IN (%s)%s AND tier = ?
			AND timestamp >= ? AND timestamp <= ?
			ORDER BY resource_id, metric_type, timestamp ASC
		`, inClause, metricClause)
	} else {
		params = make([]interface{}, 0, len(resourceIDs)+len(metricArgs)+4)
		params = append(params, resourceType)
		for _, id := range resourceIDs {
			params = append(params, id)
		}
		params = append(params, metricArgs...)
		params = append(params, string(tier), start.Unix(), end.Unix())

		sqlQuery = fmt.Sprintf(`
			SELECT resource_id, metric_type, timestamp, value, COALESCE(min_value, value), COALESCE(max_value, value)
			FROM metrics
			WHERE resource_type = ? AND resource_id IN (%s)%s AND tier = ?
			AND timestamp >= ? AND timestamp <= ?
			ORDER BY resource_id, metric_type, timestamp ASC
		`, inClause, metricClause)
	}

	// Retry on SQLITE_BUSY
	var rows *sql.Rows
	var err error
	for i := 0; i < 5; i++ {
		rows, err = s.db.Query(sqlQuery, params...)
		if err == nil {
			break
		}
		if i < 4 && (err.Error() == "database is locked" || err.Error() == "sql: database is closed") {
			time.Sleep(time.Duration(100*(i+1)) * time.Millisecond)
			continue
		}
		return nil, fmt.Errorf("failed to batch query metrics: %w", err)
	}
	defer rows.Close()

	result := make(map[string]map[string][]MetricPoint, len(resourceIDs))
	seriesCapacity := estimateQueryAllBatchSeriesCapacity(start, end, stepSecs)
	if stepSecs > 1 {
		type bucketAggregate struct {
			active      bool
			resourceID  string
			metricType  string
			bucketStart int64
			sum         float64
			min         float64
			max         float64
			count       int
		}

		appendBucketPoint := func(resourceID, metricType string, point MetricPoint) {
			if _, exists := result[resourceID]; !exists {
				result[resourceID] = make(map[string][]MetricPoint, 8)
			}
			if _, exists := result[resourceID][metricType]; !exists {
				result[resourceID][metricType] = make([]MetricPoint, 0, seriesCapacity)
			}
			result[resourceID][metricType] = append(result[resourceID][metricType], point)
		}

		var aggregate bucketAggregate
		flushAggregate := func() {
			if !aggregate.active || aggregate.count == 0 {
				return
			}
			appendBucketPoint(aggregate.resourceID, aggregate.metricType, MetricPoint{
				Timestamp: time.Unix(aggregate.bucketStart+(stepSecs/2), 0),
				Value:     aggregate.sum / float64(aggregate.count),
				Min:       aggregate.min,
				Max:       aggregate.max,
			})
			aggregate = bucketAggregate{}
		}

		for rows.Next() {
			var resourceID, metricType string
			var ts int64
			var value, minVal, maxVal float64
			if err := rows.Scan(&resourceID, &metricType, &ts, &value, &minVal, &maxVal); err != nil {
				log.Warn().Err(err).Msg("Failed to scan batch metric row")
				continue
			}

			bucketStart := (ts / stepSecs) * stepSecs
			if !aggregate.active {
				aggregate = bucketAggregate{
					active:      true,
					resourceID:  resourceID,
					metricType:  metricType,
					bucketStart: bucketStart,
					sum:         value,
					min:         minVal,
					max:         maxVal,
					count:       1,
				}
				continue
			}

			if aggregate.resourceID != resourceID || aggregate.metricType != metricType || aggregate.bucketStart != bucketStart {
				flushAggregate()
				aggregate = bucketAggregate{
					active:      true,
					resourceID:  resourceID,
					metricType:  metricType,
					bucketStart: bucketStart,
					sum:         value,
					min:         minVal,
					max:         maxVal,
					count:       1,
				}
				continue
			}

			aggregate.sum += value
			if minVal < aggregate.min {
				aggregate.min = minVal
			}
			if maxVal > aggregate.max {
				aggregate.max = maxVal
			}
			aggregate.count++
		}

		if err := rows.Err(); err != nil {
			return nil, err
		}
		flushAggregate()
		return result, nil
	}

	for rows.Next() {
		var resourceID, metricType string
		var ts int64
		var p MetricPoint
		if err := rows.Scan(&resourceID, &metricType, &ts, &p.Value, &p.Min, &p.Max); err != nil {
			log.Warn().Err(err).Msg("Failed to scan batch metric row")
			continue
		}
		p.Timestamp = time.Unix(ts, 0)

		if _, exists := result[resourceID]; !exists {
			result[resourceID] = make(map[string][]MetricPoint, 8)
		}
		if _, exists := result[resourceID][metricType]; !exists {
			result[resourceID][metricType] = make([]MetricPoint, 0, seriesCapacity)
		}
		result[resourceID][metricType] = append(result[resourceID][metricType], p)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return result, nil
}

func estimateQueryAllBatchSeriesCapacity(start, end time.Time, stepSecs int64) int {
	if stepSecs <= 1 {
		return 0
	}
	if !end.After(start) {
		return 1
	}

	stepDuration := time.Duration(stepSecs) * time.Second
	buckets := int(end.Sub(start)/stepDuration) + 2
	if buckets < 1 {
		return 1
	}
	return buckets
}

// selectTier chooses the appropriate data tier based on time range
// Note: Tier selection uses fixed thresholds to ensure queries use tiers with complete data:
// - Raw: up to 2 hours (high-resolution real-time data)
// - Minute: up to 24 hours (recent detailed data)
// - Hourly: up to 7 days (medium-term with mock/seeded data coverage)
// - Daily: beyond 7 days (long-term historical data)
func (s *Store) selectTier(duration time.Duration) Tier {
	const (
		rawThreshold    = 2 * time.Hour
		minuteThreshold = 24 * time.Hour
		hourlyThreshold = 7 * 24 * time.Hour
	)

	switch {
	case duration <= rawThreshold:
		return TierRaw
	case duration <= minuteThreshold:
		return TierMinute
	case duration <= hourlyThreshold:
		return TierHourly
	default:
		return TierDaily
	}
}

func (s *Store) tierFallbacks(duration time.Duration) []Tier {
	switch s.selectTier(duration) {
	case TierRaw:
		// Fall back to coarser tiers when raw is empty (e.g., mock mode with seeded data)
		return []Tier{TierRaw, TierMinute, TierHourly}
	case TierMinute:
		// Fall back to coarser tiers when minute is empty (e.g., mock mode with seeded data)
		return []Tier{TierMinute, TierRaw, TierHourly}
	case TierHourly:
		return []Tier{TierHourly, TierMinute, TierRaw}
	case TierDaily:
		return []Tier{TierDaily, TierHourly, TierMinute, TierRaw}
	default:
		return []Tier{TierRaw}
	}
}

// backgroundWorker owns buffered metric ingestion. Maintenance work runs on a
// separate scheduler so a long startup migration, rollup, or retention pass
// cannot leave write requests unconsumed until the bounded channel overflows.
func (s *Store) backgroundWorker() {
	defer close(s.doneCh)

	flushTicker := time.NewTicker(s.config.FlushInterval)
	defer flushTicker.Stop()

	for {
		select {
		case <-s.stopCh:
			var remaining []writeRequest
			if batch := s.drainBuffer(); len(batch) > 0 {
				remaining = append(remaining, writeRequest{metrics: batch})
			}
			close(s.writeCh)
			for req := range s.writeCh {
				remaining = append(remaining, req)
			}
			s.processWriteRequests(remaining)
			return

		case req, ok := <-s.writeCh:
			if !ok {
				return
			}
			s.processWriteRequests(s.coalesceQueuedRequests(req))

		case <-flushTicker.C:
			s.flushBufferedAsync()
		}
	}
}

// maintenanceWorker owns startup maintenance and periodic rollup/retention
// scheduling. Its database writes may still wait behind live metric writes,
// but its CPU or read work cannot stop the ingestion worker from draining the
// write channel.
func (s *Store) maintenanceWorker() {
	defer close(s.maintenanceDoneCh)

	rollupTicker := time.NewTicker(s.config.RollupInterval)
	retentionTicker := time.NewTicker(1 * time.Hour)
	defer rollupTicker.Stop()
	defer retentionTicker.Stop()

	for {
		select {
		case <-s.stopCh:
			return

		case maintenance := <-s.maintenanceCh:
			if maintenance.run != nil {
				maintenance.run()
			}
			if maintenance.done != nil {
				close(maintenance.done)
			}

		case <-rollupTicker.C:
			s.runRollup()

		case <-retentionTicker.C:
			s.runRetention()
		}
	}
}

// Flush writes any buffered metrics to the database and waits for all queued
// writes ahead of the flush barrier to become visible.
func (s *Store) Flush() {
	if s == nil || s.stopping.Load() {
		return
	}
	s.enqueueAndWait(writeRequest{metrics: s.drainBuffer()})
}

// runRollup aggregates raw data into higher tiers
func (s *Store) runRollup() {
	start := time.Now()

	// Rollup raw -> minute (for data older than 5 minutes)
	s.rollupTier(TierRaw, TierMinute, time.Minute, 5*time.Minute)

	// Rollup minute -> hourly (for data older than 1 hour)
	s.rollupTier(TierMinute, TierHourly, time.Hour, time.Hour)

	// Rollup hourly -> daily (for data older than 24 hours)
	s.rollupTier(TierHourly, TierDaily, 24*time.Hour, 24*time.Hour)

	log.Debug().Dur("duration", time.Since(start)).Msg("Metrics rollup completed")
}

// rollupTier aggregates data from one tier to another.
// Uses a single INSERT...SELECT...GROUP BY to batch-rollup all resource/metric
// combinations at once, replacing the previous N+1 pattern (candidate discovery
// query + per-candidate INSERT transaction).
func (s *Store) rollupTier(fromTier, toTier Tier, bucketSize, minAge time.Duration) {
	cutoff := time.Now().Add(-minAge).Unix()
	bucketSecs := int64(bucketSize.Seconds())
	if bucketSecs <= 0 {
		return
	}
	cutoffBucket := (cutoff / bucketSecs) * bucketSecs
	if cutoffBucket <= 0 {
		return
	}

	metaKey := fmt.Sprintf("rollup:%s:%s", fromTier, toTier)
	lastBucket, ok := s.getMetaInt(metaKey)
	if !ok {
		if maxTs, ok := s.getMaxTimestampForTier(toTier); ok {
			lastBucket = (maxTs / bucketSecs) * bucketSecs
			_ = s.setMetaInt(metaKey, lastBucket)
		}
	}

	if cutoffBucket <= lastBucket {
		return
	}

	chunkWindow := rollupChunkWindow(bucketSize)
	chunkSecs := int64(chunkWindow.Seconds())
	if chunkSecs <= 0 {
		chunkSecs = bucketSecs
	}

	windowStart := lastBucket
	processedChunks := 0
	processedAny := false

	for windowStart < cutoffBucket {
		nextBucket, hasSource := s.nextSourceRollupBucket(fromTier, windowStart, cutoffBucket, bucketSecs)
		if !hasSource {
			if processedAny {
				if err := s.setMetaInt(metaKey, cutoffBucket); err != nil {
					log.Warn().Err(err).Str("tier", string(fromTier)).Msg("Failed to persist rollup checkpoint")
				}
			}
			return
		}

		if nextBucket > windowStart {
			windowStart = nextBucket
		}
		windowEnd := windowStart + chunkSecs
		if windowEnd > cutoffBucket {
			windowEnd = cutoffBucket
		}
		if windowEnd <= windowStart {
			return
		}

		if !s.rollupTierWindow(fromTier, toTier, bucketSecs, windowStart, windowEnd) {
			return
		}

		windowStart = windowEnd
		processedChunks++
		processedAny = true

		if err := s.setMetaInt(metaKey, windowStart); err != nil {
			log.Warn().Err(err).Str("tier", string(fromTier)).Msg("Failed to persist rollup checkpoint")
			return
		}
		if processedChunks >= maxRollupChunksPerRun && windowStart < cutoffBucket {
			log.Debug().
				Str("from", string(fromTier)).
				Str("to", string(toTier)).
				Int("chunks", processedChunks).
				Msg("Metrics rollup paused after bounded chunk budget")
			return
		}
	}
}

func rollupChunkWindow(bucketSize time.Duration) time.Duration {
	if bucketSize <= 0 {
		return maxRollupChunkWindow
	}
	if bucketSize >= maxRollupChunkWindow {
		return bucketSize
	}
	buckets := int64(maxRollupChunkWindow / bucketSize)
	if buckets < 1 {
		buckets = 1
	}
	return time.Duration(buckets) * bucketSize
}

func (s *Store) nextSourceRollupBucket(tier Tier, startBucket, cutoffBucket, bucketSecs int64) (int64, bool) {
	var next sql.NullInt64
	if err := s.db.QueryRow(`
		SELECT MIN(timestamp)
		FROM metrics
		WHERE tier = ? AND timestamp >= ? AND timestamp < ?
	`, string(tier), startBucket, cutoffBucket).Scan(&next); err != nil {
		log.Warn().Err(err).Str("tier", string(tier)).Msg("Failed to locate next rollup source window")
		return 0, false
	}
	if !next.Valid {
		return 0, false
	}
	nextBucket := (next.Int64 / bucketSecs) * bucketSecs
	if nextBucket < startBucket {
		nextBucket = startBucket
	}
	return nextBucket, true
}

func (s *Store) rollupTierWindow(fromTier, toTier Tier, bucketSecs, startTs, endTs int64) bool {
	tx, err := s.db.Begin()
	if err != nil {
		log.Error().Err(err).Str("tier", string(fromTier)).Msg("Failed to begin rollup transaction")
		return false
	}
	defer func() { _ = tx.Rollback() }()

	_, err = tx.Exec(`
		INSERT OR IGNORE INTO metrics (resource_type, resource_id, metric_type, value, min_value, max_value, timestamp, tier)
		SELECT
			resource_type,
			resource_id,
			metric_type,
			AVG(value) as value,
			MIN(value) as min_value,
			MAX(value) as max_value,
			(timestamp / ?) * ? as bucket_ts,
			?
		FROM metrics
		WHERE tier = ? AND timestamp >= ? AND timestamp < ?
		GROUP BY resource_type, resource_id, metric_type, bucket_ts
	`, bucketSecs, bucketSecs, string(toTier), string(fromTier), startTs, endTs)
	if err != nil {
		log.Warn().Err(err).
			Str("from", string(fromTier)).
			Str("to", string(toTier)).
			Msg("Failed to batch rollup metrics")
		return false
	}

	if err := tx.Commit(); err != nil {
		log.Warn().Err(err).
			Str("from", string(fromTier)).
			Str("to", string(toTier)).
			Msg("Failed to commit rollup transaction")
		return false
	}
	return true
}

// rollupCandidate aggregates a single resource/metric from one tier to another
func (s *Store) rollupCandidate(resourceType, resourceID, metricType string, fromTier, toTier Tier, bucketSecs, startTs, endTs int64) {
	if startTs >= endTs {
		return
	}
	itx, err := s.db.Begin()
	if err != nil {
		return
	}
	defer func() { _ = itx.Rollback() }()

	// Aggregate data into buckets
	_, err = itx.Exec(`
		INSERT OR IGNORE INTO metrics (resource_type, resource_id, metric_type, value, min_value, max_value, timestamp, tier)
		SELECT 
			resource_type, 
			resource_id, 
			metric_type,
			AVG(value) as value,
			MIN(value) as min_value,
			MAX(value) as max_value,
			(timestamp / ?) * ? as bucket_ts,
			?
		FROM metrics
		WHERE resource_type = ? AND resource_id = ? AND metric_type = ? 
		AND tier = ? AND timestamp >= ? AND timestamp < ?
		GROUP BY resource_type, resource_id, metric_type, bucket_ts
	`, bucketSecs, bucketSecs, string(toTier), resourceType, resourceID, metricType, string(fromTier), startTs, endTs)

	if err != nil {
		log.Warn().Err(err).
			Str("resource", resourceID).
			Str("from", string(fromTier)).
			Str("to", string(toTier)).
			Msg("Failed to rollup metrics")
		return
	}

	if err := itx.Commit(); err != nil {
		log.Warn().Err(err).
			Str("resource", resourceID).
			Str("from", string(fromTier)).
			Str("to", string(toTier)).
			Msg("Failed to commit rollup transaction")
	}
}

func (s *Store) getMetaInt(key string) (int64, bool) {
	var value string
	err := s.db.QueryRow(`SELECT value FROM metrics_meta WHERE key = ?`, key).Scan(&value)
	if err == sql.ErrNoRows {
		return 0, false
	}
	if err != nil {
		log.Warn().Err(err).Str("key", key).Msg("Failed to read metrics metadata")
		return 0, false
	}
	parsed, err := strconv.ParseInt(value, 10, 64)
	if err != nil {
		log.Warn().Err(err).Str("key", key).Msg("Invalid metrics metadata value")
		return 0, false
	}
	return parsed, true
}

func (s *Store) setMetaInt(key string, value int64) error {
	_, err := s.db.Exec(`
		INSERT INTO metrics_meta (key, value)
		VALUES (?, ?)
		ON CONFLICT(key) DO UPDATE SET value = excluded.value
	`, key, strconv.FormatInt(value, 10))
	return err
}

func (s *Store) getMaxTimestampForTier(tier Tier) (int64, bool) {
	var maxTs sql.NullInt64
	if err := s.db.QueryRow(`SELECT MAX(timestamp) FROM metrics WHERE tier = ?`, string(tier)).Scan(&maxTs); err != nil {
		log.Warn().Err(err).Str("tier", string(tier)).Msg("Failed to read metrics max timestamp")
		return 0, false
	}
	if !maxTs.Valid || maxTs.Int64 <= 0 {
		return 0, false
	}
	return maxTs.Int64, true
}

// MaxTimestampsForTier returns the newest stored timestamp for every series
// present in the given tier. Backfill seeders use it to write only the gap
// between existing coverage and now instead of re-writing whole windows.
func (s *Store) MaxTimestampsForTier(tier Tier) (map[SeriesKey]time.Time, error) {
	rows, err := s.db.Query(`
		SELECT resource_type, resource_id, metric_type, MAX(timestamp)
		FROM metrics
		WHERE tier = ?
		GROUP BY resource_type, resource_id, metric_type
	`, string(tier))
	if err != nil {
		return nil, fmt.Errorf("failed to query metrics coverage for tier %q: %w", tier, err)
	}
	defer rows.Close()

	coverage := make(map[SeriesKey]time.Time)
	for rows.Next() {
		var key SeriesKey
		var ts int64
		if err := rows.Scan(&key.ResourceType, &key.ResourceID, &key.MetricType, &ts); err != nil {
			return nil, fmt.Errorf("failed to scan metrics coverage row: %w", err)
		}
		coverage[key] = time.Unix(ts, 0).UTC()
	}
	return coverage, rows.Err()
}

// runRetention deletes data older than retention period
func (s *Store) runRetention() {
	start := time.Now()
	now := time.Now()

	// Delete old data for each tier
	tiers := []struct {
		tier      Tier
		retention time.Duration
	}{
		{TierRaw, s.effectiveRetention(s.config.RetentionRaw, now)},
		{TierMinute, s.effectiveRetention(s.config.RetentionMinute, now)},
		{TierHourly, s.effectiveRetention(s.config.RetentionHourly, now)},
		{TierDaily, s.effectiveRetention(s.config.RetentionDaily, now)},
	}

	var totalDeleted int64
	for _, t := range tiers {
		cutoff := now.Add(-t.retention).Unix()
		result, err := s.db.Exec(`DELETE FROM metrics WHERE tier = ? AND timestamp < ?`, string(t.tier), cutoff)
		if err != nil {
			log.Warn().Err(err).Str("tier", string(t.tier)).Msg("Failed to prune metrics")
			continue
		}
		if affected, _ := result.RowsAffected(); affected > 0 {
			totalDeleted += affected
		}
	}

	if totalDeleted > 0 {
		log.Info().
			Int64("deleted", totalDeleted).
			Dur("duration", time.Since(start)).
			Msg("Metrics retention cleanup completed")
	}

	// Reclaim freed pages every cycle, even in an hour where nothing was
	// deleted, so a pre-existing freelist backlog still drains over time.
	s.reclaimFreePages()
}

// maxReclaimPages bounds how many freed SQLite pages reclaimFreePages returns
// to the OS per retention cycle (~50k pages * 4KiB ≈ 200MiB). A large backlog
// is drained over several hourly cycles instead of holding the write lock for
// minutes at once, while steady-state freelists (well under the cap) drain
// fully every cycle.
const maxReclaimPages = 50000

// reclaimFreePages returns freed pages to the OS. auto_vacuum is INCREMENTAL,
// so deleted-row pages sit on the freelist until reclaimed here. A previous
// fixed 5000-page batch could not keep up on busy instances: when an hourly
// retention pass frees more pages than the batch reclaims, the freelist grows
// net-positive every cycle and the database file bloats unboundedly (5GB+ of
// free pages over ~60MB of live data) even though row retention is working.
// Draining proportionally to the freelist, capped, fixes that.
func (s *Store) reclaimFreePages() {
	var freelist int64
	if err := s.db.QueryRow(`PRAGMA freelist_count`).Scan(&freelist); err != nil {
		log.Debug().Err(err).Msg("Failed to read freelist_count")
		return
	}
	if freelist == 0 {
		// Nothing to reclaim: skip the checkpoint too so steady-state WAL
		// cadence is no more aggressive than before this drained every cycle.
		return
	}
	pages := freelist
	if pages > maxReclaimPages {
		pages = maxReclaimPages
	}
	if _, err := s.db.Exec(fmt.Sprintf(`PRAGMA incremental_vacuum(%d)`, pages)); err != nil {
		log.Debug().Err(err).Msg("Incremental vacuum failed")
	}
	if _, err := s.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil {
		log.Debug().Err(err).Msg("WAL checkpoint failed")
	}
}

// SetMaxOpenConns sets the maximum number of open connections to the database.
func (s *Store) SetMaxOpenConns(n int) {
	s.db.SetMaxOpenConns(n)
	s.db.SetMaxIdleConns(n)
}

// Close shuts down the store gracefully
func (s *Store) Close() error {
	s.stopOnce.Do(func() {
		s.stopping.Store(true)
		close(s.stopCh)
	})

	// Wait for both the ingestion and maintenance workers to finish before
	// closing the shared database pool.
	shutdownTimer := time.NewTimer(5 * time.Second)
	defer shutdownTimer.Stop()

	select {
	case <-s.doneCh:
	case <-shutdownTimer.C:
		log.Warn().Msg("Metrics store ingestion shutdown timed out")
		return s.db.Close()
	}

	select {
	case <-s.maintenanceDoneCh:
	case <-shutdownTimer.C:
		log.Warn().Msg("Metrics store maintenance shutdown timed out")
	}

	return s.db.Close()
}

func ensureOwnerOnlyDir(dir string) error {
	if err := os.MkdirAll(dir, privateDirPerm); err != nil {
		return err
	}
	return os.Chmod(dir, privateDirPerm)
}

func rejectSymlinkOrNonRegular(path string) error {
	info, err := os.Lstat(path)
	if err != nil {
		return err
	}
	if info.Mode()&os.ModeSymlink != 0 {
		return fmt.Errorf("unsafe sqlite path %q: symlink is not allowed", path)
	}
	if !info.Mode().IsRegular() {
		return fmt.Errorf("unsafe sqlite path %q: non-regular file is not allowed", path)
	}
	return nil
}

func hardenSQLiteFile(path string) error {
	if err := rejectSymlinkOrNonRegular(path); err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return nil
		}
		return err
	}
	return os.Chmod(path, privateFilePerm)
}

func hardenSQLiteArtifacts(dbPath string) error {
	for _, path := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} {
		if err := hardenSQLiteFile(path); err != nil {
			if errors.Is(err, os.ErrNotExist) {
				continue
			}
			return err
		}
	}
	return nil
}

// Clear removes all stored metrics data.
func (s *Store) Clear() error {
	s.Flush()
	if _, err := s.db.Exec("DELETE FROM metrics"); err != nil {
		return err
	}
	_, _ = s.db.Exec("DELETE FROM metrics_meta")
	return nil
}

// Stats holds metrics store statistics
type Stats struct {
	DBSize        int64     `json:"dbSize"`
	RawCount      int64     `json:"rawCount"`
	MinuteCount   int64     `json:"minuteCount"`
	HourlyCount   int64     `json:"hourlyCount"`
	DailyCount    int64     `json:"dailyCount"`
	TotalWrites   int64     `json:"totalWrites"`
	BufferSize    int       `json:"bufferSize"`
	LastFlush     time.Time `json:"lastFlush"`
	LastRollup    time.Time `json:"lastRollup"`
	LastRetention time.Time `json:"lastRetention"`
}

// GetStats returns storage statistics
func (s *Store) GetStats() Stats {
	stats := Stats{}

	// Count by tier
	rows, err := s.db.Query(`SELECT tier, COUNT(*) FROM metrics GROUP BY tier`)
	if err == nil {
		defer rows.Close()
		for rows.Next() {
			var tier string
			var count int64
			if err := rows.Scan(&tier, &count); err == nil {
				switch tier {
				case "raw":
					stats.RawCount = count
				case "minute":
					stats.MinuteCount = count
				case "hourly":
					stats.HourlyCount = count
				case "daily":
					stats.DailyCount = count
				}
			}
		}
	}

	// Get database size
	if fi, err := os.Stat(s.config.DBPath); err == nil {
		stats.DBSize = fi.Size()
	}

	// Get buffer size
	s.bufferMu.Lock()
	stats.BufferSize = len(s.buffer)
	s.bufferMu.Unlock()

	return stats
}