mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-11-01 08:20:01 +00:00
5213b794b2
Keep continuous and late replay held without spending a provider attempt. Split mixed batches atomically so eligible alerts retain admitted destinations, occurrence links and retry budgets, and bind saved monitor policy before activating persisted work. Add local receipt, cancellation, rollback, reconstruction and race regression coverage with the owning contracts and verification routes. Change-source: pulse-maintainer
181 lines
6.4 KiB
Go
181 lines
6.4 KiB
Go
package notifications
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"database/sql"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
|
|
)
|
|
|
|
// prepareQuietHoursDelivery runs under the original batch's delivery gates,
|
|
// before claiming a provider attempt. Reloading the row preserves cancellation
|
|
// rewrites made after GetPending. Held work keeps the original row; a mixed
|
|
// batch gets one ready child, committed atomically with the held remainder.
|
|
// A hold is scheduling, not an attempt, failure, receipt or lifecycle change.
|
|
func (nq *NotificationQueue) prepareQuietHoursDelivery(
|
|
notif *QueuedNotification,
|
|
policy func(*alerts.Alert, time.Time) *time.Time,
|
|
) (bool, error) {
|
|
nq.mu.Lock()
|
|
defer nq.mu.Unlock()
|
|
now := time.Now()
|
|
if nq.now != nil {
|
|
now = nq.now()
|
|
}
|
|
current, err := nq.scanNotification(nq.db.QueryRow(`
|
|
SELECT id, type, method, status, alerts, config, attempts, max_attempts,
|
|
last_attempt, last_error, created_at, next_retry_at, completed_at,
|
|
payload_bytes, operational_links
|
|
FROM notification_queue WHERE id = ? AND status = 'pending'`, notif.ID))
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, fmt.Errorf("reload quiet-hours notification: %w", err)
|
|
}
|
|
// Another worker may already have postponed this row since GetPending.
|
|
if current.NextRetryAt != nil && current.NextRetryAt.After(now) {
|
|
return false, nil
|
|
}
|
|
var ready, held []*alerts.Alert
|
|
var replayAt *time.Time
|
|
for _, alert := range current.Alerts {
|
|
at := policy(alert, now)
|
|
if at == nil || !at.After(now) {
|
|
ready = append(ready, alert)
|
|
continue
|
|
}
|
|
held = append(held, alert)
|
|
if replayAt == nil || at.Before(*replayAt) {
|
|
value := at.UTC()
|
|
replayAt = &value
|
|
}
|
|
}
|
|
if len(held) == 0 {
|
|
*notif = *current
|
|
return true, nil
|
|
}
|
|
// The queue stores second precision. Never round a policy deadline down
|
|
// into a provider send (or a busy retry loop) before it becomes eligible.
|
|
deadline := replayAt.Truncate(time.Second)
|
|
if deadline.Before(*replayAt) {
|
|
deadline = deadline.Add(time.Second)
|
|
}
|
|
method := current.Method
|
|
if strings.HasPrefix(method, "resolved-group") {
|
|
// This batch has reached its admitted grouping deadline. It must not
|
|
// reopen that window and absorb later recoveries while quiet-held.
|
|
method = "quiet-hours-replay"
|
|
}
|
|
if len(ready) == 0 {
|
|
_, err := nq.db.Exec(`UPDATE notification_queue
|
|
SET next_retry_at = ?, method = ? WHERE id = ? AND status = 'pending'`, deadline.Unix(), method, current.ID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("postpone quiet-hours notification: %w", err)
|
|
}
|
|
current.NextRetryAt = &deadline
|
|
current.Method = method
|
|
*notif = *current
|
|
return false, nil
|
|
}
|
|
|
|
readyJSON, err := json.Marshal(ready)
|
|
if err != nil {
|
|
return false, fmt.Errorf("encode ready quiet-hours alerts: %w", err)
|
|
}
|
|
heldJSON, err := json.Marshal(held)
|
|
if err != nil {
|
|
return false, fmt.Errorf("encode held quiet-hours alerts: %w", err)
|
|
}
|
|
// Include the parent identity and exact ready payload. Restarts and duplicate
|
|
// worker snapshots cannot generate a second child for the same partition.
|
|
sum := sha256.Sum256(append([]byte(current.ID+"\x00"), readyJSON...))
|
|
childID := current.ID + ":quiet:" + hex.EncodeToString(sum[:8])
|
|
readyLinks, heldLinks, err := partitionQuietHoursLinks(current.Links, ready, held, childID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
readyLinksJSON, err := json.Marshal(readyLinks)
|
|
if err != nil {
|
|
return false, fmt.Errorf("encode ready quiet-hours links: %w", err)
|
|
}
|
|
heldLinksJSON, err := json.Marshal(heldLinks)
|
|
if err != nil {
|
|
return false, fmt.Errorf("encode held quiet-hours links: %w", err)
|
|
}
|
|
|
|
tx, err := nq.db.Begin()
|
|
if err != nil {
|
|
return false, fmt.Errorf("begin quiet-hours partition: %w", err)
|
|
}
|
|
defer tx.Rollback()
|
|
// Copy the admitted destination bytes and attempt budget, not the current
|
|
// destination configuration. Previous attempts remain in the original audit
|
|
// trail; splitting never resets the retry budget or fabricates a new audit.
|
|
_, err = tx.Exec(`INSERT INTO notification_queue
|
|
(id, type, method, status, alerts, operational_links, config, attempts,
|
|
max_attempts, last_attempt, last_error, created_at, next_retry_at,
|
|
completed_at, payload_bytes)
|
|
SELECT ?, type, method, status, ?, ?, config, attempts, max_attempts,
|
|
last_attempt, last_error, created_at, NULL, completed_at, payload_bytes
|
|
FROM notification_queue WHERE id = ? AND status = 'pending'`, childID, string(readyJSON), string(readyLinksJSON), current.ID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("persist ready quiet-hours partition: %w", err)
|
|
}
|
|
_, err = tx.Exec(`UPDATE notification_queue
|
|
SET alerts = ?, operational_links = ?, next_retry_at = ?, method = ?
|
|
WHERE id = ? AND status = 'pending'`, string(heldJSON), string(heldLinksJSON), deadline.Unix(), method, current.ID)
|
|
if err != nil {
|
|
return false, fmt.Errorf("persist held quiet-hours partition: %w", err)
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return false, fmt.Errorf("commit quiet-hours partition: %w", err)
|
|
}
|
|
current.ID = childID
|
|
current.Alerts = ready
|
|
current.Links = readyLinks
|
|
current.NextRetryAt = nil
|
|
*notif = *current
|
|
return true, nil
|
|
}
|
|
|
|
func partitionQuietHoursLinks(
|
|
links []operationaltrust.NotificationLink,
|
|
ready, held []*alerts.Alert,
|
|
childID string,
|
|
) ([]operationaltrust.NotificationLink, []operationaltrust.NotificationLink, error) {
|
|
keys := func(batch []*alerts.Alert) map[string]bool {
|
|
result := make(map[string]bool, len(batch))
|
|
for _, alert := range batch {
|
|
if alert != nil && alert.OperationalRecord != nil && alert.LatestTransition != nil {
|
|
result[alert.OperationalRecord.ID+"\x00"+alert.LatestTransition.ID] = true
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
readyKeys, heldKeys := keys(ready), keys(held)
|
|
var readyLinks, heldLinks []operationaltrust.NotificationLink
|
|
for _, link := range links {
|
|
key := link.OperationalRecordID + "\x00" + link.TransitionID
|
|
switch {
|
|
case readyKeys[key] && !heldKeys[key]:
|
|
link = link.Clone()
|
|
link.NotificationID = childID
|
|
readyLinks = append(readyLinks, link)
|
|
case heldKeys[key] && !readyKeys[key]:
|
|
heldLinks = append(heldLinks, link.Clone())
|
|
default:
|
|
// Do not silently discard or misattribute a historical linkage.
|
|
return nil, nil, fmt.Errorf("quiet-hours partition has an ambiguous operational link")
|
|
}
|
|
}
|
|
return readyLinks, heldLinks, nil
|
|
}
|