tidaldb/docs/ops/prometheus-alerts.yaml

# =============================================================================
# TIDALDB ALERT CONTRACT
# =============================================================================
# The orchard9 fleet runs these expressions through vmalert from:
#   ../k3s-fleet/deployments/k8s/base/observability/alerting-rules.yaml
#
# This product-side copy documents the emitted metrics and alert thresholds.
# Keep every expression and threshold aligned with the fleet source when the
# contract changes. The fleet source adds routing labels (`capability`,
# `surface`) and runbook URLs required by the shared Alertmanager pipeline.
#
# Every metric referenced below is emitted by tidalDB today. Standalone rules
# stay inactive when no standalone instance is running; cluster-only series
# activate only in cluster mode. The shared fleet rules additionally cover
# target loss, unavailable replicas, image-pull failures, restarts, PVCs, and
# Kubernetes Jobs.
# =============================================================================
groups:
  - name: tidaldb
    interval: 30s
    rules:
      - alert: TidalDBDown
        expr: tidaldb_health_ok == 0
        for: 1m
        labels: { severity: critical }
        annotations:
          summary: "tidalDB is unhealthy"
          description: "tidaldb_health_ok is 0 — database is unhealthy or shut down."

      - alert: TidalDBCheckpointStale
        expr: tidaldb_checkpoint_age_seconds > 300
        for: 2m
        labels: { severity: warning }
        annotations:
          summary: "Signal checkpoint not running"
          description: "{{ $value }}s since last checkpoint (threshold: 300s). Signal durability at risk."

      - alert: TidalDBCheckpointFailures
        expr: increase(tidaldb_checkpoint_failures_total[5m]) > 0
        labels: { severity: warning }
        annotations:
          summary: "Signal checkpoint failures detected"
          description: "Checkpoint failures in last 5m. Check disk space and storage errors."

      - alert: TidalDBWALDiskPressure
        expr: tidaldb_wal_lag_bytes > 1000000000
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "WAL disk usage exceeds 1GB"
          description: "{{ $value | humanize1024 }}B of WAL uncompacted. Compaction may be stuck."

      - alert: TidalDBSignalBacklog
        expr: tidaldb_signal_hot_entries > 4000000
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "Signal ledger over 80% of capacity"
          description: "{{ $value }} hot entries (threshold: 4M / 80% of 5M budget)."

      - alert: TidalDBDegradedRanking
        expr: tidaldb_degradation_level > 0
        for: 2m
        labels: { severity: warning }
        annotations:
          summary: "Ranking quality degraded"
          description: "Degradation level {{ $value }} active. Scale up or reduce load."

      - alert: TidalDBSessionLeak
        # tidaldb_active_sessions is a GAUGE, so rate()/increase() are invalid
        # (those operators only count counter resets). deriv() fits a least-squares
        # slope over the window and is the gauge-correct way to detect sustained
        # growth: > 0.03 sessions/sec is ~> 9 new sessions over a 5m window.
        expr: deriv(tidaldb_active_sessions[5m]) > 0.03 and tidaldb_active_sessions > 100
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "Active session count growing steadily"
          description: "{{ $value }} active sessions and trending up (deriv > 0.03/s). Agents may not be closing sessions."

      - alert: TidalDBHighRateLimiting
        expr: rate(tidaldb_rate_limited_total[5m]) > 100
        for: 5m
        labels: { severity: info }
        annotations:
          summary: "Sustained rate limiting"
          description: "{{ $value }}/s rate-limited writes. Review agent rate limit config."

      - alert: TidalDBTantivySegmentBloat
        expr: tidaldb_tantivy_segment_count > 30
        for: 10m
        labels: { severity: warning }
        annotations:
          summary: "Tantivy segment count elevated"
          description: "{{ $value }} segments (threshold: 30). Text syncer may be stalled."

      - alert: TidalDBSlowRetrieve
        expr: histogram_quantile(0.95, rate(tidaldb_retrieve_latency_us_bucket[5m])) > 500000
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "Retrieve p95 latency exceeds 500ms"
          description: "p95 retrieve latency is {{ $value | humanizeDuration }}. Check signal ledger load and degradation level."

      - alert: TidalDBSlowSearch
        expr: histogram_quantile(0.95, rate(tidaldb_search_latency_us_bucket[5m])) > 1000000
        for: 5m
        labels: { severity: warning }
        annotations:
          summary: "Search p95 latency exceeds 1s"
          description: "p95 search latency is {{ $value | humanizeDuration }}. Check Tantivy segment count and ANN index health."

  # ---------------------------------------------------------------------------
  # Cluster-mode rules (m11p8). Every metric below is emitted by
  # tidal/src/db/metrics/cluster.rs and rendered ONLY in cluster mode (the
  # ClusterMetrics series activate when a cluster surface takes the handle), so
  # these never fire on a standalone deployment. Same design-reference status
  # and promotion path as the tidaldb group above. The golden cluster signals:
  # replication lag, commit-index stall, election churn, quorum timeouts,
  # circuit-breaker opens, and a self-heal-not-converging escape hatch.
  # ---------------------------------------------------------------------------
  - name: tidaldb-cluster
    interval: 30s
    rules:
      - alert: TidalDBClusterReplicationLagHigh
        expr: (tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq) > 1000
        for: 2m
        labels: { severity: warning }
        annotations:
          summary: "Quorum replication lag elevated"
          description: "{{ $value }} events between the leader's flushed frontier and the quorum commit index for 2m (SLA < 2s of writes). A follower is lagging — check per-peer ship queue depth and breaker state."

      - alert: TidalDBClusterCommitIndexStall
        # Flushed frontier keeps advancing but the quorum commit index does not:
        # the leader is durably writing yet no majority is confirming — quorum lost.
        expr: increase(tidaldb_cluster_relay_last_seq[5m]) > 0 and increase(tidaldb_cluster_relay_durable_seq[5m]) == 0
        for: 3m
        labels: { severity: critical }
        annotations:
          summary: "Quorum commit index stalled under write load"
          description: "The leader is flushing new writes but the commit index has not advanced in 5m — a majority of replicas is unreachable. ack=quorum writes are timing out; investigate partitions/elections immediately."

      - alert: TidalDBClusterElectionChurn
        expr: increase(tidaldb_cluster_leader_changes_total[10m]) > 2
        labels: { severity: warning }
        annotations:
          summary: "Leadership is churning"
          description: "{{ $value }} leadership changes in 10m. Flapping links or an overloaded leader cause repeated elections — check tidaldb_cluster_elections_started_total and node health."

      - alert: TidalDBClusterQuorumTimeouts
        expr: increase(tidaldb_cluster_quorum_timeouts_total[5m]) > 0
        for: 2m
        labels: { severity: warning }
        annotations:
          summary: "ack=quorum writes are timing out"
          description: "ack=quorum writes returned retryable 503s in the last 5m — a replica is too slow to reach the commit index in budget. Identify the laggard via per-peer ship queue depth."

      - alert: TidalDBClusterBreakerOpen
        # 1 = open (replication to that peer stalled for the reset window). 2 =
        # half-open is transient and expected during recovery, so alert on == 1.
        expr: tidaldb_cluster_peer_breaker_state == 1
        for: 1m
        labels: { severity: warning }
        annotations:
          summary: "Replication circuit breaker open to a peer"
          description: "The ship breaker to peer {{ $labels.peer_shard }} has been open for 1m — replication to it is stalled. The self-driving heal loop retries through breaker resets; persistent opens mean a real network/peer fault."

      - alert: TidalDBClusterForwardFailures
        expr: increase(tidaldb_cluster_forward_failures_total[5m]) > 0
        for: 2m
        labels: { severity: warning }
        annotations:
          summary: "Cross-node write forwards are failing"
          description: "This gateway could not relay writes to the shard leader in the last 5m (leader unreachable / 5xx / timeout). An election may be in flight or the leader is partitioned."

      - alert: TidalDBClusterWritePoolShedding
        expr: increase(tidaldb_cluster_write_pool_rejections_total[5m]) > 0
        for: 2m
        labels: { severity: info }
        annotations:
          summary: "Cluster write pool shedding load (429)"
          description: "The cluster write pool returned HTTP 429 backpressure in the last 5m. Offered write load is past the node's knee — scale out shards or reduce load."

      - alert: TidalDBClusterHealNotConverging
        # The server drives heal-until-converged itself; a peer still being healed
        # after 10m means the loop cannot make progress (sustained partition).
        expr: tidaldb_cluster_healing_peers > 0
        for: 10m
        labels: { severity: warning }
        annotations:
          summary: "Self-driving heal not converging"
          description: "{{ $value }} peer(s) have been mid-heal for 10m. The heal loop retries through breaker resets automatically; a peer stuck this long is a real partition or a dead node — investigate the link, do NOT re-issue heal by hand."

      - alert: TidalDBClusterPeerShipStall
        # A peer stops accepting batches (its acked frontier flatlines) while
        # flushed events queue behind it: deriv() is the gauge-correct way to
        # detect a flat acked seqno (rate()/increase() are invalid on gauges).
        expr: deriv(tidaldb_cluster_peer_acked_seqno[2m]) == 0 and tidaldb_cluster_peer_ship_queue_depth > 0
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: "Replication to a peer has stalled with events queued"
          description: "Peer {{ $labels.peer_shard }} stopped accepting batches while events queue behind it (partition, dead peer, or paused sender). Check /cluster/status and the self-heal loop."

      - alert: TidalDBClusterQuorumLag
        expr: (tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq) > 10000
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: "Quorum commit index is far behind the leader frontier"
          description: "{{ $value }} events between the leader's flushed frontier and the quorum commit index (threshold 10000). A majority of the replica set is not confirming durability (down/partitioned followers, or follower apply throughput exhausted). ack=quorum writes will 503; the bodies name the laggards."

      - alert: TidalDBClusterDivergenceQuarantine
        # 1 = the node fenced itself from the data plane after detecting a
        # divergent suffix. Since m11p5 it auto-reseeds on next boot; a persistent
        # latch means the reseed is not completing.
        expr: tidaldb_cluster_divergence_quarantined == 1
        for: 2m
        labels: { severity: critical }
        annotations:
          summary: "A node has quarantined itself for a divergent suffix"
          description: "A node fenced itself from the data plane after detecting a divergent suffix. Since m11p5 it auto-reseeds on its next boot; if the latch persists, the reseed is not completing — check tidaldb_cluster_reseed_required and the snapshot path."

      - alert: TidalDBClusterReseedPending
        # Durable reseed-marker latch. Expected briefly after a quarantine or a
        # behind-a-compacted-leader restart; a persistent latch means the snapshot
        # fetch is failing.
        expr: tidaldb_cluster_reseed_required == 1
        for: 10m
        labels: { severity: warning }
        annotations:
          summary: "A node has a reseed pending but has not completed it"
          description: "A node has latched the reseed marker but has not completed a snapshot reseed in 10m. Expected briefly after a quarantine or a behind-a-compacted-leader restart; a persistent latch means the snapshot fetch is failing (no reachable leader, capability gate, or staging fault)."

      - alert: TidalDBClusterSnapshotPinForceDrop
        # A staged-snapshot retention pin was force-dropped past the hard cap — a
        # joiner started a reseed and never released (died mid-fetch). The dropped
        # pin protects compaction.
        expr: increase(tidaldb_cluster_snapshot_pin_force_drops_total[1h]) > 0
        labels: { severity: warning }
        annotations:
          summary: "A staged-snapshot retention pin was force-dropped"
          description: "A staged-snapshot retention pin was force-dropped past the hard cap in the last hour — a joiner started a reseed and never released (died mid-fetch). The dropped pin protects compaction; the stranded joiner must be re-driven or removed."