# ============================================================================= # DESIGN-REFERENCE RULE SET — NOT LOADED BY ANY ALERTMANAGER TODAY. # ============================================================================= # This file is reference/design config. It is NOT wired into any running # alerting pipeline: # - vmalert (pilot/dev) loads ONLY ops/vmalert/rules/*.yaml # - prod CRDs live ONLY in infra/k8s/prom/prometheus-rules/*.yaml # Nothing mounts or imports docs/ops/prometheus-alerts.yaml. # Do not read these as live pager rules. # # Provenance: every metric referenced below (tidaldb_health_ok, # tidaldb_checkpoint_age_seconds, tidaldb_checkpoint_failures_total, # tidaldb_wal_lag_bytes, tidaldb_signal_hot_entries, tidaldb_degradation_level, # tidaldb_active_sessions, tidaldb_rate_limited_total, # tidaldb_tantivy_segment_count, tidaldb_retrieve_latency_us_bucket, # tidaldb_search_latency_us_bucket) IS really emitted today by # tidal/src/db/metrics/mod.rs. So these rules are promotable — they # are not orphaned against phantom metrics. # # Promotion path (the right long-term fix — do it deliberately, do NOT # hand-copy these as live pager rules without the steps below): # 1. Add a vmalert rule file: ops/vmalert/rules/tidaldb.yaml, translating # each rule and stamping the canonical labels every routed rule carries — # severity + capability + surface (+ optional component). Map per the # canonical severity taxonomy in ops/alerting.md: # labels {severity: critical} = pager class (TidalDBDown only here), # {severity: warning} = non-paging / business-hours, # {severity: info} = pure-FYI. # Add capability: tidaldb and a surface label per rule, and a # runbook_url annotation: # https://github.com/orchard9/tidaldb/blob/main/docs/runbooks/.md # 2. Add the prod CRD twin: infra/k8s/prom/prometheus-rules/tidaldb.yaml, # kept byte-aligned in expr/threshold with the vmalert file. # 3. Verify the alerts fire against real metrics (scrape tidaldb, force a # degraded state) before declaring them on-call-ready. # Until steps 1–3 land, this file stays a design reference only. # ============================================================================= groups: - name: tidaldb interval: 30s rules: - alert: TidalDBDown expr: tidaldb_health_ok == 0 for: 1m labels: { severity: critical } annotations: summary: "tidalDB is unhealthy" description: "tidaldb_health_ok is 0 — database is unhealthy or shut down." - alert: TidalDBCheckpointStale expr: tidaldb_checkpoint_age_seconds > 300 for: 2m labels: { severity: warning } annotations: summary: "Signal checkpoint not running" description: "{{ $value }}s since last checkpoint (threshold: 300s). Signal durability at risk." - alert: TidalDBCheckpointFailures expr: increase(tidaldb_checkpoint_failures_total[5m]) > 0 labels: { severity: warning } annotations: summary: "Signal checkpoint failures detected" description: "Checkpoint failures in last 5m. Check disk space and storage errors." - alert: TidalDBWALDiskPressure expr: tidaldb_wal_lag_bytes > 1000000000 for: 5m labels: { severity: warning } annotations: summary: "WAL disk usage exceeds 1GB" description: "{{ $value | humanize1024 }}B of WAL uncompacted. Compaction may be stuck." - alert: TidalDBSignalBacklog expr: tidaldb_signal_hot_entries > 4000000 for: 5m labels: { severity: warning } annotations: summary: "Signal ledger over 80% of capacity" description: "{{ $value }} hot entries (threshold: 4M / 80% of 5M budget)." - alert: TidalDBDegradedRanking expr: tidaldb_degradation_level > 0 for: 2m labels: { severity: warning } annotations: summary: "Ranking quality degraded" description: "Degradation level {{ $value }} active. Scale up or reduce load." - alert: TidalDBSessionLeak # tidaldb_active_sessions is a GAUGE, so rate()/increase() are invalid # (those operators only count counter resets). deriv() fits a least-squares # slope over the window and is the gauge-correct way to detect sustained # growth: > 0.03 sessions/sec is ~> 9 new sessions over a 5m window. expr: deriv(tidaldb_active_sessions[5m]) > 0.03 and tidaldb_active_sessions > 100 for: 5m labels: { severity: warning } annotations: summary: "Active session count growing steadily" description: "{{ $value }} active sessions and trending up (deriv > 0.03/s). Agents may not be closing sessions." - alert: TidalDBHighRateLimiting expr: rate(tidaldb_rate_limited_total[5m]) > 100 for: 5m labels: { severity: info } annotations: summary: "Sustained rate limiting" description: "{{ $value }}/s rate-limited writes. Review agent rate limit config." - alert: TidalDBTantivySegmentBloat expr: tidaldb_tantivy_segment_count > 30 for: 10m labels: { severity: warning } annotations: summary: "Tantivy segment count elevated" description: "{{ $value }} segments (threshold: 30). Text syncer may be stalled." - alert: TidalDBSlowRetrieve expr: histogram_quantile(0.95, rate(tidaldb_retrieve_latency_us_bucket[5m])) > 500000 for: 5m labels: { severity: warning } annotations: summary: "Retrieve p95 latency exceeds 500ms" description: "p95 retrieve latency is {{ $value | humanizeDuration }}. Check signal ledger load and degradation level." - alert: TidalDBSlowSearch expr: histogram_quantile(0.95, rate(tidaldb_search_latency_us_bucket[5m])) > 1000000 for: 5m labels: { severity: warning } annotations: summary: "Search p95 latency exceeds 1s" description: "p95 search latency is {{ $value | humanizeDuration }}. Check Tantivy segment count and ANN index health." # --------------------------------------------------------------------------- # Cluster-mode rules (m11p8). Every metric below is emitted by # tidal/src/db/metrics/cluster.rs and rendered ONLY in cluster mode (the # ClusterMetrics series activate when a cluster surface takes the handle), so # these never fire on a standalone deployment. Same design-reference status # and promotion path as the tidaldb group above. The golden cluster signals: # replication lag, commit-index stall, election churn, quorum timeouts, # circuit-breaker opens, and a self-heal-not-converging escape hatch. # --------------------------------------------------------------------------- - name: tidaldb-cluster interval: 30s rules: - alert: TidalDBClusterReplicationLagHigh expr: (tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq) > 1000 for: 2m labels: { severity: warning } annotations: summary: "Quorum replication lag elevated" description: "{{ $value }} events between the leader's flushed frontier and the quorum commit index for 2m (SLA < 2s of writes). A follower is lagging — check per-peer ship queue depth and breaker state." - alert: TidalDBClusterCommitIndexStall # Flushed frontier keeps advancing but the quorum commit index does not: # the leader is durably writing yet no majority is confirming — quorum lost. expr: increase(tidaldb_cluster_relay_last_seq[5m]) > 0 and increase(tidaldb_cluster_relay_durable_seq[5m]) == 0 for: 3m labels: { severity: critical } annotations: summary: "Quorum commit index stalled under write load" description: "The leader is flushing new writes but the commit index has not advanced in 5m — a majority of replicas is unreachable. ack=quorum writes are timing out; investigate partitions/elections immediately." - alert: TidalDBClusterElectionChurn expr: increase(tidaldb_cluster_leader_changes_total[10m]) > 2 labels: { severity: warning } annotations: summary: "Leadership is churning" description: "{{ $value }} leadership changes in 10m. Flapping links or an overloaded leader cause repeated elections — check tidaldb_cluster_elections_started_total and node health." - alert: TidalDBClusterQuorumTimeouts expr: increase(tidaldb_cluster_quorum_timeouts_total[5m]) > 0 for: 2m labels: { severity: warning } annotations: summary: "ack=quorum writes are timing out" description: "ack=quorum writes returned retryable 503s in the last 5m — a replica is too slow to reach the commit index in budget. Identify the laggard via per-peer ship queue depth." - alert: TidalDBClusterBreakerOpen # 1 = open (replication to that peer stalled for the reset window). 2 = # half-open is transient and expected during recovery, so alert on == 1. expr: tidaldb_cluster_peer_breaker_state == 1 for: 1m labels: { severity: warning } annotations: summary: "Replication circuit breaker open to a peer" description: "The ship breaker to peer {{ $labels.peer_shard }} has been open for 1m — replication to it is stalled. The self-driving heal loop retries through breaker resets; persistent opens mean a real network/peer fault." - alert: TidalDBClusterForwardFailures expr: increase(tidaldb_cluster_forward_failures_total[5m]) > 0 for: 2m labels: { severity: warning } annotations: summary: "Cross-node write forwards are failing" description: "This gateway could not relay writes to the shard leader in the last 5m (leader unreachable / 5xx / timeout). An election may be in flight or the leader is partitioned." - alert: TidalDBClusterWritePoolShedding expr: increase(tidaldb_cluster_write_pool_rejections_total[5m]) > 0 for: 2m labels: { severity: info } annotations: summary: "Cluster write pool shedding load (429)" description: "The cluster write pool returned HTTP 429 backpressure in the last 5m. Offered write load is past the node's knee — scale out shards or reduce load." - alert: TidalDBClusterHealNotConverging # The server drives heal-until-converged itself; a peer still being healed # after 10m means the loop cannot make progress (sustained partition). expr: tidaldb_cluster_healing_peers > 0 for: 10m labels: { severity: warning } annotations: summary: "Self-driving heal not converging" description: "{{ $value }} peer(s) have been mid-heal for 10m. The heal loop retries through breaker resets automatically; a peer stuck this long is a real partition or a dead node — investigate the link, do NOT re-issue heal by hand."