# ============================================================================= # TIDALDB ALERT CONTRACT # ============================================================================= # The orchard9 fleet runs these expressions through vmalert from: # ../k3s-fleet/deployments/k8s/base/observability/alerting-rules.yaml # # This product-side copy documents the emitted metrics and alert thresholds. # Keep every expression and threshold aligned with the fleet source when the # contract changes. The fleet source adds routing labels (`capability`, # `surface`) and runbook URLs required by the shared Alertmanager pipeline. # # Every metric referenced below is emitted by tidalDB today. Standalone rules # stay inactive when no standalone instance is running; cluster-only series # activate only in cluster mode. The shared fleet rules additionally cover # target loss, unavailable replicas, image-pull failures, restarts, PVCs, and # Kubernetes Jobs. # ============================================================================= groups: - name: tidaldb interval: 30s rules: - alert: TidalDBDown expr: tidaldb_health_ok == 0 for: 1m labels: { severity: critical } annotations: summary: "tidalDB is unhealthy" description: "tidaldb_health_ok is 0 — database is unhealthy or shut down." - alert: TidalDBCheckpointStale expr: tidaldb_checkpoint_age_seconds > 300 for: 2m labels: { severity: warning } annotations: summary: "Signal checkpoint not running" description: "{{ $value }}s since last checkpoint (threshold: 300s). Signal durability at risk." - alert: TidalDBCheckpointFailures expr: increase(tidaldb_checkpoint_failures_total[5m]) > 0 labels: { severity: warning } annotations: summary: "Signal checkpoint failures detected" description: "Checkpoint failures in last 5m. Check disk space and storage errors." - alert: TidalDBWALDiskPressure expr: tidaldb_wal_lag_bytes > 1000000000 for: 5m labels: { severity: warning } annotations: summary: "WAL disk usage exceeds 1GB" description: "{{ $value | humanize1024 }}B of WAL uncompacted. Compaction may be stuck." - alert: TidalDBSignalBacklog expr: tidaldb_signal_hot_entries > 4000000 for: 5m labels: { severity: warning } annotations: summary: "Signal ledger over 80% of capacity" description: "{{ $value }} hot entries (threshold: 4M / 80% of 5M budget)." - alert: TidalDBDegradedRanking expr: tidaldb_degradation_level > 0 for: 2m labels: { severity: warning } annotations: summary: "Ranking quality degraded" description: "Degradation level {{ $value }} active. Scale up or reduce load." - alert: TidalDBSessionLeak # tidaldb_active_sessions is a GAUGE, so rate()/increase() are invalid # (those operators only count counter resets). deriv() fits a least-squares # slope over the window and is the gauge-correct way to detect sustained # growth: > 0.03 sessions/sec is ~> 9 new sessions over a 5m window. expr: deriv(tidaldb_active_sessions[5m]) > 0.03 and tidaldb_active_sessions > 100 for: 5m labels: { severity: warning } annotations: summary: "Active session count growing steadily" description: "{{ $value }} active sessions and trending up (deriv > 0.03/s). Agents may not be closing sessions." - alert: TidalDBHighRateLimiting expr: rate(tidaldb_rate_limited_total[5m]) > 100 for: 5m labels: { severity: info } annotations: summary: "Sustained rate limiting" description: "{{ $value }}/s rate-limited writes. Review agent rate limit config." - alert: TidalDBTantivySegmentBloat expr: tidaldb_tantivy_segment_count > 30 for: 10m labels: { severity: warning } annotations: summary: "Tantivy segment count elevated" description: "{{ $value }} segments (threshold: 30). Text syncer may be stalled." - alert: TidalDBSlowRetrieve expr: histogram_quantile(0.95, rate(tidaldb_retrieve_latency_us_bucket[5m])) > 500000 for: 5m labels: { severity: warning } annotations: summary: "Retrieve p95 latency exceeds 500ms" description: "p95 retrieve latency is {{ $value | humanizeDuration }}. Check signal ledger load and degradation level." - alert: TidalDBSlowSearch expr: histogram_quantile(0.95, rate(tidaldb_search_latency_us_bucket[5m])) > 1000000 for: 5m labels: { severity: warning } annotations: summary: "Search p95 latency exceeds 1s" description: "p95 search latency is {{ $value | humanizeDuration }}. Check Tantivy segment count and ANN index health." # --------------------------------------------------------------------------- # Cluster-mode rules (m11p8). Every metric below is emitted by # tidal/src/db/metrics/cluster.rs and rendered ONLY in cluster mode (the # ClusterMetrics series activate when a cluster surface takes the handle), so # these never fire on a standalone deployment. Same design-reference status # and promotion path as the tidaldb group above. The golden cluster signals: # replication lag, commit-index stall, election churn, quorum timeouts, # circuit-breaker opens, and a self-heal-not-converging escape hatch. # --------------------------------------------------------------------------- - name: tidaldb-cluster interval: 30s rules: - alert: TidalDBClusterReplicationLagHigh expr: (tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq) > 1000 for: 2m labels: { severity: warning } annotations: summary: "Quorum replication lag elevated" description: "{{ $value }} events between the leader's flushed frontier and the quorum commit index for 2m (SLA < 2s of writes). A follower is lagging — check per-peer ship queue depth and breaker state." - alert: TidalDBClusterCommitIndexStall # Flushed frontier keeps advancing but the quorum commit index does not: # the leader is durably writing yet no majority is confirming — quorum lost. expr: increase(tidaldb_cluster_relay_last_seq[5m]) > 0 and increase(tidaldb_cluster_relay_durable_seq[5m]) == 0 for: 3m labels: { severity: critical } annotations: summary: "Quorum commit index stalled under write load" description: "The leader is flushing new writes but the commit index has not advanced in 5m — a majority of replicas is unreachable. ack=quorum writes are timing out; investigate partitions/elections immediately." - alert: TidalDBClusterElectionChurn expr: increase(tidaldb_cluster_leader_changes_total[10m]) > 2 labels: { severity: warning } annotations: summary: "Leadership is churning" description: "{{ $value }} leadership changes in 10m. Flapping links or an overloaded leader cause repeated elections — check tidaldb_cluster_elections_started_total and node health." - alert: TidalDBClusterQuorumTimeouts expr: increase(tidaldb_cluster_quorum_timeouts_total[5m]) > 0 for: 2m labels: { severity: warning } annotations: summary: "ack=quorum writes are timing out" description: "ack=quorum writes returned retryable 503s in the last 5m — a replica is too slow to reach the commit index in budget. Identify the laggard via per-peer ship queue depth." - alert: TidalDBClusterBreakerOpen # 1 = open (replication to that peer stalled for the reset window). 2 = # half-open is transient and expected during recovery, so alert on == 1. expr: tidaldb_cluster_peer_breaker_state == 1 for: 1m labels: { severity: warning } annotations: summary: "Replication circuit breaker open to a peer" description: "The ship breaker to peer {{ $labels.peer_shard }} has been open for 1m — replication to it is stalled. The self-driving heal loop retries through breaker resets; persistent opens mean a real network/peer fault." - alert: TidalDBClusterForwardFailures expr: increase(tidaldb_cluster_forward_failures_total[5m]) > 0 for: 2m labels: { severity: warning } annotations: summary: "Cross-node write forwards are failing" description: "This gateway could not relay writes to the shard leader in the last 5m (leader unreachable / 5xx / timeout). An election may be in flight or the leader is partitioned." - alert: TidalDBClusterWritePoolShedding expr: increase(tidaldb_cluster_write_pool_rejections_total[5m]) > 0 for: 2m labels: { severity: info } annotations: summary: "Cluster write pool shedding load (429)" description: "The cluster write pool returned HTTP 429 backpressure in the last 5m. Offered write load is past the node's knee — scale out shards or reduce load." - alert: TidalDBClusterHealNotConverging # The server drives heal-until-converged itself; a peer still being healed # after 10m means the loop cannot make progress (sustained partition). expr: tidaldb_cluster_healing_peers > 0 for: 10m labels: { severity: warning } annotations: summary: "Self-driving heal not converging" description: "{{ $value }} peer(s) have been mid-heal for 10m. The heal loop retries through breaker resets automatically; a peer stuck this long is a real partition or a dead node — investigate the link, do NOT re-issue heal by hand." - alert: TidalDBClusterPeerShipStall # A peer stops accepting batches (its acked frontier flatlines) while # flushed events queue behind it: deriv() is the gauge-correct way to # detect a flat acked seqno (rate()/increase() are invalid on gauges). expr: deriv(tidaldb_cluster_peer_acked_seqno[2m]) == 0 and tidaldb_cluster_peer_ship_queue_depth > 0 for: 2m labels: { severity: critical } annotations: summary: "Replication to a peer has stalled with events queued" description: "Peer {{ $labels.peer_shard }} stopped accepting batches while events queue behind it (partition, dead peer, or paused sender). Check /cluster/status and the self-heal loop." - alert: TidalDBClusterQuorumLag expr: (tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq) > 10000 for: 2m labels: { severity: critical } annotations: summary: "Quorum commit index is far behind the leader frontier" description: "{{ $value }} events between the leader's flushed frontier and the quorum commit index (threshold 10000). A majority of the replica set is not confirming durability (down/partitioned followers, or follower apply throughput exhausted). ack=quorum writes will 503; the bodies name the laggards." - alert: TidalDBClusterDivergenceQuarantine # 1 = the node fenced itself from the data plane after detecting a # divergent suffix. Since m11p5 it auto-reseeds on next boot; a persistent # latch means the reseed is not completing. expr: tidaldb_cluster_divergence_quarantined == 1 for: 2m labels: { severity: critical } annotations: summary: "A node has quarantined itself for a divergent suffix" description: "A node fenced itself from the data plane after detecting a divergent suffix. Since m11p5 it auto-reseeds on its next boot; if the latch persists, the reseed is not completing — check tidaldb_cluster_reseed_required and the snapshot path." - alert: TidalDBClusterReseedPending # Durable reseed-marker latch. Expected briefly after a quarantine or a # behind-a-compacted-leader restart; a persistent latch means the snapshot # fetch is failing. expr: tidaldb_cluster_reseed_required == 1 for: 10m labels: { severity: warning } annotations: summary: "A node has a reseed pending but has not completed it" description: "A node has latched the reseed marker but has not completed a snapshot reseed in 10m. Expected briefly after a quarantine or a behind-a-compacted-leader restart; a persistent latch means the snapshot fetch is failing (no reachable leader, capability gate, or staging fault)." - alert: TidalDBClusterSnapshotPinForceDrop # A staged-snapshot retention pin was force-dropped past the hard cap — a # joiner started a reseed and never released (died mid-fetch). The dropped # pin protects compaction. expr: increase(tidaldb_cluster_snapshot_pin_force_drops_total[1h]) > 0 labels: { severity: warning } annotations: summary: "A staged-snapshot retention pin was force-dropped" description: "A staged-snapshot retention pin was force-dropped past the hard cap in the last hour — a joiner started a reseed and never released (died mid-fetch). The dropped pin protects compaction; the stranded joiner must be re-driven or removed."