4051077cff
- API.md: document `similar_to`/`region`/`unavailable_shards` on /feed and /search, the new POST /vector_search k-NN probe, and the cluster-node-only routes (/cluster/*, /sharded/*, /hardnegs) - CHANGELOG.md: M12 entries — multi-vector preference + ANN candidate-gen, idle-readiness + TLS scale-up (m12p5/p6), sharded ingestion (m12p4) - ROADMAP.md: mark M11 + M12 COMPLETE; restate the v1.0 bar (30-day-green nightly calendar + Ref-A/k3s throughput re-runs) - prometheus-alerts.yaml: add ship-stall, quorum-lag, divergence-quarantine, reseed-pending, and snapshot-pin-force-drop cluster alerts - check-docs.sh: self-updating milestone-status freshness guard derived from ROADMAP's latest COMPLETE milestone - refresh specs (00-14), ai-lookup, guides, and runbooks to M0-M12
264 lines
14 KiB
YAML
264 lines
14 KiB
YAML
# =============================================================================
|
||
# DESIGN-REFERENCE RULE SET — NOT LOADED BY ANY ALERTMANAGER TODAY.
|
||
# =============================================================================
|
||
# This file is reference/design config. It is NOT wired into any running
|
||
# alerting pipeline:
|
||
# - vmalert (pilot/dev) loads ONLY ops/vmalert/rules/*.yaml
|
||
# - prod CRDs live ONLY in infra/k8s/prom/prometheus-rules/*.yaml
|
||
# Nothing mounts or imports docs/ops/prometheus-alerts.yaml.
|
||
# Do not read these as live pager rules.
|
||
#
|
||
# Provenance: every metric referenced below (tidaldb_health_ok,
|
||
# tidaldb_checkpoint_age_seconds, tidaldb_checkpoint_failures_total,
|
||
# tidaldb_wal_lag_bytes, tidaldb_signal_hot_entries, tidaldb_degradation_level,
|
||
# tidaldb_active_sessions, tidaldb_rate_limited_total,
|
||
# tidaldb_tantivy_segment_count, tidaldb_retrieve_latency_us_bucket,
|
||
# tidaldb_search_latency_us_bucket) IS really emitted today by
|
||
# tidal/src/db/metrics/mod.rs. So these rules are promotable — they
|
||
# are not orphaned against phantom metrics.
|
||
#
|
||
# Promotion path (the right long-term fix — do it deliberately, do NOT
|
||
# hand-copy these as live pager rules without the steps below):
|
||
# 1. Add a vmalert rule file: ops/vmalert/rules/tidaldb.yaml, translating
|
||
# each rule and stamping the canonical labels every routed rule carries —
|
||
# severity + capability + surface (+ optional component). Map per the
|
||
# canonical severity taxonomy in ops/alerting.md:
|
||
# labels {severity: critical} = pager class (TidalDBDown only here),
|
||
# {severity: warning} = non-paging / business-hours,
|
||
# {severity: info} = pure-FYI.
|
||
# Add capability: tidaldb and a surface label per rule, and a
|
||
# runbook_url annotation:
|
||
# https://github.com/orchard9/tidaldb/blob/main/docs/runbooks/<slug>.md
|
||
# 2. Add the prod CRD twin: infra/k8s/prom/prometheus-rules/tidaldb.yaml,
|
||
# kept byte-aligned in expr/threshold with the vmalert file.
|
||
# 3. Verify the alerts fire against real metrics (scrape tidaldb, force a
|
||
# degraded state) before declaring them on-call-ready.
|
||
# Until steps 1–3 land, this file stays a design reference only.
|
||
# =============================================================================
|
||
groups:
|
||
- name: tidaldb
|
||
interval: 30s
|
||
rules:
|
||
- alert: TidalDBDown
|
||
expr: tidaldb_health_ok == 0
|
||
for: 1m
|
||
labels: { severity: critical }
|
||
annotations:
|
||
summary: "tidalDB is unhealthy"
|
||
description: "tidaldb_health_ok is 0 — database is unhealthy or shut down."
|
||
|
||
- alert: TidalDBCheckpointStale
|
||
expr: tidaldb_checkpoint_age_seconds > 300
|
||
for: 2m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Signal checkpoint not running"
|
||
description: "{{ $value }}s since last checkpoint (threshold: 300s). Signal durability at risk."
|
||
|
||
- alert: TidalDBCheckpointFailures
|
||
expr: increase(tidaldb_checkpoint_failures_total[5m]) > 0
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Signal checkpoint failures detected"
|
||
description: "Checkpoint failures in last 5m. Check disk space and storage errors."
|
||
|
||
- alert: TidalDBWALDiskPressure
|
||
expr: tidaldb_wal_lag_bytes > 1000000000
|
||
for: 5m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "WAL disk usage exceeds 1GB"
|
||
description: "{{ $value | humanize1024 }}B of WAL uncompacted. Compaction may be stuck."
|
||
|
||
- alert: TidalDBSignalBacklog
|
||
expr: tidaldb_signal_hot_entries > 4000000
|
||
for: 5m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Signal ledger over 80% of capacity"
|
||
description: "{{ $value }} hot entries (threshold: 4M / 80% of 5M budget)."
|
||
|
||
- alert: TidalDBDegradedRanking
|
||
expr: tidaldb_degradation_level > 0
|
||
for: 2m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Ranking quality degraded"
|
||
description: "Degradation level {{ $value }} active. Scale up or reduce load."
|
||
|
||
- alert: TidalDBSessionLeak
|
||
# tidaldb_active_sessions is a GAUGE, so rate()/increase() are invalid
|
||
# (those operators only count counter resets). deriv() fits a least-squares
|
||
# slope over the window and is the gauge-correct way to detect sustained
|
||
# growth: > 0.03 sessions/sec is ~> 9 new sessions over a 5m window.
|
||
expr: deriv(tidaldb_active_sessions[5m]) > 0.03 and tidaldb_active_sessions > 100
|
||
for: 5m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Active session count growing steadily"
|
||
description: "{{ $value }} active sessions and trending up (deriv > 0.03/s). Agents may not be closing sessions."
|
||
|
||
- alert: TidalDBHighRateLimiting
|
||
expr: rate(tidaldb_rate_limited_total[5m]) > 100
|
||
for: 5m
|
||
labels: { severity: info }
|
||
annotations:
|
||
summary: "Sustained rate limiting"
|
||
description: "{{ $value }}/s rate-limited writes. Review agent rate limit config."
|
||
|
||
- alert: TidalDBTantivySegmentBloat
|
||
expr: tidaldb_tantivy_segment_count > 30
|
||
for: 10m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Tantivy segment count elevated"
|
||
description: "{{ $value }} segments (threshold: 30). Text syncer may be stalled."
|
||
|
||
- alert: TidalDBSlowRetrieve
|
||
expr: histogram_quantile(0.95, rate(tidaldb_retrieve_latency_us_bucket[5m])) > 500000
|
||
for: 5m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Retrieve p95 latency exceeds 500ms"
|
||
description: "p95 retrieve latency is {{ $value | humanizeDuration }}. Check signal ledger load and degradation level."
|
||
|
||
- alert: TidalDBSlowSearch
|
||
expr: histogram_quantile(0.95, rate(tidaldb_search_latency_us_bucket[5m])) > 1000000
|
||
for: 5m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Search p95 latency exceeds 1s"
|
||
description: "p95 search latency is {{ $value | humanizeDuration }}. Check Tantivy segment count and ANN index health."
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Cluster-mode rules (m11p8). Every metric below is emitted by
|
||
# tidal/src/db/metrics/cluster.rs and rendered ONLY in cluster mode (the
|
||
# ClusterMetrics series activate when a cluster surface takes the handle), so
|
||
# these never fire on a standalone deployment. Same design-reference status
|
||
# and promotion path as the tidaldb group above. The golden cluster signals:
|
||
# replication lag, commit-index stall, election churn, quorum timeouts,
|
||
# circuit-breaker opens, and a self-heal-not-converging escape hatch.
|
||
# ---------------------------------------------------------------------------
|
||
- name: tidaldb-cluster
|
||
interval: 30s
|
||
rules:
|
||
- alert: TidalDBClusterReplicationLagHigh
|
||
expr: (tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq) > 1000
|
||
for: 2m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Quorum replication lag elevated"
|
||
description: "{{ $value }} events between the leader's flushed frontier and the quorum commit index for 2m (SLA < 2s of writes). A follower is lagging — check per-peer ship queue depth and breaker state."
|
||
|
||
- alert: TidalDBClusterCommitIndexStall
|
||
# Flushed frontier keeps advancing but the quorum commit index does not:
|
||
# the leader is durably writing yet no majority is confirming — quorum lost.
|
||
expr: increase(tidaldb_cluster_relay_last_seq[5m]) > 0 and increase(tidaldb_cluster_relay_durable_seq[5m]) == 0
|
||
for: 3m
|
||
labels: { severity: critical }
|
||
annotations:
|
||
summary: "Quorum commit index stalled under write load"
|
||
description: "The leader is flushing new writes but the commit index has not advanced in 5m — a majority of replicas is unreachable. ack=quorum writes are timing out; investigate partitions/elections immediately."
|
||
|
||
- alert: TidalDBClusterElectionChurn
|
||
expr: increase(tidaldb_cluster_leader_changes_total[10m]) > 2
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Leadership is churning"
|
||
description: "{{ $value }} leadership changes in 10m. Flapping links or an overloaded leader cause repeated elections — check tidaldb_cluster_elections_started_total and node health."
|
||
|
||
- alert: TidalDBClusterQuorumTimeouts
|
||
expr: increase(tidaldb_cluster_quorum_timeouts_total[5m]) > 0
|
||
for: 2m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "ack=quorum writes are timing out"
|
||
description: "ack=quorum writes returned retryable 503s in the last 5m — a replica is too slow to reach the commit index in budget. Identify the laggard via per-peer ship queue depth."
|
||
|
||
- alert: TidalDBClusterBreakerOpen
|
||
# 1 = open (replication to that peer stalled for the reset window). 2 =
|
||
# half-open is transient and expected during recovery, so alert on == 1.
|
||
expr: tidaldb_cluster_peer_breaker_state == 1
|
||
for: 1m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Replication circuit breaker open to a peer"
|
||
description: "The ship breaker to peer {{ $labels.peer_shard }} has been open for 1m — replication to it is stalled. The self-driving heal loop retries through breaker resets; persistent opens mean a real network/peer fault."
|
||
|
||
- alert: TidalDBClusterForwardFailures
|
||
expr: increase(tidaldb_cluster_forward_failures_total[5m]) > 0
|
||
for: 2m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Cross-node write forwards are failing"
|
||
description: "This gateway could not relay writes to the shard leader in the last 5m (leader unreachable / 5xx / timeout). An election may be in flight or the leader is partitioned."
|
||
|
||
- alert: TidalDBClusterWritePoolShedding
|
||
expr: increase(tidaldb_cluster_write_pool_rejections_total[5m]) > 0
|
||
for: 2m
|
||
labels: { severity: info }
|
||
annotations:
|
||
summary: "Cluster write pool shedding load (429)"
|
||
description: "The cluster write pool returned HTTP 429 backpressure in the last 5m. Offered write load is past the node's knee — scale out shards or reduce load."
|
||
|
||
- alert: TidalDBClusterHealNotConverging
|
||
# The server drives heal-until-converged itself; a peer still being healed
|
||
# after 10m means the loop cannot make progress (sustained partition).
|
||
expr: tidaldb_cluster_healing_peers > 0
|
||
for: 10m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "Self-driving heal not converging"
|
||
description: "{{ $value }} peer(s) have been mid-heal for 10m. The heal loop retries through breaker resets automatically; a peer stuck this long is a real partition or a dead node — investigate the link, do NOT re-issue heal by hand."
|
||
|
||
- alert: TidalDBClusterPeerShipStall
|
||
# A peer stops accepting batches (its acked frontier flatlines) while
|
||
# flushed events queue behind it: deriv() is the gauge-correct way to
|
||
# detect a flat acked seqno (rate()/increase() are invalid on gauges).
|
||
expr: deriv(tidaldb_cluster_peer_acked_seqno[2m]) == 0 and tidaldb_cluster_peer_ship_queue_depth > 0
|
||
for: 2m
|
||
labels: { severity: critical }
|
||
annotations:
|
||
summary: "Replication to a peer has stalled with events queued"
|
||
description: "Peer {{ $labels.peer_shard }} stopped accepting batches while events queue behind it (partition, dead peer, or paused sender). Check /cluster/status and the self-heal loop."
|
||
|
||
- alert: TidalDBClusterQuorumLag
|
||
expr: (tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq) > 10000
|
||
for: 2m
|
||
labels: { severity: critical }
|
||
annotations:
|
||
summary: "Quorum commit index is far behind the leader frontier"
|
||
description: "{{ $value }} events between the leader's flushed frontier and the quorum commit index (threshold 10000). A majority of the replica set is not confirming durability (down/partitioned followers, or follower apply throughput exhausted). ack=quorum writes will 503; the bodies name the laggards."
|
||
|
||
- alert: TidalDBClusterDivergenceQuarantine
|
||
# 1 = the node fenced itself from the data plane after detecting a
|
||
# divergent suffix. Since m11p5 it auto-reseeds on next boot; a persistent
|
||
# latch means the reseed is not completing.
|
||
expr: tidaldb_cluster_divergence_quarantined == 1
|
||
for: 2m
|
||
labels: { severity: critical }
|
||
annotations:
|
||
summary: "A node has quarantined itself for a divergent suffix"
|
||
description: "A node fenced itself from the data plane after detecting a divergent suffix. Since m11p5 it auto-reseeds on its next boot; if the latch persists, the reseed is not completing — check tidaldb_cluster_reseed_required and the snapshot path."
|
||
|
||
- alert: TidalDBClusterReseedPending
|
||
# Durable reseed-marker latch. Expected briefly after a quarantine or a
|
||
# behind-a-compacted-leader restart; a persistent latch means the snapshot
|
||
# fetch is failing.
|
||
expr: tidaldb_cluster_reseed_required == 1
|
||
for: 10m
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "A node has a reseed pending but has not completed it"
|
||
description: "A node has latched the reseed marker but has not completed a snapshot reseed in 10m. Expected briefly after a quarantine or a behind-a-compacted-leader restart; a persistent latch means the snapshot fetch is failing (no reachable leader, capability gate, or staging fault)."
|
||
|
||
- alert: TidalDBClusterSnapshotPinForceDrop
|
||
# A staged-snapshot retention pin was force-dropped past the hard cap — a
|
||
# joiner started a reseed and never released (died mid-fetch). The dropped
|
||
# pin protects compaction.
|
||
expr: increase(tidaldb_cluster_snapshot_pin_force_drops_total[1h]) > 0
|
||
labels: { severity: warning }
|
||
annotations:
|
||
summary: "A staged-snapshot retention pin was force-dropped"
|
||
description: "A staged-snapshot retention pin was force-dropped past the hard cap in the last hour — a joiner started a reseed and never released (died mid-fetch). The dropped pin protects compaction; the stranded joiner must be re-driven or removed."
|