d5d1e7d81a
m11p8 closes G-O + §1.4-3: - Cluster metrics: breaker state, forwards, self-heal on /metrics; multi-shard sibling render (shard="N") - Grafana cluster row + 8-rule Prometheus alert group - Request-id / TraceLayer on both cluster routers; id rides forward hop - Truthful status: flushed leader applied_events frontier; post-promote ShardId(0) keying fix - Self-driving heal: tick_self_heal re-arms stuck-peer backlog every ~3s - WAL PITR: wal.archive_dir, archive-before-delete gap-free - tidalctl backup/restore with BLAKE3 content-hash verification - Rolling-upgrade build_version handshake (N/N+1, never rejects) + Woodpecker release gate perf-sweep wave 2 T2: one-get-per-type pre-pass in ranking executor - signal_values.rs pre-fetches all signal kinds before scoring loop - Eliminates per-item repeated DashMap lookups: −18.8% for_you, −31% under writes - Byte-identical output verified with A/B test harness
672 lines
21 KiB
JSON
672 lines
21 KiB
JSON
{
|
|
"uid": "tidaldb-overview",
|
|
"title": "tidalDB Overview",
|
|
"description": "Operational dashboard covering all 20 tidalDB metrics including retrieve and search latency histograms.",
|
|
"schemaVersion": 38,
|
|
"version": 2,
|
|
"refresh": "30s",
|
|
"time": { "from": "now-1h", "to": "now" },
|
|
"timepicker": {},
|
|
"tags": ["tidaldb"],
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "Health Overview",
|
|
"gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 },
|
|
"collapsed": false
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Health",
|
|
"gridPos": { "x": 0, "y": 1, "w": 4, "h": 4 },
|
|
"targets": [
|
|
{
|
|
"expr": "tidaldb_health_ok",
|
|
"legendFormat": "health_ok"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"mappings": [
|
|
{ "type": "value", "options": { "0": { "text": "UNHEALTHY", "color": "red" } } },
|
|
{ "type": "value", "options": { "1": { "text": "OK", "color": "green" } } }
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "red", "value": 0 },
|
|
{ "color": "green", "value": 1 }
|
|
]
|
|
},
|
|
"color": { "mode": "thresholds" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "colorMode": "background" }
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Uptime",
|
|
"gridPos": { "x": 4, "y": 1, "w": 4, "h": 4 },
|
|
"targets": [
|
|
{
|
|
"expr": "tidaldb_uptime_seconds",
|
|
"legendFormat": "uptime_seconds"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "colorMode": "value" }
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Degradation Level",
|
|
"gridPos": { "x": 8, "y": 1, "w": 4, "h": 4 },
|
|
"targets": [
|
|
{
|
|
"expr": "tidaldb_degradation_level",
|
|
"legendFormat": "degradation_level"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": 0 },
|
|
{ "color": "red", "value": 1 }
|
|
]
|
|
},
|
|
"color": { "mode": "thresholds" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "colorMode": "background" }
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "stat",
|
|
"title": "Version",
|
|
"gridPos": { "x": 12, "y": 1, "w": 4, "h": 4 },
|
|
"targets": [
|
|
{
|
|
"expr": "tidaldb_info",
|
|
"legendFormat": "{{version}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "fixed", "fixedColor": "text" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "colorMode": "none", "textMode": "name" }
|
|
},
|
|
|
|
{
|
|
"id": 10,
|
|
"type": "row",
|
|
"title": "Signal Throughput",
|
|
"gridPos": { "x": 0, "y": 5, "w": 24, "h": 1 },
|
|
"collapsed": false
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "timeseries",
|
|
"title": "Signal Write Rate (per second)",
|
|
"gridPos": { "x": 0, "y": 6, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{
|
|
"expr": "rate(tidaldb_signal_writes_total[5m])",
|
|
"legendFormat": "writes/s"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": { "mode": "palette-classic" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "timeseries",
|
|
"title": "Signal Write Latency (µs)",
|
|
"gridPos": { "x": 8, "y": 6, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{
|
|
"expr": "tidaldb_signal_write_latency_us",
|
|
"legendFormat": "latency_us"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "µs",
|
|
"color": { "mode": "palette-classic" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"type": "gauge",
|
|
"title": "Signal Hot Entries",
|
|
"gridPos": { "x": 16, "y": 6, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{
|
|
"expr": "tidaldb_signal_hot_entries",
|
|
"legendFormat": "hot_entries"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"min": 0,
|
|
"max": 5000000,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": 0 },
|
|
{ "color": "yellow", "value": 4000000 },
|
|
{ "color": "red", "value": 5000000 }
|
|
]
|
|
},
|
|
"color": { "mode": "thresholds" }
|
|
}
|
|
}
|
|
},
|
|
|
|
{
|
|
"id": 14,
|
|
"type": "timeseries",
|
|
"title": "Retrieve Latency Percentiles (µs)",
|
|
"gridPos": { "x": 0, "y": 12, "w": 12, "h": 6 },
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.50, rate(tidaldb_retrieve_latency_us_bucket[$__rate_interval]))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.95, rate(tidaldb_retrieve_latency_us_bucket[$__rate_interval]))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, rate(tidaldb_retrieve_latency_us_bucket[$__rate_interval]))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "µs",
|
|
"color": { "mode": "palette-classic" },
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": 0 },
|
|
{ "color": "yellow", "value": 500000 }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 15,
|
|
"type": "timeseries",
|
|
"title": "Search Latency Percentiles (µs)",
|
|
"gridPos": { "x": 12, "y": 12, "w": 12, "h": 6 },
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.50, rate(tidaldb_search_latency_us_bucket[$__rate_interval]))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.95, rate(tidaldb_search_latency_us_bucket[$__rate_interval]))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, rate(tidaldb_search_latency_us_bucket[$__rate_interval]))",
|
|
"legendFormat": "p99"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "µs",
|
|
"color": { "mode": "palette-classic" },
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": 0 },
|
|
{ "color": "yellow", "value": 1000000 }
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
|
|
{
|
|
"id": 20,
|
|
"type": "row",
|
|
"title": "Durability",
|
|
"gridPos": { "x": 0, "y": 19, "w": 24, "h": 1 },
|
|
"collapsed": false
|
|
},
|
|
{
|
|
"id": 21,
|
|
"type": "timeseries",
|
|
"title": "Checkpoint Age (seconds)",
|
|
"gridPos": { "x": 0, "y": 20, "w": 6, "h": 6 },
|
|
"targets": [
|
|
{
|
|
"expr": "tidaldb_checkpoint_age_seconds",
|
|
"legendFormat": "checkpoint_age_seconds"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": 0 },
|
|
{ "color": "red", "value": 300 }
|
|
]
|
|
},
|
|
"color": { "mode": "thresholds" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 22,
|
|
"type": "stat",
|
|
"title": "Checkpoint Failures",
|
|
"gridPos": { "x": 6, "y": 20, "w": 6, "h": 6 },
|
|
"targets": [
|
|
{
|
|
"expr": "tidaldb_checkpoint_failures_total",
|
|
"legendFormat": "checkpoint_failures_total"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": 0 },
|
|
{ "color": "red", "value": 1 }
|
|
]
|
|
},
|
|
"color": { "mode": "thresholds" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" }
|
|
},
|
|
{
|
|
"id": 23,
|
|
"type": "timeseries",
|
|
"title": "WAL Lag (bytes)",
|
|
"gridPos": { "x": 12, "y": 20, "w": 6, "h": 6 },
|
|
"targets": [
|
|
{
|
|
"expr": "tidaldb_wal_lag_bytes",
|
|
"legendFormat": "wal_lag_bytes"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": 0 },
|
|
{ "color": "yellow", "value": 500000000 },
|
|
{ "color": "red", "value": 1000000000 }
|
|
]
|
|
},
|
|
"color": { "mode": "palette-classic" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 24,
|
|
"type": "timeseries",
|
|
"title": "WAL Compacted Segments (rate)",
|
|
"gridPos": { "x": 18, "y": 20, "w": 6, "h": 6 },
|
|
"targets": [
|
|
{
|
|
"expr": "rate(tidaldb_wal_compacted_segments_total[5m])",
|
|
"legendFormat": "compacted/s"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "cps",
|
|
"color": { "mode": "palette-classic" }
|
|
}
|
|
}
|
|
},
|
|
|
|
{
|
|
"id": 30,
|
|
"type": "row",
|
|
"title": "Index Health",
|
|
"gridPos": { "x": 0, "y": 26, "w": 24, "h": 1 },
|
|
"collapsed": false
|
|
},
|
|
{
|
|
"id": 31,
|
|
"type": "stat",
|
|
"title": "Tantivy Indexed Docs",
|
|
"gridPos": { "x": 0, "y": 27, "w": 4, "h": 4 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_tantivy_indexed_docs", "legendFormat": "indexed_docs" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value" }
|
|
},
|
|
{
|
|
"id": 32,
|
|
"type": "gauge",
|
|
"title": "Tantivy Segment Count",
|
|
"gridPos": { "x": 4, "y": 27, "w": 4, "h": 4 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_tantivy_segment_count", "legendFormat": "segment_count" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"min": 0,
|
|
"max": 50,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": 0 },
|
|
{ "color": "yellow", "value": 20 },
|
|
{ "color": "red", "value": 30 }
|
|
]
|
|
},
|
|
"color": { "mode": "thresholds" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 33,
|
|
"type": "stat",
|
|
"title": "uSearch Vector Count",
|
|
"gridPos": { "x": 8, "y": 27, "w": 4, "h": 4 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_usearch_vector_count", "legendFormat": "vector_count" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value" }
|
|
},
|
|
{
|
|
"id": 34,
|
|
"type": "stat",
|
|
"title": "uSearch Index Size",
|
|
"gridPos": { "x": 12, "y": 27, "w": 4, "h": 4 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_usearch_index_size_bytes", "legendFormat": "index_size_bytes" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes",
|
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value" }
|
|
},
|
|
{
|
|
"id": 35,
|
|
"type": "stat",
|
|
"title": "Bitmap Index Cardinality",
|
|
"gridPos": { "x": 16, "y": 27, "w": 4, "h": 4 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_bitmap_index_cardinality", "legendFormat": "bitmap_cardinality" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": { "mode": "fixed", "fixedColor": "blue" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value" }
|
|
},
|
|
|
|
{
|
|
"id": 40,
|
|
"type": "row",
|
|
"title": "Sessions",
|
|
"gridPos": { "x": 0, "y": 31, "w": 24, "h": 1 },
|
|
"collapsed": false
|
|
},
|
|
{
|
|
"id": 41,
|
|
"type": "timeseries",
|
|
"title": "Active Sessions",
|
|
"gridPos": { "x": 0, "y": 32, "w": 6, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_active_sessions", "legendFormat": "active_sessions" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": { "mode": "palette-classic" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 42,
|
|
"type": "timeseries",
|
|
"title": "Session Close Rate (per second)",
|
|
"gridPos": { "x": 6, "y": 32, "w": 6, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "rate(tidaldb_closed_sessions_total[5m])", "legendFormat": "closes/s" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"color": { "mode": "palette-classic" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 43,
|
|
"type": "stat",
|
|
"title": "Auto-Closed Sessions",
|
|
"gridPos": { "x": 12, "y": 32, "w": 4, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_session_auto_closed_total", "legendFormat": "auto_closed_total" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"color": { "mode": "fixed", "fixedColor": "yellow" }
|
|
}
|
|
},
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value" }
|
|
},
|
|
{
|
|
"id": 44,
|
|
"type": "timeseries",
|
|
"title": "Rate Limited (per second)",
|
|
"gridPos": { "x": 16, "y": 32, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "rate(tidaldb_rate_limited_total[5m])", "legendFormat": "rate_limited/s" }
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": 0 },
|
|
{ "color": "red", "value": 100 }
|
|
]
|
|
},
|
|
"color": { "mode": "palette-classic" }
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 49,
|
|
"type": "row",
|
|
"title": "Cluster Replication (m11p8)",
|
|
"description": "Rendered only in cluster mode (tidaldb_cluster_* series). One scrape target per node; co-located shard groups carry a shard=\"N\" label.",
|
|
"gridPos": { "x": 0, "y": 39, "w": 24, "h": 1 },
|
|
"collapsed": false
|
|
},
|
|
{
|
|
"id": 50,
|
|
"type": "timeseries",
|
|
"title": "Quorum Replication Lag (events)",
|
|
"description": "flushed frontier minus quorum commit index — how far the durable majority trails the leader's log. SLA < 2s of writes.",
|
|
"gridPos": { "x": 0, "y": 40, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq", "legendFormat": "quorum lag (events)" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 51,
|
|
"type": "timeseries",
|
|
"title": "Commit Progress (flushed vs committed)",
|
|
"description": "Leader flushed high-water mark and the quorum commit index. A flat commit index while flushed climbs = quorum lost.",
|
|
"gridPos": { "x": 8, "y": 40, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_cluster_relay_last_seq", "legendFormat": "flushed seq" },
|
|
{ "expr": "tidaldb_cluster_relay_durable_seq", "legendFormat": "commit index" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 52,
|
|
"type": "timeseries",
|
|
"title": "Per-Peer Ship Queue Depth",
|
|
"description": "Unacked seqnos outstanding to each follower. A single peer climbing = that link is the laggard.",
|
|
"gridPos": { "x": 16, "y": 40, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_cluster_peer_ship_queue_depth", "legendFormat": "peer {{peer_shard}}" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 53,
|
|
"type": "timeseries",
|
|
"title": "Ship RTT p99 (µs)",
|
|
"description": "Replication batch ship round-trip time, 99th percentile across peers.",
|
|
"gridPos": { "x": 0, "y": 46, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "histogram_quantile(0.99, sum by (le) (rate(tidaldb_cluster_ship_rtt_us_bucket[5m])))", "legendFormat": "ship rtt p99" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "µs", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 54,
|
|
"type": "timeseries",
|
|
"title": "WAL Group-Commit fsync p99 (µs)",
|
|
"description": "Leader and follower WAL group-commit fsync latency, 99th percentile. The platform durability floor.",
|
|
"gridPos": { "x": 8, "y": 46, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "histogram_quantile(0.99, sum by (le) (rate(tidaldb_cluster_wal_fsync_us_bucket[5m])))", "legendFormat": "fsync p99" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "µs", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 55,
|
|
"type": "timeseries",
|
|
"title": "Group-Commit Batch Fill (median events)",
|
|
"description": "Median events amortized per fsync. Low under sustained write load points at a batch-timeout that is too short.",
|
|
"gridPos": { "x": 16, "y": 46, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "histogram_quantile(0.5, sum by (le) (rate(tidaldb_cluster_group_commit_events_bucket[5m])))", "legendFormat": "events/fsync (p50)" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 56,
|
|
"type": "timeseries",
|
|
"title": "Election Term & Leader Changes",
|
|
"description": "Current term (gauge) and the rate of observed leadership changes. Steady term + zero changes = stable cluster.",
|
|
"gridPos": { "x": 0, "y": 52, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_cluster_election_term", "legendFormat": "term" },
|
|
{ "expr": "rate(tidaldb_cluster_leader_changes_total[5m])", "legendFormat": "leader changes/s" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 57,
|
|
"type": "timeseries",
|
|
"title": "Elections Started & Quorum Timeouts (per second)",
|
|
"description": "Election attempts and ack=quorum write timeouts. Sustained nonzero = unstable leadership or a slow replica.",
|
|
"gridPos": { "x": 8, "y": 52, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "rate(tidaldb_cluster_elections_started_total[5m])", "legendFormat": "elections/s" },
|
|
{ "expr": "rate(tidaldb_cluster_quorum_timeouts_total[5m])", "legendFormat": "quorum timeouts/s" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 58,
|
|
"type": "timeseries",
|
|
"title": "Write Pool: Depth & Shedding",
|
|
"description": "Queued cluster write jobs and the rate of HTTP 429 backpressure rejections. Rising 429s = offered load past the knee.",
|
|
"gridPos": { "x": 16, "y": 52, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_cluster_write_pool_depth", "legendFormat": "queue depth" },
|
|
{ "expr": "rate(tidaldb_cluster_write_pool_rejections_total[5m])", "legendFormat": "429/s" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 59,
|
|
"type": "timeseries",
|
|
"title": "Per-Peer Circuit-Breaker State",
|
|
"description": "0 closed, 1 open, 2 half-open. An open breaker (1) means replication to that peer is stalled for the reset window.",
|
|
"gridPos": { "x": 0, "y": 58, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_cluster_peer_breaker_state", "legendFormat": "peer {{peer_shard}}" },
|
|
{ "expr": "rate(tidaldb_cluster_breaker_opens_total[5m])", "legendFormat": "opens/s" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 60,
|
|
"type": "timeseries",
|
|
"title": "Cross-Node Write Forwards (per second)",
|
|
"description": "Writes this gateway relayed to a shard leader, and forwards that failed (leader unreachable). High forward rate at one node = client routing imbalance.",
|
|
"gridPos": { "x": 8, "y": 58, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "rate(tidaldb_cluster_forwards_total[5m])", "legendFormat": "forwards/s" },
|
|
{ "expr": "rate(tidaldb_cluster_forward_failures_total[5m])", "legendFormat": "failures/s" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" } } }
|
|
},
|
|
{
|
|
"id": 61,
|
|
"type": "timeseries",
|
|
"title": "Self-Driving Heal",
|
|
"description": "Peers being driven back to convergence (gauge) and heal reconcile attempt/success rates. healing_peers > 0 for minutes = investigate.",
|
|
"gridPos": { "x": 16, "y": 58, "w": 8, "h": 6 },
|
|
"targets": [
|
|
{ "expr": "tidaldb_cluster_healing_peers", "legendFormat": "healing peers" },
|
|
{ "expr": "rate(tidaldb_cluster_heal_attempts_total[5m])", "legendFormat": "attempts/s" },
|
|
{ "expr": "rate(tidaldb_cluster_heal_successes_total[5m])", "legendFormat": "successes/s" }
|
|
],
|
|
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } }
|
|
}
|
|
]
|
|
}
|