{
  "annotations": {
    "list": []
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "id": 1,
      "type": "row",
      "title": "HTTP surface \u2014 usage and errors",
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "panels": []
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "Request rate by route",
      "description": "Requests per second by matched route TEMPLATE. Before tidaldb_http_requests_total existed there was no metric anywhere that could answer 'how much traffic are we serving'.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 40,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "reqps",
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum by (route) (rate(tidaldb_http_requests_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))",
          "legendFormat": "{{route}}",
          "range": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "Requests by status",
      "description": "Exact status codes, not classes: 401 (no/bad credential) vs 403 (valid data bearer, not an operator) vs 429 (rate limited) vs 503 (quorum timeout) are different incidents.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 1
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 40,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "reqps",
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum by (status) (rate(tidaldb_http_requests_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))",
          "legendFormat": "{{status}}",
          "range": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 4,
      "type": "stat",
      "title": "5xx ratio",
      "description": "Server-fault share of served requests. clamp_min keeps an idle cluster from dividing by zero and painting red.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 6,
        "w": 4,
        "x": 0,
        "y": 9
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.01
              },
              {
                "color": "red",
                "value": 0.05
              }
            ]
          },
          "mappings": [],
          "unit": "percentunit",
          "decimals": 2,
          "noValue": "no requests"
        },
        "overrides": []
      },
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum(rate(tidaldb_http_requests_total{namespace=~\"$namespace\",pod=~\"$pod\",status=~\"5..\"}[$__rate_interval])) / clamp_min(sum(rate(tidaldb_http_requests_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval])), 0.001)",
          "legendFormat": "5xx share",
          "range": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "Auth rejections (401 / 403)",
      "description": "401 = missing/invalid credential. 403 = a valid DATA bearer used against an operator verb, i.e. a client trying to reach /cluster/*. A sustained 403 rate is a client misconfigured with the wrong key, or someone probing.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 6,
        "w": 10,
        "x": 4,
        "y": 9
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 8,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "none"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "reqps",
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum by (status, route) (rate(tidaldb_http_requests_total{namespace=~\"$namespace\",pod=~\"$pod\",status=~\"401|403\"}[$__rate_interval]))",
          "legendFormat": "{{status}} {{route}}",
          "range": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "HTTP p99 by route",
      "description": "End-to-end latency as the CLIENT sees it, including auth, queueing behind the concurrency cap, and the leader forward. The engine's own latency series exclude all of that.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 6,
        "w": 10,
        "x": 14,
        "y": 9
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 8,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "none"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ms",
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.99, sum by (route, le) (rate(tidaldb_http_request_duration_us_bucket{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))) / 1000",
          "legendFormat": "{{route}}",
          "range": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 7,
      "type": "row",
      "title": "Query engine latency",
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 15
      },
      "panels": []
    },
    {
      "id": 8,
      "type": "timeseries",
      "title": "Search latency",
      "description": "Search percentiles published directly by the engine.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 16
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 8,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "none"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ms",
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "tidaldb_search_latency_us_p50{namespace=~\"$namespace\",pod=~\"$pod\"} / 1000",
          "legendFormat": "{{pod}} p50",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "tidaldb_search_latency_us_p95{namespace=~\"$namespace\",pod=~\"$pod\"} / 1000",
          "legendFormat": "{{pod}} p95",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "tidaldb_search_latency_us_p99{namespace=~\"$namespace\",pod=~\"$pod\"} / 1000",
          "legendFormat": "{{pod}} p99",
          "range": true,
          "refId": "C"
        }
      ]
    },
    {
      "id": 9,
      "type": "timeseries",
      "title": "Retrieve latency",
      "description": "Retrieve percentiles published directly by the engine.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 16
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 8,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "none"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ms",
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "tidaldb_retrieve_latency_us_p50{namespace=~\"$namespace\",pod=~\"$pod\"} / 1000",
          "legendFormat": "{{pod}} p50",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "tidaldb_retrieve_latency_us_p95{namespace=~\"$namespace\",pod=~\"$pod\"} / 1000",
          "legendFormat": "{{pod}} p95",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "tidaldb_retrieve_latency_us_p99{namespace=~\"$namespace\",pod=~\"$pod\"} / 1000",
          "legendFormat": "{{pod}} p99",
          "range": true,
          "refId": "C"
        }
      ]
    },
    {
      "id": 10,
      "type": "timeseries",
      "title": "Signal write p99",
      "description": "Write path. WAL group-commit fsync is plotted alongside because it is usually the reason a write p99 moves.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 16
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 8,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "none"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "ms",
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.99, sum by (le) (rate(tidaldb_signal_write_latency_us_bucket{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))) / 1000",
          "legendFormat": "p99",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "histogram_quantile(0.99, sum by (le) (rate(tidaldb_cluster_wal_fsync_us_bucket{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))) / 1000",
          "legendFormat": "WAL fsync p99",
          "range": true,
          "refId": "B"
        }
      ]
    },
    {
      "id": 11,
      "type": "row",
      "title": "Cluster correctness",
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "panels": []
    },
    {
      "id": 14,
      "type": "stat",
      "title": "Cluster health",
      "description": "min(tidaldb_health_ok) across the selected pods. DOWN if any node is unhealthy.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 4,
        "w": 8,
        "x": 0,
        "y": 24
      },
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "textMode": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "mappings": [
            {
              "type": "value",
              "options": {
                "0": {
                  "text": "DOWN",
                  "color": "red",
                  "index": 0
                },
                "1": {
                  "text": "OK",
                  "color": "green",
                  "index": 1
                }
              }
            }
          ]
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "min(tidaldb_health_ok{namespace=~\"$namespace\",pod=~\"$pod\"})",
          "legendFormat": "Cluster health",
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          }
        }
      ]
    },
    {
      "id": 18,
      "type": "stat",
      "title": "Reseed pending",
      "description": "max(tidaldb_cluster_reseed_required). REQUIRED means a node is rebuilding or must.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 4,
        "w": 8,
        "x": 8,
        "y": 24
      },
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "textMode": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          },
          "mappings": [
            {
              "type": "value",
              "options": {
                "0": {
                  "text": "none",
                  "color": "green",
                  "index": 0
                },
                "1": {
                  "text": "REQUIRED",
                  "color": "red",
                  "index": 1
                }
              }
            }
          ]
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "max(tidaldb_cluster_reseed_required{namespace=~\"$namespace\",pod=~\"$pod\"})",
          "legendFormat": "Reseed pending",
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          }
        }
      ]
    },
    {
      "id": 19,
      "type": "stat",
      "title": "Indexed vectors",
      "description": "max(tidaldb_usearch_vector_count) \u2014 corpus size on the fullest node.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 4,
        "w": 8,
        "x": 16,
        "y": 24
      },
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "textMode": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              }
            ]
          },
          "mappings": [],
          "unit": "short"
        },
        "overrides": []
      },
      "targets": [
        {
          "refId": "A",
          "expr": "max(tidaldb_usearch_vector_count{namespace=~\"$namespace\",pod=~\"$pod\"})",
          "legendFormat": "Indexed vectors",
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          }
        }
      ]
    },
    {
      "id": 12,
      "type": "timeseries",
      "title": "Replication lag",
      "description": "Events this replica is behind the leader. Sustained non-zero means a follower is not keeping up; a step change usually follows a leader change.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 28
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 8,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "none"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "tidaldb_replication_lag_seqno{namespace=~\"$namespace\",pod=~\"$pod\"}",
          "legendFormat": "{{pod}}",
          "range": true,
          "refId": "A"
        }
      ]
    },
    {
      "id": 13,
      "type": "timeseries",
      "title": "Correctness counters",
      "description": "scatter degraded is the one to watch: a 200 answered with PARTIAL results because a shard errored or missed its deadline. It is a CORRECTNESS signal that looks like success to the client.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 28
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 8,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "none"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum(rate(tidaldb_cluster_quorum_timeouts_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))",
          "legendFormat": "quorum timeouts",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum(rate(tidaldb_cluster_forward_failures_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))",
          "legendFormat": "forward failures",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum(rate(tidaldb_cluster_scatter_degraded_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))",
          "legendFormat": "scatter degraded",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum(rate(tidaldb_cluster_leader_changes_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))",
          "legendFormat": "leader changes",
          "range": true,
          "refId": "D"
        }
      ]
    },
    {
      "id": 15,
      "type": "row",
      "title": "Capacity and backpressure",
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 35
      },
      "panels": []
    },
    {
      "id": 16,
      "type": "timeseries",
      "title": "Shed load",
      "description": "Load actively shed. NOTE: tidalDB's own limiter is UNLIMITED unless TIDAL_RATE_LIMIT_RPS is set, and it keys per PRINCIPAL \u2014 with one shared bearer every caller is one principal, so this is an aggregate cap, not fairness.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 36
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 8,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "none"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "reqps",
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum(rate(tidaldb_rate_limited_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))",
          "legendFormat": "rate limited",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum(rate(tidaldb_cluster_write_pool_rejections_total{namespace=~\"$namespace\",pod=~\"$pod\"}[$__rate_interval]))",
          "legendFormat": "write pool rejections",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "sum by (status) (rate(tidaldb_http_requests_total{namespace=~\"$namespace\",pod=~\"$pod\",status=~\"429|408\"}[$__rate_interval]))",
          "legendFormat": "http {{status}}",
          "range": true,
          "refId": "C"
        }
      ]
    },
    {
      "id": 17,
      "type": "timeseries",
      "title": "Durability backlog",
      "description": "Uncompacted WAL and time since the last successful checkpoint. A checkpoint age that only climbs means the checkpoint thread is wedged \u2014 recovery time grows with it.",
      "datasource": {
        "type": "prometheus",
        "uid": "victoriametrics"
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 36
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 8,
            "lineWidth": 2,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "none"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "min": 0
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "tidaldb_wal_lag_bytes{namespace=~\"$namespace\",pod=~\"$pod\"}",
          "legendFormat": "WAL lag bytes {{pod}}",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "victoriametrics"
          },
          "editorMode": "code",
          "expr": "tidaldb_checkpoint_age_seconds{namespace=~\"$namespace\",pod=~\"$pod\"}",
          "legendFormat": "checkpoint age {{pod}}",
          "range": true,
          "refId": "B"
        }
      ]
    }
  ],
  "refresh": "30s",
  "schemaVersion": 39,
  "tags": [
    "tidaldb",
    "database"
  ],
  "templating": {
    "list": [
      {
        "current": {
          "selected": true,
          "text": [
            "tidaldb-cluster"
          ],
          "value": [
            "tidaldb-cluster"
          ]
        },
        "datasource": {
          "type": "prometheus",
          "uid": "victoriametrics"
        },
        "definition": "label_values(tidaldb_health_ok, namespace)",
        "hide": 0,
        "includeAll": true,
        "label": "namespace",
        "multi": true,
        "name": "namespace",
        "options": [],
        "query": {
          "query": "label_values(tidaldb_health_ok, namespace)",
          "refId": "A"
        },
        "refresh": 2,
        "sort": 1,
        "type": "query"
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "victoriametrics"
        },
        "definition": "label_values(tidaldb_health_ok{namespace=~\"$namespace\"}, pod)",
        "hide": 0,
        "includeAll": true,
        "label": "pod",
        "multi": true,
        "name": "pod",
        "options": [],
        "query": {
          "query": "label_values(tidaldb_health_ok{namespace=~\"$namespace\"}, pod)",
          "refId": "A"
        },
        "refresh": 2,
        "sort": 1,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "utc",
  "title": "tidalDB \u2014 usage, errors, and cluster health",
  "uid": "tidaldb-overview",
  "version": 1,
  "weekStart": "",
  "description": "tidalDB usage, errors, and cluster health. PRODUCT-SIDE SOURCE OF TRUTH: this file lives in the tidalDB repo at docs/ops/grafana-tidaldb.json and is mirrored into the fleet as the tidaldb-overview.json key of the grafana-database-dashboards ConfigMap (deployments/k8s/base/observability/database-dashboards.yaml), the same convention docs/ops/prometheus-alerts.yaml follows. Edit here, then mirror. Every expression was validated against the live TSDB; the label set is namespace/pod/container/partition_id as attached by vmagent's annotation-based pod discovery."
}