3e7eddc074
This commit implements comprehensive production hardening across multiple layers to prepare StemeDB for enterprise pilot deployments: ## API Layer - Add rate limiting middleware with configurable limits per endpoint - Enhance error handling with detailed context and proper HTTP status codes - Add security hardening tests for input validation and boundary conditions - Create store_helpers module for defensive storage access patterns ## Storage & WAL - Optimize group commit batching for higher throughput - Add defensive error handling in hybrid backend with proper fallbacks - Enhance WAL journal durability guarantees with fsync validation - Improve index store query performance with better caching ## Operations & Deployment - Add comprehensive operations documentation (deployment, monitoring, DR) - Create systemd units for backup, WAL archival, and verification - Add monitoring configs (Prometheus alerts, metrics exporters) - Implement backup/restore scripts with verification and S3 archival - Add DR drill automation and runbook procedures - Create load balancer configs (nginx, envoy) with health checks ## Documentation - Update CLAUDE.md with operations and troubleshooting guides - Expand roadmap with production readiness milestones - Add pilot success criteria and deployment reference architecture - Document TLS setup, monitoring integration, and incident response ## Configuration - Add .env.example with all required environment variables - Document resource sizing for different deployment scales - Add configuration examples for various deployment topologies This positions StemeDB for successful enterprise pilots with proper operational discipline, monitoring, backup/DR, and security hardening. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
151 lines
3.9 KiB
JSON
151 lines
3.9 KiB
JSON
{
|
|
"dashboard": {
|
|
"title": "StemeDB - Cluster Overview",
|
|
"tags": ["stemedb", "cluster", "distributed"],
|
|
"timezone": "browser",
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Node Status",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"expr": "stemedb_cluster_nodes_alive",
|
|
"legendFormat": "Alive"
|
|
},
|
|
{
|
|
"expr": "stemedb_cluster_nodes_suspect",
|
|
"legendFormat": "Suspect"
|
|
},
|
|
{
|
|
"expr": "stemedb_cluster_nodes_dead",
|
|
"legendFormat": "Dead"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"value": 0, "color": "green"},
|
|
{"value": 1, "color": "red"}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 0}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Replication Lag (by peer)",
|
|
"type": "graph",
|
|
"targets": [
|
|
{
|
|
"expr": "stemedb_sync_lag_seconds",
|
|
"legendFormat": "{{peer_id}}"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "s", "label": "Lag"},
|
|
{"format": "short"}
|
|
],
|
|
"gridPos": {"h": 8, "w": 16, "x": 8, "y": 0},
|
|
"alert": {
|
|
"conditions": [
|
|
{
|
|
"evaluator": {"params": [300], "type": "gt"},
|
|
"operator": {"type": "and"},
|
|
"query": {"params": ["A", "5m", "now"]},
|
|
"reducer": {"type": "avg"}
|
|
}
|
|
],
|
|
"name": "High Replication Lag"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Sync Operations/sec",
|
|
"type": "graph",
|
|
"targets": [
|
|
{
|
|
"expr": "rate(stemedb_sync_operations_total[5m])",
|
|
"legendFormat": "{{operation}}"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "ops", "label": "Operations/sec"},
|
|
{"format": "short"}
|
|
],
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "Merkle Diff Size (by peer)",
|
|
"type": "graph",
|
|
"targets": [
|
|
{
|
|
"expr": "stemedb_merkle_diff_size",
|
|
"legendFormat": "{{peer_id}}"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "short", "label": "Diff Size"},
|
|
{"format": "short"}
|
|
],
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Cluster Convergence State",
|
|
"type": "gauge",
|
|
"targets": [
|
|
{
|
|
"expr": "stemedb_cluster_convergence_ratio",
|
|
"legendFormat": "Convergence %"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"thresholds": {
|
|
"mode": "percentage",
|
|
"steps": [
|
|
{"value": 0, "color": "red"},
|
|
{"value": 0.9, "color": "yellow"},
|
|
{"value": 0.99, "color": "green"}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 16}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "Gossip Message Rate",
|
|
"type": "graph",
|
|
"targets": [
|
|
{
|
|
"expr": "rate(stemedb_gossip_messages_sent_total[5m])",
|
|
"legendFormat": "Sent"
|
|
},
|
|
{
|
|
"expr": "rate(stemedb_gossip_messages_received_total[5m])",
|
|
"legendFormat": "Received"
|
|
}
|
|
],
|
|
"yaxes": [
|
|
{"format": "msgs", "label": "Messages/sec"},
|
|
{"format": "short"}
|
|
],
|
|
"gridPos": {"h": 8, "w": 16, "x": 8, "y": 16}
|
|
}
|
|
],
|
|
"refresh": "10s",
|
|
"schemaVersion": 30,
|
|
"version": 1
|
|
}
|
|
}
|