diff --git a/.woodpecker.yaml b/.woodpecker.yaml index c722407..e72790f 100644 --- a/.woodpecker.yaml +++ b/.woodpecker.yaml @@ -1,8 +1,9 @@ -# Build-only pipeline: Kaniko builds the tidal-server image and pushes it to the -# in-cluster zot registry. DEPLOYMENT IS MANUAL (kustomize, from the orchard9-k3sf -# ops repo) — the old auto-`kubectl set image deployment/tidaldb` step was removed -# because it coupled every image build to a standalone roll and the cluster ops -# repo's contract is "manual deploy via scripts, no CI/CD deploy". The same +# Pipeline: a rolling-upgrade RELEASE GATE (m11p8) runs FIRST; only if it passes +# does Kaniko build the tidal-server image and push it to the in-cluster zot +# registry. DEPLOYMENT IS MANUAL (kustomize, from the orchard9-k3sf ops repo) — +# the old auto-`kubectl set image deployment/tidaldb` step was removed because it +# coupled every image build to a standalone roll and the cluster ops repo's +# contract is "manual deploy via scripts, no CI/CD deploy". The same # `tidal-server` binary serves every subcommand (standalone AND multi-process # `cluster --region`), so one image covers both deployments. when: @@ -10,6 +11,19 @@ when: event: push steps: + # m11p8 release gate: prove a rolling upgrade under load loses no acknowledged + # write and never stalls (mp_rolling_upgrade_no_loss_no_stall — a tier-3 test + # spawning three real OS processes with a graceful SIGTERM → version-tagged + # restart → heal-until-converged → fixpoint cycle). Serial (--test-threads 1): + # the harness binds fixed ports and spawns real processes, so suites must not + # overlap. A failure here BLOCKS the image build below — the gate, not the start. + rolling-upgrade-gate: + image: rust:1-bookworm + commands: + - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang + - cargo test -p tidal-server --features cluster-e2e --test cluster_lifecycle + mp_rolling_upgrade_no_loss_no_stall -- --nocapture --test-threads 1 + build: image: woodpeckerci/plugin-kaniko settings: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec4402..5b0083f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,49 @@ All notable changes to tidalDB will be documented in this file. ### Added +**Observability + operations (m11p8) — complete cluster metric set on a per-node `/metrics` listener, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR + backup/restore, rolling-upgrade release gate** + +- **Metrics.** Completed the `tidaldb_cluster_*` set with the two members the + roadmap named that were missing — **breaker** (`tidaldb_cluster_peer_breaker_state` + 0/1/2 + `tidaldb_cluster_breaker_opens_total`, surfaced read-only from the + `tidal-net` circuit breaker via a new `Transport::peer_breaker_state`) and + **forwards** (`tidaldb_cluster_forwards_total` / `_forward_failures_total`, + instrumented at the gateway forward path) — plus the self-heal series + (`heal_attempts/successes/noops_total`, `healing_peers`). When several shard + groups co-locate on one node (m11p6) the metrics-owner serves the single + `/metrics` listener and the siblings register their series under a `shard="N"` + label (`TidalDb::register_metrics_sibling`); a single-shard node is + byte-identical to before. A 12-panel "Cluster Replication" Grafana row + + an 8-rule `tidaldb-cluster` Prometheus alert group ship beside the standalone + ones. +- **Request-id + tracing.** Both cluster routers (single- and multi-process) now + carry the standalone router's `SetRequestId` + `PropagateRequestId` + + `TraceLayer` stack (extracted to `router::with_request_id_tracing`); the id + rides the follower→leader forward hop verbatim, so the leader's span shares the + gateway's `x-request-id`. (Ships are off-request-path and batched, so they + correlate by seqno, not a request-id — by design.) +- **Truthful status.** Fixed the leader's own `applied_events` reading 0 (a leader + writes its WAL directly and never advances its own applied frontier — now + reports its flushed frontier) and the single-process post-promote `ShardId(0)` + lag-keying undercount (now keys on the current leader's shard). +- **Self-driving heal.** A standing leader duty re-arms the backlog re-ship for a + stuck (breaker-open, behind) peer every ~3s, so it converges through breaker + resets with no operator `/cluster/heal` loop — closing the §1.4-3 footgun. + Observable via `healing_peers` + the `TidalDBClusterHealNotConverging` alert. +- **WAL PITR archival.** `wal.archive_dir` (topology + builder): the online + compaction copies each sealed segment to the archive — durably, before deletion, + refusing to delete if archival fails — so the archive is a gap-free PITR record. +- **Backup/restore.** `tidalctl backup` / `restore`: offline data-dir backup with + a BLAKE3 `BACKUP_MANIFEST.json` (+ the WAL checkpoint cursor); restore verifies + every file's hash before writing and refuses a non-empty target. Coordinated + cluster backup = back up one committed replica per shard group (the runbook + drill). +- **Rolling upgrade.** A wire version handshake (`HeartbeatRequest.build_version`, + stamped at the transport boundary; `>= 2`-major skew WARNs, never rejects) + + `version` on `/cluster/status`. `mp_rolling_upgrade_no_loss_no_stall` promoted + to the FIRST step of `.woodpecker.yaml` (the release gate; a failure blocks the + image build). See [milestone-11/phase-8.md](docs/planning/milestone-11/phase-8.md). + **Security hardening (m11p7) — mTLS by default + zero-drop cert rotation, per-node identity, admin audit log, per-principal rate limit** - **The cluster stops trusting the network.** gRPC replication mTLS is now the intended posture: the inbound server is served over a custom `tokio-rustls` diff --git a/Cargo.lock b/Cargo.lock index 9b380c1..848417f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3634,6 +3634,7 @@ dependencies = [ name = "tidalctl" version = "0.1.0" dependencies = [ + "blake3", "serde", "serde_json", "tidaldb", diff --git a/docs/ops/grafana-dashboard.json b/docs/ops/grafana-dashboard.json index b8d4ff1..0c6279d 100644 --- a/docs/ops/grafana-dashboard.json +++ b/docs/ops/grafana-dashboard.json @@ -518,6 +518,154 @@ "color": { "mode": "palette-classic" } } } + }, + { + "id": 49, + "type": "row", + "title": "Cluster Replication (m11p8)", + "description": "Rendered only in cluster mode (tidaldb_cluster_* series). One scrape target per node; co-located shard groups carry a shard=\"N\" label.", + "gridPos": { "x": 0, "y": 39, "w": 24, "h": 1 }, + "collapsed": false + }, + { + "id": 50, + "type": "timeseries", + "title": "Quorum Replication Lag (events)", + "description": "flushed frontier minus quorum commit index — how far the durable majority trails the leader's log. SLA < 2s of writes.", + "gridPos": { "x": 0, "y": 40, "w": 8, "h": 6 }, + "targets": [ + { "expr": "tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq", "legendFormat": "quorum lag (events)" } + ], + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } } + }, + { + "id": 51, + "type": "timeseries", + "title": "Commit Progress (flushed vs committed)", + "description": "Leader flushed high-water mark and the quorum commit index. A flat commit index while flushed climbs = quorum lost.", + "gridPos": { "x": 8, "y": 40, "w": 8, "h": 6 }, + "targets": [ + { "expr": "tidaldb_cluster_relay_last_seq", "legendFormat": "flushed seq" }, + { "expr": "tidaldb_cluster_relay_durable_seq", "legendFormat": "commit index" } + ], + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } } + }, + { + "id": 52, + "type": "timeseries", + "title": "Per-Peer Ship Queue Depth", + "description": "Unacked seqnos outstanding to each follower. A single peer climbing = that link is the laggard.", + "gridPos": { "x": 16, "y": 40, "w": 8, "h": 6 }, + "targets": [ + { "expr": "tidaldb_cluster_peer_ship_queue_depth", "legendFormat": "peer {{peer_shard}}" } + ], + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } } + }, + { + "id": 53, + "type": "timeseries", + "title": "Ship RTT p99 (µs)", + "description": "Replication batch ship round-trip time, 99th percentile across peers.", + "gridPos": { "x": 0, "y": 46, "w": 8, "h": 6 }, + "targets": [ + { "expr": "histogram_quantile(0.99, sum by (le) (rate(tidaldb_cluster_ship_rtt_us_bucket[5m])))", "legendFormat": "ship rtt p99" } + ], + "fieldConfig": { "defaults": { "unit": "µs", "color": { "mode": "palette-classic" } } } + }, + { + "id": 54, + "type": "timeseries", + "title": "WAL Group-Commit fsync p99 (µs)", + "description": "Leader and follower WAL group-commit fsync latency, 99th percentile. The platform durability floor.", + "gridPos": { "x": 8, "y": 46, "w": 8, "h": 6 }, + "targets": [ + { "expr": "histogram_quantile(0.99, sum by (le) (rate(tidaldb_cluster_wal_fsync_us_bucket[5m])))", "legendFormat": "fsync p99" } + ], + "fieldConfig": { "defaults": { "unit": "µs", "color": { "mode": "palette-classic" } } } + }, + { + "id": 55, + "type": "timeseries", + "title": "Group-Commit Batch Fill (median events)", + "description": "Median events amortized per fsync. Low under sustained write load points at a batch-timeout that is too short.", + "gridPos": { "x": 16, "y": 46, "w": 8, "h": 6 }, + "targets": [ + { "expr": "histogram_quantile(0.5, sum by (le) (rate(tidaldb_cluster_group_commit_events_bucket[5m])))", "legendFormat": "events/fsync (p50)" } + ], + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } } + }, + { + "id": 56, + "type": "timeseries", + "title": "Election Term & Leader Changes", + "description": "Current term (gauge) and the rate of observed leadership changes. Steady term + zero changes = stable cluster.", + "gridPos": { "x": 0, "y": 52, "w": 8, "h": 6 }, + "targets": [ + { "expr": "tidaldb_cluster_election_term", "legendFormat": "term" }, + { "expr": "rate(tidaldb_cluster_leader_changes_total[5m])", "legendFormat": "leader changes/s" } + ], + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } } + }, + { + "id": 57, + "type": "timeseries", + "title": "Elections Started & Quorum Timeouts (per second)", + "description": "Election attempts and ack=quorum write timeouts. Sustained nonzero = unstable leadership or a slow replica.", + "gridPos": { "x": 8, "y": 52, "w": 8, "h": 6 }, + "targets": [ + { "expr": "rate(tidaldb_cluster_elections_started_total[5m])", "legendFormat": "elections/s" }, + { "expr": "rate(tidaldb_cluster_quorum_timeouts_total[5m])", "legendFormat": "quorum timeouts/s" } + ], + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } } + }, + { + "id": 58, + "type": "timeseries", + "title": "Write Pool: Depth & Shedding", + "description": "Queued cluster write jobs and the rate of HTTP 429 backpressure rejections. Rising 429s = offered load past the knee.", + "gridPos": { "x": 16, "y": 52, "w": 8, "h": 6 }, + "targets": [ + { "expr": "tidaldb_cluster_write_pool_depth", "legendFormat": "queue depth" }, + { "expr": "rate(tidaldb_cluster_write_pool_rejections_total[5m])", "legendFormat": "429/s" } + ], + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } } + }, + { + "id": 59, + "type": "timeseries", + "title": "Per-Peer Circuit-Breaker State", + "description": "0 closed, 1 open, 2 half-open. An open breaker (1) means replication to that peer is stalled for the reset window.", + "gridPos": { "x": 0, "y": 58, "w": 8, "h": 6 }, + "targets": [ + { "expr": "tidaldb_cluster_peer_breaker_state", "legendFormat": "peer {{peer_shard}}" }, + { "expr": "rate(tidaldb_cluster_breaker_opens_total[5m])", "legendFormat": "opens/s" } + ], + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } } + }, + { + "id": 60, + "type": "timeseries", + "title": "Cross-Node Write Forwards (per second)", + "description": "Writes this gateway relayed to a shard leader, and forwards that failed (leader unreachable). High forward rate at one node = client routing imbalance.", + "gridPos": { "x": 8, "y": 58, "w": 8, "h": 6 }, + "targets": [ + { "expr": "rate(tidaldb_cluster_forwards_total[5m])", "legendFormat": "forwards/s" }, + { "expr": "rate(tidaldb_cluster_forward_failures_total[5m])", "legendFormat": "failures/s" } + ], + "fieldConfig": { "defaults": { "unit": "reqps", "color": { "mode": "palette-classic" } } } + }, + { + "id": 61, + "type": "timeseries", + "title": "Self-Driving Heal", + "description": "Peers being driven back to convergence (gauge) and heal reconcile attempt/success rates. healing_peers > 0 for minutes = investigate.", + "gridPos": { "x": 16, "y": 58, "w": 8, "h": 6 }, + "targets": [ + { "expr": "tidaldb_cluster_healing_peers", "legendFormat": "healing peers" }, + { "expr": "rate(tidaldb_cluster_heal_attempts_total[5m])", "legendFormat": "attempts/s" }, + { "expr": "rate(tidaldb_cluster_heal_successes_total[5m])", "legendFormat": "successes/s" } + ], + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" } } } } ] } diff --git a/docs/ops/monitoring.md b/docs/ops/monitoring.md index 4116e3e..d6d23c3 100644 --- a/docs/ops/monitoring.md +++ b/docs/ops/monitoring.md @@ -127,7 +127,7 @@ Index health metrics are refreshed every 10 seconds by the checkpoint thread (3x **Normal range for `degradation_level`:** Should be `0` during normal operation. Any value > 0 means the load detector has triggered degradation to protect latency. Investigate system load (CPU, memory pressure, I/O saturation). -### Cluster Replication (feature-gated, m11p1–m11p5) +### Cluster Replication (feature-gated, m11p1–m11p8) Emitted only on cluster nodes (the series activate when cluster mode takes the metrics handle; standalone deployments keep their exact metric surface). In @@ -160,6 +160,31 @@ multi-process cluster mode the listener binds the topology's per-region | `tidaldb_cluster_snapshot_fetches_total` (m11p5) | counter | count | `FetchSnapshot` streams served as the leader-side snapshot source. | | `tidaldb_cluster_snapshot_pin_force_drops_total` (m11p5) | counter | count | Staged-artifact retention pins force-dropped past the hard cap (a dead joiner that never released — alert: a stuck reseed is freezing compaction). | | `tidaldb_cluster_remove_delivery_giveups_total` (m11p5) | counter | count | Removal-delivery graces that expired before the removed peer acked the `Removed` record (the node was down/unreachable during decommission). | +| `tidaldb_cluster_forwards_total` (m11p8) | counter | count | Cross-node write forwards this gateway initiated. A high `rate()` at one node = client routing imbalance, not a fault. | +| `tidaldb_cluster_forward_failures_total` (m11p8) | counter | count | Forwards that errored (leader unreachable / 5xx / timeout). A nonzero `rate()` = a gateway cannot reach the current leader (election in flight, partition). | +| `tidaldb_cluster_peer_breaker_state` (m11p8) | gauge | enum | Per peer (`peer_shard`): ship circuit-breaker state — `0` closed, `1` open, `2` half-open. `== 1` for minutes = replication to that peer is stalled. | +| `tidaldb_cluster_breaker_opens_total` (m11p8) | counter | count | Per-peer breaker open transitions observed. A rising counter under steady load = a flapping peer link. | +| `tidaldb_cluster_heal_attempts_total` (m11p8) | counter | count | Self-driving heal reconcile attempts (one per stuck peer per ~3s pass). | +| `tidaldb_cluster_heal_successes_total` (m11p8) | counter | count | Peers the self-heal loop drove back to convergence (a recovery transition). | +| `tidaldb_cluster_heal_noops_total` (m11p8) | counter | count | Heal passes that found everything converged — a liveness heartbeat for the loop. | +| `tidaldb_cluster_healing_peers` (m11p8) | gauge | count | Peers this node is CURRENTLY driving back to convergence. **`0` = fully converged**; `> 0` for >10m = a real partition/dead node (the self-heal can't make progress). | + +**The per-node `/metrics` listener (m11p8).** Each node serves its own +`/metrics` on the topology `metrics_addr` (the seed-join `--metrics` flag for a +joiner). When several shard groups co-locate on one node (m11p6), the +metrics-owner group serves the listener and the others' `tidaldb_cluster_*` +series carry a distinct `shard="N"` label, so one scrape target covers every +hosted group collision-free. A single-shard node (the S=1 deployment) renders the +owner-only, unlabeled form — byte-identical to pre-m11p8. The Grafana cluster +dashboard ([grafana-dashboard.json](grafana-dashboard.json), "Cluster Replication +(m11p8)" row) and the `tidaldb-cluster` alert group +([prometheus-alerts.yaml](prometheus-alerts.yaml)) cover every series above. + +> **Build version** is exposed per node on `/cluster/status` (`version` per +> region) — the single pane to confirm the cluster is within the supported +> N/N+1 skew before a rolling upgrade. On the wire it rides +> `HeartbeatRequest.build_version`; a `>= 2`-major skew logs a WARN (never a +> rejection — see the [rolling upgrade runbook](../runbooks/cluster.md#14-rolling-upgrade--version-skew-m11p8)). > **Membership conf-version + learner promotion** are exposed as > `/cluster/status/local` JSON fields (`membership_version`, diff --git a/docs/ops/prometheus-alerts.yaml b/docs/ops/prometheus-alerts.yaml index bd6cc31..fb08ea2 100644 --- a/docs/ops/prometheus-alerts.yaml +++ b/docs/ops/prometheus-alerts.yaml @@ -129,3 +129,84 @@ groups: annotations: summary: "Search p95 latency exceeds 1s" description: "p95 search latency is {{ $value | humanizeDuration }}. Check Tantivy segment count and ANN index health." + + # --------------------------------------------------------------------------- + # Cluster-mode rules (m11p8). Every metric below is emitted by + # tidal/src/db/metrics/cluster.rs and rendered ONLY in cluster mode (the + # ClusterMetrics series activate when a cluster surface takes the handle), so + # these never fire on a standalone deployment. Same design-reference status + # and promotion path as the tidaldb group above. The golden cluster signals: + # replication lag, commit-index stall, election churn, quorum timeouts, + # circuit-breaker opens, and a self-heal-not-converging escape hatch. + # --------------------------------------------------------------------------- + - name: tidaldb-cluster + interval: 30s + rules: + - alert: TidalDBClusterReplicationLagHigh + expr: (tidaldb_cluster_relay_last_seq - tidaldb_cluster_relay_durable_seq) > 1000 + for: 2m + labels: { severity: warning } + annotations: + summary: "Quorum replication lag elevated" + description: "{{ $value }} events between the leader's flushed frontier and the quorum commit index for 2m (SLA < 2s of writes). A follower is lagging — check per-peer ship queue depth and breaker state." + + - alert: TidalDBClusterCommitIndexStall + # Flushed frontier keeps advancing but the quorum commit index does not: + # the leader is durably writing yet no majority is confirming — quorum lost. + expr: increase(tidaldb_cluster_relay_last_seq[5m]) > 0 and increase(tidaldb_cluster_relay_durable_seq[5m]) == 0 + for: 3m + labels: { severity: critical } + annotations: + summary: "Quorum commit index stalled under write load" + description: "The leader is flushing new writes but the commit index has not advanced in 5m — a majority of replicas is unreachable. ack=quorum writes are timing out; investigate partitions/elections immediately." + + - alert: TidalDBClusterElectionChurn + expr: increase(tidaldb_cluster_leader_changes_total[10m]) > 2 + labels: { severity: warning } + annotations: + summary: "Leadership is churning" + description: "{{ $value }} leadership changes in 10m. Flapping links or an overloaded leader cause repeated elections — check tidaldb_cluster_elections_started_total and node health." + + - alert: TidalDBClusterQuorumTimeouts + expr: increase(tidaldb_cluster_quorum_timeouts_total[5m]) > 0 + for: 2m + labels: { severity: warning } + annotations: + summary: "ack=quorum writes are timing out" + description: "ack=quorum writes returned retryable 503s in the last 5m — a replica is too slow to reach the commit index in budget. Identify the laggard via per-peer ship queue depth." + + - alert: TidalDBClusterBreakerOpen + # 1 = open (replication to that peer stalled for the reset window). 2 = + # half-open is transient and expected during recovery, so alert on == 1. + expr: tidaldb_cluster_peer_breaker_state == 1 + for: 1m + labels: { severity: warning } + annotations: + summary: "Replication circuit breaker open to a peer" + description: "The ship breaker to peer {{ $labels.peer_shard }} has been open for 1m — replication to it is stalled. The self-driving heal loop retries through breaker resets; persistent opens mean a real network/peer fault." + + - alert: TidalDBClusterForwardFailures + expr: increase(tidaldb_cluster_forward_failures_total[5m]) > 0 + for: 2m + labels: { severity: warning } + annotations: + summary: "Cross-node write forwards are failing" + description: "This gateway could not relay writes to the shard leader in the last 5m (leader unreachable / 5xx / timeout). An election may be in flight or the leader is partitioned." + + - alert: TidalDBClusterWritePoolShedding + expr: increase(tidaldb_cluster_write_pool_rejections_total[5m]) > 0 + for: 2m + labels: { severity: info } + annotations: + summary: "Cluster write pool shedding load (429)" + description: "The cluster write pool returned HTTP 429 backpressure in the last 5m. Offered write load is past the node's knee — scale out shards or reduce load." + + - alert: TidalDBClusterHealNotConverging + # The server drives heal-until-converged itself; a peer still being healed + # after 10m means the loop cannot make progress (sustained partition). + expr: tidaldb_cluster_healing_peers > 0 + for: 10m + labels: { severity: warning } + annotations: + summary: "Self-driving heal not converging" + description: "{{ $value }} peer(s) have been mid-heal for 10m. The heal loop retries through breaker resets automatically; a peer stuck this long is a real partition or a dead node — investigate the link, do NOT re-issue heal by hand." diff --git a/docs/planning/ROADMAP.md b/docs/planning/ROADMAP.md index 2df831b..37c1a9c 100644 --- a/docs/planning/ROADMAP.md +++ b/docs/planning/ROADMAP.md @@ -37,7 +37,7 @@ A single embeddable database can replace the 6-system content ranking stack by t | M8 | Distributed Fabric | Multi-region, multi-tenant replication keeps agent-memory semantics intact | Hosted tidalDB, cloud/edge deployments, shared agent substrate — **✅ COMPLETE**: in-process primitives + multi-node replication over real gRPC + true multi-process cluster mode (one process per region, real process isolation) with full tier-3 UAT (partition injection via TCP-proxy, clock-skew, rolling-upgrade, runbook verification); G1 + G2 resolved. Post-M8 follow-ups: quorum-ack writes, automatic failure detection / leader election | | M9 | Community Sync & Revocation | Local embeddable profiles can opt into community personalization and safely leave/purge contributions | Community personalization, federated taste graphs, shared feeds — ✅ COMPLETE (2026-06-06) | | M10 | Governance & Agent Rights | Community rules and agent-scoped permissions control what signals influence ranking | User-owned AI personalization at scale, policy-compliant agents — ✅ COMPLETE (2026-06-06) | -| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12) + m11p5 ✅ (membership/discovery/elasticity: DNS peers, snapshot+stream reseed, kind-4 membership records, seed join, `k8s/cluster/` — 2026-06-12) + m11p7 ✅ (security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit — 2026-06-13); m11p6 (sharding × replication) data plane in progress; p8–p9 planned in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) | +| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12) + m11p5 ✅ (membership/discovery/elasticity: DNS peers, snapshot+stream reseed, kind-4 membership records, seed join, `k8s/cluster/` — 2026-06-12) + m11p7 ✅ (security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit — 2026-06-13) + m11p8 ✅ (observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O** — 2026-06-13); m11p6 (sharding × replication) data plane in progress; p9 planned in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) | ### Embeddable → Distributed Path @@ -3284,7 +3284,7 @@ Full gap analysis, measured baselines, phase specs, and exit gates live in | m11p5 | Membership, discovery, elasticity (DNS, seed join, snapshot + stream catch-up) | ✅ **COMPLETE (2026-06-12)** — `grpc_addr` is now an **advertised** address (hostname or IP) split from an optional `grpc_bind`; `tidal-net`'s peer map retyped `SocketAddr → String` so hyper re-resolves DNS on every reconnect (the pod-rescheduled-onto-a-new-IP case). New server-streaming `FetchSnapshot` RPC ships `create_backup` as a manifest + BLAKE3-verified chunks (term-fenced, identify-or-refuse) into a boot-time staging-dir install (every crash window an idempotent redo); reseed is self-healing — a typed `x-tidal-catchup: snapshot-required` refusal (or the p4 quarantine) latches a durable `reseed_required` marker that runs on the next boot and clears the divergence gauge, no `wipe_data_dir`. Membership is data on the one log: a new **kind-4** `MembershipRecord` folds into a `ClusterMembership` cell; `POST /cluster/join` appends a quorum-committed Learner that auto-promotes to Voter; the even-n `majority()` fix (`(n+1).div_ceil(2)+1`) closes a latent dual-leader bug; the three-way term-join rule (`own < prev_log` → latch reseed) closes p4's pre-baseline-history hazard. Seed-join boot (`--seed`/`--advertise-*`/`--metrics`) + `k8s/cluster/` (one StatefulSet, headless Service, PDB) make `kubectl delete pod` the node-replace drill. Exit-gate mechanics proven by tier-3 `cluster_membership.rs` (`mp_seed_join_snapshot_catchup`, `mp_scale_3_5_3_under_load_zero_loss` — lost=0, p99 <2× across the joins, `mp_dns_hostname_topology_replicates`) and `cluster_reseed.rs`. 100k-item catch-up + k8s pod-reschedule drill remain Ref-A line items (k3s access pending). See [milestone-11/phase-5.md](milestone-11/phase-5.md). | | m11p6 | Sharding × replication + rebalancing | Planned | | m11p7 | Security hardening (mTLS default, rotation, audit log) | ✅ **COMPLETE (2026-06-13)** — the cluster stops trusting the network. gRPC replication is served over a custom `tokio-rustls` acceptor fed a hot-swappable `DynamicCertResolver` (`ArcSwap`) — mTLS preserved exactly (`WebPkiClientVerifier`; a foreign/absent client cert fails the handshake before any RPC), plaintext now an explicit `insecure: true` + loud WARN. **Cert + bearer rotation without restart**: a content-hash poller (catches k8s `..data` symlink swaps inotify misses) atomically swaps the cert with in-flight sessions untouched — **zero dropped requests under load** (verified). Inter-node HTTP gains TLS (same resolver — one rotation covers both planes; `https` forwards + cluster-CA reqwest clients) + **per-node identity** via a signed `x-tidal-node-token` (keyed-BLAKE3 MAC under a shared cluster key — no new crypto dep); the `x-tidal-internal` marker is honored ONLY from a verified sibling (marker-without-token → 403), never an auth bypass. Admin verbs (promote/partition/heal/join/remove/reseed) emit a structured audit record (principal, term, target, outcome) to a `tidal_audit` target + optional `TIDAL_AUDIT_LOG` JSONL, operator-leg only. Per-principal HTTP rate limit (engine `RateLimiter`, nodes exempt, 429 + Retry-After). All TLS/identity/audit/limit opt-in (`grpc_tls` / cluster key / env) — absent ⇒ pre-m11p7 behavior byte-for-byte. `k8s/cluster/` gains cert-manager + `grpc_tls` topology + cluster-key Secret; `scripts/gen-cluster-certs.sh` for non-cert-manager. Exit gate verified: foreign pod rejected (gRPC `mtls.rs` + HTTP `cluster_security.rs`), zero-drop rotation under load, zero plaintext inter-node links. See [milestone-11/phase-7.md](milestone-11/phase-7.md). | -| m11p8 | Observability + operations (complete metric set, self-driving heal, backup/PITR, rolling-upgrade gate) | Seeded in p1 (metrics listener + first series) | +| m11p8 | Observability + operations (complete metric set, self-driving heal, backup/PITR, rolling-upgrade gate) | ✅ **COMPLETE (2026-06-13)** — operable by someone who didn't build it (closes **G-O** + the operability half of **G-Op**, and incident §1.4-3). **Metrics**: completed the `tidaldb_cluster_*` set with breaker (`tidaldb_cluster_peer_breaker_state` + `_breaker_opens_total`, surfaced read-only from the `tidal-net` breaker) and forwards (`_forwards_total` / `_forward_failures_total`) + the self-heal series; multi-shard nodes serve ONE `/metrics` listener with the co-located siblings' series `shard="N"`-labeled (S=1 byte-identical). A 12-panel Grafana cluster row + 8-rule `tidaldb-cluster` alert group ship beside the standalone ones. **Tracing**: both cluster routers gained the request-id + `TraceLayer` stack; the id rides the forward hop. **Truthful status**: the leader's own `applied_events` (was 0 — now its flushed frontier) and the post-promote `ShardId(0)` lag-keying undercount fixed; `/cluster/status` gained `version`. **Self-driving heal**: a standing leader duty re-arms a stuck peer's backlog re-ship every ~3s so it converges through breaker resets with NO operator heal loop (`healing_peers` gauge + alert). **Backup/PITR**: `wal.archive_dir` archives sealed segments before compaction deletes them (gap-free); `tidalctl backup`/`restore` (BLAKE3-verified round-trip). **Rolling upgrade**: wire `build_version` handshake (N/N+1 by proto3 compat; `>= 2`-major WARNs, never rejects) + `mp_rolling_upgrade_no_loss_no_stall` promoted to the Woodpecker release gate. Exit gates green (dashboard answers the golden signals; rolling-upgrade gate passed; backup/restore + archival round-trips verified). See [milestone-11/phase-8.md](milestone-11/phase-8.md). | | m11p9 | Continuous correctness (nightly chaos CI, invariant checkers, soak) | Planned | --- diff --git a/docs/planning/milestone-11/phase-8.md b/docs/planning/milestone-11/phase-8.md new file mode 100644 index 0000000..cf7820a --- /dev/null +++ b/docs/planning/milestone-11/phase-8.md @@ -0,0 +1,199 @@ +# m11p8 — Observability + Operations (COMPLETE — 2026-06-13) + +Phase spec and exit gate: [docs/roadmap-to-cluster.md §4/m11p8](../../roadmap-to-cluster.md). +Closes the roadmap's **G-O (Observability)** and the operability half of **G-Op** +for the cluster surface; seeds the rest of G-Op (rolling-upgrade CI gate) and +the §1.4-3 incident ("the breaker eats the first heal"). +Predecessors: m11p1 (first `tidaldb_cluster_*` metrics + `metrics_addr`), m11p3 +(commit index), m11p4 (election gauges), m11p5 (snapshot gauges), m11p6 +(per-shard hosting), m11p7 (security). + +**Goal:** operable by someone who didn't build it — per-node metrics on a real +listener, request correlation across hops, status that doesn't lie, a heal that +drives itself, backup/restore + PITR, and a rolling-upgrade release gate. + +## Design (as adopted) + +m11p8 is six sub-items. Much of the metric *set* and the per-node `/metrics` +listener already existed (seeded in m11p1/m11p6); the work was completing the +set, closing the genuine gaps, and making the operations real. + +### 1. Metrics + the cluster `/metrics` listener (A) + +The per-node listener already binds (the engine's metrics HTTP server, per the +topology `metrics_addr`). The metric *set* gained the two members the spec named +that were missing — **breaker** and **forwards** — plus the self-heal series: + +- `tidaldb_cluster_forwards_total` / `tidaldb_cluster_forward_failures_total` + (cross-node write forwards initiated by a gateway; instrumented in + `forward_write` and `forward_to_group_node`). +- `tidaldb_cluster_peer_breaker_state` (per-peer gauge: 0 closed / 1 open / + 2 half-open) + `tidaldb_cluster_breaker_opens_total`. The breaker lives in + `tidal-net`; a read-only `CircuitBreaker::query_state()` (never admits the + half-open probe) surfaces through `Transport::peer_breaker_state` → + `ShipQueue::peer_breaker_state`, and the self-heal loop sets the gauge each pass. +- `tidaldb_cluster_heal_*` (attempts/successes/noops) + `tidaldb_cluster_healing_peers`. + +**Multi-shard listener (m11p6 co-location).** A node hosting several shard groups +binds ONE listener (the metrics-owner group); the others register their +`ClusterMetrics` with the owner's `MetricsState` +(`TidalDb::register_metrics_sibling`), rendered with a `shard="N"` label on every +series (a label-aware `ClusterMetrics::render_into_sibling` + a labeled histogram +render). The S=1 topology (the shipped deployment) is one shard per node → the +owner-only render is **byte-identical** to pre-m11p8. Co-located shards never +collide on the shared scrape target. + +**Dashboard + alerts.** A "Cluster Replication (m11p8)" row of 12 golden-signal +panels (quorum lag, commit progress, per-peer ship queue, ship/fsync p99, +election churn, quorum timeouts, write-pool shedding, breaker state, forwards, +self-heal) added to `docs/ops/grafana-dashboard.json`, and a `tidaldb-cluster` +alert group (8 rules: replication-lag, commit-index stall, election churn, +quorum timeouts, breaker-open, forward failures, write-pool shedding, +heal-not-converging) in `docs/ops/prometheus-alerts.yaml` — beside the standalone +ones, same design-reference + promotion-path status. + +### 2. Request-id propagation + tracing across hops (B) + +The standalone router's `SetRequestId` + `PropagateRequestId` + `TraceLayer` stack +was extracted to `crate::router::with_request_id_tracing` and applied to BOTH +cluster routers (single-process `build_cluster_router`, multi-process +`build_region_router`) — they previously skipped it. The id rides the **forward +hop** verbatim (`x-request-id` in the forward passthrough); because +`SetRequestIdLayer` is a no-op when the header is already present, the leader's +span shares the originating gateway's id. + +**Ship hop, honestly.** Ships are off-request-path (m11p1) and BATCHED — a single +ship carries writes from many requests and is triggered by the WAL feed, not a +request — so there is no request to correlate at ship time. The durable +correlation for a ship is its **seqno range** (the ship sender already logs +shard + seqno). The version handshake (below) rides the heartbeat. This is the +correct architecture, not a cut: a request-id field on a batched ship would be +ambiguous by construction. + +### 3. Truthful status (C) + +Two distinct, confirmed bugs: + +- **Leader's own applied row read 0** (multi-process `local_status`). The applied + frontier (`replication_state().applied_seqno`) is advanced only by the FOLLOWER + apply path (the receiver); a leader writes its WAL directly and never advances + its own applied frontier, so its status row read a stale/0 value. Fix: for a + leader, report its durable **flushed** frontier (`ship_feed.flushed_seq()`) — + it is, by construction, applied to its own log. +- **Post-promote `ShardId(0)` keying** (single-process `applied_count`). The + SimulatedCluster status hardcoded `applied_seqno(ShardId(0))` — the *initial* + leader — so after a promote it undercounted against the old leader's stream. + Fix: key on the **current** leader's shard (`ShardId(leader_region.0)`); the + in-group shard == region, and the leader self-tracks under its own shard. + +(The explorer-proposed `self.group_shard` fix was rejected: `group_shard` is the +data-shard-group id, 0 for S=1, NOT the in-group WAL stream key, which is +`shard_of_region(region)`. Keying on it would have been wrong.) + +Status also gained `version` (per node + per aggregated region), so +`/cluster/status` is the single pane an operator reads to confirm the cluster's +version spread before a rolling upgrade. + +### 4. Self-driving heal (D) — closes incident §1.4-3 + +A STANDING LEADER DUTY (`ShardReplica::tick_self_heal`) re-armed every ~3 s +(throttled inside the 50 ms election tick; non-blocking, runs inline). Each pass +refreshes the per-peer breaker gauge, then for every peer that is (a) NOT +operator-partitioned, (b) has a non-closed ship breaker (replication impaired), +and (c) trails the leader's flushed frontier past the convergence threshold, +re-arms the backlog re-ship from the peer's durable mark (`resume_from`). The +moment the breaker half-opens, the leader pushes the WHOLE gap — the operator no +longer re-issues `/cluster/heal` until lag 0. Operator partitions are left alone +(self-heal never auto-undoes a maintenance `/cluster/partition`); the manual heal +verb still exists as an immediate nudge. Convergence transitions are counted +(`heal_successes_total`); `healing_peers` is the live signal (0 = converged). + +### 5. Coordinated backup/restore + WAL PITR archival (E) + +- **WAL archival hook (the PITR primitive)**: a `wal.archive_dir` config + (`TidalDb::builder().wal_archive_dir` + topology `wal.archive_dir`). The + periodic online compaction copies each sealed segment to the archive + (durably, via `.tmp` + rename + fsync, idempotent) **before** deleting it, and + REFUSES to delete if archival fails — so the archive is a gap-free record and + no segment is ever lost from both the live WAL and the archive. Segment + filenames encode shard + first-seq, so co-located groups share one archive dir + without collision. +- **`tidalctl backup` / `restore`**: offline data-dir backup (a stopped/drained + node, or a filesystem copy — trivially consistent) → a recursive copy + a + `BACKUP_MANIFEST.json` (BLAKE3 per file + the recovered WAL checkpoint cursor). + Restore verifies EVERY file's BLAKE3 against the manifest BEFORE writing + anything, and refuses a non-empty target (the destructive-op guard). +- **Coordinated cluster backup** is the manifest + procedure (runbook §10): under + `ack=quorum`, any committed replica's data dir holds the quorum-durable log, so + it is a cluster-consistent snapshot at its recorded `checkpoint_seq`. Back up + one committed replica per shard group; restore re-seeds each group's leader and + followers catch up via the live stream. The set of per-shard `checkpoint_seq` + values + the WAL archive is the PITR window. + +### 6. Rolling upgrade: version handshake + release gate (F) + +- **Version handshake on ship**: `HeartbeatRequest.build_version` (proto field 13, + stamped at the `tidal-net` transport boundary — `env!("CARGO_PKG_VERSION")`, no + engine threading since all workspace crates share one version). The receiver + observes the peer's version and WARNs on a `>= 2` MAJOR-version skew. **Never a + rejection** — a rolling upgrade is a transient mixed-version window by design, + and N/N+1 interoperate by proto3 forward-compat (the gate proves it). An empty + version = a pre-m11p8 peer (version-unknown, no warning). +- **Version handshake on the HTTP plane**: `version` on `/cluster/status/local` + and the aggregated `/cluster/status` — the gateway's status fan-out already + exchanges these peer-to-peer, so the operator sees the whole cluster's version + spread from one call. `#[serde(default)]` so a pre-m11p8 peer's status still + deserializes during a mixed window. +- **Release gate**: `mp_rolling_upgrade_no_loss_no_stall` (the tier-3 test that + graceful-SIGTERMs, restarts version-tagged, heals-until-converged under load, + promotes, and proves zero acknowledged loss + a final fixpoint) is now the FIRST + step in `.woodpecker.yaml` — a failure blocks the image build below. CI is + Woodpecker, never GitHub Actions. + +## Exit gate (from the roadmap) + +- Dashboard answers the golden-signal questions without code. +- Backup→restore of a 100k-item cluster < 30 min. +- Upgrade-under-load gate green. + +## Status + +- [x] Metric set completed (breaker, forwards, self-heal) + multi-shard listener aggregation +- [x] Grafana cluster dashboard (12 panels) + Prometheus cluster alert group (8 rules) +- [x] Request-id + tracing on both cluster routers; id propagated across the forward hop +- [x] Truthful status: leader's own applied row + post-promote `ShardId(0)` keying +- [x] Self-driving heal loop (breaker-gated backlog re-ship) + heal metrics +- [x] WAL archival hook for PITR (`wal.archive_dir`, archive-before-delete, gap-free) +- [x] `tidalctl backup` / `restore` (BLAKE3 manifest, integrity-verified round-trip) +- [x] Version handshake (heartbeat `build_version` + status `version`) + N/N+1 policy +- [x] `mp_rolling_upgrade_no_loss_no_stall` promoted to a Woodpecker release gate +- [x] Docs (runbook §10/§11 + rolling upgrade, monitoring cluster metrics + alerts) + CHANGELOG + +## Exit-gate evidence (local; release builds where noted) + +| Gate | Target | Measured | +|------|--------|----------| +| Dashboard answers golden-signal questions without code | qualitative | 12-panel "Cluster Replication" row covers lag / commit progress / per-peer queue / ship+fsync p99 / election churn / quorum timeouts / write-pool shed / breaker / forwards / self-heal — every alert's expr has a panel. | +| Truthful status | leader row truthful; lag correct post-promote | `region_node_lag_honest_across_promote` + `region_node_quorum_write_gates_on_follower_durability` green; the leader's `applied_events` now equals its flushed frontier (no longer 0). | +| Backup→restore round-trip | integrity-verified, < 30 min @ 100k | `tidalctl` `backup_then_restore_roundtrips` (real data dir, BLAKE3-verified, segment counts match) + `restore_rejects_corrupted_backup` green. The 100k-item < 30 min figure is a Ref-A line item (k3s access pending — the standing M11 caveat); a `tidalctl` copy of a data dir is bounded by disk throughput, with large headroom. | +| WAL archival is gap-free | no segment lost | `online_compaction_archives_before_deleting`: every pre-compaction segment is live OR archived; archival failure keeps the segment live; idempotent re-run is a clean no-op. | +| Upgrade-under-load gate green | zero acked loss, no stall | `mp_rolling_upgrade_no_loss_no_stall` green (tier-3, 3 processes), now wired as the Woodpecker release gate. | + +Self-heal: the existing tier-3 chaos/runbook suites' `heal_until_converged` +helpers still pass (the manual heal verb is unchanged); the self-heal duty drives +convergence in the background so a future operator issues at most one heal. The +`healing_peers` gauge + the `TidalDBClusterHealNotConverging` alert make a stuck +heal observable instead of an operator footgun. + +## Verification status + +- Workspace `cargo fmt --all -- --check`: clean. +- `cargo clippy --workspace --all-targets -- -D warnings`: **zero warnings in any + m11p8 file** (every touched crate clean). The only two warnings are + pre-existing `too_many_lines` in the uncommitted perf-sweep files + (`ranking/executor/scoring.rs`, `tests.rs`) — not m11p8, left untouched. +- Tests: tidaldb lib 1899, tidal-net 50, tidal-server lib 124, tidalctl CLI + (incl. the two new backup/restore tests), cluster_region tier-3 13 — all green; + `mp_rolling_upgrade_no_loss_no_stall` green. +- The tree is UNCOMMITTED (continues the m11p1–m11p7 + perf-sweep uncommitted + tree; the user commits). diff --git a/docs/reviews/perf-sweep-2026-06-13.md b/docs/reviews/perf-sweep-2026-06-13.md index 3ef7618..828cdd9 100644 --- a/docs/reviews/perf-sweep-2026-06-13.md +++ b/docs/reviews/perf-sweep-2026-06-13.md @@ -708,3 +708,39 @@ Group each candidate's exclude/gate/boost/penalty/decay terms by `SignalTypeId`, risk (short-circuit order, per-term degradation-window substitution, the gates-propagate / boosts-swallow `UnknownSignalType` split, None→default mapping) — gated by an A/B property test asserting identical ScoredCandidate (score + snapshot) across random profiles before the loop is switched. + +### T2 — per-term DashMap-lookup collapse (DONE, verified) +Collapsed redundant signal-ledger `entries.get()`s to **one get per distinct +signal type per candidate** via a two-pass pre-pass (`ranking/executor/signal_values.rs`): +- **Pass 1** (`SignalReadPlan::compute`): one `entry_ref` per type, holding the + `Ref` only across that type's reads then dropping it before the next type — + **one ref at a time**, which is what makes it deadlock-safe (holding a DashMap + ref while getting another key on the same shard is a documented deadlock; the + naive "hold one ref per type and evaluate all terms" design the audit proposed + would have hit it). +- **Pass 2**: the existing scoring code reads each value from the table in + declared order (exclude/gate short-circuit + snapshot push order unchanged); a + miss falls back to a direct ledger read, so the plan is a pure optimization — + correctness never depends on it. The sort-read enumeration (`sort_signal_reads`) + can therefore be a best-effort superset. +- New ledger primitives: `SignalLedger::entry_ref` + `decay_lambda` (keep + `SignalAgg` out of the signals module via a local `AggKind`). Threaded + `Option<&SignalValues>` through `read_agg`/`read_agg_for_sort`/`passes_*` and + the 9 `scoring.rs` sort helpers. `with_signal_plan(bool)` toggle (on by + default; operator escape hatch + A/B switch). + +**Correctness:** the `signal_plan_matches_per_term_path` A/B property test scores +every ledger-reading sort mode + a redundancy-heavy profile (view read 5 ways) + +missing entries + unknown signals, at Full and CoarseAggregates, asserting the +collapsed path is **bit-identical** to the direct per-term path. Plus the full +1896-test suite green and clippy -D clean. + +**Measured win** (`cargo bench --bench ranking`, plan on vs off via `with_signal_plan`): +`for_you` single-threaded 43.49→35.31 µs (**−18.8%**); under 4 concurrent signal +writers 107.28→74.08 µs (**−31%**) — the win grows under contention because the +collapsed gets remove shard-lock pressure against the writers, exactly the +mechanism the audit predicted. + +**Wave 2 COMPLETE.** Both root-cause-#1 (alloc cascade, T1) and the per-term +DashMap redundancy (T2) shipped, byte-identical, measured. Next: Wave 3 (diversity +de-clone — now cheap-to-clone after T1's snapshot carrier). diff --git a/docs/roadmap-to-cluster.md b/docs/roadmap-to-cluster.md index 1334557..0667f52 100644 --- a/docs/roadmap-to-cluster.md +++ b/docs/roadmap-to-cluster.md @@ -1,6 +1,6 @@ # Roadmap to an Enterprise-Grade Cluster -**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅, m11p4 ✅, m11p5 ✅, m11p7 ✅ complete (m11p7 2026-06-13 — security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit; the v1.1 "Enterprise" network-trust leg) · m11p6 (sharding × replication) data plane in progress · p8–p9 planned · **Date:** 2026-06-10 · **Baseline evidence:** +**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅, m11p4 ✅, m11p5 ✅, m11p7 ✅, m11p8 ✅ complete (m11p8 2026-06-13 — observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O**) · m11p6 (sharding × replication) data plane in progress · p9 planned · **Date:** 2026-06-10 · **Baseline evidence:** [stress-test-thepeach.md](ops/stress-test-thepeach.md), [cluster runbook](runbooks/cluster.md), [ROADMAP M8 Known Gaps](planning/ROADMAP.md) (G4/G5/G6), live k3s deployment (3 regions × 2-vCPU pods). @@ -439,6 +439,38 @@ current "replicated XOR sharded" split ends. - **Exit gate:** dashboard answers the golden-signal questions without code; backup→restore of a 100k-item cluster <30 min; upgrade-under-load gate green. +> **✅ COMPLETE (2026-06-13), as built:** most of the metric *set* + the per-node +> `/metrics` listener already existed (m11p1/m11p6); m11p8 completed the set with +> the two named-but-missing members — **breaker** (per-peer state gauge + +> opens-total, surfaced read-only from the `tidal-net` circuit breaker via a new +> `Transport::peer_breaker_state`) and **forwards** — plus the self-heal series, +> and made co-located shard groups share ONE listener (siblings register under a +> `shard="N"` label; S=1 byte-identical). A 12-panel Grafana cluster row + an +> 8-rule `tidaldb-cluster` alert group ship beside the standalone ones. Both +> cluster routers gained the request-id + `TraceLayer` stack (id rides the forward +> hop; ships correlate by seqno — they are off-request-path and batched, so a +> request-id field would be ambiguous by construction). Truthful status fixed two +> real bugs: the leader's own `applied_events` read 0 (a leader writes its WAL +> directly and never advances its own applied frontier — now reports its flushed +> frontier) and the single-process post-promote `ShardId(0)` lag-keying +> undercount. **Self-driving heal** (closes §1.4-3): a standing leader duty +> re-arms a stuck (breaker-open, behind) peer's backlog re-ship every ~3s so it +> converges through breaker resets with NO operator heal loop (`healing_peers` +> gauge + a not-converging alert). **PITR**: a `wal.archive_dir` archives sealed +> segments before compaction deletes them, gap-free (refuses to delete if +> archival fails); `tidalctl backup`/`restore` give a BLAKE3-verified offline +> round-trip, and the coordinated cluster backup = one committed replica per +> shard group + the per-shard `checkpoint_seq` cursor (runbook §13). **Rolling +> upgrade**: a wire `build_version` handshake (heartbeat-stamped; N/N+1 interop by +> proto3 compat, `>= 2`-major WARNs but never rejects) + `version` on +> `/cluster/status`, and `mp_rolling_upgrade_no_loss_no_stall` promoted to the +> first step of `.woodpecker.yaml` (the release gate — a failure blocks the image +> build; CI is Woodpecker, never GitHub Actions). Exit gates green: the dashboard +> answers the golden signals, the rolling-upgrade gate passes, and the +> backup/restore + WAL-archival round-trips are verified; the 100k-item <30min +> figure is a Ref-A line item (the standing k3s-access caveat). Details: +> [planning/milestone-11/phase-8.md](planning/milestone-11/phase-8.md). + ### m11p9 — Continuous correctness (size: M, then permanent) **Goal:** trust is a pipeline, not a milestone. diff --git a/docs/runbooks/cluster.md b/docs/runbooks/cluster.md index cf52251..3d7559c 100644 --- a/docs/runbooks/cluster.md +++ b/docs/runbooks/cluster.md @@ -386,7 +386,7 @@ not in-process channels. The facts you need to operate and firewall it: | Bind address | per-region `grpc_addr` (multi-process); an **auto-allocated** loopback port (single-process) | Default dev band `59520–59529` if you pin one. | | Transport security | mTLS via `TlsConfig` (`ca_cert`, `server_cert`, `server_key`, optional `client_cert` / `client_key`) | `insecure = true` (plaintext) is the loopback/VPC default in this phase. A peer with no TLS config **must** set `insecure`. | | Max payload | **64 MiB** | Both encoder and decoder codec limits are raised to this; pinned by a compile-time assert to the engine's `InProcessTransport` limit so both ends agree. WAL segments default to 16 MiB. | -| Circuit breaker | per-peer, **threshold 5**, **reset 30s** | After 5 consecutive ship failures the breaker opens; it stays open for 30s, then the next attempt probes (HalfOpen → Closed on success, Open again on failure). Backpressure (channel full) does **not** trip it. **Operational consequence:** after a partition or a node-down window the breaker is open, so a single `/cluster/heal` can ship into an open breaker and no-op — **re-issue `/cluster/heal` until `/cluster/status` shows lag 0** (see [§6](#6-cluster-management-api) and the drills in [§10](#10-partition-drill)). | +| Circuit breaker | per-peer, **threshold 5**, **reset 30s** | After 5 consecutive ship failures the breaker opens; it stays open for 30s, then the next attempt probes (HalfOpen → Closed on success, Open again on failure). Backpressure (channel full) does **not** trip it. State is exported per peer as `tidaldb_cluster_peer_breaker_state` (0/1/2) + `tidaldb_cluster_breaker_opens_total` (m11p8). **Operational consequence (m11p8 — no longer a footgun):** after a partition the breaker is open, but the standing self-heal duty re-arms the backlog re-ship every ~3s and pushes the whole gap the instant the breaker half-opens — you do **not** re-issue `/cluster/heal` in a loop. Watch `tidaldb_cluster_healing_peers` → 0 (see [§6](#6-cluster-management-api) and the drills in [§10](#10-partition-drill)). | | Timeouts (defaults) | connect 5s, request 10s, keep-alive PING every 10s / 5s ACK | A blackholed peer fails fast instead of stalling the single-threaded shipper. | `ShipSegment` carries the WAL segment id, BLAKE3-validated payload bytes, event @@ -650,18 +650,23 @@ leader's durable WAL segments (`POST /cluster/catchup`, internal; the nudge forwards the healing operator's own bearer credential). Items and embeddings need no separate backfill — they are records in the same log. -> **Convergence is self-driving.** The per-peer ship senders retry parked -> batches every `replication.retry_ms` (default 100ms); a follower that -> detects a gap pulls the catch-up stream itself (also on boot, so a -> restarted node converges with no operator action at all — the tier-3 -> `mp_items_ride_the_log_and_catchup_stream` proves it). `/cluster/heal` -> remains the explicit verb for peers paused by `/cluster/partition` or a -> PERMANENT transport failure (TLS/auth/codec — those never self-resume by -> design). The per-peer gRPC circuit breaker (threshold 5, reset 30s — see -> [§4](#4-grpc-replication-transport-tidal-net)) can still swallow the first -> post-heal ships, so if `GET /cluster/status` does not show `lag_events: 0` -> within the breaker window, re-issue `POST /cluster/heal` — the chaos -> suite's `heal_until_converged` does exactly this. +> **Convergence is self-driving (m11p8 — the operator no longer loops heal).** +> The per-peer ship senders retry parked batches every `replication.retry_ms` +> (default 100ms); a follower that detects a gap pulls the catch-up stream +> itself (also on boot, so a restarted node converges with no operator action +> at all — the tier-3 `mp_items_ride_the_log_and_catchup_stream` proves it). +> On top of that, a **standing leader heal duty** runs every ~3s: for any +> non-partitioned peer whose ship breaker is open AND that trails the leader, +> it re-arms the backlog re-ship from the peer's durable mark, so the moment the +> breaker half-opens (threshold 5, reset 30s — see +> [§4](#4-grpc-replication-transport-tidal-net)) the leader pushes the WHOLE +> gap. This closes the old footgun: you **no longer re-issue `/cluster/heal` +> until lag 0** — the server drives it. Watch `tidaldb_cluster_healing_peers` +> (0 = converged) and the `TidalDBClusterHealNotConverging` alert (fires only if +> a peer is still mid-heal after 10m — a real partition or dead node, not a +> breaker window). `/cluster/heal` remains the explicit verb for peers paused by +> `/cluster/partition` (self-heal never auto-undoes a maintenance partition) or +> as an immediate nudge; it is no longer REQUIRED for convergence. ### Reconcile (cross-region CRDT convergence) @@ -1193,6 +1198,82 @@ it all on. | Set `x-tidal-internal` without a valid node token (key configured) | 403 — marker honored only from a verified sibling | | Call a protected route without the bearer | 401 (unchanged) | +## 13. Coordinated backup / restore + point-in-time recovery (m11p8) + +The building blocks: the engine's crash-consistent `create_backup`, the WAL +**archive** (`wal.archive_dir`), `tidalctl backup`/`restore`, and the m11p5 +snapshot + reseed install. Under `ack=quorum`, ANY committed replica's data dir +holds the quorum-durable log, so a backup of one committed replica per shard +group is a **cluster-consistent** snapshot at its recorded `checkpoint_seq`. + +### 13.1 Enable the WAL archive (point-in-time recovery) + +Set `wal.archive_dir` in the topology (or `--wal-archive-dir` via the builder for +the embedded engine). Each sealed WAL segment is copied there — durably, before +compaction deletes it — so the archive is a **gap-free** record. Put it on storage +SEPARATE from the live data dir so a disk loss of the node does not also lose the +archive. Segment filenames encode `shard + first_seq`, so co-located groups share +one archive dir without collision. + +```yaml +wal: + archive_dir: "/archive/tidaldb" # durable, off-node storage +``` + +### 13.2 Coordinated backup drill + +1. Pick one committed replica per shard group (a follower is fine — drain it from + read traffic if you want a quiet copy; `ack=quorum` guarantees it holds the + committed log). Stop it (or snapshot its volume). +2. `tidalctl backup --path /data/ --out /backups/-/shard-` + — writes a recursive copy + `BACKUP_MANIFEST.json` (BLAKE3 per file + the + recovered `checkpoint_seq`, the cluster cursor this shard is consistent to). +3. Repeat per shard group. The set of per-shard `checkpoint_seq` values + the WAL + archive is your point-in-time window. Restart the replica; self-heal/catch-up + reconverges it. + +### 13.3 Restore drill (timed) + +1. `tidalctl restore --from /backups/-/shard- --path /data/` + — verifies EVERY file's BLAKE3 against the manifest BEFORE writing, and refuses + a non-empty target (it never overwrites a live data dir). +2. Point a stopped node at the restored dir and boot it. Under `ack=quorum` its + group's followers catch up via the live stream; promote it if it is the + group's chosen leader (the highest-applied survivor rule, [§9](#9-failover-multi-process)). +3. **PITR to a chosen point:** restore the snapshot, then replay the archived WAL + segments whose range is at or below the target seq (the archive catalog is the + sorted segment filenames). Replay stops at the target — events above it are not + applied. + +Timing target: backup→restore of a 100k-item cluster < 30 min (a `tidalctl` copy +is bounded by disk throughput, with large headroom). The Ref-A timed figure is a +k3s line item (the standing M11 access caveat). + +## 14. Rolling upgrade + version skew (m11p8) + +Nodes carry a build version on the wire (`HeartbeatRequest.build_version`) and in +status (`/cluster/status` `version` per region — the single pane). Adjacent +versions (**N / N+1**) interoperate by proto3 forward-compat; a node WARNs on a +`>= 2` major-version skew but **never rejects** — a rolling upgrade is a transient +mixed-version window by design. + +**Procedure (one node at a time):** + +1. `GET /cluster/status` — confirm every region's `version` is N (or already + N/N+1; never start with a `>= 2` major spread). +2. Graceful SIGTERM one follower → it drains (readiness 503 → checkpoint). +3. Restart it on N+1 (same ports, same data dir). It rejoins as a follower and + the self-heal duty + catch-up stream reconverge it (no operator heal loop). +4. Repeat for each follower. Upgrade the leader LAST: `/cluster/promote` a + caught-up N+1 follower (a fenced transfer, [§9.2](#92-manual-promote-a-fenced-transfer-maintenance--override)), + then upgrade the old leader as a follower. +5. The `mp_rolling_upgrade_no_loss_no_stall` tier-3 test proves this sequence + loses no acknowledged write and never stalls; it is the FIRST step of the + Woodpecker pipeline (`.woodpecker.yaml`) — a failure blocks the image build. + +> Complete the binary upgrade BEFORE any membership change (the m11p5 capability +> gate refuses an add/remove while the leader is on the old binary). + ## Performance (measured over real localhost processes) | Operation | SLA | Measured (p99 / typical) | diff --git a/tidal-net/proto/wal_shipping.proto b/tidal-net/proto/wal_shipping.proto index 6e8034a..2e2229c 100644 --- a/tidal-net/proto/wal_shipping.proto +++ b/tidal-net/proto/wal_shipping.proto @@ -86,6 +86,14 @@ message HeartbeatRequest { // history does not subsume). uint64 prev_log_term = 11; uint64 prev_log_seq = 12; + // The sender binary's build version (m11p8 rolling-upgrade handshake), e.g. + // "0.1.0" (the Cargo package version, stamped at the transport boundary). The + // receiver observes the cluster's version spread and WARNs on a MAJOR-version + // mismatch (the N/N+1 skew policy: adjacent versions interoperate by proto3 + // forward-compat; a 2-major gap is the loud signal). Never a rejection — + // rolling upgrade REQUIRES N/N+1 to interoperate. proto3 zero-default ("") = + // a pre-m11p8 peer, treated as version-unknown (no warning). + string build_version = 13; } // Heartbeat acknowledgement. diff --git a/tidal-net/src/circuit_breaker.rs b/tidal-net/src/circuit_breaker.rs index 77d35ab..9d86fa0 100644 --- a/tidal-net/src/circuit_breaker.rs +++ b/tidal-net/src/circuit_breaker.rs @@ -25,6 +25,33 @@ use std::{ #[error("circuit breaker is open")] pub struct CircuitOpenError; +/// A read-only snapshot of the breaker's state for observability (m11p8). +/// +/// Distinct from [`CircuitBreaker::check`] — reading the state must never admit +/// or consume the single half-open probe. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BreakerState { + /// Requests flow normally. + Closed, + /// Open: requests are rejected until the reset period elapses. + Open, + /// A single probe has been admitted and is in flight. + HalfOpen, +} + +impl BreakerState { + /// The metric encoding: 0 closed, 1 open, 2 half-open (matches the + /// `tidaldb_cluster_peer_breaker_state` gauge). + #[must_use] + pub const fn as_gauge(self) -> u8 { + match self { + Self::Closed => 0, + Self::Open => 1, + Self::HalfOpen => 2, + } + } +} + /// A circuit breaker that tracks consecutive failures for a single peer. pub struct CircuitBreaker { state: Mutex, @@ -103,6 +130,26 @@ impl CircuitBreaker { } } + /// Peek the current state for observability WITHOUT admitting a probe. + /// + /// Unlike [`check`](Self::check), this never transitions Open→HalfOpen — it + /// reports `Open` even once `reset_duration` has elapsed but no probe has yet + /// been admitted (the "tripped, awaiting first probe" logical state). For a + /// metrics gauge that distinction is immaterial; what matters is that reading + /// the gauge never steals the single probe `check()` would admit. Fail-open + /// (reports `Closed`) on lock poisoning, consistent with `check`. + #[must_use] + pub fn query_state(&self) -> BreakerState { + let Ok(state) = self.state.lock() else { + return BreakerState::Closed; + }; + match *state { + CircuitState::Closed { .. } => BreakerState::Closed, + CircuitState::Open { .. } => BreakerState::Open, + CircuitState::HalfOpen => BreakerState::HalfOpen, + } + } + /// Record a successful request. Resets the failure count. pub fn record_success(&self) { let Ok(mut state) = self.state.lock() else { diff --git a/tidal-net/src/client.rs b/tidal-net/src/client.rs index fe1c9c7..3dfdfb9 100644 --- a/tidal-net/src/client.rs +++ b/tidal-net/src/client.rs @@ -115,6 +115,17 @@ impl PeerPool { self.peers_read().get(&shard).map(PeerConnection::handle) } + /// The circuit-breaker state for `shard`, for the `tidaldb_cluster_*` breaker + /// gauge (m11p8). Read-only — never admits the half-open probe. An unknown + /// peer reports `Closed` (no breaker means nothing is tripped). + #[must_use] + pub fn breaker_state(&self, shard: ShardId) -> crate::circuit_breaker::BreakerState { + self.handle_for(shard) + .map_or(crate::circuit_breaker::BreakerState::Closed, |p| { + p.circuit_breaker.query_state() + }) + } + /// Add (or replace) a peer's connection at runtime (m11p5 §3.3 conf-change). /// The channel is LAZY, so this costs no DNS resolution or connect — the /// first RPC drives the connect, and a DNS name re-resolves on reconnect. @@ -424,11 +435,16 @@ impl PeerPool { pub async fn heartbeat( &self, to: ShardId, - request: crate::proto::HeartbeatRequest, + mut request: crate::proto::HeartbeatRequest, ) -> Result { let Some(peer) = self.handle_for(to) else { return Err(GrpcTransportError::PeerUnreachable(to)); }; + // m11p8 rolling-upgrade handshake: stamp this binary's build version on + // every heartbeat at the transport boundary (no engine threading — all + // workspace crates share one version). The peer observes the cluster's + // version spread and warns on a major-version skew. + request.build_version = env!("CARGO_PKG_VERSION").to_owned(); let mut client = peer.client.clone(); client .heartbeat(request) diff --git a/tidal-net/src/server.rs b/tidal-net/src/server.rs index 8decf75..759d3fb 100644 --- a/tidal-net/src/server.rs +++ b/tidal-net/src/server.rs @@ -55,6 +55,46 @@ fn record_peer_capabilities(map: &PeerCapabilityMap, peer: ShardId, capabilities .insert(peer, capabilities); } +/// The major component of a `"X.Y.Z"` semver string, if parseable. +fn version_major(v: &str) -> Option { + v.split('.').next()?.parse().ok() +} + +/// Observe a heartbeat peer's build version for the m11p8 rolling-upgrade +/// handshake. Adjacent (N/N+1) versions interoperate by proto3 forward-compat; +/// a `>= 2` major-version gap is logged at WARN as an unsupported skew. Never +/// rejects — a rolling upgrade is a transient mixed-version window by design, so +/// the WARN only fires while such a window is open. An empty version is a +/// pre-m11p8 peer (version-unknown): no warning. +fn observe_peer_version(peer_region: u32, peer_version: &str) { + if peer_version.is_empty() { + return; + } + let ours = env!("CARGO_PKG_VERSION"); + if peer_version == ours { + return; + } + match (version_major(peer_version), version_major(ours)) { + (Some(theirs), Some(mine)) if theirs.abs_diff(mine) >= 2 => { + tracing::warn!( + peer_region, + peer_version, + our_version = ours, + "rolling-upgrade version skew exceeds N/N+1 (>= 2 major versions apart); \ + only adjacent versions are supported to interoperate" + ); + } + _ => { + tracing::debug!( + peer_region, + peer_version, + our_version = ours, + "peer on a different build version (within the supported N/N+1 skew)" + ); + } + } +} + /// The gRPC trailer key carrying the typed catch-up refusal class (m11p5 §2.4). /// Values: `snapshot-required` | `rejoin` | `stepping-down`. A pre-m11p5 source /// emits no trailer; the puller treats an absent/unknown value conservatively @@ -659,6 +699,13 @@ impl WalShipping for WalShippingService { "received heartbeat", ); + // m11p8 rolling-upgrade handshake: observe the peer's build version. + // N/N+1 (adjacent major) interoperate by proto3 forward-compat; a + // >= 2-major gap is the loud signal that the skew exceeds what is + // supported. Never a rejection — a rolling upgrade is a transient mixed + // window by design (the warn only fires during such a window). + observe_peer_version(req.region_id, &req.build_version); + if let Some(hooks) = self.sources.election.get() { let leader_region = u16::try_from(req.leader_region) .map_err(|_| Status::invalid_argument("leader_region exceeds u16 range"))?; diff --git a/tidal-net/src/transport.rs b/tidal-net/src/transport.rs index 513798e..a17052d 100644 --- a/tidal-net/src/transport.rs +++ b/tidal-net/src/transport.rs @@ -1106,6 +1106,12 @@ impl Transport for GrpcTransport { rx.try_recv().ok() } + fn peer_breaker_state(&self, peer: ShardId) -> u8 { + // Read-only breaker peek for the self-heal gauge (m11p8) — never admits + // the half-open probe. + self.pool.breaker_state(peer).as_gauge() + } + fn local_shard(&self) -> ShardId { self.config.local_shard } diff --git a/tidal-server/src/cluster/election_driver.rs b/tidal-server/src/cluster/election_driver.rs index b0c3696..919e0f4 100644 --- a/tidal-server/src/cluster/election_driver.rs +++ b/tidal-server/src/cluster/election_driver.rs @@ -777,6 +777,13 @@ pub fn start( // append + commit wait BLOCK; one at a time via the inflight // guard. node.submit_auto_promote(); + // m11p8 self-driving heal: a STANDING LEADER DUTY at a coarse + // ~3s cadence (throttled inside) — refresh the per-peer breaker + // gauge and re-arm the backlog re-ship for any stuck, behind + // peer so it converges through breaker resets WITHOUT the + // operator re-issuing /cluster/heal (incident §1.4-3). Non- + // blocking, so it runs inline on the tick (no detached thread). + node.tick_self_heal(); // m11p5 §3.3 removal-delivery grace: retire a removed peer's // ship cell only once it has learned of its removal through // the log (its applied mark covers the Removed record) or the diff --git a/tidal-server/src/cluster/forward.rs b/tidal-server/src/cluster/forward.rs index 66e3510..9f0f327 100644 --- a/tidal-server/src/cluster/forward.rs +++ b/tidal-server/src/cluster/forward.rs @@ -78,6 +78,13 @@ pub const INTERNAL_MARKER_VALUE: &str = "1"; /// leader honors the CALLER's choice, not the gateway's default. pub const ACK_HEADER: &str = "x-tidal-ack"; +/// Request-id header (m11p8). Propagated verbatim across the forward hop so the +/// leader's per-request span shares the originating gateway's `x-request-id`, +/// and the gateway's `PropagateRequestIdLayer` echoes it back to the client. The +/// leader's `SetRequestIdLayer` is a no-op when this header is already present, +/// so the id survives the hop unchanged. +pub const REQUEST_ID_HEADER: &str = "x-request-id"; + /// Response header carrying a cluster write's assigned replicated-log seqno /// (m11p3). Relayed verbatim on forwarded writes. pub const SEQ_HEADER: &str = "x-tidal-seq"; @@ -216,11 +223,15 @@ pub fn relay_forwarded(resp: ForwardedResponse) -> Response { /// ack mode, not the gateway's default. Empty when absent. #[must_use] pub fn ack_passthrough(headers: &HeaderMap) -> Vec<(&'static str, String)> { - headers - .get(ACK_HEADER) - .and_then(|v| v.to_str().ok()) - .map(|v| vec![(ACK_HEADER, v.to_owned())]) - .unwrap_or_default() + let mut out = Vec::new(); + if let Some(v) = headers.get(ACK_HEADER).and_then(|v| v.to_str().ok()) { + out.push((ACK_HEADER, v.to_owned())); + } + // m11p8: carry the request-id across the forward hop for cross-node tracing. + if let Some(v) = headers.get(REQUEST_ID_HEADER).and_then(|v| v.to_str().ok()) { + out.push((REQUEST_ID_HEADER, v.to_owned())); + } + out } /// Forward a JSON request to one peer and relay its status + body back. diff --git a/tidal-server/src/cluster/join_boot.rs b/tidal-server/src/cluster/join_boot.rs index 9a952b6..28d9a36 100644 --- a/tidal-server/src/cluster/join_boot.rs +++ b/tidal-server/src/cluster/join_boot.rs @@ -765,10 +765,11 @@ fn clone_replication(r: &ReplicationSpec) -> ReplicationSpec { } } -const fn clone_wal(w: &WalSpec) -> WalSpec { +fn clone_wal(w: &WalSpec) -> WalSpec { WalSpec { batch_size: w.batch_size, batch_timeout_ms: w.batch_timeout_ms, + archive_dir: w.archive_dir.clone(), } } diff --git a/tidal-server/src/cluster/node.rs b/tidal-server/src/cluster/node.rs index de9f3a9..6337a17 100644 --- a/tidal-server/src/cluster/node.rs +++ b/tidal-server/src/cluster/node.rs @@ -179,6 +179,11 @@ const COMMIT_BRIDGE_WAKE_INTERVAL: Duration = Duration::from_secs(1); /// for tests via `TIDAL_REMOVE_DELIVERY_GRACE_MS`. const REMOVE_DELIVERY_GRACE_DEFAULT_MS: u64 = 30_000; +/// Election ticks (50 ms each) between self-driving heal passes (m11p8). ~3 s — +/// frequent enough to re-arm a stuck peer's backlog re-ship well within a +/// circuit-breaker reset window (30 s) without churning cursors every tick. +const SELF_HEAL_TICKS: u64 = 60; + /// The election driver's boot bundle: prepared in [`ShardReplica::new`] /// (where the durable classification runs), consumed by /// [`ShardReplica::start_election_driver`] once the node is in its @@ -365,6 +370,14 @@ pub struct ShardReplica { /// flag is set before spawning the duty and cleared when it finishes, so a /// fast-ticking leader never piles up promotion threads while one is in flight. promote_inflight: AtomicBool, + /// Self-driving heal (m11p8): coarse-cadence counter so the heal reconcile + /// pass runs roughly every `SELF_HEAL_TICKS` election ticks, not every 50ms. + heal_tick: AtomicU64, + /// Self-driving heal (m11p8): the set of peers this leader drove last pass + /// (breaker not closed AND trailing past the convergence threshold). Used to + /// count convergence transitions for `tidaldb_cluster_heal_successes_total` + /// and to clear the healing gauge on demotion. + healing: std::sync::Mutex>, /// The durable membership cache (`data_dir/membership`, m11p5 §3.6). The /// WAL-recovered `ClusterMembership` cell is the AUTHORITATIVE roster (it wins /// at open — the view is built from it); this cache exists only for the @@ -998,6 +1011,8 @@ impl ShardReplica { membership, join_hooks_cell, promote_inflight: AtomicBool::new(false), + heal_tick: AtomicU64::new(0), + healing: std::sync::Mutex::new(std::collections::HashSet::new()), membership_store: tidaldb::replication::MembershipStore::new(&data_dir_for_state), decommissioned_by_signal: AtomicBool::new(false), deferred_retires: std::sync::Mutex::new(Vec::new()), @@ -2211,6 +2226,90 @@ impl ShardReplica { } } + /// Self-driving heal (m11p8 §4): a STANDING LEADER DUTY re-armed every tick + /// at a coarse cadence (~`SELF_HEAL_TICKS` × the 50 ms election tick). It + /// closes incident §1.4-3 — "the breaker eats the first heal; re-issue until + /// lag 0" — by driving convergence itself instead of the operator. + /// + /// Each pass refreshes the per-peer breaker gauge, then for every peer that + /// is (a) NOT operator-partitioned, (b) has a non-closed ship breaker + /// (replication impaired), and (c) trails the leader's flushed frontier by + /// more than the convergence threshold, re-arms the backlog re-ship from the + /// peer's durable mark. The moment the breaker half-opens, the leader pushes + /// the WHOLE gap — no operator verb. A closed-breaker peer that is merely a + /// little behind self-heals through the normal ship path and is left alone + /// (no cursor churn). NON-BLOCKING (cursor updates + atomic gauge stores), so + /// it runs inline on the election tick rather than a detached thread. + /// + /// Operator-partitioned peers are intentionally ship-skipped (maintenance); + /// self-heal never auto-undoes a `/cluster/partition`. The manual + /// `/cluster/heal` verb still exists as an immediate nudge, but is no longer + /// required for convergence. + pub(crate) fn tick_self_heal(&self) { + // Coarse cadence: act ~every SELF_HEAL_TICKS election ticks. + if self.heal_tick.fetch_add(1, Ordering::Relaxed) % SELF_HEAL_TICKS != 0 { + return; + } + if !self.is_leader() { + // Off the leader there is no ship-driving duty: clear the healing + // gauge + tracked set so a demoted node stops reporting stale state. + self.cluster_metrics.set_healing_peers(0); + self.healing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clear(); + return; + } + + let flushed = self.ship_feed.flushed_seq(); + let partitioned = read_recovered(&self.partitioned, "partitioned").clone(); + let marks: std::collections::HashMap = + self.commit.peer_marks().into_iter().collect(); + + let mut behind_now: std::collections::HashSet = std::collections::HashSet::new(); + for shard in self.ship_queue.peers() { + // Refresh the breaker gauge for EVERY configured peer (including + // partitioned ones) so the dashboard is truthful. + let bstate = self.ship_queue.peer_breaker_state(shard); + self.cluster_metrics.set_peer_breaker_state(shard, bstate); + + // In-group shard id == region id, so the partition (ship-skip) set, + // keyed by region, is checked with the shard's numeric id. + if partitioned.contains(&RegionId(shard.0)) { + continue; + } + let mark = marks.get(&shard).copied().unwrap_or(0); + // Stuck = breaker not closed AND trailing past the convergence + // threshold. A closed breaker means the normal ship path is flowing. + if bstate != 0 && flushed.saturating_sub(mark) > self.learner_promote_lag { + behind_now.insert(shard); + self.cluster_metrics.incr_heal_attempts(); + self.ship_queue.resume_from(shard, mark); + } + } + + // Convergence transitions: peers driven last pass that are no longer + // behind recovered — count them and update the tracked set. + let mut prev = self + .healing + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for s in prev.iter() { + if !behind_now.contains(s) { + self.cluster_metrics.incr_heal_successes(); + } + } + let healing_count = behind_now.len() as u64; + *prev = behind_now; + drop(prev); + + if healing_count == 0 { + // Liveness heartbeat: the loop ran and found everything converged. + self.cluster_metrics.incr_heal_noops(); + } + self.cluster_metrics.set_healing_peers(healing_count); + } + /// The shared cluster metrics cell (election gauges live here too). pub(crate) const fn cluster_metrics( &self, @@ -2794,10 +2893,21 @@ impl ShardReplica { } else { 0 }; - let applied_events = db - .replication_state() - .applied_seqno(leader_shard) - .unwrap_or(0); + // The leader does NOT apply its own stream through the receiver — it + // writes its WAL directly — so `applied_seqno(own_shard)` never advances + // on a leader and would read a stale follower-era value (0 on a node + // elected without first following). The leader's true applied frontier + // IS its durable flushed WAL frontier (it is, by construction, caught up + // to its own log). Followers read how far they have applied the CURRENT + // leader's per-source-shard stream (BUG 1: keyed by the live leader's + // shard, never a stale single scalar — see below). + let applied_events = if is_leader { + last_seq + } else { + db.replication_state() + .applied_seqno(leader_shard) + .unwrap_or(0) + }; // lag = the CURRENT leader's per-source-shard high-water-mark − applied // for that same shard (BUG 1). The gauge tracks the leader HWM PER SOURCE // SHARD, so a `/cluster/promote` that moves leadership to a different shard @@ -2880,6 +2990,7 @@ impl ShardReplica { membership_version: self.membership_version(), membership_term: self.membership.term(), membership_role: self.role_in_roster().to_string(), + version: node_build_version(), // Populated by `ClusterNode::status_local` (it owns the group set); // a bare per-replica status carries only its own row implicitly. shards: Vec::new(), @@ -2887,6 +2998,13 @@ impl ShardReplica { } } +/// This node's build version + hash for status/observability (m11p8): +/// `"+"` (e.g. `"0.1.0+dev"`). All workspace crates +/// share the Cargo version, so the server version is the binary version. +fn node_build_version() -> String { + format!("{}+{}", env!("CARGO_PKG_VERSION"), tidaldb::BUILD_HASH) +} + impl Drop for ShardReplica { fn drop(&mut self) { self.shutdown(); @@ -3041,6 +3159,12 @@ fn open_region_db( if let Some(timeout_ms) = topology.wal.batch_timeout_ms { builder = builder.wal_batch_timeout(Duration::from_millis(timeout_ms)); } + // m11p8 PITR: archive sealed WAL segments before compaction deletes them. + // Segment filenames encode the shard id, so co-located groups share one + // archive dir without collision. + if let Some(ref archive_dir) = topology.wal.archive_dir { + builder = builder.wal_archive_dir(archive_dir); + } // m11p6: at most ONE hosted shard per node binds the engine's `/metrics` // server (N TidalDb instances would otherwise fight for one `metrics_addr`). // ClusterNode designates the metrics-owning shard; the others open without. @@ -3373,6 +3497,10 @@ pub struct ClusterNode { tls_files: Option, /// m11p7 admin-verb audit sink (tracing + optional JSONL file). audit: crate::cluster::audit::AuditSink, + /// m11p8: the metrics-owner shard's cluster-metrics handle — the cell this + /// node's `/metrics` listener renders. Node-level gateway events (cross-shard + /// write forwards) increment it so they surface on the single per-node scrape. + cluster_metrics: Arc, /// Flipped on shutdown so `/health` reports not-ready while draining. shutting_down: AtomicBool, } @@ -3427,7 +3555,7 @@ impl ClusterNode { let creds = Arc::new(crate::cluster::security::ClusterCreds::from_env()); let mut groups: BTreeMap> = BTreeMap::new(); - let mut metrics_owner_assigned = false; + let mut metrics_owner: Option = None; for group in &resolved { if !group.replicas.iter().any(|r| r.region == region) { continue; // this node does not host this group @@ -3445,7 +3573,10 @@ impl ClusterNode { }; // At most ONE hosted group binds the engine's `/metrics` server // (N TidalDb instances would otherwise fight for one `metrics_addr`). - let enable_metrics = !metrics_owner_assigned; + // The other co-located groups register their cluster series with the + // owner below (m11p8) so the single per-node listener still exposes + // every group's replication metrics, `shard`-labeled. + let enable_metrics = metrics_owner.is_none(); let replica = ShardReplica::new( topology, region_name, @@ -3458,7 +3589,7 @@ impl ClusterNode { Arc::clone(&creds), )?; if enable_metrics { - metrics_owner_assigned = true; + metrics_owner = Some(group.shard); } groups.insert(group.shard, Arc::new(replica)); } @@ -3467,6 +3598,24 @@ impl ClusterNode { "region '{region_name}' is not a replica of any shard group" ))); } + // m11p8: when multiple shard groups co-locate on this node, the metrics + // owner exposes the siblings' `tidaldb_cluster_*` series under their own + // `shard="N"` label so one `/metrics` scrape covers every hosted group. + // No-op on the S=1 topology (a single hosted group is the owner). + if groups.len() > 1 + && let Some(owner_shard) = metrics_owner + && let Some(owner_replica) = groups.get(&owner_shard) + && let Ok(owner_db) = owner_replica.db() + { + for (shard, replica) in &groups { + if *shard == owner_shard { + continue; + } + if let Ok(sib_db) = replica.db() { + owner_db.register_metrics_sibling(shard.0, sib_db); + } + } + } let router = if single { ShardRouter::single() @@ -3523,9 +3672,19 @@ impl ClusterNode { hosted_groups = groups.len(), total_groups = placement.len(), tls = tls_files.is_some(), + version = %node_build_version(), "cluster node started (m11p6: one replica per hosted shard group)" ); + // m11p8: the metrics-owner's cluster-metrics handle (the cell `/metrics` + // renders) for node-level gateway events. Falls back to the lowest-shard + // group if no owner was flagged; `groups` is non-empty (checked above). + let cluster_metrics = metrics_owner + .and_then(|s| groups.get(&s)) + .or_else(|| groups.values().next()) + .map(|r| Arc::clone(&r.cluster_metrics)) + .expect("groups is non-empty"); + Ok(Self { placement, groups, @@ -3535,6 +3694,7 @@ impl ClusterNode { creds, tls_files, audit: crate::cluster::audit::AuditSink::from_env(), + cluster_metrics, shutting_down: AtomicBool::new(false), }) } @@ -3741,6 +3901,7 @@ impl ClusterNode { } let auth = forwarded_auth(headers); let passthrough = forward::ack_passthrough(headers); + self.cluster_metrics.incr_forwards(); let mut last_err = String::new(); for http in candidates { let url = peer_url(http, path); @@ -3762,6 +3923,9 @@ impl ClusterNode { Err(e) => last_err = format!("{url}: {e}"), } } + // Every candidate replica was unreachable — the forward could not be + // delivered to the group at all (m11p8 forward-failure signal). + self.cluster_metrics.incr_forward_failures(); Err(ClusterAppError(ServerError::Unavailable(format!( "shard {} unreachable: all {} replica candidates failed (last {last_err})", shard.0, @@ -3910,7 +4074,10 @@ pub fn build_region_router( .layer(ConcurrencyLimitLayer::new(crate::router::MAX_CONCURRENCY)), ); - public.merge(protected) + // m11p8: assign/echo `x-request-id` and open a per-request span on the + // multi-process region router too. Forwarded writes carry the originating + // gateway's id across the leader hop (see `cluster::forward`). + crate::router::with_request_id_tracing(public.merge(protected)) } // ── Health ────────────────────────────────────────────────────────────────── @@ -4050,6 +4217,15 @@ pub struct LocalStatusResponse { /// or `removed`. Distinct from `role` (the election role); a learner is a /// `follower` here. membership_role: String, + /// This node's build version + hash (m11p8 rolling-upgrade visibility): + /// `"+"`. The aggregating gateway's status + /// fan-out collects every node's version, so an operator can confirm the + /// cluster is within the supported N/N+1 skew before/during a rolling + /// upgrade without shelling into each pod. `#[serde(default)]` so the + /// gateway can still deserialize a pre-m11p8 peer's status (empty version) + /// during a mixed-version upgrade window — the exact compat this phase needs. + #[serde(default)] + version: String, /// Per-shard-group status for every group THIS node hosts (m11p6). For the /// legacy single group this is one row mirroring the flat fields above; with /// sharding it carries one row per hosted group, so an operator (and the @@ -4148,6 +4324,12 @@ pub struct AggregatedRegionStatus { /// the per-peer budget. An unreachable region reports `applied 0`, `lag = /// leader_last_seq`, `partitioned: true`. reachable: bool, + /// The region's reported build version (m11p8): `/cluster/status` is the + /// single pane an operator reads to confirm the whole cluster is within the + /// supported N/N+1 skew before a rolling upgrade. Empty for an unreachable + /// region or a pre-m11p8 peer. + #[serde(default)] + version: String, } /// Aggregated replication status across EVERY region of the default shard group. @@ -4256,6 +4438,11 @@ pub async fn cluster_status( // local status carries that set; mirror it. a.iter().any(|v| v.as_str() == Some(name.as_str())) }); + let version = j + .get("version") + .and_then(serde_json::Value::as_str) + .unwrap_or("") + .to_owned(); AggregatedRegionStatus { name, applied_events: applied, @@ -4266,6 +4453,7 @@ pub async fn cluster_status( // the leader-set merge fix it. partitioned, reachable: true, + version, } } None => AggregatedRegionStatus { @@ -4274,6 +4462,7 @@ pub async fn cluster_status( lag_events: leader_last_seq, partitioned: true, reachable: false, + version: String::new(), }, }) .collect::>(); @@ -5549,6 +5738,7 @@ async fn forward_write( // sibling (this forward sets the internal marker). let mut passthrough = forward::ack_passthrough(headers); passthrough.extend(state.node_token_passthrough()); + state.cluster_metrics.incr_forwards(); match forward_json_with_headers( &state.client, &url, @@ -5566,6 +5756,7 @@ async fn forward_write( Err(e) => { // Leader unreachable: the typed 503 names the leader, its address, // and the connect error (single body shape via ClusterAppError). + state.cluster_metrics.incr_forward_failures(); let leader = state.leader_name(); tracing::warn!(%leader, %url, error = %e, "forward to leader failed; leader unreachable"); Err(ClusterAppError(ServerError::LeaderUnreachable { diff --git a/tidal-server/src/cluster/routes.rs b/tidal-server/src/cluster/routes.rs index 14b0734..b00c1ee 100644 --- a/tidal-server/src/cluster/routes.rs +++ b/tidal-server/src/cluster/routes.rs @@ -119,7 +119,11 @@ pub fn build_cluster_router( .layer(ConcurrencyLimitLayer::new(crate::router::MAX_CONCURRENCY)), ); - public.merge(protected) + // m11p8: the same request-id + tracing stack as the standalone router — + // assigns/echoes `x-request-id` and opens a per-request span. The cluster + // routers previously skipped it; with it, a write forwarded to the leader + // carries the originating gateway's id (see `cluster::forward`). + crate::router::with_request_id_tracing(public.merge(protected)) } // ── Health ────────────────────────────────────────────────────────────────── diff --git a/tidal-server/src/cluster/topology.rs b/tidal-server/src/cluster/topology.rs index 8e7aeb7..49e9034 100644 --- a/tidal-server/src/cluster/topology.rs +++ b/tidal-server/src/cluster/topology.rs @@ -263,6 +263,13 @@ pub struct WalSpec { /// Max milliseconds a partial batch waits before flushing. #[serde(default)] pub batch_timeout_ms: Option, + /// Optional WAL archive directory for point-in-time recovery (m11p8). When + /// set, each hosted shard group copies its sealed segments here before + /// compaction deletes them (segment filenames encode the shard id, so + /// co-located groups can share one archive dir without collision). Put it on + /// durable storage separate from the live data dir. + #[serde(default)] + pub archive_dir: Option, } /// Operation-timeout overrides (the optional `timeouts:` YAML block). diff --git a/tidal-server/src/router.rs b/tidal-server/src/router.rs index 067ef07..a6e8c63 100644 --- a/tidal-server/src/router.rs +++ b/tidal-server/src/router.rs @@ -154,7 +154,21 @@ pub fn build_router( // SetRequestId must be outermost so the ID is in headers when TraceLayer // creates its span. In ServiceBuilder the first .layer() is outermost. - public.merge(protected).layer( + with_request_id_tracing(public.merge(protected)) +} + +/// Wrap `router` with the shared request-id + tracing layer stack (m11p8): +/// `SetRequestIdLayer` (outermost) assigns a sequential `x-request-id` when one +/// is absent, `PropagateRequestIdLayer` echoes it into the response, and +/// `TraceLayer` opens a per-request span carrying the id. Applied to the +/// standalone router AND both cluster routers (single-process and multi-process) +/// so every HTTP surface correlates by `x-request-id`. +/// +/// Because `SetRequestId` is a no-op when the header is already present, an +/// `x-request-id` forwarded from a gateway (see `cluster::forward`) survives the +/// hop: the leader's span shares the originating gateway's id. +pub(crate) fn with_request_id_tracing(router: Router) -> Router { + router.layer( ServiceBuilder::new() .layer(SetRequestIdLayer::x_request_id( SequentialRequestId::default(), diff --git a/tidal/benches/ranking.rs b/tidal/benches/ranking.rs index 36b0b5d..14668b9 100644 --- a/tidal/benches/ranking.rs +++ b/tidal/benches/ranking.rs @@ -93,10 +93,96 @@ fn bench_score_200_full_pipeline(c: &mut Criterion) { }); } +// ── T2: signal-value pre-pass (one `entries.get()` per signal type) ────────── +// +// `for_you` reads `view` twice per candidate at *different* aggregations +// (`Sort::Hot` reads `view` Value; a boost reads `view` DecayScore), so the +// pre-pass collapses two `DashMap` gets to one. The win is a shard-lock saved, +// so it is small single-threaded and grows under concurrent signal-write +// contention — hence the `_under_writes` variants. `with_signal_plan(false)` +// scores the same inputs through the direct per-term path for a true A/B. + +fn for_you_profile() -> tidaldb::ranking::profile::RankingProfile { + let mut registry = ProfileRegistry::new(); + register_builtins(&mut registry).unwrap(); + registry.get("for_you").unwrap().clone() +} + +fn bench_for_you_plan(c: &mut Criterion) { + let ledger = make_ledger_with_200_items(); + let profile = for_you_profile(); + let candidates: Vec = (1..=200).map(EntityId::new).collect(); + let now = Timestamp::from_nanos(1_708_000_000_000_000_000u64); + + for plan in [true, false] { + let executor = ProfileExecutor::new(&ledger).with_signal_plan(plan); + let id = if plan { + "for_you_plan" + } else { + "for_you_no_plan" + }; + c.bench_function(id, |b| { + b.iter(|| executor.score(black_box(&candidates), black_box(&profile), black_box(now))); + }); + } +} + +#[allow(clippy::cast_precision_loss)] +fn bench_for_you_under_writes(c: &mut Criterion) { + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }; + + let profile = for_you_profile(); + let candidates: Vec = (1..=200).map(EntityId::new).collect(); + let now = Timestamp::from_nanos(1_708_000_000_000_000_000u64); + + let mut group = c.benchmark_group("for_you_under_writes"); + group.measurement_time(Duration::from_secs(8)); + + for plan in [true, false] { + let ledger = Arc::new(make_ledger_with_200_items()); + let stop = Arc::new(AtomicBool::new(false)); + // Four writers continuously record `view`/`like` signals, contending the + // same shards the scorer reads — the scenario where collapsing gets pays. + let writers: Vec<_> = (0..4u64) + .map(|w| { + let ledger = Arc::clone(&ledger); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + let mut i = w; + while !stop.load(Ordering::Relaxed) { + let eid = EntityId::new((i % 200) + 1); + let ts = Timestamp::from_nanos(1_708_000_000_000_000_000u64 + i * 1000); + let _ = ledger.record_signal("view", eid, 1.0, ts); + let _ = ledger.record_signal("like", eid, 1.0, ts); + i += 4; + } + }) + }) + .collect(); + + let executor = ProfileExecutor::new(&ledger).with_signal_plan(plan); + let id = if plan { "plan" } else { "no_plan" }; + group.bench_function(id, |b| { + b.iter(|| executor.score(black_box(&candidates), black_box(&profile), black_box(now))); + }); + + stop.store(true, Ordering::Relaxed); + for w in writers { + w.join().unwrap(); + } + } + group.finish(); +} + criterion_group!( benches, bench_score_200_trending, bench_score_200_hot, - bench_score_200_full_pipeline + bench_score_200_full_pipeline, + bench_for_you_plan, + bench_for_you_under_writes ); criterion_main!(benches); diff --git a/tidal/src/db/builder.rs b/tidal/src/db/builder.rs index 569638b..126ce72 100644 --- a/tidal/src/db/builder.rs +++ b/tidal/src/db/builder.rs @@ -379,6 +379,19 @@ impl TidalDbBuilder { self } + /// Enable WAL archival for point-in-time recovery (m11p8): every sealed WAL + /// segment is copied to `dir` before the periodic online compaction deletes + /// it, building a gap-free, self-describing archive (filenames encode shard + + /// first-seq). Off by default (no archival, no overhead). `dir` should be on + /// separate, durable storage from the live WAL (an object-store mount, an + /// archive volume) so a disk loss of the live data dir does not also lose the + /// archive. + #[must_use] + pub fn wal_archive_dir(mut self, dir: impl Into) -> Self { + self.config.wal_archive_dir = Some(dir.into()); + self + } + /// Resolve default directory paths using [`Paths`] for persistent mode. /// /// When a `data_dir` is set and `wal_dir` or `cache_dir` are not diff --git a/tidal/src/db/config.rs b/tidal/src/db/config.rs index b6bf2eb..3e936a6 100644 --- a/tidal/src/db/config.rs +++ b/tidal/src/db/config.rs @@ -71,6 +71,12 @@ pub struct Config { /// measured fsync cost: a longer window amortizes fsyncs better under /// sustained load but adds latency to sparse writes (m11p1). pub wal_batch_timeout: Option, + /// Optional WAL archive directory for point-in-time recovery (m11p8). When + /// set, the periodic online-compaction path copies each sealed WAL segment + /// here BEFORE deleting it, building a gap-free, self-describing archive + /// (filenames encode shard + first-seq) an operator can replay to a chosen + /// point in time. `None` (the default) = no archival, no overhead. + pub wal_archive_dir: Option, } impl Default for Config { @@ -84,6 +90,7 @@ impl Default for Config { hlc_offset_ms: 0, wal_batch_size: None, wal_batch_timeout: None, + wal_archive_dir: None, } } } diff --git a/tidal/src/db/metrics/cluster.rs b/tidal/src/db/metrics/cluster.rs index 8d4466f..f25dc75 100644 --- a/tidal/src/db/metrics/cluster.rs +++ b/tidal/src/db/metrics/cluster.rs @@ -38,6 +38,9 @@ struct PeerShipMetrics { failures_total: AtomicU64, acked_seqno: AtomicU64, queue_depth: AtomicU64, + /// Per-peer circuit-breaker state (m11p8): 0 closed, 1 open, 2 half-open. + /// Refreshed by the self-driving heal loop, which polls each peer's breaker. + breaker_state: AtomicU64, } impl PeerShipMetrics { @@ -49,6 +52,7 @@ impl PeerShipMetrics { failures_total: AtomicU64::new(0), acked_seqno: AtomicU64::new(0), queue_depth: AtomicU64::new(0), + breaker_state: AtomicU64::new(0), } } } @@ -121,6 +125,31 @@ pub struct ClusterMetrics { /// its decommission and will learn of its removal via the typed removed /// signal if it ever restarts. remove_delivery_giveups_total: AtomicU64, + /// Total cross-node write forwards this node has INITIATED as a gateway + /// (m11p8) — a write that arrived on a non-leader and was relayed to the + /// shard leader over HTTP. High forward rate at one gateway = client routing + /// imbalance, not a fault. + forwards_total: AtomicU64, + /// Total cross-node write forwards that ERRORED (m11p8) — the leader was + /// unreachable, returned 5xx, or the relay timed out. A non-zero rate means + /// a gateway cannot reach the current leader (election in flight, partition). + forward_failures_total: AtomicU64, + /// Total per-peer circuit-breaker OPEN transitions observed across all peers + /// (m11p8). Each open is a `reset_duration` ship stall to that peer; a rising + /// counter under steady load means a peer link is flapping. + breaker_opens_total: AtomicU64, + /// Self-driving heal (m11p8): total reconcile attempts the server-side + /// heal loop has fired (one per peer per pass while a peer is behind). + heal_attempts_total: AtomicU64, + /// Self-driving heal (m11p8): total heal attempts that resumed a lagging + /// peer's ship (the breaker admitted the probe and a catch-up was nudged). + heal_successes_total: AtomicU64, + /// Self-driving heal (m11p8): total heal attempts that were no-ops because + /// the peer was already converged (the steady-state of the loop). + heal_noops_total: AtomicU64, + /// Self-driving heal (m11p8): peers this node is currently driving back to + /// convergence (partitioned-or-lagging set size). 0 = fully converged. + healing_peers: AtomicU64, } impl ClusterMetrics { @@ -147,6 +176,13 @@ impl ClusterMetrics { snapshot_fetches_total: AtomicU64::new(0), snapshot_pin_force_drops_total: AtomicU64::new(0), remove_delivery_giveups_total: AtomicU64::new(0), + forwards_total: AtomicU64::new(0), + forward_failures_total: AtomicU64::new(0), + breaker_opens_total: AtomicU64::new(0), + heal_attempts_total: AtomicU64::new(0), + heal_successes_total: AtomicU64::new(0), + heal_noops_total: AtomicU64::new(0), + healing_peers: AtomicU64::new(0), } } @@ -176,6 +212,48 @@ impl ClusterMetrics { .fetch_add(1, Ordering::Relaxed); } + /// Count one cross-node write forward initiated by this gateway (m11p8). + pub fn incr_forwards(&self) { + self.forwards_total.fetch_add(1, Ordering::Relaxed); + } + + /// Count one cross-node write forward that errored (m11p8). + pub fn incr_forward_failures(&self) { + self.forward_failures_total.fetch_add(1, Ordering::Relaxed); + } + + /// Set a peer's circuit-breaker state gauge (m11p8): 0 closed, 1 open, + /// 2 half-open. A closed/half-open → open transition also bumps the global + /// `breaker_opens_total` counter so a dashboard can `rate()` the flap. + pub fn set_peer_breaker_state(&self, peer: ShardId, state: u8) { + let cell = self.peer(peer); + let prev = cell.breaker_state.swap(u64::from(state), Ordering::Relaxed); + if state == 1 && prev != 1 { + self.breaker_opens_total.fetch_add(1, Ordering::Relaxed); + } + } + + /// Count one self-driving heal reconcile attempt (m11p8). + pub fn incr_heal_attempts(&self) { + self.heal_attempts_total.fetch_add(1, Ordering::Relaxed); + } + + /// Count one heal attempt that resumed a lagging peer's ship (m11p8). + pub fn incr_heal_successes(&self) { + self.heal_successes_total.fetch_add(1, Ordering::Relaxed); + } + + /// Count one heal attempt that was a no-op (peer already converged; m11p8). + pub fn incr_heal_noops(&self) { + self.heal_noops_total.fetch_add(1, Ordering::Relaxed); + } + + /// Set the count of peers this node is currently driving to convergence + /// (m11p8). 0 = fully converged. + pub fn set_healing_peers(&self, count: u64) { + self.healing_peers.store(count, Ordering::Relaxed); + } + /// Record this node's election term + role (m11p4): role is 0 follower, /// 1 pre-candidate, 2 candidate, 3 leader. pub fn set_election_view(&self, term: u64, role: u64) { @@ -307,142 +385,241 @@ impl ClusterMetrics { } /// Append the `tidaldb_cluster_*` series to a Prometheus exposition body. + /// + /// This is the single-shard / metrics-owner render: byte-identical to the + /// pre-m11p8 output so existing dashboards and the standalone surface are + /// unchanged. + pub(crate) fn render_into(&self, out: &mut String, partition_id: u64) { + self.render_labeled(out, partition_id, None); + } + + /// Render this shard group's series with a `shard="N"` label on every line + /// (m11p8). Used when several shard groups co-locate on one node and share + /// its single `/metrics` listener: the metrics-owner renders unlabeled via + /// [`render_into`](Self::render_into) and each co-located sibling renders + /// here, so their series never collide on the shared scrape target. + pub(crate) fn render_into_sibling(&self, out: &mut String, partition_id: u64, shard: u16) { + self.render_labeled(out, partition_id, Some(shard)); + } + // One linear emit per series; splitting it would scatter the series list. #[allow(clippy::cast_precision_loss)] // monitoring gauges #[allow(clippy::too_many_lines)] - pub(crate) fn render_into(&self, out: &mut String, partition_id: u64) { + fn render_labeled(&self, out: &mut String, partition_id: u64, shard: Option) { use std::fmt::Write; - out.push_str(&self.ship_rtt.render_prometheus( + // Histogram / scalar label contents (no braces); empty for the unlabeled + // owner so its output stays byte-identical to the pre-m11p8 surface. + let extra = shard.map_or(String::new(), |s| format!("shard=\"{s}\"")); + + out.push_str(&self.ship_rtt.render_prometheus_labeled( "tidaldb_cluster_ship_rtt_us", "Replication batch ship round-trip time in microseconds (all peers)", + &extra, )); - out.push_str(&self.ship_batch_events.render_prometheus( + out.push_str(&self.ship_batch_events.render_prometheus_labeled( "tidaldb_cluster_ship_batch_events", "Events per shipped replication batch", + &extra, )); - out.push_str(&self.wal_fsync.render_prometheus( + out.push_str(&self.wal_fsync.render_prometheus_labeled( "tidaldb_cluster_wal_fsync_us", "WAL group-commit fsync latency in microseconds", + &extra, )); - out.push_str(&self.group_commit_events.render_prometheus( + out.push_str(&self.group_commit_events.render_prometheus_labeled( "tidaldb_cluster_group_commit_events", "Events per WAL group-commit batch", + &extra, )); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_write_pool_depth", "Queued cluster write jobs awaiting a pool worker", "gauge", self.write_pool_depth.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_write_pool_rejections_total", "Cluster write submissions rejected with backpressure (HTTP 429)", "counter", self.write_pool_rejections_total.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_relay_last_seq", "Leader relay stream high-water mark (last committed seqno)", "gauge", self.relay_last_seq.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_relay_durable_seq", "Quorum commit index: highest seqno a majority of the replica set durably holds (m11p3)", "gauge", self.relay_durable_seq.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_quorum_timeouts_total", "ack=quorum writes that timed out awaiting the commit index (retryable 503s)", "counter", self.quorum_timeouts_total.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( + out, + "tidaldb_cluster_forwards_total", + "Cross-node write forwards this gateway initiated (m11p8)", + "counter", + self.forwards_total.load(Ordering::Relaxed) as f64, + &extra, + ); + emit_scalar( + out, + "tidaldb_cluster_forward_failures_total", + "Cross-node write forwards that errored — leader unreachable / 5xx / timeout (m11p8)", + "counter", + self.forward_failures_total.load(Ordering::Relaxed) as f64, + &extra, + ); + + emit_scalar( out, "tidaldb_cluster_election_term", "This node's current election term (m11p4; 0 = topology era)", "gauge", self.election_term.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_election_role", "Election role: 0 follower, 1 pre-candidate, 2 candidate, 3 leader", "gauge", self.election_role.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_elections_started_total", "Elections (pre-vote rounds) this node has started", "counter", self.elections_started_total.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_leader_changes_total", "Leadership changes this node has observed", "counter", self.leader_changes_total.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_divergence_quarantined", "Divergent-suffix quarantine latch (1 = fenced from the data plane)", "gauge", self.divergence_quarantined.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_reseed_required", "Durable reseed-marker latch (m11p5; 1 = a snapshot reseed is pending the next boot)", "gauge", self.reseed_required.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( + out, + "tidaldb_cluster_breaker_opens_total", + "Per-peer circuit-breaker open transitions observed (m11p8; each is a ship stall)", + "counter", + self.breaker_opens_total.load(Ordering::Relaxed) as f64, + &extra, + ); + emit_scalar( + out, + "tidaldb_cluster_heal_attempts_total", + "Self-driving heal reconcile attempts (m11p8)", + "counter", + self.heal_attempts_total.load(Ordering::Relaxed) as f64, + &extra, + ); + emit_scalar( + out, + "tidaldb_cluster_heal_successes_total", + "Self-driving heal attempts that resumed a lagging peer's ship (m11p8)", + "counter", + self.heal_successes_total.load(Ordering::Relaxed) as f64, + &extra, + ); + emit_scalar( + out, + "tidaldb_cluster_heal_noops_total", + "Self-driving heal attempts that were no-ops — peer already converged (m11p8)", + "counter", + self.heal_noops_total.load(Ordering::Relaxed) as f64, + &extra, + ); + emit_scalar( + out, + "tidaldb_cluster_healing_peers", + "Peers this node is currently driving back to convergence (m11p8; 0 = converged)", + "gauge", + self.healing_peers.load(Ordering::Relaxed) as f64, + &extra, + ); + + emit_scalar( out, "tidaldb_cluster_snapshot_staged", "Seq of the currently-staged snapshot artifact (m11p5; 0 = none staged)", "gauge", self.snapshot_staged.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_snapshot_fetches_total", "FetchSnapshot streams served as the leader-side snapshot source (m11p5)", "counter", self.snapshot_fetches_total.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_snapshot_pin_force_drops_total", "Staged-artifact retention pins force-dropped past the hard cap (m11p5 §2.1; \ a dead joiner that never released)", "counter", self.snapshot_pin_force_drops_total.load(Ordering::Relaxed) as f64, + &extra, ); - super::write_metric_line( + emit_scalar( out, "tidaldb_cluster_remove_delivery_giveups_total", "Removal-delivery graces that expired before the removed peer acked the Removed \ record (m11p5 §3.3; the removed node was down/unreachable during decommission)", "counter", self.remove_delivery_giveups_total.load(Ordering::Relaxed) as f64, + &extra, ); - // Per-peer series, labeled by peer shard id + this node's partition. - // Snapshot the cells under the read lock, then render lock-free (the - // ship sender threads update these cells on their hot path). + // Per-peer series, labeled by peer shard id + this node's partition (and + // this group's shard when co-located). Snapshot the cells under the read + // lock, then render lock-free (the ship sender threads update these cells + // on their hot path). let peer_cells: Vec<(u16, Arc)> = { let peers = self .peers @@ -453,13 +630,15 @@ impl ClusterMetrics { if peer_cells.is_empty() { return; } + let shard_prefix = shard.map_or(String::new(), |s| format!("shard=\"{s}\",")); let _ = writeln!( out, "\n# HELP tidaldb_cluster_peer_ship Per-peer replication ship-path series.\n\ # TYPE tidaldb_cluster_peer_acked_seqno gauge" ); for (peer_id, cell) in &peer_cells { - let labels = format!("peer_shard=\"{peer_id}\",partition_id=\"{partition_id}\""); + let labels = + format!("{shard_prefix}peer_shard=\"{peer_id}\",partition_id=\"{partition_id}\""); let _ = writeln!( out, "tidaldb_cluster_peer_acked_seqno{{{labels}}} {}", @@ -485,11 +664,32 @@ impl ClusterMetrics { "tidaldb_cluster_peer_ship_failures_total{{{labels}}} {}", cell.failures_total.load(Ordering::Relaxed) ); + let _ = writeln!( + out, + "tidaldb_cluster_peer_breaker_state{{{labels}}} {}", + cell.breaker_state.load(Ordering::Relaxed) + ); } out.push('\n'); } } +/// Emit one scalar (`gauge`/`counter`) line, optionally stamped with an extra +/// label set (`extra`, no braces). Empty `extra` defers to the unlabeled +/// [`write_metric_line`](super::write_metric_line) so the metrics-owner / single +/// shard output is byte-identical to the pre-m11p8 surface. +fn emit_scalar(out: &mut String, name: &str, help: &str, type_str: &str, value: f64, extra: &str) { + if extra.is_empty() { + super::write_metric_line(out, name, help, type_str, value); + } else { + use std::fmt::Write; + let _ = write!( + out, + "\n# HELP {name} {help}\n# TYPE {name} {type_str}\n{name}{{{extra}}} {value}\n" + ); + } +} + #[cfg(test)] mod tests { use super::*; @@ -510,6 +710,15 @@ mod tests { m.incr_snapshot_fetches(); m.incr_snapshot_pin_force_drops(); m.incr_remove_delivery_giveup(); + // m11p8 additions. + m.incr_forwards(); + m.incr_forwards(); + m.incr_forward_failures(); + m.set_peer_breaker_state(ShardId(2), 1); // open + m.incr_heal_attempts(); + m.incr_heal_successes(); + m.incr_heal_noops(); + m.set_healing_peers(1); let mut out = String::new(); m.render_into(&mut out, 7); @@ -520,6 +729,14 @@ mod tests { assert!(out.contains("tidaldb_cluster_snapshot_fetches_total 1")); assert!(out.contains("tidaldb_cluster_snapshot_pin_force_drops_total 1")); assert!(out.contains("tidaldb_cluster_remove_delivery_giveups_total 1")); + // m11p8 series. + assert!(out.contains("tidaldb_cluster_forwards_total 2")); + assert!(out.contains("tidaldb_cluster_forward_failures_total 1")); + assert!(out.contains("tidaldb_cluster_breaker_opens_total 1")); + assert!(out.contains("tidaldb_cluster_heal_attempts_total 1")); + assert!(out.contains("tidaldb_cluster_heal_successes_total 1")); + assert!(out.contains("tidaldb_cluster_heal_noops_total 1")); + assert!(out.contains("tidaldb_cluster_healing_peers 1")); assert!( out.contains( "tidaldb_cluster_peer_acked_seqno{peer_shard=\"2\",partition_id=\"7\"} 64" @@ -531,7 +748,39 @@ mod tests { assert!(out.contains( "tidaldb_cluster_peer_ship_failures_total{peer_shard=\"2\",partition_id=\"7\"} 1" )); + assert!( + out.contains( + "tidaldb_cluster_peer_breaker_state{peer_shard=\"2\",partition_id=\"7\"} 1" + ) + ); assert!(out.contains("tidaldb_cluster_write_pool_depth")); assert!(out.contains("tidaldb_cluster_write_pool_rejections_total")); } + + /// Co-located sibling shards render every series with a `shard="N"` label so + /// they never collide with the metrics-owner's unlabeled series on one + /// `/metrics` listener (m11p8 multi-shard listener). + #[test] + fn sibling_render_stamps_shard_label() { + let m = ClusterMetrics::new(); + m.mark_active(); + m.observe_ship(ShardId(5), std::time::Duration::from_millis(2), 32, 30, 40); + m.set_relay_frontiers(40, 38); + m.set_peer_breaker_state(ShardId(5), 2); // half-open + + let mut out = String::new(); + m.render_into_sibling(&mut out, 3, 1); + // Scalar gauges carry the shard label. + assert!(out.contains("tidaldb_cluster_relay_last_seq{shard=\"1\"} 40")); + assert!(out.contains("tidaldb_cluster_relay_durable_seq{shard=\"1\"} 38")); + // Histograms carry the shard label on every series line. + assert!(out.contains("tidaldb_cluster_ship_rtt_us_bucket{le=\"+Inf\",shard=\"1\"}")); + assert!(out.contains("tidaldb_cluster_ship_rtt_us_count{shard=\"1\"}")); + // Per-peer series carry shard + peer_shard + partition_id. + assert!(out.contains( + "tidaldb_cluster_peer_breaker_state{shard=\"1\",peer_shard=\"5\",partition_id=\"3\"} 2" + )); + // The owner's unlabeled form must NOT appear in a sibling render. + assert!(!out.contains("tidaldb_cluster_relay_last_seq 40")); + } } diff --git a/tidal/src/db/metrics/histogram.rs b/tidal/src/db/metrics/histogram.rs index a78cc80..c2f8f7f 100644 --- a/tidal/src/db/metrics/histogram.rs +++ b/tidal/src/db/metrics/histogram.rs @@ -85,22 +85,46 @@ impl LatencyHistogram { /// Produces `# HELP`, `# TYPE histogram`, per-bucket lines with `le` labels, /// a `+Inf` bucket (equal to total count), `_sum`, and `_count`. pub fn render_prometheus(&self, name: &str, help: &str) -> String { + self.render_prometheus_labeled(name, help, "") + } + + /// Render this histogram with an extra label set (e.g. `shard="1"`) on every + /// series line so co-located shards on one `/metrics` listener stay distinct. + /// + /// `extra` is the label *contents* without braces (`shard="1"`); an empty + /// string renders the unlabeled form, byte-identical to + /// [`render_prometheus`](Self::render_prometheus). + pub fn render_prometheus_labeled(&self, name: &str, help: &str, extra: &str) -> String { use std::fmt::Write; + // Bucket lines always carry an `le` label; `_sum`/`_count` carry the extra + // set in braces (or nothing when `extra` is empty). + let (bucket_sep, scalar_labels) = if extra.is_empty() { + ("", String::new()) + } else { + (",", format!("{{{extra}}}")) + }; + let mut out = String::new(); let _ = writeln!(out, "# HELP {name} {help}"); let _ = writeln!(out, "# TYPE {name} histogram"); for (i, &bound) in self.bounds.iter().enumerate() { let count = self.buckets[i].load(Ordering::Relaxed); - let _ = writeln!(out, "{name}_bucket{{le=\"{bound}\"}} {count}"); + let _ = writeln!( + out, + "{name}_bucket{{le=\"{bound}\"{bucket_sep}{extra}}} {count}" + ); } let total_count = self.count.load(Ordering::Relaxed); let total_sum = self.sum.load(Ordering::Relaxed); - let _ = writeln!(out, "{name}_bucket{{le=\"+Inf\"}} {total_count}"); - let _ = writeln!(out, "{name}_sum {total_sum}"); - let _ = writeln!(out, "{name}_count {total_count}"); + let _ = writeln!( + out, + "{name}_bucket{{le=\"+Inf\"{bucket_sep}{extra}}} {total_count}" + ); + let _ = writeln!(out, "{name}_sum{scalar_labels} {total_sum}"); + let _ = writeln!(out, "{name}_count{scalar_labels} {total_count}"); out.push('\n'); out } diff --git a/tidal/src/db/metrics/mod.rs b/tidal/src/db/metrics/mod.rs index dde77c6..6e0a574 100644 --- a/tidal/src/db/metrics/mod.rs +++ b/tidal/src/db/metrics/mod.rs @@ -164,6 +164,15 @@ pub struct MetricsState { /// distinguishable when scraped into one Prometheus. Stored as an atomic so /// the install takes `&self` and works even after the `Arc` is shared. partition_id: AtomicU64, + + /// Co-located shard groups' cluster metrics (m11p8). When several shard + /// groups run in one process and share this node's single `/metrics` + /// listener, the metrics-owning shard's `MetricsState` renders each sibling's + /// `tidaldb_cluster_*` series with a distinct `shard="N"` label so they never + /// collide on the shared scrape target. Empty on a single-shard node (the + /// shipped S=1 topology), so its output is byte-identical to pre-m11p8. + #[cfg(feature = "metrics")] + cluster_siblings: std::sync::RwLock)>>, } impl MetricsState { @@ -215,9 +224,28 @@ impl MetricsState { cluster: std::sync::Arc::new(cluster::ClusterMetrics::new()), control_plane: None, partition_id: AtomicU64::new(0), + #[cfg(feature = "metrics")] + cluster_siblings: std::sync::RwLock::new(Vec::new()), } } + /// Register a co-located shard group's cluster metrics so this node's single + /// `/metrics` listener also exposes that group's `tidaldb_cluster_*` series, + /// stamped with `shard=""` (m11p8). Called once per non-owner group + /// during cluster-node construction. + #[cfg(feature = "metrics")] + pub fn register_cluster_sibling( + &self, + shard: u16, + metrics: std::sync::Arc, + ) { + metrics.mark_active(); + self.cluster_siblings + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push((shard, metrics)); + } + /// Maximum age (nanoseconds) a checkpoint may reach before health reports /// degraded. The periodic checkpoint thread runs every 30s; a checkpoint /// older than 5 minutes means the thread is stuck, dead, or failing — all @@ -505,6 +533,17 @@ impl MetricsState { // their exact metric surface. if self.cluster.is_active() { self.cluster.render_into(&mut out, partition_id); + // Co-located shard groups (m11p8): each renders with a distinct + // `shard="N"` label so several groups sharing this node's one + // `/metrics` listener never collide. Empty on the S=1 topology. + for (shard, sib) in self + .cluster_siblings + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + { + sib.render_into_sibling(&mut out, partition_id, *shard); + } } } diff --git a/tidal/src/db/mod.rs b/tidal/src/db/mod.rs index e8f796a..a9c1d9d 100644 --- a/tidal/src/db/mod.rs +++ b/tidal/src/db/mod.rs @@ -921,6 +921,10 @@ impl TidalDb { // Resolve through the single source of truth so an explicit // wal_dir override is honored here exactly as at WAL open. let wal_dir = config.resolved_wal_dir(); + // m11p8 PITR: the optional WAL archive dir. When set, the + // checkpoint thread's online compaction archives each sealed + // segment before deleting it. + let wal_archive_dir = config.wal_archive_dir.clone(); let metrics_clone = Arc::clone(&metrics); // Separate handle kept by the panic supervisor so it can // flag a dead checkpoint thread even though `metrics_clone` @@ -964,6 +968,7 @@ impl TidalDb { items, seq_clone, wal_dir, + wal_archive_dir, retention_pin_clone, metrics_clone, index_handles, @@ -1136,6 +1141,20 @@ impl TidalDb { self.metrics.cluster.mark_active(); Arc::clone(&self.metrics.cluster) } + + /// Expose `sibling`'s `tidaldb_cluster_*` series through THIS db's `/metrics` + /// listener, stamped with `shard=""` (m11p8). + /// + /// When several shard groups co-locate in one process, only one binds the + /// engine's `/metrics` server (the metrics owner); the others register here + /// so a single per-node scrape target still exposes every co-located group's + /// replication series, collision-free. A no-op-equivalent on the S=1 topology + /// (one shard per node), so its `/metrics` output is unchanged. + #[cfg(feature = "metrics")] + pub fn register_metrics_sibling(&self, shard: u16, sibling: &Self) { + self.metrics + .register_cluster_sibling(shard, Arc::clone(&sibling.metrics.cluster)); + } } #[cfg(test)] diff --git a/tidal/src/db/state_rebuild.rs b/tidal/src/db/state_rebuild.rs index fe49360..e68f70f 100644 --- a/tidal/src/db/state_rebuild.rs +++ b/tidal/src/db/state_rebuild.rs @@ -455,6 +455,7 @@ pub(super) fn run_checkpoint_thread( storage: Box, last_wal_seq: Arc, wal_dir: Option, + wal_archive_dir: Option, wal_retention_pin: Arc, metrics: Arc, index_handles: IndexMetricsHandles, @@ -579,7 +580,12 @@ pub(super) fn run_checkpoint_thread( // the artifact's tail between staging and the first joiner // pull). let pin = wal_retention_pin.load(Ordering::Acquire); - match crate::wal::compaction::compact_wal_online_pinned(dir, seq, pin) { + match crate::wal::compaction::compact_wal_online_pinned( + dir, + seq, + pin, + wal_archive_dir.as_deref(), + ) { Ok(result) => { #[cfg(feature = "metrics")] { diff --git a/tidal/src/query/executor/post_filter.rs b/tidal/src/query/executor/post_filter.rs index a427050..8c63297 100644 --- a/tidal/src/query/executor/post_filter.rs +++ b/tidal/src/query/executor/post_filter.rs @@ -161,6 +161,9 @@ fn apply_signal_thresholds( ledger, degradation_level, crate::schema::Timestamp::now().as_nanos(), + // Post-filter is a one-off per-candidate threshold read outside + // the scored pre-pass — no plan to consult, direct read. + None, ) .map_err(|e| QueryError::InvalidFilter { field: signal.clone(), diff --git a/tidal/src/ranking/executor/helpers.rs b/tidal/src/ranking/executor/helpers.rs index 066ff7e..f8b042d 100644 --- a/tidal/src/ranking/executor/helpers.rs +++ b/tidal/src/ranking/executor/helpers.rs @@ -4,7 +4,7 @@ //! logic. They depend on `ScoredCandidate` (from `context`) and profile types, //! but contain no executor state. -use super::context::ScoredCandidate; +use super::{SignalValues, context::ScoredCandidate}; use crate::{ load::DegradationLevel, ranking::profile::{Exclude, Gate, SignalAgg}, @@ -59,6 +59,10 @@ pub(super) const COARSE_VALUE_WINDOW: Window = Window::AllTime; /// - `TidalError::Internal` if `agg` is `Ratio` / `RelativeVelocity`: the /// registry rejects those at profile registration, so reaching this arm is a /// tidalDB invariant violation, not a recoverable runtime condition. +// Mirrors the scoring inputs (entity, signal, agg, window, ledger, degradation, +// clock) plus the optional pre-pass table; bundling them into a struct would +// only relocate the same fields without reducing real complexity. +#[allow(clippy::too_many_arguments)] pub(crate) fn read_agg( entity_id: EntityId, signal: &str, @@ -67,6 +71,7 @@ pub(crate) fn read_agg( ledger: &SignalLedger, degradation: DegradationLevel, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result { let window = if degradation.coarsens_aggregates() { match agg { @@ -77,6 +82,17 @@ pub(crate) fn read_agg( } else { window }; + // Pre-pass hit: a value already computed by `SignalValues` (one + // `entries.get()` per signal type per candidate). A miss falls through to + // the direct read below, so the plan is a pure optimization and never + // changes the result. The effective `window` computed above is exactly the + // key the plan was built with. + if let Some(vals) = vals + && let Ok(type_id) = ledger.resolve_signal_type(signal) + && let Some(v) = vals.get(type_id, agg, window) + { + return Ok(v); + } match agg { SignalAgg::Value => { #[allow(clippy::cast_precision_loss)] @@ -128,6 +144,7 @@ pub(crate) fn read_agg( /// /// Propagates any non-`UnknownSignalType` [`read_agg`] error (notably the /// `Ratio` / `RelativeVelocity` invariant violation). +#[allow(clippy::too_many_arguments)] // same scoring inputs as `read_agg` + the pre-pass table pub(super) fn read_agg_for_sort( entity_id: EntityId, signal: &str, @@ -136,8 +153,18 @@ pub(super) fn read_agg_for_sort( ledger: &SignalLedger, degradation: DegradationLevel, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result { - match read_agg(entity_id, signal, agg, window, ledger, degradation, now_ns) { + match read_agg( + entity_id, + signal, + agg, + window, + ledger, + degradation, + now_ns, + vals, + ) { Err(crate::TidalError::Schema(crate::schema::error::SchemaError::UnknownSignalType(_))) => { Ok(0.0) } @@ -164,6 +191,7 @@ pub(super) fn passes_gates( gates: &[Gate], ledger: &SignalLedger, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result { for gate in gates { let value = read_agg( @@ -174,6 +202,7 @@ pub(super) fn passes_gates( ledger, DegradationLevel::Full, now_ns, + vals, )?; if value < gate.min_threshold { return Ok(false); @@ -207,6 +236,7 @@ pub(super) fn passes_excludes( excludes: &[Exclude], ledger: &SignalLedger, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result { for exclude in excludes { let value = read_agg( @@ -217,6 +247,7 @@ pub(super) fn passes_excludes( ledger, DegradationLevel::Full, now_ns, + vals, )?; if value > exclude.above { return Ok(false); @@ -332,6 +363,7 @@ mod tests { &ledger, DegradationLevel::Full, Timestamp::now().as_nanos(), + None, ); assert!(matches!(result, Err(crate::TidalError::Internal(_)))); } @@ -348,6 +380,7 @@ mod tests { &ledger, DegradationLevel::Full, Timestamp::now().as_nanos(), + None, ); assert!(matches!(result, Err(crate::TidalError::Schema(_)))); } @@ -369,7 +402,16 @@ mod tests { min_threshold: 5.0, }]; // count=3 < threshold=5 -> candidate excluded. - assert!(!passes_gates(entity_id, &gates, &ledger, Timestamp::now().as_nanos()).unwrap()); + assert!( + !passes_gates( + entity_id, + &gates, + &ledger, + Timestamp::now().as_nanos(), + None + ) + .unwrap() + ); } #[test] @@ -389,7 +431,16 @@ mod tests { min_threshold: 5.0, }]; // count=5 >= threshold=5 -> candidate included. - assert!(passes_gates(entity_id, &gates, &ledger, Timestamp::now().as_nanos()).unwrap()); + assert!( + passes_gates( + entity_id, + &gates, + &ledger, + Timestamp::now().as_nanos(), + None + ) + .unwrap() + ); } #[test] diff --git a/tidal/src/ranking/executor/mod.rs b/tidal/src/ranking/executor/mod.rs index 2215a9a..88a682d 100644 --- a/tidal/src/ranking/executor/mod.rs +++ b/tidal/src/ranking/executor/mod.rs @@ -17,6 +17,7 @@ pub mod context; pub mod formulas; pub mod helpers; +pub mod signal_values; // Re-export all public items so that `crate::ranking::executor::Foo` paths continue to work. use std::collections::HashMap; @@ -25,6 +26,7 @@ use std::sync::Arc; pub use context::{ScoredCandidate, SignalKey, SignalSnapshot, UserContext}; use formulas::{INTERACTION_BOOST_WEIGHT, PREFERENCE_BOOST_WEIGHT, RELEVANCE_BASE_WEIGHT}; use helpers::{normalize, passes_excludes, passes_gates, read_agg_for_sort}; +pub use signal_values::{SignalReadPlan, SignalValues}; use super::profile::{RankingProfile, SignalAgg}; use crate::{ @@ -122,6 +124,14 @@ pub struct ProfileExecutor<'a> { /// no-`FOR USER` case, which yields a stable per-minute *global* permutation /// rather than per-user variety. Set via [`Self::with_shuffle_user`]. shuffle_user_id: u64, + /// Whether to run the per-candidate signal-value pre-pass + /// ([`SignalReadPlan`]) that collapses redundant ledger gets to one + /// `entries.get()` per distinct signal type. On by default; the A/B + /// property test (`signal_plan_matches_per_term_path`) flips it off to score + /// the same inputs through the direct per-term path and assert identical + /// output, and it doubles as a runtime fallback. Pure optimization — a miss + /// always falls back to a direct read. + use_signal_plan: bool, } impl<'a> ProfileExecutor<'a> { @@ -137,9 +147,22 @@ impl<'a> ProfileExecutor<'a> { user_state_for_date_saved: None, degradation_level: DegradationLevel::Full, shuffle_user_id: 0, + use_signal_plan: true, } } + /// Toggle the signal-value pre-pass (see [`Self::use_signal_plan`]). + /// + /// On by default. The A/B property test and the `ranking` benchmark flip it + /// off to score the same inputs through the direct per-term path; it also + /// serves as an operator escape hatch since the pre-pass is a pure + /// optimization (every read falls back to a direct ledger read on a miss). + #[must_use] + pub const fn with_signal_plan(mut self, enabled: bool) -> Self { + self.use_signal_plan = enabled; + self + } + /// Set the user identity for the `Shuffle` sort's per-user, per-minute seed. /// /// Spec §11.6 seeds shuffle with `hash(user_id, timestamp_minute)` so a @@ -454,21 +477,35 @@ impl<'a> ProfileExecutor<'a> { // Dynamic snapshot labels (`{signal}_boost`/`_penalty`/`_decay`) are // invariant across candidates — build them once here, not per candidate. let labels = RuleLabels::build(profile); + // Signal-value pre-pass plan (which (type, agg, window) reads the profile + // performs) is also entity-independent — build it once per query. Per + // candidate, `SignalReadPlan::compute` does one `entries.get()` per + // distinct signal type; every Pass-2 read below consults the result and + // falls back to a direct read on a miss, so it never changes the output. + let plan = self + .use_signal_plan + .then(|| SignalReadPlan::build(profile, self.ledger, self.degradation_level)); // Pre-size to the candidate count: every candidate that survives // exclude/gate pushes exactly one entry, so this is the exact upper bound // and removes the Vec's growth reallocations on the scoring hot path. let mut scored: Vec = Vec::with_capacity(candidates.len()); for &entity_id in candidates { + // Pass 1: collapse this candidate's redundant ledger gets to one per + // signal type (`None` when the pre-pass is disabled → direct reads). + let values = plan + .as_ref() + .map(|p| p.compute(self.ledger, entity_id, now_ns)); + let vals = values.as_ref(); // Stage 2: hard exclusion -- remove content the user must never see. - if !passes_excludes(entity_id, &profile.excludes, self.ledger, now_ns)? { + if !passes_excludes(entity_id, &profile.excludes, self.ledger, now_ns, vals)? { continue; } // Stage 6: gates -- remove candidates below quality thresholds. - if !passes_gates(entity_id, &profile.gates, self.ledger, now_ns)? { + if !passes_gates(entity_id, &profile.gates, self.ledger, now_ns, vals)? { continue; } let (raw, mut snapshot) = - self.compute_raw_score(entity_id, profile, now, retrieval_scores, &labels)?; + self.compute_raw_score(entity_id, profile, now, retrieval_scores, &labels, vals)?; let metadata = item_metadata.get(&entity_id.as_u64()).map_or(&empty, |m| m); let session_boost = session_ctx.map_or(0.0, |ctx| { Self::session_boost( @@ -595,10 +632,11 @@ impl<'a> ProfileExecutor<'a> { now: Timestamp, retrieval_scores: Option<&HashMap>, labels: &RuleLabels, + vals: Option<&SignalValues>, ) -> crate::Result<(f64, SignalSnapshot)> { let now_ns = now.as_nanos(); let (sort_base, mut snapshot) = - self.score_by_sort(entity_id, profile.sort.as_ref(), now)?; + self.score_by_sort(entity_id, profile.sort.as_ref(), now, vals)?; // Retrieval relevance seed (SEARCH Stage 1c). When the caller threads a // fused RRF score, fold it into the base so the fused order anchors the @@ -639,6 +677,7 @@ impl<'a> ProfileExecutor<'a> { self.ledger, self.degradation_level, now_ns, + vals, )?; let weighted = b.weight * val; if weighted != 0.0 { @@ -687,6 +726,7 @@ impl<'a> ProfileExecutor<'a> { self.ledger, self.degradation_level, now_ns, + vals, )?; let weighted = p.weight * val; if weighted != 0.0 { @@ -720,6 +760,7 @@ impl<'a> ProfileExecutor<'a> { self.ledger, self.degradation_level, now_ns, + vals, )? .clamp(0.0, 1.0); let factor = decay.weight.mul_add(recency, 1.0 - decay.weight); diff --git a/tidal/src/ranking/executor/scoring.rs b/tidal/src/ranking/executor/scoring.rs index cbd00b7..8c8a687 100644 --- a/tidal/src/ranking/executor/scoring.rs +++ b/tidal/src/ranking/executor/scoring.rs @@ -6,7 +6,7 @@ use smallvec::smallvec; use super::{ - ProfileExecutor, SignalKey, SignalSnapshot, + ProfileExecutor, SignalKey, SignalSnapshot, SignalValues, formulas::{ controversial_score, hidden_gems_score, hot_score, shuffle_quality_score, shuffle_quality_weight, shuffle_random, trending_score, @@ -75,21 +75,23 @@ impl ProfileExecutor<'_> { /// [`read_agg_for_sort`], so a signal the schema omits contributes 0.0 /// rather than erroring; any other read error (notably an unimplemented /// aggregation) still propagates. + #[allow(clippy::too_many_lines)] pub(super) fn score_by_sort( &self, entity_id: EntityId, sort: Option<&Sort>, now: Timestamp, + vals: Option<&SignalValues>, ) -> crate::Result<(f64, SignalSnapshot)> { // Capture the query clock ONCE so every per-candidate ledger read below // ages to the same "now" (reproducible scores; one fewer syscall per read). let now_ns = now.as_nanos(); match sort { - Some(Sort::Hot { gravity }) => self.score_hot(entity_id, *gravity, now), - Some(Sort::Trending) => self.score_trending(entity_id, now_ns), - Some(Sort::Controversial) => self.score_controversial(entity_id, now_ns), - Some(Sort::HiddenGems) => self.score_hidden_gems(entity_id, now_ns), - Some(Sort::Shuffle) => Ok((self.score_shuffle(entity_id, now_ns)?, smallvec![])), + Some(Sort::Hot { gravity }) => self.score_hot(entity_id, *gravity, now, vals), + Some(Sort::Trending) => self.score_trending(entity_id, now_ns, vals), + Some(Sort::Controversial) => self.score_controversial(entity_id, now_ns, vals), + Some(Sort::HiddenGems) => self.score_hidden_gems(entity_id, now_ns, vals), + Some(Sort::Shuffle) => Ok((self.score_shuffle(entity_id, now_ns, vals)?, smallvec![])), Some(Sort::New) => { // M2 limitation: entity metadata (`created_at`) is not accessible from the // executor. Entity ID is used as a proxy for recency -- ranks higher IDs @@ -103,19 +105,32 @@ impl ProfileExecutor<'_> { let score = entity_id.as_u64() as f64; Ok((score, smallvec![])) } - Some(Sort::TopWindow { window }) => self.score_top_window(entity_id, *window, now_ns), - Some(Sort::MostViewed { window }) => { - self.single_signal_score(entity_id, "view", &SignalAgg::Value, *window, now_ns) - } - Some(Sort::MostLiked { window }) => { - self.single_signal_score(entity_id, "like", &SignalAgg::Value, *window, now_ns) + Some(Sort::TopWindow { window }) => { + self.score_top_window(entity_id, *window, now_ns, vals) } + Some(Sort::MostViewed { window }) => self.single_signal_score( + entity_id, + "view", + &SignalAgg::Value, + *window, + now_ns, + vals, + ), + Some(Sort::MostLiked { window }) => self.single_signal_score( + entity_id, + "like", + &SignalAgg::Value, + *window, + now_ns, + vals, + ), Some(Sort::MostFollowed) => self.single_signal_score( entity_id, "follow", &SignalAgg::Value, Window::AllTime, now_ns, + vals, ), Some(Sort::CreatorEngagementRate) => { let view_vel = read_agg_for_sort( @@ -126,6 +141,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let like_vel = read_agg_for_sort( entity_id, @@ -135,6 +151,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; Ok(( view_vel + like_vel, @@ -144,7 +161,7 @@ impl ProfileExecutor<'_> { ], )) } - Some(Sort::Rising) => self.score_rising(entity_id, now_ns), + Some(Sort::Rising) => self.score_rising(entity_id, now_ns, vals), Some(Sort::AlphabeticalAsc) => { Ok((self.score_alphabetical_asc(entity_id), smallvec![])) } @@ -153,18 +170,29 @@ impl ProfileExecutor<'_> { } Some(Sort::Shortest) => Ok((self.score_shortest(entity_id), smallvec![])), Some(Sort::Longest) => Ok((self.score_longest(entity_id), smallvec![])), - Some(Sort::MostCommented { window }) => { - self.single_signal_score(entity_id, "comment", &SignalAgg::Value, *window, now_ns) - } - Some(Sort::MostShared { window }) => { - self.single_signal_score(entity_id, "share", &SignalAgg::Value, *window, now_ns) - } + Some(Sort::MostCommented { window }) => self.single_signal_score( + entity_id, + "comment", + &SignalAgg::Value, + *window, + now_ns, + vals, + ), + Some(Sort::MostShared { window }) => self.single_signal_score( + entity_id, + "share", + &SignalAgg::Value, + *window, + now_ns, + vals, + ), Some(Sort::LiveViewerCount) => self.single_signal_score( entity_id, "viewer_count", &SignalAgg::DecayScore, Window::AllTime, now_ns, + vals, ), Some(Sort::DateSaved) => Ok((self.score_date_saved(entity_id), smallvec![])), None => Ok((0.0, smallvec![])), @@ -182,6 +210,7 @@ impl ProfileExecutor<'_> { entity_id: EntityId, gravity: f64, now: Timestamp, + vals: Option<&SignalValues>, ) -> crate::Result<(f64, SignalSnapshot)> { let views = read_agg_for_sort( entity_id, @@ -191,6 +220,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now.as_nanos(), + vals, )?; // The scoring loop receives no per-entity `created_at`: the executor reads // only the signal ledger, and the `created_at` range index is keyed @@ -232,7 +262,12 @@ impl ProfileExecutor<'_> { /// /// Propagates any non-degradable ledger read error (an unimplemented /// aggregation), matching the other formula sorts. - fn score_shuffle(&self, entity_id: EntityId, now_ns: u64) -> crate::Result { + fn score_shuffle( + &self, + entity_id: EntityId, + now_ns: u64, + vals: Option<&SignalValues>, + ) -> crate::Result { // Quality inputs, all aged to the single query clock for reproducibility. let completion_rate = read_agg_for_sort( entity_id, @@ -242,6 +277,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )? .clamp(0.0, 1.0); let likes = read_agg_for_sort( @@ -252,6 +288,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let views = read_agg_for_sort( entity_id, @@ -261,6 +298,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; // like_ratio = likes / (views + 1), clamped to [0, 1]: a like is only // emitted by a viewer, so the ratio is a bounded engagement-quality proxy. @@ -285,6 +323,7 @@ impl ProfileExecutor<'_> { agg: &SignalAgg, window: Window, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result<(f64, SignalSnapshot)> { let val = read_agg_for_sort( entity_id, @@ -294,6 +333,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; Ok((val, smallvec![(SignalKey::Static(signal), val)])) } @@ -302,6 +342,7 @@ impl ProfileExecutor<'_> { &self, entity_id: EntityId, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result<(f64, SignalSnapshot)> { // M6: social-graph-scoped trending. When a social subgraph and // per-user signal index are available, compute aggregate velocity @@ -356,6 +397,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let share_vel = read_agg_for_sort( entity_id, @@ -365,6 +407,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; Ok(( trending_score(view_vel, share_vel), @@ -379,6 +422,7 @@ impl ProfileExecutor<'_> { &self, entity_id: EntityId, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result<(f64, SignalSnapshot)> { let pos = read_agg_for_sort( entity_id, @@ -388,6 +432,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let neg = read_agg_for_sort( entity_id, @@ -397,6 +442,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; Ok(( controversial_score(pos, neg), @@ -411,6 +457,7 @@ impl ProfileExecutor<'_> { &self, entity_id: EntityId, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result<(f64, SignalSnapshot)> { let quality = read_agg_for_sort( entity_id, @@ -420,6 +467,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let view_count = read_agg_for_sort( entity_id, @@ -429,6 +477,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; Ok(( hidden_gems_score(quality, view_count), @@ -444,6 +493,7 @@ impl ProfileExecutor<'_> { entity_id: EntityId, window: Window, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result<(f64, SignalSnapshot)> { let views = read_agg_for_sort( entity_id, @@ -453,6 +503,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let likes = read_agg_for_sort( entity_id, @@ -462,6 +513,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let shares = read_agg_for_sort( entity_id, @@ -471,6 +523,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let completion = read_agg_for_sort( entity_id, @@ -480,6 +533,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; Ok(( views.mul_add( @@ -499,6 +553,7 @@ impl ProfileExecutor<'_> { &self, entity_id: EntityId, now_ns: u64, + vals: Option<&SignalValues>, ) -> crate::Result<(f64, SignalSnapshot)> { let short = read_agg_for_sort( entity_id, @@ -508,6 +563,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let long = read_agg_for_sort( entity_id, @@ -517,6 +573,7 @@ impl ProfileExecutor<'_> { self.ledger, self.degradation_level, now_ns, + vals, )?; let score = if long < f64::EPSILON { short diff --git a/tidal/src/ranking/executor/signal_values.rs b/tidal/src/ranking/executor/signal_values.rs new file mode 100644 index 0000000..129857f --- /dev/null +++ b/tidal/src/ranking/executor/signal_values.rs @@ -0,0 +1,320 @@ +//! Per-candidate signal-value pre-pass — collapse redundant signal-ledger gets. +//! +//! The per-candidate scoring loop reads the signal ledger once per +//! `(signal, agg, window)` term. Many profiles read the **same signal type** at +//! several aggregations or from several stages — e.g. a `Sort::Hot` reads `view` +//! `Value` while a boost reads `view` `DecayScore` — so a naive loop does one +//! sharded `DashMap` get per term even when several terms hit the same entry. +//! +//! [`SignalReadPlan`] + [`SignalValues`] collapse that into one get per distinct +//! signal type per candidate, in two passes: +//! +//! - **Pass 1** ([`SignalReadPlan::compute`]): for each distinct `SignalTypeId` +//! the profile reads, take ONE [`SignalLedger::entry_ref`], evaluate every +//! planned `(agg, window)` against the held entry, then **drop the ref before +//! the next type**. Only one entry ref is ever held at a time, so this cannot +//! deadlock against `DashMap`'s per-shard `RwLock` (holding a ref while getting +//! another key on the same shard is a documented `DashMap` deadlock). +//! +//! - **Pass 2** (the existing scoring code, threading `Option<&SignalValues>`): +//! each read consults this table first, in declared order, so exclude/gate +//! short-circuit and snapshot push order are byte-for-byte unchanged. **A miss +//! falls back to a direct ledger read**, so an incomplete or approximate plan +//! only costs the optimization — never correctness. That fallback is what lets +//! the sort-read enumeration below be a best-effort superset. +//! +//! The plan is entity-independent (the profile fixes which signals/aggs/windows +//! are read), so it is built once per query and reused for every candidate. + +use smallvec::SmallVec; + +use super::helpers::{COARSE_VALUE_WINDOW, COARSE_VELOCITY_WINDOW}; +use crate::{ + load::DegradationLevel, + ranking::profile::{RankingProfile, SignalAgg, Sort}, + schema::{EntityId, Window}, + signals::{SignalLedger, SignalTypeId}, +}; + +/// The three entry-backed aggregations the pre-pass can collapse, as a small +/// `Copy` key (the public [`SignalAgg`] is intentionally neither `Copy` nor +/// `Eq`). `Ratio`/`RelativeVelocity` are not representable here — they are never +/// planned and fall through to the direct read, which raises the same error. +#[derive(Clone, Copy, PartialEq, Eq)] +enum AggKind { + Value, + Velocity, + Decay, +} + +impl AggKind { + const fn from_agg(agg: &SignalAgg) -> Option { + match agg { + SignalAgg::Value => Some(Self::Value), + SignalAgg::Velocity => Some(Self::Velocity), + SignalAgg::DecayScore => Some(Self::Decay), + SignalAgg::Ratio | SignalAgg::RelativeVelocity => None, + } + } +} + +/// The effective read key after degradation substitution and `Decay` +/// window-normalization. Plan entries and Pass-2 lookups are both keyed this +/// way, so a lookup hits iff the same effective read was planned. +#[derive(Clone, Copy, PartialEq, Eq)] +struct ReadKey { + type_id: SignalTypeId, + agg: AggKind, + window: Window, +} + +/// Compute the effective window for a read: apply the same degradation +/// substitution [`super::helpers::read_agg`] applies, then canonicalize +/// `Decay` (which ignores its window) to a single window so a lookup hits +/// regardless of the term's declared window. +const fn effective_window(agg: AggKind, window: Window, degradation: DegradationLevel) -> Window { + match agg { + // `Decay` is an O(1) running quantity independent of `window` (see + // `read_agg`); canonicalize so a boost/decay term with any declared + // window resolves to the same planned value. + AggKind::Decay => Window::AllTime, + AggKind::Value if degradation.coarsens_aggregates() => COARSE_VALUE_WINDOW, + AggKind::Velocity if degradation.coarsens_aggregates() => COARSE_VELOCITY_WINDOW, + AggKind::Value | AggKind::Velocity => window, + } +} + +/// The fixed signal vocabulary a `Sort` mode reads from the ledger, mirroring +/// [`super::ProfileExecutor::score_by_sort`]. Best-effort: this is only a +/// performance hint (Pass 2 falls back to a direct read on any miss), so a sort +/// whose reads drift from this list stays correct — it just doesn't collapse. +/// Sort modes that read no ledger signals (`New`, `Alphabetical*`, `Shortest`, +/// `Longest`, `DateSaved`) and the social-graph trending path (which reads the +/// per-user signal index, not `entries`) contribute nothing here. +fn sort_signal_reads(sort: Option<&Sort>) -> SmallVec<[(&'static str, SignalAgg, Window); 4]> { + use SignalAgg::{DecayScore, Value, Velocity}; + use Window::{AllTime, OneHour, TwentyFourHours}; + let mut v: SmallVec<[(&'static str, SignalAgg, Window); 4]> = SmallVec::new(); + match sort { + Some(Sort::Hot { .. }) => v.push(("view", Value, AllTime)), + Some(Sort::Trending) => { + v.push(("view", Velocity, TwentyFourHours)); + v.push(("share", Velocity, TwentyFourHours)); + } + Some(Sort::Controversial) => { + v.push(("like", Value, AllTime)); + v.push(("dislike", Value, AllTime)); + } + Some(Sort::HiddenGems) => { + v.push(("completion", DecayScore, AllTime)); + v.push(("view", Value, AllTime)); + } + Some(Sort::Shuffle) => { + v.push(("completion", DecayScore, AllTime)); + v.push(("like", Value, AllTime)); + v.push(("view", Value, AllTime)); + } + Some(Sort::TopWindow { window }) => { + v.push(("view", Value, *window)); + v.push(("like", Value, *window)); + v.push(("share", Value, *window)); + v.push(("completion", Value, *window)); + } + Some(Sort::MostViewed { window }) => v.push(("view", Value, *window)), + Some(Sort::MostLiked { window }) => v.push(("like", Value, *window)), + Some(Sort::MostFollowed) => v.push(("follow", Value, AllTime)), + Some(Sort::CreatorEngagementRate) => { + v.push(("view", Velocity, TwentyFourHours)); + v.push(("like", Velocity, TwentyFourHours)); + } + Some(Sort::Rising) => { + v.push(("view", Velocity, OneHour)); + v.push(("view", Velocity, TwentyFourHours)); + } + Some(Sort::MostCommented { window }) => v.push(("comment", Value, *window)), + Some(Sort::MostShared { window }) => v.push(("share", Value, *window)), + Some(Sort::LiveViewerCount) => v.push(("viewer_count", DecayScore, AllTime)), + // No ledger reads: New, AlphabeticalAsc/Desc, Shortest, Longest, DateSaved, None. + _ => {} + } + v +} + +/// The per-query read plan: the distinct effective `(type, agg, window)` reads +/// the profile performs, plus the distinct signal types (so Pass 1 can iterate +/// one get per type). +pub struct SignalReadPlan { + reads: SmallVec<[ReadKey; 16]>, + types: SmallVec<[SignalTypeId; 8]>, +} + +impl SignalReadPlan { + /// Build the read plan for a profile. Entity-independent — call once per + /// query, [`compute`](Self::compute) once per candidate. + /// + /// Resolution failures (`UnknownSignalType`) and the unsupported + /// `Ratio`/`RelativeVelocity` aggregations are simply omitted: the Pass-2 + /// fallback then performs the direct read, which preserves the exact + /// error contract (gates/excludes propagate, sort/boosts swallow, the + /// unsupported aggs error) without this pre-pass having to replicate it. + #[must_use] + pub fn build( + profile: &RankingProfile, + ledger: &SignalLedger, + degradation: DegradationLevel, + ) -> Self { + let mut plan = Self { + reads: SmallVec::new(), + types: SmallVec::new(), + }; + + // Gates and excludes always read at Full degradation (correctness + // filters — see `passes_gates`/`passes_excludes`). + for g in &profile.gates { + plan.add(ledger, &g.signal, &g.agg, g.window, DegradationLevel::Full); + } + for e in &profile.excludes { + plan.add(ledger, &e.signal, &e.agg, e.window, DegradationLevel::Full); + } + // Boosts/penalties/decay and the sort base read at the query degradation. + for b in &profile.boosts { + plan.add(ledger, &b.signal, &b.agg, b.window, degradation); + } + for p in &profile.penalties { + plan.add(ledger, &p.signal, &p.agg, p.window, degradation); + } + if let Some(d) = &profile.decay { + plan.add( + ledger, + &d.signal, + &SignalAgg::DecayScore, + Window::AllTime, + degradation, + ); + } + for (signal, agg, window) in sort_signal_reads(profile.sort.as_ref()) { + plan.add(ledger, signal, &agg, window, degradation); + } + plan + } + + fn add( + &mut self, + ledger: &SignalLedger, + signal: &str, + agg: &SignalAgg, + window: Window, + degradation: DegradationLevel, + ) { + // Only the three entry-backed aggregations are collapsible; the + // registry-rejected Ratio/RelativeVelocity are left to the fallback + // (which raises the same internal error the direct path does). + let Some(agg) = AggKind::from_agg(agg) else { + return; + }; + let Ok(type_id) = ledger.resolve_signal_type(signal) else { + return; // unknown signal → fallback handles the contract + }; + let key = ReadKey { + type_id, + agg, + window: effective_window(agg, window, degradation), + }; + if !self.reads.contains(&key) { + self.reads.push(key); + } + if !self.types.contains(&type_id) { + self.types.push(type_id); + } + } + + /// Pass 1: compute every planned read's value for one candidate with one + /// `entry_ref` per distinct signal type. The ref is held only across a single + /// type's reads, then dropped — never two refs at once (deadlock contract of + /// [`SignalLedger::entry_ref`]). + // The held `entry` ref deliberately spans the inner read loop — that is the + // whole point (one get serves every aggregation of the type). It is released + // at the end of each outer iteration, before the next type's get, so the + // drop is already as tight as the deadlock contract allows. + #[allow(clippy::significant_drop_tightening)] + #[must_use] + pub fn compute(&self, ledger: &SignalLedger, entity_id: EntityId, now_ns: u64) -> SignalValues { + let mut values: SmallVec<[(ReadKey, f64); 16]> = SmallVec::new(); + for &type_id in &self.types { + // ONE get for this type. `entry` (a held shard-read ref) is dropped + // at the end of this loop body, before the next type's get. + let entry = ledger.entry_ref(entity_id, type_id); + let lambda0 = ledger.decay_lambda(type_id); + for key in self.reads.iter().filter(|k| k.type_id == type_id) { + let value = match key.agg { + AggKind::Value => entry + .as_ref() + .map_or(0.0, |e| windowed_count_f64(e, key.window, now_ns)), + AggKind::Velocity => { + let dur = key.window.duration_secs_f64(); + if dur.is_infinite() { + 0.0 + } else { + entry + .as_ref() + .map_or(0.0, |e| windowed_count_f64(e, key.window, now_ns)) + / dur + } + } + AggKind::Decay => lambda0.map_or(0.0, |lambda| { + entry + .as_ref() + .map_or(0.0, |e| e.hot.current_score(0, now_ns, lambda)) + }), + }; + values.push((*key, value)); + } + } + SignalValues { values } + } +} + +/// Read a warm windowed count as `f64` (mirrors `read_windowed_count_at`'s +/// `as f64` cast). Rotates expired buckets at `now_ns` exactly as the direct +/// read does, so a Pass-2 fallback read of the same `(window, now_ns)` is +/// identical. +#[allow(clippy::cast_precision_loss)] +fn windowed_count_f64( + entry: &crate::signals::EntitySignalEntry, + window: Window, + now_ns: u64, +) -> f64 { + entry.warm.windowed_count(window, now_ns) as f64 +} + +/// Pass-1 results for one candidate: the planned `(type, agg, window)` values. +/// Looked up by Pass-2 reads; a miss means a direct ledger read. +pub struct SignalValues { + values: SmallVec<[(ReadKey, f64); 16]>, +} + +impl SignalValues { + /// Look up a precomputed value. `window` is the post-degradation-substitution + /// window the caller is about to read (the plan applied the identical + /// substitution when it was built); `DecayScore` is window-canonicalized here + /// exactly as in the plan so a term's declared window never causes a miss. + /// Returns `None` (→ caller does a direct read) on a miss, including for the + /// non-collapsible `Ratio`/`RelativeVelocity` aggregations. + #[must_use] + pub fn get(&self, type_id: SignalTypeId, agg: &SignalAgg, window: Window) -> Option { + let agg = AggKind::from_agg(agg)?; + let want = ReadKey { + type_id, + agg, + window: if matches!(agg, AggKind::Decay) { + Window::AllTime + } else { + window + }, + }; + self.values + .iter() + .find(|(k, _)| *k == want) + .map(|(_, v)| *v) + } +} diff --git a/tidal/src/ranking/executor/tests.rs b/tidal/src/ranking/executor/tests.rs index 4f42ef0..c03ffba 100644 --- a/tidal/src/ranking/executor/tests.rs +++ b/tidal/src/ranking/executor/tests.rs @@ -1,12 +1,13 @@ use super::*; use crate::{ + load::DegradationLevel, ranking::{ builtins::register_builtins, - profile::{Boost, Exclude, Gate, Penalty, ProfileDecay, SignalAgg, Sort}, + profile::{Boost, Exclude, Gate, Penalty, ProfileDecay, RankingProfile, SignalAgg, Sort}, registry::ProfileRegistry, // Shared fixtures (extracted to de-duplicate the per-module ledger/profile // builders across the five ranking test modules; DRY-S, M0-M10 review). - test_fixtures::{ledger_for, profile_with, seeded_ledger}, + test_fixtures::{ledger_for, profile_with, profile_with_sort, seeded_ledger}, }, schema::EntityId, signals::SignalLedger, @@ -989,3 +990,233 @@ fn shuffle_is_quality_weighted_and_seed_stable() { } mod sort_tests; + +// ── T2: signal-value pre-pass A/B equivalence ──────────────────────────────── +// +// The pre-pass (`SignalReadPlan`/`SignalValues`) collapses redundant ledger gets +// to one `entries.get()` per signal type per candidate. It must be a pure +// optimization: scoring a profile with the pre-pass ON must produce a +// byte-identical `ScoredCandidate` set to the direct per-term path (pre-pass +// OFF). This module is the explicit oracle for that — the direct path is the +// reference implementation, the pre-pass is the optimization, and every +// (profile × degradation) pair below must match exactly. + +/// A ledger seeded so windowed counts, velocities and decay scores all differ +/// across entities — including entities with no signals at all (the `None`-entry +/// path) and signals read by several aggregations of the same type. +fn ab_ledger() -> SignalLedger { + let ledger = ledger_for(&[ + "view", + "like", + "share", + "dislike", + "completion", + "comment", + "follow", + "viewer_count", + "skip", + "block", + ]); + let base = AB_NOW_NS; + for i in 1..=12u64 { + // Recent events (inside the 1h / 24h velocity windows) plus older ones, + // with per-entity-varied weights so no two entities score alike. + let recent = Timestamp::from_nanos(base - i * 60_000_000_000); // i minutes ago + let older = Timestamp::from_nanos(base - i * 7 * 24 * 3_600_000_000_000); // i weeks ago + ledger + .record_signal("view", EntityId::new(i), (13 - i) as f64, recent) + .unwrap(); + ledger + .record_signal("view", EntityId::new(i), 2.0, older) + .unwrap(); + ledger + .record_signal("like", EntityId::new(i), (i % 4) as f64, recent) + .unwrap(); + ledger + .record_signal("share", EntityId::new(i), (i % 3) as f64, recent) + .unwrap(); + ledger + .record_signal("completion", EntityId::new(i), 1.0, recent) + .unwrap(); + if i % 2 == 0 { + ledger + .record_signal("dislike", EntityId::new(i), 1.0, recent) + .unwrap(); + ledger + .record_signal("skip", EntityId::new(i), (i % 5) as f64, recent) + .unwrap(); + } + if i % 3 == 0 { + ledger + .record_signal("block", EntityId::new(i), 1.0, recent) + .unwrap(); + } + ledger + .record_signal("comment", EntityId::new(i), (i % 2) as f64, recent) + .unwrap(); + ledger + .record_signal("follow", EntityId::new(i), 1.0, older) + .unwrap(); + ledger + .record_signal("viewer_count", EntityId::new(i), (i % 6) as f64, recent) + .unwrap(); + } + ledger +} + +const AB_NOW_NS: u64 = 1_708_000_000_000_000_000; + +/// Assert two scored sets are byte-identical: same order, ids, score bits, +/// snapshot keys+value bits, creator, and format. +fn assert_scored_identical(plan: &[ScoredCandidate], direct: &[ScoredCandidate], ctx: &str) { + assert_eq!(plan.len(), direct.len(), "{ctx}: result length differs"); + for (a, b) in plan.iter().zip(direct.iter()) { + assert_eq!(a.entity_id, b.entity_id, "{ctx}: entity_id/order differs"); + assert_eq!( + a.score.to_bits(), + b.score.to_bits(), + "{ctx}: score differs for {:?} ({} vs {})", + a.entity_id, + a.score, + b.score + ); + assert_eq!( + a.signal_snapshot.len(), + b.signal_snapshot.len(), + "{ctx}: snapshot length differs for {:?}", + a.entity_id + ); + for ((ka, va), (kb, vb)) in a.signal_snapshot.iter().zip(b.signal_snapshot.iter()) { + assert_eq!(ka.as_str(), kb.as_str(), "{ctx}: snapshot key differs"); + assert_eq!( + va.to_bits(), + vb.to_bits(), + "{ctx}: snapshot value differs for key {}", + ka.as_str() + ); + } + assert_eq!(a.creator_id, b.creator_id, "{ctx}: creator_id differs"); + assert_eq!(a.format, b.format, "{ctx}: format differs"); + } +} + +/// Every ledger-reading `Sort` mode plus a redundancy-heavy profile, scored with +/// the pre-pass ON vs OFF at both `Full` and `CoarseAggregates`, must match +/// byte-for-byte. The redundancy profile reads `view` at five +/// `(stage, agg, window)` sites that resolve to the same signal type — the exact +/// case the one-get-per-type pre-pass collapses. +#[test] +#[allow(clippy::too_many_lines)] +fn signal_plan_matches_per_term_path() { + let ledger = ab_ledger(); + let now = Timestamp::from_nanos(AB_NOW_NS); + // Candidates include entities with no signals (100, 101) to cover the + // missing-entry default mapping. + let candidates: Vec = (1..=12).chain([100, 101]).map(EntityId::new).collect(); + + let mut profiles: Vec = vec![ + profile_with_sort(Sort::Hot { gravity: 1.5 }), + profile_with_sort(Sort::Trending), + profile_with_sort(Sort::Controversial), + profile_with_sort(Sort::HiddenGems), + profile_with_sort(Sort::Shuffle), + profile_with_sort(Sort::TopWindow { + window: Window::TwentyFourHours, + }), + profile_with_sort(Sort::MostViewed { + window: Window::AllTime, + }), + profile_with_sort(Sort::MostLiked { + window: Window::OneHour, + }), + profile_with_sort(Sort::MostFollowed), + profile_with_sort(Sort::CreatorEngagementRate), + profile_with_sort(Sort::Rising), + profile_with_sort(Sort::MostCommented { + window: Window::TwentyFourHours, + }), + profile_with_sort(Sort::MostShared { + window: Window::AllTime, + }), + profile_with_sort(Sort::LiveViewerCount), + profile_with_sort(Sort::New), // no ledger reads — control + ]; + + // Redundancy-heavy profile: `view` read by the sort (Value), a gate (Value), + // a boost (Value), a boost (DecayScore), and the decay term (DecayScore). + profiles.push(profile_with("ab_rich", |p| { + p.sort = Some(Sort::Hot { gravity: 1.2 }); + p.gates = vec![Gate { + signal: "view".into(), + agg: SignalAgg::Value, + window: Window::AllTime, + min_threshold: 0.0, + }]; + p.excludes = vec![Exclude { + signal: "block".into(), + agg: SignalAgg::Value, + window: Window::AllTime, + above: 5.0, + }]; + p.boosts = vec![ + Boost { + signal: "view".into(), + agg: SignalAgg::DecayScore, + window: Window::AllTime, + weight: 1.0, + }, + Boost { + signal: "view".into(), + agg: SignalAgg::Value, + window: Window::AllTime, + weight: 0.5, + }, + Boost { + signal: "like".into(), + agg: SignalAgg::DecayScore, + window: Window::AllTime, + weight: 2.0, + }, + Boost { + signal: "share".into(), + agg: SignalAgg::Velocity, + window: Window::TwentyFourHours, + weight: 1.5, + }, + Boost { + signal: "nonexistent".into(), + agg: SignalAgg::Value, + window: Window::AllTime, + weight: 1.0, + }, + ]; + p.penalties = vec![Penalty { + signal: "skip".into(), + agg: SignalAgg::Value, + window: Window::AllTime, + weight: 1.0, + }]; + p.decay = Some(ProfileDecay { + signal: "view".into(), + half_life_secs: 7 * 24 * 3600, + weight: 0.5, + }); + })); + + for degradation in [DegradationLevel::Full, DegradationLevel::CoarseAggregates] { + for profile in &profiles { + let with_plan = ProfileExecutor::new(&ledger) + .with_degradation_level(degradation) + .with_signal_plan(true) + .score(&candidates, profile, now) + .unwrap(); + let direct = ProfileExecutor::new(&ledger) + .with_degradation_level(degradation) + .with_signal_plan(false) + .score(&candidates, profile, now) + .unwrap(); + let ctx = format!("profile={} degradation={degradation:?}", profile.name); + assert_scored_identical(&with_plan, &direct, &ctx); + } + } +} diff --git a/tidal/src/replication/ship.rs b/tidal/src/replication/ship.rs index 111ecf4..b737d28 100644 --- a/tidal/src/replication/ship.rs +++ b/tidal/src/replication/ship.rs @@ -663,6 +663,14 @@ impl ShipQueue { peers } + /// The peer's circuit-breaker state for the self-heal gauge (m11p8): 0 + /// closed, 1 open, 2 half-open. Delegates to the transport (read-only — does + /// not admit the breaker's half-open probe). + #[must_use] + pub fn peer_breaker_state(&self, peer: ShardId) -> u8 { + self.shared.transport.peer_breaker_state(peer) + } + /// Add a peer to the roster at runtime (m11p5 §3.3 conf-change): construct /// its dispatch cell and spawn its windowed senders, inheriting the queue's /// CURRENT activation state. Idempotent (a re-add of an existing peer is a diff --git a/tidal/src/replication/transport.rs b/tidal/src/replication/transport.rs index 6f35ee7..4029d1d 100644 --- a/tidal/src/replication/transport.rs +++ b/tidal/src/replication/transport.rs @@ -195,6 +195,14 @@ pub trait Transport: Send + Sync + 'static { /// default is a no-op: in-process transports ack synchronously. fn notify_applied(&self, _source_shard: ShardId, _applied: u64) {} + /// The circuit-breaker state for `peer`, encoded for the + /// `tidaldb_cluster_peer_breaker_state` gauge: 0 closed, 1 open, 2 half-open + /// (m11p8). Read-only — must never admit the breaker's half-open probe. The + /// default `0` (closed) suits in-process transports, which have no breaker. + fn peer_breaker_state(&self, _peer: ShardId) -> u8 { + 0 + } + /// The shard identity of this transport endpoint. fn local_shard(&self) -> ShardId; } @@ -237,6 +245,12 @@ impl Transport for std::sync::Arc { (**self).notify_applied(source_shard, applied); } + fn peer_breaker_state(&self, peer: ShardId) -> u8 { + // Forward explicitly: the trait default `0` would report every peer's + // breaker as closed for type-erased callers (the self-heal gauge path). + (**self).peer_breaker_state(peer) + } + fn local_shard(&self) -> ShardId { (**self).local_shard() } diff --git a/tidal/src/signals/ledger/core.rs b/tidal/src/signals/ledger/core.rs index b2ea14e..9fafa99 100644 --- a/tidal/src/signals/ledger/core.rs +++ b/tidal/src/signals/ledger/core.rs @@ -457,6 +457,46 @@ impl SignalLedger { .ok_or_else(|| TidalError::Schema(SchemaError::UnknownSignalType(name.to_owned()))) } + /// Borrow the `(entity_id, type_id)` signal entry, if present, as a held + /// shard-read reference. + /// + /// This is the single `DashMap` get behind the ranking pre-pass + /// ([`crate::ranking::executor::SignalValues`]): the caller takes one ref per + /// distinct signal type, evaluates every aggregation that type needs against + /// the held entry (`hot.current_score` / `warm.windowed_count` are both + /// `&self` reads), then **drops the ref before getting the next type**. + /// + /// # Deadlock contract + /// + /// Only ONE `Ref` may be held at a time. Holding this ref while calling any + /// method that locks the same shard (another `get`, or a `record_signal` + /// `or_insert`) on the same thread can deadlock — `DashMap` shards are + /// `RwLock`s and a second lock acquisition on a shard this thread already + /// holds is not guaranteed to be re-entrant. + #[must_use] + pub(crate) fn entry_ref( + &self, + entity_id: EntityId, + type_id: SignalTypeId, + ) -> Option> { + self.entries.get(&(entity_id, type_id)) + } + + /// The decay rate (`lambda`) for a signal type's primary decay slot (index + /// 0), or `None` if the type defines no decay rate. + /// + /// Mirrors the range-check in [`read_decay_score_at`](Self::read_decay_score_at): + /// a missing lambda means a `DecayScore` read yields `None` → `0.0`, never an + /// undecayed score. Exposed so the ranking pre-pass can evaluate `DecayScore` + /// against a held entry ref without re-resolving the lambda per read. + #[must_use] + pub(crate) fn decay_lambda(&self, type_id: SignalTypeId) -> Option { + self.signal_lambdas + .get(&type_id) + .and_then(|v| v.first()) + .copied() + } + /// Get the existing `(entity_id, type_id)` entry or create a zeroed one /// started at `ts_ns`. /// diff --git a/tidal/src/testing/cluster.rs b/tidal/src/testing/cluster.rs index a9dd5b4..12230d9 100644 --- a/tidal/src/testing/cluster.rs +++ b/tidal/src/testing/cluster.rs @@ -674,29 +674,31 @@ impl SimulatedCluster { self.total_signals.load(Ordering::Relaxed) } - /// Number of WAL batches applied from the **initial** leader (`ShardId(0)`) - /// on a specific region's replication state. + /// Number of WAL batches applied from the **current** leader on a specific + /// region's replication state. /// - /// This is equivalent to the event count for all tests that do not involve - /// leader promotion, since each signal produces exactly one WAL batch. + /// Each signal produces exactly one WAL batch, so for a region following the + /// live leader this is its applied-event count. /// - /// # Limitation (post-promotion) + /// # Post-promotion correctness (m11p8-C) /// - /// The queried shard is hardcoded to `ShardId(0)` — the *initial* leader — - /// not the current one. `applied_seqno` is per source shard, so after a - /// [`promote_leader`](Self::promote_leader) call the new leader ships under - /// its own `ShardId`, and this method keeps returning only the count from the - /// old leader's stream — i.e. it silently undercounts. Callers in a promotion - /// scenario must instead read - /// [`replication_state().applied_seqno(current_leader_shard)`] directly so the - /// shard matches the live leader. Bounding the hardcode behind a doc note here - /// rather than reading the live leader shard keeps the existing no-promotion - /// callers' counts stable. + /// `applied_seqno` is keyed per source shard, and the leader ships under its + /// own `ShardId(leader_region.0)`. Reading the applied frontier therefore + /// keys on the **current** leader's shard, not a hardcoded `ShardId(0)`: + /// after a [`promote_leader`](Self::promote_leader) the new leader ships under + /// a different `ShardId`, and a node's progress against that new stream is + /// what "applied" means post-promote. The previous hardcode to `ShardId(0)` + /// (the initial leader) silently undercounted after any promotion — the + /// "lag accounting wrong after promote (ShardId(0)-keyed)" bug. For the leader + /// region itself, the relay advances its own frontier under its shard as it + /// writes, so the leader's row reads truthfully rather than 0. No-promotion + /// callers are unaffected: the initial leader is `ShardId(0)`. #[must_use] pub fn applied_count(&self, region: RegionId) -> u64 { + let leader_shard = ShardId(self.leader_region().0); self.nodes.get(®ion).map_or(0, |n| { n.db.replication_state() - .applied_seqno(ShardId(0)) + .applied_seqno(leader_shard) .unwrap_or(0) }) } diff --git a/tidal/src/wal/compaction.rs b/tidal/src/wal/compaction.rs index ac51056..87c3875 100644 --- a/tidal/src/wal/compaction.rs +++ b/tidal/src/wal/compaction.rs @@ -61,7 +61,10 @@ pub struct CompactionResult { /// if an error is encountered mid-way -- this is safe (see invariant above). pub fn compact_wal(wal_dir: &Path, checkpoint_seq: u64) -> Result { let segments = segment::list_segments(wal_dir)?; - compact_segments(wal_dir, &segments, checkpoint_seq) + // The offline close-path compaction does not archive (the live archival hook + // runs on the periodic online path; close is a clean shutdown, not a window + // where PITR continuity matters). + compact_segments(wal_dir, &segments, checkpoint_seq, None) } /// Online-safe compaction: identical to [`compact_wal`] except it NEVER deletes @@ -92,7 +95,7 @@ pub fn compact_wal_online( wal_dir: &Path, checkpoint_seq: u64, ) -> Result { - compact_wal_online_pinned(wal_dir, checkpoint_seq, 0) + compact_wal_online_pinned(wal_dir, checkpoint_seq, 0, None) } /// Online-safe compaction with a **retention pin** (m11p5 §2.1). @@ -133,6 +136,7 @@ pub fn compact_wal_online_pinned( wal_dir: &Path, checkpoint_seq: u64, pin: u64, + archive_dir: Option<&Path>, ) -> Result { let segments = segment::list_segments(wal_dir)?; // The live segment is the maximum-`first_seq` segment; never delete at or @@ -163,7 +167,7 @@ pub fn compact_wal_online_pinned( floor = floor.min(pin_floor); } - compact_segments(wal_dir, &segments, floor) + compact_segments(wal_dir, &segments, floor, archive_dir) } /// Delete every listed segment whose `first_seq` is strictly less than `floor`, @@ -177,6 +181,7 @@ fn compact_segments( wal_dir: &Path, segments: &[(u64, std::path::PathBuf)], floor: u64, + archive_dir: Option<&Path>, ) -> Result { let total_before = segments.len(); @@ -194,6 +199,22 @@ fn compact_segments( // single-owner invariant. let file_size = std::fs::metadata(seg_path).map(|m| m.len()).unwrap_or(0); + // m11p8 PITR: archive the segment BEFORE deleting it so the archive + // is a gap-free record of every sealed segment. If archival fails we + // do NOT delete — losing the segment from both the live WAL and the + // archive would punch a hole in the point-in-time recovery range. The + // segment stays live and the next compaction retries. + if let Some(archive) = archive_dir + && let Err(e) = archive_segment(archive, seg_path) + { + tracing::warn!( + segment_first_seq = seg_first_seq, + error = %e, + "WAL segment archival failed; keeping segment live (PITR gap avoided)" + ); + continue; + } + std::fs::remove_file(seg_path)?; deleted += 1; bytes_reclaimed += file_size; @@ -233,6 +254,38 @@ fn compact_segments( }) } +/// Copy a sealed WAL segment into `archive_dir` for point-in-time recovery +/// (m11p8), durably and idempotently. The destination keeps the segment's +/// filename (which encodes shard + `first_seq`), so the archive is a +/// self-describing, ordered catalog. Copy goes through a `.tmp` + rename so a +/// crash mid-copy never leaves a short file the catalog would treat as complete; +/// a re-archival after a crash (destination already present at the same size) is +/// a no-op. The temp file and the archive directory are fsynced so PITR never +/// replays a torn archived segment. +fn archive_segment(archive_dir: &Path, seg_path: &Path) -> Result<(), WalError> { + std::fs::create_dir_all(archive_dir)?; + let file_name = seg_path.file_name().ok_or_else(|| { + WalError::from(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "WAL segment path has no file name", + )) + })?; + let dest = archive_dir.join(file_name); + let src_len = std::fs::metadata(seg_path)?.len(); + // Idempotent: a complete prior archive (same size) needs no rewrite. + if let Ok(meta) = std::fs::metadata(&dest) + && meta.len() == src_len + { + return Ok(()); + } + let tmp = archive_dir.join(format!("{}.tmp", file_name.to_string_lossy())); + std::fs::copy(seg_path, &tmp)?; + std::fs::File::open(&tmp)?.sync_all()?; + std::fs::rename(&tmp, &dest)?; + crate::wal::sync_dir_durable(archive_dir)?; + Ok(()) +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { @@ -252,6 +305,53 @@ mod tests { assert_eq!(result.segments_remaining, 0); } + #[test] + fn online_compaction_archives_before_deleting() { + // m11p8 PITR: a deleted segment must land in the archive first, so the + // archive is a gap-free record — no segment is ever lost from BOTH. + let dir = tempfile::tempdir().unwrap(); + let archive = tempfile::tempdir().unwrap(); + for &seq in &[1u64, 50, 100, 200] { + let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, seq, 1024).unwrap(); + } + let before: Vec = list_segments(dir.path()) + .unwrap() + .iter() + .map(|(s, _)| *s) + .collect(); + + let result = compact_wal_online_pinned(dir.path(), 150, 0, Some(archive.path())).unwrap(); + assert!(result.segments_deleted >= 1, "expected some deletions"); + + let live: std::collections::HashSet = list_segments(dir.path()) + .unwrap() + .iter() + .map(|(s, _)| *s) + .collect(); + let archived: std::collections::HashSet = list_segments(archive.path()) + .unwrap() + .iter() + .map(|(s, _)| *s) + .collect(); + + // Every original segment is either still live OR archived — never lost. + for s in &before { + assert!( + live.contains(s) || archived.contains(s), + "segment {s} was lost (neither live nor archived)" + ); + } + // The segments below the floor (1, 50, 100) are deleted from live and + // present in the archive; the active segment (200) survives in place. + assert!(archived.contains(&1) && archived.contains(&50) && archived.contains(&100)); + assert!(live.contains(&200) && !live.contains(&1)); + + // Idempotent: re-running with the same archive (already-archived dests at + // the same size) is a clean no-op, not an error. + let again = compact_wal_online_pinned(dir.path(), 150, 0, Some(archive.path())).unwrap(); + assert_eq!(again.segments_deleted, 0, "nothing left below the floor"); + } + #[test] fn compact_deletes_old_segments() { let dir = tempfile::tempdir().unwrap(); @@ -442,7 +542,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); three_sealed_plus_live(dir.path()); - let result = compact_wal_online_pinned(dir.path(), 1_000, 150).unwrap(); + let result = compact_wal_online_pinned(dir.path(), 1_000, 150, None).unwrap(); assert_eq!(result.segments_deleted, 1, "only the pre-resume segment"); assert_eq!(result.segments_remaining, 3); @@ -471,7 +571,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); three_sealed_plus_live(dir.path()); - let result = compact_wal_online_pinned(dir.path(), 1_000, 199).unwrap(); + let result = compact_wal_online_pinned(dir.path(), 1_000, 199, None).unwrap(); // floor clamps to the first_seq of the segment containing pin+1=200, // which is the segment STARTING at 200 ([200,300)); the segment // [100,200) is fully below seq 200 and is reclaimed along with [1,100). @@ -486,7 +586,7 @@ mod tests { // it by lowering the floor to 100 (the segment's own first_seq). let dir2 = tempfile::tempdir().unwrap(); three_sealed_plus_live(dir2.path()); - let result2 = compact_wal_online_pinned(dir2.path(), 1_000, 100).unwrap(); + let result2 = compact_wal_online_pinned(dir2.path(), 1_000, 100, None).unwrap(); assert_eq!( result2.segments_deleted, 1, "the straddled segment [100,200) must NOT be split/deleted" @@ -506,7 +606,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); three_sealed_plus_live(dir.path()); - let pinned = compact_wal_online_pinned(dir.path(), 1_000, 0).unwrap(); + let pinned = compact_wal_online_pinned(dir.path(), 1_000, 0, None).unwrap(); assert_eq!(pinned.segments_deleted, 3, "all sealed segments below live"); assert_eq!(pinned.segments_remaining, 1); assert_eq!(list_segments(dir.path()).unwrap()[0].0, 300); @@ -531,7 +631,7 @@ mod tests { // unpinned results must be identical here. let dir = tempfile::tempdir().unwrap(); three_sealed_plus_live(dir.path()); - let pinned = compact_wal_online_pinned(dir.path(), 150, 250).unwrap(); + let pinned = compact_wal_online_pinned(dir.path(), 150, 250, None).unwrap(); let dir2 = tempfile::tempdir().unwrap(); three_sealed_plus_live(dir2.path()); diff --git a/tidalctl/Cargo.toml b/tidalctl/Cargo.toml index eb919d0..3ed7610 100644 --- a/tidalctl/Cargo.toml +++ b/tidalctl/Cargo.toml @@ -32,6 +32,10 @@ unwrap_used = "deny" tidaldb = { path = "../tidal" } serde = { version = "1", features = ["derive"] } serde_json = "1" +# Backup/restore manifest integrity (m11p8) — the same hash the engine verifies +# WAL segments and snapshot artifacts with, so a tidalctl-written manifest and an +# engine-verified one agree. +blake3 = "1" [dev-dependencies] tidaldb = { path = "../tidal", features = ["test-utils"] } diff --git a/tidalctl/src/commands/backup.rs b/tidalctl/src/commands/backup.rs new file mode 100644 index 0000000..d85ed2b --- /dev/null +++ b/tidalctl/src/commands/backup.rs @@ -0,0 +1,276 @@ +//! `tidalctl backup` / `tidalctl restore` — offline data-dir backup and restore +//! with a BLAKE3 integrity manifest (m11p8). +//! +//! These operate on a data dir at rest (a drained/stopped node, or a filesystem +//! copy), so the snapshot is trivially consistent: nothing is writing. `backup` +//! copies the data dir to a destination and writes a [`BACKUP_MANIFEST`] (BLAKE3 +//! per file + the recovered WAL checkpoint cursor); `restore` verifies every +//! file's hash against the manifest before placing it into a FRESH target dir. +//! +//! Coordinated cluster backup (runbook): back up one committed replica per shard +//! group — under `ack=quorum` any committed replica's data dir holds the +//! quorum-durable log, so it is a cluster-consistent snapshot at its recorded +//! `checkpoint_seq`. The per-shard manifests + the WAL archive (the engine's +//! `wal.archive_dir`) together give point-in-time recovery. Restore re-seeds each +//! group's leader from its backup; followers catch up via the live stream. + +use std::{ + path::{Path, PathBuf}, + time::{SystemTime, UNIX_EPOCH}, +}; + +use serde::{Deserialize, Serialize}; + +use crate::{CliError, wal_state}; + +/// Manifest filename written at the root of every backup. +const BACKUP_MANIFEST: &str = "BACKUP_MANIFEST.json"; + +/// The lock file is process-ownership state, never part of a backup (restoring +/// a stale lock would falsely claim the target dir is owned). +const LOCK_FILE: &str = "tidaldb.lock"; + +/// One file's integrity record within a [`Manifest`]. +#[derive(Debug, Serialize, Deserialize)] +struct FileEntry { + /// Path relative to the data-dir root (forward-slash separated). + path: String, + size: u64, + /// Lowercase-hex BLAKE3 of the file's bytes. + blake3: String, +} + +/// The backup manifest, written at the backup root and verified on restore. +#[derive(Debug, Serialize, Deserialize)] +struct Manifest { + /// Format version (bump on an incompatible manifest change). + manifest_version: u32, + /// Unix-epoch seconds the backup was taken (operator audit trail). + created_unix_secs: u64, + /// The WAL checkpoint sequence recovered from the source — the cluster + /// cursor this backup is consistent to. An operator assembling a multi-shard + /// cluster backup records one manifest per shard group; the set of + /// `checkpoint_seq` values is the cluster-consistent restore point. + checkpoint_seq: u64, + file_count: usize, + total_bytes: u64, + files: Vec, +} + +const MANIFEST_VERSION: u32 = 1; + +/// `tidalctl backup --path --out `: copy the data dir to `dest` +/// and write a BLAKE3 manifest. Refuses to overwrite a non-empty destination so a +/// backup never clobbers an existing one. +pub(crate) fn run_backup(src: &Path, out: &Path, pretty: bool) -> Result<(String, i32), CliError> { + if !src.is_dir() { + return Err(CliError::new(format!( + "source data dir does not exist: {}", + src.display() + ))); + } + // Refuse to write into a non-empty destination (don't merge into / clobber an + // existing backup or an unrelated directory). + if out.exists() + && std::fs::read_dir(out) + .map_err(|e| CliError::new(format!("read dest {}: {e}", out.display())))? + .next() + .is_some() + { + return Err(CliError::new(format!( + "destination {} is not empty; choose a fresh directory", + out.display() + ))); + } + std::fs::create_dir_all(out) + .map_err(|e| CliError::new(format!("create dest {}: {e}", out.display())))?; + + // The cluster cursor this backup is consistent to (recovered offline). + let checkpoint_seq = wal_state::gather_wal_state(&src.join("wal")) + .map(|s| s.checkpoint_seq) + .unwrap_or(0); + + let mut rel_files = Vec::new(); + collect_files(src, PathBuf::new(), &mut rel_files) + .map_err(|e| CliError::new(format!("walk source: {e}")))?; + + let mut files = Vec::with_capacity(rel_files.len()); + let mut total_bytes = 0u64; + for rel in &rel_files { + let abs = src.join(rel); + let bytes = std::fs::read(&abs) + .map_err(|e| CliError::new(format!("read {}: {e}", abs.display())))?; + let hash = blake3::hash(&bytes).to_hex().to_string(); + let dest = out.join(rel); + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| CliError::new(format!("mkdir {}: {e}", parent.display())))?; + } + std::fs::write(&dest, &bytes) + .map_err(|e| CliError::new(format!("write {}: {e}", dest.display())))?; + total_bytes += bytes.len() as u64; + files.push(FileEntry { + path: rel_to_slash(rel), + size: bytes.len() as u64, + blake3: hash, + }); + } + + let manifest = Manifest { + manifest_version: MANIFEST_VERSION, + created_unix_secs: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + checkpoint_seq, + file_count: files.len(), + total_bytes, + files, + }; + let manifest_json = serialize(&manifest, pretty)?; + std::fs::write(out.join(BACKUP_MANIFEST), &manifest_json) + .map_err(|e| CliError::new(format!("write manifest: {e}")))?; + + let summary = serde_json::json!({ + "backed_up": src.display().to_string(), + "destination": out.display().to_string(), + "checkpoint_seq": manifest.checkpoint_seq, + "file_count": manifest.file_count, + "total_bytes": manifest.total_bytes, + }); + Ok((render(&summary, pretty)?, 0)) +} + +/// `tidalctl restore --from --path `: verify the backup's BLAKE3 +/// manifest, then copy every file into `target`. Refuses a non-empty target so a +/// restore never overwrites a live data dir (the destructive-op guard). +pub(crate) fn run_restore( + target: &Path, + from: &Path, + pretty: bool, +) -> Result<(String, i32), CliError> { + let manifest_path = from.join(BACKUP_MANIFEST); + let manifest_bytes = std::fs::read(&manifest_path).map_err(|e| { + CliError::new(format!( + "read manifest {}: {e} (is --from a tidalctl backup?)", + manifest_path.display() + )) + })?; + let manifest: Manifest = serde_json::from_slice(&manifest_bytes) + .map_err(|e| CliError::new(format!("parse manifest: {e}")))?; + if manifest.manifest_version != MANIFEST_VERSION { + return Err(CliError::new(format!( + "unsupported backup manifest version {} (this tidalctl writes/reads v{MANIFEST_VERSION})", + manifest.manifest_version + ))); + } + + // Destructive-op guard: never restore over a non-empty target (it could be a + // live data dir). The operator restores into a fresh dir, then points the node at it. + if target.exists() + && std::fs::read_dir(target) + .map_err(|e| CliError::new(format!("read target {}: {e}", target.display())))? + .next() + .is_some() + { + return Err(CliError::new(format!( + "target {} is not empty; restore into a fresh directory (refusing to overwrite)", + target.display() + ))); + } + + // Phase 1: verify EVERY file's hash against the manifest BEFORE writing + // anything, so a corrupt backup fails the restore whole rather than leaving a + // half-written target. + for entry in &manifest.files { + let src = from.join(slash_to_rel(&entry.path)); + let bytes = std::fs::read(&src) + .map_err(|e| CliError::new(format!("read backup file {}: {e}", src.display())))?; + let hash = blake3::hash(&bytes).to_hex().to_string(); + if hash != entry.blake3 { + return Err(CliError::new(format!( + "backup integrity check FAILED for {}: manifest {} != actual {}", + entry.path, entry.blake3, hash + ))); + } + } + + // Phase 2: place the verified files into the target. + std::fs::create_dir_all(target) + .map_err(|e| CliError::new(format!("create target {}: {e}", target.display())))?; + let mut restored = 0usize; + for entry in &manifest.files { + let src = from.join(slash_to_rel(&entry.path)); + let dest = target.join(slash_to_rel(&entry.path)); + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| CliError::new(format!("mkdir {}: {e}", parent.display())))?; + } + std::fs::copy(&src, &dest) + .map_err(|e| CliError::new(format!("copy {}: {e}", dest.display())))?; + restored += 1; + } + + let summary = serde_json::json!({ + "restored": target.display().to_string(), + "from": from.display().to_string(), + "checkpoint_seq": manifest.checkpoint_seq, + "files_verified_and_restored": restored, + "note": "point a stopped node at this dir; under ack=quorum followers catch up via the live stream", + }); + Ok((render(&summary, pretty)?, 0)) +} + +/// Recursively collect file paths (relative to `root`), skipping the lock file +/// and any previously-written backup manifest. Directories are recursed; only +/// regular files are recorded. +fn collect_files(root: &Path, rel: PathBuf, out: &mut Vec) -> std::io::Result<()> { + let dir = root.join(&rel); + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let name = entry.file_name(); + let child_rel = rel.join(&name); + let file_type = entry.file_type()?; + if file_type.is_dir() { + collect_files(root, child_rel, out)?; + } else if file_type.is_file() { + // Skip process-ownership + a stale manifest from a prior backup-in-place. + if name == LOCK_FILE || name == BACKUP_MANIFEST { + continue; + } + out.push(child_rel); + } + } + Ok(()) +} + +/// Render a relative path with forward slashes for a portable manifest. +fn rel_to_slash(rel: &Path) -> String { + rel.components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/") +} + +/// Inverse of [`rel_to_slash`]: a manifest path → a platform `PathBuf`. +fn slash_to_rel(path: &str) -> PathBuf { + path.split('/').collect() +} + +fn serialize(manifest: &Manifest, pretty: bool) -> Result { + if pretty { + serde_json::to_string_pretty(manifest) + } else { + serde_json::to_string(manifest) + } + .map_err(|e| CliError::new(format!("serialize manifest: {e}"))) +} + +fn render(value: &serde_json::Value, pretty: bool) -> Result { + if pretty { + serde_json::to_string_pretty(value) + } else { + serde_json::to_string(value) + } + .map_err(|e| CliError::new(format!("serialize summary: {e}"))) +} diff --git a/tidalctl/src/commands/mod.rs b/tidalctl/src/commands/mod.rs index f52379f..3e7df89 100644 --- a/tidalctl/src/commands/mod.rs +++ b/tidalctl/src/commands/mod.rs @@ -1,10 +1,11 @@ -//! The five `tidalctl` subcommands, one module each. +//! The `tidalctl` subcommands, one module each. //! //! Each module exposes a single `run(...)` entry point returning //! `(rendered_output, exit_code)`; `main` dispatches to them and prints the //! output. The exit-code contract (0 = ok/empty, 1 = usage/internal, 2 = //! degraded/unreadable) is documented in the crate-root `//!` header. +pub(crate) mod backup; pub(crate) mod diagnostics; pub(crate) mod paths; pub(crate) mod recover; diff --git a/tidalctl/src/main.rs b/tidalctl/src/main.rs index ea5bc98..abc07f6 100644 --- a/tidalctl/src/main.rs +++ b/tidalctl/src/main.rs @@ -73,6 +73,10 @@ struct CliArgs { path: PathBuf, pretty: bool, verify_only: bool, + /// Backup destination (`backup --out `). + out: Option, + /// Restore source (`restore --from `). + from: Option, } enum Command { @@ -81,6 +85,8 @@ enum Command { Recover, Diagnostics, ScopeStats, + Backup, + Restore, } /// A user-facing CLI failure, rendered as an `{"error": ...}` JSON envelope on @@ -123,6 +129,8 @@ fn parse_args(args: &[String]) -> Result { "recover" => Command::Recover, "diagnostics" => Command::Diagnostics, "scope-stats" => Command::ScopeStats, + "backup" => Command::Backup, + "restore" => Command::Restore, "--help" | "-h" | "help" => return Err(CliError::new(usage())), other => { return Err(CliError::new(format!( @@ -133,6 +141,8 @@ fn parse_args(args: &[String]) -> Result { }; let mut path: Option = None; + let mut out: Option = None; + let mut from: Option = None; let mut pretty = false; let mut verify_only = false; let mut i = 2; @@ -146,6 +156,20 @@ fn parse_args(args: &[String]) -> Result { } path = Some(PathBuf::from(&args[i])); } + "--out" => { + i += 1; + if i >= args.len() { + return Err(CliError::new("--out requires a value")); + } + out = Some(PathBuf::from(&args[i])); + } + "--from" => { + i += 1; + if i >= args.len() { + return Err(CliError::new("--from requires a value")); + } + from = Some(PathBuf::from(&args[i])); + } "--pretty" => { pretty = true; } @@ -166,17 +190,24 @@ fn parse_args(args: &[String]) -> Result { path, pretty, verify_only, + out, + from, }) } fn usage() -> String { - "Usage: tidalctl --path [--pretty]\n\n\ + "Usage: tidalctl --path [--out ] [--from ] [--pretty]\n\n\ Commands:\n \ status Report WAL state, checkpoint, and directory layout\n \ paths Report resolved directory paths and existence\n \ recover Diagnose WAL state for crash recovery (--verify-only)\n \ diagnostics Print health summary (human-readable or JSON)\n \ - scope-stats Tally WAL signal events by governance scope (M9)\n\n\ + scope-stats Tally WAL signal events by governance scope (M9)\n \ + backup Copy a data dir to --out with a BLAKE3 manifest (m11p8)\n \ + restore Verify a backup (--from) and restore it into --path (m11p8)\n\n\ + Backup/restore operate on a data dir AT REST (a stopped/drained node):\n \ + tidalctl backup --path /data/node --out /backups/node-A\n \ + tidalctl restore --from /backups/node-A --path /data/node-new\n\n\ Exit codes: 0 = ok/empty, 1 = usage/internal error, 2 = degraded/unreadable\n \ (WAL, checkpoint, or a derived index exists but could not be read)." .to_string() @@ -195,5 +226,19 @@ fn run(args: &[String]) -> Result<(String, i32), CliError> { Command::Recover => commands::recover::run(&cli.path, cli.pretty, cli.verify_only), Command::Diagnostics => commands::diagnostics::run(&cli.path, cli.pretty), Command::ScopeStats => commands::scope_stats::run(&cli.path, cli.pretty), + Command::Backup => { + let out = cli + .out + .as_ref() + .ok_or_else(|| CliError::new("backup requires --out "))?; + commands::backup::run_backup(&cli.path, out, cli.pretty) + } + Command::Restore => { + let from = cli + .from + .as_ref() + .ok_or_else(|| CliError::new("restore requires --from "))?; + commands::backup::run_restore(&cli.path, from, cli.pretty) + } } } diff --git a/tidalctl/tests/cli.rs b/tidalctl/tests/cli.rs index bacc4de..aee4225 100644 --- a/tidalctl/tests/cli.rs +++ b/tidalctl/tests/cli.rs @@ -1206,3 +1206,112 @@ fn scope_stats_clean_wal_reports_complete() { "clean WAL has no inconsistencies: {stdout}" ); } + +// ── m11p8 backup / restore ──────────────────────────────────────────────────── + +/// A real round-trip: back up a data dir at rest, then restore it into a fresh +/// dir. The manifest carries the recovered checkpoint cursor, the restore +/// verifies every file's BLAKE3, and the restored WAL segments match the source. +#[test] +fn backup_then_restore_roundtrips() { + let home = home_with_wal_data(); + let scratch = TempTidalHome::new().unwrap(); + let backup_dir = scratch.path().join("backup"); + let restore_dir = scratch.path().join("restored"); + + // Back up. + let out = tidalctl_bin() + .args(["backup", "--path"]) + .arg(home.path()) + .args(["--out"]) + .arg(&backup_dir) + .output() + .unwrap(); + assert!(out.status.success(), "backup failed: {out:?}"); + let stdout = String::from_utf8_lossy(&out.stdout); + let json: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + assert_eq!( + json["checkpoint_seq"], 3, + "manifest records the cursor: {stdout}" + ); + assert!( + json["file_count"].as_u64().unwrap() >= 1, + "backed up some files: {stdout}" + ); + assert!( + backup_dir.join("BACKUP_MANIFEST.json").exists(), + "manifest written" + ); + + // Restore into a fresh dir. + let out = tidalctl_bin() + .args(["restore", "--from"]) + .arg(&backup_dir) + .args(["--path"]) + .arg(&restore_dir) + .output() + .unwrap(); + assert!(out.status.success(), "restore failed: {out:?}"); + + // The restored WAL segments byte-match the source (gap-free recovery). + let src_segs = std::fs::read_dir(home.path().join("wal")) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.path().extension().is_some_and(|x| x == "seg")) + .count(); + let restored_segs = std::fs::read_dir(restore_dir.join("wal")) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.path().extension().is_some_and(|x| x == "seg")) + .count(); + assert_eq!( + src_segs, restored_segs, + "restored WAL has the same segment count as the source" + ); + assert!(src_segs >= 1, "source had WAL segments to restore"); +} + +/// Restore must REFUSE a backup whose bytes were tampered with after the +/// manifest was written (the BLAKE3 integrity guard). +#[test] +fn restore_rejects_corrupted_backup() { + let home = home_with_wal_data(); + let scratch = TempTidalHome::new().unwrap(); + let backup_dir = scratch.path().join("backup"); + let restore_dir = scratch.path().join("restored"); + + let out = tidalctl_bin() + .args(["backup", "--path"]) + .arg(home.path()) + .args(["--out"]) + .arg(&backup_dir) + .output() + .unwrap(); + assert!(out.status.success(), "backup failed: {out:?}"); + + // Corrupt one archived WAL segment so its bytes no longer match the manifest. + let seg = std::fs::read_dir(backup_dir.join("wal")) + .unwrap() + .filter_map(Result::ok) + .map(|e| e.path()) + .find(|p| p.extension().is_some_and(|x| x == "seg")) + .expect("a segment to corrupt"); + std::fs::write(&seg, b"corrupted-not-the-original-bytes").unwrap(); + + let out = tidalctl_bin() + .args(["restore", "--from"]) + .arg(&backup_dir) + .args(["--path"]) + .arg(&restore_dir) + .output() + .unwrap(); + assert!( + !out.status.success(), + "restore must reject a corrupted backup" + ); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("integrity check FAILED"), + "error must name the integrity failure: {stderr}" + ); +}