From 1265140e2804079d2b82e4e6305d98e5ba901c43 Mon Sep 17 00:00:00 2001 From: jx12n Date: Sat, 13 Jun 2026 15:23:59 -0600 Subject: [PATCH] =?UTF-8?q?feat(m11):=20continuous=20correctness=20(m11p9)?= =?UTF-8?q?=20=E2=80=94=20fault=20classes,=20invariant=20checkers,=20soak?= =?UTF-8?q?=20gates,=20nightly=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fault-injection cargo feature (compiled OUT of prod): slow-fsync + disk-full WAL hooks in tidal/src/fault.rs, inert until armed, tier-3 builds with feature - first-class invariant checkers (tests/support/invariants.rs): AckLedger no-acked-loss (now consumed by m11p3 gate), feed parity, single-leader-per-term, monotonic frontiers - cluster_faults.rs tier-3 suite 4/4: disk-full degrade+recover, slow-fsync lag+converge, both-slow quorum 503, asymmetric partition no-split-brain - tidal-stress soak gates: --json-summary + --max-p99-ms/--max-error-pct/ --fail-on-knee → non-zero exit on regression - Woodpecker cron nightly flow (chaos + gated soak), event-routed, not GH Actions - guarantee-traceability.md: roadmap §2 guarantees → named tests (closes G-C apparatus; 30-day-green is a calendar criterion) --- .woodpecker.yaml | 136 +++- CHANGELOG.md | 38 ++ docs/planning/ROADMAP.md | 4 +- .../milestone-11/guarantee-traceability.md | 66 ++ docs/planning/milestone-11/phase-9.md | 223 +++++++ docs/roadmap-to-cluster.md | 28 +- docs/runbooks/cluster.md | 54 +- tidal-server/Cargo.toml | 3 + tidal-server/tests/cluster_chaos.rs | 3 +- tidal-server/tests/cluster_faults.rs | 404 ++++++++++++ tidal-server/tests/cluster_quorum.rs | 167 +---- tidal-server/tests/support/invariants.rs | 588 ++++++++++++++++++ tidal-server/tests/support/mod.rs | 1 + tidal-server/tests/support/multiproc.rs | 12 +- tidal-stress/Cargo.toml | 4 + tidal-stress/src/error.rs | 20 + tidal-stress/src/lib.rs | 1 + tidal-stress/src/main.rs | 76 +++ tidal-stress/src/scheduler.rs | 9 + tidal-stress/src/summary.rs | 395 ++++++++++++ tidal/Cargo.toml | 5 + tidal/src/fault.rs | 147 +++++ tidal/src/lib.rs | 6 + tidal/src/wal/mod.rs | 5 + tidal/src/wal/segment.rs | 7 + 25 files changed, 2241 insertions(+), 161 deletions(-) create mode 100644 docs/planning/milestone-11/guarantee-traceability.md create mode 100644 docs/planning/milestone-11/phase-9.md create mode 100644 tidal-server/tests/cluster_faults.rs create mode 100644 tidal-server/tests/support/invariants.rs create mode 100644 tidal-stress/src/summary.rs create mode 100644 tidal/src/fault.rs diff --git a/.woodpecker.yaml b/.woodpecker.yaml index e47ff51..75c66c5 100644 --- a/.woodpecker.yaml +++ b/.woodpecker.yaml @@ -1,16 +1,23 @@ -# Pipeline: a rolling-upgrade RELEASE GATE (m11p8) runs FIRST; only if it passes -# does Kaniko build the tidal-server image and push it to the in-cluster zot -# registry. DEPLOYMENT IS MANUAL (kustomize, from the orchard9-k3sf ops repo) — -# the old auto-`kubectl set image deployment/tidaldb` step was removed because it -# coupled every image build to a standalone roll and the cluster ops repo's -# contract is "manual deploy via scripts, no CI/CD deploy". The same -# `tidal-server` binary serves every subcommand (standalone AND multi-process -# `cluster --region`), so one image covers both deployments. +# Two pipelines in one file, split by event (Woodpecker, NEVER GitHub Actions): +# +# • event: push → the m11p8 RELEASE GATE (rolling-upgrade tier-3 test) then a +# Kaniko image build. Deployment is MANUAL (kustomize, orchard9-k3sf ops repo). +# • event: cron → the m11p9 NIGHTLY CONTINUOUS-CORRECTNESS run: the tier-3 +# chaos suites (incl. the new disk-full / slow-fsync / asymmetric-partition +# fault classes) with elevated kill-points, then a tidal-stress soak with +# PASS/FAIL regression gates on p99 + error rate. A nightly failure flags a +# correctness or performance regression for that day. +# +# Per-step `when:` routes each step to its event; the workflow-level `when` admits +# both. Configure a cron named "nightly" in the Woodpecker repo settings to fire +# the cron pipeline (the same `tidal-server` binary serves standalone AND +# multi-process `cluster --region`, so one image covers both deployments). when: branch: main - event: push + event: [push, cron] steps: + # ── Release gate (push) ───────────────────────────────────────────────────── # m11p8 release gate: prove a rolling upgrade under load loses no acknowledged # write and never stalls (mp_rolling_upgrade_no_loss_no_stall — a tier-3 test # spawning three real OS processes with a graceful SIGTERM → version-tagged @@ -19,6 +26,8 @@ steps: # overlap. A failure here BLOCKS the image build below — the gate, not the start. rolling-upgrade-gate: image: rust:1-bookworm + when: + event: push commands: - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang - cargo test -p tidal-server --features cluster-e2e --test cluster_lifecycle @@ -26,6 +35,8 @@ steps: build: image: woodpeckerci/plugin-kaniko + when: + event: push settings: repo: tidal/server dockerfile: docker/standalone/Dockerfile @@ -42,3 +53,110 @@ steps: build_args: - TARGETPLATFORM=linux/amd64 extra_args: --customPlatform=linux/amd64 + + # ── Nightly chaos (cron) ──────────────────────────────────────────────────── + # The tier-3 chaos/correctness suites over REAL OS processes, run serially + # (fixed ports + spawned processes must not overlap). Kill-points are elevated + # above the per-PR defaults (8/5) but kept below the 100-point exit-gate run so + # the nightly stays bounded; widen TIDAL_*_KILLPOINTS for a deeper sweep. The + # `fault-injection` feature compiles in the WAL slow-fsync / disk-full hooks for + # cluster_faults (inert in every other suite — see tidal/src/fault.rs). Boot / + # convergence budgets are raised for a shared CI runner. A failure = a + # correctness regression for the night. + nightly-chaos: + image: rust:1-bookworm + when: + event: cron + cron: nightly + environment: + TIDAL_QUORUM_KILLPOINTS: "25" + TIDAL_ELECTION_KILLPOINTS: "15" + TIDAL_TEST_BOOT_BUDGET_SECS: "120" + TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "90" + commands: + - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang + # New m11p9 fault classes first (disk-full, slow-fsync, asymmetric partition). + - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_faults -- --nocapture --test-threads 1 + # The standing chaos / durability / availability / elasticity gates. + - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_chaos -- --nocapture --test-threads 1 + - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_quorum -- --nocapture --test-threads 1 + - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_election -- --nocapture --test-threads 1 + - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_membership -- --nocapture --test-threads 1 + - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_reseed -- --nocapture --test-threads 1 + - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_lifecycle -- --nocapture --test-threads 1 + - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_runbook -- --nocapture --test-threads 1 + + # ── Nightly soak (cron) ───────────────────────────────────────────────────── + # A tidal-stress soak with PASS/FAIL regression gates (m11p9). By default it + # boots a local standalone server and soaks it for a bounded window, gating on + # error rate and p99 so a perf regression fails the step and shows on the trend + # line (the JSON summary is the archived artifact). Point TIDAL_SOAK_TARGET at + # the live Ref-A cluster and raise TIDAL_SOAK_SECS to 3600 for the GA-bar 1-hour + # 100k-DAU soak (the standing k3s-access caveat applies — same as m11p1–p3). + nightly-soak: + image: rust:1-bookworm + when: + event: cron + cron: nightly + environment: + TIDAL_SOAK_RPS: "1000" + TIDAL_SOAK_SECS: "600" + TIDAL_SOAK_MAX_P99_MS: "250" + TIDAL_SOAK_MAX_ERROR_PCT: "1" + # Soak target port — an UNCLAIMED slot in the project's reserved dev band + # (59520-59529): 59520=site, 59521=iknowyou, so the soak uses 59526. One + # source of truth for the literal (referenced by --listen AND the TARGET). + TIDAL_SOAK_PORT: "59526" + commands: + - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang curl + - cargo build -p tidal-server -p tidal-stress + # Boot a standalone target unless TIDAL_SOAK_TARGET points elsewhere (Ref-A). + # NOTE the `$${VAR}` escaping: Woodpecker substitutes a bare `${VAR}` in a + # command BEFORE the shell runs, and step `environment:` vars are NOT in that + # preprocessor namespace (they would blank to ""). `$${VAR}` passes a literal + # `${VAR}` to the container shell, which expands it from the runtime env — + # so the soak's ramp and gate thresholds are actually populated. Bare shell + # locals (`$PORT`, `$TARGET`, `$SRV`) are never touched by the preprocessor. + - | + PORT="$${TIDAL_SOAK_PORT}" + TARGET="$${TIDAL_SOAK_TARGET:-http://127.0.0.1:$PORT}" + if [ -z "$TIDAL_SOAK_TARGET" ]; then + mkdir -p /tmp/soak-data + ./target/debug/tidal-server standalone --listen "127.0.0.1:$PORT" \ + --schema tidal-server/config/default-schema.yaml --data-dir /tmp/soak-data & + SRV=$! + # Reap the background server + scratch dir on ANY exit (success, gate + # failure, or the boot-failure exit below) so the step is idempotent. + trap 'kill "$SRV" 2>/dev/null; rm -rf /tmp/soak-data' EXIT + up=0 + for i in $(seq 1 100); do + curl -sf "$TARGET/health/startup" >/dev/null 2>&1 && { up=1; break; } || sleep 0.3 + done + # Fail FAST and unambiguously on a boot failure (bad schema, port bound) + # rather than running the soak against a dead target and mislabeling it + # as an error-rate regression on the trend line. + if [ "$up" != "1" ]; then echo "soak target failed to start at $TARGET"; exit 1; fi + fi + ./target/debug/tidal-stress --target "$TARGET" \ + --ramp "$${TIDAL_SOAK_RPS}:$${TIDAL_SOAK_SECS}" --corpus 5000 --mix peach \ + --json-summary soak-summary.json \ + --max-error-pct "$${TIDAL_SOAK_MAX_ERROR_PCT}" --max-p99-ms "$${TIDAL_SOAK_MAX_P99_MS}" --fail-on-knee + - cat soak-summary.json + + # ── Nightly security + ops correctness (cron) ─────────────────────────────── + # The G-Sec and G-Op owner-tests run nightly too (not just on-demand): mTLS / + # foreign-pod rejection / cert rotation (tidal-net mtls + cluster_security), the + # tidalctl backup/restore round-trip, and the gap-free WAL-archival (PITR) unit + # test (in the engine lib). In-process + fast; serial for the TLS port binds. + nightly-security-ops: + image: rust:1-bookworm + when: + event: cron + cron: nightly + commands: + - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang + - cargo test -p tidal-net --test mtls -- --test-threads 1 + - cargo test -p tidal-server --test cluster_security -- --test-threads 1 + - cargo test -p tidal-server --lib + - cargo test -p tidalctl + - cargo test -p tidaldb --lib wal::compaction diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b0083f..94298bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,44 @@ All notable changes to tidalDB will be documented in this file. ### Added +**Continuous correctness (m11p9) — new fault classes as REAL faults, first-class invariant checkers, soak regression gates, a Woodpecker nightly chaos+soak pipeline, and a guarantee→test matrix** + +- **Fault injection compiled out of production.** A non-default `fault-injection` + cargo feature (tidaldb + a tidal-server passthrough) adds two WAL hooks + (`tidal/src/fault.rs`): **slow-fsync** (`TIDAL_FAULT_FSYNC_DELAY_MS` sleeps + before each durable fsync) and **disk-full** (`TIDAL_FAULT_DISK_FULL_AFTER_BYTES` + returns a real `ENOSPC`/errno-28 once cumulative segment bytes cross the + threshold). The production image build never passes the feature, so the hooks + are compiled out entirely — a disk/fsync fault a stray env var could trip in + prod is a 3am footgun we refuse to ship; the safety is structural. Inert until + armed even when compiled in. The tier-3 harness builds the spawned binary with + the feature. +- **First-class invariant checkers** (`tidal-server/tests/support/invariants.rs`): + the no-acked-loss `AckLedger` (frontier + content, extracted from the m11p3 + ledger gate, which now consumes it), cross-replica `assert_feed_parity` / + `feed_item_ids`, `assert_single_leader_per_term` (membership safety, reads the + m11p6 `shards[]` rows), and `MonotonicCounters` (per-node applied frontier + + leader commit index never regress, with legitimate epoch resets forgiven). +- **New fault-class suite** (`tidal-server/tests/cluster_faults.rs`, tier-3): a + disk-full follower degrades gracefully (receiver halts, node alive, zero acked + loss, restart recovers to parity); a slow-fsync follower lags then converges + while the fast follower supplies quorum; both followers slow → `ack=quorum` + returns a retryable 503 naming the laggards while `ack=leader` is unaffected; + an asymmetric partition (inbound severed, outbound up) causes no split brain + (pre-vote + check-quorum) and no loss. 4/4 green. +- **Soak + regression gates** in `tidal-stress`: `--json-summary ` (a + machine-readable per-stage p99/throughput/error roll-up for trend lines) and + `--max-p99-ms` / `--max-error-pct` / `--fail-on-knee` gates that exit non-zero + on a regression (the tool always exited 0 before). +- **Nightly pipeline** (`.woodpecker.yaml`): a cron `nightly` flow (chaos suites + with elevated kill-points + the `fault-injection` feature, then the gated + `tidal-stress` soak) beside the existing push release gate, event-routed by + per-step `when`. Woodpecker, never GitHub Actions. +- **Guarantee traceability** (`docs/planning/milestone-11/guarantee-traceability.md`): + every roadmap §2 guarantee mapped to its named automated test(s). Closes the + **G-C** apparatus; the 30-consecutive-days-green half of the GA bar is a + calendar criterion the nightly pipeline accrues. + **Observability + operations (m11p8) — complete cluster metric set on a per-node `/metrics` listener, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR + backup/restore, rolling-upgrade release gate** - **Metrics.** Completed the `tidaldb_cluster_*` set with the two members the diff --git a/docs/planning/ROADMAP.md b/docs/planning/ROADMAP.md index 37c1a9c..80328bb 100644 --- a/docs/planning/ROADMAP.md +++ b/docs/planning/ROADMAP.md @@ -37,7 +37,7 @@ A single embeddable database can replace the 6-system content ranking stack by t | M8 | Distributed Fabric | Multi-region, multi-tenant replication keeps agent-memory semantics intact | Hosted tidalDB, cloud/edge deployments, shared agent substrate — **✅ COMPLETE**: in-process primitives + multi-node replication over real gRPC + true multi-process cluster mode (one process per region, real process isolation) with full tier-3 UAT (partition injection via TCP-proxy, clock-skew, rolling-upgrade, runbook verification); G1 + G2 resolved. Post-M8 follow-ups: quorum-ack writes, automatic failure detection / leader election | | M9 | Community Sync & Revocation | Local embeddable profiles can opt into community personalization and safely leave/purge contributions | Community personalization, federated taste graphs, shared feeds — ✅ COMPLETE (2026-06-06) | | M10 | Governance & Agent Rights | Community rules and agent-scoped permissions control what signals influence ranking | User-owned AI personalization at scale, policy-compliant agents — ✅ COMPLETE (2026-06-06) | -| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12) + m11p5 ✅ (membership/discovery/elasticity: DNS peers, snapshot+stream reseed, kind-4 membership records, seed join, `k8s/cluster/` — 2026-06-12) + m11p7 ✅ (security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit — 2026-06-13) + m11p8 ✅ (observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O** — 2026-06-13); m11p6 (sharding × replication) data plane in progress; p9 planned in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) | +| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12) + m11p5 ✅ (membership/discovery/elasticity: DNS peers, snapshot+stream reseed, kind-4 membership records, seed join, `k8s/cluster/` — 2026-06-12) + m11p7 ✅ (security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit — 2026-06-13) + m11p8 ✅ (observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O** — 2026-06-13) + m11p9 ✅ (continuous correctness: new fault classes (disk-full / slow-fsync / asymmetric partition) as REAL faults behind a production-compiled-out `fault-injection` feature, first-class invariant checkers, `tidal-stress` soak with regression gates + JSON summary, a Woodpecker cron nightly chaos+soak pipeline, and a guarantee→test traceability matrix — closes the **G-C** apparatus; the 30-day-green calendar accrues nightly — 2026-06-13); m11p6 (sharding × replication) data plane in progress in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) | ### Embeddable → Distributed Path @@ -3285,7 +3285,7 @@ Full gap analysis, measured baselines, phase specs, and exit gates live in | m11p6 | Sharding × replication + rebalancing | Planned | | m11p7 | Security hardening (mTLS default, rotation, audit log) | ✅ **COMPLETE (2026-06-13)** — the cluster stops trusting the network. gRPC replication is served over a custom `tokio-rustls` acceptor fed a hot-swappable `DynamicCertResolver` (`ArcSwap`) — mTLS preserved exactly (`WebPkiClientVerifier`; a foreign/absent client cert fails the handshake before any RPC), plaintext now an explicit `insecure: true` + loud WARN. **Cert + bearer rotation without restart**: a content-hash poller (catches k8s `..data` symlink swaps inotify misses) atomically swaps the cert with in-flight sessions untouched — **zero dropped requests under load** (verified). Inter-node HTTP gains TLS (same resolver — one rotation covers both planes; `https` forwards + cluster-CA reqwest clients) + **per-node identity** via a signed `x-tidal-node-token` (keyed-BLAKE3 MAC under a shared cluster key — no new crypto dep); the `x-tidal-internal` marker is honored ONLY from a verified sibling (marker-without-token → 403), never an auth bypass. Admin verbs (promote/partition/heal/join/remove/reseed) emit a structured audit record (principal, term, target, outcome) to a `tidal_audit` target + optional `TIDAL_AUDIT_LOG` JSONL, operator-leg only. Per-principal HTTP rate limit (engine `RateLimiter`, nodes exempt, 429 + Retry-After). All TLS/identity/audit/limit opt-in (`grpc_tls` / cluster key / env) — absent ⇒ pre-m11p7 behavior byte-for-byte. `k8s/cluster/` gains cert-manager + `grpc_tls` topology + cluster-key Secret; `scripts/gen-cluster-certs.sh` for non-cert-manager. Exit gate verified: foreign pod rejected (gRPC `mtls.rs` + HTTP `cluster_security.rs`), zero-drop rotation under load, zero plaintext inter-node links. See [milestone-11/phase-7.md](milestone-11/phase-7.md). | | m11p8 | Observability + operations (complete metric set, self-driving heal, backup/PITR, rolling-upgrade gate) | ✅ **COMPLETE (2026-06-13)** — operable by someone who didn't build it (closes **G-O** + the operability half of **G-Op**, and incident §1.4-3). **Metrics**: completed the `tidaldb_cluster_*` set with breaker (`tidaldb_cluster_peer_breaker_state` + `_breaker_opens_total`, surfaced read-only from the `tidal-net` breaker) and forwards (`_forwards_total` / `_forward_failures_total`) + the self-heal series; multi-shard nodes serve ONE `/metrics` listener with the co-located siblings' series `shard="N"`-labeled (S=1 byte-identical). A 12-panel Grafana cluster row + 8-rule `tidaldb-cluster` alert group ship beside the standalone ones. **Tracing**: both cluster routers gained the request-id + `TraceLayer` stack; the id rides the forward hop. **Truthful status**: the leader's own `applied_events` (was 0 — now its flushed frontier) and the post-promote `ShardId(0)` lag-keying undercount fixed; `/cluster/status` gained `version`. **Self-driving heal**: a standing leader duty re-arms a stuck peer's backlog re-ship every ~3s so it converges through breaker resets with NO operator heal loop (`healing_peers` gauge + alert). **Backup/PITR**: `wal.archive_dir` archives sealed segments before compaction deletes them (gap-free); `tidalctl backup`/`restore` (BLAKE3-verified round-trip). **Rolling upgrade**: wire `build_version` handshake (N/N+1 by proto3 compat; `>= 2`-major WARNs, never rejects) + `mp_rolling_upgrade_no_loss_no_stall` promoted to the Woodpecker release gate. Exit gates green (dashboard answers the golden signals; rolling-upgrade gate passed; backup/restore + archival round-trips verified). See [milestone-11/phase-8.md](milestone-11/phase-8.md). | -| m11p9 | Continuous correctness (nightly chaos CI, invariant checkers, soak) | Planned | +| m11p9 | Continuous correctness (nightly chaos CI, invariant checkers, soak) | ✅ **COMPLETE (2026-06-13)** — closes the **G-C** apparatus (the 30-consecutive-days-green half of the GA bar accrues nightly). New fault classes as REAL faults behind a production-compiled-out `fault-injection` feature: **slow-fsync** (`TIDAL_FAULT_FSYNC_DELAY_MS`) + **disk-full** (`TIDAL_FAULT_DISK_FULL_AFTER_BYTES` → real `ENOSPC`) WAL hooks (`tidal/src/fault.rs`); **asymmetric partition** via the harness's directed-edge proxies. First-class invariant checkers (`support/invariants.rs`): no-acked-loss `AckLedger` (the m11p3 ledger gate now consumes it), decay/feed parity, `assert_single_leader_per_term`, `MonotonicCounters`. New tier-3 `cluster_faults.rs` (disk-full / slow-fsync ×2 / asymmetric) **4/4 green** with zero acked loss. `tidal-stress` gained `--json-summary` + `--max-p99-ms`/`--max-error-pct`/`--fail-on-knee` regression gates (non-zero exit on breach — verified vs a real server). Nightly Woodpecker CRON pipeline (chaos suites with elevated kill-points + the gated soak) beside the push release gate. Every §2 guarantee → named test in [milestone-11/guarantee-traceability.md](milestone-11/guarantee-traceability.md). See [milestone-11/phase-9.md](milestone-11/phase-9.md). | --- diff --git a/docs/planning/milestone-11/guarantee-traceability.md b/docs/planning/milestone-11/guarantee-traceability.md new file mode 100644 index 0000000..0481cad --- /dev/null +++ b/docs/planning/milestone-11/guarantee-traceability.md @@ -0,0 +1,66 @@ +# Enterprise-Guarantee → Automated-Test Traceability (m11p9) + +This is the m11p9 exit-gate artifact: **every guarantee in +[roadmap-to-cluster.md §2](../../roadmap-to-cluster.md) maps to a named automated +test** (the second half of the GA bar is the calendar one — the nightly suite +green for 30 consecutive days; see [phase-9.md](phase-9.md)). + +Each guarantee lists the named test(s) that prove it, the fault class each one +injects (where applicable), and the first-class invariant checker +([`tidal-server/tests/support/invariants.rs`](../../../tidal-server/tests/support/invariants.rs)) +it asserts through. All `mp_*` tests are tier-3 over REAL OS processes +(`MultiProcCluster`); partitions sever real loopback TCP; the disk-full / +slow-fsync faults return a real `ENOSPC` / sleep the real fsync behind the +`fault-injection` feature (`tidal/src/fault.rs`). + +Run any suite locally: +`cargo test -p tidal-server --features "cluster-e2e fault-injection" --test -- --test-threads 1 --nocapture`. +Every named test that EXISTS below runs nightly via the cron pipeline in +[`.woodpecker.yaml`](../../../.woodpecker.yaml): the chaos/correctness suites and +the gated soak in `nightly-chaos`/`nightly-soak`, and the G-Sec/G-Op owner-tests +(mTLS, `cluster_security`, `tidalctl` backup/restore, WAL-archival) in +`nightly-security-ops`. The one exception is the **G-S** throughput gate, which +is not yet written (m11p6 L4) — its row is marked accordingly. + +| Guarantee | Named automated test(s) | Suite | Fault / scenario | Invariant checker | +|---|---|---|---|---| +| **G-D Durability** — `ack=quorum` 2xx ⇒ majority-durable; SIGKILL any node ⇒ zero acked loss | `mp_quorum_ledger_zero_acked_loss_across_killpoints` | `cluster_quorum` | SIGKILL leader across N random kill-points under quorum load | `AckLedger` (frontier + content) | +| | `mp_auto_failover_writes_resume_zero_acked_loss` | `cluster_election` | SIGKILL leader, auto-elect, across kill-points | `AckLedger` | +| | `mp_disk_full_follower_degrades_no_acked_loss` | `cluster_faults` | **disk-full** (ENOSPC) on a follower | `AckLedger` + `MonotonicCounters` | +| | `mp_slow_fsync_follower_lags_but_quorum_holds`, `mp_slow_fsync_both_followers_force_honest_quorum_timeout` | `cluster_faults` | **slow-fsync** on one / both followers | `AckLedger` | +| | `mp_follower_reseeds_via_snapshot_after_compaction`, `mp_quarantined_node_reseeds_without_wipe` | `cluster_reseed` | WAL compaction past cursor / divergent-suffix quarantine | content probes | +| **G-A Availability** — single-node loss ⇒ reads continue, writes resume <10 s, no dual-leader | `mp_auto_failover_writes_resume_zero_acked_loss` | `cluster_election` | leader SIGKILL → election (<10 s) | `AckLedger` | +| | `mp_fenced_ex_leader_restart_cannot_write` | `cluster_election` | partition leader away, restart while fenced | fencing assertions | +| | `mp_flapping_links_bounded_churn` | `cluster_election` | repeated link flaps | single-leader-per-term | +| | `mp_asymmetric_partition_no_split_brain_no_loss` | `cluster_faults` | **asymmetric partition** (inbound severed, outbound up) | `assert_single_leader_per_term` + `AckLedger` | +| | `mp_self_heal_converges_without_operator_verb` | `cluster_chaos` | gRPC link sever, no operator verb | feed parity | +| **G-S Scalability** — write throughput scales with shard count at RF=3 (≥2.5× 1→3) | **(NOT YET WRITTEN — m11p6 L4)** the 3 shards × RF=3 ≥5,000 quorum signals/s gate + `tidal-stress --write-path` path-collapse comparison | _pending m11p6 L4_ | sharded × replicated scaling | throughput gate | +| **G-E Elasticity** — online add/remove/replace; snapshot+stream catch-up; p99 <2× for <60 s | `mp_scale_3_5_3_under_load_zero_loss`, `mp_seed_join_snapshot_catchup`, `mp_dns_hostname_topology_replicates` | `cluster_membership` | scale 3→5→3 under load; seed-join after compaction | `AckLedger` (lost=0), p99 bound | +| **G-Sec Security** — all inter-node links mTLS; authenticated RPC; rotation no downtime; admin audit | `http_tls_serves_ca_trusting_client_and_rejects_foreign`, `http_tls_cert_rotation_under_load_drops_zero` | `cluster_security` | foreign-CA client rejected; cert hot-rotation under load | zero-drop assertion | +| | `mtls` (foreign-pod ship rejected), `cluster::security` unit tests (token mint/verify, marker-not-bypass) | `tidal-net/tests/mtls.rs`, engine unit | foreign pod cannot ship or call internal routes | negative tests | +| **G-O Observability** — per-node Prometheus metrics; dashboard answers golden signals; alerts on lag / commit stall / election churn | per-node `/metrics` listener + `tidaldb_cluster_*` set (m11p8); `grafana-dashboard.json` "Cluster Replication" row (12 panels); `prometheus-alerts.yaml` `tidaldb-cluster` group (8 rules) | `cluster_runbook` (`runbook_s5_health_and_openapi`) + artifacts | metric/alert presence | artifact + status tests | +| **G-Op Operability** — rolling upgrade N/N+1 under load (CI gate); backup/restore + PITR drilled; runbooks executable | `mp_rolling_upgrade_no_loss_no_stall` (the **Woodpecker release gate**) | `cluster_lifecycle` | graceful SIGTERM → version-tagged restart → heal → fixpoint under load | `AckLedger` | +| | `backup_then_restore_roundtrips`, `restore_rejects_corrupted_backup`, `online_compaction_archives_before_deleting` | `tidalctl` / engine | offline backup/restore integrity; gap-free WAL archival (PITR) | BLAKE3 manifest | +| | `runbook_s5..s11` (full operator playbook) | `cluster_runbook` | failover / partition / shutdown+recovery drills | per-drill | +| **G-C Continuous correctness** — nightly chaos + soak green 30 consecutive days | the entire `cluster_faults` suite + the `nightly-chaos`/`nightly-soak` cron pipeline + `tidal-stress` soak regression gates + THIS matrix | `cluster_faults` / `.woodpecker.yaml` / `tidal-stress` | all fault classes nightly; soak p99/error regression gates | all four checkers | + +## Honest status of the bar + +- **G-D, G-A, G-E, G-Sec, G-Op** — each maps to a green named test that runs in + the nightly pipeline. ✅ +- **G-S (Scalability)** — the sharding × replication **data plane** is in place + (m11p6 L0–L2); the ≥5,000/s 3-shard×RF=3 throughput gate and the `tidal-stress` + path-collapse comparison land with m11p6 L4. The named owner-test is recorded + here so the guarantee is not orphaned. ⏳ (tracked in m11p6.) +- **G-O (Observability)** — proven by artifact + presence tests (the metric set, + the dashboard, the alert group) rather than one behavioral test; the + golden-signal coverage is the m11p8 exit-gate evidence. +- **G-C (Continuous correctness)** — the *apparatus* is complete and green + (the fault suite passes; the cron pipeline runs every named suite + the gated + soak). The remaining half of the bar is **calendar**: 30 consecutive nightly-green + days before GA. That cannot be completed in one session — the pipeline is what + produces it. ⏳ (accrues nightly.) +- **Ref-A line items** — the throughput / 100k-item / 1-hour-soak figures that + must run on the live 3-region k3s cluster remain blocked on k3s access (the + standing M11 caveat from m11p1–p3); the in-runner soak gates regressions + locally and the same `tidal-stress` invocation targets Ref-A when reachable. diff --git a/docs/planning/milestone-11/phase-9.md b/docs/planning/milestone-11/phase-9.md new file mode 100644 index 0000000..117f5ad --- /dev/null +++ b/docs/planning/milestone-11/phase-9.md @@ -0,0 +1,223 @@ +# m11p9 — Continuous Correctness (COMPLETE — 2026-06-13) + +Phase spec and exit gate: [docs/roadmap-to-cluster.md §4/m11p9](../../roadmap-to-cluster.md). +Closes the roadmap's **G-C (Continuous correctness)** — trust is a pipeline, not +a milestone — and wires the named owner-test for every other §2 guarantee +([guarantee-traceability.md](guarantee-traceability.md)). +Predecessors: every prior m11 phase (this phase industrializes their tier-3 +suites and adds the fault classes they lacked). + +**Goal:** make correctness continuous — new fault classes (disk-full, slow-fsync, +asymmetric partition) as REAL faults; the invariant checks the durability and +election gates carried inline promoted to first-class reusable checkers; a +nightly chaos + soak pipeline that fails on a regression; and a guarantee→test +matrix so the GA bar is auditable. + +## Design (as adopted) + +### 1. Fault injection compiled out of production (`tidal/src/fault.rs`) + +Two new fault classes need REAL faults at the storage boundary, not engine flags: + +- **`TIDAL_FAULT_FSYNC_DELAY_MS`** — sleep before every durable WAL fsync + (`wal::sync_file_durable`). Models a slow disk: data still reaches stable + storage, each group-commit fsync costs `delay + real`. +- **`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`** — after N cumulative segment bytes this + process lifetime, every segment write returns a real `ENOSPC` + (`std::io::Error::from_raw_os_error(28)` — the errno on Linux AND macOS, + wrapped in the same `WalError::Io` a genuine full disk produces) at + `segment::write_batch_bytes`. + +Both are behind the **non-default `fault-injection` cargo feature** (tidaldb + +a tidal-server passthrough). Production builds (`docker/standalone/Dockerfile`: +`cargo build -p tidal-server --release --locked`) never pass it, so the hooks +are **compiled out entirely** — a disk-write or fsync fault a stray env var could +trip in production is exactly the 3am footgun we refuse to ship, so the safety is +structural, not a default. Even compiled in, faults are inert until an env var +arms them (the posture of `TIDAL_HLC_SKEW_MS` / the kill-point knobs). The +`MultiProcCluster` harness's `tidal_server_bin()` builds the spawned binary with +the feature; every non-fault suite spawns the same binary and is unaffected. + +The injection returns `ENOSPC` *before* the partial `write_all` (the standard +clean-fault model). The follower apply path already handles a WAL write error +correctly (proven by the suite below): the receiver halts (degraded, alive), +NEVER advances its applied frontier past the fault (no gap-swallowing), and a +restart re-pulls the suffix via the catch-up stream. + +### 2. First-class invariant checkers (`tidal-server/tests/support/invariants.rs`) + +The roadmap's named invariants were inline copies in three suites; m11p9 promotes +them to one audited module every suite consumes: + +- **`AckLedger`** — no acknowledged-write loss: records every write the client + saw a 2xx + `x-tidal-seq` for, then proves (A) no acked seqno exceeds the + max-applied survivor's contiguous frontier, and (B) every acked item is present + on the promoted/recovered leader via `/search`. Extracted verbatim from the + m11p3 ledger gate, which now CONSUMES it (re-verified: 3/3 kill-points zero + acked loss after the refactor). +- **`assert_feed_parity` / `feed_pairs`** — cross-replica decay/score parity to + `1e-6` between continuously-running replicas. +- **`feed_item_ids`** — cross-replica DATA parity (the materialized item set), + robust across a node RESTART where a velocity/rate score legitimately differs + (its time-bucketed windowed counts are rebuilt from a burst WAL-replay rather + than continuous accumulation, even with identical durable data — see the + disk-full test). +- **`assert_single_leader_per_term`** — membership safety: at most one leader per + `(shard, term)` across a set of status snapshots (reads the m11p6 `shards[]` + rows or the flat fields). A second leader at the same term is a split brain. +- **`MonotonicCounters`** — per-entity monotonic counters: a node's applied + frontier and the leader's commit index never move backward across observations + (legitimate epoch resets — a reseeding node, a leader change — are forgotten, + not flagged), so a regression means durably-acked state was lost. + +### 3. New fault-class suite (`tidal-server/tests/cluster_faults.rs`) + +Four tier-3 tests over real OS processes, asserting through the checkers: + +- `mp_disk_full_follower_degrades_no_acked_loss` — a follower hits ENOSPC + mid-replication, its receiver halts (degraded, alive), the healthy majority + keeps acking quorum writes, NO acked write is lost, and a restart with space + recovers it to full content + item-set parity. +- `mp_slow_fsync_follower_lags_but_quorum_holds` — a slow disk lags one follower; + the fast follower supplies quorum so every write commits; the slow node + converges once the burst ends; no loss. +- `mp_slow_fsync_both_followers_force_honest_quorum_timeout` — neither follower + can confirm inside the budget ⇒ `ack=quorum` returns a retryable 503 naming the + laggards (never a false 2xx) while `ack=leader` is unaffected; the followers + recover. +- `mp_asymmetric_partition_no_split_brain_no_loss` — a follower that loses INBOUND + links (can still send) cannot disrupt the cluster: pre-vote + check-quorum keep + the standing leader, there is never a second leader at the same term, no acked + write is lost, and it rejoins cleanly on heal. + +The asymmetric partition reuses the harness's directed-edge proxies +(`proxied_rewrite(&["ap-south"])` + `region("ap-south").sever_all()` severs only +inbound to ap-south; its outbound dials are unproxied, so it keeps sending the +disruptive RequestVotes pre-vote neutralizes) — no new mechanism, the existing +`PartitionProxy` was already directional. + +### 4. Soak + regression gates (`tidal-stress`) + +`tidal-stress` gained, for the nightly soak: + +- **`--json-summary `** — a flat, machine-readable per-stage roll-up + (p99 / throughput / error-rate / client-shed + the gate verdict), hand-rolled + (no serializer dep, matching the metrics module's posture) for trend-line + archival. +- **`--max-p99-ms`, `--max-error-pct`, `--fail-on-knee`** — PASS/FAIL regression + gates. A breach returns `StressError::Gate` → non-zero exit (the tool always + exited 0 before, so a regression scrolled past in green). Unarmed = the + original informational behavior, byte-for-byte. + +### 5. Nightly pipeline (`.woodpecker.yaml`) — Woodpecker, never GitHub Actions + +The pipeline now carries two event-routed flows (workflow-level `when: +event: [push, cron]`, per-step `when`): + +- **push** — the m11p8 rolling-upgrade release gate, then the Kaniko image build + (unchanged). +- **cron `nightly`** — `nightly-chaos` runs every tier-3 suite serially with + elevated kill-points (`TIDAL_QUORUM_KILLPOINTS=25`, `TIDAL_ELECTION_KILLPOINTS=15`) + and the `fault-injection` feature (so `cluster_faults` runs); `nightly-soak` + boots a target and runs `tidal-stress` with the regression gates, archiving the + JSON summary. Point `TIDAL_SOAK_TARGET` at the live Ref-A cluster and raise + `TIDAL_SOAK_SECS=3600` for the GA-bar 1-hour 100k-DAU soak. + +### 6. Guarantee traceability + +[guarantee-traceability.md](guarantee-traceability.md) maps every §2 guarantee +(G-D, G-A, G-S, G-E, G-Sec, G-O, G-Op, G-C) to its named automated test(s), the +fault each injects, and the checker each asserts through. + +## Exit gate (from the roadmap) + +- Every guarantee in §2 maps to a named automated test. +- Nightly suite green 30 consecutive days before GA. + +## Exit-gate evidence (local; debug builds, real OS processes) + +| Gate | Target | Measured | +|------|--------|----------| +| New fault classes pass | disk-full / slow-fsync / asymmetric | **`cluster_faults` 4/4 green** (73s): disk-full follower froze at applied 71 while node1 reached 240, recovered after restart to 120-item parity, zero acked loss; slow follower lagged 15 vs 0 then converged; both-slow → honest 503 naming `["eu-west","ap-south"]` while `ack=leader` 204'd; asymmetric → leader held us-east, single-leader-per-term, ap-south term bounded (0→0), rejoined. | +| First-class checkers faithful | extraction = no regression | `cluster_quorum::mp_quorum_ledger_zero_acked_loss_across_killpoints` refactored onto `AckLedger`: **3/3 kill-points zero acked loss** (29s) — extraction verified against the live gate. | +| Soak regression gate works | PASS exits 0, breach exits ≠0 | Against a real standalone server: a within-threshold run **exited 0** with a valid JSON summary (p99 39.7ms, 0% error); an impossible `--max-p99-ms 0.01` run **exited 1** with the breach in stderr and `"passed": false` in the JSON. | +| Every guarantee → named test | §2 coverage | [guarantee-traceability.md](guarantee-traceability.md): G-D/G-A/G-E/G-Sec/G-Op each map to a green named test; G-S to the m11p6 owner-test (in progress); G-O to the m11p8 metric/dashboard/alert artifacts; G-C to this apparatus. | +| Nightly green 30 days | calendar | **Pipeline established, not yet accrued** — this is a wall-clock criterion the cron pipeline produces over 30 nights; it cannot be completed in one session. Honestly out-of-session (like the Ref-A k3s runs). | + +## Status + +- [x] `fault-injection` feature (slow-fsync + disk-full WAL hooks), compiled out of production +- [x] First-class invariant checkers (`support/invariants.rs`): ledger, decay parity, single-leader-per-term, monotonic counters +- [x] `cluster_quorum` refactored onto `AckLedger` (extraction re-verified) +- [x] `cluster_faults.rs`: disk-full, slow-fsync (×2), asymmetric partition — 4/4 green +- [x] `tidal-stress` `--json-summary` + `--max-p99-ms` / `--max-error-pct` / `--fail-on-knee` + non-zero exit (verified vs a real server) +- [x] Woodpecker nightly cron pipeline (chaos suites + gated soak) beside the push release gate +- [x] Guarantee → automated-test traceability matrix +- [x] Docs (this phase, runbook chaos/soak section, monitoring soak note) + CHANGELOG + ROADMAP + roadmap-to-cluster as-built +- [ ] **Calendar:** nightly green for 30 consecutive days before GA (the pipeline accrues this; not an in-session deliverable) + +## Adversarial review (6-dimension, read-only) + +A read-only review fanned out over six dimensions (production-safety, checkers, +fault-tests, soak-gates, Woodpecker, doc-honesty), each finding adversarially +verified before adjudication. **10 confirmed findings, all fixed:** + +- **Woodpecker (blocker):** the soak step referenced `environment:`-block vars as + `${VAR}`, which Woodpecker's preprocessor blanks BEFORE the shell — the gate + would have run with an empty ramp/thresholds (never executing). Fixed with the + `$${VAR}` escape so the container shell expands them at runtime. +- **Gate threshold (blocker):** a `--max-p99-ms nan`/`inf` reported the gate ARMED + while every `> NaN` comparison is false — silently passing a regression. Now + rejected (finite, non-negative) at the CLI boundary. +- **JSON validity (blocker):** a non-finite `target_rps` (`--ramp "inf:30"`) emitted + bare `inf`/`NaN`, invalid JSON. Fixed at the parse boundary + a `to_json` + null-sanitizer (defense in depth). +- **Checkers (warnings):** `assert_single_leader_per_term` false-failed on a + duplicate same-region snapshot (`assert_ne!` → gated `if prev != region`); + `MonotonicCounters` now re-baselines a region's role-relative applied frontier + on a leader↔follower flip (not just reseed); `assert_items_present` distinguishes + a transport/non-2xx probe failure from true absence (no false ACKNOWLEDGED LOSS). +- **Coverage (warning):** added a `summary.rs` test module (gate breach/pass, + fail-on-knee, JSON round-trip through a real parser, null-sanitize) — 6 tests. +- **Doc honesty (warnings):** the G-Sec/G-Op owner-tests (mTLS, `cluster_security`, + `tidalctl` backup/restore, WAL-archival) were NOT in the nightly — ADDED a + `nightly-security-ops` cron step so the "runs nightly" claim is true; the G-S + row is marked "(NOT YET WRITTEN — m11p6 L4)" so it doesn't read as delivered. +- **Cron name (nit, external):** the nightly is silently-green until a cron named + `nightly` is created in Woodpecker repo settings — documented in the pipeline + header; a one-time repo-settings step. + +After the fixes the fault suite re-ran **4/4 green** and the ledger gate **3/3** +(the checker changes are faithful); the slow-fsync and asymmetric tests moved to +item-SET parity (a burst catch-up reconstructs velocity differently — the same +restart-robustness rationale as disk-full). + +### Formal seven-dimension review (`/review-code` → `fix-all`) + +A second pass ran the `code-reviewer` seven-dimension protocol (each dimension a +distinct frame, each finding adversarially verified): **0 blockers, 0 critical, 7 +warnings, 12 suggestions** (APPROVE-with-fixes). All 7 warnings + 10 of 12 +suggestions fixed; final score 90/100. Notable fixes: a real diagnostic bug +(`assert_items_present`'s two guards were swapped — a true acknowledged-loss +printed "PROBE UNREACHABLE" and vice-versa); `StressError::Gate` was overloaded +across breach / invalid-threshold / IO-write and split into `Gate`/`BadGate`/ +`Summary` so the operator-facing message names the real cause; the soak port +moved off iknowyou's 59521 to a `TIDAL_SOAK_PORT` (59526); the soak step now +fails fast on a boot failure and reaps its background server; doc-truthing the +"every suite consumes the shared checkers" claim (only `cluster_quorum` + +`cluster_faults` do — the legacy suites' migration is a tracked follow-up); +S=1-scope notes on the flat-field checkers; a real `from_stage` unit test; and +the `observe_cluster_frontiers` double-status-fetch removed. Two suggestions +deferred with justification: consolidating the `role`-vs-`is_leader` leadership +read into one helper (steady-state-equal, >5min, regression risk on green gates) +and Woodpecker YAML anchors for the repeated apt-get/cargo lines (untestable CI +refactor, benign duplication). + +## Verification status + +- Workspace `cargo fmt --all -- --check`: clean. +- `cargo clippy` `-D warnings`: clean on tidaldb (`fault-injection` on + off), tidal-server + (`cluster-e2e fault-injection`), and tidal-stress (all targets). +- Production safety: `cargo check -p tidaldb` with the feature OFF compiles the + fault module and both WAL hooks out entirely (the shipped binary is unchanged). +- The tree is UNCOMMITTED (continues the m11p1–p8 uncommitted tree; the user commits). diff --git a/docs/roadmap-to-cluster.md b/docs/roadmap-to-cluster.md index 0667f52..87cd5d4 100644 --- a/docs/roadmap-to-cluster.md +++ b/docs/roadmap-to-cluster.md @@ -1,6 +1,6 @@ # Roadmap to an Enterprise-Grade Cluster -**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅, m11p4 ✅, m11p5 ✅, m11p7 ✅, m11p8 ✅ complete (m11p8 2026-06-13 — observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O**) · m11p6 (sharding × replication) data plane in progress · p9 planned · **Date:** 2026-06-10 · **Baseline evidence:** +**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅, m11p4 ✅, m11p5 ✅, m11p7 ✅, m11p8 ✅, m11p9 ✅ complete (m11p9 2026-06-13 — continuous correctness: new fault classes (disk-full / slow-fsync / asymmetric partition) as REAL faults behind a production-compiled-out `fault-injection` feature, first-class invariant checkers (no-acked-loss ledger, decay parity, single-leader-per-term, monotonic counters), a `tidal-stress` soak with PASS/FAIL regression gates + JSON summary, a Woodpecker cron nightly chaos+soak pipeline beside the push release gate, and a guarantee→test traceability matrix — closes **G-C** apparatus; the 30-day-green calendar accrues nightly) · m11p6 (sharding × replication) data plane in progress · **Date:** 2026-06-10 · **Baseline evidence:** [stress-test-thepeach.md](ops/stress-test-thepeach.md), [cluster runbook](runbooks/cluster.md), [ROADMAP M8 Known Gaps](planning/ROADMAP.md) (G4/G5/G6), live k3s deployment (3 regions × 2-vCPU pods). @@ -485,6 +485,32 @@ current "replicated XOR sharded" split ends. - **Exit gate (GA):** every guarantee in §2 maps to a named automated test; nightly suite green 30 consecutive days. +> **✅ COMPLETE (2026-06-13), as built:** the new fault classes are REAL faults +> at the storage boundary, not engine flags — a `fault-injection` cargo feature +> (NOT default, never passed by the production image build, so compiled OUT of the +> shipped binary) adds WAL hooks for **slow-fsync** (`TIDAL_FAULT_FSYNC_DELAY_MS`, +> sleep before the durable fsync) and **disk-full** (`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`, +> a real `ENOSPC` once cumulative segment bytes cross the threshold); +> **asymmetric partition** reuses the harness's directed-edge proxies (sever +> inbound-only). The invariant checks the m11p3/m11p4 gates carried inline became +> a first-class `support/invariants.rs` (no-acked-loss `AckLedger`, decay/feed +> parity, `assert_single_leader_per_term`, monotonic frontiers) — the m11p3 ledger +> gate now consumes it (re-verified). A new tier-3 `cluster_faults.rs` proves all +> three fault classes degrade gracefully with zero acked loss (**4/4 green**: +> disk-full follower froze then recovered to parity; slow follower lagged then +> converged; both-slow → honest retryable 503 naming laggards while `ack=leader` +> kept 204'ing; asymmetric partition → leader held, single-leader-per-term, no +> split brain). `tidal-stress` gained a `--json-summary` + `--max-p99-ms` / +> `--max-error-pct` / `--fail-on-knee` regression gate (non-zero exit on breach — +> verified vs a real server). The nightly is a Woodpecker CRON pipeline (chaos +> suites with elevated kill-points + the gated soak) beside the push release gate +> — Woodpecker, never GitHub Actions. Every §2 guarantee maps to a named test in +> [guarantee-traceability.md](planning/milestone-11/guarantee-traceability.md); +> G-S's owner-test is the (in-progress) m11p6 throughput gate. The remaining GA +> half — nightly green for **30 consecutive days** — is a calendar criterion the +> pipeline now produces (out-of-session, like the Ref-A k3s runs). Details: +> [planning/milestone-11/phase-9.md](planning/milestone-11/phase-9.md). + --- ## 5. Sequencing, sizing, releases diff --git a/docs/runbooks/cluster.md b/docs/runbooks/cluster.md index 7976db3..afb3f7e 100644 --- a/docs/runbooks/cluster.md +++ b/docs/runbooks/cluster.md @@ -1303,6 +1303,53 @@ macOS, where `F_FULLFSYNC` averages ~7.4ms with a 10–50ms tail the p99 gate on the reference environment, and tune `wal.batch_timeout_ms` against the measured fsync histogram. +## 15. Continuous correctness: chaos suites + soak (m11p9) + +Correctness is a pipeline, not a one-time gate. The tier-3 suites run REAL OS +processes; the new fault classes inject REAL faults. + +**Run the chaos suites locally** (serial — they bind fixed ports and spawn +processes, so suites must not overlap): + +``` +cargo test -p tidal-server --features "cluster-e2e fault-injection" \ + --test cluster_faults -- --test-threads 1 --nocapture +``` + +`cluster_faults` covers the fault classes the partition/crash/skew suites lacked: + +| Test | Fault | Asserts | +|------|-------|---------| +| `mp_disk_full_follower_degrades_no_acked_loss` | follower WAL `ENOSPC` | receiver halts (degraded, alive), healthy majority keeps acking quorum, zero acked loss, restart recovers to parity | +| `mp_slow_fsync_follower_lags_but_quorum_holds` | one follower's fsync slowed | fast follower supplies quorum, slow node lags then converges, no loss | +| `mp_slow_fsync_both_followers_force_honest_quorum_timeout` | both followers slowed below the budget | `ack=quorum` → retryable 503 naming laggards; `ack=leader` → 204; recover | +| `mp_asymmetric_partition_no_split_brain_no_loss` | inbound to one node severed (outbound up) | pre-vote + check-quorum hold the leader; single-leader-per-term; no loss | + +**Fault knobs** (behind the `fault-injection` feature — compiled OUT of the +production image, inert until armed): `TIDAL_FAULT_FSYNC_DELAY_MS=` slows +every durable WAL fsync; `TIDAL_FAULT_DISK_FULL_AFTER_BYTES=` fails segment +writes with `ENOSPC` after `n` cumulative bytes this process lifetime (arm it on +a node at restart to fail after `n` bytes of post-restart writes). NEVER set +these on a production node. + +**Soak with regression gates** (`tidal-stress`): + +``` +tidal-stress --target http:// --ramp "3900:3600" --mix peach \ + --json-summary soak.json --max-error-pct 1 --max-p99-ms 250 --fail-on-knee +``` + +`--fail-on-knee` (built-in SLO), `--max-p99-ms`, and `--max-error-pct` make the +run exit non-zero on a regression; `--json-summary` writes a machine-readable +per-stage roll-up for trend lines. A bounded version runs nightly; the GA-bar +1-hour 100k-DAU soak points `--target` at the live Ref-A cluster (`--ramp 3900:3600`). + +**Nightly CI** (`.woodpecker.yaml`, cron `nightly` — Woodpecker, never GitHub +Actions): the chaos suites with elevated kill-points (`TIDAL_QUORUM_KILLPOINTS`, +`TIDAL_ELECTION_KILLPOINTS`) then the gated soak. A nightly failure flags the +day's correctness or performance regression. The guarantee→test map is +[docs/planning/milestone-11/guarantee-traceability.md](../planning/milestone-11/guarantee-traceability.md). + ## Cross-references - **Kubernetes deployment** — [docs/runbooks/kubernetes.md](kubernetes.md) @@ -1319,9 +1366,10 @@ against the measured fsync histogram. - **Roadmap / cluster status & known gaps** — [docs/planning/ROADMAP.md](../planning/ROADMAP.md) and [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) for the M11 cluster - status — quorum-ack writes (m11p3), automatic failover (m11p4), and - membership/discovery/elasticity (m11p5) all shipped; sharding × replication - (p6), security hardening (p7), and continuous correctness (p9) remain. + status — quorum-ack writes (m11p3), automatic failover (m11p4), + membership/discovery/elasticity (m11p5), security hardening (m11p7), + observability + operations (m11p8), and continuous correctness (m11p9) all + shipped; sharding × replication (p6) data plane in progress. - **API & schema reference** — [API.md](../../API.md), [QUICKSTART.md](../../QUICKSTART.md), and the live `/openapi.json` document. - **Scope & vision** — [VISION.md](../../VISION.md). diff --git a/tidal-server/Cargo.toml b/tidal-server/Cargo.toml index 5c7a74c..a4e75f9 100644 --- a/tidal-server/Cargo.toml +++ b/tidal-server/Cargo.toml @@ -95,6 +95,9 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus [features] cluster-e2e = [] +# m11p9 chaos testing: passes through to tidaldb's WAL fault injection. The +# tier-3 fault suite spawns a binary built with this; production never sets it. +fault-injection = ["tidaldb/fault-injection"] [dev-dependencies] tempfile = "3" diff --git a/tidal-server/tests/cluster_chaos.rs b/tidal-server/tests/cluster_chaos.rs index 0915047..e8ded12 100644 --- a/tidal-server/tests/cluster_chaos.rs +++ b/tidal-server/tests/cluster_chaos.rs @@ -94,7 +94,8 @@ const AP_SOUTH: usize = 2; /// A short partition window during which the leader keeps writing. const PARTITION_WRITES: u64 = 8; -// ── Shared seeding / feed helpers (canonical bodies in support::multiproc) ────── +// ── Feed helpers (LOCAL copies; canonical home is support::invariants — +// this suite's migration onto the shared checkers is a tracked follow-up) ────── /// A node's local-region feed as a sorted `(entity_id, score)` vector. fn feed_pairs( diff --git a/tidal-server/tests/cluster_faults.rs b/tidal-server/tests/cluster_faults.rs new file mode 100644 index 0000000..97c439f --- /dev/null +++ b/tidal-server/tests/cluster_faults.rs @@ -0,0 +1,404 @@ +//! Tier-3 fault-injection suite (m11p9 continuous correctness). +//! +//! The chaos gate (`cluster_chaos.rs`) covers partition / crash / clock-skew. +//! This suite adds the fault classes the roadmap (§4/m11p9) names as missing: +//! +//! 1. **Disk-full on a follower** (`mp_disk_full_follower_degrades_no_acked_loss`) +//! — a follower whose WAL hits `ENOSPC` mid-replication halts its receiver +//! (degraded, alive), the healthy majority keeps acking quorum writes, NO +//! acknowledged write is lost, and a restart with space recovers it to full +//! parity. +//! 2. **Slow-fsync on a follower** (`mp_slow_fsync_follower_lags_but_quorum_holds`) +//! — a slow disk lags one follower; the fast follower supplies quorum so +//! every write still commits; the slow node converges once the burst ends; no +//! loss. +//! 3. **Slow-fsync on BOTH followers** (`mp_slow_fsync_both_followers_force_honest_quorum_timeout`) +//! — when no follower can confirm inside the budget, `ack=quorum` returns a +//! retryable 503 naming the laggards (never a false success) while +//! `ack=leader` is unaffected; the followers recover. +//! 4. **Asymmetric partition** (`mp_asymmetric_partition_no_split_brain_no_loss`) +//! — a follower that loses INBOUND links (can still send) cannot disrupt the +//! cluster: pre-vote + check-quorum keep the standing leader, there is never a +//! second leader at the same term, and no acked write is lost. +//! +//! These are REAL faults exercising REAL recovery paths: the WAL hooks return a +//! genuine `ENOSPC` / sleep the real fsync (behind the `fault-injection` feature, +//! compiled out of production — see `tidal/src/fault.rs`); partitions sever real +//! loopback TCP. Every test asserts through the first-class invariant checkers +//! (`support::invariants`): the no-acked-loss ledger, cross-replica feed parity, +//! single-leader-per-term, and per-node monotonic frontiers. +//! +//! Run: `cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_faults -- --nocapture --test-threads 1` + +#![cfg(feature = "cluster-e2e")] +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::cast_possible_truncation, + clippy::cast_precision_loss, + clippy::too_many_lines +)] + +mod support; + +use std::time::{Duration, Instant}; + +use support::{ + invariants::{AckLedger, MonotonicCounters, assert_single_leader_now, feed_item_ids}, + multiproc::{BREAKER_RESET, ClusterOptions, MultiProcCluster, convergence_budget}, + partition::proxied_rewrite, +}; + +const LEADER: usize = 0; +const FSYNC_DELAY: &str = "TIDAL_FAULT_FSYNC_DELAY_MS"; +const DISK_FULL: &str = "TIDAL_FAULT_DISK_FULL_AFTER_BYTES"; + +/// Stable single-leader posture (no auto-election) + quorum default, for the +/// fault tests whose subject is the data plane, not the election. A faulted +/// node must not trigger a spurious failover that muddies the assertion. +const STABLE_QUORUM_YAML: &str = + "election:\n auto_election: false\nreplication:\n ack: quorum\n quorum_timeout_ms: 2000"; + +/// Drive `count` quorum item+view writes for ids `id_base..id_base+count` +/// through `base`, recording the client-observed acks into `ledger`. Returns +/// how many items the client saw acknowledged. +fn drive_quorum_writes(base: &str, id_base: u64, count: u64, ledger: &mut AckLedger) -> u64 { + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(4)) + .build() + .unwrap(); + let mut acked = 0; + for i in 0..count { + if ledger.write_item_and_view(&client, base, "quorum", id_base + i) { + acked += 1; + } + } + acked +} + +/// **Disk-full on a follower.** A follower whose WAL fills mid-replication must +/// degrade gracefully (halt apply, stay alive — never corrupt, never crash), +/// the healthy majority must keep acking `ack=quorum` writes, NO acknowledged +/// write may be lost, and a restart with space must recover the follower to full +/// content + score parity. +#[test] +fn mp_disk_full_follower_degrades_no_acked_loss() { + // ap-south (node 2) hits ENOSPC after ~8 KiB of post-boot WAL writes — far + // past the empty-cluster boot (it writes nothing until it applies), but + // crossed after a few dozen replicated records. + let opts = ClusterOptions::new(3) + .with_env(2, DISK_FULL, "8192") + .with_topology_extra(STABLE_QUORUM_YAML); + let mut cluster = MultiProcCluster::start_with(opts); + cluster.wait_converged_all(convergence_budget()); + + // Write a burst of quorum item+view writes through the leader. node 1 + // (healthy) supplies the one-of-two follower quorum, so every write commits + // even after node 2's disk fills and its receiver halts. + let mut ledger = AckLedger::new(); + let mut mono = MonotonicCounters::new("disk-full"); + let mut acked = 0; + for round in 0..6 { + acked += drive_quorum_writes(&cluster.node(LEADER), 10_000 + round * 20, 20, &mut ledger); + mono.observe_cluster_frontiers(&cluster); + } + assert!(acked >= 100, "expected ~120 acked writes, got {acked}"); + + // node 2 is FROZEN behind: its WAL is full, its receiver halted, so its + // applied frontier stalled while node 1 stayed current. Poll briefly to let + // node 1 finish converging and confirm the asymmetry. + let leader_seq = cluster.leader_last_seq().expect("leader serves status"); + let deadline = Instant::now() + convergence_budget(); + loop { + let n1 = cluster.local_status(1).expect("node1 status"); + let n2 = cluster.local_status(2).expect("node2 status"); + let n1_applied = n1["applied_events"].as_u64().unwrap_or(0); + let n2_applied = n2["applied_events"].as_u64().unwrap_or(0); + if n1_applied >= leader_seq && n2_applied < leader_seq { + println!( + "[disk-full] node1 caught up (applied {n1_applied}/{leader_seq}); \ + node2 frozen at applied {n2_applied} (disk full)" + ); + break; + } + assert!( + Instant::now() <= deadline, + "node1 must converge while node2 stays frozen: \ + n1_applied={n1_applied} n2_applied={n2_applied} leader_seq={leader_seq}" + ); + std::thread::sleep(Duration::from_millis(100)); + } + + // No acked loss: every acked write is durable on the HEALTHY follower's + // contiguous frontier (the real quorum-durability claim under a disk-full + // minority), and every acked item is content-present on it. + let (_keep, _) = ledger.assert_frontier_covers_acks(&cluster, &[1usize], "disk-full"); + ledger.assert_items_present(&cluster, 1, Duration::from_secs(10), "disk-full"); + + // HEAL: restart node 2 with the fault disarmed (0 = off). A fresh receiver + // catches the full log up via the stream — no operator verb beyond restart. + cluster.restart(2, &[(DISK_FULL, "0")]); + cluster.wait_converged_all(convergence_budget() + BREAKER_RESET); + + // The recovered disk-full node now holds every acked write. Convergence is + // proven three ways: the contiguous frontier matched the leader + // (wait_converged_all above), the materialized item SET matches the leader, + // and every acked item is content-present via /search. We assert feed item-SET + // parity, NOT score parity: `trending` scores are a windowed velocity + // (count / window), and a node rebuilt from a burst WAL-replay aligns its time + // buckets differently than one that accumulated continuously — so a velocity + // score legitimately differs across a restart even with identical durable data. + // Decay/score parity to 1e-6 between CONTINUOUS replicas is asserted by the + // slow-fsync and asymmetric tests (and the chaos gate). + let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200); + let recovered_ids = feed_item_ids(&cluster, 2, "trending", 200); + assert_eq!( + leader_ids, recovered_ids, + "recovered disk-full node must materialize the same item set as the leader" + ); + ledger.assert_items_present( + &cluster, + 2, + Duration::from_secs(10), + "disk-full (post-recovery)", + ); + println!( + "[disk-full] node2 recovered after restart: {acked} acked writes all present, \ + item set parity with leader ({} items)", + leader_ids.len() + ); +} + +/// **Slow-fsync on one follower.** A slow disk lags one follower; the fast +/// follower supplies quorum so every `ack=quorum` write still commits; the slow +/// node is visibly behind during the burst, converges once it ends, and loses +/// nothing. +#[test] +fn mp_slow_fsync_follower_lags_but_quorum_holds() { + // ap-south (node 2) sleeps 250ms before every durable fsync. + let opts = ClusterOptions::new(3) + .with_env(2, FSYNC_DELAY, "250") + .with_topology_extra(STABLE_QUORUM_YAML); + let cluster = MultiProcCluster::start_with(opts); + cluster.wait_converged_all(convergence_budget()); + + let mut ledger = AckLedger::new(); + let mut mono = MonotonicCounters::new("slow-fsync-follower"); + let acked = drive_quorum_writes(&cluster.node(LEADER), 20_000, 60, &mut ledger); + mono.observe_cluster_frontiers(&cluster); + assert!( + acked >= 55, + "fast follower must supply quorum: only {acked} acked" + ); + + // Right after the burst the slow follower trails the fast one (250ms/fsync + // ≫ the fast nodes' ~2ms group-commit), proving the slow disk is visible. + let n1 = cluster.local_status(1).expect("node1 status"); + let n2 = cluster.local_status(2).expect("node2 status"); + let n1_lag = n1["lag_events"].as_u64().unwrap_or(u64::MAX); + let n2_lag = n2["lag_events"].as_u64().unwrap_or(0); + println!("[slow-fsync] after burst: node1 lag={n1_lag} node2(slow) lag={n2_lag}"); + assert!( + n2_lag > n1_lag, + "the slow follower must trail the fast one: node1 lag={n1_lag} node2 lag={n2_lag}" + ); + + // A slow disk is not a broken one: the burst over, the slow follower drains + // and converges; no acked write is lost; frontiers never regressed. + cluster.wait_converged_all(convergence_budget() * 2); + mono.observe_cluster_frontiers(&cluster); + let (_keep, _) = + ledger.assert_frontier_covers_acks(&cluster, &[1usize, 2usize], "slow-fsync-follower"); + // Item-SET parity, not score: the slow follower drains its backlog as a burst + // once the write burst ends, which reconstructs its time-bucketed velocity + // differently than the leader's continuous accumulation — so a `trending` + // (velocity) score can differ across the catch-up even with identical durable + // data. The frontier + content + set checks prove data convergence robustly. + let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200); + let slow_ids = feed_item_ids(&cluster, 2, "trending", 200); + assert_eq!( + leader_ids, slow_ids, + "converged slow follower must materialize the same item set as the leader" + ); + ledger.assert_items_present(&cluster, 2, Duration::from_secs(10), "slow-fsync-follower"); + println!("[slow-fsync] {acked} acked writes, slow follower converged to parity, no loss"); +} + +/// **Slow-fsync on BOTH followers + tight budget → honest quorum timeout.** When +/// neither follower can confirm a write inside `quorum_timeout_ms`, `ack=quorum` +/// must return a retryable 503 naming the laggards — never a false 2xx — while +/// `ack=leader` on the same cluster keeps succeeding (the leader's disk is fast). +/// The slow followers are not broken, so the cluster converges once load eases. +#[test] +fn mp_slow_fsync_both_followers_force_honest_quorum_timeout() { + // Both followers sleep 1500ms per fsync; the quorum budget is 300ms — so the + // commit index physically cannot advance to a fresh write inside the budget. + let yaml = + "election:\n auto_election: false\nreplication:\n ack: quorum\n quorum_timeout_ms: 300"; + let opts = ClusterOptions::new(3) + .with_env(1, FSYNC_DELAY, "1500") + .with_env(2, FSYNC_DELAY, "1500") + .with_topology_extra(yaml); + let cluster = MultiProcCluster::start_with(opts); + cluster.wait_converged_all(convergence_budget()); + + let client = cluster.client(); + let leader = cluster.node(LEADER); + + // ack=quorum (the topology default): the budget expires before either slow + // follower fsyncs the apply → retryable 503 naming the laggards. + let q = client + .post(format!("{leader}/signals")) + .json(&serde_json::json!({ "entity_id": 30_001, "signal": "view", "weight": 1.0 })) + .send() + .expect("quorum write sends"); + assert_eq!( + q.status().as_u16(), + 503, + "ack=quorum must time out honestly against two slow followers" + ); + let body: serde_json::Value = q.json().expect("503 carries a JSON body"); + assert_eq!( + body["retryable"].as_bool(), + Some(true), + "503 must be retryable: {body}" + ); + assert!( + body["laggards"].as_array().is_some_and(|l| !l.is_empty()), + "503 must name the laggards: {body}" + ); + println!( + "[both-slow] ack=quorum → honest 503 retryable, laggards={}", + body["laggards"] + ); + + // ack=leader on the same cluster is unaffected (the leader's disk is fast): + // the caller's durability choice is the caller's, never the deployment's. + let l = client + .post(format!("{leader}/signals")) + .header("x-tidal-ack", "leader") + .json(&serde_json::json!({ "entity_id": 30_002, "signal": "view", "weight": 1.0 })) + .send() + .expect("leader write sends"); + assert_eq!( + l.status().as_u16(), + 204, + "ack=leader must keep succeeding while ack=quorum times out: {}", + l.status() + ); + println!("[both-slow] ack=leader → 204 (unaffected)"); + + // The slow followers are not broken: they DO apply, just slowly. Once the + // single in-flight write drains, the cluster converges (the timed-out write + // is in the leader's log and commits late — at-least-once, never lost). + cluster.wait_converged_all(convergence_budget() * 3); + println!("[both-slow] slow followers drained and converged"); +} + +/// **Asymmetric partition.** A follower that loses its INBOUND links (peers +/// cannot reach it; it can still send) must not disrupt the cluster: pre-vote + +/// check-quorum keep the standing leader, there is never a second leader at the +/// same term, the reachable majority keeps acking quorum writes with zero loss, +/// and the follower rejoins cleanly on heal. +#[test] +fn mp_asymmetric_partition_no_split_brain_no_loss() { + // Proxy only the edges INTO ap-south; ap-south's own outbound dials are + // unproxied (identity), so severing the inbound edges is a true ASYMMETRIC + // cut: ap-south stops RECEIVING heartbeats/ships but can still SEND the + // disruptive RequestVotes pre-vote is designed to neutralize. + let (rewrite, proxies) = proxied_rewrite(&["ap-south"]); + let fast_election = "election:\n heartbeat_interval_ms: 100\n election_timeout_min_ms: 500\n \ + election_timeout_max_ms: 1000\n leader_lease_ms: 350\nreplication:\n ack: quorum"; + let opts = ClusterOptions::new(3) + .with_rewrite(rewrite) + .with_topology_extra(fast_election); + let cluster = MultiProcCluster::start_with(opts); + cluster.wait_leader_agreed("us-east", convergence_budget()); + cluster.wait_converged_all(convergence_budget()); + + // Baseline writes; capture the standing leader's term. + let mut ledger = AckLedger::new(); + drive_quorum_writes(&cluster.node(LEADER), 40_000, 30, &mut ledger); + let base_term = cluster.local_status(LEADER).expect("leader status")["term"] + .as_u64() + .unwrap_or(0); + + // ── Asymmetric sever: cut every inbound edge to ap-south ────────────────── + proxies.region("ap-south").sever_all(); + + // The reachable majority keeps acking quorum writes (us-east + eu-west). + let acked = drive_quorum_writes(&cluster.node(LEADER), 41_000, 30, &mut ledger); + assert!( + acked >= 25, + "reachable majority must keep acking: only {acked}" + ); + + // Through several election timeouts: the standing leader is unchanged, there + // is never a second leader at any term, and ap-south's term does not explode + // (pre-vote blocks a node that can't receive grants from bumping its term). + let watch_until = Instant::now() + Duration::from_secs(12); + let mut max_apsouth_term = base_term; + while Instant::now() < watch_until { + // The shared checker snapshots every live node and asserts + // single-leader-per-term; reuse its snapshots for the rest of the checks + // (no second fetch, no hardcoded node count). + let statuses = assert_single_leader_now(&cluster, "asymmetric-partition"); + // The two reachable nodes still agree us-east leads. + for st in &statuses { + let region = st["region"].as_str().unwrap_or("?"); + if region == "us-east" || region == "eu-west" { + assert_eq!( + st["leader"].as_str(), + Some("us-east"), + "reachable node {region} must still see us-east as leader: {st}" + ); + } + if region == "ap-south" { + max_apsouth_term = max_apsouth_term.max(st["term"].as_u64().unwrap_or(0)); + } + } + std::thread::sleep(Duration::from_millis(200)); + } + assert!( + max_apsouth_term <= base_term + 2, + "ap-south term exploded under asymmetric partition ({base_term} → {max_apsouth_term}); \ + pre-vote should have neutralized the disruptive node" + ); + println!( + "[asymmetric] leader held us-east, single-leader-per-term, ap-south term bounded \ + ({base_term} → {max_apsouth_term})" + ); + + // No acked loss: every acked write is durable on the reachable follower. + let (_keep, _) = + ledger.assert_frontier_covers_acks(&cluster, &[1usize], "asymmetric-partition"); + ledger.assert_items_present( + &cluster, + LEADER, + Duration::from_secs(10), + "asymmetric-partition", + ); + + // HEAL: ap-south's inbound returns; it catches up and rejoins as a follower. + proxies.region("ap-south").heal_all(); + cluster.wait_converged_all(convergence_budget() + BREAKER_RESET); + // Item-SET parity, not score: ap-south applies the partition backlog as a + // burst on heal, reconstructing its velocity buckets differently than the + // leader's continuous accumulation — so trending (velocity) scores can differ + // across the catch-up even with identical durable data. Set + content + + // frontier prove data convergence robustly. + let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200); + let rejoined_ids = feed_item_ids(&cluster, 2, "trending", 200); + assert_eq!( + leader_ids, rejoined_ids, + "rejoined ap-south must materialize the same item set as the leader" + ); + ledger.assert_items_present( + &cluster, + 2, + Duration::from_secs(10), + "asymmetric-partition (rejoined)", + ); + println!("[asymmetric] ap-south rejoined to parity, no acked loss"); +} diff --git a/tidal-server/tests/cluster_quorum.rs b/tidal-server/tests/cluster_quorum.rs index e891ce7..51d1b76 100644 --- a/tidal-server/tests/cluster_quorum.rs +++ b/tidal-server/tests/cluster_quorum.rs @@ -53,6 +53,7 @@ use std::sync::{ use std::time::{Duration, Instant}; use support::{ + invariants::{AckLedger, post_acked}, multiproc::{BREAKER_RESET, ClusterOptions, MultiProcCluster, convergence_budget}, partition::proxied_rewrite, }; @@ -68,44 +69,6 @@ fn killpoints() -> usize { .unwrap_or(8) } -/// A unique all-alpha search token for an entity id (digits 0-9 → letters -/// a-j), so `/search?query=` is an exact item-presence probe under -/// the default tokenizer. -fn item_token(entity_id: u64) -> String { - let mut token = String::from("kpq"); - for d in entity_id.to_string().bytes() { - token.push(char::from(b'a' + (d - b'0'))); - } - token -} - -/// POST with the `x-tidal-ack` header through a dedicated client. Returns -/// `Some(seq)` only for a 2xx carrying `x-tidal-seq` — the ledger's -/// definition of "acknowledged". -fn post_acked( - client: &reqwest::blocking::Client, - base: &str, - path: &str, - ack: &str, - body: &serde_json::Value, -) -> Option { - let resp = client - .post(format!("{base}{path}")) - .header("x-tidal-ack", ack) - .json(body) - .send() - .ok()?; - if !resp.status().is_success() { - return None; - } - resp.headers() - .get("x-tidal-seq")? - .to_str() - .ok()? - .parse() - .ok() -} - /// m11p3 quorum semantics over a REAL 3-process cluster with real TCP /// partitions: /// @@ -268,7 +231,8 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() { cluster.node(1), // forwarded quorum writes through a follower ]; - // ── Concurrent quorum writers, ledger = client-observed acks ─────── + // ── Concurrent quorum writers; each builds an AckLedger of the writes + // the client saw acknowledged (the shared first-class checker). ────── let stop = Arc::new(AtomicBool::new(false)); let id_base = 1_000 * (round as u64 + 1); let mut writers = Vec::new(); @@ -280,38 +244,16 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() { .timeout(Duration::from_secs(4)) .build() .unwrap(); - // (entity_id, item_seq, view_seq-if-acked) - let mut acked: Vec<(u64, u64, Option)> = Vec::new(); + let mut ledger = AckLedger::new(); let mut n = 0u64; while !stop.load(Ordering::Acquire) { let entity_id = id_base + (w as u64) * 500 + n; n += 1; - let Some(item_seq) = post_acked( - &client, - &base, - "/items", - "quorum", - &serde_json::json!({ - "entity_id": entity_id, - "metadata": { "title": item_token(entity_id) } - }), - ) else { - // Not acknowledged (leader dying/dead, forward failed, - // or quorum timeout): by contract it owes us nothing. - continue; - }; - let view_seq = post_acked( - &client, - &base, - "/signals", - "quorum", - &serde_json::json!({ - "entity_id": entity_id, "signal": "view", "weight": 1.0 - }), - ); - acked.push((entity_id, item_seq, view_seq)); + // An un-acked item (leader dying/dead, forward failed, or + // quorum timeout) is owed nothing by contract — skip it. + let _ = ledger.write_item_and_view(&client, &base, "quorum", entity_id); } - acked + ledger.writes().to_vec() })); } @@ -322,42 +264,18 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() { cluster.kill_hard(LEADER); stop.store(true, Ordering::Release); let _ = client_drain(&leader_base); // flush any half-open socket - let mut ledger: Vec<(u64, u64, Option)> = Vec::new(); + let mut ledger = AckLedger::new(); for w in writers { - ledger.extend(w.join().expect("writer thread")); + ledger.extend(w.join().expect("writer thread").into_iter()); } - let max_acked_seq = ledger - .iter() - .map(|(_, item_seq, view_seq)| view_seq.unwrap_or(*item_seq).max(*item_seq)) - .max() - .unwrap_or(0); + let max_acked_seq = ledger.max_acked_seq(); - // ── Operator rule: promote the max-applied survivor ──────────────── - let survivors = [1usize, 2usize]; - let applied: Vec<(usize, u64)> = survivors - .iter() - .map(|&idx| { - let status = cluster - .local_status(idx) - .expect("survivor must serve status"); - (idx, status["applied_events"].as_u64().unwrap()) - }) - .collect(); - let (chosen, chosen_applied) = applied - .iter() - .copied() - .max_by_key(|&(_, a)| a) - .expect("two survivors"); - - // ── INVARIANT A (frontier): no acked seqno above the chosen - // survivor's contiguous durable frontier ─────────────────────────── - assert!( - max_acked_seq <= chosen_applied, - "round {round}: ACKNOWLEDGED LOSS — max acked seq {max_acked_seq} exceeds the \ - max-applied survivor's frontier {chosen_applied} (applied: {applied:?}, \ - {} acked writes)", - ledger.len() - ); + // ── INVARIANT A (frontier): no acked seqno above the max-applied + // survivor's contiguous durable frontier — the shared checker also + // returns the survivor the operator rule says to promote. ─────────── + let ctx = format!("round {round}"); + let (chosen, chosen_applied) = + ledger.assert_frontier_covers_acks(&cluster, &[1usize, 2usize], &ctx); // m11p4: promote is a FENCED transfer — the election's up-to-date // restriction can refuse a target that fell behind between this @@ -372,10 +290,9 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() { &serde_json::json!({ "region": new_leader }), ); if resp.status().as_u16() != 200 { - let (other, _) = applied - .iter() - .copied() - .find(|&(idx, _)| idx != chosen) + let other = [1usize, 2usize] + .into_iter() + .find(|&idx| idx != chosen) .expect("two survivors"); println!( "[ledger] round {round}: promote of {new_leader} refused (it fell \ @@ -391,50 +308,12 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() { } cluster.wait_leader_agreed(&new_leader, Duration::from_secs(10)); - // ── INVARIANT B (content): every acked item is on the new leader ─── - let client = reqwest::blocking::Client::builder() - .timeout(Duration::from_secs(4)) - .build() - .unwrap(); - // The text index auto-commits every 2s (engine default), so the FIRST - // probe polls past the commit interval; data presence is what is - // asserted, not commit timing. + // ── INVARIANT B (content): every acked item is present on the new + // leader (the shared checker polls past the text index's 2s commit). ─ let winner_idx = (0..3) .find(|&i| cluster.region_name(i) == new_leader) .expect("winner index"); - let new_leader_base = cluster.node(winner_idx); - let search_deadline = Instant::now() + Duration::from_secs(10); - for (entity_id, item_seq, _) in &ledger { - let token = item_token(*entity_id); - // Definitely assigned: the loop body's first statement writes it - // before any break can be reached. - let mut last: serde_json::Value; - let present = loop { - last = client - .get(format!("{new_leader_base}/search?query={token}&limit=5")) - .send() - .unwrap() - .json() - .unwrap(); - let hit = last["items"] - .as_array() - .unwrap_or(&Vec::new()) - .iter() - .any(|it| it["entity_id"].as_u64() == Some(*entity_id)); - if hit { - break true; - } - if Instant::now() > search_deadline { - break false; - } - std::thread::sleep(Duration::from_millis(200)); - }; - assert!( - present, - "round {round}: ACKNOWLEDGED LOSS — item {entity_id} (seq {item_seq}, \ - acked at quorum) is missing on promoted leader {new_leader}: {last}" - ); - } + ledger.assert_items_present(&cluster, winner_idx, Duration::from_secs(10), &ctx); println!( "[ledger] round {round}: kill@{kill_after:?} → {} acked writes (max seq \ {max_acked_seq}) all present on {new_leader} (applied {chosen_applied})", diff --git a/tidal-server/tests/support/invariants.rs b/tidal-server/tests/support/invariants.rs new file mode 100644 index 0000000..bc646ce --- /dev/null +++ b/tidal-server/tests/support/invariants.rs @@ -0,0 +1,588 @@ +//! First-class correctness invariant checkers for the tier-3 chaos suites (m11p9). +//! +//! Before m11p9 each suite carried its own inline copy of these checks — the +//! zero-acked-loss ledger lived in `cluster_quorum.rs`, feed-score parity in +//! `cluster_chaos.rs`, the single-leader-per-term assertion in +//! `cluster_election.rs`. m11p9 makes this module the **canonical, audited home** +//! so a new fault class proves the SAME invariants the durability gate does, and +//! a fix to a checker fixes it for every CONSUMER at once. +//! +//! Consumers today: `cluster_quorum.rs` (the m11p3 ledger gate, migrated onto +//! `AckLedger`) and the new `cluster_faults.rs`. The legacy suites +//! (`cluster_chaos`, `cluster_election`, `cluster_membership`, `cluster_reseed`, +//! `cluster_multiproc`, `cluster_lifecycle`) still carry their own pre-m11p9 +//! local copies of `item_token`/`post_acked`/`feed_pairs`/`assert_feed_parity` +//! and their inline single-leader assertions — migrating them onto this module is +//! a tracked follow-up (note: some are deliberate variants, e.g. `cluster_chaos`'s +//! `feed_pairs` panics via `.unwrap()` where the shared one defaults via +//! `.unwrap_or`, so the migration is a behavior review, not a blind swap). +//! +//! The four checkers map to the roadmap's named invariants: +//! +//! 1. [`AckLedger`] — **no acknowledged-write loss**. Records every write the +//! client saw a 2xx + `x-tidal-seq` for, then proves after a leader kill that +//! (A, frontier) no acked seqno exceeds the max-applied survivor's contiguous +//! durable frontier, and (B, content) every acked item is actually present on +//! the promoted leader. This is the m11p3 exit-gate proof, extracted verbatim. +//! 2. [`assert_feed_parity`] — **cross-replica decay parity to 1e-6**. Two +//! replicas that have applied the same log must rank identical items with +//! identical decayed scores (a doubly-applied or dropped segment inflates or +//! deflates a score past the tolerance). +//! 3. [`assert_single_leader_per_term`] — **membership safety**: at most one +//! leader per term per shard, across a set of node status snapshots. A second +//! leader at the same term is a split brain. +//! 4. [`MonotonicCounters`] — **per-entity monotonic counters**: a named u64 +//! quantity (a node's applied frontier, a leader's commit index, an entity's +//! durable visibility) must never move backward across observations. A +//! regression means durably-acknowledged state was forgotten — the failure +//! mode disk-full / slow-fsync chaos is most likely to expose. + +#![allow(dead_code)] + +use std::{ + collections::HashMap, + time::{Duration, Instant}, +}; + +use super::multiproc::MultiProcCluster; + +// ── No acknowledged-write loss (G-D durability) ────────────────────────────── + +/// A unique all-alpha search token for an entity id (digits 0-9 → letters a-j), +/// so `/search?query=` is an exact item-presence probe under the default +/// tokenizer. Stable across suites so a recorded item is probeable everywhere. +#[must_use] +pub fn item_token(entity_id: u64) -> String { + let mut token = String::from("kpq"); + for d in entity_id.to_string().bytes() { + token.push(char::from(b'a' + (d - b'0'))); + } + token +} + +/// POST with the `x-tidal-ack` header through a caller-supplied client. Returns +/// `Some(seq)` only for a 2xx carrying `x-tidal-seq` — the ledger's definition +/// of "acknowledged". A non-2xx, a missing header, or any transport error is +/// `None`: by the at-least-once contract an un-acked write is owed nothing. +#[must_use] +pub fn post_acked( + client: &reqwest::blocking::Client, + base: &str, + path: &str, + ack: &str, + body: &serde_json::Value, +) -> Option { + let resp = client + .post(format!("{base}{path}")) + .header("x-tidal-ack", ack) + .json(body) + .send() + .ok()?; + if !resp.status().is_success() { + return None; + } + resp.headers() + .get("x-tidal-seq")? + .to_str() + .ok()? + .parse() + .ok() +} + +/// One acknowledged write the ledger tracks: the entity, the item record's +/// seqno, and the view signal's seqno if it too was acked. +#[derive(Clone, Copy, Debug)] +pub struct AckedWrite { + pub entity_id: u64, + pub item_seq: u64, + pub view_seq: Option, +} + +/// The client-observed ledger of acknowledged writes — the ground truth a +/// no-acked-loss proof replays against post-recovery state. +#[derive(Default)] +pub struct AckLedger { + acked: Vec, +} + +impl AckLedger { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Write one item (+ its `view` signal) at `ack` durability through `base` + /// and record whatever the client saw acknowledged. The item is recorded + /// only if its own write was acked (an un-acked item owes nothing); the view + /// seqno is recorded as `Some` only if the view was acked too. Returns + /// whether the item was acked (so a writer loop can pace itself). + pub fn write_item_and_view( + &mut self, + client: &reqwest::blocking::Client, + base: &str, + ack: &str, + entity_id: u64, + ) -> bool { + let Some(item_seq) = post_acked( + client, + base, + "/items", + ack, + &serde_json::json!({ + "entity_id": entity_id, + "metadata": { "title": item_token(entity_id) } + }), + ) else { + return false; + }; + let view_seq = post_acked( + client, + base, + "/signals", + ack, + &serde_json::json!({ "entity_id": entity_id, "signal": "view", "weight": 1.0 }), + ); + self.acked.push(AckedWrite { + entity_id, + item_seq, + view_seq, + }); + true + } + + /// Fold another writer thread's ledger into this one. + pub fn extend(&mut self, other: impl IntoIterator) { + self.acked.extend(other); + } + + /// Record a raw acked write (for callers that post through their own path). + pub fn record(&mut self, entity_id: u64, item_seq: u64, view_seq: Option) { + self.acked.push(AckedWrite { + entity_id, + item_seq, + view_seq, + }); + } + + #[must_use] + pub const fn len(&self) -> usize { + self.acked.len() + } + + #[must_use] + pub const fn is_empty(&self) -> bool { + self.acked.is_empty() + } + + #[must_use] + pub fn writes(&self) -> &[AckedWrite] { + &self.acked + } + + /// The highest seqno the client ever saw acknowledged (item or view). + #[must_use] + pub fn max_acked_seq(&self) -> u64 { + self.acked + .iter() + .map(|w| w.view_seq.unwrap_or(w.item_seq).max(w.item_seq)) + .max() + .unwrap_or(0) + } + + /// INVARIANT A (frontier): no acked seqno exceeds the max-applied survivor's + /// contiguous durable frontier. A quorum ack for seqno S means some + /// follower's applied frontier reached S durably; the operator rule "promote + /// the max-applied survivor" therefore guarantees the promoted node holds + /// every acked seqno. Returns the chosen survivor's `(index, applied)` so the + /// caller can promote it. + /// + /// SCOPE: reads the FLAT `applied_events` field, which under m11p6 sharding + /// mirrors only the node's DEFAULT (lowest-id) hosted group — exact for S=1 + /// (the shipped topology). A multi-shard ledger must record each acked write's + /// shard and compare against that shard's `shards[]` frontier (the follow-up, + /// alongside the shard-aware path in [`assert_single_leader_per_term`]). + /// + /// # Panics + /// + /// Panics (ACKNOWLEDGED LOSS) if `max_acked_seq > max survivor applied`, or + /// if no survivor serves status. + #[must_use] + pub fn assert_frontier_covers_acks( + &self, + cluster: &MultiProcCluster, + survivors: &[usize], + ctx: &str, + ) -> (usize, u64) { + let applied: Vec<(usize, u64)> = survivors + .iter() + .map(|&idx| { + let status = cluster + .local_status(idx) + .unwrap_or_else(|| panic!("{ctx}: survivor {idx} must serve status")); + (idx, status["applied_events"].as_u64().unwrap_or(0)) + }) + .collect(); + let (chosen, chosen_applied) = applied + .iter() + .copied() + .max_by_key(|&(_, a)| a) + .unwrap_or_else(|| panic!("{ctx}: no survivors supplied")); + let max_acked = self.max_acked_seq(); + assert!( + max_acked <= chosen_applied, + "{ctx}: ACKNOWLEDGED LOSS — max acked seq {max_acked} exceeds the max-applied \ + survivor's frontier {chosen_applied} (applied: {applied:?}, {} acked writes)", + self.acked.len() + ); + (chosen, chosen_applied) + } + + /// INVARIANT B (content): every acked item is present on `leader_idx` via + /// `/search`, polling up to `budget` (the text index auto-commits every 2s, + /// so a fresh item can race the first probe — presence is asserted, not + /// commit timing). The frontier can't lie about data it lacks, but this + /// catches a frontier that lies about data it claims. + /// + /// # Panics + /// + /// Panics (ACKNOWLEDGED LOSS) if any acked item is absent within `budget`. + pub fn assert_items_present( + &self, + cluster: &MultiProcCluster, + leader_idx: usize, + budget: Duration, + ctx: &str, + ) { + let base = cluster.node(leader_idx); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(4)) + .build() + .expect("build probe client"); + let deadline = Instant::now() + budget; + for w in &self.acked { + let token = item_token(w.entity_id); + // The text index auto-commits every 2s, so a fresh item can race the + // first probe — poll up to `budget`, asserting presence not timing. + // Track whether we ever got a CLEAN 2xx-with-no-hit: a poll that only + // ever saw transport errors / non-2xx is a REACHABILITY failure (a + // just-promoted leader still warming up), NOT durability loss — it + // must not be charged as ACKNOWLEDGED LOSS. + let mut saw_clean_miss = false; + let mut last_note = String::from("no response"); + let present = loop { + match client + .get(format!("{base}/search?query={token}&limit=5")) + .send() + { + Ok(resp) if resp.status().is_success() => { + let body: serde_json::Value = + resp.json().unwrap_or(serde_json::Value::Null); + let hit = body["items"] + .as_array() + .unwrap_or(&Vec::new()) + .iter() + .any(|it| it["entity_id"].as_u64() == Some(w.entity_id)); + if hit { + break true; + } + saw_clean_miss = true; + last_note = format!("2xx, item absent: {body}"); + } + Ok(resp) => last_note = format!("HTTP {}", resp.status()), + Err(e) => last_note = format!("transport error: {e}"), + } + if Instant::now() > deadline { + break false; + } + std::thread::sleep(Duration::from_millis(200)); + }; + // A clean 2xx-with-no-hit through the whole budget is true loss; a + // budget spent only on transport/non-2xx failures is a reachability + // bug in the probe path, surfaced distinctly so it is never + // misattributed to data loss. The two guards must match their + // messages: PROBE UNREACHABLE fires only when we NEVER saw a clean + // miss (transport/non-2xx the whole time); ACKNOWLEDGED LOSS fires + // only when we DID see a clean 2xx-with-no-hit (the item is genuinely + // absent on a reachable leader). + assert!( + present || saw_clean_miss, + "{ctx}: PROBE UNREACHABLE — item {} (seq {}, acked) could not be probed on \ + leader node {leader_idx} within the budget (last: {last_note}); this is a \ + reachability failure, not necessarily data loss", + w.entity_id, + w.item_seq + ); + assert!( + present || !saw_clean_miss, + "{ctx}: ACKNOWLEDGED LOSS — item {} (seq {}, acked) is missing on leader \ + node {leader_idx} (last: {last_note})", + w.entity_id, + w.item_seq + ); + } + } +} + +// ── Cross-replica decay parity (G-D / G-C) ─────────────────────────────────── + +/// A node's local-region feed as a sorted `(entity_id, score)` vector. +/// +/// # Panics +/// +/// Panics if the feed body is not the expected shape. +#[must_use] +pub fn feed_pairs( + cluster: &MultiProcCluster, + idx: usize, + profile: &str, + limit: u32, +) -> Vec<(u64, f64)> { + let body = cluster.get_json(idx, &format!("/feed?profile={profile}&limit={limit}")); + let mut pairs: Vec<(u64, f64)> = body["items"] + .as_array() + .unwrap_or(&Vec::new()) + .iter() + .map(|it| { + ( + it["entity_id"].as_u64().unwrap_or(0), + it["score"].as_f64().unwrap_or(f64::NAN), + ) + }) + .collect(); + pairs.sort_by_key(|(id, _)| *id); + pairs +} + +/// A node's local-region feed as a sorted entity-id vector — the data-convergence +/// view (which items are materialized), independent of score. Robust across a +/// node RESTART, where a velocity/rate score (`trending`/`hot`) legitimately +/// differs because its time-bucketed windowed counts are rebuilt from a burst +/// WAL-replay rather than continuous accumulation, even though the durable data +/// is identical. Use this for cross-replica DATA parity after a restart; use +/// [`assert_feed_parity`] for decay/score parity between continuously-running +/// replicas. +/// +/// # Panics +/// +/// Panics if the feed body is not the expected shape. +#[must_use] +pub fn feed_item_ids( + cluster: &MultiProcCluster, + idx: usize, + profile: &str, + limit: u32, +) -> Vec { + feed_pairs(cluster, idx, profile, limit) + .into_iter() + .map(|(id, _)| id) + .collect() +} + +/// Assert two replicas' feed views carry the SAME items with scores equal to +/// `1e-6`. A doubly-applied segment inflates decayed scores past the tolerance; +/// a dropped one deflates them — either trips this. +/// +/// # Panics +/// +/// Panics if the item sets differ or any score differs by more than `1e-6`. +pub fn assert_feed_parity(label: &str, a: &[(u64, f64)], b: &[(u64, f64)]) { + assert_eq!( + a.iter().map(|(id, _)| *id).collect::>(), + b.iter().map(|(id, _)| *id).collect::>(), + "{label}: feed item sets differ" + ); + for ((id_a, score_a), (_, score_b)) in a.iter().zip(b.iter()) { + assert!( + (score_a - score_b).abs() <= 1e-6, + "{label}: score for item {id_a} differs: {score_a} vs {score_b}" + ); + } +} + +// ── Membership safety: single leader per term per shard (G-A availability) ──── + +/// Assert that across `statuses` (each a `/cluster/status/local` body) no two +/// nodes claim to be leader at the same term. Reads the per-shard `shards[]` +/// rows when present (m11p6 multi-shard) and falls back to the flat +/// `role`/`term` fields (S=1), so it is correct on both surfaces. A second +/// leader at the same `(shard, term)` is a split brain. +/// +/// # Panics +/// +/// Panics naming both offending nodes if a `(shard, term)` has two leaders. +pub fn assert_single_leader_per_term(statuses: &[serde_json::Value], ctx: &str) { + // (shard, term) -> the region already seen leading it. + let mut leader_of: HashMap<(u64, u64), String> = HashMap::new(); + for st in statuses { + let region = st["region"].as_str().unwrap_or("").to_string(); + let rows = st["shards"].as_array(); + let claims: Vec<(u64, u64, bool)> = match rows { + Some(rows) if !rows.is_empty() => rows + .iter() + .map(|r| { + ( + r["shard"].as_u64().unwrap_or(0), + r["term"].as_u64().unwrap_or(0), + r["role"].as_str() == Some("leader"), + ) + }) + .collect(), + _ => vec![( + 0, + st["term"].as_u64().unwrap_or(0), + st["role"].as_str() == Some("leader"), + )], + }; + for (shard, term, is_leader) in claims { + if !is_leader { + continue; + } + // A genuine split brain is TWO DISTINCT regions leading the same + // (shard, term). The same region's status observed twice in one + // snapshot (a duplicate sample, a retry-appended list) is NOT a split + // brain — gate on `prev != region` so it is idempotently ignored. + if let Some(prev) = leader_of.insert((shard, term), region.clone()) + && prev != region + { + panic!( + "{ctx}: SPLIT BRAIN — {prev} and {region} both lead shard {shard} term {term}" + ); + } + } + } +} + +/// Convenience: snapshot every live node's `/cluster/status/local` and assert +/// single-leader-per-term. Returns the snapshots for further assertions. +#[must_use] +pub fn assert_single_leader_now(cluster: &MultiProcCluster, ctx: &str) -> Vec { + let statuses: Vec = (0..cluster.len()) + .filter_map(|i| cluster.local_status(i)) + .collect(); + assert_single_leader_per_term(&statuses, ctx); + statuses +} + +// ── Per-entity monotonic counters (G-C continuous correctness) ──────────────── + +/// Asserts a set of named `u64` counters never moves backward across +/// observations. Each [`observe`](Self::observe) checks the new value against +/// the last seen for that key; a decrease panics. Used for per-node applied +/// frontiers, a leader's commit index, and per-entity durable visibility +/// (0→1, never 1→0): quantities that are monotonic by construction, so a +/// regression is durably-acknowledged state being forgotten. +pub struct MonotonicCounters { + label: String, + last: HashMap, + /// The leader region observed on the previous frontier sweep, so a leader + /// change (a new epoch, where `commit_index` legitimately resets to the + /// promote baseline) clears the commit-index tracking instead of false-firing. + last_leader: Option, + /// Per-region role (`true` = leader) observed on the previous sweep. A + /// region's `applied_events` is role-RELATIVE — a leader reports its own + /// flushed frontier, a follower reports how far it applied the CURRENT + /// leader's stream — so a leader↔follower flip re-baselines the value and is + /// NOT a regression. We re-baseline `applied:` on any role flip. + last_role: HashMap, +} + +impl MonotonicCounters { + #[must_use] + pub fn new(label: &str) -> Self { + Self { + label: label.to_string(), + last: HashMap::new(), + last_leader: None, + last_role: HashMap::new(), + } + } + + /// Observe `value` for `key`; panic if it is below the last value seen. + /// + /// # Panics + /// + /// Panics if `value` regressed below a prior observation of `key`. + pub fn observe(&mut self, key: &str, value: u64) { + if let Some(&prev) = self.last.get(key) { + assert!( + value >= prev, + "{}: NON-MONOTONIC — {key} regressed {prev} → {value} (acknowledged state lost)", + self.label + ); + } + self.last.insert(key.to_string(), value); + } + + /// Forget any tracking for `key` (e.g. when a node legitimately resets its + /// frontier by reseeding, so the next observation starts a fresh baseline). + pub fn reset(&mut self, key: &str) { + self.last.remove(key); + } + + /// Observe every live node's `applied_events` frontier (keyed by region) and + /// the current leader's `commit_index`. Call repeatedly through a fault + /// window: within a stable leadership epoch no node's durable frontier and + /// no leader's commit index may ever regress. + /// + /// Legitimate epoch resets are handled, not flagged. `applied_events` is + /// role-RELATIVE (a leader reports its own flushed frontier; a follower + /// reports how far it applied the current leader's stream), so it is + /// re-baselined — not asserted — whenever a region (a) reports + /// `reseeding`/`quarantined` (mid snapshot-reinstall) OR (b) flips role + /// leader↔follower (a clean failover, where the value changes WHAT it + /// measures). `commit_index` is leadership-scoped and re-baselined on a + /// leader-region change. What remains asserted is the real invariant: within + /// a stable role/epoch a node never forgets durably-applied data, and a leader + /// never un-commits a quorum-durable write within its term. + /// SCOPE: this reads the FLAT top-level status fields (`is_leader`, + /// `applied_events`, `commit_index`), which under m11p6 sharding mirror only + /// the node's DEFAULT (lowest-id) hosted group — so it is exact for S=1 (the + /// shipped topology) and tracks only the default group's frontier under S>1. + /// The per-group `shards[]` rows (which [`assert_single_leader_per_term`] + /// already consults) are the follow-up for multi-group frontier tracking. + pub fn observe_cluster_frontiers(&mut self, cluster: &MultiProcCluster) { + let mut leader_now: Option = None; + // Capture the leader's commit_index in the SAME sweep (no second N-request + // scan): the value is in `st` while we hold it. + let mut leader_commit: u64 = 0; + for idx in 0..cluster.len() { + let Some(st) = cluster.local_status(idx) else { + continue; + }; + let region = st["region"].as_str().unwrap_or("?").to_string(); + let is_leader = st["is_leader"].as_bool() == Some(true); + let reseeding = st["reseeding"].as_bool() == Some(true) + || st["quarantined"].as_bool() == Some(true); + // A role flip (leader↔follower) re-baselines the role-relative + // applied frontier; the first observation of a region is not a flip. + let role_flipped = self + .last_role + .insert(region.clone(), is_leader) + .is_some_and(|was| was != is_leader); + let applied_key = format!("applied:{region}"); + if reseeding || role_flipped { + self.reset(&applied_key); + // Re-seed the baseline at the current value so the NEXT sweep + // asserts from here, not from the stale pre-flip frontier. + self.last + .insert(applied_key, st["applied_events"].as_u64().unwrap_or(0)); + } else { + // A leader reports its flushed frontier as applied (m11p8); a + // follower reports its applied frontier. Both are durable. + self.observe(&applied_key, st["applied_events"].as_u64().unwrap_or(0)); + } + if is_leader { + leader_now = Some(region); + leader_commit = st["commit_index"].as_u64().unwrap_or(0); + } + } + // commit_index is leadership-scoped: clear it on a leader change. + if leader_now != self.last_leader { + self.reset("commit_index"); + self.last_leader.clone_from(&leader_now); + } + if leader_now.is_some() { + self.observe("commit_index", leader_commit); + } + } +} diff --git a/tidal-server/tests/support/mod.rs b/tidal-server/tests/support/mod.rs index 78399d5..6422ea0 100644 --- a/tidal-server/tests/support/mod.rs +++ b/tidal-server/tests/support/mod.rs @@ -14,5 +14,6 @@ #![allow(dead_code)] +pub mod invariants; pub mod multiproc; pub mod partition; diff --git a/tidal-server/tests/support/multiproc.rs b/tidal-server/tests/support/multiproc.rs index abaa035..7017354 100644 --- a/tidal-server/tests/support/multiproc.rs +++ b/tidal-server/tests/support/multiproc.rs @@ -1159,16 +1159,26 @@ fn tidal_server_bin() -> PathBuf { .or_else(|| manifest_dir.parent()) .expect("locate workspace root"); + // Build with `fault-injection` so the spawned cluster processes carry the + // m11p9 WAL slow-fsync / disk-full hooks (inert unless a `TIDAL_FAULT_*` + // env var arms them — every non-fault suite spawns the same binary and is + // unaffected). Production never passes this feature, so the shipped image + // compiles the hooks out entirely. let status = Command::new("cargo") .arg("build") .arg("-p") .arg("tidal-server") + .arg("--features") + .arg("fault-injection") .current_dir(workspace_root) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .status() .expect("run cargo build"); - assert!(status.success(), "cargo build -p tidal-server failed"); + assert!( + status.success(), + "cargo build -p tidal-server --features fault-injection failed" + ); let bin = workspace_root.join("target/debug/tidal-server"); assert!( diff --git a/tidal-stress/Cargo.toml b/tidal-stress/Cargo.toml index f0fcff2..e8db283 100644 --- a/tidal-stress/Cargo.toml +++ b/tidal-stress/Cargo.toml @@ -60,6 +60,10 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus # network): bench workload.next() construction and the latency-histogram record # path in isolation. Matches the engine crate's criterion posture. criterion = { version = "0.5", features = ["html_reports"] } +# Test-only: the summary module's JSON is hand-rolled (no serializer dep on the +# hot path), but the round-trip unit test parses it with a real strict parser so +# a malformed-JSON regression is caught. +serde_json = "1" [[bench]] name = "hotpath" diff --git a/tidal-stress/src/error.rs b/tidal-stress/src/error.rs index 26b14f5..e0fb3fe 100644 --- a/tidal-stress/src/error.rs +++ b/tidal-stress/src/error.rs @@ -35,6 +35,26 @@ pub enum StressError { /// header value (contains control or non-visible characters). #[error("invalid api key: {0}")] Auth(String), + + /// A regression gate (`--max-p99-ms`, `--max-error-pct`, `--fail-on-knee`) + /// was breached. The run completed and reported normally; this is the + /// non-zero exit that fails the nightly CI step (m11p9). + #[error("regression gate breached: {0}")] + Gate(String), + + /// A gate threshold flag was invalid (non-finite or negative) — an operator + /// CLI mistake caught before the run starts, NOT a regression. + #[error("invalid gate threshold: {0}")] + BadGate(String), + + /// The `--json-summary` artifact could not be written (an IO/disk fault, not + /// a gate breach). + #[error("could not write summary {path}: {source}")] + Summary { + path: String, + #[source] + source: std::io::Error, + }, } /// Crate-local result alias. diff --git a/tidal-stress/src/lib.rs b/tidal-stress/src/lib.rs index fbfde37..b877e3d 100644 --- a/tidal-stress/src/lib.rs +++ b/tidal-stress/src/lib.rs @@ -12,4 +12,5 @@ pub mod client; pub mod error; pub mod metrics; pub mod scheduler; +pub mod summary; pub mod workload; diff --git a/tidal-stress/src/main.rs b/tidal-stress/src/main.rs index b36d39b..29aa3df 100644 --- a/tidal-stress/src/main.rs +++ b/tidal-stress/src/main.rs @@ -20,6 +20,7 @@ use tidal_stress::client::{HttpClient, seed_corpus}; use tidal_stress::error::{Result, StressError}; use tidal_stress::metrics::{self, StageStats}; use tidal_stress::scheduler::{Stage, parse_ramp, run_stage}; +use tidal_stress::summary::{GateConfig, RunSummary, StageSummary}; use tidal_stress::workload::{OpKind, Workload, WritePath, parse_mix}; #[derive(Parser)] @@ -27,6 +28,9 @@ use tidal_stress::workload::{OpKind, Workload, WritePath, parse_mix}; version, about = "Open-loop capacity ramp for tidalDB (thepeach feed workload)" )] +// A CLI args struct: each bool is an independent flag, not a state machine — the +// standard exception to `struct_excessive_bools`. +#[allow(clippy::struct_excessive_bools)] struct Cli { /// Region gateway base URL, e.g. http://10.43.99.11:9500 . Repeat for each /// region; reads round-robin across them (each serves locally), writes too @@ -119,6 +123,26 @@ struct Cli { /// feed p99 >150ms) instead of pushing every stage. #[arg(long, default_value_t = false)] stop_on_knee: bool, + + /// Write a machine-readable JSON run summary to this path (per-stage p99 / + /// throughput / error rate + the gate verdict). For nightly soak trend lines. + #[arg(long)] + json_summary: Option, + + /// Regression gate: exit non-zero if any stage's worst-op p99 exceeds this + /// (milliseconds). Omitted = no p99 gate. + #[arg(long)] + max_p99_ms: Option, + + /// Regression gate: exit non-zero if any stage's error rate exceeds this + /// (percent, e.g. 1.0 = 1%). Omitted = no error-rate gate. + #[arg(long)] + max_error_pct: Option, + + /// Regression gate: exit non-zero if any stage breached the built-in SLO + /// (the capacity knee). The nightly soak's PASS/FAIL signal. + #[arg(long, default_value_t = false)] + fail_on_knee: bool, } // SLO thresholds for the per-stage verdict. tidalDB's own RETRIEVE SLA is p99 @@ -169,6 +193,22 @@ async fn run() -> Result<()> { "--ack must be leader|quorum, got {ack:?}" ))); } + // A gate threshold of NaN/inf would report the gate ARMED while every + // `value > NaN` comparison is false — silently letting a regression through + // (clap's default f64 parser accepts "nan"/"inf"). Reject non-finite or + // negative thresholds so the gate cannot be disarmed by a typo. + for (flag, val) in [ + ("--max-p99-ms", cli.max_p99_ms), + ("--max-error-pct", cli.max_error_pct), + ] { + if let Some(v) = val + && (!v.is_finite() || v < 0.0) + { + return Err(StressError::BadGate(format!( + "{flag} must be finite and non-negative, got {v}" + ))); + } + } let client = Arc::new(HttpClient::new( Duration::from_secs(cli.request_timeout_secs), cli.api_key.clone(), @@ -267,6 +307,7 @@ async fn run() -> Result<()> { // ── Ramp ──────────────────────────────────────────────────────────────── let mut best_pass: Option<(usize, f64, f64)> = None; // (stage idx, total ok/s, signal ok/s) let mut knee: Option = None; + let mut stage_summaries: Vec = Vec::new(); for (i, stage) in stages.iter().enumerate() { if cli.poll_status @@ -285,6 +326,12 @@ async fn run() -> Result<()> { let _ = std::io::stdout().flush(); let (passed, signal_ok_rps) = stage_verdict(&stats); + stage_summaries.push(StageSummary::from_stage( + stage.target_rps, + &stats, + passed, + signal_ok_rps, + )); if passed { best_pass = Some((i, stats.achieved_rps(), signal_ok_rps)); } else if knee.is_none() { @@ -304,6 +351,35 @@ async fn run() -> Result<()> { } print_verdict(&cli, &stages, best_pass, knee); + + // ── Machine-readable summary + regression gates (m11p9) ────────────────── + let run = RunSummary { + stages: stage_summaries, + knee_stage: knee, + gate: GateConfig { + max_p99_ms: cli.max_p99_ms, + max_error_pct: cli.max_error_pct, + fail_on_knee: cli.fail_on_knee, + }, + }; + if let Some(path) = &cli.json_summary { + std::fs::write(path, run.to_json()).map_err(|source| StressError::Summary { + path: path.clone(), + source, + })?; + println!("\nwrote JSON summary → {path}"); + } + if run.gate.is_armed() { + let breaches = run.breaches(); + if breaches.is_empty() { + println!( + "\nregression gates: PASS ({} stage(s) within thresholds)", + run.stages.len() + ); + } else { + return Err(StressError::Gate(breaches.join("; "))); + } + } Ok(()) } diff --git a/tidal-stress/src/scheduler.rs b/tidal-stress/src/scheduler.rs index fc2e52d..12fe910 100644 --- a/tidal-stress/src/scheduler.rs +++ b/tidal-stress/src/scheduler.rs @@ -191,6 +191,15 @@ pub fn parse_ramp(spec: &str, stage_secs: u64) -> Result> { .trim() .parse() .map_err(|_| StressError::Ramp(format!("bad rps '{r}'")))?; + // `f64::from_str` accepts "inf"/"nan" and overflows ("1e400") to + // +inf; a non-finite or negative rps is meaningless and would + // poison the scheduler and the JSON summary (inf/NaN are not valid + // JSON). Reject it at the boundary. + if !target_rps.is_finite() || target_rps < 0.0 { + return Err(StressError::Ramp(format!( + "rps must be finite and non-negative, got '{r}'" + ))); + } let secs: u64 = s .trim() .parse() diff --git a/tidal-stress/src/summary.rs b/tidal-stress/src/summary.rs new file mode 100644 index 0000000..33fcea2 --- /dev/null +++ b/tidal-stress/src/summary.rs @@ -0,0 +1,395 @@ +//! Machine-readable run summary + regression gates (m11p9 continuous correctness). +//! +//! The human-readable per-stage tables and the capacity verdict are for an +//! operator reading the terminal. A NIGHTLY SOAK needs two more things the +//! original tool lacked: a machine-readable artifact a trend-line job can archive +//! and diff, and a PASS/FAIL exit code so a regression (p99 or error-rate creep, +//! or the built-in SLO knee) fails the CI step instead of scrolling past in green. +//! +//! This module builds a flat JSON summary (hand-rolled — every value is a number, +//! bool, or null, so no serializer dependency is needed, matching the +//! dependency-light posture of [`crate::metrics`]) and evaluates the regression +//! gates. A breach is surfaced as a [`crate::error::StressError::Gate`], which +//! `main` turns into a non-zero exit. + +use std::fmt::Write as _; + +use crate::metrics::StageStats; +use crate::workload::OpKind; + +/// One stage's machine-readable roll-up. +pub struct StageSummary { + pub target_rps: f64, + pub achieved_rps: f64, + pub ok_per_sec: f64, + pub signal_ok_per_sec: f64, + pub error_rate: f64, + pub client_shed: u64, + /// Feed-read p99 (ms), 0 if no feed reads in the mix. + pub feed_p99_ms: f64, + /// Worst signal-write p99 (ms) across view/like/skip, 0 if none. + pub signal_p99_ms: f64, + /// Worst p99 (ms) across every op that took OK traffic — the single number + /// the `--max-p99-ms` gate compares. + pub overall_p99_ms: f64, + /// Whether this stage held the built-in SLO (error ≤1%, feed p99 ≤150ms exact, + /// zero client shed). + pub slo_passed: bool, +} + +impl StageSummary { + /// Build from a finished stage's stats and its built-in-SLO verdict. + #[must_use] + pub fn from_stage( + target_rps: f64, + stats: &StageStats, + slo_passed: bool, + signal_ok_per_sec: f64, + ) -> Self { + let p99_ms = |k: OpKind| { + let op = &stats.ops[k.idx()]; + if op.ok() == 0 { + 0.0 + } else { + op.hist.percentile(0.99).as_secs_f64() * 1000.0 + } + }; + let feed_p99_ms = p99_ms(OpKind::FeedRead); + let signal_p99_ms = [OpKind::SignalView, OpKind::SignalLike, OpKind::SignalSkip] + .into_iter() + .map(p99_ms) + .fold(0.0_f64, f64::max); + // The worst p99 across every op (zero-ok ops contribute 0.0 via the + // closure's guard) — the single number the `--max-p99-ms` gate compares. + // One source of truth for the per-op p99 math (the `p99_ms` closure). + let overall_p99_ms = OpKind::ALL.into_iter().map(p99_ms).fold(0.0_f64, f64::max); + Self { + target_rps, + achieved_rps: stats.achieved_rps(), + ok_per_sec: stats.total_ok() as f64 / stats.elapsed.as_secs_f64().max(1e-9), + signal_ok_per_sec, + error_rate: stats.error_rate(), + client_shed: stats.client_shed, + feed_p99_ms, + signal_p99_ms, + overall_p99_ms, + slo_passed, + } + } +} + +/// Regression-gate thresholds resolved from the CLI. +#[derive(Clone, Copy, Default)] +pub struct GateConfig { + /// Fail if any stage's worst-op p99 exceeds this (ms). + pub max_p99_ms: Option, + /// Fail if any stage's error rate exceeds this (percent, e.g. 1.0 = 1%). + pub max_error_pct: Option, + /// Fail if any stage breached the built-in SLO (the capacity knee). + pub fail_on_knee: bool, +} + +impl GateConfig { + /// Whether any gate is armed (if not, the run is informational and always + /// exits 0, preserving the original tool's behavior). + #[must_use] + pub const fn is_armed(&self) -> bool { + self.max_p99_ms.is_some() || self.max_error_pct.is_some() || self.fail_on_knee + } +} + +/// The whole run's machine-readable summary. +pub struct RunSummary { + pub stages: Vec, + /// First stage index that breached the built-in SLO, if any. + pub knee_stage: Option, + pub gate: GateConfig, +} + +impl RunSummary { + /// Evaluate the armed regression gates. Returns the list of human-readable + /// breach reasons (empty = passed). Unarmed gates never breach. + #[must_use] + pub fn breaches(&self) -> Vec { + let mut out = Vec::new(); + if self.gate.fail_on_knee + && let Some(k) = self.knee_stage + { + out.push(format!( + "built-in SLO breached at stage {} (target {:.0} rps) [--fail-on-knee]", + k + 1, + self.stages.get(k).map_or(0.0, |s| s.target_rps) + )); + } + if let Some(max) = self.gate.max_p99_ms { + for (i, s) in self.stages.iter().enumerate() { + if s.overall_p99_ms > max { + out.push(format!( + "stage {} p99 {:.1}ms exceeds --max-p99-ms {max:.1}", + i + 1, + s.overall_p99_ms + )); + } + } + } + if let Some(max) = self.gate.max_error_pct { + for (i, s) in self.stages.iter().enumerate() { + if s.error_rate * 100.0 > max { + out.push(format!( + "stage {} error rate {:.2}% exceeds --max-error-pct {max:.2}", + i + 1, + s.error_rate * 100.0 + )); + } + } + } + out + } + + /// Whether the run passed every armed gate. + #[must_use] + pub fn passed(&self) -> bool { + self.breaches().is_empty() + } + + /// Render the flat JSON summary (hand-rolled; all values numeric/bool/null). + #[must_use] + pub fn to_json(&self) -> String { + let mut s = String::new(); + let breaches = self.breaches(); + s.push_str("{\n"); + let _ = writeln!(s, " \"passed\": {},", self.passed()); + let _ = writeln!(s, " \"gates_armed\": {},", self.gate.is_armed()); + match self.knee_stage { + Some(k) => { + let _ = writeln!(s, " \"knee_stage\": {},", k + 1); + } + None => s.push_str(" \"knee_stage\": null,\n"), + } + // Breach reasons as a JSON string array (these strings are tool-authored, + // but escape quotes/backslashes defensively). + s.push_str(" \"breaches\": ["); + for (i, b) in breaches.iter().enumerate() { + if i > 0 { + s.push_str(", "); + } + let _ = write!(s, "\"{}\"", b.replace('\\', "\\\\").replace('"', "\\\"")); + } + s.push_str("],\n"); + s.push_str(" \"stages\": [\n"); + for (i, st) in self.stages.iter().enumerate() { + s.push_str(" {"); + let _ = write!(s, "\"target_rps\": {}, ", jnum(st.target_rps, 1)); + let _ = write!(s, "\"achieved_rps\": {}, ", jnum(st.achieved_rps, 1)); + let _ = write!(s, "\"ok_per_sec\": {}, ", jnum(st.ok_per_sec, 1)); + let _ = write!( + s, + "\"signal_ok_per_sec\": {}, ", + jnum(st.signal_ok_per_sec, 1) + ); + let _ = write!(s, "\"error_rate\": {}, ", jnum(st.error_rate, 6)); + let _ = write!(s, "\"client_shed\": {}, ", st.client_shed); + let _ = write!(s, "\"feed_p99_ms\": {}, ", jnum(st.feed_p99_ms, 3)); + let _ = write!(s, "\"signal_p99_ms\": {}, ", jnum(st.signal_p99_ms, 3)); + let _ = write!(s, "\"overall_p99_ms\": {}, ", jnum(st.overall_p99_ms, 3)); + let _ = write!(s, "\"slo_passed\": {}", st.slo_passed); + s.push('}'); + if i + 1 < self.stages.len() { + s.push(','); + } + s.push('\n'); + } + s.push_str(" ]\n}\n"); + s + } +} + +/// Format an `f64` for JSON at `decimals` precision, or `null` if it is +/// non-finite (NaN/inf are not valid JSON — a single unguarded field would +/// otherwise corrupt the whole artifact). Defense in depth: the inputs are +/// already validated finite at the CLI/parse boundary, but this guarantees the +/// emitter can never produce invalid JSON regardless of upstream. +fn jnum(v: f64, decimals: usize) -> String { + if v.is_finite() { + format!("{v:.decimals$}") + } else { + "null".to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn stage( + target_rps: f64, + error_rate: f64, + overall_p99_ms: f64, + slo_passed: bool, + ) -> StageSummary { + StageSummary { + target_rps, + achieved_rps: target_rps, + ok_per_sec: target_rps, + signal_ok_per_sec: target_rps, + error_rate, + client_shed: 0, + feed_p99_ms: overall_p99_ms, + signal_p99_ms: overall_p99_ms, + overall_p99_ms, + slo_passed, + } + } + + #[test] + fn unarmed_gate_never_breaches() { + let run = RunSummary { + stages: vec![stage(1000.0, 0.5, 9999.0, false)], + knee_stage: Some(0), + gate: GateConfig::default(), + }; + assert!(!run.gate.is_armed()); + assert!( + run.passed(), + "an unarmed run is informational — always passes" + ); + assert!(run.breaches().is_empty()); + } + + #[test] + fn p99_and_error_gates_breach_over_threshold_and_pass_within() { + let over = RunSummary { + stages: vec![stage(1000.0, 0.05, 300.0, true)], + knee_stage: None, + gate: GateConfig { + max_p99_ms: Some(250.0), + max_error_pct: Some(1.0), + fail_on_knee: false, + }, + }; + assert!(!over.passed(), "p99 300>250 and error 5%>1% must breach"); + assert_eq!(over.breaches().len(), 2); + + let within = RunSummary { + stages: vec![stage(1000.0, 0.001, 100.0, true)], + knee_stage: None, + gate: GateConfig { + max_p99_ms: Some(250.0), + max_error_pct: Some(1.0), + fail_on_knee: false, + }, + }; + assert!(within.passed(), "p99 100<=250 and error 0.1%<=1% must pass"); + } + + #[test] + fn fail_on_knee_breaches_only_with_a_knee() { + let cfg = GateConfig { + max_p99_ms: None, + max_error_pct: None, + fail_on_knee: true, + }; + let with_knee = RunSummary { + stages: vec![stage(1000.0, 0.0, 10.0, false)], + knee_stage: Some(0), + gate: cfg, + }; + assert!(!with_knee.passed()); + let no_knee = RunSummary { + stages: vec![stage(1000.0, 0.0, 10.0, true)], + knee_stage: None, + gate: cfg, + }; + assert!(no_knee.passed()); + } + + #[test] + fn json_round_trips_through_a_real_parser() { + let run = RunSummary { + stages: vec![ + stage(1500.0, 0.012, 142.5, false), + stage(3000.0, 0.2, 410.0, false), + ], + knee_stage: Some(2), + gate: GateConfig { + max_p99_ms: Some(250.0), + max_error_pct: Some(1.0), + fail_on_knee: true, + }, + }; + let parsed: serde_json::Value = + serde_json::from_str(&run.to_json()).expect("to_json must be valid JSON"); + assert_eq!(parsed["passed"], false); + assert_eq!(parsed["gates_armed"], true); + assert_eq!(parsed["knee_stage"], 3); + assert_eq!(parsed["stages"].as_array().expect("stages array").len(), 2); + assert!( + !parsed["breaches"] + .as_array() + .expect("breaches array") + .is_empty() + ); + } + + #[test] + fn non_finite_field_serializes_as_null_not_invalid_json() { + // Defense in depth: even if a non-finite value reached a field, the + // emitter must still produce parseable JSON (null), never `inf`/`NaN`. + let run = RunSummary { + stages: vec![stage(f64::INFINITY, f64::NAN, 100.0, true)], + knee_stage: None, + gate: GateConfig::default(), + }; + let json = run.to_json(); + assert!( + !json.contains("inf") && !json.contains("NaN"), + "no bare inf/NaN: {json}" + ); + let parsed: serde_json::Value = + serde_json::from_str(&json).expect("still valid JSON with null-sanitized fields"); + assert!(parsed["stages"][0]["target_rps"].is_null()); + assert!(parsed["stages"][0]["error_rate"].is_null()); + } + + #[test] + fn from_stage_overall_p99_is_worst_busy_op_ignoring_zero_ok() { + use std::time::Duration; + + use crate::metrics::StageStats; + + // A real StageStats: FeedRead busy with a ~120ms tail, SignalView busy + // with a ~40ms tail, every other op left zero-ok (no traffic). + let mut stats = StageStats::new(); + stats.elapsed = Duration::from_secs(10); + let feed = OpKind::FeedRead.idx(); + let view = OpKind::SignalView.idx(); + for _ in 0..100 { + stats.ops[feed].status[0] += 1; // count as Ok + stats.ops[feed].hist.record(Duration::from_millis(120)); + stats.ops[view].status[0] += 1; + stats.ops[view].hist.record(Duration::from_millis(40)); + } + + let s = StageSummary::from_stage(1000.0, &stats, true, 0.0); + // The histogram is bucketed (~3-4%), so assert with tolerance. + assert!( + (100.0..150.0).contains(&s.feed_p99_ms), + "feed p99 {} ~120ms", + s.feed_p99_ms + ); + assert!( + (30.0..55.0).contains(&s.signal_p99_ms), + "signal p99 {} ~40ms", + s.signal_p99_ms + ); + // overall = the WORST busy op (FeedRead), and the zero-ok ops (which would + // each contribute 0.0) never inflate or drag it. + assert!( + (s.overall_p99_ms - s.feed_p99_ms).abs() < 1e-9, + "overall p99 {} must equal the worst busy op's p99 {} (zero-ok ops excluded)", + s.overall_p99_ms, + s.feed_p99_ms + ); + assert!(s.overall_p99_ms >= s.signal_p99_ms); + } +} diff --git a/tidal/Cargo.toml b/tidal/Cargo.toml index bcc5f03..5ddfa3b 100644 --- a/tidal/Cargo.toml +++ b/tidal/Cargo.toml @@ -10,6 +10,11 @@ license = "MIT" default = ["metrics"] test-utils = ["dep:tempfile"] metrics = [] # hand-rolled HTTP, no new crate deps +# m11p9 chaos testing: WAL slow-fsync + disk-full injection (src/fault.rs). +# NOT in `default` and never passed by the production image build, so the +# shipped binary compiles it out entirely. Inert until an env var arms it even +# when compiled in. See src/fault.rs. +fault-injection = [] [dependencies] base64 = "0.22" diff --git a/tidal/src/fault.rs b/tidal/src/fault.rs new file mode 100644 index 0000000..f63cd9a --- /dev/null +++ b/tidal/src/fault.rs @@ -0,0 +1,147 @@ +//! Test-only fault injection for continuous-correctness chaos testing (m11p9). +//! +//! # Compiled out of production +//! +//! Every item in this module is behind the `fault-injection` cargo feature, +//! which is **not** in tidaldb's `default` set and is **never** passed by the +//! production image build (`docker/standalone/Dockerfile`: `cargo build -p +//! tidal-server --release --locked`). With the feature off this module does not +//! exist and the two WAL call sites that reference it +//! ([`crate::wal::sync_file_durable`] and +//! [`crate::wal::segment::SegmentWriter::write_batch_bytes`]) compile to their +//! original instructions — the production binary is byte-identical to one built +//! from a tree without this file. +//! +//! A disk-write or fsync fault that a stray env var could trip in production is +//! exactly the kind of 3am footgun we refuse to ship, so the safety is +//! structural (the code is absent), not merely a default. The chaos suites and +//! the nightly CI build the binary with `--features fault-injection`; the +//! `MultiProcCluster` harness's `tidal_server_bin()` does so explicitly. +//! +//! # Inert until armed +//! +//! Even *with* the feature compiled in, every fault is INERT until an env var +//! explicitly arms it — the exact posture of the existing `TIDAL_HLC_SKEW_MS`, +//! `TIDAL_QUORUM_KILLPOINTS`, and `TIDAL_ELECTION_KILLPOINTS` test knobs. The +//! configuration is read **once** into a process-global `OnceLock` on first +//! access, so an un-armed process pays at most one relaxed atomic load per WAL +//! batch/fsync and no env lookups on the hot path. +//! +//! # The faults +//! +//! - **`TIDAL_FAULT_FSYNC_DELAY_MS`** — sleep this many milliseconds *before* +//! every durable WAL fsync. Models a slow disk / degraded NVMe: the data still +//! reaches stable storage, but each group-commit fsync costs `delay + real`. +//! On a follower this throttles apply (it WAL-firsts every replicated batch), +//! so under `ack=quorum` the commit index lags honestly; on a leader it raises +//! write latency without ever losing an acknowledged write. +//! +//! - **`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`** — after this process has written N +//! cumulative bytes to WAL segments, every subsequent segment write fails with +//! a real `ENOSPC` (`std::io::Error::from_raw_os_error(28)`, the errno on both +//! Linux and macOS), wrapped in [`crate::wal::error::WalError::Io`] — the same +//! variant a genuine full disk produces. Arming it on a node *at restart* +//! (the cumulative counter starts at zero each process lifetime) makes the +//! threshold "fail after N bytes of post-restart writes", which is how the +//! chaos suite drives a node into disk-full mid-replication and then proves the +//! writer notifies its waiters, halts cleanly (no torn-batch corruption past +//! WAL recovery's BLAKE3 truncation), and the cluster loses no acknowledged +//! write because the healthy majority held quorum throughout. +//! +//! The injection returns `ENOSPC` *before* issuing the partial `write_all`, the +//! standard chaos model (a clean, deterministic fault). A real full disk can +//! also tear a batch mid-write; that case is already covered by the WAL's +//! crash-recovery tail truncation and is not what this knob targets. +//! +//! # Granularity (per-PROCESS, not per-WAL) +//! +//! Both knobs are PROCESS-GLOBAL: the disk-full byte counter and the fsync delay +//! apply to every WAL in the process collectively. Today (S=1, one +//! `ShardReplica`/WAL per process) that is exactly one WAL, so the model is +//! clean. Under m11p6 multi-shard co-location (a `BTreeMap` +//! in one process, each with its own `SegmentWriter`) the counter would aggregate +//! across all hosted shards and fire `ENOSPC` on whichever shard first crosses +//! the shared threshold — so a *per-shard* disk-full (fill shard 2 but not shard +//! 1) needs either one-shard-per-process or keying these statics by shard/WAL. +//! That keying is deferred until the m11p6 (S>1) fault suite actually needs it. + +use std::{ + sync::{ + OnceLock, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + +use crate::wal::error::WalError; + +/// Process-global fault configuration, parsed once from the environment. +struct FaultConfig { + /// Sleep before each durable fsync (slow-disk injection). + fsync_delay: Option, + /// Fail segment writes once cumulative bytes exceed this (disk-full). + disk_full_after: Option, +} + +impl FaultConfig { + /// Parse the fault env vars. A missing/blank/unparsable value disarms that + /// fault (never a panic — an un-armed process must boot normally). + fn from_env() -> Self { + Self { + fsync_delay: env_u64("TIDAL_FAULT_FSYNC_DELAY_MS").map(Duration::from_millis), + disk_full_after: env_u64("TIDAL_FAULT_DISK_FULL_AFTER_BYTES"), + } + } +} + +/// Read a non-negative `u64` env var, or `None` if unset/blank/zero/unparsable. +/// Zero disarms (a `*_AFTER_BYTES=0` would otherwise fail the very first write, +/// which is never the intent of an opt-in knob). +fn env_u64(var: &str) -> Option { + std::env::var(var) + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|&n| n > 0) +} + +/// The parsed config, initialized once on first access. +fn config() -> &'static FaultConfig { + static CONFIG: OnceLock = OnceLock::new(); + CONFIG.get_or_init(FaultConfig::from_env) +} + +/// Cumulative bytes this process has written to WAL segments (disk-full input). +static SEGMENT_BYTES_WRITTEN: AtomicU64 = AtomicU64::new(0); + +/// Hook called at the top of every durable WAL fsync. With the slow-fsync fault +/// armed, sleeps `TIDAL_FAULT_FSYNC_DELAY_MS` before the real sync; otherwise a +/// single `Option` check and return. +pub fn before_fsync() { + if let Some(delay) = config().fsync_delay { + std::thread::sleep(delay); + } +} + +/// Hook called at the top of every WAL segment write. With the disk-full fault +/// armed, accounts `len` against the cumulative counter and returns a real +/// `ENOSPC` once the threshold is crossed; otherwise returns `Ok` immediately +/// without even touching the counter. +/// +/// # Errors +/// +/// Returns [`WalError::Io`] wrapping `ENOSPC` once cumulative segment bytes +/// exceed `TIDAL_FAULT_DISK_FULL_AFTER_BYTES`. +pub fn before_segment_write(len: usize) -> Result<(), WalError> { + let Some(threshold) = config().disk_full_after else { + return Ok(()); + }; + let total = SEGMENT_BYTES_WRITTEN.fetch_add(len as u64, Ordering::Relaxed) + len as u64; + if total > threshold { + // ENOSPC is errno 28 on both Linux and macOS. `from_raw_os_error` + // produces an error whose `.kind()` the platform maps exactly as a real + // full-disk write would — the same `WalError::Io` the writer's error + // path already handles (notify waiters, keep serving). + return Err(WalError::Io(std::io::Error::from_raw_os_error(28))); + } + Ok(()) +} diff --git a/tidal/src/lib.rs b/tidal/src/lib.rs index 182188d..f9e542f 100644 --- a/tidal/src/lib.rs +++ b/tidal/src/lib.rs @@ -27,6 +27,12 @@ pub mod storage; pub mod text; pub mod wal; +/// Test-only WAL fault injection (slow-fsync, disk-full) for the m11p9 +/// continuous-correctness chaos suites. Behind the non-default `fault-injection` +/// feature, so production builds never compile it — see the module docs. +#[cfg(feature = "fault-injection")] +pub(crate) mod fault; + /// Build hash compiled in from the `TIDALDB_BUILD_HASH` environment variable. /// /// Falls back to `"dev"` if `TIDALDB_BUILD_HASH` is unset or `build.rs` is not diff --git a/tidal/src/wal/mod.rs b/tidal/src/wal/mod.rs index 3fefed8..2b0d29c 100644 --- a/tidal/src/wal/mod.rs +++ b/tidal/src/wal/mod.rs @@ -77,6 +77,11 @@ pub type WalOpenResult = ( /// appends, checkpoint temp). For directory-entry durability use /// [`sync_dir_durable`]. pub(crate) fn sync_file_durable(file: &std::fs::File) -> Result<(), WalError> { + // Test-only slow-disk injection (m11p9): inert unless the `fault-injection` + // feature is compiled AND `TIDAL_FAULT_FSYNC_DELAY_MS` is armed. Compiled out + // of production entirely. + #[cfg(feature = "fault-injection")] + crate::fault::before_fsync(); #[cfg(target_os = "macos")] { // F_FULLFSYNC flushes the device write cache; plain fsync on macOS does diff --git a/tidal/src/wal/segment.rs b/tidal/src/wal/segment.rs index f2d8903..cd7e3ab 100644 --- a/tidal/src/wal/segment.rs +++ b/tidal/src/wal/segment.rs @@ -327,6 +327,13 @@ impl SegmentWriter { /// /// Returns `WalError::Io` on write failure. pub fn write_batch_bytes(&mut self, bytes: &[u8]) -> Result { + // Test-only disk-full injection (m11p9): inert unless the + // `fault-injection` feature is compiled AND + // `TIDAL_FAULT_DISK_FULL_AFTER_BYTES` is armed. Returns a real ENOSPC + // before any partial write, so the writer's error path runs unchanged. + // Compiled out of production entirely. + #[cfg(feature = "fault-injection")] + crate::fault::before_segment_write(bytes.len())?; let offset = self.current_size; self.file.write_all(bytes)?; self.current_size += bytes.len() as u64;