feat(m11): continuous correctness (m11p9) — fault classes, invariant checkers, soak gates, nightly pipeline
- fault-injection cargo feature (compiled OUT of prod): slow-fsync + disk-full WAL hooks in tidal/src/fault.rs, inert until armed, tier-3 builds with feature - first-class invariant checkers (tests/support/invariants.rs): AckLedger no-acked-loss (now consumed by m11p3 gate), feed parity, single-leader-per-term, monotonic frontiers - cluster_faults.rs tier-3 suite 4/4: disk-full degrade+recover, slow-fsync lag+converge, both-slow quorum 503, asymmetric partition no-split-brain - tidal-stress soak gates: --json-summary + --max-p99-ms/--max-error-pct/ --fail-on-knee → non-zero exit on regression - Woodpecker cron nightly flow (chaos + gated soak), event-routed, not GH Actions - guarantee-traceability.md: roadmap §2 guarantees → named tests (closes G-C apparatus; 30-day-green is a calendar criterion)
This commit is contained in:
jx12n
parent
005e292cbb
commit
1265140e28
136
.woodpecker.yaml
136
.woodpecker.yaml
@ -1,16 +1,23 @@
|
||||
# Pipeline: a rolling-upgrade RELEASE GATE (m11p8) runs FIRST; only if it passes
|
||||
# does Kaniko build the tidal-server image and push it to the in-cluster zot
|
||||
# registry. DEPLOYMENT IS MANUAL (kustomize, from the orchard9-k3sf ops repo) —
|
||||
# the old auto-`kubectl set image deployment/tidaldb` step was removed because it
|
||||
# coupled every image build to a standalone roll and the cluster ops repo's
|
||||
# contract is "manual deploy via scripts, no CI/CD deploy". The same
|
||||
# `tidal-server` binary serves every subcommand (standalone AND multi-process
|
||||
# `cluster --region`), so one image covers both deployments.
|
||||
# Two pipelines in one file, split by event (Woodpecker, NEVER GitHub Actions):
|
||||
#
|
||||
# • event: push → the m11p8 RELEASE GATE (rolling-upgrade tier-3 test) then a
|
||||
# Kaniko image build. Deployment is MANUAL (kustomize, orchard9-k3sf ops repo).
|
||||
# • event: cron → the m11p9 NIGHTLY CONTINUOUS-CORRECTNESS run: the tier-3
|
||||
# chaos suites (incl. the new disk-full / slow-fsync / asymmetric-partition
|
||||
# fault classes) with elevated kill-points, then a tidal-stress soak with
|
||||
# PASS/FAIL regression gates on p99 + error rate. A nightly failure flags a
|
||||
# correctness or performance regression for that day.
|
||||
#
|
||||
# Per-step `when:` routes each step to its event; the workflow-level `when` admits
|
||||
# both. Configure a cron named "nightly" in the Woodpecker repo settings to fire
|
||||
# the cron pipeline (the same `tidal-server` binary serves standalone AND
|
||||
# multi-process `cluster --region`, so one image covers both deployments).
|
||||
when:
|
||||
branch: main
|
||||
event: push
|
||||
event: [push, cron]
|
||||
|
||||
steps:
|
||||
# ── Release gate (push) ─────────────────────────────────────────────────────
|
||||
# m11p8 release gate: prove a rolling upgrade under load loses no acknowledged
|
||||
# write and never stalls (mp_rolling_upgrade_no_loss_no_stall — a tier-3 test
|
||||
# spawning three real OS processes with a graceful SIGTERM → version-tagged
|
||||
@ -19,6 +26,8 @@ steps:
|
||||
# overlap. A failure here BLOCKS the image build below — the gate, not the start.
|
||||
rolling-upgrade-gate:
|
||||
image: rust:1-bookworm
|
||||
when:
|
||||
event: push
|
||||
commands:
|
||||
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
|
||||
- cargo test -p tidal-server --features cluster-e2e --test cluster_lifecycle
|
||||
@ -26,6 +35,8 @@ steps:
|
||||
|
||||
build:
|
||||
image: woodpeckerci/plugin-kaniko
|
||||
when:
|
||||
event: push
|
||||
settings:
|
||||
repo: tidal/server
|
||||
dockerfile: docker/standalone/Dockerfile
|
||||
@ -42,3 +53,110 @@ steps:
|
||||
build_args:
|
||||
- TARGETPLATFORM=linux/amd64
|
||||
extra_args: --customPlatform=linux/amd64
|
||||
|
||||
# ── Nightly chaos (cron) ────────────────────────────────────────────────────
|
||||
# The tier-3 chaos/correctness suites over REAL OS processes, run serially
|
||||
# (fixed ports + spawned processes must not overlap). Kill-points are elevated
|
||||
# above the per-PR defaults (8/5) but kept below the 100-point exit-gate run so
|
||||
# the nightly stays bounded; widen TIDAL_*_KILLPOINTS for a deeper sweep. The
|
||||
# `fault-injection` feature compiles in the WAL slow-fsync / disk-full hooks for
|
||||
# cluster_faults (inert in every other suite — see tidal/src/fault.rs). Boot /
|
||||
# convergence budgets are raised for a shared CI runner. A failure = a
|
||||
# correctness regression for the night.
|
||||
nightly-chaos:
|
||||
image: rust:1-bookworm
|
||||
when:
|
||||
event: cron
|
||||
cron: nightly
|
||||
environment:
|
||||
TIDAL_QUORUM_KILLPOINTS: "25"
|
||||
TIDAL_ELECTION_KILLPOINTS: "15"
|
||||
TIDAL_TEST_BOOT_BUDGET_SECS: "120"
|
||||
TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "90"
|
||||
commands:
|
||||
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
|
||||
# New m11p9 fault classes first (disk-full, slow-fsync, asymmetric partition).
|
||||
- cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_faults -- --nocapture --test-threads 1
|
||||
# The standing chaos / durability / availability / elasticity gates.
|
||||
- cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_chaos -- --nocapture --test-threads 1
|
||||
- cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_quorum -- --nocapture --test-threads 1
|
||||
- cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_election -- --nocapture --test-threads 1
|
||||
- cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_membership -- --nocapture --test-threads 1
|
||||
- cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_reseed -- --nocapture --test-threads 1
|
||||
- cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_lifecycle -- --nocapture --test-threads 1
|
||||
- cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_runbook -- --nocapture --test-threads 1
|
||||
|
||||
# ── Nightly soak (cron) ─────────────────────────────────────────────────────
|
||||
# A tidal-stress soak with PASS/FAIL regression gates (m11p9). By default it
|
||||
# boots a local standalone server and soaks it for a bounded window, gating on
|
||||
# error rate and p99 so a perf regression fails the step and shows on the trend
|
||||
# line (the JSON summary is the archived artifact). Point TIDAL_SOAK_TARGET at
|
||||
# the live Ref-A cluster and raise TIDAL_SOAK_SECS to 3600 for the GA-bar 1-hour
|
||||
# 100k-DAU soak (the standing k3s-access caveat applies — same as m11p1–p3).
|
||||
nightly-soak:
|
||||
image: rust:1-bookworm
|
||||
when:
|
||||
event: cron
|
||||
cron: nightly
|
||||
environment:
|
||||
TIDAL_SOAK_RPS: "1000"
|
||||
TIDAL_SOAK_SECS: "600"
|
||||
TIDAL_SOAK_MAX_P99_MS: "250"
|
||||
TIDAL_SOAK_MAX_ERROR_PCT: "1"
|
||||
# Soak target port — an UNCLAIMED slot in the project's reserved dev band
|
||||
# (59520-59529): 59520=site, 59521=iknowyou, so the soak uses 59526. One
|
||||
# source of truth for the literal (referenced by --listen AND the TARGET).
|
||||
TIDAL_SOAK_PORT: "59526"
|
||||
commands:
|
||||
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang curl
|
||||
- cargo build -p tidal-server -p tidal-stress
|
||||
# Boot a standalone target unless TIDAL_SOAK_TARGET points elsewhere (Ref-A).
|
||||
# NOTE the `$${VAR}` escaping: Woodpecker substitutes a bare `${VAR}` in a
|
||||
# command BEFORE the shell runs, and step `environment:` vars are NOT in that
|
||||
# preprocessor namespace (they would blank to ""). `$${VAR}` passes a literal
|
||||
# `${VAR}` to the container shell, which expands it from the runtime env —
|
||||
# so the soak's ramp and gate thresholds are actually populated. Bare shell
|
||||
# locals (`$PORT`, `$TARGET`, `$SRV`) are never touched by the preprocessor.
|
||||
- |
|
||||
PORT="$${TIDAL_SOAK_PORT}"
|
||||
TARGET="$${TIDAL_SOAK_TARGET:-http://127.0.0.1:$PORT}"
|
||||
if [ -z "$TIDAL_SOAK_TARGET" ]; then
|
||||
mkdir -p /tmp/soak-data
|
||||
./target/debug/tidal-server standalone --listen "127.0.0.1:$PORT" \
|
||||
--schema tidal-server/config/default-schema.yaml --data-dir /tmp/soak-data &
|
||||
SRV=$!
|
||||
# Reap the background server + scratch dir on ANY exit (success, gate
|
||||
# failure, or the boot-failure exit below) so the step is idempotent.
|
||||
trap 'kill "$SRV" 2>/dev/null; rm -rf /tmp/soak-data' EXIT
|
||||
up=0
|
||||
for i in $(seq 1 100); do
|
||||
curl -sf "$TARGET/health/startup" >/dev/null 2>&1 && { up=1; break; } || sleep 0.3
|
||||
done
|
||||
# Fail FAST and unambiguously on a boot failure (bad schema, port bound)
|
||||
# rather than running the soak against a dead target and mislabeling it
|
||||
# as an error-rate regression on the trend line.
|
||||
if [ "$up" != "1" ]; then echo "soak target failed to start at $TARGET"; exit 1; fi
|
||||
fi
|
||||
./target/debug/tidal-stress --target "$TARGET" \
|
||||
--ramp "$${TIDAL_SOAK_RPS}:$${TIDAL_SOAK_SECS}" --corpus 5000 --mix peach \
|
||||
--json-summary soak-summary.json \
|
||||
--max-error-pct "$${TIDAL_SOAK_MAX_ERROR_PCT}" --max-p99-ms "$${TIDAL_SOAK_MAX_P99_MS}" --fail-on-knee
|
||||
- cat soak-summary.json
|
||||
|
||||
# ── Nightly security + ops correctness (cron) ───────────────────────────────
|
||||
# The G-Sec and G-Op owner-tests run nightly too (not just on-demand): mTLS /
|
||||
# foreign-pod rejection / cert rotation (tidal-net mtls + cluster_security), the
|
||||
# tidalctl backup/restore round-trip, and the gap-free WAL-archival (PITR) unit
|
||||
# test (in the engine lib). In-process + fast; serial for the TLS port binds.
|
||||
nightly-security-ops:
|
||||
image: rust:1-bookworm
|
||||
when:
|
||||
event: cron
|
||||
cron: nightly
|
||||
commands:
|
||||
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
|
||||
- cargo test -p tidal-net --test mtls -- --test-threads 1
|
||||
- cargo test -p tidal-server --test cluster_security -- --test-threads 1
|
||||
- cargo test -p tidal-server --lib
|
||||
- cargo test -p tidalctl
|
||||
- cargo test -p tidaldb --lib wal::compaction
|
||||
|
||||
38
CHANGELOG.md
38
CHANGELOG.md
@ -6,6 +6,44 @@ All notable changes to tidalDB will be documented in this file.
|
||||
|
||||
### Added
|
||||
|
||||
**Continuous correctness (m11p9) — new fault classes as REAL faults, first-class invariant checkers, soak regression gates, a Woodpecker nightly chaos+soak pipeline, and a guarantee→test matrix**
|
||||
|
||||
- **Fault injection compiled out of production.** A non-default `fault-injection`
|
||||
cargo feature (tidaldb + a tidal-server passthrough) adds two WAL hooks
|
||||
(`tidal/src/fault.rs`): **slow-fsync** (`TIDAL_FAULT_FSYNC_DELAY_MS` sleeps
|
||||
before each durable fsync) and **disk-full** (`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`
|
||||
returns a real `ENOSPC`/errno-28 once cumulative segment bytes cross the
|
||||
threshold). The production image build never passes the feature, so the hooks
|
||||
are compiled out entirely — a disk/fsync fault a stray env var could trip in
|
||||
prod is a 3am footgun we refuse to ship; the safety is structural. Inert until
|
||||
armed even when compiled in. The tier-3 harness builds the spawned binary with
|
||||
the feature.
|
||||
- **First-class invariant checkers** (`tidal-server/tests/support/invariants.rs`):
|
||||
the no-acked-loss `AckLedger` (frontier + content, extracted from the m11p3
|
||||
ledger gate, which now consumes it), cross-replica `assert_feed_parity` /
|
||||
`feed_item_ids`, `assert_single_leader_per_term` (membership safety, reads the
|
||||
m11p6 `shards[]` rows), and `MonotonicCounters` (per-node applied frontier +
|
||||
leader commit index never regress, with legitimate epoch resets forgiven).
|
||||
- **New fault-class suite** (`tidal-server/tests/cluster_faults.rs`, tier-3): a
|
||||
disk-full follower degrades gracefully (receiver halts, node alive, zero acked
|
||||
loss, restart recovers to parity); a slow-fsync follower lags then converges
|
||||
while the fast follower supplies quorum; both followers slow → `ack=quorum`
|
||||
returns a retryable 503 naming the laggards while `ack=leader` is unaffected;
|
||||
an asymmetric partition (inbound severed, outbound up) causes no split brain
|
||||
(pre-vote + check-quorum) and no loss. 4/4 green.
|
||||
- **Soak + regression gates** in `tidal-stress`: `--json-summary <path>` (a
|
||||
machine-readable per-stage p99/throughput/error roll-up for trend lines) and
|
||||
`--max-p99-ms` / `--max-error-pct` / `--fail-on-knee` gates that exit non-zero
|
||||
on a regression (the tool always exited 0 before).
|
||||
- **Nightly pipeline** (`.woodpecker.yaml`): a cron `nightly` flow (chaos suites
|
||||
with elevated kill-points + the `fault-injection` feature, then the gated
|
||||
`tidal-stress` soak) beside the existing push release gate, event-routed by
|
||||
per-step `when`. Woodpecker, never GitHub Actions.
|
||||
- **Guarantee traceability** (`docs/planning/milestone-11/guarantee-traceability.md`):
|
||||
every roadmap §2 guarantee mapped to its named automated test(s). Closes the
|
||||
**G-C** apparatus; the 30-consecutive-days-green half of the GA bar is a
|
||||
calendar criterion the nightly pipeline accrues.
|
||||
|
||||
**Observability + operations (m11p8) — complete cluster metric set on a per-node `/metrics` listener, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR + backup/restore, rolling-upgrade release gate**
|
||||
|
||||
- **Metrics.** Completed the `tidaldb_cluster_*` set with the two members the
|
||||
|
||||
@ -37,7 +37,7 @@ A single embeddable database can replace the 6-system content ranking stack by t
|
||||
| M8 | Distributed Fabric | Multi-region, multi-tenant replication keeps agent-memory semantics intact | Hosted tidalDB, cloud/edge deployments, shared agent substrate — **✅ COMPLETE**: in-process primitives + multi-node replication over real gRPC + true multi-process cluster mode (one process per region, real process isolation) with full tier-3 UAT (partition injection via TCP-proxy, clock-skew, rolling-upgrade, runbook verification); G1 + G2 resolved. Post-M8 follow-ups: quorum-ack writes, automatic failure detection / leader election |
|
||||
| M9 | Community Sync & Revocation | Local embeddable profiles can opt into community personalization and safely leave/purge contributions | Community personalization, federated taste graphs, shared feeds — ✅ COMPLETE (2026-06-06) |
|
||||
| M10 | Governance & Agent Rights | Community rules and agent-scoped permissions control what signals influence ranking | User-owned AI personalization at scale, policy-compliant agents — ✅ COMPLETE (2026-06-06) |
|
||||
| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12) + m11p5 ✅ (membership/discovery/elasticity: DNS peers, snapshot+stream reseed, kind-4 membership records, seed join, `k8s/cluster/` — 2026-06-12) + m11p7 ✅ (security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit — 2026-06-13) + m11p8 ✅ (observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O** — 2026-06-13); m11p6 (sharding × replication) data plane in progress; p9 planned in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) |
|
||||
| M11 | Enterprise-Grade Cluster | The multi-process cluster becomes a system of record: fast unified log, quorum-durable, self-healing, horizontally scalable, secured, observable, chaos-tested | Paying-customer cluster deployments — **IN PROGRESS**: m11p1 ✅ + m11p2 ✅ + m11p3 ✅ + m11p4 ✅ (automatic failover: Raft-style election + term fencing + divergence quarantine — closes **G5**; v0.9 "Credible HA" wave complete 2026-06-12) + m11p5 ✅ (membership/discovery/elasticity: DNS peers, snapshot+stream reseed, kind-4 membership records, seed join, `k8s/cluster/` — 2026-06-12) + m11p7 ✅ (security hardening: gRPC mTLS default + zero-drop cert rotation, inter-node HTTP TLS + per-node signed tokens, admin audit log, per-principal rate limit — 2026-06-13) + m11p8 ✅ (observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O** — 2026-06-13) + m11p9 ✅ (continuous correctness: new fault classes (disk-full / slow-fsync / asymmetric partition) as REAL faults behind a production-compiled-out `fault-injection` feature, first-class invariant checkers, `tidal-stress` soak with regression gates + JSON summary, a Woodpecker cron nightly chaos+soak pipeline, and a guarantee→test traceability matrix — closes the **G-C** apparatus; the 30-day-green calendar accrues nightly — 2026-06-13); m11p6 (sharding × replication) data plane in progress in [docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) |
|
||||
|
||||
### Embeddable → Distributed Path
|
||||
|
||||
@ -3285,7 +3285,7 @@ Full gap analysis, measured baselines, phase specs, and exit gates live in
|
||||
| m11p6 | Sharding × replication + rebalancing | Planned |
|
||||
| m11p7 | Security hardening (mTLS default, rotation, audit log) | ✅ **COMPLETE (2026-06-13)** — the cluster stops trusting the network. gRPC replication is served over a custom `tokio-rustls` acceptor fed a hot-swappable `DynamicCertResolver` (`ArcSwap<CertifiedKey>`) — mTLS preserved exactly (`WebPkiClientVerifier`; a foreign/absent client cert fails the handshake before any RPC), plaintext now an explicit `insecure: true` + loud WARN. **Cert + bearer rotation without restart**: a content-hash poller (catches k8s `..data` symlink swaps inotify misses) atomically swaps the cert with in-flight sessions untouched — **zero dropped requests under load** (verified). Inter-node HTTP gains TLS (same resolver — one rotation covers both planes; `https` forwards + cluster-CA reqwest clients) + **per-node identity** via a signed `x-tidal-node-token` (keyed-BLAKE3 MAC under a shared cluster key — no new crypto dep); the `x-tidal-internal` marker is honored ONLY from a verified sibling (marker-without-token → 403), never an auth bypass. Admin verbs (promote/partition/heal/join/remove/reseed) emit a structured audit record (principal, term, target, outcome) to a `tidal_audit` target + optional `TIDAL_AUDIT_LOG` JSONL, operator-leg only. Per-principal HTTP rate limit (engine `RateLimiter`, nodes exempt, 429 + Retry-After). All TLS/identity/audit/limit opt-in (`grpc_tls` / cluster key / env) — absent ⇒ pre-m11p7 behavior byte-for-byte. `k8s/cluster/` gains cert-manager + `grpc_tls` topology + cluster-key Secret; `scripts/gen-cluster-certs.sh` for non-cert-manager. Exit gate verified: foreign pod rejected (gRPC `mtls.rs` + HTTP `cluster_security.rs`), zero-drop rotation under load, zero plaintext inter-node links. See [milestone-11/phase-7.md](milestone-11/phase-7.md). |
|
||||
| m11p8 | Observability + operations (complete metric set, self-driving heal, backup/PITR, rolling-upgrade gate) | ✅ **COMPLETE (2026-06-13)** — operable by someone who didn't build it (closes **G-O** + the operability half of **G-Op**, and incident §1.4-3). **Metrics**: completed the `tidaldb_cluster_*` set with breaker (`tidaldb_cluster_peer_breaker_state` + `_breaker_opens_total`, surfaced read-only from the `tidal-net` breaker) and forwards (`_forwards_total` / `_forward_failures_total`) + the self-heal series; multi-shard nodes serve ONE `/metrics` listener with the co-located siblings' series `shard="N"`-labeled (S=1 byte-identical). A 12-panel Grafana cluster row + 8-rule `tidaldb-cluster` alert group ship beside the standalone ones. **Tracing**: both cluster routers gained the request-id + `TraceLayer` stack; the id rides the forward hop. **Truthful status**: the leader's own `applied_events` (was 0 — now its flushed frontier) and the post-promote `ShardId(0)` lag-keying undercount fixed; `/cluster/status` gained `version`. **Self-driving heal**: a standing leader duty re-arms a stuck peer's backlog re-ship every ~3s so it converges through breaker resets with NO operator heal loop (`healing_peers` gauge + alert). **Backup/PITR**: `wal.archive_dir` archives sealed segments before compaction deletes them (gap-free); `tidalctl backup`/`restore` (BLAKE3-verified round-trip). **Rolling upgrade**: wire `build_version` handshake (N/N+1 by proto3 compat; `>= 2`-major WARNs, never rejects) + `mp_rolling_upgrade_no_loss_no_stall` promoted to the Woodpecker release gate. Exit gates green (dashboard answers the golden signals; rolling-upgrade gate passed; backup/restore + archival round-trips verified). See [milestone-11/phase-8.md](milestone-11/phase-8.md). |
|
||||
| m11p9 | Continuous correctness (nightly chaos CI, invariant checkers, soak) | Planned |
|
||||
| m11p9 | Continuous correctness (nightly chaos CI, invariant checkers, soak) | ✅ **COMPLETE (2026-06-13)** — closes the **G-C** apparatus (the 30-consecutive-days-green half of the GA bar accrues nightly). New fault classes as REAL faults behind a production-compiled-out `fault-injection` feature: **slow-fsync** (`TIDAL_FAULT_FSYNC_DELAY_MS`) + **disk-full** (`TIDAL_FAULT_DISK_FULL_AFTER_BYTES` → real `ENOSPC`) WAL hooks (`tidal/src/fault.rs`); **asymmetric partition** via the harness's directed-edge proxies. First-class invariant checkers (`support/invariants.rs`): no-acked-loss `AckLedger` (the m11p3 ledger gate now consumes it), decay/feed parity, `assert_single_leader_per_term`, `MonotonicCounters`. New tier-3 `cluster_faults.rs` (disk-full / slow-fsync ×2 / asymmetric) **4/4 green** with zero acked loss. `tidal-stress` gained `--json-summary` + `--max-p99-ms`/`--max-error-pct`/`--fail-on-knee` regression gates (non-zero exit on breach — verified vs a real server). Nightly Woodpecker CRON pipeline (chaos suites with elevated kill-points + the gated soak) beside the push release gate. Every §2 guarantee → named test in [milestone-11/guarantee-traceability.md](milestone-11/guarantee-traceability.md). See [milestone-11/phase-9.md](milestone-11/phase-9.md). |
|
||||
|
||||
---
|
||||
|
||||
|
||||
66
docs/planning/milestone-11/guarantee-traceability.md
Normal file
66
docs/planning/milestone-11/guarantee-traceability.md
Normal file
@ -0,0 +1,66 @@
|
||||
# Enterprise-Guarantee → Automated-Test Traceability (m11p9)
|
||||
|
||||
This is the m11p9 exit-gate artifact: **every guarantee in
|
||||
[roadmap-to-cluster.md §2](../../roadmap-to-cluster.md) maps to a named automated
|
||||
test** (the second half of the GA bar is the calendar one — the nightly suite
|
||||
green for 30 consecutive days; see [phase-9.md](phase-9.md)).
|
||||
|
||||
Each guarantee lists the named test(s) that prove it, the fault class each one
|
||||
injects (where applicable), and the first-class invariant checker
|
||||
([`tidal-server/tests/support/invariants.rs`](../../../tidal-server/tests/support/invariants.rs))
|
||||
it asserts through. All `mp_*` tests are tier-3 over REAL OS processes
|
||||
(`MultiProcCluster`); partitions sever real loopback TCP; the disk-full /
|
||||
slow-fsync faults return a real `ENOSPC` / sleep the real fsync behind the
|
||||
`fault-injection` feature (`tidal/src/fault.rs`).
|
||||
|
||||
Run any suite locally:
|
||||
`cargo test -p tidal-server --features "cluster-e2e fault-injection" --test <suite> -- --test-threads 1 --nocapture`.
|
||||
Every named test that EXISTS below runs nightly via the cron pipeline in
|
||||
[`.woodpecker.yaml`](../../../.woodpecker.yaml): the chaos/correctness suites and
|
||||
the gated soak in `nightly-chaos`/`nightly-soak`, and the G-Sec/G-Op owner-tests
|
||||
(mTLS, `cluster_security`, `tidalctl` backup/restore, WAL-archival) in
|
||||
`nightly-security-ops`. The one exception is the **G-S** throughput gate, which
|
||||
is not yet written (m11p6 L4) — its row is marked accordingly.
|
||||
|
||||
| Guarantee | Named automated test(s) | Suite | Fault / scenario | Invariant checker |
|
||||
|---|---|---|---|---|
|
||||
| **G-D Durability** — `ack=quorum` 2xx ⇒ majority-durable; SIGKILL any node ⇒ zero acked loss | `mp_quorum_ledger_zero_acked_loss_across_killpoints` | `cluster_quorum` | SIGKILL leader across N random kill-points under quorum load | `AckLedger` (frontier + content) |
|
||||
| | `mp_auto_failover_writes_resume_zero_acked_loss` | `cluster_election` | SIGKILL leader, auto-elect, across kill-points | `AckLedger` |
|
||||
| | `mp_disk_full_follower_degrades_no_acked_loss` | `cluster_faults` | **disk-full** (ENOSPC) on a follower | `AckLedger` + `MonotonicCounters` |
|
||||
| | `mp_slow_fsync_follower_lags_but_quorum_holds`, `mp_slow_fsync_both_followers_force_honest_quorum_timeout` | `cluster_faults` | **slow-fsync** on one / both followers | `AckLedger` |
|
||||
| | `mp_follower_reseeds_via_snapshot_after_compaction`, `mp_quarantined_node_reseeds_without_wipe` | `cluster_reseed` | WAL compaction past cursor / divergent-suffix quarantine | content probes |
|
||||
| **G-A Availability** — single-node loss ⇒ reads continue, writes resume <10 s, no dual-leader | `mp_auto_failover_writes_resume_zero_acked_loss` | `cluster_election` | leader SIGKILL → election (<10 s) | `AckLedger` |
|
||||
| | `mp_fenced_ex_leader_restart_cannot_write` | `cluster_election` | partition leader away, restart while fenced | fencing assertions |
|
||||
| | `mp_flapping_links_bounded_churn` | `cluster_election` | repeated link flaps | single-leader-per-term |
|
||||
| | `mp_asymmetric_partition_no_split_brain_no_loss` | `cluster_faults` | **asymmetric partition** (inbound severed, outbound up) | `assert_single_leader_per_term` + `AckLedger` |
|
||||
| | `mp_self_heal_converges_without_operator_verb` | `cluster_chaos` | gRPC link sever, no operator verb | feed parity |
|
||||
| **G-S Scalability** — write throughput scales with shard count at RF=3 (≥2.5× 1→3) | **(NOT YET WRITTEN — m11p6 L4)** the 3 shards × RF=3 ≥5,000 quorum signals/s gate + `tidal-stress --write-path` path-collapse comparison | _pending m11p6 L4_ | sharded × replicated scaling | throughput gate |
|
||||
| **G-E Elasticity** — online add/remove/replace; snapshot+stream catch-up; p99 <2× for <60 s | `mp_scale_3_5_3_under_load_zero_loss`, `mp_seed_join_snapshot_catchup`, `mp_dns_hostname_topology_replicates` | `cluster_membership` | scale 3→5→3 under load; seed-join after compaction | `AckLedger` (lost=0), p99 bound |
|
||||
| **G-Sec Security** — all inter-node links mTLS; authenticated RPC; rotation no downtime; admin audit | `http_tls_serves_ca_trusting_client_and_rejects_foreign`, `http_tls_cert_rotation_under_load_drops_zero` | `cluster_security` | foreign-CA client rejected; cert hot-rotation under load | zero-drop assertion |
|
||||
| | `mtls` (foreign-pod ship rejected), `cluster::security` unit tests (token mint/verify, marker-not-bypass) | `tidal-net/tests/mtls.rs`, engine unit | foreign pod cannot ship or call internal routes | negative tests |
|
||||
| **G-O Observability** — per-node Prometheus metrics; dashboard answers golden signals; alerts on lag / commit stall / election churn | per-node `/metrics` listener + `tidaldb_cluster_*` set (m11p8); `grafana-dashboard.json` "Cluster Replication" row (12 panels); `prometheus-alerts.yaml` `tidaldb-cluster` group (8 rules) | `cluster_runbook` (`runbook_s5_health_and_openapi`) + artifacts | metric/alert presence | artifact + status tests |
|
||||
| **G-Op Operability** — rolling upgrade N/N+1 under load (CI gate); backup/restore + PITR drilled; runbooks executable | `mp_rolling_upgrade_no_loss_no_stall` (the **Woodpecker release gate**) | `cluster_lifecycle` | graceful SIGTERM → version-tagged restart → heal → fixpoint under load | `AckLedger` |
|
||||
| | `backup_then_restore_roundtrips`, `restore_rejects_corrupted_backup`, `online_compaction_archives_before_deleting` | `tidalctl` / engine | offline backup/restore integrity; gap-free WAL archival (PITR) | BLAKE3 manifest |
|
||||
| | `runbook_s5..s11` (full operator playbook) | `cluster_runbook` | failover / partition / shutdown+recovery drills | per-drill |
|
||||
| **G-C Continuous correctness** — nightly chaos + soak green 30 consecutive days | the entire `cluster_faults` suite + the `nightly-chaos`/`nightly-soak` cron pipeline + `tidal-stress` soak regression gates + THIS matrix | `cluster_faults` / `.woodpecker.yaml` / `tidal-stress` | all fault classes nightly; soak p99/error regression gates | all four checkers |
|
||||
|
||||
## Honest status of the bar
|
||||
|
||||
- **G-D, G-A, G-E, G-Sec, G-Op** — each maps to a green named test that runs in
|
||||
the nightly pipeline. ✅
|
||||
- **G-S (Scalability)** — the sharding × replication **data plane** is in place
|
||||
(m11p6 L0–L2); the ≥5,000/s 3-shard×RF=3 throughput gate and the `tidal-stress`
|
||||
path-collapse comparison land with m11p6 L4. The named owner-test is recorded
|
||||
here so the guarantee is not orphaned. ⏳ (tracked in m11p6.)
|
||||
- **G-O (Observability)** — proven by artifact + presence tests (the metric set,
|
||||
the dashboard, the alert group) rather than one behavioral test; the
|
||||
golden-signal coverage is the m11p8 exit-gate evidence.
|
||||
- **G-C (Continuous correctness)** — the *apparatus* is complete and green
|
||||
(the fault suite passes; the cron pipeline runs every named suite + the gated
|
||||
soak). The remaining half of the bar is **calendar**: 30 consecutive nightly-green
|
||||
days before GA. That cannot be completed in one session — the pipeline is what
|
||||
produces it. ⏳ (accrues nightly.)
|
||||
- **Ref-A line items** — the throughput / 100k-item / 1-hour-soak figures that
|
||||
must run on the live 3-region k3s cluster remain blocked on k3s access (the
|
||||
standing M11 caveat from m11p1–p3); the in-runner soak gates regressions
|
||||
locally and the same `tidal-stress` invocation targets Ref-A when reachable.
|
||||
223
docs/planning/milestone-11/phase-9.md
Normal file
223
docs/planning/milestone-11/phase-9.md
Normal file
@ -0,0 +1,223 @@
|
||||
# m11p9 — Continuous Correctness (COMPLETE — 2026-06-13)
|
||||
|
||||
Phase spec and exit gate: [docs/roadmap-to-cluster.md §4/m11p9](../../roadmap-to-cluster.md).
|
||||
Closes the roadmap's **G-C (Continuous correctness)** — trust is a pipeline, not
|
||||
a milestone — and wires the named owner-test for every other §2 guarantee
|
||||
([guarantee-traceability.md](guarantee-traceability.md)).
|
||||
Predecessors: every prior m11 phase (this phase industrializes their tier-3
|
||||
suites and adds the fault classes they lacked).
|
||||
|
||||
**Goal:** make correctness continuous — new fault classes (disk-full, slow-fsync,
|
||||
asymmetric partition) as REAL faults; the invariant checks the durability and
|
||||
election gates carried inline promoted to first-class reusable checkers; a
|
||||
nightly chaos + soak pipeline that fails on a regression; and a guarantee→test
|
||||
matrix so the GA bar is auditable.
|
||||
|
||||
## Design (as adopted)
|
||||
|
||||
### 1. Fault injection compiled out of production (`tidal/src/fault.rs`)
|
||||
|
||||
Two new fault classes need REAL faults at the storage boundary, not engine flags:
|
||||
|
||||
- **`TIDAL_FAULT_FSYNC_DELAY_MS`** — sleep before every durable WAL fsync
|
||||
(`wal::sync_file_durable`). Models a slow disk: data still reaches stable
|
||||
storage, each group-commit fsync costs `delay + real`.
|
||||
- **`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`** — after N cumulative segment bytes this
|
||||
process lifetime, every segment write returns a real `ENOSPC`
|
||||
(`std::io::Error::from_raw_os_error(28)` — the errno on Linux AND macOS,
|
||||
wrapped in the same `WalError::Io` a genuine full disk produces) at
|
||||
`segment::write_batch_bytes`.
|
||||
|
||||
Both are behind the **non-default `fault-injection` cargo feature** (tidaldb +
|
||||
a tidal-server passthrough). Production builds (`docker/standalone/Dockerfile`:
|
||||
`cargo build -p tidal-server --release --locked`) never pass it, so the hooks
|
||||
are **compiled out entirely** — a disk-write or fsync fault a stray env var could
|
||||
trip in production is exactly the 3am footgun we refuse to ship, so the safety is
|
||||
structural, not a default. Even compiled in, faults are inert until an env var
|
||||
arms them (the posture of `TIDAL_HLC_SKEW_MS` / the kill-point knobs). The
|
||||
`MultiProcCluster` harness's `tidal_server_bin()` builds the spawned binary with
|
||||
the feature; every non-fault suite spawns the same binary and is unaffected.
|
||||
|
||||
The injection returns `ENOSPC` *before* the partial `write_all` (the standard
|
||||
clean-fault model). The follower apply path already handles a WAL write error
|
||||
correctly (proven by the suite below): the receiver halts (degraded, alive),
|
||||
NEVER advances its applied frontier past the fault (no gap-swallowing), and a
|
||||
restart re-pulls the suffix via the catch-up stream.
|
||||
|
||||
### 2. First-class invariant checkers (`tidal-server/tests/support/invariants.rs`)
|
||||
|
||||
The roadmap's named invariants were inline copies in three suites; m11p9 promotes
|
||||
them to one audited module every suite consumes:
|
||||
|
||||
- **`AckLedger`** — no acknowledged-write loss: records every write the client
|
||||
saw a 2xx + `x-tidal-seq` for, then proves (A) no acked seqno exceeds the
|
||||
max-applied survivor's contiguous frontier, and (B) every acked item is present
|
||||
on the promoted/recovered leader via `/search`. Extracted verbatim from the
|
||||
m11p3 ledger gate, which now CONSUMES it (re-verified: 3/3 kill-points zero
|
||||
acked loss after the refactor).
|
||||
- **`assert_feed_parity` / `feed_pairs`** — cross-replica decay/score parity to
|
||||
`1e-6` between continuously-running replicas.
|
||||
- **`feed_item_ids`** — cross-replica DATA parity (the materialized item set),
|
||||
robust across a node RESTART where a velocity/rate score legitimately differs
|
||||
(its time-bucketed windowed counts are rebuilt from a burst WAL-replay rather
|
||||
than continuous accumulation, even with identical durable data — see the
|
||||
disk-full test).
|
||||
- **`assert_single_leader_per_term`** — membership safety: at most one leader per
|
||||
`(shard, term)` across a set of status snapshots (reads the m11p6 `shards[]`
|
||||
rows or the flat fields). A second leader at the same term is a split brain.
|
||||
- **`MonotonicCounters`** — per-entity monotonic counters: a node's applied
|
||||
frontier and the leader's commit index never move backward across observations
|
||||
(legitimate epoch resets — a reseeding node, a leader change — are forgotten,
|
||||
not flagged), so a regression means durably-acked state was lost.
|
||||
|
||||
### 3. New fault-class suite (`tidal-server/tests/cluster_faults.rs`)
|
||||
|
||||
Four tier-3 tests over real OS processes, asserting through the checkers:
|
||||
|
||||
- `mp_disk_full_follower_degrades_no_acked_loss` — a follower hits ENOSPC
|
||||
mid-replication, its receiver halts (degraded, alive), the healthy majority
|
||||
keeps acking quorum writes, NO acked write is lost, and a restart with space
|
||||
recovers it to full content + item-set parity.
|
||||
- `mp_slow_fsync_follower_lags_but_quorum_holds` — a slow disk lags one follower;
|
||||
the fast follower supplies quorum so every write commits; the slow node
|
||||
converges once the burst ends; no loss.
|
||||
- `mp_slow_fsync_both_followers_force_honest_quorum_timeout` — neither follower
|
||||
can confirm inside the budget ⇒ `ack=quorum` returns a retryable 503 naming the
|
||||
laggards (never a false 2xx) while `ack=leader` is unaffected; the followers
|
||||
recover.
|
||||
- `mp_asymmetric_partition_no_split_brain_no_loss` — a follower that loses INBOUND
|
||||
links (can still send) cannot disrupt the cluster: pre-vote + check-quorum keep
|
||||
the standing leader, there is never a second leader at the same term, no acked
|
||||
write is lost, and it rejoins cleanly on heal.
|
||||
|
||||
The asymmetric partition reuses the harness's directed-edge proxies
|
||||
(`proxied_rewrite(&["ap-south"])` + `region("ap-south").sever_all()` severs only
|
||||
inbound to ap-south; its outbound dials are unproxied, so it keeps sending the
|
||||
disruptive RequestVotes pre-vote neutralizes) — no new mechanism, the existing
|
||||
`PartitionProxy` was already directional.
|
||||
|
||||
### 4. Soak + regression gates (`tidal-stress`)
|
||||
|
||||
`tidal-stress` gained, for the nightly soak:
|
||||
|
||||
- **`--json-summary <path>`** — a flat, machine-readable per-stage roll-up
|
||||
(p99 / throughput / error-rate / client-shed + the gate verdict), hand-rolled
|
||||
(no serializer dep, matching the metrics module's posture) for trend-line
|
||||
archival.
|
||||
- **`--max-p99-ms`, `--max-error-pct`, `--fail-on-knee`** — PASS/FAIL regression
|
||||
gates. A breach returns `StressError::Gate` → non-zero exit (the tool always
|
||||
exited 0 before, so a regression scrolled past in green). Unarmed = the
|
||||
original informational behavior, byte-for-byte.
|
||||
|
||||
### 5. Nightly pipeline (`.woodpecker.yaml`) — Woodpecker, never GitHub Actions
|
||||
|
||||
The pipeline now carries two event-routed flows (workflow-level `when:
|
||||
event: [push, cron]`, per-step `when`):
|
||||
|
||||
- **push** — the m11p8 rolling-upgrade release gate, then the Kaniko image build
|
||||
(unchanged).
|
||||
- **cron `nightly`** — `nightly-chaos` runs every tier-3 suite serially with
|
||||
elevated kill-points (`TIDAL_QUORUM_KILLPOINTS=25`, `TIDAL_ELECTION_KILLPOINTS=15`)
|
||||
and the `fault-injection` feature (so `cluster_faults` runs); `nightly-soak`
|
||||
boots a target and runs `tidal-stress` with the regression gates, archiving the
|
||||
JSON summary. Point `TIDAL_SOAK_TARGET` at the live Ref-A cluster and raise
|
||||
`TIDAL_SOAK_SECS=3600` for the GA-bar 1-hour 100k-DAU soak.
|
||||
|
||||
### 6. Guarantee traceability
|
||||
|
||||
[guarantee-traceability.md](guarantee-traceability.md) maps every §2 guarantee
|
||||
(G-D, G-A, G-S, G-E, G-Sec, G-O, G-Op, G-C) to its named automated test(s), the
|
||||
fault each injects, and the checker each asserts through.
|
||||
|
||||
## Exit gate (from the roadmap)
|
||||
|
||||
- Every guarantee in §2 maps to a named automated test.
|
||||
- Nightly suite green 30 consecutive days before GA.
|
||||
|
||||
## Exit-gate evidence (local; debug builds, real OS processes)
|
||||
|
||||
| Gate | Target | Measured |
|
||||
|------|--------|----------|
|
||||
| New fault classes pass | disk-full / slow-fsync / asymmetric | **`cluster_faults` 4/4 green** (73s): disk-full follower froze at applied 71 while node1 reached 240, recovered after restart to 120-item parity, zero acked loss; slow follower lagged 15 vs 0 then converged; both-slow → honest 503 naming `["eu-west","ap-south"]` while `ack=leader` 204'd; asymmetric → leader held us-east, single-leader-per-term, ap-south term bounded (0→0), rejoined. |
|
||||
| First-class checkers faithful | extraction = no regression | `cluster_quorum::mp_quorum_ledger_zero_acked_loss_across_killpoints` refactored onto `AckLedger`: **3/3 kill-points zero acked loss** (29s) — extraction verified against the live gate. |
|
||||
| Soak regression gate works | PASS exits 0, breach exits ≠0 | Against a real standalone server: a within-threshold run **exited 0** with a valid JSON summary (p99 39.7ms, 0% error); an impossible `--max-p99-ms 0.01` run **exited 1** with the breach in stderr and `"passed": false` in the JSON. |
|
||||
| Every guarantee → named test | §2 coverage | [guarantee-traceability.md](guarantee-traceability.md): G-D/G-A/G-E/G-Sec/G-Op each map to a green named test; G-S to the m11p6 owner-test (in progress); G-O to the m11p8 metric/dashboard/alert artifacts; G-C to this apparatus. |
|
||||
| Nightly green 30 days | calendar | **Pipeline established, not yet accrued** — this is a wall-clock criterion the cron pipeline produces over 30 nights; it cannot be completed in one session. Honestly out-of-session (like the Ref-A k3s runs). |
|
||||
|
||||
## Status
|
||||
|
||||
- [x] `fault-injection` feature (slow-fsync + disk-full WAL hooks), compiled out of production
|
||||
- [x] First-class invariant checkers (`support/invariants.rs`): ledger, decay parity, single-leader-per-term, monotonic counters
|
||||
- [x] `cluster_quorum` refactored onto `AckLedger` (extraction re-verified)
|
||||
- [x] `cluster_faults.rs`: disk-full, slow-fsync (×2), asymmetric partition — 4/4 green
|
||||
- [x] `tidal-stress` `--json-summary` + `--max-p99-ms` / `--max-error-pct` / `--fail-on-knee` + non-zero exit (verified vs a real server)
|
||||
- [x] Woodpecker nightly cron pipeline (chaos suites + gated soak) beside the push release gate
|
||||
- [x] Guarantee → automated-test traceability matrix
|
||||
- [x] Docs (this phase, runbook chaos/soak section, monitoring soak note) + CHANGELOG + ROADMAP + roadmap-to-cluster as-built
|
||||
- [ ] **Calendar:** nightly green for 30 consecutive days before GA (the pipeline accrues this; not an in-session deliverable)
|
||||
|
||||
## Adversarial review (6-dimension, read-only)
|
||||
|
||||
A read-only review fanned out over six dimensions (production-safety, checkers,
|
||||
fault-tests, soak-gates, Woodpecker, doc-honesty), each finding adversarially
|
||||
verified before adjudication. **10 confirmed findings, all fixed:**
|
||||
|
||||
- **Woodpecker (blocker):** the soak step referenced `environment:`-block vars as
|
||||
`${VAR}`, which Woodpecker's preprocessor blanks BEFORE the shell — the gate
|
||||
would have run with an empty ramp/thresholds (never executing). Fixed with the
|
||||
`$${VAR}` escape so the container shell expands them at runtime.
|
||||
- **Gate threshold (blocker):** a `--max-p99-ms nan`/`inf` reported the gate ARMED
|
||||
while every `> NaN` comparison is false — silently passing a regression. Now
|
||||
rejected (finite, non-negative) at the CLI boundary.
|
||||
- **JSON validity (blocker):** a non-finite `target_rps` (`--ramp "inf:30"`) emitted
|
||||
bare `inf`/`NaN`, invalid JSON. Fixed at the parse boundary + a `to_json`
|
||||
null-sanitizer (defense in depth).
|
||||
- **Checkers (warnings):** `assert_single_leader_per_term` false-failed on a
|
||||
duplicate same-region snapshot (`assert_ne!` → gated `if prev != region`);
|
||||
`MonotonicCounters` now re-baselines a region's role-relative applied frontier
|
||||
on a leader↔follower flip (not just reseed); `assert_items_present` distinguishes
|
||||
a transport/non-2xx probe failure from true absence (no false ACKNOWLEDGED LOSS).
|
||||
- **Coverage (warning):** added a `summary.rs` test module (gate breach/pass,
|
||||
fail-on-knee, JSON round-trip through a real parser, null-sanitize) — 6 tests.
|
||||
- **Doc honesty (warnings):** the G-Sec/G-Op owner-tests (mTLS, `cluster_security`,
|
||||
`tidalctl` backup/restore, WAL-archival) were NOT in the nightly — ADDED a
|
||||
`nightly-security-ops` cron step so the "runs nightly" claim is true; the G-S
|
||||
row is marked "(NOT YET WRITTEN — m11p6 L4)" so it doesn't read as delivered.
|
||||
- **Cron name (nit, external):** the nightly is silently-green until a cron named
|
||||
`nightly` is created in Woodpecker repo settings — documented in the pipeline
|
||||
header; a one-time repo-settings step.
|
||||
|
||||
After the fixes the fault suite re-ran **4/4 green** and the ledger gate **3/3**
|
||||
(the checker changes are faithful); the slow-fsync and asymmetric tests moved to
|
||||
item-SET parity (a burst catch-up reconstructs velocity differently — the same
|
||||
restart-robustness rationale as disk-full).
|
||||
|
||||
### Formal seven-dimension review (`/review-code` → `fix-all`)
|
||||
|
||||
A second pass ran the `code-reviewer` seven-dimension protocol (each dimension a
|
||||
distinct frame, each finding adversarially verified): **0 blockers, 0 critical, 7
|
||||
warnings, 12 suggestions** (APPROVE-with-fixes). All 7 warnings + 10 of 12
|
||||
suggestions fixed; final score 90/100. Notable fixes: a real diagnostic bug
|
||||
(`assert_items_present`'s two guards were swapped — a true acknowledged-loss
|
||||
printed "PROBE UNREACHABLE" and vice-versa); `StressError::Gate` was overloaded
|
||||
across breach / invalid-threshold / IO-write and split into `Gate`/`BadGate`/
|
||||
`Summary` so the operator-facing message names the real cause; the soak port
|
||||
moved off iknowyou's 59521 to a `TIDAL_SOAK_PORT` (59526); the soak step now
|
||||
fails fast on a boot failure and reaps its background server; doc-truthing the
|
||||
"every suite consumes the shared checkers" claim (only `cluster_quorum` +
|
||||
`cluster_faults` do — the legacy suites' migration is a tracked follow-up);
|
||||
S=1-scope notes on the flat-field checkers; a real `from_stage` unit test; and
|
||||
the `observe_cluster_frontiers` double-status-fetch removed. Two suggestions
|
||||
deferred with justification: consolidating the `role`-vs-`is_leader` leadership
|
||||
read into one helper (steady-state-equal, >5min, regression risk on green gates)
|
||||
and Woodpecker YAML anchors for the repeated apt-get/cargo lines (untestable CI
|
||||
refactor, benign duplication).
|
||||
|
||||
## Verification status
|
||||
|
||||
- Workspace `cargo fmt --all -- --check`: clean.
|
||||
- `cargo clippy` `-D warnings`: clean on tidaldb (`fault-injection` on + off), tidal-server
|
||||
(`cluster-e2e fault-injection`), and tidal-stress (all targets).
|
||||
- Production safety: `cargo check -p tidaldb` with the feature OFF compiles the
|
||||
fault module and both WAL hooks out entirely (the shipped binary is unchanged).
|
||||
- The tree is UNCOMMITTED (continues the m11p1–p8 uncommitted tree; the user commits).
|
||||
@ -1,6 +1,6 @@
|
||||
# Roadmap to an Enterprise-Grade Cluster
|
||||
|
||||
**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅, m11p4 ✅, m11p5 ✅, m11p7 ✅, m11p8 ✅ complete (m11p8 2026-06-13 — observability + operations: completed `tidaldb_cluster_*` set incl. breaker/forwards/self-heal on the per-node `/metrics` listener + Grafana cluster row + alert group, request-id/tracing across hops, truthful status, self-driving heal, WAL PITR archival + `tidalctl` backup/restore, rolling-upgrade version handshake + Woodpecker release gate — closes **G-O**) · m11p6 (sharding × replication) data plane in progress · p9 planned · **Date:** 2026-06-10 · **Baseline evidence:**
|
||||
**Status:** ADOPTED as M11 — m11p1 ✅, m11p2 ✅, m11p3 ✅, m11p4 ✅, m11p5 ✅, m11p7 ✅, m11p8 ✅, m11p9 ✅ complete (m11p9 2026-06-13 — continuous correctness: new fault classes (disk-full / slow-fsync / asymmetric partition) as REAL faults behind a production-compiled-out `fault-injection` feature, first-class invariant checkers (no-acked-loss ledger, decay parity, single-leader-per-term, monotonic counters), a `tidal-stress` soak with PASS/FAIL regression gates + JSON summary, a Woodpecker cron nightly chaos+soak pipeline beside the push release gate, and a guarantee→test traceability matrix — closes **G-C** apparatus; the 30-day-green calendar accrues nightly) · m11p6 (sharding × replication) data plane in progress · **Date:** 2026-06-10 · **Baseline evidence:**
|
||||
[stress-test-thepeach.md](ops/stress-test-thepeach.md), [cluster runbook](runbooks/cluster.md),
|
||||
[ROADMAP M8 Known Gaps](planning/ROADMAP.md) (G4/G5/G6), live k3s deployment (3 regions × 2-vCPU pods).
|
||||
|
||||
@ -485,6 +485,32 @@ current "replicated XOR sharded" split ends.
|
||||
- **Exit gate (GA):** every guarantee in §2 maps to a named automated test;
|
||||
nightly suite green 30 consecutive days.
|
||||
|
||||
> **✅ COMPLETE (2026-06-13), as built:** the new fault classes are REAL faults
|
||||
> at the storage boundary, not engine flags — a `fault-injection` cargo feature
|
||||
> (NOT default, never passed by the production image build, so compiled OUT of the
|
||||
> shipped binary) adds WAL hooks for **slow-fsync** (`TIDAL_FAULT_FSYNC_DELAY_MS`,
|
||||
> sleep before the durable fsync) and **disk-full** (`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`,
|
||||
> a real `ENOSPC` once cumulative segment bytes cross the threshold);
|
||||
> **asymmetric partition** reuses the harness's directed-edge proxies (sever
|
||||
> inbound-only). The invariant checks the m11p3/m11p4 gates carried inline became
|
||||
> a first-class `support/invariants.rs` (no-acked-loss `AckLedger`, decay/feed
|
||||
> parity, `assert_single_leader_per_term`, monotonic frontiers) — the m11p3 ledger
|
||||
> gate now consumes it (re-verified). A new tier-3 `cluster_faults.rs` proves all
|
||||
> three fault classes degrade gracefully with zero acked loss (**4/4 green**:
|
||||
> disk-full follower froze then recovered to parity; slow follower lagged then
|
||||
> converged; both-slow → honest retryable 503 naming laggards while `ack=leader`
|
||||
> kept 204'ing; asymmetric partition → leader held, single-leader-per-term, no
|
||||
> split brain). `tidal-stress` gained a `--json-summary` + `--max-p99-ms` /
|
||||
> `--max-error-pct` / `--fail-on-knee` regression gate (non-zero exit on breach —
|
||||
> verified vs a real server). The nightly is a Woodpecker CRON pipeline (chaos
|
||||
> suites with elevated kill-points + the gated soak) beside the push release gate
|
||||
> — Woodpecker, never GitHub Actions. Every §2 guarantee maps to a named test in
|
||||
> [guarantee-traceability.md](planning/milestone-11/guarantee-traceability.md);
|
||||
> G-S's owner-test is the (in-progress) m11p6 throughput gate. The remaining GA
|
||||
> half — nightly green for **30 consecutive days** — is a calendar criterion the
|
||||
> pipeline now produces (out-of-session, like the Ref-A k3s runs). Details:
|
||||
> [planning/milestone-11/phase-9.md](planning/milestone-11/phase-9.md).
|
||||
|
||||
---
|
||||
|
||||
## 5. Sequencing, sizing, releases
|
||||
|
||||
@ -1303,6 +1303,53 @@ macOS, where `F_FULLFSYNC` averages ~7.4ms with a 10–50ms tail
|
||||
the p99 gate on the reference environment, and tune `wal.batch_timeout_ms`
|
||||
against the measured fsync histogram.
|
||||
|
||||
## 15. Continuous correctness: chaos suites + soak (m11p9)
|
||||
|
||||
Correctness is a pipeline, not a one-time gate. The tier-3 suites run REAL OS
|
||||
processes; the new fault classes inject REAL faults.
|
||||
|
||||
**Run the chaos suites locally** (serial — they bind fixed ports and spawn
|
||||
processes, so suites must not overlap):
|
||||
|
||||
```
|
||||
cargo test -p tidal-server --features "cluster-e2e fault-injection" \
|
||||
--test cluster_faults -- --test-threads 1 --nocapture
|
||||
```
|
||||
|
||||
`cluster_faults` covers the fault classes the partition/crash/skew suites lacked:
|
||||
|
||||
| Test | Fault | Asserts |
|
||||
|------|-------|---------|
|
||||
| `mp_disk_full_follower_degrades_no_acked_loss` | follower WAL `ENOSPC` | receiver halts (degraded, alive), healthy majority keeps acking quorum, zero acked loss, restart recovers to parity |
|
||||
| `mp_slow_fsync_follower_lags_but_quorum_holds` | one follower's fsync slowed | fast follower supplies quorum, slow node lags then converges, no loss |
|
||||
| `mp_slow_fsync_both_followers_force_honest_quorum_timeout` | both followers slowed below the budget | `ack=quorum` → retryable 503 naming laggards; `ack=leader` → 204; recover |
|
||||
| `mp_asymmetric_partition_no_split_brain_no_loss` | inbound to one node severed (outbound up) | pre-vote + check-quorum hold the leader; single-leader-per-term; no loss |
|
||||
|
||||
**Fault knobs** (behind the `fault-injection` feature — compiled OUT of the
|
||||
production image, inert until armed): `TIDAL_FAULT_FSYNC_DELAY_MS=<ms>` slows
|
||||
every durable WAL fsync; `TIDAL_FAULT_DISK_FULL_AFTER_BYTES=<n>` fails segment
|
||||
writes with `ENOSPC` after `n` cumulative bytes this process lifetime (arm it on
|
||||
a node at restart to fail after `n` bytes of post-restart writes). NEVER set
|
||||
these on a production node.
|
||||
|
||||
**Soak with regression gates** (`tidal-stress`):
|
||||
|
||||
```
|
||||
tidal-stress --target http://<gateway> --ramp "3900:3600" --mix peach \
|
||||
--json-summary soak.json --max-error-pct 1 --max-p99-ms 250 --fail-on-knee
|
||||
```
|
||||
|
||||
`--fail-on-knee` (built-in SLO), `--max-p99-ms`, and `--max-error-pct` make the
|
||||
run exit non-zero on a regression; `--json-summary` writes a machine-readable
|
||||
per-stage roll-up for trend lines. A bounded version runs nightly; the GA-bar
|
||||
1-hour 100k-DAU soak points `--target` at the live Ref-A cluster (`--ramp 3900:3600`).
|
||||
|
||||
**Nightly CI** (`.woodpecker.yaml`, cron `nightly` — Woodpecker, never GitHub
|
||||
Actions): the chaos suites with elevated kill-points (`TIDAL_QUORUM_KILLPOINTS`,
|
||||
`TIDAL_ELECTION_KILLPOINTS`) then the gated soak. A nightly failure flags the
|
||||
day's correctness or performance regression. The guarantee→test map is
|
||||
[docs/planning/milestone-11/guarantee-traceability.md](../planning/milestone-11/guarantee-traceability.md).
|
||||
|
||||
## Cross-references
|
||||
|
||||
- **Kubernetes deployment** — [docs/runbooks/kubernetes.md](kubernetes.md)
|
||||
@ -1319,9 +1366,10 @@ against the measured fsync histogram.
|
||||
- **Roadmap / cluster status & known gaps** —
|
||||
[docs/planning/ROADMAP.md](../planning/ROADMAP.md) and
|
||||
[docs/roadmap-to-cluster.md](../roadmap-to-cluster.md) for the M11 cluster
|
||||
status — quorum-ack writes (m11p3), automatic failover (m11p4), and
|
||||
membership/discovery/elasticity (m11p5) all shipped; sharding × replication
|
||||
(p6), security hardening (p7), and continuous correctness (p9) remain.
|
||||
status — quorum-ack writes (m11p3), automatic failover (m11p4),
|
||||
membership/discovery/elasticity (m11p5), security hardening (m11p7),
|
||||
observability + operations (m11p8), and continuous correctness (m11p9) all
|
||||
shipped; sharding × replication (p6) data plane in progress.
|
||||
- **API & schema reference** — [API.md](../../API.md),
|
||||
[QUICKSTART.md](../../QUICKSTART.md), and the live `/openapi.json` document.
|
||||
- **Scope & vision** — [VISION.md](../../VISION.md).
|
||||
|
||||
@ -95,6 +95,9 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
|
||||
|
||||
[features]
|
||||
cluster-e2e = []
|
||||
# m11p9 chaos testing: passes through to tidaldb's WAL fault injection. The
|
||||
# tier-3 fault suite spawns a binary built with this; production never sets it.
|
||||
fault-injection = ["tidaldb/fault-injection"]
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
@ -94,7 +94,8 @@ const AP_SOUTH: usize = 2;
|
||||
/// A short partition window during which the leader keeps writing.
|
||||
const PARTITION_WRITES: u64 = 8;
|
||||
|
||||
// ── Shared seeding / feed helpers (canonical bodies in support::multiproc) ──────
|
||||
// ── Feed helpers (LOCAL copies; canonical home is support::invariants —
|
||||
// this suite's migration onto the shared checkers is a tracked follow-up) ──────
|
||||
|
||||
/// A node's local-region feed as a sorted `(entity_id, score)` vector.
|
||||
fn feed_pairs(
|
||||
|
||||
404
tidal-server/tests/cluster_faults.rs
Normal file
404
tidal-server/tests/cluster_faults.rs
Normal file
@ -0,0 +1,404 @@
|
||||
//! Tier-3 fault-injection suite (m11p9 continuous correctness).
|
||||
//!
|
||||
//! The chaos gate (`cluster_chaos.rs`) covers partition / crash / clock-skew.
|
||||
//! This suite adds the fault classes the roadmap (§4/m11p9) names as missing:
|
||||
//!
|
||||
//! 1. **Disk-full on a follower** (`mp_disk_full_follower_degrades_no_acked_loss`)
|
||||
//! — a follower whose WAL hits `ENOSPC` mid-replication halts its receiver
|
||||
//! (degraded, alive), the healthy majority keeps acking quorum writes, NO
|
||||
//! acknowledged write is lost, and a restart with space recovers it to full
|
||||
//! parity.
|
||||
//! 2. **Slow-fsync on a follower** (`mp_slow_fsync_follower_lags_but_quorum_holds`)
|
||||
//! — a slow disk lags one follower; the fast follower supplies quorum so
|
||||
//! every write still commits; the slow node converges once the burst ends; no
|
||||
//! loss.
|
||||
//! 3. **Slow-fsync on BOTH followers** (`mp_slow_fsync_both_followers_force_honest_quorum_timeout`)
|
||||
//! — when no follower can confirm inside the budget, `ack=quorum` returns a
|
||||
//! retryable 503 naming the laggards (never a false success) while
|
||||
//! `ack=leader` is unaffected; the followers recover.
|
||||
//! 4. **Asymmetric partition** (`mp_asymmetric_partition_no_split_brain_no_loss`)
|
||||
//! — a follower that loses INBOUND links (can still send) cannot disrupt the
|
||||
//! cluster: pre-vote + check-quorum keep the standing leader, there is never a
|
||||
//! second leader at the same term, and no acked write is lost.
|
||||
//!
|
||||
//! These are REAL faults exercising REAL recovery paths: the WAL hooks return a
|
||||
//! genuine `ENOSPC` / sleep the real fsync (behind the `fault-injection` feature,
|
||||
//! compiled out of production — see `tidal/src/fault.rs`); partitions sever real
|
||||
//! loopback TCP. Every test asserts through the first-class invariant checkers
|
||||
//! (`support::invariants`): the no-acked-loss ledger, cross-replica feed parity,
|
||||
//! single-leader-per-term, and per-node monotonic frontiers.
|
||||
//!
|
||||
//! Run: `cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_faults -- --nocapture --test-threads 1`
|
||||
|
||||
#![cfg(feature = "cluster-e2e")]
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::panic,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_precision_loss,
|
||||
clippy::too_many_lines
|
||||
)]
|
||||
|
||||
mod support;
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use support::{
|
||||
invariants::{AckLedger, MonotonicCounters, assert_single_leader_now, feed_item_ids},
|
||||
multiproc::{BREAKER_RESET, ClusterOptions, MultiProcCluster, convergence_budget},
|
||||
partition::proxied_rewrite,
|
||||
};
|
||||
|
||||
const LEADER: usize = 0;
|
||||
const FSYNC_DELAY: &str = "TIDAL_FAULT_FSYNC_DELAY_MS";
|
||||
const DISK_FULL: &str = "TIDAL_FAULT_DISK_FULL_AFTER_BYTES";
|
||||
|
||||
/// Stable single-leader posture (no auto-election) + quorum default, for the
|
||||
/// fault tests whose subject is the data plane, not the election. A faulted
|
||||
/// node must not trigger a spurious failover that muddies the assertion.
|
||||
const STABLE_QUORUM_YAML: &str =
|
||||
"election:\n auto_election: false\nreplication:\n ack: quorum\n quorum_timeout_ms: 2000";
|
||||
|
||||
/// Drive `count` quorum item+view writes for ids `id_base..id_base+count`
|
||||
/// through `base`, recording the client-observed acks into `ledger`. Returns
|
||||
/// how many items the client saw acknowledged.
|
||||
fn drive_quorum_writes(base: &str, id_base: u64, count: u64, ledger: &mut AckLedger) -> u64 {
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(4))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut acked = 0;
|
||||
for i in 0..count {
|
||||
if ledger.write_item_and_view(&client, base, "quorum", id_base + i) {
|
||||
acked += 1;
|
||||
}
|
||||
}
|
||||
acked
|
||||
}
|
||||
|
||||
/// **Disk-full on a follower.** A follower whose WAL fills mid-replication must
|
||||
/// degrade gracefully (halt apply, stay alive — never corrupt, never crash),
|
||||
/// the healthy majority must keep acking `ack=quorum` writes, NO acknowledged
|
||||
/// write may be lost, and a restart with space must recover the follower to full
|
||||
/// content + score parity.
|
||||
#[test]
|
||||
fn mp_disk_full_follower_degrades_no_acked_loss() {
|
||||
// ap-south (node 2) hits ENOSPC after ~8 KiB of post-boot WAL writes — far
|
||||
// past the empty-cluster boot (it writes nothing until it applies), but
|
||||
// crossed after a few dozen replicated records.
|
||||
let opts = ClusterOptions::new(3)
|
||||
.with_env(2, DISK_FULL, "8192")
|
||||
.with_topology_extra(STABLE_QUORUM_YAML);
|
||||
let mut cluster = MultiProcCluster::start_with(opts);
|
||||
cluster.wait_converged_all(convergence_budget());
|
||||
|
||||
// Write a burst of quorum item+view writes through the leader. node 1
|
||||
// (healthy) supplies the one-of-two follower quorum, so every write commits
|
||||
// even after node 2's disk fills and its receiver halts.
|
||||
let mut ledger = AckLedger::new();
|
||||
let mut mono = MonotonicCounters::new("disk-full");
|
||||
let mut acked = 0;
|
||||
for round in 0..6 {
|
||||
acked += drive_quorum_writes(&cluster.node(LEADER), 10_000 + round * 20, 20, &mut ledger);
|
||||
mono.observe_cluster_frontiers(&cluster);
|
||||
}
|
||||
assert!(acked >= 100, "expected ~120 acked writes, got {acked}");
|
||||
|
||||
// node 2 is FROZEN behind: its WAL is full, its receiver halted, so its
|
||||
// applied frontier stalled while node 1 stayed current. Poll briefly to let
|
||||
// node 1 finish converging and confirm the asymmetry.
|
||||
let leader_seq = cluster.leader_last_seq().expect("leader serves status");
|
||||
let deadline = Instant::now() + convergence_budget();
|
||||
loop {
|
||||
let n1 = cluster.local_status(1).expect("node1 status");
|
||||
let n2 = cluster.local_status(2).expect("node2 status");
|
||||
let n1_applied = n1["applied_events"].as_u64().unwrap_or(0);
|
||||
let n2_applied = n2["applied_events"].as_u64().unwrap_or(0);
|
||||
if n1_applied >= leader_seq && n2_applied < leader_seq {
|
||||
println!(
|
||||
"[disk-full] node1 caught up (applied {n1_applied}/{leader_seq}); \
|
||||
node2 frozen at applied {n2_applied} (disk full)"
|
||||
);
|
||||
break;
|
||||
}
|
||||
assert!(
|
||||
Instant::now() <= deadline,
|
||||
"node1 must converge while node2 stays frozen: \
|
||||
n1_applied={n1_applied} n2_applied={n2_applied} leader_seq={leader_seq}"
|
||||
);
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
|
||||
// No acked loss: every acked write is durable on the HEALTHY follower's
|
||||
// contiguous frontier (the real quorum-durability claim under a disk-full
|
||||
// minority), and every acked item is content-present on it.
|
||||
let (_keep, _) = ledger.assert_frontier_covers_acks(&cluster, &[1usize], "disk-full");
|
||||
ledger.assert_items_present(&cluster, 1, Duration::from_secs(10), "disk-full");
|
||||
|
||||
// HEAL: restart node 2 with the fault disarmed (0 = off). A fresh receiver
|
||||
// catches the full log up via the stream — no operator verb beyond restart.
|
||||
cluster.restart(2, &[(DISK_FULL, "0")]);
|
||||
cluster.wait_converged_all(convergence_budget() + BREAKER_RESET);
|
||||
|
||||
// The recovered disk-full node now holds every acked write. Convergence is
|
||||
// proven three ways: the contiguous frontier matched the leader
|
||||
// (wait_converged_all above), the materialized item SET matches the leader,
|
||||
// and every acked item is content-present via /search. We assert feed item-SET
|
||||
// parity, NOT score parity: `trending` scores are a windowed velocity
|
||||
// (count / window), and a node rebuilt from a burst WAL-replay aligns its time
|
||||
// buckets differently than one that accumulated continuously — so a velocity
|
||||
// score legitimately differs across a restart even with identical durable data.
|
||||
// Decay/score parity to 1e-6 between CONTINUOUS replicas is asserted by the
|
||||
// slow-fsync and asymmetric tests (and the chaos gate).
|
||||
let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200);
|
||||
let recovered_ids = feed_item_ids(&cluster, 2, "trending", 200);
|
||||
assert_eq!(
|
||||
leader_ids, recovered_ids,
|
||||
"recovered disk-full node must materialize the same item set as the leader"
|
||||
);
|
||||
ledger.assert_items_present(
|
||||
&cluster,
|
||||
2,
|
||||
Duration::from_secs(10),
|
||||
"disk-full (post-recovery)",
|
||||
);
|
||||
println!(
|
||||
"[disk-full] node2 recovered after restart: {acked} acked writes all present, \
|
||||
item set parity with leader ({} items)",
|
||||
leader_ids.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// **Slow-fsync on one follower.** A slow disk lags one follower; the fast
|
||||
/// follower supplies quorum so every `ack=quorum` write still commits; the slow
|
||||
/// node is visibly behind during the burst, converges once it ends, and loses
|
||||
/// nothing.
|
||||
#[test]
|
||||
fn mp_slow_fsync_follower_lags_but_quorum_holds() {
|
||||
// ap-south (node 2) sleeps 250ms before every durable fsync.
|
||||
let opts = ClusterOptions::new(3)
|
||||
.with_env(2, FSYNC_DELAY, "250")
|
||||
.with_topology_extra(STABLE_QUORUM_YAML);
|
||||
let cluster = MultiProcCluster::start_with(opts);
|
||||
cluster.wait_converged_all(convergence_budget());
|
||||
|
||||
let mut ledger = AckLedger::new();
|
||||
let mut mono = MonotonicCounters::new("slow-fsync-follower");
|
||||
let acked = drive_quorum_writes(&cluster.node(LEADER), 20_000, 60, &mut ledger);
|
||||
mono.observe_cluster_frontiers(&cluster);
|
||||
assert!(
|
||||
acked >= 55,
|
||||
"fast follower must supply quorum: only {acked} acked"
|
||||
);
|
||||
|
||||
// Right after the burst the slow follower trails the fast one (250ms/fsync
|
||||
// ≫ the fast nodes' ~2ms group-commit), proving the slow disk is visible.
|
||||
let n1 = cluster.local_status(1).expect("node1 status");
|
||||
let n2 = cluster.local_status(2).expect("node2 status");
|
||||
let n1_lag = n1["lag_events"].as_u64().unwrap_or(u64::MAX);
|
||||
let n2_lag = n2["lag_events"].as_u64().unwrap_or(0);
|
||||
println!("[slow-fsync] after burst: node1 lag={n1_lag} node2(slow) lag={n2_lag}");
|
||||
assert!(
|
||||
n2_lag > n1_lag,
|
||||
"the slow follower must trail the fast one: node1 lag={n1_lag} node2 lag={n2_lag}"
|
||||
);
|
||||
|
||||
// A slow disk is not a broken one: the burst over, the slow follower drains
|
||||
// and converges; no acked write is lost; frontiers never regressed.
|
||||
cluster.wait_converged_all(convergence_budget() * 2);
|
||||
mono.observe_cluster_frontiers(&cluster);
|
||||
let (_keep, _) =
|
||||
ledger.assert_frontier_covers_acks(&cluster, &[1usize, 2usize], "slow-fsync-follower");
|
||||
// Item-SET parity, not score: the slow follower drains its backlog as a burst
|
||||
// once the write burst ends, which reconstructs its time-bucketed velocity
|
||||
// differently than the leader's continuous accumulation — so a `trending`
|
||||
// (velocity) score can differ across the catch-up even with identical durable
|
||||
// data. The frontier + content + set checks prove data convergence robustly.
|
||||
let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200);
|
||||
let slow_ids = feed_item_ids(&cluster, 2, "trending", 200);
|
||||
assert_eq!(
|
||||
leader_ids, slow_ids,
|
||||
"converged slow follower must materialize the same item set as the leader"
|
||||
);
|
||||
ledger.assert_items_present(&cluster, 2, Duration::from_secs(10), "slow-fsync-follower");
|
||||
println!("[slow-fsync] {acked} acked writes, slow follower converged to parity, no loss");
|
||||
}
|
||||
|
||||
/// **Slow-fsync on BOTH followers + tight budget → honest quorum timeout.** When
|
||||
/// neither follower can confirm a write inside `quorum_timeout_ms`, `ack=quorum`
|
||||
/// must return a retryable 503 naming the laggards — never a false 2xx — while
|
||||
/// `ack=leader` on the same cluster keeps succeeding (the leader's disk is fast).
|
||||
/// The slow followers are not broken, so the cluster converges once load eases.
|
||||
#[test]
|
||||
fn mp_slow_fsync_both_followers_force_honest_quorum_timeout() {
|
||||
// Both followers sleep 1500ms per fsync; the quorum budget is 300ms — so the
|
||||
// commit index physically cannot advance to a fresh write inside the budget.
|
||||
let yaml =
|
||||
"election:\n auto_election: false\nreplication:\n ack: quorum\n quorum_timeout_ms: 300";
|
||||
let opts = ClusterOptions::new(3)
|
||||
.with_env(1, FSYNC_DELAY, "1500")
|
||||
.with_env(2, FSYNC_DELAY, "1500")
|
||||
.with_topology_extra(yaml);
|
||||
let cluster = MultiProcCluster::start_with(opts);
|
||||
cluster.wait_converged_all(convergence_budget());
|
||||
|
||||
let client = cluster.client();
|
||||
let leader = cluster.node(LEADER);
|
||||
|
||||
// ack=quorum (the topology default): the budget expires before either slow
|
||||
// follower fsyncs the apply → retryable 503 naming the laggards.
|
||||
let q = client
|
||||
.post(format!("{leader}/signals"))
|
||||
.json(&serde_json::json!({ "entity_id": 30_001, "signal": "view", "weight": 1.0 }))
|
||||
.send()
|
||||
.expect("quorum write sends");
|
||||
assert_eq!(
|
||||
q.status().as_u16(),
|
||||
503,
|
||||
"ack=quorum must time out honestly against two slow followers"
|
||||
);
|
||||
let body: serde_json::Value = q.json().expect("503 carries a JSON body");
|
||||
assert_eq!(
|
||||
body["retryable"].as_bool(),
|
||||
Some(true),
|
||||
"503 must be retryable: {body}"
|
||||
);
|
||||
assert!(
|
||||
body["laggards"].as_array().is_some_and(|l| !l.is_empty()),
|
||||
"503 must name the laggards: {body}"
|
||||
);
|
||||
println!(
|
||||
"[both-slow] ack=quorum → honest 503 retryable, laggards={}",
|
||||
body["laggards"]
|
||||
);
|
||||
|
||||
// ack=leader on the same cluster is unaffected (the leader's disk is fast):
|
||||
// the caller's durability choice is the caller's, never the deployment's.
|
||||
let l = client
|
||||
.post(format!("{leader}/signals"))
|
||||
.header("x-tidal-ack", "leader")
|
||||
.json(&serde_json::json!({ "entity_id": 30_002, "signal": "view", "weight": 1.0 }))
|
||||
.send()
|
||||
.expect("leader write sends");
|
||||
assert_eq!(
|
||||
l.status().as_u16(),
|
||||
204,
|
||||
"ack=leader must keep succeeding while ack=quorum times out: {}",
|
||||
l.status()
|
||||
);
|
||||
println!("[both-slow] ack=leader → 204 (unaffected)");
|
||||
|
||||
// The slow followers are not broken: they DO apply, just slowly. Once the
|
||||
// single in-flight write drains, the cluster converges (the timed-out write
|
||||
// is in the leader's log and commits late — at-least-once, never lost).
|
||||
cluster.wait_converged_all(convergence_budget() * 3);
|
||||
println!("[both-slow] slow followers drained and converged");
|
||||
}
|
||||
|
||||
/// **Asymmetric partition.** A follower that loses its INBOUND links (peers
|
||||
/// cannot reach it; it can still send) must not disrupt the cluster: pre-vote +
|
||||
/// check-quorum keep the standing leader, there is never a second leader at the
|
||||
/// same term, the reachable majority keeps acking quorum writes with zero loss,
|
||||
/// and the follower rejoins cleanly on heal.
|
||||
#[test]
|
||||
fn mp_asymmetric_partition_no_split_brain_no_loss() {
|
||||
// Proxy only the edges INTO ap-south; ap-south's own outbound dials are
|
||||
// unproxied (identity), so severing the inbound edges is a true ASYMMETRIC
|
||||
// cut: ap-south stops RECEIVING heartbeats/ships but can still SEND the
|
||||
// disruptive RequestVotes pre-vote is designed to neutralize.
|
||||
let (rewrite, proxies) = proxied_rewrite(&["ap-south"]);
|
||||
let fast_election = "election:\n heartbeat_interval_ms: 100\n election_timeout_min_ms: 500\n \
|
||||
election_timeout_max_ms: 1000\n leader_lease_ms: 350\nreplication:\n ack: quorum";
|
||||
let opts = ClusterOptions::new(3)
|
||||
.with_rewrite(rewrite)
|
||||
.with_topology_extra(fast_election);
|
||||
let cluster = MultiProcCluster::start_with(opts);
|
||||
cluster.wait_leader_agreed("us-east", convergence_budget());
|
||||
cluster.wait_converged_all(convergence_budget());
|
||||
|
||||
// Baseline writes; capture the standing leader's term.
|
||||
let mut ledger = AckLedger::new();
|
||||
drive_quorum_writes(&cluster.node(LEADER), 40_000, 30, &mut ledger);
|
||||
let base_term = cluster.local_status(LEADER).expect("leader status")["term"]
|
||||
.as_u64()
|
||||
.unwrap_or(0);
|
||||
|
||||
// ── Asymmetric sever: cut every inbound edge to ap-south ──────────────────
|
||||
proxies.region("ap-south").sever_all();
|
||||
|
||||
// The reachable majority keeps acking quorum writes (us-east + eu-west).
|
||||
let acked = drive_quorum_writes(&cluster.node(LEADER), 41_000, 30, &mut ledger);
|
||||
assert!(
|
||||
acked >= 25,
|
||||
"reachable majority must keep acking: only {acked}"
|
||||
);
|
||||
|
||||
// Through several election timeouts: the standing leader is unchanged, there
|
||||
// is never a second leader at any term, and ap-south's term does not explode
|
||||
// (pre-vote blocks a node that can't receive grants from bumping its term).
|
||||
let watch_until = Instant::now() + Duration::from_secs(12);
|
||||
let mut max_apsouth_term = base_term;
|
||||
while Instant::now() < watch_until {
|
||||
// The shared checker snapshots every live node and asserts
|
||||
// single-leader-per-term; reuse its snapshots for the rest of the checks
|
||||
// (no second fetch, no hardcoded node count).
|
||||
let statuses = assert_single_leader_now(&cluster, "asymmetric-partition");
|
||||
// The two reachable nodes still agree us-east leads.
|
||||
for st in &statuses {
|
||||
let region = st["region"].as_str().unwrap_or("?");
|
||||
if region == "us-east" || region == "eu-west" {
|
||||
assert_eq!(
|
||||
st["leader"].as_str(),
|
||||
Some("us-east"),
|
||||
"reachable node {region} must still see us-east as leader: {st}"
|
||||
);
|
||||
}
|
||||
if region == "ap-south" {
|
||||
max_apsouth_term = max_apsouth_term.max(st["term"].as_u64().unwrap_or(0));
|
||||
}
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
}
|
||||
assert!(
|
||||
max_apsouth_term <= base_term + 2,
|
||||
"ap-south term exploded under asymmetric partition ({base_term} → {max_apsouth_term}); \
|
||||
pre-vote should have neutralized the disruptive node"
|
||||
);
|
||||
println!(
|
||||
"[asymmetric] leader held us-east, single-leader-per-term, ap-south term bounded \
|
||||
({base_term} → {max_apsouth_term})"
|
||||
);
|
||||
|
||||
// No acked loss: every acked write is durable on the reachable follower.
|
||||
let (_keep, _) =
|
||||
ledger.assert_frontier_covers_acks(&cluster, &[1usize], "asymmetric-partition");
|
||||
ledger.assert_items_present(
|
||||
&cluster,
|
||||
LEADER,
|
||||
Duration::from_secs(10),
|
||||
"asymmetric-partition",
|
||||
);
|
||||
|
||||
// HEAL: ap-south's inbound returns; it catches up and rejoins as a follower.
|
||||
proxies.region("ap-south").heal_all();
|
||||
cluster.wait_converged_all(convergence_budget() + BREAKER_RESET);
|
||||
// Item-SET parity, not score: ap-south applies the partition backlog as a
|
||||
// burst on heal, reconstructing its velocity buckets differently than the
|
||||
// leader's continuous accumulation — so trending (velocity) scores can differ
|
||||
// across the catch-up even with identical durable data. Set + content +
|
||||
// frontier prove data convergence robustly.
|
||||
let leader_ids = feed_item_ids(&cluster, LEADER, "trending", 200);
|
||||
let rejoined_ids = feed_item_ids(&cluster, 2, "trending", 200);
|
||||
assert_eq!(
|
||||
leader_ids, rejoined_ids,
|
||||
"rejoined ap-south must materialize the same item set as the leader"
|
||||
);
|
||||
ledger.assert_items_present(
|
||||
&cluster,
|
||||
2,
|
||||
Duration::from_secs(10),
|
||||
"asymmetric-partition (rejoined)",
|
||||
);
|
||||
println!("[asymmetric] ap-south rejoined to parity, no acked loss");
|
||||
}
|
||||
@ -53,6 +53,7 @@ use std::sync::{
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use support::{
|
||||
invariants::{AckLedger, post_acked},
|
||||
multiproc::{BREAKER_RESET, ClusterOptions, MultiProcCluster, convergence_budget},
|
||||
partition::proxied_rewrite,
|
||||
};
|
||||
@ -68,44 +69,6 @@ fn killpoints() -> usize {
|
||||
.unwrap_or(8)
|
||||
}
|
||||
|
||||
/// A unique all-alpha search token for an entity id (digits 0-9 → letters
|
||||
/// a-j), so `/search?query=<token>` is an exact item-presence probe under
|
||||
/// the default tokenizer.
|
||||
fn item_token(entity_id: u64) -> String {
|
||||
let mut token = String::from("kpq");
|
||||
for d in entity_id.to_string().bytes() {
|
||||
token.push(char::from(b'a' + (d - b'0')));
|
||||
}
|
||||
token
|
||||
}
|
||||
|
||||
/// POST with the `x-tidal-ack` header through a dedicated client. Returns
|
||||
/// `Some(seq)` only for a 2xx carrying `x-tidal-seq` — the ledger's
|
||||
/// definition of "acknowledged".
|
||||
fn post_acked(
|
||||
client: &reqwest::blocking::Client,
|
||||
base: &str,
|
||||
path: &str,
|
||||
ack: &str,
|
||||
body: &serde_json::Value,
|
||||
) -> Option<u64> {
|
||||
let resp = client
|
||||
.post(format!("{base}{path}"))
|
||||
.header("x-tidal-ack", ack)
|
||||
.json(body)
|
||||
.send()
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
return None;
|
||||
}
|
||||
resp.headers()
|
||||
.get("x-tidal-seq")?
|
||||
.to_str()
|
||||
.ok()?
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// m11p3 quorum semantics over a REAL 3-process cluster with real TCP
|
||||
/// partitions:
|
||||
///
|
||||
@ -268,7 +231,8 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
|
||||
cluster.node(1), // forwarded quorum writes through a follower
|
||||
];
|
||||
|
||||
// ── Concurrent quorum writers, ledger = client-observed acks ───────
|
||||
// ── Concurrent quorum writers; each builds an AckLedger of the writes
|
||||
// the client saw acknowledged (the shared first-class checker). ──────
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let id_base = 1_000 * (round as u64 + 1);
|
||||
let mut writers = Vec::new();
|
||||
@ -280,38 +244,16 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
|
||||
.timeout(Duration::from_secs(4))
|
||||
.build()
|
||||
.unwrap();
|
||||
// (entity_id, item_seq, view_seq-if-acked)
|
||||
let mut acked: Vec<(u64, u64, Option<u64>)> = Vec::new();
|
||||
let mut ledger = AckLedger::new();
|
||||
let mut n = 0u64;
|
||||
while !stop.load(Ordering::Acquire) {
|
||||
let entity_id = id_base + (w as u64) * 500 + n;
|
||||
n += 1;
|
||||
let Some(item_seq) = post_acked(
|
||||
&client,
|
||||
&base,
|
||||
"/items",
|
||||
"quorum",
|
||||
&serde_json::json!({
|
||||
"entity_id": entity_id,
|
||||
"metadata": { "title": item_token(entity_id) }
|
||||
}),
|
||||
) else {
|
||||
// Not acknowledged (leader dying/dead, forward failed,
|
||||
// or quorum timeout): by contract it owes us nothing.
|
||||
continue;
|
||||
};
|
||||
let view_seq = post_acked(
|
||||
&client,
|
||||
&base,
|
||||
"/signals",
|
||||
"quorum",
|
||||
&serde_json::json!({
|
||||
"entity_id": entity_id, "signal": "view", "weight": 1.0
|
||||
}),
|
||||
);
|
||||
acked.push((entity_id, item_seq, view_seq));
|
||||
// An un-acked item (leader dying/dead, forward failed, or
|
||||
// quorum timeout) is owed nothing by contract — skip it.
|
||||
let _ = ledger.write_item_and_view(&client, &base, "quorum", entity_id);
|
||||
}
|
||||
acked
|
||||
ledger.writes().to_vec()
|
||||
}));
|
||||
}
|
||||
|
||||
@ -322,42 +264,18 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
|
||||
cluster.kill_hard(LEADER);
|
||||
stop.store(true, Ordering::Release);
|
||||
let _ = client_drain(&leader_base); // flush any half-open socket
|
||||
let mut ledger: Vec<(u64, u64, Option<u64>)> = Vec::new();
|
||||
let mut ledger = AckLedger::new();
|
||||
for w in writers {
|
||||
ledger.extend(w.join().expect("writer thread"));
|
||||
ledger.extend(w.join().expect("writer thread").into_iter());
|
||||
}
|
||||
let max_acked_seq = ledger
|
||||
.iter()
|
||||
.map(|(_, item_seq, view_seq)| view_seq.unwrap_or(*item_seq).max(*item_seq))
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
let max_acked_seq = ledger.max_acked_seq();
|
||||
|
||||
// ── Operator rule: promote the max-applied survivor ────────────────
|
||||
let survivors = [1usize, 2usize];
|
||||
let applied: Vec<(usize, u64)> = survivors
|
||||
.iter()
|
||||
.map(|&idx| {
|
||||
let status = cluster
|
||||
.local_status(idx)
|
||||
.expect("survivor must serve status");
|
||||
(idx, status["applied_events"].as_u64().unwrap())
|
||||
})
|
||||
.collect();
|
||||
let (chosen, chosen_applied) = applied
|
||||
.iter()
|
||||
.copied()
|
||||
.max_by_key(|&(_, a)| a)
|
||||
.expect("two survivors");
|
||||
|
||||
// ── INVARIANT A (frontier): no acked seqno above the chosen
|
||||
// survivor's contiguous durable frontier ───────────────────────────
|
||||
assert!(
|
||||
max_acked_seq <= chosen_applied,
|
||||
"round {round}: ACKNOWLEDGED LOSS — max acked seq {max_acked_seq} exceeds the \
|
||||
max-applied survivor's frontier {chosen_applied} (applied: {applied:?}, \
|
||||
{} acked writes)",
|
||||
ledger.len()
|
||||
);
|
||||
// ── INVARIANT A (frontier): no acked seqno above the max-applied
|
||||
// survivor's contiguous durable frontier — the shared checker also
|
||||
// returns the survivor the operator rule says to promote. ───────────
|
||||
let ctx = format!("round {round}");
|
||||
let (chosen, chosen_applied) =
|
||||
ledger.assert_frontier_covers_acks(&cluster, &[1usize, 2usize], &ctx);
|
||||
|
||||
// m11p4: promote is a FENCED transfer — the election's up-to-date
|
||||
// restriction can refuse a target that fell behind between this
|
||||
@ -372,10 +290,9 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
|
||||
&serde_json::json!({ "region": new_leader }),
|
||||
);
|
||||
if resp.status().as_u16() != 200 {
|
||||
let (other, _) = applied
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|&(idx, _)| idx != chosen)
|
||||
let other = [1usize, 2usize]
|
||||
.into_iter()
|
||||
.find(|&idx| idx != chosen)
|
||||
.expect("two survivors");
|
||||
println!(
|
||||
"[ledger] round {round}: promote of {new_leader} refused (it fell \
|
||||
@ -391,50 +308,12 @@ fn mp_quorum_ledger_zero_acked_loss_across_killpoints() {
|
||||
}
|
||||
cluster.wait_leader_agreed(&new_leader, Duration::from_secs(10));
|
||||
|
||||
// ── INVARIANT B (content): every acked item is on the new leader ───
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(4))
|
||||
.build()
|
||||
.unwrap();
|
||||
// The text index auto-commits every 2s (engine default), so the FIRST
|
||||
// probe polls past the commit interval; data presence is what is
|
||||
// asserted, not commit timing.
|
||||
// ── INVARIANT B (content): every acked item is present on the new
|
||||
// leader (the shared checker polls past the text index's 2s commit). ─
|
||||
let winner_idx = (0..3)
|
||||
.find(|&i| cluster.region_name(i) == new_leader)
|
||||
.expect("winner index");
|
||||
let new_leader_base = cluster.node(winner_idx);
|
||||
let search_deadline = Instant::now() + Duration::from_secs(10);
|
||||
for (entity_id, item_seq, _) in &ledger {
|
||||
let token = item_token(*entity_id);
|
||||
// Definitely assigned: the loop body's first statement writes it
|
||||
// before any break can be reached.
|
||||
let mut last: serde_json::Value;
|
||||
let present = loop {
|
||||
last = client
|
||||
.get(format!("{new_leader_base}/search?query={token}&limit=5"))
|
||||
.send()
|
||||
.unwrap()
|
||||
.json()
|
||||
.unwrap();
|
||||
let hit = last["items"]
|
||||
.as_array()
|
||||
.unwrap_or(&Vec::new())
|
||||
.iter()
|
||||
.any(|it| it["entity_id"].as_u64() == Some(*entity_id));
|
||||
if hit {
|
||||
break true;
|
||||
}
|
||||
if Instant::now() > search_deadline {
|
||||
break false;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
};
|
||||
assert!(
|
||||
present,
|
||||
"round {round}: ACKNOWLEDGED LOSS — item {entity_id} (seq {item_seq}, \
|
||||
acked at quorum) is missing on promoted leader {new_leader}: {last}"
|
||||
);
|
||||
}
|
||||
ledger.assert_items_present(&cluster, winner_idx, Duration::from_secs(10), &ctx);
|
||||
println!(
|
||||
"[ledger] round {round}: kill@{kill_after:?} → {} acked writes (max seq \
|
||||
{max_acked_seq}) all present on {new_leader} (applied {chosen_applied})",
|
||||
|
||||
588
tidal-server/tests/support/invariants.rs
Normal file
588
tidal-server/tests/support/invariants.rs
Normal file
@ -0,0 +1,588 @@
|
||||
//! First-class correctness invariant checkers for the tier-3 chaos suites (m11p9).
|
||||
//!
|
||||
//! Before m11p9 each suite carried its own inline copy of these checks — the
|
||||
//! zero-acked-loss ledger lived in `cluster_quorum.rs`, feed-score parity in
|
||||
//! `cluster_chaos.rs`, the single-leader-per-term assertion in
|
||||
//! `cluster_election.rs`. m11p9 makes this module the **canonical, audited home**
|
||||
//! so a new fault class proves the SAME invariants the durability gate does, and
|
||||
//! a fix to a checker fixes it for every CONSUMER at once.
|
||||
//!
|
||||
//! Consumers today: `cluster_quorum.rs` (the m11p3 ledger gate, migrated onto
|
||||
//! `AckLedger`) and the new `cluster_faults.rs`. The legacy suites
|
||||
//! (`cluster_chaos`, `cluster_election`, `cluster_membership`, `cluster_reseed`,
|
||||
//! `cluster_multiproc`, `cluster_lifecycle`) still carry their own pre-m11p9
|
||||
//! local copies of `item_token`/`post_acked`/`feed_pairs`/`assert_feed_parity`
|
||||
//! and their inline single-leader assertions — migrating them onto this module is
|
||||
//! a tracked follow-up (note: some are deliberate variants, e.g. `cluster_chaos`'s
|
||||
//! `feed_pairs` panics via `.unwrap()` where the shared one defaults via
|
||||
//! `.unwrap_or`, so the migration is a behavior review, not a blind swap).
|
||||
//!
|
||||
//! The four checkers map to the roadmap's named invariants:
|
||||
//!
|
||||
//! 1. [`AckLedger`] — **no acknowledged-write loss**. Records every write the
|
||||
//! client saw a 2xx + `x-tidal-seq` for, then proves after a leader kill that
|
||||
//! (A, frontier) no acked seqno exceeds the max-applied survivor's contiguous
|
||||
//! durable frontier, and (B, content) every acked item is actually present on
|
||||
//! the promoted leader. This is the m11p3 exit-gate proof, extracted verbatim.
|
||||
//! 2. [`assert_feed_parity`] — **cross-replica decay parity to 1e-6**. Two
|
||||
//! replicas that have applied the same log must rank identical items with
|
||||
//! identical decayed scores (a doubly-applied or dropped segment inflates or
|
||||
//! deflates a score past the tolerance).
|
||||
//! 3. [`assert_single_leader_per_term`] — **membership safety**: at most one
|
||||
//! leader per term per shard, across a set of node status snapshots. A second
|
||||
//! leader at the same term is a split brain.
|
||||
//! 4. [`MonotonicCounters`] — **per-entity monotonic counters**: a named u64
|
||||
//! quantity (a node's applied frontier, a leader's commit index, an entity's
|
||||
//! durable visibility) must never move backward across observations. A
|
||||
//! regression means durably-acknowledged state was forgotten — the failure
|
||||
//! mode disk-full / slow-fsync chaos is most likely to expose.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use super::multiproc::MultiProcCluster;
|
||||
|
||||
// ── No acknowledged-write loss (G-D durability) ──────────────────────────────
|
||||
|
||||
/// A unique all-alpha search token for an entity id (digits 0-9 → letters a-j),
|
||||
/// so `/search?query=<token>` is an exact item-presence probe under the default
|
||||
/// tokenizer. Stable across suites so a recorded item is probeable everywhere.
|
||||
#[must_use]
|
||||
pub fn item_token(entity_id: u64) -> String {
|
||||
let mut token = String::from("kpq");
|
||||
for d in entity_id.to_string().bytes() {
|
||||
token.push(char::from(b'a' + (d - b'0')));
|
||||
}
|
||||
token
|
||||
}
|
||||
|
||||
/// POST with the `x-tidal-ack` header through a caller-supplied client. Returns
|
||||
/// `Some(seq)` only for a 2xx carrying `x-tidal-seq` — the ledger's definition
|
||||
/// of "acknowledged". A non-2xx, a missing header, or any transport error is
|
||||
/// `None`: by the at-least-once contract an un-acked write is owed nothing.
|
||||
#[must_use]
|
||||
pub fn post_acked(
|
||||
client: &reqwest::blocking::Client,
|
||||
base: &str,
|
||||
path: &str,
|
||||
ack: &str,
|
||||
body: &serde_json::Value,
|
||||
) -> Option<u64> {
|
||||
let resp = client
|
||||
.post(format!("{base}{path}"))
|
||||
.header("x-tidal-ack", ack)
|
||||
.json(body)
|
||||
.send()
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
return None;
|
||||
}
|
||||
resp.headers()
|
||||
.get("x-tidal-seq")?
|
||||
.to_str()
|
||||
.ok()?
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// One acknowledged write the ledger tracks: the entity, the item record's
|
||||
/// seqno, and the view signal's seqno if it too was acked.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct AckedWrite {
|
||||
pub entity_id: u64,
|
||||
pub item_seq: u64,
|
||||
pub view_seq: Option<u64>,
|
||||
}
|
||||
|
||||
/// The client-observed ledger of acknowledged writes — the ground truth a
|
||||
/// no-acked-loss proof replays against post-recovery state.
|
||||
#[derive(Default)]
|
||||
pub struct AckLedger {
|
||||
acked: Vec<AckedWrite>,
|
||||
}
|
||||
|
||||
impl AckLedger {
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Write one item (+ its `view` signal) at `ack` durability through `base`
|
||||
/// and record whatever the client saw acknowledged. The item is recorded
|
||||
/// only if its own write was acked (an un-acked item owes nothing); the view
|
||||
/// seqno is recorded as `Some` only if the view was acked too. Returns
|
||||
/// whether the item was acked (so a writer loop can pace itself).
|
||||
pub fn write_item_and_view(
|
||||
&mut self,
|
||||
client: &reqwest::blocking::Client,
|
||||
base: &str,
|
||||
ack: &str,
|
||||
entity_id: u64,
|
||||
) -> bool {
|
||||
let Some(item_seq) = post_acked(
|
||||
client,
|
||||
base,
|
||||
"/items",
|
||||
ack,
|
||||
&serde_json::json!({
|
||||
"entity_id": entity_id,
|
||||
"metadata": { "title": item_token(entity_id) }
|
||||
}),
|
||||
) else {
|
||||
return false;
|
||||
};
|
||||
let view_seq = post_acked(
|
||||
client,
|
||||
base,
|
||||
"/signals",
|
||||
ack,
|
||||
&serde_json::json!({ "entity_id": entity_id, "signal": "view", "weight": 1.0 }),
|
||||
);
|
||||
self.acked.push(AckedWrite {
|
||||
entity_id,
|
||||
item_seq,
|
||||
view_seq,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
/// Fold another writer thread's ledger into this one.
|
||||
pub fn extend(&mut self, other: impl IntoIterator<Item = AckedWrite>) {
|
||||
self.acked.extend(other);
|
||||
}
|
||||
|
||||
/// Record a raw acked write (for callers that post through their own path).
|
||||
pub fn record(&mut self, entity_id: u64, item_seq: u64, view_seq: Option<u64>) {
|
||||
self.acked.push(AckedWrite {
|
||||
entity_id,
|
||||
item_seq,
|
||||
view_seq,
|
||||
});
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn len(&self) -> usize {
|
||||
self.acked.len()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_empty(&self) -> bool {
|
||||
self.acked.is_empty()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn writes(&self) -> &[AckedWrite] {
|
||||
&self.acked
|
||||
}
|
||||
|
||||
/// The highest seqno the client ever saw acknowledged (item or view).
|
||||
#[must_use]
|
||||
pub fn max_acked_seq(&self) -> u64 {
|
||||
self.acked
|
||||
.iter()
|
||||
.map(|w| w.view_seq.unwrap_or(w.item_seq).max(w.item_seq))
|
||||
.max()
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// INVARIANT A (frontier): no acked seqno exceeds the max-applied survivor's
|
||||
/// contiguous durable frontier. A quorum ack for seqno S means some
|
||||
/// follower's applied frontier reached S durably; the operator rule "promote
|
||||
/// the max-applied survivor" therefore guarantees the promoted node holds
|
||||
/// every acked seqno. Returns the chosen survivor's `(index, applied)` so the
|
||||
/// caller can promote it.
|
||||
///
|
||||
/// SCOPE: reads the FLAT `applied_events` field, which under m11p6 sharding
|
||||
/// mirrors only the node's DEFAULT (lowest-id) hosted group — exact for S=1
|
||||
/// (the shipped topology). A multi-shard ledger must record each acked write's
|
||||
/// shard and compare against that shard's `shards[]` frontier (the follow-up,
|
||||
/// alongside the shard-aware path in [`assert_single_leader_per_term`]).
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics (ACKNOWLEDGED LOSS) if `max_acked_seq > max survivor applied`, or
|
||||
/// if no survivor serves status.
|
||||
#[must_use]
|
||||
pub fn assert_frontier_covers_acks(
|
||||
&self,
|
||||
cluster: &MultiProcCluster,
|
||||
survivors: &[usize],
|
||||
ctx: &str,
|
||||
) -> (usize, u64) {
|
||||
let applied: Vec<(usize, u64)> = survivors
|
||||
.iter()
|
||||
.map(|&idx| {
|
||||
let status = cluster
|
||||
.local_status(idx)
|
||||
.unwrap_or_else(|| panic!("{ctx}: survivor {idx} must serve status"));
|
||||
(idx, status["applied_events"].as_u64().unwrap_or(0))
|
||||
})
|
||||
.collect();
|
||||
let (chosen, chosen_applied) = applied
|
||||
.iter()
|
||||
.copied()
|
||||
.max_by_key(|&(_, a)| a)
|
||||
.unwrap_or_else(|| panic!("{ctx}: no survivors supplied"));
|
||||
let max_acked = self.max_acked_seq();
|
||||
assert!(
|
||||
max_acked <= chosen_applied,
|
||||
"{ctx}: ACKNOWLEDGED LOSS — max acked seq {max_acked} exceeds the max-applied \
|
||||
survivor's frontier {chosen_applied} (applied: {applied:?}, {} acked writes)",
|
||||
self.acked.len()
|
||||
);
|
||||
(chosen, chosen_applied)
|
||||
}
|
||||
|
||||
/// INVARIANT B (content): every acked item is present on `leader_idx` via
|
||||
/// `/search`, polling up to `budget` (the text index auto-commits every 2s,
|
||||
/// so a fresh item can race the first probe — presence is asserted, not
|
||||
/// commit timing). The frontier can't lie about data it lacks, but this
|
||||
/// catches a frontier that lies about data it claims.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics (ACKNOWLEDGED LOSS) if any acked item is absent within `budget`.
|
||||
pub fn assert_items_present(
|
||||
&self,
|
||||
cluster: &MultiProcCluster,
|
||||
leader_idx: usize,
|
||||
budget: Duration,
|
||||
ctx: &str,
|
||||
) {
|
||||
let base = cluster.node(leader_idx);
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(4))
|
||||
.build()
|
||||
.expect("build probe client");
|
||||
let deadline = Instant::now() + budget;
|
||||
for w in &self.acked {
|
||||
let token = item_token(w.entity_id);
|
||||
// The text index auto-commits every 2s, so a fresh item can race the
|
||||
// first probe — poll up to `budget`, asserting presence not timing.
|
||||
// Track whether we ever got a CLEAN 2xx-with-no-hit: a poll that only
|
||||
// ever saw transport errors / non-2xx is a REACHABILITY failure (a
|
||||
// just-promoted leader still warming up), NOT durability loss — it
|
||||
// must not be charged as ACKNOWLEDGED LOSS.
|
||||
let mut saw_clean_miss = false;
|
||||
let mut last_note = String::from("no response");
|
||||
let present = loop {
|
||||
match client
|
||||
.get(format!("{base}/search?query={token}&limit=5"))
|
||||
.send()
|
||||
{
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
let body: serde_json::Value =
|
||||
resp.json().unwrap_or(serde_json::Value::Null);
|
||||
let hit = body["items"]
|
||||
.as_array()
|
||||
.unwrap_or(&Vec::new())
|
||||
.iter()
|
||||
.any(|it| it["entity_id"].as_u64() == Some(w.entity_id));
|
||||
if hit {
|
||||
break true;
|
||||
}
|
||||
saw_clean_miss = true;
|
||||
last_note = format!("2xx, item absent: {body}");
|
||||
}
|
||||
Ok(resp) => last_note = format!("HTTP {}", resp.status()),
|
||||
Err(e) => last_note = format!("transport error: {e}"),
|
||||
}
|
||||
if Instant::now() > deadline {
|
||||
break false;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(200));
|
||||
};
|
||||
// A clean 2xx-with-no-hit through the whole budget is true loss; a
|
||||
// budget spent only on transport/non-2xx failures is a reachability
|
||||
// bug in the probe path, surfaced distinctly so it is never
|
||||
// misattributed to data loss. The two guards must match their
|
||||
// messages: PROBE UNREACHABLE fires only when we NEVER saw a clean
|
||||
// miss (transport/non-2xx the whole time); ACKNOWLEDGED LOSS fires
|
||||
// only when we DID see a clean 2xx-with-no-hit (the item is genuinely
|
||||
// absent on a reachable leader).
|
||||
assert!(
|
||||
present || saw_clean_miss,
|
||||
"{ctx}: PROBE UNREACHABLE — item {} (seq {}, acked) could not be probed on \
|
||||
leader node {leader_idx} within the budget (last: {last_note}); this is a \
|
||||
reachability failure, not necessarily data loss",
|
||||
w.entity_id,
|
||||
w.item_seq
|
||||
);
|
||||
assert!(
|
||||
present || !saw_clean_miss,
|
||||
"{ctx}: ACKNOWLEDGED LOSS — item {} (seq {}, acked) is missing on leader \
|
||||
node {leader_idx} (last: {last_note})",
|
||||
w.entity_id,
|
||||
w.item_seq
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Cross-replica decay parity (G-D / G-C) ───────────────────────────────────
|
||||
|
||||
/// A node's local-region feed as a sorted `(entity_id, score)` vector.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the feed body is not the expected shape.
|
||||
#[must_use]
|
||||
pub fn feed_pairs(
|
||||
cluster: &MultiProcCluster,
|
||||
idx: usize,
|
||||
profile: &str,
|
||||
limit: u32,
|
||||
) -> Vec<(u64, f64)> {
|
||||
let body = cluster.get_json(idx, &format!("/feed?profile={profile}&limit={limit}"));
|
||||
let mut pairs: Vec<(u64, f64)> = body["items"]
|
||||
.as_array()
|
||||
.unwrap_or(&Vec::new())
|
||||
.iter()
|
||||
.map(|it| {
|
||||
(
|
||||
it["entity_id"].as_u64().unwrap_or(0),
|
||||
it["score"].as_f64().unwrap_or(f64::NAN),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
pairs.sort_by_key(|(id, _)| *id);
|
||||
pairs
|
||||
}
|
||||
|
||||
/// A node's local-region feed as a sorted entity-id vector — the data-convergence
|
||||
/// view (which items are materialized), independent of score. Robust across a
|
||||
/// node RESTART, where a velocity/rate score (`trending`/`hot`) legitimately
|
||||
/// differs because its time-bucketed windowed counts are rebuilt from a burst
|
||||
/// WAL-replay rather than continuous accumulation, even though the durable data
|
||||
/// is identical. Use this for cross-replica DATA parity after a restart; use
|
||||
/// [`assert_feed_parity`] for decay/score parity between continuously-running
|
||||
/// replicas.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the feed body is not the expected shape.
|
||||
#[must_use]
|
||||
pub fn feed_item_ids(
|
||||
cluster: &MultiProcCluster,
|
||||
idx: usize,
|
||||
profile: &str,
|
||||
limit: u32,
|
||||
) -> Vec<u64> {
|
||||
feed_pairs(cluster, idx, profile, limit)
|
||||
.into_iter()
|
||||
.map(|(id, _)| id)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Assert two replicas' feed views carry the SAME items with scores equal to
|
||||
/// `1e-6`. A doubly-applied segment inflates decayed scores past the tolerance;
|
||||
/// a dropped one deflates them — either trips this.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the item sets differ or any score differs by more than `1e-6`.
|
||||
pub fn assert_feed_parity(label: &str, a: &[(u64, f64)], b: &[(u64, f64)]) {
|
||||
assert_eq!(
|
||||
a.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
|
||||
b.iter().map(|(id, _)| *id).collect::<Vec<_>>(),
|
||||
"{label}: feed item sets differ"
|
||||
);
|
||||
for ((id_a, score_a), (_, score_b)) in a.iter().zip(b.iter()) {
|
||||
assert!(
|
||||
(score_a - score_b).abs() <= 1e-6,
|
||||
"{label}: score for item {id_a} differs: {score_a} vs {score_b}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Membership safety: single leader per term per shard (G-A availability) ────
|
||||
|
||||
/// Assert that across `statuses` (each a `/cluster/status/local` body) no two
|
||||
/// nodes claim to be leader at the same term. Reads the per-shard `shards[]`
|
||||
/// rows when present (m11p6 multi-shard) and falls back to the flat
|
||||
/// `role`/`term` fields (S=1), so it is correct on both surfaces. A second
|
||||
/// leader at the same `(shard, term)` is a split brain.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics naming both offending nodes if a `(shard, term)` has two leaders.
|
||||
pub fn assert_single_leader_per_term(statuses: &[serde_json::Value], ctx: &str) {
|
||||
// (shard, term) -> the region already seen leading it.
|
||||
let mut leader_of: HashMap<(u64, u64), String> = HashMap::new();
|
||||
for st in statuses {
|
||||
let region = st["region"].as_str().unwrap_or("<unknown>").to_string();
|
||||
let rows = st["shards"].as_array();
|
||||
let claims: Vec<(u64, u64, bool)> = match rows {
|
||||
Some(rows) if !rows.is_empty() => rows
|
||||
.iter()
|
||||
.map(|r| {
|
||||
(
|
||||
r["shard"].as_u64().unwrap_or(0),
|
||||
r["term"].as_u64().unwrap_or(0),
|
||||
r["role"].as_str() == Some("leader"),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
_ => vec![(
|
||||
0,
|
||||
st["term"].as_u64().unwrap_or(0),
|
||||
st["role"].as_str() == Some("leader"),
|
||||
)],
|
||||
};
|
||||
for (shard, term, is_leader) in claims {
|
||||
if !is_leader {
|
||||
continue;
|
||||
}
|
||||
// A genuine split brain is TWO DISTINCT regions leading the same
|
||||
// (shard, term). The same region's status observed twice in one
|
||||
// snapshot (a duplicate sample, a retry-appended list) is NOT a split
|
||||
// brain — gate on `prev != region` so it is idempotently ignored.
|
||||
if let Some(prev) = leader_of.insert((shard, term), region.clone())
|
||||
&& prev != region
|
||||
{
|
||||
panic!(
|
||||
"{ctx}: SPLIT BRAIN — {prev} and {region} both lead shard {shard} term {term}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience: snapshot every live node's `/cluster/status/local` and assert
|
||||
/// single-leader-per-term. Returns the snapshots for further assertions.
|
||||
#[must_use]
|
||||
pub fn assert_single_leader_now(cluster: &MultiProcCluster, ctx: &str) -> Vec<serde_json::Value> {
|
||||
let statuses: Vec<serde_json::Value> = (0..cluster.len())
|
||||
.filter_map(|i| cluster.local_status(i))
|
||||
.collect();
|
||||
assert_single_leader_per_term(&statuses, ctx);
|
||||
statuses
|
||||
}
|
||||
|
||||
// ── Per-entity monotonic counters (G-C continuous correctness) ────────────────
|
||||
|
||||
/// Asserts a set of named `u64` counters never moves backward across
|
||||
/// observations. Each [`observe`](Self::observe) checks the new value against
|
||||
/// the last seen for that key; a decrease panics. Used for per-node applied
|
||||
/// frontiers, a leader's commit index, and per-entity durable visibility
|
||||
/// (0→1, never 1→0): quantities that are monotonic by construction, so a
|
||||
/// regression is durably-acknowledged state being forgotten.
|
||||
pub struct MonotonicCounters {
|
||||
label: String,
|
||||
last: HashMap<String, u64>,
|
||||
/// The leader region observed on the previous frontier sweep, so a leader
|
||||
/// change (a new epoch, where `commit_index` legitimately resets to the
|
||||
/// promote baseline) clears the commit-index tracking instead of false-firing.
|
||||
last_leader: Option<String>,
|
||||
/// Per-region role (`true` = leader) observed on the previous sweep. A
|
||||
/// region's `applied_events` is role-RELATIVE — a leader reports its own
|
||||
/// flushed frontier, a follower reports how far it applied the CURRENT
|
||||
/// leader's stream — so a leader↔follower flip re-baselines the value and is
|
||||
/// NOT a regression. We re-baseline `applied:<region>` on any role flip.
|
||||
last_role: HashMap<String, bool>,
|
||||
}
|
||||
|
||||
impl MonotonicCounters {
|
||||
#[must_use]
|
||||
pub fn new(label: &str) -> Self {
|
||||
Self {
|
||||
label: label.to_string(),
|
||||
last: HashMap::new(),
|
||||
last_leader: None,
|
||||
last_role: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe `value` for `key`; panic if it is below the last value seen.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `value` regressed below a prior observation of `key`.
|
||||
pub fn observe(&mut self, key: &str, value: u64) {
|
||||
if let Some(&prev) = self.last.get(key) {
|
||||
assert!(
|
||||
value >= prev,
|
||||
"{}: NON-MONOTONIC — {key} regressed {prev} → {value} (acknowledged state lost)",
|
||||
self.label
|
||||
);
|
||||
}
|
||||
self.last.insert(key.to_string(), value);
|
||||
}
|
||||
|
||||
/// Forget any tracking for `key` (e.g. when a node legitimately resets its
|
||||
/// frontier by reseeding, so the next observation starts a fresh baseline).
|
||||
pub fn reset(&mut self, key: &str) {
|
||||
self.last.remove(key);
|
||||
}
|
||||
|
||||
/// Observe every live node's `applied_events` frontier (keyed by region) and
|
||||
/// the current leader's `commit_index`. Call repeatedly through a fault
|
||||
/// window: within a stable leadership epoch no node's durable frontier and
|
||||
/// no leader's commit index may ever regress.
|
||||
///
|
||||
/// Legitimate epoch resets are handled, not flagged. `applied_events` is
|
||||
/// role-RELATIVE (a leader reports its own flushed frontier; a follower
|
||||
/// reports how far it applied the current leader's stream), so it is
|
||||
/// re-baselined — not asserted — whenever a region (a) reports
|
||||
/// `reseeding`/`quarantined` (mid snapshot-reinstall) OR (b) flips role
|
||||
/// leader↔follower (a clean failover, where the value changes WHAT it
|
||||
/// measures). `commit_index` is leadership-scoped and re-baselined on a
|
||||
/// leader-region change. What remains asserted is the real invariant: within
|
||||
/// a stable role/epoch a node never forgets durably-applied data, and a leader
|
||||
/// never un-commits a quorum-durable write within its term.
|
||||
/// SCOPE: this reads the FLAT top-level status fields (`is_leader`,
|
||||
/// `applied_events`, `commit_index`), which under m11p6 sharding mirror only
|
||||
/// the node's DEFAULT (lowest-id) hosted group — so it is exact for S=1 (the
|
||||
/// shipped topology) and tracks only the default group's frontier under S>1.
|
||||
/// The per-group `shards[]` rows (which [`assert_single_leader_per_term`]
|
||||
/// already consults) are the follow-up for multi-group frontier tracking.
|
||||
pub fn observe_cluster_frontiers(&mut self, cluster: &MultiProcCluster) {
|
||||
let mut leader_now: Option<String> = None;
|
||||
// Capture the leader's commit_index in the SAME sweep (no second N-request
|
||||
// scan): the value is in `st` while we hold it.
|
||||
let mut leader_commit: u64 = 0;
|
||||
for idx in 0..cluster.len() {
|
||||
let Some(st) = cluster.local_status(idx) else {
|
||||
continue;
|
||||
};
|
||||
let region = st["region"].as_str().unwrap_or("?").to_string();
|
||||
let is_leader = st["is_leader"].as_bool() == Some(true);
|
||||
let reseeding = st["reseeding"].as_bool() == Some(true)
|
||||
|| st["quarantined"].as_bool() == Some(true);
|
||||
// A role flip (leader↔follower) re-baselines the role-relative
|
||||
// applied frontier; the first observation of a region is not a flip.
|
||||
let role_flipped = self
|
||||
.last_role
|
||||
.insert(region.clone(), is_leader)
|
||||
.is_some_and(|was| was != is_leader);
|
||||
let applied_key = format!("applied:{region}");
|
||||
if reseeding || role_flipped {
|
||||
self.reset(&applied_key);
|
||||
// Re-seed the baseline at the current value so the NEXT sweep
|
||||
// asserts from here, not from the stale pre-flip frontier.
|
||||
self.last
|
||||
.insert(applied_key, st["applied_events"].as_u64().unwrap_or(0));
|
||||
} else {
|
||||
// A leader reports its flushed frontier as applied (m11p8); a
|
||||
// follower reports its applied frontier. Both are durable.
|
||||
self.observe(&applied_key, st["applied_events"].as_u64().unwrap_or(0));
|
||||
}
|
||||
if is_leader {
|
||||
leader_now = Some(region);
|
||||
leader_commit = st["commit_index"].as_u64().unwrap_or(0);
|
||||
}
|
||||
}
|
||||
// commit_index is leadership-scoped: clear it on a leader change.
|
||||
if leader_now != self.last_leader {
|
||||
self.reset("commit_index");
|
||||
self.last_leader.clone_from(&leader_now);
|
||||
}
|
||||
if leader_now.is_some() {
|
||||
self.observe("commit_index", leader_commit);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -14,5 +14,6 @@
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod invariants;
|
||||
pub mod multiproc;
|
||||
pub mod partition;
|
||||
|
||||
@ -1159,16 +1159,26 @@ fn tidal_server_bin() -> PathBuf {
|
||||
.or_else(|| manifest_dir.parent())
|
||||
.expect("locate workspace root");
|
||||
|
||||
// Build with `fault-injection` so the spawned cluster processes carry the
|
||||
// m11p9 WAL slow-fsync / disk-full hooks (inert unless a `TIDAL_FAULT_*`
|
||||
// env var arms them — every non-fault suite spawns the same binary and is
|
||||
// unaffected). Production never passes this feature, so the shipped image
|
||||
// compiles the hooks out entirely.
|
||||
let status = Command::new("cargo")
|
||||
.arg("build")
|
||||
.arg("-p")
|
||||
.arg("tidal-server")
|
||||
.arg("--features")
|
||||
.arg("fault-injection")
|
||||
.current_dir(workspace_root)
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.expect("run cargo build");
|
||||
assert!(status.success(), "cargo build -p tidal-server failed");
|
||||
assert!(
|
||||
status.success(),
|
||||
"cargo build -p tidal-server --features fault-injection failed"
|
||||
);
|
||||
|
||||
let bin = workspace_root.join("target/debug/tidal-server");
|
||||
assert!(
|
||||
|
||||
@ -60,6 +60,10 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
|
||||
# network): bench workload.next() construction and the latency-histogram record
|
||||
# path in isolation. Matches the engine crate's criterion posture.
|
||||
criterion = { version = "0.5", features = ["html_reports"] }
|
||||
# Test-only: the summary module's JSON is hand-rolled (no serializer dep on the
|
||||
# hot path), but the round-trip unit test parses it with a real strict parser so
|
||||
# a malformed-JSON regression is caught.
|
||||
serde_json = "1"
|
||||
|
||||
[[bench]]
|
||||
name = "hotpath"
|
||||
|
||||
@ -35,6 +35,26 @@ pub enum StressError {
|
||||
/// header value (contains control or non-visible characters).
|
||||
#[error("invalid api key: {0}")]
|
||||
Auth(String),
|
||||
|
||||
/// A regression gate (`--max-p99-ms`, `--max-error-pct`, `--fail-on-knee`)
|
||||
/// was breached. The run completed and reported normally; this is the
|
||||
/// non-zero exit that fails the nightly CI step (m11p9).
|
||||
#[error("regression gate breached: {0}")]
|
||||
Gate(String),
|
||||
|
||||
/// A gate threshold flag was invalid (non-finite or negative) — an operator
|
||||
/// CLI mistake caught before the run starts, NOT a regression.
|
||||
#[error("invalid gate threshold: {0}")]
|
||||
BadGate(String),
|
||||
|
||||
/// The `--json-summary` artifact could not be written (an IO/disk fault, not
|
||||
/// a gate breach).
|
||||
#[error("could not write summary {path}: {source}")]
|
||||
Summary {
|
||||
path: String,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
}
|
||||
|
||||
/// Crate-local result alias.
|
||||
|
||||
@ -12,4 +12,5 @@ pub mod client;
|
||||
pub mod error;
|
||||
pub mod metrics;
|
||||
pub mod scheduler;
|
||||
pub mod summary;
|
||||
pub mod workload;
|
||||
|
||||
@ -20,6 +20,7 @@ use tidal_stress::client::{HttpClient, seed_corpus};
|
||||
use tidal_stress::error::{Result, StressError};
|
||||
use tidal_stress::metrics::{self, StageStats};
|
||||
use tidal_stress::scheduler::{Stage, parse_ramp, run_stage};
|
||||
use tidal_stress::summary::{GateConfig, RunSummary, StageSummary};
|
||||
use tidal_stress::workload::{OpKind, Workload, WritePath, parse_mix};
|
||||
|
||||
#[derive(Parser)]
|
||||
@ -27,6 +28,9 @@ use tidal_stress::workload::{OpKind, Workload, WritePath, parse_mix};
|
||||
version,
|
||||
about = "Open-loop capacity ramp for tidalDB (thepeach feed workload)"
|
||||
)]
|
||||
// A CLI args struct: each bool is an independent flag, not a state machine — the
|
||||
// standard exception to `struct_excessive_bools`.
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
struct Cli {
|
||||
/// Region gateway base URL, e.g. http://10.43.99.11:9500 . Repeat for each
|
||||
/// region; reads round-robin across them (each serves locally), writes too
|
||||
@ -119,6 +123,26 @@ struct Cli {
|
||||
/// feed p99 >150ms) instead of pushing every stage.
|
||||
#[arg(long, default_value_t = false)]
|
||||
stop_on_knee: bool,
|
||||
|
||||
/// Write a machine-readable JSON run summary to this path (per-stage p99 /
|
||||
/// throughput / error rate + the gate verdict). For nightly soak trend lines.
|
||||
#[arg(long)]
|
||||
json_summary: Option<String>,
|
||||
|
||||
/// Regression gate: exit non-zero if any stage's worst-op p99 exceeds this
|
||||
/// (milliseconds). Omitted = no p99 gate.
|
||||
#[arg(long)]
|
||||
max_p99_ms: Option<f64>,
|
||||
|
||||
/// Regression gate: exit non-zero if any stage's error rate exceeds this
|
||||
/// (percent, e.g. 1.0 = 1%). Omitted = no error-rate gate.
|
||||
#[arg(long)]
|
||||
max_error_pct: Option<f64>,
|
||||
|
||||
/// Regression gate: exit non-zero if any stage breached the built-in SLO
|
||||
/// (the capacity knee). The nightly soak's PASS/FAIL signal.
|
||||
#[arg(long, default_value_t = false)]
|
||||
fail_on_knee: bool,
|
||||
}
|
||||
|
||||
// SLO thresholds for the per-stage verdict. tidalDB's own RETRIEVE SLA is p99
|
||||
@ -169,6 +193,22 @@ async fn run() -> Result<()> {
|
||||
"--ack must be leader|quorum, got {ack:?}"
|
||||
)));
|
||||
}
|
||||
// A gate threshold of NaN/inf would report the gate ARMED while every
|
||||
// `value > NaN` comparison is false — silently letting a regression through
|
||||
// (clap's default f64 parser accepts "nan"/"inf"). Reject non-finite or
|
||||
// negative thresholds so the gate cannot be disarmed by a typo.
|
||||
for (flag, val) in [
|
||||
("--max-p99-ms", cli.max_p99_ms),
|
||||
("--max-error-pct", cli.max_error_pct),
|
||||
] {
|
||||
if let Some(v) = val
|
||||
&& (!v.is_finite() || v < 0.0)
|
||||
{
|
||||
return Err(StressError::BadGate(format!(
|
||||
"{flag} must be finite and non-negative, got {v}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let client = Arc::new(HttpClient::new(
|
||||
Duration::from_secs(cli.request_timeout_secs),
|
||||
cli.api_key.clone(),
|
||||
@ -267,6 +307,7 @@ async fn run() -> Result<()> {
|
||||
// ── Ramp ────────────────────────────────────────────────────────────────
|
||||
let mut best_pass: Option<(usize, f64, f64)> = None; // (stage idx, total ok/s, signal ok/s)
|
||||
let mut knee: Option<usize> = None;
|
||||
let mut stage_summaries: Vec<StageSummary> = Vec::new();
|
||||
|
||||
for (i, stage) in stages.iter().enumerate() {
|
||||
if cli.poll_status
|
||||
@ -285,6 +326,12 @@ async fn run() -> Result<()> {
|
||||
let _ = std::io::stdout().flush();
|
||||
|
||||
let (passed, signal_ok_rps) = stage_verdict(&stats);
|
||||
stage_summaries.push(StageSummary::from_stage(
|
||||
stage.target_rps,
|
||||
&stats,
|
||||
passed,
|
||||
signal_ok_rps,
|
||||
));
|
||||
if passed {
|
||||
best_pass = Some((i, stats.achieved_rps(), signal_ok_rps));
|
||||
} else if knee.is_none() {
|
||||
@ -304,6 +351,35 @@ async fn run() -> Result<()> {
|
||||
}
|
||||
|
||||
print_verdict(&cli, &stages, best_pass, knee);
|
||||
|
||||
// ── Machine-readable summary + regression gates (m11p9) ──────────────────
|
||||
let run = RunSummary {
|
||||
stages: stage_summaries,
|
||||
knee_stage: knee,
|
||||
gate: GateConfig {
|
||||
max_p99_ms: cli.max_p99_ms,
|
||||
max_error_pct: cli.max_error_pct,
|
||||
fail_on_knee: cli.fail_on_knee,
|
||||
},
|
||||
};
|
||||
if let Some(path) = &cli.json_summary {
|
||||
std::fs::write(path, run.to_json()).map_err(|source| StressError::Summary {
|
||||
path: path.clone(),
|
||||
source,
|
||||
})?;
|
||||
println!("\nwrote JSON summary → {path}");
|
||||
}
|
||||
if run.gate.is_armed() {
|
||||
let breaches = run.breaches();
|
||||
if breaches.is_empty() {
|
||||
println!(
|
||||
"\nregression gates: PASS ({} stage(s) within thresholds)",
|
||||
run.stages.len()
|
||||
);
|
||||
} else {
|
||||
return Err(StressError::Gate(breaches.join("; ")));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@ -191,6 +191,15 @@ pub fn parse_ramp(spec: &str, stage_secs: u64) -> Result<Vec<Stage>> {
|
||||
.trim()
|
||||
.parse()
|
||||
.map_err(|_| StressError::Ramp(format!("bad rps '{r}'")))?;
|
||||
// `f64::from_str` accepts "inf"/"nan" and overflows ("1e400") to
|
||||
// +inf; a non-finite or negative rps is meaningless and would
|
||||
// poison the scheduler and the JSON summary (inf/NaN are not valid
|
||||
// JSON). Reject it at the boundary.
|
||||
if !target_rps.is_finite() || target_rps < 0.0 {
|
||||
return Err(StressError::Ramp(format!(
|
||||
"rps must be finite and non-negative, got '{r}'"
|
||||
)));
|
||||
}
|
||||
let secs: u64 = s
|
||||
.trim()
|
||||
.parse()
|
||||
|
||||
395
tidal-stress/src/summary.rs
Normal file
395
tidal-stress/src/summary.rs
Normal file
@ -0,0 +1,395 @@
|
||||
//! Machine-readable run summary + regression gates (m11p9 continuous correctness).
|
||||
//!
|
||||
//! The human-readable per-stage tables and the capacity verdict are for an
|
||||
//! operator reading the terminal. A NIGHTLY SOAK needs two more things the
|
||||
//! original tool lacked: a machine-readable artifact a trend-line job can archive
|
||||
//! and diff, and a PASS/FAIL exit code so a regression (p99 or error-rate creep,
|
||||
//! or the built-in SLO knee) fails the CI step instead of scrolling past in green.
|
||||
//!
|
||||
//! This module builds a flat JSON summary (hand-rolled — every value is a number,
|
||||
//! bool, or null, so no serializer dependency is needed, matching the
|
||||
//! dependency-light posture of [`crate::metrics`]) and evaluates the regression
|
||||
//! gates. A breach is surfaced as a [`crate::error::StressError::Gate`], which
|
||||
//! `main` turns into a non-zero exit.
|
||||
|
||||
use std::fmt::Write as _;
|
||||
|
||||
use crate::metrics::StageStats;
|
||||
use crate::workload::OpKind;
|
||||
|
||||
/// One stage's machine-readable roll-up.
|
||||
pub struct StageSummary {
|
||||
pub target_rps: f64,
|
||||
pub achieved_rps: f64,
|
||||
pub ok_per_sec: f64,
|
||||
pub signal_ok_per_sec: f64,
|
||||
pub error_rate: f64,
|
||||
pub client_shed: u64,
|
||||
/// Feed-read p99 (ms), 0 if no feed reads in the mix.
|
||||
pub feed_p99_ms: f64,
|
||||
/// Worst signal-write p99 (ms) across view/like/skip, 0 if none.
|
||||
pub signal_p99_ms: f64,
|
||||
/// Worst p99 (ms) across every op that took OK traffic — the single number
|
||||
/// the `--max-p99-ms` gate compares.
|
||||
pub overall_p99_ms: f64,
|
||||
/// Whether this stage held the built-in SLO (error ≤1%, feed p99 ≤150ms exact,
|
||||
/// zero client shed).
|
||||
pub slo_passed: bool,
|
||||
}
|
||||
|
||||
impl StageSummary {
|
||||
/// Build from a finished stage's stats and its built-in-SLO verdict.
|
||||
#[must_use]
|
||||
pub fn from_stage(
|
||||
target_rps: f64,
|
||||
stats: &StageStats,
|
||||
slo_passed: bool,
|
||||
signal_ok_per_sec: f64,
|
||||
) -> Self {
|
||||
let p99_ms = |k: OpKind| {
|
||||
let op = &stats.ops[k.idx()];
|
||||
if op.ok() == 0 {
|
||||
0.0
|
||||
} else {
|
||||
op.hist.percentile(0.99).as_secs_f64() * 1000.0
|
||||
}
|
||||
};
|
||||
let feed_p99_ms = p99_ms(OpKind::FeedRead);
|
||||
let signal_p99_ms = [OpKind::SignalView, OpKind::SignalLike, OpKind::SignalSkip]
|
||||
.into_iter()
|
||||
.map(p99_ms)
|
||||
.fold(0.0_f64, f64::max);
|
||||
// The worst p99 across every op (zero-ok ops contribute 0.0 via the
|
||||
// closure's guard) — the single number the `--max-p99-ms` gate compares.
|
||||
// One source of truth for the per-op p99 math (the `p99_ms` closure).
|
||||
let overall_p99_ms = OpKind::ALL.into_iter().map(p99_ms).fold(0.0_f64, f64::max);
|
||||
Self {
|
||||
target_rps,
|
||||
achieved_rps: stats.achieved_rps(),
|
||||
ok_per_sec: stats.total_ok() as f64 / stats.elapsed.as_secs_f64().max(1e-9),
|
||||
signal_ok_per_sec,
|
||||
error_rate: stats.error_rate(),
|
||||
client_shed: stats.client_shed,
|
||||
feed_p99_ms,
|
||||
signal_p99_ms,
|
||||
overall_p99_ms,
|
||||
slo_passed,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Regression-gate thresholds resolved from the CLI.
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub struct GateConfig {
|
||||
/// Fail if any stage's worst-op p99 exceeds this (ms).
|
||||
pub max_p99_ms: Option<f64>,
|
||||
/// Fail if any stage's error rate exceeds this (percent, e.g. 1.0 = 1%).
|
||||
pub max_error_pct: Option<f64>,
|
||||
/// Fail if any stage breached the built-in SLO (the capacity knee).
|
||||
pub fail_on_knee: bool,
|
||||
}
|
||||
|
||||
impl GateConfig {
|
||||
/// Whether any gate is armed (if not, the run is informational and always
|
||||
/// exits 0, preserving the original tool's behavior).
|
||||
#[must_use]
|
||||
pub const fn is_armed(&self) -> bool {
|
||||
self.max_p99_ms.is_some() || self.max_error_pct.is_some() || self.fail_on_knee
|
||||
}
|
||||
}
|
||||
|
||||
/// The whole run's machine-readable summary.
|
||||
pub struct RunSummary {
|
||||
pub stages: Vec<StageSummary>,
|
||||
/// First stage index that breached the built-in SLO, if any.
|
||||
pub knee_stage: Option<usize>,
|
||||
pub gate: GateConfig,
|
||||
}
|
||||
|
||||
impl RunSummary {
|
||||
/// Evaluate the armed regression gates. Returns the list of human-readable
|
||||
/// breach reasons (empty = passed). Unarmed gates never breach.
|
||||
#[must_use]
|
||||
pub fn breaches(&self) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
if self.gate.fail_on_knee
|
||||
&& let Some(k) = self.knee_stage
|
||||
{
|
||||
out.push(format!(
|
||||
"built-in SLO breached at stage {} (target {:.0} rps) [--fail-on-knee]",
|
||||
k + 1,
|
||||
self.stages.get(k).map_or(0.0, |s| s.target_rps)
|
||||
));
|
||||
}
|
||||
if let Some(max) = self.gate.max_p99_ms {
|
||||
for (i, s) in self.stages.iter().enumerate() {
|
||||
if s.overall_p99_ms > max {
|
||||
out.push(format!(
|
||||
"stage {} p99 {:.1}ms exceeds --max-p99-ms {max:.1}",
|
||||
i + 1,
|
||||
s.overall_p99_ms
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(max) = self.gate.max_error_pct {
|
||||
for (i, s) in self.stages.iter().enumerate() {
|
||||
if s.error_rate * 100.0 > max {
|
||||
out.push(format!(
|
||||
"stage {} error rate {:.2}% exceeds --max-error-pct {max:.2}",
|
||||
i + 1,
|
||||
s.error_rate * 100.0
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether the run passed every armed gate.
|
||||
#[must_use]
|
||||
pub fn passed(&self) -> bool {
|
||||
self.breaches().is_empty()
|
||||
}
|
||||
|
||||
/// Render the flat JSON summary (hand-rolled; all values numeric/bool/null).
|
||||
#[must_use]
|
||||
pub fn to_json(&self) -> String {
|
||||
let mut s = String::new();
|
||||
let breaches = self.breaches();
|
||||
s.push_str("{\n");
|
||||
let _ = writeln!(s, " \"passed\": {},", self.passed());
|
||||
let _ = writeln!(s, " \"gates_armed\": {},", self.gate.is_armed());
|
||||
match self.knee_stage {
|
||||
Some(k) => {
|
||||
let _ = writeln!(s, " \"knee_stage\": {},", k + 1);
|
||||
}
|
||||
None => s.push_str(" \"knee_stage\": null,\n"),
|
||||
}
|
||||
// Breach reasons as a JSON string array (these strings are tool-authored,
|
||||
// but escape quotes/backslashes defensively).
|
||||
s.push_str(" \"breaches\": [");
|
||||
for (i, b) in breaches.iter().enumerate() {
|
||||
if i > 0 {
|
||||
s.push_str(", ");
|
||||
}
|
||||
let _ = write!(s, "\"{}\"", b.replace('\\', "\\\\").replace('"', "\\\""));
|
||||
}
|
||||
s.push_str("],\n");
|
||||
s.push_str(" \"stages\": [\n");
|
||||
for (i, st) in self.stages.iter().enumerate() {
|
||||
s.push_str(" {");
|
||||
let _ = write!(s, "\"target_rps\": {}, ", jnum(st.target_rps, 1));
|
||||
let _ = write!(s, "\"achieved_rps\": {}, ", jnum(st.achieved_rps, 1));
|
||||
let _ = write!(s, "\"ok_per_sec\": {}, ", jnum(st.ok_per_sec, 1));
|
||||
let _ = write!(
|
||||
s,
|
||||
"\"signal_ok_per_sec\": {}, ",
|
||||
jnum(st.signal_ok_per_sec, 1)
|
||||
);
|
||||
let _ = write!(s, "\"error_rate\": {}, ", jnum(st.error_rate, 6));
|
||||
let _ = write!(s, "\"client_shed\": {}, ", st.client_shed);
|
||||
let _ = write!(s, "\"feed_p99_ms\": {}, ", jnum(st.feed_p99_ms, 3));
|
||||
let _ = write!(s, "\"signal_p99_ms\": {}, ", jnum(st.signal_p99_ms, 3));
|
||||
let _ = write!(s, "\"overall_p99_ms\": {}, ", jnum(st.overall_p99_ms, 3));
|
||||
let _ = write!(s, "\"slo_passed\": {}", st.slo_passed);
|
||||
s.push('}');
|
||||
if i + 1 < self.stages.len() {
|
||||
s.push(',');
|
||||
}
|
||||
s.push('\n');
|
||||
}
|
||||
s.push_str(" ]\n}\n");
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
/// Format an `f64` for JSON at `decimals` precision, or `null` if it is
|
||||
/// non-finite (NaN/inf are not valid JSON — a single unguarded field would
|
||||
/// otherwise corrupt the whole artifact). Defense in depth: the inputs are
|
||||
/// already validated finite at the CLI/parse boundary, but this guarantees the
|
||||
/// emitter can never produce invalid JSON regardless of upstream.
|
||||
fn jnum(v: f64, decimals: usize) -> String {
|
||||
if v.is_finite() {
|
||||
format!("{v:.decimals$}")
|
||||
} else {
|
||||
"null".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn stage(
|
||||
target_rps: f64,
|
||||
error_rate: f64,
|
||||
overall_p99_ms: f64,
|
||||
slo_passed: bool,
|
||||
) -> StageSummary {
|
||||
StageSummary {
|
||||
target_rps,
|
||||
achieved_rps: target_rps,
|
||||
ok_per_sec: target_rps,
|
||||
signal_ok_per_sec: target_rps,
|
||||
error_rate,
|
||||
client_shed: 0,
|
||||
feed_p99_ms: overall_p99_ms,
|
||||
signal_p99_ms: overall_p99_ms,
|
||||
overall_p99_ms,
|
||||
slo_passed,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unarmed_gate_never_breaches() {
|
||||
let run = RunSummary {
|
||||
stages: vec![stage(1000.0, 0.5, 9999.0, false)],
|
||||
knee_stage: Some(0),
|
||||
gate: GateConfig::default(),
|
||||
};
|
||||
assert!(!run.gate.is_armed());
|
||||
assert!(
|
||||
run.passed(),
|
||||
"an unarmed run is informational — always passes"
|
||||
);
|
||||
assert!(run.breaches().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn p99_and_error_gates_breach_over_threshold_and_pass_within() {
|
||||
let over = RunSummary {
|
||||
stages: vec![stage(1000.0, 0.05, 300.0, true)],
|
||||
knee_stage: None,
|
||||
gate: GateConfig {
|
||||
max_p99_ms: Some(250.0),
|
||||
max_error_pct: Some(1.0),
|
||||
fail_on_knee: false,
|
||||
},
|
||||
};
|
||||
assert!(!over.passed(), "p99 300>250 and error 5%>1% must breach");
|
||||
assert_eq!(over.breaches().len(), 2);
|
||||
|
||||
let within = RunSummary {
|
||||
stages: vec![stage(1000.0, 0.001, 100.0, true)],
|
||||
knee_stage: None,
|
||||
gate: GateConfig {
|
||||
max_p99_ms: Some(250.0),
|
||||
max_error_pct: Some(1.0),
|
||||
fail_on_knee: false,
|
||||
},
|
||||
};
|
||||
assert!(within.passed(), "p99 100<=250 and error 0.1%<=1% must pass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_on_knee_breaches_only_with_a_knee() {
|
||||
let cfg = GateConfig {
|
||||
max_p99_ms: None,
|
||||
max_error_pct: None,
|
||||
fail_on_knee: true,
|
||||
};
|
||||
let with_knee = RunSummary {
|
||||
stages: vec![stage(1000.0, 0.0, 10.0, false)],
|
||||
knee_stage: Some(0),
|
||||
gate: cfg,
|
||||
};
|
||||
assert!(!with_knee.passed());
|
||||
let no_knee = RunSummary {
|
||||
stages: vec![stage(1000.0, 0.0, 10.0, true)],
|
||||
knee_stage: None,
|
||||
gate: cfg,
|
||||
};
|
||||
assert!(no_knee.passed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_round_trips_through_a_real_parser() {
|
||||
let run = RunSummary {
|
||||
stages: vec![
|
||||
stage(1500.0, 0.012, 142.5, false),
|
||||
stage(3000.0, 0.2, 410.0, false),
|
||||
],
|
||||
knee_stage: Some(2),
|
||||
gate: GateConfig {
|
||||
max_p99_ms: Some(250.0),
|
||||
max_error_pct: Some(1.0),
|
||||
fail_on_knee: true,
|
||||
},
|
||||
};
|
||||
let parsed: serde_json::Value =
|
||||
serde_json::from_str(&run.to_json()).expect("to_json must be valid JSON");
|
||||
assert_eq!(parsed["passed"], false);
|
||||
assert_eq!(parsed["gates_armed"], true);
|
||||
assert_eq!(parsed["knee_stage"], 3);
|
||||
assert_eq!(parsed["stages"].as_array().expect("stages array").len(), 2);
|
||||
assert!(
|
||||
!parsed["breaches"]
|
||||
.as_array()
|
||||
.expect("breaches array")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_finite_field_serializes_as_null_not_invalid_json() {
|
||||
// Defense in depth: even if a non-finite value reached a field, the
|
||||
// emitter must still produce parseable JSON (null), never `inf`/`NaN`.
|
||||
let run = RunSummary {
|
||||
stages: vec![stage(f64::INFINITY, f64::NAN, 100.0, true)],
|
||||
knee_stage: None,
|
||||
gate: GateConfig::default(),
|
||||
};
|
||||
let json = run.to_json();
|
||||
assert!(
|
||||
!json.contains("inf") && !json.contains("NaN"),
|
||||
"no bare inf/NaN: {json}"
|
||||
);
|
||||
let parsed: serde_json::Value =
|
||||
serde_json::from_str(&json).expect("still valid JSON with null-sanitized fields");
|
||||
assert!(parsed["stages"][0]["target_rps"].is_null());
|
||||
assert!(parsed["stages"][0]["error_rate"].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn from_stage_overall_p99_is_worst_busy_op_ignoring_zero_ok() {
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::metrics::StageStats;
|
||||
|
||||
// A real StageStats: FeedRead busy with a ~120ms tail, SignalView busy
|
||||
// with a ~40ms tail, every other op left zero-ok (no traffic).
|
||||
let mut stats = StageStats::new();
|
||||
stats.elapsed = Duration::from_secs(10);
|
||||
let feed = OpKind::FeedRead.idx();
|
||||
let view = OpKind::SignalView.idx();
|
||||
for _ in 0..100 {
|
||||
stats.ops[feed].status[0] += 1; // count as Ok
|
||||
stats.ops[feed].hist.record(Duration::from_millis(120));
|
||||
stats.ops[view].status[0] += 1;
|
||||
stats.ops[view].hist.record(Duration::from_millis(40));
|
||||
}
|
||||
|
||||
let s = StageSummary::from_stage(1000.0, &stats, true, 0.0);
|
||||
// The histogram is bucketed (~3-4%), so assert with tolerance.
|
||||
assert!(
|
||||
(100.0..150.0).contains(&s.feed_p99_ms),
|
||||
"feed p99 {} ~120ms",
|
||||
s.feed_p99_ms
|
||||
);
|
||||
assert!(
|
||||
(30.0..55.0).contains(&s.signal_p99_ms),
|
||||
"signal p99 {} ~40ms",
|
||||
s.signal_p99_ms
|
||||
);
|
||||
// overall = the WORST busy op (FeedRead), and the zero-ok ops (which would
|
||||
// each contribute 0.0) never inflate or drag it.
|
||||
assert!(
|
||||
(s.overall_p99_ms - s.feed_p99_ms).abs() < 1e-9,
|
||||
"overall p99 {} must equal the worst busy op's p99 {} (zero-ok ops excluded)",
|
||||
s.overall_p99_ms,
|
||||
s.feed_p99_ms
|
||||
);
|
||||
assert!(s.overall_p99_ms >= s.signal_p99_ms);
|
||||
}
|
||||
}
|
||||
@ -10,6 +10,11 @@ license = "MIT"
|
||||
default = ["metrics"]
|
||||
test-utils = ["dep:tempfile"]
|
||||
metrics = [] # hand-rolled HTTP, no new crate deps
|
||||
# m11p9 chaos testing: WAL slow-fsync + disk-full injection (src/fault.rs).
|
||||
# NOT in `default` and never passed by the production image build, so the
|
||||
# shipped binary compiles it out entirely. Inert until an env var arms it even
|
||||
# when compiled in. See src/fault.rs.
|
||||
fault-injection = []
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.22"
|
||||
|
||||
147
tidal/src/fault.rs
Normal file
147
tidal/src/fault.rs
Normal file
@ -0,0 +1,147 @@
|
||||
//! Test-only fault injection for continuous-correctness chaos testing (m11p9).
|
||||
//!
|
||||
//! # Compiled out of production
|
||||
//!
|
||||
//! Every item in this module is behind the `fault-injection` cargo feature,
|
||||
//! which is **not** in tidaldb's `default` set and is **never** passed by the
|
||||
//! production image build (`docker/standalone/Dockerfile`: `cargo build -p
|
||||
//! tidal-server --release --locked`). With the feature off this module does not
|
||||
//! exist and the two WAL call sites that reference it
|
||||
//! ([`crate::wal::sync_file_durable`] and
|
||||
//! [`crate::wal::segment::SegmentWriter::write_batch_bytes`]) compile to their
|
||||
//! original instructions — the production binary is byte-identical to one built
|
||||
//! from a tree without this file.
|
||||
//!
|
||||
//! A disk-write or fsync fault that a stray env var could trip in production is
|
||||
//! exactly the kind of 3am footgun we refuse to ship, so the safety is
|
||||
//! structural (the code is absent), not merely a default. The chaos suites and
|
||||
//! the nightly CI build the binary with `--features fault-injection`; the
|
||||
//! `MultiProcCluster` harness's `tidal_server_bin()` does so explicitly.
|
||||
//!
|
||||
//! # Inert until armed
|
||||
//!
|
||||
//! Even *with* the feature compiled in, every fault is INERT until an env var
|
||||
//! explicitly arms it — the exact posture of the existing `TIDAL_HLC_SKEW_MS`,
|
||||
//! `TIDAL_QUORUM_KILLPOINTS`, and `TIDAL_ELECTION_KILLPOINTS` test knobs. The
|
||||
//! configuration is read **once** into a process-global `OnceLock` on first
|
||||
//! access, so an un-armed process pays at most one relaxed atomic load per WAL
|
||||
//! batch/fsync and no env lookups on the hot path.
|
||||
//!
|
||||
//! # The faults
|
||||
//!
|
||||
//! - **`TIDAL_FAULT_FSYNC_DELAY_MS`** — sleep this many milliseconds *before*
|
||||
//! every durable WAL fsync. Models a slow disk / degraded NVMe: the data still
|
||||
//! reaches stable storage, but each group-commit fsync costs `delay + real`.
|
||||
//! On a follower this throttles apply (it WAL-firsts every replicated batch),
|
||||
//! so under `ack=quorum` the commit index lags honestly; on a leader it raises
|
||||
//! write latency without ever losing an acknowledged write.
|
||||
//!
|
||||
//! - **`TIDAL_FAULT_DISK_FULL_AFTER_BYTES`** — after this process has written N
|
||||
//! cumulative bytes to WAL segments, every subsequent segment write fails with
|
||||
//! a real `ENOSPC` (`std::io::Error::from_raw_os_error(28)`, the errno on both
|
||||
//! Linux and macOS), wrapped in [`crate::wal::error::WalError::Io`] — the same
|
||||
//! variant a genuine full disk produces. Arming it on a node *at restart*
|
||||
//! (the cumulative counter starts at zero each process lifetime) makes the
|
||||
//! threshold "fail after N bytes of post-restart writes", which is how the
|
||||
//! chaos suite drives a node into disk-full mid-replication and then proves the
|
||||
//! writer notifies its waiters, halts cleanly (no torn-batch corruption past
|
||||
//! WAL recovery's BLAKE3 truncation), and the cluster loses no acknowledged
|
||||
//! write because the healthy majority held quorum throughout.
|
||||
//!
|
||||
//! The injection returns `ENOSPC` *before* issuing the partial `write_all`, the
|
||||
//! standard chaos model (a clean, deterministic fault). A real full disk can
|
||||
//! also tear a batch mid-write; that case is already covered by the WAL's
|
||||
//! crash-recovery tail truncation and is not what this knob targets.
|
||||
//!
|
||||
//! # Granularity (per-PROCESS, not per-WAL)
|
||||
//!
|
||||
//! Both knobs are PROCESS-GLOBAL: the disk-full byte counter and the fsync delay
|
||||
//! apply to every WAL in the process collectively. Today (S=1, one
|
||||
//! `ShardReplica`/WAL per process) that is exactly one WAL, so the model is
|
||||
//! clean. Under m11p6 multi-shard co-location (a `BTreeMap<ShardId, ShardReplica>`
|
||||
//! in one process, each with its own `SegmentWriter`) the counter would aggregate
|
||||
//! across all hosted shards and fire `ENOSPC` on whichever shard first crosses
|
||||
//! the shared threshold — so a *per-shard* disk-full (fill shard 2 but not shard
|
||||
//! 1) needs either one-shard-per-process or keying these statics by shard/WAL.
|
||||
//! That keying is deferred until the m11p6 (S>1) fault suite actually needs it.
|
||||
|
||||
use std::{
|
||||
sync::{
|
||||
OnceLock,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use crate::wal::error::WalError;
|
||||
|
||||
/// Process-global fault configuration, parsed once from the environment.
|
||||
struct FaultConfig {
|
||||
/// Sleep before each durable fsync (slow-disk injection).
|
||||
fsync_delay: Option<Duration>,
|
||||
/// Fail segment writes once cumulative bytes exceed this (disk-full).
|
||||
disk_full_after: Option<u64>,
|
||||
}
|
||||
|
||||
impl FaultConfig {
|
||||
/// Parse the fault env vars. A missing/blank/unparsable value disarms that
|
||||
/// fault (never a panic — an un-armed process must boot normally).
|
||||
fn from_env() -> Self {
|
||||
Self {
|
||||
fsync_delay: env_u64("TIDAL_FAULT_FSYNC_DELAY_MS").map(Duration::from_millis),
|
||||
disk_full_after: env_u64("TIDAL_FAULT_DISK_FULL_AFTER_BYTES"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a non-negative `u64` env var, or `None` if unset/blank/zero/unparsable.
|
||||
/// Zero disarms (a `*_AFTER_BYTES=0` would otherwise fail the very first write,
|
||||
/// which is never the intent of an opt-in knob).
|
||||
fn env_u64(var: &str) -> Option<u64> {
|
||||
std::env::var(var)
|
||||
.ok()
|
||||
.and_then(|v| v.trim().parse::<u64>().ok())
|
||||
.filter(|&n| n > 0)
|
||||
}
|
||||
|
||||
/// The parsed config, initialized once on first access.
|
||||
fn config() -> &'static FaultConfig {
|
||||
static CONFIG: OnceLock<FaultConfig> = OnceLock::new();
|
||||
CONFIG.get_or_init(FaultConfig::from_env)
|
||||
}
|
||||
|
||||
/// Cumulative bytes this process has written to WAL segments (disk-full input).
|
||||
static SEGMENT_BYTES_WRITTEN: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Hook called at the top of every durable WAL fsync. With the slow-fsync fault
|
||||
/// armed, sleeps `TIDAL_FAULT_FSYNC_DELAY_MS` before the real sync; otherwise a
|
||||
/// single `Option` check and return.
|
||||
pub fn before_fsync() {
|
||||
if let Some(delay) = config().fsync_delay {
|
||||
std::thread::sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
/// Hook called at the top of every WAL segment write. With the disk-full fault
|
||||
/// armed, accounts `len` against the cumulative counter and returns a real
|
||||
/// `ENOSPC` once the threshold is crossed; otherwise returns `Ok` immediately
|
||||
/// without even touching the counter.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`WalError::Io`] wrapping `ENOSPC` once cumulative segment bytes
|
||||
/// exceed `TIDAL_FAULT_DISK_FULL_AFTER_BYTES`.
|
||||
pub fn before_segment_write(len: usize) -> Result<(), WalError> {
|
||||
let Some(threshold) = config().disk_full_after else {
|
||||
return Ok(());
|
||||
};
|
||||
let total = SEGMENT_BYTES_WRITTEN.fetch_add(len as u64, Ordering::Relaxed) + len as u64;
|
||||
if total > threshold {
|
||||
// ENOSPC is errno 28 on both Linux and macOS. `from_raw_os_error`
|
||||
// produces an error whose `.kind()` the platform maps exactly as a real
|
||||
// full-disk write would — the same `WalError::Io` the writer's error
|
||||
// path already handles (notify waiters, keep serving).
|
||||
return Err(WalError::Io(std::io::Error::from_raw_os_error(28)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@ -27,6 +27,12 @@ pub mod storage;
|
||||
pub mod text;
|
||||
pub mod wal;
|
||||
|
||||
/// Test-only WAL fault injection (slow-fsync, disk-full) for the m11p9
|
||||
/// continuous-correctness chaos suites. Behind the non-default `fault-injection`
|
||||
/// feature, so production builds never compile it — see the module docs.
|
||||
#[cfg(feature = "fault-injection")]
|
||||
pub(crate) mod fault;
|
||||
|
||||
/// Build hash compiled in from the `TIDALDB_BUILD_HASH` environment variable.
|
||||
///
|
||||
/// Falls back to `"dev"` if `TIDALDB_BUILD_HASH` is unset or `build.rs` is not
|
||||
|
||||
@ -77,6 +77,11 @@ pub type WalOpenResult = (
|
||||
/// appends, checkpoint temp). For directory-entry durability use
|
||||
/// [`sync_dir_durable`].
|
||||
pub(crate) fn sync_file_durable(file: &std::fs::File) -> Result<(), WalError> {
|
||||
// Test-only slow-disk injection (m11p9): inert unless the `fault-injection`
|
||||
// feature is compiled AND `TIDAL_FAULT_FSYNC_DELAY_MS` is armed. Compiled out
|
||||
// of production entirely.
|
||||
#[cfg(feature = "fault-injection")]
|
||||
crate::fault::before_fsync();
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
// F_FULLFSYNC flushes the device write cache; plain fsync on macOS does
|
||||
|
||||
@ -327,6 +327,13 @@ impl SegmentWriter {
|
||||
///
|
||||
/// Returns `WalError::Io` on write failure.
|
||||
pub fn write_batch_bytes(&mut self, bytes: &[u8]) -> Result<u64, WalError> {
|
||||
// Test-only disk-full injection (m11p9): inert unless the
|
||||
// `fault-injection` feature is compiled AND
|
||||
// `TIDAL_FAULT_DISK_FULL_AFTER_BYTES` is armed. Returns a real ENOSPC
|
||||
// before any partial write, so the writer's error path runs unchanged.
|
||||
// Compiled out of production entirely.
|
||||
#[cfg(feature = "fault-injection")]
|
||||
crate::fault::before_segment_write(bytes.len())?;
|
||||
let offset = self.current_size;
|
||||
self.file.write_all(bytes)?;
|
||||
self.current_size += bytes.len() as u64;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user