# Two pipelines in one file, split by event (Woodpecker, NEVER GitHub Actions): # # • event: push → the m11p8 RELEASE GATE (rolling-upgrade tier-3 test) then a # Kaniko image build. Deployment is MANUAL (kustomize, orchard9-k3sf ops repo). # • event: cron → the m11p9 NIGHTLY CONTINUOUS-CORRECTNESS run: the tier-3 # chaos suites (incl. the new disk-full / slow-fsync / asymmetric-partition # fault classes) with elevated kill-points, then a tidal-stress soak with # PASS/FAIL regression gates on p99 + error rate. A nightly failure flags a # correctness or performance regression for that day. # # Per-step `when:` routes each step to its event; the workflow-level `when` admits # both. Configure a cron named "nightly" in the Woodpecker repo settings to fire # the cron pipeline (the same `tidal-server` binary serves standalone AND # multi-process `cluster --region`, so one image covers both deployments). when: branch: main event: [push, cron] steps: # ── Release gate (push) ───────────────────────────────────────────────────── # m11p8 release gate: prove a rolling upgrade under load loses no acknowledged # write and never stalls (mp_rolling_upgrade_no_loss_no_stall — a tier-3 test # spawning three real OS processes with a graceful SIGTERM → version-tagged # restart → heal-until-converged → fixpoint cycle). Serial (--test-threads 1): # the harness binds fixed ports and spawns real processes, so suites must not # overlap. A failure here BLOCKS the image build below — the gate, not the start. rolling-upgrade-gate: image: rust:1-bookworm when: event: push commands: - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang - cargo test -p tidal-server --features cluster-e2e --test cluster_lifecycle mp_rolling_upgrade_no_loss_no_stall -- --nocapture --test-threads 1 build: image: woodpeckerci/plugin-kaniko when: event: push settings: repo: tidal/server dockerfile: docker/standalone/Dockerfile context: . # Tag by immutable identity only: `latest` plus the commit SHA (the durable # handle). A per-milestone literal tag drifts every milestone and lies about # the image's vintage (a hand-edited `m8p10` once tagged m11 code) — the SHA # never goes stale and the version-skew machinery keys off the binary's # CARGO_PKG_VERSION/BUILD_HASH, not the image tag. tags: - latest - ${CI_COMMIT_SHA} registry: registry.threesix.ai build_args: - TARGETPLATFORM=linux/amd64 extra_args: --customPlatform=linux/amd64 # ── Nightly chaos (cron) ──────────────────────────────────────────────────── # The tier-3 chaos/correctness suites over REAL OS processes, run serially # (fixed ports + spawned processes must not overlap). Kill-points are elevated # above the per-PR defaults (8/5) but kept below the 100-point exit-gate run so # the nightly stays bounded; widen TIDAL_*_KILLPOINTS for a deeper sweep. The # `fault-injection` feature compiles in the WAL slow-fsync / disk-full hooks for # cluster_faults (inert in every other suite — see tidal/src/fault.rs). Boot / # convergence budgets are raised for a shared CI runner. A failure = a # correctness regression for the night. nightly-chaos: image: rust:1-bookworm when: event: cron cron: nightly environment: TIDAL_QUORUM_KILLPOINTS: "25" TIDAL_ELECTION_KILLPOINTS: "15" TIDAL_TEST_BOOT_BUDGET_SECS: "120" TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "90" commands: - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang # New m11p9 fault classes first (disk-full, slow-fsync, asymmetric partition). - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_faults -- --nocapture --test-threads 1 # The standing chaos / durability / availability / elasticity gates. - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_chaos -- --nocapture --test-threads 1 - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_quorum -- --nocapture --test-threads 1 - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_election -- --nocapture --test-threads 1 - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_membership -- --nocapture --test-threads 1 - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_reseed -- --nocapture --test-threads 1 - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_lifecycle -- --nocapture --test-threads 1 - cargo test -p tidal-server --features "cluster-e2e fault-injection" --test cluster_runbook -- --nocapture --test-threads 1 # ── Nightly soak (cron) ───────────────────────────────────────────────────── # A tidal-stress soak with PASS/FAIL regression gates (m11p9). By default it # boots a local standalone server and soaks it for a bounded window, gating on # error rate and p99 so a perf regression fails the step and shows on the trend # line (the JSON summary is the archived artifact). Point TIDAL_SOAK_TARGET at # the live Ref-A cluster and raise TIDAL_SOAK_SECS to 3600 for the GA-bar 1-hour # 100k-DAU soak (the standing k3s-access caveat applies — same as m11p1–p3). nightly-soak: image: rust:1-bookworm when: event: cron cron: nightly environment: TIDAL_SOAK_RPS: "1000" TIDAL_SOAK_SECS: "600" TIDAL_SOAK_MAX_P99_MS: "250" TIDAL_SOAK_MAX_ERROR_PCT: "1" # Soak target port — an UNCLAIMED slot in the project's reserved dev band # (59520-59529): 59520=site, 59521=iknowyou, so the soak uses 59526. One # source of truth for the literal (referenced by --listen AND the TARGET). TIDAL_SOAK_PORT: "59526" commands: - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang curl - cargo build -p tidal-server -p tidal-stress # Boot a standalone target unless TIDAL_SOAK_TARGET points elsewhere (Ref-A). # NOTE the `$${VAR}` escaping: Woodpecker substitutes a bare `${VAR}` in a # command BEFORE the shell runs, and step `environment:` vars are NOT in that # preprocessor namespace (they would blank to ""). `$${VAR}` passes a literal # `${VAR}` to the container shell, which expands it from the runtime env — # so the soak's ramp and gate thresholds are actually populated. Bare shell # locals (`$PORT`, `$TARGET`, `$SRV`) are never touched by the preprocessor. - | PORT="$${TIDAL_SOAK_PORT}" TARGET="$${TIDAL_SOAK_TARGET:-http://127.0.0.1:$PORT}" if [ -z "$TIDAL_SOAK_TARGET" ]; then mkdir -p /tmp/soak-data ./target/debug/tidal-server standalone --listen "127.0.0.1:$PORT" \ --schema tidal-server/config/default-schema.yaml --data-dir /tmp/soak-data & SRV=$! # Reap the background server + scratch dir on ANY exit (success, gate # failure, or the boot-failure exit below) so the step is idempotent. trap 'kill "$SRV" 2>/dev/null; rm -rf /tmp/soak-data' EXIT up=0 for i in $(seq 1 100); do curl -sf "$TARGET/health/startup" >/dev/null 2>&1 && { up=1; break; } || sleep 0.3 done # Fail FAST and unambiguously on a boot failure (bad schema, port bound) # rather than running the soak against a dead target and mislabeling it # as an error-rate regression on the trend line. if [ "$up" != "1" ]; then echo "soak target failed to start at $TARGET"; exit 1; fi fi ./target/debug/tidal-stress --target "$TARGET" \ --ramp "$${TIDAL_SOAK_RPS}:$${TIDAL_SOAK_SECS}" --corpus 5000 --mix peach \ --json-summary soak-summary.json \ --max-error-pct "$${TIDAL_SOAK_MAX_ERROR_PCT}" --max-p99-ms "$${TIDAL_SOAK_MAX_P99_MS}" --fail-on-knee - cat soak-summary.json # ── Nightly security + ops correctness (cron) ─────────────────────────────── # The G-Sec and G-Op owner-tests run nightly too (not just on-demand): mTLS / # foreign-pod rejection / cert rotation (tidal-net mtls + cluster_security), the # tidalctl backup/restore round-trip, and the gap-free WAL-archival (PITR) unit # test (in the engine lib). In-process + fast; serial for the TLS port binds. nightly-security-ops: image: rust:1-bookworm when: event: cron cron: nightly commands: - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang - cargo test -p tidal-net --test mtls -- --test-threads 1 - cargo test -p tidal-server --test cluster_security -- --test-threads 1 - cargo test -p tidal-server --lib - cargo test -p tidalctl - cargo test -p tidaldb --lib wal::compaction