From 25361bb66019bb9aa4d4f8688fcb9f397d6b7634 Mon Sep 17 00:00:00 2001 From: jordan Date: Mon, 31 Aug 2026 01:31:03 -0600 Subject: [PATCH] ci: disable incremental compilation, surface the nested build's error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline #8 failed and told us nothing. Both fixes here address that. WHY IT FAILED: Woodpecker gives each workflow a fresh 10Gi workspace PVC. The release gate alone fit (pipelines #5 and #6 passed), but the new `fast-suites` step builds eight test binaries ahead of it, and the gate's nested `cargo build --features fault-injection` then had no room. CARGO_INCREMENTAL=0 now applies to every Rust step: incremental artifacts are pure waste in CI since nothing is ever reused across pipelines, and they measured 11G of a 27G target tree locally — 41%. WHY IT SAID NOTHING: tests/support/multiproc.rs:1686 built with `stderr(Stdio::null())`, so the assert fired with "cargo build ... failed" and no compiler error, no ENOSPC, no exit code. A build failure whose reason is discarded costs more than the build. stderr is now captured and included in the panic message; the output is only read on the failure path. The env block is anchored on its first consuming step rather than a top-level `variables:` key. I initially used the top-level form and reverted it: the file is schema-validated and `when: branch: main` means only main triggers a pipeline, so a rejected key could not be caught on a throwaway branch and would break every push until reverted. Same rule the resource shapes already follow. --- .woodpecker.yaml | 24 ++++++++++++++++++++++++ tidal-server/tests/support/multiproc.rs | 19 ++++++++++++++----- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/.woodpecker.yaml b/.woodpecker.yaml index 606873b..5edfcce 100644 --- a/.woodpecker.yaml +++ b/.woodpecker.yaml @@ -33,6 +33,17 @@ when: branch: main event: [push, cron] +# CARGO_INCREMENTAL=0 for every Rust step. Incremental compilation is pure waste +# here: Woodpecker gives each workflow a FRESH 10Gi workspace PVC, so nothing is +# ever reused across pipelines, and the artifacts are enormous. Measured locally, +# `target/debug/incremental` was 11G of a 27G target tree — 41%. +# +# It is also what broke pipeline #8. The 10Gi workspace held the release gate +# alone (pipelines #5 and #6 both passed), but once `fast-suites` built eight test +# binaries ahead of it, the gate's nested `cargo build --features fault-injection` +# ran out of room and failed with its stderr discarded, which is why the failure +# said nothing (now fixed in tests/support/multiproc.rs). + steps: # ── Fast in-process suites (push) ─────────────────────────────────────────── # ADDED 2026-08-31 after a survey found that 14 of 23 integration suites were @@ -57,6 +68,14 @@ steps: resources: requests: { cpu: "500m", memory: 1Gi } limits: { cpu: "2", memory: 4Gi } + # Declared ONCE here (first consumer) and aliased by every later Rust step — + # same technique as the resource shapes, and for the same reason: a custom + # top-level key would be a schema risk, and because `when: branch: main` means + # only main triggers a pipeline, a parse error could not be caught on a + # throwaway branch and would break every push until reverted. + environment: &cargo_env + CARGO_INCREMENTAL: "0" + CARGO_TERM_COLOR: never commands: - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang - cargo test -p tidaldb --lib @@ -120,6 +139,8 @@ steps: # already sets it "for a shared CI runner" — and this is the step that BLOCKS # the image build, so its flake cost is the highest of any step in the file. environment: + CARGO_INCREMENTAL: "0" + CARGO_TERM_COLOR: never TIDAL_TEST_BOOT_BUDGET_SECS: "300" TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "180" commands: @@ -170,6 +191,8 @@ steps: # Same three-process shape as the release gate, so the same resource shape. backend_options: *resources-heavy environment: + CARGO_INCREMENTAL: "0" + CARGO_TERM_COLOR: never TIDAL_QUORUM_KILLPOINTS: "25" TIDAL_ELECTION_KILLPOINTS: "15" # Raised from 120/90 to match the release gate. These budgets were TIGHTER @@ -221,6 +244,7 @@ steps: cron: nightly # In-process and fast; the light shape is sufficient. backend_options: *resources-light + environment: *cargo_env commands: - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang - cargo test -p tidal-net --test mtls -- --test-threads 1 diff --git a/tidal-server/tests/support/multiproc.rs b/tidal-server/tests/support/multiproc.rs index 8a7310e..2d0f48e 100644 --- a/tidal-server/tests/support/multiproc.rs +++ b/tidal-server/tests/support/multiproc.rs @@ -1675,7 +1675,12 @@ fn tidal_server_bin() -> PathBuf { // env var arms them — every non-fault suite spawns the same binary and is // unaffected). Production never passes this feature, so the shipped image // compiles the hooks out entirely. - let status = Command::new("cargo") + // stderr is CAPTURED, not discarded. It used to be `Stdio::null()`, and on + // 2026-08-31 this assert fired in CI with the message below and nothing else — + // no compiler error, no ENOSPC, no OOM, nothing to act on. A build failure + // whose reason is thrown away costs more than the build. The output is only + // read on the failure path, so a passing run is unchanged. + let out = Command::new("cargo") .arg("build") .arg("-p") .arg("tidal-server") @@ -1683,12 +1688,16 @@ fn tidal_server_bin() -> PathBuf { .arg("fault-injection") .current_dir(workspace_root) .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() + .stderr(std::process::Stdio::piped()) + .output() .expect("run cargo build"); assert!( - status.success(), - "cargo build -p tidal-server --features fault-injection failed" + out.status.success(), + "cargo build -p tidal-server --features fault-injection failed \ + (exit {:?}) in {}:\n{}", + out.status.code(), + workspace_root.display(), + String::from_utf8_lossy(&out.stderr) ); let bin = workspace_root.join("target/debug/tidal-server");