ci: disable incremental compilation, surface the nested build's error
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed

Pipeline #8 failed and told us nothing. Both fixes here address that.

WHY IT FAILED: Woodpecker gives each workflow a fresh 10Gi workspace PVC. The
release gate alone fit (pipelines #5 and #6 passed), but the new `fast-suites`
step builds eight test binaries ahead of it, and the gate's nested
`cargo build --features fault-injection` then had no room. CARGO_INCREMENTAL=0
now applies to every Rust step: incremental artifacts are pure waste in CI since
nothing is ever reused across pipelines, and they measured 11G of a 27G target
tree locally — 41%.

WHY IT SAID NOTHING: tests/support/multiproc.rs:1686 built with
`stderr(Stdio::null())`, so the assert fired with "cargo build ... failed" and no
compiler error, no ENOSPC, no exit code. A build failure whose reason is discarded
costs more than the build. stderr is now captured and included in the panic
message; the output is only read on the failure path.

The env block is anchored on its first consuming step rather than a top-level
`variables:` key. I initially used the top-level form and reverted it: the file is
schema-validated and `when: branch: main` means only main triggers a pipeline, so
a rejected key could not be caught on a throwaway branch and would break every
push until reverted. Same rule the resource shapes already follow.
This commit is contained in:
jordan 2026-08-31 01:31:03 -06:00
parent a6f663f002
commit 25361bb660
2 changed files with 38 additions and 5 deletions

View File

@ -33,6 +33,17 @@ when:
branch: main branch: main
event: [push, cron] event: [push, cron]
# CARGO_INCREMENTAL=0 for every Rust step. Incremental compilation is pure waste
# here: Woodpecker gives each workflow a FRESH 10Gi workspace PVC, so nothing is
# ever reused across pipelines, and the artifacts are enormous. Measured locally,
# `target/debug/incremental` was 11G of a 27G target tree — 41%.
#
# It is also what broke pipeline #8. The 10Gi workspace held the release gate
# alone (pipelines #5 and #6 both passed), but once `fast-suites` built eight test
# binaries ahead of it, the gate's nested `cargo build --features fault-injection`
# ran out of room and failed with its stderr discarded, which is why the failure
# said nothing (now fixed in tests/support/multiproc.rs).
steps: steps:
# ── Fast in-process suites (push) ─────────────────────────────────────────── # ── Fast in-process suites (push) ───────────────────────────────────────────
# ADDED 2026-08-31 after a survey found that 14 of 23 integration suites were # ADDED 2026-08-31 after a survey found that 14 of 23 integration suites were
@ -57,6 +68,14 @@ steps:
resources: resources:
requests: { cpu: "500m", memory: 1Gi } requests: { cpu: "500m", memory: 1Gi }
limits: { cpu: "2", memory: 4Gi } limits: { cpu: "2", memory: 4Gi }
# Declared ONCE here (first consumer) and aliased by every later Rust step —
# same technique as the resource shapes, and for the same reason: a custom
# top-level key would be a schema risk, and because `when: branch: main` means
# only main triggers a pipeline, a parse error could not be caught on a
# throwaway branch and would break every push until reverted.
environment: &cargo_env
CARGO_INCREMENTAL: "0"
CARGO_TERM_COLOR: never
commands: commands:
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
- cargo test -p tidaldb --lib - cargo test -p tidaldb --lib
@ -120,6 +139,8 @@ steps:
# already sets it "for a shared CI runner" — and this is the step that BLOCKS # already sets it "for a shared CI runner" — and this is the step that BLOCKS
# the image build, so its flake cost is the highest of any step in the file. # the image build, so its flake cost is the highest of any step in the file.
environment: environment:
CARGO_INCREMENTAL: "0"
CARGO_TERM_COLOR: never
TIDAL_TEST_BOOT_BUDGET_SECS: "300" TIDAL_TEST_BOOT_BUDGET_SECS: "300"
TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "180" TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "180"
commands: commands:
@ -170,6 +191,8 @@ steps:
# Same three-process shape as the release gate, so the same resource shape. # Same three-process shape as the release gate, so the same resource shape.
backend_options: *resources-heavy backend_options: *resources-heavy
environment: environment:
CARGO_INCREMENTAL: "0"
CARGO_TERM_COLOR: never
TIDAL_QUORUM_KILLPOINTS: "25" TIDAL_QUORUM_KILLPOINTS: "25"
TIDAL_ELECTION_KILLPOINTS: "15" TIDAL_ELECTION_KILLPOINTS: "15"
# Raised from 120/90 to match the release gate. These budgets were TIGHTER # Raised from 120/90 to match the release gate. These budgets were TIGHTER
@ -221,6 +244,7 @@ steps:
cron: nightly cron: nightly
# In-process and fast; the light shape is sufficient. # In-process and fast; the light shape is sufficient.
backend_options: *resources-light backend_options: *resources-light
environment: *cargo_env
commands: commands:
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang - apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
- cargo test -p tidal-net --test mtls -- --test-threads 1 - cargo test -p tidal-net --test mtls -- --test-threads 1

View File

@ -1675,7 +1675,12 @@ fn tidal_server_bin() -> PathBuf {
// env var arms them — every non-fault suite spawns the same binary and is // env var arms them — every non-fault suite spawns the same binary and is
// unaffected). Production never passes this feature, so the shipped image // unaffected). Production never passes this feature, so the shipped image
// compiles the hooks out entirely. // compiles the hooks out entirely.
let status = Command::new("cargo") // stderr is CAPTURED, not discarded. It used to be `Stdio::null()`, and on
// 2026-08-31 this assert fired in CI with the message below and nothing else —
// no compiler error, no ENOSPC, no OOM, nothing to act on. A build failure
// whose reason is thrown away costs more than the build. The output is only
// read on the failure path, so a passing run is unchanged.
let out = Command::new("cargo")
.arg("build") .arg("build")
.arg("-p") .arg("-p")
.arg("tidal-server") .arg("tidal-server")
@ -1683,12 +1688,16 @@ fn tidal_server_bin() -> PathBuf {
.arg("fault-injection") .arg("fault-injection")
.current_dir(workspace_root) .current_dir(workspace_root)
.stdout(std::process::Stdio::null()) .stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null()) .stderr(std::process::Stdio::piped())
.status() .output()
.expect("run cargo build"); .expect("run cargo build");
assert!( assert!(
status.success(), out.status.success(),
"cargo build -p tidal-server --features fault-injection failed" "cargo build -p tidal-server --features fault-injection failed \
(exit {:?}) in {}:\n{}",
out.status.code(),
workspace_root.display(),
String::from_utf8_lossy(&out.stderr)
); );
let bin = workspace_root.join("target/debug/tidal-server"); let bin = workspace_root.join("target/debug/tidal-server");