ci: disable incremental compilation, surface the nested build's error
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Pipeline #8 failed and told us nothing. Both fixes here address that. WHY IT FAILED: Woodpecker gives each workflow a fresh 10Gi workspace PVC. The release gate alone fit (pipelines #5 and #6 passed), but the new `fast-suites` step builds eight test binaries ahead of it, and the gate's nested `cargo build --features fault-injection` then had no room. CARGO_INCREMENTAL=0 now applies to every Rust step: incremental artifacts are pure waste in CI since nothing is ever reused across pipelines, and they measured 11G of a 27G target tree locally — 41%. WHY IT SAID NOTHING: tests/support/multiproc.rs:1686 built with `stderr(Stdio::null())`, so the assert fired with "cargo build ... failed" and no compiler error, no ENOSPC, no exit code. A build failure whose reason is discarded costs more than the build. stderr is now captured and included in the panic message; the output is only read on the failure path. The env block is anchored on its first consuming step rather than a top-level `variables:` key. I initially used the top-level form and reverted it: the file is schema-validated and `when: branch: main` means only main triggers a pipeline, so a rejected key could not be caught on a throwaway branch and would break every push until reverted. Same rule the resource shapes already follow.
This commit is contained in:
parent
a6f663f002
commit
25361bb660
@ -33,6 +33,17 @@ when:
|
||||
branch: main
|
||||
event: [push, cron]
|
||||
|
||||
# CARGO_INCREMENTAL=0 for every Rust step. Incremental compilation is pure waste
|
||||
# here: Woodpecker gives each workflow a FRESH 10Gi workspace PVC, so nothing is
|
||||
# ever reused across pipelines, and the artifacts are enormous. Measured locally,
|
||||
# `target/debug/incremental` was 11G of a 27G target tree — 41%.
|
||||
#
|
||||
# It is also what broke pipeline #8. The 10Gi workspace held the release gate
|
||||
# alone (pipelines #5 and #6 both passed), but once `fast-suites` built eight test
|
||||
# binaries ahead of it, the gate's nested `cargo build --features fault-injection`
|
||||
# ran out of room and failed with its stderr discarded, which is why the failure
|
||||
# said nothing (now fixed in tests/support/multiproc.rs).
|
||||
|
||||
steps:
|
||||
# ── Fast in-process suites (push) ───────────────────────────────────────────
|
||||
# ADDED 2026-08-31 after a survey found that 14 of 23 integration suites were
|
||||
@ -57,6 +68,14 @@ steps:
|
||||
resources:
|
||||
requests: { cpu: "500m", memory: 1Gi }
|
||||
limits: { cpu: "2", memory: 4Gi }
|
||||
# Declared ONCE here (first consumer) and aliased by every later Rust step —
|
||||
# same technique as the resource shapes, and for the same reason: a custom
|
||||
# top-level key would be a schema risk, and because `when: branch: main` means
|
||||
# only main triggers a pipeline, a parse error could not be caught on a
|
||||
# throwaway branch and would break every push until reverted.
|
||||
environment: &cargo_env
|
||||
CARGO_INCREMENTAL: "0"
|
||||
CARGO_TERM_COLOR: never
|
||||
commands:
|
||||
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
|
||||
- cargo test -p tidaldb --lib
|
||||
@ -120,6 +139,8 @@ steps:
|
||||
# already sets it "for a shared CI runner" — and this is the step that BLOCKS
|
||||
# the image build, so its flake cost is the highest of any step in the file.
|
||||
environment:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
CARGO_TERM_COLOR: never
|
||||
TIDAL_TEST_BOOT_BUDGET_SECS: "300"
|
||||
TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "180"
|
||||
commands:
|
||||
@ -170,6 +191,8 @@ steps:
|
||||
# Same three-process shape as the release gate, so the same resource shape.
|
||||
backend_options: *resources-heavy
|
||||
environment:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
CARGO_TERM_COLOR: never
|
||||
TIDAL_QUORUM_KILLPOINTS: "25"
|
||||
TIDAL_ELECTION_KILLPOINTS: "15"
|
||||
# Raised from 120/90 to match the release gate. These budgets were TIGHTER
|
||||
@ -221,6 +244,7 @@ steps:
|
||||
cron: nightly
|
||||
# In-process and fast; the light shape is sufficient.
|
||||
backend_options: *resources-light
|
||||
environment: *cargo_env
|
||||
commands:
|
||||
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
|
||||
- cargo test -p tidal-net --test mtls -- --test-threads 1
|
||||
|
||||
@ -1675,7 +1675,12 @@ fn tidal_server_bin() -> PathBuf {
|
||||
// env var arms them — every non-fault suite spawns the same binary and is
|
||||
// unaffected). Production never passes this feature, so the shipped image
|
||||
// compiles the hooks out entirely.
|
||||
let status = Command::new("cargo")
|
||||
// stderr is CAPTURED, not discarded. It used to be `Stdio::null()`, and on
|
||||
// 2026-08-31 this assert fired in CI with the message below and nothing else —
|
||||
// no compiler error, no ENOSPC, no OOM, nothing to act on. A build failure
|
||||
// whose reason is thrown away costs more than the build. The output is only
|
||||
// read on the failure path, so a passing run is unchanged.
|
||||
let out = Command::new("cargo")
|
||||
.arg("build")
|
||||
.arg("-p")
|
||||
.arg("tidal-server")
|
||||
@ -1683,12 +1688,16 @@ fn tidal_server_bin() -> PathBuf {
|
||||
.arg("fault-injection")
|
||||
.current_dir(workspace_root)
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.output()
|
||||
.expect("run cargo build");
|
||||
assert!(
|
||||
status.success(),
|
||||
"cargo build -p tidal-server --features fault-injection failed"
|
||||
out.status.success(),
|
||||
"cargo build -p tidal-server --features fault-injection failed \
|
||||
(exit {:?}) in {}:\n{}",
|
||||
out.status.code(),
|
||||
workspace_root.display(),
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
|
||||
let bin = workspace_root.join("target/debug/tidal-server");
|
||||
|
||||
Loading…
Reference in New Issue
Block a user