ci: disable incremental compilation, surface the nested build's error
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
Pipeline #8 failed and told us nothing. Both fixes here address that. WHY IT FAILED: Woodpecker gives each workflow a fresh 10Gi workspace PVC. The release gate alone fit (pipelines #5 and #6 passed), but the new `fast-suites` step builds eight test binaries ahead of it, and the gate's nested `cargo build --features fault-injection` then had no room. CARGO_INCREMENTAL=0 now applies to every Rust step: incremental artifacts are pure waste in CI since nothing is ever reused across pipelines, and they measured 11G of a 27G target tree locally — 41%. WHY IT SAID NOTHING: tests/support/multiproc.rs:1686 built with `stderr(Stdio::null())`, so the assert fired with "cargo build ... failed" and no compiler error, no ENOSPC, no exit code. A build failure whose reason is discarded costs more than the build. stderr is now captured and included in the panic message; the output is only read on the failure path. The env block is anchored on its first consuming step rather than a top-level `variables:` key. I initially used the top-level form and reverted it: the file is schema-validated and `when: branch: main` means only main triggers a pipeline, so a rejected key could not be caught on a throwaway branch and would break every push until reverted. Same rule the resource shapes already follow.
This commit is contained in:
parent
a6f663f002
commit
25361bb660
@ -33,6 +33,17 @@ when:
|
|||||||
branch: main
|
branch: main
|
||||||
event: [push, cron]
|
event: [push, cron]
|
||||||
|
|
||||||
|
# CARGO_INCREMENTAL=0 for every Rust step. Incremental compilation is pure waste
|
||||||
|
# here: Woodpecker gives each workflow a FRESH 10Gi workspace PVC, so nothing is
|
||||||
|
# ever reused across pipelines, and the artifacts are enormous. Measured locally,
|
||||||
|
# `target/debug/incremental` was 11G of a 27G target tree — 41%.
|
||||||
|
#
|
||||||
|
# It is also what broke pipeline #8. The 10Gi workspace held the release gate
|
||||||
|
# alone (pipelines #5 and #6 both passed), but once `fast-suites` built eight test
|
||||||
|
# binaries ahead of it, the gate's nested `cargo build --features fault-injection`
|
||||||
|
# ran out of room and failed with its stderr discarded, which is why the failure
|
||||||
|
# said nothing (now fixed in tests/support/multiproc.rs).
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
# ── Fast in-process suites (push) ───────────────────────────────────────────
|
# ── Fast in-process suites (push) ───────────────────────────────────────────
|
||||||
# ADDED 2026-08-31 after a survey found that 14 of 23 integration suites were
|
# ADDED 2026-08-31 after a survey found that 14 of 23 integration suites were
|
||||||
@ -57,6 +68,14 @@ steps:
|
|||||||
resources:
|
resources:
|
||||||
requests: { cpu: "500m", memory: 1Gi }
|
requests: { cpu: "500m", memory: 1Gi }
|
||||||
limits: { cpu: "2", memory: 4Gi }
|
limits: { cpu: "2", memory: 4Gi }
|
||||||
|
# Declared ONCE here (first consumer) and aliased by every later Rust step —
|
||||||
|
# same technique as the resource shapes, and for the same reason: a custom
|
||||||
|
# top-level key would be a schema risk, and because `when: branch: main` means
|
||||||
|
# only main triggers a pipeline, a parse error could not be caught on a
|
||||||
|
# throwaway branch and would break every push until reverted.
|
||||||
|
environment: &cargo_env
|
||||||
|
CARGO_INCREMENTAL: "0"
|
||||||
|
CARGO_TERM_COLOR: never
|
||||||
commands:
|
commands:
|
||||||
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
|
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
|
||||||
- cargo test -p tidaldb --lib
|
- cargo test -p tidaldb --lib
|
||||||
@ -120,6 +139,8 @@ steps:
|
|||||||
# already sets it "for a shared CI runner" — and this is the step that BLOCKS
|
# already sets it "for a shared CI runner" — and this is the step that BLOCKS
|
||||||
# the image build, so its flake cost is the highest of any step in the file.
|
# the image build, so its flake cost is the highest of any step in the file.
|
||||||
environment:
|
environment:
|
||||||
|
CARGO_INCREMENTAL: "0"
|
||||||
|
CARGO_TERM_COLOR: never
|
||||||
TIDAL_TEST_BOOT_BUDGET_SECS: "300"
|
TIDAL_TEST_BOOT_BUDGET_SECS: "300"
|
||||||
TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "180"
|
TIDAL_TEST_CONVERGENCE_BUDGET_SECS: "180"
|
||||||
commands:
|
commands:
|
||||||
@ -170,6 +191,8 @@ steps:
|
|||||||
# Same three-process shape as the release gate, so the same resource shape.
|
# Same three-process shape as the release gate, so the same resource shape.
|
||||||
backend_options: *resources-heavy
|
backend_options: *resources-heavy
|
||||||
environment:
|
environment:
|
||||||
|
CARGO_INCREMENTAL: "0"
|
||||||
|
CARGO_TERM_COLOR: never
|
||||||
TIDAL_QUORUM_KILLPOINTS: "25"
|
TIDAL_QUORUM_KILLPOINTS: "25"
|
||||||
TIDAL_ELECTION_KILLPOINTS: "15"
|
TIDAL_ELECTION_KILLPOINTS: "15"
|
||||||
# Raised from 120/90 to match the release gate. These budgets were TIGHTER
|
# Raised from 120/90 to match the release gate. These budgets were TIGHTER
|
||||||
@ -221,6 +244,7 @@ steps:
|
|||||||
cron: nightly
|
cron: nightly
|
||||||
# In-process and fast; the light shape is sufficient.
|
# In-process and fast; the light shape is sufficient.
|
||||||
backend_options: *resources-light
|
backend_options: *resources-light
|
||||||
|
environment: *cargo_env
|
||||||
commands:
|
commands:
|
||||||
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
|
- apt-get update && apt-get install -y --no-install-recommends protobuf-compiler cmake clang
|
||||||
- cargo test -p tidal-net --test mtls -- --test-threads 1
|
- cargo test -p tidal-net --test mtls -- --test-threads 1
|
||||||
|
|||||||
@ -1675,7 +1675,12 @@ fn tidal_server_bin() -> PathBuf {
|
|||||||
// env var arms them — every non-fault suite spawns the same binary and is
|
// env var arms them — every non-fault suite spawns the same binary and is
|
||||||
// unaffected). Production never passes this feature, so the shipped image
|
// unaffected). Production never passes this feature, so the shipped image
|
||||||
// compiles the hooks out entirely.
|
// compiles the hooks out entirely.
|
||||||
let status = Command::new("cargo")
|
// stderr is CAPTURED, not discarded. It used to be `Stdio::null()`, and on
|
||||||
|
// 2026-08-31 this assert fired in CI with the message below and nothing else —
|
||||||
|
// no compiler error, no ENOSPC, no OOM, nothing to act on. A build failure
|
||||||
|
// whose reason is thrown away costs more than the build. The output is only
|
||||||
|
// read on the failure path, so a passing run is unchanged.
|
||||||
|
let out = Command::new("cargo")
|
||||||
.arg("build")
|
.arg("build")
|
||||||
.arg("-p")
|
.arg("-p")
|
||||||
.arg("tidal-server")
|
.arg("tidal-server")
|
||||||
@ -1683,12 +1688,16 @@ fn tidal_server_bin() -> PathBuf {
|
|||||||
.arg("fault-injection")
|
.arg("fault-injection")
|
||||||
.current_dir(workspace_root)
|
.current_dir(workspace_root)
|
||||||
.stdout(std::process::Stdio::null())
|
.stdout(std::process::Stdio::null())
|
||||||
.stderr(std::process::Stdio::null())
|
.stderr(std::process::Stdio::piped())
|
||||||
.status()
|
.output()
|
||||||
.expect("run cargo build");
|
.expect("run cargo build");
|
||||||
assert!(
|
assert!(
|
||||||
status.success(),
|
out.status.success(),
|
||||||
"cargo build -p tidal-server --features fault-injection failed"
|
"cargo build -p tidal-server --features fault-injection failed \
|
||||||
|
(exit {:?}) in {}:\n{}",
|
||||||
|
out.status.code(),
|
||||||
|
workspace_root.display(),
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
);
|
);
|
||||||
|
|
||||||
let bin = workspace_root.join("target/debug/tidal-server");
|
let bin = workspace_root.join("target/debug/tidal-server");
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user