From c9adec040dc206a77f30d90e3d9a3d17f881c278 Mon Sep 17 00:00:00 2001 From: jordan Date: Sat, 22 Aug 2026 19:49:37 -0600 Subject: [PATCH] k8s(cluster): declare the limits production actually runs The manifest said cpu 2 / memory 6Gi while the live StatefulSet ran cpu 3 / memory 7Gi, so `kubectl apply -k k8s/cluster/` DOWNGRADED production every time it was run - a silent capacity regression on the write path, delivered by the very command used to deploy a fix. Verified before and after with kubectl diff: the cpu/memory hunks are now absent, so an apply is a no-op on resources. The live values are the intended ones; this file is now the source of truth for them. The comment keeps the reasoning that matters - the limit exists to keep the measured query/apply burst reachable without reserving it, and when neighbours burst too the answer is more REQUEST, not more limit, because CFS throttling is how CockroachDB was pushed into multi-second Raft stalls on this fleet with nodes 70% idle. --- k8s/cluster/statefulset.yaml | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index e3066d0..839c6f8 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -287,17 +287,28 @@ spec: # footprint, not a gate artifact: it stays at 4 GiB. memory: 4Gi limits: - # Two cores keeps the measured query/apply burst reachable on the - # tightest node without reserving it. Note the ratio: a burstable - # pod whose neighbours are also bursting gets CFS-throttled, which - # is precisely how CockroachDB was pushed into multi-second Raft - # stalls on this fleet with nodes 70% idle. The quorum alerts above - # are the detector for that; more request is the fix. - cpu: "2" - # Four independent OOMKills occurred at 3.97-4.00 GiB. Six GiB is - # measured peak plus 50% recovery/profiling headroom; the exact - # internal growth source still requires heap/allocation profiling. - memory: 6Gi + # 3 cores / 7Gi, matching what production actually runs. + # + # These were 2 cores / 6Gi here while the live StatefulSet ran 3/7, + # so `kubectl apply -k k8s/cluster/` DOWNGRADED production every + # time it was run - a silent capacity regression on the write path, + # applied by the very command used to deploy a fix. The live values + # are the intended ones and this file is now the source of truth for + # them; raise BOTH together or not at all. + # + # Why a limit above the request at all: it keeps the measured + # query/apply burst reachable on the tightest node without reserving + # it. Note the ratio - a burstable pod whose neighbours are also + # bursting gets CFS-throttled, which is precisely how CockroachDB was + # pushed into multi-second Raft stalls on this fleet with nodes 70% + # idle. The quorum alerts above are the detector for that; more + # REQUEST is the fix, not more limit. + cpu: "3" + # Four independent OOMKills occurred at 3.97-4.00 GiB. Six GiB was + # measured peak plus 50% headroom; production carries 7Gi and that is + # what is declared here. The exact internal growth source still + # requires heap/allocation profiling. + memory: 7Gi securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true # writes only /data (PVC) and /tmp (emptyDir)