diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index e3066d0..839c6f8 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -287,17 +287,28 @@ spec: # footprint, not a gate artifact: it stays at 4 GiB. memory: 4Gi limits: - # Two cores keeps the measured query/apply burst reachable on the - # tightest node without reserving it. Note the ratio: a burstable - # pod whose neighbours are also bursting gets CFS-throttled, which - # is precisely how CockroachDB was pushed into multi-second Raft - # stalls on this fleet with nodes 70% idle. The quorum alerts above - # are the detector for that; more request is the fix. - cpu: "2" - # Four independent OOMKills occurred at 3.97-4.00 GiB. Six GiB is - # measured peak plus 50% recovery/profiling headroom; the exact - # internal growth source still requires heap/allocation profiling. - memory: 6Gi + # 3 cores / 7Gi, matching what production actually runs. + # + # These were 2 cores / 6Gi here while the live StatefulSet ran 3/7, + # so `kubectl apply -k k8s/cluster/` DOWNGRADED production every + # time it was run - a silent capacity regression on the write path, + # applied by the very command used to deploy a fix. The live values + # are the intended ones and this file is now the source of truth for + # them; raise BOTH together or not at all. + # + # Why a limit above the request at all: it keeps the measured + # query/apply burst reachable on the tightest node without reserving + # it. Note the ratio - a burstable pod whose neighbours are also + # bursting gets CFS-throttled, which is precisely how CockroachDB was + # pushed into multi-second Raft stalls on this fleet with nodes 70% + # idle. The quorum alerts above are the detector for that; more + # REQUEST is the fix, not more limit. + cpu: "3" + # Four independent OOMKills occurred at 3.97-4.00 GiB. Six GiB was + # measured peak plus 50% headroom; production carries 7Gi and that is + # what is declared here. The exact internal growth source still + # requires heap/allocation profiling. + memory: 7Gi securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true # writes only /data (PVC) and /tmp (emptyDir)