From 34cc5f08cac6c366743b9d61f9a38725d40c4b24 Mon Sep 17 00:00:00 2001 From: jx12n Date: Thu, 3 Sep 2026 05:15:35 -0600 Subject: [PATCH] fix(k8s): reconcile tidaldb StatefulSet resources with live cluster (300m/4Gi requests, 3/7Gi limits) --- k8s/cluster/statefulset.yaml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index 99f190a..408e85a 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -309,8 +309,9 @@ spec: # # 300m is what the tightest pinned node (k3s-agent-1, 355m free) # can actually reserve for a voter, and it is honest for a FIRST - # CONSUMER's load - not for the gate. The limit below keeps the - # measured burst reachable without reserving it. + # CONSUMER's load - not for the gate. It also matches measured p95 + # (302/235/233m per pod). The limit below keeps the measured burst + # reachable without reserving it. # # This is provisioned optimism with named detectors: if real write # volume approaches the knee, TidalDBClusterQuorumLag, @@ -321,7 +322,10 @@ spec: cpu: "300m" # Baseline working set was ~3.6 GiB before load, and full placement # means every pod holds the WHOLE 1536-D corpus. This is a resident - # footprint, not a gate artifact: it stays at 4 GiB. + # footprint, not a gate artifact: it stays at 4 GiB. An 11d + # observation put pods at 4.4-5.7Gi, so the 1Gi this file once + # declared would make every replica a Burstable-over-request + # eviction candidate. memory: 4Gi limits: # 3 cores / 7Gi, matching what production actually runs. @@ -333,6 +337,14 @@ spec: # are the intended ones and this file is now the source of truth for # them; raise BOTH together or not at all. # + # Why 3 and not 2 (m12 read-SLA fix): the cgroup cpu quota is what + # the engine's available_parallelism() reads (SEARCH_GATE / + # worker_threads sizing). At "2" a cross-shard search burst starved + # the async reactor plus the election/heartbeat/apply loops - reads + # hung to the 30s route timeout, and the starved control plane + # churned elections into reseed self-exit. "3" leaves ~1 core for + # kubelet/system on the 4-core nodes. + # # Why a limit above the request at all: it keeps the measured # query/apply burst reachable on the tightest node without reserving # it. Note the ratio - a burstable pod whose neighbours are also @@ -343,8 +355,9 @@ spec: cpu: "3" # Four independent OOMKills occurred at 3.97-4.00 GiB. Six GiB was # measured peak plus 50% headroom; production carries 7Gi and that is - # what is declared here. The exact internal growth source still - # requires heap/allocation profiling. + # what is declared here - tidaldb-0 held 5751Mi, 94% of the old 6Gi + # ceiling. The exact internal growth source still requires + # heap/allocation profiling. memory: 7Gi securityContext: allowPrivilegeEscalation: false