From df0e1b98dedb81bd082c076f305c3ba47a0d2f42 Mon Sep 17 00:00:00 2001 From: jx12n Date: Sat, 13 Jun 2026 18:39:30 -0600 Subject: [PATCH] =?UTF-8?q?feat(deploy+stress):=20m11p6/mTLS=20deploy=20fi?= =?UTF-8?q?xes=20=E2=80=94=20HTTPS=20probes=20+=20TLS-aware=20generator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploying the m11-44b768b image (p6 sharding + p7 mTLS + p8 ops + p9 correctness) surfaced two blockers; both fixed here. 1. statefulset.yaml: the m11p7 change made the :9500 HTTP plane serve TLS, but the startup/liveness/readiness probes still used scheme HTTP — kubelet got a TLS handshake back ("malformed HTTP response \x15\x03\x03") and pods never went Ready. Set scheme: HTTPS on all three probes (kubelet skips cert verification for httpGet probes, so the cert's DNS-only SANs are fine). Image pinned to the m11-44b768b amd64 digest. 2. tidal-stress: the generator's reqwest client did default cert verification and had no way to trust the cluster's private CA, so https:// targets failed. Added --ca-cert (verified TLS against the mounted tidaldb-cluster-tls ca.crt) and --insecure (skip verification, escape hatch). New StressError::CaCert for the PEM read fault. stress-job-m11p6-baseline.yaml: T2-A-equivalent quorum-write throughput run on the new stack — https:// targets, ca.crt mounted from the tidaldb-cluster-tls Secret, --ca-cert verified TLS. Drops the removed --write-path flag (m11p6 unified the write path to hash-routing). --- k8s/cluster/statefulset.yaml | 8 +- .../k8s/stress-job-m11p6-baseline.yaml | 97 +++++++++++++++++++ tidal-stress/src/client.rs | 22 ++++- tidal-stress/src/error.rs | 8 ++ tidal-stress/src/main.rs | 13 +++ 5 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 tidal-stress/k8s/stress-job-m11p6-baseline.yaml diff --git a/k8s/cluster/statefulset.yaml b/k8s/cluster/statefulset.yaml index 81bb778..ffaa76d 100644 --- a/k8s/cluster/statefulset.yaml +++ b/k8s/cluster/statefulset.yaml @@ -81,7 +81,7 @@ spec: mountPath: /data containers: - name: tidaldb - image: registry.threesix.ai/tidal/server@sha256:173e803082beea479c48f3729f998e7332766ebcef55786da12f15719648f4fa # m11p5 + image: registry.threesix.ai/tidal/server@sha256:8b136de21b969adedee37fdcb0cac15ecdd77b9beb5c5f31be3a5d15340323b1 # m11-44b768b (p6 sharding + p7 mTLS + p8 ops + p9 correctness) imagePullPolicy: IfNotPresent # The image ENTRYPOINT is the bare binary. We override the command with # a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we @@ -164,16 +164,21 @@ spec: # A restarted PVC-retained voter is Ready on today's terms (no # regression). The full predicate is documented in the kubernetes.md # runbook so probe behavior is diagnosable. + # m11p7: the HTTP plane on :9500 serves TLS (inter-node mTLS), so every + # probe must use scheme HTTPS. kubelet does NOT verify the server cert + # for httpGet probes, so the cert's DNS-only SANs (no pod IP) are fine. startupProbe: httpGet: path: /health/startup port: http + scheme: HTTPS periodSeconds: 5 failureThreshold: 60 # ~5 min for large-DB WAL replay / index load livenessProbe: httpGet: path: /health/live port: http + scheme: HTTPS periodSeconds: 10 timeoutSeconds: 3 failureThreshold: 3 @@ -181,6 +186,7 @@ spec: httpGet: path: /health # cluster-aware: 503 joiner/quarantined/draining port: http + scheme: HTTPS periodSeconds: 10 timeoutSeconds: 3 failureThreshold: 3 diff --git a/tidal-stress/k8s/stress-job-m11p6-baseline.yaml b/tidal-stress/k8s/stress-job-m11p6-baseline.yaml new file mode 100644 index 0000000..d2755fb --- /dev/null +++ b/tidal-stress/k8s/stress-job-m11p6-baseline.yaml @@ -0,0 +1,97 @@ +# m11p6 baseline — quorum-write throughput over the mTLS plane (m11p7). +# First run on the m11-44b768b image: confirms the HTTPS :9500 client plane, +# cluster-key auth, and m11p6 single-group replication all carry a real write +# load end-to-end, and re-establishes the T2-A capacity number on the new stack. +# +# TLS: the :9500 plane serves a private-CA cert (tidaldb-cluster-tls). The +# generator trusts it via the mounted ca.crt (--ca-cert) — verified TLS, not +# --insecure. Targets MUST be https:// and use the cert's DNS SANs (pod-DNS / VIP), +# never pod IPs (not in the SAN list). +# +# Apply: kubectl apply -f tidal-stress/k8s/stress-job-m11p6-baseline.yaml +# Watch: kubectl logs -f job/tidal-stress-m11p6 -n tidaldb-cluster +# Rearm: kubectl delete job tidal-stress-m11p6 -n tidaldb-cluster +apiVersion: batch/v1 +kind: Job +metadata: + name: tidal-stress-m11p6 + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 7200 + template: + metadata: + labels: + app.kubernetes.io/name: tidal-stress + app.kubernetes.io/part-of: tidaldb + spec: + restartPolicy: Never + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: stress + image: registry.threesix.ai/tidal/stress@sha256:e130aa871f5df17a14a9e13e7df606c602b95a03d8eba49490ff7481e6e2b2b3 # m11-44b768b (TLS-aware) + imagePullPolicy: IfNotPresent + args: + - --target + - https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --target + - https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500 + - --leader-url + - https://tidaldb.tidaldb-cluster.svc.cluster.local:9500 # VIP — routes to any ready pod; m11p6 hash-routes writes to the group leader + - --ca-cert + - /etc/tidaldb/tls/ca.crt + - --ack + - quorum + - --ramp + - peach-100k + - --stage-secs + - "120" + - --mix + - writes + - --corpus + - "20000" + - --users + - "100000" + - --poll-status + env: + - name: TIDAL_API_KEY + valueFrom: + secretKeyRef: + name: tidaldb-credentials + key: TIDAL_API_KEY + - name: TIDAL_STRESS_LOG + value: warn + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "3" + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: cluster-tls + mountPath: /etc/tidaldb/tls + readOnly: true + volumes: + - name: cluster-tls + secret: + secretName: tidaldb-cluster-tls + items: + - key: ca.crt + path: ca.crt diff --git a/tidal-stress/src/client.rs b/tidal-stress/src/client.rs index ed8dbba..ac84049 100644 --- a/tidal-stress/src/client.rs +++ b/tidal-stress/src/client.rs @@ -45,8 +45,10 @@ impl HttpClient { request_timeout: Duration, api_key: Option, ack: Option, + ca_cert: Option, + insecure: bool, ) -> Result { - let inner = reqwest::Client::builder() + let mut builder = reqwest::Client::builder() .timeout(request_timeout) .connect_timeout(Duration::from_secs(5)) // Warm-connection pool sized to comfortably saturate the server's @@ -54,9 +56,21 @@ impl HttpClient { // 4 host-pools (≈4×256 idle ceiling). .pool_max_idle_per_host(256) .pool_idle_timeout(Duration::from_secs(90)) - .tcp_nodelay(true) - .build() - .map_err(StressError::Client)?; + .tcp_nodelay(true); + // m11p7: the cluster's :9500 plane serves TLS from a private CA. Trust it + // via the mounted ca.crt (verified), or skip verification with --insecure. + if let Some(path) = ca_cert { + let pem = std::fs::read(&path).map_err(|source| StressError::CaCert { + path: path.clone(), + source, + })?; + let cert = reqwest::Certificate::from_pem(&pem).map_err(StressError::Client)?; + builder = builder.add_root_certificate(cert); + } + if insecure { + builder = builder.danger_accept_invalid_certs(true); + } + let inner = builder.build().map_err(StressError::Client)?; // Build the Authorization header once. reqwest's `bearer_auth` does a fresh // `format!("Bearer {token}")` allocation plus a HeaderValue validation pass // on EVERY call, over a token that never changes — pure per-request waste. diff --git a/tidal-stress/src/error.rs b/tidal-stress/src/error.rs index e0fb3fe..13c1c71 100644 --- a/tidal-stress/src/error.rs +++ b/tidal-stress/src/error.rs @@ -31,6 +31,14 @@ pub enum StressError { #[error("http client build failed: {0}")] Client(#[source] reqwest::Error), + /// The `--ca-cert` PEM bundle could not be read (an IO/disk fault). + #[error("could not load --ca-cert {path}: {source}")] + CaCert { + path: String, + #[source] + source: std::io::Error, + }, + /// The provided API key could not be turned into an HTTP `Authorization` /// header value (contains control or non-visible characters). #[error("invalid api key: {0}")] diff --git a/tidal-stress/src/main.rs b/tidal-stress/src/main.rs index 7c09333..9c7b2cc 100644 --- a/tidal-stress/src/main.rs +++ b/tidal-stress/src/main.rs @@ -49,6 +49,17 @@ struct Cli { #[arg(long, env = "TIDAL_API_KEY")] api_key: Option, + /// PEM CA bundle to trust for HTTPS targets (m11p7: the cluster's :9500 plane + /// serves TLS from a private CA). Point at the mounted `ca.crt` from the + /// `tidaldb-cluster-tls` Secret. Verified TLS — preferred over --insecure. + #[arg(long)] + ca_cert: Option, + + /// Skip TLS certificate verification for HTTPS targets. An escape hatch for + /// quick in-cluster runs; use --ca-cert for real verification. + #[arg(long, default_value_t = false)] + insecure: bool, + /// Write acknowledgment mode sent as `x-tidal-ack` on every write /// (m11p3: leader|quorum). Omitted = the cluster's topology default. #[arg(long)] @@ -201,6 +212,8 @@ async fn run() -> Result<()> { Duration::from_secs(cli.request_timeout_secs), cli.api_key.clone(), cli.ack.clone(), + cli.ca_cert.clone(), + cli.insecure, )?); // The seed URL anchors corpus seeding (any gateway accepts items and