feat(deploy+stress): m11p6/mTLS deploy fixes — HTTPS probes + TLS-aware generator
Deploying the m11-44b768b image (p6 sharding + p7 mTLS + p8 ops + p9 correctness)
surfaced two blockers; both fixed here.
1. statefulset.yaml: the m11p7 change made the :9500 HTTP plane serve TLS, but the
startup/liveness/readiness probes still used scheme HTTP — kubelet got a TLS
handshake back ("malformed HTTP response \x15\x03\x03") and pods never went
Ready. Set scheme: HTTPS on all three probes (kubelet skips cert verification
for httpGet probes, so the cert's DNS-only SANs are fine). Image pinned to the
m11-44b768b amd64 digest.
2. tidal-stress: the generator's reqwest client did default cert verification and
had no way to trust the cluster's private CA, so https:// targets failed. Added
--ca-cert <pem> (verified TLS against the mounted tidaldb-cluster-tls ca.crt)
and --insecure (skip verification, escape hatch). New StressError::CaCert for
the PEM read fault.
stress-job-m11p6-baseline.yaml: T2-A-equivalent quorum-write throughput run on the
new stack — https:// targets, ca.crt mounted from the tidaldb-cluster-tls Secret,
--ca-cert verified TLS. Drops the removed --write-path flag (m11p6 unified the
write path to hash-routing).
This commit is contained in:
parent
44b768b8c6
commit
df0e1b98de
@ -81,7 +81,7 @@ spec:
|
|||||||
mountPath: /data
|
mountPath: /data
|
||||||
containers:
|
containers:
|
||||||
- name: tidaldb
|
- name: tidaldb
|
||||||
image: registry.threesix.ai/tidal/server@sha256:173e803082beea479c48f3729f998e7332766ebcef55786da12f15719648f4fa # m11p5
|
image: registry.threesix.ai/tidal/server@sha256:8b136de21b969adedee37fdcb0cac15ecdd77b9beb5c5f31be3a5d15340323b1 # m11-44b768b (p6 sharding + p7 mTLS + p8 ops + p9 correctness)
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
# The image ENTRYPOINT is the bare binary. We override the command with
|
# The image ENTRYPOINT is the bare binary. We override the command with
|
||||||
# a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we
|
# a tiny /bin/sh wrapper (the bookworm-slim runtime HAS a shell) so we
|
||||||
@ -164,16 +164,21 @@ spec:
|
|||||||
# A restarted PVC-retained voter is Ready on today's terms (no
|
# A restarted PVC-retained voter is Ready on today's terms (no
|
||||||
# regression). The full predicate is documented in the kubernetes.md
|
# regression). The full predicate is documented in the kubernetes.md
|
||||||
# runbook so probe behavior is diagnosable.
|
# runbook so probe behavior is diagnosable.
|
||||||
|
# m11p7: the HTTP plane on :9500 serves TLS (inter-node mTLS), so every
|
||||||
|
# probe must use scheme HTTPS. kubelet does NOT verify the server cert
|
||||||
|
# for httpGet probes, so the cert's DNS-only SANs (no pod IP) are fine.
|
||||||
startupProbe:
|
startupProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /health/startup
|
path: /health/startup
|
||||||
port: http
|
port: http
|
||||||
|
scheme: HTTPS
|
||||||
periodSeconds: 5
|
periodSeconds: 5
|
||||||
failureThreshold: 60 # ~5 min for large-DB WAL replay / index load
|
failureThreshold: 60 # ~5 min for large-DB WAL replay / index load
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /health/live
|
path: /health/live
|
||||||
port: http
|
port: http
|
||||||
|
scheme: HTTPS
|
||||||
periodSeconds: 10
|
periodSeconds: 10
|
||||||
timeoutSeconds: 3
|
timeoutSeconds: 3
|
||||||
failureThreshold: 3
|
failureThreshold: 3
|
||||||
@ -181,6 +186,7 @@ spec:
|
|||||||
httpGet:
|
httpGet:
|
||||||
path: /health # cluster-aware: 503 joiner/quarantined/draining
|
path: /health # cluster-aware: 503 joiner/quarantined/draining
|
||||||
port: http
|
port: http
|
||||||
|
scheme: HTTPS
|
||||||
periodSeconds: 10
|
periodSeconds: 10
|
||||||
timeoutSeconds: 3
|
timeoutSeconds: 3
|
||||||
failureThreshold: 3
|
failureThreshold: 3
|
||||||
|
|||||||
97
tidal-stress/k8s/stress-job-m11p6-baseline.yaml
Normal file
97
tidal-stress/k8s/stress-job-m11p6-baseline.yaml
Normal file
@ -0,0 +1,97 @@
|
|||||||
|
# m11p6 baseline — quorum-write throughput over the mTLS plane (m11p7).
|
||||||
|
# First run on the m11-44b768b image: confirms the HTTPS :9500 client plane,
|
||||||
|
# cluster-key auth, and m11p6 single-group replication all carry a real write
|
||||||
|
# load end-to-end, and re-establishes the T2-A capacity number on the new stack.
|
||||||
|
#
|
||||||
|
# TLS: the :9500 plane serves a private-CA cert (tidaldb-cluster-tls). The
|
||||||
|
# generator trusts it via the mounted ca.crt (--ca-cert) — verified TLS, not
|
||||||
|
# --insecure. Targets MUST be https:// and use the cert's DNS SANs (pod-DNS / VIP),
|
||||||
|
# never pod IPs (not in the SAN list).
|
||||||
|
#
|
||||||
|
# Apply: kubectl apply -f tidal-stress/k8s/stress-job-m11p6-baseline.yaml
|
||||||
|
# Watch: kubectl logs -f job/tidal-stress-m11p6 -n tidaldb-cluster
|
||||||
|
# Rearm: kubectl delete job tidal-stress-m11p6 -n tidaldb-cluster
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: tidal-stress-m11p6
|
||||||
|
namespace: tidaldb-cluster
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: tidal-stress
|
||||||
|
app.kubernetes.io/part-of: tidaldb
|
||||||
|
spec:
|
||||||
|
backoffLimit: 0
|
||||||
|
ttlSecondsAfterFinished: 7200
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: tidal-stress
|
||||||
|
app.kubernetes.io/part-of: tidaldb
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 1000
|
||||||
|
runAsGroup: 1000
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
containers:
|
||||||
|
- name: stress
|
||||||
|
image: registry.threesix.ai/tidal/stress@sha256:e130aa871f5df17a14a9e13e7df606c602b95a03d8eba49490ff7481e6e2b2b3 # m11-44b768b (TLS-aware)
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
args:
|
||||||
|
- --target
|
||||||
|
- https://tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
|
||||||
|
- --target
|
||||||
|
- https://tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
|
||||||
|
- --target
|
||||||
|
- https://tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
|
||||||
|
- --leader-url
|
||||||
|
- https://tidaldb.tidaldb-cluster.svc.cluster.local:9500 # VIP — routes to any ready pod; m11p6 hash-routes writes to the group leader
|
||||||
|
- --ca-cert
|
||||||
|
- /etc/tidaldb/tls/ca.crt
|
||||||
|
- --ack
|
||||||
|
- quorum
|
||||||
|
- --ramp
|
||||||
|
- peach-100k
|
||||||
|
- --stage-secs
|
||||||
|
- "120"
|
||||||
|
- --mix
|
||||||
|
- writes
|
||||||
|
- --corpus
|
||||||
|
- "20000"
|
||||||
|
- --users
|
||||||
|
- "100000"
|
||||||
|
- --poll-status
|
||||||
|
env:
|
||||||
|
- name: TIDAL_API_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: tidaldb-credentials
|
||||||
|
key: TIDAL_API_KEY
|
||||||
|
- name: TIDAL_STRESS_LOG
|
||||||
|
value: warn
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 250m
|
||||||
|
memory: 256Mi
|
||||||
|
limits:
|
||||||
|
cpu: "3"
|
||||||
|
memory: 1Gi
|
||||||
|
securityContext:
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
readOnlyRootFilesystem: true
|
||||||
|
capabilities:
|
||||||
|
drop: ["ALL"]
|
||||||
|
volumeMounts:
|
||||||
|
- name: cluster-tls
|
||||||
|
mountPath: /etc/tidaldb/tls
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: cluster-tls
|
||||||
|
secret:
|
||||||
|
secretName: tidaldb-cluster-tls
|
||||||
|
items:
|
||||||
|
- key: ca.crt
|
||||||
|
path: ca.crt
|
||||||
@ -45,8 +45,10 @@ impl HttpClient {
|
|||||||
request_timeout: Duration,
|
request_timeout: Duration,
|
||||||
api_key: Option<String>,
|
api_key: Option<String>,
|
||||||
ack: Option<String>,
|
ack: Option<String>,
|
||||||
|
ca_cert: Option<String>,
|
||||||
|
insecure: bool,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let inner = reqwest::Client::builder()
|
let mut builder = reqwest::Client::builder()
|
||||||
.timeout(request_timeout)
|
.timeout(request_timeout)
|
||||||
.connect_timeout(Duration::from_secs(5))
|
.connect_timeout(Duration::from_secs(5))
|
||||||
// Warm-connection pool sized to comfortably saturate the server's
|
// Warm-connection pool sized to comfortably saturate the server's
|
||||||
@ -54,9 +56,21 @@ impl HttpClient {
|
|||||||
// 4 host-pools (≈4×256 idle ceiling).
|
// 4 host-pools (≈4×256 idle ceiling).
|
||||||
.pool_max_idle_per_host(256)
|
.pool_max_idle_per_host(256)
|
||||||
.pool_idle_timeout(Duration::from_secs(90))
|
.pool_idle_timeout(Duration::from_secs(90))
|
||||||
.tcp_nodelay(true)
|
.tcp_nodelay(true);
|
||||||
.build()
|
// m11p7: the cluster's :9500 plane serves TLS from a private CA. Trust it
|
||||||
.map_err(StressError::Client)?;
|
// via the mounted ca.crt (verified), or skip verification with --insecure.
|
||||||
|
if let Some(path) = ca_cert {
|
||||||
|
let pem = std::fs::read(&path).map_err(|source| StressError::CaCert {
|
||||||
|
path: path.clone(),
|
||||||
|
source,
|
||||||
|
})?;
|
||||||
|
let cert = reqwest::Certificate::from_pem(&pem).map_err(StressError::Client)?;
|
||||||
|
builder = builder.add_root_certificate(cert);
|
||||||
|
}
|
||||||
|
if insecure {
|
||||||
|
builder = builder.danger_accept_invalid_certs(true);
|
||||||
|
}
|
||||||
|
let inner = builder.build().map_err(StressError::Client)?;
|
||||||
// Build the Authorization header once. reqwest's `bearer_auth` does a fresh
|
// Build the Authorization header once. reqwest's `bearer_auth` does a fresh
|
||||||
// `format!("Bearer {token}")` allocation plus a HeaderValue validation pass
|
// `format!("Bearer {token}")` allocation plus a HeaderValue validation pass
|
||||||
// on EVERY call, over a token that never changes — pure per-request waste.
|
// on EVERY call, over a token that never changes — pure per-request waste.
|
||||||
|
|||||||
@ -31,6 +31,14 @@ pub enum StressError {
|
|||||||
#[error("http client build failed: {0}")]
|
#[error("http client build failed: {0}")]
|
||||||
Client(#[source] reqwest::Error),
|
Client(#[source] reqwest::Error),
|
||||||
|
|
||||||
|
/// The `--ca-cert` PEM bundle could not be read (an IO/disk fault).
|
||||||
|
#[error("could not load --ca-cert {path}: {source}")]
|
||||||
|
CaCert {
|
||||||
|
path: String,
|
||||||
|
#[source]
|
||||||
|
source: std::io::Error,
|
||||||
|
},
|
||||||
|
|
||||||
/// The provided API key could not be turned into an HTTP `Authorization`
|
/// The provided API key could not be turned into an HTTP `Authorization`
|
||||||
/// header value (contains control or non-visible characters).
|
/// header value (contains control or non-visible characters).
|
||||||
#[error("invalid api key: {0}")]
|
#[error("invalid api key: {0}")]
|
||||||
|
|||||||
@ -49,6 +49,17 @@ struct Cli {
|
|||||||
#[arg(long, env = "TIDAL_API_KEY")]
|
#[arg(long, env = "TIDAL_API_KEY")]
|
||||||
api_key: Option<String>,
|
api_key: Option<String>,
|
||||||
|
|
||||||
|
/// PEM CA bundle to trust for HTTPS targets (m11p7: the cluster's :9500 plane
|
||||||
|
/// serves TLS from a private CA). Point at the mounted `ca.crt` from the
|
||||||
|
/// `tidaldb-cluster-tls` Secret. Verified TLS — preferred over --insecure.
|
||||||
|
#[arg(long)]
|
||||||
|
ca_cert: Option<String>,
|
||||||
|
|
||||||
|
/// Skip TLS certificate verification for HTTPS targets. An escape hatch for
|
||||||
|
/// quick in-cluster runs; use --ca-cert for real verification.
|
||||||
|
#[arg(long, default_value_t = false)]
|
||||||
|
insecure: bool,
|
||||||
|
|
||||||
/// Write acknowledgment mode sent as `x-tidal-ack` on every write
|
/// Write acknowledgment mode sent as `x-tidal-ack` on every write
|
||||||
/// (m11p3: leader|quorum). Omitted = the cluster's topology default.
|
/// (m11p3: leader|quorum). Omitted = the cluster's topology default.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
@ -201,6 +212,8 @@ async fn run() -> Result<()> {
|
|||||||
Duration::from_secs(cli.request_timeout_secs),
|
Duration::from_secs(cli.request_timeout_secs),
|
||||||
cli.api_key.clone(),
|
cli.api_key.clone(),
|
||||||
cli.ack.clone(),
|
cli.ack.clone(),
|
||||||
|
cli.ca_cert.clone(),
|
||||||
|
cli.insecure,
|
||||||
)?);
|
)?);
|
||||||
|
|
||||||
// The seed URL anchors corpus seeding (any gateway accepts items and
|
// The seed URL anchors corpus seeding (any gateway accepts items and
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user