From 087b83154a0ca2fc07a7306db529ee8e8320c512 Mon Sep 17 00:00:00 2001 From: jordan Date: Fri, 21 Aug 2026 23:14:55 -0600 Subject: [PATCH] k8s(cluster): publish the client surface over public TLS, data routes only tidaldb.threesix.ai now serves the cluster's data surface over a Let's Encrypt cert, verified end-to-end from the internet: 401 without a bearer, 401 with a wrong one, 200 with the real key, and a quorum-acked write returning 201 on all three node IPs. Three things this had to get right, each of which failed first: * The backend is HTTPS, not HTTP. Pods serve :9500 over TLS with the internal cluster CA whenever grpc_tls is configured, so a plaintext backend dial answers 500. Added a ServersTransport that VERIFIES that hop - every pod mounts the same tidaldb-cluster-tls leaf and its SANs include the client-Service DNS name, so serverName pinning validates it without insecureSkipVerify. * `service.*` annotations are read from the Service, not the Ingress. Putting serversscheme/serverstransport on the Ingress is silently ignored and presents exactly as a broken backend. * http01 cannot be used behind any gateway gate that rejects unknown callers, because it rejects the ACME challenge too. Uses the Cloudflare dns01 solver. Deliberately unpublished: /cluster/* (every mutating admin verb shares the SAME single bearer as the data routes, so a client key could remove members or transfer shards), /cluster/status (unauthenticated - leaks leader, membership, seqnos), /openapi.json (unauthenticated, enumerates the admin routes), and /metrics (only on the headless peers Service, unreachable here). Documents two controls that are NOT available and why: an IP allowlist cannot work while the shared Traefik Service runs externalTrafficPolicy=Cluster (svclb SNATs the client address), and Traefik basicAuth cannot stack in front of the bearer because both occupy the Authorization header. --- k8s/cluster/ingress.yaml | 122 ++++++++++++++++++++++++++++++++ k8s/cluster/kustomization.yaml | 2 + k8s/cluster/service-client.yaml | 8 +++ 3 files changed, 132 insertions(+) create mode 100644 k8s/cluster/ingress.yaml diff --git a/k8s/cluster/ingress.yaml b/k8s/cluster/ingress.yaml new file mode 100644 index 0000000..8c2d5e9 --- /dev/null +++ b/k8s/cluster/ingress.yaml @@ -0,0 +1,122 @@ +# Public client exposure for the tidalDB cluster (Traefik + Let's Encrypt). +# +# OPTIONAL: this file is what publishes the cluster on the open internet. Comment +# it out of kustomization.yaml for an internal-only deployment. +# +# WHAT THIS DELIBERATELY DOES NOT PUBLISH +# * `/cluster/*` - every mutating admin verb (`promote`, `partition`, `heal`, +# `members/remove`, `join`, `reseed`, `shards/{id}/transfer`) sits behind the +# SAME single bearer token as the data routes (tidal-server cluster/node.rs +# `protected`). There is no operator/data credential split, so publishing +# these would let any client key destroy the cluster. +# * `/cluster/status` + `/cluster/status/local` - these are UNAUTHENTICATED +# (they live in the `public` router next to the health probes) and report +# leader identity, membership, term, and per-shard applied/lag/commit seqnos. +# * `/openapi.json` - unauthenticated, and enumerates the admin routes above. +# * `/metrics` - never reachable here: :9091 is published only on the headless +# `tidaldb-peers` Service, not on the client Service this Ingress targets. +# +# CREDENTIALS: the ONLY external credential is tidalDB's bearer token +# (`TIDAL_API_KEY`). A Traefik `basicAuth` middleware CANNOT be stacked in front +# of it - both occupy the `Authorization` header, so a client can only ever send +# one of them. If a second factor is required, use gateway mTLS (a `TLSOption` +# with `clientAuth`), which occupies a different layer. +# +# IP ALLOWLISTING IS NOT AVAILABLE on this fleet: the shared Traefik Service runs +# `externalTrafficPolicy: Cluster`, so k3s svclb SNATs the client address and an +# `ipAllowList` sees an internal IP and rejects everyone. Preserving the source IP +# would require flipping that Service to `Local`, which is fleet-wide and unsafe +# here (Traefik runs 2 replicas across 3 nodes, so one node IP would blackhole, +# and 35 Ingresses share it). +--- +# Traefik must dial the backend over TLS with the internal CA, and can VERIFY it: +# every pod mounts the same `tidaldb-cluster-tls` leaf, whose SANs include the +# client-Service DNS name, so pinning `serverName` to that validates the hop while +# connecting to a pod IP. No `insecureSkipVerify`. +apiVersion: traefik.io/v1alpha1 +kind: ServersTransport +metadata: + name: tidaldb-internal + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidaldb + app.kubernetes.io/part-of: tidaldb +spec: + serverName: tidaldb.tidaldb-cluster.svc.cluster.local + rootCAsSecrets: + - tidaldb-cluster-ca +--- +# Gateway-level cap. tidalDB's OWN limiter is unlimited unless +# `TIDAL_RATE_LIMIT_RPS` is set, and it keys per PRINCIPAL - with one shared +# bearer every caller is the same principal, so it is a single global bucket +# either way. Because svclb SNATs the source, this Traefik limit is likewise +# effectively global rather than per-client. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: tidaldb-ratelimit + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidaldb + app.kubernetes.io/part-of: tidaldb +spec: + rateLimit: + average: 200 + burst: 400 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: tidaldb + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidaldb + app.kubernetes.io/part-of: tidaldb + annotations: + # dns01, NOT http01: any gateway middleware that rejects unknown callers also + # rejects Let's Encrypt's http01 challenge. The Cloudflare DNS solver has no + # such dependency. `letsencrypt-prod` carries the threesix.ai dns01 solver. + cert-manager.io/cluster-issuer: letsencrypt-prod + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + traefik.ingress.kubernetes.io/router.middlewares: tidaldb-cluster-tidaldb-ratelimit@kubernetescrd +spec: + ingressClassName: traefik + tls: + - hosts: + - tidaldb.threesix.ai + secretName: tidaldb-public-tls + rules: + - host: tidaldb.threesix.ai + http: + paths: + - path: /items + pathType: Prefix + backend: &client + service: + name: tidaldb + port: + number: 9500 + - path: /embeddings + pathType: Prefix + backend: *client + - path: /signals + pathType: Prefix + backend: *client + - path: /hardnegs + pathType: Prefix + backend: *client + - path: /feed + pathType: Prefix + backend: *client + - path: /search + pathType: Prefix + backend: *client + - path: /vector_search + pathType: Prefix + backend: *client + # Unauthenticated by design (probe contract). It does report leader and + # region names; drop this path if that is unacceptable externally. + - path: /health + pathType: Exact + backend: *client diff --git a/k8s/cluster/kustomization.yaml b/k8s/cluster/kustomization.yaml index b4a404b..0dd6638 100644 --- a/k8s/cluster/kustomization.yaml +++ b/k8s/cluster/kustomization.yaml @@ -30,6 +30,8 @@ resources: - service-client.yaml - statefulset.yaml - poddisruptionbudget.yaml + # Public exposure (Traefik + Let's Encrypt). Remove for internal-only. + - ingress.yaml labels: - pairs: diff --git a/k8s/cluster/service-client.yaml b/k8s/cluster/service-client.yaml index 282223f..43951c8 100644 --- a/k8s/cluster/service-client.yaml +++ b/k8s/cluster/service-client.yaml @@ -24,6 +24,14 @@ metadata: labels: app.kubernetes.io/name: tidaldb app.kubernetes.io/part-of: tidaldb + annotations: + # Traefik dials this backend over TLS. The pods serve :9500 as HTTPS whenever + # grpc_tls is configured (tidal-server `http_tls`), so a plaintext backend + # dial answers 500. These MUST live on the Service - `service.*` annotations + # are read from the Service, NOT the Ingress (putting them on the Ingress is + # silently ignored and looks exactly like a broken backend). + traefik.ingress.kubernetes.io/service.serversscheme: https + traefik.ingress.kubernetes.io/service.serverstransport: tidaldb-cluster-tidaldb-internal@kubernetescrd spec: selector: app.kubernetes.io/name: tidaldb