diff --git a/k8s/cluster/kustomization.yaml b/k8s/cluster/kustomization.yaml index 0dd6638..f469c47 100644 --- a/k8s/cluster/kustomization.yaml +++ b/k8s/cluster/kustomization.yaml @@ -30,6 +30,7 @@ resources: - service-client.yaml - statefulset.yaml - poddisruptionbudget.yaml + - networkpolicy.yaml # Public exposure (Traefik + Let's Encrypt). Remove for internal-only. - ingress.yaml diff --git a/k8s/cluster/networkpolicy.yaml b/k8s/cluster/networkpolicy.yaml new file mode 100644 index 0000000..1b21c78 --- /dev/null +++ b/k8s/cluster/networkpolicy.yaml @@ -0,0 +1,62 @@ +# Ingress isolation for the tidalDB cluster. +# +# WHY: before this existed, ANY pod in the k3s cluster could read :9091 metrics +# (unauthenticated - corpus size, seqnos, leader identity) and reach the peer gRPC +# plane. Verified by scraping tidaldb-0:9091 from an unrelated pod in another +# namespace. +# +# INGRESS ONLY, deliberately. Egress is left unrestricted: the WAL-archival / S3 +# paths and the peer dial-out surface are not fully enumerated here, and a wrong +# egress rule partitions the cluster rather than merely blocking a scrape. +# +# :9500 IS INTENTIONALLY LEFT OPEN, and stays that way. It is the only port the +# kubelet probes - startup /health/startup, readiness /health, AND liveness +# /health/live all target it - and probes originate from the NODE, not a pod. +# Node-to-pod handling differs between CNIs, so a wrong rule here fails liveness +# (threshold 6 x 10s) and RESTARTS every pod. +# +# The exposure this would have closed is `/cluster/status`, which is +# unauthenticated by design and leaks leader identity, membership and seqnos to +# any in-cluster pod. That is an ENGINE defect, not a network one, and network +# rules are the wrong layer to fix it: the route is moved behind the bearer in +# tidal-server instead. Every data and admin route on this port is already +# bearer-gated. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: tidaldb + namespace: tidaldb-cluster + labels: + app.kubernetes.io/name: tidaldb + app.kubernetes.io/part-of: tidaldb +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: tidaldb + app.kubernetes.io/component: cluster-node + policyTypes: + - Ingress + ingress: + # 1. Same namespace, every port: the sibling replication plane (gRPC 9601-9603 + # plus the inter-node HTTP forwards on 9500) and the soak monitor. + - from: + - podSelector: {} + # 2. The client + health port from anywhere. See the header for why this is + # not narrowed to kube-system: kubelet probes come from the node. + # To tighten later, add an ipBlock for the node InternalIPs + # (208.122.204.172/173/174) alongside a kube-system namespaceSelector, then + # confirm readiness holds on all three pods before trusting it. + - ports: + - protocol: TCP + port: 9500 + # 3. Metrics: the scraper only. tidalDB's :9091 has no authentication, so this + # is the rule that actually closes the leak. Pods carry + # prometheus.io/scrape, so discovery is annotation-based from vmagent in + # ns observability - there is no ServiceMonitor. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + ports: + - protocol: TCP + port: 9091