tidaldb/k8s/cluster/topology-configmap.yaml

# The ONE bootstrap topology file, shared by EVERY pod (m11p5 §4).
#
# Before m11p5, k8s cluster mode needed a per-pod ConfigMap variant: a node bound
# its own `grpc_addr` LITERALLY and a pod cannot bind a Service ClusterIP, so each
# pod's self-entry had to be a bindable address while peers' entries stayed the
# routable ones — defeating the "every process parses the SAME file" contract.
#
# The m11p5 bind/advertise split kills that hack:
#   - `grpc_addr` is the ADVERTISED address siblings DIAL (a per-pod DNS name);
#     with a DNS name tonic re-resolves on every reconnect, so a rescheduled pod
#     on a new IP is reachable again with NO peer restart.
#   - `grpc_bind` is the LOCAL socket — `0.0.0.0:9601` on every pod. The local
#     socket never tries to bind the DNS name.
# So one file names all three initial regions by their stable headless-Service
# DNS names, and every pod mounts THIS file unmodified.
#
# Region declaration order == RegionId (topology.rs:24-28). The three regions are
# named after the StatefulSet's stable pod identities tidaldb-{0,1,2}. The pod's
# `--region` comes from POD_NAME (statefulset.yaml fieldRef), so pod `tidaldb-0`
# is region `tidaldb-0`, etc. — the names line up by construction.
#
# SCALING PAST 3 DOES NOT EDIT THIS FILE: pod N>=3 boots with `--seed` and learns
# its roster/id/term from a seed (membership is data, on the replicated log). It
# still mounts THIS file for the behavioral knob blocks only — `--seed` boots
# REQUIRE a local config for `replication`/`wal`/`election`/`timeouts`/`grpc_tls`
# (m11p5 §3.5); the `regions:` list below is IGNORED for the roster of a seed
# joiner (the join response is authoritative), it supplies only those knobs.
#
# DNS form: <pod>.<headless-service>.<namespace>.svc.cluster.local
#   tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local
apiVersion: v1
kind: ConfigMap
metadata:
  name: tidaldb-cluster-topology
  namespace: tidaldb-cluster
  labels:
    app.kubernetes.io/name: tidaldb
    app.kubernetes.io/part-of: tidaldb
data:
  cluster-topology.yaml: |
    # Three initial regions, named by their per-pod headless-Service DNS.
    # grpc_addr  = ADVERTISED gRPC (DNS, port 9601) — what peers dial.
    # grpc_bind  = LOCAL gRPC bind (0.0.0.0:9601)   — the pod can't bind the DNS.
    # http_addr  = ADVERTISED HTTP (DNS, port 9500) — peers forward writes/status.
    # metrics_addr = per-region Prometheus listener (0.0.0.0:9091).
    # grpc_tls (m11p7): inter-node TLS material, mounted from the cert-manager
    # Secret `tidaldb-cluster-tls` at /etc/tidaldb/tls (see certs.yaml +
    # statefulset.yaml). The SAME shared node cert is mounted into every pod, so
    # all three blocks point at identical paths; each node reads its OWN block.
    # The cert doubles as the gRPC mTLS client identity (client_cert/client_key)
    # and the gRPC + inter-node-HTTP server identity. Removing these blocks reverts
    # the cluster to PLAINTEXT inter-node links (a loud startup WARN), acceptable
    # only on a fully trusted network.
    regions:
      - name: tidaldb-0
        grpc_addr: tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9601
        grpc_bind: 0.0.0.0:9601
        http_addr: tidaldb-0.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
        metrics_addr: 0.0.0.0:9091
        grpc_tls: &grpc_tls
          ca_cert: /etc/tidaldb/tls/ca.crt
          server_cert: /etc/tidaldb/tls/tls.crt
          server_key: /etc/tidaldb/tls/tls.key
          client_cert: /etc/tidaldb/tls/tls.crt
          client_key: /etc/tidaldb/tls/tls.key
      - name: tidaldb-1
        grpc_addr: tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9601
        grpc_bind: 0.0.0.0:9601
        http_addr: tidaldb-1.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
        metrics_addr: 0.0.0.0:9091
        grpc_tls: *grpc_tls
      - name: tidaldb-2
        grpc_addr: tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9601
        grpc_bind: 0.0.0.0:9601
        http_addr: tidaldb-2.tidaldb-peers.tidaldb-cluster.svc.cluster.local:9500
        metrics_addr: 0.0.0.0:9091
        grpc_tls: *grpc_tls
    # Term-0 bootstrap leader only — post-election this field is dead config
    # (durable election_state governs; a restart always boots a follower).
    leader: tidaldb-0
    # ── Sharding × replication (m11p6 / m12p4 T5) ────────────────────────────
    # Absent `shards:` ⇒ ONE group, RF = all pods. We ENABLE 3 groups × RF=3 so
    # the write path scales horizontally: each group is an independent
    # replication group with its OWN elected leader, and the three leaders are
    # balanced one-per-pod (tidaldb-0 leads shard 0, tidaldb-1 shard 1,
    # tidaldb-2 shard 2). The entity space is hash-partitioned across the three
    # groups, so three leaders absorb writes in parallel instead of one — the
    # ~S× write-throughput claim m12p4/T5 proves.
    #
    # Full placement: every pod replicates EVERY group, so any pod can serve a
    # corpus-wide read locally (no cross-node read fan-out needed at this shape)
    # and a single pod loss never loses a group's quorum (2 of 3 survive per
    # group). A pod hosting all three groups binds one gRPC port per group:
    # `replicas[].grpc_addr` is omitted, so each is DERIVED as
    # `node base port + shard id` (tidaldb-N binds 9601 for shard 0, 9602 for
    # shard 1, 9603 for shard 2 — matching the extra containerPorts in
    # statefulset.yaml). Each group's data lives under
    # <data_dir>/shard-{id:05}/ in the one PVC. Operators rebalance with
    # `POST /cluster/shards/{id}/transfer` and `/replicas` (see runbook §6a).
    shards:
      - { id: 0, leader: tidaldb-0, replicas: [ {node: tidaldb-0}, {node: tidaldb-1}, {node: tidaldb-2} ] }
      - { id: 1, leader: tidaldb-1, replicas: [ {node: tidaldb-0}, {node: tidaldb-1}, {node: tidaldb-2} ] }
      - { id: 2, leader: tidaldb-2, replicas: [ {node: tidaldb-0}, {node: tidaldb-1}, {node: tidaldb-2} ] }
    replication:
      # ack=quorum: a write succeeds once a MAJORITY of the replica set durably
      # holds it (m11p3). Callers can still override per-request with x-tidal-ack.
      ack: quorum
      # reseed_self_restart=true (m11p5 §2.4): a node that latches the durable
      # `reseed_required` marker drains and exits(0) so the StatefulSet restarts
      # it and the boot-time install re-runs. The exit is REFUSED (loudly, in
      # status + gauge) when the remaining voters can't sustain quorum without
      # this node — exiting during a 2-voter window would be a total write outage.
      reseed_self_restart: true
    election:
      # Defaults (300ms heartbeats, 1500-3000ms election timeout, 900ms lease,
      # auto-election on) are right for an in-cluster (low-RTT) deployment. The
      # C2 safety invariant lease+heartbeat < election_timeout_min holds for the
      # defaults; spell the block out so an operator widening WAN budgets edits
      # here, not the compiled-in defaults.
      heartbeat_interval_ms: 300
      election_timeout_min_ms: 1500
      election_timeout_max_ms: 3000
      leader_lease_ms: 900
      auto_election: true