5ed2edb211
ack=quorum gates replicated writes on a majority of the replica set durably holding them: followers push their durably-applied frontier (ReportApplied, once per apply round, decoupled from ship acks), the leader folds frontier reports + ship-ack hints + heal resumes into a leadership-scoped CommitIndex (k-th-largest durable mark), and handlers await it through an async watch-channel bridge (zero parked threads per waiter). Honest timeouts: retryable 503 naming the laggards; x-tidal-seq on every cluster write. Follower blob applies are batched under group-commit fsyncs (22x seeding). Exit gate: 167/167 leader-SIGKILL kill points, zero acked-write loss. Seven-dimension review pass (all confirmed findings fixed): - WAL blob drain now ABORTS on the first write failure instead of reusing the failed seqno mid-drain (a torn record buried mid-segment would truncate every later acked record on replay) - apply_replicated_blobs waits every staged append even after a mid-batch failure, parses metadata once, and moves records into Arcs shared with the WAL writer (no deep clone per record on the follower apply path) - CommitIndex: zero-peer fast path now respects demotion (active checked under lock before the single-replica return), k-th-largest uses select_nth over a reused scratch buffer - await_quorum: re-reads the index once after the deadline fires (no false 503 for a write that committed in the race window), warns when the commit-watch bridge dies outside shutdown, zero-peer path checks active - notify_applied report failures: WARN on the first failure of a streak, INFO on recovery (a silently stalling frontier reads as unexplained quorum 503s); receiver skips re-notifying unadvanced frontiers - x-tidal-deduplicated: 1 marks dedup-suppressed signal writes (relayed through forwards) so durability cursors can tell dedup from no-seqno - docs: 167/167 kill-point record corrected in CHANGELOG; rolling-upgrade order (leader first — a pre-m11p3 leader silently downgrades quorum requests to leader-ack) in CHANGELOG + runbook §8; monitoring note for report-loss diagnosis on the quorum-timeout alert Verified: workspace clippy -D warnings (incl. cluster-e2e targets), full tidaldb/tidal-net/tidal-server/tidalctl suites green, tier-3 multi-process quorum suite green (8/8 kill points, zero acked loss, partition gate/recover).
98 lines
3.4 KiB
Protocol Buffer
98 lines
3.4 KiB
Protocol Buffer
syntax = "proto3";
|
|
package tidal.replication.v1;
|
|
|
|
// Globally unique identifier for a WAL segment.
|
|
message WalSegmentId {
|
|
uint32 region_id = 1;
|
|
uint32 shard_id = 2;
|
|
uint64 seqno = 3;
|
|
}
|
|
|
|
// A WAL segment ready for shipping to a peer shard.
|
|
message ShipSegmentRequest {
|
|
WalSegmentId id = 1;
|
|
bytes payload = 2;
|
|
uint64 event_count = 3;
|
|
// The segment's authoritative last WAL sequence number, computed by the
|
|
// leader from the ORIGINAL (pre-community-overlay-filter) bytes. Lets the
|
|
// receiver advance its replication-lag leader high-water-mark even for an
|
|
// all-local segment that filters to an empty payload (obs-REPL-1). A 0 value
|
|
// (e.g. from an older sender that omits this field) means "unknown" and the
|
|
// receiver falls back to the per-batch boundaries it decodes.
|
|
uint64 leader_last_seq = 4;
|
|
// The shipping stream's baseline (m11p2): the WAL seqno at which this
|
|
// leader's stream STARTED. Non-zero only on catch-up stream chunks from a
|
|
// promoted leader; seqnos at or below it are pre-stream history the
|
|
// receiver jumps its frontier past instead of treating as a gap. 0 (the
|
|
// default, and what every live unary ship carries) means "stream from the
|
|
// beginning".
|
|
uint64 stream_baseline = 5;
|
|
}
|
|
|
|
// Response to a segment shipment.
|
|
message ShipSegmentResponse {
|
|
bool accepted = 1;
|
|
// The receiver's contiguous applied seqno for the request's source shard at
|
|
// acceptance time (m11p2): a monotonic hint the sender folds into its acked
|
|
// frontier so retries of already-applied data prune and heal needs no
|
|
// separate status fetch. 0 = unknown (older peer / no applied source wired).
|
|
uint64 applied_seqno = 2;
|
|
}
|
|
|
|
// Request to stream segments from a given sequence number.
|
|
message StreamRequest {
|
|
uint32 shard_id = 1;
|
|
uint64 from_seqno = 2;
|
|
}
|
|
|
|
// Heartbeat request matching ControlPlane's ShardStats.
|
|
message HeartbeatRequest {
|
|
uint32 shard_id = 1;
|
|
uint32 region_id = 2;
|
|
uint64 entity_count = 3;
|
|
double signal_throughput_eps = 4;
|
|
uint64 disk_bytes = 5;
|
|
// Replication lag per peer region (region_id -> lag in events).
|
|
map<uint32, uint64> replication_lag = 6;
|
|
uint64 last_heartbeat_ns = 7;
|
|
}
|
|
|
|
// Heartbeat acknowledgement.
|
|
message HeartbeatResponse {
|
|
bool acknowledged = 1;
|
|
}
|
|
|
|
// A follower's self-report of its durable frontier (m11p3).
|
|
//
|
|
// Pushed by the receiver once per apply round — fully decoupled from ship
|
|
// acks, so the leader's quorum commit index stays fresh even when its
|
|
// outbound ships stall (gap-parked follower, quiet leader, pull catch-up).
|
|
message AppliedReport {
|
|
// The reporting node's shard id.
|
|
uint32 reporter_shard = 1;
|
|
// The stream's source shard (the leader being reported to).
|
|
uint32 source_shard = 2;
|
|
// The reporter's contiguous durably-applied seqno for that stream.
|
|
uint64 applied_seqno = 3;
|
|
}
|
|
|
|
// Applied-report acknowledgement.
|
|
message AppliedReportAck {
|
|
bool acknowledged = 1;
|
|
}
|
|
|
|
// WAL segment shipping service between tidalDB shards.
|
|
service WalShipping {
|
|
// Ship a single WAL segment to a peer shard (unary).
|
|
rpc ShipSegment(ShipSegmentRequest) returns (ShipSegmentResponse);
|
|
|
|
// Stream WAL segments from a given sequence number (server-streaming).
|
|
rpc StreamSegments(StreamRequest) returns (stream ShipSegmentRequest);
|
|
|
|
// Periodic health check for the ControlPlane.
|
|
rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse);
|
|
|
|
// Follower -> leader durable-frontier report (m11p3 quorum acks).
|
|
rpc ReportApplied(AppliedReport) returns (AppliedReportAck);
|
|
}
|