225751d34d
m11p1 — decoupled ack/ship path: staged writes (seqno+WAL+relay-push, microseconds) separate from group-commit fsync; ShipQueue batches+windows outbound segments; receiver coalesces inbound chunks before applying. Adds first tidaldb_cluster_* metrics. m11p2 — leader WAL is now THE replicated log: fsynced batches feed a bounded WalShipFeed and ship byte-identical to followers; WAL seqnos survive restarts (relay-reset hazard gone). Item metadata and embeddings journal kind-1/2 blob records on the same stream as signals; the m8p10 HTTP broadcast is deleted. StreamSegments catch-up is follower-pulled via server-streaming RPC, triggered on gap detection, follower boot, and leader heal nudge. Promote carries a stream baseline so peers skip pre-stream history.
76 lines
2.7 KiB
Protocol Buffer
76 lines
2.7 KiB
Protocol Buffer
syntax = "proto3";
|
|
package tidal.replication.v1;
|
|
|
|
// Globally unique identifier for a WAL segment.
|
|
message WalSegmentId {
|
|
uint32 region_id = 1;
|
|
uint32 shard_id = 2;
|
|
uint64 seqno = 3;
|
|
}
|
|
|
|
// A WAL segment ready for shipping to a peer shard.
|
|
message ShipSegmentRequest {
|
|
WalSegmentId id = 1;
|
|
bytes payload = 2;
|
|
uint64 event_count = 3;
|
|
// The segment's authoritative last WAL sequence number, computed by the
|
|
// leader from the ORIGINAL (pre-community-overlay-filter) bytes. Lets the
|
|
// receiver advance its replication-lag leader high-water-mark even for an
|
|
// all-local segment that filters to an empty payload (obs-REPL-1). A 0 value
|
|
// (e.g. from an older sender that omits this field) means "unknown" and the
|
|
// receiver falls back to the per-batch boundaries it decodes.
|
|
uint64 leader_last_seq = 4;
|
|
// The shipping stream's baseline (m11p2): the WAL seqno at which this
|
|
// leader's stream STARTED. Non-zero only on catch-up stream chunks from a
|
|
// promoted leader; seqnos at or below it are pre-stream history the
|
|
// receiver jumps its frontier past instead of treating as a gap. 0 (the
|
|
// default, and what every live unary ship carries) means "stream from the
|
|
// beginning".
|
|
uint64 stream_baseline = 5;
|
|
}
|
|
|
|
// Response to a segment shipment.
|
|
message ShipSegmentResponse {
|
|
bool accepted = 1;
|
|
// The receiver's contiguous applied seqno for the request's source shard at
|
|
// acceptance time (m11p2): a monotonic hint the sender folds into its acked
|
|
// frontier so retries of already-applied data prune and heal needs no
|
|
// separate status fetch. 0 = unknown (older peer / no applied source wired).
|
|
uint64 applied_seqno = 2;
|
|
}
|
|
|
|
// Request to stream segments from a given sequence number.
|
|
message StreamRequest {
|
|
uint32 shard_id = 1;
|
|
uint64 from_seqno = 2;
|
|
}
|
|
|
|
// Heartbeat request matching ControlPlane's ShardStats.
|
|
message HeartbeatRequest {
|
|
uint32 shard_id = 1;
|
|
uint32 region_id = 2;
|
|
uint64 entity_count = 3;
|
|
double signal_throughput_eps = 4;
|
|
uint64 disk_bytes = 5;
|
|
// Replication lag per peer region (region_id -> lag in events).
|
|
map<uint32, uint64> replication_lag = 6;
|
|
uint64 last_heartbeat_ns = 7;
|
|
}
|
|
|
|
// Heartbeat acknowledgement.
|
|
message HeartbeatResponse {
|
|
bool acknowledged = 1;
|
|
}
|
|
|
|
// WAL segment shipping service between tidalDB shards.
|
|
service WalShipping {
|
|
// Ship a single WAL segment to a peer shard (unary).
|
|
rpc ShipSegment(ShipSegmentRequest) returns (ShipSegmentResponse);
|
|
|
|
// Stream WAL segments from a given sequence number (server-streaming).
|
|
rpc StreamSegments(StreamRequest) returns (stream ShipSegmentRequest);
|
|
|
|
// Periodic health check for the ControlPlane.
|
|
rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse);
|
|
}
|