225751d34d
m11p1 — decoupled ack/ship path: staged writes (seqno+WAL+relay-push, microseconds) separate from group-commit fsync; ShipQueue batches+windows outbound segments; receiver coalesces inbound chunks before applying. Adds first tidaldb_cluster_* metrics. m11p2 — leader WAL is now THE replicated log: fsynced batches feed a bounded WalShipFeed and ship byte-identical to followers; WAL seqnos survive restarts (relay-reset hazard gone). Item metadata and embeddings journal kind-1/2 blob records on the same stream as signals; the m8p10 HTTP broadcast is deleted. StreamSegments catch-up is follower-pulled via server-streaming RPC, triggered on gap detection, follower boot, and leader heal nudge. Promote carries a stream baseline so peers skip pre-stream history.
98 lines
3.6 KiB
Rust
98 lines
3.6 KiB
Rust
// Integration-test exemptions (same posture as the tidaldb integration tests):
|
|
// unwrap on known-good fixtures and short-lived read guards are idiomatic here.
|
|
#![allow(clippy::unwrap_used, clippy::significant_drop_tightening)]
|
|
//! Regression test for the tonic default 4 MiB codec message-size limit.
|
|
//!
|
|
//! tonic 0.12 defaults BOTH the encoder and decoder message-size limits to
|
|
//! 4 MiB. `GrpcTransportConfig::max_payload_bytes` defaults to 64 MiB and the
|
|
//! WAL default sealed segment is 16 MiB, so a full-size segment is 4x the codec
|
|
//! default. If the limits are not raised on both ends, the FIRST full-size
|
|
//! segment a leader ships fails inside the codec, maps to `TransportError::
|
|
//! Closed`, and the shipper retries the same seqno forever (silent replication
|
|
//! stall). This test ships a payload strictly between 4 MiB and the configured
|
|
//! max and asserts it round-trips byte-for-byte, so the regression can never
|
|
//! return undetected.
|
|
|
|
use std::{collections::HashMap, net::SocketAddr, thread, time::Duration};
|
|
|
|
use tidal_net::{GrpcTransport, config::GrpcTransportConfig};
|
|
use tidaldb::replication::{
|
|
WalSegmentId,
|
|
shard::{RegionId, ShardId},
|
|
transport::{Transport, WalSegmentPayload},
|
|
};
|
|
|
|
/// Get a unique listen address using port 0 (OS-assigned).
|
|
fn free_addr() -> SocketAddr {
|
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
|
listener.local_addr().unwrap()
|
|
}
|
|
|
|
fn make_config(
|
|
shard: ShardId,
|
|
listen: SocketAddr,
|
|
peers: HashMap<ShardId, SocketAddr>,
|
|
) -> GrpcTransportConfig {
|
|
GrpcTransportConfig {
|
|
local_shard: shard,
|
|
listen_addr: listen,
|
|
peers,
|
|
insecure: true,
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn ships_payload_larger_than_tonic_default_codec_limit() {
|
|
// Sanity: the test is only meaningful if the payload exceeds tonic's 4 MiB
|
|
// default AND fits inside the configured ceiling. The payload-exceeds-default
|
|
// half is a compile-time invariant; the fits-in-ceiling half is runtime.
|
|
const FOUR_MIB: usize = 4 * 1024 * 1024;
|
|
const EIGHT_MIB: usize = 8 * 1024 * 1024;
|
|
const _: () = assert!(EIGHT_MIB > FOUR_MIB, "payload must exceed tonic default");
|
|
|
|
let addr0 = free_addr();
|
|
let addr1 = free_addr();
|
|
|
|
let config0 = make_config(ShardId(0), addr0, HashMap::from([(ShardId(1), addr1)]));
|
|
let config1 = make_config(ShardId(1), addr1, HashMap::from([(ShardId(0), addr0)]));
|
|
|
|
assert!(
|
|
EIGHT_MIB <= config0.max_payload_bytes,
|
|
"payload must fit configured max"
|
|
);
|
|
|
|
let t0 = GrpcTransport::new(config0).expect("transport 0");
|
|
let t1 = GrpcTransport::new(config1).expect("transport 1");
|
|
|
|
// Give the servers a moment to start.
|
|
thread::sleep(Duration::from_millis(150));
|
|
|
|
// A deterministic, verifiable 8 MiB body (a repeating byte pattern keyed on
|
|
// index so a silent truncation or corruption is caught, not just length).
|
|
let body: Vec<u8> = (0..EIGHT_MIB).map(|i| (i % 251) as u8).collect();
|
|
let payload = WalSegmentPayload {
|
|
id: WalSegmentId::new(RegionId::SINGLE, ShardId(0), 7),
|
|
bytes: body.clone(),
|
|
event_count: 3,
|
|
leader_last_seq: 7,
|
|
stream_baseline: 0,
|
|
};
|
|
|
|
t0.send_segment(ShardId(1), payload)
|
|
.expect("8 MiB segment must ship past the raised codec limit");
|
|
|
|
let received = t1
|
|
.recv_segment()
|
|
.expect("follower must receive the 8 MiB segment");
|
|
|
|
assert_eq!(received.id.seqno, 7);
|
|
assert_eq!(received.event_count, 3);
|
|
assert_eq!(
|
|
received.bytes.len(),
|
|
EIGHT_MIB,
|
|
"body length must round-trip"
|
|
);
|
|
assert_eq!(received.bytes, body, "body bytes must round-trip exactly");
|
|
}
|