b55ad70141
Resolves the 142 findings from tidal/docs/reviews/CODE_REVIEW_m0-m10.md across the engine, server, net, and CLI surfaces: - WAL/session-journal durability, checkpoint format, and crash-recovery hardening - Replication shipper/receiver, tenant isolation, and migration paths - Cluster scatter-gather, router, standalone server + health/offload endpoints - tidalctl refactored into command modules with JSON output and WAL-state tooling - Cohort, governance, signal-ledger, and vector-registry correctness fixes - Expanded UAT/integration/durability test coverage across all milestones
112 lines
3.4 KiB
Rust
112 lines
3.4 KiB
Rust
// Integration-test exemptions (same posture as the tidaldb integration tests):
|
|
// unwrap on known-good fixtures and short-lived read guards are idiomatic here.
|
|
#![allow(clippy::unwrap_used, clippy::significant_drop_tightening)]
|
|
//! Tests that `GrpcTransport` reconnects after a peer restarts.
|
|
|
|
use std::{
|
|
collections::HashMap,
|
|
net::SocketAddr,
|
|
thread,
|
|
time::{Duration, Instant},
|
|
};
|
|
|
|
use tidal_net::{GrpcTransport, config::GrpcTransportConfig};
|
|
use tidaldb::replication::{
|
|
WalSegmentId,
|
|
shard::{RegionId, ShardId},
|
|
transport::{Transport, WalSegmentPayload},
|
|
};
|
|
|
|
fn free_addr() -> SocketAddr {
|
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
|
listener.local_addr().unwrap()
|
|
}
|
|
|
|
fn make_config(
|
|
shard: ShardId,
|
|
listen: SocketAddr,
|
|
peers: HashMap<ShardId, SocketAddr>,
|
|
) -> GrpcTransportConfig {
|
|
GrpcTransportConfig {
|
|
local_shard: shard,
|
|
listen_addr: listen,
|
|
peers,
|
|
insecure: true,
|
|
// Shorter circuit breaker for testing.
|
|
circuit_breaker_threshold: 3,
|
|
circuit_breaker_reset: Duration::from_millis(500),
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
fn make_payload(seqno: u64) -> WalSegmentPayload {
|
|
WalSegmentPayload {
|
|
id: WalSegmentId::new(RegionId::SINGLE, ShardId(0), seqno),
|
|
bytes: vec![0xCD; 50],
|
|
event_count: 1,
|
|
leader_last_seq: seqno,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn reconnects_after_peer_restart() {
|
|
let addr_sender = free_addr();
|
|
let addr_receiver = free_addr();
|
|
|
|
// Start sender (shard 0) that knows about receiver (shard 1).
|
|
let sender_config = make_config(
|
|
ShardId(0),
|
|
addr_sender,
|
|
HashMap::from([(ShardId(1), addr_receiver)]),
|
|
);
|
|
let sender = GrpcTransport::new(sender_config).expect("sender");
|
|
|
|
// Start receiver (shard 1).
|
|
let receiver_config = make_config(
|
|
ShardId(1),
|
|
addr_receiver,
|
|
HashMap::from([(ShardId(0), addr_sender)]),
|
|
);
|
|
let receiver = GrpcTransport::new(receiver_config).expect("receiver");
|
|
thread::sleep(Duration::from_millis(100));
|
|
|
|
// Send successfully.
|
|
sender.send_segment(ShardId(1), make_payload(1)).unwrap();
|
|
let first_segment = receiver.recv_segment().unwrap();
|
|
assert_eq!(first_segment.id.seqno, 1);
|
|
|
|
// Drop the receiver to simulate crash.
|
|
drop(receiver);
|
|
thread::sleep(Duration::from_millis(200));
|
|
|
|
// Sends may fail now (peer is down). That's expected.
|
|
let _ = sender.send_segment(ShardId(1), make_payload(2));
|
|
|
|
// Restart receiver on the same address.
|
|
let receiver_config2 = make_config(
|
|
ShardId(1),
|
|
addr_receiver,
|
|
HashMap::from([(ShardId(0), addr_sender)]),
|
|
);
|
|
let receiver2 = GrpcTransport::new(receiver_config2).expect("receiver2");
|
|
thread::sleep(Duration::from_millis(200));
|
|
|
|
// Wait for circuit breaker to allow probes again.
|
|
thread::sleep(Duration::from_millis(500));
|
|
|
|
// Should reconnect and deliver.
|
|
let start = Instant::now();
|
|
let mut delivered = false;
|
|
while start.elapsed() < Duration::from_secs(5) {
|
|
if sender.send_segment(ShardId(1), make_payload(3)).is_ok()
|
|
&& let Some(seg) = receiver2.recv_segment()
|
|
{
|
|
assert_eq!(seg.id.seqno, 3);
|
|
delivered = true;
|
|
break;
|
|
}
|
|
thread::sleep(Duration::from_millis(200));
|
|
}
|
|
assert!(delivered, "failed to reconnect within 5s");
|
|
}
|