// Integration-test exemptions (same posture as the tidaldb integration tests): // unwrap on known-good fixtures and short-lived read guards are idiomatic here. #![allow(clippy::unwrap_used, clippy::significant_drop_tightening)] //! Tests that `GrpcTransport` reconnects after a peer restarts. use std::{ collections::HashMap, net::SocketAddr, thread, time::{Duration, Instant}, }; use tidal_net::{GrpcTransport, config::GrpcTransportConfig}; use tidaldb::replication::{ WalSegmentId, shard::{RegionId, ShardId}, transport::{Transport, WalSegmentPayload}, }; fn free_addr() -> SocketAddr { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); listener.local_addr().unwrap() } fn make_config( shard: ShardId, listen: SocketAddr, peers: HashMap, ) -> GrpcTransportConfig { GrpcTransportConfig { local_shard: shard, listen_addr: listen, peers, insecure: true, // Shorter circuit breaker for testing. circuit_breaker_threshold: 3, circuit_breaker_reset: Duration::from_millis(500), ..Default::default() } } fn make_payload(seqno: u64) -> WalSegmentPayload { WalSegmentPayload { id: WalSegmentId::new(RegionId::SINGLE, ShardId(0), seqno), bytes: vec![0xCD; 50], event_count: 1, leader_last_seq: seqno, } } #[test] fn reconnects_after_peer_restart() { let addr_sender = free_addr(); let addr_receiver = free_addr(); // Start sender (shard 0) that knows about receiver (shard 1). let sender_config = make_config( ShardId(0), addr_sender, HashMap::from([(ShardId(1), addr_receiver)]), ); let sender = GrpcTransport::new(sender_config).expect("sender"); // Start receiver (shard 1). let receiver_config = make_config( ShardId(1), addr_receiver, HashMap::from([(ShardId(0), addr_sender)]), ); let receiver = GrpcTransport::new(receiver_config).expect("receiver"); thread::sleep(Duration::from_millis(100)); // Send successfully. sender.send_segment(ShardId(1), make_payload(1)).unwrap(); let first_segment = receiver.recv_segment().unwrap(); assert_eq!(first_segment.id.seqno, 1); // Drop the receiver to simulate crash. drop(receiver); thread::sleep(Duration::from_millis(200)); // Sends may fail now (peer is down). That's expected. let _ = sender.send_segment(ShardId(1), make_payload(2)); // Restart receiver on the same address. let receiver_config2 = make_config( ShardId(1), addr_receiver, HashMap::from([(ShardId(0), addr_sender)]), ); let receiver2 = GrpcTransport::new(receiver_config2).expect("receiver2"); thread::sleep(Duration::from_millis(200)); // Wait for circuit breaker to allow probes again. thread::sleep(Duration::from_millis(500)); // Should reconnect and deliver. let start = Instant::now(); let mut delivered = false; while start.elapsed() < Duration::from_secs(5) { if sender.send_segment(ShardId(1), make_payload(3)).is_ok() && let Some(seg) = receiver2.recv_segment() { assert_eq!(seg.id.seqno, 3); delivered = true; break; } thread::sleep(Duration::from_millis(200)); } assert!(delivered, "failed to reconnect within 5s"); }