fe711870be
Delivers the distributed fabric's network layer and HTTP cluster surface: **m8p7: tidal-net crate (gRPC transport)** - GrpcTransport implementing Transport trait via tonic 0.12 - Per-peer circuit breaker (Closed/Open/HalfOpen), mutual TLS via rustls - Boxed error types (clippy-clean), graceful mutex recovery, debug_assert against calling block_on from tokio context - Proto: WalShipping service (ShipSegment, StreamSegments stub, Heartbeat) - 19 tests: contract, mTLS, reconnection, multi-node UAT, benchmarks **m8p8: cluster subcommand + HTTP routes** - ClusterState wrapping SimulatedCluster with region name mapping - Routes: /health, /cluster/status, /cluster/promote, /partition, /heal - Data routes: /items, /embeddings, /signals, /feed, /search (region-aware) - Ranking profiles wired through ClusterConfig to all cluster nodes - Topology YAML config, docker/cluster/Dockerfile (ENTRYPOINT+CMD, non-root) **m8p9: scatter-gather query routing** - Entity-sharded writes via Knuth multiplicative hash - Scatter-gather RETRIEVE and SEARCH with deadline propagation (50ms-5ms) - Partial failure: degraded=true with unavailable_shards metadata - 6 tests: distribution, determinism, multi-shard retrieve, degraded partial results, deadline propagation, scatter-gather search **m8p10: gRPC transport integration tests** - 8 tests over real gRPC: replication convergence, idempotent replay, mixed signals, 3-node fan-out, partition/heal, degraded follower, perf - Documented as tier-2 (in-process+gRPC); tier-3 multi-process pending
106 lines
3.2 KiB
Rust
106 lines
3.2 KiB
Rust
//! Tests that GrpcTransport reconnects after a peer restarts.
|
|
|
|
use std::collections::HashMap;
|
|
use std::net::SocketAddr;
|
|
use std::thread;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use tidaldb::replication::WalSegmentId;
|
|
use tidaldb::replication::shard::{RegionId, ShardId};
|
|
use tidaldb::replication::transport::{Transport, WalSegmentPayload};
|
|
|
|
use tidal_net::GrpcTransport;
|
|
use tidal_net::config::GrpcTransportConfig;
|
|
|
|
fn free_addr() -> SocketAddr {
|
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
|
listener.local_addr().unwrap()
|
|
}
|
|
|
|
fn make_config(
|
|
shard: ShardId,
|
|
listen: SocketAddr,
|
|
peers: HashMap<ShardId, SocketAddr>,
|
|
) -> GrpcTransportConfig {
|
|
GrpcTransportConfig {
|
|
local_shard: shard,
|
|
listen_addr: listen,
|
|
peers,
|
|
insecure: true,
|
|
// Shorter circuit breaker for testing.
|
|
circuit_breaker_threshold: 3,
|
|
circuit_breaker_reset: Duration::from_millis(500),
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
fn make_payload(seqno: u64) -> WalSegmentPayload {
|
|
WalSegmentPayload {
|
|
id: WalSegmentId::new(RegionId::SINGLE, ShardId(0), seqno),
|
|
bytes: vec![0xCD; 50],
|
|
event_count: 1,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn reconnects_after_peer_restart() {
|
|
let addr_sender = free_addr();
|
|
let addr_receiver = free_addr();
|
|
|
|
// Start sender (shard 0) that knows about receiver (shard 1).
|
|
let sender_config = make_config(
|
|
ShardId(0),
|
|
addr_sender,
|
|
HashMap::from([(ShardId(1), addr_receiver)]),
|
|
);
|
|
let sender = GrpcTransport::new(sender_config).expect("sender");
|
|
|
|
// Start receiver (shard 1).
|
|
let receiver_config = make_config(
|
|
ShardId(1),
|
|
addr_receiver,
|
|
HashMap::from([(ShardId(0), addr_sender)]),
|
|
);
|
|
let receiver = GrpcTransport::new(receiver_config).expect("receiver");
|
|
thread::sleep(Duration::from_millis(100));
|
|
|
|
// Send successfully.
|
|
sender.send_segment(ShardId(1), make_payload(1)).unwrap();
|
|
let received = receiver.recv_segment().unwrap();
|
|
assert_eq!(received.id.seqno, 1);
|
|
|
|
// Drop the receiver to simulate crash.
|
|
drop(receiver);
|
|
thread::sleep(Duration::from_millis(200));
|
|
|
|
// Sends may fail now (peer is down). That's expected.
|
|
let _ = sender.send_segment(ShardId(1), make_payload(2));
|
|
|
|
// Restart receiver on the same address.
|
|
let receiver_config2 = make_config(
|
|
ShardId(1),
|
|
addr_receiver,
|
|
HashMap::from([(ShardId(0), addr_sender)]),
|
|
);
|
|
let receiver2 = GrpcTransport::new(receiver_config2).expect("receiver2");
|
|
thread::sleep(Duration::from_millis(200));
|
|
|
|
// Wait for circuit breaker to allow probes again.
|
|
thread::sleep(Duration::from_millis(500));
|
|
|
|
// Should reconnect and deliver.
|
|
let start = Instant::now();
|
|
let mut delivered = false;
|
|
while start.elapsed() < Duration::from_secs(5) {
|
|
if sender.send_segment(ShardId(1), make_payload(3)).is_ok() {
|
|
if let Some(seg) = receiver2.recv_segment() {
|
|
assert_eq!(seg.id.seqno, 3);
|
|
delivered = true;
|
|
break;
|
|
}
|
|
}
|
|
thread::sleep(Duration::from_millis(200));
|
|
}
|
|
assert!(delivered, "failed to reconnect within 5s");
|
|
}
|