WAL segment format: 8-byte TSEG header (magic + version byte + 3 reserved)
prepended to every new segment. Legacy headerless segments (m0-m11p3) read
as implicit v0 — no migration. Unknown magic/version surfaces as
WalError::SegmentFormatUnknown at open time; foreign files are never
repaired or truncated (fixes the silent data-loss path from the p3 rollout
incident where torn-tail repair zeroed a follower's unreadable segments).
Catch-up transport: FAILED_PRECONDITION ("snapshot required") and stream
errors that skip the shard now arm a timer retry (re-arm-on-skip is the
load-bearing liveness fix — without it a skipped pull never re-fires and
the follower stays permanently behind). Single retry pending per shard;
CatchupRunner owns the Arc'd state shared between the retry tasks and the
transport. Test: tidal-net/tests/catchup_retry.rs covers the retry path.
Stress: k8s stress-job-t2a/t2b yaml + ops/stress-test-p3-t2 runbook.
376 lines
13 KiB
Rust
376 lines
13 KiB
Rust
use super::*;
|
|
|
|
#[test]
|
|
fn segment_filename_format() {
|
|
assert_eq!(
|
|
segment_filename(ShardId::SINGLE, 1),
|
|
"wal-00000000000000000001.seg"
|
|
);
|
|
assert_eq!(
|
|
segment_filename(ShardId::SINGLE, 0),
|
|
"wal-00000000000000000000.seg"
|
|
);
|
|
assert_eq!(
|
|
segment_filename(ShardId::SINGLE, u64::MAX),
|
|
"wal-18446744073709551615.seg"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_segment_seq_valid() {
|
|
assert_eq!(parse_segment_seq("wal-00000000000000000001.seg"), Some(1));
|
|
assert_eq!(parse_segment_seq("wal-00000000000000000000.seg"), Some(0));
|
|
}
|
|
|
|
#[test]
|
|
fn parse_segment_seq_invalid() {
|
|
assert_eq!(parse_segment_seq("not-a-segment.txt"), None);
|
|
assert_eq!(parse_segment_seq("wal-.seg"), None);
|
|
assert_eq!(parse_segment_seq("wal-abc.seg"), None);
|
|
assert_eq!(parse_segment_seq(""), None);
|
|
}
|
|
|
|
#[test]
|
|
fn write_and_check_size() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let mut writer =
|
|
SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024).expect("open should succeed");
|
|
// A fresh segment starts at the versioned header's size (m11p4).
|
|
assert_eq!(writer.current_size(), SEGMENT_HEADER_SIZE as u64);
|
|
|
|
let data = [0xABu8; 100];
|
|
writer
|
|
.write_batch_bytes(&data)
|
|
.expect("write should succeed");
|
|
assert_eq!(writer.current_size(), SEGMENT_HEADER_SIZE as u64 + 100);
|
|
}
|
|
|
|
#[test]
|
|
fn rotation_creates_new_file() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let mut writer =
|
|
SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 100).expect("open should succeed");
|
|
|
|
writer
|
|
.write_batch_bytes(&[0u8; 50])
|
|
.expect("write should succeed");
|
|
writer.rotate(100).expect("rotate should succeed");
|
|
|
|
// The rotated-to segment starts at its own versioned header.
|
|
assert_eq!(writer.current_size(), SEGMENT_HEADER_SIZE as u64);
|
|
assert_eq!(writer.first_seq(), 100);
|
|
|
|
let segments = list_segments(dir.path()).expect("list should succeed");
|
|
assert_eq!(segments.len(), 2);
|
|
assert_eq!(segments[0].0, 1);
|
|
assert_eq!(segments[1].0, 100);
|
|
}
|
|
|
|
#[test]
|
|
fn needs_rotation_threshold() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let mut writer =
|
|
SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 100).expect("open should succeed");
|
|
assert!(!writer.needs_rotation());
|
|
|
|
writer
|
|
.write_batch_bytes(&[0u8; 100])
|
|
.expect("write should succeed");
|
|
assert!(writer.needs_rotation());
|
|
}
|
|
|
|
#[test]
|
|
fn list_segments_sorted() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
// Create segments out of order
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 300, 1024);
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 100, 1024);
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 200, 1024);
|
|
|
|
let segments = list_segments(dir.path()).expect("list should succeed");
|
|
assert_eq!(segments.len(), 3);
|
|
assert_eq!(segments[0].0, 100);
|
|
assert_eq!(segments[1].0, 200);
|
|
assert_eq!(segments[2].0, 300);
|
|
}
|
|
|
|
#[test]
|
|
fn list_segments_empty_dir() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let segments = list_segments(dir.path()).expect("list should succeed");
|
|
assert!(segments.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn list_segments_ignores_non_segment_files() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
fs::write(dir.path().join("checkpoint.meta"), "seq=1\nts=1\n").expect("write should succeed");
|
|
fs::write(dir.path().join("random.txt"), "hello").expect("write should succeed");
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024);
|
|
|
|
let segments = list_segments(dir.path()).expect("list should succeed");
|
|
assert_eq!(segments.len(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn delete_segments_before_removes_older() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024);
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 100, 1024);
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 200, 1024);
|
|
|
|
let deleted = delete_segments_before(dir.path(), 200).expect("delete should succeed");
|
|
assert_eq!(deleted, 2);
|
|
|
|
let remaining = list_segments(dir.path()).expect("list should succeed");
|
|
assert_eq!(remaining.len(), 1);
|
|
assert_eq!(remaining[0].0, 200);
|
|
}
|
|
|
|
#[test]
|
|
fn delete_segments_before_none_to_delete() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 100, 1024);
|
|
|
|
let deleted = delete_segments_before(dir.path(), 50).expect("delete should succeed");
|
|
assert_eq!(deleted, 0);
|
|
}
|
|
|
|
#[test]
|
|
fn sync_does_not_error() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let writer =
|
|
SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024).expect("open should succeed");
|
|
writer.sync().expect("sync should succeed");
|
|
}
|
|
|
|
// -- Multi-shard naming tests --
|
|
|
|
#[test]
|
|
fn segment_filename_multi_shard() {
|
|
let name = segment_filename(ShardId(3), 42);
|
|
assert_eq!(name, "wal-s00003-00000000000000000042.seg");
|
|
}
|
|
|
|
#[test]
|
|
fn segment_filename_single_shard_backward_compat() {
|
|
// ShardId::SINGLE retains old format exactly
|
|
assert_eq!(
|
|
segment_filename(ShardId::SINGLE, 1),
|
|
"wal-00000000000000000001.seg"
|
|
);
|
|
assert_eq!(
|
|
segment_filename(ShardId(0), 42),
|
|
"wal-00000000000000000042.seg"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_segment_filename_both_formats() {
|
|
assert_eq!(
|
|
parse_segment_filename("wal-00000000000000000001.seg"),
|
|
Some((ShardId::SINGLE, 1))
|
|
);
|
|
assert_eq!(
|
|
parse_segment_filename("wal-s00003-00000000000000000042.seg"),
|
|
Some((ShardId(3), 42))
|
|
);
|
|
assert_eq!(parse_segment_filename("not-a-segment.txt"), None);
|
|
}
|
|
|
|
#[test]
|
|
fn parse_segment_filename_edge_cases() {
|
|
// Empty filename
|
|
assert_eq!(parse_segment_filename(""), None);
|
|
// Missing .seg suffix
|
|
assert_eq!(parse_segment_filename("wal-00000000000000000001"), None);
|
|
// Shard format but missing seq
|
|
assert_eq!(parse_segment_filename("wal-s00003.seg"), None);
|
|
// Non-numeric shard
|
|
assert_eq!(
|
|
parse_segment_filename("wal-sXYZ-00000000000000000001.seg"),
|
|
None
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn list_segments_for_shard_filters_correctly() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
|
|
// Create single-shard segment files (v1 format)
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024);
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 100, 1024);
|
|
|
|
// Create multi-shard segment files (v2 format) manually
|
|
let s3_name = segment_filename(ShardId(3), 50);
|
|
fs::write(dir.path().join(s3_name), []).expect("write should succeed");
|
|
let s3_name2 = segment_filename(ShardId(3), 200);
|
|
fs::write(dir.path().join(s3_name2), []).expect("write should succeed");
|
|
let s5_name = segment_filename(ShardId(5), 75);
|
|
fs::write(dir.path().join(s5_name), []).expect("write should succeed");
|
|
|
|
// list_segments returns ALL segment files (backward compat)
|
|
let all = list_segments(dir.path()).expect("list should succeed");
|
|
assert_eq!(all.len(), 5);
|
|
|
|
// list_segments_for_shard returns only matching shard
|
|
let single = list_segments_for_shard(dir.path(), ShardId::SINGLE).expect("list should succeed");
|
|
assert_eq!(single.len(), 2);
|
|
assert_eq!(single[0].0, 1);
|
|
assert_eq!(single[1].0, 100);
|
|
|
|
let shard3 = list_segments_for_shard(dir.path(), ShardId(3)).expect("list should succeed");
|
|
assert_eq!(shard3.len(), 2);
|
|
assert_eq!(shard3[0].0, 50);
|
|
assert_eq!(shard3[1].0, 200);
|
|
|
|
let shard5 = list_segments_for_shard(dir.path(), ShardId(5)).expect("list should succeed");
|
|
assert_eq!(shard5.len(), 1);
|
|
assert_eq!(shard5[0].0, 75);
|
|
|
|
// Non-existent shard returns empty
|
|
let empty = list_segments_for_shard(dir.path(), ShardId(99)).expect("list should succeed");
|
|
assert!(empty.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn list_segments_for_shard_missing_dir() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let missing = dir.path().join("does-not-exist");
|
|
let result = list_segments_for_shard(&missing, ShardId::SINGLE)
|
|
.expect("should handle missing dir gracefully");
|
|
assert!(result.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn segment_writer_stores_shard_id() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let writer = SegmentWriter::open(dir.path(), ShardId(7), 1, 1024).expect("open should succeed");
|
|
assert_eq!(writer.shard_id(), ShardId(7));
|
|
|
|
// Verify the on-disk filename uses the v2 format.
|
|
let expected = segment_filename(ShardId(7), 1);
|
|
assert!(
|
|
dir.path().join(&expected).exists(),
|
|
"expected file {expected} to exist in WAL dir"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn rotation_preserves_shard_id_in_filename() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let mut writer =
|
|
SegmentWriter::open(dir.path(), ShardId(3), 1, 100).expect("open should succeed");
|
|
writer
|
|
.write_batch_bytes(&[0u8; 50])
|
|
.expect("write should succeed");
|
|
writer.rotate(100).expect("rotate should succeed");
|
|
|
|
let seg1 = segment_filename(ShardId(3), 1);
|
|
let seg2 = segment_filename(ShardId(3), 100);
|
|
assert!(
|
|
dir.path().join(&seg1).exists(),
|
|
"first segment {seg1} must exist"
|
|
);
|
|
assert!(
|
|
dir.path().join(&seg2).exists(),
|
|
"rotated segment {seg2} must exist"
|
|
);
|
|
assert_eq!(writer.shard_id(), ShardId(3));
|
|
}
|
|
|
|
// ── Segment format versioning (m11p4) ──────────────────────────────────────
|
|
|
|
#[test]
|
|
fn new_segment_leads_with_versioned_header() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024).expect("open");
|
|
|
|
let bytes = fs::read(dir.path().join(segment_filename(ShardId::SINGLE, 1))).expect("read");
|
|
assert_eq!(
|
|
bytes,
|
|
segment_header(),
|
|
"fresh segment = exactly the header"
|
|
);
|
|
assert_eq!(&bytes[..4], b"TSEG");
|
|
assert_eq!(bytes[4], SEGMENT_FORMAT_VERSION);
|
|
assert_eq!(&bytes[5..8], &[0, 0, 0], "reserved bytes must be zero");
|
|
}
|
|
|
|
#[test]
|
|
fn reopening_existing_segment_writes_no_second_header() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
{
|
|
let mut writer =
|
|
SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024).expect("open should succeed");
|
|
writer.write_batch_bytes(&[0xAB; 10]).expect("write");
|
|
}
|
|
let writer =
|
|
SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024).expect("reopen should succeed");
|
|
assert_eq!(
|
|
writer.current_size(),
|
|
SEGMENT_HEADER_SIZE as u64 + 10,
|
|
"reopen must append, not re-write the header"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn reopening_legacy_headerless_segment_stays_legacy() {
|
|
// A pre-m11p4 file (batch bytes at offset 0): appends must NOT inject a
|
|
// header mid-file — the segment keeps its legacy layout.
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let path = dir.path().join(segment_filename(ShardId::SINGLE, 1));
|
|
fs::write(&path, [0xCD; 32]).expect("write should succeed");
|
|
|
|
let writer =
|
|
SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024).expect("open should succeed");
|
|
assert_eq!(writer.current_size(), 32);
|
|
let bytes = fs::read(&path).expect("read");
|
|
assert_eq!(&bytes[..4], &[0xCD; 4], "legacy leading bytes untouched");
|
|
}
|
|
|
|
#[test]
|
|
fn unparseable_seg_filename_is_format_unknown() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let _ = SegmentWriter::open(dir.path(), ShardId::SINGLE, 1, 1024).expect("open");
|
|
fs::write(dir.path().join("wal-mystery.seg"), [0u8; 8]).expect("write should succeed");
|
|
|
|
assert!(
|
|
matches!(
|
|
list_segments(dir.path()),
|
|
Err(WalError::SegmentFormatUnknown { .. })
|
|
),
|
|
"an unparseable .seg filename must refuse loudly, not read as absent"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
list_segments_for_shard(dir.path(), ShardId::SINGLE),
|
|
Err(WalError::SegmentFormatUnknown { .. })
|
|
),
|
|
"the per-shard listing must refuse identically"
|
|
);
|
|
}
|
|
|
|
mod proptests {
|
|
use proptest::prelude::*;
|
|
|
|
use super::*;
|
|
|
|
proptest! {
|
|
#[test]
|
|
fn filename_roundtrip(seq: u64) {
|
|
let name = segment_filename(ShardId::SINGLE, seq);
|
|
let parsed = parse_segment_seq(&name);
|
|
prop_assert_eq!(parsed, Some(seq));
|
|
}
|
|
|
|
#[test]
|
|
fn shard_filename_roundtrip(shard_id in 0u16..100u16, seq in proptest::num::u64::ANY) {
|
|
let shard = ShardId(shard_id);
|
|
let name = segment_filename(shard, seq);
|
|
let parsed = parse_segment_filename(&name);
|
|
prop_assert_eq!(parsed, Some((shard, seq)));
|
|
}
|
|
}
|
|
}
|