Implements the foundation of tidalDB's data pipeline: **Phase 1 – Schema primitives** - EntityId newtype (u64, big-endian ordering) - SignalTypeDefinition with pre-computed decay λ, deduped/sorted windows - SchemaBuilder with full constraint validation (duplicates, identifiers, half-life, windows, velocity) - LumenError wrapping all subsystems with required From impls **Phase 2 – Write-Ahead Log** - Length-prefixed, BLAKE3-protected entry format - Group-commit writer (batch up to 100 events / 10 ms) - Double-buffered content-hash deduplication - Checkpoint, truncation, and crash-recovery with full replay - Integration, property, and UAT tests (incl. 5,500-event deterministic UAT) - Proptest coverage scaled to 10 000 events/run (was ≤500) to meet acceptance criterion; cases reduced 100→10 to keep runtime comparable **Phase 3 – Storage engine** - StorageEngine trait (get/put/delete/scan/batch/flush) - Key encoding: [EntityId][0x00][Tag][suffix] with ordering/prefix helpers - InMemoryBackend (BTreeMap + RwLock) - FjallStorage with three isolated keyspaces and atomic batch helper - Property tests for key ordering and round-trip correctness Also adds planning docs for phases 4-5, research docs, architecture overview, and roadmap updates. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
175 lines
6.3 KiB
Rust
175 lines
6.3 KiB
Rust
use std::fs;
|
|
use std::path::Path;
|
|
|
|
use super::error::WalError;
|
|
|
|
/// File name for the checkpoint metadata.
|
|
const CHECKPOINT_FILE: &str = "checkpoint.meta";
|
|
|
|
/// Temporary file used for atomic write.
|
|
const CHECKPOINT_TMP: &str = "checkpoint.meta.tmp";
|
|
|
|
/// Manages checkpoint metadata for the WAL.
|
|
///
|
|
/// A checkpoint marks the sequence number through which all signal events
|
|
/// have been materialized to durable storage. On recovery, the WAL only
|
|
/// needs to replay events after the checkpoint.
|
|
///
|
|
/// Checkpoint writes are atomic: write to a temp file, fsync, then rename.
|
|
pub struct CheckpointManager;
|
|
|
|
impl CheckpointManager {
|
|
/// Write a checkpoint with the given sequence number and timestamp.
|
|
///
|
|
/// Uses write-to-temp-then-rename for atomicity on POSIX systems.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns `WalError::Io` on filesystem failure.
|
|
pub fn write(dir: &Path, seq: u64, ts: u64) -> Result<(), WalError> {
|
|
let content = format!("seq={seq}\nts={ts}\n");
|
|
let tmp_path = dir.join(CHECKPOINT_TMP);
|
|
let final_path = dir.join(CHECKPOINT_FILE);
|
|
|
|
fs::write(&tmp_path, content.as_bytes())?;
|
|
|
|
// fsync the temp file to ensure contents are durable before rename
|
|
let file = fs::File::open(&tmp_path)?;
|
|
file.sync_all()?;
|
|
drop(file);
|
|
|
|
// Atomic rename (POSIX guarantees)
|
|
fs::rename(&tmp_path, &final_path)?;
|
|
|
|
// Fsync the directory to ensure the rename (directory entry update)
|
|
// is durable. Without this, a crash after rename but before the
|
|
// directory metadata is flushed could lose the checkpoint file.
|
|
let dir_fd = fs::File::open(dir)?;
|
|
dir_fd.sync_all()?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Read the checkpoint metadata.
|
|
///
|
|
/// Returns `None` if the checkpoint file does not exist (fresh WAL).
|
|
/// Returns `Some((seq, ts))` on success.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns `WalError::Io` on read failure, or `WalError::Corruption`
|
|
/// if the file exists but cannot be parsed.
|
|
pub fn read(dir: &Path) -> Result<Option<(u64, u64)>, WalError> {
|
|
let path = dir.join(CHECKPOINT_FILE);
|
|
let content = match fs::read_to_string(&path) {
|
|
Ok(c) => c,
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
|
Err(e) => return Err(WalError::Io(e)),
|
|
};
|
|
|
|
let mut seq: Option<u64> = None;
|
|
let mut ts: Option<u64> = None;
|
|
|
|
for line in content.lines() {
|
|
let line = line.trim();
|
|
if line.is_empty() {
|
|
continue;
|
|
}
|
|
if let Some(val) = line.strip_prefix("seq=") {
|
|
seq = Some(val.parse::<u64>().map_err(|_| WalError::Corruption {
|
|
message: format!("invalid seq value in checkpoint: '{val}'"),
|
|
})?);
|
|
} else if let Some(val) = line.strip_prefix("ts=") {
|
|
ts = Some(val.parse::<u64>().map_err(|_| WalError::Corruption {
|
|
message: format!("invalid ts value in checkpoint: '{val}'"),
|
|
})?);
|
|
}
|
|
}
|
|
|
|
match (seq, ts) {
|
|
(Some(s), Some(t)) => Ok(Some((s, t))),
|
|
_ => Err(WalError::Corruption {
|
|
message: "checkpoint file missing seq or ts field".into(),
|
|
}),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn write_and_read_roundtrip() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
CheckpointManager::write(dir.path(), 1000, 2_000_000_000).expect("write should succeed");
|
|
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
|
|
assert_eq!(result, Some((1000, 2_000_000_000)));
|
|
}
|
|
|
|
#[test]
|
|
fn read_missing_returns_none() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
|
|
assert_eq!(result, None);
|
|
}
|
|
|
|
#[test]
|
|
fn overwrite_updates_values() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
CheckpointManager::write(dir.path(), 100, 200).expect("write should succeed");
|
|
CheckpointManager::write(dir.path(), 500, 600).expect("write should succeed");
|
|
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
|
|
assert_eq!(result, Some((500, 600)));
|
|
}
|
|
|
|
#[test]
|
|
fn corrupt_file_returns_error() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let path = dir.path().join("checkpoint.meta");
|
|
fs::write(&path, "garbage data").expect("write should succeed");
|
|
let result = CheckpointManager::read(dir.path());
|
|
assert!(result.is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn partial_file_returns_error() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
let path = dir.path().join("checkpoint.meta");
|
|
fs::write(&path, "seq=100\n").expect("write should succeed"); // missing ts
|
|
let result = CheckpointManager::read(dir.path());
|
|
assert!(result.is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn max_u64_values() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
CheckpointManager::write(dir.path(), u64::MAX, u64::MAX).expect("write should succeed");
|
|
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
|
|
assert_eq!(result, Some((u64::MAX, u64::MAX)));
|
|
}
|
|
|
|
#[test]
|
|
fn zero_values() {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
CheckpointManager::write(dir.path(), 0, 0).expect("write should succeed");
|
|
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
|
|
assert_eq!(result, Some((0, 0)));
|
|
}
|
|
|
|
mod proptests {
|
|
use super::*;
|
|
use proptest::prelude::*;
|
|
|
|
proptest! {
|
|
#[test]
|
|
fn roundtrip(seq: u64, ts: u64) {
|
|
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
|
|
CheckpointManager::write(dir.path(), seq, ts)?;
|
|
let result = CheckpointManager::read(dir.path())?;
|
|
prop_assert_eq!(result, Some((seq, ts)));
|
|
}
|
|
}
|
|
}
|
|
}
|