tidaldb/tidal/src/wal/checkpoint.rs
jordan 29400d48db feat: implement Milestone 1 phases 1-3 — schema, WAL, and storage layer
Implements the foundation of tidalDB's data pipeline:

**Phase 1 – Schema primitives**
- EntityId newtype (u64, big-endian ordering)
- SignalTypeDefinition with pre-computed decay λ, deduped/sorted windows
- SchemaBuilder with full constraint validation (duplicates, identifiers,
  half-life, windows, velocity)
- LumenError wrapping all subsystems with required From impls

**Phase 2 – Write-Ahead Log**
- Length-prefixed, BLAKE3-protected entry format
- Group-commit writer (batch up to 100 events / 10 ms)
- Double-buffered content-hash deduplication
- Checkpoint, truncation, and crash-recovery with full replay
- Integration, property, and UAT tests (incl. 5,500-event deterministic UAT)
- Proptest coverage scaled to 10 000 events/run (was ≤500) to meet
  acceptance criterion; cases reduced 100→10 to keep runtime comparable

**Phase 3 – Storage engine**
- StorageEngine trait (get/put/delete/scan/batch/flush)
- Key encoding: [EntityId][0x00][Tag][suffix] with ordering/prefix helpers
- InMemoryBackend (BTreeMap + RwLock)
- FjallStorage with three isolated keyspaces and atomic batch helper
- Property tests for key ordering and round-trip correctness

Also adds planning docs for phases 4-5, research docs, architecture
overview, and roadmap updates.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-02-20 16:43:24 -07:00

175 lines
6.3 KiB
Rust

use std::fs;
use std::path::Path;
use super::error::WalError;
/// File name for the checkpoint metadata.
const CHECKPOINT_FILE: &str = "checkpoint.meta";
/// Temporary file used for atomic write.
const CHECKPOINT_TMP: &str = "checkpoint.meta.tmp";
/// Manages checkpoint metadata for the WAL.
///
/// A checkpoint marks the sequence number through which all signal events
/// have been materialized to durable storage. On recovery, the WAL only
/// needs to replay events after the checkpoint.
///
/// Checkpoint writes are atomic: write to a temp file, fsync, then rename.
pub struct CheckpointManager;
impl CheckpointManager {
/// Write a checkpoint with the given sequence number and timestamp.
///
/// Uses write-to-temp-then-rename for atomicity on POSIX systems.
///
/// # Errors
///
/// Returns `WalError::Io` on filesystem failure.
pub fn write(dir: &Path, seq: u64, ts: u64) -> Result<(), WalError> {
let content = format!("seq={seq}\nts={ts}\n");
let tmp_path = dir.join(CHECKPOINT_TMP);
let final_path = dir.join(CHECKPOINT_FILE);
fs::write(&tmp_path, content.as_bytes())?;
// fsync the temp file to ensure contents are durable before rename
let file = fs::File::open(&tmp_path)?;
file.sync_all()?;
drop(file);
// Atomic rename (POSIX guarantees)
fs::rename(&tmp_path, &final_path)?;
// Fsync the directory to ensure the rename (directory entry update)
// is durable. Without this, a crash after rename but before the
// directory metadata is flushed could lose the checkpoint file.
let dir_fd = fs::File::open(dir)?;
dir_fd.sync_all()?;
Ok(())
}
/// Read the checkpoint metadata.
///
/// Returns `None` if the checkpoint file does not exist (fresh WAL).
/// Returns `Some((seq, ts))` on success.
///
/// # Errors
///
/// Returns `WalError::Io` on read failure, or `WalError::Corruption`
/// if the file exists but cannot be parsed.
pub fn read(dir: &Path) -> Result<Option<(u64, u64)>, WalError> {
let path = dir.join(CHECKPOINT_FILE);
let content = match fs::read_to_string(&path) {
Ok(c) => c,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(WalError::Io(e)),
};
let mut seq: Option<u64> = None;
let mut ts: Option<u64> = None;
for line in content.lines() {
let line = line.trim();
if line.is_empty() {
continue;
}
if let Some(val) = line.strip_prefix("seq=") {
seq = Some(val.parse::<u64>().map_err(|_| WalError::Corruption {
message: format!("invalid seq value in checkpoint: '{val}'"),
})?);
} else if let Some(val) = line.strip_prefix("ts=") {
ts = Some(val.parse::<u64>().map_err(|_| WalError::Corruption {
message: format!("invalid ts value in checkpoint: '{val}'"),
})?);
}
}
match (seq, ts) {
(Some(s), Some(t)) => Ok(Some((s, t))),
_ => Err(WalError::Corruption {
message: "checkpoint file missing seq or ts field".into(),
}),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn write_and_read_roundtrip() {
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
CheckpointManager::write(dir.path(), 1000, 2_000_000_000).expect("write should succeed");
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
assert_eq!(result, Some((1000, 2_000_000_000)));
}
#[test]
fn read_missing_returns_none() {
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
assert_eq!(result, None);
}
#[test]
fn overwrite_updates_values() {
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
CheckpointManager::write(dir.path(), 100, 200).expect("write should succeed");
CheckpointManager::write(dir.path(), 500, 600).expect("write should succeed");
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
assert_eq!(result, Some((500, 600)));
}
#[test]
fn corrupt_file_returns_error() {
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
let path = dir.path().join("checkpoint.meta");
fs::write(&path, "garbage data").expect("write should succeed");
let result = CheckpointManager::read(dir.path());
assert!(result.is_err());
}
#[test]
fn partial_file_returns_error() {
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
let path = dir.path().join("checkpoint.meta");
fs::write(&path, "seq=100\n").expect("write should succeed"); // missing ts
let result = CheckpointManager::read(dir.path());
assert!(result.is_err());
}
#[test]
fn max_u64_values() {
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
CheckpointManager::write(dir.path(), u64::MAX, u64::MAX).expect("write should succeed");
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
assert_eq!(result, Some((u64::MAX, u64::MAX)));
}
#[test]
fn zero_values() {
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
CheckpointManager::write(dir.path(), 0, 0).expect("write should succeed");
let result = CheckpointManager::read(dir.path()).expect("read should succeed");
assert_eq!(result, Some((0, 0)));
}
mod proptests {
use super::*;
use proptest::prelude::*;
proptest! {
#[test]
fn roundtrip(seq: u64, ts: u64) {
let dir = tempfile::tempdir().expect("tempdir creation should succeed");
CheckpointManager::write(dir.path(), seq, ts)?;
let result = CheckpointManager::read(dir.path())?;
prop_assert_eq!(result, Some((seq, ts)));
}
}
}
}