f4cfd6c81f
Milestone 8 (phases 1-4): - Shard-aware WAL segment naming, BatchHeader v2, ShardRouter - Transport trait, InProcessTransport, WalShipper, FollowerDb - HLC, PNCounter, LWWRegister, CrdtSignalState, ReconciliationEngine - Session replication bridge with SeqNo/HWM, idempotency store Forage application: - Multi-source discovery engine with MAB exploration - Embedding-based label system, server handlers, UI refresh Other: - QUICKSTART.md, README.md, milestone-8 planning docs - Hard negative union semantics, RLHF export enhancements - Recovery benchmark and visibility test expansions - Split 8 oversized source files per CODING_GUIDELINES §9 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
582 lines
18 KiB
Rust
582 lines
18 KiB
Rust
// ── Session journal record types ────────────────────────────────────────────
|
|
|
|
/// Record type discriminant for session start events.
|
|
pub const SESSION_RECORD_START: u8 = 0x01;
|
|
/// Record type discriminant for session signal events.
|
|
pub const SESSION_RECORD_SIGNAL: u8 = 0x02;
|
|
/// Record type discriminant for session close events.
|
|
pub const SESSION_RECORD_CLOSE: u8 = 0x03;
|
|
|
|
/// Monotonic sequence number for session writes.
|
|
///
|
|
/// Incremented once per session write operation (preference signal,
|
|
/// annotation, search query). Used by the receiver to enforce idempotent
|
|
/// replay: a write with seqno <= hwm is a no-op.
|
|
#[derive(
|
|
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize,
|
|
)]
|
|
pub struct SessionSeqNo(pub u64);
|
|
|
|
impl SessionSeqNo {
|
|
pub const ZERO: Self = Self(0);
|
|
#[must_use]
|
|
pub const fn next(self) -> Self {
|
|
Self(self.0 + 1)
|
|
}
|
|
}
|
|
|
|
impl std::fmt::Display for SessionSeqNo {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
write!(f, "ssn:{}", self.0)
|
|
}
|
|
}
|
|
|
|
/// A session event decoded from the session journal.
|
|
///
|
|
/// These events are stored in a separate append-only file (`sessions.log`)
|
|
/// alongside the signal WAL. They are used to restore active sessions on
|
|
/// crash recovery.
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub enum SessionWalEvent {
|
|
/// A session was started.
|
|
Start {
|
|
session_id: u64,
|
|
user_id: u64,
|
|
started_at_ns: u64,
|
|
agent_id: String,
|
|
policy_name: String,
|
|
},
|
|
/// A signal was written within a session.
|
|
Signal {
|
|
session_id: u64,
|
|
entity_id: u64,
|
|
weight: f32,
|
|
ts_ns: u64,
|
|
signal_name: String,
|
|
annotation: Option<String>,
|
|
// m8p4 extensions (default None for legacy events)
|
|
session_seqno: Option<SessionSeqNo>,
|
|
idempotency_key: Option<u128>,
|
|
},
|
|
/// A session was closed.
|
|
Close { session_id: u64 },
|
|
}
|
|
|
|
/// Encode a session event to bytes for the session journal.
|
|
///
|
|
/// Format: `[len: u32 LE][type: u8][payload bytes]`
|
|
///
|
|
/// **Start payload**: `[session_id: u64 LE][user_id: u64 LE][started_at_ns: u64 LE]`
|
|
/// `[agent_id_len: u16 LE][agent_id: bytes][policy_name_len: u16 LE][policy_name: bytes]`
|
|
///
|
|
/// **Signal payload**: `[session_id: u64 LE][entity_id: u64 LE][weight: f32 LE][ts_ns: u64 LE]`
|
|
/// `[signal_name_len: u16 LE][signal_name: bytes][has_annotation: u8]`
|
|
/// `[if has_annotation: annotation_len: u16 LE, annotation: bytes]`
|
|
///
|
|
/// **Close payload**: `[session_id: u64 LE]`
|
|
#[must_use]
|
|
#[allow(clippy::cast_possible_truncation)]
|
|
pub fn encode_session_event(event: &SessionWalEvent) -> Vec<u8> {
|
|
// Encode the payload first, then prepend the length+type header.
|
|
let mut payload = Vec::new();
|
|
match event {
|
|
SessionWalEvent::Start {
|
|
session_id,
|
|
user_id,
|
|
started_at_ns,
|
|
agent_id,
|
|
policy_name,
|
|
} => {
|
|
payload.push(SESSION_RECORD_START);
|
|
payload.extend_from_slice(&session_id.to_le_bytes());
|
|
payload.extend_from_slice(&user_id.to_le_bytes());
|
|
payload.extend_from_slice(&started_at_ns.to_le_bytes());
|
|
payload.extend_from_slice(&(agent_id.len() as u16).to_le_bytes());
|
|
payload.extend_from_slice(agent_id.as_bytes());
|
|
payload.extend_from_slice(&(policy_name.len() as u16).to_le_bytes());
|
|
payload.extend_from_slice(policy_name.as_bytes());
|
|
}
|
|
SessionWalEvent::Signal {
|
|
session_id,
|
|
entity_id,
|
|
weight,
|
|
ts_ns,
|
|
signal_name,
|
|
annotation,
|
|
session_seqno,
|
|
idempotency_key,
|
|
} => {
|
|
payload.push(SESSION_RECORD_SIGNAL);
|
|
payload.extend_from_slice(&session_id.to_le_bytes());
|
|
payload.extend_from_slice(&entity_id.to_le_bytes());
|
|
payload.extend_from_slice(&weight.to_le_bytes());
|
|
payload.extend_from_slice(&ts_ns.to_le_bytes());
|
|
payload.extend_from_slice(&(signal_name.len() as u16).to_le_bytes());
|
|
payload.extend_from_slice(signal_name.as_bytes());
|
|
match annotation {
|
|
Some(ann) => {
|
|
payload.push(1u8);
|
|
payload.extend_from_slice(&(ann.len() as u16).to_le_bytes());
|
|
payload.extend_from_slice(ann.as_bytes());
|
|
}
|
|
None => {
|
|
payload.push(0u8);
|
|
}
|
|
}
|
|
// m8p4: session_seqno (optional u64)
|
|
match session_seqno {
|
|
Some(seqno) => {
|
|
payload.push(1u8);
|
|
payload.extend_from_slice(&seqno.0.to_le_bytes());
|
|
}
|
|
None => {
|
|
payload.push(0u8);
|
|
}
|
|
}
|
|
// m8p4: idempotency_key (optional u128)
|
|
match idempotency_key {
|
|
Some(key) => {
|
|
payload.push(1u8);
|
|
payload.extend_from_slice(&key.to_le_bytes());
|
|
}
|
|
None => {
|
|
payload.push(0u8);
|
|
}
|
|
}
|
|
}
|
|
SessionWalEvent::Close { session_id } => {
|
|
payload.push(SESSION_RECORD_CLOSE);
|
|
payload.extend_from_slice(&session_id.to_le_bytes());
|
|
}
|
|
}
|
|
|
|
let len = payload.len() as u32;
|
|
let mut buf = Vec::with_capacity(4 + payload.len());
|
|
buf.extend_from_slice(&len.to_le_bytes());
|
|
buf.extend(payload);
|
|
buf
|
|
}
|
|
|
|
/// Decode all session events from a session journal file's contents.
|
|
///
|
|
/// Stops at the first truncated or malformed record. This is the correct
|
|
/// behavior for crash recovery: a torn write at the end of the file is
|
|
/// simply ignored.
|
|
#[must_use]
|
|
pub fn decode_session_events(bytes: &[u8]) -> Vec<SessionWalEvent> {
|
|
let mut events = Vec::new();
|
|
let mut pos = 0;
|
|
|
|
while pos + 4 <= bytes.len() {
|
|
let record_len =
|
|
u32::from_le_bytes([bytes[pos], bytes[pos + 1], bytes[pos + 2], bytes[pos + 3]])
|
|
as usize;
|
|
pos += 4;
|
|
|
|
if pos + record_len > bytes.len() || record_len == 0 {
|
|
// Truncated or zero-length record -- stop.
|
|
break;
|
|
}
|
|
|
|
let record_end = pos + record_len;
|
|
let record_type = bytes[pos];
|
|
pos += 1;
|
|
|
|
match record_type {
|
|
SESSION_RECORD_START => {
|
|
if let Some(event) = decode_start_record(bytes, &mut pos, record_end) {
|
|
events.push(event);
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
SESSION_RECORD_SIGNAL => {
|
|
if let Some(event) = decode_signal_record(bytes, &mut pos, record_end) {
|
|
events.push(event);
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
SESSION_RECORD_CLOSE => {
|
|
if pos + 8 > record_end {
|
|
break;
|
|
}
|
|
let session_id = read_u64_le(bytes, &mut pos);
|
|
events.push(SessionWalEvent::Close { session_id });
|
|
}
|
|
_ => {
|
|
// Unknown record type -- skip handled by `pos = record_end` below.
|
|
}
|
|
}
|
|
|
|
// Ensure pos is at the expected record boundary.
|
|
pos = record_end;
|
|
}
|
|
|
|
events
|
|
}
|
|
|
|
/// Helper: read a little-endian u64 from `bytes` at `*pos`, advancing `*pos`.
|
|
fn read_u64_le(bytes: &[u8], pos: &mut usize) -> u64 {
|
|
let v = u64::from_le_bytes([
|
|
bytes[*pos],
|
|
bytes[*pos + 1],
|
|
bytes[*pos + 2],
|
|
bytes[*pos + 3],
|
|
bytes[*pos + 4],
|
|
bytes[*pos + 5],
|
|
bytes[*pos + 6],
|
|
bytes[*pos + 7],
|
|
]);
|
|
*pos += 8;
|
|
v
|
|
}
|
|
|
|
/// Helper: read a little-endian u16 from `bytes` at `*pos`, advancing `*pos`.
|
|
fn read_u16_le(bytes: &[u8], pos: &mut usize) -> u16 {
|
|
let v = u16::from_le_bytes([bytes[*pos], bytes[*pos + 1]]);
|
|
*pos += 2;
|
|
v
|
|
}
|
|
|
|
/// Decode a Start record from the payload region.
|
|
fn decode_start_record(bytes: &[u8], pos: &mut usize, end: usize) -> Option<SessionWalEvent> {
|
|
if *pos + 24 > end {
|
|
return None;
|
|
}
|
|
let session_id = read_u64_le(bytes, pos);
|
|
let user_id = read_u64_le(bytes, pos);
|
|
let started_at_ns = read_u64_le(bytes, pos);
|
|
|
|
if *pos + 2 > end {
|
|
return None;
|
|
}
|
|
let agent_len = read_u16_le(bytes, pos) as usize;
|
|
if *pos + agent_len > end {
|
|
return None;
|
|
}
|
|
let agent_id = String::from_utf8_lossy(&bytes[*pos..*pos + agent_len]).to_string();
|
|
*pos += agent_len;
|
|
|
|
if *pos + 2 > end {
|
|
return None;
|
|
}
|
|
let policy_len = read_u16_le(bytes, pos) as usize;
|
|
if *pos + policy_len > end {
|
|
return None;
|
|
}
|
|
let policy_name = String::from_utf8_lossy(&bytes[*pos..*pos + policy_len]).to_string();
|
|
*pos += policy_len;
|
|
|
|
Some(SessionWalEvent::Start {
|
|
session_id,
|
|
user_id,
|
|
started_at_ns,
|
|
agent_id,
|
|
policy_name,
|
|
})
|
|
}
|
|
|
|
/// Decode a Signal record from the payload region.
|
|
fn decode_signal_record(bytes: &[u8], pos: &mut usize, end: usize) -> Option<SessionWalEvent> {
|
|
// session_id(8) + entity_id(8) + weight(4) + ts_ns(8) = 28
|
|
if *pos + 28 > end {
|
|
return None;
|
|
}
|
|
let session_id = read_u64_le(bytes, pos);
|
|
let entity_id = read_u64_le(bytes, pos);
|
|
let weight = f32::from_le_bytes([
|
|
bytes[*pos],
|
|
bytes[*pos + 1],
|
|
bytes[*pos + 2],
|
|
bytes[*pos + 3],
|
|
]);
|
|
*pos += 4;
|
|
let ts_ns = read_u64_le(bytes, pos);
|
|
|
|
if *pos + 2 > end {
|
|
return None;
|
|
}
|
|
let sig_len = read_u16_le(bytes, pos) as usize;
|
|
if *pos + sig_len > end {
|
|
return None;
|
|
}
|
|
let signal_name = String::from_utf8_lossy(&bytes[*pos..*pos + sig_len]).to_string();
|
|
*pos += sig_len;
|
|
|
|
if *pos + 1 > end {
|
|
return None;
|
|
}
|
|
let has_annotation = bytes[*pos] != 0;
|
|
*pos += 1;
|
|
|
|
let annotation = if has_annotation {
|
|
if *pos + 2 > end {
|
|
return None;
|
|
}
|
|
let ann_len = read_u16_le(bytes, pos) as usize;
|
|
if *pos + ann_len > end {
|
|
return None;
|
|
}
|
|
let ann = String::from_utf8_lossy(&bytes[*pos..*pos + ann_len]).to_string();
|
|
*pos += ann_len;
|
|
Some(ann)
|
|
} else {
|
|
None
|
|
};
|
|
|
|
// m8p4: session_seqno (backward-compatible — absent in legacy records)
|
|
let session_seqno = if *pos < end {
|
|
let has_seqno = bytes[*pos];
|
|
*pos += 1;
|
|
if has_seqno != 0 && *pos + 8 <= end {
|
|
let v = read_u64_le(bytes, pos);
|
|
Some(SessionSeqNo(v))
|
|
} else {
|
|
None
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
// m8p4: idempotency_key (backward-compatible — absent in legacy records)
|
|
let idempotency_key = if *pos < end {
|
|
let has_key = bytes[*pos];
|
|
*pos += 1;
|
|
if has_key != 0 && *pos + 16 <= end {
|
|
let v = u128::from_le_bytes([
|
|
bytes[*pos],
|
|
bytes[*pos + 1],
|
|
bytes[*pos + 2],
|
|
bytes[*pos + 3],
|
|
bytes[*pos + 4],
|
|
bytes[*pos + 5],
|
|
bytes[*pos + 6],
|
|
bytes[*pos + 7],
|
|
bytes[*pos + 8],
|
|
bytes[*pos + 9],
|
|
bytes[*pos + 10],
|
|
bytes[*pos + 11],
|
|
bytes[*pos + 12],
|
|
bytes[*pos + 13],
|
|
bytes[*pos + 14],
|
|
bytes[*pos + 15],
|
|
]);
|
|
*pos += 16;
|
|
Some(v)
|
|
} else {
|
|
None
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
Some(SessionWalEvent::Signal {
|
|
session_id,
|
|
entity_id,
|
|
weight,
|
|
ts_ns,
|
|
signal_name,
|
|
annotation,
|
|
session_seqno,
|
|
idempotency_key,
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[allow(clippy::unwrap_used)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn session_start_roundtrip() {
|
|
let event = SessionWalEvent::Start {
|
|
session_id: 42,
|
|
user_id: 100,
|
|
started_at_ns: 1_000_000_000,
|
|
agent_id: "test-agent".to_string(),
|
|
policy_name: "default_policy".to_string(),
|
|
};
|
|
let encoded = encode_session_event(&event);
|
|
let decoded = decode_session_events(&encoded);
|
|
assert_eq!(decoded.len(), 1);
|
|
assert_eq!(decoded[0], event);
|
|
}
|
|
|
|
#[test]
|
|
fn session_signal_roundtrip_with_annotation() {
|
|
let event = SessionWalEvent::Signal {
|
|
session_id: 7,
|
|
entity_id: 999,
|
|
weight: 1.5,
|
|
ts_ns: 2_000_000_000,
|
|
signal_name: "reward".to_string(),
|
|
annotation: Some("jazz fusion".to_string()),
|
|
session_seqno: None,
|
|
idempotency_key: None,
|
|
};
|
|
let encoded = encode_session_event(&event);
|
|
let decoded = decode_session_events(&encoded);
|
|
assert_eq!(decoded.len(), 1);
|
|
assert_eq!(decoded[0], event);
|
|
}
|
|
|
|
#[test]
|
|
fn session_signal_roundtrip_without_annotation() {
|
|
let event = SessionWalEvent::Signal {
|
|
session_id: 7,
|
|
entity_id: 999,
|
|
weight: 1.5,
|
|
ts_ns: 2_000_000_000,
|
|
signal_name: "view".to_string(),
|
|
annotation: None,
|
|
session_seqno: None,
|
|
idempotency_key: None,
|
|
};
|
|
let encoded = encode_session_event(&event);
|
|
let decoded = decode_session_events(&encoded);
|
|
assert_eq!(decoded.len(), 1);
|
|
assert_eq!(decoded[0], event);
|
|
}
|
|
|
|
#[test]
|
|
fn session_close_roundtrip() {
|
|
let event = SessionWalEvent::Close { session_id: 42 };
|
|
let encoded = encode_session_event(&event);
|
|
let decoded = decode_session_events(&encoded);
|
|
assert_eq!(decoded.len(), 1);
|
|
assert_eq!(decoded[0], event);
|
|
}
|
|
|
|
#[test]
|
|
fn session_multiple_events_roundtrip() {
|
|
let events = vec![
|
|
SessionWalEvent::Start {
|
|
session_id: 1,
|
|
user_id: 10,
|
|
started_at_ns: 100,
|
|
agent_id: "agent-a".to_string(),
|
|
policy_name: "policy-1".to_string(),
|
|
},
|
|
SessionWalEvent::Signal {
|
|
session_id: 1,
|
|
entity_id: 42,
|
|
weight: 1.0,
|
|
ts_ns: 200,
|
|
signal_name: "view".to_string(),
|
|
annotation: None,
|
|
session_seqno: None,
|
|
idempotency_key: None,
|
|
},
|
|
SessionWalEvent::Signal {
|
|
session_id: 1,
|
|
entity_id: 43,
|
|
weight: 2.0,
|
|
ts_ns: 300,
|
|
signal_name: "reward".to_string(),
|
|
annotation: Some("good content".to_string()),
|
|
session_seqno: None,
|
|
idempotency_key: None,
|
|
},
|
|
SessionWalEvent::Close { session_id: 1 },
|
|
];
|
|
|
|
let mut all_bytes = Vec::new();
|
|
for event in &events {
|
|
all_bytes.extend(encode_session_event(event));
|
|
}
|
|
|
|
let decoded = decode_session_events(&all_bytes);
|
|
assert_eq!(decoded.len(), events.len());
|
|
for (orig, dec) in events.iter().zip(decoded.iter()) {
|
|
assert_eq!(orig, dec);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn session_decode_truncated_stops_cleanly() {
|
|
let event = SessionWalEvent::Start {
|
|
session_id: 1,
|
|
user_id: 10,
|
|
started_at_ns: 100,
|
|
agent_id: "agent".to_string(),
|
|
policy_name: "policy".to_string(),
|
|
};
|
|
let encoded = encode_session_event(&event);
|
|
|
|
// Truncate the record mid-way.
|
|
let truncated = &encoded[..encoded.len() / 2];
|
|
let decoded = decode_session_events(truncated);
|
|
assert!(decoded.is_empty(), "truncated record should be skipped");
|
|
}
|
|
|
|
#[test]
|
|
fn session_decode_partial_second_record_stops() {
|
|
let e1 = SessionWalEvent::Close { session_id: 1 };
|
|
let e2 = SessionWalEvent::Start {
|
|
session_id: 2,
|
|
user_id: 20,
|
|
started_at_ns: 200,
|
|
agent_id: "agent".to_string(),
|
|
policy_name: "policy".to_string(),
|
|
};
|
|
let mut all_bytes = encode_session_event(&e1);
|
|
let e2_bytes = encode_session_event(&e2);
|
|
// Add partial second record.
|
|
all_bytes.extend_from_slice(&e2_bytes[..e2_bytes.len() / 2]);
|
|
|
|
let decoded = decode_session_events(&all_bytes);
|
|
assert_eq!(decoded.len(), 1);
|
|
assert_eq!(decoded[0], e1);
|
|
}
|
|
|
|
#[test]
|
|
fn session_decode_empty_bytes() {
|
|
let decoded = decode_session_events(&[]);
|
|
assert!(decoded.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn session_seqno_ordering() {
|
|
assert!(SessionSeqNo(1) > SessionSeqNo(0));
|
|
assert_eq!(SessionSeqNo(5).next(), SessionSeqNo(6));
|
|
assert_eq!(SessionSeqNo::ZERO, SessionSeqNo(0));
|
|
}
|
|
|
|
#[test]
|
|
fn session_signal_with_seqno_roundtrip() {
|
|
let event = SessionWalEvent::Signal {
|
|
session_id: 1,
|
|
entity_id: 42,
|
|
weight: 1.0,
|
|
ts_ns: 1000,
|
|
signal_name: "view".to_string(),
|
|
annotation: None,
|
|
session_seqno: Some(SessionSeqNo(7)),
|
|
idempotency_key: Some(0xDEAD_BEEF_CAFE_BABE_u128),
|
|
};
|
|
let encoded = encode_session_event(&event);
|
|
let decoded = decode_session_events(&encoded);
|
|
assert_eq!(decoded.len(), 1);
|
|
assert_eq!(decoded[0], event);
|
|
}
|
|
|
|
#[test]
|
|
fn session_signal_legacy_roundtrip_no_seqno() {
|
|
// Legacy events (seqno=None) decode correctly.
|
|
let event = SessionWalEvent::Signal {
|
|
session_id: 2,
|
|
entity_id: 99,
|
|
weight: 2.0,
|
|
ts_ns: 2000,
|
|
signal_name: "like".to_string(),
|
|
annotation: None,
|
|
session_seqno: None,
|
|
idempotency_key: None,
|
|
};
|
|
let encoded = encode_session_event(&event);
|
|
let decoded = decode_session_events(&encoded);
|
|
assert_eq!(decoded[0], event);
|
|
}
|
|
}
|