//! Background worker that tails the WAL and updates the KV store.
//!
//! The worker reads records from the Write-Ahead Log and persists them
//! to the storage engine using content-addressed keys.
//!
//! # Storage Layout
//!
//! Following the architecture spec, records are stored with these key prefixes:
//! - `H:{hash}` - Assertions (content-addressed by BLAKE3 hash)
//! - `V:{assertion_hash}:{vote_hash}` - Votes on assertions
//! - `E:{hash}` - Epochs (paradigm definitions)
//! - `S:{subject}` - Subject adjacency index (list of assertion hashes)

use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use stemedb_core::types::HlcTimestamp;
use stemedb_storage::{GenericIndexStore, GenericVoteStore, KVStore, VectorIndex, VisualIndex};
use stemedb_wal::{Journal, HEADER_SIZE};
use tokio::sync::{Mutex, Notify};
use tracing::{debug, info, warn};

use crate::error::{IngestError, Result};
use crate::gossip::GossipBroadcast;

// Module declarations
mod processing;
mod record_types;
mod run;
mod storage;

#[cfg(test)]
mod tests;

// Re-exports
pub use record_types::{serialize_assertion, serialize_epoch, serialize_vote, RecordType};

/// Background worker that tails the WAL and updates the KV store.
pub struct IngestWorker<S> {
    journal: Arc<Mutex<Journal>>,
    store: Arc<S>,
    index_store: GenericIndexStore<Arc<S>>,
    vote_store: GenericVoteStore<Arc<S>>,
    current_offset: u64,
    /// Optional notification channel for event-driven materialization.
    /// When set, the worker signals this after each successful ingestion
    /// so downstream consumers (e.g., the Materializer) can react immediately.
    notify: Option<Arc<Notify>>,
    /// Optional vector index for semantic similarity search.
    /// When set, assertions with embedding vectors are indexed on ingestion.
    vector_index: Option<Arc<dyn VectorIndex>>,
    /// Optional visual index for perceptual hash similarity search.
    /// When set, assertions with visual_hash are indexed on ingestion.
    visual_index: Option<Arc<dyn VisualIndex>>,
    /// Shutdown signal shared with Ingestor.
    /// When set to true, the run() loop exits gracefully.
    shutdown: Arc<AtomicBool>,
    /// Hybrid Logical Clock for distributed causal ordering.
    ///
    /// Used to generate HLC timestamps for supersessions and epoch
    /// ingestion. Provides causal ordering guarantees across distributed
    /// nodes, even with clock skew.
    hlc: uhlc::HLC,
    /// Optional gossip broadcaster for distributed replication.
    ///
    /// When set, the worker broadcasts newly ingested assertions to peer nodes.
    gossip_broadcaster: Option<Arc<dyn GossipBroadcast>>,
}

impl<S: KVStore + 'static> IngestWorker<S> {
    /// Create a new ingest worker, resuming from the last persisted cursor.
    ///
    /// If a cursor checkpoint exists in the KV store, the worker resumes
    /// from that offset. Otherwise, it starts from the beginning of the WAL
    /// (after the file header).
    pub async fn new(journal: Arc<Mutex<Journal>>, store: Arc<S>) -> Result<Self> {
        let index_store = GenericIndexStore::new(store.clone());
        let vote_store = GenericVoteStore::new(store.clone());
        let cursor_key = stemedb_storage::key_codec::cursor_key();
        let current_offset = match store.get(&cursor_key).await? {
            Some(bytes) if bytes.len() == 8 => {
                let offset =
                    u64::from_le_bytes(bytes.try_into().map_err(|_| {
                        IngestError::Serialization("Invalid cursor bytes".to_string())
                    })?);
                info!(offset, "Resuming ingestion from persisted cursor");
                offset
            }
            Some(bytes) => {
                warn!(
                    len = bytes.len(),
                    "Corrupt cursor value (expected 8 bytes), starting from beginning"
                );
                HEADER_SIZE as u64
            }
            None => {
                debug!("No cursor found, starting from WAL header");
                HEADER_SIZE as u64
            }
        };
        // Initialize HLC with random node ID
        let hlc = uhlc::HLCBuilder::new().build();

        Ok(Self {
            journal,
            store,
            index_store,
            vote_store,
            current_offset,
            notify: None,
            vector_index: None,
            visual_index: None,
            shutdown: Arc::new(AtomicBool::new(false)),
            hlc,
            gossip_broadcaster: None,
        })
    }

    /// Create a new ingest worker with a shared shutdown signal.
    ///
    /// This is used by the Ingestor to coordinate shutdown between the
    /// manager and the background task.
    pub async fn with_shutdown(
        journal: Arc<Mutex<Journal>>,
        store: Arc<S>,
        shutdown: Arc<AtomicBool>,
    ) -> Result<Self> {
        let mut worker = Self::new(journal, store).await?;
        worker.shutdown = shutdown;
        Ok(worker)
    }

    /// Check if shutdown has been requested.
    pub fn is_shutdown(&self) -> bool {
        self.shutdown.load(Ordering::Relaxed)
    }

    /// Attach a notification channel for event-driven downstream consumers.
    ///
    /// After each successful record ingestion, the worker will signal this
    /// `Notify` so consumers like the Materializer can react immediately
    /// instead of polling on a fixed interval.
    pub fn with_notify(mut self, notify: Arc<Notify>) -> Self {
        self.notify = Some(notify);
        self
    }

    /// Attach a notification channel for event-driven WAL reading.
    ///
    /// When the GroupCommitBuffer flushes new data to the WAL, it signals
    /// this `Notify` so the worker can immediately refresh its segment list
    /// and process the new records. This is the counterpart to the downstream
    /// `with_notify` - this one is for upstream signaling from the writer.
    ///
    /// Note: This uses the same internal field as `with_notify` since we
    /// want to wake up on both upstream writes and downstream requests.
    /// The run loop handles both cases by refreshing segments and processing.
    pub fn with_flush_notify(mut self, notify: Arc<Notify>) -> Self {
        self.notify = Some(notify);
        self
    }

    /// Attach a vector index for semantic similarity search.
    ///
    /// When set, assertions with embedding vectors (`vector` field) are
    /// automatically indexed during ingestion, enabling k-NN queries.
    ///
    /// # Example
    /// ```ignore
    /// let vector_index = Arc::new(HnswVectorIndex::new(128));
    /// let worker = IngestWorker::new(journal, store)
    ///     .await?
    ///     .with_vector_index(vector_index);
    /// ```
    pub fn with_vector_index(mut self, index: Arc<dyn VectorIndex>) -> Self {
        self.vector_index = Some(index);
        self
    }

    /// Attach a visual index for perceptual hash similarity search.
    ///
    /// When set, assertions with visual hashes (`visual_hash` field) are
    /// automatically indexed during ingestion, enabling visual similarity queries.
    ///
    /// # Example
    /// ```ignore
    /// let visual_index = Arc::new(BkTreeVisualIndex::new());
    /// let worker = IngestWorker::new(journal, store)
    ///     .await?
    ///     .with_visual_index(visual_index);
    /// ```
    pub fn with_visual_index(mut self, index: Arc<dyn VisualIndex>) -> Self {
        self.visual_index = Some(index);
        self
    }

    /// Configure the HLC with a specific node ID.
    ///
    /// Use this when running multiple nodes in a distributed cluster to ensure
    /// each node has a unique identifier for total ordering of concurrent events.
    ///
    /// # Example
    /// ```ignore
    /// let node_id = uhlc::ID::try_from(&node_uuid.as_bytes()[..]).unwrap();
    /// let worker = IngestWorker::new(journal, store)
    ///     .await?
    ///     .with_node_id(node_id);
    /// ```
    pub fn with_node_id(mut self, node_id: uhlc::ID) -> Self {
        self.hlc = uhlc::HLCBuilder::new().with_id(node_id).build();
        self
    }

    /// Attach a gossip broadcaster for distributed replication.
    ///
    /// When set, newly ingested assertions are broadcast to peer nodes
    /// for low-latency replication. The gossip layer is best-effort:
    /// failures are logged but don't block the ingestion pipeline.
    ///
    /// # Example
    /// ```ignore
    /// let broadcaster = GossipBroadcaster::new(peers).await?;
    /// let worker = IngestWorker::new(journal, store)
    ///     .await?
    ///     .with_gossip_broadcaster(Arc::new(broadcaster));
    /// ```
    pub fn with_gossip_broadcaster(mut self, broadcaster: Arc<dyn GossipBroadcast>) -> Self {
        self.gossip_broadcaster = Some(broadcaster);
        self
    }

    /// Returns the gossip broadcaster if configured.
    pub fn gossip_broadcaster(&self) -> Option<&Arc<dyn GossipBroadcast>> {
        self.gossip_broadcaster.as_ref()
    }

    /// Generates a new HLC timestamp.
    ///
    /// The returned timestamp is guaranteed to be greater than all previously
    /// generated timestamps from this worker, even if the system clock goes
    /// backwards.
    ///
    /// Use this when creating supersessions or other records that need
    /// causal ordering across distributed nodes.
    pub fn generate_hlc_timestamp(&self) -> HlcTimestamp {
        HlcTimestamp::now(&self.hlc)
    }

    /// Updates the HLC with a timestamp from a remote node.
    ///
    /// Call this when receiving data from another node to ensure the local
    /// clock stays synchronized. The HLC will advance to at least the
    /// remote timestamp, maintaining causal ordering.
    ///
    /// # Arguments
    ///
    /// * `remote` - HLC timestamp received from a remote node
    ///
    /// # Returns
    ///
    /// Ok(()) if the clock was updated, Err if the timestamp is too far
    /// in the future (clock skew protection).
    pub fn update_hlc_from_remote(&self, remote: &HlcTimestamp) -> Result<()> {
        if let Some(ts) = remote.to_uhlc() {
            self.hlc.update_with_timestamp(&ts).map_err(|e| {
                warn!(
                    remote_time = remote.time_ntp64,
                    error = %e,
                    "Failed to update HLC from remote timestamp (clock skew?)"
                );
                IngestError::InputValidation(format!("HLC update failed: {}", e))
            })?;
        }
        Ok(())
    }

    /// Returns the current HLC node ID as bytes.
    ///
    /// Useful for including in CRDT state or other distributed data structures.
    pub fn hlc_node_id(&self) -> [u8; 16] {
        self.hlc.get_id().to_le_bytes()
    }
}
