pub mod labels;
pub mod mab;
pub mod schema;
pub mod seed;
pub mod sources;

use std::collections::{HashMap, HashSet, VecDeque};
use std::path::Path;

// ── Tuneable constants ────────────────────────────────────────────────────────

/// Minimum number of seeded items before `seed_default_corpus` considers the
/// corpus already populated and skips re-seeding (idempotency guard).
const SEED_CORPUS_MIN: u64 = 100;

/// Dwell duration after which a read is treated as article completion (ms).
/// 15 seconds of continuous reading signals strong positive engagement.
const COMPLETION_DWELL_MS: u64 = 15_000;

/// Maximum number of discovered item IDs held in the in-memory ring buffer.
/// Oldest entries are evicted first when the cap is reached.
const MAX_DISCOVERED_IDS: usize = 1_000;

/// Number of ANN candidates retrieved when searching for a bridge item.
const BRIDGE_CANDIDATE_LIMIT: u32 = 10;

/// Minimum per-dimension score in the user's preference vector for a category
/// to be considered "active" when selecting bridge item categories.
const BRIDGE_MIN_CATEGORY_SCORE: f32 = 0.05;

/// HTTP request timeout for calls to the embedding sidecar (seconds).
const EMBEDDER_TIMEOUT_SECS: u64 = 10;

use tidaldb::TidalDb;
use tidaldb::schema::{EntityId, Timestamp};

pub use labels::{ForageItem, ItemLabel};
pub use mab::{ExplorationStats, MabConfig};
pub use schema::{DEFAULT_DIM, REAL_DIM};
pub use seed::{SeedItem, url_to_item_id};

/// A single topic the discovery agent should browse for.
#[derive(Debug, Clone, serde::Serialize)]
pub struct BrowseTopic {
    /// Category name (e.g. "jazz").
    pub name: String,
    /// 0.0--1.0 weight derived from user's preference vector.
    pub priority: f32,
    /// Source URLs to navigate (from the static source registry).
    pub sources: Vec<String>,
}

/// The full browse plan returned by `GET /browse-tasks`.
#[derive(Debug, Clone, serde::Serialize)]
pub struct BrowsePlan {
    /// Whether the agent should run a discovery cycle now.
    /// False when last discovery was recent and the feed has enough items.
    pub should_run: bool,
    /// How many minutes to wait between cycles.
    pub interval_minutes: u32,
    /// Topics ordered by priority descending.
    pub topics: Vec<BrowseTopic>,
    /// Max articles to capture per source per cycle.
    pub limit_per_topic: usize,
    /// Top tags from the user's saved/dwelled items.
    /// The agent uses these to prefer articles matching the user's subtopics.
    pub tag_hints: Vec<String>,
}

/// Input for registering a dynamically discovered page as a Forage item.
#[derive(Debug, Clone)]
pub struct ForageItemInput {
    pub url: String,
    pub title: String,
    /// Hostname of the page (e.g. `"news.ycombinator.com"`).
    pub source: String,
    /// Content category hint (e.g. `"technology"`). Defaults to `"discovered"`.
    pub category: String,
    pub reading_time_min: u32,
    pub description: String,
    /// Specific subtopics (e.g. `["modal jazz", "music theory"]`).
    pub tags: Vec<String>,
    /// Named entities (e.g. `["John Coltrane", "Blue Note"]`).
    pub entities: Vec<String>,
    /// Content type classification (e.g. `"analysis"`, `"tutorial"`, `"news"`).
    pub content_type: String,
    /// Claude's 2-sentence summary of the article.
    pub summary: String,
}

#[derive(Debug, thiserror::Error)]
pub enum ForageError {
    #[error("tidaldb: {0}")]
    Db(#[from] tidaldb::TidalError),
    #[error("query: {0}")]
    Query(String),
    #[error("seed: {0}")]
    Seed(String),
}

pub type Result<T> = std::result::Result<T, ForageError>;

/// The core engine wrapping tidalDB with Forage-specific logic.
pub struct ForageEngine {
    pub(crate) db: TidalDb,
    #[allow(dead_code)]
    pub(crate) mab: MabConfig,
    pub(crate) seed_items: Vec<SeedItem>,
    /// IDs of items added via `add_item` (in-memory; reset on server restart).
    /// These are injected into every feed pool so they surface even before they
    /// accumulate enough global signal to rank on their own.
    /// Uses `VecDeque` for O(1) front-eviction when the cap is reached.
    discovered_ids: std::sync::Mutex<VecDeque<u64>>,
    /// URL of the embedding sidecar (e.g. `http://localhost:4243`).
    /// When `Some`, `add_item` and `seed_default_corpus` call `POST /embed`
    /// to obtain a real semantic vector.  When `None`, a neutral unit vector
    /// is written instead (P0/P1 behaviour).
    embed_url: Option<String>,
    /// Reusable HTTP client for calls to the embedding sidecar.
    ///
    /// Created once in `ForageEngineBuilder::open()` so connection pooling is
    /// preserved across `add_item` calls.  `None` when no embedder is configured.
    embed_client: Option<reqwest::blocking::Client>,
    /// Dimensionality of the embedding space.  Must match the sidecar output
    /// and the schema's `embedding_slot` declaration.
    embed_dim: usize,
    /// Per-user exploration stats, persisted to disk in persistent mode.
    ///
    /// Tracks `exploration_hits` / `exploration_total` → hit rate, which drives
    /// the adaptive exploration ratio, and per-category signal counts for UCB1.
    ///
    /// Loaded from `stats_path` on `open()` and written back after every
    /// `track_signal_stats()` call so profiles survive server restarts.
    exploration_stats: std::sync::Mutex<HashMap<u64, mab::ExplorationStats>>,
    /// Path to the exploration-stats JSON file (`None` in ephemeral mode).
    stats_path: Option<std::path::PathBuf>,
    /// The set of item IDs labeled `Exploring` in each user's last `feed()` call.
    ///
    /// Used by `signal()` to detect when a user engages with an exploration slot
    /// and record the outcome into `exploration_stats`.
    last_explore_items: std::sync::Mutex<HashMap<u64, HashSet<u64>>>,
    /// Per-user saved item IDs.
    ///
    /// Maintained at the Forage engine level (not tidalDB's `UserStateIndex`)
    /// because tidalDB's saved bitmap uses `RoaringBitmap<u32>` which truncates
    /// Forage's u64 FNV-hash item IDs for discovered items.  This set preserves
    /// full u64 IDs so `top_tags()` can correctly look up item metadata.
    saved_items: std::sync::Mutex<HashMap<u64, HashSet<u64>>>,
    /// Per-user item IDs where the user dwelled ≥15 seconds (article completion).
    ///
    /// Used by `top_tags()` alongside `saved_items` so that strong-dwell reads
    /// (spec: "save + strong dwell ≥15s") contribute to tag affinity.
    dwelled_items: std::sync::Mutex<HashMap<u64, HashSet<u64>>>,
    /// Path to the `user_state.json` file that persists `saved_items` and
    /// `dwelled_items` across server restarts (`None` in ephemeral mode).
    user_state_path: Option<std::path::PathBuf>,
}

/// Fluent builder for `ForageEngine`.
///
/// ```no_run
/// use forage_engine::ForageEngine;
/// let engine = ForageEngine::builder()
///     .ephemeral()
///     .with_embedder("http://localhost:4243")
///     .open()
///     .unwrap();
/// ```
pub struct ForageEngineBuilder {
    ephemeral: bool,
    data_dir: Option<std::path::PathBuf>,
    embed_url: Option<String>,
}

impl ForageEngineBuilder {
    /// Use in-memory (ephemeral) storage.  Mutually exclusive with `data_dir`.
    pub fn ephemeral(mut self) -> Self {
        self.ephemeral = true;
        self
    }

    /// Use persistent storage at the given path.
    pub fn data_dir(mut self, path: impl Into<std::path::PathBuf>) -> Self {
        self.data_dir = Some(path.into());
        self
    }

    /// Connect to an embedding sidecar at `url`.
    /// When set, `add_item` and `seed_default_corpus` will POST to `{url}/embed`
    /// to obtain a 1536-dimensional semantic vector for each item.
    pub fn with_embedder(mut self, url: impl Into<String>) -> Self {
        self.embed_url = Some(url.into());
        self
    }

    /// Build and open the engine.
    pub fn open(self) -> Result<ForageEngine> {
        let embed_dim = if self.embed_url.is_some() {
            REAL_DIM
        } else {
            DEFAULT_DIM
        };
        let schema = schema::build(embed_dim);
        let (db, stats_path, user_state_path) = if self.ephemeral {
            (
                TidalDb::builder().ephemeral().with_schema(schema).open()?,
                None,
                None,
            )
        } else {
            let dir = self
                .data_dir
                .ok_or_else(|| ForageError::Seed("data_dir required for persistent mode".into()))?;
            std::fs::create_dir_all(&dir)
                .map_err(|e| ForageError::Seed(format!("create data dir {:?}: {e}", dir)))?;
            let db = TidalDb::builder()
                .with_data_dir(&dir)
                .with_schema(schema)
                .open()?;
            let stats = dir.join("exploration_stats.json");
            let user_state = dir.join("user_state.json");
            (db, Some(stats), Some(user_state))
        };
        // Load persisted exploration stats from disk (persistent mode only).
        let exploration_stats_map = stats_path
            .as_ref()
            .and_then(|p| load_exploration_stats(p))
            .unwrap_or_default();
        // Load persisted saved/dwelled item sets.
        let (saved_items_map, dwelled_items_map) = user_state_path
            .as_ref()
            .map(|p| load_user_item_state(p))
            .unwrap_or_default();
        // Build a reusable HTTP client for the embedding sidecar if configured.
        // Creating the client once preserves the connection pool across all
        // add_item and seed_default_corpus calls.
        let embed_client = if self.embed_url.is_some() {
            Some(
                reqwest::blocking::Client::builder()
                    .timeout(std::time::Duration::from_secs(EMBEDDER_TIMEOUT_SECS))
                    .build()
                    .map_err(|e| ForageError::Seed(format!("build HTTP client: {e}")))?,
            )
        } else {
            None
        };
        Ok(ForageEngine {
            db,
            mab: MabConfig::default(),
            seed_items: seed::default_corpus(),
            discovered_ids: std::sync::Mutex::new(VecDeque::new()),
            embed_url: self.embed_url,
            embed_client,
            embed_dim,
            exploration_stats: std::sync::Mutex::new(exploration_stats_map),
            last_explore_items: std::sync::Mutex::new(HashMap::new()),
            saved_items: std::sync::Mutex::new(saved_items_map),
            dwelled_items: std::sync::Mutex::new(dwelled_items_map),
            user_state_path,
            stats_path,
        })
    }
}

impl ForageEngine {
    /// Returns a fluent builder for constructing a `ForageEngine`.
    pub fn builder() -> ForageEngineBuilder {
        ForageEngineBuilder {
            ephemeral: false,
            data_dir: None,
            embed_url: None,
        }
    }

    /// Open an ephemeral (in-memory) engine with category-axis embeddings.
    /// For real semantic embeddings use `ForageEngine::builder().ephemeral().with_embedder(url).open()`.
    pub fn ephemeral() -> Result<Self> {
        ForageEngine::builder().ephemeral().open()
    }

    /// Open a persistent engine backed by the given directory with category-axis embeddings.
    /// For real semantic embeddings use `ForageEngine::builder().data_dir(path).with_embedder(url).open()`.
    pub fn persistent(data_dir: &Path) -> Result<Self> {
        ForageEngine::builder().data_dir(data_dir).open()
    }

    // ── Internal: call the embedding sidecar ─────────────────────────────────

    /// POST `{embed_url}/embed` and return the vector.
    ///
    /// Falls back to a neutral unit vector on any network or parse error so
    /// that `add_item` never fails because the sidecar is temporarily down.
    fn embed_text(&self, text: &str) -> Vec<f32> {
        let (Some(url), Some(client)) = (&self.embed_url, &self.embed_client) else {
            return neutral_vector(self.embed_dim);
        };
        match call_embedder(client, url, text) {
            Ok(v) if v.len() == self.embed_dim => v,
            Ok(v) => {
                eprintln!(
                    "[forage-engine] embedder returned {} dims, expected {}; using neutral vector",
                    v.len(),
                    self.embed_dim
                );
                neutral_vector(self.embed_dim)
            }
            Err(e) => {
                eprintln!("[forage-engine] embedder error: {e}; using neutral vector");
                neutral_vector(self.embed_dim)
            }
        }
    }

    /// Seed the default corpus (idempotent — skips if items already present).
    ///
    /// When an embedder is configured, each seed item's title + description is
    /// sent to the sidecar and the resulting semantic vector is stored.  Without
    /// an embedder, category-axis vectors are used as in P0/P1.
    pub fn seed_default_corpus(&self) -> Result<()> {
        if self.db.item_count() >= SEED_CORPUS_MIN {
            return Ok(());
        }

        for item in &self.seed_items {
            let mut meta = HashMap::new();
            meta.insert("title".to_owned(), item.title.clone());
            meta.insert("source".to_owned(), item.source.clone());
            meta.insert("category".to_owned(), item.category.clone());
            meta.insert(
                "reading_time_min".to_owned(),
                item.reading_time_min.to_string(),
            );
            meta.insert("description".to_owned(), item.description.clone());
            meta.insert("url".to_owned(), item.url.clone());

            let id = EntityId::new(item.id);
            self.db.write_item_with_metadata(id, &meta)?;

            let vec = if self.embed_url.is_some() {
                let text = format!("{} — {}", item.title, item.description);
                self.embed_text(&text)
            } else {
                seed::category_vector(&item.category, item.id)
            };
            self.db.write_item_embedding(id, &vec)?;
        }

        seed::seed_users(&self.db)?;
        Ok(())
    }

    /// Record a signal for a user--item interaction.
    ///
    /// For `Save` signals, additionally:
    /// - Marks the item in the user's saved bitmap (`UserStateIndex::add_save`)
    ///   so that `top_tags()` and `browse_tasks()` can aggregate tags from
    ///   saved items.
    /// - Emits a secondary `"share"` signal so that tidalDB's preference vector
    ///   update path (which gates on `is_positive_engagement_signal`) is
    ///   triggered.  Save is strong positive intent and should shift the user's
    ///   taste vector.
    pub fn signal(&self, user_id: u64, item_id: u64, kind: SignalKind) -> Result<()> {
        let signal_type = match kind {
            SignalKind::View => "view",
            SignalKind::Save => "save",
            SignalKind::Skip => "skip",
            SignalKind::Share => "share",
        };
        self.db.signal_with_context(
            signal_type,
            EntityId::new(item_id),
            1.0,
            Timestamp::now(),
            Some(user_id),
            None,
        )?;

        // Save-specific side effects: tidalDB's signal_with_context does not
        // populate the saved bitmap or update the preference vector for "save"
        // signals (only "like", "share", "completion", "search_click" are
        // positive-engagement triggers).  Forage treats save as strong positive
        // intent, so we bridge the gap here.
        if matches!(kind, SignalKind::Save) {
            // Track in Forage-level saved set (u64-safe; see field doc).
            self.saved_items
                .lock()
                .unwrap()
                .entry(user_id)
                .or_default()
                .insert(item_id);

            // Also populate tidalDB's u32 bitmap for query-path compatibility.
            #[allow(clippy::cast_possible_truncation)]
            let item_u32 = item_id as u32;
            self.db.user_state().add_save(user_id, item_u32);

            // Emit a secondary "share" signal so the preference vector update
            // fires.  Weight is lower (0.5) to distinguish from an explicit share.
            self.db.signal_with_context(
                "share",
                EntityId::new(item_id),
                0.5,
                Timestamp::now(),
                Some(user_id),
                None,
            )?;
            // Persist the updated saved set so it survives server restarts.
            self.save_user_item_state();
        }

        let is_positive = matches!(
            kind,
            SignalKind::View | SignalKind::Save | SignalKind::Share
        );
        self.track_signal_stats(user_id, item_id, is_positive);
        Ok(())
    }

    /// Record a dwell signal with duration weight.
    pub fn signal_dwell(&self, user_id: u64, item_id: u64, duration_ms: u64) -> Result<()> {
        // Normalize: 1.0 per 10 seconds, capped at 5.0.
        let weight = (duration_ms as f64 / 10_000.0).clamp(0.1, 5.0);
        self.db.signal_with_context(
            "dwell",
            EntityId::new(item_id),
            weight,
            Timestamp::now(),
            Some(user_id),
            None,
        )?;
        // 15+ second read is recognized as article completion. tidalDB treats
        // "completion" as a positive engagement signal, which updates the user's
        // preference vector and closes the behavioral ranking loop.
        if duration_ms >= COMPLETION_DWELL_MS {
            self.db.signal_with_context(
                "completion",
                EntityId::new(item_id),
                1.0,
                Timestamp::now(),
                Some(user_id),
                None,
            )?;
            // Track in the dwell set so top_tags() includes completion-read items
            // per spec: "save + strong dwell ≥15s" contribute to tag affinity.
            self.dwelled_items
                .lock()
                .unwrap()
                .entry(user_id)
                .or_default()
                .insert(item_id);
            self.save_user_item_state();
        }
        // Dwell is always a positive engagement signal.
        self.track_signal_stats(user_id, item_id, true);
        Ok(())
    }

    /// Track category signals and exploration outcomes for adaptive MAB.
    ///
    /// Called after every signal write.  Increments the per-category signal count
    /// for UCB1 bonus computation, and if the item was in the user's last
    /// exploration set, records the outcome in `exploration_stats`.
    fn track_signal_stats(&self, user_id: u64, item_id: u64, is_positive: bool) {
        // Look up the item's category: seed corpus first, then DB (for discovered items).
        let category = self
            .seed_items
            .iter()
            .find(|s| s.id == item_id)
            .map(|s| s.category.clone())
            .or_else(|| {
                self.db
                    .get_item_metadata(EntityId::new(item_id))
                    .ok()
                    .flatten()
                    .and_then(|m| m.get("category").cloned())
            });

        let was_exploring = self
            .last_explore_items
            .lock()
            .unwrap()
            .get(&user_id)
            .is_some_and(|ids| ids.contains(&item_id));

        // Update under the lock, then clone the map for serialization outside
        // the critical section so I/O does not block concurrent feed() callers.
        let snapshot_data = {
            let mut stats = self.exploration_stats.lock().unwrap();
            let entry = stats.entry(user_id).or_default();
            if let Some(cat) = &category {
                entry.record_category_signal(cat);
            }
            if was_exploring {
                entry.record_outcome(is_positive);
            }
            if self.stats_path.is_some() {
                Some(stats.clone())
            } else {
                None
            }
        };
        // Serialize outside the lock — allocation and string formatting do not
        // belong inside a critical section.
        let snapshot = snapshot_data.and_then(|data| serde_json::to_string(&data).ok());

        // Atomic disk write: temp file + rename so a crash mid-write leaves
        // the previous file intact.
        if let Some(json) = snapshot
            && let Some(ref path) = self.stats_path
        {
            let tmp = path.with_extension("tmp");
            if std::fs::write(&tmp, &json).is_ok() {
                let _ = std::fs::rename(&tmp, path);
            }
        }
    }

    /// Register a dynamically-discovered web page as a Forage item.
    ///
    /// Uses a stable FNV-1a hash of the **canonicalized** URL as the item ID,
    /// so the operation is idempotent: AMP variants and trailing-slash duplicates
    /// of the same article map to the same item.  Calling `add_item` twice with
    /// the same canonical URL returns the same ID without writing to tidalDB.
    ///
    /// When an embedder is configured (`ForageEngine::builder().with_embedder(url)`),
    /// the page title + description are sent to the sidecar and the semantic vector
    /// is stored.  Without an embedder, a neutral unit vector is written so the
    /// item participates in ANN retrieval without biasing any centroid.
    pub fn add_item(&self, item: ForageItemInput) -> Result<u64> {
        let canonical = canonicalize_url(&item.url);
        let id = url_to_item_id(&canonical);
        let entity_id = EntityId::new(id);

        // Idempotent: if the item already exists, verify the URL matches and
        // return its ID.  A mismatch indicates an FNV-1a hash collision — this
        // is practically impossible at this scale (p < 10^-13 at 1000 items)
        // but we surface it rather than silently returning the wrong item ID.
        if let Some(existing) = self.db.get_item_metadata(entity_id)? {
            let stored_url = existing.get("url").map(String::as_str).unwrap_or("");
            if stored_url == canonical {
                return Ok(id);
            }
            return Err(ForageError::Seed(format!(
                "FNV-1a hash collision: url '{}' maps to id {id} already occupied by '{stored_url}'",
                canonical
            )));
        }

        let category = if item.category.is_empty() {
            "discovered".to_string()
        } else {
            item.category
        };

        let mut meta = HashMap::new();
        meta.insert("title".to_owned(), item.title.clone());
        meta.insert("source".to_owned(), item.source);
        meta.insert("category".to_owned(), category);
        meta.insert(
            "reading_time_min".to_owned(),
            item.reading_time_min.to_string(),
        );
        meta.insert("description".to_owned(), item.description.clone());
        meta.insert("url".to_owned(), canonical);
        meta.insert("tags".to_owned(), item.tags.join(","));
        meta.insert("entities".to_owned(), item.entities.join(","));
        meta.insert("content_type".to_owned(), item.content_type);
        meta.insert("summary".to_owned(), item.summary);
        self.db.write_item_with_metadata(entity_id, &meta)?;

        // Obtain embedding: call sidecar if configured, else neutral unit vector.
        let embedding = if self.embed_url.is_some() {
            let text = format!("{} — {}", item.title, item.description);
            self.embed_text(&text)
        } else {
            neutral_vector(self.embed_dim)
        };
        self.db.write_item_embedding(entity_id, &embedding)?;

        // Track so this item is injected into every feed pool regardless of signal score.
        // Cap at 1000 entries: oldest discoveries are evicted first (O(1) with VecDeque).
        let mut disc = self.discovered_ids.lock().unwrap();
        if disc.len() >= MAX_DISCOVERED_IDS {
            disc.pop_front();
        }
        disc.push_back(id);

        Ok(id)
    }

    /// Return the top-3 category names from the user's preference vector,
    /// ordered by dimension score descending.
    ///
    /// Returns an empty vec for cold-start users (no preference vector yet).
    pub fn top_categories(&self, user_id: u64) -> Vec<String> {
        const CATS: &[&str] = &[
            "technology",
            "science",
            "jazz",
            "travel",
            "cooking",
            "design",
            "history",
            "health",
        ];
        let Some(pref) = self.db.preference_vectors().get(user_id) else {
            return vec![];
        };
        let mut pairs: Vec<(&str, f32)> = CATS.iter().copied().zip(pref.iter().copied()).collect();
        pairs.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
        pairs
            .into_iter()
            .take(3)
            .filter(|(_, score)| *score > 0.0)
            .map(|(name, _)| name.to_string())
            .collect()
    }

    /// Return the top `limit` tags from items the user has positively engaged with.
    ///
    /// Scans both the user's **saved** item IDs and items where the user dwelled
    /// ≥15 seconds (article completion).  Per spec: "save + strong dwell ≥15s"
    /// are the positive engagement sources for tag affinity.
    ///
    /// Returns an empty vec for cold-start users or items with no enriched tags.
    pub fn top_tags(&self, user_id: u64, limit: usize) -> Vec<String> {
        // Collect item IDs from both saved and strong-dwell (completion) sets.
        // Use Forage-level sets (full u64 IDs) rather than tidalDB's
        // RoaringBitmap<u32> which truncates FNV-hash IDs for discovered items.
        let mut item_ids: HashSet<u64> = HashSet::new();
        {
            let saved = self.saved_items.lock().unwrap();
            if let Some(ids) = saved.get(&user_id) {
                item_ids.extend(ids.iter().copied());
            }
        }
        {
            let dwelled = self.dwelled_items.lock().unwrap();
            if let Some(ids) = dwelled.get(&user_id) {
                item_ids.extend(ids.iter().copied());
            }
        }
        if item_ids.is_empty() {
            return vec![];
        }

        let mut freq: HashMap<String, usize> = HashMap::new();
        for id in &item_ids {
            if let Ok(Some(m)) = self.db.get_item_metadata(EntityId::new(*id))
                && let Some(tags_str) = m.get("tags")
                && !tags_str.is_empty()
            {
                for tag in tags_str.split(',') {
                    *freq.entry(tag.to_string()).or_insert(0) += 1;
                }
            }
        }

        if freq.is_empty() {
            return vec![];
        }

        let mut pairs: Vec<(String, usize)> = freq.into_iter().collect();
        pairs.sort_by(|a, b| b.1.cmp(&a.1));
        pairs.into_iter().take(limit).map(|(tag, _)| tag).collect()
    }

    /// Build a browse plan for the discovery agent.
    ///
    /// Topics are weighted by the user's preference vector (from `top_categories`).
    /// Cold-start users receive equal weight across all 8 categories.
    /// `tag_hints` comes from `top_tags`.
    ///
    /// `item_count` is used to set `should_run: true` when the corpus is sparse.
    pub fn browse_tasks(&self, user_id: u64, limit_per_topic: usize) -> BrowsePlan {
        let top_cats = self.top_categories(user_id);

        let mut topics: Vec<BrowseTopic> = if top_cats.is_empty() {
            // Cold start: all 8 categories at equal priority.
            let equal_priority = 1.0 / sources::SOURCES.len() as f32;
            sources::SOURCES
                .iter()
                .map(|(name, srcs)| BrowseTopic {
                    name: (*name).to_string(),
                    priority: equal_priority,
                    sources: srcs.iter().map(|s| (*s).to_string()).collect(),
                })
                .collect()
        } else {
            // Warm user: top categories get linearly distributed priority,
            // remaining categories get a low exploration priority.
            let n = top_cats.len();
            let top_set: HashSet<&str> = top_cats.iter().map(String::as_str).collect();

            let mut all_topics: Vec<BrowseTopic> = Vec::with_capacity(sources::SOURCES.len());

            // Top categories: linearly decreasing priority from 1.0 down.
            for (rank, cat) in top_cats.iter().enumerate() {
                let priority = 1.0 - (rank as f32 / n as f32);
                let srcs = sources::sources_for(cat);
                all_topics.push(BrowseTopic {
                    name: cat.clone(),
                    priority,
                    sources: srcs.iter().map(|s| (*s).to_string()).collect(),
                });
            }

            // Remaining categories at low exploration priority.
            for (name, srcs) in sources::SOURCES {
                if !top_set.contains(*name) {
                    all_topics.push(BrowseTopic {
                        name: (*name).to_string(),
                        priority: 0.1,
                        sources: srcs.iter().map(|s| (*s).to_string()).collect(),
                    });
                }
            }

            all_topics
        };

        // Sort by priority descending.
        topics.sort_by(|a, b| {
            b.priority
                .partial_cmp(&a.priority)
                .unwrap_or(std::cmp::Ordering::Equal)
        });

        let tag_hints = self.top_tags(user_id, 5);

        BrowsePlan {
            should_run: true,
            interval_minutes: 30,
            topics,
            limit_per_topic,
            tag_hints,
        }
    }

    /// Retrieve a personalized feed for the given user.
    ///
    /// - **Warm path** (user has signaled ≥1 item): ANN query using the user's
    ///   learned preference vector, filtered to unseen items, reranked via `search`.
    /// - **Cold path** (no preference vector): global `for_you` hot ranking via
    ///   `retrieve`.
    ///
    /// Both paths widen the pool to 7× the requested limit, then pass through
    /// the epsilon-greedy MAB to blend exploit and explore slots.
    pub fn feed(&self, user_id: u64, limit: usize) -> Result<Vec<ForageItem>> {
        use tidaldb::query::retrieve::RetrieveResult;
        use tidaldb::storage::indexes::filter::FilterExpr;

        let pool_size = (limit * 7).min(100);
        // Build owned metadata map; discovered items are loaded from DB below.
        let mut meta_map: HashMap<u64, SeedItem> =
            self.seed_items.iter().map(|s| (s.id, s.clone())).collect();

        // Prefer ANN-guided retrieval when the user has a preference vector.
        let mut pool: Vec<RetrieveResult> =
            if let Some(pref_vec) = self.db.preference_vectors().get(user_id) {
                use tidaldb::query::search::Search;
                use tidaldb::schema::EntityKind;

                let query = Search::builder()
                    .entity_kind(EntityKind::Item)
                    .vector(pref_vec)
                    .for_user(user_id)
                    .filter(FilterExpr::Unseen(user_id))
                    .limit(pool_size as u32)
                    .build()
                    .map_err(|e| ForageError::Query(e.to_string()))?;

                let results = self.db.search(&query)?;
                // Blend semantic similarity with rank-position score using
                // `semantic_boost` as the weighting coefficient.
                //
                //   score = semantic_boost * semantic_similarity
                //         + (1 - semantic_boost) * rank_score
                //
                // This ensures items semantically near the user's preference
                // centroid score higher while still respecting retrieval rank.
                let result_count = results.items.len();
                let boost = f64::from(self.mab.semantic_boost);
                let mut pool: Vec<RetrieveResult> = results
                    .items
                    .into_iter()
                    .map(|r| {
                        // semantic_score is L2 distance; invert to similarity.
                        let semantic_sim = r
                            .semantic_score
                            .map(|d| 1.0_f64 / (1.0_f64 + f64::from(d)))
                            .unwrap_or(0.0);
                        // Rank-based score: position 0 → 1.0, last → ≈ 0.0.
                        let rank_score = if result_count > 1 {
                            1.0 - (r.rank as f64 / result_count as f64)
                        } else {
                            1.0
                        };
                        let score = boost * semantic_sim + (1.0 - boost) * rank_score;
                        RetrieveResult {
                            entity_id: r.entity_id,
                            score,
                            rank: r.rank,
                            signals: vec![],
                        }
                    })
                    .collect();
                // Sort descending by blended score.
                pool.sort_by(|a, b| {
                    b.score
                        .partial_cmp(&a.score)
                        .unwrap_or(std::cmp::Ordering::Equal)
                });
                pool
            } else {
                // Cold start: fall back to global hot ranking.
                use tidaldb::query::retrieve::{ProfileRef, Retrieve, RetrieveBuilder};
                use tidaldb::schema::EntityKind;

                let query: Retrieve =
                    RetrieveBuilder::new(EntityKind::Item, ProfileRef::new("for_you"))
                        .for_user(user_id)
                        .filter(FilterExpr::Unseen(user_id))
                        .limit(pool_size)
                        .build()
                        .map_err(|e| ForageError::Query(e.to_string()))?;

                // Interleave cold-start pool to ensure category diversity.
                // Without this, global signal dominance keeps one category
                // (e.g. technology) filling all exploit slots.
                let raw = self.db.retrieve(&query)?.items;
                interleave_categories(raw, &meta_map, pool_size)
            };

        // When `similar_to_saved` is enabled *and* a real embedder is configured,
        // augment the candidate pool with items semantically similar to the user's
        // saved content.  With category-axis vectors (no embedder) the ANN results
        // are not semantically meaningful, so we skip this augmentation to avoid
        // polluting the pool with misleading similarity candidates.
        if self.mab.similar_to_saved && self.embed_url.is_some() {
            use tidaldb::query::search::Search;
            use tidaldb::schema::EntityKind;

            let saved = self.db.user_state().saved_bitmap(user_id);
            if !saved.is_empty() {
                let mut in_pool: std::collections::HashSet<u64> =
                    pool.iter().map(|r| r.entity_id.as_u64()).collect();

                // Limit to first 3 saved items to keep latency bounded.
                for saved_id in saved.iter().take(3) {
                    let Ok(q) = Search::builder()
                        .entity_kind(EntityKind::Item)
                        .similar_to(EntityId::new(u64::from(saved_id)))
                        .filter(FilterExpr::Unseen(user_id))
                        .limit((pool_size / 3).max(1) as u32)
                        .build()
                    else {
                        continue;
                    };

                    let Ok(similar) = self.db.search(&q) else {
                        continue;
                    };

                    for r in similar.items {
                        let id = r.entity_id.as_u64();
                        if in_pool.insert(id) {
                            let score = r
                                .semantic_score
                                .map(|d| f64::from(1.0_f32 / (1.0 + d)))
                                .unwrap_or(0.4);
                            pool.push(RetrieveResult {
                                entity_id: r.entity_id,
                                score,
                                rank: r.rank,
                                signals: vec![],
                            });
                        }
                    }
                }

                // Re-sort after augmentation so MAB sees consistent ordering.
                pool.sort_by(|a, b| {
                    b.score
                        .partial_cmp(&a.score)
                        .unwrap_or(std::cmp::Ordering::Equal)
                });
            }
        }

        // Inject discovered items that aren't already in the pool.
        // Discovered items have very few signals and won't rank into the top-N on
        // their own; pinning them ensures they surface before accumulating history.
        {
            let disc_ids = self.discovered_ids.lock().unwrap().clone();
            let in_pool: std::collections::HashSet<u64> =
                pool.iter().map(|r| r.entity_id.as_u64()).collect();
            for disc_id in disc_ids {
                if !in_pool.contains(&disc_id) {
                    pool.push(tidaldb::query::retrieve::RetrieveResult {
                        entity_id: EntityId::new(disc_id),
                        score: 0.5,
                        rank: 0,
                        signals: vec![],
                    });
                }
            }
        }

        // Load metadata for any discovered items (IDs > 100_000) that are in the
        // pool but not in the seed corpus map.  This ensures they appear in the
        // feed rather than being silently skipped by the MAB.
        let seed_ids: std::collections::HashSet<u64> =
            self.seed_items.iter().map(|s| s.id).collect();
        for rr in &pool {
            let id = rr.entity_id.as_u64();
            if !seed_ids.contains(&id)
                && !meta_map.contains_key(&id)
                && let Ok(Some(m)) = self.db.get_item_metadata(rr.entity_id)
            {
                meta_map.insert(
                    id,
                    SeedItem {
                        id,
                        title: m.get("title").cloned().unwrap_or_default(),
                        source: m.get("source").cloned().unwrap_or_default(),
                        category: m
                            .get("category")
                            .cloned()
                            .unwrap_or_else(|| "discovered".to_string()),
                        reading_time_min: m
                            .get("reading_time_min")
                            .and_then(|s| s.parse().ok())
                            .unwrap_or(5),
                        description: m.get("description").cloned().unwrap_or_default(),
                        url: m.get("url").cloned().unwrap_or_default(),
                    },
                );
            }
        }

        // Compute per-user adaptive exploration ratio and snapshot UCB1 stats.
        // UCB1 is applied only within the exploration slot candidates in
        // mab::select() — the exploit pool order is never modified, so user
        // preference signals always govern exploit ranking.
        let (adaptive_ratio, ucb1_stats) = {
            let stats = self.exploration_stats.lock().unwrap();
            match stats.get(&user_id) {
                Some(s) => (s.adaptive_ratio(), Some(s.clone())),
                None => (self.mab.exploration_ratio, None),
            }
        };

        let mut items = mab::select(&pool, &meta_map, limit, adaptive_ratio, ucb1_stats.as_ref());

        // P4: Inject 1 bridge item at the semantic intersection of the user's
        // top-2 active categories.  Replaces the last *non-Exploring* item so
        // the Exploring budget from the MAB is preserved.
        // Warm users with ≥2 active categories receive a bridge; cold-start
        // users (no preference vector or only 1 active category) do not.
        if items.len() == limit {
            let in_feed: std::collections::HashSet<u64> = items.iter().map(|fi| fi.id).collect();
            if let Some(bridge) = self.make_bridge_item(user_id, &in_feed, &meta_map, limit) {
                // Replace the last non-Exploring item so MAB Exploring slots
                // are not displaced by the bridge injection.
                let replace_idx = items
                    .iter()
                    .rposition(|fi| !matches!(fi.label, ItemLabel::Exploring));
                if let Some(idx) = replace_idx {
                    items[idx] = bridge;
                    // Swap bridge to mid-feed position for natural appearance,
                    // but only if the target slot is also non-Exploring.
                    let mid = limit / 2;
                    if mid != idx && !matches!(items[mid].label, ItemLabel::Exploring) {
                        items.swap(mid, idx);
                    }
                }
            }
        } else {
            // Pool returned fewer items than the requested limit — bridge injection
            // is skipped because there are no surplus non-Exploring slots to replace.
            // This can happen after heavy unseen-item filtering on a sparse corpus.
            eprintln!(
                "[forage-engine] bridge injection skipped for user {user_id}: \
                 pool returned {} items (limit {limit})",
                items.len()
            );
        }

        // Load enrichment for discovered items (non-seed) after MAB selection.
        // mab::select() constructs ForageItem from SeedItem which has no enrichment
        // fields — we hydrate here from DB metadata for any discovered item.
        let seed_id_set: HashSet<u64> = self.seed_items.iter().map(|s| s.id).collect();
        for item in &mut items {
            if !seed_id_set.contains(&item.id)
                && item.tags.is_empty()
                && item.summary.is_empty()
                && let Ok(Some(m)) = self.db.get_item_metadata(EntityId::new(item.id))
            {
                item.tags = m
                    .get("tags")
                    .map(|s| {
                        if s.is_empty() {
                            vec![]
                        } else {
                            s.split(',').map(str::to_string).collect()
                        }
                    })
                    .unwrap_or_default();
                item.entities = m
                    .get("entities")
                    .map(|s| {
                        if s.is_empty() {
                            vec![]
                        } else {
                            s.split(',').map(str::to_string).collect()
                        }
                    })
                    .unwrap_or_default();
                item.content_type = m.get("content_type").cloned().unwrap_or_default();
                item.summary = m.get("summary").cloned().unwrap_or_default();
            }
        }

        // Record which item IDs were labeled Exploring so signal() can detect
        // exploration outcomes and update adaptive MAB state.
        {
            let exploring: HashSet<u64> = items
                .iter()
                .filter(|fi| matches!(fi.label, ItemLabel::Exploring))
                .map(|fi| fi.id)
                .collect();
            self.last_explore_items
                .lock()
                .unwrap()
                .insert(user_id, exploring);
        }

        Ok(items)
    }

    /// Return the per-user exploration stats for observability.
    ///
    /// Returns default (zeroed) stats for users with no exploration history.
    pub fn exploration_stats(&self, user_id: u64) -> mab::ExplorationStats {
        self.exploration_stats
            .lock()
            .unwrap()
            .get(&user_id)
            .cloned()
            .unwrap_or_default()
    }

    pub fn all_items(&self) -> &[SeedItem] {
        &self.seed_items
    }

    /// Semantic text search over the item corpus.
    ///
    /// When an embedder is configured (`ForageEngine::builder().with_embedder(url)`),
    /// the query text is sent to the sidecar and the resulting vector is used for
    /// ANN retrieval — enabling genuine meaning-based search (e.g. `"jazz theory"`
    /// returns jazz and music theory articles, not just items with those words).
    ///
    /// Without an embedder, the query falls back to a neutral unit vector, which
    /// produces unordered results.  Use the embedder sidecar for meaningful results.
    pub fn semantic_search(&self, text: &str, limit: usize) -> Result<Vec<ForageItem>> {
        use tidaldb::query::search::Search;
        use tidaldb::schema::EntityKind;

        let vec = self.embed_text(text);

        let query = Search::builder()
            .entity_kind(EntityKind::Item)
            .vector(vec)
            .limit(limit as u32)
            .build()
            .map_err(|e| ForageError::Query(e.to_string()))?;

        let results = self.db.search(&query)?;
        Ok(self.search_results_to_items(&results.items, ItemLabel::Match))
    }

    /// Return items semantically similar to the item with `item_id`.
    ///
    /// Reads the stored embedding for `item_id` and runs ANN retrieval using it
    /// as the query vector.  The source item is excluded from results.
    ///
    /// Returns an error if `item_id` is not found or has no stored embedding.
    pub fn similar_to(&self, item_id: u64, limit: usize) -> Result<Vec<ForageItem>> {
        use tidaldb::query::search::Search;
        use tidaldb::schema::EntityKind;

        let query = Search::builder()
            .entity_kind(EntityKind::Item)
            .similar_to(EntityId::new(item_id))
            .limit(limit as u32)
            .build()
            .map_err(|e| ForageError::Query(e.to_string()))?;

        let results = self.db.search(&query)?;
        Ok(self.search_results_to_items(&results.items, ItemLabel::Match))
    }

    // ── Internal helpers ──────────────────────────────────────────────────────

    /// Compute the P4 bridge item for the user's feed.
    ///
    /// Reads the user's preference vector, identifies the top-2 active category
    /// dimensions, computes the normalized midpoint vector, and runs an ANN query
    /// to find items at the semantic intersection of those two interests.
    ///
    /// Returns `None` when:
    /// - The user has no preference vector (cold start).
    /// - Fewer than 2 dimensions have meaningful signal (score < 0.05).
    /// - The ANN query returns no candidates outside `already_in_feed`.
    fn make_bridge_item(
        &self,
        user_id: u64,
        already_in_feed: &std::collections::HashSet<u64>,
        meta_map: &HashMap<u64, SeedItem>,
        limit: usize,
    ) -> Option<ForageItem> {
        let _ = limit; // reserved for future pool-size control
        use tidaldb::query::search::Search;
        use tidaldb::schema::EntityKind;
        use tidaldb::storage::indexes::filter::FilterExpr;

        const CATS: &[&str] = &[
            "technology",
            "science",
            "jazz",
            "travel",
            "cooking",
            "design",
            "history",
            "health",
        ];

        let pref = self.db.preference_vectors().get(user_id)?;

        // Build the midpoint vector and extract category names.
        // Two cases: category-axis 8-dim embeddings (P0–P3) vs real embedder.
        let (cat_a, cat_b, midpoint) = if pref.len() == CATS.len() {
            // 8-dim: find the top-2 dimensions by preference score.
            let mut indexed: Vec<(usize, f32)> = pref.iter().copied().enumerate().collect();
            indexed.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
            let (dim_a, score_a) = indexed[0];
            let (dim_b, score_b) = indexed[1];

            // Require both categories to have meaningful signal.
            if score_a < BRIDGE_MIN_CATEGORY_SCORE || score_b < BRIDGE_MIN_CATEGORY_SCORE {
                return None;
            }

            let cat_a = CATS.get(dim_a).copied()?.to_string();
            let cat_b = CATS.get(dim_b).copied()?.to_string();

            // Midpoint: equal weight on both category dimensions, L2-normalised.
            let mut mid = vec![0.0f32; pref.len()];
            mid[dim_a] = 1.0;
            mid[dim_b] = 1.0;
            let mag = mid.iter().map(|x| x * x).sum::<f32>().sqrt();
            if mag > 0.0 {
                for v in &mut mid {
                    *v /= mag;
                }
            }
            (cat_a, cat_b, mid)
        } else {
            // Real embedder (1536-dim): compute true per-category centroids by
            // averaging stored item embeddings for each top-2 category, then use
            // their normalized arithmetic midpoint as the ANN query vector.
            //
            //   midpoint = normalise((centroid_a + centroid_b) / 2)
            //
            // This places the query at the semantic intersection of the two
            // strongest interest directions, not biased toward either one.
            let cats = self.top_categories(user_id);
            if cats.len() < 2 {
                return None;
            }
            let cat_a = cats[0].clone();
            let cat_b = cats[1].clone();

            let centroid_a =
                compute_category_centroid(&self.db, &cat_a, &self.seed_items, meta_map)?;
            let centroid_b =
                compute_category_centroid(&self.db, &cat_b, &self.seed_items, meta_map)?;

            // Arithmetic midpoint, then L2-normalise.
            let mut mid: Vec<f32> = centroid_a
                .iter()
                .zip(centroid_b.iter())
                .map(|(a, b)| (a + b) * 0.5)
                .collect();
            let mag = mid.iter().map(|x| x * x).sum::<f32>().sqrt();
            if mag < f32::EPSILON {
                return None;
            }
            for v in &mut mid {
                *v /= mag;
            }

            (cat_a, cat_b, mid)
        };

        let query = Search::builder()
            .entity_kind(EntityKind::Item)
            .vector(midpoint)
            .filter(FilterExpr::Unseen(user_id))
            .limit(BRIDGE_CANDIDATE_LIMIT)
            .build()
            .ok()?;

        let results = self.db.search(&query).ok()?;

        // Return the first result not already showing in the feed.
        let seed_id_set: std::collections::HashSet<u64> =
            self.seed_items.iter().map(|s| s.id).collect();
        for r in results.items {
            let id = r.entity_id.as_u64();
            if already_in_feed.contains(&id) {
                continue;
            }
            if let Some(meta) = meta_map.get(&id) {
                let score = r.semantic_score.map(|d| 1.0_f32 / (1.0 + d)).unwrap_or(0.6);
                // Load enrichment from DB for discovered (non-seed) items.
                // Seed items never have tags/entities/content_type/summary.
                let (tags, entities, content_type, summary) = if seed_id_set.contains(&id) {
                    (vec![], vec![], String::new(), String::new())
                } else {
                    self.db
                        .get_item_metadata(EntityId::new(id))
                        .ok()
                        .flatten()
                        .map(|m| {
                            let parse_list = |key: &str| {
                                m.get(key)
                                    .map(|s| {
                                        if s.is_empty() {
                                            vec![]
                                        } else {
                                            s.split(',').map(str::to_string).collect()
                                        }
                                    })
                                    .unwrap_or_default()
                            };
                            (
                                parse_list("tags"),
                                parse_list("entities"),
                                m.get("content_type").cloned().unwrap_or_default(),
                                m.get("summary").cloned().unwrap_or_default(),
                            )
                        })
                        .unwrap_or_default()
                };
                return Some(ForageItem {
                    id,
                    title: meta.title.clone(),
                    source: meta.source.clone(),
                    category: meta.category.clone(),
                    reading_time_min: meta.reading_time_min,
                    description: meta.description.clone(),
                    label: ItemLabel::Bridge { cat_a, cat_b },
                    score,
                    url: meta.url.clone(),
                    tags,
                    entities,
                    content_type,
                    summary,
                });
            }
        }
        None
    }

    /// Convert search result items into `ForageItem`s by looking up metadata
    /// from the seed corpus (fast path) or live DB (for discovered items).
    fn search_results_to_items(
        &self,
        results: &[tidaldb::query::search::SearchResultItem],
        label: ItemLabel,
    ) -> Vec<ForageItem> {
        let seed_map: HashMap<u64, &SeedItem> = self.seed_items.iter().map(|s| (s.id, s)).collect();

        results
            .iter()
            .filter_map(|r| {
                let id = r.entity_id.as_u64();
                let score = r.semantic_score.map(|d| 1.0_f32 / (1.0 + d)).unwrap_or(0.5);

                if let Some(meta) = seed_map.get(&id) {
                    Some(ForageItem {
                        id,
                        title: meta.title.clone(),
                        source: meta.source.clone(),
                        category: meta.category.clone(),
                        reading_time_min: meta.reading_time_min,
                        description: meta.description.clone(),
                        label: label.clone(),
                        score,
                        url: meta.url.clone(),
                        tags: vec![],
                        entities: vec![],
                        content_type: String::new(),
                        summary: String::new(),
                    })
                } else {
                    self.db
                        .get_item_metadata(r.entity_id)
                        .ok()
                        .flatten()
                        .map(|m| ForageItem {
                            id,
                            title: m.get("title").cloned().unwrap_or_default(),
                            source: m.get("source").cloned().unwrap_or_default(),
                            category: m
                                .get("category")
                                .cloned()
                                .unwrap_or_else(|| "discovered".to_string()),
                            reading_time_min: m
                                .get("reading_time_min")
                                .and_then(|s| s.parse().ok())
                                .unwrap_or(5),
                            description: m.get("description").cloned().unwrap_or_default(),
                            label: label.clone(),
                            score,
                            url: m.get("url").cloned().unwrap_or_default(),
                            tags: m
                                .get("tags")
                                .map(|s| {
                                    if s.is_empty() {
                                        vec![]
                                    } else {
                                        s.split(',').map(str::to_string).collect()
                                    }
                                })
                                .unwrap_or_default(),
                            entities: m
                                .get("entities")
                                .map(|s| {
                                    if s.is_empty() {
                                        vec![]
                                    } else {
                                        s.split(',').map(str::to_string).collect()
                                    }
                                })
                                .unwrap_or_default(),
                            content_type: m.get("content_type").cloned().unwrap_or_default(),
                            summary: m.get("summary").cloned().unwrap_or_default(),
                        })
                }
            })
            .collect()
    }
}

#[derive(Debug, Clone, Copy)]
pub enum SignalKind {
    View,
    Save,
    Skip,
    Share,
}

// ── Persistence helpers ───────────────────────────────────────────────────────

/// Persist the saved and dwelled item sets to `user_state.json`.
///
/// Uses an atomic temp-file+rename pattern so a crash mid-write leaves the
/// previous file intact.  Called after every save/completion-dwell event.
impl ForageEngine {
    fn save_user_item_state(&self) {
        let Some(ref path) = self.user_state_path else {
            return;
        };
        // Snapshot both sets outside the I/O path to keep lock hold time short.
        let saved_snapshot: HashMap<String, Vec<u64>> = {
            let guard = self.saved_items.lock().unwrap();
            guard
                .iter()
                .map(|(k, v)| (k.to_string(), v.iter().copied().collect()))
                .collect()
        };
        let dwelled_snapshot: HashMap<String, Vec<u64>> = {
            let guard = self.dwelled_items.lock().unwrap();
            guard
                .iter()
                .map(|(k, v)| (k.to_string(), v.iter().copied().collect()))
                .collect()
        };
        let Ok(json) = serde_json::to_string(&serde_json::json!({
            "saved": saved_snapshot,
            "dwelled": dwelled_snapshot,
        })) else {
            return;
        };
        let tmp = path.with_extension("tmp");
        if std::fs::write(&tmp, &json).is_ok() {
            let _ = std::fs::rename(&tmp, path);
        }
    }
}

/// Load saved and dwelled item sets from `user_state.json`.
///
/// Returns two empty maps if the file does not exist.  Logs a warning and
/// returns empty maps if the file is corrupt, so the engine starts fresh
/// rather than refusing to open.
fn load_user_item_state(
    path: &std::path::Path,
) -> (HashMap<u64, HashSet<u64>>, HashMap<u64, HashSet<u64>>) {
    let Some(bytes) = std::fs::read(path).ok() else {
        return (HashMap::new(), HashMap::new());
    };
    let Ok(json): std::result::Result<serde_json::Value, _> = serde_json::from_slice(&bytes) else {
        eprintln!(
            "[forage-engine] user_state.json at {:?} is corrupt; starting with empty state",
            path
        );
        return (HashMap::new(), HashMap::new());
    };
    let parse_map = |v: &serde_json::Value| -> HashMap<u64, HashSet<u64>> {
        v.as_object()
            .map(|obj| {
                obj.iter()
                    .filter_map(|(k, ids)| {
                        let user_id: u64 = k.parse().ok()?;
                        let items: HashSet<u64> = ids
                            .as_array()?
                            .iter()
                            .filter_map(serde_json::Value::as_u64)
                            .collect();
                        Some((user_id, items))
                    })
                    .collect()
            })
            .unwrap_or_default()
    };
    (parse_map(&json["saved"]), parse_map(&json["dwelled"]))
}

/// Deserialize the per-user `ExplorationStats` map from the JSON file at `path`.
///
/// Returns `None` if the file does not exist.  Logs a warning and returns `None`
/// if the file exists but cannot be parsed, so the engine starts fresh rather than
/// failing to open.  A parse failure is worth surfacing because it can indicate
/// data corruption after an unclean shutdown.
fn load_exploration_stats(path: &std::path::Path) -> Option<HashMap<u64, mab::ExplorationStats>> {
    let bytes = std::fs::read(path).ok()?;
    serde_json::from_slice(&bytes)
        .map_err(|e| {
            eprintln!(
                "[forage-engine] exploration_stats.json at {:?} is corrupt ({e}); \
                 starting with empty exploration stats",
                path
            );
        })
        .ok()
}

// ── Bridge centroid helpers ───────────────────────────────────────────────────

/// Compute the L2-normalised centroid vector for all items in `category`.
///
/// Reads stored embeddings from `db` for every item in `category` (using
/// `seed_items` as the primary source and `meta_map` as a fallback for
/// discovered items), averages them component-wise, and L2-normalises the
/// result.  Returns `None` if no embeddings are found for the category.
///
/// Used by `make_bridge_item` in real-embedder mode to compute the true
/// semantic midpoint between two interest regions rather than using the
/// user's blended preference vector as a proxy.
fn compute_category_centroid(
    db: &TidalDb,
    category: &str,
    seed_items: &[SeedItem],
    meta_map: &HashMap<u64, SeedItem>,
) -> Option<Vec<f32>> {
    // Collect item IDs whose category matches, preferring the seed corpus.
    let ids: Vec<u64> = seed_items
        .iter()
        .filter(|s| s.category == category)
        .map(|s| s.id)
        .chain(
            meta_map
                .values()
                .filter(|m| m.category == category && !seed_items.iter().any(|s| s.id == m.id))
                .map(|m| m.id),
        )
        .collect();

    if ids.is_empty() {
        return None;
    }

    let mut sum: Option<Vec<f32>> = None;
    let mut count = 0u32;

    for id in ids {
        if let Ok(Some(vec)) = db.read_item_embedding(tidaldb::schema::EntityId::new(id)) {
            match &mut sum {
                None => sum = Some(vec),
                Some(s) => {
                    for (a, b) in s.iter_mut().zip(vec.iter()) {
                        *a += b;
                    }
                }
            }
            count += 1;
        }
    }

    let mut centroid = sum?;
    if count > 1 {
        let n = count as f32;
        for v in &mut centroid {
            *v /= n;
        }
    }

    // L2-normalise.
    let mag = centroid.iter().map(|x| x * x).sum::<f32>().sqrt();
    if mag < f32::EPSILON {
        return None;
    }
    for v in &mut centroid {
        *v /= mag;
    }
    Some(centroid)
}

// ── Embedding helpers ─────────────────────────────────────────────────────────

/// Return a neutral unit vector of the given dimension.
/// All components are equal (1/√dim), so the vector neither attracts nor
/// repels any centroid in the ANN index.
fn neutral_vector(dim: usize) -> Vec<f32> {
    let inv_sqrt = 1.0_f32 / (dim as f32).sqrt();
    vec![inv_sqrt; dim]
}

/// Synchronous HTTP call to the embedding sidecar.
///
/// Sends `POST {url}/embed` with `{ "text": text }` and parses the response
/// `{ "vector": [f32, ...] }`.
///
/// Takes a pre-built `client` so connection pooling is preserved across calls.
fn call_embedder(
    client: &reqwest::blocking::Client,
    base_url: &str,
    text: &str,
) -> std::result::Result<Vec<f32>, String> {
    let url = format!("{base_url}/embed");
    let resp = client
        .post(&url)
        .json(&serde_json::json!({ "text": text }))
        .send()
        .map_err(|e| format!("POST {url}: {e}"))?;

    if !resp.status().is_success() {
        let status = resp.status();
        let body = resp.text().unwrap_or_default();
        return Err(format!("embedder {status}: {body}"));
    }

    let json: serde_json::Value = resp.json().map_err(|e| format!("parse: {e}"))?;
    let vector = json["vector"]
        .as_array()
        .ok_or("missing 'vector' field")?
        .iter()
        .map(|v| v.as_f64().unwrap_or(0.0) as f32)
        .collect();
    Ok(vector)
}

// ── URL canonicalization ──────────────────────────────────────────────────────

/// Normalise a URL so that AMP variants and trailing-slash duplicates of the
/// same article map to a single stable item ID.
///
/// Rules applied in order:
/// 1. Strip `amp.` subdomain: `https://amp.example.com/…` → `https://example.com/…`
/// 2. Remove AMP query parameters (`amp`, `amp_tf`).
/// 3. Strip a trailing slash from the path unless the path is the root `/`.
fn canonicalize_url(url: &str) -> String {
    // 1. Strip amp. subdomain
    let s = if let Some(rest) = url.strip_prefix("https://amp.") {
        format!("https://{rest}")
    } else if let Some(rest) = url.strip_prefix("http://amp.") {
        format!("http://{rest}")
    } else {
        url.to_owned()
    };

    // 2. Separate path from query (ignore fragment)
    let (base, query_str) = match s.find('?') {
        Some(i) => (s[..i].to_owned(), Some(&s[i + 1..])),
        None => match s.find('#') {
            Some(i) => (s[..i].to_owned(), None),
            None => (s.clone(), None),
        },
    };

    // 3. Strip trailing slash only when there is a real path beyond the root
    //    "https://example.com/" → keep (root slash)
    //    "https://example.com/article/" → strip
    let base = {
        // Find position of the first slash after "://"
        let after_scheme = base.find("://").map_or(0, |i| i + 3);
        let first_path_slash = base[after_scheme..].find('/');
        let has_real_path = first_path_slash.is_some_and(|j| base.len() > after_scheme + j + 1);
        if has_real_path && base.ends_with('/') {
            base[..base.len() - 1].to_owned()
        } else {
            base
        }
    };

    // 4. Filter AMP query params
    let filtered = query_str.map(|q| {
        let kept: Vec<&str> = q
            .split('&')
            .filter(|p| {
                let k = p.split('=').next().unwrap_or("").to_lowercase();
                k != "amp" && k != "amp_tf"
            })
            .collect();
        if kept.is_empty() {
            None
        } else {
            Some(kept.join("&"))
        }
    });

    match filtered.flatten() {
        Some(q) => format!("{base}?{q}"),
        None => base,
    }
}

#[cfg(test)]
mod canon_tests {
    use super::canonicalize_url;

    #[test]
    fn strips_amp_subdomain_https() {
        assert_eq!(
            canonicalize_url("https://amp.example.com/article/123"),
            "https://example.com/article/123"
        );
    }

    #[test]
    fn strips_amp_subdomain_http() {
        assert_eq!(
            canonicalize_url("http://amp.cnn.com/story"),
            "http://cnn.com/story"
        );
    }

    #[test]
    fn removes_amp_query_param_standalone() {
        assert_eq!(
            canonicalize_url("https://example.com/article?amp=1"),
            "https://example.com/article"
        );
    }

    #[test]
    fn removes_amp_tf_query_param_among_others() {
        assert_eq!(
            canonicalize_url("https://example.com/a?q=rust&amp_tf=1&page=2"),
            "https://example.com/a?q=rust&page=2"
        );
    }

    #[test]
    fn strips_trailing_slash_from_real_path() {
        assert_eq!(
            canonicalize_url("https://example.com/article/"),
            "https://example.com/article"
        );
    }

    #[test]
    fn preserves_root_slash() {
        assert_eq!(
            canonicalize_url("https://example.com/"),
            "https://example.com/"
        );
    }

    #[test]
    fn preserves_meaningful_query_params() {
        let url = "https://example.com/search?q=rust&lang=en";
        assert_eq!(canonicalize_url(url), url);
    }

    #[test]
    fn amp_subdomain_plus_amp_param_both_stripped() {
        assert_eq!(
            canonicalize_url("https://amp.example.com/post/?amp=1"),
            "https://example.com/post"
        );
    }
}

/// Round-robin interleave items by category to ensure the cold-start exploit
/// pool spans ≥3 categories. Preserves score ordering within each category.
///
/// Algorithm: group items by category (preserving input order), then pull
/// one item at a time from each group until `limit` items are collected.
fn interleave_categories(
    items: Vec<tidaldb::query::retrieve::RetrieveResult>,
    meta_map: &HashMap<u64, SeedItem>,
    limit: usize,
) -> Vec<tidaldb::query::retrieve::RetrieveResult> {
    use std::collections::BTreeMap;

    // Group by category, preserving within-group order (best score first).
    let mut by_cat: BTreeMap<String, Vec<tidaldb::query::retrieve::RetrieveResult>> =
        BTreeMap::new();
    let mut no_meta: Vec<tidaldb::query::retrieve::RetrieveResult> = Vec::new();
    for item in items {
        if let Some(seed) = meta_map.get(&item.entity_id.as_u64()) {
            by_cat.entry(seed.category.clone()).or_default().push(item);
        } else {
            no_meta.push(item);
        }
    }

    // Round-robin across categories (BTreeMap gives deterministic order).
    let mut result = Vec::with_capacity(limit);
    let mut iterators: Vec<std::vec::IntoIter<tidaldb::query::retrieve::RetrieveResult>> =
        by_cat.into_values().map(|v| v.into_iter()).collect();
    if iterators.is_empty() {
        // All items lacked metadata; fall back to uncategorized list.
        result.extend(no_meta.into_iter().take(limit));
        return result;
    }
    let mut idx = 0;
    while result.len() < limit {
        let start = idx;
        let mut advanced = false;
        loop {
            if let Some(item) = iterators[idx].next() {
                result.push(item);
                advanced = true;
            }
            idx = (idx + 1) % iterators.len();
            if idx == start {
                break;
            }
        }
        if !advanced {
            break; // all iterators exhausted
        }
    }

    // Append any uncategorized items to fill if needed.
    for item in no_meta {
        if result.len() >= limit {
            break;
        }
        result.push(item);
    }

    result
}
