//! RETRIEVE query execution pipeline.
//!
//! Contains the `execute()` method that drives the 6-stage pipeline.
//! Stage 3 (signal scoring) is delegated to `stage3_score` in `mod.rs`.

#![allow(clippy::too_many_lines)]

use std::collections::HashSet;
use std::time::Instant;

use roaring::RoaringBitmap;

use crate::db::deserialize_metadata as deserialize_item_metadata;
use crate::query::retrieve::{Cursor, QueryError, Results, Retrieve, RetrieveResult, Signal};
use crate::query::stats::QueryStats;
use crate::ranking::diversity::DiversitySelector;
use crate::ranking::profile::{CandidateStrategy, Sort};
use crate::schema::{EntityId, Timestamp};
use crate::storage::indexes::bitmap::BitmapIndex;
use crate::storage::indexes::filter::{FilterEvaluator, FilterExpr, FilterResult};
use crate::storage::indexes::range::RangeIndex;
use crate::storage::{Tag, encode_key};

use super::RetrieveExecutor;
use super::candidate_gen;
use super::social_filter;
use super::user_filter;

impl RetrieveExecutor<'_> {
    /// Execute a RETRIEVE query through the 6-stage pipeline.
    ///
    /// # Errors
    ///
    /// Returns `QueryError` on validation failure, missing profile, or
    /// unsupported candidate strategy.
    #[allow(clippy::too_many_lines)]
    pub fn execute(&self, query: &Retrieve) -> Result<Results, QueryError> {
        let query_start = Instant::now();
        let mut stats = QueryStats::new(query.profile.name.clone());
        stats.degradation_level = self.degradation_level as u8;

        // Validate the query against the profile registry.
        query.validate(self.profile_registry)?;

        tracing::trace!(
            profile = %query.profile.name,
            limit = query.limit,
            has_filter = query.combined_filter().is_some(),
            has_diversity = query.diversity.is_some(),
            "retrieve query start"
        );

        // Resolve the profile.
        #[allow(clippy::option_if_let_else)]
        let profile = match query.profile.version {
            Some(v) => self
                .profile_registry
                .get_version(&query.profile.name, v)
                .map_err(|_| QueryError::ProfileNotFound(query.profile.name.clone()))?,
            None => self
                .profile_registry
                .get(&query.profile.name)
                .map_err(|_| QueryError::ProfileNotFound(query.profile.name.clone()))?,
        };

        let mut warnings: Vec<String> = Vec::new();

        // ── Stage 1: Candidate Generation ───────────────────────────────
        // When for_creator is specified, restrict candidates to that creator's
        // items using the CreatorItemsBitmap. This is O(k) where k = the
        // creator's item count, avoiding the O(N) scan cap that would miss
        // items at high IDs in a large catalog.
        let has_user_context = query.for_user.is_some();
        let mut candidates = if let Some(creator_id) = query.for_creator
            && let Some(creator_items) = self.creator_items
        {
            creator_items
                .get(creator_id.as_u64())
                .map_or_else(Vec::new, |bm| {
                    bm.iter()
                        .map(|id_u32| EntityId::new(u64::from(id_u32)))
                        .collect()
                })
        } else {
            match &profile.candidate_strategy {
                CandidateStrategy::Scan { .. } => {
                    candidate_gen::scan_candidates(self.universe, query.limit, has_user_context)
                }
                CandidateStrategy::SignalRanked { signal, .. } => {
                    candidate_gen::signal_ranked_candidates(self.ledger, signal, query.limit)
                }
                CandidateStrategy::Ann { .. } => {
                    // ANN candidate strategy falls back to scan with a warning.
                    warnings.push(
                        "ANN candidate strategy not yet wired; falling back to scan".to_string(),
                    );
                    candidate_gen::scan_candidates(self.universe, query.limit, has_user_context)
                }
                CandidateStrategy::Relationship => {
                    // M3: source candidates from the user's followed creators.
                    // Uses the follows index in user_state + creator_items bitmap to build
                    // a candidate set from items produced by followed creators.
                    if let (Some(user_id), Some(user_state), Some(creator_items)) =
                        (query.for_user, self.user_state, self.creator_items)
                    {
                        let followed = user_state.followed_creators_vec(user_id);
                        if followed.is_empty() {
                            warnings.push(
                            "Relationship strategy: user follows no creators; falling back to scan"
                                .to_string(),
                        );
                            candidate_gen::scan_candidates(self.universe, query.limit, true)
                        } else {
                            let bitmap = creator_items.union_for(&followed);
                            if bitmap.is_empty() {
                                warnings.push(
                                "Relationship strategy: followed creators have no items; falling back to scan"
                                    .to_string(),
                            );
                                candidate_gen::scan_candidates(self.universe, query.limit, true)
                            } else {
                                bitmap
                                    .iter()
                                    .map(|id_u32| EntityId::new(u64::from(id_u32)))
                                    .collect()
                            }
                        }
                    } else {
                        warnings.push(
                            "Relationship strategy requires FOR USER clause; falling back to scan"
                                .to_string(),
                        );
                        candidate_gen::scan_candidates(self.universe, query.limit, false)
                    }
                }
                other => {
                    return Err(QueryError::UnsupportedStrategy(format!(
                        "{other:?} requires M4+ infrastructure"
                    )));
                }
            }
        };

        // Apply exclude list.
        if !query.exclude.is_empty() {
            let exclude_set: HashSet<u64> = query.exclude.iter().map(|id| id.as_u64()).collect();
            candidates.retain(|id| !exclude_set.contains(&id.as_u64()));
        }

        // M7p2: ReducedCandidates — cap scan set to reduce Stage 3 scoring work.
        if self.degradation_level.reduces_candidates() {
            let cap = (query.limit * 4).max(100);
            candidates.truncate(cap);
        }

        stats.candidates_considered = candidates.len();

        tracing::trace!(
            candidates = candidates.len(),
            "stage 1: candidates generated"
        );

        // ── Stage 2: Filter Evaluation ──────────────────────────────────
        if let Some(filter_expr) = query.combined_filter() {
            // Build a per-query FilterEvaluator from whatever indexes are available.
            //
            // When an index is `None` (no items have been written to it yet), the
            // empty fallback causes filters on that field to match nothing -- which
            // is the correct semantic: if no items have a "category" field indexed,
            // then `CategoryEq("jazz")` should return zero results.
            let empty_bitmap = BitmapIndex::new("_empty");
            let empty_dur = RangeIndex::<u32>::new("_empty");
            let empty_ts = RangeIndex::<u64>::new("_empty");
            let empty_universe = RoaringBitmap::new();

            let cat = self.category_index.unwrap_or_else(|| {
                tracing::debug!(
                    "category_index unavailable; CategoryEq/In filters will return no matches"
                );
                &empty_bitmap
            });
            let fmt = self.format_index.unwrap_or_else(|| {
                tracing::debug!(
                    "format_index unavailable; FormatEq/In filters will return no matches"
                );
                &empty_bitmap
            });
            let cre = self.creator_index.unwrap_or(&empty_bitmap);
            let tag = self.tag_index.unwrap_or(&empty_bitmap);
            let dur = self.duration_index.unwrap_or(&empty_dur);
            let ts = self.created_at_index.unwrap_or(&empty_ts);

            // Read-lock the universe bitmap, fall back to empty if unavailable.
            let universe_guard;
            #[allow(clippy::option_if_let_else)]
            let universe_ref = match self.universe {
                Some(u) => match u.read() {
                    Ok(guard) => {
                        universe_guard = guard;
                        &*universe_guard
                    }
                    Err(_) => &empty_universe,
                },
                None => &empty_universe,
            };

            let evaluator = FilterEvaluator::new(cat, fmt, cre, tag, dur, ts, universe_ref);
            let filter_result = evaluator.evaluate(&filter_expr);

            match filter_result {
                FilterResult::Bitmap(bitmap) => {
                    // SAFETY: all entity IDs stored in RoaringBitmaps are 32-bit;
                    // truncation is intentional and correct throughout this module.
                    #[allow(clippy::cast_possible_truncation)]
                    candidates.retain(|id| bitmap.contains(id.as_u64() as u32));
                }
                FilterResult::Predicate(pred) => {
                    candidates.retain(|id| pred(id.as_u64()));
                }
            }
        }

        tracing::trace!(candidates = candidates.len(), "stage 2: filter applied");

        // ── Stage 2.2: InCollection Post-Filter (M6p4) ──────────────────
        if let Some(ref filter_expr) = query.combined_filter() {
            let coll_filters = user_filter::extract_collection_filters(filter_expr);
            if !coll_filters.is_empty()
                && let Some(coll_idx) = self.collection_index
            {
                for cf in &coll_filters {
                    if let FilterExpr::InCollection(cid) = cf {
                        if let Some(bm) = coll_idx.bitmap(*cid) {
                            #[allow(clippy::cast_possible_truncation)]
                            candidates.retain(|id| bm.contains(id.as_u64() as u32));
                        } else {
                            // Collection not found or empty -- retain nothing.
                            candidates.clear();
                        }
                    }
                }
                tracing::trace!(
                    candidates = candidates.len(),
                    "stage 2.2: InCollection filters applied"
                );
            }
        }

        // ── Stage 2.3: Signal Threshold Post-Filters (M6p3) ─────────────
        if let Some(ref filter_expr) = query.combined_filter() {
            let sig_filters = user_filter::extract_signal_threshold_filters(filter_expr);
            if !sig_filters.is_empty() {
                for sf in &sig_filters {
                    match sf {
                        FilterExpr::MinSignal { signal, threshold } => {
                            candidates.retain(|&eid| {
                                let val = crate::ranking::executor::helpers::read_agg(
                                    eid,
                                    signal,
                                    &crate::ranking::profile::SignalAgg::Value,
                                    crate::schema::Window::AllTime,
                                    self.ledger,
                                    self.degradation_level,
                                );
                                val >= *threshold
                            });
                        }
                        FilterExpr::MaxSignal { signal, threshold } => {
                            candidates.retain(|&eid| {
                                let val = crate::ranking::executor::helpers::read_agg(
                                    eid,
                                    signal,
                                    &crate::ranking::profile::SignalAgg::Value,
                                    crate::schema::Window::AllTime,
                                    self.ledger,
                                    self.degradation_level,
                                );
                                val <= *threshold
                            });
                        }
                        _ => {}
                    }
                }
                tracing::trace!(
                    candidates = candidates.len(),
                    "stage 2.3: signal threshold filters applied"
                );
            }
        }

        // ── Stage 2.4: Geographic Post-Filter (M6p3) ────────────────────
        if let Some(ref filter_expr) = query.combined_filter() {
            let geo_filters = user_filter::extract_geo_filters(filter_expr);
            if !geo_filters.is_empty()
                && let Some(storage) = self.items_storage
            {
                candidates.retain(|&eid| {
                    let key = encode_key(eid, Tag::Meta, b"");
                    let meta = storage
                        .get(&key)
                        .ok()
                        .flatten()
                        .map(|b| deserialize_item_metadata(&b))
                        .unwrap_or_default();
                    geo_filters.iter().all(|gf| match gf {
                        FilterExpr::NearLocation {
                            lat,
                            lng,
                            radius_km,
                        } => {
                            let item_lat = meta.get("latitude").and_then(|v| v.parse::<f64>().ok());
                            let item_lng =
                                meta.get("longitude").and_then(|v| v.parse::<f64>().ok());
                            match (item_lat, item_lng) {
                                (Some(ilat), Some(ilng)) => {
                                    crate::ranking::executor::formulas::haversine_km(
                                        *lat, *lng, ilat, ilng,
                                    ) <= *radius_km
                                }
                                _ => false,
                            }
                        }
                        _ => true,
                    })
                });
                tracing::trace!(
                    candidates = candidates.len(),
                    "stage 2.4: geographic filters applied"
                );
            }
        }

        // ── Stage 2.5: User-Context Filtering (M3) ─────────────────────
        // When FOR USER is specified, exclude seen items, hidden items,
        // items from blocked creators, and hard negatives.
        if let Some(user_id) = query.for_user {
            if let Some(user_state) = self.user_state {
                // Exclude seen items.
                let seen = user_state.seen_bitmap(user_id);
                #[allow(clippy::cast_possible_truncation)]
                candidates.retain(|id| !seen.contains(id.as_u64() as u32));

                // Exclude hidden items.
                let hidden = user_state.hidden_items(user_id);
                #[allow(clippy::cast_possible_truncation)]
                candidates.retain(|id| !hidden.contains(id.as_u64() as u32));

                // Exclude items from blocked creators.
                if let Some(creator_items) = self.creator_items {
                    let blocked_creators = user_state.blocked_creators(user_id);
                    if !blocked_creators.is_empty() {
                        let mut blocked_items = RoaringBitmap::new();
                        for &cid in &blocked_creators {
                            if let Some(bm) = creator_items.get(cid) {
                                blocked_items |= &bm;
                            }
                        }
                        #[allow(clippy::cast_possible_truncation)]
                        candidates.retain(|id| !blocked_items.contains(id.as_u64() as u32));
                    }
                }

                // ── User-state inclusion filters (Saved, Liked, InProgress) ──
                // These are INCLUSION filters: retain only matching items, unlike
                // the exclusion filters above that remove seen/blocked/hard-neg.
                if let Some(ref filter_expr) = query.combined_filter() {
                    let user_filters = user_filter::extract_user_state_filters(filter_expr);
                    for uf in &user_filters {
                        match uf {
                            FilterExpr::Saved(uid) => {
                                let saved = user_state.saved_bitmap(*uid);
                                #[allow(clippy::cast_possible_truncation)]
                                candidates.retain(|id| saved.contains(id.as_u64() as u32));
                            }
                            FilterExpr::Liked(uid) => {
                                let liked = user_state.liked_bitmap(*uid);
                                #[allow(clippy::cast_possible_truncation)]
                                candidates.retain(|id| liked.contains(id.as_u64() as u32));
                            }
                            FilterExpr::InProgress {
                                user_id: uid,
                                threshold,
                            } => {
                                candidates.retain(|id| {
                                    user_state.is_in_progress(*uid, id.as_u64(), *threshold)
                                });
                            }
                            FilterExpr::SocialGraph {
                                user_id: uid,
                                depth,
                            } => {
                                // Social graph: constrain to items from followed creators
                                // (depth=1) or expanded community items (depth=2).
                                if let Some(ci) = self.creator_items {
                                    let sg_bitmap = social_filter::social_graph_bitmap(
                                        *uid, *depth, user_state, ci,
                                    );
                                    #[allow(clippy::cast_possible_truncation)]
                                    candidates.retain(|id| sg_bitmap.contains(id.as_u64() as u32));
                                }
                            }
                            // Other variants are not user-state inclusion filters.
                            _ => {}
                        }
                    }
                }
            }

            // Exclude hard negatives.
            if let Some(hard_neg) = self.hard_negatives {
                let neg_bitmap = hard_neg.bitmap(user_id);
                if !neg_bitmap.is_empty() {
                    #[allow(clippy::cast_possible_truncation)]
                    candidates.retain(|id| !neg_bitmap.contains(id.as_u64() as u32));
                }
            }

            tracing::trace!(
                candidates = candidates.len(),
                user_id = user_id,
                "stage 2.5: user-context filter applied"
            );
        }

        stats.candidates_after_filter = candidates.len();
        stats.filters_applied = query.filters.len();

        // ── DateSaved validation (M6p3) ─────────────────────────────────
        // DateSaved sort requires FOR USER context. Fail early with a clear
        // error rather than silently returning bad ordering.
        if matches!(profile.sort, Some(Sort::DateSaved)) && query.for_user.is_none() {
            return Err(QueryError::InvalidFilter {
                field: "sort".to_string(),
                reason: "DateSaved sort requires FOR USER context".to_string(),
            });
        }

        // ── Stage 3: Signal Scoring ─────────────────────────────────────
        let scoring_start = Instant::now();
        let now = Timestamp::now();
        let (scored, _item_metadata) =
            self.stage3_score(&candidates, profile, query, now, &mut stats, &mut warnings)?;
        let total_scored = scored.len();
        stats.scoring_time_us = scoring_start.elapsed().as_micros() as u64;
        tracing::trace!(scored = scored.len(), "stage 3: scored");

        // ── Stage 4: Diversity Enforcement ──────────────────────────────
        // Use explicit query diversity if set; otherwise fall back to the profile's
        // DiversitySpec so that built-in profiles (e.g. notification: max_per_creator=1)
        // are always enforced even when the caller does not set query.diversity.
        let profile_diversity: Option<crate::ranking::diversity::DiversityConstraints> = {
            let spec = &profile.diversity;
            if spec.max_per_creator.is_some() || spec.format_mix_max_fraction.is_some() {
                Some(crate::ranking::diversity::DiversityConstraints {
                    max_per_creator: spec.max_per_creator,
                    format_mix_max_fraction: spec.format_mix_max_fraction,
                    ..crate::ranking::diversity::DiversityConstraints::default()
                })
            } else {
                None
            }
        };
        let effective_diversity = query.diversity.as_ref().or(profile_diversity.as_ref());
        // M7p2: NoDiversity — skip diversity enforcement entirely under load.
        let diversity_start = Instant::now();
        let (final_candidates, constraints_satisfied) = if self.degradation_level.skips_diversity()
        {
            warnings.push("diversity enforcement skipped due to load degradation".to_string());
            (scored, true)
        } else if let Some(diversity) = effective_diversity {
            // Pass the full scored set (not query.limit) so diversity constraints are
            // applied before pagination slicing. This ensures the page boundary does
            // not artificially inflate or reduce the diversity-enforced count.
            let result = DiversitySelector::select(&scored, diversity, scored.len());
            let satisfied = result.constraints_satisfied;
            if !satisfied {
                for v in &result.violations {
                    warnings.push(format!(
                        "diversity constraint '{}' relaxed: {}",
                        v.constraint, v.detail
                    ));
                }
            }
            (result.selected, satisfied)
        } else {
            (scored, true)
        };

        stats.diversity_time_us = diversity_start.elapsed().as_micros() as u64;
        stats.candidates_after_diversity = final_candidates.len();

        tracing::trace!(
            final_count = final_candidates.len(),
            "stage 4: diversity enforced"
        );

        // ── Stage 4.5: Notification Cap Enforcement (M6p6) ──────────────
        let final_candidates =
            if let (Some(caps), Some(user_id)) = (query.notification_caps, query.for_user) {
                self.apply_notification_caps(final_candidates, caps, user_id, &query.profile.name)
            } else {
                final_candidates
            };

        tracing::trace!(
            final_count = final_candidates.len(),
            "stage 4.5: notification caps applied"
        );

        // ── Stage 5: Result Assembly ────────────────────────────────────
        let offset = query.cursor.as_ref().map_or(0, Cursor::offset);
        let end = (offset + query.limit).min(final_candidates.len());
        let page = if offset < final_candidates.len() {
            &final_candidates[offset..end]
        } else {
            &[]
        };

        let items: Vec<RetrieveResult> = page
            .iter()
            .enumerate()
            .map(|(i, c)| {
                let signals: Vec<Signal> = c
                    .signal_snapshot
                    .iter()
                    .map(|(name, value)| Signal {
                        name: name.clone(),
                        value: *value,
                        source: "decay_score".to_string(),
                    })
                    .collect();

                RetrieveResult {
                    entity_id: c.entity_id,
                    score: c.score,
                    rank: offset + i + 1,
                    signals,
                }
            })
            .collect();

        // M2: offset-based pagination. The cursor encodes a list position rather
        // than a (score, entity_id) keyset. This is stable only when the ranked
        // list does not change between page requests (no concurrent signal writes
        // during iteration). Replace with keyset pagination in M3.
        let next_cursor = if end < final_candidates.len() {
            Some(Cursor::from_offset(end))
        } else {
            None
        };

        tracing::trace!(
            returned = items.len(),
            has_next_cursor = next_cursor.is_some(),
            "stage 5: results assembled"
        );

        stats.total_time_us = query_start.elapsed().as_micros() as u64;

        Ok(Results {
            items,
            next_cursor,
            total_candidates: total_scored,
            constraints_satisfied,
            warnings,
            session_snapshot: self.session_snapshot.clone(),
            degradation_level: self.degradation_level,
            stats,
        })
    }
}
