//! Evaluation harness for running LLM extraction against fixtures.
//!
//! The harness orchestrates:
//! - Loading fixtures
//! - Running extraction (with bounded concurrency)
//! - Matching results against expectations
//! - Computing metrics
//! - Generating reports

use std::path::PathBuf;
use std::time::Instant;

use serde::{Deserialize, Serialize};
use tracing::{debug, info, instrument, warn};
use uuid::Uuid;

use super::fixture::{BaselineMetrics, CorpusManifest, ExpectedClaim, Fixture, FixtureLoader};
use super::matcher::{count_false_positives, ClaimMatcher};
use super::metrics::{
    estimate_cost, BaselineComparison, FixtureResult, Metrics, UnmatchedExpectation,
    ViolationDetail,
};
use crate::config::EvalConfig;
use crate::error::Result;
use crate::llm::ontology::{AuthorityConcept, OntologyVocabulary, ValueType};
use crate::llm::{GeminiClient, LlmCache, LlmExtractor};
use crate::types::{ExtractedClaim, Language};

/// Configuration for an evaluation run.
#[derive(Debug, Clone)]
pub struct EvalRunConfig {
    /// Path to fixtures directory.
    pub fixtures_dir: PathBuf,

    /// Categories to evaluate (None = all).
    pub categories: Option<Vec<String>>,

    /// Maximum fixtures to run (for smoke tests).
    pub max_fixtures: Option<usize>,

    /// Evaluation mode.
    pub mode: EvalMode,

    /// Baseline file to compare against.
    pub baseline: Option<PathBuf>,

    /// Whether to save observations to the database.
    pub save_observations: bool,

    /// Maximum concurrent LLM calls.
    pub max_concurrent: usize,

    /// Regression threshold (e.g., 0.05 = 5%).
    pub regression_threshold: f64,

    /// LLM model identifier for reporting.
    pub model: String,

    /// Prompt version for tracking.
    pub prompt_version: String,
}

/// Current prompt version (update when prompt changes significantly).
pub const PROMPT_VERSION: &str = "1.0.0";

impl EvalRunConfig {
    /// Create config from EvalConfig with defaults.
    pub fn from_config(config: &EvalConfig, model: &str) -> Self {
        Self {
            fixtures_dir: config.fixtures_dir.clone(),
            categories: None,
            max_fixtures: None,
            mode: EvalMode::Cached,
            baseline: None,
            save_observations: config.save_observations,
            max_concurrent: config.max_concurrent,
            regression_threshold: config.regression_threshold,
            model: model.to_string(),
            prompt_version: PROMPT_VERSION.to_string(),
        }
    }
}

/// Evaluation mode.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EvalMode {
    /// Use real LLM API (costs money, tests actual prompt).
    Live,
    /// Use cached responses only (fast, deterministic, for CI).
    Cached,
    /// Skip LLM, return empty claims (for testing harness itself).
    Mock,
}

impl std::str::FromStr for EvalMode {
    type Err = String;

    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
        match s.to_lowercase().as_str() {
            "live" => Ok(EvalMode::Live),
            "cached" => Ok(EvalMode::Cached),
            "mock" => Ok(EvalMode::Mock),
            _ => Err(format!("Unknown eval mode: {}. Use: live, cached, mock", s)),
        }
    }
}

/// Result of an evaluation run.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EvalResult {
    /// Unique run identifier.
    pub run_id: Uuid,

    /// When the run started (RFC3339).
    pub started_at: String,

    /// When the run completed (RFC3339).
    pub completed_at: String,

    /// Evaluation mode used.
    pub mode: String,

    /// Prompt version evaluated.
    pub prompt_version: String,

    /// Model used.
    pub model: String,

    /// Aggregate metrics.
    pub metrics: Metrics,

    /// Per-fixture results.
    #[serde(skip_serializing)]
    pub fixture_results: Vec<FixtureResult>,

    /// Baseline comparison (if baseline provided).
    pub baseline_comparison: Option<BaselineComparison>,

    /// Overall verdict.
    pub verdict: EvalVerdict,
}

/// Verdict of an evaluation run.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum EvalVerdict {
    /// All checks passed.
    Pass,
    /// Some regressions detected.
    Regression,
    /// Review recommended (no regression but some failures).
    Review,
    /// Evaluation failed (errors prevented completion).
    Error,
}

impl std::fmt::Display for EvalVerdict {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            EvalVerdict::Pass => write!(f, "PASS"),
            EvalVerdict::Regression => write!(f, "REGRESSION"),
            EvalVerdict::Review => write!(f, "REVIEW"),
            EvalVerdict::Error => write!(f, "ERROR"),
        }
    }
}

/// Build an `OntologyVocabulary` from fixture expectations.
///
/// This extracts the subject/predicate/value_type from all `must_contain` claims
/// across fixtures, creating a vocabulary that constrains LLM output to match
/// the expected claims.
fn build_vocabulary_from_fixtures(fixtures: &[Fixture]) -> OntologyVocabulary {
    let mut seen = std::collections::HashSet::new();
    let mut concepts = Vec::new();

    for fixture in fixtures {
        for expected in &fixture.expected.must_contain {
            // Deduplicate by (subject, predicate)
            let key = (expected.subject.clone(), expected.predicate.clone());
            if seen.contains(&key) {
                continue;
            }
            seen.insert(key);

            let concept = expected_claim_to_concept(expected);
            concepts.push(concept);
        }

        // Also include acceptable_variants to allow LLM to use those
        for variant in &fixture.expected.acceptable_variants {
            let key = (variant.subject.clone(), variant.predicate.clone());
            if seen.contains(&key) {
                continue;
            }
            seen.insert(key);

            let concept = expected_claim_to_concept(variant);
            concepts.push(concept);
        }
    }

    debug!(concept_count = concepts.len(), "Built vocabulary from fixture expectations");
    OntologyVocabulary { concepts }
}

/// Convert an ExpectedClaim to an AuthorityConcept.
fn expected_claim_to_concept(expected: &ExpectedClaim) -> AuthorityConcept {
    let (value_type, example_value) = infer_value_type(&expected.value);

    AuthorityConcept {
        subject: expected.subject.clone(), // Use subject as full path too
        leaf_path: expected.subject.clone(),
        predicate: expected.predicate.clone(),
        value_type,
        example_value,
        description: expected
            .rationale
            .clone()
            .unwrap_or_else(|| format!("{} {}", expected.subject, expected.predicate)),
    }
}

/// Infer the value type from a serde_json::Value.
fn infer_value_type(value: &serde_json::Value) -> (ValueType, String) {
    match value {
        serde_json::Value::Bool(b) => (ValueType::Boolean, b.to_string()),
        serde_json::Value::Number(n) => (ValueType::Number, n.to_string()),
        serde_json::Value::String(s) => (ValueType::Text, s.clone()),
        _ => (ValueType::Text, value.to_string()),
    }
}

/// The evaluation harness.
pub struct EvalHarness {
    /// Configuration.
    config: EvalRunConfig,
    /// Fixture loader.
    loader: FixtureLoader,
    /// Claim matcher.
    matcher: ClaimMatcher,
    /// LLM extractor (optional, None in Mock mode).
    extractor: Option<LlmExtractor>,
    /// Loaded fixtures (cached after initial load).
    fixtures: Vec<Fixture>,
}

impl EvalHarness {
    /// Create a new evaluation harness.
    ///
    /// In Live mode, this loads fixtures first to build an ontology vocabulary,
    /// ensuring the LLM extractor outputs claims that match fixture expectations.
    pub fn new(config: EvalRunConfig) -> Result<Self> {
        let loader = FixtureLoader::new(&config.fixtures_dir);
        let matcher = ClaimMatcher::new();

        // Load fixtures first (needed for vocabulary extraction in Live mode)
        let categories = config.categories.as_deref();
        let mut fixtures = loader.load_all(categories)?;

        // Apply max_fixtures limit
        if let Some(max) = config.max_fixtures {
            fixtures.truncate(max);
        }

        info!(count = fixtures.len(), "Loaded fixtures for evaluation");

        // Create extractor for Live and Cached modes (not Mock)
        let extractor = if config.mode != EvalMode::Mock {
            // Build vocabulary from fixture expectations
            let vocabulary = build_vocabulary_from_fixtures(&fixtures);

            // Create LLM config - disable high_value_only filter for eval
            let llm_config = crate::config::LlmConfig {
                enabled: true,
                high_value_only: false, // Eval all fixtures, not just high-value files
                ..Default::default()
            };

            let cache_dir = dirs::cache_dir()
                .ok_or_else(|| {
                    crate::AphoriaError::Config(
                        "Cannot determine cache directory. Set $HOME or XDG_CACHE_HOME".to_string(),
                    )
                })?
                .join("aphoria")
                .join("llm_cache");
            let cache = LlmCache::new(cache_dir);

            if config.mode == EvalMode::Live {
                // Live mode: create client for API calls
                GeminiClient::new(&llm_config)?.map(|client| {
                    LlmExtractor::with_vocabulary(client, cache, llm_config, vocabulary)
                })
            } else {
                // Cached mode: use cache-only extractor (no API calls)
                Some(LlmExtractor::with_vocabulary_cached(cache, llm_config, vocabulary))
            }
        } else {
            None
        };

        Ok(Self { config, loader, matcher, extractor, fixtures })
    }

    /// Run the evaluation.
    #[instrument(skip(self), fields(mode = ?self.config.mode))]
    pub fn run(&self) -> Result<EvalResult> {
        let run_id = Uuid::new_v4();
        let started_at = chrono::Utc::now();
        info!(run_id = %run_id, "Starting evaluation run");

        // Fixtures are already loaded in new() - use cached fixtures
        info!(count = self.fixtures.len(), "Using cached fixtures");

        // Run evaluations
        let results: Vec<FixtureResult> =
            self.fixtures.iter().map(|fixture| self.evaluate_fixture(fixture)).collect();

        let completed_at = chrono::Utc::now();

        // Compute metrics
        let metrics = Metrics::compute(&results);

        // Load baseline for comparison if provided
        let baseline_comparison = self.load_and_compare_baseline(&metrics)?;

        // Determine verdict
        let verdict = self.determine_verdict(&metrics, &baseline_comparison);

        let result = EvalResult {
            run_id,
            started_at: started_at.to_rfc3339(),
            completed_at: completed_at.to_rfc3339(),
            mode: format!("{:?}", self.config.mode),
            prompt_version: self.config.prompt_version.clone(),
            model: self.config.model.clone(),
            metrics,
            fixture_results: results,
            baseline_comparison,
            verdict,
        };

        info!(
            verdict = %result.verdict,
            precision = %format!("{:.2}", result.metrics.precision),
            recall = %format!("{:.2}", result.metrics.recall),
            "Evaluation complete"
        );

        Ok(result)
    }

    /// Evaluate a single fixture.
    fn evaluate_fixture(&self, fixture: &Fixture) -> FixtureResult {
        let start = Instant::now();
        debug!(fixture_id = %fixture.metadata.id, "Evaluating fixture");

        // Extract claims based on mode
        let (claims, tokens, parse_success) = match self.config.mode {
            EvalMode::Mock => (Vec::new(), 0, true),
            EvalMode::Cached | EvalMode::Live => self.extract_claims(fixture),
        };

        let latency = start.elapsed().as_millis() as u64;

        // Match claims against expectations
        let must_contain_result =
            self.matcher.check_must_contain(&claims, &fixture.expected.must_contain);

        let violations =
            self.matcher.check_must_not_contain(&claims, &fixture.expected.must_not_contain);

        let false_positives = count_false_positives(
            &claims,
            &fixture.expected.must_contain,
            &fixture.expected.acceptable_variants,
            &self.matcher,
        );

        let tp = must_contain_result.true_positives();
        let fn_ = must_contain_result.false_negatives();
        let violation_count = violations.len();

        let cost = estimate_cost(tokens / 2, tokens / 2); // Rough split

        let mut result = FixtureResult::success(
            fixture.metadata.id.clone(),
            fixture.metadata.category.clone(),
            tp,
            false_positives,
            fn_,
            violation_count,
            tokens,
            cost,
            latency,
        );

        // Add details for unmatched expectations
        let unmatched: Vec<UnmatchedExpectation> = must_contain_result
            .unmatched
            .iter()
            .map(|exp| UnmatchedExpectation {
                subject: exp.subject.clone(),
                predicate: exp.predicate.clone(),
                expected_value: exp.value.clone(),
                rationale: exp.rationale.clone(),
            })
            .collect();

        // Add violation details
        let violation_details: Vec<ViolationDetail> = violations
            .iter()
            .map(|(exp, found)| ViolationDetail {
                subject: exp.subject.clone(),
                predicate: exp.predicate.clone(),
                found_value: format!("{:?}", found.value),
            })
            .collect();

        result = result.with_unmatched(unmatched).with_violations(violation_details);

        if !parse_success {
            result.parse_success = false;
        }

        debug!(
            fixture_id = %fixture.metadata.id,
            status = ?result.status,
            tp = tp,
            fp = false_positives,
            fn_ = fn_,
            "Fixture evaluated"
        );

        result
    }

    /// Extract claims from fixture content.
    fn extract_claims(&self, fixture: &Fixture) -> (Vec<ExtractedClaim>, usize, bool) {
        // In cached/live mode, we would call the LLM extractor
        // For now, return empty (mock behavior) until LLM is integrated
        if let Some(extractor) = &self.extractor {
            let language = Language::from_path(std::path::Path::new(&fixture.input.filename));

            let claims = extractor.extract(
                &[], // path segments
                &fixture.input.content,
                language,
                &fixture.input.filename,
            );

            let tokens = extractor.tokens_used();
            (claims, tokens, true)
        } else {
            // Mock mode: return empty claims
            (Vec::new(), 0, true)
        }
    }

    /// Load baseline and compare metrics.
    fn load_and_compare_baseline(&self, metrics: &Metrics) -> Result<Option<BaselineComparison>> {
        // Try to load baseline from manifest
        let manifest = match self.loader.load_manifest() {
            Ok(m) => m,
            Err(e) => {
                warn!(error = %e, "Could not load manifest for baseline comparison");
                return Ok(None);
            }
        };

        if let Some(baseline) = &manifest.baseline {
            let comparison =
                BaselineComparison::compare(metrics, baseline, self.config.regression_threshold);
            return Ok(Some(comparison));
        }

        Ok(None)
    }

    /// Determine the verdict based on metrics and baseline.
    fn determine_verdict(
        &self,
        metrics: &Metrics,
        baseline_comparison: &Option<BaselineComparison>,
    ) -> EvalVerdict {
        // Check for errors first
        if metrics.errored > 0 && metrics.errored == metrics.total_fixtures {
            return EvalVerdict::Error;
        }

        // Check for regression
        if let Some(comparison) = baseline_comparison {
            if comparison.has_regression {
                return EvalVerdict::Regression;
            }
        }

        // Check if all passed
        if metrics.failed == 0 && metrics.errored == 0 {
            return EvalVerdict::Pass;
        }

        // Some failures but no regression
        EvalVerdict::Review
    }

    /// Get the fixture loader (for listing, validation).
    pub fn loader(&self) -> &FixtureLoader {
        &self.loader
    }
}

/// Update the baseline in the manifest.
pub fn update_baseline(fixtures_dir: &std::path::Path, metrics: &Metrics) -> Result<()> {
    let manifest_path = fixtures_dir.join("manifest.toml");

    let mut manifest = if manifest_path.exists() {
        let content = std::fs::read_to_string(&manifest_path).map_err(|e| {
            crate::error::AphoriaError::Config(format!("Failed to read manifest: {}", e))
        })?;
        toml::from_str::<CorpusManifest>(&content).map_err(|e| {
            crate::error::AphoriaError::Config(format!("Failed to parse manifest: {}", e))
        })?
    } else {
        CorpusManifest {
            corpus: super::fixture::CorpusMetadata {
                version: "1.0.0".to_string(),
                created: Some(chrono::Utc::now().format("%Y-%m-%d").to_string()),
                description: Some("LLM extraction evaluation fixtures".to_string()),
            },
            categories: std::collections::HashMap::new(),
            baseline: None,
        }
    };

    manifest.baseline = Some(BaselineMetrics {
        precision: metrics.precision,
        recall: metrics.recall,
        f1: metrics.f1,
        total_fixtures: metrics.total_fixtures,
        prompt_version: "1.0.0".to_string(),
        model: "gemini-2.0-flash".to_string(),
        measured_at: chrono::Utc::now().to_rfc3339(),
    });

    let content = toml::to_string_pretty(&manifest).map_err(|e| {
        crate::error::AphoriaError::Config(format!("Failed to serialize manifest: {}", e))
    })?;

    std::fs::write(&manifest_path, content).map_err(|e| {
        crate::error::AphoriaError::Config(format!("Failed to write manifest: {}", e))
    })?;

    info!(path = %manifest_path.display(), "Updated baseline in manifest");
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::TempDir;

    fn setup_fixture_dir() -> TempDir {
        let temp_dir = TempDir::new().expect("temp dir");

        // Create manifest
        let manifest = r#"
[corpus]
version = "1.0.0"
description = "Test corpus"

[categories.tls]
fixtures = 1
description = "TLS fixtures"
"#;
        std::fs::write(temp_dir.path().join("manifest.toml"), manifest).expect("write manifest");

        // Create tls category
        let tls_dir = temp_dir.path().join("tls");
        std::fs::create_dir(&tls_dir).expect("create tls dir");

        // Create fixture
        let fixture = r#"
[metadata]
id = "tls-001"
name = "Test TLS fixture"
category = "tls"
language = "python"

[input]
filename = "client.py"
content = "verify=False"

[expected]
must_contain = [
    { subject = "tls/cert_verification", predicate = "enabled", value = false }
]
"#;
        std::fs::write(tls_dir.join("test.toml"), fixture).expect("write fixture");

        temp_dir
    }

    #[test]
    fn test_harness_mock_mode() {
        let temp_dir = setup_fixture_dir();

        let config = EvalRunConfig {
            fixtures_dir: temp_dir.path().to_path_buf(),
            categories: None,
            max_fixtures: None,
            mode: EvalMode::Mock,
            baseline: None,
            save_observations: false,
            max_concurrent: 1,
            regression_threshold: 0.05,
            model: "test-model".to_string(),
            prompt_version: PROMPT_VERSION.to_string(),
        };

        let harness = EvalHarness::new(config).expect("create harness");
        let result = harness.run().expect("run evaluation");

        assert_eq!(result.fixture_results.len(), 1);
        // In mock mode with no claims, all expectations fail
        assert_eq!(result.metrics.false_negatives, 1);
    }

    #[test]
    fn test_eval_mode_parsing() {
        assert_eq!("live".parse::<EvalMode>().unwrap(), EvalMode::Live);
        assert_eq!("cached".parse::<EvalMode>().unwrap(), EvalMode::Cached);
        assert_eq!("mock".parse::<EvalMode>().unwrap(), EvalMode::Mock);
        assert!("invalid".parse::<EvalMode>().is_err());
    }

    #[test]
    fn test_verdict_determination() {
        let temp_dir = setup_fixture_dir();

        let config = EvalRunConfig {
            fixtures_dir: temp_dir.path().to_path_buf(),
            categories: None,
            max_fixtures: None,
            mode: EvalMode::Mock,
            baseline: None,
            save_observations: false,
            max_concurrent: 1,
            regression_threshold: 0.05,
            model: "test-model".to_string(),
            prompt_version: PROMPT_VERSION.to_string(),
        };

        let harness = EvalHarness::new(config).expect("create harness");

        // With no baseline, failed fixtures -> Review
        let metrics = Metrics { failed: 1, ..Default::default() };
        let verdict = harness.determine_verdict(&metrics, &None);
        assert_eq!(verdict, EvalVerdict::Review);

        // All passed -> Pass
        let metrics =
            Metrics { total_fixtures: 1, passed: 1, failed: 0, errored: 0, ..Default::default() };
        let verdict = harness.determine_verdict(&metrics, &None);
        assert_eq!(verdict, EvalVerdict::Pass);
    }

    #[test]
    fn test_build_vocabulary_from_fixtures() {
        let fixtures = vec![
            Fixture {
                metadata: super::super::fixture::FixtureMetadata {
                    id: "tls-001".to_string(),
                    name: "TLS test".to_string(),
                    category: "tls".to_string(),
                    language: "python".to_string(),
                    difficulty: "easy".to_string(),
                    source: "test".to_string(),
                    created: None,
                    updated: None,
                    notes: None,
                },
                input: super::super::fixture::FixtureInput {
                    filename: "test.py".to_string(),
                    content: "verify=False".to_string(),
                },
                expected: super::super::fixture::FixtureExpected {
                    must_contain: vec![ExpectedClaim {
                        subject: "tls/cert_verification".to_string(),
                        predicate: "enabled".to_string(),
                        value: serde_json::json!(false),
                        min_confidence: None,
                        rationale: Some("TLS verification disabled".to_string()),
                    }],
                    must_not_contain: vec![],
                    acceptable_variants: vec![],
                },
                scoring: Default::default(),
            },
            Fixture {
                metadata: super::super::fixture::FixtureMetadata {
                    id: "secrets-001".to_string(),
                    name: "Secrets test".to_string(),
                    category: "secrets".to_string(),
                    language: "python".to_string(),
                    difficulty: "easy".to_string(),
                    source: "test".to_string(),
                    created: None,
                    updated: None,
                    notes: None,
                },
                input: super::super::fixture::FixtureInput {
                    filename: "config.py".to_string(),
                    content: "API_KEY = 'secret'".to_string(),
                },
                expected: super::super::fixture::FixtureExpected {
                    must_contain: vec![ExpectedClaim {
                        subject: "secrets/api_key".to_string(),
                        predicate: "hardcoded".to_string(),
                        value: serde_json::json!(true),
                        min_confidence: None,
                        rationale: Some("API key is hardcoded".to_string()),
                    }],
                    must_not_contain: vec![],
                    acceptable_variants: vec![],
                },
                scoring: Default::default(),
            },
        ];

        let vocab = build_vocabulary_from_fixtures(&fixtures);

        // Should have 2 concepts
        assert_eq!(vocab.concepts.len(), 2);

        // Check TLS concept
        let tls = vocab.find_by_leaf("tls/cert_verification");
        assert!(tls.is_some());
        let tls = tls.unwrap();
        assert_eq!(tls.predicate, "enabled");
        assert_eq!(tls.value_type, ValueType::Boolean);

        // Check secrets concept
        let secrets = vocab.find_by_leaf("secrets/api_key");
        assert!(secrets.is_some());
        let secrets = secrets.unwrap();
        assert_eq!(secrets.predicate, "hardcoded");
        assert_eq!(secrets.value_type, ValueType::Boolean);
    }

    #[test]
    fn test_infer_value_type() {
        let (vt, ex) = infer_value_type(&serde_json::json!(true));
        assert_eq!(vt, ValueType::Boolean);
        assert_eq!(ex, "true");

        let (vt, ex) = infer_value_type(&serde_json::json!(42));
        assert_eq!(vt, ValueType::Number);
        assert_eq!(ex, "42");

        let (vt, ex) = infer_value_type(&serde_json::json!("hello"));
        assert_eq!(vt, ValueType::Text);
        assert_eq!(ex, "hello");

        let (vt, ex) = infer_value_type(&serde_json::json!("sk-live-*"));
        assert_eq!(vt, ValueType::Text);
        assert_eq!(ex, "sk-live-*");
    }
}
