//! Promotion pipeline for converting learned patterns to declarative extractors. //! //! Orchestrates the full promotion flow: candidates → regex generation → validation → YAML output. use std::path::PathBuf; use tracing::{debug, info, warn}; use uuid::Uuid; /// Result of smart autonomous promotion. #[derive(Debug, Default)] pub struct SmartPromotionResult { /// Number of patterns auto-promoted (no human review). pub auto_promoted: usize, /// Number of patterns that require human review. pub requires_review: usize, /// Paths to promoted YAML files. pub promoted_files: Vec, /// Errors encountered during processing. pub errors: Vec, } use super::audit::AutonomousAuditLog; use super::regex_gen::RegexGenerator; use super::types::{PromotionCandidate, PromotionStats, ValidationResult}; use super::validator::ExtractorValidator; use super::writer::YamlWriter; use crate::config::{AutonomousConfig, GovernanceConfig, PromotionConfig}; use crate::governance::{ApprovalStatus, GovernanceStateMachine, GovernanceStore}; use crate::learning::{LearnedPattern, PatternStore}; use crate::llm::GeminiClient; use crate::AphoriaError; /// The promotion pipeline orchestrates pattern-to-extractor conversion. pub struct PromotionPipeline<'a, S: PatternStore> { /// Pattern store for fetching candidates. store: &'a S, /// LLM client for regex generation. client: Option<&'a GeminiClient>, /// Configuration for promotion thresholds. config: &'a PromotionConfig, /// Validator for testing generated extractors. validator: ExtractorValidator, /// YAML writer for output. writer: Option, } impl<'a, S: PatternStore> PromotionPipeline<'a, S> { /// Create a new promotion pipeline. /// /// If `output_dir` is None, uses the default `.aphoria/extractors/learned/`. pub fn new( store: &'a S, client: Option<&'a GeminiClient>, config: &'a PromotionConfig, output_dir: Option, ) -> Result { let writer = if let Some(dir) = output_dir { Some(YamlWriter::new(dir)?) } else { None }; Ok(Self { store, client, config, validator: ExtractorValidator::default(), writer }) } /// Get patterns eligible for promotion. /// /// Returns patterns that meet the configured thresholds for project count /// and confidence. pub fn get_candidates(&self) -> Vec { self.store.get_promotion_candidates(self.config.min_projects, self.config.min_confidence) } /// Generate a promotion candidate from a learned pattern. /// /// Uses the LLM to generate a regex pattern and validates it. pub fn generate_candidate( &self, pattern: &LearnedPattern, ) -> Result { let client = self.client.ok_or_else(|| { AphoriaError::Promotion("LLM client not configured for regex generation".to_string()) })?; // Generate extractor definition using LLM let generator = RegexGenerator::new(client); let extractor_def = generator.generate(pattern)?; // Validate the generated extractor let validation = self.validator.validate(&extractor_def, pattern)?; Ok(PromotionCandidate::new(pattern.clone(), extractor_def, validation)) } /// Promote a candidate by writing it to YAML and marking the pattern as promoted. /// /// Returns the path to the written YAML file. /// /// If governance is enabled, this will check for an approved governance request. /// If no request exists or it's not approved, the promotion will be blocked. pub fn promote(&self, candidate: &PromotionCandidate) -> Result { self.promote_with_governance(candidate, None) } /// Promote with optional governance configuration. /// /// When `governance_config` is provided and enabled, checks for governance approval. pub fn promote_with_governance( &self, candidate: &PromotionCandidate, governance_config: Option<&GovernanceConfig>, ) -> Result { // Check if candidate is ready if !candidate.is_ready() { return Err(AphoriaError::Promotion(format!( "Candidate {} is not ready for promotion: validation={}, performance={}", candidate.pattern_id(), candidate.validation.passed, candidate.validation.performance_ok ))); } // Check governance if enabled if let Some(gov_config) = governance_config { if gov_config.enabled { self.check_governance_approval(candidate, gov_config)?; } } // Get or create writer let writer = if let Some(ref w) = self.writer { w } else { return Err(AphoriaError::Promotion("YAML writer not configured".to_string())); }; // Check if already exists if writer.exists(candidate.extractor_name()) { return Err(AphoriaError::Promotion(format!( "Extractor '{}' already exists", candidate.extractor_name() ))); } // Write YAML file let path = writer.write(&candidate.extractor_def, &candidate.pattern)?; // Mark pattern as promoted self.store.mark_promoted(&candidate.pattern_id(), candidate.extractor_name())?; info!( pattern_id = %candidate.pattern_id(), extractor = %candidate.extractor_name(), path = %path.display(), "Pattern promoted to extractor" ); Ok(path) } /// Check if a pattern has governance approval for promotion. fn check_governance_approval( &self, candidate: &PromotionCandidate, governance_config: &GovernanceConfig, ) -> Result<(), AphoriaError> { let store = GovernanceStore::open_default()?; let pattern_id = candidate.pattern_id(); match store.get_request_by_pattern(&pattern_id)? { Some(request) => { match &request.status { ApprovalStatus::Approved => { // Approved - can proceed with promotion debug!( pattern_id = %pattern_id, request_id = %request.id, "Governance approval verified" ); Ok(()) } ApprovalStatus::Pending { stage } => Err(AphoriaError::Promotion(format!( "Pattern awaiting governance approval at stage '{}'. Request ID: {}", stage, request.id ))), ApprovalStatus::Rejected { stage, reason } => { Err(AphoriaError::Promotion(format!( "Pattern was rejected at stage '{}': {}. Request ID: {}", stage, reason, request.id ))) } ApprovalStatus::Escalated { from_stage, to_stage } => { Err(AphoriaError::Promotion(format!( "Pattern was escalated from '{}' to '{}'. Request ID: {}", from_stage, to_stage, request.id ))) } ApprovalStatus::Expired => Err(AphoriaError::Promotion(format!( "Pattern approval request expired. Create a new request. Request ID: {}", request.id ))), } } None => { // No approval request exists - create one self.create_governance_request(candidate, governance_config)?; Err(AphoriaError::Promotion( "Approval request created. Pattern awaiting governance review.".to_string(), )) } } } /// Create a governance approval request for a pattern. fn create_governance_request( &self, candidate: &PromotionCandidate, governance_config: &GovernanceConfig, ) -> Result<(), AphoriaError> { let sm = GovernanceStateMachine::open_default(governance_config.clone())?; // Get the appropriate workflow let workflow = sm.get_workflow_for_pattern(&candidate.pattern).ok_or_else(|| { AphoriaError::Promotion( "No governance workflow configured for this pattern".to_string(), ) })?; // Create the request let creator = whoami::username(); let request = sm.create_request(&candidate.pattern, &workflow, &creator)?; info!( pattern_id = %candidate.pattern_id(), request_id = %request.id, workflow = %workflow.name, "Created governance approval request" ); Ok(()) } /// Process all eligible patterns and return promotion candidates. /// /// Generates and validates extractors for each eligible pattern. /// Does not actually promote (write YAML) - use `promote()` for that. pub fn process_all(&self) -> Vec> { let patterns = self.get_candidates(); debug!(count = patterns.len(), "Processing promotion candidates"); patterns.iter().map(|pattern| self.generate_candidate(pattern)).collect() } /// Auto-promote all ready candidates. /// /// Only runs if `auto_promote` is enabled in config. /// Returns the number of patterns promoted and any errors. pub fn auto_promote_all(&self) -> (usize, Vec) { if !self.config.auto_promote { warn!("auto_promote is disabled in config"); return (0, vec![]); } let candidates = self.process_all(); let mut promoted = 0; let mut errors = Vec::new(); for result in candidates { match result { Ok(candidate) if candidate.is_ready() => match self.promote(&candidate) { Ok(_) => promoted += 1, Err(e) => errors.push(e), }, Ok(candidate) => { debug!( pattern_id = %candidate.pattern_id(), "Candidate not ready for auto-promotion" ); } Err(e) => errors.push(e), } } (promoted, errors) } /// Smart auto-promote with autonomous decision logic. /// /// Unlike `auto_promote_all()` which uses the basic `auto_promote` flag, /// this method applies stricter thresholds from `AutonomousConfig` and /// logs all decisions to an audit trail. /// /// # Returns /// /// A tuple of (auto_promoted_count, requires_review_count, errors). /// /// # Behavior /// /// For each eligible candidate: /// 1. Checks `should_auto_promote()` against autonomous thresholds /// 2. If eligible: promotes and logs "auto_promoted" decision /// 3. If not eligible: logs "requires_review" decision with blockers pub fn smart_auto_promote_all( &self, autonomous_config: &AutonomousConfig, ) -> Result { let mut result = SmartPromotionResult::default(); // Check kill switch if !autonomous_config.enabled { warn!("Autonomous promotion is disabled (kill switch is off)"); return Ok(result); } // Create audit log if enabled let audit_log = if autonomous_config.audit_log { Some(AutonomousAuditLog::new(autonomous_config.audit_dir.as_ref())?) } else { None }; // Process all candidates let candidates = self.process_all(); for candidate_result in candidates { match candidate_result { Ok(candidate) => { if candidate.should_auto_promote(autonomous_config) { // Promote autonomously match self.promote_autonomous(&candidate) { Ok(path) => { result.auto_promoted += 1; result.promoted_files.push(path.clone()); // Log the decision if let Some(ref log) = audit_log { if let Err(e) = log.record_promoted(&candidate, autonomous_config, path) { warn!(error = %e, "Failed to record audit log"); } } info!( pattern_id = %candidate.pattern_id(), extractor = %candidate.extractor_name(), "Autonomously promoted pattern" ); } Err(e) => { result.errors.push(e); } } } else { // Requires human review result.requires_review += 1; // Log the decision with blockers if let Some(ref log) = audit_log { if let Err(e) = log.record_requires_review(&candidate, autonomous_config) { warn!(error = %e, "Failed to record audit log"); } } debug!( pattern_id = %candidate.pattern_id(), blockers = ?candidate.auto_promotion_blockers(autonomous_config), "Pattern requires human review" ); } } Err(e) => { result.errors.push(e); } } } Ok(result) } /// Promote a candidate autonomously (sets auto_promoted metadata). fn promote_autonomous(&self, candidate: &PromotionCandidate) -> Result { // Check if candidate is ready if !candidate.is_ready() { return Err(AphoriaError::Promotion(format!( "Candidate {} is not ready for promotion: validation={}, performance={}", candidate.pattern_id(), candidate.validation.passed, candidate.validation.performance_ok ))); } // Get or create writer let writer = if let Some(ref w) = self.writer { w } else { return Err(AphoriaError::Promotion("YAML writer not configured".to_string())); }; // Check if already exists if writer.exists(candidate.extractor_name()) { return Err(AphoriaError::Promotion(format!( "Extractor '{}' already exists", candidate.extractor_name() ))); } // Write YAML file with autonomous metadata let path = writer.write_autonomous(&candidate.extractor_def, &candidate.pattern)?; // Mark pattern as promoted self.store.mark_promoted(&candidate.pattern_id(), candidate.extractor_name())?; Ok(path) } /// Get statistics about the promotion pipeline. pub fn stats(&self) -> PromotionStats { let all_patterns: Vec = self.store.get_promotion_candidates(0, 0.0); // Get all patterns let eligible = self.get_candidates(); let promoted: Vec<_> = all_patterns.iter().filter(|p| p.promoted).collect(); let avg_confidence = if eligible.is_empty() { 0.0 } else { eligible.iter().map(|p| p.avg_confidence).sum::() / eligible.len() as f32 }; let avg_projects = if eligible.is_empty() { 0.0 } else { eligible.iter().map(|p| p.project_count() as f32).sum::() / eligible.len() as f32 }; PromotionStats { total_patterns: all_patterns.len(), eligible_patterns: eligible.len(), promoted_patterns: promoted.len(), pending_review: eligible.len().saturating_sub(promoted.len()), avg_confidence, avg_projects, } } /// Promote a specific pattern by ID. pub fn promote_by_id(&self, pattern_id: &Uuid) -> Result { // Find the pattern let candidates = self.get_candidates(); let pattern = candidates.iter().find(|p| &p.id == pattern_id).ok_or_else(|| { AphoriaError::Promotion(format!("Pattern {} not found in candidates", pattern_id)) })?; // Generate and validate let candidate = self.generate_candidate(pattern)?; // Promote self.promote(&candidate) } /// Validate a pattern without promoting it. /// /// Returns the validation result for inspection. pub fn validate_pattern( &self, pattern: &LearnedPattern, ) -> Result { let client = self.client.ok_or_else(|| { AphoriaError::Promotion("LLM client not configured for regex generation".to_string()) })?; let generator = RegexGenerator::new(client); let extractor_def = generator.generate(pattern)?; self.validator.validate(&extractor_def, pattern) } } #[cfg(test)] mod tests { use super::*; use crate::config::PromotionConfig; use crate::learning::{ClaimTemplate, LocalPatternStore, ValueType}; use crate::types::Language; use chrono::Utc; use tempfile::TempDir; fn create_test_store(temp: &TempDir) -> LocalPatternStore { LocalPatternStore::new(temp.path()).expect("create store") } fn create_eligible_pattern() -> LearnedPattern { let mut pattern = LearnedPattern::new( "verify_ssl = false", "verify_ssl = ", ClaimTemplate::new("ssl/verify", "enabled", ValueType::Boolean, "SSL verification"), Language::Python, "project1", 0.9, ); // Add enough projects to meet threshold for i in 2..=6 { pattern.record_observation(format!("project{}", i), 0.85, Utc::now()); } pattern } #[test] fn test_pipeline_creation() { let temp = TempDir::new().expect("temp dir"); let store = create_test_store(&temp); let config = PromotionConfig::default(); let pipeline = PromotionPipeline::new(&store, None, &config, Some(temp.path().to_path_buf())); assert!(pipeline.is_ok()); } #[test] fn test_get_candidates_empty() { let temp = TempDir::new().expect("temp dir"); let store = create_test_store(&temp); let config = PromotionConfig::default(); let pipeline = PromotionPipeline::new(&store, None, &config, None).expect("create pipeline"); let candidates = pipeline.get_candidates(); assert!(candidates.is_empty()); } #[test] fn test_get_candidates_with_eligible() { let temp = TempDir::new().expect("temp dir"); let store = create_test_store(&temp); let config = PromotionConfig::default(); // Add eligible pattern let pattern = create_eligible_pattern(); store.record_pattern(&pattern, None).expect("record"); let pipeline = PromotionPipeline::new(&store, None, &config, None).expect("create pipeline"); let candidates = pipeline.get_candidates(); assert_eq!(candidates.len(), 1); } #[test] fn test_stats_empty_store() { let temp = TempDir::new().expect("temp dir"); let store = create_test_store(&temp); let config = PromotionConfig::default(); let pipeline = PromotionPipeline::new(&store, None, &config, None).expect("create pipeline"); let stats = pipeline.stats(); assert_eq!(stats.total_patterns, 0); assert_eq!(stats.eligible_patterns, 0); assert_eq!(stats.promoted_patterns, 0); } #[test] fn test_stats_with_patterns() { let temp = TempDir::new().expect("temp dir"); let store = create_test_store(&temp); let config = PromotionConfig::default(); // Add eligible pattern let pattern = create_eligible_pattern(); store.record_pattern(&pattern, None).expect("record"); // Add non-eligible pattern (not enough projects) let small_pattern = LearnedPattern::new( "test = true", "test = ", ClaimTemplate::new("test", "value", ValueType::Boolean, "Test"), Language::Rust, "project1", 0.9, ); store.record_pattern(&small_pattern, None).expect("record"); let pipeline = PromotionPipeline::new(&store, None, &config, None).expect("create pipeline"); let stats = pipeline.stats(); assert_eq!(stats.eligible_patterns, 1); assert_eq!(stats.pending_review, 1); } #[test] fn test_generate_candidate_requires_client() { let temp = TempDir::new().expect("temp dir"); let store = create_test_store(&temp); let config = PromotionConfig::default(); let pattern = create_eligible_pattern(); let pipeline = PromotionPipeline::new(&store, None, &config, None).expect("create pipeline"); let result = pipeline.generate_candidate(&pattern); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("LLM client not configured")); } #[test] fn test_auto_promote_disabled() { let temp = TempDir::new().expect("temp dir"); let store = create_test_store(&temp); let config = PromotionConfig { auto_promote: false, ..Default::default() }; let pipeline = PromotionPipeline::new(&store, None, &config, None).expect("create pipeline"); let (promoted, errors) = pipeline.auto_promote_all(); assert_eq!(promoted, 0); assert!(errors.is_empty()); } }