"""
AI Learning Service - Self-learning and pattern recognition system
Learns from conversations, stores patterns, and improves responses over time
"""

from typing import Dict, List, Optional, Any, Tuple
from datetime import datetime, timedelta
from sqlalchemy.orm import Session
from sqlalchemy import and_, or_, func, desc
import json
import re
import logging
from collections import Counter

from ..models.ai_conversation import (
    AIConversation, AIConversationFeedback, AILearnedPattern,
    AIKnowledgeEntry, AITrainingMetrics
)
from ...shared.config.logging_config import get_logger

logger = get_logger(__name__)


class AILearningService:
    """Service for AI self-learning and pattern recognition"""
    
    def __init__(self, db: Session):
        self.db = db
    
    def save_conversation(
        self,
        user_id: int,
        user_query: str,
        ai_response: str,
        intent: str,
        context_used: Optional[Dict] = None,
        user_role: Optional[str] = None,
        session_id: Optional[str] = None,
        response_time_ms: Optional[int] = None
    ) -> AIConversation:
        """Save conversation for learning"""
        try:
            conversation = AIConversation(
                user_id=user_id,
                session_id=session_id,
                user_query=user_query,
                ai_response=ai_response,
                intent=intent,
                context_used=context_used or {},
                user_role=user_role,
                response_time_ms=response_time_ms
            )
            self.db.add(conversation)
            self.db.commit()
            self.db.refresh(conversation)
            
            # Trigger learning process in background
            self._learn_from_conversation(conversation)
            
            return conversation
        except Exception as e:
            logger.error(f"Error saving conversation: {str(e)}", exc_info=True)
            self.db.rollback()
            raise
    
    def save_feedback(
        self,
        conversation_id: int,
        user_id: int,
        rating: Optional[int] = None,
        is_helpful: Optional[bool] = None,
        is_correct: Optional[bool] = None,
        feedback_text: Optional[str] = None,
        correction: Optional[str] = None
    ) -> AIConversationFeedback:
        """Save user feedback for learning"""
        try:
            feedback = AIConversationFeedback(
                conversation_id=conversation_id,
                user_id=user_id,
                rating=rating,
                is_helpful=is_helpful,
                is_correct=is_correct,
                feedback_text=feedback_text,
                correction=correction
            )
            self.db.add(feedback)
            
            # Update conversation helpfulness
            conversation = self.db.query(AIConversation).filter(
                AIConversation.id == conversation_id
            ).first()
            if conversation:
                conversation.is_helpful = is_helpful
            
            self.db.commit()
            self.db.refresh(feedback)
            
            # Trigger learning from feedback
            if feedback.is_correct == False or correction:
                self._learn_from_feedback(conversation, feedback)
            
            return feedback
        except Exception as e:
            logger.error(f"Error saving feedback: {str(e)}", exc_info=True)
            self.db.rollback()
            raise
    
    def find_similar_queries(self, query: str, intent: Optional[str] = None, limit: int = 5) -> List[Dict[str, Any]]:
        """Find similar past queries to learn from"""
        try:
            # Extract keywords from query
            keywords = self._extract_keywords(query)
            
            # Search for similar queries
            query_obj = self.db.query(AIConversation).filter(
                AIConversation.user_query.ilike(f"%{query[:50]}%")
            )
            
            if intent:
                query_obj = query_obj.filter(AIConversation.intent == intent)
            
            similar = query_obj.order_by(desc(AIConversation.created_at)).limit(limit).all()
            
            results = []
            for conv in similar:
                results.append({
                    "query": conv.user_query,
                    "response": conv.ai_response,
                    "intent": conv.intent,
                    "is_helpful": conv.is_helpful,
                    "created_at": conv.created_at.isoformat()
                })
            
            return results
        except Exception as e:
            logger.error(f"Error finding similar queries: {str(e)}", exc_info=True)
            return []
    
    def get_learned_pattern(self, query: str, user_role: Optional[str] = None) -> Optional[AILearnedPattern]:
        """Get learned pattern that matches query"""
        try:
            keywords = self._extract_keywords(query)
            
            # Search patterns by keywords
            patterns = self.db.query(AILearnedPattern).filter(
                and_(
                    AILearnedPattern.is_active == True,
                    or_(*[AILearnedPattern.pattern_keywords.contains(kw) for kw in keywords[:5]])
                )
            )
            
            if user_role:
                patterns = patterns.filter(
                    or_(
                        AILearnedPattern.user_role == user_role,
                        AILearnedPattern.user_role.is_(None)
                    )
                )
            
            # Order by confidence and usage
            patterns = patterns.order_by(
                desc(AILearnedPattern.confidence_score),
                desc(AILearnedPattern.usage_count)
            ).limit(1).all()
            
            if patterns:
                pattern = patterns[0]
                # Update usage stats
                pattern.usage_count += 1
                pattern.last_used_at = datetime.utcnow()
                self.db.commit()
                return pattern
            
            return None
        except Exception as e:
            logger.error(f"Error getting learned pattern: {str(e)}", exc_info=True)
            return None
    
    def get_knowledge_entry(self, query: str, user_role: Optional[str] = None) -> Optional[Dict[str, Any]]:
        """Get knowledge base entry that matches query"""
        try:
            keywords = self._extract_keywords(query)
            query_lower = query.lower()
            
            # Search knowledge entries
            entries = self.db.query(AIKnowledgeEntry).filter(
                and_(
                    or_(
                        AIKnowledgeEntry.question.ilike(f"%{query[:50]}%"),
                        *[AIKnowledgeEntry.keywords.contains([kw]) for kw in keywords[:3]]
                    ),
                    or_(
                        AIKnowledgeEntry.is_verified == True,
                        AIKnowledgeEntry.confidence >= 70.0
                    )
                )
            )
            
            if user_role:
                entries = entries.filter(
                    or_(
                        AIKnowledgeEntry.user_role == user_role,
                        AIKnowledgeEntry.user_role.is_(None)
                    )
                )
            
            entries = entries.order_by(
                desc(AIKnowledgeEntry.confidence),
                desc(AIKnowledgeEntry.success_count)
            ).limit(1).all()
            
            if entries:
                entry = entries[0]
                # Update usage stats
                entry.usage_count += 1
                entry.last_used_at = datetime.utcnow()
                self.db.commit()
                
                return {
                    "topic": entry.topic,
                    "question": entry.question,
                    "answer": entry.answer,
                    "confidence": float(entry.confidence)
                }
            
            return None
        except Exception as e:
            logger.error(f"Error getting knowledge entry: {str(e)}", exc_info=True)
            return None
    
    def _learn_from_conversation(self, conversation: AIConversation):
        """Learn patterns from a conversation - automatically learns from all conversations"""
        try:
            # Always extract keywords and patterns (learn from all conversations)
            keywords = self._extract_keywords(conversation.user_query)
            
            # Check if similar pattern exists
            existing_pattern = self.get_learned_pattern(
                conversation.user_query,
                conversation.user_role
            )
            
            if existing_pattern:
                # Update existing pattern confidence
                existing_pattern.success_count += 1
                existing_pattern.usage_count += 1
                
                # Adjust confidence based on helpfulness
                if conversation.is_helpful is True:
                    existing_pattern.confidence_score = min(
                        100.0,
                        existing_pattern.confidence_score + 1.0
                    )
                elif conversation.is_helpful is False:
                    existing_pattern.confidence_score = max(
                        0.0,
                        existing_pattern.confidence_score - 2.0
                    )
                else:
                    # Neutral - slight increase
                    existing_pattern.confidence_score = min(
                        100.0,
                        existing_pattern.confidence_score + 0.3
                    )
                
                existing_pattern.last_used_at = datetime.utcnow()
            else:
                # Create new pattern - learn from all conversations
                initial_confidence = 60.0 if conversation.is_helpful is True else 50.0
                if conversation.is_helpful is False:
                    initial_confidence = 30.0  # Lower confidence for unhelpful patterns
                
                pattern = AILearnedPattern(
                    pattern_keywords=json.dumps(keywords),
                    query_pattern=conversation.user_query[:200],
                    intent=conversation.intent or "general_query",
                    response_template=conversation.ai_response[:500],
                    context_keys=conversation.context_used,
                    confidence_score=initial_confidence,
                    usage_count=1,
                    success_count=1 if conversation.is_helpful is not False else 0,
                    source_conversation_id=conversation.id,
                    user_role=conversation.user_role,
                    last_used_at=datetime.utcnow()
                )
                self.db.add(pattern)
            
            # Automatically extract knowledge from every conversation
            # This enables the AI to build its knowledge base autonomously
            self._auto_extract_knowledge(conversation)
            
            self.db.commit()
        except Exception as e:
            logger.error(f"Error learning from conversation: {str(e)}", exc_info=True)
            self.db.rollback()
    
    def _learn_from_feedback(self, conversation: AIConversation, feedback: AIConversationFeedback):
        """Learn from user feedback and corrections"""
        try:
            # If user provided correction, create knowledge entry
            if feedback.correction:
                knowledge = AIKnowledgeEntry(
                    topic=self._extract_topic(conversation.user_query),
                    question=conversation.user_query,
                    answer=feedback.correction,
                    keywords=self._extract_keywords(conversation.user_query),
                    related_intent=conversation.intent,
                    source='user_feedback',
                    confidence=75.0,
                    user_role=conversation.user_role,
                    created_by_user_id=feedback.user_id
                )
                self.db.add(knowledge)
            
            # Update pattern confidence based on feedback
            patterns = self.db.query(AILearnedPattern).filter(
                AILearnedPattern.source_conversation_id == conversation.id
            ).all()
            
            for pattern in patterns:
                if feedback.is_helpful:
                    pattern.confidence_score = min(100.0, pattern.confidence_score + 2.0)
                    pattern.success_count += 1
                elif feedback.is_helpful == False:
                    pattern.confidence_score = max(0.0, pattern.confidence_score - 5.0)
                
                if feedback.is_correct == False:
                    pattern.is_active = False  # Deactivate incorrect patterns
            
            self.db.commit()
        except Exception as e:
            logger.error(f"Error learning from feedback: {str(e)}", exc_info=True)
            self.db.rollback()
    
    def _extract_keywords(self, text: str) -> List[str]:
        """Extract keywords from text"""
        # Remove common stop words
        stop_words = {
            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
            'of', 'with', 'by', 'from', 'as', 'is', 'are', 'was', 'were', 'be',
            'been', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
            'should', 'could', 'may', 'might', 'can', 'this', 'that', 'these',
            'those', 'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him',
            'her', 'us', 'them', 'my', 'your', 'his', 'her', 'its', 'our', 'their',
            'what', 'which', 'who', 'whom', 'whose', 'where', 'when', 'why', 'how',
            'show', 'tell', 'give', 'get', 'find', 'search', 'list', 'all'
        }
        
        # Extract words
        words = re.findall(r'\b[a-z]+\b', text.lower())
        keywords = [w for w in words if w not in stop_words and len(w) > 2]
        
        # Count frequency and return top keywords
        word_freq = Counter(keywords)
        return [word for word, _ in word_freq.most_common(10)]
    
    def _extract_topic(self, query: str) -> str:
        """Extract topic from query"""
        # Simple topic extraction - can be enhanced
        keywords = self._extract_keywords(query)
        if keywords:
            return keywords[0].title()
        return "General"
    
    def train_from_history(self, days: int = 7) -> Dict[str, Any]:
        """Train AI from recent conversation history"""
        try:
            cutoff_date = datetime.utcnow() - timedelta(days=days)
            
            # Get conversations from last N days
            conversations = self.db.query(AIConversation).filter(
                AIConversation.created_at >= cutoff_date
            ).all()
            
            trained_patterns = 0
            trained_knowledge = 0
            improved_patterns = 0
            
            for conv in conversations:
                # Learn from helpful conversations
                if conv.is_helpful is True:
                    self._learn_from_conversation(conv)
                    trained_patterns += 1
                
                # Get feedback for this conversation
                feedbacks = self.db.query(AIConversationFeedback).filter(
                    AIConversationFeedback.conversation_id == conv.id
                ).all()
                
                for feedback in feedbacks:
                    if feedback.correction:
                        # Create knowledge entry from correction
                        knowledge = AIKnowledgeEntry(
                            topic=self._extract_topic(conv.user_query),
                            question=conv.user_query,
                            answer=feedback.correction,
                            keywords=self._extract_keywords(conv.user_query),
                            related_intent=conv.intent,
                            source='training',
                            confidence=80.0,
                            user_role=conv.user_role
                        )
                        self.db.add(knowledge)
                        trained_knowledge += 1
            
            self.db.commit()
            
            # Update training metrics
            self._update_training_metrics()
            
            return {
                "conversations_analyzed": len(conversations),
                "patterns_trained": trained_patterns,
                "knowledge_entries_created": trained_knowledge,
                "improved_patterns": improved_patterns
            }
        except Exception as e:
            logger.error(f"Error in training: {str(e)}", exc_info=True)
            self.db.rollback()
            return {"error": str(e)}
    
    def _update_training_metrics(self):
        """Update training metrics"""
        try:
            today = datetime.utcnow().date()
            
            # Calculate metrics
            total_conversations = self.db.query(AIConversation).count()
            total_patterns = self.db.query(AILearnedPattern).filter(
                AILearnedPattern.is_active == True
            ).count()
            total_knowledge = self.db.query(AIKnowledgeEntry).filter(
                AIKnowledgeEntry.is_verified == True
            ).count()
            
            # Average response time
            avg_response_time = self.db.query(
                func.avg(AIConversation.response_time_ms)
            ).filter(
                AIConversation.response_time_ms.isnot(None)
            ).scalar()
            
            # Average rating
            avg_rating = self.db.query(
                func.avg(AIConversationFeedback.rating)
            ).filter(
                AIConversationFeedback.rating.isnot(None)
            ).scalar()
            
            # Helpful rate
            total_with_feedback = self.db.query(AIConversation).filter(
                AIConversation.is_helpful.isnot(None)
            ).count()
            helpful_count = self.db.query(AIConversation).filter(
                AIConversation.is_helpful == True
            ).count()
            helpful_rate = (helpful_count / total_with_feedback * 100) if total_with_feedback > 0 else None
            
            # Create or update metrics
            metrics = self.db.query(AITrainingMetrics).filter(
                func.date(AITrainingMetrics.metric_date) == today
            ).first()
            
            if not metrics:
                metrics = AITrainingMetrics(metric_date=datetime.utcnow())
                self.db.add(metrics)
            
            metrics.total_conversations = total_conversations
            metrics.total_patterns_learned = total_patterns
            metrics.total_knowledge_entries = total_knowledge
            metrics.average_response_time_ms = int(avg_response_time) if avg_response_time else None
            metrics.average_rating = float(avg_rating) if avg_rating else None
            metrics.helpful_rate = float(helpful_rate) if helpful_rate else None
            
            self.db.commit()
        except Exception as e:
            logger.error(f"Error updating training metrics: {str(e)}", exc_info=True)
            self.db.rollback()
    
    def auto_train_from_all_conversations(self) -> Dict[str, Any]:
        """Automatically train from all conversations - self-learning method"""
        try:
            logger.info("Starting automatic self-training from all conversations...")
            
            # Get all conversations that haven't been processed for learning
            # Process conversations from last 30 days by default
            cutoff_date = datetime.utcnow() - timedelta(days=30)
            conversations = self.db.query(AIConversation).filter(
                AIConversation.created_at >= cutoff_date
            ).order_by(desc(AIConversation.created_at)).all()
            
            trained_patterns = 0
            created_knowledge = 0
            improved_patterns = 0
            analyzed_conversations = 0
            
            for conv in conversations:
                analyzed_conversations += 1
                
                # Always learn from conversations (even without explicit feedback)
                # The AI learns from patterns in successful responses
                if conv.is_helpful is not False:  # Learn from helpful or neutral conversations
                    # Extract and save knowledge automatically
                    self._auto_extract_knowledge(conv)
                    created_knowledge += 1
                
                # Learn patterns from all conversations
                self._learn_from_conversation(conv)
                trained_patterns += 1
            
            # Auto-optimize patterns based on usage and success
            improved_patterns = self._auto_optimize_patterns()
            
            # Auto-merge similar patterns
            merged = self._auto_merge_similar_patterns()
            
            # Update metrics
            self._update_training_metrics()
            
            logger.info(f"Auto-training completed: {analyzed_conversations} conversations analyzed, "
                       f"{trained_patterns} patterns trained, {created_knowledge} knowledge entries created, "
                       f"{improved_patterns} patterns improved, {merged} patterns merged")
            
            return {
                "status": "success",
                "conversations_analyzed": analyzed_conversations,
                "patterns_trained": trained_patterns,
                "knowledge_entries_created": created_knowledge,
                "patterns_improved": improved_patterns,
                "patterns_merged": merged,
                "timestamp": datetime.utcnow().isoformat()
            }
        except Exception as e:
            logger.error(f"Error in auto-training: {str(e)}", exc_info=True)
            self.db.rollback()
            return {"status": "error", "error": str(e)}
    
    def _auto_extract_knowledge(self, conversation: AIConversation):
        """Automatically extract knowledge from conversations"""
        try:
            # Check if similar knowledge already exists
            keywords = self._extract_keywords(conversation.user_query)
            existing = self.db.query(AIKnowledgeEntry).filter(
                and_(
                    AIKnowledgeEntry.question.ilike(f"%{conversation.user_query[:50]}%"),
                    or_(
                        AIKnowledgeEntry.user_role == conversation.user_role,
                        AIKnowledgeEntry.user_role.is_(None)
                    )
                )
            ).first()
            
            if not existing:
                # Create new knowledge entry (only for helpful or neutral conversations)
                # Don't learn from explicitly unhelpful responses
                if conversation.is_helpful is not False:
                    knowledge = AIKnowledgeEntry(
                        topic=self._extract_topic(conversation.user_query),
                        question=conversation.user_query,
                        answer=conversation.ai_response,
                        keywords=keywords,
                        related_intent=conversation.intent,
                        source='auto_learned',
                        confidence=60.0 if conversation.is_helpful is True else 50.0,
                        user_role=conversation.user_role,
                        is_verified=False  # Auto-learned entries need verification
                    )
                    self.db.add(knowledge)
            else:
                # Update existing knowledge if this response is better
                # Increase confidence if response is marked as helpful
                existing.usage_count += 1
                if conversation.is_helpful is True:
                    existing.confidence = min(100.0, existing.confidence + 5.0)
                    existing.success_count += 1
                    # Update answer if confidence is high enough
                    if existing.confidence >= 80.0:
                        existing.answer = conversation.ai_response
                        existing.updated_at = datetime.utcnow()
                elif conversation.is_helpful is False:
                    existing.confidence = max(20.0, existing.confidence - 3.0)
                else:
                    # Neutral - slight increase
                    existing.confidence = min(100.0, existing.confidence + 1.0)
                    existing.success_count += 1
                
                existing.last_used_at = datetime.utcnow()
            
            # Don't commit here - let the caller commit
        except Exception as e:
            logger.error(f"Error auto-extracting knowledge: {str(e)}", exc_info=True)
            # Don't rollback here - let the caller handle it
    
    def _auto_optimize_patterns(self) -> int:
        """Automatically optimize patterns based on performance"""
        try:
            improved = 0
            
            # Get all active patterns
            patterns = self.db.query(AILearnedPattern).filter(
                AILearnedPattern.is_active == True
            ).all()
            
            for pattern in patterns:
                # Calculate success rate
                if pattern.usage_count > 0:
                    success_rate = (pattern.success_count / pattern.usage_count) * 100
                    
                    # Adjust confidence based on success rate
                    if success_rate >= 80:
                        # High success rate - increase confidence
                        pattern.confidence_score = min(100.0, pattern.confidence_score + 1.0)
                        improved += 1
                    elif success_rate < 50 and pattern.usage_count >= 5:
                        # Low success rate - decrease confidence or deactivate
                        pattern.confidence_score = max(0.0, pattern.confidence_score - 2.0)
                        if pattern.confidence_score < 30:
                            pattern.is_active = False
                            logger.info(f"Deactivated low-performing pattern {pattern.id}")
                        improved += 1
                    
                    # Update last used timestamp
                    pattern.updated_at = datetime.utcnow()
            
            self.db.commit()
            return improved
        except Exception as e:
            logger.error(f"Error optimizing patterns: {str(e)}", exc_info=True)
            self.db.rollback()
            return 0
    
    def _auto_merge_similar_patterns(self) -> int:
        """Automatically merge similar patterns to reduce redundancy"""
        try:
            merged = 0
            
            # Get patterns with same intent and role
            patterns_by_intent = {}
            all_patterns = self.db.query(AILearnedPattern).filter(
                AILearnedPattern.is_active == True
            ).all()
            
            for pattern in all_patterns:
                key = (pattern.intent, pattern.user_role)
                if key not in patterns_by_intent:
                    patterns_by_intent[key] = []
                patterns_by_intent[key].append(pattern)
            
            # Merge similar patterns within each intent group
            for (intent, role), patterns in patterns_by_intent.items():
                if len(patterns) < 2:
                    continue
                
                # Sort by confidence and usage
                patterns.sort(key=lambda p: (p.confidence_score, p.usage_count), reverse=True)
                
                # Keep the best pattern and merge others into it
                best_pattern = patterns[0]
                for pattern in patterns[1:]:
                    # Check if patterns are similar (same keywords or similar query pattern)
                    best_keywords = set(json.loads(best_pattern.pattern_keywords) if best_pattern.pattern_keywords else [])
                    pattern_keywords = set(json.loads(pattern.pattern_keywords) if pattern.pattern_keywords else [])
                    
                    # If 70% keywords overlap, merge them
                    if best_keywords and pattern_keywords:
                        overlap = len(best_keywords & pattern_keywords) / len(best_keywords | pattern_keywords)
                        if overlap >= 0.7:
                            # Merge: combine usage counts and update confidence
                            best_pattern.usage_count += pattern.usage_count
                            best_pattern.success_count += pattern.success_count
                            best_pattern.confidence_score = (
                                (best_pattern.confidence_score * best_pattern.usage_count + 
                                 pattern.confidence_score * pattern.usage_count) / 
                                (best_pattern.usage_count + pattern.usage_count)
                            )
                            
                            # Deactivate the merged pattern
                            pattern.is_active = False
                            merged += 1
                            logger.info(f"Merged pattern {pattern.id} into {best_pattern.id}")
            
            self.db.commit()
            return merged
        except Exception as e:
            logger.error(f"Error merging patterns: {str(e)}", exc_info=True)
            self.db.rollback()
            return 0
    
    def auto_analyze_and_improve(self) -> Dict[str, Any]:
        """Automatically analyze performance and improve - self-improvement method"""
        try:
            logger.info("Starting automatic self-analysis and improvement...")
            
            # Analyze conversation quality
            recent_conversations = self.db.query(AIConversation).filter(
                AIConversation.created_at >= datetime.utcnow() - timedelta(days=7)
            ).all()
            
            # Find patterns in successful vs unsuccessful conversations
            successful_queries = []
            unsuccessful_queries = []
            
            for conv in recent_conversations:
                if conv.is_helpful is True:
                    successful_queries.append(conv.user_query)
                elif conv.is_helpful is False:
                    unsuccessful_queries.append(conv.user_query)
            
            # Extract common patterns from successful queries
            if successful_queries:
                common_success_keywords = self._find_common_keywords(successful_queries)
                logger.info(f"Common keywords in successful queries: {common_success_keywords[:10]}")
            
            # Identify problematic patterns
            if unsuccessful_queries:
                common_failure_keywords = self._find_common_keywords(unsuccessful_queries)
                logger.info(f"Common keywords in unsuccessful queries: {common_failure_keywords[:10]}")
            
            # Improve knowledge entries based on usage
            improved_knowledge = self._improve_knowledge_entries()
            
            # Clean up old inactive patterns
            cleaned = self._cleanup_old_patterns()
            
            # Update metrics
            self._update_training_metrics()
            
            return {
                "status": "success",
                "knowledge_entries_improved": improved_knowledge,
                "old_patterns_cleaned": cleaned,
                "timestamp": datetime.utcnow().isoformat()
            }
        except Exception as e:
            logger.error(f"Error in auto-analysis: {str(e)}", exc_info=True)
            return {"status": "error", "error": str(e)}
    
    def _find_common_keywords(self, queries: List[str]) -> List[str]:
        """Find common keywords across multiple queries"""
        all_keywords = []
        for query in queries:
            keywords = self._extract_keywords(query)
            all_keywords.extend(keywords)
        
        # Count frequency
        keyword_freq = Counter(all_keywords)
        return [word for word, _ in keyword_freq.most_common(20)]
    
    def _improve_knowledge_entries(self) -> int:
        """Improve knowledge entries based on usage statistics"""
        try:
            improved = 0
            
            # Get knowledge entries with usage data
            entries = self.db.query(AIKnowledgeEntry).filter(
                AIKnowledgeEntry.usage_count > 0
            ).all()
            
            for entry in entries:
                # Calculate success rate
                if entry.usage_count > 0:
                    success_rate = (entry.success_count / entry.usage_count) * 100
                    
                    # Auto-verify entries with high success rate
                    if success_rate >= 85 and entry.usage_count >= 5 and not entry.is_verified:
                        entry.is_verified = True
                        entry.confidence = min(100.0, entry.confidence + 10.0)
                        improved += 1
                        logger.info(f"Auto-verified knowledge entry {entry.id} (success rate: {success_rate:.1f}%)")
                    
                    # Increase confidence for successful entries
                    elif success_rate >= 70:
                        entry.confidence = min(100.0, entry.confidence + 2.0)
                        improved += 1
                    
                    # Decrease confidence for unsuccessful entries
                    elif success_rate < 40 and entry.usage_count >= 3:
                        entry.confidence = max(20.0, entry.confidence - 5.0)
                        improved += 1
            
            self.db.commit()
            return improved
        except Exception as e:
            logger.error(f"Error improving knowledge entries: {str(e)}", exc_info=True)
            self.db.rollback()
            return 0
    
    def _cleanup_old_patterns(self) -> int:
        """Clean up old inactive or low-performing patterns"""
        try:
            cutoff_date = datetime.utcnow() - timedelta(days=90)
            
            # Find patterns that haven't been used in 90 days and have low confidence
            old_patterns = self.db.query(AILearnedPattern).filter(
                and_(
                    or_(
                        AILearnedPattern.last_used_at < cutoff_date,
                        AILearnedPattern.last_used_at.is_(None)
                    ),
                    AILearnedPattern.confidence_score < 40.0,
                    AILearnedPattern.is_active == True
                )
            ).all()
            
            cleaned = 0
            for pattern in old_patterns:
                pattern.is_active = False
                cleaned += 1
            
            self.db.commit()
            logger.info(f"Cleaned up {cleaned} old inactive patterns")
            return cleaned
        except Exception as e:
            logger.error(f"Error cleaning up patterns: {str(e)}", exc_info=True)
            self.db.rollback()
            return 0

