enhanced_product_chatbot.py

"""
Enhanced ProductChatbot with dynamic category detection and unified data sources.

This module builds on the original ProductChatbot to provide:
1. Dynamic category detection from both JSON and Word document sources
2. Consistent user experience whether a Word document is uploaded or not
3. Improved language detection and response generation
4. Enhanced conversation memory and context building
"""
import os
import json
import logging
import requests
import time
import re
import mammoth
from typing import Dict, List, Any, Optional, Tuple
from difflib import SequenceMatcher

# Import our new modules
from product_data_integrator import ProductDataIntegrator
from dynamic_category_extractor import DynamicCategoryExtractor

# Configure logging
logger = logging.getLogger(__name__)

class EnhancedProductChatbot:
    def __init__(self, api_key: str, data_dir: str = None):
        """
        Initialize the enhanced chatbot with OpenAI API key and data directory
        
        Args:
            api_key: OpenAI API key
            data_dir: Directory to store data files (defaults to current directory)
        """
        self.api_key = api_key
        self.data_dir = data_dir or os.path.dirname(os.path.abspath(__file__))
        
        # Initialize data sources
        self.data_integrator = ProductDataIntegrator(self.data_dir)
        self.category_extractor = DynamicCategoryExtractor(self.data_integrator)
        
        # Conversation memory
        self.conversation_memory = {}  # Store conversation history by session_id
        
        logger.info("Enhanced ProductChatbot initialized")
    
    def process_document(self, docx_path: str) -> bool:
        """
        Process a Word document and integrate its data
        
        Args:
            docx_path: Path to the Word document
            
        Returns:
            bool: True if successful, False otherwise
        """
        try:
            logger.info(f"Processing document: {docx_path}")
            
            # Import the word_parser module
            import importlib.util
            spec = importlib.util.spec_from_file_location("word_parser", 
                                                          os.path.join(self.data_dir, "word_parser.py"))
            word_parser = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(word_parser)
            
            # Parse the document
            parsed_data = word_parser.parse_word_document(docx_path)
            
            # Check if we got data
            if not parsed_data:
                logger.error("No data extracted from document")
                return False
            
            # Integrate the parsed data
            success = self.data_integrator.integrate_word_data(parsed_data)
            
            if success:
                # Rebuild category mapping
                self.category_extractor.build_category_mapping()
                
                # Save the integrated data
                self.data_integrator.save_unified_data()
                
                logger.info(f"Successfully processed document and integrated {len(parsed_data)} items")
                return True
            else:
                logger.error("Failed to integrate Word document data")
                return False
            
        except Exception as e:
            logger.error(f"Error processing document: {e}", exc_info=True)
            return False
    
    def detect_language(self, text: str) -> str:
        """
        Detect whether text is in Dutch, French, or English
        
        Args:
            text: Text to analyze
            
        Returns:
            str: 'nl' for Dutch, 'fr' for French, 'en' for English
        """
        text_lower = text.lower()
        
        # Common Dutch words
        dutch_words = [
            'de', 'het', 'een', 'en', 'van', 'in', 'is', 'dat', 'op', 'te',
            'voor', 'met', 'zijn', 'niet', 'aan', 'er', 'ook', 'als', 'bij',
            'door', 'maar', 'naar', 'dan', 'ze', 'uit', 'wel', 'nog', 'al'
        ]
        
        # Common French words
        french_words = [
            'le', 'la', 'les', 'un', 'une', 'des', 'et', 'est', 'que', 'en',
            'à', 'pour', 'dans', 'ce', 'qui', 'pas', 'sur', 'plus', 'avec',
            'vous', 'au', 'par', 'mais', 'nous', 'sont', 'du', 'comme', 'je'
        ]
        
        # Common English words
        english_words = [
            'the', 'a', 'an', 'and', 'is', 'are', 'was', 'were', 'be', 'been',
            'to', 'of', 'for', 'in', 'on', 'at', 'by', 'with', 'from', 'about',
            'that', 'this', 'these', 'those', 'which', 'what', 'where', 'who',
            'when', 'how', 'why', 'not', 'as', 'if'
        ]
        
        # Count matches
        dutch_count = sum(1 for word in dutch_words if f" {word} " in f" {text_lower} ")
        french_count = sum(1 for word in french_words if f" {word} " in f" {text_lower} ")
        english_count = sum(1 for word in english_words if f" {word} " in f" {text_lower} ")
        
        # Determine the most likely language
        if dutch_count > french_count and dutch_count > english_count:
            return 'nl'
        elif french_count > dutch_count and french_count > english_count:
            return 'fr'
        else:
            return 'en'  # Default to English if uncertain or equal counts
    
    def get_conversation_memory(self, session_id: str) -> List[Dict[str, str]]:
        """
        Get the conversation history for a specific session
        
        Args:
            session_id: Unique identifier for the session
            
        Returns:
            List of message dictionaries
        """
        if session_id not in self.conversation_memory:
            self.conversation_memory[session_id] = []
        
        # Clean up old conversations (older than 1 hour)
        current_time = time.time()
        for sid in list(self.conversation_memory.keys()):
            history = self.conversation_memory[sid]
            if history and 'timestamp' in history[0]:
                if current_time - history[0]['timestamp'] > 3600:  # 1 hour
                    del self.conversation_memory[sid]
        
        return self.conversation_memory[session_id]
    
    def add_to_conversation(self, session_id: str, role: str, content: str):
        """
        Add a message to the conversation history
        
        Args:
            session_id: Unique identifier for the session
            role: The role of the message sender ('user' or 'assistant')
            content: The message content
        """
        if session_id not in self.conversation_memory:
            self.conversation_memory[session_id] = []
        
        self.conversation_memory[session_id].append({
            'role': role,
            'content': content,
            'timestamp': time.time()
        })
        
        # Keep conversation history to a reasonable size (last 10 messages)
        if len(self.conversation_memory[session_id]) > 10:
            self.conversation_memory[session_id] = self.conversation_memory[session_id][-10:]
    
    def _extract_conversation_context(self, conversation_history: List[Dict]) -> Dict:
        """
        Extract context from conversation history
        
        Args:
            conversation_history: List of conversation messages
            
        Returns:
            Dict: Extracted context
        """
        context = {
            'user_preferences': [],
            'assistant_questions': [],
            'user_confirmations': [],
            'identified_categories': [],
            'detected_products': []
        }
        
        # Track categories mentioned throughout the conversation
        all_categories = []
        
        for i, msg in enumerate(conversation_history):
            if msg['role'] == 'user':
                # Extract user preferences from their messages
                context['user_preferences'].append(msg['content'])
                
                # Detect categories from user messages
                detected_categories = self.category_extractor.detect_categories(msg['content'])
                if detected_categories:
                    # Add top categories with scores to context
                    for category_data in detected_categories[:3]:  # Top 3 categories
                        all_categories.append(category_data)
                
                # Check if this is a confirmation to a previous question
                if i > 0 and conversation_history[i-1]['role'] == 'assistant' and '?' in conversation_history[i-1]['content']:
                    # Simple confirmation detection
                    confirmation_words = ['yes', 'oui', 'ja', 'correct', 'exactement', 'juist', 'indeed']
                    rejection_words = ['no', 'non', 'nee', 'incorrect', 'faux']
                    
                    msg_lower = msg['content'].lower()
                    
                    if any(word in msg_lower for word in confirmation_words):
                        context['user_confirmations'].append({
                            'question': conversation_history[i-1]['content'],
                            'answer': msg['content'],
                            'is_positive': True
                        })
                    elif any(word in msg_lower for word in rejection_words):
                        context['user_confirmations'].append({
                            'question': conversation_history[i-1]['content'],
                            'answer': msg['content'],
                            'is_positive': False
                        })
                        
            elif msg['role'] == 'assistant' and '?' in msg['content']:
                # Store questions asked by the assistant
                context['assistant_questions'].append(msg['content'])
        
        # Consolidate category scores
        category_scores = {}
        for category_data in all_categories:
            category = category_data["category"]
            score = category_data["score"]
            
            if category not in category_scores:
                category_scores[category] = 0
            
            category_scores[category] += score
        
        # Add top categories to context
        sorted_categories = [
            {"category": category, "score": score}
            for category, score in category_scores.items()
        ]
        sorted_categories.sort(key=lambda x: x["score"], reverse=True)
        
        context['identified_categories'] = sorted_categories[:5]  # Top 5 categories
        
        return context
    
    def generate_response(self, 
                        query: str, 
                        matcher: Any, 
                        session_id: str = "default",
                        top_products: int = 10) -> str:  # Increased from 3 to 10
        """
        Generate a concise response with strict product category enforcement
        
        Args:
            query: User's question
            matcher: Instance of your trained model
            session_id: Unique identifier for the session
            top_products: Number of top products to include
            
        Returns:
            str: Response to the user
        """
        try:
            # Add the user's message to conversation history
            self.add_to_conversation(session_id, 'user', query)
            
            # Get conversation history
            conversation_history = self.get_conversation_memory(session_id)
            
            # Detect language
            language = self.detect_language(query)
            
            # Language consistency with previous messages
            if len(conversation_history) > 1:
                previous_messages = [msg['content'] for msg in conversation_history[:-1]]
                previous_text = " ".join(previous_messages)
                previous_language = self.detect_language(previous_text)
                
                if previous_language != language and previous_language in ['fr', 'nl']:
                    language = previous_language
            
            language_name = "Dutch" if language == 'nl' else "French" if language == 'fr' else "English"
            
            # Extract key category terms from all user messages
            all_user_queries = " ".join([msg['content'] for msg in conversation_history if msg['role'] == 'user'])
            
            # Get predictions from your trained model using COMBINED approach
            model_products = []
            model_groups = set()
            model_subgroups = set()
            property_terms = set()  # New: collect important property terms
            
            if matcher and hasattr(matcher, 'predict_item_number'):
                # Get top predictions with the combined ranking approach
                predictions, _ = matcher.predict_item_number(all_user_queries, top_n=top_products)
                
                # Extract common properties across top results
                common_properties = {}
                for pred in predictions[:5]:  # Analyze top 5 results
                    props = pred.get("properties", "").lower().split()
                    for prop in props:
                        if len(prop) > 3:  # Only meaningful terms
                            common_properties[prop] = common_properties.get(prop, 0) + 1
                
                # Get most common properties (appearing in at least 2 products)
                property_terms = {term for term, count in common_properties.items() if count >= 2}
                
                # Format the products for the prompt
                for pred in predictions:
                    product = {
                        "item_number": pred.get("item_number", ""),
                        "description": pred.get("description", ""),
                        "brand": pred.get("brand", ""),
                        "group": pred.get("group", ""),
                        "subgroup": pred.get("subgroup", ""),
                        "properties": pred.get("properties", ""),
                        "probability": pred.get("probability", ""),
                        "combined_score": pred.get("combined_score", ""),  # Include the combined score
                        "key_terms": pred.get("corresponding_words", [])  # Include matching terms
                    }
                    
                    model_products.append(product)
                    
                    # Collect unique groups and subgroups
                    if "group" in pred and pred["group"]:
                        model_groups.add(pred["group"])
                    if "subgroup" in pred and pred["subgroup"]:
                        model_subgroups.add(pred["subgroup"])
            
            # Create a context using information from your model
            conversation_context = self._extract_conversation_context(conversation_history)
            
            # Add the primary category and category terms to context for emphasis
            primary_category = self._get_primary_category(all_user_queries, predictions)
            category_terms = self._extract_category_terms(all_user_queries)
            
            # Determine conversation stage (1st, 2nd, or 3rd+ interaction)
            user_message_count = sum(1 for msg in conversation_history if msg['role'] == 'user')
            
            context = {
                "products": model_products,
                "product_groups": list(model_groups),
                "product_subgroups": list(model_subgroups),
                "conversation_context": conversation_context,
                "primary_category": primary_category if primary_category else "",
                "category_terms": category_terms,
                "common_properties": list(property_terms),  # Add common properties
                "conversation_stage": user_message_count,  # Add conversation stage
                "top_probability": predictions[0].get("probability", "") if predictions else ""  # Top match probability
            }
            
            # Create system prompt with emphasis on guiding to prediction
            system_prompt = f"""
            You are a confident product specialist who helps customers find exactly what they need.

            Respond in {language_name}.
            Based on the conversation, here is product information:
            {json.dumps(context, ensure_ascii=False, indent=2)}


            INSTRUCTIONS CRITIQUES :
                        1. CRITICAL: EXTREMELY BRIEF RESPONSES ONLY - 10 WORDS MAXIMUM PER QUESTION
                        2. Étape : {user_message_count}/3
                        3. Se concentrer uniquement sur la CATÉGORIE PRIMAIRE : « {primary_category} » et les meilleurs résultats
                        4. Les questions doivent porter sur les caractéristiques distinctives des premiers résultats.
                        5. Si la probabilité est > 70 %, dites seulement « Cliquez sur PREDICT pour voir votre produit »
                        6. N'utilisez que des termes tirés de données réelles sur les produits
                        7. N'incluez JAMAIS d'expressions telles que « Parmi les principaux résultats » ou « Aimeriez-vous »
                        8. Formulez les questions comme suit : « Serrure à un point ou à trois points ? » PAS « Préférez-vous une serrure à un point ou à trois points ? »

                        Exemple de bonnes réponses :
                        - « Porte intérieure ou porte coupe-feu ? »
                        - Verrouillage à un ou trois points ?
                        - Serrure mécanique ou électromécanique ?

                        Objectif : trouver le produit exact en 4 interactions maximum.

            """
            
            # Prepare messages for the API call
            messages = [{"role": "system", "content": system_prompt}]
            
            # Add conversation history
            for msg in conversation_history:
                if msg['role'] in ['user', 'assistant']:
                    messages.append({
                        "role": msg['role'],
                        "content": msg['content']
                    })
            
            # Call OpenAI API
            headers = {
                'Content-Type': 'application/json',
                'Authorization': f'Bearer {self.api_key}'
            }
            
            payload = {
                'model': 'gpt-3.5-turbo',
                'messages': messages,
                'max_tokens': 75,
                'temperature': 0.3
            }
            
            response = requests.post(
                'https://api.openai.com/v1/chat/completions',
                headers=headers,
                json=payload,
                timeout=10
            )
            
            response_data = response.json()
            
            if 'choices' in response_data and len(response_data['choices']) > 0:
                response_text = response_data['choices'][0]['message']['content'].strip()
                self.add_to_conversation(session_id, 'assistant', response_text)
                return response_text
            else:
                logger.error(f"Invalid OpenAI response structure: {response_data}")
                error_message = "Je suis désolé, j'ai eu un problème en traitant votre demande."
                if language == 'nl':
                    error_message = "Het spijt me, ik had een probleem bij het verwerken van uw verzoek."
                elif language == 'en':
                    error_message = "I'm sorry, I had trouble processing your request."
                
                self.add_to_conversation(session_id, 'assistant', error_message)
                return error_message
                
        except Exception as e:
            logger.error(f"Error generating response: {e}", exc_info=True)
            language = self.detect_language(query)
            error_message = "Je suis désolé, j'ai rencontré une erreur."
            if language == 'nl':
                error_message = "Het spijt me, ik ben een fout tegengekomen."
            elif language == 'en':
                error_message = "I'm sorry, I encountered an error."
            
            self.add_to_conversation(session_id, 'assistant', error_message)
            return error_message