product_data_integrator.py

"""
ProductDataIntegrator - Unified data management for product chatbot

This module handles the integration of product data from multiple sources:
1. JSON data (primary source)
2. Word document data (supplementary source)

It maintains a unified product repository and provides fuzzy matching capabilities
to handle variations in product descriptions and item numbers.
"""
import os
import json
import logging
from typing import Dict, List, Any, Optional
import re
from difflib import SequenceMatcher

# Configure logging
logger = logging.getLogger(__name__)

class ProductDataIntegrator:
    def __init__(self, data_dir: str = None):
        """
        Initialize the product data integrator
        
        Args:
            data_dir: Directory to store data files (defaults to current directory)
        """
        self.data_dir = data_dir or os.path.dirname(os.path.abspath(__file__))
        self.json_data = {}  # Store original JSON data
        self.word_data = {}  # Store data extracted from Word documents
        self.unified_data = {}  # Store merged product data
        self.category_map = {}  # Maps category terms to product groups
        
        # Load JSON data if it exists
        json_data_path = os.path.join(self.data_dir, "product_data.json")
        if os.path.exists(json_data_path):
            try:
                with open(json_data_path, 'r', encoding='utf-8') as f:
                    self.json_data = json.load(f)
                    logger.info(f"Loaded JSON data with {len(self.json_data.get('products', []))} products")
                    # Initialize unified data with JSON data
                    self.unified_data = self._process_json_data(self.json_data)
                    # Build initial category mappings
                    self._build_category_mapping()
            except Exception as e:
                logger.error(f"Error loading JSON data: {e}")
    
    def _process_json_data(self, json_data: Dict) -> Dict:
        """
        Process raw JSON data into unified format
        
        Args:
            json_data: Raw JSON data
            
        Returns:
            Dict: Processed data in unified format
        """
        unified = {
            "products": [],
            "categories": {},
            "brands": set(),
            "item_number_map": {}  # Maps item numbers to product indices
        }
        
        products = json_data.get("products", [])
        
        for product in products:
            # Extract key fields
            item_number = product.get("item_number", "")
            group = product.get("group", "")
            subgroup = product.get("subgroup", "")
            description = product.get("description", "")
            brand = product.get("brand", "")
            
            # Skip products without an item number
            if not item_number:
                continue
            
            # Add to unified products
            product_index = len(unified["products"])
            unified["products"].append({
                "item_number": item_number,
                "group": group,
                "subgroup": subgroup,
                "description": description,
                "brand": brand,
                "properties": product.get("properties", {}),
                "long_description": "",  # Will be filled by Word data if available
                "sources": ["json"]  # Track data sources for this product
            })
            
            # Update item number map
            unified["item_number_map"][item_number] = product_index
            
            # Update categories
            if group:
                if group not in unified["categories"]:
                    unified["categories"][group] = {"subgroups": {}, "products": []}
                unified["categories"][group]["products"].append(product_index)
                
                if subgroup:
                    if subgroup not in unified["categories"][group]["subgroups"]:
                        unified["categories"][group]["subgroups"][subgroup] = []
                    unified["categories"][group]["subgroups"][subgroup].append(product_index)
            
            # Update brands
            if brand:
                unified["brands"].add(brand)
        
        # Convert brands set to list for JSON serialization
        unified["brands"] = list(unified["brands"])
        
        return unified
    
    def integrate_word_data(self, word_data: List[Dict]) -> bool:
        """
        Integrate data extracted from Word document
        
        Args:
            word_data: List of dictionaries with Word document data
            
        Returns:
            bool: True if successful
        """
        try:
            logger.info(f"Integrating Word data with {len(word_data)} entries")
            self.word_data = word_data
            
            # Map from Word document field names to unified format
            field_mapping = {
                "Group_title": "group",
                "Subgroup_title": "subgroup",
                "Item_title_NL": "item_title",
                "Description_NL": "description",
                "LongDescription": "long_description",
                "Item_Number": "item_number",
                "Brand": "brand",
                "Measuring_State": "measuring_state"
            }
            
            # Process each Word data entry
            for entry in word_data:
                # Extract mapped fields
                mapped_entry = {}
                for word_field, unified_field in field_mapping.items():
                    if word_field in entry:
                        mapped_entry[unified_field] = entry[word_field]
                
                item_number = mapped_entry.get("item_number", "")
                
                # Skip entries without an item number
                if not item_number:
                    continue
                
                # Try to find a matching product in our unified data
                product_index = self._find_matching_product(mapped_entry)
                
                if product_index is not None:
                    # Update existing product with Word data
                    self._update_product_with_word_data(product_index, mapped_entry)
                else:
                    # Add new product from Word data
                    self._add_new_product_from_word_data(mapped_entry)
            
            # Rebuild category mappings with the new data
            self._build_category_mapping()
            
            return True
        except Exception as e:
            logger.error(f"Error integrating Word data: {e}")
            return False
    
    def _find_matching_product(self, word_entry: Dict) -> Optional[int]:
        """
        Find matching product in unified data
        
        Args:
            word_entry: Word document entry
            
        Returns:
            Optional[int]: Index of matching product or None
        """
        item_number = word_entry.get("item_number", "")
        
        # Try exact match first
        if item_number in self.unified_data["item_number_map"]:
            return self.unified_data["item_number_map"][item_number]
        
        # Try fuzzy matching on item number (allowing for small variations)
        for existing_item, index in self.unified_data["item_number_map"].items():
            # If they're similar (same prefix, just minor differences)
            if (existing_item.startswith(item_number[:3]) and 
                SequenceMatcher(None, existing_item, item_number).ratio() > 0.8):
                return index
        
        # Try matching on description and brand if there's no item number match
        description = word_entry.get("description", "")
        brand = word_entry.get("brand", "")
        
        if description and brand:
            for i, product in enumerate(self.unified_data["products"]):
                if (product["brand"] == brand and 
                    SequenceMatcher(None, product["description"], description).ratio() > 0.7):
                    return i
        
        return None
    
    def _update_product_with_word_data(self, product_index: int, word_entry: Dict):
        """
        Update existing product with data from Word document
        
        Args:
            product_index: Index of product to update
            word_entry: Word document entry data
        """
        product = self.unified_data["products"][product_index]
        
        # Update long description if present in Word data
        if "long_description" in word_entry and word_entry["long_description"]:
            product["long_description"] = word_entry["long_description"]
        
        # Update description if Word data has a better one
        if ("description" in word_entry and word_entry["description"] and 
            (not product["description"] or 
             len(word_entry["description"]) > len(product["description"]))):
            product["description"] = word_entry["description"]
        
        # Add any additional fields from Word data
        for field, value in word_entry.items():
            if field not in product and value:
                product[field] = value
        
        # Update sources to include Word data
        if "word" not in product["sources"]:
            product["sources"].append("word")
    
    def _add_new_product_from_word_data(self, word_entry: Dict):
        """
        Add a new product from Word data
        
        Args:
            word_entry: Word document entry data
        """
        item_number = word_entry.get("item_number", "")
        group = word_entry.get("group", "")
        subgroup = word_entry.get("subgroup", "")
        
        # Create new product entry
        product_index = len(self.unified_data["products"])
        self.unified_data["products"].append({
            "item_number": item_number,
            "group": group,
            "subgroup": subgroup,
            "description": word_entry.get("description", ""),
            "brand": word_entry.get("brand", ""),
            "long_description": word_entry.get("long_description", ""),
            "item_title": word_entry.get("item_title", ""),
            "measuring_state": word_entry.get("measuring_state", ""),
            "properties": {},
            "sources": ["word"]
        })
        
        # Update item number map
        self.unified_data["item_number_map"][item_number] = product_index
        
        # Update categories
        if group:
            if group not in self.unified_data["categories"]:
                self.unified_data["categories"][group] = {"subgroups": {}, "products": []}
            
            self.unified_data["categories"][group]["products"].append(product_index)
            
            if subgroup:
                if subgroup not in self.unified_data["categories"][group]["subgroups"]:
                    self.unified_data["categories"][group]["subgroups"][subgroup] = []
                
                self.unified_data["categories"][group]["subgroups"][subgroup].append(product_index)
        
        # Update brands
        brand = word_entry.get("brand", "")
        if brand and brand not in self.unified_data["brands"]:
            self.unified_data["brands"].append(brand)
    
    def _build_category_mapping(self):
        """
        Build mapping from common terms to product categories
        """
        self.category_map = {}
        
        # First, extract terms from group names
        for group_name in self.unified_data["categories"].keys():
            # Split the group name into terms
            terms = self._extract_category_terms(group_name)
            
            # Map each term to this group
            for term in terms:
                if term not in self.category_map:
                    self.category_map[term] = {"groups": [], "weight": 0}
                
                # Add group with a high weight (it's a direct match)
                if group_name not in self.category_map[term]["groups"]:
                    self.category_map[term]["groups"].append(group_name)
                    self.category_map[term]["weight"] += 3
        
        # Then, extract terms from subgroup names
        for group_name, group_data in self.unified_data["categories"].items():
            for subgroup_name in group_data["subgroups"].keys():
                # Skip numeric-only subgroups
                if re.match(r'^[\d\.\s]+$', subgroup_name):
                    continue
                
                # Split the subgroup name into terms
                terms = self._extract_category_terms(subgroup_name)
                
                # Map each term to its parent group
                for term in terms:
                    if term not in self.category_map:
                        self.category_map[term] = {"groups": [], "weight": 0}
                    
                    # Add group with a medium weight (it's from a subgroup)
                    if group_name not in self.category_map[term]["groups"]:
                        self.category_map[term]["groups"].append(group_name)
                        self.category_map[term]["weight"] += 2
        
        # Finally, extract terms from product descriptions
        for product_index, product in enumerate(self.unified_data["products"]):
            group_name = product.get("group", "")
            if not group_name:
                continue
            
            # Extract terms from description
            description = product.get("description", "")
            if description:
                terms = self._extract_category_terms(description)
                
                # Map each term to the product's group
                for term in terms:
                    if term not in self.category_map:
                        self.category_map[term] = {"groups": [], "weight": 0}
                    
                    # Add group with a low weight (it's from a description)
                    if group_name not in self.category_map[term]["groups"]:
                        self.category_map[term]["groups"].append(group_name)
                        self.category_map[term]["weight"] += 1
            
            # Extract terms from long description
            long_description = product.get("long_description", "")
            if long_description:
                terms = self._extract_category_terms(long_description)
                
                # Map each term to the product's group
                for term in terms:
                    if term not in self.category_map:
                        self.category_map[term] = {"groups": [], "weight": 0}
                    
                    # Add group with a low weight (it's from a description)
                    if group_name not in self.category_map[term]["groups"]:
                        self.category_map[term]["groups"].append(group_name)
                        self.category_map[term]["weight"] += 1
        
        logger.info(f"Built category mapping with {len(self.category_map)} terms")
    
    def _extract_category_terms(self, text: str) -> List[str]:
        """
        Extract meaningful terms from text for category mapping
        
        Args:
            text: Text to extract terms from
            
        Returns:
            List[str]: Extracted terms
        """
        if not text:
            return []
        
        # Convert to lowercase
        text_lower = text.lower()
        
        # Remove numbers and common words
        text_clean = re.sub(r'\d+', '', text_lower)
        
        # Split into words
        words = re.findall(r'\b\w{3,}\b', text_clean)
        
        # Remove common stopwords in Dutch, French and English
        stopwords = {
            # Dutch stopwords
            'een', 'de', 'het', 'en', 'van', 'in', 'is', 'dat', 'op', 'te', 
            'voor', 'met', 'zijn', 'niet', 'aan', 'er', 'ook', 'als', 'bij',
            'door', 'maar', 'naar', 'dan', 'ze', 'uit', 'wel', 'nog', 'al',
            # French stopwords
            'le', 'la', 'les', 'un', 'une', 'des', 'et', 'est', 'que', 'en',
            'à', 'pour', 'dans', 'ce', 'qui', 'pas', 'sur', 'plus', 'avec',
            'vous', 'au', 'par', 'mais', 'nous', 'sont', 'du', 'comme', 'je',
            # English stopwords
            'the', 'a', 'an', 'and', 'is', 'are', 'was', 'were', 'be', 'been',
            'to', 'of', 'for', 'in', 'on', 'at', 'by', 'with', 'from', 'about'
        }
        
        terms = [word for word in words if word not in stopwords]
        
        # Also extract 2-word phrases (bigrams) that might be meaningful
        if len(words) >= 2:
            for i in range(len(words) - 1):
                if words[i] not in stopwords and words[i+1] not in stopwords:
                    terms.append(f"{words[i]} {words[i+1]}")
        
        return terms
    
    def find_categories_for_query(self, query: str) -> List[Dict]:
        """
        Find relevant product categories for a user query
        
        Args:
            query: User query
            
        Returns:
            List[Dict]: Relevant categories with their scores
        """
        # Extract terms from the query
        query_terms = self._extract_category_terms(query)
        
        # Score each category based on matching terms
        category_scores = {}
        
        for term in query_terms:
            if term in self.category_map:
                for group in self.category_map[term]["groups"]:
                    if group not in category_scores:
                        category_scores[group] = 0
                    
                    category_scores[group] += self.category_map[term]["weight"]
        
        # Convert scores to a sorted list
        scored_categories = [
            {"category": category, "score": score}
            for category, score in category_scores.items()
        ]
        
        # Sort by score in descending order
        scored_categories.sort(key=lambda x: x["score"], reverse=True)
        
        return scored_categories
    
    def find_products_by_category(self, category: str, limit: int = 10) -> List[Dict]:
        """
        Find products in a specific category
        
        Args:
            category: Category name
            limit: Maximum number of products to return
            
        Returns:
            List[Dict]: Matching products
        """
        if category not in self.unified_data["categories"]:
            return []
        
        product_indices = self.unified_data["categories"][category]["products"]
        products = [self.unified_data["products"][i] for i in product_indices[:limit]]
        
        return products
    
    def find_products_by_query(self, query: str, limit: int = 10) -> List[Dict]:
        """
        Find products matching a user query
        
        Args:
            query: User query
            limit: Maximum number of products to return
            
        Returns:
            List[Dict]: Matching products with scores
        """
        # First, find relevant categories
        categories = self.find_categories_for_query(query)
        
        # If we have categories, get products from them
        if categories:
            # Get products from top categories
            all_products = []
            remaining = limit
            
            for category_data in categories:
                category = category_data["category"]
                products = self.find_products_by_category(category, remaining)
                all_products.extend(products)
                remaining = limit - len(all_products)
                
                if remaining <= 0:
                    break
            
            # Add scores to products based on category score
            for product in all_products:
                for category_data in categories:
                    if product["group"] == category_data["category"]:
                        product["score"] = category_data["score"]
                        break
            
            return all_products[:limit]
        
        # If no categories found, try direct text matching
        scored_products = []
        
        for product in self.unified_data["products"]:
            score = 0
            
            # Match on description
            description = product.get("description", "")
            if description and query.lower() in description.lower():
                score += 3
            
            # Match on long description
            long_description = product.get("long_description", "")
            if long_description and query.lower() in long_description.lower():
                score += 2
            
            # Match on item number
            item_number = product.get("item_number", "")
            if item_number and query.lower() in item_number.lower():
                score += 5
            
            # Match on brand
            brand = product.get("brand", "")
            if brand and query.lower() in brand.lower():
                score += 4
            
            if score > 0:
                product_copy = product.copy()
                product_copy["score"] = score
                scored_products.append(product_copy)
        
        # Sort by score
        scored_products.sort(key=lambda x: x["score"], reverse=True)
        
        return scored_products[:limit]
    
    def get_product_details(self, item_number: str) -> Optional[Dict]:
        """
        Get detailed information for a specific product
        
        Args:
            item_number: Product item number
            
        Returns:
            Optional[Dict]: Product details or None if not found
        """
        if item_number in self.unified_data["item_number_map"]:
            index = self.unified_data["item_number_map"][item_number]
            return self.unified_data["products"][index]
        
        # Try fuzzy matching
        for existing_item, index in self.unified_data["item_number_map"].items():
            # If they're similar (same prefix, just minor differences)
            if (existing_item.startswith(item_number[:3]) and 
                SequenceMatcher(None, existing_item, item_number).ratio() > 0.8):
                return self.unified_data["products"][index]
        
        return None
    
    def save_unified_data(self) -> bool:
        """
        Save the unified data to disk
        
        Returns:
            bool: True if successful
        """
        try:
            unified_data_path = os.path.join(self.data_dir, "unified_product_data.json")
            
            with open(unified_data_path, 'w', encoding='utf-8') as f:
                # Create a copy with brands as list for JSON serialization
                save_data = self.unified_data.copy()
                
                json.dump(save_data, f, ensure_ascii=False, indent=2)
            
            logger.info(f"Saved unified data with {len(self.unified_data['products'])} products")
            return True
        except Exception as e:
            logger.error(f"Error saving unified data: {e}")
            return False
    
    def load_unified_data(self) -> bool:
        """
        Load the unified data from disk
        
        Returns:
            bool: True if successful
        """
        try:
            unified_data_path = os.path.join(self.data_dir, "unified_product_data.json")
            
            if os.path.exists(unified_data_path):
                with open(unified_data_path, 'r', encoding='utf-8') as f:
                    self.unified_data = json.load(f)
                
                logger.info(f"Loaded unified data with {len(self.unified_data['products'])} products")
                return True
            
            return False
        except Exception as e:
            logger.error(f"Error loading unified data: {e}")
            return False