train_model.py

import os
import re
import time
import unicodedata
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import NMF
from sklearn.metrics.pairwise import cosine_similarity
import joblib
from tqdm import tqdm
import spacy
import tempfile
from rank_bm25 import BM25Okapi
import numpy as np
# from nltk.corpus import wordnet  # Removed

import logging

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

class AdvancedTextMatcher:
    def __init__(self, file_path: str):
        self.file_path = file_path
        self.df = None
        self.vectorizer = None
        self.nmf_model = None
        self.topic_matrix = None
        self.nmf_model_path = 'nmf_model.joblib'
        self.vectorizer_path = 'vectorizer.joblib'
        self.unique_groups = []
        self.unique_brands = []
        self.french_stopwords = self.load_stopwords()
        self.french_stopwords = {
            unicodedata.normalize('NFKD', word.lower()).encode('ascii', 'ignore').decode('utf-8')
            for word in self.french_stopwords
        }
        self.french_stopwords = set(filter(None, self.french_stopwords))

    def load_stopwords(self):
        """Load stopwords from a CSV file"""
        stopwords_file = os.path.join(os.path.dirname(__file__), 'french_stopwords.csv')
        try:
            with open(stopwords_file, 'r', encoding='utf-8') as file:
                stopwords = {line.strip() for line in file}
            return stopwords
        except FileNotFoundError:
            logger.error(f"Stopwords file not found: {stopwords_file}")
            raise
        except Exception as e:
            logger.error(f"Error loading stopwords: {e}")
            raise

    def basic_preprocess(self, text):
        """Enhanced basic preprocessing method"""
        if not isinstance(text, str):
            text = str(text)
        text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8')
        text = text.lower()
        text = re.sub(r'[^a-z0-9\s]', '', text)
        text = re.sub(r'\s+', ' ', text).strip()
        return text

    def preprocess_text(self, text):
        """Enhanced text preprocessing method with explicit stopword removal"""
        text = self.basic_preprocess(text)
        # tokens = safe_tokenize(text)  # REPLACED
        tokens = text.split()  # Use str.split() directly
        tokens = [token.lower() for token in tokens if token.lower() not in self.french_stopwords]
        return ' '.join(tokens)

    def load_and_preprocess_dataset(self):
        """Load and preprocess the dataset"""
        try:
            encodings = ['utf-8', 'utf-8-sig', 'latin1', 'iso-8859-1']
            for encoding in encodings:
                try:
                    df = pd.read_csv(self.file_path, delimiter=";", encoding=encoding)
                    print(f"Successfully loaded with encoding: {encoding}")
                    print(df.head())
                    break
                except UnicodeDecodeError:
                    logger.warning(f"Failed to load with encoding: {encoding}")
                    continue
                except Exception as e:
                    logger.error(f"Error loading with encoding {encoding}: {e}")
                    continue

            else:
                raise ValueError("Unable to read CSV file")

            df = df.fillna('')

            df['Consolidated_Text'] = (
                df['Group_FR'].astype(str) + " " +
                df['Subgroup_FR'].astype(str) + " " +
                df['Description_FR'].astype(str) + " " +
                df['Properties_FR'].astype(str) + " " +
                df['CSC_Subgroup_title_FR'].astype(str) + " " +
                df['CSC_Item_title_FR'].astype(str) + " " +
                df['Series'].astype(str) + " " +
                df['Item_Number'].astype(str) + " " +
                df['Brand'].astype(str)
            )
            print(df['Consolidated_Text'].head())

            df['Preprocessed_Text'] = df['Consolidated_Text'].apply(self.preprocess_text)
            self.unique_groups = sorted(df['Group_FR'].unique().tolist())
            self.unique_brands = sorted(df['Brand'].unique().tolist())
            self.df = df
            return df
        except Exception as e:
            logger.error(f"Dataset loading error: {e}")
            raise

    def _tfidf_preprocessor(self, text):
        """Preprocessor for the TfidfVectorizer"""
        return self.preprocess_text(text)

    def train_topic_model(self, num_topics=180, min_df=2, max_df=0.95, max_features=15000, max_iter=500):
        """Enhanced topic model training"""
        try:
            custom_weights = {}
            for group_text in tqdm(self.df['Group_FR'], desc="Processing Group_FR"):
                group_tokens = self.preprocess_text(group_text).split()
                for token in group_tokens:
                    current_weight = custom_weights.get(token, 1.0)
                    custom_weights[token] = min(current_weight * 2.0, 7.0)
            for subgroup_text in tqdm(self.df['Subgroup_FR'], desc="Processing Subgroup_FR"):
                subgroup_tokens = self.preprocess_text(subgroup_text).split()
                for token in subgroup_tokens:
                    current_weight = custom_weights.get(token, 1.0)
                    custom_weights[token] = min(current_weight * 1.5, 4.0)            
            for brand_text in tqdm(self.df['Brand'], desc="Processing Brand"):
                brand_tokens = self.preprocess_text(brand_text).split()
                for token in brand_tokens:
                    current_weight = custom_weights.get(token, 1.0)
                    custom_weights[token] = min(current_weight * 1.8, 5.0)

            self.vectorizer = TfidfVectorizer(
                max_features=max_features,
                min_df=min_df,
                max_df=max_df,
                ngram_range=(1, 4),
                preprocessor=self._tfidf_preprocessor,
                use_idf=True,
                smooth_idf=True,
                sublinear_tf=True
            )

            logger.info("Starting document vectorization...")
            X = self.vectorizer.fit_transform(self.df['Consolidated_Text'])
            feature_names = self.vectorizer.get_feature_names_out()
            logger.info(f"Vectorization complete. Shape: {X.shape}")

            feature_weights = np.ones(len(feature_names))
            for i, feature in enumerate(feature_names):
                if feature in custom_weights:
                    feature_weights[i] *= custom_weights[feature]

            feature_weights = feature_weights / np.mean(feature_weights)
            X_weighted = X.multiply(feature_weights)

            self.nmf_model = NMF(
                n_components=num_topics,
                random_state=42,
                max_iter=max_iter,
                init='random',  # Consider 'nndsvda'
                beta_loss='kullback-leibler',
                solver='mu',
                tol=1e-4
            )

            logger.info("Starting NMF model fitting...")
            self.nmf_model.fit(X_weighted)
            self.topic_matrix = self.nmf_model.transform(X_weighted)
            reconstruction_error = self.nmf_model.reconstruction_err_
            logger.info(f"Model fitting complete. Reconstruction error: {reconstruction_error}")

            top_words_per_topic = self._get_top_words(self.nmf_model, feature_names)
            avg_topic_coherence = self._calculate_topic_coherence(X, top_words_per_topic)
            logger.info(f"Average topic coherence: {avg_topic_coherence}")

            model_metadata = {
                'num_topics': num_topics,
                'max_features': max_features,
                'reconstruction_error': reconstruction_error,
                'avg_topic_coherence': avg_topic_coherence,
                'feature_weights': feature_weights.tolist(),
                'training_date': time.strftime('%Y-%m-%d %H:%M:%S')
            }
            joblib.dump((self.nmf_model, model_metadata), self.nmf_model_path)
            joblib.dump(self.vectorizer, self.vectorizer_path)
            logger.info("Model saving complete.")

            return {
                'vectorizer': self.vectorizer,
                'topic_model': self.nmf_model,
                'topic_matrix': self.topic_matrix,
                'top_words': top_words_per_topic,
                'reconstruction_error': reconstruction_error,
                'topic_coherence': avg_topic_coherence,
                'metadata': model_metadata
            }
        except Exception as e:
            logger.error(f"Topic modeling error: {e}")
            raise

    def _calculate_topic_coherence(self, X, top_words_per_topic, num_words=10):
        """Calculate topic coherence score"""
        coherence_scores = []
        dtm = X.tocsr()
        for topic_words in top_words_per_topic:
            topic_words = topic_words[:num_words]
            score = 0
            pairs = 0
            for i in range(len(topic_words)):
                for j in range(i + 1, len(topic_words)):
                    word1_idx = self.vectorizer.vocabulary_.get(topic_words[i])
                    word2_idx = self.vectorizer.vocabulary_.get(topic_words[j])
                    if word1_idx is not None and word2_idx is not None:
                        word1_doc = dtm[:, word1_idx].toarray().flatten()
                        word2_doc = dtm[:, word2_idx].toarray().flatten()
                        co_occur = np.sum(np.logical_and(word1_doc > 0, word2_doc > 0))
                        occur1 = np.sum(word1_doc > 0)
                        occur2 = np.sum(word2_doc > 0)
                        if co_occur > 0:
                            pmi = np.log((co_occur * dtm.shape[0]) / (occur1 * occur2))
                            score += pmi
                            pairs += 1
            if pairs > 0:
                coherence_scores.append(score / pairs)
        return np.mean(coherence_scores) if coherence_scores else 0.0

    def load_model(self):
        """Load saved models"""
        try:
            if not os.path.exists(self.nmf_model_path) or os.path.getsize(self.nmf_model_path) == 0:
                logger.warning(f"NMF model file not found or empty: {self.nmf_model_path}")
                return False
            if not os.path.exists(self.vectorizer_path) or os.path.getsize(self.vectorizer_path) == 0:
                logger.warning(f"Vectorizer file not found or empty: {self.vectorizer_path}")
                return False

            loaded_data = joblib.load(self.nmf_model_path)
            self.nmf_model = loaded_data[0]
            model_metadata = loaded_data[1]
            self.vectorizer = joblib.load(self.vectorizer_path)
            return True
        except Exception as e:
            logger.error(f"Error loading model: {e}")
            return False

    def _get_top_words(self, model, feature_names, n_top_words=20, topic_ids=None):
        """Extract top words for each topic"""
        top_words = []
        if topic_ids is None:
            topic_ids = range(model.n_components)
        for topic_id in topic_ids:
            topic = model.components_[topic_id]
            topic_words = [feature_names[i] for i in topic.argsort()[:-n_top_words - 1:-1]]
            top_words.append(topic_words)
        return top_words

    def filter_dataframe(self, selected_group, selected_brand):
        """Filters the DataFrame based on selected group and brand."""
        filtered_df = self.df.copy()
        if selected_group:
            filtered_df = filtered_df[filtered_df['Group_FR'] == selected_group]
        if selected_brand:
            filtered_df = filtered_df[filtered_df['Brand'] == selected_brand]
        return filtered_df

    def predict_item_number(self, new_description, top_n=10, df=None):
        """Predicts the item number based on a new description with combined BM25 and topic modeling approach."""
        if df is None:
            df = self.df  # Use the class's DataFrame if none is provided

        cleaned_description = self.preprocess_text(new_description)
        keywords = [word.lower() for word in cleaned_description.split()
                    if word.lower() not in self.french_stopwords]
        unique_keywords = list(set(keywords))

        # Initialize scores array
        all_scores = np.zeros(len(df))
        
        # 1. Keyword-based matching for short queries
        if len(unique_keywords) <= 10:
            keyword_scores = []
            for text in df['Preprocessed_Text']:
                text_words = set(word.lower() for word in text.split()
                                if word.lower() not in self.french_stopwords)
                score = len(set(keywords).intersection(text_words)) / len(set(keywords)) if keywords else 0
                keyword_scores.append(score)
            
            keyword_scores_array = np.array(keyword_scores)
            all_scores += keyword_scores_array * 0.4  # 40% weight to keyword matching
        
        # 2. TF-IDF and NMF-based matching
        description_vector = self.vectorizer.transform([cleaned_description])
        topic_distribution = self.nmf_model.transform(description_vector)[0]
        dataset_vectors = self.vectorizer.transform(df['Preprocessed_Text'])
        similarities = cosine_similarity(description_vector, dataset_vectors)[0]
        
        # Scale similarities to 0-1 range (should already be in this range)
        topic_scores = similarities * topic_distribution.max()  # Weight by topic relevance
        all_scores += topic_scores * 0.3  # 30% weight to topic modeling
        
        # 3. BM25 ranking
        tokenized_corpus = [doc.split() for doc in df['Preprocessed_Text']]
        bm25 = BM25Okapi(tokenized_corpus)
        bm25_scores = np.array(bm25.get_scores(keywords))
        
        # Normalize BM25 scores to 0-1 range
        if bm25_scores.max() > 0:  # Avoid division by zero
            bm25_scores = bm25_scores / bm25_scores.max()
        
        all_scores += bm25_scores * 0.3  # 30% weight to BM25
        
        # Get final ranking
        ranked_indices = all_scores.argsort()[::-1]
        
        # Process results
        results = []
        seen_items = set()  # Keep track of item numbers to avoid duplicates

        for idx in ranked_indices:
            if len(results) >= top_n:
                break  # Stop after finding enough results

            item_number = df.iloc[idx]['Item_Number']
            if item_number in seen_items:
                continue  # Skip if item number has already been added
            seen_items.add(item_number)

            original_description = df.iloc[idx]['Description_FR']
            preprocessed_original_description = self.preprocess_text(original_description)

            # Ensure consistent stopword removal and case handling
            text_keywords = set(word.lower() for word in preprocessed_original_description.split()
                                if word.lower() not in self.french_stopwords)

            unique_words = list(set(keywords) - text_keywords)  # Words in input, but not in item
            corresponding_words = list(set(keywords).intersection(text_keywords))  # Matching words

            # Calculate percentage based on filtered keywords
            keyword_match_percentage = (len(corresponding_words) / len(set(keywords)) * 100
                                        if keywords else 0)

            # Calculate individual scores for transparency
            item_keyword_score = keyword_scores_array[idx] if len(unique_keywords) <= 10 else 0
            item_topic_score = topic_scores[idx]
            item_bm25_score = bm25_scores[idx]
            
            # Overall combined score (same weights as above)
            combined_score = all_scores[idx]
            display_percentage = combined_score * 100

            results.append({
                "rank": len(results) + 1,
                "item_number": item_number,
                "brand": df.iloc[idx]['Brand'],
                "probability": f"{display_percentage:.2f}",
                "combined_score": f"{combined_score:.4f}",
                "keyword_score": f"{item_keyword_score:.4f}",
                "topic_score": f"{item_topic_score:.4f}",
                "bm25_score": f"{item_bm25_score:.4f}",
                "group": df.iloc[idx]['Group_FR'],
                "subgroup": df.iloc[idx]['Subgroup_FR'],
                "description": original_description,
                "properties": df.iloc[idx]['Properties_FR'],
                "unique_words": unique_words,
                "corresponding_words": corresponding_words,
            })

        # Sort by combined score (already reflected in ranked_indices)
        return results, cleaned_description

if __name__ == "__main__":
    print(os.getcwd()) # Verify working directory
    # download_nltk_resources()  # REMOVED - No longer needed
    # print("NLTK Data Path:", nltk.data.path)  # REMOVE THIS LINE

    # REMOVED - No longer needed
    # try:
    #     # Try loading the French Punkt parameters directly
    #     nltk.data.load('tokenizers/punkt/french.pickle')
    #     print("Successfully loaded french.pickle")

    #     # Try loading the tagger
    #     nltk.data.load('taggers/averaged_perceptron_tagger/averaged_perceptron_tagger.pickle')
    #     print("Successfully loaded averaged_perceptron_tagger.pickle")

    # except LookupError as e:
    #     print(f"ERROR: Could not load NLTK resource: {e}")
    #     # This is a critical error, so exit the program
    #     import sys
    #     sys.exit(1)

    base_path = os.path.dirname(os.path.abspath(__file__))
    file_path = os.path.join(base_path, 'dataset.csv')  # Use FULL dataset
    print(f"Loading dataset from: {file_path}")  # VERIFY PATH
    matcher = AdvancedTextMatcher(file_path)
    matcher.df = matcher.load_and_preprocess_dataset()
    results = matcher.train_topic_model(
        num_topics=180,
        min_df=2,
        max_df=0.8,
        max_features=15000,
        max_iter=500
    )
    logger.info("Model training and saving complete.")