topic_modeling_script.py

from flask import Flask, render_template, request, jsonify, session, send_from_directory
import pytesseract
from PIL import Image
import os
import pandas as pd
import nltk
import unicodedata
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import NMF
import joblib
import numpy as np
import logging
from dotenv import load_dotenv
from sklearn.metrics.pairwise import cosine_similarity
import requests
import time
import csv
import tempfile
import json
from tqdm import tqdm
from chatbot_module import ProductChatbot
from product_data_integrator import ProductDataIntegrator
from dynamic_category_extractor import DynamicCategoryExtractor
from enhanced_product_chatbot import EnhancedProductChatbot

app = Flask(__name__)

import platform

# Only set Tesseract path on Windows
if platform.system() == 'Windows':
    pytesseract.pytesseract.tesseract_cmd = r'C:\Users\matcol\AppData\Local\Programs\Tesseract-OCR\tesseract.exe'
# On Linux (Render/Railway), Tesseract is already in PATH

load_dotenv()  # Load environment variables from .env file

# OpenAI API configuration
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
OPENAI_API_URL = 'https://api.openai.com/v1/completions'

chatbot = ProductChatbot(api_key=OPENAI_API_KEY)
enhanced_chatbot = EnhancedProductChatbot(api_key=OPENAI_API_KEY)
# Configure logging
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s - %(levelname)s: %(message)s')

product_relationships = {}
try:
    with open('product_relationships.json', 'r', encoding='utf-8') as f:
        product_relationships = json.load(f)
    logger.info(f"Loaded product relationships for {len(product_relationships)} products")
except Exception as e:
    logger.warning(f"Could not load product relationships: {e}")

def safe_tokenize(text):
    """Safe tokenization method with fallback"""
    try:
        return nltk.word_tokenize(text, language='french')
    except Exception:
        return text.split()

def calculate_missing_words(input_keywords, item_keywords):
    """Calculates words present in item_keywords but not in input_keywords,
       preserving the order from item_keywords.
    """
    missing_words = []
    for word in item_keywords:  # Iterate through item_keywords (which is a list)
        if word not in input_keywords:  # Check membership in input_keywords (also a list)
            missing_words.append(word)
    return missing_words

class AdvancedTextMatcher:
    def __init__(self, file_path: str):
        self.file_path = file_path
        self.df = None
        self.vectorizer = None
        self.nmf_model = None
        self.topic_matrix = None
        self.nmf_model_path = 'nmf_model.joblib'
        self.vectorizer_path = 'vectorizer.joblib'
        self.unique_groups = []
        self.unique_brands = []

        # Load stopwords from CSV
        self.french_stopwords = self.load_stopwords()

        # Normalize stopwords: lowercase and remove accents
        self.french_stopwords = {
            unicodedata.normalize('NFKD', word.lower()).encode('ascii', 'ignore').decode('utf-8')
            for word in self.french_stopwords
        }
        
        # Remove any empty strings that might have been created
        self.french_stopwords = set(filter(None, self.french_stopwords))

        try:
            nltk_download_dir = os.path.join(os.path.dirname(__file__), 'nltk_data')
            os.makedirs(nltk_download_dir, exist_ok=True)
            nltk.download('punkt', download_dir=nltk_download_dir)
            nltk.download('stopwords', download_dir=nltk_download_dir)
            nltk.data.path.append(nltk_download_dir)
        except Exception as e:
            logger.warning(f"NLTK resource download failed: {e}")

    def load_stopwords(self):
        """Load stopwords from a CSV file"""
        stopwords_file = os.path.join(os.path.dirname(__file__), 'french_stopwords.csv')
        try:
            with open(stopwords_file, 'r', encoding='utf-8') as file:
                stopwords = {line.strip() for line in file}
            return stopwords
        except FileNotFoundError:
            logger.error(f"Stopwords file not found: {stopwords_file}")
            raise
        except Exception as e:
            logger.error(f"Error loading stopwords: {e}")
            raise

    def basic_preprocess(self, text):
        """Enhanced basic preprocessing method"""
        if not isinstance(text, str):
            text = str(text)

        # Normalize unicode characters
        text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8')
        
        # Convert to lowercase
        text = text.lower()
        
        # Remove special characters but keep spaces
        text = re.sub(r'[^a-z0-9\s]', '', text)
        
        # Normalize whitespace
        text = re.sub(r'\s+', ' ', text).strip()
        
        return text

    def preprocess_text(self, text):
        """Enhanced text preprocessing method with more explicit stopword removal"""
        # Basic preprocessing first
        text = self.basic_preprocess(text)
        
        # Split into tokens
        tokens = safe_tokenize(text)
        
        # Remove stopwords - make sure to use lower case comparison
        tokens = [token.lower() for token in tokens 
                  if token.lower() not in self.french_stopwords]
        
        # Join back into text
        return ' '.join(tokens)

    def load_and_preprocess_dataset(self):
        """Load and preprocess the dataset"""
        try:
            encodings = ['utf-8', 'latin1', 'iso-8859-1']
            for encoding in encodings:
                try:
                    df = pd.read_csv(self.file_path, delimiter=";", encoding=encoding)
                    break
                except Exception:
                    continue
            else:
                raise ValueError("Unable to read CSV file")

            df = df.fillna('')

            df['Consolidated_Text'] = (
                df['Group_FR'].astype(str) + " " +
                df['Subgroup_FR'].astype(str) + " " +
                df['Properties_FR'].astype(str) + " " +
                df['Description_FR'].astype(str) + " " +
                df['Series'].astype(str) + " " +
                df['Item_Number'].astype(str) + " " +
                df['Brand'].astype(str)
            )

            df['Preprocessed_Text'] = df['Consolidated_Text'].apply(self.preprocess_text)
            self.unique_groups = sorted(df['Group_FR'].unique().tolist())
            self.unique_brands = sorted(df['Brand'].unique().tolist())
            self.df = df  # Store df in the instance
            return df

        except Exception as e:
            logger.error(f"Dataset loading error: {e}")
            raise

    def _tfidf_preprocessor(self, text):
        """Preprocessor for the TfidfVectorizer"""
        return self.preprocess_text(text)

    def train_topic_model(self, num_topics=180, min_df=2, max_df=0.95, max_features=10000, max_iter=500):
        """
        Enhanced topic model training with more parameters and optimization

        Parameters:
        -----------
        num_topics : int
            Number of topics to extract
        min_df : int or float
            Minimum document frequency for TF-IDF
        max_df : float
            Maximum document frequency for TF-IDF
        max_features : int
            Maximum number of features for TF-IDF
        """
        try:
            # Create custom weights with enhanced weighting scheme
            custom_weights = {}

            # Enhanced group weighting
            for group_text in tqdm(self.df['Group_FR'], desc="Processing Group_FR"):
                group_tokens = self.preprocess_text(group_text).split()
                for token in group_tokens:
                    current_weight = custom_weights.get(token, 1.0)
                    # Increased weight for group terms
                    custom_weights[token] = min(current_weight * 2.0, 7.0)

            # Enhanced brand weighting
            for brand_text in tqdm(self.df['Brand'], desc="Processing Brand"):
                brand_tokens = self.preprocess_text(brand_text).split()
                for token in brand_tokens:
                    current_weight = custom_weights.get(token, 1.0)
                    # Increased weight for brand terms
                    custom_weights[token] = min(current_weight * 1.8, 5.0)

            # Enhanced subgroup weighting
            for subgroup_text in tqdm(self.df['Subgroup_FR'], desc="Processing Subgroup_FR"):
                subgroup_tokens = self.preprocess_text(subgroup_text).split()
                for token in subgroup_tokens:
                    current_weight = custom_weights.get(token, 1.0)
                    custom_weights[token] = min(current_weight * 1.5, 4.0)

            # Initialize vectorizer with modified parameters
            self.vectorizer = TfidfVectorizer(
                max_features=max_features,
                min_df=min_df,
                max_df=max_df,
                ngram_range=(1, 3),
                preprocessor=self._tfidf_preprocessor,
                use_idf=True,
                smooth_idf=True,
                sublinear_tf=True
            )

            print("Starting document vectorization...")
            X = self.vectorizer.fit_transform(self.df['Consolidated_Text'])
            feature_names = self.vectorizer.get_feature_names_out()
            print(f"Vectorization complete. Shape: {X.shape}")

            # Apply custom weights with normalization
            feature_weights = np.ones(len(feature_names))
            for i, feature in enumerate(feature_names):
                if feature in custom_weights:
                    feature_weights[i] *= custom_weights[feature]

            # Normalize weights
            feature_weights = feature_weights / np.mean(feature_weights)
            X_weighted = X.multiply(feature_weights)

            # Initialize NMF with enhanced parameters
            self.nmf_model = NMF(
                n_components=num_topics,
                random_state=42,
                max_iter=max_iter,  # Increased iterations
                init='random',  # Better initialization
                beta_loss='kullback-leibler',  # Better for text data
                solver='mu',  # Multiplicative update solver
                tol=1e-4  # Tighter convergence tolerance
            )

            print("Starting NMF model fitting...")
            for i in tqdm(range(1, 501), desc="Fitting NMF Model"):
                self.nmf_model.fit(X_weighted)
            self.topic_matrix = self.nmf_model.transform(X_weighted)

            reconstruction_error = self.nmf_model.reconstruction_err_
            print(f"Model fitting complete. Reconstruction error: {reconstruction_error}")

            # Calculate and print topic coherence
            top_words_per_topic = self._get_top_words(self.nmf_model, feature_names)
            avg_topic_coherence = self._calculate_topic_coherence(X, top_words_per_topic)
            print(f"Average topic coherence: {avg_topic_coherence}")

            # Save models with metadata
            model_metadata = {
                'num_topics': num_topics,
                'max_features': max_features,
                'reconstruction_error': reconstruction_error,
                'avg_topic_coherence': avg_topic_coherence,
                'feature_weights': feature_weights.tolist(),
                'training_date': time.strftime('%Y-%m-%d %H:%M:%S')
            }

            joblib.dump((self.nmf_model, model_metadata), self.nmf_model_path)
            joblib.dump(self.vectorizer, self.vectorizer_path)
            print("Model saving complete.")

            return {
                'vectorizer': self.vectorizer,
                'topic_model': self.nmf_model,
                'topic_matrix': self.topic_matrix,
                'top_words': top_words_per_topic,
                'reconstruction_error': reconstruction_error,
                'topic_coherence': avg_topic_coherence,
                'metadata': model_metadata
            }

        except Exception as e:
            logger.error(f"Topic modeling error: {e}")
            raise

    def _calculate_topic_coherence(self, X, top_words_per_topic, num_words=10):
        """
        Calculate topic coherence score
        """
        coherence_scores = []

        # Get document-term matrix
        dtm = X.tocsr()

        for topic_words in top_words_per_topic:
            topic_words = topic_words[:num_words]
            score = 0
            pairs = 0

            for i in range(len(topic_words)):
                for j in range(i + 1, len(topic_words)):
                    word1_idx = self.vectorizer.vocabulary_.get(topic_words[i])
                    word2_idx = self.vectorizer.vocabulary_.get(topic_words[j])

                    if word1_idx is not None and word2_idx is not None:
                        # Calculate co-occurrence
                        word1_doc = dtm[:, word1_idx].toarray().flatten()
                        word2_doc = dtm[:, word2_idx].toarray().flatten()

                        co_occur = np.sum(np.logical_and(word1_doc > 0, word2_doc > 0))
                        occur1 = np.sum(word1_doc > 0)
                        occur2 = np.sum(word2_doc > 0)

                        # Calculate PMI score
                        if co_occur > 0:
                            pmi = np.log((co_occur * len(dtm.toarray())) / (occur1 * occur2))
                            score += pmi
                            pairs += 1

            if pairs > 0:
                coherence_scores.append(score / pairs)

        return np.mean(coherence_scores) if coherence_scores else 0.0

    def load_model(self):
        """Load saved models"""
        try:
            if not os.path.exists(self.nmf_model_path) or os.path.getsize(self.nmf_model_path) == 0:
                logger.warning(f"NMF model file not found or empty: {self.nmf_model_path}")
                return False
            if not os.path.exists(self.vectorizer_path) or os.path.getsize(self.vectorizer_path) == 0:
                logger.warning(f"Vectorizer file not found or empty: {self.vectorizer_path}")
                return False

            loaded_data = joblib.load(self.nmf_model_path)  # Load the tuple
            self.nmf_model = loaded_data[0] #NMF model object
            model_metadata = loaded_data[1] #Metadata dictionary

            self.vectorizer = joblib.load(self.vectorizer_path)
            return True
        except Exception as e:
            logger.error(f"Error loading model: {e}")
            return False

    def _get_top_words(self, model, feature_names, n_top_words=20, topic_ids=None):
        """Extract top words for each topic"""
        top_words = []
        if topic_ids is None:
            topic_ids = range(model.n_components)
        for topic_id in topic_ids:
            topic = model.components_[topic_id]
            topic_words = [feature_names[i] for i in topic.argsort()[:-n_top_words - 1:-1]]
            top_words.append(topic_words)
        return top_words

    def filter_dataframe(self, selected_group, selected_brand):
        filtered_df = self.df.copy()

        if selected_group:
            filtered_df = filtered_df[filtered_df['Group_FR'] == selected_group]

        if selected_brand:
            filtered_df = filtered_df[filtered_df['Brand'] == selected_brand]

        return filtered_df

    def predict_item_number(self, new_description, top_n=100, df=None):
        """Predicts the item number based on a new description, preserving word order."""
        if df is None:
            df = self.df  # Use the class's DataFrame if none is provided

        cleaned_description = self.preprocess_text(new_description)
        # Use a list comprehension to preserve order, and only add unique words.
        keywords = []
        for word in cleaned_description.split():
            word = word.lower()
            if word not in self.french_stopwords and word not in keywords:
                keywords.append(word)

        if len(keywords) <= 5:
            # Keyword-based matching
            keyword_scores = []
            for text in df['Preprocessed_Text']:  # Iterate over PREPROCESSED text
                # Also preserve order of words in the preprocessed text
                text_words = []
                for word in text.split():
                    word = word.lower()
                    if word not in self.french_stopwords and word not in text_words:
                        text_words.append(word)
                # Calculate a score based on the *order* of matching words
                score = 0
                keyword_index = 0
                for text_word in text_words:
                    if keyword_index < len(keywords) and text_word == keywords[keyword_index]:
                        score += (len(keywords) - keyword_index)  # Add order weighting
                        keyword_index += 1
                    elif text_word in keywords: # bonus for all keywords, even out of order
                        score += 1
                keyword_scores.append(score / len(keywords) if keywords else 0)

            ranked_indices = np.argsort(keyword_scores)[::-1]
        else:
            # TF-IDF and NMF-based matching
            description_vector = self.vectorizer.transform([cleaned_description])
            topic_distribution = self.nmf_model.transform(description_vector)[0]
            dataset_vectors = self.vectorizer.transform(df['Preprocessed_Text'])  # Use PREPROCESSED text
            similarities = cosine_similarity(description_vector, dataset_vectors)[0]
            ranking_scores = similarities * topic_distribution.max() # Weigh by topic relevance
            ranked_indices = ranking_scores.argsort()[::-1]  # Descending order

        results = []
        seen_items = set()  # Keep track of item numbers to avoid duplicates

        for idx in ranked_indices:
            if len(results) >= top_n:
                break  # Stop after finding enough results

            item_number = df.iloc[idx]['Item_Number']
            if item_number in seen_items:
                continue  # Skip if item number has already been added
            seen_items.add(item_number)

            original_description = df.iloc[idx]['Description_FR']  # Still get original for display
            preprocessed_original_description = df.iloc[idx]['Preprocessed_Text']
            text_words = []
            for word in preprocessed_original_description.split():
                word.lower()
                if word not in self.french_stopwords and word not in text_words:
                    text_words.append(word)

            # Find corresponding, unique, and missing words, *preserving order*.
            corresponding_words = []
            unique_words = []
            for word in keywords:  # Iterate through *input* keywords in order
                if word in text_words:
                    corresponding_words.append(word)
                else:
                    unique_words.append(word)

            missing_words = calculate_missing_words(keywords, text_words)

            keyword_match_percentage = (len(corresponding_words) / len(keywords) * 100
                                        if keywords else 0)
            image_url = df.iloc[idx]['IMAGE']  # Get the image URL
            datasheet_url = df.iloc[idx]['URL_FR'] # Get the datasheet URL
            results.append({
                "rank": len(results) + 1,
                "item_number": item_number,
                "brand": df.iloc[idx]['Brand'],
                "probability": f"{keyword_match_percentage:.2f}% Keyword Match",
                "group": df.iloc[idx]['Group_FR'],
                "series": df.iloc[idx]['Series'],
                "subgroup": df.iloc[idx]['Subgroup_FR'],
                "description": original_description,
                "properties": df.iloc[idx]['Properties_FR'],
                "unique_words": unique_words,
                "corresponding_words": corresponding_words,
                "missing_words": missing_words,
                "keyword_match": len(keywords) <= 10,
                "image_url": image_url,  # Add the image URL
                "datasheet_url": datasheet_url  # Add the datasheet URL
            })

        results.sort(key=lambda x: float(x['probability'].split('%')[0]), reverse=True)
        return results, cleaned_description

def translate_text(text, target_language='fr'):
    """Translate text using OpenAI API
    
    Parameters:
    text (str): Text to translate
    target_language (str): Target language code ('fr' for French, 'nl' for Dutch)
    
    Returns:
    str: Translated text
    """
    try:
        language_name = "French" if target_language == 'fr' else "Dutch"
        
        # Prepare request to OpenAI API
        headers = {
            'Content-Type': 'application/json',
            'Authorization': f'Bearer {OPENAI_API_KEY}'
        }
        
        prompt = f"Translate the following text to {language_name}:\n\n{text}"
        
        payload = {
            'model': 'gpt-3.5-turbo-instruct',  # Using instruct model for this simpler task
            'prompt': prompt,
            'max_tokens': 500,
            'temperature': 0.3  # Lower temperature for more accurate translations
        }
        
        response = requests.post(
            'https://api.openai.com/v1/completions',
            headers=headers,
            json=payload,
            timeout=10
        )
        
        response_data = response.json()
        
        if 'choices' in response_data and len(response_data['choices']) > 0:
            translated_text = response_data['choices'][0]['text'].strip()
            logger.info(f"Translated text from {'Dutch to French' if target_language == 'fr' else 'French to Dutch'}")
            return translated_text
        else:
            logger.error(f"Invalid OpenAI response for translation: {response_data}")
            return text  # Return original text if translation fails
            
    except Exception as e:
        logger.error(f"Error translating text: {e}")
        return text  # Return original text if translation fails

def create_app(file_path: str) -> Flask:
    """
    Creates and configures the Flask application.

    Args:
        file_path (str): The path to the dataset CSV file.

    Returns:
        Flask: The configured Flask application instance.
    """
    app = Flask(__name__)
    matcher = AdvancedTextMatcher(file_path)
    _first_request = True  # Flag for initial setup
    app.secret_key = os.urandom(24)

    app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY', 'YourVerySecretKey')

    @app.route('/api/enhanced_chat', methods=['POST'])
    def enhanced_chat_api():
        """Handle enhanced chatbot API request"""
        data = request.json
        user_message = data.get('message', '')
        session_id = data.get('session_id', request.remote_addr)
        
        if not user_message:
            return jsonify({"error": "No message provided"}), 400
        
        # Log the incoming message
        logger.info(f"Received enhanced chat message from session {session_id}: {user_message[:50]}...")
        
        # Get the matcher instance
        matcher = app.config.get('matcher')
        
        # Generate response using the enhanced chatbot
        response = enhanced_chatbot.generate_response(
            query=user_message,
            matcher=matcher,
            session_id=session_id
        )
        
        return jsonify({"response": response, "session_id": session_id})

    @app.route('/chatbot.html')
    def chatbot_page():
        """Serve the chatbot HTML page"""
        return render_template('chatbot.html')
    
    @app.route('/get_prediction_context', methods=['POST'])
    def get_prediction_context():
        """Get the full context for prediction"""
        data = request.json
        session_id = data.get('session_id', '')
        
        if not session_id:
            return jsonify({"prediction_query": ""}), 400
        
        # Get prediction query from enhanced chatbot
        prediction_query = enhanced_chatbot.get_prediction_query(session_id)
        
        return jsonify({"prediction_query": prediction_query})

    @app.route('/upload-specs', methods=['POST'])
    def upload_specs():
        """Handle Word document upload with both chatbots"""
        if 'file' not in request.files:
            logger.error("No file part in the request")
            return jsonify({"error": "No file part"}), 400
        
        file = request.files['file']
        if file.filename == '':
            logger.error("No selected file")
            return jsonify({"error": "No selected file"}), 400
        
        if file and file.filename.endswith('.docx'):
            # Create a temporary file
            import tempfile
            temp_dir = tempfile.gettempdir()
            temp_file = os.path.join(temp_dir, f"temp_specs_{int(time.time())}.docx")
            
            try:
                logger.info(f"Saving uploaded file to {temp_file}")
                file.save(temp_file)
                
                # Process with original chatbot (keep existing functionality)
                from word_parser import parse_word_document
                parsed_data = parse_word_document(temp_file)
                original_success = chatbot.process_document(temp_file)
                chatbot.store_parsed_data(parsed_data)
                
                # Also process with enhanced chatbot
                enhanced_success = enhanced_chatbot.process_document(temp_file)
                
                # Count the number of items in the list
                product_count = len(parsed_data)
                logger.info(f"Processed document with {product_count} products")
                
                return jsonify({
                    "message": "File processed successfully",
                    "products_count": product_count,
                    "original_success": original_success,
                    "enhanced_success": enhanced_success
                })
            except Exception as e:
                logger.error(f"Error processing file: {e}")
                import traceback
                logger.error(traceback.format_exc())
                return jsonify({"error": str(e)}), 500
            finally:
                # Clean up temporary file
                if os.path.exists(temp_file):
                    os.remove(temp_file)
                    logger.info(f"Removed temporary file {temp_file}")
        
        logger.error("Invalid file format")
        return jsonify({"error": "Invalid file format. Please upload a .docx file"}), 400

    @app.route('/api/chat', methods=['POST'])
    def chat_api():
        """Handle chatbot API request with conversation memory"""
        data = request.json
        user_message = data.get('message', '')
        session_id = data.get('session_id', request.remote_addr)  # Use IP as default session ID
        
        if not user_message:
            return jsonify({"error": "No message provided"}), 400
        
        # Log the incoming message
        logger.info(f"Received chat message from session {session_id}: {user_message[:50]}...")
        
        # Get the matcher instance from app config, or create one if it doesn't exist
        matcher = app.config.get('matcher')
        if not matcher:
            try:
                # Create an instance of your matcher
                base_path = os.path.dirname(os.path.abspath(__file__))
                file_path = os.path.join(base_path, 'dataset.csv')
                matcher = AdvancedTextMatcher(file_path)
                
                # Load the model
                if not matcher.load_model():
                    # If loading fails, train a new model
                    logger.warning("Could not load existing model, training new model...")
                    matcher.df = matcher.load_and_preprocess_dataset()
                    matcher.train_topic_model()
                else:
                    # Load dataset for predictions
                    matcher.df = matcher.load_and_preprocess_dataset()
                
                app.config['matcher'] = matcher
            except Exception as e:
                logger.error(f"Error initializing matcher: {e}")
                return jsonify({"response": "I'm sorry, I'm having trouble with my product database."}), 500
        
        # Generate response using the chatbot module
        response = chatbot.generate_response(user_message, matcher, session_id=session_id)
        
        return jsonify({"response": response, "session_id": session_id})

    @app.route('/analyze_image', methods=['POST'])
    def analyze_image():
        if 'file' not in request.files:
            return jsonify({'error': 'No file part'})

        file = request.files['file']

        if file.filename == '':
            return jsonify({'error': 'No selected file'})

        if file:
            try:
                # Read the image file
                image = Image.open(file.stream)

                # Use pytesseract to do OCR on the image
                text = pytesseract.image_to_string(image)

                logger.info(f"Extracted text: {text}")
                return jsonify({'text': text})
            except Exception as e:
                logger.error(f"Error processing image: {e}")
                return jsonify({'error': str(e)})

    @app.route('/translate', methods=['POST'])
    def translate():
        data = request.get_json()
        text_to_translate = data.get('text', '')

        if not text_to_translate:
            return jsonify({'error': 'No text provided'}), 400

        try:
            # Limit the text to 500 tokens
            words = text_to_translate.split()
            limited_text = ' '.join(words[:500])

            response = requests.post(
                OPENAI_API_URL,
                headers={
                    'Content-Type': 'application/json',
                    'Authorization': f'Bearer {OPENAI_API_KEY}'
                },
                json={
                    'model': 'gpt-3.5-turbo-instruct',
                    'prompt': f'Translate the following text to French (technical): {limited_text}',
                    'max_tokens': 500,
                    'temperature': 0.2
                },
                timeout=10  # Add timeout to prevent hanging requests
            )

            # Log the response for debugging
            print(f"OpenAI API response status: {response.status_code}")
            print(f"OpenAI API response content: {response.text[:200]}...")  # Print first 200 chars of response

            response.raise_for_status()
            response_data = response.json()
            
            if 'choices' in response_data and len(response_data['choices']) > 0:
                translated_text = response_data['choices'][0]['text'].strip()
                return jsonify({'translated_text': translated_text})
            else:
                print(f"Invalid OpenAI response structure: {response_data}")
                return jsonify({'error': 'Invalid response from translation service'}), 500

        except requests.exceptions.RequestException as e:
            print(f"Translation request error: {str(e)}")
            return jsonify({'error': str(e)}), 500
        except ValueError as e:  # JSON parsing error
            print(f"JSON parsing error: {str(e)}")
            return jsonify({'error': 'Invalid response format'}), 500
        except Exception as e:
            print(f"Unexpected error in translation: {str(e)}")
            return jsonify({'error': 'An unexpected error occurred'}), 500

    @app.route('/')
    def index():
        nonlocal _first_request
        if _first_request:
            matcher.df = matcher.load_and_preprocess_dataset()
            model_loaded = matcher.load_model()  # Try to load pre-trained models
            if not model_loaded:
                logger.warning("Models not loaded. ML search disabled.")
                matcher.vectorizer = None
                matcher.nmf_model = None
            _first_request = False
        return render_template('indexx.html', unique_groups=matcher.unique_groups, unique_brands=matcher.unique_brands)
    
    def load_spider_chart_data(file_path):
        with open(file_path, 'r') as f:  # Corrected variable name from `file_path` to `file_path`
            return json.load(f)
    # Load spider chart data
    spider_chart_data = load_spider_chart_data('spider_chart_data.json')

    @app.route('/predict', methods=['POST'])
    def predict():
        """Handles prediction requests."""
        # Get description from form
        description = request.form.get('description', '').strip()
        
        # Get the group and brand filters from the form
        selected_group = request.form.get('group_filter', '')
        selected_brand = request.form.get('brand_filter', '')
        
        logger.info(f"Received description for prediction: {description}")
        
        # EARLY CHECK: If description is empty and no filters selected, show browse mode
        if not description and not selected_group and not selected_brand:
            try:
                # Use pandas DataFrame directly, avoiding any matcher methods
                browse_df = matcher.df.head(750)  # Get first 100 rows
                
                # Load spider chart data
                spider_chart_data = load_spider_chart_data('spider_chart_data.json')
                
                # Create predictions list manually
                predictions = []
                
                for _, row in browse_df.iterrows():
                    # IMPORTANT: Make sure probability is a string ending with % for the template
                    prediction = {
                        'item_number': str(row.get('Item_Number', '')),
                        'group': str(row.get('Group_FR', '')),
                        'subgroup': str(row.get('Subgroup_FR', '')),
                        'brand': str(row.get('Brand', '')),
                        'series': str(row.get('Series', '')),
                        'description': str(row.get('Description_FR', '')),
                        'properties': str(row.get('Properties_FR', '')),
                        'probability': "100%",  # Use string with % sign to avoid split error
                        'image_url': str(row.get('IMAGE', '/static/images/placeholder.jpg')),
                        'datasheet_url': str(row.get('URL_FR', '#')),
                        'corresponding_words': [],
                        'unique_words': [],
                        'missing_words': [],
                        'top_features': []
                    }
                    
                    # Add spider chart data if available
                    item_number = prediction['item_number']
                    spider_data = spider_chart_data.get(item_number, {})
                    if spider_data and 'categories' in spider_data and 'values' in spider_data:
                        if len(spider_data['categories']) == len(spider_data['values']):
                            prediction['top_features'] = sorted(
                                zip(spider_data['categories'], spider_data['values']),
                                key=lambda x: x[1],
                                reverse=True
                            )[:5]
                        else:
                            prediction['top_features'] = []
                    
                    predictions.append(prediction)
                
                # Return browse mode template with data
                return render_template('resultss.html',
                                    predictions=predictions,
                                    cleaned_description="Browse all products",
                                    selected_group='',
                                    selected_brand='',
                                    spider_chart_data=spider_chart_data,
                                    is_browse_all=True)
            except Exception as e:
                logger.error(f"Error in browse mode: {str(e)}")
                import traceback
                logger.error(traceback.format_exc())  # Log the full stack trace
                return f"An error occurred in browse mode: {str(e)}", 500
        
        # Continue with normal prediction for non-empty description or selected filters
        try:
            filtered_df = matcher.filter_dataframe(selected_group, selected_brand)
            
            if filtered_df.empty:
                return render_template('resultss.html',
                                    predictions=[],
                                    cleaned_description=description,
                                    selected_group=selected_group,
                                    selected_brand=selected_brand,
                                    spider_chart_data={})
            
            # Use a dummy/placeholder description if original is empty
            # This can help avoid the split() error if that's happening in the predict_item_number method
            if not description and (selected_group or selected_brand):
                description = "placeholder"  # Use a non-empty placeholder
                
            # Normal prediction pathway
            predictions, cleaned_description = matcher.predict_item_number(description, df=filtered_df)
            
            # Load spider chart data
            spider_chart_data = load_spider_chart_data('spider_chart_data.json')
            
            # Add spider chart data to each prediction
            for prediction in predictions:
                item_number = prediction['item_number']
                spider_data = spider_chart_data.get(item_number, {})
                
                if spider_data and 'categories' in spider_data and 'values' in spider_data:
                    if len(spider_data['categories']) == len(spider_data['values']):
                        prediction['top_features'] = sorted(
                            zip(spider_data['categories'], spider_data['values']),
                            key=lambda x: x[1],
                            reverse=True
                        )[:5]
                    else:
                        prediction['top_features'] = []
                else:
                    prediction['top_features'] = []
            
            # Store unique words for feedback
            item_unique_words = {p['item_number']: p['unique_words'] for p in predictions}
            with tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix=".json") as temp_file:
                temp_file_name = temp_file.name
                json.dump(item_unique_words, temp_file)
            session['temp_file_name'] = temp_file_name

            # Add related products to each prediction
            for prediction in predictions:
                item_number = str(prediction['item_number'])
                if item_number in product_relationships:
                    prediction['related_products'] = product_relationships[item_number]

            return render_template('resultss.html',
                                predictions=predictions,
                                cleaned_description=cleaned_description,
                                selected_group=selected_group,
                                selected_brand=selected_brand,
                                spider_chart_data=spider_chart_data)
            
        except Exception as e:
            logger.error(f"Error in prediction: {str(e)}")
            return f"An error occurred: {str(e)}", 500
        

    @app.route('/save_feedback', methods=['POST'])
    def save_feedback():
        try:
            data = request.get_json()
            if not data:
                return jsonify({'success': False, 'message': 'No data received'}), 400

            feedback = data.get('feedback')
            item_number = data.get('item_number')
            description = data.get('description')
            selected_group = data.get('selected_group', '')  # Default to empty string if not provided
            selected_brand = data.get('selected_brand', '')  # Default to empty string if not provided

            if not all([feedback, item_number, description]):
                missing_fields = [field for field in ['feedback', 'item_number', 'description'] if not data.get(field)]
                return jsonify({'success': False, 'message': f'Missing required fields: {", ".join(missing_fields)}'}), 400

            preprocessed_description = matcher.preprocess_text(description)
            input_keywords = preprocessed_description.split()

            item_number_str = str(item_number).strip()
            matcher.df['Item_Number'] = matcher.df['Item_Number'].astype(str)
            item_data = matcher.df[matcher.df['Item_Number'] == item_number_str]

            if item_data.empty:
                return jsonify({'success': False, 'message': f'Item number not found: {item_number_str}'}), 404

            original_text = item_data.iloc[0]['Description_FR']
            preprocessed_original_text = item_data.iloc[0]['Preprocessed_Text']
            item_keywords = preprocessed_original_text.split()

            unique_words = [word for word in input_keywords if word not in item_keywords]
            corresponding_words = [word for word in input_keywords if word in item_keywords]
            missing_words = calculate_missing_words(input_keywords, item_keywords)

            csv_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'feedback_data.csv')
            timestamp = time.strftime('%Y-%m-%d %H:%M:%S')
            new_data = {
                'item_number': item_number_str,
                'unique_words': ' '.join(unique_words),
                'corresponding_words': ' '.join(corresponding_words),
                'missing_words': ' '.join(missing_words),
                'feedback': feedback,
                'timestamp': timestamp,
                'original_description': description,
                'preprocessed_description': preprocessed_description,
                #'selected_group': selected_group,
                #'selected_brand': selected_brand
            }

            try:
                df_existing = pd.read_csv(csv_file_path)
                df_existing['item_number'] = df_existing['item_number'].astype(str)

                # Convert 'selected_group' and 'selected_brand' columns to object type
                df_existing['selected_group'] = df_existing['selected_group'].astype(object)
                df_existing['selected_brand'] = df_existing['selected_brand'].astype(object)

            except FileNotFoundError:
                df_existing = pd.DataFrame(columns=[
                    'item_number', 'unique_words', 'corresponding_words', 'missing_words',
                    'feedback', 'timestamp', 'original_description', 'preprocessed_description',
                    'selected_group', 'selected_brand'
                ])

            if item_number_str in df_existing['item_number'].values:
                for col, value in new_data.items():
                    df_existing.loc[df_existing['item_number'] == item_number_str, col] = value
            else:
                df_existing = pd.concat([df_existing, pd.DataFrame([new_data])], ignore_index=True)

            os.makedirs(os.path.dirname(csv_file_path), exist_ok=True)
            df_existing.to_csv(csv_file_path, index=False)

            return jsonify({
                'success': True,
                'message': 'Feedback saved successfully',
                'corresponding_words': corresponding_words,
                'unique_words': unique_words,
                'missing_words': missing_words
            })

        except Exception as e:
            logger.error(f"Error in save_feedback: {str(e)}")
            return jsonify({'success': False, 'message': f'Server error: {str(e)}'}), 500
        
    @app.route('/get_item_details/<item_number>')
    def get_item_details(item_number):
        """
        Retrieve details for a specific item number with unique word analysis.
        """
        try:
            # Convert item_number to string for comparison
            item_number_str = str(item_number).strip()
            matcher.df['Item_Number'] = matcher.df['Item_Number'].astype(str)
            
            # Find the item in the DataFrame
            item_data = matcher.df[matcher.df['Item_Number'] == item_number_str]
            
            if item_data.empty:
                return jsonify({'error': 'Item not found'}), 404
            
            # Get the first (and should be only) matching row
            item = item_data.iloc[0]
            
            # Combine relevant text fields for analysis
            item_text = f"{item['Description_FR']} {item['Properties_FR']}"
            
            # Preprocess the text using the existing preprocess_text method
            processed_text = matcher.preprocess_text(item_text)
            
            # Split into individual words and remove duplicates while preserving order
            words = []
            [words.append(word) for word in processed_text.split() if word not in words]
            
            return jsonify({
                'item_number': item_number,
                'brand': item['Brand'],
                'group': item['Group_FR'],
                'subgroup': item['Subgroup_FR'],
                'description': item['Description_FR'],
                'properties': item['Properties_FR'],
                'image_url': item['IMAGE'],
                'datasheet_url': item['URL_FR'],
                'words': words  # Include the processed words for comparison
            })
            
        except Exception as e:
            logger.error(f"Error retrieving item details: {e}")
            return jsonify({'error': str(e)}), 500    

    @app.route('/import_article_list.html')
    def import_article_list():
        return render_template('import_article_list.html')

    @app.route('/validate_items', methods=['POST'])
    def validate_items():
        """
        Validates multiple item numbers against the database.
        
        Expected JSON format:
        {
            "items": ["123456", "789012", ...]
        }
        
        Returns:
        {
            "valid_items": [
                {"item_number": "123456", "details": {...}},
                ...
            ],
            "invalid_items": ["789012", ...]
        }
        """
        try:
            data = request.get_json()
            if not data or 'items' not in data:
                return jsonify({'error': 'No items provided'}), 400
                
            items = data['items']
            
            # Convert to strings for comparison
            matcher.df['Item_Number'] = matcher.df['Item_Number'].astype(str)
            valid_items = []
            invalid_items = []
            
            for item_number in items:
                item_number_str = str(item_number).strip()
                item_data = matcher.df[matcher.df['Item_Number'] == item_number_str]
                
                if not item_data.empty:
                    # Item exists
                    item = item_data.iloc[0]
                    valid_items.append({
                        'item_number': item_number_str,
                        'details': {
                            'brand': item['Brand'],
                            'group': item['Group_FR'],
                            'subgroup': item['Subgroup_FR'],
                            'description': item['Description_FR'],
                            'properties': item['Properties_FR'],
                            'image_url': item['IMAGE'],
                            'datasheet_url': item['URL_FR']
                        }
                    })
                else:
                    # Item does not exist
                    invalid_items.append(item_number_str)
            
            return jsonify({
                'valid_items': valid_items,
                'invalid_items': invalid_items
            })
            
        except Exception as e:
            logger.error(f"Error validating items: {e}")
            return jsonify({'error': str(e)}), 500


    @app.route('/get_multiple_item_details', methods=['POST'])
    def get_multiple_item_details():
        """
        Retrieve details for multiple item numbers for spec sheet generation.
        """
        try:
            data = request.get_json()
            if not data or 'items' not in data:
                return jsonify({'error': 'No items provided'}), 400
                
            item_numbers = data['items']
            
            # Convert to strings for comparison
            matcher.df['Item_Number'] = matcher.df['Item_Number'].astype(str)
            item_details = []
            
            for item_number in item_numbers:
                item_number_str = str(item_number).strip()
                item_data = matcher.df[matcher.df['Item_Number'] == item_number_str]
                
                if not item_data.empty:
                    # Get the first (and should be only) matching row
                    item = item_data.iloc[0]
                    
                    # Series field might not exist in all datasets
                    series = item['Series'] if 'Series' in matcher.df.columns else ''
                    
                    item_details.append({
                        'item_number': item_number_str,
                        'brand': item['Brand'],
                        'series': series,
                        'group': item['Group_FR'],
                        'subgroup': item['Subgroup_FR'],
                        'description': item['Description_FR'],
                        'properties': item['Properties_FR'],
                        'image_url': item['IMAGE'],
                        'datasheet_url': item['URL_FR']
                    })
            
            return jsonify(item_details)
            
        except Exception as e:
            logger.error(f"Error retrieving item details: {e}")
            return jsonify({'error': str(e)}), 500

    @app.route('/save_description_feedback', methods=['POST'])
    def save_description_feedback():
        """
        Save description or properties feedback to a CSV file, including highlighted text.
        """
        try:
            data = request.get_json()
            if not data:
                return jsonify({'success': False, 'message': 'No data received'}), 400

            item_number = data.get('item_number')
            field_type = data.get('field_type')
            feedback = data.get('feedback')
            highlighted_text = data.get('highlighted_text', '')  # Get highlighted text, default to empty string

            if not all([item_number, field_type, feedback]):
                missing_fields = [field for field in ['item_number', 'field_type', 'feedback'] if not data.get(field)]
                return jsonify({'success': False, 'message': f'Missing required fields: {", ".join(missing_fields)}'}), 400

            # Validate field_type
            if field_type not in ['description', 'properties']:
                return jsonify({'success': False, 'message': 'Invalid field type'}), 400

            # Create CSV file path
            csv_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'description_feedback.csv')
            
            # Create the feedback data
            timestamp = time.strftime('%Y-%m-%d %H:%M:%S')
            
            # Prepare data row
            new_data = {
                'item_number': str(item_number),
                'timestamp': timestamp,
                'field_type': field_type,
                'feedback': feedback,
                'highlighted_text': highlighted_text  # Add highlighted text to data
            }

            # Check if file exists and read it
            try:
                df_existing = pd.read_csv(csv_file_path)
            except FileNotFoundError:
                # Create new DataFrame with these columns
                df_existing = pd.DataFrame(columns=[
                    'item_number', 'timestamp', 'field_type', 'feedback', 'highlighted_text'
                ])

            # Append the new data
            df_existing = pd.concat([df_existing, pd.DataFrame([new_data])], ignore_index=True)

            # Save to CSV
            os.makedirs(os.path.dirname(csv_file_path), exist_ok=True)
            df_existing.to_csv(csv_file_path, index=False)

            return jsonify({
                'success': True,
                'message': 'Feedback saved successfully'
            })

        except Exception as e:
            logger.error(f"Error in save_description_feedback: {str(e)}")
            return jsonify({'success': False, 'message': f'Server error: {str(e)}'}), 500

    @app.route('/compare.html')
    def compare():
        return render_template('compare.html')
    
    @app.route('/basket.html')
    def basket():
        return render_template('basket.html')
    
    @app.route('/get_basket_item_details', methods=['POST'])
    def get_basket_item_details():
        try:
            # Get items from the request
            data = request.get_json()
            items = data.get('items', [])

            if not items:
                return jsonify({'error': 'No items provided'}), 400

            # Find item details in the dataset
            item_details = []
            for item_number in items:
                item = matcher.df[matcher.df['Item_Number'] == str(item_number)]
                
                if not item.empty:
                    item = item.iloc[0]
                    
                    # Add Series if column exists, otherwise use an empty string
                    series = item['Series'] if 'Series' in matcher.df.columns else ''
                    
                    item_details.append({
                        'item_number': item['Item_Number'],
                        'brand': item['Brand'],
                        'series': series,  # New addition
                        'group': item['Group_FR'],
                        'subgroup': item['Subgroup_FR'],
                        'description': item['Description_FR'],
                        'properties': item['Properties_FR'],
                        'image_url': item['IMAGE'],
                        'datasheet_url': item['URL_FR']
                    })

            return jsonify(item_details)
        
        except Exception as e:
            return jsonify({'error': str(e)}), 500

    return app

def main():
    import platform
    
    # Fix Tesseract path (line 30 fix should already be done)
    
    base_path = os.path.dirname(os.path.abspath(__file__))
    
    # Try multiple dataset paths
    possible_paths = [
        os.path.join(base_path, 'dataset.csv'),
        os.path.join(base_path, 'moederbestek', 'dataset_MAR2025_FR.csv'),
        os.path.join(base_path, 'moederbestek', 'specifications_catalog_full.csv')
    ]
    
    file_path = None
    for path in possible_paths:
        if os.path.exists(path):
            file_path = path
            print(f"Using dataset: {path}")
            break
    
    if not file_path:
        raise FileNotFoundError("No dataset.csv found!")
    
    app = create_app(file_path)
    
    # Use PORT from environment (Render provides this)
    port = int(os.environ.get('PORT', 5000))
    app.run(host='0.0.0.0', debug=False, port=port)

if __name__ == "__main__":
    main()

# Copyright (C) 2025 Mattias Colla. Alle rechten voorbehouden.