word_parser.py

"""
Word Document Parser for Specifications Catalog

This script parses a structured Word document containing product specifications
and outputs the data to a CSV file.

The script handles the following hierarchical structure from the document:
1. Group_title - Top level category in UPPERCASE (e.g., "MECHANISCHE SLOTEN")
2. Subgroup_title - Second level with number format "00.00.00" (e.g., "00.00.00 Mechanische éénpuntsloten")
3. Item_title_NL - Specific product category with number and brand (e.g., "00.00.00 Standaard klavierslot... |FH| st Litto")
4. Description_NL - Detailed text description of the product category
5. LongDescription - Specific product description in purple text
6. Item_Number - Reference code (e.g., "A13E1")
7. Brand - Brand name extracted from Item_title_NL (e.g., "Litto")
8. Measuring_State - Special format text (e.g., "|FH| st")

Usage:
  python word_parser.py -i input.docx -o output.csv [--debug]

Requirements:
  - python-docx
  - re
  - csv
"""

import os
import re
import csv
from docx import Document
from docx.shared import RGBColor
from docx.text.run import Run

def is_purple_text(run):
    """Check if text is in purple color."""
    if not hasattr(run, 'font') or not hasattr(run.font, 'color') or not hasattr(run.font.color, 'rgb'):
        return False
    
    # Check for the specific purple color you mentioned (RGB 112, 48, 160)
    # Allow some tolerance in the values for potential slight variations
    rgb = run.font.color.rgb
    if rgb:
        r, g, b = rgb[0], rgb[1], rgb[2]
        
        # Check if the color is close to your specific purple
        r_match = 90 <= r <= 130  # Around 112
        g_match = 30 <= g <= 70   # Around 48
        b_match = 140 <= b <= 180 # Around 160
        
        return r_match and g_match and b_match
    
    return False

def extract_bullet_text(text):
    """Extract text after bullet point characters."""
    # List of possible bullet characters
    bullets = ['•', '-', '>', '*', '◦', '○', '■', '□', '▪', '▫', '⁃', '‣']
    
    # Check if text starts with any bullet character
    for bullet in bullets:
        if text.lstrip().startswith(bullet):
            # Return text after the bullet, stripped of leading/trailing whitespace
            return text.lstrip().split(bullet, 1)[1].strip()
    
    # Handle indented bullet points with a leading space
    if text.lstrip().startswith('    ') and any(b in text for b in bullets):
        for bullet in bullets:
            if bullet in text:
                return text.split(bullet, 1)[1].strip()
    
    # No bullet found, return the original text
    return text

def find_long_description(doc, start_idx):
    """
    Find the LongDescription with the specific characteristics:
    - Located before a REFERENTIE line
    - Has purple text (RGB 112, 48, 160)
    - Often starts with a bullet point
    """
    # Search up to 5 paragraphs before the reference line
    max_lookback = min(start_idx, 5)
    
    for offset in range(1, max_lookback + 1):
        idx = start_idx - offset
        para = doc.paragraphs[idx]
        text = para.text.strip()
        
        # Skip empty paragraphs
        if not text:
            continue
        
        # Check if this paragraph has purple text
        has_purple = False
        for run in para.runs:
            if is_purple_text(run):
                has_purple = True
                break
        
        # If it has purple text, it's likely our LongDescription
        if has_purple:
            # If it starts with a bullet, clean it up
            return extract_bullet_text(text)
        
        # Check if it's a bullet point that might not have been detected as purple
        if text.startswith('•') or text.startswith('-'):
            # Double-check nearby paragraphs to ensure we're not in the middle of a list
            if idx > 0 and idx < len(doc.paragraphs) - 1:
                prev_text = doc.paragraphs[idx - 1].text.strip()
                next_text = doc.paragraphs[idx + 1].text.strip()
                
                # If this is a standalone bullet point or the last in a list before REFERENTIE
                if not (prev_text.startswith('•') or prev_text.startswith('-')) or \
                   (next_text.startswith('REFERENTIE') or 'REFERENTIE' in next_text):
                    return extract_bullet_text(text)
    
    # If nothing found, try one more approach - look for the last bullet point before reference
    for offset in range(1, max_lookback + 1):
        idx = start_idx - offset
        para = doc.paragraphs[idx]
        text = para.text.strip()
        
        if not text:
            continue
            
        # Simple check for bullet points
        if text.startswith('•') or text.startswith('-') or '>':
            return extract_bullet_text(text)
    
    return ""

def is_different_font(run, normal_font):
    """Check if text has a different font than the normal document font."""
    if not hasattr(run, 'font') or not hasattr(run.font, 'name'):
        return False
    
    return run.font.name != normal_font

def extract_brand(text):
    """Extract brand from text with a special format."""
    # Look for text between '|FH| st ' or '|FH| ST ' and the end or other delimiter
    # Now also handles bold formatting markers ** around the brand name
    pattern = r'\|FH\|\s+(?:st|ST)\s+(?:\*\*)?([^\]\*]+)(?:\*\*)?'
    match = re.search(pattern, text, re.IGNORECASE)
    if match:
        return match.group(1).strip()
    return ""

def is_group_title(text, doc, index):
    """Check if the text is likely a Group_title."""
    # Group titles are ALL CAPS
    if not text.isupper():
        return False
    
    # Exclude obvious non-titles
    if text.startswith("00.00.00") or "REFERENTIE" in text:
        return False
    
    # Check if it's a known Group_title
    known_titles = ["MECHANISCHE SLOTEN", "ELEKTROMECHANISCHE SLOTEN", "DIGITAL ACCESS SOLUTIONS"]
    if any(title in text for title in known_titles):
        return True
    
    # Check the surrounding context
    # If it's followed by a Subgroup_title (00.00.00), it's likely a Group_title
    if index < len(doc.paragraphs) - 1:
        next_text = doc.paragraphs[index + 1].text.strip()
        if next_text.startswith("00.00.00"):
            return True
    
    # If it's short and all caps, and not a common technical term, it's likely a title
    if len(text.split()) <= 3 and not any(term in text for term in ["ISO", "EN", "PC", "CE"]):
        return True
        
    return False


def extract_measuring_state(text):
    """Extract measuring state from text."""
    # Look for '|FH| st' or '|FH| ST' pattern, case insensitive
    match = re.search(r'\|FH\|\s+(?:st|ST)', text, re.IGNORECASE)
    if match:
        return match.group(0).strip()
    return ""

def extract_item_title_without_metadata(text):
    """Extract the clean item title without measuring state and brand."""
    # Remove the |FH| st and brand part
    title = re.sub(r'\[\[\\?\|FH\|\\?\s+st\s+[^\]]+\]\]', '', text)
    # Remove any other formatting markers
    title = re.sub(r'\[\[|\]\]|\{[^}]+\}', '', title)
    # Clean up any extra spaces
    title = re.sub(r'\s+', ' ', title).strip()
    return title

def extract_item_number(text):
    """Extract item number from reference text.
    
    Now handles both formats:
    1. "REFERENTIE: A13E1 OF EQUIVALENT" 
    2. "REFERENTIE: A13E1" (without "OF EQUIVALENT")
    """
    # First try the original pattern with "OF EQUIVALENT"
    pattern1 = r'(?:REFERENTIE|[Rr]eferentie)\s*:\s*([^\s]+(?:\s+[^\s]+)*?)\s*(?:OF|[Oo]f)\s+(?:EQUIVALENT|[Ee]quivalent)'
    match = re.search(pattern1, text)
    
    if match:
        # Clean up any formatting markers
        item_number = match.group(1).strip()
        item_number = re.sub(r'\[\[|\]\]|\{[^}]+\}', '', item_number)
        return item_number.strip()
    
    # If not found, try without "OF EQUIVALENT"
    pattern2 = r'(?:REFERENTIE|[Rr]eferentie)\s*:\s*([^\s]+(?:\s+[^\s]+)*?)(?:\s*$|\s*[,.])'
    match = re.search(pattern2, text)
    
    if match:
        # Clean up any formatting markers
        item_number = match.group(1).strip()
        item_number = re.sub(r'\[\[|\]\]|\{[^}]+\}', '', item_number)
        return item_number.strip()
    
    # As a fallback, try a more general pattern
    pattern3 = r'(?:REFERENTIE|[Rr]eferentie)\s*:\s*(\S+)'
    match = re.search(pattern3, text)
    
    if match:
        item_number = match.group(1).strip()
        item_number = re.sub(r'\[\[|\]\]|\{[^}]+\}', '', item_number)
        return item_number.strip()
    
    return ""

def is_reference_line(text):
    """Check if this text line contains a reference."""
    # Check for various ways references might appear in the document
    pattern = r'[Rr][Ee][Ff][Ee][Rr][Ee][Nn][Tt][Ii][Ee]'
    if re.search(pattern, text):
        return True
    return False

def clean_text(text):
    """Clean up text by removing problematic characters and normalizing whitespace."""
    if not text:
        return ""
    
    # Replace tab characters, multiple spaces, etc.
    cleaned = re.sub(r'\s+', ' ', text)
    # Remove any control characters
    cleaned = ''.join(c for c in cleaned if c >= ' ' or c in '\n\r\t')
    return cleaned.strip()

def parse_word_document(file_path):
    """
    Parse the Word document and extract structured data.
    
    Args:
        file_path: Path to the Word document
        
    Returns:
        list: List of dictionaries with extracted data
    """
    doc = Document(file_path)
    
    # Determine the normal font used in the document
    normal_font = "Times New Roman"  # Default, will be updated if possible
    if doc.styles['Normal'].font.name:
        normal_font = doc.styles['Normal'].font.name
    
    # Initialize variables to track hierarchy
    current_group = ""
    current_subgroup = ""
    current_item_title = ""
    current_description = ""
    current_brand = ""
    current_measuring_state = ""
    
    # List to store all extracted data
    extracted_data = []
    
    # Flag to track if we're collecting description text
    collecting_description = False
    description_buffer = []
    
    # Process each paragraph
    for i, para in enumerate(doc.paragraphs):
        text = para.text.strip()
        
        # Skip empty paragraphs
        if not text:
            continue
        
        # Check if this is a Group_title
        if text.isupper() and not text.startswith("00.00.00") and "REFERENTIE" not in text:
            if is_group_title(text, doc, i):  # Pass the document and index
                current_group = clean_text(text)
                
                # If we were collecting description, save it before moving to new group
                if collecting_description and description_buffer:
                    current_description = clean_text(" ".join(description_buffer))
                    description_buffer = []
                    
                collecting_description = False
                continue
        
        # Check if this is a Subgroup_title (starts with numbers, no special format)
        if text.startswith("00.00.00") and not any("|FH|" in run.text for run in para.runs):
            current_subgroup = clean_text(text)
            
            # If we were collecting description, save it before moving to new subgroup
            if collecting_description and description_buffer:
                current_description = clean_text(" ".join(description_buffer))
                description_buffer = []
                
            collecting_description = False
            continue
        
        # Check if this is an Item_title_NL
        if text.startswith("00.00.00") and any("|FH|" in run.text for run in para.runs):
            # If we were collecting description, save it before moving to new item
            if collecting_description and description_buffer:
                current_description = clean_text(" ".join(description_buffer))
                description_buffer = []
            
            current_item_title = clean_text(text)
            # Extract brand and measuring state
            current_brand = clean_text(extract_brand(text))
            current_measuring_state = clean_text(extract_measuring_state(text))
            
            # Start collecting description
            collecting_description = True
            description_buffer = []  # Reset the buffer
            continue
        
        # Check if this is an Item_Number reference
        if "REFERENTIE" in text or "Referentie" in text:
            item_number = clean_text(extract_item_number(text))
            
            # Get the LongDescription and print it for debugging
            long_description = clean_text(find_long_description(doc, i))
            print(f"Found Reference: {item_number}")
            print(f"LongDescription: {long_description}")
            
            # If we were collecting description, finalize it
            if collecting_description and description_buffer:
                current_description = clean_text(" ".join(description_buffer))
            
            # Add data to our list
            extracted_data.append({
                "Group_title": current_group,
                "Subgroup_title": current_subgroup,
                "Item_title_NL": current_item_title,
                "Description_NL": current_description,
                "LongDescription": long_description,
                "Item_Number": item_number,
                "Brand": current_brand,
                "Measuring_State": current_measuring_state
            })
            
            # Don't reset the description since multiple item numbers might share it
            continue
        
        # If we're collecting description text, add to buffer
        if collecting_description:
            # Skip bullet points and other formatting that might be part of the description
            description_buffer.append(text)
    
    return extracted_data

def save_to_csv(data, output_file):
    """Save extracted data to CSV file."""
    if not data:
        print("No data to save.")
        return
    
    # Define the fieldnames for the CSV
    fieldnames = [
        "Group_title", 
        "Subgroup_title", 
        "Item_title_NL", 
        "Description_NL", 
        "LongDescription", 
        "Item_Number", 
        "Brand", 
        "Measuring_State"
    ]
    
    # Write to CSV with UTF-8-BOM encoding and semicolon delimiter for Excel
    with open(output_file, 'w', newline='', encoding='utf-8-sig') as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames, delimiter=';', quotechar='"', quoting=csv.QUOTE_ALL)
        writer.writeheader()
        writer.writerows(data)
    
    print(f"Data saved to {output_file}")

def main():
    """Main function to run the parser."""
    # Accept command line arguments for input and output files
    import argparse
    
    parser = argparse.ArgumentParser(description="Parse Word document with product specifications")
    parser.add_argument("-i", "--input", default="specifications_catalog.docx", 
                        help="Input Word document file path")
    parser.add_argument("-o", "--output", default="specifications_catalog.csv",
                        help="Output CSV file path")
    parser.add_argument("--debug", action="store_true", help="Print debug information")
    
    args = parser.parse_args()
    
    input_file = args.input
    output_file = args.output
    
    print(f"Parsing {input_file}...")
    data = parse_word_document(input_file)
    
    # Print some debug info if requested
    if args.debug and data:
        print("\nSample of extracted data:")
        for field in data[0].keys():
            print(f"{field}: {data[0][field][:50]}..." if len(str(data[0][field])) > 50 else f"{field}: {data[0][field]}")
    
    print(f"\nExtracted {len(data)} items.")
    save_to_csv(data, output_file)
    
    print(f"Data successfully saved to {output_file}")
    print("Done!")

if __name__ == "__main__":
    main()