From 951297bb367a7189e5ac1decbfc58745b06b5865 Mon Sep 17 00:00:00 2001 From: Casey Becking Date: Mon, 27 Oct 2025 07:18:11 -0700 Subject: [PATCH] Separate transaction fields into individual database columns This update separates the transaction description into distinct fields for better data integrity and querying: - Added merchant, original_statement, notes, and tags as separate database columns - Updated CSV import to store fields individually instead of concatenating them - Enhanced frontend edit modal to support all transaction fields - Improved error reporting in CSV import to show first 50 errors with categorized summary - Updated all utility scripts to work from scripts/ directory with proper path handling - Added database migration scripts for new columns This change enables better filtering, searching, and reporting on transaction data by maintaining field separation throughout the system. --- api/transaction/controllers.py | 67 ++++++++++++++++----- api/transaction/models.py | 18 +++++- app/static/js/transactions/import.js | 25 ++++++-- app/static/js/transactions/transactions.js | 12 ++++ app/templates/transactions/index.html | 30 +++++++++- scripts/add_merchant_column.py | 29 +++++++++ scripts/add_transaction_fields.py | 37 ++++++++++++ scripts/clear_tables.py | 53 +++++++++++++++++ scripts/clear_transactions.py | 28 +++++++++ scripts/convert_monarch_categories.py | 69 ++++++++++++++++++++++ 10 files changed, 345 insertions(+), 23 deletions(-) create mode 100644 scripts/add_merchant_column.py create mode 100644 scripts/add_transaction_fields.py create mode 100644 scripts/clear_tables.py create mode 100644 scripts/clear_transactions.py create mode 100644 scripts/convert_monarch_categories.py diff --git a/api/transaction/controllers.py b/api/transaction/controllers.py index 1fb736f..fbfbbe3 100644 --- a/api/transaction/controllers.py +++ b/api/transaction/controllers.py @@ -22,6 +22,10 @@ 'transaction_type': fields.String(required=True, description='Transaction Type'), 'external_id': fields.String(description='External ID'), 'external_date': fields.DateTime(description='External Date'), + 'merchant': fields.String(description='Merchant Name'), + 'original_statement': fields.String(description='Original Statement'), + 'notes': fields.String(description='Notes'), + 'tags': fields.String(description='Tags'), 'description': fields.String(description='Description') }) @@ -37,6 +41,10 @@ def post(self): transaction_type = data.get('transaction_type') external_id = data.get('external_id') external_date = data.get('external_date') + merchant = data.get('merchant') + original_statement = data.get('original_statement') + notes = data.get('notes') + tags = data.get('tags') description = data.get('description') # Validate required fields @@ -61,6 +69,10 @@ def post(self): transaction_type=transaction_type, external_id=external_id, external_date=external_date, + merchant=merchant, + original_statement=original_statement, + notes=notes, + tags=tags, description=description ) new_transaction.save() @@ -210,20 +222,11 @@ def post(self): error_count += 1 continue - # Build description from available fields - description_parts = [] - if merchant: - description_parts.append(f"Merchant: {merchant}") - if original_statement: - description_parts.append(f"Statement: {original_statement}") - if notes: - description_parts.append(f"Notes: {notes}") - if tags: - description_parts.append(f"Tags: {tags}") + # Store fields separately (no joining) + # Description can be used for additional custom info if needed + description = None # Keep empty unless specifically provided - description = " | ".join(description_parts) if description_parts else merchant - - # Create transaction + # Create transaction with all fields separate self.create_transaction( user_id=user_id, categories_id=category_id, @@ -232,6 +235,10 @@ def post(self): transaction_type=_transaction_type, external_id=external_id, external_date=formatted_timestamp, + merchant=merchant, + original_statement=original_statement, + notes=notes, + tags=tags, description=description ) created_count += 1 @@ -251,8 +258,24 @@ def post(self): 'errors': error_count } - if errors and len(errors) <= 10: # Only include first 10 errors - response_data['error_details'] = errors[:10] + if errors: + # Show first 50 errors to understand patterns + response_data['error_details'] = errors[:50] + # Also include a summary of error types + error_summary = {} + for error in errors: + # Extract error type + if "Category" in error and "not found" in error: + error_summary['category_not_found'] = error_summary.get('category_not_found', 0) + 1 + elif "Could not create account" in error: + error_summary['account_creation_failed'] = error_summary.get('account_creation_failed', 0) + 1 + elif "Could not parse" in error: + error_summary['date_parse_error'] = error_summary.get('date_parse_error', 0) + 1 + elif "Invalid amount" in error: + error_summary['invalid_amount'] = error_summary.get('invalid_amount', 0) + 1 + else: + error_summary['other'] = error_summary.get('other', 0) + 1 + response_data['error_summary'] = error_summary return make_response(jsonify(response_data), 201 if created_count > 0 else 200) @@ -347,7 +370,7 @@ def ensure_account_exists_smart(self, account_name, merchant_name): return account.id - def create_transaction(self,user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, description): + def create_transaction(self,user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, merchant=None, original_statement=None, notes=None, tags=None, description=None): print(f"Creating transaction {external_id}...") transaction = TransactionModel( user_id=user_id, @@ -357,6 +380,10 @@ def create_transaction(self,user_id, categories_id, account_id, amount, transact transaction_type=transaction_type, external_id=external_id, external_date=external_date, + merchant=merchant, + original_statement=original_statement, + notes=notes, + tags=tags, description=description ) db.session.add(transaction) @@ -408,6 +435,14 @@ def put(self, id): }), 400) # Update fields if provided + if 'merchant' in data: + transaction.merchant = data['merchant'] + if 'original_statement' in data: + transaction.original_statement = data['original_statement'] + if 'notes' in data: + transaction.notes = data['notes'] + if 'tags' in data: + transaction.tags = data['tags'] if 'description' in data: transaction.description = data['description'] if 'amount' in data: diff --git a/api/transaction/models.py b/api/transaction/models.py index 8b07d6a..93f8503 100644 --- a/api/transaction/models.py +++ b/api/transaction/models.py @@ -25,9 +25,13 @@ class TransactionModel(Base): transaction_type = db.Column(db.String(255), nullable=False) external_id = db.Column(db.String(255), nullable=False) external_date = db.Column(db.DateTime, nullable=True) + merchant = db.Column(db.String(255), nullable=True) + original_statement = db.Column(db.String(255), nullable=True) + notes = db.Column(db.Text, nullable=True) + tags = db.Column(db.String(255), nullable=True) description = db.Column(db.String(255), nullable=True) - def __init__(self, user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, description): + def __init__(self, user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, merchant=None, original_statement=None, notes=None, tags=None, description=None): """ Initialize a TransactionModel instance. @@ -39,6 +43,10 @@ def __init__(self, user_id, categories_id, account_id, amount, transaction_type, transaction_type (str): The transaction_type of the transaction. external_id (str): The external ID of the transaction. external_date (datetime): The external date of the transaction. + merchant (str): The merchant name for the transaction. + original_statement (str): The original statement text from the bank. + notes (str): User notes for the transaction. + tags (str): Tags for categorizing the transaction. description (str): The description of the transaction. """ self.user_id = user_id @@ -48,6 +56,10 @@ def __init__(self, user_id, categories_id, account_id, amount, transaction_type, self.transaction_type = transaction_type self.external_id = external_id self.external_date = external_date + self.merchant = merchant + self.original_statement = original_statement + self.notes = notes + self.tags = tags self.description = description def __repr__(self): @@ -77,6 +89,10 @@ def to_dict(self): 'transaction_type': self.transaction_type, 'external_id': self.external_id, 'external_date': self.external_date, + 'merchant': self.merchant, + 'original_statement': self.original_statement, + 'notes': self.notes, + 'tags': self.tags, 'description': self.description, 'created_at': self.created_at, 'updated_at': self.updated_at diff --git a/app/static/js/transactions/import.js b/app/static/js/transactions/import.js index 0e5d30d..76e840a 100644 --- a/app/static/js/transactions/import.js +++ b/app/static/js/transactions/import.js @@ -117,13 +117,30 @@ uploadBtn.addEventListener('click', async () => { function showSuccess(data) { let errorDetailsHtml = ''; if (data.error_details && data.error_details.length > 0) { + let errorSummaryHtml = ''; + if (data.error_summary) { + errorSummaryHtml = ` +
+ Error Summary: + +
+ `; + } + errorDetailsHtml = `
-
Import Errors (showing first 10):
- +
Import Errors (showing first 50):
+ ${errorSummaryHtml} +
+
    + ${data.error_details.map(err => `
  • ${err}
    • `).join('')} +
+
`; } diff --git a/app/static/js/transactions/transactions.js b/app/static/js/transactions/transactions.js index 88e5337..c438662 100644 --- a/app/static/js/transactions/transactions.js +++ b/app/static/js/transactions/transactions.js @@ -88,6 +88,10 @@ async function editTransaction(transactionId) { if (response.ok) { // Populate the edit form document.getElementById('edit_transaction_id').value = transactionId; + document.getElementById('edit_transactionMerchant').value = data.transaction.merchant || ''; + document.getElementById('edit_transactionOriginalStatement').value = data.transaction.original_statement || ''; + document.getElementById('edit_transactionNotes').value = data.transaction.notes || ''; + document.getElementById('edit_transactionTags').value = data.transaction.tags || ''; document.getElementById('edit_transactionDescription').value = data.transaction.description || ''; document.getElementById('edit_transactionAmount').value = data.transaction.amount || ''; document.getElementById('edit_transactionCategory').value = data.transaction.categories_id || ''; @@ -109,6 +113,10 @@ function updateTransaction(event) { event.preventDefault(); const transactionId = document.getElementById('edit_transaction_id').value; + const merchant = document.getElementById('edit_transactionMerchant').value; + const originalStatement = document.getElementById('edit_transactionOriginalStatement').value; + const notes = document.getElementById('edit_transactionNotes').value; + const tags = document.getElementById('edit_transactionTags').value; const description = document.getElementById('edit_transactionDescription').value; const amount = document.getElementById('edit_transactionAmount').value; const categoryId = document.getElementById('edit_transactionCategory').value; @@ -116,6 +124,10 @@ function updateTransaction(event) { const user_id = document.getElementById('edit_transaction_user_id').value; const data = { + merchant: merchant, + original_statement: originalStatement, + notes: notes, + tags: tags, description: description, amount: parseFloat(amount), categories_id: categoryId, diff --git a/app/templates/transactions/index.html b/app/templates/transactions/index.html index d7e8115..acaea6e 100644 --- a/app/templates/transactions/index.html +++ b/app/templates/transactions/index.html @@ -107,6 +107,7 @@
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
- +
diff --git a/scripts/add_merchant_column.py b/scripts/add_merchant_column.py new file mode 100644 index 0000000..4889887 --- /dev/null +++ b/scripts/add_merchant_column.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +""" +Migration script to add merchant column to transaction table +""" +import os +import sys + +# Change to project root directory (parent of scripts directory) +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +os.chdir(project_root) +sys.path.insert(0, project_root) + +from app import create_app, db +from sqlalchemy import text + +app = create_app() + +with app.app_context(): + try: + # Add the merchant column if it doesn't exist + db.session.execute(text(""" + ALTER TABLE transaction + ADD COLUMN IF NOT EXISTS merchant VARCHAR(255); + """)) + db.session.commit() + print("✓ Successfully added merchant column to transaction table") + except Exception as e: + print(f"✗ Error adding merchant column: {e}") + db.session.rollback() diff --git a/scripts/add_transaction_fields.py b/scripts/add_transaction_fields.py new file mode 100644 index 0000000..b3b2f97 --- /dev/null +++ b/scripts/add_transaction_fields.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +""" +Migration script to add original_statement, notes, and tags columns to transaction table +""" +import os +import sys + +# Change to project root directory (parent of scripts directory) +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +os.chdir(project_root) +sys.path.insert(0, project_root) + +from app import create_app, db +from sqlalchemy import text + +app = create_app() + +with app.app_context(): + try: + # Add the new columns if they don't exist + db.session.execute(text(""" + ALTER TABLE transaction + ADD COLUMN IF NOT EXISTS original_statement VARCHAR(255); + """)) + db.session.execute(text(""" + ALTER TABLE transaction + ADD COLUMN IF NOT EXISTS notes TEXT; + """)) + db.session.execute(text(""" + ALTER TABLE transaction + ADD COLUMN IF NOT EXISTS tags VARCHAR(255); + """)) + db.session.commit() + print("✓ Successfully added original_statement, notes, and tags columns to transaction table") + except Exception as e: + print(f"✗ Error adding columns: {e}") + db.session.rollback() diff --git a/scripts/clear_tables.py b/scripts/clear_tables.py new file mode 100644 index 0000000..5131601 --- /dev/null +++ b/scripts/clear_tables.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +""" +Script to clear all tables except user table +""" +import os +import sys + +# Change to project root directory (parent of scripts directory) +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +os.chdir(project_root) +sys.path.insert(0, project_root) + +from app import create_app, db +from sqlalchemy import text + +app = create_app() + +with app.app_context(): + try: + # Clear tables in correct order (respecting foreign key constraints) + print("Clearing tables...") + + # Delete transactions first (has foreign keys to categories and accounts) + result = db.session.execute(text("DELETE FROM transaction")) + print(f"✓ Cleared transaction table ({result.rowcount} rows)") + + # Delete accounts (has foreign key to institution) + result = db.session.execute(text("DELETE FROM account")) + print(f"✓ Cleared account table ({result.rowcount} rows)") + + # Delete institutions + result = db.session.execute(text("DELETE FROM institution")) + print(f"✓ Cleared institution table ({result.rowcount} rows)") + + # Delete categories (has foreign keys to categories_group and categories_type) + result = db.session.execute(text("DELETE FROM categories")) + print(f"✓ Cleared categories table ({result.rowcount} rows)") + + # Delete categories_group (has foreign key to categories_type) + result = db.session.execute(text("DELETE FROM categories_group")) + print(f"✓ Cleared categories_group table ({result.rowcount} rows)") + + # Delete categories_type + result = db.session.execute(text("DELETE FROM categories_type")) + print(f"✓ Cleared categories_type table ({result.rowcount} rows)") + + db.session.commit() + print("\n✓ Successfully cleared all tables (except user table)") + print("\nYou can now start fresh with importing categories and transactions!") + + except Exception as e: + print(f"✗ Error clearing tables: {e}") + db.session.rollback() diff --git a/scripts/clear_transactions.py b/scripts/clear_transactions.py new file mode 100644 index 0000000..67b4a31 --- /dev/null +++ b/scripts/clear_transactions.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +""" +Script to clear only the transactions table +""" +import os +import sys + +# Change to project root directory (parent of scripts directory) +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +os.chdir(project_root) +sys.path.insert(0, project_root) + +from app import create_app, db +from sqlalchemy import text + +app = create_app() + +with app.app_context(): + try: + # Delete all transactions + result = db.session.execute(text("DELETE FROM transaction")) + db.session.commit() + print(f"✓ Cleared transaction table ({result.rowcount} rows)") + print("\nTransactions table is now empty. Ready for fresh import!") + + except Exception as e: + print(f"✗ Error clearing transactions: {e}") + db.session.rollback() diff --git a/scripts/convert_monarch_categories.py b/scripts/convert_monarch_categories.py new file mode 100644 index 0000000..d0bc014 --- /dev/null +++ b/scripts/convert_monarch_categories.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +""" +Convert Monarch categories JSON to CSV format for OSPF import +""" +import json +import csv +import os +import sys + +# Change to project root directory (parent of scripts directory) +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +os.chdir(project_root) + +# Read the JSON file +with open('data/monarch-categories.json', 'r') as f: + data = json.load(f) + +# Extract categories +categories = data['data']['categories'] + +# Create a set to track unique combinations (to avoid duplicates) +seen = set() +rows = [] + +for category in categories: + # Skip disabled categories + if category.get('isDisabled', False): + continue + + category_name = category['name'] + group_name = category['group']['name'] + group_type = category['group']['type'] + + # Capitalize first letter of type for consistency with existing data + if group_type == 'income': + type_name = 'Income' + elif group_type == 'expense': + type_name = 'Expense' + elif group_type == 'transfer': + type_name = 'Transfer' + else: + type_name = group_type.capitalize() + + # Create unique key + key = (category_name, group_name, type_name) + + # Only add if not seen before + if key not in seen: + seen.add(key) + rows.append({ + 'categories': category_name, + 'categories_group': group_name, + 'categories_type': type_name + }) + +# Sort by type, then group, then category +rows.sort(key=lambda x: (x['categories_type'], x['categories_group'], x['categories'])) + +# Write to CSV +with open('data/monarch-categories-data.csv', 'w', newline='', encoding='utf-8') as f: + writer = csv.DictWriter(f, fieldnames=['categories', 'categories_group', 'categories_type']) + writer.writeheader() + writer.writerows(rows) + +print(f"✓ Successfully created monarch-categories-data.csv with {len(rows)} categories") +print(f"\nBreakdown:") +print(f" - Income categories: {sum(1 for r in rows if r['categories_type'] == 'Income')}") +print(f" - Expense categories: {sum(1 for r in rows if r['categories_type'] == 'Expense')}") +print(f" - Transfer categories: {sum(1 for r in rows if r['categories_type'] == 'Transfer')}")