diff --git a/api/transaction/controllers.py b/api/transaction/controllers.py
index 1fb736f..fbfbbe3 100644
--- a/api/transaction/controllers.py
+++ b/api/transaction/controllers.py
@@ -22,6 +22,10 @@
'transaction_type': fields.String(required=True, description='Transaction Type'),
'external_id': fields.String(description='External ID'),
'external_date': fields.DateTime(description='External Date'),
+ 'merchant': fields.String(description='Merchant Name'),
+ 'original_statement': fields.String(description='Original Statement'),
+ 'notes': fields.String(description='Notes'),
+ 'tags': fields.String(description='Tags'),
'description': fields.String(description='Description')
})
@@ -37,6 +41,10 @@ def post(self):
transaction_type = data.get('transaction_type')
external_id = data.get('external_id')
external_date = data.get('external_date')
+ merchant = data.get('merchant')
+ original_statement = data.get('original_statement')
+ notes = data.get('notes')
+ tags = data.get('tags')
description = data.get('description')
# Validate required fields
@@ -61,6 +69,10 @@ def post(self):
transaction_type=transaction_type,
external_id=external_id,
external_date=external_date,
+ merchant=merchant,
+ original_statement=original_statement,
+ notes=notes,
+ tags=tags,
description=description
)
new_transaction.save()
@@ -210,20 +222,11 @@ def post(self):
error_count += 1
continue
- # Build description from available fields
- description_parts = []
- if merchant:
- description_parts.append(f"Merchant: {merchant}")
- if original_statement:
- description_parts.append(f"Statement: {original_statement}")
- if notes:
- description_parts.append(f"Notes: {notes}")
- if tags:
- description_parts.append(f"Tags: {tags}")
+ # Store fields separately (no joining)
+ # Description can be used for additional custom info if needed
+ description = None # Keep empty unless specifically provided
- description = " | ".join(description_parts) if description_parts else merchant
-
- # Create transaction
+ # Create transaction with all fields separate
self.create_transaction(
user_id=user_id,
categories_id=category_id,
@@ -232,6 +235,10 @@ def post(self):
transaction_type=_transaction_type,
external_id=external_id,
external_date=formatted_timestamp,
+ merchant=merchant,
+ original_statement=original_statement,
+ notes=notes,
+ tags=tags,
description=description
)
created_count += 1
@@ -251,8 +258,24 @@ def post(self):
'errors': error_count
}
- if errors and len(errors) <= 10: # Only include first 10 errors
- response_data['error_details'] = errors[:10]
+ if errors:
+ # Show first 50 errors to understand patterns
+ response_data['error_details'] = errors[:50]
+ # Also include a summary of error types
+ error_summary = {}
+ for error in errors:
+ # Extract error type
+ if "Category" in error and "not found" in error:
+ error_summary['category_not_found'] = error_summary.get('category_not_found', 0) + 1
+ elif "Could not create account" in error:
+ error_summary['account_creation_failed'] = error_summary.get('account_creation_failed', 0) + 1
+ elif "Could not parse" in error:
+ error_summary['date_parse_error'] = error_summary.get('date_parse_error', 0) + 1
+ elif "Invalid amount" in error:
+ error_summary['invalid_amount'] = error_summary.get('invalid_amount', 0) + 1
+ else:
+ error_summary['other'] = error_summary.get('other', 0) + 1
+ response_data['error_summary'] = error_summary
return make_response(jsonify(response_data), 201 if created_count > 0 else 200)
@@ -347,7 +370,7 @@ def ensure_account_exists_smart(self, account_name, merchant_name):
return account.id
- def create_transaction(self,user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, description):
+ def create_transaction(self,user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, merchant=None, original_statement=None, notes=None, tags=None, description=None):
print(f"Creating transaction {external_id}...")
transaction = TransactionModel(
user_id=user_id,
@@ -357,6 +380,10 @@ def create_transaction(self,user_id, categories_id, account_id, amount, transact
transaction_type=transaction_type,
external_id=external_id,
external_date=external_date,
+ merchant=merchant,
+ original_statement=original_statement,
+ notes=notes,
+ tags=tags,
description=description
)
db.session.add(transaction)
@@ -408,6 +435,14 @@ def put(self, id):
}), 400)
# Update fields if provided
+ if 'merchant' in data:
+ transaction.merchant = data['merchant']
+ if 'original_statement' in data:
+ transaction.original_statement = data['original_statement']
+ if 'notes' in data:
+ transaction.notes = data['notes']
+ if 'tags' in data:
+ transaction.tags = data['tags']
if 'description' in data:
transaction.description = data['description']
if 'amount' in data:
diff --git a/api/transaction/models.py b/api/transaction/models.py
index 8b07d6a..93f8503 100644
--- a/api/transaction/models.py
+++ b/api/transaction/models.py
@@ -25,9 +25,13 @@ class TransactionModel(Base):
transaction_type = db.Column(db.String(255), nullable=False)
external_id = db.Column(db.String(255), nullable=False)
external_date = db.Column(db.DateTime, nullable=True)
+ merchant = db.Column(db.String(255), nullable=True)
+ original_statement = db.Column(db.String(255), nullable=True)
+ notes = db.Column(db.Text, nullable=True)
+ tags = db.Column(db.String(255), nullable=True)
description = db.Column(db.String(255), nullable=True)
- def __init__(self, user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, description):
+ def __init__(self, user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, merchant=None, original_statement=None, notes=None, tags=None, description=None):
"""
Initialize a TransactionModel instance.
@@ -39,6 +43,10 @@ def __init__(self, user_id, categories_id, account_id, amount, transaction_type,
transaction_type (str): The transaction_type of the transaction.
external_id (str): The external ID of the transaction.
external_date (datetime): The external date of the transaction.
+ merchant (str): The merchant name for the transaction.
+ original_statement (str): The original statement text from the bank.
+ notes (str): User notes for the transaction.
+ tags (str): Tags for categorizing the transaction.
description (str): The description of the transaction.
"""
self.user_id = user_id
@@ -48,6 +56,10 @@ def __init__(self, user_id, categories_id, account_id, amount, transaction_type,
self.transaction_type = transaction_type
self.external_id = external_id
self.external_date = external_date
+ self.merchant = merchant
+ self.original_statement = original_statement
+ self.notes = notes
+ self.tags = tags
self.description = description
def __repr__(self):
@@ -77,6 +89,10 @@ def to_dict(self):
'transaction_type': self.transaction_type,
'external_id': self.external_id,
'external_date': self.external_date,
+ 'merchant': self.merchant,
+ 'original_statement': self.original_statement,
+ 'notes': self.notes,
+ 'tags': self.tags,
'description': self.description,
'created_at': self.created_at,
'updated_at': self.updated_at
diff --git a/app/static/js/transactions/import.js b/app/static/js/transactions/import.js
index 0e5d30d..76e840a 100644
--- a/app/static/js/transactions/import.js
+++ b/app/static/js/transactions/import.js
@@ -117,13 +117,30 @@ uploadBtn.addEventListener('click', async () => {
function showSuccess(data) {
let errorDetailsHtml = '';
if (data.error_details && data.error_details.length > 0) {
+ let errorSummaryHtml = '';
+ if (data.error_summary) {
+ errorSummaryHtml = `
+
+
Error Summary:
+
+ ${Object.entries(data.error_summary).map(([type, count]) =>
+ `- ${type.replace(/_/g, ' ')}: ${count}
`
+ ).join('')}
+
+
+ `;
+ }
+
errorDetailsHtml = `
-
Import Errors (showing first 10):
-
- ${data.error_details.map(err => `- ${err}
`).join('')}
-
+
Import Errors (showing first 50):
+ ${errorSummaryHtml}
+
+
+ ${data.error_details.map(err => `- ${err}
`).join('')}
+
+
`;
}
diff --git a/app/static/js/transactions/transactions.js b/app/static/js/transactions/transactions.js
index 88e5337..c438662 100644
--- a/app/static/js/transactions/transactions.js
+++ b/app/static/js/transactions/transactions.js
@@ -88,6 +88,10 @@ async function editTransaction(transactionId) {
if (response.ok) {
// Populate the edit form
document.getElementById('edit_transaction_id').value = transactionId;
+ document.getElementById('edit_transactionMerchant').value = data.transaction.merchant || '';
+ document.getElementById('edit_transactionOriginalStatement').value = data.transaction.original_statement || '';
+ document.getElementById('edit_transactionNotes').value = data.transaction.notes || '';
+ document.getElementById('edit_transactionTags').value = data.transaction.tags || '';
document.getElementById('edit_transactionDescription').value = data.transaction.description || '';
document.getElementById('edit_transactionAmount').value = data.transaction.amount || '';
document.getElementById('edit_transactionCategory').value = data.transaction.categories_id || '';
@@ -109,6 +113,10 @@ function updateTransaction(event) {
event.preventDefault();
const transactionId = document.getElementById('edit_transaction_id').value;
+ const merchant = document.getElementById('edit_transactionMerchant').value;
+ const originalStatement = document.getElementById('edit_transactionOriginalStatement').value;
+ const notes = document.getElementById('edit_transactionNotes').value;
+ const tags = document.getElementById('edit_transactionTags').value;
const description = document.getElementById('edit_transactionDescription').value;
const amount = document.getElementById('edit_transactionAmount').value;
const categoryId = document.getElementById('edit_transactionCategory').value;
@@ -116,6 +124,10 @@ function updateTransaction(event) {
const user_id = document.getElementById('edit_transaction_user_id').value;
const data = {
+ merchant: merchant,
+ original_statement: originalStatement,
+ notes: notes,
+ tags: tags,
description: description,
amount: parseFloat(amount),
categories_id: categoryId,
diff --git a/app/templates/transactions/index.html b/app/templates/transactions/index.html
index d7e8115..acaea6e 100644
--- a/app/templates/transactions/index.html
+++ b/app/templates/transactions/index.html
@@ -107,6 +107,7 @@ Add
| Date |
+ Merchant |
Description |
Category |
Amount |
@@ -127,9 +128,10 @@ Add
| {{ transaction.external_date|format_datetime('short') }} |
+ {{ transaction.merchant }} |
{{ transaction.description }} |
- {{ transaction.amount }} |
{{ transaction.categories.name }} |
+ {{ transaction.amount }} |
{{ transaction.account.name }} |
{{ transaction.account.institution.name }} |
@@ -162,10 +164,34 @@ Edit Transaction
+
+
+
+
+
+
+
+
+
diff --git a/scripts/add_merchant_column.py b/scripts/add_merchant_column.py
new file mode 100644
index 0000000..4889887
--- /dev/null
+++ b/scripts/add_merchant_column.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""
+Migration script to add merchant column to transaction table
+"""
+import os
+import sys
+
+# Change to project root directory (parent of scripts directory)
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+os.chdir(project_root)
+sys.path.insert(0, project_root)
+
+from app import create_app, db
+from sqlalchemy import text
+
+app = create_app()
+
+with app.app_context():
+ try:
+ # Add the merchant column if it doesn't exist
+ db.session.execute(text("""
+ ALTER TABLE transaction
+ ADD COLUMN IF NOT EXISTS merchant VARCHAR(255);
+ """))
+ db.session.commit()
+ print("✓ Successfully added merchant column to transaction table")
+ except Exception as e:
+ print(f"✗ Error adding merchant column: {e}")
+ db.session.rollback()
diff --git a/scripts/add_transaction_fields.py b/scripts/add_transaction_fields.py
new file mode 100644
index 0000000..b3b2f97
--- /dev/null
+++ b/scripts/add_transaction_fields.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""
+Migration script to add original_statement, notes, and tags columns to transaction table
+"""
+import os
+import sys
+
+# Change to project root directory (parent of scripts directory)
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+os.chdir(project_root)
+sys.path.insert(0, project_root)
+
+from app import create_app, db
+from sqlalchemy import text
+
+app = create_app()
+
+with app.app_context():
+ try:
+ # Add the new columns if they don't exist
+ db.session.execute(text("""
+ ALTER TABLE transaction
+ ADD COLUMN IF NOT EXISTS original_statement VARCHAR(255);
+ """))
+ db.session.execute(text("""
+ ALTER TABLE transaction
+ ADD COLUMN IF NOT EXISTS notes TEXT;
+ """))
+ db.session.execute(text("""
+ ALTER TABLE transaction
+ ADD COLUMN IF NOT EXISTS tags VARCHAR(255);
+ """))
+ db.session.commit()
+ print("✓ Successfully added original_statement, notes, and tags columns to transaction table")
+ except Exception as e:
+ print(f"✗ Error adding columns: {e}")
+ db.session.rollback()
diff --git a/scripts/clear_tables.py b/scripts/clear_tables.py
new file mode 100644
index 0000000..5131601
--- /dev/null
+++ b/scripts/clear_tables.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""
+Script to clear all tables except user table
+"""
+import os
+import sys
+
+# Change to project root directory (parent of scripts directory)
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+os.chdir(project_root)
+sys.path.insert(0, project_root)
+
+from app import create_app, db
+from sqlalchemy import text
+
+app = create_app()
+
+with app.app_context():
+ try:
+ # Clear tables in correct order (respecting foreign key constraints)
+ print("Clearing tables...")
+
+ # Delete transactions first (has foreign keys to categories and accounts)
+ result = db.session.execute(text("DELETE FROM transaction"))
+ print(f"✓ Cleared transaction table ({result.rowcount} rows)")
+
+ # Delete accounts (has foreign key to institution)
+ result = db.session.execute(text("DELETE FROM account"))
+ print(f"✓ Cleared account table ({result.rowcount} rows)")
+
+ # Delete institutions
+ result = db.session.execute(text("DELETE FROM institution"))
+ print(f"✓ Cleared institution table ({result.rowcount} rows)")
+
+ # Delete categories (has foreign keys to categories_group and categories_type)
+ result = db.session.execute(text("DELETE FROM categories"))
+ print(f"✓ Cleared categories table ({result.rowcount} rows)")
+
+ # Delete categories_group (has foreign key to categories_type)
+ result = db.session.execute(text("DELETE FROM categories_group"))
+ print(f"✓ Cleared categories_group table ({result.rowcount} rows)")
+
+ # Delete categories_type
+ result = db.session.execute(text("DELETE FROM categories_type"))
+ print(f"✓ Cleared categories_type table ({result.rowcount} rows)")
+
+ db.session.commit()
+ print("\n✓ Successfully cleared all tables (except user table)")
+ print("\nYou can now start fresh with importing categories and transactions!")
+
+ except Exception as e:
+ print(f"✗ Error clearing tables: {e}")
+ db.session.rollback()
diff --git a/scripts/clear_transactions.py b/scripts/clear_transactions.py
new file mode 100644
index 0000000..67b4a31
--- /dev/null
+++ b/scripts/clear_transactions.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+"""
+Script to clear only the transactions table
+"""
+import os
+import sys
+
+# Change to project root directory (parent of scripts directory)
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+os.chdir(project_root)
+sys.path.insert(0, project_root)
+
+from app import create_app, db
+from sqlalchemy import text
+
+app = create_app()
+
+with app.app_context():
+ try:
+ # Delete all transactions
+ result = db.session.execute(text("DELETE FROM transaction"))
+ db.session.commit()
+ print(f"✓ Cleared transaction table ({result.rowcount} rows)")
+ print("\nTransactions table is now empty. Ready for fresh import!")
+
+ except Exception as e:
+ print(f"✗ Error clearing transactions: {e}")
+ db.session.rollback()
diff --git a/scripts/convert_monarch_categories.py b/scripts/convert_monarch_categories.py
new file mode 100644
index 0000000..d0bc014
--- /dev/null
+++ b/scripts/convert_monarch_categories.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""
+Convert Monarch categories JSON to CSV format for OSPF import
+"""
+import json
+import csv
+import os
+import sys
+
+# Change to project root directory (parent of scripts directory)
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+os.chdir(project_root)
+
+# Read the JSON file
+with open('data/monarch-categories.json', 'r') as f:
+ data = json.load(f)
+
+# Extract categories
+categories = data['data']['categories']
+
+# Create a set to track unique combinations (to avoid duplicates)
+seen = set()
+rows = []
+
+for category in categories:
+ # Skip disabled categories
+ if category.get('isDisabled', False):
+ continue
+
+ category_name = category['name']
+ group_name = category['group']['name']
+ group_type = category['group']['type']
+
+ # Capitalize first letter of type for consistency with existing data
+ if group_type == 'income':
+ type_name = 'Income'
+ elif group_type == 'expense':
+ type_name = 'Expense'
+ elif group_type == 'transfer':
+ type_name = 'Transfer'
+ else:
+ type_name = group_type.capitalize()
+
+ # Create unique key
+ key = (category_name, group_name, type_name)
+
+ # Only add if not seen before
+ if key not in seen:
+ seen.add(key)
+ rows.append({
+ 'categories': category_name,
+ 'categories_group': group_name,
+ 'categories_type': type_name
+ })
+
+# Sort by type, then group, then category
+rows.sort(key=lambda x: (x['categories_type'], x['categories_group'], x['categories']))
+
+# Write to CSV
+with open('data/monarch-categories-data.csv', 'w', newline='', encoding='utf-8') as f:
+ writer = csv.DictWriter(f, fieldnames=['categories', 'categories_group', 'categories_type'])
+ writer.writeheader()
+ writer.writerows(rows)
+
+print(f"✓ Successfully created monarch-categories-data.csv with {len(rows)} categories")
+print(f"\nBreakdown:")
+print(f" - Income categories: {sum(1 for r in rows if r['categories_type'] == 'Income')}")
+print(f" - Expense categories: {sum(1 for r in rows if r['categories_type'] == 'Expense')}")
+print(f" - Transfer categories: {sum(1 for r in rows if r['categories_type'] == 'Transfer')}")
|