Separate transaction fields into individual database columns

This update separates the transaction description into distinct fields for better data integrity and querying:
- Added merchant, original_statement, notes, and tags as separate database columns
- Updated CSV import to store fields individually instead of concatenating them
- Enhanced frontend edit modal to support all transaction fields
- Improved error reporting in CSV import to show first 50 errors with categorized summary
- Updated all utility scripts to work from scripts/ directory with proper path handling
- Added database migration scripts for new columns

This change enables better filtering, searching, and reporting on transaction data by maintaining field separation throughout the system.

Author: caseybecking

api/transaction/controllers.py

@@ -22,6 +22,10 @@
     'transaction_type': fields.String(required=True, description='Transaction Type'),
     'external_id': fields.String(description='External ID'),
     'external_date': fields.DateTime(description='External Date'),
+    'merchant': fields.String(description='Merchant Name'),
+    'original_statement': fields.String(description='Original Statement'),
+    'notes': fields.String(description='Notes'),
+    'tags': fields.String(description='Tags'),
     'description': fields.String(description='Description')
 })
 
@@ -37,6 +41,10 @@ def post(self):
         transaction_type = data.get('transaction_type')
         external_id = data.get('external_id')
         external_date = data.get('external_date')
+        merchant = data.get('merchant')
+        original_statement = data.get('original_statement')
+        notes = data.get('notes')
+        tags = data.get('tags')
         description = data.get('description')
 
         # Validate required fields
@@ -61,6 +69,10 @@ def post(self):
             transaction_type=transaction_type,
             external_id=external_id,
             external_date=external_date,
+            merchant=merchant,
+            original_statement=original_statement,
+            notes=notes,
+            tags=tags,
             description=description
         )
         new_transaction.save()
@@ -210,20 +222,11 @@ def post(self):
                             error_count += 1
                             continue
 
-                        # Build description from available fields
-                        description_parts = []
-                        if merchant:
-                            description_parts.append(f"Merchant: {merchant}")
-                        if original_statement:
-                            description_parts.append(f"Statement: {original_statement}")
-                        if notes:
-                            description_parts.append(f"Notes: {notes}")
-                        if tags:
-                            description_parts.append(f"Tags: {tags}")
+                        # Store fields separately (no joining)
+                        # Description can be used for additional custom info if needed
+                        description = None  # Keep empty unless specifically provided
 
-                        description = " | ".join(description_parts) if description_parts else merchant
-
-                        # Create transaction
+                        # Create transaction with all fields separate
                         self.create_transaction(
                             user_id=user_id,
                             categories_id=category_id,
@@ -232,6 +235,10 @@ def post(self):
                             transaction_type=_transaction_type,
                             external_id=external_id,
                             external_date=formatted_timestamp,
+                            merchant=merchant,
+                            original_statement=original_statement,
+                            notes=notes,
+                            tags=tags,
                             description=description
                         )
                         created_count += 1
@@ -251,8 +258,24 @@ def post(self):
                 'errors': error_count
             }
 
-            if errors and len(errors) <= 10:  # Only include first 10 errors
-                response_data['error_details'] = errors[:10]
+            if errors:
+                # Show first 50 errors to understand patterns
+                response_data['error_details'] = errors[:50]
+                # Also include a summary of error types
+                error_summary = {}
+                for error in errors:
+                    # Extract error type
+                    if "Category" in error and "not found" in error:
+                        error_summary['category_not_found'] = error_summary.get('category_not_found', 0) + 1
+                    elif "Could not create account" in error:
+                        error_summary['account_creation_failed'] = error_summary.get('account_creation_failed', 0) + 1
+                    elif "Could not parse" in error:
+                        error_summary['date_parse_error'] = error_summary.get('date_parse_error', 0) + 1
+                    elif "Invalid amount" in error:
+                        error_summary['invalid_amount'] = error_summary.get('invalid_amount', 0) + 1
+                    else:
+                        error_summary['other'] = error_summary.get('other', 0) + 1
+                response_data['error_summary'] = error_summary
 
             return make_response(jsonify(response_data), 201 if created_count > 0 else 200)
 
@@ -347,7 +370,7 @@ def ensure_account_exists_smart(self, account_name, merchant_name):
 
         return account.id
 
-    def create_transaction(self,user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, description):
+    def create_transaction(self,user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, merchant=None, original_statement=None, notes=None, tags=None, description=None):
         print(f"Creating transaction {external_id}...")
         transaction = TransactionModel(
             user_id=user_id,
@@ -357,6 +380,10 @@ def create_transaction(self,user_id, categories_id, account_id, amount, transact
             transaction_type=transaction_type,
             external_id=external_id,
             external_date=external_date,
+            merchant=merchant,
+            original_statement=original_statement,
+            notes=notes,
+            tags=tags,
             description=description
         )
         db.session.add(transaction)
@@ -408,6 +435,14 @@ def put(self, id):
                 }), 400)
 
         # Update fields if provided
+        if 'merchant' in data:
+            transaction.merchant = data['merchant']
+        if 'original_statement' in data:
+            transaction.original_statement = data['original_statement']
+        if 'notes' in data:
+            transaction.notes = data['notes']
+        if 'tags' in data:
+            transaction.tags = data['tags']
         if 'description' in data:
             transaction.description = data['description']
         if 'amount' in data:

api/transaction/models.py

@@ -25,9 +25,13 @@ class TransactionModel(Base):
     transaction_type = db.Column(db.String(255), nullable=False)
     external_id = db.Column(db.String(255), nullable=False)
     external_date = db.Column(db.DateTime, nullable=True)
+    merchant = db.Column(db.String(255), nullable=True)
+    original_statement = db.Column(db.String(255), nullable=True)
+    notes = db.Column(db.Text, nullable=True)
+    tags = db.Column(db.String(255), nullable=True)
     description = db.Column(db.String(255), nullable=True)
 
-    def __init__(self, user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, description):
+    def __init__(self, user_id, categories_id, account_id, amount, transaction_type, external_id, external_date, merchant=None, original_statement=None, notes=None, tags=None, description=None):
         """
         Initialize a TransactionModel instance.
 
@@ -39,6 +43,10 @@ def __init__(self, user_id, categories_id, account_id, amount, transaction_type,
             transaction_type (str): The transaction_type of the transaction.
             external_id (str): The external ID of the transaction.
             external_date (datetime): The external date of the transaction.
+            merchant (str): The merchant name for the transaction.
+            original_statement (str): The original statement text from the bank.
+            notes (str): User notes for the transaction.
+            tags (str): Tags for categorizing the transaction.
             description (str): The description of the transaction.
         """
         self.user_id = user_id
@@ -48,6 +56,10 @@ def __init__(self, user_id, categories_id, account_id, amount, transaction_type,
         self.transaction_type = transaction_type
         self.external_id = external_id
         self.external_date = external_date
+        self.merchant = merchant
+        self.original_statement = original_statement
+        self.notes = notes
+        self.tags = tags
         self.description = description
 
     def __repr__(self):
@@ -77,6 +89,10 @@ def to_dict(self):
             'transaction_type': self.transaction_type,
             'external_id': self.external_id,
             'external_date': self.external_date,
+            'merchant': self.merchant,
+            'original_statement': self.original_statement,
+            'notes': self.notes,
+            'tags': self.tags,
             'description': self.description,
             'created_at': self.created_at,
             'updated_at': self.updated_at

app/static/js/transactions/import.js

@@ -117,13 +117,30 @@ uploadBtn.addEventListener('click', async () => {
 function showSuccess(data) {
     let errorDetailsHtml = '';
     if (data.error_details && data.error_details.length > 0) {
+        let errorSummaryHtml = '';
+        if (data.error_summary) {
+            errorSummaryHtml = `
+                <div class="mb-3">
+                    <strong>Error Summary:</strong>
+                    <ul>
+                        ${Object.entries(data.error_summary).map(([type, count]) =>
+                            `<li>${type.replace(/_/g, ' ')}: ${count}</li>`
+                        ).join('')}
+                    </ul>
+                </div>
+            `;
+        }
+
         errorDetailsHtml = `
             <hr>
             <div class="alert alert-warning">
-                <h6 class="alert-heading">Import Errors (showing first 10):</h6>
-                <ul class="mb-0">
-                    ${data.error_details.map(err => `<li>${err}</li>`).join('')}
-                </ul>
+                <h6 class="alert-heading">Import Errors (showing first 50):</h6>
+                ${errorSummaryHtml}
+                <div style="max-height: 300px; overflow-y: auto;">
+                    <ul class="mb-0">
+                        ${data.error_details.map(err => `<li><small>${err}</small></li>`).join('')}
+                    </ul>
+                </div>
             </div>
         `;
     }

app/static/js/transactions/transactions.js

@@ -88,6 +88,10 @@ async function editTransaction(transactionId) {
         if (response.ok) {
             // Populate the edit form
             document.getElementById('edit_transaction_id').value = transactionId;
+            document.getElementById('edit_transactionMerchant').value = data.transaction.merchant || '';
+            document.getElementById('edit_transactionOriginalStatement').value = data.transaction.original_statement || '';
+            document.getElementById('edit_transactionNotes').value = data.transaction.notes || '';
+            document.getElementById('edit_transactionTags').value = data.transaction.tags || '';
             document.getElementById('edit_transactionDescription').value = data.transaction.description || '';
             document.getElementById('edit_transactionAmount').value = data.transaction.amount || '';
             document.getElementById('edit_transactionCategory').value = data.transaction.categories_id || '';
@@ -109,13 +113,21 @@ function updateTransaction(event) {
     event.preventDefault();
 
     const transactionId = document.getElementById('edit_transaction_id').value;
+    const merchant = document.getElementById('edit_transactionMerchant').value;
+    const originalStatement = document.getElementById('edit_transactionOriginalStatement').value;
+    const notes = document.getElementById('edit_transactionNotes').value;
+    const tags = document.getElementById('edit_transactionTags').value;
     const description = document.getElementById('edit_transactionDescription').value;
     const amount = document.getElementById('edit_transactionAmount').value;
     const categoryId = document.getElementById('edit_transactionCategory').value;
     const accountId = document.getElementById('edit_transactionAccount').value;
     const user_id = document.getElementById('edit_transaction_user_id').value;
 
     const data = {
+        merchant: merchant,
+        original_statement: originalStatement,
+        notes: notes,
+        tags: tags,
         description: description,
         amount: parseFloat(amount),
         categories_id: categoryId,

app/templates/transactions/index.html

@@ -107,6 +107,7 @@ <h5 class="modal-title" id="TransactionsModalgridLabel">Add
                                                     </div>
                                                 </th>
                                                 <th scope="col">Date</th>
+                                                <th scope="col">Merchant</th>
                                                 <th scope="col">Description</th>
                                                 <th scope="col">Category</th>
                                                 <th scope="col">Amount</th>
@@ -127,9 +128,10 @@ <h5 class="modal-title" id="TransactionsModalgridLabel">Add
                                                     </div>
                                                 </td>
                                                 <td>{{ transaction.external_date|format_datetime('short') }}</td>
+                                                <td>{{ transaction.merchant }}</td>
                                                 <td>{{ transaction.description }}</td>
-                                                <td>{{ transaction.amount }}</td>
                                                 <td>{{ transaction.categories.name }}</td>
+                                                <td>{{ transaction.amount }}</td>
                                                 <td>{{ transaction.account.name }}</td>
                                                 <td>{{ transaction.account.institution.name }}</td>
                                                 <td>
@@ -162,10 +164,34 @@ <h5 class="modal-title" id="EditTransactionModalLabel">Edit Transaction</h5>
                                         <input type="hidden" id="edit_transaction_id">
                                         <input type="hidden" id="edit_transaction_user_id" value="{{ user_id }}">
                                         <div class="row g-3">
+                                            <div class="col-xxl-12">
+                                                <div>
+                                                    <label for="edit_transactionMerchant" class="form-label">Merchant</label>
+                                                    <input type="text" class="form-control" id="edit_transactionMerchant" placeholder="Enter Merchant Name">
+                                                </div>
+                                            </div>
+                                            <div class="col-xxl-12">
+                                                <div>
+                                                    <label for="edit_transactionOriginalStatement" class="form-label">Original Statement</label>
+                                                    <input type="text" class="form-control" id="edit_transactionOriginalStatement" placeholder="Original Bank Statement">
+                                                </div>
+                                            </div>
+                                            <div class="col-xxl-12">
+                                                <div>
+                                                    <label for="edit_transactionNotes" class="form-label">Notes</label>
+                                                    <textarea class="form-control" id="edit_transactionNotes" rows="2" placeholder="Enter Notes"></textarea>
+                                                </div>
+                                            </div>
+                                            <div class="col-xxl-12">
+                                                <div>
+                                                    <label for="edit_transactionTags" class="form-label">Tags</label>
+                                                    <input type="text" class="form-control" id="edit_transactionTags" placeholder="Enter Tags (comma separated)">
+                                                </div>
+                                            </div>
                                             <div class="col-xxl-12">
                                                 <div>
                                                     <label for="edit_transactionDescription" class="form-label">Description</label>
-                                                    <input type="text" class="form-control" id="edit_transactionDescription" placeholder="Enter Transaction Description">
+                                                    <input type="text" class="form-control" id="edit_transactionDescription" placeholder="Additional Description">
                                                 </div>
                                             </div>
                                             <div class="col-xxl-12">

scripts/add_merchant_column.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""
+Migration script to add merchant column to transaction table
+"""
+import os
+import sys
+
+# Change to project root directory (parent of scripts directory)
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+os.chdir(project_root)
+sys.path.insert(0, project_root)
+
+from app import create_app, db
+from sqlalchemy import text
+
+app = create_app()
+
+with app.app_context():
+    try:
+        # Add the merchant column if it doesn't exist
+        db.session.execute(text("""
+            ALTER TABLE transaction
+            ADD COLUMN IF NOT EXISTS merchant VARCHAR(255);
+        """))
+        db.session.commit()
+        print("✓ Successfully added merchant column to transaction table")
+    except Exception as e:
+        print(f"✗ Error adding merchant column: {e}")
+        db.session.rollback()

scripts/add_transaction_fields.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""
+Migration script to add original_statement, notes, and tags columns to transaction table
+"""
+import os
+import sys
+
+# Change to project root directory (parent of scripts directory)
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+os.chdir(project_root)
+sys.path.insert(0, project_root)
+
+from app import create_app, db
+from sqlalchemy import text
+
+app = create_app()
+
+with app.app_context():
+    try:
+        # Add the new columns if they don't exist
+        db.session.execute(text("""
+            ALTER TABLE transaction
+            ADD COLUMN IF NOT EXISTS original_statement VARCHAR(255);
+        """))
+        db.session.execute(text("""
+            ALTER TABLE transaction
+            ADD COLUMN IF NOT EXISTS notes TEXT;
+        """))
+        db.session.execute(text("""
+            ALTER TABLE transaction
+            ADD COLUMN IF NOT EXISTS tags VARCHAR(255);
+        """))
+        db.session.commit()
+        print("✓ Successfully added original_statement, notes, and tags columns to transaction table")
+    except Exception as e:
+        print(f"✗ Error adding columns: {e}")
+        db.session.rollback()