diff --git a/app/routes.py b/app/routes.py index 81ccae9..e501754 100644 --- a/app/routes.py +++ b/app/routes.py @@ -3,9 +3,9 @@ import os from datetime import datetime +import secrets from flask import request, current_app, render_template, redirect, \ make_response, send_from_directory, session -import secrets class Routes: @@ -383,12 +383,12 @@ def uploadIban(iban): Returns: json: Informationen zur Datei und Ergebnis der Untersuchung. """ - input_file = request.files.get('file-input') + input_file = request.files.get('file-batch') if not input_file: return {'error': 'Es wurde keine Datei übermittelt.'}, 400 # Store Upload file to tmp - path = '/tmp/transactions.tmp' + path = f"/tmp/{secrets.token_hex(12)}" content_type, size = parent.mv_fileupload(input_file, path) # Daten einlesen und in Object speichern (Bank und Format default bzw. wird geraten) @@ -416,7 +416,7 @@ def uploadIban(iban): insert_result = parent.db_handler.insert(parsed_data, iban) inserted = insert_result.get('inserted') - except (KeyError, ValueError, NotImplementedError) as ex: + except (KeyError, ValueError) as ex: return { "error": ( "Die hochgeladene Datei konnte nicht verarbeitet werden, " diff --git a/app/static/css/style.css b/app/static/css/style.css index 5881f53..c1b2287 100644 --- a/app/static/css/style.css +++ b/app/static/css/style.css @@ -104,10 +104,23 @@ button.info { } /* Tables */ +.transactions th { + cursor: pointer; +} .transactions .amount { white-space: nowrap; text-align: right; } + .transactions th.amount { + text-align: center; + } + .transactions th.sorted-asc::after, .transactions th.sorted-desc::after { + position: absolute; + margin-left: 4px; + color: var(--pico-primary); + } + .transactions th.sorted-asc::after { content: "\2193"; } + .transactions th.sorted-desc:after { content: "\2191"; } #dynamic-results tr th { font-weight: bold; @@ -115,9 +128,7 @@ button.info { /* S : Slim button in transaction table */ @media (max-width: 767px) { - .transactions th.amount { - text-align: center; - } + .transactions tr { border-bottom: var(--pico-border-width) solid var(--pico-table-border-color); } diff --git a/app/static/js/functions.js b/app/static/js/functions.js index 72265ea..a2ae3d3 100644 --- a/app/static/js/functions.js +++ b/app/static/js/functions.js @@ -166,6 +166,156 @@ function removeTagBullet(element, hiddenInputId) { } + +/** + * Show a popup with a result message from an operation + * + * @param {string} heading Heading for the PopUp + * @param {list} domlist List of DOMELements to add in the "content" section + */ +function responsePopUp(heading, domlist) { + const popup = document.getElementById('response-popup'); + popup.querySelector('header h2').textContent = heading; + + const content = document.getElementById('response-content'); + content.innerHTML = ''; + domlist.forEach(dom => { + content.appendChild(dom); + }); + + // Close actual Popup if open and open Response PopUp + const openPopUp = document.querySelector('dialog[open] header button') + if (openPopUp) { + openPopUp.click(); + } + openModal(popup, { 'currentTarget': { 'dataset': {} } }); +} + +/** Show a popup with an error message from an operation + * + * @param {string} heading Heading for the PopUp + * @param {number} error Error code + * @param {string} responseText Response text from the operation + */ +function errorPopUp(heading, error, responseText) { + const reason1 = document.createElement('p'); + reason1.innerHTML = "Fehler " + error; + reason1.className = 'error'; + const reason2 = document.createElement('pre'); + reason2.innerHTML = responseText; + responsePopUp(heading, [reason1, reason2]); +} + +/** + * Sort Table Rows + * + * @param {string} table_id The ID of the tabel to sort rows in + * + * @param {number} col_index The index of the column to compare rows content with + */ +function sortTable(table_id, col_index){ + const table = document.getElementById(table_id); + if (!table){ + console.error("Table '", table_id, "' to sort was not found"); + } + if (typeof col_index == 'undefined'){ + col_index = 0; + } + + var switching = true; + var switchcount = 0; + var dir = "asc"; + + while (switching) { + // Loop until nothing else was switched + switching = false; + var rows = table.rows; + + /* + Loop through the actual order of rows and + compare if one pair should be switched. + Break if a pair is found and switch it later. + This loop will be started again after the switch. + */ + for (var i = 1; i < (rows.length - 1); i++) { + var should_switch = false; + var x = rows[i].getElementsByTagName('td')[col_index]; + var y = rows[i + 1].getElementsByTagName('td')[col_index]; + + // Compare values with special handling for numbers + if (col_index == 0) { + // Compare timestamp from dataset + var x_val = Number(x.getElementsByTagName('input')[0].dataset.txdate); + var y_val = Number(y.getElementsByTagName('input')[0].dataset.txdate); + + } else if (col_index == 5){ + // Compare amount (numbers) + var x_val = Number(x.innerText.slice(0, -4)); + var y_val = Number(y.innerText.slice(0, -4)); + + } else { + // Compare normal innerText + var x_val = x.innerText.toLowerCase(); + var y_val = y.innerText.toLowerCase(); + } + + // Compare (depending on direction) + if (dir == "asc"){ + + if (x_val > y_val) { + // Switch needed: Brake! + should_switch = true; + break; + } + + } else if (dir == "desc") { + + if (x_val < y_val) { + // Switch needed: Brake! + should_switch = true; + break; + } + + } + } + + // Was a needed switch found in the loop? + if (should_switch) { + // Yes: Switch it now + rows[i].parentNode.insertBefore(rows[i + 1], rows[i]); + + // Remember that we at least switched one time + switchcount ++; + + // Reset switching-var to get into the next iteration of the while loop + switching = true; + + } else { + // There is no need for a switch... + if (switchcount == 0 && dir == "asc") { + // ...and there weren't any switches before: + // Change the direction of the sorting if it is still default 'asc' + dir = "desc"; + + // Reset switching-var to get into the next iteration of the while loop + switching = true; + } + } + + } + + const table_headings = rows[0].getElementsByTagName('th'); + for (var i=0; i < table_headings.length; i++){ + table_headings[i].classList.remove('sorted-asc'); + table_headings[i].classList.remove('sorted-desc'); + } + if (col_index == 0){ + table_headings[1].classList.add('sorted-' + dir); + } else { + table_headings[col_index].classList.add('sorted-' + dir); + } +} + // ---------------------------------------------------------------------------- // -- AJAX Functions ---------------------------------------------------------- // ---------------------------------------------------------------------------- diff --git a/app/static/js/iban.js b/app/static/js/iban.js index 89c0f21..b8ac3db 100644 --- a/app/static/js/iban.js +++ b/app/static/js/iban.js @@ -228,26 +228,41 @@ function tagAndCat(operation) { } apiSubmit(api_url + IBAN, payload, function (responseText, error) { + const heading = operation == 'tag' ? 'Tagging' : 'Kategorisierung'; if (error) { - alert(operation + ' failed: ' + '(' + error + ')' + responseText); + errorPopUp( + operation.substring(1) + operation.substring(0, 1).toUpperCase() + ' fehlgeschlagen', + error, responseText + ); } else { - alert(operation + ' successful!' + responseText); - if (dry_run) { - // List UUIDs which would have been tagged/cat - const r = JSON.parse(responseText) - let txt_list = "" - r.entries.forEach(element => { - txt_list += "\n- " + element - }); - alert("Folgende UUIDs würden geändert werden:\n" + txt_list); + const reason1 = document.createElement('p'); + reason1.innerHTML = dry_run ? "Folgende Transaktionen wären geändert worden:" : "Folgende Transaktionen wurden geändert:"; - } else { - // Tagged/Cat -> Reload - window.location.reload(); + const reason2 = document.createElement('ul'); + let r = JSON.parse(responseText) + + if (r.entries.length == 0) { + const li = document.createElement('li'); + li.innerHTML = '(keine)'; + reason2.appendChild(li); } + r.entries.forEach(element => { + let li = document.createElement('li'); + let a = document.createElement('a'); + a.href = '/' + IBAN + '/' + element; + a.target = '_blank'; + a.innerHTML = element; + li.appendChild(a); + reason2.appendChild(li); + }); + + responsePopUp( + heading + ' erfolgreich', + [reason1, reason2]); } + }, false); } @@ -270,11 +285,12 @@ function removeTags() { apiSubmit(api_function, tags, function (responseText, error) { if (error) { - alert('Tag removal failed: ' + '(' + error + ')' + responseText); + errorPopUp('Tag entfernen fehlgeschlagen', error, responseText); } else { - alert('Entries tags deleted successfully!' + responseText); - window.location.reload(); + const p = document.createElement('p'); + p.innerHTML = 'Die Tags der ausgewählten Einträge wurden erfolgreich entfernt.'; + responsePopUp('Tags entfernt', [p]); } }, false); @@ -300,11 +316,12 @@ function removeCats() { apiSubmit(api_function, payload, function (responseText, error) { if (error) { - alert('Cat removal failed: ' + '(' + error + ')' + responseText); + errorPopUp('Kategorie entfernen fehlgeschlagen', error, responseText); } else { - alert('Entries category deleted successfully!' + responseText); - window.location.reload(); + const p = document.createElement('p'); + p.innerHTML = 'Die Kategorie der ausgewählten Einträge wurden erfolgreich entfernt.'; + responsePopUp('Kategorie entfernt', [p]); } }, false); @@ -344,7 +361,7 @@ function addCat() { function getInfo(uuid, callback = alert) { apiGet('/' + IBAN + '/' + uuid, {}, function (responseText, error) { if (error) { - alert('getTx failed: ' + '(' + error + ')' + responseText); + errorPopUp('Transaktionsabruf fehlgeschlagen', error, responseText); } else { callback(responseText); diff --git a/app/static/js/index.js b/app/static/js/index.js index c3b7f34..471e1b1 100644 --- a/app/static/js/index.js +++ b/app/static/js/index.js @@ -13,7 +13,7 @@ document.addEventListener('DOMContentLoaded', function () { fileInput.addEventListener('change', () => { if (fileInput.files.length > 0) { - fileLabel.textContent = fileInput.files[0].name; + fileLabel.textContent = fileInput.files.length + " Datei(en) ausgewählt"; } else { fileLabel.textContent = '.csv,.pdf,.html'; } @@ -28,7 +28,7 @@ document.addEventListener('DOMContentLoaded', function () { const files = e.dataTransfer.files; if (files.length > 0) { fileInput.files = files; - fileLabel.textContent = files[0].name; + fileLabel.textContent = files.length + " Datei(en) ausgewählt"; } }); @@ -231,10 +231,12 @@ function importSettings() { } /** - * Sends a file to the server for upload. - * The file is selected via the file input element 'file-input'. + * Sends transactions in a file or a batch of files to the server for upload. + * The file is selected via the file input element 'file-input' (multiple) + * but every entry is send step-by-step to get results per call directly. + * Therefore this methods differ from the global `apiSubmit()` function. */ -function uploadFile() { +function uploadIban() { const iban = document.getElementById('iban-input').value; if (!iban) { alert("Keine IBAN angegeben!"); @@ -242,30 +244,80 @@ function uploadFile() { } const bank_id = document.getElementById('bank-type').value - const fileInput = document.getElementById('file-input'); if (fileInput.files.length === 0) { alert('Es wurde keine Datei ausgewählt.'); return; } - const params = { file: 'file-input', 'bank': bank_id}; // The value of 'file' corresponds to the input element's ID - apiSubmit('upload/' + iban, params, function (responseText, error) { - if (error) { - showAjaxError(error, responseText); + const upload_modal = document.getElementById('upload-list'); + + // Prepare List entry for clone in loop + const open_btn = document.querySelector('#upload-list footer a'); + open_btn.setAttribute('disabled', 'true'); + const list_table = document.querySelector('#upload-list table'); + list_table.innerHTML = ""; + const list_tr = document.createElement('tr'); + const cell1 = document.createElement('td'); + const cell2 = document.createElement('td'); + const span = document.createElement('span'); + span.setAttribute('aria-busy', "true"); + cell2.appendChild(span); + list_tr.appendChild(cell1); + list_tr.appendChild(cell2); + + //TODO: May need to create Promises per Loop + for (let i = 0; i < fileInput.files.length; i++) { + // DOM + const tr = list_tr.cloneNode(true); + const td2 = tr.querySelector('td:last-child'); + const td1 = tr.querySelector('td:first-child'); + let file_name = fileInput.files[i].name.slice(-30); + if (fileInput.files[i].name.length > 30) { + file_name = '...' + file_name; + } + td1.innerHTML = file_name + '
 '; + list_table.appendChild(tr); + + // Form + const fileFormData = new FormData(); + fileFormData.append('bank', bank_id) + fileFormData.append('file-batch', fileInput.files[i]); + + const ajax = createAjax(function (responseText, error) { + + // Update List of uploads with results per File + let result = JSON.parse(responseText); + if (error) { + console.warn(fileInput.files[i].name, error, responseText); + td2.setAttribute('aria-busy', 'false'); + td2.classList.add('error'); + td2.innerHTML = '×'; + result = result.error; - } else { - let success_msg = JSON.parse(responseText); - success_msg = 'Es wurden ' + success_msg.inserted + ' Transaktionen aus der ' + - Math.round(success_msg.size / 1024 * 100) / 100 + - ' KB großen Datei importiert.\n\nMöchtest du das Konto jetzt aufrufen?' - if (confirm(success_msg)) { - window.location.href = '/' + iban; } else { - prepareAddModal('add-iban', null, iban); + td2.setAttribute('aria-busy', 'false'); + td2.innerHTML = '✔'; + result = result.inserted + ' Transaktionen importiert'; + } - } - }, true); + td1.querySelector('small').innerHTML = result; + + }); + + // Show Upload Modal + document.querySelector('#add-iban header button').click(); + openModal(upload_modal, { 'currentTarget': { 'dataset': {} } }); + + // Send request(s) + ajax.open("POST", "/api/upload/" + iban, true); + ajax.send(fileFormData); + + } + + //TODO: Show an overall OK or confirm() for opening IBAN + upload_modal.querySelector('footer a').href = '/' + iban; + open_btn.removeAttribute('disabled'); } /** diff --git a/app/templates/iban.html b/app/templates/iban.html index fdac2fd..3ed06ea 100644 --- a/app/templates/iban.html +++ b/app/templates/iban.html @@ -34,15 +34,17 @@

- +
- - - - - + + + + + @@ -76,6 +78,20 @@

+ + + +
diff --git a/app/templates/index.html b/app/templates/index.html index b885618..3d37187 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -105,9 +105,9 @@

Konten verwalten

PDF / CSV / HTML - +
- +
Öffnen @@ -115,6 +115,22 @@

Konten verwalten

+ +
+
+ +

Import

+
+ +

Datum
(Valuta)
BuchungstextKategorieTagsBetrag + Datum
(Valuta) +
BuchungstextKategorieTagsBetrag  
+
+ + + + +
diff --git a/handler/TinyDb.py b/handler/TinyDb.py index 9917242..e92be84 100644 --- a/handler/TinyDb.py +++ b/handler/TinyDb.py @@ -6,12 +6,35 @@ import operator import logging import re -from tinydb import TinyDB, Query, where +from tinydb import TinyDB, Query, where, JSONStorage, middlewares from flask import current_app +import portalocker from handler.BaseDb import BaseDb +class FileLockMiddleware(middlewares.Middleware): + """ + Middleware Klasse für die TinyDB Instanz, die vor jeder Operation ihre Methoden ausführen kann + (siehe: https://tinydb.readthedocs.io/en/latest/_modules/tinydb/middlewares.html). + Sie wird hier für ein Datei-Locking benötigt, um parallele (Flask) Requests + auf die TindyDB zu ermöglichen. + """ + def __init__(self, storage_cls): + super().__init__(storage_cls) + self.lock_file = os.path.join(current_app.config['DATABASE_URI'], 'db.lock') + + def read(self): + """Hook into the database read operation with file locking.""" + with portalocker.Lock(self.lock_file, timeout=5): + return self.storage.read() + + def write(self, data): + """Hook into the database write operation with file locking.""" + with portalocker.Lock(self.lock_file, timeout=5): + self.storage.write(data) + + class TinyDbHandler(BaseDb): """ Handler für die Interaktion mit einer TinyDB Datenbank. @@ -23,9 +46,11 @@ def __init__(self): logging.info("Starting TinyDB Handler...") try: self.connection = TinyDB(os.path.join( - current_app.config['DATABASE_URI'], - current_app.config['DATABASE_NAME'] - )) + current_app.config['DATABASE_URI'], + current_app.config['DATABASE_NAME'] + ), + storage=FileLockMiddleware(JSONStorage) + ) if not hasattr(self, 'connection'): raise IOError('Es konnte kein Connection Objekt erstellt werden') diff --git a/reader/Comdirect.py b/reader/Comdirect.py index 26ab0f6..9d4472e 100644 --- a/reader/Comdirect.py +++ b/reader/Comdirect.py @@ -119,7 +119,6 @@ def from_pdf(self, filepath): for i, row in enumerated_table: if re_datecheck.match(row[0]) is None: - print("skip row", row) continue # Skip Header and unvalid Rows amount = float(row[4].replace('.', '').replace(',', '.')) diff --git a/requirements.txt b/requirements.txt index 745ceba..ff192c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ tinydb==4.7.1 pymongo==4.3.3 flask==3.1.0 natsort==8.4.0 -camelot-py[base]==1.0.9 \ No newline at end of file +camelot-py[base]==1.0.9 +portalocker==3.2.0 \ No newline at end of file diff --git a/tests/test_integ_basics.py b/tests/test_integ_basics.py index 03b19f3..0d913c0 100644 --- a/tests/test_integ_basics.py +++ b/tests/test_integ_basics.py @@ -53,7 +53,7 @@ def test_upload_csv_commerzbank(test_app): # Prepare File content = get_testfile_contents(EXAMPLE_CSV, binary=True) files = { - 'file-input': (io.BytesIO(content), 'input_commerzbank.csv'), + 'file-batch': (io.BytesIO(content), 'input_commerzbank.csv'), 'bank': 'Commerzbank' } # Post File @@ -137,7 +137,7 @@ def test_double_upload(test_app): # Prepare File content = get_testfile_contents(EXAMPLE_CSV, binary=True) files = { - 'file-input': (io.BytesIO(content), 'input_commerzbank.csv'), + 'file-batch': (io.BytesIO(content), 'input_commerzbank.csv'), 'bank': 'Commerzbank' } # Post File 1 @@ -154,7 +154,7 @@ def test_double_upload(test_app): # Post File 2 files = { - 'file-input': (io.BytesIO(content), 'input_commerzbank.csv'), + 'file-batch': (io.BytesIO(content), 'input_commerzbank.csv'), 'bank': 'Commerzbank' } result = client.post( diff --git a/tests/test_integ_more_routes.py b/tests/test_integ_more_routes.py index 64dfcd9..5049fa9 100644 --- a/tests/test_integ_more_routes.py +++ b/tests/test_integ_more_routes.py @@ -28,7 +28,7 @@ def test_upload_file_route(test_app): # Prepare File content = get_testfile_contents(EXAMPLE_CSV, binary=True) files = { - 'file-input': (io.BytesIO(content), 'input_commerzbank.csv'), + 'file-batch': (io.BytesIO(content), 'input_commerzbank.csv'), 'bank': 'Commerzbank' } @@ -65,7 +65,7 @@ def test_get_error_messages(test_app): # - Prepare faulty File content = get_testfile_contents(EXAMPLE_CSV, binary=True) files = { - 'file-input': (io.BytesIO(content), 'input_commerzbank.csv') + 'file-batch': (io.BytesIO(content), 'input_commerzbank.csv') } # Post faulty File (missing 'bank' field for Generic importer)