Feat. Overrides, Artwork Uploads, and Snapshot Caching

app/app.py

@@ -1,15 +1,19 @@
-from flask import Flask, render_template, request, redirect, url_for, jsonify, send_from_directory, Response
-from flask_login import LoginManager
+from flask import (
+    Flask,
+    render_template,
+    request,
+    jsonify,
+    send_from_directory,
+    Response
+)
 from scheduler import init_scheduler
 from functools import wraps
 from file_watcher import Watcher
 import threading
 import logging
 import sys
 import copy
-import flask.cli
-from datetime import timedelta
-flask.cli.show_server_banner = lambda *args: None
+import datetime
 from constants import *
 from settings import *
 from db import *
@@ -18,6 +22,8 @@
 import titles
 from utils import *
 from library import *
+from overrides import *
+from cache import regenerate_all_caches
 import titledb
 import os
 
@@ -67,10 +73,10 @@ def scan_library_job():
                 logger.info("Skipping scheduled library scan: update_titledb job is currently in progress. Rescheduling in 5 minutes.")
                 # Reschedule the job for 5 minutes later
                 app.scheduler.add_job(
-                    job_id=f'scan_library_rescheduled_{datetime.now().timestamp()}', # Unique ID
+                    job_id=f'scan_library_rescheduled_{datetime.datetime.now().timestamp()}', # Unique ID
                     func=scan_library_job,
                     run_once=True,
-                    start_date=datetime.now().replace(microsecond=0) + timedelta(minutes=5)
+                    start_date=datetime.datetime.now().replace(microsecond=0) + datetime.timedelta(minutes=5)
                 )
                 return
         logger.info("Starting scheduled library scan job...")
@@ -101,12 +107,14 @@ def update_db_and_scan_job():
     app.scheduler.add_job(
         job_id='update_db_and_scan',
         func=update_db_and_scan_job,
-        interval=timedelta(hours=2),
+        interval=datetime.timedelta(hours=2),
         run_first=True
     )
 
 os.makedirs(CONFIG_DIR, exist_ok=True)
 os.makedirs(DATA_DIR, exist_ok=True)
+os.makedirs(BANNERS_UPLOAD_DIR, exist_ok=True)
+os.makedirs(ICONS_UPLOAD_DIR, exist_ok=True)
 
 ## Global variables
 app_settings = {}
@@ -150,6 +158,33 @@ def reload_conf():
     global watcher
     app_settings = load_settings()
 
+def _update_titledb_before_manual_scan() -> None:
+    """
+    Run a lightweight TitleDB update before a manual scan so new metadata is visible
+    without waiting for the scheduled job. Uses the existing update lock so we never
+    overlap with the scheduler.
+    """
+    global is_titledb_update_running
+    if not titledb_update_lock.acquire(blocking=False):
+        logger.info("Skipping manual TitleDB update: scheduler lock unavailable.")
+        return
+    try:
+        if is_titledb_update_running:
+            logger.info("Skipping manual TitleDB update: another update is already in progress.")
+            return
+        is_titledb_update_running = True
+        try:
+            logger.info("Checking TitleDB before manual library scan...")
+            current_settings = load_settings()
+            titledb.update_titledb(current_settings)
+            logger.info("TitleDB check for manual scan completed.")
+        except Exception as exc:
+            logger.error(f"Error updating TitleDB before manual scan: {exc}")
+        finally:
+            is_titledb_update_running = False
+    finally:
+        titledb_update_lock.release()
+
 def on_library_change(events):
     # TODO refactor: group modified and created together
     with app.app_context():
@@ -188,12 +223,17 @@ def create_app():
     # TODO: generate random secret_key
     app.config['SECRET_KEY'] = '8accb915665f11dfa15c2db1a4e8026905f57716'
     app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+    app.config.setdefault("BANNERS_UPLOAD_DIR", BANNERS_UPLOAD_DIR)
+    app.config.setdefault("BANNERS_UPLOAD_URL_PREFIX", BANNERS_UPLOAD_URL_PREFIX)
+    app.config.setdefault("ICONS_UPLOAD_DIR", ICONS_UPLOAD_DIR)
+    app.config.setdefault("ICONS_UPLOAD_URL_PREFIX", ICONS_UPLOAD_URL_PREFIX)
 
     db.init_app(app)
     migrate.init_app(app, db)
     login_manager.init_app(app)
 
     app.register_blueprint(auth_blueprint)
+    app.register_blueprint(overrides_blueprint)
 
     return app
 
@@ -266,7 +306,13 @@ def _tinfoil_access(*args, **kwargs):
     return _tinfoil_access
 
 def access_shop():
-    return render_template('index.html', title='Library', admin_account_created=admin_account_created(), valid_keys=app_settings['titles']['valid_keys'])
+    return render_template(
+        'index.html',
+        title='Library',
+        admin_account_created=admin_account_created(),
+        valid_keys=app_settings['titles']['valid_keys'],
+        placeholder_text=app_settings['shop']['placeholder_text']
+    )
 
 @access_required('shop')
 def access_shop_auth():
@@ -275,6 +321,36 @@ def access_shop_auth():
 @app.route('/')
 def index():
 
+    def _build_tinfoil_shop_response():
+        """
+        Uses the cached shop snapshot (files + titledb) and wraps it with:
+          - success MOTD
+          - optional referrer (when host verified)
+        Returns a conditional (ETag) response, JSON or encrypted bytes depending on settings.
+        """
+        payload, etag = generate_shop()
+
+        # Wrap with MOTD and referrer (host verification happens in @tinfoil_access)
+        shop = {
+            "success": app_settings['shop']['motd']
+        }
+        if getattr(request, "verified_host", None):
+            shop["referrer"] = f"https://{request.verified_host}"
+
+        # Merge the cached payload
+        shop.update(payload)
+
+        if app_settings['shop']['encrypt']:
+            blob = encrypt_shop(shop)
+            resp = Response(blob, mimetype='application/octet-stream')
+        else:
+            resp = jsonify(shop)
+
+        # Enable cheap 304s
+        resp.set_etag(etag)
+        resp.headers["Cache-Control"] = "no-cache, private"
+        return resp.make_conditional(request)
+
     @tinfoil_access
     def access_tinfoil_shop():
         shop = {
@@ -285,12 +361,7 @@ def access_tinfoil_shop():
             # enforce client side host verification
             shop["referrer"] = f"https://{request.verified_host}"
 
-        shop["files"] = gen_shop_files(db)
-
-        if app_settings['shop']['encrypt']:
-            return Response(encrypt_shop(shop), mimetype='application/octet-stream')
-
-        return jsonify(shop)
+        return _build_tinfoil_shop_response()
 
     if all(header in request.headers for header in TINFOIL_HEADERS):
     # if True:
@@ -328,9 +399,21 @@ def get_settings_api():
 @app.post('/api/settings/titles')
 @access_required('admin')
 def set_titles_settings_api():
-    settings = request.json
-    region = settings['region']
-    language = settings['language']
+    settings_payload = request.get_json(silent=True) or {}
+    region = settings_payload.get('region')
+    language = settings_payload.get('language')
+    if isinstance(region, str):
+        region = region.strip()
+    if isinstance(language, str):
+        language = language.strip()
+    if not region or not language:
+        return jsonify({
+            'success': False,
+            'errors': [{
+                'path': 'titles',
+                'error': "Both 'region' and 'language' are required."
+            }]
+        }), 400
     with open(os.path.join(TITLEDB_DIR, 'languages.json')) as f:
         languages = json.load(f)
         languages = dict(sorted(languages.items()))
@@ -357,8 +440,10 @@ def set_titles_settings_api():
 
 @app.post('/api/settings/shop')
 def set_shop_settings_api():
-    data = request.json
-    set_shop_settings(data)
+    payload = request.get_json(silent=True)
+    if payload is None:
+        return jsonify({'success': False, 'errors': ['Request body must be JSON']}), 400
+    set_shop_settings(payload)
     reload_conf()
     resp = {
         'success': True,
@@ -371,8 +456,11 @@ def set_shop_settings_api():
 def library_paths_api():
     global watcher
     if request.method == 'POST':
-        data = request.json
-        success, errors = add_library_complete(app, watcher, data['path'])
+        payload = request.get_json(silent=True) or {}
+        path = payload.get('path')
+        if not path:
+            return jsonify({'success': False, 'errors': ['Library path is required']}), 400
+        success, errors = add_library_complete(app, watcher, path)
         if success:
             reload_conf()
             post_library_change()
@@ -388,8 +476,11 @@ def library_paths_api():
             'paths': app_settings['library']['paths']
         }    
     elif request.method == 'DELETE':
-        data = request.json
-        success, errors = remove_library_complete(app, watcher, data['path'])
+        payload = request.get_json(silent=True) or {}
+        path = payload.get('path')
+        if not path:
+            return jsonify({'success': False, 'errors': ['Library path is required']}), 400
+        success, errors = remove_library_complete(app, watcher, path)
         if success:
             reload_conf()
             post_library_change()
@@ -402,8 +493,10 @@ def library_paths_api():
 @app.post('/api/settings/library/management')
 @access_required('admin')
 def set_library_management_settings_api():
-    data = request.json
-    set_library_management_settings(data)
+    payload = request.get_json(silent=True)
+    if payload is None:
+        return jsonify({'success': False, 'errors': ['Request body must be JSON']}), 400
+    set_library_management_settings(payload)
     reload_conf()
     post_library_change()
     resp = {
@@ -440,16 +533,28 @@ def upload_file():
     } 
     return jsonify(resp)
 
+@app.route("/uploads/banners/<path:filename>")
+def uploaded_banners(filename):
+    return send_from_directory(app.config["BANNERS_UPLOAD_DIR"], filename, conditional=True)
+
+@app.route("/uploads/icons/<path:filename>")
+def uploaded_icons(filename):
+    return send_from_directory(app.config["ICONS_UPLOAD_DIR"], filename, conditional=True)
 
 @app.route('/api/titles', methods=['GET'])
 @access_required('shop')
 def get_all_titles_api():
-    titles_library = generate_library()
-
-    return jsonify({
+    titles_library, etag_hash = generate_library_snapshot()
+    payload = {
         'total': len(titles_library),
         'games': titles_library
-    })
+    }
+
+    resp = jsonify(payload)
+    resp.set_etag(etag_hash)
+    resp.headers["Vary"] = "Authorization"
+    resp.headers["Cache-Control"] = "no-cache, private"
+    return resp.make_conditional(request)
 
 @app.route('/api/get_game/<int:id>')
 @tinfoil_access
@@ -463,24 +568,25 @@ def serve_game(id):
 @debounce(10)
 def post_library_change():
     with app.app_context():
-        titles.load_titledb()
-        process_library_identification(app)
-        add_missing_apps_to_db()
-        update_titles() # Ensure titles are updated after identification
-        # remove missing files
-        remove_missing_files_from_db()
-        process_library_organization(app, watcher) # Pass the watcher instance to skip organizer move/delete events
-        # The process_library_identification already handles updating titles and generating library
-        # So, we just need to ensure titles_library is updated from the generated library
-        generate_library()
-        titles.identification_in_progress_count -= 1
-        titles.unload_titledb()
+        with titles.titledb_session("post_library_change"):
+            process_library_identification(app)
+            add_missing_apps_to_db()
+            update_titles()  # Ensure titles are updated after identification
+            # remove missing files
+            remove_missing_files_from_db()
+            process_library_organization(app, watcher) # Pass the watcher instance to skip organizer move/delete events
+
+        # refresh caches after leaving the titledb session
+        regenerate_all_caches()
+
 
 @app.post('/api/library/scan')
 @access_required('admin')
 def scan_library_api():
-    data = request.json
-    path = data['path']
+    payload = request.get_json(silent=True) or {}
+    path = payload.get('path') if payload else None
+    if isinstance(path, str) and not path.strip():
+        path = None
     success = True
     errors = []
 
@@ -493,6 +599,7 @@ def scan_library_api():
     scan_in_progress = True
 
     try:
+        _update_titledb_before_manual_scan()
         if path is None:
             scan_library()
         else: