From 6477a87bc80378365a529cab28885763a6d50d42 Mon Sep 17 00:00:00 2001 From: Harley Bartles Date: Wed, 29 Oct 2025 18:20:55 +0000 Subject: [PATCH 1/2] Combined commit for new features. --- app/app.py | 167 +- app/cache.py | 226 +++ app/constants.py | 15 +- app/db.py | 196 ++- app/images.py | 184 +++ app/library.py | 1416 +++++++++++++---- .../b52221b8c1e8_add_title_overrides_table.py | 47 + app/overrides.py | 495 ++++++ app/shop.py | 485 +++++- app/static/js/overrides.js | 1028 ++++++++++++ app/static/js/pagination.js | 183 +++ app/static/style.css | 139 +- app/templates/index.html | 765 +++++---- app/templates/override_modal.html | 137 ++ app/templates/settings.html | 71 +- app/titles.py | 396 ++++- app/utils.py | 99 +- requirements.txt | 1 + 18 files changed, 5219 insertions(+), 831 deletions(-) create mode 100644 app/cache.py create mode 100644 app/images.py create mode 100644 app/migrations/versions/b52221b8c1e8_add_title_overrides_table.py create mode 100644 app/overrides.py create mode 100644 app/static/js/overrides.js create mode 100644 app/static/js/pagination.js create mode 100644 app/templates/override_modal.html diff --git a/app/app.py b/app/app.py index 8c10c91c..5241964f 100644 --- a/app/app.py +++ b/app/app.py @@ -1,5 +1,11 @@ -from flask import Flask, render_template, request, redirect, url_for, jsonify, send_from_directory, Response -from flask_login import LoginManager +from flask import ( + Flask, + render_template, + request, + jsonify, + send_from_directory, + Response +) from scheduler import init_scheduler from functools import wraps from file_watcher import Watcher @@ -7,9 +13,7 @@ import logging import sys import copy -import flask.cli -from datetime import timedelta -flask.cli.show_server_banner = lambda *args: None +import datetime from constants import * from settings import * from db import * @@ -18,6 +22,8 @@ import titles from utils import * from library import * +from overrides import * +from cache import regenerate_all_caches import titledb import os @@ -67,10 +73,10 @@ def scan_library_job(): logger.info("Skipping scheduled library scan: update_titledb job is currently in progress. Rescheduling in 5 minutes.") # Reschedule the job for 5 minutes later app.scheduler.add_job( - job_id=f'scan_library_rescheduled_{datetime.now().timestamp()}', # Unique ID + job_id=f'scan_library_rescheduled_{datetime.datetime.now().timestamp()}', # Unique ID func=scan_library_job, run_once=True, - start_date=datetime.now().replace(microsecond=0) + timedelta(minutes=5) + start_date=datetime.datetime.now().replace(microsecond=0) + datetime.timedelta(minutes=5) ) return logger.info("Starting scheduled library scan job...") @@ -101,12 +107,14 @@ def update_db_and_scan_job(): app.scheduler.add_job( job_id='update_db_and_scan', func=update_db_and_scan_job, - interval=timedelta(hours=2), + interval=datetime.timedelta(hours=2), run_first=True ) os.makedirs(CONFIG_DIR, exist_ok=True) os.makedirs(DATA_DIR, exist_ok=True) +os.makedirs(BANNERS_UPLOAD_DIR, exist_ok=True) +os.makedirs(ICONS_UPLOAD_DIR, exist_ok=True) ## Global variables app_settings = {} @@ -188,12 +196,17 @@ def create_app(): # TODO: generate random secret_key app.config['SECRET_KEY'] = '8accb915665f11dfa15c2db1a4e8026905f57716' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + app.config.setdefault("BANNERS_UPLOAD_DIR", BANNERS_UPLOAD_DIR) + app.config.setdefault("BANNERS_UPLOAD_URL_PREFIX", BANNERS_UPLOAD_URL_PREFIX) + app.config.setdefault("ICONS_UPLOAD_DIR", ICONS_UPLOAD_DIR) + app.config.setdefault("ICONS_UPLOAD_URL_PREFIX", ICONS_UPLOAD_URL_PREFIX) db.init_app(app) migrate.init_app(app, db) login_manager.init_app(app) app.register_blueprint(auth_blueprint) + app.register_blueprint(overrides_blueprint) return app @@ -266,7 +279,13 @@ def _tinfoil_access(*args, **kwargs): return _tinfoil_access def access_shop(): - return render_template('index.html', title='Library', admin_account_created=admin_account_created(), valid_keys=app_settings['titles']['valid_keys']) + return render_template( + 'index.html', + title='Library', + admin_account_created=admin_account_created(), + valid_keys=app_settings['titles']['valid_keys'], + placeholder_text=app_settings['shop']['placeholder_text'] + ) @access_required('shop') def access_shop_auth(): @@ -275,6 +294,36 @@ def access_shop_auth(): @app.route('/') def index(): + def _build_tinfoil_shop_response(): + """ + Uses the cached shop snapshot (files + titledb) and wraps it with: + - success MOTD + - optional referrer (when host verified) + Returns a conditional (ETag) response, JSON or encrypted bytes depending on settings. + """ + payload, etag = generate_shop() + + # Wrap with MOTD and referrer (host verification happens in @tinfoil_access) + shop = { + "success": app_settings['shop']['motd'] + } + if getattr(request, "verified_host", None): + shop["referrer"] = f"https://{request.verified_host}" + + # Merge the cached payload + shop.update(payload) + + if app_settings['shop']['encrypt']: + blob = encrypt_shop(shop) + resp = Response(blob, mimetype='application/octet-stream') + else: + resp = jsonify(shop) + + # Enable cheap 304s + resp.set_etag(etag) + resp.headers["Cache-Control"] = "no-cache, private" + return resp.make_conditional(request) + @tinfoil_access def access_tinfoil_shop(): shop = { @@ -285,12 +334,7 @@ def access_tinfoil_shop(): # enforce client side host verification shop["referrer"] = f"https://{request.verified_host}" - shop["files"] = gen_shop_files(db) - - if app_settings['shop']['encrypt']: - return Response(encrypt_shop(shop), mimetype='application/octet-stream') - - return jsonify(shop) + return _build_tinfoil_shop_response() if all(header in request.headers for header in TINFOIL_HEADERS): # if True: @@ -328,9 +372,21 @@ def get_settings_api(): @app.post('/api/settings/titles') @access_required('admin') def set_titles_settings_api(): - settings = request.json - region = settings['region'] - language = settings['language'] + settings_payload = request.get_json(silent=True) or {} + region = settings_payload.get('region') + language = settings_payload.get('language') + if isinstance(region, str): + region = region.strip() + if isinstance(language, str): + language = language.strip() + if not region or not language: + return jsonify({ + 'success': False, + 'errors': [{ + 'path': 'titles', + 'error': "Both 'region' and 'language' are required." + }] + }), 400 with open(os.path.join(TITLEDB_DIR, 'languages.json')) as f: languages = json.load(f) languages = dict(sorted(languages.items())) @@ -357,8 +413,10 @@ def set_titles_settings_api(): @app.post('/api/settings/shop') def set_shop_settings_api(): - data = request.json - set_shop_settings(data) + payload = request.get_json(silent=True) + if payload is None: + return jsonify({'success': False, 'errors': ['Request body must be JSON']}), 400 + set_shop_settings(payload) reload_conf() resp = { 'success': True, @@ -371,8 +429,11 @@ def set_shop_settings_api(): def library_paths_api(): global watcher if request.method == 'POST': - data = request.json - success, errors = add_library_complete(app, watcher, data['path']) + payload = request.get_json(silent=True) or {} + path = payload.get('path') + if not path: + return jsonify({'success': False, 'errors': ['Library path is required']}), 400 + success, errors = add_library_complete(app, watcher, path) if success: reload_conf() post_library_change() @@ -388,8 +449,11 @@ def library_paths_api(): 'paths': app_settings['library']['paths'] } elif request.method == 'DELETE': - data = request.json - success, errors = remove_library_complete(app, watcher, data['path']) + payload = request.get_json(silent=True) or {} + path = payload.get('path') + if not path: + return jsonify({'success': False, 'errors': ['Library path is required']}), 400 + success, errors = remove_library_complete(app, watcher, path) if success: reload_conf() post_library_change() @@ -402,8 +466,10 @@ def library_paths_api(): @app.post('/api/settings/library/management') @access_required('admin') def set_library_management_settings_api(): - data = request.json - set_library_management_settings(data) + payload = request.get_json(silent=True) + if payload is None: + return jsonify({'success': False, 'errors': ['Request body must be JSON']}), 400 + set_library_management_settings(payload) reload_conf() post_library_change() resp = { @@ -440,16 +506,28 @@ def upload_file(): } return jsonify(resp) +@app.route("/uploads/banners/") +def uploaded_banners(filename): + return send_from_directory(app.config["BANNERS_UPLOAD_DIR"], filename, conditional=True) + +@app.route("/uploads/icons/") +def uploaded_icons(filename): + return send_from_directory(app.config["ICONS_UPLOAD_DIR"], filename, conditional=True) @app.route('/api/titles', methods=['GET']) @access_required('shop') def get_all_titles_api(): - titles_library = generate_library() - - return jsonify({ + titles_library, etag_hash = generate_library_snapshot() + payload = { 'total': len(titles_library), 'games': titles_library - }) + } + + resp = jsonify(payload) + resp.set_etag(etag_hash) + resp.headers["Vary"] = "Authorization" + resp.headers["Cache-Control"] = "no-cache, private" + return resp.make_conditional(request) @app.route('/api/get_game/') @tinfoil_access @@ -463,24 +541,25 @@ def serve_game(id): @debounce(10) def post_library_change(): with app.app_context(): - titles.load_titledb() - process_library_identification(app) - add_missing_apps_to_db() - update_titles() # Ensure titles are updated after identification - # remove missing files - remove_missing_files_from_db() - process_library_organization(app, watcher) # Pass the watcher instance to skip organizer move/delete events - # The process_library_identification already handles updating titles and generating library - # So, we just need to ensure titles_library is updated from the generated library - generate_library() - titles.identification_in_progress_count -= 1 - titles.unload_titledb() + with titles.titledb_session("post_library_change"): + process_library_identification(app) + add_missing_apps_to_db() + update_titles() # Ensure titles are updated after identification + # remove missing files + remove_missing_files_from_db() + process_library_organization(app, watcher) # Pass the watcher instance to skip organizer move/delete events + + # refresh caches after leaving the titledb session + regenerate_all_caches() + @app.post('/api/library/scan') @access_required('admin') def scan_library_api(): - data = request.json - path = data['path'] + payload = request.get_json(silent=True) or {} + path = payload.get('path') if payload else None + if isinstance(path, str) and not path.strip(): + path = None success = True errors = [] diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..496e3db3 --- /dev/null +++ b/app/cache.py @@ -0,0 +1,226 @@ +import datetime +import hashlib +import json +import logging +import os +from typing import Callable, Dict, Optional + +from constants import LIBRARY_CACHE_FILE, LIBRARY_SNAPSHOT_VERSION, OVERRIDES_CACHE_FILE, SHOP_CACHE_FILE +from db import AppOverrides, Files, db, get_all_apps +from utils import load_json +import titles as titles_lib + +logger = logging.getLogger("main") + +CacheValidator = Callable[[Optional[dict]], bool] + + +def compute_library_apps_hash() -> str: + """ + Computes a hash of all Apps table content to detect changes in library state. + """ + hash_md5 = hashlib.md5() + apps = get_all_apps() + + for app in sorted(apps, key=lambda x: (x["app_id"] or "", x["app_version"] or "")): + hash_md5.update((app["app_id"] or "").encode()) + hash_md5.update((app["app_version"] or "").encode()) + hash_md5.update((app["app_type"] or "").encode()) + hash_md5.update(str(app["owned"] or False).encode()) + hash_md5.update((app["title_id"] or "").encode()) + return hash_md5.hexdigest() + + +def is_library_snapshot_current(saved_library: Optional[dict]) -> bool: + if not saved_library or not saved_library.get("hash"): + return False + + if saved_library.get("snapshot_version") != LIBRARY_SNAPSHOT_VERSION: + return False + + current_apps_hash = compute_library_apps_hash() + if saved_library.get("hash") != current_apps_hash: + return False + + current_tdb = titles_lib.get_titledb_commit_hash() or "" + saved_tdb = saved_library.get("titledb_commit") + if saved_tdb is None: + return False + + return saved_tdb == current_tdb + + +def compute_overrides_fingerprint_rows() -> list[tuple]: + rows = ( + db.session.query( + AppOverrides.id, + AppOverrides.updated_at, + AppOverrides.corrected_title_id, + AppOverrides.banner_path, + AppOverrides.icon_path, + AppOverrides.enabled, + ) + .order_by(AppOverrides.id.asc()) + .all() + ) + + normalized = [] + for row in rows: + updated_at = row[1] + if isinstance(updated_at, datetime.datetime): + updated_at_str = updated_at.isoformat(timespec="seconds") + else: + updated_at_str = str(updated_at) + + normalized.append( + ( + row[0], + updated_at_str, + row[2] or None, + row[3] or None, + row[4] or None, + bool(row[5]), + ) + ) + return normalized + + +def compute_overrides_snapshot_hash() -> str: + payload_for_hash = { + "rows": compute_overrides_fingerprint_rows(), + "titledb_commit": titles_lib.get_titledb_commit_hash(), + } + return hashlib.sha256( + json.dumps(payload_for_hash, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def is_overrides_snapshot_current(saved_snapshot: Optional[dict]) -> bool: + if not saved_snapshot or not isinstance(saved_snapshot, dict): + return False + stored_hash = saved_snapshot.get("hash") + if not stored_hash: + return False + return stored_hash == compute_overrides_snapshot_hash() + + +def compute_shop_files_fingerprint_rows() -> list[tuple[int, int, str]]: + rows = ( + db.session.query(Files.id, Files.size, Files.filepath) + .order_by(Files.id.asc()) + .all() + ) + fingerprint = [] + for fid, size, path in rows: + base = os.path.basename(path or "") if path else "" + fingerprint.append((int(fid), int(size or 0), base)) + return fingerprint + + +def compute_shop_snapshot_hash() -> str: + from overrides import load_or_generate_overrides_snapshot + + overrides_snapshot = load_or_generate_overrides_snapshot() or {} + ov_hash = overrides_snapshot.get("hash") or "" + + library_snapshot = load_json(LIBRARY_CACHE_FILE) or {} + lib_hash = library_snapshot.get("hash") or "" + + files_fp = compute_shop_files_fingerprint_rows() + + payload = { + "overrides_hash": ov_hash, + "library_hash": lib_hash, + "files": files_fp, + } + return hashlib.sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def is_shop_snapshot_current(saved_snapshot: Optional[dict]) -> bool: + if not saved_snapshot or not isinstance(saved_snapshot, dict): + return False + stored_hash = saved_snapshot.get("hash") + if not stored_hash: + return False + return stored_hash == compute_shop_snapshot_hash() + + +_CACHE_VALIDATORS: Dict[str, CacheValidator] = { + LIBRARY_CACHE_FILE: is_library_snapshot_current, + OVERRIDES_CACHE_FILE: is_overrides_snapshot_current, + SHOP_CACHE_FILE: is_shop_snapshot_current, +} + +def generate_snapshot(path: str): + """ + Regenerate a known cache snapshot given its file path. + Dispatches to the correct builder so the cache is warm for next request. + """ + try: + if path == LIBRARY_CACHE_FILE: + from library import load_or_generate_library_snapshot + + load_or_generate_library_snapshot() + logger.info(f"Regenerated library snapshot: {path}") + elif path == OVERRIDES_CACHE_FILE: + from overrides import load_or_generate_overrides_snapshot + + load_or_generate_overrides_snapshot() + logger.info(f"Regenerated overrides snapshot: {path}") + elif path == SHOP_CACHE_FILE: + from shop import load_or_generate_shop_snapshot + + load_or_generate_shop_snapshot() + logger.info(f"Regenerated shop snapshot: {path}") + else: + logger.warning(f"Unknown snapshot path: {path}") + except Exception as e: + logger.error(f"Failed to regenerate {path}: {e}") + + +def regenerate_cache(*paths: str): + """ + Force regeneration of one or more known cache snapshots. + + Accepts either a sequence of paths, or a single iterable of paths. The + existing cache file is left in place until the snapshot builder finishes, + so callers keep a fallback if regeneration fails. + """ + if len(paths) == 1 and not isinstance(paths[0], str): + candidate_paths = paths[0] + else: + candidate_paths = paths + + for path in candidate_paths: + if not isinstance(path, str): + logger.warning(f"Skipping non-string cache path: {path!r}") + continue + generate_snapshot(path) + + +def regenerate_all_caches(): + """ + Ensure all known cache snapshots are up-to-date without forcing rebuilds. + """ + for path in (LIBRARY_CACHE_FILE, OVERRIDES_CACHE_FILE, SHOP_CACHE_FILE): + validator = _CACHE_VALIDATORS.get(path) + if not validator: + logger.warning(f"No validator registered for {path}; forcing regeneration.") + generate_snapshot(path) + continue + + name = os.path.basename(path) + try: + saved = load_json(path, default=None) + except Exception as exc: + logger.warning(f"Failed to load cache snapshot {path}: {exc}") + saved = None + + if validator(saved): + logger.debug(f"{name} cache is up-to-date; skipping regeneration.") + continue + + logger.info(f"Refreshing {name}") + generate_snapshot(path) diff --git a/app/constants.py b/app/constants.py index 737e2312..882f3589 100644 --- a/app/constants.py +++ b/app/constants.py @@ -1,15 +1,27 @@ import os +import re APP_DIR = os.path.dirname(os.path.abspath(__file__)) DATA_DIR = os.path.join(APP_DIR, 'data') CONFIG_DIR = os.path.join(APP_DIR, 'config') +BANNERS_UPLOAD_DIR = os.path.join(DATA_DIR, 'uploads', 'banners') +BANNERS_UPLOAD_URL_PREFIX = '/uploads/banners' +ICONS_UPLOAD_DIR = os.path.join(DATA_DIR, 'uploads', 'icons') +ICONS_UPLOAD_URL_PREFIX = "/uploads/icons" DB_FILE = os.path.join(CONFIG_DIR, 'ownfoil.db') CONFIG_FILE = os.path.join(CONFIG_DIR, 'settings.yaml') KEYS_FILE = os.path.join(CONFIG_DIR, 'keys.txt') CACHE_DIR = os.path.join(DATA_DIR, 'cache') LIBRARY_CACHE_FILE = os.path.join(CACHE_DIR, 'library.json') +LIBRARY_SNAPSHOT_VERSION = 2 +OVERRIDES_CACHE_FILE = os.path.join(CACHE_DIR, 'overrides.json') +SHOP_CACHE_FILE = os.path.join(CACHE_DIR, 'shop.json') ALEMBIC_DIR = os.path.join(APP_DIR, 'migrations') ALEMBIC_CONF = os.path.join(ALEMBIC_DIR, 'alembic.ini') +TITLE_ID_RE = re.compile(r'^[0-9A-F]{16}$') +APP_ID_RE = re.compile(r"^(?:[0-9A-F]{16}|[0-9A-F]{32})$") # For validating raw IDs supplied by APIs/DB: 16 (title/app) or 32 (content) hex chars +FILENAME_APP_ID_RE = re.compile(r"\[([0-9A-Fa-f]{16})\]") # For filename parsing like "... [0100ABCDEF123456] ...": +VERSION_RE = re.compile(r"\[v(\d+)\]") TITLEDB_DIR = os.path.join(DATA_DIR, 'titledb') TITLEDB_URL = 'https://github.com/blawar/titledb.git' TITLEDB_ARTEFACTS_URL = 'https://nightly.link/a1ex4/ownfoil/workflows/region_titles/master/titledb.zip' @@ -53,6 +65,7 @@ "clientCertKey": "-----BEGIN PRIVATE KEY-----", "host": "", "hauth": "", + "placeholder_text": "Image Unavailable", } } @@ -80,4 +93,4 @@ 128: APP_TYPE_BASE, 129: APP_TYPE_UPD, 130: APP_TYPE_DLC -} \ No newline at end of file +} diff --git a/app/db.py b/app/db.py index 9ce0bf5d..2f86eecb 100644 --- a/app/db.py +++ b/app/db.py @@ -37,7 +37,7 @@ def get_current_db_version(): def create_db_backup(): current_revision = get_current_db_version() - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + timestamp = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S") backup_filename = f".backup_v{current_revision}_{timestamp}.db" backup_path = os.path.join(CONFIG_DIR, backup_filename) shutil.copy2(DB_FILE, backup_path) @@ -79,7 +79,7 @@ class Files(db.Model): identification_type = db.Column(db.String) identification_error = db.Column(db.String) identification_attempts = db.Column(db.Integer, default=0) - last_attempt = db.Column(db.DateTime, default=datetime.datetime.now()) + last_attempt = db.Column(db.DateTime, default=datetime.datetime.utcnow) library = db.relationship('Libraries', backref=db.backref('files', lazy=True, cascade="all, delete-orphan")) @@ -107,6 +107,15 @@ class Apps(db.Model): title = db.relationship('Titles', backref=db.backref('apps', lazy=True, cascade="all, delete-orphan")) files = db.relationship('Files', secondary=app_files, backref=db.backref('apps', lazy='select')) + # One-to-one: delete override automatically when an Apps row is deleted + override = db.relationship( + 'AppOverrides', + back_populates='app', + cascade='all, delete-orphan', + passive_deletes=True, + uselist=False + ) + __table_args__ = (db.UniqueConstraint('app_id', 'app_version', name='uq_apps_app_version'),) class User(UserMixin, db.Model): @@ -138,6 +147,64 @@ def has_access(self, access): elif access == 'backup': return self.has_backup_access() +class AppOverrides(db.Model): + __tablename__ = 'app_overrides' + + id = db.Column(db.Integer, primary_key=True) + + # FK to Apps (one-to-one), cascades on app delete + app_fk = db.Column(db.Integer, db.ForeignKey('apps.id', ondelete='CASCADE'), nullable=False, unique=True) + + # ORM relationship + app = db.relationship('Apps', back_populates='override') + + # ---- Overridable metadata (all optional) ---- + name = db.Column(db.String(512), nullable=True) + release_date = db.Column(db.Date, nullable=True) + region = db.Column(db.String(32), nullable=True) + description = db.Column(db.Text, nullable=True) + content_type = db.Column(db.String(64), nullable=True) # e.g., Base/Update/DLC + version = db.Column(db.String(64), nullable=True) + + # ---- Artwork (relative paths under /static/...) ---- + icon_path = db.Column(db.String(1024), nullable=True) + banner_path = db.Column(db.String(1024), nullable=True) + + # ---- Corrected Title ID (for TitleDB lookups/merge only) ---- + corrected_title_id = db.Column(db.String(16), index=True, nullable=True) # NEW + + enabled = db.Column(db.Boolean, nullable=False, default=True) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) + updated_at = db.Column( + db.DateTime, + nullable=False, + default=datetime.datetime.utcnow, + onupdate=datetime.datetime.utcnow + ) + + # Convenience: expose app.app_id without storing it in this table + @property + def app_id(self): + return self.app.app_id if self.app else None + + def as_dict(self): + return { + 'id': self.id, + 'app_id': self.app.app_id if self.app else None, + 'name': self.name, + 'release_date': self.release_date.isoformat() if self.release_date else None, + 'region': self.region, + 'description': self.description, + 'content_type': self.content_type, + 'version': self.version, + 'icon_path': self.icon_path, + 'banner_path': self.banner_path, + 'corrected_title_id': self.corrected_title_id, + 'enabled': self.enabled, + 'created_at': self.created_at.isoformat() if self.created_at else None, + 'updated_at': self.updated_at.isoformat() if self.updated_at else None, + } + def init_db(app): with app.app_context(): # Ensure foreign keys are enforced when the SQLite connection is opened @@ -160,6 +227,9 @@ def set_sqlite_pragma(dbapi_connection, connection_record): upgrade() logger.info("Database migration applied successfully.") + # cleanup any stale banner/icon files + _garbage_collect_orphaned_art() + def file_exists_in_db(filepath): return Files.query.filter_by(filepath=filepath).first() is not None @@ -215,18 +285,34 @@ def get_all_files_without_identification(identification): results = Files.query.filter(Files.identification_type != identification).all() return[to_dict(r)['filepath'] for r in results] -def get_all_apps(): - apps_list = [ - { +from sqlalchemy.orm import joinedload + +def get_all_apps(include_files: bool = False): + q = Apps.query.options(db.joinedload(Apps.title)) + if include_files: + q = q.options(joinedload(Apps.files)) + + apps_list = [] + for app in q.all(): + row = { "id": app.id, - "title_id": app.title.title_id, # Access the actual title_id from Titles + "title_id": app.title.title_id, "app_id": app.app_id, "app_version": app.app_version, "app_type": app.app_type, - "owned": app.owned + "owned": app.owned, } - for app in Apps.query.options(db.joinedload(Apps.title)).all() # Optimized with joinedload - ] + if include_files: + row["files"] = [{ + "id": f.id, + "filename": getattr(f, "filename", None), + "filepath": getattr(f, "filepath", None), + "identification_type": (getattr(f, "identification_type", None) or "").lower(), + "last_attempt": getattr(f, "last_attempt", None), + "created_at": getattr(f, "created_at", None), + } for f in (app.files or [])] + apps_list.append(row) + return apps_list def get_all_non_identified_files_from_library(library_id): @@ -236,31 +322,12 @@ def get_files_with_identification_from_library(library_id, identification_type): return Files.query.filter_by(library_id=library_id, identification_type=identification_type).all() def get_shop_files(): - shop_files = [] - results = Files.query.options(db.joinedload(Files.apps).joinedload(Apps.title)).all() - - for file in results: - if file.identified: - # Get the first app associated with this file using the many-to-many relationship - app = file.apps[0] if file.apps else None - - if app: - if file.multicontent or file.extension.startswith('x'): - title_id = app.title.title_id - final_filename = f"[{title_id}].{file.extension}" - else: - final_filename = f"[{app.app_id}][v{app.app_version}].{file.extension}" - else: - final_filename = file.filename.replace(f'.{file.extension}', '') + ' (unidentified).' + file.extension - else: - final_filename = file.filename.replace(f'.{file.extension}', '') + ' (unidentified).' + file.extension - - shop_files.append({ - "id": file.id, - "filename": final_filename, - "size": file.size - }) - + results = Files.query.all() + shop_files = [{ + "id": file.id, + "filename": file.filename, + "size": file.size + } for file in results] return shop_files def get_libraries(): @@ -281,6 +348,7 @@ def delete_library(library): db.session.delete(get_library(library)) db.session.commit() + _garbage_collect_orphaned_art() def get_library(library_id): return Libraries.query.filter_by(id=library_id).first() @@ -304,7 +372,7 @@ def get_library_file_paths(library_id): def set_library_scan_time(library_id, scan_time=None): library = get_library(library_id) - library.last_scan = scan_time or datetime.datetime.now() + library.last_scan = scan_time or datetime.datetime.utcnow() db.session.commit() def get_all_titles(): @@ -315,7 +383,7 @@ def get_title(title_id): def get_title_id_db_id(title_id): title = get_title(title_id) - return title.id + return title.id if title else None def add_title_id_in_db(title_id): existing_title = Titles.query.filter_by(title_id=title_id).first() @@ -327,7 +395,7 @@ def add_title_id_in_db(title_id): def get_all_title_apps(title_id): title = Titles.query.options(joinedload(Titles.apps)).filter_by(title_id=title_id).first() - return[to_dict(a) for a in title.apps] + return [] if title is None else [to_dict(a) for a in title.apps] def get_app_by_id_and_version(app_id, app_version): """Get app entry for a specific app_id and version (unique due to constraint)""" @@ -343,17 +411,28 @@ def is_app_owned(app_id, app_version): app = get_app_by_id_and_version(app_id, app_version) return app.owned if app else False -def add_file_to_app(app_id, app_version, file_id): - """Add a file to an existing app using many-to-many relationship""" - app = get_app_by_id_and_version(app_id, app_version) - if app: - file_obj = get_file_from_db(file_id) - if file_obj and file_obj not in app.files: - app.files.append(file_obj) - app.owned = True - db.session.commit() - return True - return False +def add_file_to_app(app_id, app_version, file_id, *, commit=True): + """Add a file to an existing app using many-to-many relationship (idempotent).""" + ver = str(app_version if app_version is not None else "0") + app = get_app_by_id_and_version(app_id, ver) + if not app: + return False + + file_obj = get_file_from_db(file_id) + if not file_obj: + return False + + # Link if missing + if file_obj not in app.files: + app.files.append(file_obj) + + # Ensure owned reflects reality once any file is linked + if not app.owned: + app.owned = True + + if commit: + db.session.commit() + return True def remove_file_from_apps(file_id): """Remove a file from all apps that reference it and update owned status""" @@ -399,6 +478,10 @@ def remove_titles_without_owned_apps(): db.session.delete(title) titles_removed += 1 + if titles_removed: + db.session.commit() + _garbage_collect_orphaned_art() + return titles_removed def delete_files_by_library(library_path): @@ -406,7 +489,7 @@ def delete_files_by_library(library_path): errors = [] try: # Find all files with the given library - files_to_delete = Files.query.filter_by(library=library_path).all() + files_to_delete = Files.query.filter_by(library_id=get_library_id(library_path)).all() # Update Apps table before deleting files total_apps_updated = 0 @@ -420,6 +503,7 @@ def delete_files_by_library(library_path): # Commit the changes db.session.commit() + _garbage_collect_orphaned_art() logger.info(f"All entries with library '{library_path}' have been deleted.") if total_apps_updated > 0: @@ -450,7 +534,8 @@ def delete_file_by_filepath(filepath): # Commit the changes db.session.commit() - + _garbage_collect_orphaned_art() + logger.info(f"File '{filepath}' removed from database.") if apps_updated > 0: logger.info(f"Updated {apps_updated} app entries to remove file reference.") @@ -489,6 +574,7 @@ def remove_missing_files_from_db(): Files.query.filter(Files.id.in_(ids_to_delete)).delete(synchronize_session=False) db.session.commit() + _garbage_collect_orphaned_art() logger.info(f"Deleted {len(ids_to_delete)} files from the database.") if total_apps_updated > 0: @@ -500,3 +586,13 @@ def remove_missing_files_from_db(): except Exception as e: db.session.rollback() # Rollback in case of an error logger.error(f"An error occurred while removing missing files: {str(e)}") + +def _garbage_collect_orphaned_art(): + try: + # Local import to avoid circular imports + from images import garbage_collect_orphan_art_files + from flask import current_app + with current_app.app_context(): + garbage_collect_orphan_art_files() + except Exception as e: + logger.warning(f"GC of orphan override art failed: {e}") diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..100f1528 --- /dev/null +++ b/app/images.py @@ -0,0 +1,184 @@ +import os + +from flask import current_app +from io import BytesIO +from PIL import Image, ImageOps +from werkzeug.exceptions import BadRequest +from werkzeug.utils import secure_filename + +from utils import * +import logging +logger = logging.getLogger('main') + +ALLOWED_IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.webp'} + +def validate_upload(file_storage) -> None: + filename = secure_filename(file_storage.filename or "") + if not _allowed_image(filename): + ext = _ext_for_content_type(getattr(file_storage, "mimetype", "") or "") + if not ext or ext not in ALLOWED_IMAGE_EXTS: + raise BadRequest("Unsupported file type. Allowed: .jpg .jpeg .png .webp") + +def read_upload_bytes(file_storage) -> bytes: + stream = getattr(file_storage, "stream", None) + try: + if stream: stream.seek(0) + except Exception: + pass + if hasattr(file_storage, "read"): + data = file_storage.read() + elif stream and hasattr(stream, "read"): + data = stream.read() + else: + raise BadRequest("Invalid upload object.") + try: + if stream: stream.seek(0) + except Exception: + pass + return data + +def save_art_from_bytes(app_id: str, raw: bytes, kind: str) -> str: + """ + Save banner or icon artwork from raw bytes. + + Args: + app_id: 16- or 32-character hex app id (title/content id). + raw: Raw image data. + kind: Either "banner" or "icon". + + Returns: + The public URL path to the saved image. + """ + if kind not in ("banner", "icon"): + raise ValueError(f"Unknown art kind: {kind}. Expected 'banner' or 'icon'.") + + # Determine target parameters + if kind == "banner": + out_name = f"{app_id}_banner.png" + upload_dir = current_app.config["BANNERS_UPLOAD_DIR"] + url_prefix = current_app.config["BANNERS_UPLOAD_URL_PREFIX"].rstrip('/') + target_w, target_h = 400, 225 + else: # icon + out_name = f"{app_id}_icon.png" + upload_dir = current_app.config.get("ICONS_UPLOAD_DIR") or current_app.config["BANNERS_UPLOAD_DIR"] + url_prefix = (current_app.config.get("ICONS_UPLOAD_URL_PREFIX") + or current_app.config["BANNERS_UPLOAD_URL_PREFIX"]).rstrip('/') + target_w = target_h = 400 + + logger.info(f"Saving {kind} for app_id={app_id} → dir={upload_dir}") + os.makedirs(upload_dir, exist_ok=True) + dst_path = os.path.join(upload_dir, out_name) + + # Remove any older variants (e.g., .jpg/.webp/.png) + for old_ext in ALLOWED_IMAGE_EXTS.union({".png"}): + old_path = os.path.join(upload_dir, f"{app_id}_{kind}{old_ext}") + if old_path != dst_path and os.path.exists(old_path): + try: + os.remove(old_path) + except OSError as e: + logger.error(f"[images] failed to remove old variant {old_path}: {e}") + + # Open, resize, crop, and save + with Image.open(BytesIO(raw)) as im: + im = ImageOps.exif_transpose(im) + src_w, src_h = im.size + if src_w == 0 or src_h == 0: + raise BadRequest("Invalid image.") + + # Compute scale and resize + scale = max(target_w / src_w, target_h / src_h) + new_w, new_h = int(round(src_w * scale)), int(round(src_h * scale)) + if (new_w, new_h) != (src_w, src_h): + im = im.resize((new_w, new_h), Image.Resampling.LANCZOS) + + # Center-crop to target size + left = max(0, (im.width - target_w) // 2) + top = max(0, (im.height - target_h) // 2) + im = im.crop((left, top, left + target_w, target_h + top)) + + # Convert to RGB(A) + if im.mode not in ("RGB", "RGBA"): + im = im.convert("RGBA" if "A" in im.getbands() else "RGB") + + im.save(dst_path, format="PNG", optimize=True, compress_level=9) + + logger.info(f"Wrote {kind} → {dst_path}") + return f"{url_prefix}/{out_name}" + +def delete_art_file_if_owned(public_path: str, kind: str) -> None: + """ + Delete a banner or icon file if the given public URL points inside our managed upload area. + + Args: + public_path: The public URL of the art file (banner/icon) to delete. + kind: Either "banner" or "icon" (determines directory/prefix rules). + """ + if not public_path: + return + + # Determine the correct URL prefix and directory + if kind == "banner": + prefix = current_app.config['BANNERS_UPLOAD_URL_PREFIX'].rstrip('/') + '/' + upload_dir = current_app.config['BANNERS_UPLOAD_DIR'] + elif kind == "icon": + prefix = (current_app.config.get('ICONS_UPLOAD_URL_PREFIX') + or current_app.config['BANNERS_UPLOAD_URL_PREFIX']).rstrip('/') + '/' + upload_dir = current_app.config.get('ICONS_UPLOAD_DIR') or current_app.config['BANNERS_UPLOAD_DIR'] + else: + raise ValueError(f"Unknown kind: {kind}. Expected 'banner' or 'icon'.") + + # Only delete if the file lives inside our configured prefix + if not public_path.startswith(prefix): + return + + rel_name = public_path[len(prefix):] + file_path = os.path.join(upload_dir, rel_name) + + if os.path.exists(file_path): + try: + os.remove(file_path) + except OSError: + pass + +def garbage_collect_orphan_art_files(): + """Remove banner/icon files on disk with no matching override record.""" + from db import AppOverrides # local import to avoid cycles + existing = {ov.banner_path for ov in AppOverrides.query if ov.banner_path} \ + | {ov.icon_path for ov in AppOverrides.query if ov.icon_path} + + logger.info("Garbage collection started.") + for kind, dir_key, url_key in ( + ("banner", "BANNERS_UPLOAD_DIR", "BANNERS_UPLOAD_URL_PREFIX"), + ("icon", "ICONS_UPLOAD_DIR", "ICONS_UPLOAD_URL_PREFIX"), + ): + upload_dir = current_app.config.get(dir_key) or current_app.config["BANNERS_UPLOAD_DIR"] + if not os.path.isdir(upload_dir): + continue + logger.info(f"Scanning {kind } directory: {upload_dir}... {len(existing)} images found.") + for name in os.listdir(upload_dir): + if not name.endswith((".png", ".jpg", ".jpeg", ".webp")): + continue + path = os.path.join(upload_dir, name) + url_prefix = (current_app.config.get(url_key) or current_app.config["BANNERS_UPLOAD_URL_PREFIX"]).rstrip("/") + public = f"{url_prefix}/{name}" + if public not in existing: + logger.info(f"Removing unused {kind} image: {path}.") + try: os.remove(path) + except OSError: pass + logger.info("Garbage collection completed.") + +def _allowed_image(filename: str) -> bool: + if not filename: + return False + _, ext = os.path.splitext(filename.lower()) + return ext in ALLOWED_IMAGE_EXTS + +def _ext_for_content_type(content_type: str) -> str: + # Best-effort fallback if the filename extension is missing/untrusted + if content_type == 'image/jpeg': + return '.jpg' + if content_type == 'image/png': + return '.png' + if content_type == 'image/webp': + return '.webp' + return '' diff --git a/app/library.py b/app/library.py index 0f6a0c6d..669ea4f8 100644 --- a/app/library.py +++ b/app/library.py @@ -1,14 +1,27 @@ +import datetime import hashlib import os import shutil +import unicodedata +from collections import defaultdict +from typing import Dict, Optional + +from sqlalchemy import or_ + +from cache import compute_library_apps_hash, is_library_snapshot_current from constants import * from db import * +from overrides import build_override_index +from settings import load_settings import titles as titles_lib -import datetime -from pathlib import Path from utils import * -from settings import load_settings -from db import update_file_path # Import update_file_path + +def _normalize_sort_text(value): + if value is None: + return "" + text = unicodedata.normalize("NFKD", str(value)) + stripped = "".join(ch for ch in text if not unicodedata.combining(ch)) + return stripped.casefold() def organize_file(file_obj, library_path, organizer_settings, watcher): try: @@ -114,7 +127,6 @@ def _get_template_for_file(file_obj, app, templates): return templates.get(template_key) + '.{extension}' - def add_library_complete(app, watcher, path): """Add a library to settings, database, and watchdog""" from settings import add_library_path_to_settings @@ -194,45 +206,96 @@ def init_libraries(app, watcher, paths): watcher.add_directory(path) def add_files_to_library(library, files): - if isinstance(library, int) or library.isdigit(): - library_id = library + """ + Upsert files into the Files table. + - Always inserts a row even when file_info is None. + - Marks unidentified files via identification_type="filename". + - Updates existing rows in place. + - Commits in batches of 100. + """ + nb_to_identify = len(files) + + # Resolve library_id/path + if isinstance(library, int) or (isinstance(library, str) and library.isdigit()): + library_id = int(library) library_path = get_library_path(library_id) else: library_path = library library_id = get_library_id(library_path) library_path = get_library_path(library_id) - - # Get existing file paths in the library - filepaths_in_db = get_library_file_paths(library_id) - - # Filter out files that are already in the database - new_files_to_add = [f for f in files if f not in filepaths_in_db] - - if not new_files_to_add: - return + for n, filepath in enumerate(files): + file_rel = filepath.replace(library_path, "") + logger.info(f'Getting file info ({n+1}/{nb_to_identify}): {file_rel}') - nb_to_identify = len(new_files_to_add) - for n, filepath in enumerate(new_files_to_add): - file = filepath.replace(library_path, "") - logger.info(f'Getting file info ({n+1}/{nb_to_identify}): {file}') - - file_info = titles_lib.get_file_info(filepath) - - if file_info is None: - logger.error(f'Failed to get info for file: {file} - file will be skipped.') - # in the future save identification error to be displayed and inspected in the UI - continue - - new_file = Files( - filepath = filepath, - library_id = library_id, - folder = file_info["filedir"], - filename = file_info["filename"], - extension = file_info["extension"], - size = file_info["size"], - ) - db.session.add(new_file) + # Basic FS info + filename = os.path.basename(filepath) + extension = os.path.splitext(filename)[1].lstrip('.').lower() + try: + size = os.path.getsize(filepath) + except Exception: + size = None + + # Best-effort metadata probe (non-final) + try: + file_info = titles_lib.get_file_info(filepath) + except Exception as e: + logger.exception(f"Error getting info for file {file_rel}: {e}") + file_info = None + + # Upsert by unique filepath + existing = Files.query.filter_by(filepath=filepath).first() + if existing: + # Refresh core info (prefer parsed values when present) + existing.folder = (file_info.get("filedir") if file_info else os.path.dirname(filepath)) + existing.filename = (file_info.get("filename") if file_info else filename) + existing.extension = (file_info.get("extension") if file_info else extension) + existing.size = (file_info.get("size") if (file_info and "size" in file_info) else size) + + # STAGE for deep identification (don't finalize here) + if file_info is None: + existing.identified = False + existing.identification_type = "unidentified" + existing.identification_error = "Failed to parse file info" + else: + existing.identified = False + existing.identification_type = "filename" + existing.identification_error = None + + existing.identification_attempts = (existing.identification_attempts or 0) + 1 + existing.last_attempt = datetime.datetime.utcnow() + + else: + # Insert new staged row + if file_info is None: + new_file = Files( + filepath=filepath, + library_id=library_id, + folder=os.path.dirname(filepath), + filename=filename, + extension=extension, + size=size, + identified=False, + identification_type="unidentified", + identification_error="Failed to parse file info", + identification_attempts=1, + last_attempt=datetime.datetime.utcnow(), + ) + else: + new_file = Files( + filepath=filepath, + library_id=library_id, + folder=file_info.get("filedir", os.path.dirname(filepath)), + filename=file_info.get("filename", filename), + extension=file_info.get("extension", extension), + size=file_info.get("size", size), + identified=False, # <- STAGED, not final + identification_type="filename", # <- STAGED marker + identification_error=None, + identification_attempts=1, + last_attempt=datetime.datetime.utcnow(), + ) + db.session.add(new_file) # Commit every 100 files to avoid excessive memory use if (n + 1) % 100 == 0: @@ -247,182 +310,484 @@ def scan_library_path(library_path): if not os.path.isdir(library_path): logger.warning(f'Library path {library_path} does not exists.') return - _, files = titles_lib.getDirsAndFiles(library_path) + _, files = titles_lib.get_dirs_and_files(library_path) filepaths_in_library = get_library_file_paths(library_id) new_files = [f for f in files if f not in filepaths_in_library] add_files_to_library(library_id, new_files) set_library_scan_time(library_id) -def get_files_to_identify(library_id): - non_identified_files = get_all_non_identified_files_from_library(library_id) - if titles_lib.Keys.keys_loaded: - files_to_identify_with_cnmt = get_files_with_identification_from_library(library_id, 'filename') - non_identified_files = list(set(non_identified_files).union(files_to_identify_with_cnmt)) - return non_identified_files +def get_files_to_identify(library_id, *, force_all: bool = False): + q = ( + Files.query + .filter(Files.library_id == library_id) + .options(db.joinedload(Files.apps).joinedload(Apps.override)) + ) + + if force_all: + return q.order_by(Files.last_attempt.asc().nullsfirst()).all() + + staged = ("filename", "titles_lib", "not_in_titledb") # staged markers + seven_days_ago = datetime.datetime.utcnow() - datetime.timedelta(days=7) + + q = q.filter( + or_( + Files.identified.is_(False), # not yet identified + Files.identification_type.in_(staged), # staged by first pass + Files.last_attempt.is_(None), # never attempted + Files.last_attempt < seven_days_ago, # stale + ) + ) + + candidates = q.order_by(Files.last_attempt.asc().nullsfirst()).all() + + filtered = [] + for file in candidates: + ident_type = (getattr(file, "identification_type", "") or "").lower() + if ident_type == "not_in_titledb": + app_types = { + (getattr(app, "app_type", "") or "").upper() + for app in getattr(file, "apps", []) + } + if APP_TYPE_UPD in app_types: + continue + has_override = any( + getattr(app, "override", None) + and getattr(app.override, "enabled", True) + for app in getattr(file, "apps", []) + ) + if has_override: + continue + filtered.append(file) + + return filtered def identify_library_files(library): - if isinstance(library, int) or library.isdigit(): - library_id = library + # Resolve library_id / path + if isinstance(library, int) or (isinstance(library, str) and library.isdigit()): + library_id = int(library) library_path = get_library_path(library_id) else: library_path = library library_id = get_library_id(library_path) + files_to_identify = get_files_to_identify(library_id) + not_in_titledb_count = sum( + 1 for f in files_to_identify + if (getattr(f, "identification_type", "") or "").lower() == "not_in_titledb" + ) + + if not_in_titledb_count: + logger.info( + "Re-identifying %s not-in-TitleDB file(s) for library %s to refresh overrides.", + not_in_titledb_count, + library_path, + ) nb_to_identify = len(files_to_identify) - for n, file in enumerate(files_to_identify): - try: - file_id = file.id - filepath = file.filepath - filename = file.filename - if not os.path.exists(filepath): - logger.warning(f'Identifying file ({n+1}/{nb_to_identify}): {filename} no longer exists, deleting from database.') - Files.query.filter_by(id=file_id).delete(synchronize_session=False) - continue + # Load TitleDB once so we can check presence of title_ids quickly + with titles_lib.titledb_session("identify_library_files"): + name_lookup_cache: Dict[str, Optional[str]] = {} + for n, file in enumerate(files_to_identify): + try: + file_id = file.id + filepath = file.filepath + filename = file.filename + + if not os.path.exists(filepath): + logger.warning(f'Identifying file ({n+1}/{nb_to_identify}): {filename} no longer exists, deleting from database.') + Files.query.filter_by(id=file_id).delete(synchronize_session=False) + continue + + logger.info(f'Identifying file ({n+1}/{nb_to_identify}): {filename}') + identification, success, file_contents, error = titles_lib.identify_file(filepath) + + if success and file_contents and not error: + # Unique title_ids present in this file + title_ids = list(dict.fromkeys([c['title_id'] for c in file_contents])) + + # Ensure Titles table has those IDs + for title_id in title_ids: + add_title_id_in_db(title_id) + + file_basename = os.path.splitext(filename)[0] + clean_display_name = titles_lib.clean_display_name(file_basename) + normalized_display_name = titles_lib.normalize_display_name(file_basename) + + nb_content = 0 + auto_override_candidates = [] + for file_content in file_contents: + logger.info( + f'Identifying file ({n+1}/{nb_to_identify}) - ' + f'Found content Title ID: {file_content["title_id"]} ' + f'App ID : {file_content["app_id"]} ' + f'Title Type: {file_content["type"]} ' + f'Version: {file_content["version"]}' + ) + # Upsert Apps and attach file via M2M + title_id_in_db = get_title_id_db_id(file_content["title_id"]) + + # Idempotent get-or-create, then link file and flip owned=True + if file_content["type"] == APP_TYPE_BASE: + app_row, _ = _ensure_single_latest_base( + app_id=file_content["app_id"], + detected_version=file_content["version"], + title_db_id=title_id_in_db + ) + else: + app_row, _ = _get_or_create_app( + app_id=file_content["app_id"], + app_version=file_content["version"], + app_type=file_content["type"], + title_db_id=title_id_in_db + ) + + # Ensure a newly-added row is visible to the session query used by add_file_to_app + db.session.flush() + linked = add_file_to_app(file_content["app_id"], file_content["version"], file_id, commit=False) + if not linked: + file_obj = get_file_from_db(file_id) + if file_obj and file_obj not in app_row.files: + app_row.files.append(file_obj) + app_row.owned = True + + auto_override_candidates.append((app_row, file_content)) + nb_content += 1 + + # Update multi-content flags + file.multicontent = nb_content > 1 + file.nb_content = nb_content + + for app_row, file_content in auto_override_candidates: + _auto_create_metadata_override( + app_row=app_row, + file_content=file_content, + clean_display_name=clean_display_name, + normalized_display_name=normalized_display_name, + name_lookup_cache=name_lookup_cache, + ) - logger.info(f'Identifying file ({n+1}/{nb_to_identify}): {filename}') - identification, success, file_contents, error = titles_lib.identify_file(filepath) - if success and file_contents and not error: - # find all unique Titles ID to add to the Titles db - title_ids = list(dict.fromkeys([c['title_id'] for c in file_contents])) + # Determine if any of this file's title_ids are unknown to TitleDB + needs_override = any(_title_metadata_missing(tid) for tid in title_ids) + + # - identified=True means "we parsed/understood the file (CNMT etc.)" + # - recognition in TitleDB is tracked via identification_type + file.identified = True + file.identification_type = "not_in_titledb" if needs_override else identification # e.g., 'cnmt' + file.identification_error = None # not an error condition + + else: + # Failed to identify contents + logger.warning(f"Error identifying file {filename}: {error}") + file.identification_error = error + file.identified = False + if not getattr(file, "identification_type", None): + file.identification_type = "exception" + + except Exception as e: + logger.warning(f"Error identifying file {getattr(file, 'filename', '')}: {e}") + file.identification_error = str(e) + file.identified = False + # keep identification_type as-is if set earlier; otherwise mark generic + if not getattr(file, "identification_type", None): + file.identification_type = "exception" + + # finally update attempts/time + file.identification_attempts = (file.identification_attempts or 0) + 1 + file.last_attempt = datetime.datetime.utcnow() + + # Commit every 100 files to avoid excessive memory use + if (n + 1) % 100 == 0: + db.session.commit() + + # Final commit for the batch + db.session.commit() + + if not_in_titledb_count: + logger.info( + "Finished re-identifying %s not-in-TitleDB file(s) for library %s.", + not_in_titledb_count, + library_path, + ) - for title_id in title_ids: - add_title_id_in_db(title_id) +def _lookup_title_id_by_normalized_name(normalized_name: str, cache: Dict[str, Optional[str]]) -> Optional[str]: + """ + Resolve a normalized display name to a Title ID using cached lookups. + """ + if not normalized_name: + return None + key = normalized_name.strip().upper() + if not key: + return None + if key in cache: + return cache[key] + match = titles_lib.find_title_id_by_normalized_name(key) + cache[key] = match + return match - nb_content = 0 - for file_content in file_contents: - logger.info(f'Identifying file ({n+1}/{nb_to_identify}) - Found content Title ID: {file_content["title_id"]} App ID : {file_content["app_id"]} Title Type: {file_content["type"]} Version: {file_content["version"]}') - # now add the content to Apps - title_id_in_db = get_title_id_db_id(file_content["title_id"]) - - # Check if app already exists - existing_app = get_app_by_id_and_version( - file_content["app_id"], - file_content["version"] - ) - - if existing_app: - # Add file to existing app using many-to-many relationship - add_file_to_app(file_content["app_id"], file_content["version"], file_id) - else: - # Create new app entry and add file using many-to-many relationship - new_app = Apps( - app_id=file_content["app_id"], - app_version=file_content["version"], - app_type=file_content["type"], - owned=True, - title_id=title_id_in_db - ) - db.session.add(new_app) - db.session.flush() # Flush to get the app ID - - # Add the file to the new app - file_obj = get_file_from_db(file_id) - if file_obj: - new_app.files.append(file_obj) - - nb_content += 1 +def _title_metadata_missing(title_id: Optional[str]) -> bool: + """ + True when TitleDB lacks a usable entry for the given Title ID. + """ + tid = normalize_id(title_id, "title") + if not tid: + return True + if not titles_lib.title_id_exists(tid): + return True + info = titles_lib.get_game_info(tid) or {} + existing_name = (info.get("name") or "").strip().lower() + return not existing_name or existing_name in {"unrecognized", "unidentified"} + +def _auto_create_metadata_override( + *, + app_row, + file_content: dict, + clean_display_name: str, + normalized_display_name: str, + name_lookup_cache: Dict[str, Optional[str]], +) -> None: + """ + Ensure an override exists when TitleDB lacks the detected Title ID. + """ + if not app_row or not getattr(app_row, "id", None): + return - if nb_content > 1: - file.multicontent = True - file.nb_content = nb_content - file.identified = True - else: - logger.warning(f"Error identifying file {filename}: {error}") - file.identification_error = error - file.identified = False + # Skip non-base/DLC entries; overrides are tied to those families. + if getattr(app_row, "app_type", None) not in (APP_TYPE_BASE, APP_TYPE_DLC): + return - file.identification_type = identification + if AppOverrides.query.filter_by(app_fk=app_row.id).first(): + return - except Exception as e: - logger.warning(f"Error identifying file {filename}: {e}") - file.identification_error = str(e) - file.identified = False + title_id = normalize_id(file_content.get("title_id"), "title") + if not title_id: + return - # and finally update the File with identification info - file.identification_attempts += 1 - file.last_attempt = datetime.datetime.now() + if not _title_metadata_missing(title_id): + return - # Commit every 100 files to avoid excessive memory use - if (n + 1) % 100 == 0: - db.session.commit() + corrected_title_id = _lookup_title_id_by_normalized_name(normalized_display_name, name_lookup_cache) + if corrected_title_id: + corrected_title_id = normalize_id(corrected_title_id, "title") + if not corrected_title_id or corrected_title_id == title_id: + return + _create_override(app_row, corrected_title_id=corrected_title_id) + logger.info( + "Auto-created redirect override for app %s → TitleID %s", + app_row.app_id, + corrected_title_id, + ) + return - # Final commit - db.session.commit() + clean_name = clean_display_name.strip() if clean_display_name else "" + if not clean_name: + return + + _create_override(app_row, name=clean_name) + logger.info( + "Auto-created name override for app %s with name '%s'", + app_row.app_id, + clean_name, + ) + +def _create_override(app_row, *, corrected_title_id: Optional[str] = None, name: Optional[str] = None) -> None: + """ + Persist a new AppOverrides row attached to the provided app. + """ + ov = AppOverrides(app=app_row) + ov.enabled = True + ov.created_at = datetime.datetime.utcnow() + ov.updated_at = datetime.datetime.utcnow() + if corrected_title_id: + ov.corrected_title_id = corrected_title_id + elif name: + ov.name = name + db.session.add(ov) def add_missing_apps_to_db(): logger.info('Adding missing apps to database...') - titles = get_all_titles() apps_added = 0 - - for n, title in enumerate(titles): - title_id = title.title_id - title_db_id = get_title_id_db_id(title_id) - - # Add base game if not present - existing_base = get_app_by_id_and_version(title_id, "0") - - if not existing_base: - new_base_app = Apps( - app_id=title_id, + commit_every = 250 + + def _chunked(sequence, size): + for idx in range(0, len(sequence), size): + yield sequence[idx:idx + size] + + def _map_title_ids(title_ids): + if not title_ids: + return {} + mapping = {} + normalized = [tid.upper() for tid in set(title_ids) if isinstance(tid, str)] + chunk_size = 900 # stay below sqlite variable limit + for chunk in _chunked(normalized, chunk_size): + rows = ( + db.session.query(Titles.title_id, Titles.id) + .filter(Titles.title_id.in_(chunk)) + .all() + ) + for title_id, db_id in rows: + if title_id: + mapping[title_id.upper()] = db_id + return mapping + + def _ensure_missing_base_apps(): + added = 0 + pending_commits = 0 + missing_base_rows = ( + db.session.query(Titles.id, Titles.title_id) + .filter(~Titles.apps.any(Apps.app_type == APP_TYPE_BASE)) + .all() + ) + for title_db_id, title_id in missing_base_rows: + if not title_id: + continue + _, created = _get_or_create_app( + app_id=title_id.upper(), app_version="0", app_type=APP_TYPE_BASE, - owned=False, - title_id=title_db_id + title_db_id=title_db_id ) - db.session.add(new_base_app) - apps_added += 1 - logger.debug(f'Added missing base app: {title_id}') - - # Add missing update versions - title_versions = titles_lib.get_all_existing_versions(title_id) - for version_info in title_versions: - version = str(version_info['version']) - update_app_id = title_id[:-3] + '800' # Convert base ID to update ID - - existing_update = get_app_by_id_and_version(update_app_id, version) - - if not existing_update: - new_update_app = Apps( + if created: + added += 1 + pending_commits += 1 + logger.debug(f'Added missing base app placeholder v0: {title_id}') + if pending_commits >= commit_every: + db.session.commit() + pending_commits = 0 + if pending_commits: + db.session.commit() + return added + + def _ensure_missing_update_apps(): + versions_db = getattr(titles_lib, "_versions_db", {}) or {} + if not versions_db: + return 0 + + title_map = _map_title_ids([tid.upper() for tid in versions_db.keys()]) + if not title_map: + return 0 + + existing_updates = defaultdict(set) + for app_id, app_version in ( + db.session.query(Apps.app_id, Apps.app_version) + .filter(Apps.app_type == APP_TYPE_UPD) + ): + if app_id: + existing_updates[app_id.upper()].add(str(app_version)) + + added = 0 + pending_commits = 0 + for title_lower, version_entries in versions_db.items(): + title_id = title_lower.upper() + if len(title_id) != 16: + continue + title_db_id = title_map.get(title_id) + if not title_db_id: + continue + + update_app_id = (title_id[:-3] + '800').upper() + known_versions = existing_updates.setdefault(update_app_id, set()) + + for version_key in version_entries.keys(): + version_str = str(version_key) + if version_str in known_versions: + continue + _, created = _get_or_create_app( app_id=update_app_id, - app_version=version, + app_version=version_str, app_type=APP_TYPE_UPD, - owned=False, - title_id=title_db_id + title_db_id=title_db_id ) - db.session.add(new_update_app) - apps_added += 1 - logger.debug(f'Added missing update app: {update_app_id} v{version}') - - # Add missing DLC - title_dlc_ids = titles_lib.get_all_existing_dlc(title_id) - for dlc_app_id in title_dlc_ids: - dlc_versions = titles_lib.get_all_app_existing_versions(dlc_app_id) - if dlc_versions: - for dlc_version in dlc_versions: - existing_dlc = get_app_by_id_and_version(dlc_app_id, str(dlc_version)) - - if not existing_dlc: - new_dlc_app = Apps( - app_id=dlc_app_id, - app_version=str(dlc_version), - app_type=APP_TYPE_DLC, - owned=False, - title_id=title_db_id - ) - db.session.add(new_dlc_app) - apps_added += 1 - logger.debug(f'Added missing DLC app: {dlc_app_id} v{dlc_version}') - - # Commit every 100 titles to avoid excessive memory use - if (n + 1) % 100 == 0: + if created: + known_versions.add(version_str) + added += 1 + pending_commits += 1 + logger.debug(f'Added missing update app: {update_app_id} v{version_str}') + if pending_commits >= commit_every: + db.session.commit() + pending_commits = 0 + if pending_commits: db.session.commit() - logger.info(f'Processed {n + 1}/{len(titles)} titles, added {apps_added} missing apps so far') - - # Final commit - db.session.commit() - logger.info(f'Finished adding missing apps to database. Total apps added: {apps_added}') + return added + + def _ensure_missing_dlc_apps(): + cnmts_db = getattr(titles_lib, "_cnmts_db", {}) or {} + if not cnmts_db: + return 0 + + dlc_index = defaultdict(lambda: defaultdict(set)) + for app_id_lower, version_map in cnmts_db.items(): + app_id = app_id_lower.upper() + for version_key, metadata in version_map.items(): + if not isinstance(metadata, dict): + continue + if metadata.get('titleType') != 130: + continue + base_tid = metadata.get('otherApplicationId') + if base_tid: + base_tid = base_tid.upper() + else: + try: + base_tid = titles_lib.get_title_id_from_app_id(app_id, APP_TYPE_DLC) + except Exception: + base_tid = None + if base_tid: + base_tid = base_tid.upper() + if not base_tid or len(base_tid) != 16: + continue + dlc_index[base_tid][app_id].add(str(version_key)) + + if not dlc_index: + return 0 + + title_map = _map_title_ids(dlc_index.keys()) + if not title_map: + return 0 + + existing_dlcs = defaultdict(set) + for app_id, app_version in ( + db.session.query(Apps.app_id, Apps.app_version) + .filter(Apps.app_type == APP_TYPE_DLC) + ): + if app_id: + existing_dlcs[app_id.upper()].add(str(app_version)) + + added = 0 + pending_commits = 0 + for base_title_id, dlc_apps in dlc_index.items(): + title_db_id = title_map.get(base_title_id) + if not title_db_id: + continue + for dlc_app_id, versions in dlc_apps.items(): + dlc_app_id_upper = dlc_app_id.upper() + known_versions = existing_dlcs.setdefault(dlc_app_id_upper, set()) + for version_str in versions: + version_str = str(version_str) + if version_str in known_versions: + continue + _, created = _get_or_create_app( + app_id=dlc_app_id_upper, + app_version=version_str, + app_type=APP_TYPE_DLC, + title_db_id=title_db_id + ) + if created: + known_versions.add(version_str) + added += 1 + pending_commits += 1 + logger.debug(f'Added missing DLC app: {dlc_app_id_upper} v{version_str}') + if pending_commits >= commit_every: + db.session.commit() + pending_commits = 0 + if pending_commits: + db.session.commit() + return added + + with titles_lib.titledb_session("add_missing_apps_to_db"): + apps_added += _ensure_missing_base_apps() + apps_added += _ensure_missing_update_apps() + apps_added += _ensure_missing_dlc_apps() + logger.info(f'Finished adding missing apps. Total apps added: {apps_added}') def process_library_identification(app): logger.info(f"Starting library identification process for all libraries...") @@ -547,9 +912,9 @@ def update_titles(): have_base = len(owned_base_apps) > 0 # check up_to_date - find highest owned update version - owned_update_apps = [app for app in title_apps if app.get('app_type') == APP_TYPE_UPD and app.get('owned')] - available_update_apps = [app for app in title_apps if app.get('app_type') == APP_TYPE_UPD] - + available_update_apps = [a for a in title_apps if a.get('app_type') == APP_TYPE_UPD] + owned_update_apps = [a for a in available_update_apps if a.get('owned')] + if not available_update_apps: # No updates available, consider up to date up_to_date = True @@ -558,8 +923,8 @@ def update_titles(): up_to_date = False else: # Find highest available version and highest owned version - highest_available_version = max(int(app['app_version']) for app in available_update_apps) - highest_owned_version = max(int(app['app_version']) for app in owned_update_apps) + highest_available_version = max(int(app['app_version'] or 0) for app in available_update_apps) + highest_owned_version = max(int(app['app_version'] or 0) for app in owned_update_apps) up_to_date = highest_owned_version >= highest_available_version # check complete - latest version of all available DLC are owned @@ -593,183 +958,532 @@ def update_titles(): db.session.commit() -def get_library_status(title_id): - title = get_title(title_id) - title_apps = get_all_title_apps(title_id) +def generate_library_snapshot(): + """ + Public entry-point for routes: + - Load library from disk if unchanged, + - Add a strong ETag derived from the snapshot's content identifiers (apps hash + TitleDB commit), + - Return the list used by the API layer. + """ + # Load library from disk or regenerate if hash changed + saved = load_or_generate_library_snapshot() # {'hash': ..., 'titledb_commit': ..., 'library': [...]} + if not saved: + empty_etag = hashlib.sha256(b":").hexdigest() + return [], empty_etag + + library = saved.get('library') or [] + payload_hash = saved.get('hash') or "" + titledb_commit = saved.get('titledb_commit') or "" + etag_source = f"{payload_hash}:{titledb_commit}".encode("utf-8") + etag = hashlib.sha256(etag_source).hexdigest() + return library, etag + +def load_or_generate_library_snapshot(): + """ + Load the BASE library (no overrides) from disk if hash unchanged. + Otherwise, regenerate and save. + """ + saved = load_json(LIBRARY_CACHE_FILE) + if saved and is_library_snapshot_current(saved): + return saved - available_versions = titles_lib.get_all_existing_versions(title_id) - for version in available_versions: - if len(list(filter(lambda x: x.get('app_type') == APP_TYPE_UPD and str(x.get('app_version')) == str(version['version']), title_apps))): - version['owned'] = True - else: - version['owned'] = False + # Hash changed or cache missing/corrupt -> regenerate + return _generate_library_snapshot() - library_status = { - 'has_base': title.have_base, - 'has_latest_version': title.up_to_date, - 'version': available_versions, - 'has_all_dlcs': title.complete - } - return library_status +def _generate_library_snapshot(): + """Generate the BASE/DLC library from Apps table and cache to disk.""" + logger.info('Generating library snapshot...') -def compute_apps_hash(): - """ - Computes a hash of all Apps table content to detect changes in library state. - """ - hash_md5 = hashlib.md5() - apps = get_all_apps() - - # Sort apps with safe handling of None values - for app in sorted(apps, key=lambda x: (x['app_id'] or '', x['app_version'] or '')): - hash_md5.update((app['app_id'] or '').encode()) - hash_md5.update((app['app_version'] or '').encode()) - hash_md5.update((app['app_type'] or '').encode()) - hash_md5.update(str(app['owned'] or False).encode()) - hash_md5.update((app['title_id'] or '').encode()) - return hash_md5.hexdigest() - -def is_library_unchanged(): - cache_path = Path(LIBRARY_CACHE_FILE) - if not cache_path.exists(): - return False - - saved_library = load_library_from_disk() - if not saved_library: - return False - - if not saved_library.get('hash'): - return False - - current_hash = compute_apps_hash() - return saved_library['hash'] == current_hash - -def save_library_to_disk(library_data): - cache_path = Path(LIBRARY_CACHE_FILE) - # Ensure cache directory exists - cache_path.parent.mkdir(parents=True, exist_ok=True) - safe_write_json(cache_path, library_data) - -def load_library_from_disk(): - cache_path = Path(LIBRARY_CACHE_FILE) - if not cache_path.exists(): - return None + with titles_lib.titledb_session("generate_library"): + titles = get_all_apps(include_files=True) + games_info = [] + processed_dlc_apps = set() # Track processed DLC app_ids to avoid duplicates - try: - with cache_path.open("r", encoding="utf-8") as f: - return json.load(f) - except: - return None + for title in titles: + has_none_value = any(value is None for value in title.values()) + if has_none_value: + logger.warning(f'File contains None value, it will be skipped: {title}') + continue + if title['app_type'] == APP_TYPE_UPD: + continue -def generate_library(): - """Generate the game library from Apps table, using cached version if unchanged""" - if is_library_unchanged(): - saved_library = load_library_from_disk() - if saved_library: - return saved_library['library'] - - logger.info(f'Generating library ...') - titles_lib.load_titledb() - titles = get_all_apps() - games_info = [] - processed_dlc_apps = set() # Track processed DLC app_ids to avoid duplicates - - for title in titles: - has_none_value = any(value is None for value in title.values()) - if has_none_value: - logger.warning(f'File contains None value, it will be skipped: {title}') - continue - if title['app_type'] == APP_TYPE_UPD: - continue - - # Get title info from titledb - info_from_titledb = titles_lib.get_game_info(title['app_id']) - if info_from_titledb is None: - logger.warning(f'Info not found for game: {title}') - continue - title.update(info_from_titledb) - - if title['app_type'] == APP_TYPE_BASE: - # Get title status from Titles table (already calculated by update_titles) - title_obj = get_title(title['title_id']) - if title_obj: - title['has_base'] = title_obj.have_base - title['has_latest_version'] = title_obj.up_to_date - title['has_all_dlcs'] = title_obj.complete + # Use DLC app_id for DLC metadata; BASE keeps family/base title_id. + lookup_id = title['app_id'] if title['app_type'] == APP_TYPE_DLC else title['title_id'] + info_from_titledb = titles_lib.get_game_info(lookup_id) + if info_from_titledb is None: + logger.warning(f'Info not found for game: {title}') + continue + metadata_missing = _title_metadata_missing(lookup_id) + + title.update(info_from_titledb) + title['has_title_db'] = not metadata_missing + title['is_unrecognized'] = metadata_missing + + # Stable sort/display key: + # - BASE: use its own (family) name + # - DLC : use the family/base title name (so DLCs sort alongside their bases) + if title['app_type'] == APP_TYPE_DLC: + family_info = titles_lib.get_game_info(title['title_id']) # family/base lookup + if family_info: + # Use the family's artwork when this DLC lacks its own assets so cards don’t show the gray placeholder. + if not (title.get("bannerUrl") or title.get("banner_path")): + fallback_banner = family_info.get("bannerUrl") + if fallback_banner: + title["bannerUrl"] = fallback_banner + if not (title.get("iconUrl") or title.get("icon_path")): + fallback_icon = family_info.get("iconUrl") + if fallback_icon: + title["iconUrl"] = fallback_icon + family_name = (family_info or {}).get('name') or title.get('name') + title['title_id_name'] = family_name or 'Unrecognized' else: - title['has_base'] = False - title['has_latest_version'] = False - title['has_all_dlcs'] = False - - # Get version info from Apps table and add release dates from versions_db - title_apps = get_all_title_apps(title['title_id']) - update_apps = [app for app in title_apps if app.get('app_type') == APP_TYPE_UPD] - - # Get release date information from external source - available_versions = titles_lib.get_all_existing_versions(title['title_id']) - version_release_dates = {v['version']: v['release_date'] for v in available_versions} - - version_list = [] - for update_app in update_apps: - app_version = int(update_app['app_version']) - version_list.append({ - 'version': app_version, - 'owned': update_app.get('owned', False), - 'release_date': version_release_dates.get(app_version, 'Unknown') - }) - - title['version'] = sorted(version_list, key=lambda x: x['version']) - title['title_id_name'] = title['name'] - - elif title['app_type'] == APP_TYPE_DLC: - # Skip if we've already processed this DLC app_id - if title['app_id'] in processed_dlc_apps: + title['title_id_name'] = title.get('name') or 'Unrecognized' + + if title['app_type'] == APP_TYPE_BASE: + # Status flags from Titles table (computed by update_titles) + title_obj = get_title(title['title_id']) + if title_obj: + title['has_base'] = title_obj.have_base + # Only mark as up to date if the base itself is owned and up_to_date + title['has_latest_version'] = (title_obj.have_base and title_obj.up_to_date) + title['has_all_dlcs'] = title_obj.complete + else: + title['has_base'] = False + title['has_latest_version'] = False + title['has_all_dlcs'] = False + + # Version list for BASE using Apps + versions DB release dates + title_apps = get_all_title_apps(title['title_id']) + update_apps = [a for a in title_apps if a.get('app_type') == APP_TYPE_UPD] + + available_versions = titles_lib.get_all_existing_versions(title['title_id']) + version_release_dates = {v['version']: v['release_date'] for v in available_versions} + + version_list = [] + for update_app in update_apps: + app_version = int(update_app['app_version']) + rd = version_release_dates.get(app_version) + version_list.append({ + 'version': app_version, + 'owned': update_app.get('owned', False), + 'release_date': rd.isoformat() if isinstance(rd, (datetime.datetime, datetime.date)) else rd + }) + + title['version'] = sorted(version_list, key=lambda x: x['version']) + + elif title['app_type'] == APP_TYPE_DLC: + # Skip if we've already processed this DLC app_id + app_id = title['app_id'] + if app_id in processed_dlc_apps: + continue + processed_dlc_apps.add(app_id) + + # Get all versions for this DLC app_id + title_apps = get_all_title_apps(title['title_id']) + dlc_apps = [app for app in title_apps if app.get('app_type') == APP_TYPE_DLC and app['app_id'] == app_id] + + # Create version list for this DLC + version_list = [] + for dlc_app in dlc_apps: + version_list.append({ + 'version': int(dlc_app['app_version']), + 'owned': dlc_app.get('owned', False), + 'release_date': 'Unknown' # DLC release dates not available in versions_db + }) + + title['version'] = sorted(version_list, key=lambda x: x['version']) + title['owned'] = any(app.get('owned') for app in dlc_apps) + + # Check if this DLC has latest version + if dlc_apps: + highest_version = max(int(app['app_version']) for app in dlc_apps) + owned_versions = [int(app['app_version']) for app in dlc_apps if app.get('owned')] + # Only true if at least one version is OWNED and the highest owned >= highest available + title['has_latest_version'] = len(owned_versions) > 0 and max(owned_versions) >= highest_version + else: + # No local rows → nothing to update + title['has_latest_version'] = True + + # File basename hint for organizer/UI + title['file_basename'] = _best_file_basename(title.get('files')) + # We don't need to send the full files payload to the client cache, and it can carry datetimes which are not serializable + title.pop('files', None) + + games_info.append(title) + + _add_files_without_apps(games_info) + + def _normalized_id(value, kind: str) -> str: + if not isinstance(value, str): + return "" + trimmed = value.strip() + if not trimmed: + return "" + try: + normalized = normalize_id(trimmed, kind) + except Exception: + normalized = None + if normalized: + return normalized.upper() + return trimmed.upper() + + override_index = build_override_index(include_disabled=False) or {} + raw_overrides = override_index.get("by_app") if isinstance(override_index, dict) else {} + overrides_by_app: dict[str, dict] = {} + if isinstance(raw_overrides, dict): + for raw_app_id, payload in raw_overrides.items(): + if not isinstance(payload, dict): + continue + normalized_app_id = _normalized_id(raw_app_id, "app") + if normalized_app_id: + overrides_by_app[normalized_app_id] = payload + + def _first_nonempty(*values): + for val in values: + if isinstance(val, str): + trimmed = val.strip() + if trimmed: + return trimmed + return None + + def _override_for_app(app_id): + key = _normalized_id(app_id, "app") + return overrides_by_app.get(key) + + base_sort_name_by_id: dict[str, str] = {} + for game in games_info: + app_type = (game.get('app_type') or '').upper() + if app_type != APP_TYPE_BASE: continue - processed_dlc_apps.add(title['app_id']) - - # Get all versions for this DLC app_id - title_apps = get_all_title_apps(title['title_id']) - dlc_apps = [app for app in title_apps if app.get('app_type') == APP_TYPE_DLC and app['app_id'] == title['app_id']] - - # Create version list for this DLC - version_list = [] - for dlc_app in dlc_apps: - app_version = int(dlc_app['app_version']) - version_list.append({ - 'version': app_version, - 'owned': dlc_app.get('owned', False), - 'release_date': 'Unknown' # DLC release dates not available in versions_db - }) - - title['version'] = sorted(version_list, key=lambda x: x['version']) - - # Check if this DLC has latest version - if dlc_apps: - highest_version = max(int(app['app_version']) for app in dlc_apps) - highest_owned_version = max((int(app['app_version']) for app in dlc_apps if app.get('owned')), default=0) - title['has_latest_version'] = highest_owned_version >= highest_version + + override = _override_for_app(game.get('app_id')) + override_name = _first_nonempty(override.get('name')) if override else None + display_name = _first_nonempty( + override_name, + game.get('title_id_name'), + game.get('name') + ) or 'Unrecognized' + + for candidate in ( + _normalized_id(game.get('title_id'), "title"), + _normalized_id(game.get('app_id'), "app"), + _normalized_id(game.get('corrected_title_id'), "title") + ): + if candidate: + base_sort_name_by_id[candidate] = display_name + + if override: + corrected = _normalized_id(override.get('corrected_title_id'), "title") + if corrected: + base_sort_name_by_id[corrected] = display_name + + def _compute_sort_tuple(game: dict) -> tuple[str, str, int, str, str]: + app_type = (game.get('app_type') or '').upper() + app_id_norm = _normalized_id(game.get('app_id'), "app") + title_id_norm = _normalized_id(game.get('title_id'), "title") + override = overrides_by_app.get(app_id_norm) + override_name = _first_nonempty(override.get('name')) if override else None + + if app_type == APP_TYPE_DLC: + base_sort_name = base_sort_name_by_id.get(title_id_norm) + if not base_sort_name and override: + corrected = _normalized_id(override.get('corrected_title_id'), "title") + if corrected: + base_sort_name = base_sort_name_by_id.get(corrected) + if not base_sort_name: + corrected = _normalized_id(game.get('corrected_title_id'), "title") + if corrected: + base_sort_name = base_sort_name_by_id.get(corrected) + + sort_name = _first_nonempty( + base_sort_name, + game.get('title_id_name'), + override_name, + game.get('name') + ) or 'Unrecognized' + base_key = title_id_norm or app_id_norm + sort_kind = 1 + elif app_type == APP_TYPE_BASE: + sort_name = base_sort_name_by_id.get(title_id_norm) or ( + _first_nonempty( + override_name, + game.get('title_id_name'), + game.get('name') + ) or 'Unrecognized' + ) + base_key = title_id_norm or app_id_norm + sort_kind = 0 else: - title['has_latest_version'] = True - - # Get title name for DLC - titleid_info = titles_lib.get_game_info(title['title_id']) - title['title_id_name'] = titleid_info['name'] if titleid_info else 'Unrecognized' - - games_info.append(title) - - library_data = { - 'hash': compute_apps_hash(), - 'library': sorted(games_info, key=lambda x: ( - "title_id_name" not in x, - x.get("title_id_name", "Unrecognized") or "Unrecognized", - x.get('app_id', "") or "" - )) - } + sort_name = _first_nonempty( + override_name, + game.get('title_id_name'), + game.get('name') + ) or 'Unrecognized' + base_key = title_id_norm or app_id_norm + sort_kind = 2 + + fallback = game.get('file_basename') or '' + return sort_name, base_key, sort_kind, app_id_norm, fallback + + def _library_sort_key(record: dict) -> tuple: + sort_name, base_key, sort_kind, app_id_norm, fallback = _compute_sort_tuple(record) + fallback_key = fallback.upper() if isinstance(fallback, str) else '' + return ( + 0 if sort_name else 1, + _normalize_sort_text(sort_name), + base_key, + sort_kind, + app_id_norm, + fallback_key, + ) + + sorted_games = sorted(games_info, key=_library_sort_key) + + library_data = { + 'hash': compute_library_apps_hash(), + 'titledb_commit': titles_lib.get_titledb_commit_hash() or "", + 'snapshot_version': LIBRARY_SNAPSHOT_VERSION, + 'library': sorted_games + } + + # Persist snapshot to disk + save_json(library_data, LIBRARY_CACHE_FILE, default=_json_default) + logger.info('Generating library snapshot done.') + return library_data - save_library_to_disk(library_data) +def _best_file_basename(files): + if not files: + return None - titles_lib.identification_in_progress_count -= 1 - titles_lib.unload_titledb() + ext_rank = {".nsz": 5, ".xcz": 5, ".nsp": 4, ".xci": 4, ".zip": 2, ".rar": 1} + strong_id = {"cnmt", "tik", "cert"} + + def _ext_score(name): + _, ext = os.path.splitext((name or "").lower()) + return ext_rank.get(ext, 0) + + def score(f): + name_for_ext = f.get("filename") or os.path.basename(f.get("filepath") or "") + return ( + 1 if f.get("identification_type") in strong_id else 0, # strong ID first + _ext_score(name_for_ext), # prefer compressed + f.get("last_attempt") or datetime.datetime.min, # recent work + f.get("created_at") or datetime.datetime.min, # stable tiebreaker + f.get("id") or 0, # deterministic + ) - logger.info(f'Generating library done.') + best = max(files, key=score) + if best.get("filename"): + return best["filename"] + return os.path.basename(best.get("filepath") or "") or None - return library_data['library'] +def _add_files_without_apps(games_info): + unid_files = Files.query.filter( + or_( + Files.identified.is_(False), + Files.identification_type.in_(["unidentified", "exception"]) + ) + ).all() + + for f in unid_files: + # If this file is already linked to an App it will already be represented; skip here + if getattr(f, "apps", None): + try: + if len(f.apps) > 0: + continue + except Exception: + # if relationship not configured to be immediately usable, just continue + pass + + fname = f.filename or (os.path.basename(f.filepath) if f.filepath else None) + + games_info.append({ + # No app or title linkage + 'name': None, + 'app_id': None, + 'app_version': None, + 'app_type': APP_TYPE_BASE, # treat as base-ish for filtering + 'title_id': None, + 'title_id_name': None, + + # Identification flags + 'identified': False, + 'identification_type': (f.identification_type or 'unidentified'), + + # What the UI needs + 'file_basename': fname, + 'filename': fname, + + # Other fields the UI expects to exist + 'owned': True, + 'has_latest_version': True, + 'has_all_dlcs': True, + 'version': [], + }) + +def _ensure_single_latest_base(app_id: str, detected_version, title_db_id=None): + """ + Guarantee a single BASE row for app_id holding the HIGHEST version. + - If none exists, create one at detected_version. + - If one exists at lower version, upgrade that row (in-place) to detected_version. + - If multiple exist, pick highest as winner, migrate files & overrides to it, delete losers. + Returns: (winner_app, created_or_upgraded: bool) + """ + Vnew = _normalize_version_int(detected_version) + + # If we weren't handed a Titles FK, resolve it from app_id (BASE app_id == TitleID) + if title_db_id is None: + try: + add_title_id_in_db(app_id) # idempotent (ensures Titles row exists) + title_db_id = get_title_id_db_id(app_id) # integer FK (or None if something’s off) + except Exception: + title_db_id = None # fail-soft; we’ll still create/upgrade the app row + + rows = Apps.query.filter_by(app_id=app_id, app_type=APP_TYPE_BASE).all() + + if not rows: + # Create brand-new BASE + winner, _ = _get_or_create_app( + app_id=app_id, + app_version=str(Vnew), + app_type=APP_TYPE_BASE, + title_db_id=title_db_id + ) + # ensure FK is correct if resolver succeeded + if title_db_id is not None: + winner.title_id = title_db_id + # sanity: make sure version is exactly Vnew + winner.app_version = str(Vnew) + db.session.flush() + return winner, True + + # If exactly one exists, upgrade in-place if needed + if len(rows) == 1: + winner = rows[0] + Vold = _normalize_version_int(winner.app_version) + if Vnew > Vold: + winner.app_version = str(Vnew) + if title_db_id is not None: + winner.title_id = title_db_id + db.session.flush() + return winner, True + else: + return winner, False + + # Multiple exist → collapse + rows_sorted = sorted(rows, key=lambda r: _normalize_version_int(r.app_version), reverse=True) + winner = rows_sorted[0] + Vwin = _normalize_version_int(winner.app_version) + + # If detected version is higher than current winner, upgrade winner in-place + if Vnew > Vwin: + winner.app_version = str(Vnew) + if title_db_id is not None: + winner.title_id = title_db_id + db.session.flush() + + # Migrate files & overrides from losers → winner + losers = rows_sorted[1:] + for loser in losers: + # move Files relationships + for f in list(loser.files): + if winner not in f.apps: + f.apps.append(winner) + # move overrides + for ov in AppOverrides.query.filter_by(app_fk=loser.id).all(): + ov.app_fk = winner.id + # delete loser + db.session.delete(loser) + + # update owned based on files presence + winner.owned = bool(getattr(winner, "files", [])) + db.session.flush() + + return winner, True + +def _get_or_create_app(app_id: str, app_version, app_type: str, title_db_id: Optional[int] = None): + """ + Get or create an Apps row by (app_id, app_version). + - app_version is normalized to str + - If row exists, fill in missing fields (title_id/app_type) but DO NOT + overwrite 'owned' or other set fields. + Returns: (app, created_bool) + """ + ver = str(app_version if app_version is not None else "0") + + # Defensive: resolve family/base Titles FK if missing + if not title_db_id: + base_tid = None + if isinstance(app_id, str): + try: + if app_type == APP_TYPE_BASE: + base_tid = app_id + elif app_type in (APP_TYPE_UPD, APP_TYPE_DLC) and len(app_id) >= 16: + base_tid = titles_lib.get_title_id_from_app_id(app_id, app_type) + except Exception: + base_tid = None + + base_tid = normalize_id(base_tid, "title") if base_tid else None + if base_tid: + try: + add_title_id_in_db(base_tid) # idempotent + except Exception as exc: + logger.debug("Failed to ensure Title row for %s: %s", base_tid, exc) + title_db_id = get_title_id_db_id(base_tid) + + if not title_db_id: + # We *cannot* create a new Apps row without a Titles FK (nullable=False). + # Check if a row already exists; if not, log and bail clearly. + existing = Apps.query.filter_by(app_id=app_id, app_version=ver).first() + if existing: + return existing, False + + logger.warning(f"Cannot resolve Titles FK for app_id={app_id} v{ver} ({app_type}); skipping create.") + # Returning the non-existent row would break callers; raise clearly instead. + raise RuntimeError(f"Missing Titles FK for app_id={app_id} v{ver} ({app_type})") + + app = Apps.query.filter_by(app_id=app_id, app_version=ver).first() + if app: + # Backfill minimal fields if missing + if not app.title_id and title_db_id: + app.title_id = title_db_id + + # Only fill in app_type if missing — warn if a conflicting one is detected + if not app.app_type and app_type: + app.app_type = app_type + elif app.app_type and app_type and app.app_type != app_type: + logger.warning( + f"Conflicting app_type for app_id={app_id} v{ver}: " + f"existing={app.app_type}, new={app_type}. Keeping existing value." + ) + + return app, False + + # No existing row — create a new one + app = Apps( + app_id=app_id, + app_version=ver, + app_type=app_type, + owned=False, + title_id=title_db_id + ) + db.session.add(app) + return app, True + +def _normalize_version_int(s) -> int: + # Accepts int or str like "65536" / "0" / "v0" + if s is None: + return 0 + if isinstance(s, int): + return s + s = str(s).lower().lstrip("v") + try: + return int(s, 10) + except Exception: + return 0 + +def _json_default(o): + if isinstance(o, (datetime.datetime, datetime.date)): + return o.isoformat() + # Let json raise for anything else non-serializable so we don’t hide bugs + raise TypeError(f'Object of type {o.__class__.__name__} is not JSON serializable') diff --git a/app/migrations/versions/b52221b8c1e8_add_title_overrides_table.py b/app/migrations/versions/b52221b8c1e8_add_title_overrides_table.py new file mode 100644 index 00000000..de105b0b --- /dev/null +++ b/app/migrations/versions/b52221b8c1e8_add_title_overrides_table.py @@ -0,0 +1,47 @@ +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "b52221b8c1e8" +down_revision = "78c33e9bffce" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "app_overrides", + sa.Column("id", sa.Integer(), primary_key=True), + + # One-to-one to apps.id; cascade on delete + sa.Column("app_fk", sa.Integer(), sa.ForeignKey("apps.id", ondelete="CASCADE"), nullable=False, unique=True), + + # Overridable metadata + sa.Column("name", sa.String(length=512), nullable=True), + sa.Column("release_date", sa.Date(), nullable=True), + sa.Column("region", sa.String(length=32), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("content_type", sa.String(length=64), nullable=True), + sa.Column("version", sa.String(length=64), nullable=True), + + # Artwork + sa.Column("icon_path", sa.String(length=1024), nullable=True), + sa.Column("banner_path", sa.String(length=1024), nullable=True), + + # Corrected Title ID (for TitleDB lookup) + sa.Column("corrected_title_id", sa.String(length=16), nullable=True), + + sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.text("1")), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + ) + + # Helpful indexes + op.create_index("ix_app_overrides_app_fk", "app_overrides", ["app_fk"]) + op.create_index("ix_app_overrides_corrected_title_id", "app_overrides", ["corrected_title_id"]) + + +def downgrade(): + op.drop_index("ix_app_overrides_corrected_title_id", table_name="app_overrides") + op.drop_index("ix_app_overrides_app_fk", table_name="app_overrides") + op.drop_table("app_overrides") diff --git a/app/overrides.py b/app/overrides.py new file mode 100644 index 00000000..91b2d4ac --- /dev/null +++ b/app/overrides.py @@ -0,0 +1,495 @@ +import json +import os +import threading + +import datetime +from typing import Optional +from flask import abort, Blueprint, request, jsonify, current_app +from sqlalchemy.orm import joinedload +from werkzeug.exceptions import BadRequest, Conflict, NotFound + +from auth import access_required +from db import db, Apps, AppOverrides +import titles as titles_lib +from utils import * +from images import * +from constants import * +from cache import ( + compute_overrides_snapshot_hash, + regenerate_cache, + is_overrides_snapshot_current, +) +import logging +logger = logging.getLogger('main') + +# --- api blueprint --------------------------------------------------------- +overrides_blueprint = Blueprint("overrides_blueprint", __name__, url_prefix="/api/overrides") + + +# --- routes ---------------------------------------------------------------- +@overrides_blueprint.route("", methods=["GET"]) +@access_required("shop") +def list_overrides(): + payload, etag_hash = generate_overrides() + + resp = jsonify(payload) + # Use the same ETag semantics as library: enable cheap 304 revalidation by clients + resp.set_etag(etag_hash) + resp.headers["Vary"] = "Authorization" + resp.headers["Cache-Control"] = "no-cache, private" + return resp.make_conditional(request) + + +@overrides_blueprint.get("/") +@access_required('shop') +def get_override(oid: int): + ov = AppOverrides.query.options(joinedload(AppOverrides.app)).get(oid) + if not ov: + raise NotFound("Override not found.") + return jsonify(_serialize_with_art_urls(ov)) + + +@overrides_blueprint.post("") +@access_required('admin') +def create_override(): + data, banner_file, banner_remove, icon_file, icon_remove = _parse_payload() + + data = data or {} + data.setdefault("enabled", True) + if "enabled" in data and isinstance(data["enabled"], str): + data["enabled"] = data["enabled"].lower() in ("1", "true", "yes", "on") + + # Empty strings → None for text fields + for k in ("app_id", "name", "region", "description", "content_type", "version", "icon_path", "banner_path", "release_date", "corrected_title_id"): + if k in data and isinstance(data[k], str) and not data[k].strip(): + data[k] = None + + # Normalize release_date + if "release_date" in data: + data["release_date"] = _parse_iso_date_or_none(data["release_date"]) + + # Normalize corrected_title_id + if "corrected_title_id" in data: + data["corrected_title_id"] = normalize_id(data["corrected_title_id"]) + + # Require app_id (string) from client, but we map it to the Apps row + app_id = data.get("app_id") + if not app_id: + raise BadRequest("app_id is required.") + + app_id = normalize_id(app_id, 'app') + if not app_id: + raise BadRequest("Invalid app_id format.") + + # Find the target Apps row for this logical app_id + app = _resolve_target_app(app_id) + if not app: + raise NotFound("No app found for the given app_id.") + + # Enforce uniqueness per Apps row (one override per app_fk) + if AppOverrides.query.filter_by(app_fk=app.id).first(): + raise Conflict("An override already exists for this app.") + + # Create the override attached to the Apps row + ov = AppOverrides(app=app) + _apply_fields(ov, data) + + # Handle explicit removals + if banner_remove and ov.banner_path: + delete_art_file_if_owned(ov.banner_path, "banner") + ov.banner_path = None + if icon_remove and ov.icon_path: + delete_art_file_if_owned(ov.icon_path, "icon") + ov.icon_path = None + + banner_raw = None + icon_raw = None + + # Validate & read first + if banner_file: + validate_upload(banner_file) + banner_raw = read_upload_bytes(banner_file) + + if icon_file: + validate_upload(icon_file) + icon_raw = read_upload_bytes(icon_file) + + # Save uploaded assets (use the related app's app_id for filenames) + if banner_raw: + ov.banner_path = save_art_from_bytes(ov.app.app_id, banner_raw, "banner") + if icon_raw: + ov.icon_path = save_art_from_bytes(ov.app.app_id, icon_raw, "icon") + + # Derive counterpart if missing + if banner_raw and not ov.icon_path and not icon_remove: + ov.icon_path = save_art_from_bytes(ov.app.app_id, banner_raw, "icon") + if icon_raw and not ov.banner_path and not banner_remove: + ov.banner_path = save_art_from_bytes(ov.app.app_id, icon_raw, "banner") + + # Timestamps + ov.created_at = datetime.datetime.utcnow() + ov.updated_at = datetime.datetime.utcnow() + + db.session.add(ov) + try: + db.session.commit() + _refresh_caches() + except Exception: + logger.error("Create override failed") + db.session.rollback() + raise BadRequest("Could not create override.") + + return jsonify(_serialize_with_art_urls(ov)), 201 + + +@overrides_blueprint.put("/") +@overrides_blueprint.patch("/") +@access_required('admin') +def update_override(oid: int): + data, banner_file, banner_remove, icon_file, icon_remove = _parse_payload() + data = data or {} + + if "enabled" in data and isinstance(data["enabled"], str): + data["enabled"] = data["enabled"].lower() in ("1", "true", "yes", "on") + + # Empty strings → None for text fields + for k in ( + "name", "region", "description", "content_type", + "version", "icon_path", "banner_path", "release_date", "corrected_title_id" + ): + if k in data and isinstance(data[k], str) and not data[k].strip(): + data[k] = None + + # app_id is immutable; reject attempts to change it + if "app_id" in data: + abort(400, description="app_id is read-only and cannot be changed.") + + # Normalize release_date using helper + if "release_date" in data: + data["release_date"] = _parse_iso_date_or_none(data["release_date"]) + + # Normalize corrected_title_id + if "corrected_title_id" in data: + data["corrected_title_id"] = normalize_id(data["corrected_title_id"]) + + ov = AppOverrides.query.get(oid) + if not ov: + raise NotFound("Override not found.") + + _apply_fields(ov, data) + + # Handle explicit removals + if banner_remove and ov.banner_path: + delete_art_file_if_owned(ov.banner_path, "banner") + ov.banner_path = None + if icon_remove and ov.icon_path: + delete_art_file_if_owned(ov.icon_path, "icon") + ov.icon_path = None + + banner_raw = None + icon_raw = None + + if banner_file: + validate_upload(banner_file) + banner_raw = read_upload_bytes(banner_file) + if icon_file: + validate_upload(icon_file) + icon_raw = read_upload_bytes(icon_file) + + # Save uploaded assets + if banner_raw: + ov.banner_path = save_art_from_bytes(ov.app.app_id, banner_raw, "banner") + if icon_raw: + ov.icon_path = save_art_from_bytes(ov.app.app_id, icon_raw, "icon") + + # Derive counterpart if missing + if banner_raw and not ov.icon_path and not icon_remove: + ov.icon_path = save_art_from_bytes(ov.app.app_id, banner_raw, "icon") + if icon_raw and not ov.banner_path and not banner_remove: + ov.banner_path = save_art_from_bytes(ov.app.app_id, icon_raw, "banner") + + ov.updated_at = datetime.datetime.utcnow() + + try: + db.session.commit() + _refresh_caches() + except Exception: + logger.error("Update override failed") + db.session.rollback() + raise BadRequest("Could not update override.") + + return jsonify(_serialize_with_art_urls(ov)) + + +@overrides_blueprint.delete("/") +@access_required('admin') +def delete_override(oid: int): + ov = AppOverrides.query.get(oid) + if not ov: + raise NotFound("Override not found.") + + if ov.banner_path: + delete_art_file_if_owned(ov.banner_path, "banner") + if ov.icon_path: + delete_art_file_if_owned(ov.icon_path, "icon") + + try: + db.session.delete(ov) + db.session.commit() + _refresh_caches() + except Exception: + logger.error("Delete override failed") + db.session.rollback() + raise BadRequest("Could not delete override.") + return jsonify({"ok": True, "deleted_id": oid}) + +def generate_overrides(): + """ + Public entry-point for routes. + Always returns the latest cached payload (regenerating when needed). + """ + snap = load_or_generate_overrides_snapshot() + return snap["payload"], snap["hash"] + +def load_or_generate_overrides_snapshot(): + """ + Load from disk if hash unchanged, otherwise regenerate + save. + """ + saved = load_json(OVERRIDES_CACHE_FILE) + if saved and is_overrides_snapshot_current(saved): + return saved + + # Cache missing or stale → regenerate + return _generate_overrides_snapshot() + +def _generate_overrides_snapshot(): + """ + Build the final payload: + { + "items": [...override rows serialized...], + "redirects": { + "": { + "corrected_title_id": "...", + "projection": {...} # from TitleDB + }, + ... + } + } + + Includes TitleDB projections; writes to disk with a 'hash' top-level key. + """ + logger.info("Generating overrides snapshot...") + + with titles_lib.titledb_session("generate_overrides"): + # Query rows once + rows = ( + db.session.query(AppOverrides) + .order_by(AppOverrides.created_at.desc()) + .all() + ) + items = [_serialize_with_art_urls(r) for r in rows] + redirects = {} + for ov in rows: + if not getattr(ov, "enabled", False): + continue + corr = getattr(ov, "corrected_title_id", None) + appid = getattr(getattr(ov, "app", None), "app_id", None) + if not (appid and corr): + continue + projection = _project_titledb_block(corr) + redirects[appid] = { + "corrected_title_id": corr, + "projection": projection, + } + + current_hash = compute_overrides_snapshot_hash() + snapshot = { + "hash": current_hash, + "payload": { + "items": items, + "redirects": redirects, + } + } + save_json(snapshot, OVERRIDES_CACHE_FILE) + logger.info("Generating overrides snapshot done.") + return snapshot + +def build_override_index(include_disabled: bool = False) -> dict: + """ + Build a lightweight index of overrides keyed by app_id. + Only fields needed by the library merge path are included. + Structure: + { + "by_app": { + "": { + "id": , + "app_fk": , + "enabled": true/false, + "corrected_title_id": "0100....", + # (optional) a few display fields if you want them downstream: + "name": "...", + "description": "...", + "release_date": "yyyy-mm-dd" | None, + "banner_path": "...", + "icon_path": "..." + }, + ... + }, + "count": + } + """ + q = AppOverrides.query.options(joinedload(AppOverrides.app)) + if not include_disabled: + q = q.filter(AppOverrides.enabled.is_(True)) + + by_app = {} + for ov in q.all(): + app_id = ov.app.app_id if ov.app else None + if not app_id: + continue + by_app[app_id] = { + "id": ov.id, + "app_fk": ov.app_fk, + "enabled": bool(ov.enabled), + "corrected_title_id": ov.corrected_title_id, + # optional extras that can be handy for UI merges (not required): + "name": ov.name, + "description": ov.description, + "release_date": ov.release_date.isoformat() if ov.release_date else None, + "banner_path": ov.banner_path, + "icon_path": ov.icon_path, + } + + return {"by_app": by_app, "count": len(by_app)} + +def _refresh_caches(): + """ + Regenerate overrides + shop caches without blocking the request. + Falls back to synchronous regeneration if we have no app context. + """ + cache_paths = (OVERRIDES_CACHE_FILE, SHOP_CACHE_FILE) + + try: + app = current_app._get_current_object() + except RuntimeError: + regenerate_cache(*cache_paths) + return + + def _job(): + with app.app_context(): + try: + regenerate_cache(*cache_paths) + except Exception: + logger.exception("Background cache regeneration failed.") + + threading.Thread(target=_job, name="refresh-caches", daemon=True).start() + +# Note: UI sends multipart only when a banner or icon upload/removal is requested; otherwise JSON. +# This keeps existing JSON flows working while enabling binary upload. +def _parse_payload(): + """ + Accept either JSON (application/json) or multipart/form-data. + Returns: (data_dict, banner_file, banner_remove, icon_file, icon_remove) + """ + def _to_bool(value): + if isinstance(value, bool): + return value + if value is None: + return False + s = str(value).strip().lower() + return s in {"1", "true", "yes", "on"} + + if request.is_json: + data = request.get_json(silent=True) or {} + banner_file = None + icon_file = None + banner_remove = _to_bool(data.get("banner_remove")) + icon_remove = _to_bool(data.get("icon_remove")) + else: + data = request.form.to_dict() + banner_file = request.files.get("banner_file") + icon_file = request.files.get("icon_file") + banner_remove = _to_bool(data.get("banner_remove")) + icon_remove = _to_bool(data.get("icon_remove")) + + # Conflict resolution: if a new file is uploaded, ignore the corresponding remove flag + if banner_file: + banner_remove = False + if icon_file: + icon_remove = False + + return data, banner_file, banner_remove, icon_file, icon_remove + +def _apply_fields(ov: AppOverrides, data: dict): + # Only touch known fields; ignore extras to keep it robust. + fields = [ + "name", "release_date", "region", "description", "content_type", "version", + "enabled", "corrected_title_id", + ] + for f in fields: + if f in data: + setattr(ov, f, data[f]) + +def _parse_iso_date_or_none(value): + if not value: + return None + try: + # Accept strict yyyy-MM-dd + return datetime.date.fromisoformat(value) + except Exception: + abort(400, description="Invalid release_date. Expected format: yyyy-MM-dd.") + +def _resolve_target_app(app_id: str) -> Optional[Apps]: + """ + Map a logical 16/32-hex app_id string to a specific Apps row. + Preference order: + 1) highest numeric app_version among rows with app_type == 'BASE' + 2) otherwise highest numeric app_version among all rows + """ + q = Apps.query.filter(Apps.app_id == app_id) + + # Try BASE first + base_rows = q.filter((Apps.app_type == 'BASE') | (Apps.app_type == 'Base')).all() + if base_rows: + def v(a): + try: return int(a.app_version or 0) + except: return 0 + return sorted(base_rows, key=v, reverse=True)[0] + + rows = q.all() + if not rows: + return None + + def v2(a): + try: return int(a.app_version or 0) + except: return 0 + return sorted(rows, key=v2, reverse=True)[0] + +def _project_titledb_block(corrected_id: str) -> dict: + """ + Build the projected block for a corrected TitleID using TitleDB. + Includes: name, description, region, normalized release_date, bannerUrl, iconUrl, category. + """ + info = titles_lib.get_game_info(corrected_id) or {} + + return { + "name": (info.get("name") or "").strip() or None, + "description": info.get("description"), + "region": info.get("region"), + "release_date": info.get("release_date"), + "bannerUrl": info.get("bannerUrl"), + "iconUrl": info.get("iconUrl"), + "category": info.get("category"), + } + +def _serialize_with_art_urls(ov: AppOverrides) -> dict: + d = ov.as_dict() + # Ensure app_id string is present even though the model uses app_fk + try: + if "app_id" not in d or not d["app_id"]: + d["app_id"] = ov.app.app_id if ov.app else None + except Exception: + d.setdefault("app_id", None) + d["bannerUrl"] = d.get("banner_path") + d["iconUrl"] = d.get("icon_path") + return d diff --git a/app/shop.py b/app/shop.py index 5612c9f8..b4089ed5 100644 --- a/app/shop.py +++ b/app/shop.py @@ -1,11 +1,27 @@ +from constants import * from db import * +from overrides import ( + build_override_index, + load_or_generate_overrides_snapshot +) +import titles as titles_lib +from utils import load_json, save_json from Crypto.PublicKey import RSA from Crypto.Cipher import PKCS1_OAEP from Crypto.Hash import SHA256 from Crypto.Cipher import AES +from sqlalchemy import func +from urllib.parse import quote import zstandard as zstd import random +import re import json +import os +import hashlib +import logging +from typing import Optional + +logger = logging.getLogger('main') # https://github.com/blawar/tinfoil/blob/master/docs/files/public.key 1160174fa2d7589831f74d149bc403711f3991e4 TINFOIL_PUBLIC_KEY = '''-----BEGIN PUBLIC KEY----- @@ -18,15 +34,18 @@ CQIDAQAB -----END PUBLIC KEY-----''' -def gen_shop_files(db): - shop_files = [] - files = get_shop_files() - for file in files: - shop_files.append({ - "url": f'/api/get_game/{file["id"]}#{file["filename"]}', - 'size': file["size"] - }) - return shop_files +_TITLE_ID_BRACKET = re.compile(r"\[[0-9A-Fa-f]{16}\]") + +def generate_shop(): + snap = load_or_generate_shop_snapshot() + return snap["payload"], snap["hash"] + +def load_or_generate_shop_snapshot(): + saved = load_json(SHOP_CACHE_FILE) + current_hash = _current_shop_hash() + if saved and saved.get("hash") == current_hash: + return saved + return _generate_shop_snapshot() def encrypt_shop(shop): input = json.dumps(shop).encode('utf-8') @@ -50,3 +69,451 @@ def encrypt_shop(shop): binary_data = b'TINFOIL' + flag.to_bytes(1, byteorder='little') + sessionKey + sz.to_bytes(8, 'little') + buf return binary_data + +def _generate_shop_snapshot(): + # Build only what Tinfoil needs + logger.info("Generating shop snapshot...") + + with titles_lib.titledb_session("generate_shop"): + files = _gen_shop_files() + titledb_map = _build_titledb_from_overrides() + + payload = { + "files": files, + "titledb": titledb_map, + } + + + snap = {"hash": _current_shop_hash(), "payload": payload} + save_json(snap, SHOP_CACHE_FILE) + logger.info("Generating shop snapshot done.") + return snap + +def _gen_shop_files(): + """ + Build the 'files' section for the custom index. + If a single-content file’s linked app has an enabled override with + corrected_title_id, present the URL with that [TITLEID] token so Tinfoil + discovers it under the corrected ID. + """ + shop_files = [] + + # Preload relationships to avoid N+1 + rows = ( + db.session.query(Files) + .options( + db.joinedload(Files.apps).joinedload(Apps.override), + db.joinedload(Files.apps).joinedload(Apps.title), + ) + .all() + ) + + for f in rows: + presented_name = f.filename or os.path.basename(f.filepath) or "file.nsp" + presented_tid = None + # Only attempt an ID correction when we can unambiguously pick + # the single linked app. + if getattr(f, 'apps', None) and len(f.apps) == 1: + app = f.apps[0] + app_type = getattr(app, "app_type", None) + + # Choose TitleID token per type to avoid collisions in Tinfoil: + # - BASE: base/family TitleID (optionally corrected via override) + # - UPD: its own update TitleID (NEVER inherit base corrected id) + # - DLC: its own DLC TitleID + if app_type == titles_lib.APP_TYPE_DLC: + corr = _effective_corrected_title_id_for_file(f) # DLC may use corrected id + presented_tid = corr or app.app_id + elif app_type == titles_lib.APP_TYPE_UPD: + # Try to find a corrected base TitleID to mirror (+0x800) + base_tid = getattr(getattr(app, "title", None), "title_id", None) + base_corr = None + if base_tid: + base_app = ( + db.session + .query(Apps) + .options(db.joinedload(Apps.override)) + .filter(Apps.app_id == base_tid, Apps.app_type == titles_lib.APP_TYPE_BASE) + .first() + ) + if base_app: + base_ov = getattr(base_app, "override", None) + if base_ov and getattr(base_ov, "enabled", False) and getattr(base_ov, "corrected_title_id", None): + base_corr = base_ov.corrected_title_id.strip().upper() + + if base_corr: + # compute the update-family TitleID (+0x800) + try: + presented_tid = f"{int(base_corr, 16) + 0x800:016X}" + except ValueError: + presented_tid = app.app_id # fallback to real app id if malformed + else: + # no corrected base; just use the update's real app id + presented_tid = app.app_id + else: + corr = _effective_corrected_title_id_for_file(f) # BASE may use corrected id + presented_tid = corr or (app.title.title_id if getattr(app, "title", None) else app.app_id) + + if presented_tid: + presented_name = _with_title_id(presented_name, presented_tid) + + shop_files.append({ + "url": f"/api/get_game/{f.id}#{quote(presented_name)}", + "size": f.size or 0 + }) + + return shop_files + +def _build_titledb_from_overrides(): + """ + Build `titledb` from enabled AppOverrides, using on disk cache. + + Rules: + - BASE override: + * Keyed by corrected_title_id if provided, else by the base TitleID from Titles. + * entry["id"] == that same key (Tinfoil expects base id == TitleID). + - DLC override: + * Keyed by corrected_title_id if provided, else by the DLC's app_id (DLC TitleID). + * entry["id"] == that same key (use the DLC's own TitleID). + - Include any overridden fields: name, version (int), region, releaseDate (yyyymmdd), description, size. + - One node per override; DLCs are NOT nested under the base. + """ + def _yyyymmdd_int(iso_date_or_none): + # Accepts 'YYYY-MM-DD' or date/datetime; returns int yyyymmdd or None + if not iso_date_or_none: + return None + rd = iso_date_or_none + if hasattr(rd, "strftime"): + return int(rd.strftime("%Y%m%d")) + # cheap normalization: 'YYYY-MM-DD' -> 'YYYYMMDD' + s = str(rd).replace("-", "") + return int(s) if s.isdigit() and len(s) == 8 else None + + def _version_to_int(v): + return _version_str_to_int(v) + + def _first_value(*values): + """ + Return the first non-empty/non-null value, preserving the original text. + """ + for v in values: + if v is None: + continue + if isinstance(v, str): + if v.strip(): + return v + else: + return v + return None + + # Try to pull from the cached overrides snapshot if available + overrides_by_app = {} + redirect_meta_by_app = {} + try: + snap = load_or_generate_overrides_snapshot() + payload = (snap or {}).get("payload", {}) or {} + items = payload.get("items", []) or [] + redirects_payload = payload.get("redirects", {}) or {} + + for raw_app_id, redirect_info in redirects_payload.items(): + app_id = (raw_app_id or "").strip().upper() + if not app_id or not isinstance(redirect_info, dict): + continue + corr = redirect_info.get("corrected_title_id") or redirect_info.get("correctedTitleId") + corr = (corr or "").strip().upper() or None + projection = redirect_info.get("projection") if isinstance(redirect_info.get("projection"), dict) else {} + redirect_meta_by_app[app_id] = { + "corrected_title_id": corr, + "projection": projection, + } + + for it in items: + if it.get("enabled") is False: + continue + app_id = (it.get("app_id") or "").strip().upper() + if not app_id: + continue + overrides_by_app[app_id] = { + "corrected_title_id": _first_value( + it.get("corrected_title_id"), + it.get("correctedTitleId"), + redirect_meta_by_app.get(app_id, {}).get("corrected_title_id"), + ), + "name": it.get("name"), + "version": it.get("version"), + "region": it.get("region"), + "release_date": _first_value(it.get("release_date"), it.get("releaseDate")), + "description": it.get("description"), + "bannerUrl": _first_value(it.get("bannerUrl"), it.get("banner_path")), + "iconUrl": _first_value(it.get("iconUrl"), it.get("icon_path")), + "category": it.get("category"), + } + except Exception: + # Snapshot unavailable/corrupt; we'll fall back + overrides_by_app = {} + redirect_meta_by_app = {} + + # Fallback (or augment) from the lightweight index if needed + if not overrides_by_app: + idx = build_override_index(include_disabled=False) + for app_id, ov in idx.get("by_app", {}).items(): + app_id_u = (app_id or "").strip().upper() + if not app_id_u: + continue + overrides_by_app[app_id_u] = { + "corrected_title_id": ov.get("corrected_title_id"), + "name": ov.get("name"), + "version": ov.get("version"), # may be None if index doesn't include it + "region": ov.get("region"), + "release_date": ov.get("release_date"), + "description": ov.get("description"), + "bannerUrl": ov.get("banner_path"), + "iconUrl": ov.get("icon_path"), + "category": ov.get("category"), + } + corr = (ov.get("corrected_title_id") or "").strip().upper() + if corr: + info = titles_lib.get_game_info(corr) or {} + redirect_meta_by_app[app_id_u] = { + "corrected_title_id": corr, + "projection": { + "name": (info.get("name") or "").strip() or None, + "description": info.get("description"), + "region": info.get("region"), + "release_date": info.get("release_date"), + "bannerUrl": info.get("bannerUrl"), + "iconUrl": info.get("iconUrl"), + "category": info.get("category"), + }, + } + + if not overrides_by_app: + return {} + + app_ids = list(overrides_by_app.keys()) + + # One bulk query for app_type + base TitleID + meta_rows = ( + db.session.query( + Apps.app_id, + Apps.app_type, + Titles.title_id, # may be None for DLC/homebrew without a Titles row + ) + .outerjoin(Apps.title) + .filter(Apps.app_id.in_(app_ids)) + .all() + ) + meta_by_app = { + (app_id or "").strip().upper(): ( + app_type, + (title_id or "").strip().upper() if title_id else None, + ) + for app_id, app_type, title_id in meta_rows + } + + # Bulk aggregate sizes per app + size_rows = ( + db.session + .query(Apps.app_id, func.sum(Files.size)) + .join(Apps.files) # Apps -> Files + .filter(Files.size.isnot(None)) + .filter(Apps.app_id.in_(app_ids)) + .group_by(Apps.app_id) + .all() + ) + sizes_by_app = { + (app_id or "").strip().upper(): int(total or 0) + for app_id, total in size_rows + } + + # Build the map + titledb_map = {} + + for app_id_u, ov in overrides_by_app.items(): + app_type, base_tid = meta_by_app.get(app_id_u, (None, None)) + if app_type not in (titles_lib.APP_TYPE_BASE, titles_lib.APP_TYPE_DLC): + # Unknown type; skip to avoid guessing + continue + + redirect_meta = redirect_meta_by_app.get(app_id_u, {}) + corr_tid = _first_value( + (ov.get("corrected_title_id") or None), + redirect_meta.get("corrected_title_id"), + ) + corr_tid = (corr_tid or "").strip().upper() or None + + if app_type == titles_lib.APP_TYPE_BASE: + # BASE → prefer corrected_title_id, else Titles.title_id, else app_id as last resort + tid_emit = corr_tid or base_tid or app_id_u + else: + # DLC → prefer corrected_title_id, else its own app_id + tid_emit = corr_tid or app_id_u + + if not tid_emit: + continue + + projection = redirect_meta.get("projection") if isinstance(redirect_meta.get("projection"), dict) else {} + + entry = {"id": tid_emit} + + # Optional overridden fields + name = _first_value(ov.get("name"), projection.get("name")) + if name: + entry["name"] = name + + vnum = _version_to_int(_first_value(ov.get("version"), projection.get("version"))) + if vnum is not None: + entry["version"] = vnum + + region = _first_value(ov.get("region"), projection.get("region")) + if region: + entry["region"] = region + + rd_int = _yyyymmdd_int(_first_value(ov.get("release_date"), projection.get("release_date"))) + if rd_int: + entry["releaseDate"] = rd_int + + description = _first_value(ov.get("description"), projection.get("description")) + if description: + entry["description"] = description + + banner_url = _first_value(ov.get("bannerUrl"), projection.get("bannerUrl")) + if banner_url: + entry["bannerUrl"] = banner_url + + icon_url = _first_value(ov.get("iconUrl"), projection.get("iconUrl")) + if icon_url: + entry["iconUrl"] = icon_url + + category = _first_value(ov.get("category"), projection.get("category")) + if category: + entry["category"] = category + + total_bytes = sizes_by_app.get(app_id_u, 0) + if total_bytes: + entry["size"] = total_bytes + + titledb_map[tid_emit] = entry # last-writer wins if collisions + + # Also publish metadata keyed by the original (non-redirected) TitleID so + # Tinfoil can resolve installed titles that retain their original IDs. + if corr_tid: + if app_type == titles_lib.APP_TYPE_BASE: + source_tid = (base_tid or app_id_u or "").strip().upper() + else: + source_tid = app_id_u # DLC original id is its app_id + source_tid = (source_tid or "").strip().upper() + if source_tid and source_tid != tid_emit: + source_entry = dict(entry) + source_entry["id"] = source_tid + titledb_map[source_tid] = source_entry + + return titledb_map + +def _version_str_to_int(version_str): + """ + Convert '1.2.3' -> 10203 (A*10000 + B*100 + C). + Returns None if not parseable. Tinfoil wants numeric `version`. + """ + if not version_str: + return None + parts = re.findall(r"\d+", str(version_str)) + if not parts: + return None + a, b, c = (int(p) for p in (parts + ["0", "0"])[:3]) + return a * 10000 + b * 100 + c + +def _effective_corrected_title_id_for_file(f: Files) -> Optional[str]: + """ + Return the corrected TitleID to present for this file, if any. + Rules: + - If the single linked App has an enabled override with corrected_title_id → use it. + - If the App is an UPDATE, inherit the BASE app's override (same Title family). + - DLCs do NOT inherit from BASE (only use their own override). + """ + if not getattr(f, "apps", None) or len(f.apps) != 1: + return None + app = f.apps[0] + + # direct override on this app? + ov = getattr(app, "override", None) + if ov and getattr(ov, "enabled", False) and getattr(ov, "corrected_title_id", None): + return ov.corrected_title_id.strip().upper() + + # UPDATE inherits BASE override + if getattr(app, "app_type", None) == titles_lib.APP_TYPE_UPD: + base = None + # We already joined Titles; ask it for the base id and fetch the BASE app row + base_tid = getattr(getattr(app, "title", None), "title_id", None) + if base_tid: + base = ( + db.session.query(Apps) + .options(db.joinedload(Apps.override)) + .filter(Apps.app_id == base_tid, Apps.app_type == titles_lib.APP_TYPE_BASE) + .first() + ) + if base: + bov = getattr(base, "override", None) + if bov and getattr(bov, "enabled", False) and getattr(bov, "corrected_title_id", None): + return bov.corrected_title_id.strip().upper() + + # DLCs do not inherit base override + return None + +def _with_title_id(presented_name: str, tid: str) -> str: + """ + Ensure the presented filename contains [TITLE_ID] before the extension. + If a [16-hex] token already exists, replace it; else insert it. + """ + if not presented_name or not tid: + return presented_name + tid = tid.strip().upper() + if not re.fullmatch(r"[0-9A-F]{16}", tid): + return presented_name # refuse to write a bad token + + root, ext = os.path.splitext(presented_name) + token = f"[{tid}]" + if _TITLE_ID_BRACKET.search(presented_name): + return _TITLE_ID_BRACKET.sub(token, presented_name) + # insert before extension (handles no-ext too) + sep = "" if not root else " " + return f"{root}{sep}{token}{ext or ''}" + +def _compute_files_fingerprint_rows(): + """ + Tiny, stable summary of things that affect the shop 'files' section: + - Files.id (stable order key) + - size (emitted in the feed) + - basename (affects the presented URL fragment) + """ + rows = ( + db.session.query(Files.id, Files.size, Files.filepath) + .order_by(Files.id.asc()) + .all() + ) + fp = [] + for fid, size, path in rows: + base = os.path.basename(path or "") if path else "" + fp.append((int(fid), int(size or 0), base)) + return fp + +def _current_shop_hash(): + # Overrides snapshot (hash + titledb_commit inside it) + ov_snap = load_or_generate_overrides_snapshot() + ov_hash = ov_snap.get("hash") or "" + + # Library snapshot (hash + titledb_commit inside it) + lib_snap = load_json(LIBRARY_CACHE_FILE) or {} + lib_hash = lib_snap.get("hash") or "" + + # Files fingerprint (sizes & basenames) + files_fp = _compute_files_fingerprint_rows() + + shop_hash = { + "overrides_hash": ov_hash, + "library_hash": lib_hash, + "files": files_fp, + } + return hashlib.sha256( + json.dumps(shop_hash, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() diff --git a/app/static/js/overrides.js b/app/static/js/overrides.js new file mode 100644 index 00000000..2199342f --- /dev/null +++ b/app/static/js/overrides.js @@ -0,0 +1,1028 @@ +'use strict'; + +// Global helper for Overrides functionality. +// Attaches as window.Ownfoil.Overrides and requires jQuery. + +((global, $) => { + if (!$) { + console.warn('Ownfoil Overrides requires jQuery.'); + return; + } + + const namespace = global.Ownfoil = global.Ownfoil || {}; + const PLACEHOLDER_TEXT = () => window.PLACEHOLDER_TEXT || "Image Unavailable"; + const DEFAULT_BANNER = () => window.DEFAULT_BANNER || `https://placehold.co/400x225/png?text=${encodeURIComponent(PLACEHOLDER_TEXT())}`; + const DEFAULT_ICON = () => window.DEFAULT_ICON || `https://placehold.co/400x400/png?text=${encodeURIComponent(PLACEHOLDER_TEXT())}`; + + // Forces browsers to refetch updated artwork after saves/resets + const ARTWORK_BUSTERS = new Map(); // app_id -> integer + const getBuster = (appId) => ARTWORK_BUSTERS.get(appId) || 0; + const bumpBuster = (appId) => ARTWORK_BUSTERS.set(appId, getBuster(appId) + 1); + const hex16 = (s) => /^[0-9A-F]{16}$/i.test((s || '').toString().trim()); + + // --- Overrides state --- + // key: app_id -> override object + const overridesByKey = new Map(); + + // --- Redirects state --- + // key: app_id -> { corrected_title_id, projection } + const redirectsByAppId = new Map(); + + // external environment (supplied by index.html) + const env = { + getGames: null, // () => games array + applyFilters: null // () => void + }; + + const $overrideModalEl = $('#overrideEditorModal'); + const overrideModal = () => { + const el = $overrideModalEl.get(0); + if (!el) return null; + return bootstrap.Modal.getOrCreateInstance(el); + }; + + // ----------------- Helpers ----------------- + const _trimmedOrNull = (u) => (typeof u === 'string' && u.trim().length) ? u.trim() : null; + const trimOrNull = (v) => _trimmedOrNull((v ?? '').toString()); + const numOrNull = (v) => { + const t = trimOrNull(v); + if (t === null) return null; + const n = Number(t); + return Number.isFinite(n) ? n : null; + }; + const addBuster = (url, buster = 0) => + !url || (/^(data:|blob:)/i.test(url)) || !buster + ? url + : `${url}${url.includes('?') ? '&' : '?'}b=${buster}`; + const appKey = (gameOrOverride) => gameOrOverride?.app_id || ''; + + // Derive Version for the modal: + // - BASE -> latest OWNED update version (fallback: latest available; fallback: app_version) + // - DLC -> app_version of the DLC itself + const deriveTitleDbVersion = (game) => { + if (!game) return null; + + const parseNum = (x) => { + if (typeof x === 'number' && Number.isFinite(x)) return x; + if (typeof x === 'string' && /^\d+$/.test(x)) return Number(x); + return null; + }; + + const type = (game.app_type || '').toUpperCase(); + + if (type === 'DLC') { + // DLCs: show the app's own version (app_version) + return parseNum(game.app_version); + } + + // BASE (or anything else): prefer latest OWNED update version + const updates = Array.isArray(game.version) ? game.version : []; + + const pickMaxVersion = (arr) => { + let max = null; + for (const item of arr) { + const n = parseNum(item?.version); + if (n !== null && (max === null || n > max)) max = n; + } + return max; + }; + + if (updates.length) { + const owned = updates.filter(u => u && u.owned === true); + const maxOwned = owned.length ? pickMaxVersion(owned) : null; + if (maxOwned !== null) return maxOwned; + + const maxAny = pickMaxVersion(updates); + if (maxAny !== null) return maxAny; + } + + // No updates? fall back to the app's own version (often 0 for v0 base) + return parseNum(game.app_version); + }; + + // Recognition flags (stable against overrides) + const computeRecognitionFlags = (game) => { + if (!game || typeof game !== 'object') return { isUnrecognized: true, hasTitleDb: false }; + + const origName = (game?._orig?.title_id_name ?? '').trim(); + const curName = (game?.title_id_name ?? game?.name ?? '').trim(); + const anyName = origName || curName; + + // Prefer server-provided boolean if available + const explicit = + (typeof game.has_title_db === 'boolean') ? game.has_title_db : + (typeof game.hasTitleDb === 'boolean') ? game.hasTitleDb : + null; + + // Heuristic name check (treat literal "Unrecognized"/"Unidentified" as unrecognized) + const looksNamed = !!anyName && !/^(unrecognized|unidentified)$/i.test(anyName); + + // A plausible 16-hex TitleID anywhere we usually carry it + const idGuess = (game.app_id ?? game.title_id ?? game.id ?? '').toString().trim(); + const hasHexId = hex16(idGuess); + + // Decision: explicit boolean wins; otherwise require both a real-looking name and a 16-hex id + const hasTitleDb = (explicit !== null) ? explicit : (looksNamed && hasHexId); + + return { isUnrecognized: !hasTitleDb, hasTitleDb }; + }; + + const isUnrecognizedGame = (game) => { + if (!game) return false; + if (typeof game.isUnrecognized === 'boolean' && typeof game.hasTitleDb === 'boolean') { + return game.isUnrecognized || !game.hasTitleDb; + } + const flags = computeRecognitionFlags(game); + game.isUnrecognized = flags.isUnrecognized; // cache + game.hasTitleDb = flags.hasTitleDb; // cache + return flags.isUnrecognized; + }; + + // Find the base game for a DLC by TitleID prefix (first 12 hex chars are shared). + const findBaseForDlc = (dlcGame, allGames) => { + if (!dlcGame || !Array.isArray(allGames)) return null; + if ((dlcGame.app_type || '').toUpperCase() !== 'DLC') return null; + + const appId = (dlcGame.app_id || '').toUpperCase(); + if (!hex16(appId)) return null; + + const familyPrefix = appId.slice(0, 12); // e.g. 010056E00853 + const base = allGames.find(g => + (g.app_type || '').toUpperCase() === 'BASE' && + typeof g.app_id === 'string' && + g.app_id.toUpperCase().startsWith(familyPrefix) + ); + return base || null; + }; + + // return the correct display title for the big card header. + const displayTitleFor = (game, allGames) => { + const type = (game?.app_type || '').toUpperCase(); + if (type === 'DLC') { + // Always show the BASE title for DLC cards + const base = findBaseForDlc(game, allGames); + if (base) return (base.name || base.title_id_name || 'Unrecognized'); + } + // For BASE (and anything else), use TitleDB name then fallback + return (game?.name || game?.title_id_name || 'Unrecognized'); + }; + + const pickTidForDisplay = (game, ovr) => { + const candidates = [ + ovr?.corrected_title_id, // explicit override first + game?.corrected_title_id, // server-computed corrected id if present + game?.app_id, // app-specific id (BASE or DLC) + game?.dlc_title_id, // distinct DLC id if existing + game?.title_id, // family/base id (fallback) + game?.id // last resort + ]; + for (const c of candidates) { + const t = (c || '').toString().trim(); + if (hex16(t)) return t.toUpperCase(); + } + return ''; + }; + + const pickNameForEdit = (game, ovr) => (ovr && typeof ovr.name === 'string' && ovr.name.trim()) + ? ovr.name.trim() + : (game?.name || game?.title_id_name || '').trim(); + + // ----------------- Overlay helpers ----------------- + // Apply (or remove) a single override onto matching games in memory. + const applyOverrideToGamesByKey = (key, games) => { + if (!key || !Array.isArray(games)) return; + const ovr = overridesByKey.get(key) || null; + const affecteds = games.filter(g => appKey(g) === key); + + affecteds.forEach(g => { + if (!g._orig) g._orig = { name: g.name, title_id_name: g.title_id_name, release_date: g.release_date ?? null, app_type: g.app_type }; + const type = (g.app_type || '').toUpperCase(); + + if (ovr && ovr.enabled) { + // --- NAME override --- + if (ovr.name && typeof ovr.name === 'string' && ovr.name.trim().length) { + const ovrName = ovr.name.trim(); + g.name = ovrName; + if (type !== 'DLC') + g.title_id_name = ovrName; + } else { + // No explicit name override: do not touch g.name/title_id_name. + // This preserves any redirect projection already applied. + } + + // apply release_date if present (allow clearing with null) + if (ovr.release_date && typeof ovr.release_date === 'string' && ovr.release_date.trim().length) { + g.release_date = ovr.release_date.trim(); + } + } else { + // Override disabled/absent -> restore originals + if (g._orig) { + g.title_id_name = g._orig.title_id_name; + g.name = g._orig.name; + g.release_date = g._orig.release_date ?? null; + } + } + }); + }; + + const reapplyAllOverridesToGames = (games) => { + if (!Array.isArray(games)) return; + const keys = new Set(); + games.forEach(g => keys.add(appKey(g))); + keys.forEach(k => applyOverrideToGamesByKey(k, games)); + } + + // Redirect helpers + const getRedirectForApp = (appId) => { + const k = (appId || '').trim(); + return k ? (redirectsByAppId.get(k) || null) : null; + }; + + // Overlay a redirect projection onto a game (identifiers unchanged). + // Mutates the game object for render-time fields only; does NOT touch g._orig. + const applyRedirectToGame = (game) => { + if (!game || !game.app_id) return game; + const r = getRedirectForApp(game.app_id); + if (!r || !r.projection) return game; + + const proj = r.projection; + + // Mark correction context (useful for badges/logic) + game.corrected_title_id = r.corrected_title_id || game.corrected_title_id || null; + game.recognized_via_correction = true; + + // Overlay display metadata (leave identifiers alone) + if (typeof proj.name === 'string') game.name = proj.name; + if (typeof proj.description === 'string') game.description = proj.description; + if (typeof proj.region === 'string') game.region = proj.region; + if (typeof proj.release_date === 'string') game.release_date = proj.release_date; + + if (proj.bannerUrl) game.bannerUrl = proj.bannerUrl; + if (proj.iconUrl) game.iconUrl = proj.iconUrl; + if (Array.isArray(proj.category)) game.category = proj.category.slice(); + + return game; + }; + + // Apply redirects to an array of games (in-place overlay for render-time fields) + const applyRedirectsToGames = (gamesArray) => { + if (!Array.isArray(gamesArray) || redirectsByAppId.size === 0) return; + for (const g of gamesArray) applyRedirectToGame(g); + }; + + // ----------------- Fetching ----------------- + const fetchOverrides = async () => { + try { + const list = await $.ajax({ + url: '/api/overrides', + method: 'GET', + dataType: 'json', + ifModified: true + }); + + // If 304, jQuery resolves but `list` can be undefined/null → no change + if (!list) return { overridesChanged: false, redirectsChanged: false }; + + overridesByKey.clear(); + (Array.isArray(list.items) ? list.items : []).forEach(o => { + const k = appKey(o); + if (k) overridesByKey.set(k, o); + }); + + // load redirects + redirectsByAppId.clear(); + const r = list && list.redirects && typeof list.redirects === 'object' ? list.redirects : null; + if (r) { + Object.entries(r).forEach(([appId, val]) => { + if (!appId) return; + if (val && (typeof val === 'object') && (val.corrected_title_id || val.projection)) { + redirectsByAppId.set(appId, { + corrected_title_id: val.corrected_title_id || null, + projection: (val.projection && typeof val.projection === 'object') ? val.projection : null + }); + } + }); + } + + // if some other view uses reapply immediately: + if (env.getGames) reapplyAllOverridesToGames(env.getGames()); + + return { overridesChanged: true, redirectsChanged: true }; + } catch (e) { + overridesByKey.clear(); + redirectsByAppId.clear(); + } + }; + + + // ----------------- Derived artwork URLs ----------------- + const bannerUrlFor = (game) => { + const ovr = getOverrideForGame(game); + const ovrUrl = _trimmedOrNull(ovr?.banner_path) || _trimmedOrNull(ovr?.bannerUrl); + if (ovrUrl) return addBuster(ovrUrl, getBuster(game.app_id)); + + return ( + _trimmedOrNull(game.banner_path) || _trimmedOrNull(game.bannerUrl) || _trimmedOrNull(game.banner) || + _trimmedOrNull(game.iconUrl) || DEFAULT_BANNER() + ); + } + + const iconUrlFor = (game) =>{ + const ovr = getOverrideForGame(game); + const ovrUrl = _trimmedOrNull(ovr?.icon_path) || _trimmedOrNull(ovr?.iconUrl); + if (ovrUrl) return addBuster(ovrUrl, getBuster(game.app_id)); + + return ( + _trimmedOrNull(game.iconUrl) || _trimmedOrNull(game.icon) || + _trimmedOrNull(game.banner_path) || _trimmedOrNull(game.bannerUrl) || _trimmedOrNull(game.banner) || + DEFAULT_ICON() + ); + } + + const getOverrideForGame = (game) => { const k = appKey(game); return k ? overridesByKey.get(k) : null; } + + const hasActiveOverride = (game) => { const o = getOverrideForGame(game); return !!(o && o.enabled !== false); } + + // ----------------- Cropping helpers ----------------- + const cropBannerFileToDataURL = (file, callback) => { + const TARGET_W = 400, TARGET_H = 225; + const img = new Image(); + const url = URL.createObjectURL(file); + + img.onload = () => { + try { + const canvas = document.createElement('canvas'); + canvas.width = TARGET_W; + canvas.height = TARGET_H; + const ctx = canvas.getContext('2d'); + + const srcW = img.naturalWidth || img.width; + const srcH = img.naturalHeight || img.height; + if (!srcW || !srcH) { URL.revokeObjectURL(url); return callback(null); } + + const scale = Math.max(TARGET_W / srcW, TARGET_H / srcH); + const drawW = Math.round(srcW * scale); + const drawH = Math.round(srcH * scale); + const dx = Math.round((TARGET_W - drawW) / 2); + const dy = Math.round((TARGET_H - drawH) / 2); + + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.clearRect(0, 0, TARGET_W, TARGET_H); + ctx.drawImage(img, dx, dy, drawW, drawH); + + const dataURL = canvas.toDataURL('image/png'); + URL.revokeObjectURL(url); + callback(dataURL); + } catch (e) { + URL.revokeObjectURL(url); + callback(null); + } + }; + + img.onerror = () => { URL.revokeObjectURL(url); callback(null); }; + img.src = url; + } + + const cropIconFileToDataURL = (file, callback) => { + const TARGET = 400; + const img = new Image(); + const url = URL.createObjectURL(file); + + img.onload = () => { + try { + const canvas = document.createElement('canvas'); + canvas.width = TARGET; canvas.height = TARGET; + const ctx = canvas.getContext('2d'); + + const srcW = img.naturalWidth || img.width; + const srcH = img.naturalHeight || img.height; + if (!srcW || !srcH) { URL.revokeObjectURL(url); return callback(null); } + + const scale = Math.max(TARGET / srcW, TARGET / srcH); + const drawW = Math.round(srcW * scale); + const drawH = Math.round(srcH * scale); + const dx = Math.round((TARGET - drawW) / 2); + const dy = Math.round((TARGET - drawH) / 2); + + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.clearRect(0, 0, TARGET, TARGET); + ctx.drawImage(img, dx, dy, drawW, drawH); + + const dataURL = canvas.toDataURL('image/png'); + URL.revokeObjectURL(url); + callback(dataURL); + } catch (e) { + URL.revokeObjectURL(url); + callback(null); + } + }; + + img.onerror = () => { URL.revokeObjectURL(url); callback(null); }; + img.src = url; + } + + // ----------------- Modal open / Save / Reset ----------------- + const openOverrideEditor = (game) => { + if (!window.IS_ADMIN) return; + const k = appKey(game); + const ovr = k ? overridesByKey.get(k) : null; + + $('#ovr-id').val(ovr?.id || ''); + $('#ovr-app-id').val(game.app_id || ''); + $('#ovr-file-name').text(game.file_basename || ''); + + // --- TitleDB baselines for the 4 fields --- + const tdReleaseDate = trimOrNull(game.release_date); + const tdRegion = trimOrNull(game.region); + const tdDescription = trimOrNull(game.description); + const tdVersionNum = deriveTitleDbVersion(game); // number or null + + // Name + $('#ovr-name').val(pickNameForEdit(game, ovr)); + $('#ovr-name') + .data('origName', ovr?.name ?? (game.title_id_name || game.name || '')) + .data('everEdited', false); + + // Region + const initialRegion = (ovr?.region != null) ? trimOrNull(ovr.region) : tdRegion; + $('#ovr-region') + .val(initialRegion ?? '') + .data('origVal', initialRegion ?? '') + .data('everEdited', false); + + // Release date + const initialReleaseDate = (ovr?.release_date != null) ? trimOrNull(ovr.release_date) : tdReleaseDate; + $('#ov-release-date') + .val(initialReleaseDate ?? '') + .data('origVal', initialReleaseDate ?? '') + .data('everEdited', false); + + // Description + const initialDescription = (ovr?.description != null) ? trimOrNull(ovr.description) : tdDescription; + $('#ovr-description') + .val(initialDescription ?? '') + .data('origVal', initialDescription ?? '') + .data('everEdited', false); + + // Version (numeric) + const initialVersion = (ovr?.version != null) ? ovr.version : tdVersionNum; + $('#ovr-version') + .val((initialVersion ?? '') === '' ? '' : String(initialVersion)) + .data('origVal', (initialVersion ?? '') === '' ? '' : String(initialVersion)) + .data('everEdited', false); + + // TID display/edit + const displayTid = pickTidForDisplay(game, ovr); + $('#ovTitleIdDisplay').text(displayTid || '(none)'); + $('#ovCorrectedTitleId') + .val(displayTid) + .data('origTid', displayTid || '') + .data('everEdited', false); + + $('#ovTitleIdEditRow').addClass('d-none'); + $('#ovTitleIdEditBtn').removeClass('d-none'); + + $('#btn-reset-override').toggle(!!ovr?.id); + + // clear file inputs + $('#ovr-banner-file').val(''); + $('#ovr-icon-file').val(''); + $('#ovr-banner-file').data('pending', null); + $('#ovr-icon-file').data('pending', null); + $('#ovr-banner-remove').hide(); + $('#ovr-icon-remove').hide(); + + // Determine sources + const ovrBanner = ovr?.banner_path || ovr?.bannerUrl || null; + const ovrIcon = ovr?.icon_path || ovr?.iconUrl || null; + + const gameBanner = game.banner_path || game.bannerUrl || game.banner || null; + const gameIcon = game.iconUrl || null; + + let currentBanner = ovrBanner ? addBuster(ovrBanner, getBuster(game.app_id)) : (gameBanner || null); + let currentIcon = ovrIcon ? addBuster(ovrIcon, getBuster(game.app_id)) : (gameIcon || null); + + if (!currentBanner && currentIcon) { + const img = new Image(); + img.crossOrigin = 'anonymous'; // allow CORS-safe draw if server sends ACAO + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = 400; canvas.height = 225; + const ctx = canvas.getContext('2d'); + const scale = Math.max(400 / img.width, 225 / img.height); + const w = Math.round(img.width * scale); + const h = Math.round(img.height * scale); + const dx = Math.round((400 - w) / 2); + const dy = Math.round((225 - h) / 2); + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.clearRect(0, 0, 400, 225); + ctx.drawImage(img, dx, dy, w, h); + try { + const dataURL = canvas.toDataURL('image/png'); + $('#ovr-banner-preview-img').attr('src', dataURL); + } catch (_) { + // Canvas tainted (no CORS) — fall back to the original image URL or default + $('#ovr-banner-preview-img').attr('src', currentIcon || DEFAULT_BANNER()); + } + $('#ovr-banner-preview-img').data('ovr', !!ovrBanner); + $('#ovr-banner-remove').toggle(!!ovrBanner); + }; + img.onerror = () => { + $('#ovr-banner-preview-img').attr('src', DEFAULT_BANNER()); + $('#ovr-banner-preview-img').data('ovr', !!ovrBanner); + $('#ovr-banner-remove').toggle(!!ovrBanner); + }; + img.src = currentIcon; + } else { + $('#ovr-banner-preview-img').attr('src', currentBanner || DEFAULT_BANNER()); + $('#ovr-banner-preview-img').data('ovr', !!ovrBanner); + $('#ovr-banner-remove').toggle(!!ovrBanner); + } + + if (!currentIcon && currentBanner) { + const img = new Image(); + img.crossOrigin = 'anonymous'; + img.onload = () => { + const canvas = document.createElement('canvas'); + canvas.width = 400; canvas.height = 400; + const ctx = canvas.getContext('2d'); + const scale = Math.max(400 / img.width, 400 / img.height); + const w = Math.round(img.width * scale); + const h = Math.round(img.height * scale); + const dx = Math.round((400 - w) / 2); + const dy = Math.round((400 - h) / 2); + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.clearRect(0, 0, 400, 400); + ctx.drawImage(img, dx, dy, w, h); + + try { + const dataURL = canvas.toDataURL('image/png'); + $('#ovr-icon-preview-img').attr('src', dataURL); + } catch (_) { + $('#ovr-icon-preview-img').attr('src', currentBanner || DEFAULT_ICON()); + } + $('#ovr-icon-preview-img').data('ovr', !!ovrIcon); + $('#ovr-icon-remove').toggle(!!ovrIcon); + }; + img.onerror = () => { + $('#ovr-icon-preview-img').attr('src', DEFAULT_ICON()); + $('#ovr-icon-preview-img').data('ovr', !!ovrIcon); + $('#ovr-icon-remove').toggle(!!ovrIcon); + }; + img.src = currentBanner; + } else { + $('#ovr-icon-preview-img').attr('src', currentIcon || DEFAULT_ICON()); + $('#ovr-icon-preview-img').data('ovr', !!ovrIcon); + $('#ovr-icon-remove').toggle(!!ovrIcon); + } + + const modalInstance = overrideModal(); + if (modalInstance) { + modalInstance.show(); + } + } + + const finishOverrideMutationAndRefresh = (modifiedKey) => { + if (env.getGames) applyOverrideToGamesByKey(modifiedKey, env.getGames()); + if (env.applyFilters) env.applyFilters(); + const modalInstance = overrideModal(); + if (modalInstance) { + modalInstance.hide(); + } + } + + const saveOverride = async () => { + const id = $('#ovr-id').val().trim(); + const app_id = $('#ovr-app-id').val().trim(); + + const payload = { + enabled: true + }; + + // --- Name (send only if edited & changed; allow explicit clearing) + const $name = $('#ovr-name'); + const origName = (trimOrNull($name.data('origName')) || ''); + const nameEdited = $name.data('everEdited') === true; + const nameVal = trimOrNull($name.val()); + + if (nameEdited) { + if (nameVal && nameVal !== origName) { + // user changed to a new non-empty value -> set it + payload.name = nameVal; + } else if (!nameVal && origName) { + // user cleared it -> explicitly clear on backend + payload.name = null; + } + } + + // --- Region (send only if edited & changed; allow explicit clearing) + const $region = $('#ovr-region'); + const regionEdited = $region.data('everEdited') === true; + const regionOrig = trimOrNull($region.data('origVal')) || ''; + const regionVal = trimOrNull($region.val()); + + if (regionEdited) { + if ((regionVal || '') !== regionOrig) { + payload.region = (regionVal ?? null); + } + } + + // --- Release date (yyyy-MM-dd) same rules + const $rd = $('#ov-release-date'); + const rdEdited = $rd.data('everEdited') === true; + const rdOrig = trimOrNull($rd.data('origVal')) || ''; + const rdVal = trimOrNull($rd.val()); // or null + + if (rdEdited) { + if ((rdVal || '') !== rdOrig) { + payload.release_date = (rdVal ?? null); + } + } + + // --- Description (send only if edited & changed; allow clearing) + const $desc = $('#ovr-description'); + const descEdited = $desc.data('everEdited') === true; + const descOrig = trimOrNull($desc.data('origVal')) || ''; + const descVal = trimOrNull($desc.val()); + + if (descEdited) { + if ((descVal || '') !== descOrig) { + payload.description = (descVal ?? null); + } + } + + // --- Version (number) — send only if edited & changed; allow clearing + const $ver = $('#ovr-version'); + const verEdited = $ver.data('everEdited') === true; + const verOrigStr = ($ver.data('origVal') ?? '').toString(); + const verOrigNum = numOrNull(verOrigStr); + const verValNum = numOrNull($ver.val()); + + if (verEdited) { + // Note: treat NaN/null as "cleared" + const changed = + (verValNum === null && verOrigNum !== null) || + (verValNum !== null && verOrigNum === null) || + (verValNum !== null && verOrigNum !== null && verValNum !== verOrigNum); + if (changed) { + payload.version = (verValNum === null ? null : verValNum); + } + } + + // --- Title ID override logic (include only if user edited AND changed) --- + const $tid = $('#ovCorrectedTitleId'); + const origTid = ($tid.data('origTid') || '').toUpperCase(); + const everEdited = $tid.data('everEdited') === true; + + let correctedTitleId = trimOrNull($tid.val()); + if (correctedTitleId) { + correctedTitleId = correctedTitleId.toUpperCase(); + if (correctedTitleId.startsWith('0X')) correctedTitleId = correctedTitleId.slice(2); + } + + if (everEdited) { + // If edited, only send when valid AND different from the original shown + if (correctedTitleId) { + if (!/^[0-9A-F]{16}$/.test(correctedTitleId)) { + // invalid -> abort + alert('Corrected Title ID must be exactly 16 hex characters (optionally prefixed by 0x).'); + return; + } + if (correctedTitleId !== origTid) { + // changed -> include + payload.corrected_title_id = correctedTitleId; + } + } + } + + if (!id) payload.app_id = app_id; + + // Pending artwork (set by change/drop handlers) + const bannerFileToUpload = $('#ovr-banner-file').data('pending') || null; + const bannerRemoveRequested = $('#ovr-banner-remove').data('remove') === true; + const iconFileToUpload = $('#ovr-icon-file').data('pending') || null; + const iconRemoveRequested = $('#ovr-icon-remove').data('remove') === true; + + const needMultipart = !!(bannerFileToUpload || bannerRemoveRequested || iconFileToUpload || iconRemoveRequested); + + let url, method, options; + if (id) { url = `/api/overrides/${id}`; method = 'PUT'; } + else { url = '/api/overrides'; method = 'POST'; } + + if (needMultipart) { + const fd = new FormData(); + Object.entries(payload).forEach(([k, v]) => { if (v !== undefined && v !== null) fd.append(k, String(v)); }); + if (bannerFileToUpload) fd.append('banner_file', bannerFileToUpload); + if (bannerRemoveRequested) fd.append('banner_remove', 'true'); + if (iconFileToUpload) fd.append('icon_file', iconFileToUpload); + if (iconRemoveRequested) fd.append('icon_remove', 'true'); + options = { method, body: fd }; + } else { + options = { method, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }; + } + + let res; + const isFD = (options.body instanceof FormData); + try { + res = await $.ajax({ + url, + type: options.method, + data: isFD ? options.body : options.body, + processData: false, + contentType: isFD ? false : 'application/json', + dataType: 'json' + }); + } catch { + alert('Failed to save override'); + return; + } + + // clear pending flags + $('#ovr-banner-file').val('').data('pending', null); + $('#ovr-icon-file').val('').data('pending', null); + $('#ovr-banner-remove').data('remove', false); + $('#ovr-icon-remove').data('remove', false); + + if (res && res.app_id) { + const key = appKey(res); + if (key) { + overridesByKey.set(key, res); + if (needMultipart) bumpBuster(key); // refresh override artwork if changed + + // If a corrected ID is present, hint the current in-memory game for a badge + if (res.corrected_title_id && env.getGames) { + const games = env.getGames(); + const g = games.find(x => x.app_id === key); + if (g) g.recognized_via_correction = true; + } + finishOverrideMutationAndRefresh(key); + return; + } + } + + await fetchOverrides(); + if (needMultipart) bumpBuster(app_id); + finishOverrideMutationAndRefresh(app_id); + } + + const resetOverride = async () => { + const id = $('#ovr-id').val().trim(); + if (!id) return; + if (!confirm('Remove override and revert to default metadata?')) return; + + try { + await $.ajax({ + url: `/api/overrides/${id}`, + method: 'DELETE' + }); + } catch { + alert('Failed to delete override'); + return; + } + + const app_id = $('#ovr-app-id').val().trim(); + + if (app_id && overridesByKey.has(app_id)) { + overridesByKey.delete(app_id); + } else { + await fetchOverrides(); + } + + bumpBuster(app_id); + finishOverrideMutationAndRefresh(app_id); + } + + // ----------------- DOM bindings for modal & DnD ----------------- + const initDomBindings = () => { + // Banner change + $('#ovr-banner-file').off('change').on('change', function () { + const f = this.files && this.files[0]; + $(this).data('pending', f || null); + $('#ovr-banner-remove').data('remove', false); + + if (!f) { + const current = $('#ovr-banner-preview-img').data('ovr') ? $('#ovr-banner-preview-img').attr('src') : null; + $('#ovr-banner-preview-img').attr('src', current || DEFAULT_BANNER()); + $('#ovr-banner-remove').toggle(!!$('#ovr-banner-preview-img').data('ovr')); + return; + } + + cropBannerFileToDataURL(f, (dataURL) => { + if (!dataURL) { + const fr = new FileReader(); + fr.onload = e => { + $('#ovr-banner-preview-img').attr('src', e.target.result); + $('#ovr-banner-remove').show(); + $('#ovr-banner-preview-img').removeData('ovr'); + }; + fr.readAsDataURL(f); + } else { + $('#ovr-banner-preview-img').attr('src', dataURL); + $('#ovr-banner-remove').show(); + $('#ovr-banner-preview-img').removeData('ovr'); + } + + if (!$('#ovr-icon-file').data('pending') && !$('#ovr-icon-preview-img').data('ovr')) { + cropIconFileToDataURL(f, (iconURL) => { + $('#ovr-icon-preview-img').attr('src', iconURL || DEFAULT_ICON()); + if (iconURL) { $('#ovr-icon-remove').show(); $('#ovr-icon-preview-img').removeData('ovr'); } + }); + } + }); + }); + + // Icon change + $('#ovr-icon-file').off('change').on('change', function () { + const f = this.files && this.files[0]; + $(this).data('pending', f || null); + $('#ovr-icon-remove').data('remove', false); + + if (!f) { + const current = $('#ovr-icon-preview-img').data('ovr') ? $('#ovr-icon-preview-img').attr('src') : null; + $('#ovr-icon-preview-img').attr('src', current || DEFAULT_ICON()); + $('#ovr-icon-remove').toggle(!!$('#ovr-icon-preview-img').data('ovr')); + return; + } + + cropIconFileToDataURL(f, (dataURL) => { + if (!dataURL) { + const fr = new FileReader(); + fr.onload = e => { + $('#ovr-icon-preview-img').attr('src', e.target.result); + $('#ovr-icon-remove').show(); + $('#ovr-icon-preview-img').removeData('ovr'); + }; + fr.readAsDataURL(f); + } else { + $('#ovr-icon-preview-img').attr('src', dataURL); + $('#ovr-icon-remove').show(); + $('#ovr-icon-preview-img').removeData('ovr'); + } + + if (!$('#ovr-banner-file').data('pending') && !$('#ovr-banner-preview-img').data('ovr')) { + cropBannerFileToDataURL(f, (bannerURL) => { + $('#ovr-banner-preview-img').attr('src', bannerURL || DEFAULT_BANNER()); + if (bannerURL) { $('#ovr-banner-remove').show(); $('#ovr-banner-preview-img').removeData('ovr'); } + }); + } + }); + }); + + // Remove buttons + $('#ovr-banner-remove').off('click').on('click', function () { + $('#ovr-banner-file').data('pending', null).val(''); + $(this).data('remove', true); + $('#ovr-banner-preview-img').removeData('ovr').attr('src', DEFAULT_BANNER()); + $(this).hide(); + }); + + $('#ovr-icon-remove').off('click').on('click', function () { + $('#ovr-icon-file').data('pending', null).val(''); + $(this).data('remove', true); + $('#ovr-icon-preview-img').removeData('ovr').attr('src', DEFAULT_ICON()); + $(this).hide(); + }); + + // Pencil/edit buttons + $('#ovr-banner-edit').off('click').on('click', () => $('#ovr-banner-file').trigger('click')); + $('#ovr-icon-edit').off('click').on('click', () => $('#ovr-icon-file').trigger('click')); + + // Drag & drop zones + const wireDropZone = ($wrap, kind) => { + const over = () => $wrap.addClass('dragover'); + const out = () => $wrap.removeClass('dragover'); + + $wrap.on('dragenter dragover', (e) => { e.preventDefault(); e.stopPropagation(); over(); }); + $wrap.on('dragleave dragend drop', (e) => { e.preventDefault(); e.stopPropagation(); out(); }); + + $wrap.on('drop', (e) => { + const dt = e.originalEvent.dataTransfer; + if (!dt || !dt.files || !dt.files.length) return; + + const file = dt.files[0]; + if (!file || !file.type || !file.type.startsWith('image/')) return; + + if (kind === 'banner') { + try { $('#ovr-banner-file')[0].files = dt.files; $('#ovr-banner-file').trigger('change'); } + catch { + $('#ovr-banner-file').data('pending', file); + $('#ovr-banner-remove').data('remove', false); + cropBannerFileToDataURL(file, (dataURL) => { + if (dataURL) { + $('#ovr-banner-preview-img').attr('src', dataURL).removeData('ovr'); + $('#ovr-banner-remove').show(); + } + if (!$('#ovr-icon-preview-img').data('ovr')) { + cropIconFileToDataURL(file, (iconURL) => { + $('#ovr-icon-preview-img').attr('src', iconURL || DEFAULT_ICON()); + if (iconURL) { $('#ovr-icon-remove').show(); $('#ovr-icon-preview-img').removeData('ovr'); } + }); + } + }); + } + } else { + try { $('#ovr-icon-file')[0].files = dt.files; $('#ovr-icon-file').trigger('change'); } + catch { + $('#ovr-icon-file').data('pending', file); + $('#ovr-icon-remove').data('remove', false); + cropIconFileToDataURL(file, (dataURL) => { + if (dataURL) { + $('#ovr-icon-preview-img').attr('src', dataURL).removeData('ovr'); + $('#ovr-icon-remove').show(); + } + if (!$('#ovr-banner-preview-img').data('ovr')) { + cropBannerFileToDataURL(file, (bannerURL) => { + $('#ovr-banner-preview-img').attr('src', bannerURL || DEFAULT_BANNER()); + if (bannerURL) { $('#ovr-banner-remove').show(); $('#ovr-banner-preview-img').removeData('ovr'); } + }); + } + }); + } + } + }); + } + + wireDropZone($('#ovr-banner-preview'), 'banner'); + wireDropZone($('#ovr-icon-preview'), 'icon'); + + // Save/Reset buttons + $('#btn-save-override').off('click').on('click', saveOverride); + $('#btn-reset-override').off('click').on('click', resetOverride); + + // Date picker (if supported) + $('#ov-release-date').off('click').on('click', function() { this.showPicker?.(); }); + + // Mark "everEdited" for region/description/version/release_date + $('#ovr-region').off('input').on('input', function(){ $(this).data('everEdited', true); }); + $('#ovr-description').off('input').on('input', function(){ $(this).data('everEdited', true); }); + $('#ovr-version').off('input').on('input', function(){ $(this).data('everEdited', true); }); + $('#ov-release-date').off('change input').on('change input', function(){ $(this).data('everEdited', true); }); + + // Title ID "click to edit" + $('#ovTitleIdEditBtn').off('click').on('click', function () { + $('#ovTitleIdEditRow').removeClass('d-none'); + $('#ovTitleIdEditBtn').addClass('d-none'); + // Mark that the user intentionally entered edit mode + $('#ovCorrectedTitleId').data('everEdited', true); + // Focus input and place caret at end + const $inp = $('#ovCorrectedTitleId'); + const v = $inp.val() || ''; + $inp.focus().val('').val(v); + }); + + // Also allow clicking the displayed Title ID to enter edit mode + $('#ovTitleIdDisplay').off('click').on('click', function () { + $('#ovTitleIdEditBtn').trigger('click'); + }); + + // Cancel returns to collapsed view (discard any unsaved edits) + $('#ovTitleIdCancelBtn').off('click').on('click', function () { + const orig = $('#ovCorrectedTitleId').data('origTid') || ''; + $('#ovCorrectedTitleId').val(orig); + // User backed out; treat as never-edited for saving + $('#ovCorrectedTitleId').data('everEdited', false); + $('#ovTitleIdEditRow').addClass('d-none'); + $('#ovTitleIdEditBtn').removeClass('d-none'); + }); + + // Mark that the Name field was intentionally edited + $('#ovr-name').off('input').on('input', function () { + $(this).data('everEdited', true); + }); + } + + // ----------------- Public API ----------------- + const overridesApi = { + // wiring + bindEnvironment(opts = {}) { + env.getGames = opts.getGames || null; + env.applyFilters = opts.applyFilters || null; + }, + initDomBindings, + + // fetching/overlay + fetchOverrides, + reapplyAllOverridesToGames, + hasActiveOverride, + bannerUrlFor, + iconUrlFor, + openOverrideEditor, + + // flags helpers to compute once per game + computeRecognitionFlags, + isUnrecognizedGame, + pickTidForDisplay, + getOverrideForGame, + displayTitleFor, + + // redirects + getRedirectForApp, + applyRedirectToGame, + applyRedirectsToGames, + }; + namespace.Overrides = overridesApi; +})(window, window.jQuery); diff --git a/app/static/js/pagination.js b/app/static/js/pagination.js new file mode 100644 index 00000000..f1069126 --- /dev/null +++ b/app/static/js/pagination.js @@ -0,0 +1,183 @@ +'use strict'; + +(function (global, $) { + if (!$) { + console.warn('Ownfoil Pagination requires jQuery.'); + return; + } + + const clamp = (value, min, max) => Math.min(Math.max(value, min), max); + const toPositiveInteger = (value, fallback) => { + const num = Number(value); + if (Number.isFinite(num) && num > 0) { + return Math.floor(num); + } + return fallback; + }; + + const ns = global.Ownfoil = global.Ownfoil || {}; + + ns.Pagination = { + create(options = {}) { + const { + container, + getCurrentPage = () => 1, + setCurrentPage = () => {}, + getItemsPerPage = () => 1, + onPageChange = () => {}, + maxVisiblePages = 5, + labels = {} + } = options; + + const $container = typeof container === 'string' ? $(container) : $(container); + if (!$container || !$container.length) { + console.warn('Ownfoil Pagination: container not found.', container); + return { update: () => {} }; + } + + const resolvedLabels = { + first: labels.first || 'First page', + previous: labels.previous || 'Previous page', + next: labels.next || 'Next page', + last: labels.last || 'Last page', + go: labels.go || 'Go', + goToPage: labels.goToPage || 'Go to page' + }; + + const maxVisible = Math.max(1, toPositiveInteger(maxVisiblePages, 5)); + + return { + update(nbDisplayedGames) { + const itemsPerPage = toPositiveInteger(getItemsPerPage(), 1); + const totalPages = Math.max(1, Math.ceil(nbDisplayedGames / itemsPerPage)); + const hasResults = nbDisplayedGames > 0; + + const originalPage = toPositiveInteger(getCurrentPage(), 1); + let currentPage = clamp(originalPage, 1, totalPages); + + if (currentPage !== originalPage) { + setCurrentPage(currentPage); + } + + const goToPage = (page) => { + const nextPage = clamp(page, 1, totalPages); + if (nextPage === currentPage) return; + currentPage = nextPage; + setCurrentPage(nextPage); + onPageChange(nextPage); + }; + + const appendControlButton = (targetPage, html, ariaLabel, disabled) => { + const item = $('
  • '); + if (disabled) item.addClass('disabled'); + + const link = $(`${html}`); + if (ariaLabel) link.attr('aria-label', ariaLabel); + link.on('click', (event) => { + event.preventDefault(); + if (disabled) return; + goToPage(targetPage); + }); + item.append(link); + $container.append(item); + }; + + const appendPageNumber = (page) => { + const isActive = page === currentPage; + const item = $('
  • '); + if (isActive) item.addClass('active'); + + const link = $(`${page}`); + if (isActive) link.attr('aria-current', 'page'); + link.on('click', (event) => { + event.preventDefault(); + if (isActive) return; + goToPage(page); + }); + item.append(link); + $container.append(item); + }; + + const appendEllipsis = () => { + const item = $('
  • '); + item.append('…'); + $container.append(item); + }; + + $container.empty(); + + appendControlButton(1, '', resolvedLabels.first, currentPage === 1 || !hasResults); + appendControlButton(currentPage - 1, '', resolvedLabels.previous, currentPage === 1 || !hasResults); + + if (totalPages <= maxVisible + 2) { + for (let page = 1; page <= totalPages; page++) { + appendPageNumber(page); + } + } else { + appendPageNumber(1); + + const halfWindow = Math.floor(maxVisible / 2); + let start = Math.max(2, currentPage - halfWindow); + let end = Math.min(totalPages - 1, currentPage + halfWindow); + + if (currentPage <= halfWindow + 1) { + start = 2; + end = start + maxVisible - 1; + } else if (currentPage >= totalPages - halfWindow) { + end = totalPages - 1; + start = end - maxVisible + 1; + } + + start = Math.max(2, start); + end = Math.min(totalPages - 1, end); + + if (start > 2) appendEllipsis(); + + for (let page = start; page <= end; page++) { + appendPageNumber(page); + } + + if (end < totalPages - 1) appendEllipsis(); + + appendPageNumber(totalPages); + } + + appendControlButton(currentPage + 1, '', resolvedLabels.next, currentPage === totalPages || !hasResults); + appendControlButton(totalPages, '', resolvedLabels.last, currentPage === totalPages || !hasResults); + + if (hasResults && totalPages > 1) { + const jumpItem = $('
  • '); + const jumpGroup = $(` +
    + + +
    + `); + + const jumpInput = jumpGroup.find('input'); + const jumpButton = jumpGroup.find('button'); + const commitJump = () => { + const rawValue = parseInt(jumpInput.val(), 10); + if (!Number.isFinite(rawValue)) return; + goToPage(rawValue); + }; + + jumpButton.on('click', (event) => { + event.preventDefault(); + commitJump(); + }); + jumpInput.on('keydown', (event) => { + if (event.key === 'Enter') { + event.preventDefault(); + commitJump(); + } + }); + + jumpItem.append(jumpGroup); + $container.append(jumpItem); + } + } + }; + } + }; +})(window, window.jQuery); diff --git a/app/static/style.css b/app/static/style.css index 99650b36..31b7152d 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -25,6 +25,10 @@ margin-right: 5px; } +.game-tag:last-child { + margin-right: 0; +} + .card { border-radius: 8px !important; /* linear-gradient(to top, rgba(0, 0, 0, 0.7), transparent 50%), */ @@ -121,4 +125,137 @@ p.game-description { .dropdown-item:focus { background: none; box-shadow: none; -} \ No newline at end of file +} + +/* Override styles */ +.override-pill { + font-size: .85rem; + line-height: 1; +} +.override-pill:hover { opacity: .95; } + +.ovr-preview-grid { + display: grid; + gap: .75rem; + grid-template-columns: 1fr 1fr; + align-items: start; +} +.ovr-preview-wrap { + position: relative; + background: rgba(255,255,255,.03); + padding: .5rem; + border-radius: .5rem; + border: 1px solid rgba(255,255,255,.15); + transition: border-color .15s ease, background .15s ease; +} +.ovr-preview-wrap.dragover { + border-color: #0d6efd; + background: rgba(13,110,253,.1); +} +.ovr-preview-banner { + width: 100%; + aspect-ratio: 16 / 9; /* 400x225 */ + object-fit: cover; + display: block; +} +.ovr-preview-icon { + width: 100%; + aspect-ratio: 1 / 1; /* 400x400 */ + object-fit: cover; + display: block; +} +.ovr-caption { + font-size: .8rem; + opacity: .75; + margin-top: .25rem; + text-align: center; +} + +/* remove button: top-left */ +.ovr-remove-x { + position: absolute; + top: .35rem; + left: .35rem; + width: 28px; + height: 28px; + line-height: 24px; + text-align: center; + border-radius: 50%; + border: 2px solid #000; + background: #fff; + color: #000; + font-weight: 800; + font-size: 18px; + cursor: pointer; + opacity: .9; + box-shadow: 0 1px 2px rgba(0,0,0,.4); + padding: 0; +} +.ovr-remove-x:hover { opacity: 1; } + +/* edit button: top-right */ +.ovr-edit-btn { + position: absolute; + top: .35rem; + right: .35rem; + width: 30px; + height: 30px; + border-radius: 8px; + border: none; + display: inline-flex; + align-items: center; + justify-content: center; + background: rgba(255,255,255,.9); + color: #000; + cursor: pointer; + box-shadow: 0 1px 2px rgba(0,0,0,.4); +} +.ovr-edit-btn:hover { + background: #fff; +} + +#ovTitleIdDisplay { user-select: text; } + +/* Pagination controls */ +#paginationControls { + flex-wrap: wrap; + gap: .35rem; +} + +#paginationControls .page-item { + margin: 0; +} + +#paginationControls .page-link { + min-width: 2.25rem; + text-align: center; +} + +#paginationControls .page-ellipsis .page-link { + pointer-events: none; +} + +#paginationControls .page-jump { + align-self: stretch; +} + +#paginationControls .page-jump .page-jump-group { + max-width: 150px; + height: 100%; +} + +#paginationControls .page-jump input { + min-width: 0; +} + +@media (max-width: 576px) { + #paginationControls .page-jump { + flex: 1 0 100%; + margin-left: 0 !important; + } + + #paginationControls .page-jump .page-jump-group { + max-width: none; + width: 100%; + } +} diff --git a/app/templates/index.html b/app/templates/index.html index 49d85712..3d533792 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -1,7 +1,12 @@ {% extends "base.html" %} - {% block content %} + {% include 'nav.html' %}
    @@ -80,6 +85,23 @@
    +
  • +
    +
    • +
  • +
    +
    + + + + + + + + +
    +
    +
    • + diff --git a/app/templates/settings.html b/app/templates/settings.html index 2b199a79..84fde94a 100644 --- a/app/templates/settings.html +++ b/app/templates/settings.html @@ -352,6 +352,22 @@

    Shop

    successfully loading your shop. + +
    + + +
    + Used to build the default banner/icon when artwork is missing (max 30 characters). +
    +
    +
    +
    @@ -365,7 +381,8 @@

    Shop