diff --git a/echeapi/__init__.py b/echeapi/__init__.py index 7f7d58f..4f3fd69 100644 --- a/echeapi/__init__.py +++ b/echeapi/__init__.py @@ -5,6 +5,7 @@ from jinja_markdown import MarkdownExtension from flask import Flask +from flask_caching import Cache from echeapi import settings from echeapi.interface.views import discover @@ -22,4 +23,10 @@ app.secret_key = settings.SECRET_KEY app.logger.addHandler(file_handler) +cache = Cache(config={ + 'CACHE_KEY_PREFIX': __name__, + **settings.CACHE_CONFIG, +}) +cache.init_app(app) + discover() diff --git a/echeapi/interface/views/api.py b/echeapi/interface/views/api.py index 8aa7b7d..4a5c149 100644 --- a/echeapi/interface/views/api.py +++ b/echeapi/interface/views/api.py @@ -3,7 +3,7 @@ from flask import jsonify, request, Response -from echeapi import app, settings +from echeapi import app, cache, settings from echeapi.utils import api @@ -54,10 +54,15 @@ def eche_list(key=None, value=None): filter = (key, value) - try: - body = api.as_json(fields=fields, filter=filter) - except Exception: - app.logger.exception('Error while fetching API data') - raise ApiError(500, 'Server error') + _key = f'{",".join(fields)}:{key}:{value}' + body = cache.get(_key) + if body is None: + try: + body = api.as_json(fields=fields, filter=filter) + except Exception: + app.logger.exception('Error while fetching API data') + raise ApiError(500, 'Server error') + else: + cache.set(_key, body, timeout=settings.DATA_CACHE_TIMEOUT) return Response(body, mimetype='application/json') diff --git a/echeapi/interface/views/pages.py b/echeapi/interface/views/pages.py index d95cf14..24c22c6 100644 --- a/echeapi/interface/views/pages.py +++ b/echeapi/interface/views/pages.py @@ -1,11 +1,13 @@ from flask import flash, Markup, render_template +from flask_cachecontrol import cache_for -from echeapi import app, settings +from echeapi import app, cache, settings from echeapi.utils import api, doctree @app.route("/") +@cache_for(seconds=settings.CACHE_CONTROL_MAX_AGE) def index(): return render_template( 'page/home.html', @@ -15,6 +17,7 @@ def index(): @app.route("/docs/") @app.route("/docs/") +@cache_for(seconds=settings.CACHE_CONTROL_MAX_AGE) def docs(params=''): if not params: params = settings.DOCS_DEFAULT @@ -23,7 +26,7 @@ def docs(params=''): htag = 'h1' main = Markup(render_template('snippets/docs-placeholder.html')) - if display == doctree.DISPLAY_MD: + if display == doctree.Display.FILE: htag = 'h5' main = Markup( render_template( @@ -32,11 +35,11 @@ def docs(params=''): ), ) - elif display == doctree.DISPLAY_DIR: + elif display == doctree.Display.DIR: if params: flash(content, category='warning') - elif display == doctree.DISPLAY_ERR: + elif display == doctree.Display.ERROR: flash(f'Path is invalid: {params}', category='error') else: @@ -65,18 +68,26 @@ def docs(params=''): @app.route("/explore/") +@cache_for(seconds=settings.CACHE_CONTROL_MAX_AGE) def explore(): - content = Markup(api.as_html( - table_id='echeTable', - classes=['table', 'table-striped', 'small'], - )) + _key = 'explore-table' + + table = cache.get(_key) + if table is None: + table = Markup(api.as_html( + table_id='echeTable', + classes=['table', 'table-striped', 'small'], + )) + cache.set(_key, table, timeout=settings.DATA_CACHE_TIMEOUT) + return render_template( 'page/explore.html', menu_parent='explore', - content=content, + content=table, ) @app.route("/openapi/") +@cache_for(seconds=settings.CACHE_CONTROL_MAX_AGE) def openapi(): return render_template('redoc/index.html') diff --git a/echeapi/scripts/initialize.py b/echeapi/scripts/initialize.py index 60e1775..74f2443 100644 --- a/echeapi/scripts/initialize.py +++ b/echeapi/scripts/initialize.py @@ -1,8 +1,9 @@ -from echeapi import settings +from echeapi import cache, settings from echeapi.utils import db def main(*args): db.initialize() + cache.clear() print(f'Created empty database {settings.DB_FILENAME}') diff --git a/echeapi/scripts/populate.py b/echeapi/scripts/populate.py index ac40264..d09b890 100644 --- a/echeapi/scripts/populate.py +++ b/echeapi/scripts/populate.py @@ -2,7 +2,7 @@ import os import sys -from echeapi import settings +from echeapi import cache, settings from echeapi.processing import country, erasmus from echeapi.utils import db, eche @@ -30,4 +30,7 @@ def main(*args): # Save DataFrame in the database. db.save(df) + # Clear cached data. + cache.clear() + print(f'ECHE data loaded to {settings.DB_FILENAME}') diff --git a/echeapi/settings/base.py b/echeapi/settings/base.py index ee9ce47..5c7f905 100644 --- a/echeapi/settings/base.py +++ b/echeapi/settings/base.py @@ -54,3 +54,17 @@ 'erasmusCodeCountryCode', 'countryCode', ] + +# Cache control max age for static pages. +CACHE_CONTROL_MAX_AGE = 600 + +# Cache control max age for data pages. +DATA_CACHE_TIMEOUT = 60 * 60 * 24 * 365 + +# Cache configuration. +CACHE_CONFIG = { + 'CACHE_TYPE': 'SimpleCache', + # 'CACHE_TYPE': 'RedisCache', + # 'CACHE_REDIS_URL': 'redis://localhost:6379/0', + 'CACHE_DEFAULT_TIMEOUT': 300, +} diff --git a/echeapi/utils/doctree.py b/echeapi/utils/doctree.py index ffc7aed..acb9893 100644 --- a/echeapi/utils/doctree.py +++ b/echeapi/utils/doctree.py @@ -1,25 +1,28 @@ import os +from enum import Enum from echeapi import settings -DISPLAY_DIR = 'DISPLAY_DIR' -DISPLAY_MD = 'DISPLAY_MD' -DISPLAY_ERR = 'display_error' + +class Display(Enum): + DIR = 'dir' + FILE = 'file' + ERROR = 'error' def fetch(args): path = os.path.join(settings.DOCS_DIR, *args) if os.path.isdir(path): - display = DISPLAY_DIR + display = Display.DIR content = f"This is a directory: docs/{os.path.relpath(path, settings.DOCS_DIR)}" elif os.path.isfile(path): - display = DISPLAY_MD + display = Display.FILE with open(path) as f: content = f.read() else: - display = DISPLAY_ERR + display = Display.ERROR content = "Cannot display the requested content." return display, content diff --git a/requirements-top-level.txt b/requirements-top-level.txt new file mode 100644 index 0000000..2cbb294 --- /dev/null +++ b/requirements-top-level.txt @@ -0,0 +1,8 @@ +Flask +Flask-CacheControl +Flask-Caching +jinja-markdown +Markdown +openpyxl +pandas +redis diff --git a/requirements.txt b/requirements.txt index 007d2af..a94b845 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,26 @@ +async-timeout==4.0.2 +cachelib==0.8.0 click==8.1.3 +Deprecated==1.2.13 et-xmlfile==1.1.0 Flask==2.1.2 -importlib-metadata==4.11.4 +Flask-CacheControl==0.3.0 +Flask-Caching==1.11.1 itsdangerous==2.1.2 jinja-markdown==1.210911 Jinja2==3.1.2 Markdown==3.3.7 MarkupSafe==2.1.1 -numpy==1.22.4 +numpy==1.23.0 openpyxl==3.0.10 -pandas==1.4.2 +packaging==21.3 +pandas==1.4.3 Pygments==2.12.0 -pymdown-extensions==9.4 +pymdown-extensions==9.5 +pyparsing==3.0.9 python-dateutil==2.8.2 pytz==2022.1 +redis==4.3.3 six==1.16.0 Werkzeug==2.1.2 -zipp==3.8.0 +wrapt==1.14.1 diff --git a/setup.cfg b/setup.cfg index b9febd9..57b0725 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,10 @@ ignore = [isort] combine_as_imports = True -known_flask = flask +known_flask = + flask + flask_caching + flask_cachecontrol known_first_party = echeapi line_length = 130 multi_line_output = 2