diff --git a/.gitignore b/.gitignore index 908cac1..3d11aed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ venv __pycache__ +.idea web/**/*.sqlite3 **/.env diff --git a/Makefile b/Makefile index 7436995..643a8c9 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,13 @@ +VENV_DIR := venv + +prepare-venv: + @command -v pyenv >/dev/null 2>&1 || { echo "pyenv not installed"; exit 1; } + pyenv install -s + python -m venv $(VENV_DIR) + $(VENV_DIR)/bin/pip install --upgrade pip + @echo "In order to use venv python, please run 'source venv/bin/activate', then install dependencies by running \ + 'make install-dev'." + prepare-web: pip install -r web/requirements.txt cp web/.env.example web/.env @@ -30,3 +40,9 @@ fix-files: python3 -m black . python3 -m isort . python3 -m flake8 . + +create-migrations: + python ./web/manage.py makemigrations + +migrate: + python ./web/manage.py migrate diff --git a/README.md b/README.md index 97beebb..2b8c154 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,18 @@ For a demonstration of a page with at least one link, see for example `{baseurl} ## Notes on installation and usage +To prepare pyenv (`venv`) make sure to have `pyenv` installed, then run +```bash +make prepare-env +source venv/bin/activate +``` + +By default, the Mathswitch uses sqlite3 database, in order to use postgresql there is +a provided `.env.example` config and Docker compose to run the database: +```bash +docker compose up -d +``` + To install all the necessary Python packages, run: ```bash @@ -22,19 +34,19 @@ cp web/.env.example web/.env Next, to create a database, run: ```bash -python manage.py migrate +python web/manage.py migrate ``` In order to use the administrative interface, you need to create an admin user: ```bash -python manage.py createsuperuser +python web/manage.py createsuperuser ``` Finally, to populate the database, run ```bash -python manage.py import_wikidata +python web/manage.py import_wikidata # OR make populate-db ``` @@ -52,7 +64,7 @@ make populate-db If you ever want to repopulate the database, you can clear it using ```bash -python manage.py clear_wikidata +python web/manage.py clear_wikidata ``` ### To run the categorizer @@ -100,12 +112,16 @@ black . && isort . && flake8 Each time after you change a model, make sure to create the appropriate migrations: ```bash -python manage.py makemigrations +python web/manage.py makemigrations +# OR +make create-migrations ``` To update the database with the new model, run: ```bash - python manage.py migrate +python web/manage.py migrate +# OR +make migrate ``` ## Instructions for Katja to update the live version diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b4e20d7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +volumes: + postgres_data: + +services: + db: + image: postgres:17-alpine + restart: unless-stopped + env_file: + - ./web/.env + ports: + - '${POSTGRES_PORT:-5432}:5432' + volumes: + - postgres_data:/var/lib/postgresql/data + profiles: + - '' + - db \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 28866a6..cc1bbec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,8 @@ [tool.black] extend-exclude = 'migrations' -target-version = ['py310'] +target-version = ['py312'] + [tool.isort] extend_skip_glob = ["*/migrations/*"] profile = "black" -py_version = 310 \ No newline at end of file +py_version = 312 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6fbba70..54cb991 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ black~=25.9.0 -isort~=5.12.0 +isort~=7.0.0 flake8~=7.3.0 -r ./web/requirements.txt diff --git a/web/.env.example b/web/.env.example index 40e274f..4f7a267 100644 --- a/web/.env.example +++ b/web/.env.example @@ -1,2 +1,10 @@ SECRET_KEY="django-insecure-9wy9w#vf^tde0262doyy_j19=64c()_qub!1)f+fh-b^=7ndw*" -WIKIPEDIA_CONTACT_EMAIL=my@email.com \ No newline at end of file +WIKIPEDIA_CONTACT_EMAIL=my@email.com + +# DB +USE_POSTGRES=True +POSTGRES_DB=mathsdb +POSTGRES_USER=mathsuser +POSTGRES_PASSWORD=mathspass +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 diff --git a/web/concepts/migrations/0015_alter_categorizerresult_llm_type.py b/web/concepts/migrations/0015_alter_categorizerresult_llm_type.py new file mode 100644 index 0000000..62475c9 --- /dev/null +++ b/web/concepts/migrations/0015_alter_categorizerresult_llm_type.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.27 on 2025-12-16 23:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("concepts", "0014_categorizerresult"), + ] + + operations = [ + migrations.AlterField( + model_name="categorizerresult", + name="llm_type", + field=models.CharField(max_length=100), + ), + ] diff --git a/web/requirements.txt b/web/requirements.txt index 4d6c286..6d19a56 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -1,4 +1,5 @@ Django~=4.2.6 +psycopg2-binary~=2.9.11 requests~=2.32.5 spacy~=3.7.0 --prefer-binary scispacy~=0.6.2 diff --git a/web/web/settings.py b/web/web/settings.py index 8b33153..1585c52 100644 --- a/web/web/settings.py +++ b/web/web/settings.py @@ -31,6 +31,23 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True +USE_POSTGRES = config("USE_POSTGRES", default=False, cast=bool) + +POSTGRES_DB = config("POSTGRES_DB", default=None) +POSTGRES_USER = config("POSTGRES_USER", default=None) +POSTGRES_PASSWORD = config("POSTGRES_PASSWORD", default=None) +POSTGRES_HOST = config("POSTGRES_HOST", default=None) +POSTGRES_PORT = config("POSTGRES_PORT", default="5432") + +IS_POSTGRES_CONFIGURED = all( + [ + POSTGRES_DB, + POSTGRES_USER, + POSTGRES_PASSWORD, + POSTGRES_HOST, + ] +) + ALLOWED_HOSTS = ["*"] @@ -90,6 +107,18 @@ } } +if USE_POSTGRES and IS_POSTGRES_CONFIGURED: + DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": POSTGRES_DB, + "USER": POSTGRES_USER, + "PASSWORD": POSTGRES_PASSWORD, + "HOST": POSTGRES_HOST, + "PORT": POSTGRES_PORT, + } + } + # Password validation # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators