diff --git a/.lando.yml b/.lando.yml index 5220465420..255997a470 100644 --- a/.lando.yml +++ b/.lando.yml @@ -3,33 +3,66 @@ name: janeway # CAREFUL with this env file, Docker env files don't support interpolation! env_file: - - dockerfiles/lando_defaults.env - - dockerfiles/lando_local.env + - etc/defaults.env + +proxy: + appserver: + - janeway.lndo.site services: appserver: type: python:3.9 - command: python /app/src/manage.py runserver 0.0.0.0:8000 -v 3 + port: 8000 + command: /bin/sh /app/dockerfiles/lando-runserver-loop.sh build_as_root: - - grep '^deb ' /etc/apt/sources.list | perl -pe 's/deb /deb-src /' >> /etc/apt/sources.list - - apt-get update -qq && apt-get install -y python3-lxml pylint libxml2-dev libxslt1-dev python3-dev zlib1g-dev lib32z1-dev libffi-dev libssl-dev libjpeg-dev && apt-get build-dep -y lxml + - | + rm -f /etc/apt/sources.list.d/debian-src.sources + if [ -f /etc/apt/sources.list.d/debian.sources ]; then + sed 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources > /etc/apt/sources.list.d/debian-with-src.sources + rm -f /etc/apt/sources.list.d/debian.sources + mv /etc/apt/sources.list.d/debian-with-src.sources /etc/apt/sources.list.d/debian.sources + fi + - | + set -eu + packages="python3-lxml pylint libxml2-dev libxslt1-dev python3-dev zlib1g-dev libffi-dev libssl-dev libjpeg-dev" + arch="$(dpkg --print-architecture)" + if [ "$arch" = "amd64" ] || [ "$arch" = "i386" ]; then + packages="$packages lib32z1-dev" + fi + apt-get update -qq + apt-get install -y $packages + apt-get build-dep -y lxml build: - - cd /app && pip install --upgrade pip && pip install Cython && pip install psycopg2-binary~=2.8.0 && pip install -r requirements.txt && pip install -r dev-requirements.txt - scanner: true + - cd /app && pip install --upgrade pip && pip install Cython && pip install psycopg2-binary~=2.8.0 && pip install -r requirements.txt && pip install -r dev-requirements.txt + scanner: + path: / overrides: - ports: - - '8000:8000' environment: - PYTHONPATH: "/app/src" - JANEWAY_SETTINGS_MODULE: "core.settings" # this is the path to the Janeway settings file, which is in src/core/settings.py + DB_HOST: "db" + # Include vendored editable package roots so Django app labels resolve + # to the actual package instead of a namespace package shell. + PYTHONPATH: "/app/src:/app/src/foundationform" + JANEWAY_SETTINGS_MODULE: "core.dev_settings" + labels: + # Janeway runs its Django development server on port 8000, but Lando's + # default proxy wiring for python services assumes port 80. Keep an + # explicit high-priority route to the real application port. + traefik.enable: "true" + traefik.docker.network: landoproxyhyperion5000gandalfedition_edge + traefik.http.routers.janeway.entrypoints: http + traefik.http.routers.janeway.rule: Host(`janeway.lndo.site`) + traefik.http.routers.janeway.priority: "100" + traefik.http.routers.janeway.service: janeway-service + traefik.http.services.janeway-service.loadbalancer.server.port: "8000" moreHttpPorts: - '8000' + db: type: postgres:15 portforward: true creds: user: postgres - password: + password: database: janeway tooling: @@ -37,24 +70,29 @@ tooling: service: appserver cmd: python dir: /app/src + pip: service: appserver description: run pip commands to manage Python package installation in this environment cmd: pip dir: /app/src + manage: service: appserver cmd: python manage.py dir: /app/src + restart-runserver: service: appserver - description: Quickly restarts the Django runserver process, follows logs, and enables interactive debugging - cmd: killall python || python /app/src/manage.py runserver 0.0.0.0:8000 -v 3 + description: Restarts the Django runserver by stopping it and letting the appserver command relaunch it + cmd: pkill -f "/app/src/manage.py runserver" || true dir: /app/src user: root + psql: service: db cmd: psql -U postgres + 'db-import ': service: :host description: Imports a dump file into a database service diff --git a/README.md b/README.md index 172cf49afc..f65e09a518 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ Janeway is written in Python (3.10+) and utilises the Django framework (4.2). # Installation instructions Developer installation [instructions are available in our documentation site](https://janeway.readthedocs.io/en/latest/dev/installation.html). +For local development with Lando, see the repository guide at [docs/source/dev/lando.rst](docs/source/dev/lando.rst). + A guide for installing on the live environment with [apache and mod_wsgi](https://github.com/openlibhums/janeway/wiki/Janeway%2C-Apache-and-WSGI) is also available. ## Running Janeway with docker @@ -16,13 +18,13 @@ Janeway's development server can be run within a docker container, avoiding the Docker is compatible with multiple architectures and Operating systems, if you need help installing docker, have a look at the [docker documentation](https://docs.docker.com/install/). Simarly to the native installation, Janeway can be installed in a docker environment by running ``make install`` and following the installation steps described [above](https://github.com/openlibhums/janeway/wiki/Installation). As a result, a database volume will be populated under janeway/db/postgres-data -Once installation is completed, just type ``make janeway`` to run janeway with a postgres backend (default behaviour). +Once installation is completed, run ``make janeway`` to run janeway with a postgres backend (default behaviour). If a change to the dependencies for Janeway is required, the Janeway container can be re-created with ``make rebuild``. The database volume will be preserved. In order to run a different RDBMS, the environment variable ``DB_VENDOR`` can be set to one of ``postgres``, ``mysql`` or ``sqlite``. e.g: ``DB_VENDOR=mysql make install && make`` -Uninstalling Janeway is as simple as running ``make uninstall`` which will delete all docker related containers as well as wipe the database volume. +To uninstall Janeway, run ``make uninstall``. This deletes the Docker-related containers and wipes the database volume. # Janeway design principles 1. No code should appear to work "by magic". Readability is key. diff --git a/dockerfiles/lando-runserver-loop.sh b/dockerfiles/lando-runserver-loop.sh new file mode 100644 index 0000000000..001ce090a1 --- /dev/null +++ b/dockerfiles/lando-runserver-loop.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -u + +while true; do + python3 /app/src/manage.py runserver 0.0.0.0:8000 -v 3 --noreload + echo "Janeway runserver exited, restarting in 2 seconds..." + sleep 2 +done diff --git a/docs/source/dev/installation.rst b/docs/source/dev/installation.rst index 004cbc5430..4b66c7e7f4 100644 --- a/docs/source/dev/installation.rst +++ b/docs/source/dev/installation.rst @@ -1,9 +1,16 @@ -Installation Guide -================== -There are a number of ways to get Janeway up and running. For development we recommend you use Docker with Postgres as the DB_VENDOR. - -Running Janeway with Docker and docker compose ----------------------------------------------- +Installation Guide +================== +There are a number of ways to get Janeway up and running. For development we recommend you use Docker with Postgres as the DB_VENDOR. + +If you want to develop Janeway with Lando, use the dedicated guide: + +.. toctree:: + :maxdepth: 1 + + lando + +Running Janeway with Docker and docker compose +---------------------------------------------- 1. Install ``docker`` and ``GNU Make``. 2. From the /path/to/janeway directory run ``make install``. 3. A docker environment will be provisioned, and shortly after the janeway install script will run. Follow the instructions on screen to complete the installation. @@ -33,11 +40,10 @@ The Makefile can be configured with a number of variables to change the way in w * By default, the database backend will come with a database named ``janeway``. If you want to Janeway against a different database (e.g.: you have multiple local databases) you can set the DB_NAME variable (e.g.: ``make install DB_NAME="janeway_staging"`` or ``make run DB_NAME=janeway_production`` * The ``JANEWAY_PORT`` variable allows you to change the port to which the Janeway development server will be bound to on your host (set this if port 8000 is already in use by another service on your host) -If you want to install custom python libraries for development, you can drop them into the dev_requirements.txt file and run ``make rebuild``. Rebuilding the container takes some time, so it is also possible to install python libraries in development mode. When installed in this manner, the library is mounted as a volume into the janeway container when you first run `make rebuild` and you will be able to make changes to the library without having to run ``make rebuild``. In order to install a library in development mode, copy the code to ``/path/to/janeway/lib/`` and run ``make rebuild`` once. - - -Native Install --------------- +If you want to install custom python libraries for development, you can drop them into the dev_requirements.txt file and run ``make rebuild``. Rebuilding the container takes some time, so it is also possible to install python libraries in development mode. When installed in this manner, the library is mounted as a volume into the janeway container when you first run `make rebuild` and you will be able to make changes to the library without having to run ``make rebuild``. In order to install a library in development mode, copy the code to ``/path/to/janeway/lib/`` and run ``make rebuild`` once. + +Native Install +-------------- The following is for Debian/Ubuntu-based systems (16.04). @@ -51,11 +57,14 @@ The following is for Debian/Ubuntu-based systems (16.04). 2. Install system dependencies. -On Ubuntu systems: -``sudo apt-get install libxml2-dev libxslt1-dev python3-dev zlib1g-dev lib32z1-dev libffi-dev libssl-dev libjpeg-dev libmysqlclient-dev`` - -On Debian systems: -``sudo apt-get install libxml2-dev libxslt1-dev python3-dev zlib1g-dev lib32z1-dev libffi-dev libssl-dev libjpeg-dev`` +On Ubuntu systems: +``sudo apt-get install libxml2-dev libxslt1-dev python3-dev zlib1g-dev libffi-dev libssl-dev libjpeg-dev libmysqlclient-dev`` + +On Debian systems: +``sudo apt-get install libxml2-dev libxslt1-dev python3-dev zlib1g-dev libffi-dev libssl-dev libjpeg-dev`` + +On ``amd64`` systems, you may also need ``lib32z1-dev``. It is not generally +available on ``arm64`` systems such as Apple Silicon. 3. Clone the janeway repo to your local machine: ``git clone https://github.com/BirkbeckCTP/janeway.git`` diff --git a/docs/source/dev/lando.rst b/docs/source/dev/lando.rst new file mode 100644 index 0000000000..12b46b322f --- /dev/null +++ b/docs/source/dev/lando.rst @@ -0,0 +1,399 @@ +Lando Development Setup +======================= + +This guide describes a working local Janeway development setup using Lando. +It is intended for contributors who want a repeatable setup on macOS, Linux, + or Windows without maintaining a separate native Python environment. + +Why this guide exists +--------------------- + +Janeway can run in either path mode or domain mode. For the Lando setup in +this repository, the most reliable local configuration is: + +* Janeway served at ``http://janeway.lndo.site/`` +* Janeway development settings loaded from ``core.dev_settings`` +* ``URL_CONFIG = "domain"`` in ``src/core/dev_settings.py`` + +Using path mode locally would require browsing to a URL like +``/Journal/``. The current Lando setup is intentionally configured for the +root-domain variant instead. + +The local Lando appserver also uses a small wrapper script, +``dockerfiles/lando-runserver-loop.sh``, to keep the Django development server +available after ``lando restart-runserver`` or a transient local crash. This +script is only used by the Lando appserver command in ``.lando.yml`` and does +not affect non-Lando installs or production deployments. + +Prerequisites +------------- + +1. Install Docker Desktop or Docker Engine. +2. Install Lando: https://lando.dev/ +3. Clone the Janeway repository. +4. Ensure port ``80`` is available on your machine for the Lando proxy. + +Recommended versions +-------------------- + +The setup has been validated with these versions: + +* Lando 3.25.x +* Docker Desktop 4.63.0 on macOS +* Docker Engine versions compatible with the above Lando release should also work on Linux + +Other versions may also work, but if you hit unexpected startup or proxy +behaviour, first compare your local Lando and Docker versions with the tested +combination above. + +What "multi-arch safe" means +---------------------------- + +Computers do not all use the same processor architecture. In local Docker +development, the two common ones are: + +* ``amd64`` for most Intel and AMD machines +* ``arm64`` for Apple Silicon Macs and many ARM-based Linux machines + +A multi-arch safe setup works on both without requiring contributors to keep +private overrides just to get through the initial build. The shared Janeway +Lando configuration installs ``lib32z1-dev`` only on ``amd64``-style +containers, because that package is not generally available on ``arm64``. + +First-time setup +---------------- + +1. Clone the repository and enter the project directory. + + .. code-block:: bash + + git clone https://github.com/openlibhums/janeway.git + cd janeway + +2. Start or rebuild the Lando app. + + .. code-block:: bash + + lando rebuild -y + +3. Run database migrations. + + .. code-block:: bash + + lando manage migrate + +4. Run the Janeway installer. + + To run a non-interactive local install, use: + + .. code-block:: bash + + lando manage install_janeway --use-defaults + + In some environments, the installer defaults may save ``localhost`` as the + Press domain and leave the first Journal domain blank. If that happens, + align both with the Lando host: + + .. code-block:: bash + + lando manage shell -c "from press.models import Press; from journal.models import Journal; p=Press.objects.first(); j=Journal.objects.first(); p.domain='janeway.lndo.site'; p.save(); j.domain='janeway.lndo.site'; j.save()" + +5. Create a superuser if you need access to Django admin features. + + .. code-block:: bash + + lando manage createsuperuser + +6. Open Janeway: + + .. code-block:: text + + http://janeway.lndo.site/ + + Use ``http`` for this local setup. ``https://janeway.lndo.site`` is not + configured as the primary development URL here and may return a proxy-level + error page. + +Useful commands +--------------- + +Start or rebuild the environment: + +.. code-block:: bash + + lando rebuild -y + +Open a Django shell: + +.. code-block:: bash + + lando manage shell + +Run a management command: + +.. code-block:: bash + + lando manage + +Restart only the Django development server: + +.. code-block:: bash + + lando restart-runserver + +This command intentionally stops the current ``runserver`` process and relies +on the Lando appserver wrapper script to bring it back automatically. + +Inspect the active URLs: + +.. code-block:: bash + + lando info + +Expected local settings +----------------------- + +The Lando setup in this repository expects the following local configuration: + +* ``src/core/dev_settings.py`` +* ``DEFAULT_HOST = "http://janeway.lndo.site"`` +* ``URL_CONFIG = "domain"`` + +Why ``URL_CONFIG = "domain"`` is needed +--------------------------------------- + +It is required for this Lando setup. + +The local proxy is configured so that Janeway is available at: + +.. code-block:: text + + http://janeway.lndo.site/ + +In path mode, Janeway expects URLs like: + +.. code-block:: text + + http://host/Journal/ + +That does not match the current Lando proxy design. Keeping local development +in domain mode avoids this mismatch and makes the local URL simpler. + +Platform notes +-------------- + +macOS +~~~~~ + +On some macOS systems, ``*.lndo.site`` may not resolve correctly in the +browser. If ``http://janeway.lndo.site/`` does not resolve, add a hosts entry: + +.. code-block:: text + + 127.0.0.1 janeway.lndo.site + +You will need administrator privileges to edit the hosts file. + +The hosts file path on macOS is: + +.. code-block:: text + + /etc/hosts + +If you are on Apple Silicon and still hit a Docker image or package issue that +does not occur on Intel, you can add a temporary local override: + +.. code-block:: yaml + + services: + appserver: + overrides: + platform: linux/amd64 + +Save that as ``.lando.local.yml`` in the project root before running +``lando rebuild -y``. This file is intentionally local-only and should not be +committed. + +Linux +~~~~~ + +If ``janeway.lndo.site`` does not resolve, add the same entry to: + +.. code-block:: text + + /etc/hosts + +using an account with permission to edit that file. + +Windows +~~~~~~~ + +If ``janeway.lndo.site`` does not resolve, add the same entry to: + +.. code-block:: text + + C:\Windows\System32\drivers\etc\hosts + +You will need to edit the file with administrator privileges. + +Troubleshooting +--------------- + +First boot after ``lando rebuild -y`` can be noisy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On a fresh setup, the first boot can show transient startup errors while: + +* Python dependencies are still finishing installation +* the database container is still becoming ready +* Django starts before migrations or initial Janeway data have been applied + +This does not necessarily mean the setup is broken. A normal first-run +sequence is: + +1. Run ``lando rebuild -y``. +2. Wait a few seconds after the rebuild completes. +3. Check whether the app is responding: + + .. code-block:: bash + + curl -I http://janeway.lndo.site/ + +4. If the appserver started but the database is not initialized yet, run: + + .. code-block:: bash + + lando manage migrate + lando manage install_janeway --use-defaults + +5. If Django started too early during the rebuild, restart only the runserver: + + .. code-block:: bash + + lando restart-runserver + +During first boot, ``502 Bad Gateway``, ``ECONNRESET``, or database errors such +as ``relation "journal_journal" does not exist`` usually mean the services are +still settling or Janeway has not been installed yet, not that the Lando setup +itself is invalid. + +If the installer has completed but the site still does not resolve correctly, +check whether the stored Press and Journal domains match the Lando host: + +.. code-block:: bash + + lando manage shell -c "from press.models import Press; from journal.models import Journal; print('press=', Press.objects.first().domain); print('journal=', Journal.objects.first().domain)" + +For this setup, both should resolve to: + +.. code-block:: text + + janeway.lndo.site + +``lando rebuild -y`` ends with ``502 Bad Gateway`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The proxy can be healthy before Django has fully restarted. Wait a few seconds, +then retry: + +.. code-block:: bash + + curl -I http://janeway.lndo.site/ + +If needed, restart the development server: + +.. code-block:: bash + + lando restart-runserver + +The appserver command uses a small restart loop so that this command can stop +the current Django process without leaving the container permanently without a +development server. + +``curl -I http://janeway.lndo.site/`` returns ``404 page not found`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Check that local development is using domain mode: + +* ``URL_CONFIG = "domain"`` +* ``DEFAULT_HOST = "http://janeway.lndo.site"`` + +This setup is not intended to use ``/Journal/`` URLs locally. + +``curl -I http://janeway.lndo.site/`` returns ``502 Bad Gateway`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Check whether Django is running: + +.. code-block:: bash + + docker logs --tail 120 janeway_appserver_1 + +Then check whether the hostname resolves inside the app container: + +.. code-block:: bash + + docker exec janeway_appserver_1 sh -lc "getent hosts db" + +If the database host resolves but Django started too early, restarting the +development server is often enough: + +.. code-block:: bash + + lando restart-runserver + +If ``lando restart-runserver`` leaves the site unavailable, ensure your +checkout includes the Lando-only wrapper script at +``dockerfiles/lando-runserver-loop.sh`` and the corresponding ``command`` in +``.lando.yml``. + +``https://janeway.lndo.site`` returns ``404`` or another proxy error +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This setup is documented and tested against: + +.. code-block:: text + + http://janeway.lndo.site/ + +If you open the same hostname over ``https``, the local proxy may return an +error page. Use the documented ``http`` URL unless you are explicitly testing +local HTTPS behaviour. + +On Safari, a cached HSTS policy can also cause the browser to reopen the same +hostname over ``https`` even after you saved the ``http`` version as a +bookmark. If that happens, make sure the ``lndo.site`` entry is not open in an +active Safari tab, then remove the stored website data and reopen the ``http`` +URL: + +1. Open ``Safari`` -> ``Settings``. +2. Go to ``Privacy``. +3. Click ``Manage Website Data...``. +4. Search for ``lndo.site``. +5. Close any open ``janeway.lndo.site`` tab or window. +6. Remove the stored ``lndo.site`` entry. +7. Open ``http://janeway.lndo.site/`` again in a new tab or window. + +``janeway.lndo.site`` does not resolve +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add a hosts entry for your platform: + +.. code-block:: text + + 127.0.0.1 janeway.lndo.site + +Why this should live in the open source repository +-------------------------------------------------- + +This guide belongs in the repository because it documents repository-specific +behaviour: + +* the required ``env_file`` path +* the Lando proxy hostname +* the current local URL mode +* the platform-specific hostname workaround + +Keeping it in versioned documentation makes it easier for contributors to +reproduce the setup and easier for maintainers to update the instructions when +the local development workflow changes. diff --git a/src/core/dev_settings.py b/src/core/dev_settings.py index fe1d3fd178..b08b91a8f4 100644 --- a/src/core/dev_settings.py +++ b/src/core/dev_settings.py @@ -7,7 +7,7 @@ SECRET_KEY = "uxprsdhk^gzd-r=_287byolxn)$k6tsd8_cepl^s^tms2w1qrv" # This is the default redirect if no other sites are found. -DEFAULT_HOST = "https://www.example.org" +DEFAULT_HOST = "http://janeway.lndo.site" EMAIL_BACKEND = ( os.environ.get( "JANEWAY_EMAIL_BACKEND", @@ -15,7 +15,7 @@ or "django.core.mail.backends.console.EmailBackend" ) -URL_CONFIG = "path" # path or domain +URL_CONFIG = "domain" # path or domain MIDDLEWARE = ( "utils.middleware.TimeMonitoring",