From 523f1dc5097cf6fbf1c25f3cf3861469bcbd85fb Mon Sep 17 00:00:00 2001 From: Neraste Date: Sun, 28 Dec 2025 00:07:54 +0100 Subject: [PATCH 01/16] Apply good ideas from #170 --- .github/workflows/ci.yml | 2 +- README.md | 33 ++++++++++++++++--- dakara_server/dakara_server/settings/base.py | 14 ++++---- .../dakara_server/settings/development.py | 29 ++++++++-------- .../dakara_server/settings/production.py | 19 ++++++----- 5 files changed, 59 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31ca7bb4..f068b865 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: run: pip install -r requirements.txt -r requirements_dev.txt - name: Run tests - run: python -m pytest -v + run: python -m pytest -v --cov src - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 diff --git a/README.md b/README.md index 40945c06..8b4edfb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Dakara server +[![Python versions](https://img.shields.io/badge/python-3.10%20|%203.11%20|%203.12%20|%203.13-blue)](https://github.com/DakaraProject/dakara-server) +[![License](https://img.shields.io/github/license/mashape/apistatus.svg)](https://github.com/DakaraProject/dakara-server?tab=MIT-1-ov-file#readme) [![Tests status](https://github.com/DakaraProject/dakara-server/actions/workflows/ci.yml/badge.svg)](https://github.com/DakaraProject/dakara-server/actions/workflows/ci.yml) [![Codecov coverage analysis](https://codecov.io/gh/DakaraProject/dakara-server/branch/develop/graph/badge.svg)](https://codecov.io/gh/DakaraProject/dakara-server) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) @@ -11,7 +13,7 @@ Server for the Dakara project. ## Installation To install Dakara completely, you have to get all the parts of the project. -Installation guidelines are provided over here: +Installation guidelines are provided here: * [Dakara web client](https://github.com/DakaraProject/dakara-client-web/); * [Dakara player VLC](https://github.com/DakaraProject/dakara-player-vlc/); @@ -19,7 +21,7 @@ Installation guidelines are provided over here: ### System requirements -* Python3, to make everything up and running (supported versions: 3.10, 3.11, 3.12, and 3.13). +* Python3, to make everything up and running (supported versions: see above). Linux, Mac and Windows are supported. @@ -41,6 +43,27 @@ Install dependencies, at the root level of the repo (in the virtual environment) pip install -r requirements.txt ``` +## Setup + +### Settings presets + +The project provides settings presets: + +- "development": for development purpose only. Uses a SQLite database, has debug mode enabled, an in-terminal pseudo mail backend, and security features turned off. Do not use this preset for production! +- "test": for test purpose only. Uses an in-memory SQLite database, and has security features turned off. Do not use this preset for production! +- "production": for use in the provided Docker image. + +By default, the development preset is used. + +You can create your own settings preset by duplicating the production file. + +To select a preset, set the `DJANGO_SETTINGS_MODULE` environment variable accordingly, by instance for production: + + +```sh +export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" +``` + ### Setting up the server Let's create the server database, after loading the virtual environment, do: @@ -67,12 +90,12 @@ dakara_server/manage.py runserver The server part is now set up correctly. -### Web client, Feeder and player +### Web client, feeder and player Now setup the [web client](https://github.com/DakaraProject/dakara-client-web), [feeder](https://github.com/DakaraProject/dakara-feeder) and [player](https://github.com/DakaraProject/dakara-player-vlc) according to their respective documentations. -The feeder can authenticate to the server using a token, or a couple login/password, of a playlist manager account. +The feeder can authenticate to the server using a token or a couple login/password of a playlist manager account. The player can authenticate using a special token that only a playlist manager can generate. -Both token can be obtained from the web interface. +Both tokens can be obtained from the web interface. After all of this is setup, just grab some friends and have fun! diff --git a/dakara_server/dakara_server/settings/base.py b/dakara_server/dakara_server/settings/base.py index bb18c39e..b92f2c32 100644 --- a/dakara_server/dakara_server/settings/base.py +++ b/dakara_server/dakara_server/settings/base.py @@ -2,22 +2,22 @@ Django base settings for the Dakara server project. For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ +https://docs.djangoproject.com/en/5.1/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ +https://docs.djangoproject.com/en/5.1/ref/settings/ This file should not be modified if you are not a dev. """ -import os +from pathlib import Path from decouple import config from dakara_server.version import __date__ as DATE # noqa F401 from dakara_server.version import __version__ as VERSION # noqa F401 -BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +PASE_DIR = Path(__file__).resolve().parent.parent.parent # Application definition @@ -94,7 +94,7 @@ # Internationalization -# https://docs.djangoproject.com/en/2.2/topics/i18n/ +# https://docs.djangoproject.com/en/5.1/topics/i18n/ USE_I18N = True @@ -104,12 +104,10 @@ # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/2.2/howto/static-files/ +# https://docs.djangoproject.com/en/5.1/howto/static-files/ STATIC_URL = "/static/" -STATICFILES_DIRS = [os.path.join(BASE_DIR, "static")] - # Django REST config REST_FRAMEWORK = { "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), diff --git a/dakara_server/dakara_server/settings/development.py b/dakara_server/dakara_server/settings/development.py index 37d8682e..36902d97 100644 --- a/dakara_server/dakara_server/settings/development.py +++ b/dakara_server/dakara_server/settings/development.py @@ -2,22 +2,20 @@ Django local settings for the Dakara server project. For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ +https://docs.djangoproject.com/en/5.1/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ +https://docs.djangoproject.com/en/5.1/ref/settings/ -You should not modify this file directly. -To modify config values, set them as environment variables, -or in a config file in the dakara root directory: -either in a `.env` file -or in a `settings.ini` with a single `[settings]` section. +You should not modify this file directly. To modify config values, set them as +environment variables, or in a config file in the current working directory: +either in a `.env` file or in a `settings.ini` with a single `[settings]` +section. """ import os from decouple import config -from dj_database_url import parse as db_url os.environ.setdefault("HOST_URL", "http://localhost:3000") @@ -25,24 +23,25 @@ from dakara_server.settings.base import BASE_DIR # noqa E402 # Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ +# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ SECRET_KEY = "YourSecretKey" DEBUG = True ALLOWED_HOSTS = ["*"] # Django password security policy -# https://docs.djangoproject.com/en/2.2/topics/auth/passwords/#module-django.contrib.auth.password_validation +# https://docs.djangoproject.com/en/5.1/topics/auth/passwords/#module-django.contrib.auth.password_validation AUTH_PASSWORD_VALIDATORS = [] # Database -# https://docs.djangoproject.com/en/2.2/ref/settings/#databases -# `DATABASE_URL` is specified according to dj-databse-url plugin -# https://github.com/kennethreitz/dj-database-url#url-schema +# https://docs.djangoproject.com/en/5.1/ref/settings/#databases DATABASES = { - "default": db_url("sqlite:///" + os.path.join(BASE_DIR, "db.sqlite3")), + "default": { + "NAME": config("DATABASE_FILE", default=BASE_DIR / "db.sqlite3"), + "ENGINE": "django.db.backends.sqlite3", + } } # Channels @@ -51,7 +50,7 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}} # Internationalization -# https://docs.djangoproject.com/en/2.2/topics/i18n/ +# https://docs.djangoproject.com/en/5.1/topics/i18n/ LANGUAGE_CODE = "en-us" diff --git a/dakara_server/dakara_server/settings/production.py b/dakara_server/dakara_server/settings/production.py index 3b8b0c0c..53f95c0f 100644 --- a/dakara_server/dakara_server/settings/production.py +++ b/dakara_server/dakara_server/settings/production.py @@ -2,16 +2,17 @@ Django local settings for the Dakara server project. For more information on this file, see -https://docs.djangoproject.com/en/2.2/topics/settings/ +https://docs.djangoproject.com/en/5.1/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/2.2/ref/settings/ +https://docs.djangoproject.com/en/5.1/ref/settings/ -You should not modify this file directly. -To modify config values, set them as environment variables, -or in a config file in the dakara root directory: -either in a `.env` file -or in a `settings.ini` with a single `[settings]` section. +You should not modify this file directly. To modify config values, set them as +environment variables, or in a config file in the current worknig directory: +either in a `.env` file or in a `settings.ini` with a single `[settings]` +section. + +If you want to customize this file more, duplicate it under a different name. """ from decouple import Csv, config @@ -24,7 +25,7 @@ ALLOWED_HOSTS = config("ALLOWED_HOSTS", cast=Csv()) # Database -# https://docs.djangoproject.com/en/2.2/ref/settings/#databases +# https://docs.djangoproject.com/en/5.1/ref/settings/#databases # `DATABASE_URL` is specified according to dj-databse-url plugin # https://github.com/kennethreitz/dj-database-url#url-schema @@ -40,7 +41,7 @@ STATIC_ROOT = config("STATIC_ROOT") # Internationalization -# https://docs.djangoproject.com/en/2.2/topics/i18n/ +# https://docs.djangoproject.com/en/5.1/topics/i18n/ LANGUAGE_CODE = config("LANGUAGE_CODE", default="en-us") From 4d09e46c362188c19461cfe39ababfe1c2cc4761 Mon Sep 17 00:00:00 2001 From: Neraste Date: Mon, 29 Dec 2025 00:49:19 +0100 Subject: [PATCH 02/16] Create Docker configuration --- .dockerignore | 6 ++ Dockerfile | 52 +++++++++++++++++ dakara_server/dakara_server/settings/base.py | 4 +- .../dakara_server/settings/production.py | 23 +++++--- deployment/bin/daphne.sh | 9 +++ deployment/bin/gunicorn.sh | 38 +++++++++++++ deployment/bin/nginx.sh | 17 ++++++ deployment/bin/run.sh | 11 ++++ deployment/nginx/nginx.conf | 57 +++++++++++++++++++ deployment/supervisor/daphne.ini | 8 +++ deployment/supervisor/gunicorn.ini | 8 +++ deployment/supervisor/logging.ini | 5 ++ deployment/supervisor/nginx.ini | 8 +++ requirements.txt | 5 +- 14 files changed, 239 insertions(+), 12 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 deployment/bin/daphne.sh create mode 100755 deployment/bin/gunicorn.sh create mode 100755 deployment/bin/nginx.sh create mode 100755 deployment/bin/run.sh create mode 100644 deployment/nginx/nginx.conf create mode 100644 deployment/supervisor/daphne.ini create mode 100644 deployment/supervisor/gunicorn.ini create mode 100644 deployment/supervisor/logging.ini create mode 100644 deployment/supervisor/nginx.ini diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..91a8879e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +*.sqlite3 +*_cache +.git +.github +.venv +venv diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c5c65783 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,52 @@ +FROM alpine:3.23 + +ARG FRONT_VERSION="1.9.2" + +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +RUN apk add --no-cache \ + nginx \ + py3-pip \ + python3 \ + supervisor \ + unzip \ + wget + +COPY requirements.txt /app/ + +# install dependencies +RUN pip install \ + --no-cache-dir \ + --root-user-action ignore \ + --break-system-packages \ + -r /app/requirements.txt + +# get the front archive +RUN FRONT_ARCHIVE="dakara-client-web_$FRONT_VERSION.zip" && \ + wget \ + -P /tmp \ + https://github.com/DakaraProject/dakara-client-web/releases/download/$FRONT_VERSION/$FRONT_ARCHIVE && \ + unzip \ + /tmp/$FRONT_ARCHIVE \ + -d /app && \ + rm -rf \ + /tmp/$FRONT_ARCHIVE \ + /tmp/front + +COPY . /app + +COPY deployment/supervisor/daphne.ini /etc/supervisor.d/daphne.ini +COPY deployment/supervisor/gunicorn.ini /etc/supervisor.d/gunicorn.ini +COPY deployment/supervisor/logging.ini /etc/supervisor.d/logging.ini +COPY deployment/supervisor/nginx.ini /etc/supervisor.d/nginx.ini + +COPY deployment/nginx/nginx.conf /etc/nginx/nginx.conf + +EXPOSE 80 +VOLUME /data + +WORKDIR / + +CMD ["sh", "/app/deployment/bin/run.sh"] diff --git a/dakara_server/dakara_server/settings/base.py b/dakara_server/dakara_server/settings/base.py index b92f2c32..96a670e3 100644 --- a/dakara_server/dakara_server/settings/base.py +++ b/dakara_server/dakara_server/settings/base.py @@ -17,7 +17,7 @@ from dakara_server.version import __date__ as DATE # noqa F401 from dakara_server.version import __version__ as VERSION # noqa F401 -PASE_DIR = Path(__file__).resolve().parent.parent.parent +BASE_DIR = Path(__file__).resolve().parent.parent.parent # Application definition @@ -131,7 +131,7 @@ SENDER_EMAIL = config("SENDER_EMAIL", default="no-reply@example.com") -HOST_URL = config("HOST_URL") +HOST_URL = config("HOST_URL", default="http://example.com") EMAIL_ENABLED = config("EMAIL_ENABLED", default=True, cast=bool) diff --git a/dakara_server/dakara_server/settings/production.py b/dakara_server/dakara_server/settings/production.py index 53f95c0f..a4755a16 100644 --- a/dakara_server/dakara_server/settings/production.py +++ b/dakara_server/dakara_server/settings/production.py @@ -20,16 +20,21 @@ from dakara_server.settings.base import * # noqa F403 -SECRET_KEY = config("SECRET_KEY") +SECRET_KEY = config("SECRET_KEY", default="secret_key") DEBUG = config("DEBUG", cast=bool, default=False) -ALLOWED_HOSTS = config("ALLOWED_HOSTS", cast=Csv()) +ALLOWED_HOSTS = config("ALLOWED_HOSTS", cast=Csv(), default="[]") +CSRF_TRUSTED_ORIGINS = ["http://localhost"] # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases # `DATABASE_URL` is specified according to dj-databse-url plugin # https://github.com/kennethreitz/dj-database-url#url-schema -DATABASES = {"default": config("DATABASE_URL", cast=db_url)} +DATABASES = { + "default": config( + "DATABASE_URL", cast=db_url, default="mysql://dakara:dakara@mysql/dakara" + ) +} # Channels # http://channels.readthedocs.io/en/latest/topics/channel_layers.html @@ -38,12 +43,12 @@ # Static root # Should point to the static directory served by nginx -STATIC_ROOT = config("STATIC_ROOT") +STATIC_ROOT = config("STATIC_ROOT", "/app/static") # Internationalization # https://docs.djangoproject.com/en/5.1/topics/i18n/ -LANGUAGE_CODE = config("LANGUAGE_CODE", default="en-us") +LANGUAGE_CODE = config("LANGUAGE", default="en-us") TIME_ZONE = config("TIME_ZONE", default="UTC") @@ -73,9 +78,9 @@ "logfile": { "level": "DEBUG", "class": "logging.handlers.RotatingFileHandler", - "filename": config("LOG_FILE_PATH"), - "maxBytes": config("LOG_FILE_MAX_SIZE", cast=int), - "backupCount": config("LOG_FILE_BACKUP_COUNT", cast=int), + "filename": config("LOG_FILE_PATH", default="/data/logs/dakara_server.log"), + "maxBytes": config("LOG_FILE_MAX_SIZE", cast=int, default=1000000), + "backupCount": config("LOG_FILE_BACKUP_COUNT", cast=int, default=2), "formatter": "default", }, }, @@ -99,7 +104,7 @@ } # email backend -EMAIL_HOST = config("EMAIL_HOST") +EMAIL_HOST = config("EMAIL_HOST", default="postfix") EMAIL_PORT = config("EMAIL_PORT", cast=int, default="25") EMAIL_HOST_USER = config("EMAIL_HOST_USER", default="") EMAIL_HOST_PASSWORD = config("EMAIL_HOST_PASSWORD", default="") diff --git a/deployment/bin/daphne.sh b/deployment/bin/daphne.sh new file mode 100755 index 00000000..26374600 --- /dev/null +++ b/deployment/bin/daphne.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +# production preset +export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" + +# run daphne +daphne -b 0.0.0.0 -p 8001 dakara_server.asgi:application diff --git a/deployment/bin/gunicorn.sh b/deployment/bin/gunicorn.sh new file mode 100755 index 00000000..6465ea2a --- /dev/null +++ b/deployment/bin/gunicorn.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +set -e + +# production preset +export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" + +# create config file if needed +if [[ ! -f /data/config/gunicorn.conf.py ]] + echo "Create default custom configuration file for gunicorn" +then + cat >/data/config/gunicorn.conf.py </data/config/nginx-custom.conf <=3.9.1,<3.10.0 # to remove after updating channels APScheduler>=3.11.0,<3.12.0 +asgiref>=3.9.1,<3.10.0 # to remove after updating channels channels>=4.2.0,<4.3.0 daphne>=4.1.2,<4.2.0 dj-database-url>=2.3.0,<2.4.0 @@ -10,6 +10,9 @@ django-rest-registration>=0.9.0,<0.10.0 Django>=5.1.6,<5.2.0 djangorestframework>=3.15.2,<3.16.0 drf-spectacular==0.28.0 +gunicorn>=23.0.0,<23.1.0 packaging>=24.2,<24.3 python-decouple>=3.8,<3.9 setuptools>=75.8 +supervisor-stdlog>=0.7.9,<0.8.0 +tzdata>=2025.3,<2025.4 From c599a488e64b4a62b75436df172a726f6773a036 Mon Sep 17 00:00:00 2001 From: Neraste Date: Tue, 30 Dec 2025 14:14:09 +0100 Subject: [PATCH 03/16] Better custom configuration management --- Dockerfile | 12 +++++----- deployment/bin/daphne.sh | 24 +++++++++++++++++++- deployment/bin/gunicorn.sh | 16 ++++++------- deployment/bin/nginx.sh | 12 ++++------ deployment/config/daphne.conf | 2 ++ deployment/config/gunicorn.conf.py | 3 +++ deployment/config/nginx.conf | 3 +++ deployment/{ => etc}/nginx/nginx.conf | 0 deployment/{ => etc}/supervisor/daphne.ini | 0 deployment/{ => etc}/supervisor/gunicorn.ini | 0 deployment/{ => etc}/supervisor/logging.ini | 0 deployment/{ => etc}/supervisor/nginx.ini | 0 12 files changed, 50 insertions(+), 22 deletions(-) create mode 100644 deployment/config/daphne.conf create mode 100644 deployment/config/gunicorn.conf.py create mode 100644 deployment/config/nginx.conf rename deployment/{ => etc}/nginx/nginx.conf (100%) rename deployment/{ => etc}/supervisor/daphne.ini (100%) rename deployment/{ => etc}/supervisor/gunicorn.ini (100%) rename deployment/{ => etc}/supervisor/logging.ini (100%) rename deployment/{ => etc}/supervisor/nginx.ini (100%) diff --git a/Dockerfile b/Dockerfile index c5c65783..5828f026 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,14 +35,14 @@ RUN FRONT_ARCHIVE="dakara-client-web_$FRONT_VERSION.zip" && \ /tmp/$FRONT_ARCHIVE \ /tmp/front -COPY . /app +COPY deployment/etc/supervisor/daphne.ini /etc/supervisor.d/daphne.ini +COPY deployment/etc/supervisor/gunicorn.ini /etc/supervisor.d/gunicorn.ini +COPY deployment/etc/supervisor/logging.ini /etc/supervisor.d/logging.ini +COPY deployment/etc/supervisor/nginx.ini /etc/supervisor.d/nginx.ini -COPY deployment/supervisor/daphne.ini /etc/supervisor.d/daphne.ini -COPY deployment/supervisor/gunicorn.ini /etc/supervisor.d/gunicorn.ini -COPY deployment/supervisor/logging.ini /etc/supervisor.d/logging.ini -COPY deployment/supervisor/nginx.ini /etc/supervisor.d/nginx.ini +COPY deployment/etc/nginx/nginx.conf /etc/nginx/nginx.conf -COPY deployment/nginx/nginx.conf /etc/nginx/nginx.conf +COPY . /app EXPOSE 80 VOLUME /data diff --git a/deployment/bin/daphne.sh b/deployment/bin/daphne.sh index 26374600..ed6149a4 100755 --- a/deployment/bin/daphne.sh +++ b/deployment/bin/daphne.sh @@ -5,5 +5,27 @@ set -e # production preset export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" +# create config file once +if [[ ! -f /data/config/daphne.conf ]] +then + echo "Create default custom configuration file for daphne" + cp \ + /app/deployment/config/daphne.conf \ + /data/config/daphne.conf +fi + +# read config file +# remove comments and concat to one line +arguments=$(\ + sed \ + -e 's/[[:space:]]*#.*// ;/^[[:space:]]*$/d' \ + /data/config/daphne.conf \ + | awk '{printf("%s ", $0)}' \ + ) + # run daphne -daphne -b 0.0.0.0 -p 8001 dakara_server.asgi:application +daphne \ + -b 0.0.0.0 \ + -p 8001 \ + $arguments \ + dakara_server.asgi:application diff --git a/deployment/bin/gunicorn.sh b/deployment/bin/gunicorn.sh index 6465ea2a..9c062e40 100755 --- a/deployment/bin/gunicorn.sh +++ b/deployment/bin/gunicorn.sh @@ -5,15 +5,13 @@ set -e # production preset export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" -# create config file if needed +# create config file once if [[ ! -f /data/config/gunicorn.conf.py ]] - echo "Create default custom configuration file for gunicorn" then - cat >/data/config/gunicorn.conf.py </data/config/nginx-custom.conf < Date: Tue, 30 Dec 2025 15:03:47 +0100 Subject: [PATCH 04/16] Harmonize log formats --- .../dakara_server/settings/production.py | 19 ++++--------------- deployment/bin/daphne.sh | 3 ++- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/dakara_server/dakara_server/settings/production.py b/dakara_server/dakara_server/settings/production.py index a4755a16..8a9a1b5a 100644 --- a/dakara_server/dakara_server/settings/production.py +++ b/dakara_server/dakara_server/settings/production.py @@ -57,8 +57,10 @@ "version": 1, "disable_existing_loggers": False, "formatters": { - "default": {"format": "[%(asctime)s] %(levelname)s %(message)s"}, - "no_time": {"format": "%(levelname)s %(message)s"}, + "default": { + "format": "[%(asctime)s] [%(process)d] %(levelname)s %(message)s", + "datefmt": "[%Y-%m-%d %H:%M:%S %z]", + }, }, "filters": { "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}, @@ -70,11 +72,6 @@ "formatter": "default", "filters": ["require_debug_true"], }, - "console_playlist": {"class": "logging.StreamHandler", "formatter": "default"}, - "console_interactive": { - "class": "logging.StreamHandler", - "formatter": "no_time", - }, "logfile": { "level": "DEBUG", "class": "logging.handlers.RotatingFileHandler", @@ -88,14 +85,6 @@ "playlist.views": {"handlers": ["logfile"], "level": "INFO"}, "playlist.date_stop": {"handlers": ["logfile"], "level": "INFO"}, "playlist.consumers": {"handlers": ["logfile"], "level": "INFO"}, - "library.management.commands.feed": { - "handlers": ["console_interactive"], - "level": "INFO", - }, - "library.management.commands.createworks": { - "handlers": ["console_interactive"], - "level": "INFO", - }, "django": { "handlers": ["logfile"], "level": config("DJANGO_LOG_LEVEL", default="INFO"), diff --git a/deployment/bin/daphne.sh b/deployment/bin/daphne.sh index ed6149a4..f7f8687e 100755 --- a/deployment/bin/daphne.sh +++ b/deployment/bin/daphne.sh @@ -25,7 +25,8 @@ arguments=$(\ # run daphne daphne \ + $arguments \ + --log-fmt "[%(asctime)s] [%(process)d] %(levelname)s %(message)s" \ -b 0.0.0.0 \ -p 8001 \ - $arguments \ dakara_server.asgi:application From 338bd466ede3b7ebb05cf35ee3d445db992603be Mon Sep 17 00:00:00 2001 From: Neraste Date: Tue, 30 Dec 2025 16:49:34 +0100 Subject: [PATCH 05/16] Manage APScheduler --- Dockerfile | 1 + dakara_server/dakara_server/settings/base.py | 1 + .../dakara_server/settings/production.py | 8 ++ dakara_server/playlist/date_stop.py | 37 +---- .../management/commands/runapscheduler.py | 60 ++++++++ dakara_server/playlist/signals.py | 9 -- .../playlist/tests/test_date_stop.py | 129 +----------------- dakara_server/playlist/tests/test_karaoke.py | 91 ------------ dakara_server/playlist/views.py | 26 ---- deployment/bin/apscheduler.sh | 9 ++ deployment/etc/supervisor/apscheduler.ini | 8 ++ requirements.txt | 2 +- 12 files changed, 96 insertions(+), 285 deletions(-) create mode 100644 dakara_server/playlist/management/commands/runapscheduler.py create mode 100644 deployment/bin/apscheduler.sh create mode 100644 deployment/etc/supervisor/apscheduler.ini diff --git a/Dockerfile b/Dockerfile index 5828f026..de59edcf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ RUN FRONT_ARCHIVE="dakara-client-web_$FRONT_VERSION.zip" && \ /tmp/$FRONT_ARCHIVE \ /tmp/front +COPY deployment/etc/supervisor/apscheduler.ini /etc/supervisor.d/apscheduler.ini COPY deployment/etc/supervisor/daphne.ini /etc/supervisor.d/daphne.ini COPY deployment/etc/supervisor/gunicorn.ini /etc/supervisor.d/gunicorn.ini COPY deployment/etc/supervisor/logging.ini /etc/supervisor.d/logging.ini diff --git a/dakara_server/dakara_server/settings/base.py b/dakara_server/dakara_server/settings/base.py index 96a670e3..67c9e0c2 100644 --- a/dakara_server/dakara_server/settings/base.py +++ b/dakara_server/dakara_server/settings/base.py @@ -30,6 +30,7 @@ "django.contrib.messages", "daphne", "django.contrib.staticfiles", + "django_apscheduler", "rest_framework", "rest_framework.authtoken", "drf_spectacular", diff --git a/dakara_server/dakara_server/settings/production.py b/dakara_server/dakara_server/settings/production.py index 8a9a1b5a..d32c3cc8 100644 --- a/dakara_server/dakara_server/settings/production.py +++ b/dakara_server/dakara_server/settings/production.py @@ -72,6 +72,10 @@ "formatter": "default", "filters": ["require_debug_true"], }, + "console_interactive": { + "class": "logging.StreamHandler", + "formatter": "default", + }, "logfile": { "level": "DEBUG", "class": "logging.handlers.RotatingFileHandler", @@ -85,6 +89,10 @@ "playlist.views": {"handlers": ["logfile"], "level": "INFO"}, "playlist.date_stop": {"handlers": ["logfile"], "level": "INFO"}, "playlist.consumers": {"handlers": ["logfile"], "level": "INFO"}, + "playlist.management.commands.runapscheduler": { + "handlers": ["logfile"], + "level": "INFO", + }, "django": { "handlers": ["logfile"], "level": config("DJANGO_LOG_LEVEL", default="INFO"), diff --git a/dakara_server/playlist/date_stop.py b/dakara_server/playlist/date_stop.py index 492386ec..41fad808 100644 --- a/dakara_server/playlist/date_stop.py +++ b/dakara_server/playlist/date_stop.py @@ -1,8 +1,6 @@ import logging from datetime import datetime -from apscheduler.schedulers.background import BackgroundScheduler -from django.core.cache import cache from django.db.utils import OperationalError from django.utils import timezone @@ -12,44 +10,21 @@ tz = timezone.get_default_timezone() logger = logging.getLogger(__name__) -scheduler = BackgroundScheduler() -scheduler.start() def clear_date_stop(): """Clear stop date and disable can add to playlist.""" - karaoke = Karaoke.objects.get_object() - if not karaoke.date_stop or karaoke.date_stop > datetime.now(tz): - logger.error("Clear date stop was called when it should not") - return - - karaoke.can_add_to_playlist = False - karaoke.date_stop = None - karaoke.save() - logger.info("Date stop was cleared and can add to playlist was disabled") - - -def check_date_stop_on_app_ready(): - """Check if date stop has expired and clear or schedule job accordingly.""" try: karaoke = Karaoke.objects.get_object() # if database does not exist when checking date stop, abort the function - # this case occurs on startup before running tests except OperationalError: return - if karaoke.date_stop is not None: - if karaoke.date_stop < datetime.now(tz): - # Date stop has already expired - clear_date_stop() - return - - # Re-schedule date stop clear if not scheduled yet - # Since this method may be called several times at startup - if cache.get(KARAOKE_JOB_NAME) is not None: - return + if not karaoke.date_stop or karaoke.date_stop > datetime.now(tz): + return - job = scheduler.add_job(clear_date_stop, "date", run_date=karaoke.date_stop) - cache.set(KARAOKE_JOB_NAME, job.id) - logger.debug("New date stop job was scheduled") + karaoke.can_add_to_playlist = False + karaoke.date_stop = None + karaoke.save() + logger.info("Date stop was cleared and can add to playlist was disabled") diff --git a/dakara_server/playlist/management/commands/runapscheduler.py b/dakara_server/playlist/management/commands/runapscheduler.py new file mode 100644 index 00000000..f5f1ae3b --- /dev/null +++ b/dakara_server/playlist/management/commands/runapscheduler.py @@ -0,0 +1,60 @@ +import logging + +from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.triggers.cron import CronTrigger +from django.conf import settings +from django.core.management.base import BaseCommand +from django_apscheduler import util +from django_apscheduler.jobstores import DjangoJobStore +from django_apscheduler.models import DjangoJobExecution + +from playlist.date_stop import clear_date_stop + +logger = logging.getLogger(__name__) + + +@util.close_old_connections +def delete_old_job_executions(max_age=604_800): + """This job deletes APScheduler job execution entries older than `max_age` + from the database. It helps to prevent the database from filling up with old + historical records that are no longer useful. + + Args: + max_age: The maximum length of time to retain historical job execution + records. Defaults to 7 days. + """ + DjangoJobExecution.objects.delete_old_job_executions(max_age) + + +class Command(BaseCommand): + help = "Runs APScheduler." + + def handle(self, *args, **options): + scheduler = BlockingScheduler(timezone=settings.TIME_ZONE) + scheduler.add_jobstore(DjangoJobStore(), "default") + + scheduler.add_job( + clear_date_stop, + trigger=CronTrigger(minute="*/5"), + id="clear_date_stop", + max_instances=1, + replace_existing=True, + ) + logger.info("Added job 'clear_date_stop'") + + scheduler.add_job( + delete_old_job_executions, + trigger=CronTrigger(day_of_week="mon", hour="00", minute="00"), + id="delete_old_job_executions", + max_instances=1, + replace_existing=True, + ) + logger.info("Added weekly job 'delete_old_job_executions'") + + try: + logger.info("Starting scheduler...") + scheduler.start() + except KeyboardInterrupt: + logger.info("Stopping scheduler...") + scheduler.shutdown() + logger.info("Scheduler shut down successfully!") diff --git a/dakara_server/playlist/signals.py b/dakara_server/playlist/signals.py index dc380d69..dda99093 100644 --- a/dakara_server/playlist/signals.py +++ b/dakara_server/playlist/signals.py @@ -3,8 +3,6 @@ from django.db.backends.signals import connection_created from django.dispatch import receiver -from internal.reloader import is_reloader - connection_created_once = Event() @@ -16,13 +14,6 @@ def handle_connection_created(connection, **kwargs): if not connection_created_once.is_set(): connection_created_once.set() - from playlist.date_stop import check_date_stop_on_app_ready from playlist.models import clean_channel_names clean_channel_names() - - if is_reloader(): - return - - # not called by the reloader - check_date_stop_on_app_ready() diff --git a/dakara_server/playlist/tests/test_date_stop.py b/dakara_server/playlist/tests/test_date_stop.py index 1fdad523..a41dd0c3 100644 --- a/dakara_server/playlist/tests/test_date_stop.py +++ b/dakara_server/playlist/tests/test_date_stop.py @@ -1,10 +1,7 @@ from datetime import datetime, timedelta -from unittest.mock import patch - -from django.db.utils import OperationalError from internal.tests.base_test import tz -from playlist.date_stop import check_date_stop_on_app_ready, clear_date_stop +from playlist.date_stop import clear_date_stop from playlist.models import Karaoke from playlist.tests.base_test import PlaylistAPITestCase @@ -45,131 +42,9 @@ def test_date_stop_not_cleared(self): karaoke.date_stop = datetime.now(tz) + timedelta(minutes=10) karaoke.save() - with self.assertLogs("playlist.date_stop", "DEBUG") as logger: - clear_date_stop() + clear_date_stop() # Check clear date stop was cleared and can add to playlist was disabled karaoke_new = Karaoke.objects.get_object() self.assertTrue(karaoke_new.can_add_to_playlist) self.assertEqual(karaoke_new.date_stop, karaoke.date_stop) - - # Check logger - self.assertListEqual( - logger.output, - ["ERROR:playlist.date_stop:Clear date stop was called when it should not"], - ) - - -class CheckDateStopOnAppReadyTestCase(PlaylistAPITestCase): - @patch("playlist.date_stop.clear_date_stop") - @patch("playlist.date_stop.scheduler") - def test_check_date_expired(self, mocked_scheduler, mocked_clear_date_stop): - """Check clear date stop is called when date has expired.""" - - # Set stop date in the past - karaoke = Karaoke.objects.get_object() - karaoke.date_stop = datetime.now(tz) - timedelta(minutes=10) - karaoke.save() - - # Call method - check_date_stop_on_app_ready() - - # Check clear date stop was called - mocked_clear_date_stop.assert_called_with() - - # Check add job was not called - mocked_scheduler.add_job.assert_not_called() - - @patch("playlist.date_stop.clear_date_stop") - @patch("playlist.date_stop.scheduler") - def test_date_not_expired(self, mocked_scheduler, mocked_clear_date_stop): - """Check job is scheduled when date not expired.""" - # Mock return value of add_job - mocked_scheduler.add_job.return_value.id = "job_id" - - # Set stop date in the future - karaoke = Karaoke.objects.get_object() - date_stop = datetime.now(tz) + timedelta(minutes=10) - karaoke.date_stop = date_stop - karaoke.save() - - # Call method - check_date_stop_on_app_ready() - - # Check clear date stop was not called - mocked_clear_date_stop.assert_not_called() - - # Check add job was called - mocked_scheduler.add_job.assert_called_with( - mocked_clear_date_stop, "date", run_date=date_stop - ) - - @patch("playlist.date_stop.clear_date_stop") - @patch("playlist.date_stop.scheduler") - def test_no_date(self, mocked_scheduler, mocked_clear_date_stop): - """Check nothing happen when date stop is not set.""" - # Assert stop date is not set - karaoke = Karaoke.objects.get_object() - self.assertIsNone(karaoke.date_stop) - - # Call method - check_date_stop_on_app_ready() - - # Check clear date stop was not called - mocked_clear_date_stop.assert_not_called() - - # Check add job was not called - mocked_scheduler.add_job.assert_not_called() - - @patch("playlist.date_stop.scheduler") - def test_date_not_expired_called_twice(self, mocked_scheduler): - """Check job is scheduled only once when date not expired.""" - # Mock return value of add_job - mocked_scheduler.add_job.return_value.id = "job_id" - - # Set stop date in the future - karaoke = Karaoke.objects.get_object() - date_stop = datetime.now(tz) + timedelta(minutes=10) - karaoke.date_stop = date_stop - karaoke.save() - - # Call method - check_date_stop_on_app_ready() - - # Check add job was called - mocked_scheduler.add_job.assert_called_with( - clear_date_stop, "date", run_date=date_stop - ) - - mocked_scheduler.reset_mock() - - # Call method a second time - check_date_stop_on_app_ready() - - # Check add job was not called - mocked_scheduler.add_job.assert_not_called() - - @patch("playlist.date_stop.Karaoke") - @patch("playlist.date_stop.clear_date_stop") - @patch("playlist.date_stop.scheduler") - def test_database_unavailable( - self, mocked_scheduler, mocked_clear_date_stop, MockedKaraoke - ): - """Check there is no crash if the database does not exist. - - We simulate a crash by raising a `django.db.utils.OperationalError` - when accessing to `Karaoke.objects.get_object`. - """ - # mock the karaoke mock to crash when invoking class method get_object - MockedKaraoke.objects.get_object.side_effect = OperationalError( - "no such table: playlist_karaoke" - ) - - # call the method - check_date_stop_on_app_ready() - - # check clear date stop was not called - mocked_clear_date_stop.assert_not_called() - - # check add job was not called - mocked_scheduler.add_job.assert_not_called() diff --git a/dakara_server/playlist/tests/test_karaoke.py b/dakara_server/playlist/tests/test_karaoke.py index f70d2afc..ff04eebc 100644 --- a/dakara_server/playlist/tests/test_karaoke.py +++ b/dakara_server/playlist/tests/test_karaoke.py @@ -6,7 +6,6 @@ from rest_framework import status from internal.tests.base_test import tz -from playlist.date_stop import KARAOKE_JOB_NAME, clear_date_stop from playlist.models import Karaoke, PlayerError, PlaylistEntry from playlist.tests.base_test import PlaylistAPITestCase @@ -263,93 +262,3 @@ def test_patch_resume_kara_playlist_empty(self, mocked_send_to_channel): # post-assertion # no command was sent to device mocked_send_to_channel.assert_not_called() - - @patch("playlist.views.scheduler") - def test_patch_karaoke_date_stop(self, mocked_scheduler): - """Test a manager can modify the kara date stop and scheduler is called.""" - # Mock return value of add_job - mocked_scheduler.add_job.return_value.id = "job_id" - - # login as manager - self.authenticate(self.manager) - - # set karaoke date stop - date_stop = datetime.now(tz) - response = self.client.patch(self.url, {"date_stop": date_stop.isoformat()}) - self.assertEqual(response.status_code, status.HTTP_200_OK) - - # Check karaoke was updated - karaoke = Karaoke.objects.get_object() - self.assertEqual(karaoke.date_stop, date_stop) - - # Check job was added - mocked_scheduler.add_job.assert_called_with( - clear_date_stop, "date", run_date=date_stop - ) - - @patch("playlist.views.scheduler") - @patch("playlist.views.cache") - def test_patch_karaoke_clear_date_stop(self, mocked_cache, mocked_scheduler): - """Test a manager can clear the kara date stop and job is cancelled.""" - # set karaoke date stop - karaoke = Karaoke.objects.get_object() - date_stop = datetime.now(tz) - karaoke.date_stop = date_stop - karaoke.save() - - # login as manager - self.authenticate(self.manager) - - # clear karaoke date stop - response = self.client.patch(self.url, {"date_stop": None}) - self.assertEqual(response.status_code, status.HTTP_200_OK) - - # Check karaoke was updated - karaoke = Karaoke.objects.get_object() - self.assertIsNone(karaoke.date_stop) - - # Check remove was called - mocked_cache.get.assert_called_with(KARAOKE_JOB_NAME) - mocked_scheduler.get_job.return_value.remove.assert_called_with() - - @patch("playlist.views.scheduler") - @patch("playlist.views.cache") - def test_patch_karaoke_clear_date_stop_existing_job_id( - self, mocked_cache, mocked_scheduler - ): - """Test a manager can clear existing date stop.""" - # create existing job in cache - mocked_cache.get.return_value = "job_id" - - # login as manager - self.authenticate(self.manager) - - # clear karaoke date stop - response = self.client.patch(self.url, {"date_stop": None}) - self.assertEqual(response.status_code, status.HTTP_200_OK) - - # Check remove was called - mocked_cache.get.assert_called_with(KARAOKE_JOB_NAME) - mocked_scheduler.get_job.return_value.remove.assert_called_with() - mocked_cache.delete.assert_not_called() - - @patch("playlist.views.scheduler") - @patch("playlist.views.cache") - def test_patch_karaoke_clear_date_stop_existing_job_id_no_job( - self, mocked_cache, mocked_scheduler - ): - """Test a manager can clear existing date stop without job.""" - # create existing job in cache - mocked_cache.get.return_value = "job_id" - mocked_scheduler.get_job.return_value = None - - # login as manager - self.authenticate(self.manager) - - # clear karaoke date stop - response = self.client.patch(self.url, {"date_stop": None}) - self.assertEqual(response.status_code, status.HTTP_200_OK) - - # Check remove was called - mocked_cache.get.assert_called_with(KARAOKE_JOB_NAME) - mocked_cache.delete.assert_called_with(KARAOKE_JOB_NAME) diff --git a/dakara_server/playlist/views.py b/dakara_server/playlist/views.py index 9e8f02e3..ea16a318 100644 --- a/dakara_server/playlist/views.py +++ b/dakara_server/playlist/views.py @@ -3,7 +3,6 @@ from django.conf import settings from django.contrib.auth import get_user_model -from django.core.cache import cache from django.shortcuts import get_object_or_404 from django.utils import timezone from django.utils.decorators import method_decorator @@ -21,7 +20,6 @@ from library import permissions as library_permissions from playlist import authentications, models, permissions, serializers from playlist.consumers import send_to_channel -from playlist.date_stop import KARAOKE_JOB_NAME, clear_date_stop, scheduler from playlist.schemes import PlayerTokenScheme # noqa F401 tz = timezone.get_default_timezone() @@ -252,30 +250,6 @@ def perform_update(self, serializer): super().perform_update(serializer) karaoke = serializer.instance - # Management of date stop - - if "date_stop" in serializer.validated_data: - # Clear existing scheduled task - existing_job_id = cache.get(KARAOKE_JOB_NAME) - if existing_job_id is not None: - existing_job = scheduler.get_job(existing_job_id) - if existing_job is not None: - existing_job.remove() - logger.debug("Existing date stop job was found and unscheduled") - - else: - cache.delete(KARAOKE_JOB_NAME) - - if karaoke.date_stop is not None: - # Schedule date stop clear - job = scheduler.add_job( - clear_date_stop, "date", run_date=karaoke.date_stop - ) - cache.set(KARAOKE_JOB_NAME, job.id) - logger.debug("New date stop job was scheduled") - - # Management of kara status Booleans change - # empty the playlist and clear the player if the kara is switched to not ongoing if "ongoing" in serializer.validated_data and not karaoke.ongoing: # request the player to be idle diff --git a/deployment/bin/apscheduler.sh b/deployment/bin/apscheduler.sh new file mode 100644 index 00000000..ec0e7f21 --- /dev/null +++ b/deployment/bin/apscheduler.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +# production preset +export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" + +# run apscheduler +/app/dakara_server/manage.py runapscheduler diff --git a/deployment/etc/supervisor/apscheduler.ini b/deployment/etc/supervisor/apscheduler.ini new file mode 100644 index 00000000..c58fcf47 --- /dev/null +++ b/deployment/etc/supervisor/apscheduler.ini @@ -0,0 +1,8 @@ +[program:apscheduler] +command=/app/deployment/bin/apscheduler.sh +directory=/app/dakara_server +user=root +autostart=true +autorestart=unexpected +stdout_events_enabled=true +stderr_events_enabled=true diff --git a/requirements.txt b/requirements.txt index eda8ac06..80e55694 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -APScheduler>=3.11.0,<3.12.0 asgiref>=3.9.1,<3.10.0 # to remove after updating channels channels>=4.2.0,<4.3.0 daphne>=4.1.2,<4.2.0 dj-database-url>=2.3.0,<2.4.0 +django-apscheduler>=0.7.0,<0.8.0 django-cache-lock>=0.2.5,<0.3.0 django-filter>=25.1,<25.2 django-ordered-model>=3.7.4,<3.8.0 From 61bc88fa55450f7181353262de8845e61b62a283 Mon Sep 17 00:00:00 2001 From: Neraste Date: Tue, 30 Dec 2025 17:12:32 +0100 Subject: [PATCH 06/16] Increase number of workers --- deployment/config/gunicorn.conf.py | 2 +- deployment/config/nginx.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/config/gunicorn.conf.py b/deployment/config/gunicorn.conf.py index e7ce822d..351ea938 100644 --- a/deployment/config/gunicorn.conf.py +++ b/deployment/config/gunicorn.conf.py @@ -1,3 +1,3 @@ # Custom configuration file for gunicorn -workers = 1 +workers = 4 diff --git a/deployment/config/nginx.conf b/deployment/config/nginx.conf index b60e2a80..2bf431ce 100644 --- a/deployment/config/nginx.conf +++ b/deployment/config/nginx.conf @@ -1,3 +1,3 @@ # Custom configuration file for nginx -worker_processes 1; +worker_processes 4; From 6b7b956183ec3517bc9abcd0b78845ce37ab882f Mon Sep 17 00:00:00 2001 From: Neraste Date: Tue, 30 Dec 2025 18:33:38 +0100 Subject: [PATCH 07/16] Add Redis and docker-compose --- README.md | 63 +++++++++++++- dakara_server/dakara_server/settings/base.py | 50 +++++------ .../dakara_server/settings/development.py | 27 ++++-- .../dakara_server/settings/production.py | 86 +++++++++++++------ dakara_server/dakara_server/settings/test.py | 23 +++-- deployment/docker-compose/docker-compose.yaml | 46 ++++++++++ requirements.txt | 2 + 7 files changed, 230 insertions(+), 67 deletions(-) create mode 100644 deployment/docker-compose/docker-compose.yaml diff --git a/README.md b/README.md index 8b4edfb3..a7441861 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,11 @@ Linux, Mac and Windows are supported. It is strongly recommended to run the Dakara server in a virtual environment. +```sh +python -m virtualenv venv +source venv/bin/activate +``` + ### Dependencies Having a recent enough versio of `pip` is required to install some dependencies properly: @@ -64,7 +69,9 @@ To select a preset, set the `DJANGO_SETTINGS_MODULE` environment variable accord export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" ``` -### Setting up the server +### Preparation of the server + +Running the server in development requires some preliminary steps. Let's create the server database, after loading the virtual environment, do: @@ -88,6 +95,13 @@ You're almost done! To start the server app, in the right virtual environment, d dakara_server/manage.py runserver ``` +In a separate terminal, also run the scheduler. +This is currently only required for the kara date stop feature (which stop the karaoke at a certain date): + +```sh +dakara_server/manage.py runapscheduler +``` + The server part is now set up correctly. ### Web client, feeder and player @@ -99,6 +113,53 @@ Both tokens can be obtained from the web interface. After all of this is setup, just grab some friends and have fun! +## Docker image + +For production, it is recommended to use the provided Docker image, which takes care of all the aspects of the execution. + +You can build the local Docker image with: + +```sh +sudo docker build . -t dakara-server +``` + +Then, run the container with: + +```sh +sudo docker run \ + -d \ + -v path/to/persistent/data:/data \ + -e DAKARA_DATABASE_URL="mysql://user:password@mysql/dakara" \ + -e DAKARA_REDIS_URL="redis://redis:6379" \ + -e DAKARA_ALLOWED_HOSTS="localhost,example.com" \ + -e DAKARA_HOST_URL="http://example.com" \ + -e DAKARA_SECRET_KEY="your-secret-key" \ + -e DAKARA_LANGUAGE_CODE="en-us" \ + -e DAKARA_TIME_ZONE="UTC" \ + -e DAKARA_LOG_LEVEL="INFO" \ + -e DAKARA_EMAIL_ENABLED= \ + -e DAKARA_EMAIL_HOST="postfix" \ + -e DAKARA_EMAIL_HOST_USER="user" \ + -e DAKARA_EMAIL_HOST_PASSWORD="password" \ + -e DAKARA_SENDER_EMAIL="no-reply@example.com" \ + -p 80:80 \ + dakara-server +``` + +A `docker-compose.yaml` file is given as an example in `deployment/docker-compose/docker-compose.yaml`. + +```sh +cp deployment/docker-compose/docker-compose.yaml ./ +# edit it as you like +sudo docker compose up -d +``` + +Then, use your browser to acces the web page: + +```sh +xdg-open http://localhost +``` + ## Development Please read the [developers documentation](CONTRIBUTING.md). diff --git a/dakara_server/dakara_server/settings/base.py b/dakara_server/dakara_server/settings/base.py index 67c9e0c2..1be3244b 100644 --- a/dakara_server/dakara_server/settings/base.py +++ b/dakara_server/dakara_server/settings/base.py @@ -84,12 +84,6 @@ } ] -CACHES = { - "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - "TIMEOUT": None, - } -} WSGI_APPLICATION = "dakara_server.wsgi.application" @@ -131,34 +125,36 @@ } -SENDER_EMAIL = config("SENDER_EMAIL", default="no-reply@example.com") -HOST_URL = config("HOST_URL", default="http://example.com") -EMAIL_ENABLED = config("EMAIL_ENABLED", default=True, cast=bool) +EMAIL_ENABLED = config("DAKARA_EMAIL_ENABLED", default=True, cast=bool) # Django rest registration config -REST_REGISTRATION = { - "LOGIN_AUTHENTICATE_SESSION": False, - "LOGIN_SERIALIZER_CLASS": "users.serializers.DakaraLoginSerializer", - "REGISTER_VERIFICATION_URL": HOST_URL + "/verify-registration/", - "RESET_PASSWORD_VERIFICATION_URL": HOST_URL + "/reset-password/", - "REGISTER_EMAIL_VERIFICATION_URL": HOST_URL + "/verify-email/", - "VERIFICATION_FROM_EMAIL": SENDER_EMAIL, - "USER_VERIFICATION_FLAG_FIELD": "validated_by_email", - "USER_LOGIN_FIELDS": ["username", "email"], - "REGISTER_VERIFICATION_ENABLED": EMAIL_ENABLED, - "REGISTER_EMAIL_VERIFICATION_ENABLED": EMAIL_ENABLED, - "RESET_PASSWORD_VERIFICATION_ENABLED": EMAIL_ENABLED, -} +def get_rest_registration(host_url, sender_email, email_enabled): + return { + "LOGIN_AUTHENTICATE_SESSION": False, + "LOGIN_SERIALIZER_CLASS": "users.serializers.DakaraLoginSerializer", + "REGISTER_VERIFICATION_URL": host_url + "/verify-registration/", + "RESET_PASSWORD_VERIFICATION_URL": host_url + "/reset-password/", + "REGISTER_EMAIL_VERIFICATION_URL": host_url + "/verify-email/", + "VERIFICATION_FROM_EMAIL": sender_email, + "USER_VERIFICATION_FLAG_FIELD": "validated_by_email", + "USER_LOGIN_FIELDS": ["username", "email"], + "REGISTER_VERIFICATION_ENABLED": email_enabled, + "REGISTER_EMAIL_VERIFICATION_ENABLED": email_enabled, + "RESET_PASSWORD_VERIFICATION_ENABLED": email_enabled, + } + AUTHENTICATION_BACKENDS = ["users.backends.DakaraModelBackend"] # Front URLs -HOST_URLS = { - "USER_EDIT_URL": HOST_URL + "/settings/users/{id}", - "LOGIN_URL": HOST_URL + "/login", -} +def get_host_urls(host_url): + return { + "USER_EDIT_URL": host_url + "/settings/users/{id}", + "LOGIN_URL": host_url + "/login", + } + # limit of the playlist size -PLAYLIST_SIZE_LIMIT = config("PLAYLIST_SIZE_LIMIT", cast=int, default=100) +PLAYLIST_SIZE_LIMIT = config("DAKARA_PLAYLIST_SIZE_LIMIT", cast=int, default=100) diff --git a/dakara_server/dakara_server/settings/development.py b/dakara_server/dakara_server/settings/development.py index 36902d97..0652bdb4 100644 --- a/dakara_server/dakara_server/settings/development.py +++ b/dakara_server/dakara_server/settings/development.py @@ -13,14 +13,18 @@ section. """ -import os - from decouple import config -os.environ.setdefault("HOST_URL", "http://localhost:3000") - from dakara_server.settings.base import * # noqa F403 -from dakara_server.settings.base import BASE_DIR # noqa E402 +from dakara_server.settings.base import ( + BASE_DIR, + EMAIL_ENABLED, + get_host_urls, + get_rest_registration, +) + +HOST_URL = "http://localhost:3000" +SENDER_EMAIL = "no-reply@localhost" # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/ @@ -49,6 +53,14 @@ CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}} + +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "TIMEOUT": None, + } +} + # Internationalization # https://docs.djangoproject.com/en/5.1/topics/i18n/ @@ -102,6 +114,5 @@ # email backend EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" -# values imported from base config -# SENDER_EMAIL is get from the environment -# HOST_URL is get from the environment +REST_REGISTRATION = get_rest_registration(HOST_URL, SENDER_EMAIL, EMAIL_ENABLED) +HOST_URLS = get_host_urls(HOST_URL) diff --git a/dakara_server/dakara_server/settings/production.py b/dakara_server/dakara_server/settings/production.py index d32c3cc8..68c59dab 100644 --- a/dakara_server/dakara_server/settings/production.py +++ b/dakara_server/dakara_server/settings/production.py @@ -19,10 +19,17 @@ from dj_database_url import parse as db_url from dakara_server.settings.base import * # noqa F403 - -SECRET_KEY = config("SECRET_KEY", default="secret_key") -DEBUG = config("DEBUG", cast=bool, default=False) -ALLOWED_HOSTS = config("ALLOWED_HOSTS", cast=Csv(), default="[]") +from dakara_server.settings.base import ( + EMAIL_ENABLED, + get_host_urls, + get_rest_registration, +) + +SENDER_EMAIL = config("DAKARA_SENDER_EMAIL", default="no-reply@example.com") +HOST_URL = config("DAKARA_HOST_URL", default="http://example.com") +SECRET_KEY = config("DAKARA_SECRET_KEY", default="secret_key") +DEBUG = config("DAKARA_DEBUG", cast=bool, default=False) +ALLOWED_HOSTS = config("DAKARA_ALLOWED_HOSTS", cast=Csv(), default="") CSRF_TRUSTED_ORIGINS = ["http://localhost"] # Database @@ -32,25 +39,44 @@ DATABASES = { "default": config( - "DATABASE_URL", cast=db_url, default="mysql://dakara:dakara@mysql/dakara" + "DAKARA_DATABASE_URL", cast=db_url, default="mysql://user:password@mysql/dakara" ) } +REDIS_URL = config("DAKARA_REDIS_URL", default="redis://redis:6379") + # Channels # http://channels.readthedocs.io/en/latest/topics/channel_layers.html -CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}} +CHANNEL_LAYERS = { + "default": { + "BACKEND": "channels_redis.core.RedisChannelLayer", + "CONFIG": {"hosts": [REDIS_URL]}, + } +} + + +CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": "{}/1".format(REDIS_URL), + "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"}, + } +} + +SESSION_ENGINE = "django.contrib.sessions.backends.cache" +SESSION_CACHE_ALIAS = "default" # Static root # Should point to the static directory served by nginx -STATIC_ROOT = config("STATIC_ROOT", "/app/static") +STATIC_ROOT = config("DAKARA_STATIC_ROOT", "/app/static") # Internationalization # https://docs.djangoproject.com/en/5.1/topics/i18n/ -LANGUAGE_CODE = config("LANGUAGE", default="en-us") +LANGUAGE_CODE = config("DAKARA_LANGUAGE_CODE", default="en-us") -TIME_ZONE = config("TIME_ZONE", default="UTC") +TIME_ZONE = config("DAKARA_TIME_ZONE", default="UTC") # Loggin config LOGGING = { @@ -86,12 +112,21 @@ }, }, "loggers": { - "playlist.views": {"handlers": ["logfile"], "level": "INFO"}, - "playlist.date_stop": {"handlers": ["logfile"], "level": "INFO"}, - "playlist.consumers": {"handlers": ["logfile"], "level": "INFO"}, + "playlist.views": { + "handlers": ["logfile"], + "level": config("DAKARA_LOG_LEVEL", default="INFO"), + }, + "playlist.date_stop": { + "handlers": ["logfile"], + "level": config("DAKARA_LOG_LEVEL", default="INFO"), + }, + "playlist.consumers": { + "handlers": ["logfile"], + "level": config("DAKARA_LOG_LEVEL", default="INFO"), + }, "playlist.management.commands.runapscheduler": { "handlers": ["logfile"], - "level": "INFO", + "level": config("DAKARA_LOG_LEVEL", default="INFO"), }, "django": { "handlers": ["logfile"], @@ -101,16 +136,15 @@ } # email backend -EMAIL_HOST = config("EMAIL_HOST", default="postfix") -EMAIL_PORT = config("EMAIL_PORT", cast=int, default="25") -EMAIL_HOST_USER = config("EMAIL_HOST_USER", default="") -EMAIL_HOST_PASSWORD = config("EMAIL_HOST_PASSWORD", default="") -EMAIL_USE_TLS = config("EMAIL_USE_TLS", cast=bool, default="false") -EMAIL_USE_SSL = config("EMAIL_USE_SSL", cast=bool, default="false") -EMAIL_TIMEOUT = config("EMAIL_TIMEOUT", cast=int, default="0") or None -EMAIL_SSL_KEYFILE = config("EMAIL_SSL_KEYFILE", default="") or None -EMAIL_SSL_CERTIFICATE = config("EMAIL_SSL_CERTIFICATE", default="") or None - -# values imported from base config -# SENDER_EMAIL is get from the environment -# HOST_URL is get from the environment +EMAIL_HOST = config("DAKARA_EMAIL_HOST", default="postfix") +EMAIL_PORT = config("DAKARA_EMAIL_PORT", cast=int, default="25") +EMAIL_HOST_USER = config("DAKARA_EMAIL_HOST_USER", default="user") +EMAIL_HOST_PASSWORD = config("DAKARA_EMAIL_HOST_PASSWORD", default="password") +EMAIL_USE_TLS = config("DAKARA_EMAIL_USE_TLS", cast=bool, default="false") +EMAIL_USE_SSL = config("DAKARA_EMAIL_USE_SSL", cast=bool, default="false") +EMAIL_TIMEOUT = config("DAKARA_EMAIL_TIMEOUT", cast=int, default="0") or None +EMAIL_SSL_KEYFILE = config("DAKARA_EMAIL_SSL_KEYFILE", default="") or None +EMAIL_SSL_CERTIFICATE = config("DAKARA_EMAIL_SSL_CERTIFICATE", default="") or None + +REST_REGISTRATION = get_rest_registration(HOST_URL, SENDER_EMAIL, EMAIL_ENABLED) +HOST_URLS = get_host_urls(HOST_URL) diff --git a/dakara_server/dakara_server/settings/test.py b/dakara_server/dakara_server/settings/test.py index 31a3b66a..7321b255 100644 --- a/dakara_server/dakara_server/settings/test.py +++ b/dakara_server/dakara_server/settings/test.py @@ -12,11 +12,16 @@ import os -os.environ.setdefault("HOST_URL", "http://frontend-host") - from dakara_server.settings.base import * # noqa F403 +from dakara_server.settings.base import ( + EMAIL_ENABLED, + get_host_urls, + get_rest_registration, +) # use test config +HOST_URL = "http://frontend-host" +SENDER_EMAIL = "no-reply@frontend-host" SECRET_KEY = "test secret key" DEBUG = True ALLOWED_HOSTS = ["*"] @@ -27,6 +32,14 @@ # use memory channels backend CHANNEL_LAYERS = {"default": {"BACKEND": "channels.layers.InMemoryChannelLayer"}} +# use memory cache +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "TIMEOUT": None, + } +} + # use faster password hasher PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) @@ -75,8 +88,8 @@ } PLAYLIST_SIZE_LIMIT = 100 + EMAIL_BACKEND = "django.core.mail.backends.dummy.EmailBackend" -# values imported from base config -# SENDER_EMAIL is get from the environment -# HOST_URL is get from the environment +REST_REGISTRATION = get_rest_registration(HOST_URL, SENDER_EMAIL, EMAIL_ENABLED) +HOST_URLS = get_host_urls(HOST_URL) diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml new file mode 100644 index 00000000..d9819e9b --- /dev/null +++ b/deployment/docker-compose/docker-compose.yaml @@ -0,0 +1,46 @@ +version: "2.2" + +services: + mysql: + image: mariadb:latest + volumes: + - mysql_data:/var/lib/mysql:Z + environment: + MARIADB_ROOT_PASSWORD: "password" + MARIADB_USER: "user" + MARIADB_PASSWORD: "password" + MARIADB_DATABASE: "dakara" + restart: "unless-stopped" + + redis: + image: redis:latest + restart: "unless-stopped" + + dakara-server: + image: dakara-server:latest + volumes: + - dakara_server_data:/data + environment: + DAKARA_DATABASE_URL: "mysql://user:password@mysql/dakara" + DAKARA_REDIS_URL: "redis://redis" + DAKARA_ALLOWED_HOSTS: "localhost,example.com" + DAKARA_HOST_URL: "http://example.com" + DAKARA_SECRET_KEY: "your-secret-key" + DAKARA_LANGUAGE_CODE: "en-us" + DAKARA_TIME_ZONE: "UTC" + DAKARA_LOG_LEVEL: "INFO" + DAKARA_EMAIL_ENABLED: true # or false + DAKARA_EMAIL_HOST: "postfix" + DAKARA_EMAIL_HOST_USER: "user" + DAKARA_EMAIL_HOST_PASSWORD: "password" + DAKARA_SENDER_EMAIL: "no-reply@example.com" + restart: "unless-stopped" + ports: + - 80 + depends_on: + - mysql + - redis + +volume: + dakara_server_data: + mysql_data: diff --git a/requirements.txt b/requirements.txt index 80e55694..c84329ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ asgiref>=3.9.1,<3.10.0 # to remove after updating channels +channels-redis>=4.3.0,<4.4.0 channels>=4.2.0,<4.3.0 daphne>=4.1.2,<4.2.0 dj-database-url>=2.3.0,<2.4.0 @@ -6,6 +7,7 @@ django-apscheduler>=0.7.0,<0.8.0 django-cache-lock>=0.2.5,<0.3.0 django-filter>=25.1,<25.2 django-ordered-model>=3.7.4,<3.8.0 +django-redis>=6.0.0,<6.1.0 django-rest-registration>=0.9.0,<0.10.0 Django>=5.1.6,<5.2.0 djangorestframework>=3.15.2,<3.16.0 From 2823dd1a1957698f774765c963c9fe62b95d344e Mon Sep 17 00:00:00 2001 From: Neraste Date: Tue, 30 Dec 2025 23:54:52 +0100 Subject: [PATCH 08/16] Fixes for docker compose --- .gitignore | 2 ++ dakara_server/dakara_server/settings/production.py | 2 +- deployment/bin/apscheduler.sh | 1 + deployment/bin/nginx.sh | 1 + deployment/docker-compose/docker-compose.yaml | 2 -- deployment/etc/nginx/nginx.conf | 2 +- 6 files changed, 6 insertions(+), 4 deletions(-) mode change 100644 => 100755 deployment/bin/apscheduler.sh diff --git a/.gitignore b/.gitignore index 6efa97be..32c12128 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,5 @@ settings.ini # Pyenv config .python-version + +/docker-compose.yaml diff --git a/dakara_server/dakara_server/settings/production.py b/dakara_server/dakara_server/settings/production.py index 68c59dab..74acf899 100644 --- a/dakara_server/dakara_server/settings/production.py +++ b/dakara_server/dakara_server/settings/production.py @@ -85,7 +85,7 @@ "formatters": { "default": { "format": "[%(asctime)s] [%(process)d] %(levelname)s %(message)s", - "datefmt": "[%Y-%m-%d %H:%M:%S %z]", + "datefmt": "%Y-%m-%d %H:%M:%S %z", }, }, "filters": { diff --git a/deployment/bin/apscheduler.sh b/deployment/bin/apscheduler.sh old mode 100644 new mode 100755 index ec0e7f21..93e8e6c4 --- a/deployment/bin/apscheduler.sh +++ b/deployment/bin/apscheduler.sh @@ -6,4 +6,5 @@ set -e export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" # run apscheduler +echo "Starting scheduler" /app/dakara_server/manage.py runapscheduler diff --git a/deployment/bin/nginx.sh b/deployment/bin/nginx.sh index 459a9d12..98d8c02c 100755 --- a/deployment/bin/nginx.sh +++ b/deployment/bin/nginx.sh @@ -12,4 +12,5 @@ then fi # run nginx +echo "Starting nginx" /usr/sbin/nginx diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml index d9819e9b..41a43891 100644 --- a/deployment/docker-compose/docker-compose.yaml +++ b/deployment/docker-compose/docker-compose.yaml @@ -1,5 +1,3 @@ -version: "2.2" - services: mysql: image: mariadb:latest diff --git a/deployment/etc/nginx/nginx.conf b/deployment/etc/nginx/nginx.conf index 5cfb3682..f42b9099 100644 --- a/deployment/etc/nginx/nginx.conf +++ b/deployment/etc/nginx/nginx.conf @@ -54,4 +54,4 @@ http { } # user custom config -include /data/config/nginx-custom.conf; +include /data/config/nginx.conf; From 9945c7749b9e09f7ee8689240ba27113772c7043 Mon Sep 17 00:00:00 2001 From: Neraste Date: Sat, 3 Jan 2026 16:37:23 +0100 Subject: [PATCH 09/16] Secure execution order --- Dockerfile | 5 ++-- .../dakara_server/settings/production.py | 15 ++++++---- .../management/commands/wait_db_ready.py | 28 +++++++++++++++++++ dakara_server/playlist/signals.py | 8 +++++- deployment/bin/apscheduler.sh | 15 ++++++++++ deployment/bin/daphne.sh | 6 ++++ deployment/bin/gunicorn.sh | 19 +++++++++---- deployment/bin/make_directories.sh | 8 ++++++ deployment/bin/run.sh | 4 +-- deployment/docker-compose/docker-compose.yaml | 4 +-- requirements.txt | 6 +--- requirements_prod.txt | 5 ++++ 12 files changed, 99 insertions(+), 24 deletions(-) create mode 100644 dakara_server/internal/management/commands/wait_db_ready.py create mode 100755 deployment/bin/make_directories.sh create mode 100644 requirements_prod.txt diff --git a/Dockerfile b/Dockerfile index de59edcf..d06770c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,14 +14,15 @@ RUN apk add --no-cache \ unzip \ wget -COPY requirements.txt /app/ +COPY requirements.txt requirements_prod.txt /app/ # install dependencies RUN pip install \ --no-cache-dir \ --root-user-action ignore \ --break-system-packages \ - -r /app/requirements.txt + -r /app/requirements.txt \ + -r /app/requirements_prod.txt # get the front archive RUN FRONT_ARCHIVE="dakara-client-web_$FRONT_VERSION.zip" && \ diff --git a/dakara_server/dakara_server/settings/production.py b/dakara_server/dakara_server/settings/production.py index 74acf899..8456f9f3 100644 --- a/dakara_server/dakara_server/settings/production.py +++ b/dakara_server/dakara_server/settings/production.py @@ -16,7 +16,8 @@ """ from decouple import Csv, config -from dj_database_url import parse as db_url +from dj_database_url import config as config_db +from dj_database_url import register from dakara_server.settings.base import * # noqa F403 from dakara_server.settings.base import ( @@ -32,14 +33,18 @@ ALLOWED_HOSTS = config("DAKARA_ALLOWED_HOSTS", cast=Csv(), default="") CSRF_TRUSTED_ORIGINS = ["http://localhost"] +# register mysql-connector for database URL +register("mysql-connector", "mysql.connector.django") + # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases -# `DATABASE_URL` is specified according to dj-databse-url plugin +# `DAKARA_DATABASE_URL` is specified according to dj-databse-url plugin # https://github.com/kennethreitz/dj-database-url#url-schema DATABASES = { - "default": config( - "DAKARA_DATABASE_URL", cast=db_url, default="mysql://user:password@mysql/dakara" + "default": config_db( + "DAKARA_DATABASE_URL", + default="mysql-connector://user:password@mysql:3306/dakara", ) } @@ -59,7 +64,7 @@ CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": "{}/1".format(REDIS_URL), + "LOCATION": f"{REDIS_URL}/1", "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"}, } } diff --git a/dakara_server/internal/management/commands/wait_db_ready.py b/dakara_server/internal/management/commands/wait_db_ready.py new file mode 100644 index 00000000..76415d01 --- /dev/null +++ b/dakara_server/internal/management/commands/wait_db_ready.py @@ -0,0 +1,28 @@ +import time + +from django.core.management.base import BaseCommand +from django.db.utils import InterfaceError, OperationalError + + +class Command(BaseCommand): + help = "Wait for the database to be ready" + + def handle(self, *args, **options): + """Entrypoint for command.""" + self.stdout.write("Waiting for database...") + db_up = False + try: + while db_up is False: + try: + self.check(databases=["default"]) + db_up = True + + except (OperationalError, InterfaceError): + self.stdout.write("Database unavailable, waiting for 5 second...") + time.sleep(5) + + except KeyboardInterrupt: + self.stdout.write("Aborted by user") + return + + self.stdout.write(self.style.SUCCESS("Database available!")) diff --git a/dakara_server/playlist/signals.py b/dakara_server/playlist/signals.py index dda99093..ca7ef3ea 100644 --- a/dakara_server/playlist/signals.py +++ b/dakara_server/playlist/signals.py @@ -1,6 +1,7 @@ from threading import Event from django.db.backends.signals import connection_created +from django.db.utils import ProgrammingError from django.dispatch import receiver connection_created_once = Event() @@ -16,4 +17,9 @@ def handle_connection_created(connection, **kwargs): from playlist.models import clean_channel_names - clean_channel_names() + try: + clean_channel_names() + + except ProgrammingError: + # database not yet created + pass diff --git a/deployment/bin/apscheduler.sh b/deployment/bin/apscheduler.sh index 93e8e6c4..1dadd4f3 100755 --- a/deployment/bin/apscheduler.sh +++ b/deployment/bin/apscheduler.sh @@ -2,9 +2,24 @@ set -e +# populating data volume +/app/deployment/bin/make_directories.sh + # production preset export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" +# wait for database +./manage.py wait_db_ready + +# wait for migrations to be done +echo "Wating for migrations to be done..." +while ! ./manage.py migrate --check +do + echo "Migrations not done, waiting for 5 seconds..." + sleep 5 +done +echo "Migrations done!" + # run apscheduler echo "Starting scheduler" /app/dakara_server/manage.py runapscheduler diff --git a/deployment/bin/daphne.sh b/deployment/bin/daphne.sh index f7f8687e..518479ac 100755 --- a/deployment/bin/daphne.sh +++ b/deployment/bin/daphne.sh @@ -2,6 +2,9 @@ set -e +# populating data volume +/app/deployment/bin/make_directories.sh + # production preset export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" @@ -23,6 +26,9 @@ arguments=$(\ | awk '{printf("%s ", $0)}' \ ) +# wait for database +./manage.py wait_db_ready + # run daphne daphne \ $arguments \ diff --git a/deployment/bin/gunicorn.sh b/deployment/bin/gunicorn.sh index 9c062e40..a1ec4932 100755 --- a/deployment/bin/gunicorn.sh +++ b/deployment/bin/gunicorn.sh @@ -2,6 +2,9 @@ set -e +# populating data volume +/app/deployment/bin/make_directories.sh + # production preset export DJANGO_SETTINGS_MODULE="dakara_server.settings.production" @@ -15,20 +18,24 @@ then fi # collect static files -/app/dakara_server/manage.py collectstatic --noinput +./manage.py collectstatic --noinput + +# wait for database +./manage.py wait_db_ready # apply migrations -/app/dakara_server/manage.py makemigrations -/app/dakara_server/manage.py migrate +./manage.py makemigrations +./manage.py migrate # create superuser once -if [[ ! -f /data/NOT_FIRST_RUN_GUNICORN ]] +if [[ ! -f /data/state/gunicorn_first_superuser ]] then + # create the superuser with a dummy password echo \ "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser('root', 'root@localhost', 'root')" \ - | /app/dakara_server/manage.py shell + | ./manage.py shell - touch /data/NOT_FIRST_RUN_GUNICORN + touch /data/state/gunicorn_first_superuser fi # run gunicorn diff --git a/deployment/bin/make_directories.sh b/deployment/bin/make_directories.sh new file mode 100755 index 00000000..c259b975 --- /dev/null +++ b/deployment/bin/make_directories.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -e + +mkdir -pv /data +mkdir -pv /data/logs +mkdir -pv /data/config +mkdir -pv /data/state diff --git a/deployment/bin/run.sh b/deployment/bin/run.sh index c0d7791a..3be58e48 100755 --- a/deployment/bin/run.sh +++ b/deployment/bin/run.sh @@ -3,9 +3,7 @@ set -e # populating data volume -mkdir -pv /data -mkdir -pv /data/logs -mkdir -pv /data/config +/app/deployment/bin/make_directories.sh # run supervisor exec /usr/bin/supervisord -n diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml index 41a43891..cdd9612f 100644 --- a/deployment/docker-compose/docker-compose.yaml +++ b/deployment/docker-compose/docker-compose.yaml @@ -19,8 +19,8 @@ services: volumes: - dakara_server_data:/data environment: - DAKARA_DATABASE_URL: "mysql://user:password@mysql/dakara" - DAKARA_REDIS_URL: "redis://redis" + DAKARA_DATABASE_URL: "mysql-connector://user:password@mysql:3306/dakara" + DAKARA_REDIS_URL: "redis://redis:6379" DAKARA_ALLOWED_HOSTS: "localhost,example.com" DAKARA_HOST_URL: "http://example.com" DAKARA_SECRET_KEY: "your-secret-key" diff --git a/requirements.txt b/requirements.txt index c84329ba..f2a2d0d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,16 @@ asgiref>=3.9.1,<3.10.0 # to remove after updating channels -channels-redis>=4.3.0,<4.4.0 channels>=4.2.0,<4.3.0 daphne>=4.1.2,<4.2.0 -dj-database-url>=2.3.0,<2.4.0 +dj-database-url>=3.0.1,<3.1.0 django-apscheduler>=0.7.0,<0.8.0 django-cache-lock>=0.2.5,<0.3.0 django-filter>=25.1,<25.2 django-ordered-model>=3.7.4,<3.8.0 -django-redis>=6.0.0,<6.1.0 django-rest-registration>=0.9.0,<0.10.0 Django>=5.1.6,<5.2.0 djangorestframework>=3.15.2,<3.16.0 drf-spectacular==0.28.0 -gunicorn>=23.0.0,<23.1.0 packaging>=24.2,<24.3 python-decouple>=3.8,<3.9 setuptools>=75.8 -supervisor-stdlog>=0.7.9,<0.8.0 tzdata>=2025.3,<2025.4 diff --git a/requirements_prod.txt b/requirements_prod.txt new file mode 100644 index 00000000..467ea47f --- /dev/null +++ b/requirements_prod.txt @@ -0,0 +1,5 @@ +channels-redis>=4.3.0,<4.4.0 +django-redis>=6.0.0,<6.1.0 +gunicorn>=23.0.0,<23.1.0 +mysql-connector-python>=9.5.0,<9.6.0 +supervisor-stdlog>=0.7.9,<0.8.0 From 9e131f1d0d22aff81e3741f58fb85e9f4525eb7e Mon Sep 17 00:00:00 2001 From: Neraste Date: Sat, 3 Jan 2026 16:39:13 +0100 Subject: [PATCH 10/16] Update documentation --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a7441861..27720636 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Server for the Dakara project. -## Installation +## Local installation To install Dakara completely, you have to get all the parts of the project. Installation guidelines are provided here: @@ -48,6 +48,12 @@ Install dependencies, at the root level of the repo (in the virtual environment) pip install -r requirements.txt ``` +For production, you will need some extra dependencies: + +```sh +pip install -r requirements_prod.txt +``` + ## Setup ### Settings presets @@ -96,7 +102,7 @@ dakara_server/manage.py runserver ``` In a separate terminal, also run the scheduler. -This is currently only required for the kara date stop feature (which stop the karaoke at a certain date): +This is currently only required for the kara date stop feature (which stops the karaoke at a certain date): ```sh dakara_server/manage.py runapscheduler From a511a53080dd1640923924980b92a2cbc341b343 Mon Sep 17 00:00:00 2001 From: Neraste Date: Sat, 3 Jan 2026 16:40:27 +0100 Subject: [PATCH 11/16] Put development Sqlite file in the project directory --- dakara_server/dakara_server/settings/base.py | 1 + dakara_server/dakara_server/settings/development.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dakara_server/dakara_server/settings/base.py b/dakara_server/dakara_server/settings/base.py index 1be3244b..e5087516 100644 --- a/dakara_server/dakara_server/settings/base.py +++ b/dakara_server/dakara_server/settings/base.py @@ -18,6 +18,7 @@ from dakara_server.version import __version__ as VERSION # noqa F401 BASE_DIR = Path(__file__).resolve().parent.parent.parent +PROJECT_DIR = BASE_DIR.parent # Application definition diff --git a/dakara_server/dakara_server/settings/development.py b/dakara_server/dakara_server/settings/development.py index 0652bdb4..56ed1f32 100644 --- a/dakara_server/dakara_server/settings/development.py +++ b/dakara_server/dakara_server/settings/development.py @@ -17,8 +17,8 @@ from dakara_server.settings.base import * # noqa F403 from dakara_server.settings.base import ( - BASE_DIR, EMAIL_ENABLED, + PROJECT_DIR, get_host_urls, get_rest_registration, ) @@ -43,7 +43,7 @@ DATABASES = { "default": { - "NAME": config("DATABASE_FILE", default=BASE_DIR / "db.sqlite3"), + "NAME": config("DATABASE_FILE", default=PROJECT_DIR / "db.sqlite3"), "ENGINE": "django.db.backends.sqlite3", } } From e200980cce1834a573348436052ca14c8e6deff9 Mon Sep 17 00:00:00 2001 From: Neraste Date: Sat, 3 Jan 2026 17:58:04 +0100 Subject: [PATCH 12/16] Remove coverage by default --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 7c370439..91df2a5e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,6 @@ omit = */tests/*, */conftest.py [tool:pytest] -addopts = --cov=dakara_server asyncio_mode = strict asyncio_default_fixture_loop_scope = function From 783df8e992652527dd97b192a12aeec7ea9ba6f3 Mon Sep 17 00:00:00 2001 From: Neraste Date: Sat, 3 Jan 2026 18:01:42 +0100 Subject: [PATCH 13/16] Set working directory in deployment scritps --- deployment/bin/apscheduler.sh | 2 ++ deployment/bin/daphne.sh | 2 ++ deployment/bin/gunicorn.sh | 2 ++ deployment/etc/supervisor/apscheduler.ini | 1 - deployment/etc/supervisor/daphne.ini | 1 - deployment/etc/supervisor/gunicorn.ini | 1 - 6 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deployment/bin/apscheduler.sh b/deployment/bin/apscheduler.sh index 1dadd4f3..eb596dcc 100755 --- a/deployment/bin/apscheduler.sh +++ b/deployment/bin/apscheduler.sh @@ -2,6 +2,8 @@ set -e +cd /app/dakara_server + # populating data volume /app/deployment/bin/make_directories.sh diff --git a/deployment/bin/daphne.sh b/deployment/bin/daphne.sh index 518479ac..d5650a01 100755 --- a/deployment/bin/daphne.sh +++ b/deployment/bin/daphne.sh @@ -2,6 +2,8 @@ set -e +cd /app/dakara_server + # populating data volume /app/deployment/bin/make_directories.sh diff --git a/deployment/bin/gunicorn.sh b/deployment/bin/gunicorn.sh index a1ec4932..7b4b6602 100755 --- a/deployment/bin/gunicorn.sh +++ b/deployment/bin/gunicorn.sh @@ -2,6 +2,8 @@ set -e +cd /app/dakara_server + # populating data volume /app/deployment/bin/make_directories.sh diff --git a/deployment/etc/supervisor/apscheduler.ini b/deployment/etc/supervisor/apscheduler.ini index c58fcf47..2ab687f6 100644 --- a/deployment/etc/supervisor/apscheduler.ini +++ b/deployment/etc/supervisor/apscheduler.ini @@ -1,6 +1,5 @@ [program:apscheduler] command=/app/deployment/bin/apscheduler.sh -directory=/app/dakara_server user=root autostart=true autorestart=unexpected diff --git a/deployment/etc/supervisor/daphne.ini b/deployment/etc/supervisor/daphne.ini index 5207c184..f96f0170 100644 --- a/deployment/etc/supervisor/daphne.ini +++ b/deployment/etc/supervisor/daphne.ini @@ -1,6 +1,5 @@ [program:daphne] command=/app/deployment/bin/daphne.sh -directory=/app/dakara_server user=root autostart=true autorestart=true diff --git a/deployment/etc/supervisor/gunicorn.ini b/deployment/etc/supervisor/gunicorn.ini index d2a3106e..cdf3e7e1 100644 --- a/deployment/etc/supervisor/gunicorn.ini +++ b/deployment/etc/supervisor/gunicorn.ini @@ -1,6 +1,5 @@ [program:gunicorn] command=/app/deployment/bin/gunicorn.sh -directory=/app/dakara_server user=root autostart=true autorestart=true From 878c8f72ccd288fc97c373bd9a881ab91295a2ae Mon Sep 17 00:00:00 2001 From: Neraste Date: Sat, 3 Jan 2026 18:15:36 +0100 Subject: [PATCH 14/16] Remove Player migration scheme as it is not stored in the database --- .../playlist/migrations/0013_player.py | 31 ++----------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/dakara_server/playlist/migrations/0013_player.py b/dakara_server/playlist/migrations/0013_player.py index 120dcf18..8147a08b 100644 --- a/dakara_server/playlist/migrations/0013_player.py +++ b/dakara_server/playlist/migrations/0013_player.py @@ -1,10 +1,6 @@ # Generated by Django 2.2.17 on 2021-09-23 10:42 -import datetime - -from django.db import migrations, models - -import internal.cache_model +from django.db import migrations class Migration(migrations.Migration): @@ -13,27 +9,4 @@ class Migration(migrations.Migration): ("playlist", "0012_playlist_entry_use_instrumental"), ] - operations = [ - migrations.CreateModel( - name="Player", - fields=[ - ( - "karaoke", - internal.cache_model.OneToOneField( - on_delete=internal.cache_model.CASCADE, - primary_key=True, - serialize=False, - to="playlist.Karaoke", - ), - ), - ("timing", models.DurationField(default=datetime.timedelta(0))), - ("paused", models.BooleanField(default=False)), - ("in_transition", models.BooleanField(default=False)), - ("date", models.DateTimeField(auto_now=True)), - ], - options={ - "abstract": False, - "managed": False, - }, - ), - ] + operations = [] From 843c05a273bfa6009a5385c97454f12ef3c9fef5 Mon Sep 17 00:00:00 2001 From: Neraste Date: Sat, 3 Jan 2026 18:34:46 +0100 Subject: [PATCH 15/16] Clean old jobs execution everyday --- .../playlist/management/commands/runapscheduler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dakara_server/playlist/management/commands/runapscheduler.py b/dakara_server/playlist/management/commands/runapscheduler.py index f5f1ae3b..5e494a45 100644 --- a/dakara_server/playlist/management/commands/runapscheduler.py +++ b/dakara_server/playlist/management/commands/runapscheduler.py @@ -14,14 +14,14 @@ @util.close_old_connections -def delete_old_job_executions(max_age=604_800): +def delete_old_job_executions(max_age=3600 * 24): """This job deletes APScheduler job execution entries older than `max_age` from the database. It helps to prevent the database from filling up with old historical records that are no longer useful. Args: max_age: The maximum length of time to retain historical job execution - records. Defaults to 7 days. + records. Defaults to 1 day. """ DjangoJobExecution.objects.delete_old_job_executions(max_age) @@ -44,12 +44,12 @@ def handle(self, *args, **options): scheduler.add_job( delete_old_job_executions, - trigger=CronTrigger(day_of_week="mon", hour="00", minute="00"), + trigger=CronTrigger(hour="00", minute="00"), id="delete_old_job_executions", max_instances=1, replace_existing=True, ) - logger.info("Added weekly job 'delete_old_job_executions'") + logger.info("Added daily job 'delete_old_job_executions'") try: logger.info("Starting scheduler...") From 28d329bd47b6139e515731f135367050fe43b9f3 Mon Sep 17 00:00:00 2001 From: Neraste Date: Sat, 3 Jan 2026 19:43:29 +0100 Subject: [PATCH 16/16] Use environment variables for superuser creation --- deployment/bin/gunicorn.sh | 10 ++++++---- deployment/docker-compose/docker-compose.yaml | 5 +++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/deployment/bin/gunicorn.sh b/deployment/bin/gunicorn.sh index 7b4b6602..9c739bc7 100755 --- a/deployment/bin/gunicorn.sh +++ b/deployment/bin/gunicorn.sh @@ -32,10 +32,12 @@ fi # create superuser once if [[ ! -f /data/state/gunicorn_first_superuser ]] then - # create the superuser with a dummy password - echo \ - "from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser('root', 'root@localhost', 'root')" \ - | ./manage.py shell + export DJANGO_SUPERUSER_USERNAME=${DAKARA_SUPERUSER_USERNAME:-root} + export DJANGO_SUPERUSER_EMAIL=${DAKARA_SUPERUSER_EMAIL:-root@localhost} + export DJANGO_SUPERUSER_PASSWORD=${DAKARA_SUPERUSER_PASSWORD:-root} + + ./manage.py createsuperuser --no-input + echo "Superuser created; you should create admin accounts and remove the superuser account as soon as possible for security reasons" touch /data/state/gunicorn_first_superuser fi diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml index cdd9612f..3bdb41a0 100644 --- a/deployment/docker-compose/docker-compose.yaml +++ b/deployment/docker-compose/docker-compose.yaml @@ -1,3 +1,7 @@ +# This file is an example and should be adapted to your needs. +# You may be interested to use an `.env` file to keep secrets: +# https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/ + services: mysql: image: mariadb:latest @@ -24,6 +28,7 @@ services: DAKARA_ALLOWED_HOSTS: "localhost,example.com" DAKARA_HOST_URL: "http://example.com" DAKARA_SECRET_KEY: "your-secret-key" + DAKARA_SUPERUSER_PASSWORD: "root-password" DAKARA_LANGUAGE_CODE: "en-us" DAKARA_TIME_ZONE: "UTC" DAKARA_LOG_LEVEL: "INFO"