diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..526c8a3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf \ No newline at end of file diff --git a/.gitignore b/.gitignore index d3cea46..34cc5ea 100644 --- a/.gitignore +++ b/.gitignore @@ -123,7 +123,7 @@ celerybeat.pid *.sage.py # Environments -.env +.env* .venv env/ venv/ @@ -162,8 +162,10 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -dev/ +/dev/* .vscode/settings.json */node_modules/* -records \ No newline at end of file +records + +/backend/alembic/versions/*.py \ No newline at end of file diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..3b93f1f --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +/alembic/versions/* \ No newline at end of file diff --git a/.example_env b/backend/.example_env similarity index 73% rename from .example_env rename to backend/.example_env index f4674ab..888f70e 100644 --- a/.example_env +++ b/backend/.example_env @@ -1,4 +1,6 @@ -# project envs +# This file is used to set up the environment variables for the backend +# This file should be copied to .env and edited with the correct values + # Domain # This would be set to the production domain with an env var on deployment # used by Traefik to transmit traffic and aqcuire TLS certificates @@ -6,14 +8,10 @@ DOMAIN="localhost" # To test the local Traefik config # DOMAIN=localhost.tiangolo.com -# Used by the backend to generate links in emails to the frontend -FRONTEND_HOST="http://localhost:5173" -# In staging and production, set this env var to the frontend host, e.g. -# FRONTEND_HOST=https://dashboard.example.com - # Environment: local, staging, production ENVIRONMENT="local" +API_V1_STR="/api/v1" PROJECT_NAME="EmonTools" STACK_NAME="full-stack-emontools-project" @@ -22,24 +20,19 @@ DATA_BASE_PATH="/opt/emon_tools/data" STATIC_BASE_PATH="/opt/emon_tools/static" # Backend +# Used by the backend to generate links in emails to the frontend +FRONTEND_HOST="http://localhost:5173" +# In staging and production, set this env var to the frontend host, e.g. +# FRONTEND_HOST=https://dashboard.example.com BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,https://localhost,https://localhost:5173" -SECRET_KEY= -FIRST_SUPERUSER=admin@example.com + +SECRET_KEY=changethis +FIRST_SUPERUSER=changethis FIRST_SUPERUSER_PASSWORD=changethis # Mysql -MYSQL_SERVER="127.0.0.1" +MYSQL_HOST="127.0.0.1" MYSQL_PORT=3306 -MYSQL_DB="app" +MYSQL_DB="emontools" MYSQL_USER="emontools" MYSQL_PASSWORD=changethis - -# EmonCms envs -# local emoncms instance -EMONCMS_URL="http://127.0.0.1:8080" -# emoncms apikey -API_KEY=changethis -# local phpfina paths -EMON_FINA_PATH="/var/lib/phpfina" -# phpfina archives directory -ARCHIVE_FINA_PATH="./datas" diff --git a/backend/.example_env_docker b/backend/.example_env_docker new file mode 100644 index 0000000..bb4112b --- /dev/null +++ b/backend/.example_env_docker @@ -0,0 +1,38 @@ +# This file is used to set up the environment variables for the backend +# This file should be copied to .env.docker and edited with the correct values + +# Domain +# This would be set to the production domain with an env var on deployment +# used by Traefik to transmit traffic and aqcuire TLS certificates +DOMAIN="localhost" +# To test the local Traefik config +# DOMAIN=localhost.tiangolo.com + +# Environment: local, staging, production +ENVIRONMENT="local" + +API_V1_STR="/api/v1" +PROJECT_NAME="EmonTools" +STACK_NAME="full-stack-emontools-project" + +# Data Path +DATA_BASE_PATH="/opt/emon_tools/data" +STATIC_BASE_PATH="/opt/emon_tools/static" + +# Backend +# Used by the backend to generate links in emails to the frontend +FRONTEND_HOST="http://localhost:5173" +# In staging and production, set this env var to the frontend host, e.g. +# FRONTEND_HOST=https://dashboard.example.com +BACKEND_CORS_ORIGINS="http://localhost,http://localhost:5173,https://localhost,https://localhost:5173" + +SECRET_KEY=changethis +FIRST_SUPERUSER=changethis +FIRST_SUPERUSER_PASSWORD=changethis + +# Mysql +MYSQL_HOST="127.0.0.1" +MYSQL_PORT=3306 +MYSQL_DB="emontools" +MYSQL_USER="emontools" +MYSQL_PASSWORD=changethis diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..c8186d7 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,65 @@ +# Use Python 3.11 slim base image +FROM python:3.11-slim + +# Set environment variables +ENV PYTHONUNBUFFERED=1 \ + # Add the local user's bin directory to PATH + PATH="/home/appuser/.local/bin:$PATH" + +# Create a non-root user and group +RUN groupadd --gid 1000 appuser && \ + useradd --uid 1000 --gid appuser --create-home appuser + +# Set working directory +WORKDIR /opt/emon_tools + +# Install build dependencies if needed (e.g., for any compiled packages) +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + dos2unix \ + default-mysql-client \ + python3-dev \ + default-libmysqlclient-dev \ + build-essential \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements and install them as the non-root user +COPY --chown=appuser:appuser requirements-docker-dev.txt . + +# Create the datas and static directories for storing data +RUN mkdir -p /opt/emon_tools/datas /opt/emon_tools/static /opt/emon_tools/backend \ + && chown -R appuser:appuser /opt/emon_tools/datas /opt/emon_tools/static /opt/emon_tools/backend + +# Switch to the non-root user +USER appuser + +# Install Python dependencies +# Use --no-cache-dir to avoid caching the packages in the image +# Use --user to install packages in the user's home directory +RUN pip install --no-cache-dir --user --upgrade pip && \ + pip install --no-cache-dir --user -r requirements-docker-dev.txt + +# Copy the backend source code +COPY --chown=appuser:appuser . ./backend + +# Replace env file with the one for Docker +COPY --chown=appuser:appuser .env.docker ./backend/.env + +RUN dos2unix /opt/emon_tools/backend/scripts/docker_start.sh +RUN dos2unix /opt/emon_tools/backend/scripts/pre_start.sh +RUN dos2unix /opt/emon_tools/backend/scripts/wait_for_db.sh + +# Ensure scripts have execution permission +RUN chmod +x /opt/emon_tools/backend/scripts/docker_start.sh \ + && chmod +x /opt/emon_tools/backend/scripts/pre_start.sh \ + && chmod +x /opt/emon_tools/backend/scripts/wait_for_db.sh + +# Set the PYTHONPATH to ensure the backend module is found +ENV PYTHONPATH=/opt/emon_tools + +# Expose FastAPI port (use 8000) +EXPOSE 8000 + +# Use the entrypoint script as CMD, which will run migrations and then launch uvicorn +CMD ["./backend/scripts/docker_start.sh"] \ No newline at end of file diff --git a/backend/alembic.ini b/backend/alembic.ini index 903c3e0..eb3f8b9 100644 --- a/backend/alembic.ini +++ b/backend/alembic.ini @@ -3,7 +3,7 @@ [alembic] # path to migration scripts # Use forward slashes (/) also on windows to provide an os agnostic path -script_location = ./emon_tools/fastapi/alembic +script_location = ./backend/alembic # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time diff --git a/backend/core/config.py b/backend/core/config.py index 2cadde8..6447e91 100644 --- a/backend/core/config.py +++ b/backend/core/config.py @@ -1,87 +1,216 @@ -"""Configuration settings for the FastAPI application.""" +""" +Configuration settings for the FastAPI application. +This module loads environment settings securely and applies best practices. +""" import secrets -from pathlib import Path -from os.path import join as join_path from urllib.parse import quote_plus -import warnings -from typing import Annotated, Any, Literal +from typing import Annotated, List, Literal from pydantic import ( - AnyUrl, - BeforeValidator, EmailStr, HttpUrl, MySQLDsn, + SecretStr, computed_field, + constr, model_validator, + field_validator ) -from typing_extensions import Self from pydantic_settings import BaseSettings from pydantic_settings import SettingsConfigDict +from backend.models.shared import ValidationConstants +from backend.utils.paths import ENV_PATH # pylint: disable=invalid-name -# ✅ Define the base directory as the root of the `emon_tools/` package -# Moves up to `emon_tools/` -BASE_DIR = Path(__file__).resolve().resolve().parents[2] -# ✅ Ensure `.env` is correctly detected in `emon_tools/` -env_file_path = join_path(BASE_DIR, ".env") +def parse_cors(value: str) -> List[str]: + """ + Convert a comma-separated string into a list of CORS origins, + ensuring that each origin is valid (using a simplified host pattern). -def parse_cors(v: Any) -> list[str] | str: - """Parse cors string""" - if isinstance(v, str) and not v.startswith("["): - return [i.strip() for i in v.split(",")] - elif isinstance(v, list | str): - return v - raise ValueError(v) + Each origin is stripped of extra whitespace and trailing slashes. + """ + # Note: Here, you may reuse the HTTP_HOST_REGEX from ValidationConstants. + host_pattern = ValidationConstants.HTTP_HOST_REGEX + origins = [] + for origin in value.split(","): + origin = origin.strip().rstrip("/") + if not origin: + continue + if not host_pattern.match(origin): + raise ValueError( + f"Invalid host in BACKEND_CORS_ORIGINS: '{origin}'" + ) + origins.append(origin) + if not origins: + raise ValueError( + "BACKEND_CORS_ORIGINS must contain at least one valid origin" + ) + return origins + + +REQUIRE_UPDATE_FIELDS = [ + "SECRET_KEY", "FIRST_SUPERUSER", "FIRST_SUPERUSER_PASSWORD", + "MYSQL_PASSWORD" +] class Settings(BaseSettings): - """Settings class for the FastAPI application.""" + """ + Settings for the FastAPI application. + Loads configurations from environment variables and from an env file. + """ model_config = SettingsConfigDict( - # Use top level .env file (one level above ./fastapi/) - env_file=env_file_path, - env_ignore_empty=True, - extra="ignore", + env_file=ENV_PATH, + env_file_encoding="utf-8", + env_ignore_empty=False, + extra="forbid", ) - print("Env file path: %s", model_config) - API_V1_STR: str = "/api/v1" - SECRET_KEY: str = secrets.token_urlsafe(32) + # Basic settings + DOMAIN: Annotated[ + str, + constr(pattern=ValidationConstants.HOST_REGEX.pattern) + ] + ENVIRONMENT: Literal["local", "development", "staging", "production"] + API_V1_STR: Annotated[ + str, + constr(pattern=ValidationConstants.URL_PATH_STR_REGEX.pattern) + ] + PROJECT_NAME: Annotated[ + str, + constr(pattern=ValidationConstants.KEY_REGEX.pattern) + ] + STACK_NAME: Annotated[ + str, + constr(pattern=ValidationConstants.SLUG_REGEX.pattern) + ] + DATA_BASE_PATH: Annotated[ + str, + constr(pattern=ValidationConstants.UNIX_PATH_REGEX.pattern) + ] + STATIC_BASE_PATH: Annotated[ + str, + constr(pattern=ValidationConstants.UNIX_PATH_REGEX.pattern) + ] + + # Frontend and CORS configuration + FRONTEND_HOST: HttpUrl + # BACKEND_CORS_ORIGINS should be provided + # as a comma-separated string in the env file. + BACKEND_CORS_ORIGINS: str + # ALLOWED_ORIGINS: List[str] + + @classmethod + @field_validator("BACKEND_CORS_ORIGINS", mode="before") + def validate_cors_origins(cls, v: str) -> List[str]: + """ + Validate the BACKEND_CORS_ORIGINS environment variable. + If it's a string, parse it into a list of valid origins. + """ + if not isinstance(v, str): + raise ValueError( + "BACKEND_CORS_ORIGINS must be a comma-separated string." + ) + return isinstance(parse_cors(v), list) + + @computed_field + @property + def ALLOWED_ORIGINS(self) -> List[str]: + """ + Combine validated BACKEND_CORS_ORIGINS (as a list) + with FRONTEND_HOST (host only, without scheme). + FRONTEND_HOST is stripped of its scheme and trailing slash. + """ + frontend = self.FRONTEND_HOST + origins = parse_cors(self.BACKEND_CORS_ORIGINS) + if frontend not in origins: + origins.append(frontend) + return origins + + # Security keys & tokens + # It is important to change these values for production. + SECRET_KEY: SecretStr = secrets.token_urlsafe(32) + FIRST_SUPERUSER: EmailStr + FIRST_SUPERUSER_PASSWORD: SecretStr TOKEN_ALGORITHM: str = "HS256" - # 30 minutes ACCESS_TOKEN_EXPIRE_MINUTES: int = 15 REFRESH_TOKEN_EXPIRE_MINUTES: int = 30 REFRESH_TOKEN_COOKIE_EXPIRE_SECONDS: int = 60 * 30 - DATA_BASE_PATH: str - STATIC_BASE_PATH: str - - FRONTEND_HOST: str = "http://localhost:5173" - ENVIRONMENT: Literal["local", "staging", "production"] = "local" - - BACKEND_CORS_ORIGINS: Annotated[ - list[AnyUrl] | str, BeforeValidator(parse_cors) - ] = [] + @classmethod + @field_validator("SECRET_KEY", mode="before") + def validate_secret_key(cls, v: SecretStr) -> SecretStr: + """ + Validate the SECRET_KEY environment variable. + """ + raw = v.get_secret_value() + if not ValidationConstants.SECRET_KEY_REGEX.match(raw): + raise ValueError( + "SECRET_KEY must be at least 32 characters " + "and include uppercase, lowercase, " + "digit, and special character." + ) + return v - @computed_field # type: ignore[prop-decorator] - @property - def all_cors_origins(self) -> list[str]: - """Get all cors origin""" - return [ - str(origin).rstrip("/") - for origin in self.BACKEND_CORS_ORIGINS - ] + [ - self.FRONTEND_HOST - ] + @classmethod + @field_validator("FIRST_SUPERUSER_PASSWORD", mode="before") + def validate_superuser_password(cls, v: SecretStr) -> SecretStr: + """ + Validate the FIRST_SUPERUSER_PASSWORD environment variable. + """ + raw = v.get_secret_value() + if not ValidationConstants.PASSWORD_REGEX.match(raw): + raise ValueError( + "FIRST_SUPERUSER_PASSWORD must be at least 8 characters " + "and include at least one lowercase letter, " + "one uppercase letter, one digit, and one special character." + ) + return v - PROJECT_NAME: str = "EmonTools" SENTRY_DSN: HttpUrl | None = None SELECTED_DB: Literal["Mysql", "Postgres"] = "Mysql" - MYSQL_SERVER: str - MYSQL_PORT: int = 3306 - MYSQL_DB: str - MYSQL_USER: str = "" - MYSQL_PASSWORD: str = "" + MYSQL_HOST: Annotated[ + str, + constr(pattern=ValidationConstants.HOST_REGEX.pattern) + ] + MYSQL_PORT: int + MYSQL_DB: Annotated[ + str, + constr(pattern=ValidationConstants.KEY_REGEX.pattern) + ] + MYSQL_USER: Annotated[ + str, + constr(pattern=ValidationConstants.KEY_REGEX.pattern) + ] + MYSQL_PASSWORD: SecretStr + + @classmethod + @field_validator("MYSQL_PORT", mode="before") + def validate_mysql_port(cls, v: int) -> int: + """ + Validate the MYSQL_PORT environment variable. + Ensure it is between 1 and 65535. + """ + if v < 1 or v > 65535: + raise ValueError("MYSQL_PORT must be between 1 and 65535") + return v + + @classmethod + @field_validator("MYSQL_PASSWORD", mode="before") + def validate_mysql_password(cls, v: SecretStr) -> SecretStr: + """ + Validate the MYSQL_PASSWORD environment variable. + Ensure it meets the password policy. + """ + raw = v.get_secret_value() + # Enforce a password policy similar to FIRST_SUPERUSER_PASSWORD + if not ValidationConstants.PASSWORD_REGEX.match(raw): + raise ValueError( + "MYSQL_PASSWORD must be at least 8 characters " + "and include at least one lowercase letter, " + "one uppercase letter, one digit, and one special character." + ) + return v # type: ignore[prop-decorator, C0103] @computed_field @@ -89,10 +218,12 @@ def all_cors_origins(self) -> list[str]: # Union[PostgresDsn, MySQLDsn]: def SQLALCHEMY_DATABASE_URI(self) -> MySQLDsn: """Set SqlAlchemy db url""" - encoded_password = quote_plus(self.MYSQL_PASSWORD) + encoded_password = quote_plus( + self.MYSQL_PASSWORD.get_secret_value() + ) return ( - f"mysql://{self.MYSQL_USER}:{encoded_password}@" - f"{self.MYSQL_SERVER}:{self.MYSQL_PORT}/{self.MYSQL_DB}" + f"mysql+pymysql://{self.MYSQL_USER}:{encoded_password}@" + f"{self.MYSQL_HOST}:{self.MYSQL_PORT}/{self.MYSQL_DB}" ) @computed_field # type: ignore[prop-decorator] @@ -102,35 +233,56 @@ def emails_enabled(self) -> bool: return bool(self.SMTP_HOST and self.EMAILS_FROM_EMAIL) EMAIL_TEST_USER: EmailStr = "test@example.com" - FIRST_SUPERUSER: EmailStr - FIRST_SUPERUSER_PASSWORD: str - - def _check_default_secret(self, var_name: str, value: str | None) -> None: - if value == "changethis": - message = ( - f'The value of {var_name} is "changethis", ' - "for security, please change it, at least for deployments." - ) - if self.ENVIRONMENT == "local": - warnings.warn(message, stacklevel=1) - else: - raise ValueError(message) + @classmethod @model_validator(mode="after") - def _enforce_non_default_secrets(self) -> Self: - self._check_default_secret("SECRET_KEY", self.SECRET_KEY) - self._check_default_secret("MYSQL_PASSWORD", self.MYSQL_PASSWORD) - self._check_default_secret( - "FIRST_SUPERUSER_PASSWORD", self.FIRST_SUPERUSER_PASSWORD - ) - - return self + def enforce_secure_and_required_values(cls, values: dict) -> dict: + """ + Ensure all required string fields are non-empty after stripping. + """ + insecure_default = "changethis" + # List of fields that must be non-empty. + required_fields = [ + "DOMAIN", "API_V1_STR", "PROJECT_NAME", "STACK_NAME", + "DATA_BASE_PATH", "STATIC_BASE_PATH", "FRONTEND_HOST", + "MYSQL_HOST", "MYSQL_DB", "MYSQL_USER", + "SECRET_KEY", "FIRST_SUPERUSER", "FIRST_SUPERUSER_PASSWORD", + "MYSQL_PASSWORD" + ] + for field_item in required_fields: + if not values.get(field_item)\ + or (isinstance(values.get(field_item), str) + and not values[field_item].strip()): + raise ValueError( + f"The environment variable '{field_item}' " + "must be provided and not be empty." + ) + # Enforce that secret values are changed. + for field_item in REQUIRE_UPDATE_FIELDS: + val = values[field_item] + if hasattr(val, "get_secret_value"): + raw_val = val.get_secret_value() + else: + raw_val = val + if raw_val.strip().lower() == insecure_default: + raise ValueError( + "Insecure value for '{field_item}' " + f"(found '{insecure_default}'). " + f"Please set a strong, unique value for {field_item}." + ) + return values - EMONCMS_URL: str = "" - API_KEY: str = "" - APP_API_KEY: str = "" - EMON_FINA_PATH: str = "/var/lib/phpfina" - ARCHIVE_FINA_PATH: str = "" +try: + settings = Settings() +except Exception as e: + # Raise with a clear error message if validation fails. + raise RuntimeError( + f"Configuration validation error:\n {str(e)}") from e -settings = Settings() +if __name__ == "__main__": + # For debugging, print out public settings without exposing secrets. + public_settings = settings.model_dump() + for field in REQUIRE_UPDATE_FIELDS: + public_settings.pop(field, None) + print(public_settings) diff --git a/backend/core/database.py b/backend/core/database.py index f5ab567..e7bfdba 100644 --- a/backend/core/database.py +++ b/backend/core/database.py @@ -1,4 +1,5 @@ -from backend.core.config import settings +"""Initialise the database connection.""" from sqlmodel import create_engine +from backend.core.config import settings -engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) \ No newline at end of file +engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) diff --git a/backend/core/db.py b/backend/core/db.py index f1055fe..68fe453 100644 --- a/backend/core/db.py +++ b/backend/core/db.py @@ -47,7 +47,7 @@ def init_db(session: Session) -> None: if not user: user_in = UserCreate( email=settings.FIRST_SUPERUSER, - password=settings.FIRST_SUPERUSER_PASSWORD, + password=settings.FIRST_SUPERUSER_PASSWORD.get_secret_value(), is_superuser=True, ) user = UserController.create_user(session=session, user_create=user_in) diff --git a/backend/core/deps.py b/backend/core/deps.py index 1717cc7..060dc75 100644 --- a/backend/core/deps.py +++ b/backend/core/deps.py @@ -64,7 +64,8 @@ def get_current_user( """ try: payload = jwt.decode( - token, settings.SECRET_KEY, algorithms=[settings.TOKEN_ALGORITHM] + token, settings.SECRET_KEY.get_secret_value(), + algorithms=[settings.TOKEN_ALGORITHM] ) token_data = TokenPayload(**payload) except (InvalidTokenError, ValidationError) as ex: diff --git a/backend/core/security.py b/backend/core/security.py index df21909..1a89457 100644 --- a/backend/core/security.py +++ b/backend/core/security.py @@ -48,7 +48,7 @@ def create_access_token( } encoded_jwt = jwt.encode( to_encode, - settings.SECRET_KEY, + settings.SECRET_KEY.get_secret_value(), algorithm=settings.TOKEN_ALGORITHM ) return encoded_jwt @@ -79,7 +79,7 @@ def create_refresh_token( } encoded_jwt = jwt.encode( to_encode, - settings.SECRET_KEY, + settings.SECRET_KEY.get_secret_value(), algorithm=settings.TOKEN_ALGORITHM ) return encoded_jwt @@ -91,7 +91,8 @@ def decode_refresh_token(token: str) -> uuid.UUID: """ try: payload = jwt.decode( - token, settings.SECRET_KEY, algorithms=[settings.TOKEN_ALGORITHM]) + token, settings.SECRET_KEY.get_secret_value(), + algorithms=[settings.TOKEN_ALGORITHM]) if payload.get("type") != "refresh": raise InvalidToken("Not a refresh token") user_id = uuid.UUID(payload.get("sub")) diff --git a/backend/backend_pre_start.py b/backend/fastapi_pre_start.py similarity index 100% rename from backend/backend_pre_start.py rename to backend/fastapi_pre_start.py diff --git a/backend/main.py b/backend/main.py index d6108c3..9151d68 100644 --- a/backend/main.py +++ b/backend/main.py @@ -30,21 +30,13 @@ def custom_generate_unique_id(route: APIRoute) -> str: generate_unique_id_function=custom_generate_unique_id, ) -origins = [ - "http://127.0.0.1:8000", - "http://localhost:8000", - "http://127.0.0.1:5173", - "http://localhost:5173", - "http://127.0.0.1:5174", - "http://localhost:5174", -] - app.add_middleware( CORSMiddleware, - allow_origins=origins, + allow_origins=settings.ALLOWED_ORIGINS, allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"] + allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], + allow_headers=["Authorization", "Content-Type", "X-Requested-With"], + max_age=3600, # cache preflight requests for 1 hour ) app.mount( diff --git a/backend/models/shared.py b/backend/models/shared.py new file mode 100644 index 0000000..4e58948 --- /dev/null +++ b/backend/models/shared.py @@ -0,0 +1,40 @@ +""" +Shared models and constants for the application. +""" +import re + +# pylint: disable=line-too-long + + +class ValidationConstants: + """ + A collection of regular expression constants used for validating configuration values. + + These include patterns for validating host/domain names, URLs, paths, project names, + slugs, passwords, secret keys, and MySQL names. + """ + #: Matches a host/domain (e.g., "localhost", an IPv4 address with an optional port, or a standard domain) + HOST_REGEX: re.Pattern = re.compile( + r"^(localhost|((\d{1,3}\.){3}\d{1,3}(:\d{1,5})?)|([A-Za-z0-9-]+\.[A-Za-z0-9-.]+))$" + ) + #: Matches an HTTP or HTTPS URL with a valid host/domain and an optional port. + HTTP_HOST_REGEX: re.Pattern = re.compile( + r"^https?:\/\/(?:localhost|(?:\d{1,3}\.){3}\d{1,3}|(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,})(?::\d{1,5})?$" + ) + #: Matches a URL path that must start with a slash and can contain allowed URL characters. + URL_PATH_STR_REGEX: re.Pattern = re.compile(r"^\/[A-Za-z0-9\-._~\/]*$") + #: Matches an alphanumeric key that can include underscores and hyphens (used for project names, etc.). + KEY_REGEX: re.Pattern = re.compile(r"^[A-Za-z0-9_-]+$") + #: Matches a slug: lowercase letters, digits, and hyphens (e.g., "my-project-123"). + SLUG_REGEX: re.Pattern = re.compile(r"^[a-z0-9-]+$") + #: Matches a Unix-style absolute path containing letters, digits, underscores, hyphens, and slashes. + UNIX_PATH_REGEX: re.Pattern = re.compile(r"^\/[A-Za-z0-9_\-\/]+$") + #: Validates that a password has at least 8 characters, including at least one lowercase letter, + #: one uppercase letter, one digit, and one special character. + PASSWORD_REGEX: re.Pattern = re.compile(r"^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[\W_]).{8,}$") + #: Validates that a secret key is at least 32 characters long and includes mixed-case letters, + #: digits, and special characters. + SECRET_KEY_REGEX: re.Pattern = re.compile(r"^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[\W_]).{32,}$") + #: Matches MySQL names (database and user names), + # which may only contain letters, digits, and underscores. + MYSQL_NAME_REGEX: re.Pattern = re.compile(r"^[A-Za-z0-9_]+$") diff --git a/backend/requirements-docker-dev.txt b/backend/requirements-docker-dev.txt new file mode 100644 index 0000000..68d3e50 --- /dev/null +++ b/backend/requirements-docker-dev.txt @@ -0,0 +1,30 @@ +setuptools>=75.8.0 +wheel>=0.45.1 +build>=1.2.2.post1 +emon-tools>=0.3.0 +pytest>=8.3.4 +pytest-cov>=6.0.0 +coverage>=7.6.9 +requests>=2.32.3 +aiohttp>=3.8.1 +pytest-aiohttp>=1.0.5 +pandas>=2.2.3 +requests>=2.32.3 +simplejson>=3.19.3 +numpy>=2.2.0 +pydantic >= 2.10.6 +pydantic-settings >= 2.7.1 +fastapi>=0.115.7 +uvicorn>=0.34.0 +sqlmodel>=0.0.22 +alembic>=1.14.1 +tenacity>=9.0.0 +dnspython>=2.7.0 +email-validator>=2.2.0 +PyJWT>=2.10.1 +passlib>=1.7.4 +python-multipart>=0.0.20 +python-slugify>=8.0.4 +mysqlclient>=2.0.3 +pymysql>=1.1.1 +sqlalchemy>=2.0.38 diff --git a/backend/scripts/docker_start.sh b/backend/scripts/docker_start.sh new file mode 100644 index 0000000..8ae736f --- /dev/null +++ b/backend/scripts/docker_start.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -e + +# wait for the database service to be available +./backend/scripts/wait_for_db.sh "${MYSQL_SERVER}" "${MYSQL_PORT}" "${MYSQL_USER}" "${MYSQL_PASSWORD}" + +# Run migrations +# Check if alembic/versions is empty +if [ -z "$(ls -A ./backend/alembic/versions)" ]; then + echo "Generating Alembic migration..." + alembic -c ./backend/alembic.ini revision --autogenerate -m "Initial migration" +fi + +# Run any pre-start tasks +echo "Initialyse DB and data..." +echo "Checking if pre_start.sh exists at $(pwd)/backend/scripts/pre_start.sh" +ls -l $(pwd)/backend/scripts/pre_start.sh +if ! ./backend/scripts/pre_start.sh; then + echo "Failed to initialise DB and data" + exit 1 # Ensure the script exits if needed +fi + +# Start the FastAPI server +if ! uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload; then + echo "Uvicorn failed to start. Dropping to a shell for debugging." + exit 1 # Ensure the script exits if needed +fi \ No newline at end of file diff --git a/backend/scripts/pre_start.sh b/backend/scripts/pre_start.sh index 743f5b3..871be66 100644 --- a/backend/scripts/pre_start.sh +++ b/backend/scripts/pre_start.sh @@ -1,13 +1,19 @@ -#! /usr/bin/env bash - +#!/bin/sh set -e set -x -# export PYTHONPATH="$(pwd)/..:$PYTHONPATH" -# Let the DB start conda run -n emon-tools-dev-fast -python -m backend.fastapi_pre_start + +# Ensure PYTHONPATH is set correctly +export PYTHONPATH=/opt/emon_tools + +echo "Current working directory: $(pwd)" + +echo "Initialysing DB..." +python -m backend.fastapi_pre_start || { echo "Failed to initialise DB"; exit 1; } # Run migrations -alembic -c emon_tools/fastapi/alembic.ini upgrade head +echo "Run Migrations" +alembic -c ./backend/alembic.ini upgrade head || { echo "Migration failed"; exit 1; } -# Create initial data in DB conda run -n emon-tools-dev-fast -python -m backend.initial_data \ No newline at end of file +# Create initial data in DB +echo "Create initial data in DB" +python -m backend.initial_data || { echo "Failed to create initial data"; exit 1; } diff --git a/backend/scripts/wait_for_db.sh b/backend/scripts/wait_for_db.sh new file mode 100644 index 0000000..245ac81 --- /dev/null +++ b/backend/scripts/wait_for_db.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -e + +# Usage: ./wait-for-db.sh +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " + exit 1 +fi + +DB_HOST="$1" +DB_PORT="$2" +DB_USER="$3" +DB_PASSWORD="$4" +MAX_ATTEMPTS=15 +SLEEP_SECONDS=5 +attempt_num=1 + +echo "Waiting for MySQL database at ${DB_HOST}:${DB_PORT}..." + +until mysqladmin ping --host="${DB_HOST}" --port="${DB_PORT}" --user="${DB_USER}" --password="${DB_PASSWORD}" --silent; do + if [ ${attempt_num} -ge ${MAX_ATTEMPTS} ]; then + echo "Database ${DB_HOST}:${DB_PORT} did not become available after ${MAX_ATTEMPTS} attempts." + exit 1 + fi + echo "Attempt ${attempt_num}/${MAX_ATTEMPTS}: Database ${DB_HOST}:${DB_PORT} is not available yet. Waiting ${SLEEP_SECONDS} seconds..." + attempt_num=$((attempt_num + 1)) + sleep ${SLEEP_SECONDS} +done + +echo "Database ${DB_HOST}:${DB_PORT} is available." \ No newline at end of file diff --git a/backend/utils/paths.py b/backend/utils/paths.py new file mode 100644 index 0000000..f6b727f --- /dev/null +++ b/backend/utils/paths.py @@ -0,0 +1,18 @@ +# backend/utils/paths.py + +from pathlib import Path +from os.path import join as join_path + +# This assumes `backend/` is your top-level package +ROOT_DIR = Path(__file__).resolve() + + +def find_dotenv(start_path: Path = ROOT_DIR) -> Path: + """Find the .env file in the directory tree starting from start_path.""" + for parent in [start_path] + list(start_path.parents): + potential = parent / '.env' + if potential.exists(): + return parent + raise FileNotFoundError(".env file not found") + +ENV_PATH = join_path(find_dotenv(), ".env") diff --git a/docker-compose/.gitignore b/docker-compose/.gitignore new file mode 100644 index 0000000..f586bea --- /dev/null +++ b/docker-compose/.gitignore @@ -0,0 +1,2 @@ +mysql_data/ +alembic/ \ No newline at end of file diff --git a/docker-compose/dev/.example_env b/docker-compose/dev/.example_env new file mode 100644 index 0000000..fb2b2e7 --- /dev/null +++ b/docker-compose/dev/.example_env @@ -0,0 +1,7 @@ +# all those values must same as .env.docker +MYSQL_HOST="changethis" +MYSQL_PORT=3306 +MYSQL_DB="changethis" +MYSQL_USER="changethis" +MYSQL_PASSWORD="changethis" +MYSQL_ROOT_PASSWORD="changethis" \ No newline at end of file diff --git a/docker-compose/dev/README.md b/docker-compose/dev/README.md new file mode 100644 index 0000000..f9630a4 --- /dev/null +++ b/docker-compose/dev/README.md @@ -0,0 +1,11 @@ +project-root/ +├── emon-app/ +│ ├── Dockerfile +│ └── ... +├── docker-compose/ +│ ├── dev +│ | ├── docker-compose.yml +│ | ├── requirements.txt +└── backend/ + ├── Dockerfile + └── ... \ No newline at end of file diff --git a/docker-compose/dev/docker-compose.yml b/docker-compose/dev/docker-compose.yml new file mode 100644 index 0000000..1f05b70 --- /dev/null +++ b/docker-compose/dev/docker-compose.yml @@ -0,0 +1,48 @@ +services: + emon_api_db: + image: bitnami/mariadb:latest + volumes: + - ./mysql_data:/bitnami/mariadb + environment: + - ALLOW_EMPTY_PASSWORD=no + - MARIADB_PORT_NUMBER=${MYSQL_PORT} + - MARIADB_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MARIADB_USER=${MYSQL_USER} + - MARIADB_PASSWORD=${MYSQL_PASSWORD} + - MARIADB_DATABASE=${MYSQL_DB} + networks: + - emon_tools_network + + emon_api: + build: + context: ../../backend + dockerfile: Dockerfile + volumes: + - ../../backend:/opt/emon_tools/backend + - ./alembic:/opt/emon_tools/backend/alembic/versions + - ../../datas:/opt/emon_tools/datas + - ../../static:/opt/emon_tools/static + ports: + - "8000:8000" + depends_on: + - emon_api_db + networks: + - emon_tools_network + + emon_app: + build: + context: ../../emon-app + dockerfile: Dockerfile + # Bind mount the source so that changes reflect immediately in the container. + volumes: + - ../../emon-app:/opt/app + # Optionally, add an anonymous volume to prevent overwriting container's node_modules: + - /opt/app/node_modules + ports: + - "5173:5173" + depends_on: + - emon_api + +networks: + emon_tools_network: + driver: bridge \ No newline at end of file diff --git a/emon-app/.dockerignore b/emon-app/.dockerignore new file mode 100644 index 0000000..b4910a3 --- /dev/null +++ b/emon-app/.dockerignore @@ -0,0 +1,2 @@ +node_modules/* +Dockerfile \ No newline at end of file diff --git a/emon-app/Dockerfile b/emon-app/Dockerfile new file mode 100644 index 0000000..74fc79f --- /dev/null +++ b/emon-app/Dockerfile @@ -0,0 +1,29 @@ +# Use slim Node base image +FROM bitnami/node:latest + +# Create non-root user +RUN useradd -m -u 1001 appuser + +# Set working directory +WORKDIR /opt/app + +# Copy package files first to leverage Docker cache +COPY package.json package-lock.json* ./ + +# Install dependencies +RUN npm install + +# Copy the rest of the app +COPY . . + +# Change ownership to the non-root user +RUN chown -R appuser:appuser /opt/app + +# Switch to non-root user +USER appuser + +# Expose Vite dev server port +EXPOSE 5173 + +# Start Vite in dev mode with hot reload +CMD [ "npm", "run", "dev", "--", "--host", "--port", "5173" ] \ No newline at end of file diff --git a/emon-app/vite.config.mts b/emon-app/vite.config.mts index 0cd6fa7..655a4b7 100644 --- a/emon-app/vite.config.mts +++ b/emon-app/vite.config.mts @@ -5,12 +5,11 @@ import react from '@vitejs/plugin-react' // https://vite.dev/config/ export default defineConfig(({ mode }) => ({ plugins: [react()], - //envDir: "/media/tepochtli/zino/Dev/python/mano8_repo/emon_tools/", server: { - host: 'localhost', + host: true, // 👈 Required for Docker port: 5173, + strictPort: true, }, - base: "http://localhost:5173", resolve: { alias: { "@": path.resolve(__dirname, "./src"), diff --git a/requirements-fastapi.txt b/requirements-fastapi.txt index 3a4350b..abbfc52 100644 --- a/requirements-fastapi.txt +++ b/requirements-fastapi.txt @@ -9,4 +9,5 @@ dnspython>=2.7.0 email-validator>=2.2.0 PyJWT>=2.10.1 passlib>=1.7.4 -python-multipart>=0.0.20 \ No newline at end of file +python-multipart>=0.0.20 +pymysql>=1.1.1 \ No newline at end of file