diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 1f0f82ccd..af4647fb4 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -1,8 +1,47 @@ -{:output {:exclude-files [".*resources/public/js/compiled.*" +{;; Prevent analysis of compiled CLJS output -- source copies live there + ;; and cause "redefined var" when IDE analyzes both src/ and compiled/. + :exclude-files "resources/public/js/compiled" + :output {:exclude-files [".*resources/public/js/compiled.*" ".*docker/scripts/.*"]} - :linters {:shadowed-fn-param {:level :off} - :shadowed-var {:level :off} + :linters {;; Enabled at :warning so NEW accidental shadows are caught. + ;; :exclude covers established patterns: core vars used as param names + ;; (name, key, type, etc.) and domain terms used as both defs and params + ;; (level, ability, armor, etc.) across modifier/option/character code. + :shadowed-fn-param {:level :warning + :exclude [source character]} + :shadowed-var {:level :warning + :exclude [;; clojure.core / cljs.core vars used as parameter names + name key type val num first last next ns fn comp + int char chars bytes str time atom ref set class + list map range min max mod count filter flatten + keys vals identity ancestors comparator cond + list? sequential? set? + ;; re-frame.core/path — used as param in route handlers + path + ;; Domain terms used as both ns-level defs and fn params + ;; in modifiers.cljc, options.cljc, character.cljc, etc. + level levels level-key ability abilities armor weapon weapons + action alignment cls equipment size subclass-name + skill-options skill-expertise tool-options weapon-proficiency-options + spellcasting-template feat-selections cantrip-selections + sorted-items selected-plugin-options built-template + ;; One-off project-var shadows — established code patterns + ;; where a fn/let binding intentionally shadows its ns def + actual-path all-armor-inventory app-header + available-selections builder-page character-summary + children critical-hit-values custom-equipment entity + first-class? following-usernames i instant item-adder + languages levels-selection message mod-cfg mod-key + name-result parties party-owner prepared-spells-by-class + prepares-spells query-map search-results source speed + spells-known style view]} + ;; 213 warnings. Many namespaces are required purely for side effects: + ;; re-frame event/sub registrations, modifier macros, spec loading. + ;; Remaining are test cleanup debt (genuinely unused test requires). :unused-namespace {:level :off} + ;; 567 warnings. Destructured map bindings used for documentation or + ;; structure (e.g. {:keys [name level key]} where only some are used + ;; in the body). Renaming to _ would lose semantic readability. :unused-binding {:level :off} :missing-else-branch {:level :warning} :clojure-lsp/unused-public-var @@ -14,7 +53,7 @@ orcpub.dnd.e5.options/spell-tags orcpub.dnd.e5.options/potent-spellcasting ;; Live callers exist but LSP can't trace them - orcpub.common/dissoc-in ; events.cljs + orcpub.common/dissoc-in ; events.cljs:2976 orcpub.dnd.e5.character/add-ability-namespaces ; test ;; Cross-file refs: used in template.cljc but defined in spell_subs.cljs orcpub.dnd.e5.spell-subs/sunlight-sensitivity @@ -22,9 +61,9 @@ ;; re-frame event handlers are dispatched via keyword, not var reference. ;; LSP can't connect reg-event-db registration to (dispatch [:keyword]). :exclude-when-defined-by #{re-frame.core/reg-event-db - re-frame.core/reg-event-fx - re-frame.core/reg-sub - re-frame.core/reg-sub-raw}} + re-frame.core/reg-event-fx + re-frame.core/reg-sub + re-frame.core/reg-sub-raw}} ;; garden.selectors vars are generated by macros (defselector, ;; defpseudoclass, gen-pseudo-class-defs, etc.) at compile time. ;; clj-kondo can't resolve macro-generated vars statically and @@ -34,8 +73,9 @@ ;; errors.cljc macros behind #?(:clj) reader conditional — ;; one kondo instance can't resolve them orcpub.errors]} - ;; read-string is a valid cljs.core symbol that clj-kondo - ;; doesn't recognize in its ClojureScript analysis data. + ;; Macros that introduce bindings kondo can't resolve statically. + ;; Each (ns/macro) entry suppresses unresolved symbols inside that + ;; macro's body. Modifier macros use defmacro with gensym bindings. :unresolved-symbol {:exclude [read-string @@ -66,7 +106,16 @@ (orcpub.entity-spec/make-entity) (orcpub.routes-test/with-conn) (orcpub.routes.folder-test/with-conn) + (orcpub.email-change-test/with-conn) (user/with-db)]}} + ;; native/cljs and web/cljs are separate source roots; kondo doesn't know + ;; about them so ns names appear to mismatch their file paths. + :config-in-ns {orcpub.core {:linters {:namespace-name-mismatch {:level :off}}} + orcpub.views {:linters {:namespace-name-mismatch {:level :off}}} + orcpub.dnd.e5.native-views {:linters {:namespace-name-mismatch {:level :off}}}} + ;; with-conn macros use bare symbol bindings — handled via + ;; :unresolved-symbol :exclude above (macroexpand hooks can't find + ;; the test namespaces and produce noisy warnings). :lint-as {reagent.core/with-let clojure.core/let hiccup.def/defhtml clojure.core/defn user/with-db clojure.core/let diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..b14ae8418 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,30 @@ +FROM clojure:temurin-21-lein + +# Install useful dev tools (including unzip for Datomic Pro) +RUN apt-get update \ + && apt-get install -y --no-install-recommends rlwrap git curl tmux make git-lfs lsof iproute2 unzip rsync maven \ + libfreetype6 fontconfig fonts-dejavu-core fonts-liberation libxrender1 libxext6 libxi6 libxrandr2 \ + && rm -rf /var/lib/apt/lists/* + +# Ensure git-lfs is configured system-wide so LFS files are available in Codespaces +RUN git lfs install --system || true + +# Install Leiningen so `lein` is available for LSP and project tasks +RUN curl -fsSL https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein -o /usr/local/bin/lein \ + && chmod +x /usr/local/bin/lein \ + && /usr/local/bin/lein --version || true + +WORKDIR /workspace + +# Copy project source (lib/ includes pdfbox vendor deps from host) +COPY . /workspace + +# Pre-cache Datomic Pro zip so post-create.sh skips the download. +# The actual install (extract, flatten, maven-install) is done by post-create.sh +# because the volume mount overwrites /workspace during container creation. +ARG DATOMIC_VERSION=1.0.7482 +RUN curl --fail -L -o "/tmp/datomic-pro-${DATOMIC_VERSION}.zip" \ + "https://datomic-pro-downloads.s3.amazonaws.com/${DATOMIC_VERSION}/datomic-pro-${DATOMIC_VERSION}.zip" + +# Default shell to bash for VS Code terminal +SHELL ["/bin/bash", "-c"] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..d1194a929 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,45 @@ +{ + "name": "ClojureScript Lein-Figwheel Dev", + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + "features": { + "ghcr.io/devcontainers/features/sshd:1": { + "version": "latest" + } + }, + "forwardPorts": [8890, 3449, 4334], + "portsAttributes": { + "8890": { + "label": "Backend Server", + "onAutoForward": "notify" + }, + "3449": { + "label": "Figwheel", + "onAutoForward": "silent" + }, + "4334": { + "label": "Datomic", + "onAutoForward": "silent" + } + }, + "postCreateCommand": [ + "bash", + "./.devcontainer/post-create.sh" + ], + "containerEnv": { + "POST_CREATE_VERBOSE": "1", + "DATOMIC_VERSION": "1.0.7482", + "DATOMIC_URL": "datomic:dev://localhost:4334/orcpub" + }, + "customizations": { + "vscode": { + "extensions": [ + "betterthantomorrow.calva", + "borkdude.clj-kondo" + ], + "settings": {} + } + } +} diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh new file mode 100755 index 000000000..832fb00a9 --- /dev/null +++ b/.devcontainer/post-create.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# +# Datomic Pro install script for devcontainer +# +# 1. Always remove lib/com/datomic/datomic-{type}/ if it exists +# 2. Unzip the Datomic zip (from /lib or /tmp, download if missing) into target dir +# 3. Flatten top-level subdir if present (some zips nest contents) +# 4. Run vendor maven-install from bin/ +# +# This script does NOT cherry-pick, rename, or check for specific files before extraction. +# All contents of the zip are placed in the target directory, overwriting any previous install. +# + +set -euo pipefail + +# Source .env from repo root if present (authoritative config) +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +if [ -f "$REPO_ROOT/.env" ]; then + set -a + # shellcheck disable=SC1090 + . "$REPO_ROOT/.env" + set +a +fi + +# Redirect all output to persistent logs for visibility during Codespace creation +LOG="/tmp/orcpub-post-create.log" +WORKSPACE_LOG="$REPO_ROOT/.devcontainer/post-create.log" +# Ensure workspace log exists and is writable (best-effort) +mkdir -p "$(dirname "$WORKSPACE_LOG")" 2>/dev/null || true +touch "$WORKSPACE_LOG" 2>/dev/null || true +# Tee to both /tmp and workspace log so it's inspectable in Codespaces UI +exec > >(tee -a "$LOG" "$WORKSPACE_LOG") 2>&1 + +# Optional verbose tracing: set POST_CREATE_VERBOSE=1 to enable `set -x` +if [ "${POST_CREATE_VERBOSE:-0}" = "1" ]; then + echo "[POST-CREATE] Verbose tracing enabled" + set -x +fi + +# Timestamp helper +ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; } +log() { echo "$(ts) [POST-CREATE] $*"; } + +log "Starting postCreateCommand... (logging to $LOG and $WORKSPACE_LOG)" + +# Configuration with defaults +DATOMIC_TYPE="${DATOMIC_TYPE:-pro}" +RAW_DATOMIC_VERSION="${DATOMIC_VERSION:-1.0.7482}" +# basename in case a path was provided (e.g., /tmp/datomic-pro-1.0.7482.zip) +RAW_DATOMIC_VERSION="$(basename "$RAW_DATOMIC_VERSION")" +# strip leading prefix and trailing .zip if present +DATOMIC_VERSION="${RAW_DATOMIC_VERSION#datomic-${DATOMIC_TYPE}-}" +DATOMIC_VERSION="${DATOMIC_VERSION%.zip}" + +log "Using DATOMIC_TYPE=$DATOMIC_TYPE, DATOMIC_VERSION=$DATOMIC_VERSION" + +# Paths +TARGET_DIR="lib/com/datomic/datomic-${DATOMIC_TYPE}/${DATOMIC_VERSION}" +ZIP_NAME="datomic-${DATOMIC_TYPE}-${DATOMIC_VERSION}.zip" +DOWNLOAD_URL="https://datomic-pro-downloads.s3.amazonaws.com/${DATOMIC_VERSION}/${ZIP_NAME}" + +# Clean and prepare target directory +if [ -d "${TARGET_DIR}" ]; then + log "Removing existing installation at ${TARGET_DIR}" + rm -rf "${TARGET_DIR}" +fi +mkdir -p "${TARGET_DIR}" + +# Find Datomic zip in /lib/ or /tmp/, download if missing +ZIP_PATH="/lib/${ZIP_NAME}" +if [ ! -f "$ZIP_PATH" ]; then + ZIP_PATH="/tmp/${ZIP_NAME}" + if [ ! -f "$ZIP_PATH" ]; then + log "Downloading Datomic from $DOWNLOAD_URL" + curl --fail --location --progress-bar -o "$ZIP_PATH" "$DOWNLOAD_URL" + fi +fi + +# Verify zip integrity (handles corrupt/incomplete downloads) +if ! unzip -t "$ZIP_PATH" >/dev/null 2>&1; then + log "Corrupt or incomplete zip detected, removing and re-downloading..." + rm -f "$ZIP_PATH" + curl --fail --location --progress-bar -o "$ZIP_PATH" "$DOWNLOAD_URL" + + # Verify the re-download + if ! unzip -t "$ZIP_PATH" >/dev/null 2>&1; then + log "ERROR: Re-downloaded zip is still corrupt. Check network connection or URL." + log "URL: $DOWNLOAD_URL" + exit 1 + fi +fi + +# Unzip Datomic distribution into target dir +log "Extracting $ZIP_PATH to $TARGET_DIR" +unzip -q "$ZIP_PATH" -d "${TARGET_DIR}" + +# Flatten if needed (some zips nest contents in a subdirectory) +TOP_SUBDIR=$(find "${TARGET_DIR}" -mindepth 1 -maxdepth 1 -type d -print -quit || true) +if [ -n "${TOP_SUBDIR}" ] && [ -z "$(find "${TARGET_DIR}" -maxdepth 1 -type f -print -quit)" ]; then + log "Flattening nested directory structure" + mv "${TOP_SUBDIR}"/* "${TARGET_DIR}/" + rmdir "${TOP_SUBDIR}" +fi + +# Run vendor maven-install +if [ -x "${TARGET_DIR}/bin/maven-install" ]; then + log "Running maven-install..." + (cd "${TARGET_DIR}" && bash bin/maven-install) +else + log "ERROR: bin/maven-install not found or not executable in ${TARGET_DIR}/bin" + exit 1 +fi + +log "Datomic ${DATOMIC_TYPE} ${DATOMIC_VERSION} installed successfully to ${TARGET_DIR}" diff --git a/.dockerignore b/.dockerignore index 2be7f65ff..46d72bf57 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,4 +8,8 @@ test/* *.md data/* log/* -backups/* \ No newline at end of file +backups/* +backup/* + +# Datomic Pro distribution (~280MB) — Docker images download their own from S3 +lib/com/datomic/ \ No newline at end of file diff --git a/.env.example b/.env.example index 4444c8eba..37bf73b63 100644 --- a/.env.example +++ b/.env.example @@ -12,17 +12,30 @@ PORT=8890 # --- Datomic Database --- +# Datomic Pro with dev storage protocol (required for Java 21 support) # ADMIN_PASSWORD secures the Datomic admin interface # DATOMIC_PASSWORD is used by the application to connect to Datomic # The password in DATOMIC_URL must match DATOMIC_PASSWORD ADMIN_PASSWORD=change-me-admin DATOMIC_PASSWORD=change-me-datomic -DATOMIC_URL=datomic:free://datomic:4334/orcpub?password=change-me-datomic +DATOMIC_URL=datomic:dev://datomic:4334/orcpub?password=change-me-datomic # --- Security --- -# Secret used to sign JWT tokens (20+ characters recommended) +# REQUIRED: JWT signing secret. Authentication will fail without this. +# 20+ random characters recommended (e.g., openssl rand -hex 16) SIGNATURE=change-me-to-something-unique-and-long +# Content Security Policy (strict|permissive|none) +CSP_POLICY=strict + +# Dev mode: CSP violations are logged (Report-Only) instead of blocked, +# allowing Figwheel hot-reload scripts to execute. +DEV_MODE=true + +# --- Logs --- +# Defaults to project logs/ if unset +LOG_DIR= + # --- Email (SMTP) --- # Leave EMAIL_SERVER_URL empty to disable email functionality EMAIL_SERVER_URL= diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 56db5e5f7..698c1f670 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -164,6 +164,7 @@ jobs: - name: Post PR comment with results if: github.event_name == 'pull_request' && always() + continue-on-error: true # Fork PRs get read-only GITHUB_TOKEN uses: actions/github-script@v7 with: script: | diff --git a/.github/workflows/docker-integration.yml b/.github/workflows/docker-integration.yml index 0c9c86c65..39c9a89f1 100644 --- a/.github/workflows/docker-integration.yml +++ b/.github/workflows/docker-integration.yml @@ -1,6 +1,10 @@ -# Docker integration tests: script validation always runs, container tests -# only run when Docker Hub images are available (orcpub/orcpub:latest may -# not exist for all release cycles). +# Dual-stack Docker integration tests: auto-detects Java 8 (develop) vs Java 21 +# (breaking/) based on whether dev.cljs.edn exists in the checkout. +# +# Java 8 (legacy): Pulls pre-built images from Docker Hub — skipped gracefully +# when images are unavailable. +# Java 21 (modern): Builds from source using docker-compose-build.yaml with +# Datomic Pro and eclipse-temurin:21. name: Docker Integration Test @@ -17,10 +21,40 @@ on: workflow_dispatch: jobs: + detect-stack: + name: Detect Stack + runs-on: ubuntu-latest + outputs: + build-mode: ${{ steps.detect.outputs.build-mode }} + compose-file: ${{ steps.detect.outputs.compose-file }} + stack-label: ${{ steps.detect.outputs.stack-label }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Detect stack from project files + id: detect + run: | + if [ -f "dev.cljs.edn" ]; then + echo "Detected: Java 21 / Datomic Pro / figwheel-main → build from source" + echo "build-mode=build" >> $GITHUB_OUTPUT + echo "compose-file=docker-compose-build.yaml" >> $GITHUB_OUTPUT + echo "stack-label=Java 21 / Datomic Pro (build)" >> $GITHUB_OUTPUT + else + echo "Detected: Java 8 / Datomic Free / cljsbuild → pull pre-built" + echo "build-mode=pull" >> $GITHUB_OUTPUT + echo "compose-file=docker-compose.yaml" >> $GITHUB_OUTPUT + echo "stack-label=Java 8 / Datomic Free (pre-built)" >> $GITHUB_OUTPUT + fi + docker-test: - name: Docker Setup & User Management + name: Docker Setup & User Management (${{ needs.detect-stack.outputs.stack-label }}) runs-on: ubuntu-latest - timeout-minutes: 10 + needs: detect-stack + timeout-minutes: 25 + + env: + COMPOSE_FILE: ${{ needs.detect-stack.outputs.compose-file }} steps: - name: Checkout @@ -77,21 +111,37 @@ jobs: fi echo "OK: --force regenerated .env with consistent passwords" - # ── Container tests (only if images are available) ─────────── + # ── Container image acquisition ───────────────────────────── + # Java 21: build from source (no pre-built images for new stack) + # Java 8: pull pre-built images from Docker Hub (skip if unavailable) - - name: Pull container images + - name: Build container images (Java 21) + id: build + if: needs.detect-stack.outputs.build-mode == 'build' + run: | + echo "Building from source with $COMPOSE_FILE..." + docker compose build + echo "Build complete" + + - name: Pull container images (Java 8) id: pull + if: needs.detect-stack.outputs.build-mode == 'pull' continue-on-error: true run: docker compose pull + # ── Container tests ───────────────────────────────────────── + # Run if images were built (Java 21) or successfully pulled (Java 8). + # steps.build.outcome is only set when the step runs; default to 'skipped'. + - name: Start datomic (no deps) - if: steps.pull.outcome == 'success' + id: start-datomic + if: steps.build.outcome == 'success' || steps.pull.outcome == 'success' run: | docker compose up -d --no-deps datomic echo "Datomic container started, waiting for health..." - name: Wait for datomic healthy - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | for i in $(seq 1 90); do CID=$(docker compose ps -q datomic 2>/dev/null) || true @@ -124,13 +174,13 @@ jobs: done - name: Start orcpub and web - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | docker compose up -d docker compose ps - name: Wait for orcpub healthy - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | for i in $(seq 1 90); do CID=$(docker compose ps -q orcpub 2>/dev/null) || true @@ -161,13 +211,13 @@ jobs: docker compose ps - name: Test — create user - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | ./docker-user.sh create testadmin admin@test.local SecurePass123 echo "Exit code: $?" - name: Test — check user exists - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | OUTPUT=$(./docker-user.sh check testadmin) echo "$OUTPUT" @@ -176,14 +226,14 @@ jobs: echo "$OUTPUT" | grep -q "true" # verified - name: Test — list includes user - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | OUTPUT=$(./docker-user.sh list) echo "$OUTPUT" echo "$OUTPUT" | grep -q "testadmin" - name: Test — duplicate user fails - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | if ./docker-user.sh create testadmin admin@test.local SecurePass123 2>&1; then echo "FAIL: Should have rejected duplicate user" @@ -192,11 +242,11 @@ jobs: echo "OK: Duplicate user correctly rejected" - name: Test — create second user - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: ./docker-user.sh create player2 player2@test.local AnotherPass456 - name: Test — list shows both users - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | OUTPUT=$(./docker-user.sh list) echo "$OUTPUT" @@ -204,14 +254,14 @@ jobs: echo "$OUTPUT" | grep -q "player2" - name: Test — verify already-verified user is idempotent - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | OUTPUT=$(./docker-user.sh verify testadmin) echo "$OUTPUT" echo "$OUTPUT" | grep -q "already verified" - name: Test — batch create users (with duplicates) - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | cat > /tmp/test-users.txt <<'TXT' # Test batch file @@ -231,7 +281,7 @@ jobs: echo "OK: Batch created 2 new, skipped 1 duplicate" - name: Test — batch users appear in list - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | OUTPUT=$(./docker-user.sh list) echo "$OUTPUT" @@ -239,7 +289,7 @@ jobs: echo "$OUTPUT" | grep -q "batch2" - name: Test — init creates admin from .env - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | # Append INIT_ADMIN_* vars to .env printf '\nINIT_ADMIN_USER=initadmin\nINIT_ADMIN_EMAIL=initadmin@test.local\nINIT_ADMIN_PASSWORD=InitPass789\n' >> .env @@ -258,7 +308,7 @@ jobs: echo "OK: init created admin from .env" - name: Test — init is idempotent (re-run skips existing) - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | # Running init again should not fail — duplicate is handled if ./docker-user.sh init 2>&1; then @@ -268,7 +318,7 @@ jobs: echo "OK: init correctly reports duplicate on re-run" - name: Test — check nonexistent user fails - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | if ./docker-user.sh check nobody@nowhere.com 2>&1; then echo "FAIL: Should have reported user not found" @@ -277,7 +327,7 @@ jobs: echo "OK: Nonexistent user correctly not found" - name: Test — created user can log in via HTTP - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | # Use nginx (port 443) since orcpub:8890 is not exposed to host RESPONSE=$(curl -sk -X POST https://localhost/login \ @@ -301,7 +351,7 @@ jobs: fi - name: Test — wrong password is rejected - if: steps.pull.outcome == 'success' + if: steps.start-datomic.outcome == 'success' run: | HTTP_CODE=$(curl -sk -o /dev/null -w "%{http_code}" \ -X POST https://localhost/login \ diff --git a/.gitignore b/.gitignore index 80f856836..7375787d0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +# Ignore environment file (secrets/config) +.env .editorconfig .gitattributes /resources/public/css/compiled @@ -7,6 +9,10 @@ figwheel_server.log pom.xml *jar /lib/ +# Vendored snapshot allowed: pdfbox is required locally (vendor snapshot not available upstream) +!lib/org/apache/pdfbox/ +!lib/org/apache/pdfbox/pdfbox/2.1.0-SNAPSHOT/ +!lib/org/apache/pdfbox/pdfbox/2.1.0-SNAPSHOT/* /classes/ /out/ /target/ @@ -50,5 +56,32 @@ cljs-test-runner-out # As created by some LSP-protocol tooling, e.g. nvim-lsp .lsp -# Claude Code local session data +# Datomic local files +/.datomic/ + +# Local audit outputs +/audit/ + +# Cursor IDE local config (contains user-specific paths) +.cursor/ + +# Claude Code local data (conversation history, credentials) .claude-data/ + +# Ignore all log files in logs/ +logs/ + +*/*.log +/workspaces/.codespaces/.persistedshare/creation.log +.devcontainer/post-create.log + +scripts/analyze/ + +# Test user credentials log (created by ./menu user create / dev-setup.sh) +.test-users + +# cljsjs extracted externs (build artifact from classpath) +cljsjs/ + +# Local devcontainer setup (superseded by dotfiles) +.devcontainer/local-setup.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 98c9ccec0..54de2eccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,190 +1,252 @@ -# Changelog: Error Handling, Import Validation & Content Reconciliation +# Changelog -All changes target `develop` from `feature/error-handling-import-validation`. +All notable changes to this project will be documented in this file. +Format: per-commit entries grouped by category, newest first. + +## [breaking/2026-stack-modernization] + +### Infrastructure + +- **2026 full-stack modernization** (`22823da`) + Java 8 → 21, Datomic Free → Pro, Pedestal 0.5 → 0.7.0, React 15 → 18, + Reagent 0.6 → 2.0, re-frame 0.x → 1.4.4, PDFBox 2 → 3, clj-time → java-time, + figwheel-main, lambdaisland/garden, Jackson/Guava pinning. + +- **Consolidate dev tooling** (`6249565`) + Unified `user.clj` with lazy figwheel, nREPL helpers, lein aliases + (`fig:dev`, `fig:watch`, `fig:build`, `fig:test`), operational scripts + (`start.sh`, `stop.sh`, `menu`), `:dev`/`:uberjar`/`:lint`/`:init-db` profiles. + +- **Merge develop** (`1d50782`) + Integrate character folders, weapon builder (special/loading properties), + docker-compose updates from `origin/develop` (24 commits). + +### Bug Fixes + +- **`:class-name` → `:class`** (`263f290`) + Reagent 2.x overwrites hiccup tag classes with `:class-name`. Converted all + UI uses to `:class`; 18 remaining `:class-name` are D&D data keys (correct). + +- **Subscribe-outside-reactive-context — phase 1** (`c2290ca`) + 42 fixes across events.cljs, options.cljc, classes.cljc, core.cljs. + Patterns: direct db read, plugin-data map, track! template cache, SSOT pure fns. + +- **Subscribe-outside-reactive-context — phase 2** (`09d7e4c`) + 14 fixes across options.cljc, pdf_spec.cljc, equipment_subs.cljs, views.cljs. + Patterns: plugin-data threading, reg-sub-raw, move to render scope. + +- **Prereq subscribes → pure character fns** (`9cbc25a`) + 22 prereq-fn lambdas in options.cljc converted from `@(subscribe)` to pure + `(fn [character] ...)` functions. + +- **Multiclass/wizard prereqs** (`3249f88`) + 7 multiclass and spell-mastery prereqs in classes.cljc converted to pure fns. + +- **`def` + `partial` → `defn`** (`f578cdb`) + `option-language-proficiency-choice` captured subscribe at load time via + `partial`. Converted to `defn` for proper reactive context. + +### Cleanup + +- **Remove 11 orphaned subscriptions** (`bb2400d`) + 4 static map wrappers deleted (superseded by homebrew-aware versions). + 7 unused subs reader-discarded (`#_`) with comments: `all-melee-weapons`, + `item`, `base-spells-map`, `spell-option`, `spell-options`, + `filtered-monster-names`, `has-prof?`. Pre-existing tech debt, not caused + by subscribe refactor. + +- **Fix 591 missing-else-branch lint warnings** (`29c9f28`, via `fix/lint-missing-else`) + Mechanical `if→when`, `if-let→when-let`, `if-not→when-not` across 33 files. + Scripted fix (`scripts/fix-missing-else.py`) with column-precise substitution. + Also fixed 2 pre-existing bugs: `when` used instead of `if` for two-branch + conditionals in classes.cljc:1808 and options.cljc:463. + +- **Fix forward-reference lint error** (`792fe3c`) + `show-generic-error` used before its `def` alias in events.cljs. Changed to + fully-qualified `event-utils/show-generic-error`. + +- **Consolidate lint config** (`7476f10`) + All linter settings moved from project.clj `:lint` profile to + `.clj-kondo/config.edn` (single source of truth for IDE + CLI). Lint scope + expanded to cover `native/`, `test/`, `web/`. clj-kondo bumped to 2026.01.19. + LSP false-positive suppression via `:exclude-when-defined-by` for re-frame. + +- **Dead code cleanup — ~92 vars** (`6bbcd9a`, `b68b917`) + `#_` reader-discard on dead defs across 10 source files: deprecated ua/scag + refs, superseded template UI (ability roller, amazon frames), 17 never-dispatched + event handlers, dead style defs, duplicate constants. Includes cascade cleanup + (helpers that lost all callers). Each `#_` has a comment explaining why. + +- **Redundant expression fixes** (in `6bbcd9a`, `b68b917`, `429152e`) + Remove nested `(str (str ...))`, flatten `(and (and ...))`, remove duplicate + destructuring param, remove unused refers, narrow test `:refer` lists, + fix unreachable code in registration.cljc. + +### Enhancements + +- **Input debounce** (`d108134`) + Moved debounce from component-level `input-field` to `debounced-build-sub` + in subs.cljs (leading+trailing edge, 500ms). Eliminates per-keystroke + entity/build recomputation. + +- **Folder hardening** (`f28f58f`) + `on-folder-failure` event re-fetches server state on HTTP error. Client + + server blank-name validation. `check-folder-owner` wrapped with + `interceptor/interceptor`, returns 404 for missing folders. Named tempid + `"new-folder"` + `d/resolve-tempid`. `case` default clause in folders sub. + CSS class fix (`builder-dropdown` → `builder-option-dropdown`). + +- **UI polish** (`d163ca9`) + Zero-warning dev/prod builds, dev-mode CSP nonce, favicon, custom + `externs.js` for React 18 advanced compilation. + +### Tests + +- **CLJS test infrastructure** (`b96b1b6`) + figwheel-main test build, `test_runner.cljs`, pure function tests for + compute, entity, character accessors. + +- **JVM tests for new code** (`6124d9f`) + `compute-all-weapons-map`, feat-prereqs, pdf_spec pure functions, folder + routes (CRUD + blank rejection + trimming). + +- **Folder validation tests** (in `f28f58f`) + Blank name → 400, whitespace trimming, nil defaults to "New Folder", + name unchanged after rejected renames. + +### Documentation + +- **Migration docs** (`026b031`) + MIGRATION-INDEX.md, JAVA-COMPATIBILITY.md, datomic-pro.md, pedestal-0.7.md, + frontend-stack.md, library-upgrades.md, dev-tooling.md, ENVIRONMENT.md, + testing.md. + +- **STACK.md** (in `f28f58f`) + Library/dependency onboarding guide: architecture diagram, all frameworks, + build system, profiles, dependency pinning rationale. + +### Current Status + +- **174 JVM tests**, 444 assertions, 0 failures +- **0 CLJS errors**, 0 warnings (dev + advanced) +- **0 subscribe warnings** in browser console +- **0 linter errors**, 0 warnings --- -## New Features +## [feature/error-handling-import-validation] (merged) + +### New Features -### Import Validation (`import_validation.cljs` -- new file) +#### Import Validation (`import_validation.cljs` -- new file) - **Unicode normalization**: Converts smart quotes, em-dashes, non-breaking spaces, and 40+ other problematic Unicode characters to ASCII equivalents on import and homebrew save. Prevents copy-paste corruption from Word/Google Docs. - **Required field detection & auto-fill**: On import, missing required fields (`:name`, `:hit-die`, `:speed`, etc.) are auto-filled with placeholder values like `[Missing Name]`. Content types covered: classes, subclasses, races, subraces, backgrounds, feats, spells, monsters, invocations, languages, encounters. - **Trait validation**: Nested `:traits` arrays are checked for missing `:name` fields and auto-filled. - **Option validation**: Empty options (`{}`) created by the UI are detected and auto-filled with unique default names ("Option 1", "Option 2", etc.). - **Multi-plugin format detection**: Distinguishes single-plugin from multi-source orcbrew files for correct processing. -### Export Validation +#### Export Validation - **Pre-export warning modal**: Before exporting homebrew, all content is validated for missing required fields. If issues are found, a modal lists them with an "Export Anyway" option. - **Specific save error messages**: `reg-save-homebrew` now extracts field names from spec failures and shows targeted messages instead of generic "You must specify a name" errors. -### Content Reconciliation (`content_reconciliation.cljs` -- new file) +#### Content Reconciliation (`content_reconciliation.cljs` -- new file) - **Missing content detection**: When a character references homebrew content that isn't loaded (e.g., deleted plugin), the system detects missing races, classes, and subclasses. - **Fuzzy key matching**: Uses prefix matching and base-keyword similarity to suggest available content that resembles missing keys (top 5 matches with similarity scores). - **Source inference**: Guesses which plugin pack a missing key likely came from based on key structure. -### Missing Content Warning UI (`character_builder.cljs`) +#### Missing Content Warning UI (`character_builder.cljs`) - **Warning banner**: Orange expandable banner appears in character builder when content is missing, showing count and details. - **Detail panel**: Lists each missing item with its content type, key, inferred source, and suggestions for similar available content. - **DOM IDs for testability**: `#missing-content-warning`, `#missing-content-details`, `.missing-content-item` with `data-key` and `data-type` attributes. -### Conflict Resolution Modal (`views/conflict_resolution.cljs`, `events.cljs`) +#### Conflict Resolution Modal (`views/conflict_resolution.cljs`, `events.cljs`) - **Duplicate key detection**: On import, detects keys that conflict with already-loaded homebrew (both internal duplicates within a file and external conflicts with existing content). - **Resolution UI**: Modal presents each conflict with rename options. Key renaming updates internal references (subclass -> parent class mappings, etc.). - **Color-coded radio options**: Rename (cyan), Keep (orange), Skip (purple) with left-border + tinted background. All styles in Garden CSS. -### Import Log Panel (`views/import_log.cljs`) +#### Import Log Panel (`views/import_log.cljs`) - **Grouped collapsible sections**: Changes grouped into Key Renames, Field Fixes, Data Cleanup, and Advanced Details (collapsed by default). Empty sections hidden automatically. - **Detailed field fix reporting**: Field Fixes section shows per-item breakdown — which item, content type, which fields were filled, how many traits/options were fixed. - **Collapsible section component**: Reusable `collapsible-section` with configurable icon, colors, and default-expanded state. -### OrcBrew CLI Debug Tool (`tools/orcbrew.clj` -- new file) +#### OrcBrew CLI Debug Tool (`tools/orcbrew.clj` -- new file) - `lein prettify-orcbrew ` -- Pretty-prints orcbrew EDN for readability. - `lein prettify-orcbrew --analyze` -- Reports potential issues: nil-nil patterns, problematic Unicode, disabled entries, missing trait names, file structure summary. ---- - -## Bug Fixes +### Bug Fixes -### nil nil Corruption (`events.cljs`) +#### nil nil Corruption (`events.cljs`) - **Root cause fix**: `set-class-path-prop` was calling `assoc-in` with a nil path, producing `{nil nil}` entries in character data. Now guards against nil path before the second `assoc-in`. -### Nil Character ID Crash (`views.cljs`) +#### Nil Character ID Crash (`views.cljs`) - Character list page crashed with "Cannot form URI without a value given for :id parameter" when characters had nil `:db/id`. Added `(when id ...)` guard to skip rendering those entries. -### Subclass Key Preservation (`options.cljc`, `spell_subs.cljs`) +#### Subclass Key Preservation (`options.cljc`, `spell_subs.cljs`) - Subclass processing now uses explicit `:key` field if present (for renamed plugins), falling back to name-generated key. Prevents renamed keys from reverting. - `plugin-subclasses` subscription preserves map keys and sets `:key` on subclass data correctly. -### Plugin Data Robustness (`spell_subs.cljs`) +#### Plugin Data Robustness (`spell_subs.cljs`) - `plugin-vals` subscription wrapped in try-catch to skip malformed plugin data instead of crashing. - `level-modifier` handles unknown modifier types gracefully (logs warning, returns nil instead of throwing). - `make-levels` filters out nil modifiers with `keep`. -### Unhandled HTTP Status Crash (`subs.cljs`, `equipment_subs.cljs`) +#### Unhandled HTTP Status Crash (`subs.cljs`, `equipment_subs.cljs`) - All 7 API-calling subscriptions used bare `case` on HTTP status with no default clause. Any unexpected status (e.g., 400) threw `No matching clause`. Replaced with `handle-api-response` HOF that logs unhandled statuses to console. -### Import Log "Renamed key nil -> nil" (`events.cljs`, `import_validation.cljs`) +#### Import Log "Renamed key nil -> nil" (`events.cljs`, `import_validation.cljs`) - Key rename change entries used `:old-key`/`:new-key` fields but display code expected `:from`/`:to`. Unified on `:from`/`:to` across creation, application, and display. ---- +### Error Handling (Backend) -## Error Handling (Backend) - -### Database (`datomic.clj`) +#### Database (`datomic.clj`) - Startup wrapped in try-catch with structured errors: `:missing-db-uri`, `:db-connection-failed`, `:schema-initialization-failed`. -### Email (`email.clj`) +#### Email (`email.clj`) - Email config parsing catches `NumberFormatException` for invalid port (`:invalid-port`). - `send-verification-email` and `send-reset-email` check postal response and raise on failure (`:verification-email-failed`). -### PDF Generation (`pdf.clj`, `pdf_spec.cljc`) +#### PDF Generation (`pdf.clj`, `pdf_spec.cljc`) - Network timeouts (10s connect, 10s read) for image loading. Specific handling for `SocketTimeoutException` and `UnknownHostException`. - Nil guards throughout `pdf_spec.cljc`: `total-length`, `trait-string`, `resistance-strings`, `profs-paragraph`, `keyword-vec-trait`, `damage-str`, spell name lookup. All use fallback strings like "(unknown)", "(Unknown Spell)", "(Unnamed Trait)". -### Routes (`routes.clj`, `routes/party.clj`) +#### Routes (`routes.clj`, `routes/party.clj`) - All mutation endpoints wrapped with error handling: verification, password reset, entity CRUD, party operations. Each uses structured error codes (`:verification-failed`, `:entity-creation-failed`, `:party-creation-failed`, etc.). -### System (`system.clj`) +#### System (`system.clj`) - PORT environment variable parsing validates numeric input (`:invalid-port`). -### Error Infrastructure (`errors.cljc` -- expanded) +#### Error Infrastructure (`errors.cljc` -- expanded) - New error code constants for auth flows. - `log-error`, `create-error` utility functions. - `with-db-error-handling`, `with-email-error-handling`, `with-validation` macros for consistent patterns. ---- - -## Supporting Changes +### Supporting Changes -### Common Utilities (`common.cljc`) +#### Common Utilities (`common.cljc`) - `kw-base`: Extracts keyword base before first dash (e.g., `:artificer-kibbles` -> `"artificer"`). - `traverse-nested`: Higher-order function for recursively walking nested option structures. -### Styles (`styles/core.clj`) +#### Styles (`styles/core.clj`) - `.bg-warning`, `.bg-warning-item` CSS classes for warning banner UI. - `.conflict-*` Garden CSS classes for conflict resolution modal (backdrop, modal, header, footer, body, radio options with color-coded variants: cyan/rename, orange/keep, purple/skip). - `.export-issue-*` Garden CSS classes for export warning modal. -### App State (`db.cljs`) +#### App State (`db.cljs`) - Added `import-log` and `conflict-resolution` state maps to re-frame db. -### Subscriptions (`subs.cljs`, `equipment_subs.cljs`) +#### Subscriptions (`subs.cljs`, `equipment_subs.cljs`) - Import log, conflict resolution, export warning, missing content report subscriptions. -- `handle-api-response` HOF (`events.cljs`) — centralizes HTTP status dispatch with sensible defaults (401 → login, 500 → generic error) and catch-all logging for unhandled statuses. Replaces bare `case` statements across 7 API-calling subscriptions. +- `handle-api-response` HOF (`event_utils.cljc`) — centralizes HTTP status dispatch with sensible defaults (401 → login, 500 → generic error) and catch-all logging for unhandled statuses. Replaces bare `case` statements across 7 API-calling subscriptions. -### Entry Point (`core.cljs`) +#### Entry Point (`core.cljs`) - Dev version logging on startup. - Import log overlay component mounted in main view wrapper. -### Linter Configuration +#### Linter Configuration - `.clj-kondo/config.edn`: Exclusions for `with-db` macro and user namespace functions. - `.lsp/config.edn` (new): Explicit source-paths to prevent clojure-lsp from scanning compiled CLJS output in `resources/public/js/compiled/out/`. ---- - -## Files Changed - -| Status | File | Category | -|--------|------|----------| -| Modified | `src/clj/orcpub/datomic.clj` | Error handling | -| Modified | `src/clj/orcpub/email.clj` | Error handling | -| Modified | `src/clj/orcpub/pdf.clj` | Error handling | -| Modified | `src/clj/orcpub/routes.clj` | Error handling | -| Modified | `src/clj/orcpub/routes/party.clj` | Error handling | -| Modified | `src/clj/orcpub/styles/core.clj` | UI styles | -| Modified | `src/clj/orcpub/system.clj` | Error handling | -| **New** | `src/clj/orcpub/tools/orcbrew.clj` | CLI tool | -| Modified | `src/cljc/orcpub/common.cljc` | Utilities | -| Modified | `src/cljc/orcpub/dnd/e5/options.cljc` | Bug fix | -| Modified | `src/cljc/orcpub/errors.cljc` | Error infrastructure | -| Modified | `src/cljc/orcpub/pdf_spec.cljc` | Nil guards | -| Modified | `src/cljs/orcpub/character_builder.cljs` | Warning UI | -| **New** | `src/cljs/orcpub/dnd/e5/content_reconciliation.cljs` | Missing content detection | -| Modified | `src/cljs/orcpub/dnd/e5/db.cljs` | App state | -| Modified | `src/cljs/orcpub/dnd/e5/events.cljs` | Import/export events | -| **New** | `src/cljs/orcpub/dnd/e5/import_validation.cljs` | Validation framework | -| Modified | `src/cljs/orcpub/dnd/e5/spell_subs.cljs` | Plugin robustness | -| Modified | `src/cljs/orcpub/dnd/e5/subs.cljs` | Subscriptions | -| Modified | `src/cljs/orcpub/dnd/e5/views.cljs` | Fuzzy matching, nil guards | -| **New** | `src/cljs/orcpub/dnd/e5/views/import_log.cljs` | Import log panel | -| **New** | `src/cljs/orcpub/dnd/e5/views/conflict_resolution.cljs` | Conflict/export modals | -| Modified | `web/cljs/orcpub/core.cljs` | Entry point | -| **New** | `test/clj/orcpub/errors_test.clj` | Unit tests | -| **New** | `test/cljc/orcpub/dnd/e5/favored_enemy_language_test.cljc` | Unit tests | -| **New** | `test/clj/orcpub/tools/orcbrew_test.clj` | Unit tests | -| **New** | `test/cljc/orcpub/pdf_spec_test.clj` | Unit tests | -| **New** | `test/cljs/orcpub/dnd/e5/content_reconciliation_test.cljs` | Unit tests | -| **New** | `test/cljs/orcpub/dnd/e5/import_validation_test.cljs` | Unit tests | -| Modified | `test/cljc/orcpub/dnd/e5/folder_test.clj` | Lint fix | -| **New** | `test/duplicate-external-a.orcbrew` | Test fixture | -| **New** | `test/duplicate-external-b.orcbrew` | Test fixture | -| Modified | `.clj-kondo/config.edn` | Linter config | -| **New** | `.lsp/config.edn` | LSP config | -| **New** | `docs/CONFLICT_RESOLUTION.md` | Feature documentation | -| **New** | `docs/CONTENT_RECONCILIATION.md` | Feature documentation | -| **New** | `docs/ERROR_HANDLING.md` | Feature documentation | -| **New** | `docs/HOMEBREW_REQUIRED_FIELDS.md` | Feature documentation | -| **New** | `docs/ORCBREW_FILE_VALIDATION.md` | Feature documentation | -| **New** | `docs/LANGUAGE_SELECTION_FIX.md` | Feature documentation | -| **New** | `docs/README.md` | Documentation index | -| Modified | `.gitignore` | Ignore patterns | - ---- - -## Documentation - -Feature documentation is included in `docs/`: - -| Document | Covers | -|----------|--------| -| [ERROR_HANDLING.md](docs/ERROR_HANDLING.md) | Backend error macros, error codes, usage patterns | -| [CONFLICT_RESOLUTION.md](docs/CONFLICT_RESOLUTION.md) | Duplicate key detection, resolution modal, reference updates | -| [CONTENT_RECONCILIATION.md](docs/CONTENT_RECONCILIATION.md) | Missing content detection, fuzzy matching strategies | -| [HOMEBREW_REQUIRED_FIELDS.md](docs/HOMEBREW_REQUIRED_FIELDS.md) | Required fields per content type, breaking code locations | -| [ORCBREW_FILE_VALIDATION.md](docs/ORCBREW_FILE_VALIDATION.md) | Import/export validation user and developer guide | -| [LANGUAGE_SELECTION_FIX.md](docs/LANGUAGE_SELECTION_FIX.md) | Ranger favored enemy language corruption fix (#296) | - -## Design Principles +### Design Principles - **Import = permissive** (auto-fix and continue), **Export = strict** (warn user, let them decide) - **Placeholder text convention**: `[Missing Name]` format (square brackets indicate auto-filled) diff --git a/README.md b/README.md index f64254102..77b360d03 100644 --- a/README.md +++ b/README.md @@ -6,449 +6,449 @@
-

A web site that provides a D&D 5e Character sheet generator.

+

A D&D 5e Character Sheet Generator

- -This is the code forked from OrcPub2, from the [original](https://github.com/larrychristensen/orcpub) repository on Jan 7, 2019 with improvements. +Forked from [OrcPub2](https://github.com/larrychristensen/orcpub) (Jan 2019) with ongoing improvements. ![GitHub language count](https://img.shields.io/github/languages/count/orcpub/orcpub) ![GitHub top language](https://img.shields.io/github/languages/top/orcpub/orcpub) ![GitHub contributors](https://img.shields.io/github/contributors/orcpub/orcpub) ![GitHub repo size](https://img.shields.io/github/repo-size/orcpub/orcpub) -![GitHub last commit (branch)](https://img.shields.io/github/last-commit/orcpub/orcpub/develop) +![GitHub last commit (branch)](https://img.shields.io/github/last-commit/orcpub/orcpub/develop) ![GitHub pull requests](https://img.shields.io/github/issues-pr/orcpub/orcpub) ![GitHub closed pull requests](https://img.shields.io/github/issues-pr-closed/orcpub/orcpub) ![GitHub issues](https://img.shields.io/github/issues/orcpub/orcpub) ![GitHub closed issues](https://img.shields.io/github/issues-closed/orcpub/orcpub) -![Docker Pulls](https://img.shields.io/docker/pulls/orcpub/orcpub) -![Docker Cloud Automated build](https://img.shields.io/docker/cloud/automated/orcpub/orcpub) -![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/orcpub/orcpub) +![CI](https://img.shields.io/github/actions/workflow/status/orcpub/orcpub/continuous-integration.yml?branch=develop&label=CI) ![Docker Pulls](https://img.shields.io/docker/pulls/orcpub/orcpub) -[About](#about) • [Getting Started](#getting-started) • [Development](#development) • [Contributing](#how-do-i-contribute?) • [Fundamentals](#Fundamentals) +[About](#about) | [Quick Start](#quick-start) | [Development](#development) | [Architecture](#architecture) | [Contributing](#contributing) | [FAQ](#faq)
## About -Dungeon Master's Vault is a web server that allows you to host your own Character Generator website for D&D 5th Edition. - - -## Getting Started - -To run your own install of Dungeon Master's Vault, there are two ways to do this. - -1. Pulls docker containers from our docker repository. -2. Build your own. -In this section we will pull from the docker repository. If you want to build your own docker containers from source, see [Development](#development) +Dungeon Master's Vault is a full-stack Clojure/ClojureScript web application for generating and managing D&D 5th Edition character sheets. You can host your own instance or contribute to development. -You will need a few tools: +### Stack -- git -- A system that can run docker, with docker-compose (windows or unix) -- A SSL certificate. Self signed or from an issuing CA. -- smtp relay -- copy of this repo (for the ./deploy directory) +| Layer | Technology | +|-------|-----------| +| Runtime | Java 21 (OpenJDK) | +| Backend | Clojure 1.12, Pedestal 0.7, Buddy auth | +| Frontend | ClojureScript, React 18, Reagent 2.0, re-frame | +| Database | Datomic Pro (dev transactor) | +| Build | Leiningen, cljsbuild, figwheel-main | +| Dev environment | VS Code devcontainer (recommended) | -### Check out this branch +--- - Clone a copy of our repository to your machine: +## Quick Start - `git clone https://github.com/Orcpub/orcpub.git` if you don't have a github account +### Development (devcontainer) - `git clone git@github.com:Orcpub/orcpub.git` if you do want to make changes to the code and make pull requests. - -### Quick Setup (Recommended) - -Run the automated setup script to generate secure passwords, SSL certificates, and all required directories: +The fastest path. Requires [VS Code](https://code.visualstudio.com/) with [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers), or [GitHub Codespaces](https://github.com/features/codespaces). ```bash -./docker-setup.sh # Interactive — prompts for each value -./docker-setup.sh --auto # Non-interactive — generates secure defaults +git clone https://github.com/orcpub/orcpub.git +cd orcpub && code . +# Click "Reopen in Container" when prompted, then: +./scripts/dev-setup.sh # Install deps, init DB, create test user +./menu # Interactive service launcher ``` -Then start the containers and create your first user: +### Development (local machine) + +Requires Java 21, Leiningen 2.9+, and Datomic Pro (free, Apache 2.0). ```bash -docker-compose up -d -./docker-user.sh create admin admin@example.com MySecurePass123 +git clone https://github.com/orcpub/orcpub.git +cd orcpub +cp .env.example .env # Dev defaults work out of the box +./scripts/dev-setup.sh # Install deps, start Datomic, init DB, create test user +./menu start server # Backend on port 8890 +./menu start figwheel # Frontend hot-reload on port 3449 ``` -The `create` command creates a **pre-verified** account — no SMTP server or email confirmation needed. For batch user creation, additional commands, and full details see the [Docker User Management](docs/docker-user-management.md) guide. - -### Edit docker-compose.yaml - -Edit the `docker-compose.yaml` and update all the environmental variables and or paths as needed. +Log in at `http://localhost:8890` with **test@test.com** / **testpass**. -The application configuration is environmental variables based, meaning that its behavior will change when modifying them at start time. +For the full walkthrough (including manual setup without scripts), see **[docs/GETTING-STARTED.md](docs/GETTING-STARTED.md)**. -To modify the variables edit the `docker-compose.yaml` or set your own in your shell/environment variables. +### Self-hosting (Docker) -Example environment variables: +For running your own production instance: -```shell -EMAIL_SERVER_URL: '' # DNS name of your smtp server -EMAIL_ACCESS_KEY: '' # User for the mail server -EMAIL_SECRET_KEY: '' # Password for the user -EMAIL_SERVER_PORT: 587 # Mail server port -EMAIL_FROM_ADDRESS: '' # Email address to send from, will default to 'no-reply@orcpub.com' if not set -EMAIL_ERRORS_TO: '' # Email address that errors will be sent to -EMAIL_SSL: 'false' # Should SSL be used? Gmail requires this. -EMAIL_TLS: 'false' # Should TLS be used? -DATOMIC_URL: datomic:free://datomic:4334/orcpub?password=yourpassword # Url for the database -ADMIN_PASSWORD: supersecretpassword #The datomic admin password (should be different than the DATOMIC_PASSWORD) -DATOMIC_PASSWORD: yourpassword #The datomic application password -SIGNATURE: '' # The Secret used to hash your password in the browser, 20+ characters recommended +```bash +git clone https://github.com/orcpub/orcpub.git && cd orcpub +./docker-setup.sh # generates .env, SSL certs, directories +docker compose up -d # pull images and start +./docker-user.sh init # create admin from .env settings ``` -The `ADMIN_PASSWORD` and `DATOMIC_PASSWORD` +Visit `https://localhost`. See the [Docker deployment section](#docker-deployment) for full details including migration from older versions. -Update the `` in the `DATOMIC_URL` to match the password used in `DATOMIC_PASSWORD`. -Create an SSL certificate using `deploy/snakeoil.sh (or bat)` or simply edit the paths to an existing SSL certificate and key in the `web` service definition. +--- -These passwords are used to secure the database server Datomic. +## Development -### Create a certificate or use an existing one +> **New to Clojure?** See [docs/migration/dev-tooling.md](docs/migration/dev-tooling.md) for an explanation of Leiningen, profiles, the REPL, and how the dev tooling is organized. For the full stack upgrade context, see [docs/MIGRATION-INDEX.md](docs/MIGRATION-INDEX.md). -You will need a webserver certificate. For a quick SSL certificate, the script at `./deploy/snakeoil.sh` (unix) or `./deploy/snakeoil.bat` (windows) will create self signed certificate you can use, or you can make a request to a CA and install one from there. +### Prerequisites -By default the certificate is named `snakeoil.crt` and `snakeoil.key` and used by the nginx container here: +If not using the devcontainer, you need: -```shell - volumes: - - ./deploy/nginx.conf:/etc/nginx/conf.d/default.conf - - ./deploy/snakeoil.crt:/etc/nginx/snakeoil.crt - - ./deploy/snakeoil.key:/etc/nginx/snakeoil.key - - ./deploy/homebrew/:/usr/share/nginx/html/homebrew/ -``` +- **Java 21** (OpenJDK recommended) - [Adoptium](https://adoptium.net/) +- **Leiningen** 2.9+ - [install guide](https://leiningen.org/#install) +- **Datomic Pro** transactor - see [docs/migration/datomic-pro.md](docs/migration/datomic-pro.md) -For windows you will need OpenSSL installed to run the `./deploy/snakeoil.bat`. +### Project Layout -OpenSSL be installed via [chocolatey](https://chocolatey.org/) `choco install openssl` +``` +src/ +├── clj/ # Server-only Clojure (JVM) +├── cljc/ # Shared code (runs on both JVM and JS) +└── cljs/ # Client-only ClojureScript +web/ +└── cljs/ # Frontend application (Reagent/re-frame) +dev/ +└── user.clj # Dev tooling hub (REPL helpers + CLI) +test/ +├── clj/ # Server-side tests +├── cljc/ # Shared tests +└── cljs/ # Frontend tests +scripts/ # Shell scripts for service management +docs/ # Technical documentation and migration guides +``` -### nginx.conf +### Starting the Dev Environment -You will need the `./deploy/nginx.conf` or roll your own. +There are two ways to run services: the **interactive menu** or **shell scripts** directly. -### Launch the docker containers +#### Interactive Menu (recommended) -```shell - docker-compose pull - docker-compose up - -or- - docker-compose up -d +```bash +./menu ``` -If all went well you should be able to hit the site via `https://localhost` - -If not - run the docker containers with `docker-compose up` which will show you the logs of the containers and troubleshoot from there. +Displays service status and lets you start/stop services with single keystrokes. -### Importing your orcbrews automatically +#### User Management -To have your orcbrew file you want to load automatically when a new client connects, place it in the `./deploy/homebrew/homebrew.orcbrew` - -All orcbrew files have to be combined into a single file named "homebrew.orcbrew". +```bash +./menu add bob pass123 # Create bob@test.com (auto-verified) +./menu add user # Interactive prompt for name/password +./menu verify bob # Verify an existing user +./menu delete bob # Delete a user +./menu user # Show all user commands +``` -### Character Data +Email auto-generates as `@test.com`. Credentials are logged to `.test-users` (gitignored). -**Data directory** +#### Shell Scripts -Character item data is held in a database provided by Datomic. Datomic stores the character and magic item information in the `./data` directory. +```bash +# 1. Start Datomic transactor +./scripts/start.sh datomic -If you want to backup the database you only need to copy the `./data` directory after Datomic is shutdown. +# 2. Initialize database (first time only) +./scripts/start.sh init-db -If you want a new database, delete the `./data` directory to start over. +# 3. Start backend REPL +./scripts/start.sh server -**Log directory** +# 4. Start Figwheel (frontend hot-reload, headless watcher) +./scripts/start.sh figwheel -The `./logs` directory contains error logs for Datomic itself and any files here can be safely removed with out affecting character data. +# 5. Start Garden (CSS auto-compilation) +./scripts/start.sh garden +``` -Watch this directory and clean up old files, it can grow quite large quickly. It is recommended to setup log rotate or some other mechanism to clean these up. +Or run the first-time setup script, which starts Datomic, initializes the database, and creates a test user (`test` / `test@test.com` / `testpass`): -## Development +```bash +./scripts/dev-setup.sh +``` -### Building your own docker images +#### First-Time Dev Setup (manual) -There are three docker containers that will be built. +If not using the devcontainer or dev-setup.sh, you can run the steps yourself using `lein` (Leiningen, the Clojure build tool): -- orcpub_datomic_1 - the database service. -- orcpub_orcpub_1 - the JRE service that is the website. -- orcpub_web_1 - the ngnix web server that reverse proxies back to the JRE service. +```bash +# Download all project dependencies +lein deps -**Dependencies** +# Start the Datomic database transactor +./scripts/start.sh datomic -- [Docker](https://docs.docker.com/install/) -- [Docker Compose](https://docs.docker.com/compose/) -- [git](https://git-scm.com/downloads) +# Create the database and apply the schema. +# "with-profile init-db" tells Leiningen to skip slow ClojureScript compilation. +# "run -m user init-db" runs the init-db command in dev/user.clj. +lein with-profile init-db run -m user init-db +# Create a test user (auto-verified, email = test@test.com) +./menu add test testpass +``` -Unix instructions [here](https://github.com/Orcpub/orcpub/wiki/Orcpub-on-Ubuntu-18.04-with-Docker) +### REPL Workflow -Windows instructions [here](https://github.com/Orcpub/orcpub/wiki/Orcpub-on-Windows-10-with-Docker) +Clojure development centers on the REPL (Read-Eval-Print Loop). Start one with: -Docker Cheat [Sheet](https://github.com/Orcpub/orcpub/wiki/Docker-Cheat-sheet) +```bash +lein repl +``` +The `user` namespace loads automatically with these helpers: -### Getting started - building the docker image from source +```clojure +;; Start/stop the web server +(start-server) +(stop-server) -There are two docker-compose example files in this repository. +;; Initialize the database (first time) +(init-database) -`docker-compose.yaml` will pull from the docker repo which the community maintains and is rebuilt with the latest code from the develop branch. **this is the default** +;; Start Figwheel from the REPL +(fig-start) +(cljs-repl) ; connect to ClojureScript REPL (after fig-start) -`docker-compose-build.yaml` is an example of how to build from the local source from a git clone. +;; Database operations (no running server needed) +(create-user! (conn) {:username "bob" :email "bob@example.com" :password "pass" :verify? true}) +(verify-user! (conn) "bob@example.com") +(delete-user! (conn) "bob@example.com") +``` -Rename docker-compose-build.yaml to docker-compose.yaml and it will build from your downloaded cloned directory. +For the full list, see [docs/migration/dev-tooling.md](docs/migration/dev-tooling.md). -1. Start by forking this repo in your own github account and checkout the **develop** branch. `git clone git@github.com:Orcpub/orcpub.git` -2. Create snakeoil (self-signed) ssl certificates by running `./deploy/snakeoil.sh | .bat` or modify the docker-compose.yaml to your certificates. -3. Modify the docker-compose.yaml and code you want to. -4. Run `docker-compose build` to create the new containers built from the source. -5. Run docker-compose `docker-compose up` or if you want to demonize it `docker-compose up -d` -6. The website should be accessible via browser in `https://localhost` +### Validation -**NOTE** +Run these before committing: -The application configuration is Environmental Variable based, meaning that its behavior will change when modifying them at start time. To modify the variables edit the `docker-compose.yaml` or `docker-compose-build.yaml` or set your own in your shell/environment. +```bash +# Server-side tests (74 tests, 237 assertions) +lein test -Example variables: +# Linter (0 errors expected; warnings are from third-party libs) +lein lint -```shell -EMAIL_SERVER_URL: '' # Url to a smtp server -EMAIL_ACCESS_KEY: '' # User for the mail server -EMAIL_SECRET_KEY: '' # Password for the user -EMAIL_SERVER_PORT: 587 # Mail server port -EMAIL_FROM_ADDRESS: '' # Email address to send from, will default to 'no-reply@orcpub.com' -EMAIL_ERRORS_TO: '' # Email address that errors will be sent to -EMAIL_SSL: 'false' # Should SSL be used? Gmail requires this. -DATOMIC_URL: datomic:free://datomic:4334/orcpub?password=yourpassword # Url for the database -ADMIN_PASSWORD: supersecretpassword -DATOMIC_PASSWORD: yourpassword #(Same as above) -SIGNATURE: '' # The Secret used to hash your password in the browser, 20+ characters recommended +# ClojureScript compilation check +lein cljsbuild once dev ``` -To change the datomic passwords you can do it through the environment variables `ADMIN_PASSWORD_OLD` and `DATOMIC_PASSWORD_OLD` start the container once, then set the `ADMIN_PASSWORD` and `DATOMIC_PASSWORD` to your new passwords. +| Command | Scope | Catches | +|---------|-------|---------| +| `lein test` | Backend (JVM) | Logic, routes, DB, PDF errors | +| `lein lint` | CLJ + CLJS | Typos, unused vars, style | +| `lein cljsbuild once dev` | Frontend (CLJS) | Reagent/re-frame API changes | +| `lein fig:build` | Full frontend | One-time CLJS compilation check | -More on these passwords here. -[ADMIN_PASSWORD](https://docs.datomic.com/on-prem/configuring-embedded.html#sec-2-1) -[DATOMIC_PASSWORD](https://docs.datomic.com/on-prem/configuring-embedded.html#sec-2-1) +### Frontend Hot-Reload -## How do I contribute? -Thank you for rolling for initiative! +| Command | Mode | Use when | +|---------|------|----------| +| `./scripts/start.sh figwheel` | Headless watcher | Background/scripted startup | +| `lein fig:dev` | Interactive REPL | You want a ClojureScript REPL in your terminal | +| `lein fig:build` | One-time build | CI or quick compilation check | -We work on forks, and branches. Fork our repo, then create a new branch for any bug or new feature that you want to work on. +**CSS** is compiled separately by Garden (`lein garden once` or `./scripts/start.sh garden` for auto-watch). Both Figwheel and Garden need to run during active frontend work. -### Get started +### Editors -- Install Java: http://openjdk.java.net/ -- or http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html -- For MacOS/Linux Download [Datomic](https://www.datomic.com/get-datomic.html), and unzip it into a directory. +Any editor with Clojure support works. Recommended options: -* For Windows [DMV Datomic](https://github.com/Orcpub/orcpub/raw/refs/heads/develop/lib/datomic-free-0.9.5703.tar.gz) - newer versions do not work on Windows. it's a known issue that the Datomic team hasn't bothered to solve. It has to do with the max characters that windows can hold for a path. +| Editor | Plugin | REPL Connection | +|--------|--------|----------------| +| **VS Code** | [Calva](https://marketplace.visualstudio.com/items?itemName=betterthantomorrow.calva) | Jack-in with Leiningen, select `:dev` profile | +| **IntelliJ** | [Cursive](https://cursive-ide.com/) | Built-in REPL support | +| **Emacs** | [CIDER](https://cider.readthedocs.io/) | `C-c M-j` to jack in | +| **Vim/Neovim** | [vim-fireplace](https://github.com/tpope/vim-fireplace) | Connect to nREPL | - Launch Datomic by going to shell/cmd prompt in the unzipped directory and run: +### Environment Variables - On Windows: +Configuration is managed through a `.env` file at the repository root. Copy `.env.example` to get started: - `bin\transactor config/samples/free-transactor-template.properties` +```bash +cp .env.example .env +``` - On Mac/Unix: +Key variables: - `bin/transactor config/samples/free-transactor-template.properties` +| Variable | Purpose | Default | +|----------|---------|---------| +| `DATOMIC_URL` | Database connection string | `datomic:dev://localhost:4334/orcpub` | +| `SIGNATURE` | JWT signing secret (**required**) | dev default in `.lein-env` | +| `PORT` | Web server port | `8890` | +| `EMAIL_SERVER_URL` | SMTP server | (optional) | +| `CSP_POLICY` | Content Security Policy mode | `strict` | +| `DEV_MODE` | Enable dev features | `true` in dev | +**How env vars are loaded:** -- Install [leiningen](https://leiningen.org/#install) - - Mac / Linux: The latest version (2.9.1 as of this writing) should work. - - Windows: 2.9.3 Can be installed with [chocolatey](https://chocolatey.org/install) using `choco install lein --version 2.9.3` +- **`./menu` and `./scripts/start.sh`** source `.env` automatically — recommended for most workflows. +- **`lein repl` / `lein run`** read dev defaults from `.lein-env` (generated by `lein-environ` from the `:dev` profile). This includes a dev-only `SIGNATURE` so auth works out of the box. +- **`.env` values** (via scripts) and **real env vars** override `.lein-env` defaults. -- Download the code from your git fork +See [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) for the full list and precedence rules. - `git clone git@github.com:yourrepo/your.git` - - Use the clone url in YOUR repo. +--- -- cd into orcpub -- create a new branch for the bug fix or feature you are about to work on `git checkout -b ` -- Pick an editor from the next steps. -- run `lein with-profile +start-server repl` -- run `lein figwheel` Once lein figwheel finishes, a browser will launch. +## Docker Deployment -You should have all three processes running: the Datomic transactor, lein repl, and lein figwheel. +For self-hosting a production instance. -On the front end, When you save changes, it will auto compile and send all changes to the browser without the -need to reload. After the compilation process is complete, you will get a Browser Connected REPL. +### Containers -An easy way to try it is: +| Container | Purpose | +|-----------|---------| +| `datomic` | Datomic Pro database transactor | +| `orcpub` | JVM application server (Java 21) | +| `web` | nginx reverse proxy with SSL termination | -```clojure -(js/alert "Am I connected?") -``` +### Fresh Install -and you should see an alert in the browser window. -On the backend (PDF generation) you will have to restart lein repl to get your changes. +```bash +git clone https://github.com/orcpub/orcpub.git && cd orcpub -Code away! and make your commits. +# Interactive setup — generates .env, SSL certs, and directories +./docker-setup.sh -When your branch is ready create a pull request on our repo for a code review and merge back into our branch. +# Pull pre-built images and start +docker compose up -d +# Create your first user (once containers are healthy) +./docker-user.sh init # from .env settings +./docker-user.sh create # or directly +``` -### Suggested Editors +Visit `https://localhost` when running. -### Emacs -Emacs with [Cider](https://cider.readthedocs.io/en/latest/) you can run the command to start the Cider REPL: +To build from source instead of pulling images: +```bash +docker compose -f docker-compose-build.yaml build +docker compose -f docker-compose-build.yaml up -d ``` -C-c M-j -``` - -### Vim -[vim-fireplace](https://github.com/tpope/vim-fireplace) provides a good way to interact with a running repl without leaving Vim. - -### IntelliJ / Cursive -You can use the community edition of [IntelliJ IDEA](https://www.jetbrains.com/idea/download/) with the [Cursive plug-in](https://cursive-ide.com/). -### VS Code -You can use the open source edition of [Visual Studio Code](https://code.visualstudio.com/Download) with the Calva: Clojure & ClojureScript Interactive Programming, Clojure Code, and Bookmarks Extensions. +For environment variable details, see [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md). -To start REPL with VS Code: -* first launch datomic in a cmd window with the transactor snippet from above: `bin\transactor config/samples/free-transactor-template.properties` - * you can also just add that to a `.ps1` file inside your project for easier reference eg. `run-datomic ps1` -* THEN jack-in using the `Leiningen + Legacy Figwheel`, `figwheel-native`, and select the `:dev` and optionally `:start-server` +### Upgrading from Datomic Free (pre-2026) -### REPL +If you have an existing deployment using the old Java 8 / Datomic Free stack, +your `./data` directory is **not compatible** with the new Datomic Pro transactor. +The storage protocols (`datomic:free://` vs `datomic:dev://`) use different formats. -Once you have a REPL, you can run this from within it to create the database, transact the database schema, and start the server: +**You must migrate your database before upgrading.** The migration tool handles this: -You only need to `(init-database)` ONCE. +**Bare metal** — use `scripts/migrate-db.sh` which wraps the `bin/datomic` CLI: -```clojure -user=> (init-database) -user=> (start-server) +```bash +./scripts/migrate-db.sh backup # With old (Free) transactor running +# ... stop Free transactor, move ./data aside, start Pro transactor ... +./scripts/migrate-db.sh restore "datomic:dev://localhost:4334/orcpub?password=..." +./scripts/migrate-db.sh verify ``` -To stop you will need to do this: +**Docker** — use `docker-migrate.sh` which runs `bin/datomic` inside containers: -```clojure -user=> (stop-server) +```bash +./docker-migrate.sh backup # With old stack running +docker compose down +docker compose -f docker-compose-build.yaml build +docker compose -f docker-compose-build.yaml up -d +./docker-migrate.sh restore # After new stack is healthy +./docker-migrate.sh verify ``` -Within Emacs you should be able to save your file (C-x C-s) and reload it into the REPL (C-c C-w) to get your server-side changes to take effect. Within Vim with `vim-fireplace` you can eval a form with `cpp`, a paragraph with `cpip`, etc; check out its help file for more information. Regardless of editor, your client-side changes will take effect immediately when you change a CLJS or CLJC file while `lein figwheel` is running. +Or run `./docker-migrate.sh full` for a guided migration. -## Fundamentals +The backup is storage-protocol-independent and writes to `./backup/`, so databases +of any size (including 20GB+) are handled. See [docs/migration/datomic-data-migration.md](docs/migration/datomic-data-migration.md) +for the full guide including disk space planning and troubleshooting. -### Overview - from the original author - Larry +### User Management -The design is based around the concept of hierarchical option selections applying modifiers to a entity. - -Consider D&D 5e as an example. In D&D 5e you build and maintain characters, which are entities, by selecting from a set of character options, such as race and class. When you select a race you will be afforded other option selections, such as subrace or subclass. - -Option selections also apply modifiers to your character, such as 'Darkvision 60'. Option selections are defined in templates. An entity is really just a record of hierarchical choices made. - -A built entity is a collection of derived attributes and functions derived from applying all of the modifiers of all the choices made. Here is some pseudocode to this more concrete: - -```clojure -user> (def character-entity {:options {:race - {:key :elf, - :options {:subrace {:key :high-elf}}}}}) - -user> (def template {:selections [{:key :race - :min 1 - :max 1 - :options [{:name "Elf" - :key :elf - :modifiers [(modifier ?dex-bonus (+ ?dex-bonus 2)) - (modifier ?race "Elf")] - :selections [{:key :subrace - :min 1 - :max 1 - :options [{:name "High Elf" - :key :high-elf - :modifiers [(modifier ?subrace "High Elf") - (modifier ?int-bonus (+ ?int-bonus 1))]}]}]}]}]} - -user> (def built-character (build-entity charater-entity template)) - -user> built-character -{:race "Elf" - :subrace "High Elf" - :dex-bonus 2 - :int-bonus 1} +```bash +./docker-user.sh create # Create a verified user +./docker-user.sh batch users.txt # Bulk create from file +./docker-user.sh list # List all users +./docker-user.sh check # Check user status +./docker-user.sh verify # Verify unverified user ``` -This may seem overly complicated, but after my work on the Original Orcpub.com, I realized that this really the only real correct solution as it models how character building actually works. +See [docs/docker-user-management.md](docs/docker-user-management.md) for details. -The original Orcpub stored characters essentially like the built-character above with a centralized set of functions to compute other derived values. This is the most straightforward solution, but this has flaws: +### Importing Homebrew Content -* You have difficulty figuring out which options have been selected and which ones still need to be selected. -* You keep having to patch your data as your application evolves. For example, say you store a character's known spells as a list of spell IDs. Then you realize later that users want to also know what their attack bonus is for each spell. At the very least you'll have to make some significant changes to every stored character. -* It is not scalable. Every time you add in more options, say from some new sourcebook, you have to pile on more conditional logic in your derived attribute functions. Believe me, this gets unmanageable very quickly. -* It's not reusable in, say, a Rifts character builder. +Place your `.orcbrew` file at `./deploy/homebrew/homebrew.orcbrew` — it loads automatically when clients connect. All homebrew must be combined into this single file. -The architecture fixes these problems: +### Data Management -* You know exactly which options have been selected, which have not, and how every modifier is arrived at, given the entity and the most up-to-date templates. -* You don't need to patch up stored characters if you find a bug since characters are stored as just a set of very generic choices. -* It's scalable and pluggable. Most logic for derived values is stored inside of the options that create or modify these derived values. Many rules within D&D 5e allow for picking the best of alternate calculations for a particular attribute (AC comes to mind). With the OrcPub2 solution you can have an option easily override or modify a calculation without any other code having to know about it. This makes for a MUCH more manageable solution and makes it easy to add options as plugins. -* The entity builder engine could be used for building any character in any system. In fact, it could be used to build any entity in any system. For example, say you have a game system with well-defined mechanics building a vehicle, the entity builder engine could do that. +- **Database**: Stored in `./data/`. Back up this directory when Datomic is stopped. Delete it to start fresh. +- **Logs**: Stored in `./logs/`. Safe to clean up; does not affect character data. Set up log rotation for production. -### Modifiers +### Scripts Reference -Character modifiers are tough to get right. As mentioned above, the naive approach is to try to centralize all logic for a calculation in one place. For example you might have a character that looks like this: +| Script | Purpose | +|--------|---------| +| `scripts/migrate-db.sh` | Migrate data from Datomic Free to Pro (bare metal) | +| `docker-migrate.sh` | Migrate data from Datomic Free to Pro (Docker) | +| `docker-setup.sh` | Generate `.env`, SSL certs, and directories | +| `docker-user.sh` | Create, verify, and list users in the database | -```clojure -{:race "Elf" - :subrace "High Elf" - :base-abilities {:int 12 - :dex 13}} -``` +--- -Given this, you might start calculating initiative as follows: +## Architecture -```clojure -(defn ability-modifier [value] ...) - -(defn race-dexterity [character] - (case (:race character) - "Elf" 2 - ...)) - -(defn subrace-dexterity [character] ...) - -(defn base-ability [character ability-key] - (get-in character [:base-abilities ability-key])) - -(defn dexterity [character] - (+ (base-ability character :dex) - (race-dexterity character) - (subrace-dexterity character))) - -(defn initiative [character] - (ability-modifier (dexterity character))) -``` +*From the original author, Larry Christensen* -Consider what happens when you need to account for the 'Improved Initiative' feat, you'll need to add the calculation to the initiative function. Okay, this is probably still manageable. +### Overview -Then consider what happens when some cool subclass comes along that gets an initiative bonus at 9th level. Now it starts getting more unmanageable. When you try to add every option from every book into the mix, each of which might have some totally new condition for modifying initiative, you quickly end up with a nauseating ball of mud that will be scary to work with. +The design is based around the concept of hierarchical option selections applying modifiers to an entity. -This method decentralizes most calculations using modifiers associated with selected character options. When you add options you also specify any modifiers associated with that option. +In D&D 5e you build characters (entities) by selecting from character options like race and class. Each selection may offer further sub-selections (subrace, subclass) and applies modifiers to the character (e.g., "Darkvision 60", "+2 Dexterity"). -For example, in the entity example above, we have the elf option: +Option selections are defined in **templates**. An entity is just a record of hierarchical choices. A **built entity** is a collection of derived attributes computed by applying all modifiers from all choices. ```clojure -{:name "Elf" - :key :elf - :modifiers [(modifier ?dex-bonus (+ ?dex-bonus 2)) - (modifier ?race "Elf")] - ...} +;; An entity records choices +(def character-entity {:options {:race + {:key :elf, + :options {:subrace {:key :high-elf}}}}}) + +;; A template defines available options and their modifiers +(def template {:selections [{:key :race + :min 1 :max 1 + :options [{:name "Elf" + :key :elf + :modifiers [(modifier ?dex-bonus (+ ?dex-bonus 2)) + (modifier ?race "Elf")] + :selections [{:key :subrace + :min 1 :max 1 + :options [{:name "High Elf" + :key :high-elf + :modifiers [(modifier ?subrace "High Elf") + (modifier ?int-bonus (+ ?int-bonus 1))]}]}]}]}]}) + +;; Building resolves all modifiers into concrete values +(def built-character (build-entity character-entity template)) +;; => {:race "Elf" :subrace "High Elf" :dex-bonus 2 :int-bonus 1} ``` -If you build a character that has this :elf option selected, the modifiers will be applied the the :dex-bonus and :race in the built character. Let's look closer at the ?dex-bonus modifier. +### Why This Architecture? -The second argument to the modifier function is a special symbol that prefixes a ? on the attribute instead of the : we'll expect on the output attribute key, in this case ?dex-bonus will be modifying the value output to the :dex-bonus attribute. +The naive approach stores characters as flat attribute maps with centralized calculation functions. This has problems: -The third argument is a modifier body. +- Hard to track which options are selected vs. still needed +- Data patches required as the application evolves +- Centralized calculation functions become unmanageable as options grow +- Not reusable across game systems -This can be any Clojure expression you like, but if you will be deriving your new value from an old value or from another attribute you must use the ? reference. In this example we updating ?dex-bonus by adding 2 to it. +The entity/template/modifier architecture fixes these: -Modifiers can be derived from attributes that are derived from other attributes, and so forth. +- You know exactly which options are selected and how every value is derived +- Characters are stored as generic choices — no data migration needed when templates change +- Logic for derived values lives with the options that create them, making it scalable and pluggable +- The engine works for any entity in any system (characters, vehicles, etc.) -For example, we may have a character whose options provide the following chain of modifiers: +### Modifiers + +Modifiers use `?` references to build dependency chains: ```clojure (modifier ?dexterity 12) @@ -456,41 +456,45 @@ For example, we may have a character whose options provide the following chain o (modifier ?dex-mod (ability-mod ?dexterity)) (modifier ?int-mod (ability-mod ?intelligence)) (modifier ?initiative ?dex-mod) -(modifier ?initiative (+ ?initiative (* 2 ?intelligence-mod))) +(modifier ?initiative (+ ?initiative (* 2 ?int-mod))) ``` -### Modifier Order is Important! +**Order matters.** Since modifiers can reference attributes set by other modifiers, the system builds a dependency graph and applies modifiers in topologically sorted order. The `?` references are what make this dependency tracking possible. -Consider what would happen if we applied the above modifiers in a different order: +--- -```clojure -(modifier ?initiative (+ ?initiative (* 2 ?int-mod))) -(modifier ?dexterity 12) -(modifier ?intelligence 15) -(modifier ?dex-mod (ability-mod ?dexterity)) -(modifier ?int-mod (ability-mod ?intelligence)) -(modifier ?initiative ?dex-mod) -``` -Either our initiative calculation would throw an error our it would be completely wrong since the other derived attributes it depends on have not been applied yet. There is no logical ordering for which options should be applied, so modifiers can very well be provided out of order. +## Contributing + +1. Fork the repository +2. Create a feature branch: `git checkout -b my-feature` +3. Make your changes +4. Run validation: `lein test && lein lint` +5. Submit a pull request against `develop` -For this reason we have to build a dependency graph of derived attributes and then apply the modifiers in topologically sorted order. Identifying these dependencies is why we use the ? references. +We cannot accept pull requests containing copyrighted D&D content. You're welcome to fork and add content for private use. -## FAQs -**Q: I'm a newb Clojure developer looking to get my feet wet, where to start?** +--- -**A:** *First I would start by getting the fundamentals down at https://4clojure.oxal.org/ From there you might add some unit tests or pick up an open issue on the "Issues" tab (and add unit tests with it).* +## FAQ +**Q: I'm new to Clojure — where do I start?** -**Q: Your DSL for defining character options is pretty cool, I can build any type of character option out there. How about I add a bunch on content from the Player's Handbook?** +A: Try [4Clojure exercises](https://4clojure.oxal.org/) for fundamentals, then pick up a small issue from the Issues tab. The [dev tooling guide](docs/migration/dev-tooling.md) explains the project-specific tooling. -**A:** *We love your enthusiasm, but we cannot accept pull requests containing copyrighted content. We do, however, encourage you to fork us and create your own private version with the full content options.* +**Q: Can I add content from the Player's Handbook?** + +A: We cannot accept PRs with copyrighted content. Fork the project and add content for your own private use. + +--- ## Disclaimer -The use of this tool is meant for use for your own use and your own content. It is only meant and should only be used on campaigns with content that you legally possess. This tool is not affiliated with Roll20, or Wizards of the Coast. +This tool is for personal use with content you legally possess. It is not affiliated with Roll20 or Wizards of the Coast. ## Credits -Larry Christensen original author of [Orcpub2](https://github.com/larrychristensen/orcpub) + +Larry Christensen — original author of [OrcPub2](https://github.com/larrychristensen/orcpub) ## License -[EPL-2.0](LICENSE) \ No newline at end of file + +[EPL-2.0](LICENSE) diff --git a/deploy/start.sh b/deploy/start.sh index 538b1dc5a..76fdde4d2 100644 --- a/deploy/start.sh +++ b/deploy/start.sh @@ -10,18 +10,18 @@ if [ -z "$DATOMIC_PASSWORD" ]; then exit 1 fi -sed "/host=datomic/a alt-host=${ALT_HOST:-127.0.0.1}" -i /datomic/transactor.properties -sed "s/# storage-admin-password=/storage-admin-password=${ADMIN_PASSWORD}/" -i /datomic/transactor.properties -sed "s/# storage-datomic-password=/storage-datomic-password=${DATOMIC_PASSWORD}/" -i /datomic/transactor.properties +sed -i "/host=datomic/a alt-host=${ALT_HOST:-127.0.0.1}" /datomic/transactor.properties +sed -i "s/# storage-admin-password=/storage-admin-password=${ADMIN_PASSWORD}/" /datomic/transactor.properties +sed -i "s/# storage-datomic-password=/storage-datomic-password=${DATOMIC_PASSWORD}/" /datomic/transactor.properties if [ -n "$ADMIN_PASSWORD_OLD" ]; then - sed "s/# old-storage-admin-password=/old-storage-admin-password=$ADMIN_PASSWORD_OLD/" -i /datomic/transactor.properties + sed -i "s/# old-storage-admin-password=/old-storage-admin-password=$ADMIN_PASSWORD_OLD/" /datomic/transactor.properties fi if [ -n "$DATOMIC_PASSWORD_OLD" ]; then - sed "s/# old-storage-datomic-password=/old-storage-datomic-password=$DATOMIC_PASSWORD_OLD/" -i /datomic/transactor.properties + sed -i "s/# old-storage-datomic-password=/old-storage-datomic-password=$DATOMIC_PASSWORD_OLD/" /datomic/transactor.properties fi -sed "s/# encrypt-channel=true/encrypt-channel=${ENCRYPT_CHANNEL:-true}/" -i /datomic/transactor.properties +sed -i "s/# encrypt-channel=true/encrypt-channel=${ENCRYPT_CHANNEL:-true}/" /datomic/transactor.properties /datomic/bin/transactor /datomic/transactor.properties diff --git a/dev.cljs.edn b/dev.cljs.edn new file mode 100644 index 000000000..dc4244ac0 --- /dev/null +++ b/dev.cljs.edn @@ -0,0 +1,13 @@ +;; Figwheel-main build: dev +;; Run with: lein fig:dev +^{:watch-dirs ["web/cljs" "src/cljc" "src/cljs"] + :css-dirs ["resources/public/css"] + :ring-server-options {:port 3449} + :open-url "http://localhost:8890"} +{:main orcpub.core + :output-to "resources/public/js/compiled/orcpub.js" + :output-dir "resources/public/js/compiled/out" + :asset-path "/js/compiled/out" + :optimizations :none + :closure-defines {goog.DEBUG true} + :preloads [devtools.preload]} diff --git a/dev/user.clj b/dev/user.clj index 694b22b19..82ba7926e 100644 --- a/dev/user.clj +++ b/dev/user.clj @@ -1,29 +1,46 @@ (ns user (:require [clojure.java.io :as io] + [clojure.string :as str] [com.stuartsierra.component :as component] - [figwheel-sidecar.repl-api :as f] [datomic.api :as datomic] + [buddy.hashers :as hashers] [orcpub.routes :as r] [orcpub.system :as s] - [orcpub.db.schema :as schema])) + [orcpub.db.schema :as schema] + [orcpub.config :as config]) + (:gen-class)) -(alter-var-root #'*print-length* (constantly 50)) - -;; user is a namespace that the Clojure runtime looks for and -;; loads if its available - -;; You can place helper functions in here. This is great for starting -;; and stopping your webserver and other development services - -;; The definitions in here will be available if you run "lein repl" or launch a -;; Clojure repl some other way +;; --------------------------------------------------------------------------- +;; Dev tooling hub — REPL helpers and CLI entrypoint. +;; +;; The `user` namespace is special in Clojure: the runtime automatically looks +;; for and loads it when starting a REPL. Any functions defined here are +;; immediately available when you run `lein repl`. +;; +;; This file lives in dev/ and is only on the classpath in :dev and :init-db +;; profiles. It is NOT included in the production uberjar — keeping dev +;; tooling (user creation, DB init, Figwheel) out of production builds. +;; +;; Two ways to use it: +;; 1. REPL: (start-server), (init-database), (create-user! (conn) {...}), etc. +;; 2. CLI: lein with-profile init-db run -m user [args] +;; (see -main at bottom for available commands) +;; +;; The :init-db profile skips ClojureScript/Garden compilation for fast CLI +;; startup. It includes dev/ in source-paths so this file is loadable. +;; --------------------------------------------------------------------------- -;; You have to ensure that the libraries you :require are listed in your dependencies +;; Lazy-load figwheel-main only when needed (avoids loading it for server-only REPL) +(def ^:private fig-api + (delay + (try + (require 'figwheel.main.api) + (find-ns 'figwheel.main.api) + (catch Exception e + (println "figwheel.main.api not available:" (.getMessage e)) + nil)))) -;; Once you start down this path -;; you will probably want to look at -;; tools.namespace https://github.com/clojure/tools.namespace -;; and Component https://github.com/stuartsierra/component +(alter-var-root #'*print-length* (constantly 50)) (defonce -server (atom nil)) @@ -72,11 +89,15 @@ (defn init-database ([] - (init-database :free)) + (init-database nil)) ([mode] - (when-not (contains? #{:free :dev :mem} mode) - (throw (IllegalArgumentException. (str "Unknown db type " mode)))) - (let [db-uri (str "datomic" mode "://localhost:4334/orcpub")] + (let [env-uri (config/datomic-env) + db-uri (if (some-> env-uri not-empty) + env-uri + (let [m (or mode :dev)] + (when-not (contains? #{:free :dev :mem} m) + (throw (IllegalArgumentException. (str "Unknown db type " m)))) + (str "datomic" m "://localhost:4334/orcpub")))] (datomic/create-database db-uri) (let [conn (datomic/connect db-uri)] (datomic/transact conn schema/all-schemas))))) @@ -106,25 +127,22 @@ (defn fig-start "This starts the figwheel server and watch based auto-compiler. + Uses figwheel-main 0.2.20 with dev.cljs.edn build config. Afterwards, call (cljs-repl) to connect." ([] (fig-start "dev")) ([build-id] - ;; this call will only work as long as your :cljsbuild and - ;; :figwheel configurations are at the top level of your project.clj - ;; and are not spread across different lein profiles - - ;; otherwise you can pass a configuration into start-figwheel! manually - (f/start-figwheel! - {:figwheel-options {} - :build-ids [build-id] - :all-builds (get-cljs-build build-id)}))) + (if-let [api @fig-api] + ((ns-resolve api 'start) build-id) + (println "figwheel-main not available. Run 'lein fig:dev' instead.")))) (defn fig-stop "Stop the figwheel server and watch based auto-compiler." [] - (f/stop-figwheel!)) + (if-let [api @fig-api] + ((ns-resolve api 'stop-all)) + (println "figwheel-main not available."))) ;; if you are in an nREPL environment you will need to make sure you ;; have setup piggieback for this to work @@ -132,5 +150,149 @@ "Launch a ClojureScript REPL that is connected to your build and host environment. (NB: Call fig-start first.)" + ([] + (cljs-repl "dev")) + ([build-id] + (if-let [api @fig-api] + ((ns-resolve api 'cljs-repl) build-id) + (println "figwheel-main not available.")))) + +(defn add-test-user + "Creates a test user for development, already marked as verified. Only runs if ORCPUB_ENV=dev." [] - (f/cljs-repl)) + (if (= (System/getenv "ORCPUB_ENV") "dev") + (let [username "test" + email "test@example.com" + password "testpass"] + (r/register {:username username :email email :password password :verified true})) + (println "add-test-user is disabled outside dev environment."))) + +;; --------------------------------------------------------------------------- +;; Standalone DB connection (no running server required) +;; --------------------------------------------------------------------------- + +(defn conn + "Connect to Datomic without starting the full server. + Uses DATOMIC_URL env or default." + ([] (conn (config/get-datomic-uri))) + ([uri] (datomic/connect uri))) + +;; --------------------------------------------------------------------------- +;; User CRUD (for CLI and REPL use — does not require a running server) +;; --------------------------------------------------------------------------- + +(defn email-exists? [db email] + (boolean (datomic/q '[:find ?e . :in $ ?email :where [?e :orcpub.user/email ?email]] + db (str/lower-case email)))) + +(defn username-exists? [db username] + (boolean (datomic/q '[:find ?e . :in $ ?username :where [?e :orcpub.user/username ?username]] + db username))) + +(defn create-user! + "Create a user directly in the database. Does not require a running server. + + Usage from REPL: + (create-user! (conn) {:username \"bob\" :email \"bob@example.com\" :password \"pass\" :verify? true}) + + Usage from CLI: + lein with-profile init-db run -m user create-user bob bob@example.com pass verify" + [conn {:keys [username email password verify? send-updates?] :or {verify? false send-updates? false}}] + (let [db (datomic/db conn) + email (when email (str/lower-case (str/trim email))) + username (when username (str/trim username))] + (cond + (and email (email-exists? db email)) + (throw (ex-info "Email already exists" {:email email})) + + (and username (username-exists? db username)) + (throw (ex-info "Username already exists" {:username username})) + + :else + (let [now (java.util.Date.) + pw (hashers/encrypt (or password "password")) + tx {:orcpub.user/email email + :orcpub.user/username username + :orcpub.user/password pw + :orcpub.user/created now + :orcpub.user/send-updates? (boolean send-updates?) + :orcpub.user/verified? (boolean verify?)}] + @(datomic/transact conn [tx]))))) + +(defn verify-user! + "Mark an existing user as verified by username or email. + Does not require a running server." + [conn username-or-email] + (let [db (datomic/db conn) + user (r/find-user-by-username-or-email db username-or-email)] + (if-not user + (throw (ex-info "User not found" {:username-or-email username-or-email})) + @(datomic/transact conn [[:db/add (:db/id user) :orcpub.user/verified? true]])))) + +(defn delete-user! + "Remove a user entity by username or email. Dev only." + [conn username-or-email] + (let [db (datomic/db conn) + user (r/find-user-by-username-or-email db username-or-email)] + (if-not user + (throw (ex-info "User not found" {:username-or-email username-or-email})) + @(datomic/transact conn [[:db/retractEntity (:db/id user)]])))) + +;; --------------------------------------------------------------------------- +;; CLI entrypoint — used by scripts/start.sh, scripts/create_dummy_user.sh +;; +;; Usage: +;; lein with-profile init-db run -m user init-db +;; lein with-profile init-db run -m user init-db --add-test-user +;; lein with-profile init-db run -m user create-user [verify] +;; lein with-profile init-db run -m user verify-user +;; lein with-profile init-db run -m user delete-user +;; --------------------------------------------------------------------------- + +(defn -main [& [cmd & args]] + (try + (case cmd + "init-db" + (let [uri (config/get-datomic-uri)] + (println "Ensuring database exists at" uri) + (datomic/create-database uri) + (let [c (datomic/connect uri)] + (println "Applying schema...") + (datomic/transact c schema/all-schemas) + (println "DB init done.") + (when (some #{"--add-test-user"} args) + (println "Creating test user...") + (add-test-user)))) + + "create-user" + (let [[username email password & flags] args + verify? (some #{"verify"} flags) + c (conn)] + (println "Creating user:" username email "verified?" (boolean verify?)) + (create-user! c {:username username :email email :password password :verify? verify?}) + (println "User created.")) + + "verify-user" + (do (verify-user! (conn) (first args)) + (println "User verified:" (first args))) + + "delete-user" + (do (delete-user! (conn) (first args)) + (println "User deleted:" (first args))) + + ;; default + (do (println "Usage: lein with-profile init-db run -m user [args]") + (println "") + (println "Commands:") + (println " init-db [--add-test-user] Create database and apply schema") + (println " create-user

[verify] Create a user") + (println " verify-user Mark user as verified") + (println " delete-user Delete a user") + (System/exit 1))) + (catch Exception e + (binding [*out* *err*] + (println "Error:" (.getMessage e))) + (System/exit 1))) + ;; Datomic peer metrics thread is non-daemon and prevents clean JVM exit. + ;; Force exit after successful command completion. + (System/exit 0)) diff --git a/docker-compose-build.yaml b/docker-compose-build.yaml index 015b6daa8..ca66cad9c 100644 --- a/docker-compose-build.yaml +++ b/docker-compose-build.yaml @@ -14,13 +14,15 @@ services: EMAIL_ERRORS_TO: ${EMAIL_ERRORS_TO:-} EMAIL_SSL: ${EMAIL_SSL:-FALSE} EMAIL_TLS: ${EMAIL_TLS:-FALSE} - DATOMIC_URL: ${DATOMIC_URL:-datomic:free://datomic:4334/orcpub?password=change-me} + # Datomic Pro with dev storage protocol (required for Java 21 support) + DATOMIC_URL: ${DATOMIC_URL:-datomic:dev://datomic:4334/orcpub?password=change-me} SIGNATURE: ${SIGNATURE:-change-me-to-something-unique} depends_on: datomic: condition: service_healthy healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8890/"] + # BusyBox wget (Alpine): only -q and --spider are supported + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8890/"] interval: 5s timeout: 3s retries: 20 diff --git a/docker-compose.yaml b/docker-compose.yaml index 08e508977..545e4bff6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,9 @@ --- services: orcpub: - image: orcpub/orcpub:latest + # No :latest tag on Docker Hub — pin to newest published version. + # Override with ORCPUB_TAG env var for custom builds. + image: orcpub/orcpub:${ORCPUB_TAG:-release-v2.5.0.27} environment: PORT: ${PORT:-8890} EMAIL_SERVER_URL: ${EMAIL_SERVER_URL:-} @@ -12,13 +14,15 @@ services: EMAIL_ERRORS_TO: ${EMAIL_ERRORS_TO:-} EMAIL_SSL: ${EMAIL_SSL:-FALSE} EMAIL_TLS: ${EMAIL_TLS:-FALSE} - DATOMIC_URL: ${DATOMIC_URL:-datomic:free://datomic:4334/orcpub?password=change-me} + # Datomic Pro with dev storage protocol (required for Java 21 support) + DATOMIC_URL: ${DATOMIC_URL:-datomic:dev://datomic:4334/orcpub?password=change-me} SIGNATURE: ${SIGNATURE:-change-me-to-something-unique} depends_on: datomic: condition: service_healthy healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8890/"] + # BusyBox wget (Alpine): only -q and --spider are supported + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8890/"] interval: 5s timeout: 3s retries: 20 diff --git a/docker-migrate.sh b/docker-migrate.sh new file mode 100755 index 000000000..d7ee74859 --- /dev/null +++ b/docker-migrate.sh @@ -0,0 +1,457 @@ +#!/usr/bin/env bash +# ============================================================================= +# docker-migrate.sh — Datomic Free → Pro migration for Docker deployments +# ============================================================================= +# Wraps the bin/datomic CLI (backup-db / restore-db) in Docker containers. +# For bare-metal, use scripts/migrate-db.sh instead. +# +# The backup format is storage-protocol-independent — a Free backup restores +# into Pro. Uses bind-mounted volumes so 20GB+ databases write directly to +# the host filesystem without filling the container overlay. +# +# Note: Passwords in database URIs are passed as command-line arguments to +# docker run, which makes them visible in process lists and docker inspect. +# This is acceptable for self-hosted/dev environments. +# +# Usage: +# ./docker-migrate.sh backup Back up the running (Free) database +# ./docker-migrate.sh restore Restore into the new (Pro) database +# ./docker-migrate.sh verify Verify backup integrity +# ./docker-migrate.sh full Guided full migration (backup → swap → restore) +# +# Prerequisites: +# - Docker Compose v2 (docker compose plugin) +# - For backup: OLD datomic container must be running +# - For restore: NEW datomic container must be running +# - .env file must exist (run docker-setup.sh first) +# +# See docs/migration/datomic-data-migration.md for the full guide. +# ============================================================================= + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BACKUP_DIR="${SCRIPT_DIR}/backup" + +# Run all docker compose commands from the repo root so project context +# is correct regardless of the caller's working directory. +cd "$SCRIPT_DIR" + +# Interrupt handler — warn about potentially incomplete operations +trap 'echo ""; error "Interrupted. If a backup or restore was in progress, it may be incomplete."; exit 130' INT TERM + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# Colors (disabled when not a terminal) +if [[ -t 1 ]] && [[ -z "${NO_COLOR:-}" ]]; then + _green='\033[0;32m' _yellow='\033[1;33m' + _red='\033[0;31m' _cyan='\033[0;36m' _nc='\033[0m' +else + _green='' _yellow='' _red='' _cyan='' _nc='' +fi + +info() { printf '%b[OK]%b %s\n' "$_green" "$_nc" "$*"; } +warn() { printf '%b[WARN]%b %s\n' "$_yellow" "$_nc" "$*"; } +error() { printf '%b[ERROR]%b %s\n' "$_red" "$_nc" "$*" >&2; } +header() { printf '\n%b=== %s ===%b\n\n' "$_cyan" "$*" "$_nc"; } + +usage() { + cat <<'USAGE' +Datomic Free → Pro Migration (Docker) + +Migrates data between Datomic storage protocols using the bin/datomic CLI +inside Docker containers. Backup/restore use bind-mounted volumes so +databases of any size are handled without filling the container overlay. + +Usage: + ./docker-migrate.sh backup Back up the running (Free) database + ./docker-migrate.sh restore Restore into the new (Pro) database + ./docker-migrate.sh verify Verify backup integrity + ./docker-migrate.sh full Guided full migration (backup → swap → restore) + +Options (must come BEFORE the command): + --compose-yaml Override compose file for rebuild (default: docker-compose-build.yaml) + --old-uri Override source database URI detection + --new-uri Override target database URI detection + --help Show this help + +Examples: + # Step-by-step (recommended for large databases) + ./docker-migrate.sh backup # With old stack running + docker compose down + docker compose -f docker-compose-build.yaml up -d + ./docker-migrate.sh restore # After new stack is healthy + ./docker-migrate.sh verify # Verify backup integrity + + # Automatic + ./docker-migrate.sh full +USAGE +} + +# Source .env for password/URI configuration. +# Uses a subshell to avoid exporting all .env vars (which could set Docker +# Compose's COMPOSE_FILE env var and change compose behavior globally). +load_env() { + local env_file="${SCRIPT_DIR}/.env" + if [[ -f "$env_file" ]]; then + # Read specific variables we need rather than exporting everything + # shellcheck disable=SC1090 + DATOMIC_PASSWORD="${DATOMIC_PASSWORD:-$(. "$env_file" && echo "${DATOMIC_PASSWORD:-}")}" + fi +} + +# Check that Docker Compose v2 is available +check_docker_compose() { + if ! docker compose version &>/dev/null; then + error "Docker Compose v2 plugin required ('docker compose')." + error "The standalone 'docker-compose' v1 is not supported." + exit 1 + fi +} + +# Detect the Docker Compose network from the running datomic container. +# Compose creates a default network that includes all services; we take the +# first network which is that default in the common single-network case. +get_compose_network() { + local cid + cid=$(docker compose ps -q datomic 2>/dev/null || true) + if [[ -z "$cid" ]]; then + error "No running datomic container found. Is the stack running?" + exit 1 + fi + docker inspect "$cid" \ + -f '{{range $k,$v := .NetworkSettings.Networks}}{{println $k}}{{end}}' \ + | head -1 +} + +# Detect the image of the running datomic container +get_datomic_image() { + local cid + cid=$(docker compose ps -q datomic 2>/dev/null || true) + if [[ -z "$cid" ]]; then + error "No running datomic container found. Is the stack running?" + exit 1 + fi + docker inspect "$cid" -f '{{.Config.Image}}' +} + +# Detect DATOMIC_URL from the running orcpub container's environment. +# The orcpub container (the peer) holds the connection URI including password; +# the datomic container (the transactor) doesn't have DATOMIC_URL. +get_container_datomic_url() { + local cid + cid=$(docker compose ps -q orcpub 2>/dev/null || true) + if [[ -z "$cid" ]]; then + return 1 + fi + docker inspect "$cid" \ + -f '{{range .Config.Env}}{{println .}}{{end}}' \ + | grep '^DATOMIC_URL=' | cut -d= -f2- || true +} + +# Wait for a compose service to become healthy +wait_healthy() { + local service="$1" + local max_wait="${2:-180}" + local waited=0 + local cid="" + local status="" + + printf "Waiting for %s to become healthy" "$service" + while [[ $waited -lt $max_wait ]]; do + cid=$(docker compose ps -q "$service" 2>/dev/null || true) + if [[ -n "$cid" ]]; then + status=$(docker inspect --format='{{.State.Health.Status}}' "$cid" 2>/dev/null || echo "starting") + if [[ "$status" == "healthy" ]]; then + echo "" + info "$service is healthy (${waited}s)" + return 0 + fi + if [[ "$status" == "unhealthy" ]]; then + echo "" + error "$service reported unhealthy" + docker compose logs "$service" | tail -20 + exit 1 + fi + fi + printf "." + sleep 2 + waited=$((waited + 2)) + done + echo "" + error "$service did not become healthy within ${max_wait}s" + error "Check: docker compose ps; docker compose logs $service" + exit 1 +} + +# Run bin/datomic CLI in a temporary container that shares the compose +# network and bind-mounts the backup directory. +# +# Uses the datomic image (which has the full Datomic distribution at +# WORKDIR=/datomic, so relative path bin/datomic resolves to /datomic/bin/datomic). +run_datomic_cli() { + local image="$1" + shift + + local network + network=$(get_compose_network) + + docker run --rm \ + --network="$network" \ + -v "${BACKUP_DIR}:/backup" \ + "$image" \ + bin/datomic "$@" +} + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + +do_backup() { + header "Backup — Datomic Database" + + local datomic_image + datomic_image=$(get_datomic_image) + info "Datomic image: $datomic_image" + + local old_url="${OLD_URI:-}" + if [[ -z "$old_url" ]]; then + old_url=$(get_container_datomic_url 2>/dev/null || true) + fi + if [[ -z "$old_url" ]]; then + old_url="datomic:free://datomic:4334/orcpub" + warn "Could not detect DATOMIC_URL; using default: $old_url" + fi + info "Source URI: $old_url" + + mkdir -p "$BACKUP_DIR" + + # Handle existing backup safely + if [[ -d "${BACKUP_DIR}/orcpub" ]]; then + warn "Existing backup found at ${BACKUP_DIR}/orcpub" + if [[ -t 0 ]]; then + read -rp "Overwrite? [y/N] " confirm + if [[ "${confirm,,}" != "y" ]]; then + echo "Aborted." + exit 0 + fi + else + warn "Non-interactive — overwriting existing backup." + fi + fi + + info "Backup destination: file:/backup/orcpub" + info "Starting backup (large databases may take 30+ minutes)..." + echo "" + + if ! run_datomic_cli "$datomic_image" \ + backup-db "$old_url" "file:/backup/orcpub"; then + error "backup-db failed. Is the source transactor running and reachable?" + exit 1 + fi + + echo "" + # Save metadata for the restore phase + echo "$old_url" > "${BACKUP_DIR}/.source-uri" + date -u +"%Y-%m-%dT%H:%M:%SZ" > "${BACKUP_DIR}/.backup-timestamp" + info "Backup complete → ${BACKUP_DIR}/orcpub" +} + +do_restore() { + header "Restore — Datomic Pro Database" + + if [[ ! -d "${BACKUP_DIR}/orcpub" ]]; then + error "No backup found at ${BACKUP_DIR}/orcpub" + error "Run './docker-migrate.sh backup' first (with the old stack running)." + exit 1 + fi + + if [[ -f "${BACKUP_DIR}/.backup-timestamp" ]]; then + info "Backup timestamp: $(cat "${BACKUP_DIR}/.backup-timestamp")" + fi + + local datomic_image + datomic_image=$(get_datomic_image) + info "Datomic image: $datomic_image" + + local new_url="${NEW_URI:-}" + if [[ -z "$new_url" ]]; then + new_url=$(get_container_datomic_url 2>/dev/null || true) + fi + if [[ -z "$new_url" ]]; then + load_env + if [[ -z "${DATOMIC_PASSWORD:-}" ]]; then + warn "DATOMIC_PASSWORD not set in .env — falling back to default 'datomic'" + fi + new_url="datomic:dev://datomic:4334/orcpub?password=${DATOMIC_PASSWORD:-datomic}" + warn "Could not detect DATOMIC_URL; constructed from DATOMIC_PASSWORD" + fi + info "Target URI: $new_url" + + info "Starting restore (large databases may take 30+ minutes)..." + echo "" + + if ! run_datomic_cli "$datomic_image" \ + restore-db "file:/backup/orcpub" "$new_url"; then + error "restore-db failed. Is the target transactor running?" + error "If 'database already exists', clear ./data and restart the transactor." + exit 1 + fi + + echo "" + info "Restore complete. Log in and verify your data." +} + +do_verify() { + header "Verify — Backup Integrity" + + if [[ ! -d "${BACKUP_DIR}/orcpub" ]]; then + error "No backup found at ${BACKUP_DIR}/orcpub" + exit 1 + fi + + local datomic_image + datomic_image=$(get_datomic_image) + + # List available backup points + info "Backup points:" + run_datomic_cli "$datomic_image" list-backups "file:/backup/orcpub" + echo "" + + # Get the latest t for verification + local latest_t + latest_t=$(run_datomic_cli "$datomic_image" list-backups "file:/backup/orcpub" 2>&1 | tail -1 || true) + + if [[ -z "$latest_t" ]]; then + error "Could not determine latest backup point." + exit 1 + fi + + info "Verifying backup at t=$latest_t (reads every segment)..." + + if ! run_datomic_cli "$datomic_image" verify-backup "file:/backup/orcpub" true "$latest_t"; then + error "Backup verification failed." + exit 1 + fi + + echo "" + info "Backup verification passed." +} + +do_full() { + header "Full Migration — Datomic Free → Pro" + + # Use a distinct variable name to avoid colliding with Docker Compose's + # COMPOSE_FILE env var (which changes docker compose's behavior globally). + local compose_yaml="${COMPOSE_YAML:-docker-compose-build.yaml}" + info "New compose file: $compose_yaml" + echo "" + + if ! [[ -t 0 ]]; then + error "Full migration requires an interactive terminal." + error "Run the steps individually — see docs/migration/datomic-data-migration.md" + exit 1 + fi + + # Phase 1: Backup from old stack + do_backup + + # Phase 2: Verify backup before we tear anything down + do_verify + + # Phase 3: Swap stacks + header "Swapping Stacks" + info "Stopping old containers..." + docker compose down + info "Old stack stopped." + echo "" + + # Move old data directory safely (avoid nesting if target exists) + if [[ -d "${SCRIPT_DIR}/data" ]]; then + if [[ -d "${SCRIPT_DIR}/data.free-backup" ]]; then + warn "data.free-backup already exists — removing stale copy" + rm -rf "${SCRIPT_DIR}/data.free-backup" + fi + warn "Moving ./data → ./data.free-backup (Free storage format)" + mv "${SCRIPT_DIR}/data" "${SCRIPT_DIR}/data.free-backup" + mkdir -p "${SCRIPT_DIR}/data" + fi + + info "Building and starting new stack with ${compose_yaml}..." + docker compose -f "$compose_yaml" build + docker compose -f "$compose_yaml" up -d + wait_healthy datomic 180 + wait_healthy orcpub 120 + + # Phase 4: Restore into new stack + do_restore + + echo "" + header "Migration Complete" + info "Old Free data preserved at: ./data.free-backup" + info "New Pro data stored in: ./data" + info "Log in and verify your data, then:" + info " rm -rf ./data.free-backup # remove old Free storage" + info " rm -rf ./backup # remove backup (or keep as insurance)" + info "" + info "To rollback: docker compose down, rm -rf data, mv data.free-backup data," + info " then restart with the original compose file." +} + +# --------------------------------------------------------------------------- +# Argument parsing — options must come BEFORE the command word +# --------------------------------------------------------------------------- + +OLD_URI="" +NEW_URI="" +COMPOSE_YAML="" +CMD="" + +# Pre-flight check +check_docker_compose + +while [[ $# -gt 0 ]]; do + case "$1" in + --compose-yaml) + if [[ $# -lt 2 ]]; then error "--compose-yaml requires a value"; exit 1; fi + COMPOSE_YAML="$2"; shift 2 ;; + --old-uri) + if [[ $# -lt 2 ]]; then error "--old-uri requires a value"; exit 1; fi + OLD_URI="$2"; shift 2 ;; + --new-uri) + if [[ $# -lt 2 ]]; then error "--new-uri requires a value"; exit 1; fi + NEW_URI="$2"; shift 2 ;; + --help|-h) + usage; exit 0 ;; + -*) + error "Unknown option: $1" + error "Options must come BEFORE the command. Run with --help for usage." + exit 1 ;; + backup|restore|verify|full) + if [[ -n "$CMD" ]]; then + error "Multiple commands given: '$CMD' and '$1'" + exit 1 + fi + CMD="$1"; shift ;; + *) + error "Unknown argument: $1" + usage + exit 1 ;; + esac +done + +if [[ -z "${CMD:-}" ]]; then + usage + exit 1 +fi + +load_env + +case "$CMD" in + backup) do_backup ;; + restore) do_restore ;; + verify) do_verify ;; + full) do_full ;; +esac diff --git a/docker-setup.sh b/docker-setup.sh index 6d97a64fa..6dbd7be63 100755 --- a/docker-setup.sh +++ b/docker-setup.sh @@ -198,7 +198,7 @@ PORT=${PORT} # The password in DATOMIC_URL must match DATOMIC_PASSWORD ADMIN_PASSWORD=${ADMIN_PASSWORD} DATOMIC_PASSWORD=${DATOMIC_PASSWORD} -DATOMIC_URL=datomic:free://datomic:4334/orcpub?password=${DATOMIC_PASSWORD} +DATOMIC_URL=datomic:dev://datomic:4334/orcpub?password=${DATOMIC_PASSWORD} # --- Security --- # Secret used to sign JWT tokens (20+ characters recommended) diff --git a/docker-user.sh b/docker-user.sh index 1ca3151e2..914ed1bc4 100755 --- a/docker-user.sh +++ b/docker-user.sh @@ -150,7 +150,8 @@ wait_for_ready() { warn "No Docker healthcheck found; polling HTTP on container port..." printf "Waiting for app" while [ $waited -lt $max_wait ]; do - if docker exec "$container" wget --no-verbose --tries=1 --spider \ + # BusyBox wget (Alpine): only -q and --spider are supported + if docker exec "$container" wget -q --spider \ "http://localhost:${PORT:-8890}/" 2>/dev/null; then echo "" return 0 diff --git a/docker/datomic/Dockerfile b/docker/datomic/Dockerfile index d3dda3182..58c2f4ff4 100644 --- a/docker/datomic/Dockerfile +++ b/docker/datomic/Dockerfile @@ -1,25 +1,30 @@ -FROM eclipse-temurin:8-jre-alpine +# Datomic Pro transactor — dev storage protocol (datomic:dev://) +FROM eclipse-temurin:21-jre-alpine-3.22 as base -ENV DATOMIC_VERSION 0.9.5703 +ENV DATOMIC_VERSION=1.0.7482 -RUN wget https://github.com/Orcpub/orcpub/raw/refs/heads/develop/lib/datomic-free-0.9.5703.tar.gz -qO /tmp/datomic.tar.gz \ - && cd /tmp \ - && tar -xvzf /tmp/datomic.tar.gz \ - && rm /tmp/datomic.tar.gz \ - && mv datomic-free-0.9.5703 /datomic +RUN apk add --no-cache curl unzip bash -WORKDIR /datomic +RUN curl --fail --location --silent --show-error \ + -o /tmp/datomic-pro.zip \ + "https://datomic-pro-downloads.s3.amazonaws.com/${DATOMIC_VERSION}/datomic-pro-${DATOMIC_VERSION}.zip" \ + && mkdir -p /datomic \ + && unzip -q /tmp/datomic-pro.zip -d /datomic \ + && mv /datomic/datomic-pro-${DATOMIC_VERSION}/* /datomic/ \ + && rmdir /datomic/datomic-pro-${DATOMIC_VERSION} \ + && rm /tmp/datomic-pro.zip -RUN cp /datomic/config/samples/free-transactor-template.properties transactor.properties +WORKDIR /datomic -RUN mkdir /data -RUN sed "s/# data-dir=data/data-dir=\/data/" -i transactor.properties +# Use dev-transactor-template (dev storage protocol for datomic:dev://) +RUN cp /datomic/config/samples/dev-transactor-template.properties transactor.properties -RUN mkdir /log -RUN sed "s/# log-dir=log/log-dir=\/log/" -i transactor.properties +RUN mkdir -p /data /log +RUN sed -i "s/# data-dir=data/data-dir=\/data/" transactor.properties +RUN sed -i "s/# log-dir=log/log-dir=\/log/" transactor.properties -RUN sed "s/host=localhost/host=0.0.0.0/" -i transactor.properties -RUN sed "s/# storage-access=local/storage-access=remote/" -i transactor.properties +RUN sed -i "s/host=localhost/host=0.0.0.0/" transactor.properties +RUN sed -i "s/# storage-access=local/storage-access=remote/" transactor.properties ADD deploy/start.sh /datomic/ RUN chmod +x /datomic/start.sh diff --git a/docker/orcpub/Dockerfile b/docker/orcpub/Dockerfile index 8c0f1cfc5..ddc72c4cd 100644 --- a/docker/orcpub/Dockerfile +++ b/docker/orcpub/Dockerfile @@ -1,7 +1,25 @@ -FROM clojure:lein as builder +FROM clojure:temurin-21-lein-alpine as builder -# Build cache layer +RUN apk add --no-cache curl unzip + +# Vendor jars (pdfbox snapshots) → Maven local repo ADD ./lib/ /root/.m2/repository/ + +# Datomic Pro peer jar isn't in public Maven repos. +# Download the distribution zip and install the peer jar + pom manually +# (bin/maven-install calls mvn, which isn't in this image). +ENV DATOMIC_VERSION=1.0.7482 +RUN curl --fail --location --silent --show-error \ + -o /tmp/datomic-pro.zip \ + "https://datomic-pro-downloads.s3.amazonaws.com/${DATOMIC_VERSION}/datomic-pro-${DATOMIC_VERSION}.zip" \ + && unzip -q /tmp/datomic-pro.zip -d /tmp \ + && mkdir -p /root/.m2/repository/com/datomic/peer/${DATOMIC_VERSION} \ + && cp /tmp/datomic-pro-${DATOMIC_VERSION}/peer-${DATOMIC_VERSION}.jar \ + /root/.m2/repository/com/datomic/peer/${DATOMIC_VERSION}/ \ + && cp /tmp/datomic-pro-${DATOMIC_VERSION}/pom.xml \ + /root/.m2/repository/com/datomic/peer/${DATOMIC_VERSION}/peer-${DATOMIC_VERSION}.pom \ + && rm -rf /tmp/datomic-pro* + WORKDIR /orcpub COPY project.clj /orcpub/ RUN lein deps @@ -10,7 +28,11 @@ ADD ./ /orcpub RUN printenv &&\ lein uberjar -FROM eclipse-temurin:8-jre-alpine as runner +# Alpine runner — BusyBox wget handles healthchecks (use -q --spider, not GNU flags) +FROM eclipse-temurin:21-jre-alpine-3.22 as runner + +# PDFBox requires fontconfig, fonts, and lcms2 for PDF character sheet generation +RUN apk add --no-cache fontconfig ttf-dejavu freetype lcms2 COPY --from=builder /orcpub/target/orcpub.jar /orcpub.jar diff --git a/docker/scripts/manage-user.clj b/docker/scripts/manage-user.clj index b02cac2ca..19cd1dde5 100644 --- a/docker/scripts/manage-user.clj +++ b/docker/scripts/manage-user.clj @@ -17,7 +17,7 @@ (def datomic-url (or (System/getenv "DATOMIC_URL") - "datomic:free://datomic:4334/orcpub?password=datomic")) + "datomic:dev://datomic:4334/orcpub?password=datomic")) (defn get-conn [] (try diff --git a/docker/scripts/migrate-db.clj b/docker/scripts/migrate-db.clj new file mode 100644 index 000000000..3daca7af8 --- /dev/null +++ b/docker/scripts/migrate-db.clj @@ -0,0 +1,95 @@ +;; Datomic Database Verification Tool +;; +;; Connects to a running Datomic database and reports statistics: basis-t, +;; user count, and entity (character) count. Used after migration to verify +;; data integrity by comparing pre- and post-migration numbers. +;; +;; Requires the uberjar on the classpath (includes Clojure + Datomic peer): +;; java -cp target/orcpub.jar clojure.main docker/scripts/migrate-db.clj verify [db-uri] +;; +;; Note: Backup and restore use the bin/datomic CLI (backup-db, restore-db), +;; NOT the Peer API — those functions do not exist in datomic.api. +;; See scripts/migrate-db.sh (bare-metal) or docker-migrate.sh (Docker). + +(ns migrate-db + (:require [datomic.api :as d])) + +;; --------------------------------------------------------------------------- +;; Configuration +;; --------------------------------------------------------------------------- + +(def ^{:doc "Database URI. Set via DATOMIC_URL env var."} + datomic-url + (or (System/getenv "DATOMIC_URL") + "datomic:dev://datomic:4334/orcpub")) + +;; --------------------------------------------------------------------------- +;; Database statistics — used for pre/post migration verification +;; --------------------------------------------------------------------------- + +(defn db-stats + "Returns a map of database statistics: basis-t, user count, entity count." + [db] + (let [users (or (d/q '[:find (count ?e) . + :where [?e :orcpub.user/username]] + db) 0) + entities (or (d/q '[:find (count ?e) . + :where [?e :orcpub.entity/owner]] + db) 0)] + {:basis-t (d/basis-t db) + :users users + :entities entities})) + +(defn print-stats + "Prints database statistics in a readable format." + [stats] + (println (format " Basis-t: %d" (:basis-t stats))) + (println (format " Users: %d" (:users stats))) + (println (format " Characters: %d" (:entities stats)))) + +;; --------------------------------------------------------------------------- +;; Verify — connects to a database and reports stats +;; --------------------------------------------------------------------------- + +(defn verify! + "Connects to a database and reports statistics." + [db-uri] + (println "Connecting to" db-uri "...") + (let [conn (d/connect db-uri) + db (d/db conn) + stats (db-stats db)] + (println "Database statistics:") + (print-stats stats) + stats)) + +;; --------------------------------------------------------------------------- +;; CLI dispatch +;; --------------------------------------------------------------------------- + +(let [args *command-line-args* + cmd (first args)] + (try + (case cmd + "verify" (let [[_ db-uri] args] + (verify! (or db-uri datomic-url))) + + ;; Default: show usage + (do + (println "Datomic Database Verification Tool") + (println) + (println "Usage:") + (println " migrate-db.clj verify [uri] Report database stats (defaults to DATOMIC_URL)") + (println) + (println "Environment:") + (println " DATOMIC_URL Database URI (default: datomic:dev://datomic:4334/orcpub)") + (println) + (println "For backup and restore, use the bin/datomic CLI:") + (println " bin/datomic backup-db ") + (println " bin/datomic restore-db ") + (System/exit 1))) + (catch Exception e + (binding [*out* *err*] + (println "FATAL:" (.getMessage e))) + (System/exit 1))) + ;; Datomic peer threads are non-daemon; force clean exit + (System/exit 0)) diff --git a/docs/CONTENT_RECONCILIATION.md b/docs/CONTENT_RECONCILIATION.md index e9e9de6f2..f79e18ed7 100644 --- a/docs/CONTENT_RECONCILIATION.md +++ b/docs/CONTENT_RECONCILIATION.md @@ -4,125 +4,120 @@ Detects when characters reference missing homebrew content and suggests alternatives using fuzzy matching. -**Why this exists:** User deletes homebrew plugin, reopens character, sees `:artificer (not loaded)` with no context. No way to know which plugin to reinstall or what similar content exists. +**Why this exists:** User deletes homebrew plugin, reopens character, sees missing content with no context. No way to know which plugin to reinstall or what similar content exists. -**Design decision:** Use multiple fuzzy matching strategies (exact key, Levenshtein distance, prefix matching) to catch common cases: typos, versioning (`:blood-hunter-v2`), and renamed content. +**Design decision:** Use multiple fuzzy matching strategies (exact key, prefix matching, keyword-base comparison, display name matching) to catch common cases: typos, versioning (`:blood-hunter-v2`), and renamed content. -**Key gotcha:** Must exclude built-in content (PHB, Xanathar's, etc.) or system suggests switching from homebrew Artificer to PHB Artificer (which doesn't exist in most books). +**Key gotcha:** Must exclude SRD built-in content. The `available-content` subscription only includes plugin content, so SRD content (hardcoded in the app) would be false-flagged as missing without explicit exclusion sets. ## How It Works ### Missing Content Detection -Scans character options tree for `::entity/key` references → Checks if key exists in loaded content → Reports missing with suggestions +Scans character options tree for `::entity/key` references, classifies each by content type, checks if key exists in loaded content, reports missing with suggestions. -**Supported types:** Classes, subclasses, races, subraces, backgrounds +**Supported types:** Classes, subclasses, races, subraces, backgrounds, feats **Implementation:** `content_reconciliation.cljs` -### Fuzzy Matching +### Content Type Detection -Four strategies find similar content: +The character entity stores options differently based on selection type: +- **Single-select** (race, background, subrace) — stored as maps, path has no index +- **Multi-select** (class, feats) — stored as vectors, path includes indices -**1. Exact key, different source** -``` -Missing: :artificer from "Serakat's Compendium" -Suggests: :artificer from "Player's Handbook" -``` +`annotate-content-type` strips integer indices from paths before matching: +- `[:class 0]` → `[:class]` → class +- `[:class 0 :martial-archetype]` → `[:class :martial-archetype]` → subclass +- `[:race]` → race (already index-free, single-select) +- `[:class 0 :levels 3 :asi-or-feat :feats 0]` → detected by `:feats` parent keyword → feat -**2. Levenshtein distance** (max 3 edits for typos) +### Fuzzy Matching + +Three strategies find similar content (`find-similar-content`): + +**1. Exact key match** (similarity 1.0) ``` -Missing: :artficer -Suggests: :artificer +Missing: :artificer +Available: :artificer (from different source) ``` -**3. Prefix matching** (min 4 chars, for versioning) +**2. Prefix matching** (similarity 0.7) ``` Missing: :battle-smith-v2 Suggests: :battle-smith ``` -**4. Display name similarity** (max 3 edits) +**3. Keyword-base comparison** (similarity 0.8 via `common/kw-base`) ``` -Missing: :drunken_master -Suggests: :drunken-master +Missing: :blood-hunter-order-of-the-lycan +Suggests: :blood-hunter (same base before first dash) ``` -**Why multiple strategies:** Single strategy missed too many cases. Levenshtein alone doesn't catch versioning (`:fighter-v2`). Prefix alone doesn't catch typos (`:artficer`). Combined approach catches ~80% of common cases. - -### Warning UI - -Displays in character builder: +**4. Display name similarity** (similarity 0.6) ``` -:missing-content (not loaded) -:missing-content (not loaded - try :suggested-content?) -:missing-content from "Plugin Name" (not loaded - try :suggested-content?) +Missing: :drunken_master +Suggests: :drunken-master (name matches after normalization) ``` -**Implementation:** `views.cljs` (display), `subs.cljs` (subscriptions) - ### Built-in Content Exclusions -Excludes PHB, Xanathar's, Tasha's, and 9 other official books from warnings. - -**Why:** Built-in content is always available. Without exclusion, system suggests "try PHB Artificer" when user's homebrew Artificer is missing (but PHB doesn't have Artificer in 5e). +Excludes **SRD-only** content from warnings. This is NOT all PHB content — only what's hardcoded in the app: -## Common Scenarios - -**Deleted plugin:** Character shows `:rune-knight from "Fighter Subclasses" (not loaded - try :eldritch-knight?)` → Re-import plugin or use suggested alternative - -**Shared character:** Friend's character uses homebrew → Warnings show which plugins needed → Ask friend for files or use suggested official alternatives - -**Renamed content:** Updated `:blood-hunter` to `:blood-hunter-v2` → Old characters suggest new version → Prefix matching catches versioning - -## Implementation - -**Key files:** -- `content_reconciliation.cljs` - Detection, fuzzy matching (`find-missing-content`, `find-suggestion`, `levenshtein-distance`) -- `subs.cljs`, `views.cljs` - UI integration (subscriptions, warning display) -- `common.cljc` - Utilities (`kw-base`, `traverse-nested`) -- `import_validation_test.cljs` - Tests - -**Data flow:** -Character loaded → `extract-character-keys` → `classify-content-type` → `find-available-content` → Missing? → `find-suggestion` → Display warning with suggestion +| Type | SRD Built-ins | Everything else | +|------|--------------|-----------------| +| Classes | All 12 base classes | — | +| Races | 9 races + their subraces | — | +| Subclasses | 1 per class (Champion, Berserker, Lore, Life, Land, Open Hand, Devotion, Hunter, Thief, Draconic, Fiend, Evocation) | All others from plugins | +| Backgrounds | Acolyte only | All others from plugins | +| Feats | None | All from plugins | -**Performance:** ~10ms detection + ~5ms per missing item for fuzzy matching. 100+ missing items may need optimization. +**Critical distinction:** Non-SRD PHB content (Battle Master, Totem Warrior, Folk Hero, etc.) comes from plugins and SHOULD be flagged when plugins are removed. Earlier versions incorrectly excluded all PHB content. -## Testing +### Warning UI -**Automated:** `import_validation_test.cljs` - Covers detection, fuzzy matching accuracy, built-in exclusions +Displays in character builder via the import log overlay. Shows missing content with type, inferred source, and suggestions. -**Critical manual tests:** -1. Delete plugin → Reopen character → Should show "(not loaded)" warning -2. Rename `:blood-hunter` to `:blood-hunter-v2` → Should suggest new version -3. PHB Wizard with Evocation → Should NOT warn (built-in exclusion) +**Implementation:** `views/conflict_resolution.cljs` (display), `subs.cljs` (subscriptions) -## Extending +## Data Flow -**Adjust thresholds:** Edit `levenshtein-distance-threshold`, `prefix-match-length`, `name-similarity-threshold` in `content_reconciliation.cljs` (defaults: 3, 4, 3) +``` +Character loaded + → extract-content-keys (walks ::entity/options tree via traverse-nested) + → annotate-content-type (classifies by path shape) + → check-content-availability (compares against available-content subscription) + → find-similar-content (fuzzy matching for suggestions) + → generate-missing-content-report + → ::char5e/missing-content-report subscription + → UI overlay +``` -**Add content types:** Add to `content-type-paths` and `content-type->field` maps +## Key Files -**Exclude sources:** Add to `built-in-sources` set +- `content_reconciliation.cljs` — Detection, classification, fuzzy matching +- `subs.cljs` — `::char5e/available-content`, `::char5e/missing-content-report` subscriptions +- `common.cljc` — `kw-base`, `traverse-nested`, `name-to-kw` utilities +- `views/conflict_resolution.cljs` — UI display -## Troubleshooting +## Common Scenarios -**"Not loaded" but exists:** Check key matches exactly (`:blood-hunter` vs `:bloodhunter`), verify plugin loaded +**Deleted plugin:** Character shows missing class/subclass/background/feat with suggestions for similar available content -**Wrong suggestions:** Adjust matching thresholds, check for duplicate keys +**Shared character:** Friend's character uses homebrew → Warnings show which content types are missing -**Built-in showing warnings:** Source name doesn't match exclusion list exactly, add variant to `built-in-sources` +**Renamed content:** Updated `:blood-hunter` to `:blood-hunter-v2` → Old characters detect missing, prefix matching suggests new version -## Future Enhancements +## Extending -**Auto-fix button:** One-click apply suggestion +**Add content types:** Add to `content-type-paths`, `content-type->field`, and `available-content` subscription in `subs.cljs` -**Smart migration:** Auto-update characters when content renamed (detect renames, prompt to update all affected characters) +**Adjust thresholds:** Edit similarity cutoff (0.3) in `find-similar-content` -**Plugin recommendations:** Suggest which plugin to install based on missing content library lookup +**Add SRD exclusions:** Only add truly hardcoded content to `builtin-*` sets. Plugin content should NOT be excluded. ## Related Documentation -- [ORCBREW_FILE_VALIDATION.md](ORCBREW_FILE_VALIDATION.md) - Import/export validation -- [CONFLICT_RESOLUTION.md](CONFLICT_RESOLUTION.md) - Duplicate key handling -- [HOMEBREW_REQUIRED_FIELDS.md](HOMEBREW_REQUIRED_FIELDS.md) - Content field requirements +- [ORCBREW_FILE_VALIDATION.md](ORCBREW_FILE_VALIDATION.md) — Import/export validation +- [CONFLICT_RESOLUTION.md](CONFLICT_RESOLUTION.md) — Duplicate key handling +- [HOMEBREW_REQUIRED_FIELDS.md](HOMEBREW_REQUIRED_FIELDS.md) — Content field requirements diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md new file mode 100644 index 000000000..4076ddb03 --- /dev/null +++ b/docs/ENVIRONMENT.md @@ -0,0 +1,82 @@ +# Environment Variables + +All configuration is managed via a `.env` file at the repository root. Copy `.env.example` to `.env` and edit as needed. + +## Precedence + +1. `.env` in repo root (authoritative — sourced by all scripts and read by `environ`) +2. `containerEnv` in `.devcontainer/devcontainer.json` (fallback defaults) +3. System environment variables + +> **Note:** `./menu` and `./scripts/start.sh` source `.env` automatically (via `scripts/common.sh`). +> Running `lein` commands directly (e.g., `lein repl`, `lein run`) reads dev defaults +> from `.lein-env` (generated by `lein-environ` from the `:dev` profile in `project.clj`). +> This includes `SIGNATURE` so auth works out of the box for local development. +> To use your `.env` values with `lein` directly: +> ```bash +> source .env && lein repl +> ``` + +## Variables + +### Datomic + +| Variable | Default | Description | +|----------|---------|-------------| +| `DATOMIC_URL` | `datomic:dev://localhost:4334/orcpub` | Database connection URI | +| `DATOMIC_VERSION` | `1.0.7482` | Datomic Pro version for installer | +| `DATOMIC_TYPE` | `pro` | Datomic distribution type | +| `DATOMIC_PASSWORD` | — | Transactor password | + +### Application + +| Variable | Default | Description | +|----------|---------|-------------| +| `PORT` | `8080` | Production web server port (dev uses 8890) | +| `SIGNATURE` | — | **Required.** JWT signing secret for authentication. All login and API calls fail without it. | +| `ADMIN_PASSWORD` | — | Admin password | + +### Security + +| Variable | Default | Description | +|----------|---------|-------------| +| `CSP_POLICY` | `strict` | Content Security Policy mode: `strict`, `permissive`, or `none` | +| `DEV_MODE` | `true` (in :dev profile) | Enables dev-mode CSP (Report-Only instead of enforcing) | + +CSP modes: +- **strict** — nonce-based CSP with `strict-dynamic`. Dev mode uses `Report-Only` header (logs violations but doesn't block). Prod uses enforcing header. +- **permissive** — allows `unsafe-inline` and `unsafe-eval`. Legacy fallback. +- **none** — disables CSP entirely. Not recommended for production. + +### Email (SMTP) + +| Variable | Default | Description | +|----------|---------|-------------| +| `EMAIL_HOST` | — | SMTP server hostname | +| `EMAIL_PORT` | — | SMTP port | +| `EMAIL_USER` | — | SMTP username | +| `EMAIL_PASSWORD` | — | SMTP password | + +### Logging + +| Variable | Default | Description | +|----------|---------|-------------| +| `LOG_DIR` | `./logs` | Directory for log files | +| `POST_CREATE_VERBOSE` | `1` (in devcontainer) | Enable verbose post-create logging | + +### Development + +| Variable | Default | Description | +|----------|---------|-------------| +| `ORCPUB_ENV` | — | Set to `dev` to enable `add-test-user` in user.clj | + +## Files That Read Environment + +| File | Variables Used | +|------|---------------| +| `src/clj/orcpub/config.clj` | `DATOMIC_URL`, `CSP_POLICY`, `DEV_MODE` | +| `src/clj/orcpub/system.clj` | `PORT` (via `System/getenv`) | +| `src/clj/orcpub/routes.clj` | `SIGNATURE`, `EMAIL_*`, `ADMIN_PASSWORD` | +| `.devcontainer/post-create.sh` | `DATOMIC_VERSION`, `DATOMIC_TYPE` | +| `scripts/start.sh` | `DATOMIC_URL`, `LOG_DIR` | +| `dev/user.clj` | `ORCPUB_ENV` (for add-test-user guard) | diff --git a/docs/GETTING-STARTED.md b/docs/GETTING-STARTED.md new file mode 100644 index 000000000..8553dfb78 --- /dev/null +++ b/docs/GETTING-STARTED.md @@ -0,0 +1,213 @@ +# Getting Started + +This guide takes you from a fresh clone to a running dev server. Two paths are covered: + +1. **Devcontainer** (recommended) - everything installs automatically +2. **Local machine** - you install Java, Leiningen, and Datomic yourself + +Both paths end at the same place: Datomic running, backend serving on port 8890, and Figwheel hot-reloading the frontend. + +--- + +## Path 1: Devcontainer + +Requires [VS Code](https://code.visualstudio.com/) with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers), or [GitHub Codespaces](https://github.com/features/codespaces). + +### 1. Open in container + +```bash +git clone https://github.com/orcpub/orcpub.git +cd orcpub +code . +``` + +When VS Code prompts **"Reopen in Container"**, click it. The container installs Java 21, Leiningen, and Datomic Pro automatically. + +### 2. Run first-time setup + +Once the container is built, open a terminal inside VS Code: + +```bash +./scripts/dev-setup.sh +``` + +This will: +- Download project dependencies (`lein deps`) +- Start the Datomic transactor +- Initialize the database schema +- Create a test user: **test** / **test@test.com** / **testpass** + +### 3. Start services + +```bash +./menu +``` + +The interactive menu shows service status and lets you start/stop with single keystrokes. Or start individually: + +```bash +./menu start server # Backend on port 8890 +./menu start figwheel # Frontend hot-reload on port 3449 +``` + +### 4. Open the app + +Navigate to `http://localhost:8890` (or the Codespaces forwarded URL). Log in with **test@test.com** / **testpass**. + +--- + +## Path 2: Local Machine + +### 1. Prerequisites + +| Tool | Version | Install | +|------|---------|---------| +| Java | 21+ (OpenJDK) | [Adoptium](https://adoptium.net/) | +| Leiningen | 2.9+ | [leiningen.org](https://leiningen.org/#install) | +| Datomic Pro | 1.0.7482 | See below | + +**Datomic Pro** is free (Apache 2.0 license). The devcontainer installer handles this automatically, but on a local machine you need to install it manually: + +```bash +# The install script downloads and extracts Datomic Pro +./.devcontainer/post-create.sh +``` + +Or follow the manual steps in [docs/migration/datomic-pro.md](migration/datomic-pro.md). Datomic should end up at `lib/com/datomic/datomic-pro/1.0.7482/`. + +### 2. Configure environment + +```bash +cp .env.example .env +``` + +Edit `.env` if needed. The defaults work for local development. The most important variable is `SIGNATURE` (JWT secret) -a dev default is provided automatically via `.lein-env`, so this step is optional for local dev. + +See [ENVIRONMENT.md](ENVIRONMENT.md) for the full variable list. + +### 3. Download dependencies + +```bash +lein deps +``` + +### 4. Start Datomic + +```bash +./scripts/start.sh datomic +``` + +Wait for "Datomic is ready" before continuing. The transactor listens on port 4334. + +### 5. Initialize the database + +First time only -creates the schema and applies seed data: + +```bash +./scripts/start.sh init-db +``` + +### 6. Create a test user + +```bash +./menu add test testpass +``` + +This creates **test@test.com** with password **testpass**, already verified. + +### 7. Start the backend + +```bash +./scripts/start.sh server +``` + +The server starts on port 8890 with an nREPL port for editor connections. + +### 8. Start Figwheel (frontend hot-reload) + +In a separate terminal: + +```bash +./scripts/start.sh figwheel +``` + +Figwheel compiles ClojureScript and serves it on port 3449. Changes to `.cljs` files are pushed to the browser automatically. + +### 9. Open the app + +Navigate to `http://localhost:8890`. Log in with **test@test.com** / **testpass**. + +--- + +## Using the REPL directly + +If you prefer `lein repl` over the shell scripts: + +```bash +lein repl +``` + +The `user` namespace loads automatically with helpers: + +```clojure +(start-server) ; Start web server on port 8890 +(stop-server) ; Stop web server +(init-database) ; Initialize DB schema (first time) +(fig-start) ; Start Figwheel from the REPL +(cljs-repl) ; Connect to ClojureScript REPL (after fig-start) +``` + +The `:dev` profile provides a dev `SIGNATURE` via `.lein-env`, so auth works without extra setup. If you need to override with your `.env` values: + +```bash +source .env && lein repl +``` + +--- + +## Verification + +After starting services, confirm everything is working: + +| Check | Expected | +|-------|----------| +| `http://localhost:8890` loads | Splash page appears | +| Log in with test@test.com / testpass | Character list page | +| Create a new character | Builder loads with race/class options | +| Edit and wait 7.5 seconds | Autosave (check Network tab for POST) | +| Upload a `.orcbrew` file (My Content page) | Homebrew options appear in builder | + +--- + +## Troubleshooting + +**Server returns 500 on login or API calls** + +The `SIGNATURE` env var is missing. If using `./menu` or `./scripts/start.sh`, check that `.env` exists and has `SIGNATURE` set. If using `lein repl`, the dev profile provides a default -but if you've overridden it with an empty value, auth will fail. + +**Datomic won't start** + +- Check that port 4334 isn't already in use: `lsof -i :4334` +- Verify Datomic is installed: `ls lib/com/datomic/datomic-pro/1.0.7482/bin/transactor` +- Check logs: `cat logs/datomic.log` + +**Figwheel compilation errors** + +- Run `lein fig:build` for a one-shot compile to see all errors +- First compilation can be slow (1-2 minutes) as it downloads ClojureScript dependencies + +**Port already in use** + +```bash +./scripts/stop.sh # Stop all services +./scripts/stop.sh server # Stop just the server +``` + +--- + +## What's next + +- [Architecture overview](../README.md#architecture) -how entities, templates, and modifiers work +- [Dev tooling guide](migration/dev-tooling.md) -Leiningen profiles, REPL helpers, build commands +- [Environment variables](ENVIRONMENT.md) -full config reference +- [Stack migration context](MIGRATION-INDEX.md) -why Java 21, Datomic Pro, React 18, etc. diff --git a/docs/JAVA-COMPATIBILITY.md b/docs/JAVA-COMPATIBILITY.md new file mode 100644 index 000000000..784514b5c --- /dev/null +++ b/docs/JAVA-COMPATIBILITY.md @@ -0,0 +1,40 @@ +# Java Compatibility + +## Requirement: Java 21 + +This project requires **Java 21** (LTS). The CI workflow, devcontainer, and all scripts are configured for Java 21. + +## Why Not Java 8 + +The original project ran on Java 8. The upgrade to Java 21 was driven by: + +1. **Datomic Pro** — the actively maintained Datomic distribution targets Java 11+ +2. **Pedestal 0.7** — uses Jetty 11, which requires Java 11+ +3. **Security** — Java 8 is past end of public updates +4. **Ecosystem** — modern Clojure libraries increasingly require Java 11+ + +## Datomic Free + Java 21: Incompatible + +Datomic Free 0.9.5703 fails on Java 21 with an SSL handshake timeout. The peer cannot connect to the transactor. This is caused by changes in Java's TLS implementation that break ActiveMQ Artemis (Datomic's internal messaging). Datomic Free is unmaintained — no fix is available. + +**Solution**: Migrate to Datomic Pro (`com.datomic/peer`), which supports Java 21. + +## Pedestal + Jetty Constraint + +Pedestal 0.7.0 uses **Jetty 11**. Pedestal 0.7.1+ uses **Jetty 12**. + +Jetty 12 removes `ScopedHandler`, causing `NoClassDefFoundError` when figwheel-main is running (figwheel-main's Ring adapter depends on Jetty 11 classes). + +**Current pin**: Pedestal 0.7.0 with Jetty 11. This constraint is about Pedestal/Jetty compatibility, not Java version. + +## javax.servlet-api + +Java 9+ removed `javax.servlet` from the default classpath. Added explicitly: + +```clojure +[javax.servlet/javax.servlet-api "4.0.1"] +``` + +## CI Configuration + +`.github/workflows/continuous-integration.yml` uses `setup-java@v4` with `temurin` distribution, Java 21. diff --git a/docs/MIGRATION-INDEX.md b/docs/MIGRATION-INDEX.md new file mode 100644 index 000000000..bb90208f6 --- /dev/null +++ b/docs/MIGRATION-INDEX.md @@ -0,0 +1,39 @@ +# Migration Index + +This branch (`breaking/2026-stack-modernization`) upgrades the full OrcPub stack from its original 2017-era dependencies to a modern 2026 baseline. Every change ships together because the upgrades are interdependent. + +## What Changed + +| Topic | Summary | Details | +|-------|---------|---------| +| Java runtime | 8 → 21 | [JAVA-COMPATIBILITY.md](JAVA-COMPATIBILITY.md) | +| Database | Datomic Free → Datomic Pro | [migration/datomic-pro.md](migration/datomic-pro.md) | +| Database data | Migrating existing Free databases | [migration/datomic-data-migration.md](migration/datomic-data-migration.md) | +| Web framework | Pedestal 0.5.1 → 0.7.0 | [migration/pedestal-0.7.md](migration/pedestal-0.7.md) | +| Frontend | React 15 / Reagent 0.6 → React 18 / Reagent 2.0 | [migration/frontend-stack.md](migration/frontend-stack.md) | +| Libraries | clj-time, PDFBox 2, Buddy 1, etc. | [migration/library-upgrades.md](migration/library-upgrades.md) | +| Dev tooling | Consolidated CLI + REPL into user.clj, profiles, scripts | [migration/dev-tooling.md](migration/dev-tooling.md) | +| Environment | `.env` pattern, new variables | [ENVIRONMENT.md](ENVIRONMENT.md) | +| Stack overview | Architecture, dependencies, build system | [STACK.md](STACK.md) | + +## Why One Branch + +These upgrades form a dependency chain: + +1. **Java 21** is required by modern Datomic Pro and Pedestal 0.7 +2. **Datomic Free** doesn't work on Java 21 (SSL handshake failure), so **Datomic Pro** is required +3. **Pedestal 0.7** requires interceptor wrapping and adds default CSP with `strict-dynamic`, which breaks inline scripts unless nonces are added +4. **Pedestal 0.7.0** specifically — 0.7.1+ uses Jetty 12, which breaks figwheel-main's Ring adapter +5. **Reagent 2.0** / **React 18** use the `createRoot` API (React 17 deprecated `render`) +6. **clj-time → java-time** because clj-time wraps Joda-Time, which is EOL on Java 21 + +Upgrading any one of these in isolation would leave the application broken. + +## Test Status + +- Backend: 206 tests, 945 assertions, 0 failures, 0 errors +- Lint: 0 errors, 0 warnings +- Dev CLJS build: 0 errors, 0 warnings +- Production CLJS build (`:advanced`): succeeds with custom `externs.js` for React 18 APIs +- Garden CSS: 0 warnings (after lambdaisland/garden upgrade) +- CI: Java 21, `lein test` + `lein lint` + `lein cljsbuild once dev` diff --git a/docs/README.md b/docs/README.md index 76590fd3a..ba1159f08 100644 --- a/docs/README.md +++ b/docs/README.md @@ -102,5 +102,3 @@ All in `src/cljs/orcpub/dnd/e5/` unless noted. **Conflicts on import:** Modal should appear automatically → Choose rename/skip/replace per item --- - -**Branch:** `feature/error-handling-import-validation` | **Last updated:** 2026-02-19 diff --git a/docs/STACK.md b/docs/STACK.md new file mode 100644 index 000000000..ccc657305 --- /dev/null +++ b/docs/STACK.md @@ -0,0 +1,209 @@ +# Stack Overview + +A guide to the libraries, frameworks, and architecture for developers getting oriented with the codebase. + +## Architecture at a Glance + +``` +Browser (ClojureScript) Server (Clojure/JVM) +┌─────────────────────┐ ┌─────────────────────┐ +│ Reagent + React 18 │ HTTP │ Pedestal 0.7 │ +│ re-frame │ ◄──────► │ Buddy (auth) │ +│ Garden (CSS) │ Transit │ PDFBox 3 (PDF gen) │ +│ Figwheel (dev) │ │ Datomic Pro (db) │ +└─────────────────────┘ └─────────────────────┘ +``` + +This is a **Clojure/ClojureScript monorepo**. Code lives in three source trees: + +| Path | Language | Runs on | +|------|----------|---------| +| `src/clj/` | Clojure | JVM only (server, routes, PDF) | +| `src/cljs/` | ClojureScript | Browser only (UI, re-frame events/subs) | +| `src/cljc/` | Clojure(Script) | Both — shared logic (character model, templates, specs) | + +## Frontend + +### React 18 + Reagent 2.0 + +[Reagent](https://reagent-project.github.io/) wraps React with ClojureScript idioms. Components are plain functions returning hiccup vectors: + +```clojure +(defn greeting [name] + [:div.hello "Welcome, " name]) +``` + +Reagent 2.0 uses React 18's `createRoot` API. The app mounts via `reagent.dom.client/render`. + +**Key gotcha**: Use `:class` (not `:class-name`) when the hiccup tag has inline classes like `[:div.foo ...]`. Reagent 2.x changed `:class-name` to overwrite rather than merge. + +### re-frame (State Management) + +[re-frame](https://day8.github.io/re-frame/) is a data-flow framework built on Reagent. It follows a unidirectional cycle: + +``` +Event → Handler → Effects → App-DB → Subscriptions → Components → Event +``` + +- **Events** (`reg-event-fx`, `reg-event-db`): Handle user actions and side effects — `src/cljs/.../events.cljs` +- **Subscriptions** (`reg-sub`): Derived views of app-db — `src/cljs/.../subs.cljs` +- **Effects** (`reg-fx`): Side effects like HTTP requests — `:http` fx in events.cljs +- **App-DB**: Single atom holding all application state + +### Garden (CSS-in-Clojure) + +[Garden](https://github.com/lambdaisland/garden) compiles Clojure data structures to CSS. Styles live in `src/clj/orcpub/styles/core.clj` and compile to `resources/public/css/compiled/styles.css`. + +```clojure +[:.builder-option-dropdown + {:background-color :transparent + :cursor :pointer}] +``` + +Compile: `lein garden once` + +We use the **lambdaisland/garden** fork (drop-in replacement) because the original noprompt/garden is unmaintained and causes `clojure.core/abs` shadowing warnings on Clojure 1.11+. + +### Figwheel (Hot Reload) + +[figwheel-main](https://figwheel.org/) provides hot code reloading during development: + +| Command | Mode | Use | +|---------|------|-----| +| `lein fig:dev` | REPL + watch | Interactive development | +| `lein fig:watch` | Watch (headless) | Background/scripted | +| `lein fig:build` | Build once | CI / compilation check | + +Config: `dev.cljs.edn` (preloads, compiler options). + +## Backend + +### Pedestal 0.7 (Web Framework) + +[Pedestal](http://pedestal.io/) is an interceptor-based web framework. Routes map URL patterns to interceptor chains: + +```clojure +["/api/folders" ^:interceptors [check-auth] + {:post `create-folder + :get `list-folders}] +``` + +Interceptors are composable middleware units (auth, parsing, ownership checks). Always wrap with `interceptor/interceptor` for proper Pedestal records. + +**Pinned at 0.7.0**: Pedestal 0.7.1+ uses Jetty 12, which breaks figwheel-main's Ring adapter. See `project.clj` comments. + +### Datomic Pro (Database) + +[Datomic](https://docs.datomic.com/) is an immutable, time-aware database. Key concepts: + +- **Entities**: Maps with `:db/id` — similar to documents +- **Transactions**: Immutable facts added over time — never UPDATE, always assert new facts +- **Pull**: Declarative data retrieval (like GraphQL) +- **Queries**: Datalog (logic programming) + +```clojure +;; Pull an entity +(d/pull db [:db/id ::folder/name] folder-id) + +;; Query +(d/q '[:find ?e :where [?e ::folder/owner "alice"]] db) +``` + +Connection uses `datomic:dev://` protocol (required for Java 21). Schema lives in `src/clj/orcpub/db/schema.clj`. + +### Buddy (Authentication) + +[Buddy](https://funcool.github.io/buddy-auth/latest/) handles JWT token signing and bcrypt password hashing: + +- `buddy-auth` — Token-based authentication middleware +- `buddy-hashers` — Password hashing (bcrypt) + +Tokens are signed with the `SIGNATURE` env var. See `src/clj/orcpub/routes.clj` for the `check-auth` interceptor. + +### PDFBox 3 (PDF Generation) + +[Apache PDFBox](https://pdfbox.apache.org/) fills PDF character sheet templates. The app loads a blank D&D character sheet PDF, fills in field values from the built character, and returns the result. + +```clojure +(with-open [doc (Loader/loadPDF input)] + ;; fill fields... + (.save doc output)) +``` + +**PDFBox 3.x API change**: Use `Loader/loadPDF` instead of `PDDocument/load`. + +## Shared Code (.cljc) + +The character model, D&D rules, templates, and entity system are all `.cljc` — they run on both JVM (for testing and PDF generation) and browser (for the UI). Key namespaces: + +| Namespace | Purpose | +|-----------|---------| +| `orcpub.entity` | Entity/template system (build, from-strict, to-strict) | +| `orcpub.dnd.e5.character` | Character accessors (abilities, HP, AC, etc.) | +| `orcpub.dnd.e5.options` | D&D class/race/feat option builders | +| `orcpub.dnd.e5.template` | Full character template (all classes, races, backgrounds) | +| `orcpub.pdf-spec` | PDF field mapping (pure — no re-frame dependency) | +| `orcpub.dnd.e5.magic-items` | Magic item/weapon computation (homebrew-aware) | + +## Build System + +### Leiningen + +The project uses [Leiningen](https://leiningen.org/) for builds, dependency management, and task running. + +Key aliases: + +| Command | What it does | +|---------|-------------| +| `lein test` | Run JVM tests (206 tests) | +| `lein fig:build` | One-shot CLJS compilation | +| `lein fig:dev` | CLJS dev with REPL + hot reload | +| `lein fig:test` | Compile + run CLJS tests in browser | +| `lein garden once` | Compile CSS | +| `lein lint` | Run clj-kondo linter | +| `lein uberjar` | Production build (AOT + advanced CLJS + CSS) | + +### Profiles + +| Profile | Purpose | +|---------|---------| +| `:dev` | Development (devtools, piggieback, re-frame-10x) | +| `:uberjar` | Production build (AOT, advanced optimizations) | +| `:lint` | clj-kondo linting | +| `:init-db` | Minimal profile for DB init scripts (no CLJS/Garden) | + +### Local JARs + +Some dependencies (Datomic Pro, PDFBox) are loaded from `lib/` via a `file:lib` local Maven repository. This avoids requiring Datomic credentials or dealing with non-standard Maven repos. + +## Data Serialization + +Client-server communication uses [Transit](https://github.com/cognitect/transit-format) (JSON-based, preserves Clojure types like keywords and sets). The `cljs-http` library handles Transit encoding/decoding automatically. + +Homebrew content (`.orcbrew` files) uses Transit for import/export. Users regularly import 2-5MB plugin files. + +## Docker + +Three services in `docker-compose.yaml`: + +| Service | Image | Purpose | +|---------|-------|---------| +| `orcpub` | Application | Clojure app server (port 8890) | +| `datomic` | Database | Datomic Pro transactor | +| `web` | nginx:alpine | Reverse proxy (ports 80/443) | + +Config via `.env` file — see `.env.example` for all variables. + +## Dependency Pinning + +Several dependencies are pinned to specific versions for compatibility: + +| Dependency | Pin | Reason | +|-----------|-----|--------| +| Pedestal | 0.7.0 | 0.7.1+ uses Jetty 12, breaks figwheel | +| Jackson | 2.15.2 | Resolves transitive conflicts and CVEs | +| Guava | 32.1.2-jre | Same | +| commons-io | 2.15.1 | Required by Pedestal 0.7 ring-middlewares | +| javax.servlet-api | 4.0.1 | Required on Java 9+ | + +See [migration/library-upgrades.md](migration/library-upgrades.md) for the full upgrade history. diff --git a/docs/TODO.md b/docs/TODO.md new file mode 100644 index 000000000..79134dada --- /dev/null +++ b/docs/TODO.md @@ -0,0 +1,47 @@ +# TODO — Tracked Issues + +## localStorage corrupt data persistence + +**Status:** Open +**Severity:** Medium +**Reported:** 2026-02-21 + +### Problem + +When `reg-local-store-cofx` reads localStorage data that fails spec validation, +it logs a warning and ignores the data — but never removes it. The corrupt data +persists across reloads, producing `INVALID ITEM FOUND, IGNORING` on every page +load. If the user never interacts with the affected feature (to trigger an +overwrite), the corrupt data stays indefinitely. + +Known corruption vector: `assoc-in` on `nil` builds maps with integer keys +instead of vectors. Example from combat tracker: + +```clojure +(assoc-in nil [:monsters 0 :monster] :adult-gold-dragon) +;; => {:monsters {0 {:monster :adult-gold-dragon}}} — MAP, not vector +``` + +This was partially fixed by guarding `set-combat-path-prop` with +`(or combat default-combat)`, but other handlers using `assoc-in` through +`path` interceptors may have the same vulnerability. + +### Proposed fix + +Scope cleanup by data criticality: + +| Category | Examples | Action on invalid | +|----------|----------|-------------------| +| Ephemeral | combat, builder state | `.removeItem` — safe to lose | +| Rebuildable | spells, monsters | `.removeItem` — regenerated from source | +| Critical | plugins, characters, user | Quarantine: rename key to `_corrupt_` | + +This preserves recovery options for irreplaceable user data (homebrew plugins +can be 2-5MB of daily imports) while cleaning up transient state that would +otherwise stubbornly persist. + +### Related + +- `src/cljs/orcpub/dnd/e5/db.cljs` — `reg-local-store-cofx` (line ~252) +- `src/cljs/orcpub/dnd/e5/events.cljs` — `set-combat-path-prop` nil guard +- All `*->local-store` serializers use `(str data)` / `reader/read-string` diff --git a/docs/migration/datomic-data-migration.md b/docs/migration/datomic-data-migration.md new file mode 100644 index 000000000..efa4572bc --- /dev/null +++ b/docs/migration/datomic-data-migration.md @@ -0,0 +1,338 @@ +# Datomic Data Migration — Free to Pro + +Existing self-hosted deployments have a Datomic Free database in `./data` that +must be migrated before upgrading to the new Java 21 / Datomic Pro Docker stack. +The storage protocols (`datomic:free://` vs `datomic:dev://`) are incompatible at +the file level — the Pro transactor cannot read Free storage, and vice versa. + +## Why Migration Is Needed + +Datomic Free and Datomic Pro use different storage protocols: + +| | Datomic Free | Datomic Pro (dev) | +|---|---|---| +| URI scheme | `datomic:free://` | `datomic:dev://` | +| Storage engine | H2 (different on-disk schema) | H2 (different on-disk schema) | +| Transport | Custom (pre-1.0) | ActiveMQ Artemis | +| Authentication | None | Password-based | +| Java support | Java 8 only | Java 11, 17, 21 | + +Despite both using H2, the Pro transactor cannot open Free's H2 files — same +data model, incompatible storage. + +## How It Works + +The migration uses Datomic's `bin/datomic` CLI — **not** the Peer API +(`datomic.api`). `backup-db` creates a portable, protocol-independent backup +from the running Free transactor. `restore-db` writes it into a new Pro database. + +``` + Free transactor (running) Pro transactor (running) + ┌──────────────────┐ ┌──────────────────┐ + │ port 4334 │ │ port 4334 │ + └────────┬─────────┘ └────────┬─────────┘ + │ │ + bin/datomic backup-db bin/datomic restore-db + (Free distribution) (Pro distribution) + │ │ + ▼ ▼ + file:./backup/orcpub ────────> file:./backup/orcpub +``` + +**Peer library matching is critical**: the Free CLI can only connect to Free +transactors, the Pro CLI can only connect to Pro transactors. Backup must use the +Free distribution's `bin/datomic`; restore must use Pro's. The migration scripts +handle this automatically. + +The backup format is storage-protocol-independent and the backup/restore cycle +is **lossless** — all datoms, full transaction history, timestamps, entity IDs, +and schema are preserved exactly. Only the storage layer changes. + +**Commands reference:** + +```bash +bin/datomic backup-db # backup (Free or Pro) +bin/datomic restore-db # restore (Free or Pro) +bin/datomic list-backups # list backup points +bin/datomic verify-backup true # verify (Pro only) +``` + +## Quick Start (Bare Metal) + +```bash +# 1. With the old (Free) transactor running: +./scripts/migrate-db.sh backup + +# 2. Stop the Free transactor, move data aside, start Pro transactor +# (see "Step-by-Step" below for details) + +# 3. Restore into the new Pro database: +./scripts/migrate-db.sh restore "datomic:dev://localhost:4334/orcpub?password=..." + +# 4. Verify backup integrity: +./scripts/migrate-db.sh verify +``` + +## Quick Start (Docker) + +```bash +# 1. With the OLD Docker stack still running: +./docker-migrate.sh backup + +# 2. Stop old stack, build and start new: +docker compose down +docker compose -f docker-compose-build.yaml build +docker compose -f docker-compose-build.yaml up -d + +# 3. After services are healthy, restore: +./docker-migrate.sh restore + +# 4. Verify backup integrity: +./docker-migrate.sh verify +``` + +Or run everything in one command: `./docker-migrate.sh full` + +## Step-by-Step (Bare Metal) + +### Prerequisites + +- Datomic Free transactor running with your existing data +- `.env` file with `DATOMIC_URL` (pointing to the Free transactor) and `DATOMIC_PASSWORD` +- Datomic Pro installed (`lib/com/datomic/datomic-pro/...`) +- Enough disk space for the backup (see [Performance](#performance)) + +### Phase 1: Backup + +With the Free transactor running: + +```bash +./scripts/migrate-db.sh backup +``` + +The script detects `datomic:free://` in your `DATOMIC_URL` and looks for the +Free distribution in `lib/`. If it can't find one interactively, it will prompt +for the path — or press Enter to extract the bundled tarball +(`lib/datomic-free-0.9.5703.tar.gz`). To skip the prompt: + +```bash +./scripts/migrate-db.sh --datomic-dir /path/to/datomic-free backup +``` + +### Phase 2: Swap Transactors + +```bash +./scripts/stop.sh datomic +mv ./data ./data.free-backup +mkdir -p ./data +./scripts/start.sh datomic +``` + +Wait for the transactor to become healthy (port 4334). + +### Phase 3: Restore + +```bash +./scripts/migrate-db.sh restore "datomic:dev://localhost:4334/orcpub?password=${DATOMIC_PASSWORD}" +``` + +**The target database must not already exist.** If the application already created +it on first connect, delete `./data/*` and restart the transactor before restoring. + +### Phase 4: Verify + +```bash +./scripts/migrate-db.sh verify + +# Optionally check DB-level stats (user/entity counts): +# java -cp target/orcpub.jar clojure.main docker/scripts/migrate-db.clj verify [db-uri] +``` + +Then log in and verify your data — characters load, homebrew content is +accessible, user accounts work. + +## Step-by-Step (Docker) + +### Prerequisites + +- Old Docker stack running (`docker compose ps` shows healthy datomic + orcpub) +- `.env` file with correct `DATOMIC_PASSWORD` +- Enough disk space (see [Performance](#performance)) +- New source code checked out (has `docker-compose-build.yaml` and migration scripts) + +Each phase launches a **temporary container** (`docker run --rm`) on the Compose +network, bind-mounting `./backup/` for I/O. No running containers are modified. + +``` + Old Stack (running) New Stack (running) +┌─────────────┐ ┌────────────┐ ┌─────────────┐ ┌────────────┐ +│ orcpub │ │ datomic │ │ orcpub │ │ datomic │ +│ (Free peer) │ │ (Free tx) │ │ (Pro peer) │ │ (Pro tx) │ +└──────────────┘ └──────┬─────┘ └──────────────┘ └──────┬─────┘ + │ │ + Compose Network Compose Network + │ │ + ┌────────┴─────────┐ ┌───────────┴────────┐ + │ Temp container │ │ Temp container │ + │ (old datomic img)│ │ (new datomic img) │ + │ backup-db ──────►│ ./backup/ │◄─────── restore-db │ + └──────────────────┘ (bind mount) └────────────────────┘ +``` + +### Phase 1: Backup + +```bash +./docker-migrate.sh backup +``` + +### Phase 2: Swap Stacks + +```bash +docker compose down +mv ./data ./data.free-backup +mkdir -p ./data +docker compose -f docker-compose-build.yaml build +docker compose -f docker-compose-build.yaml up -d +``` + +Wait for healthy: `docker compose ps` + +### Phase 3: Restore + +```bash +./docker-migrate.sh restore +``` + +URI is auto-detected from the running container, or constructed from +`DATOMIC_PASSWORD` in `.env`. + +### Phase 4: Verify + +```bash +./docker-migrate.sh verify +``` + +Test a login: + +```bash +curl -sk -X POST https://localhost/login \ + -H "Content-Type: application/json" \ + -d '{"username":"youruser","password":"yourpass"}' +``` + +## Rollback + +You can roll back at any point before deleting `./data.free-backup`: + +**Bare metal:** + +```bash +./scripts/stop.sh datomic +rm -rf ./data +mv ./data.free-backup ./data +./scripts/start.sh datomic +``` + +**Docker:** + +```bash +docker compose down +rm -rf ./data +mv ./data.free-backup ./data +docker compose up -d # OLD compose file, not docker-compose-build.yaml +``` + +The backup directory is never modified — you can re-attempt the restore as many +times as needed. Just clear `./data/*` and restart the transactor before each attempt. + +## Cleanup + +Once verified: + +```bash +rm -rf ./data.free-backup # old Free storage +rm -rf ./backup # portable backup (or keep as insurance) +``` + +## Performance + +Rough estimates — actual performance depends on disk speed, CPU, and database +complexity. + +| Database Size | Backup/Restore Time | Peak Disk Space | +|--------------|---------------------|-----------------| +| < 1 GB | 1-5 min | ~3 GB | +| 1-10 GB | 5-30 min | ~30 GB | +| 10-25 GB | 30-90 min | ~75 GB | +| 25-50 GB | 1-3 hours | ~150 GB | + +**Peak disk** = old `./data` + `./backup` + new `./data` (all coexist during +migration). Monitor progress with `du -sh ./backup/orcpub`. + +**Memory**: The default 1GB JVM heap (`bin/run`) suffices for most databases. +For 50GB+, increase with `DATOMIC_JAVA_OPTS="-Xmx4g"` before running. + +## Troubleshooting + +### "Cannot connect to Datomic" + +**Bare metal**: Ensure the transactor is running and `DATOMIC_URL` in `.env` is +correct. + +**Docker**: The temp container must be on the same Compose network. The script +auto-detects this. For unusual setups, use `--old-uri` or `--new-uri` (**before +the command word**): + +```bash +./docker-migrate.sh --old-uri "datomic:free://datomic:4334/orcpub" backup +./docker-migrate.sh --new-uri "datomic:dev://datomic:4334/orcpub?password=mypass" restore +``` + +### "Target database already exists" + +`restore-db` requires the target database to not exist. If the app auto-created +it on first boot: + +1. `docker compose down` (or stop the transactor) +2. `rm -rf ./data/*` +3. Restart, then retry restore + +### Interrupted backup + +Delete the incomplete backup directory and re-run. Backups are not resumable +mid-segment. + +### Data discrepancy after restore + +Run `./scripts/migrate-db.sh verify`, then log in and check: characters load, +homebrew content accessible, users can log in. If wrong, delete `./data`, +re-restore from the same backup. + +## FAQ + +### Can I use the Pro transactor with the Free peer library? + +No. The peer library and transactor must match — Free peer speaks +`datomic:free://` only, Pro peer speaks `datomic:dev://` only. They use +different transport protocols and are not interchangeable. + +### What happens after migration? + +The migrated database works identically to one that was always on Pro. No +ongoing migration state or compatibility mode. `backup-db` and `restore-db` +continue to work for regular backups going forward. + +## Scripts + +| Script | Environment | Purpose | +|--------|-------------|---------| +| `scripts/migrate-db.sh` | Bare metal | Primary migration tool, wraps `bin/datomic` CLI | +| `docker-migrate.sh` | Docker | Containerized wrapper, auto-detects images/networks | +| `docker/scripts/migrate-db.clj` | Either (optional) | DB-level stats verification (user/entity counts) | + +## Related + +- [datomic-pro.md](datomic-pro.md) — Code-level changes (dependency, URI, API) +- [../ENVIRONMENT.md](../ENVIRONMENT.md) — Environment variable reference +- [../../docker-setup.sh](../../docker-setup.sh) — Initial Docker setup +- [../../docker-user.sh](../../docker-user.sh) — User management after migration diff --git a/docs/migration/datomic-pro.md b/docs/migration/datomic-pro.md new file mode 100644 index 000000000..1afd3acc9 --- /dev/null +++ b/docs/migration/datomic-pro.md @@ -0,0 +1,78 @@ +# Datomic Free → Datomic Pro + +## Why + +Datomic Free 0.9.5703 does not work on Java 21. The peer-to-transactor connection fails with an SSL handshake timeout in ActiveMQ Artemis (Datomic's internal messaging layer). This is not configurable — Datomic Free is unmaintained and will never be fixed. + +Datomic Pro is free under the Apache 2.0 license and supports Java 11, 17, and 21. + +## What Changed + +### Connection URI + +``` +# Before (Datomic Free) +datomic:free://localhost:4334/orcpub + +# After (Datomic Pro dev protocol) +datomic:dev://localhost:4334/orcpub +``` + +The `dev` protocol uses the Datomic Pro dev transactor (same data model, same API, different transport). + +### Maven Dependency + +```clojure +;; Before +[com.datomic/datomic-free "0.9.5703"] + +;; After +[com.datomic/peer "1.0.7482" :exclusions [org.slf4j/slf4j-nop]] +``` + +The Datomic Pro peer library is `com.datomic/peer` (not `com.datomic/datomic-pro`). The `slf4j-nop` exclusion avoids duplicate SLF4J binding warnings. + +### Installation + +Datomic Pro is installed during container creation by `.devcontainer/post-create.sh`: + +1. Downloads the Datomic Pro zip from S3 +2. Extracts to `lib/com/datomic/datomic-pro//` +3. Runs `bin/maven-install` to populate the local Maven repository + +The peer JAR resolves via the existing `file:lib` repository pattern in `project.clj`. + +### Transactor + +The transactor is a separate process started by `scripts/start.sh datomic`. It uses a properties file generated from the template at the Datomic install path. The transactor is **not** a JAR — it's the `bin/transactor` script with a config file. + +### Configuration + +All Datomic config flows through `src/clj/orcpub/config.clj`: + +```clojure +(config/get-datomic-uri) +;; Returns DATOMIC_URL env var or default "datomic:dev://localhost:4334/orcpub" +``` + +Environment variables (see `.env.example`): +- `DATOMIC_URL` — connection URI +- `DATOMIC_VERSION` — version for installer (default: `1.0.7482`) +- `DATOMIC_TYPE` — `pro` (default) +- `DATOMIC_PASSWORD` — transactor password + +### Code Changes + +| File | Change | +|------|--------| +| `project.clj` | `datomic-free` → `com.datomic/peer`, version `1.0.7482` | +| `src/clj/orcpub/config.clj` | New — `get-datomic-uri`, `datomic-env` (SSOT for URI) | +| `src/clj/orcpub/system.clj` | Uses `config/get-datomic-uri` instead of inline logic | +| `src/clj/orcpub/datomic.clj` | Connection logic unchanged — `datomic.api` is the same | +| `dev/user.clj` | `init-database` uses `config/datomic-env` | +| `.devcontainer/post-create.sh` | Datomic Pro installer (download, extract, maven-install) | +| `.env.example` | Datomic Pro defaults | + +### Datomock Compatibility + +The test suite uses `datomock` for in-memory database testing. The original `vvvvalvalval/datomock 0.2.0` causes `AbstractMethodError` with Datomic Pro (new `transact` signature). Replaced with `org.clojars.favila/datomock 0.2.2-favila1`. diff --git a/docs/migration/dev-tooling.md b/docs/migration/dev-tooling.md new file mode 100644 index 000000000..73a46190c --- /dev/null +++ b/docs/migration/dev-tooling.md @@ -0,0 +1,178 @@ +# Dev Tooling + +## How Clojure Dev Tooling Works (Quick Primer) + +Clojure projects use **Leiningen** (`lein`) as their build tool — similar to npm for JavaScript or Maven for Java. Leiningen reads `project.clj` for configuration. + +**Profiles** are named configurations that control what code is available and how things compile. Think of them like npm scripts that also change the project's classpath: + +- `:dev` — full development mode (REPL, Figwheel, ClojureScript, all dev helpers) +- `:init-db` — minimal mode for CLI tasks (no ClojureScript, no CSS, fast startup) +- `:uberjar` — production build (AOT-compiled, no dev code) + +A **REPL** (Read-Eval-Print Loop) is an interactive Clojure session. You type expressions and get results immediately — it's how most Clojure development happens. The `user` namespace loads automatically when you start a REPL in dev mode. + +## user.clj — The Dev Tooling Hub + +**Location**: `dev/user.clj` + +This single file is the hub for all dev tooling. It serves two purposes: + +1. **REPL helpers** — functions you call interactively during development +2. **CLI entrypoint** — a `-main` function that shell scripts call for automation + +### Why dev/ and not src/ + +`user.clj` lives in the `dev/` directory, which is **only** on the classpath in development profiles (`:dev` and `:init-db`). It is **never** included in the production jar. This means none of the dev tooling (user creation, database initialization, Figwheel helpers) ships to production — a deliberate security choice. + +### REPL Functions + +When you start a REPL (`lein repl`), the `user` namespace loads automatically. These functions are available: + +| Function | What it does | Needs running server? | +|----------|-------------|----------------------| +| `(start-server)` | Start Pedestal web server + Datomic | No (starts it) | +| `(stop-server)` | Stop the server | Yes | +| `(init-database)` | Create database + apply schema | No | +| `(add-test-user)` | Create a pre-verified test user | No | +| `(conn)` | Get a raw Datomic connection | No | +| `(create-user! (conn) {...})` | Create a user with options | No | +| `(verify-user! (conn) "email")` | Mark a user as verified | No | +| `(delete-user! (conn) "email")` | Remove a user | No | +| `(verify-new-user "email")` | Verify via routes (legacy) | Yes | +| `(fig-start)` | Start Figwheel hot-reload | No | +| `(fig-stop)` | Stop Figwheel | — | +| `(cljs-repl)` | Connect to ClojureScript REPL | Figwheel running | +| `(with-db [conn db] ...)` | Ad-hoc DB access macro | Yes | + +**"Needs running server?"** — Some functions (like `create-user!`) connect directly to Datomic, so they work without the web server. Others (like `verify-new-user`) go through the route handlers and need the full system running. + +### CLI Commands + +Shell scripts call user.clj via Leiningen's `run -m` flag, which invokes the `-main` function: + +```bash +# Initialize database (create + apply schema) +lein with-profile init-db run -m user init-db + +# Initialize database AND create a test user +lein with-profile init-db run -m user init-db --add-test-user + +# Create an arbitrary user (with duplicate checking) +lein with-profile init-db run -m user create-user bob bob@example.com s3cret verify + +# Mark a user as verified (useful when emails don't send in dev) +lein with-profile init-db run -m user verify-user bob@example.com + +# Delete a user +lein with-profile init-db run -m user delete-user bob@example.com +``` + +**Why `with-profile init-db`?** Without it, Leiningen loads the full `:dev` profile which compiles ClojureScript and CSS — slow and unnecessary for database tasks. The `:init-db` profile skips all that for fast startup. + +## Shell Scripts + +You don't need to remember the `lein` commands above. Shell scripts wrap them: + +### start.sh — Service Launcher + +```bash +./scripts/start.sh datomic # Start Datomic transactor +./scripts/start.sh init-db # Initialize database (calls user.clj init-db) +./scripts/start.sh server # Start backend REPL +./scripts/start.sh figwheel # Start Figwheel hot-reload (headless watcher) +./scripts/start.sh garden # Start Garden CSS watcher +``` + +Flags: `--quiet` (less output), `--check` (pre-flight only), `--idempotent` (succeed if already running) + +### Figwheel Modes + +Figwheel-main has three modes, exposed as Leiningen aliases: + +| Alias | Command | What it does | +|-------|---------|-------------| +| `lein fig:dev` | `--build dev --repl` | Build + watch + interactive REPL (needs a terminal) | +| `lein fig:watch` | `--build dev` | Build + watch, headless (works with nohup/background) | +| `lein fig:build` | `--build-once dev` | One-time build, no watcher | + +**`start.sh figwheel`** uses `fig:watch` (headless) so it works when backgrounded. If you want an interactive ClojureScript REPL, run `lein fig:dev` directly in a terminal instead. + +### Garden CSS + +Garden compiles Clojure style definitions (`src/clj/orcpub/styles/core.clj`) into CSS (`resources/public/css/compiled/styles.css`). + +- **One-time compile**: `lein garden once` +- **Auto-watch**: `lein garden auto` (blocks the terminal — run in a separate session or use `./scripts/start.sh garden`) +- **start.sh garden**: Runs `lein garden auto` in the background + +Garden and Figwheel are independent — Figwheel hot-reloads ClojureScript, Garden compiles CSS. Both need to run during active frontend development. + +### stop.sh — Service Shutdown + +```bash +./scripts/stop.sh datomic --yes --quiet +``` + +### menu — Interactive Hub + CLI + +```bash +./menu # Interactive terminal menu with status display +./menu add bob pass123 # Create bob@test.com (auto-verified) +./menu add bob pass123 --no-verify # Create without verification +./menu add user # Interactive prompt for name/password +./menu verify bob # Verify existing user +./menu delete bob # Delete bob@test.com +./menu user # Show all user commands +``` + +Email auto-generates as `@test.com`. Credentials are logged to `.test-users` (gitignored) for easy lookup. + +### create_dummy_user.sh — Create a User (full control) + +For cases where you need a specific email address (not `@test.com`): + +```bash +./scripts/create_dummy_user.sh testuser custom@email.com s3cret verify +``` + +### dev-setup.sh — First-Time Onboarding + +Runs the full first-time setup: start Datomic, install dependencies, initialize database, and create a verified test user. + +```bash +./scripts/dev-setup.sh # full setup including test user +./scripts/dev-setup.sh --no-test-user # skip test user creation +./scripts/dev-setup.sh --skip-datomic # skip Datomic startup (if already running) +``` + +Default test user credentials: `test` / `test@test.com` / `testpass` (pre-verified, ready to log in). + +All user-creation paths log credentials to `.test-users` in the repo root (gitignored). + +## config.clj — Configuration Hub + +**Location**: `src/clj/orcpub/config.clj` + +Single source of truth for all runtime configuration. Reads environment variables via the `environ` library. Unlike user.clj, this file IS in production — it's the config layer the app uses at runtime. + +| Function | Returns | +|----------|---------| +| `(config/get-datomic-uri)` | `DATOMIC_URL` env or `"datomic:dev://localhost:4334/orcpub"` | +| `(config/get-csp-policy)` | `CSP_POLICY` env or `"strict"` | +| `(config/strict-csp?)` | `true` when CSP policy is strict | +| `(config/dev-mode?)` | `true` when `DEV_MODE` env is truthy | +| `(config/get-secure-headers-config)` | Pedestal secure-headers map based on CSP policy | + +Used by: `system.clj`, `pedestal.clj`, `user.clj` + +## Profile Security Model + +| Profile | Source paths | Includes dev/? | In production jar? | +|---------|-------------|----------------|-------------------| +| `:dev` | src/clj, src/cljc, src/cljs, web/cljs, dev | Yes | No | +| `:init-db` | src/clj, src/cljc, dev | Yes | No | +| `:uberjar` | src/clj, src/cljc, src/cljs, web/cljs | No | Yes | +| (base) | src/clj, src/cljc, src/cljs | No | — | + +The `dev/` directory (containing user.clj) is only available in development profiles. The production uberjar contains zero dev tooling. diff --git a/docs/migration/frontend-stack.md b/docs/migration/frontend-stack.md new file mode 100644 index 000000000..a117353e1 --- /dev/null +++ b/docs/migration/frontend-stack.md @@ -0,0 +1,255 @@ +# Frontend Stack Upgrade + +## React 15 → 18 + +React 18 introduces the `createRoot` API and deprecates the legacy `ReactDOM.render()`. + +### Dependencies + +```clojure +;; Before +[cljsjs/react "15.x"] +[cljsjs/react-dom "15.x"] +[reagent "0.6.x"] + +;; After +[cljsjs/react "18.3.1-1"] +[cljsjs/react-dom "18.3.1-1"] +[reagent "2.0.1"] +``` + +### createRoot Migration + +In `web/cljs/orcpub/core.cljs`, the app mounts using Reagent 2.0's `reagent.dom.client` namespace: + +```clojure +(ns orcpub.core + (:require [reagent.dom.client :as rdc] ...)) + +;; createRoot-based mounting (React 18) +``` + +This replaces the old `reagent.dom/render` call. + +### Reagent 2.x: `:class` vs `:class-name` + +Reagent 2.x changed how CSS classes merge. The `:class-name` prop **overwrites** classes set on the hiccup tag, while `:class` **merges** with them: + +```clojure +;; BAD — .white is lost, only bg-red applies +[:div.white {:class-name "bg-red"}] + +;; GOOD — both .white and .bg-red apply +[:div.white {:class "bg-red"}] +``` + +Always use `:class` (not `:class-name`) when the hiccup tag already has classes like `[:div.foo.bar ...]`. + +### Production Build: Externs for React 18 + +The `cljsjs/react-dom 18.3.1-1` package has incomplete externs. Under Closure Compiler `:advanced` optimization, two React 18 APIs get renamed, causing a runtime crash (`c0 is not a function`): + +- `ReactDOM.Root.render` — used by `reagent.dom.client/render` +- `ReactDOM.flushSync` — used by `reagent.impl.batching/react-flush` + +**Fix**: A custom `externs.js` at the repo root declares these symbols: + +```javascript +ReactDOM.Root.render = function(children) {}; +ReactDOM.flushSync = function(callback) {}; +``` + +The uberjar profile references it: + +```clojure +:compiler {:optimizations :advanced + :infer-externs true + :externs ["externs.js"]} +``` + +### re-frame + +Updated from `0.x` to `1.4.4`. The event/subscription API is unchanged — existing handlers, subscriptions, and effects work without modification. + +#### Subscribe-outside-reactive-context: Fixed + +The original codebase had **56 instances** of `@(subscribe [...])` called outside Reagent reactive context (inside event handlers, modifier conditions, top-level code, and pure `.cljc` namespaces). In re-frame 0.x these worked silently; in 1.3+ they produce console warnings and leak Reaction objects. All 56 have been fixed across two phases. + +**Fix patterns used** (stable re-frame APIs only — no `re-frame.alpha` or third-party libs): + +| Pattern | When to use | Instances | +|---------|-------------|-----------| +| Direct db read | Subscription is `(get db :key)` | 3 | +| Pass from component | Component already has the value | 4 | +| Extract pure helpers | Subscription computes derived data | 2 | +| Replace with dispatch | `reg-sub-raw` used for side effects | 1 | +| Template cache via `track!` | Autosave handler (no component context) | 1 | +| Direct db read (character) | Character already in db map | 1 | +| SSOT pure fn + `@app-db` | Modifier conditions needing dynamic data | 4 | +| Thread parameter from caller | Data already in the calling chain | 1 | +| Plugin-data map | Pure `.cljc` namespace, multiple subscribes | 7 | +| `reg-sub-raw` | Conditional subscription routing | 1 | +| Move to render scope | `@(subscribe)` in event closure | 1 | +| Pure character fns | Prereq functions with subscribe | 22 | +| `def` + `partial` → `defn` | Subscribe at load time via `partial` | 1 | + +**Phase 1** (events.cljs, options.cljc, classes.cljc, core.cljs): 42 fixes +**Phase 2** (options.cljc, pdf_spec.cljc, equipment_subs.cljs, views.cljs): 14 fixes +**Browser console**: zero subscribe warnings on fresh page load. + +**Key insight**: Most subscriptions were Layer 3 (computed/derived). Naively replacing `@(subscribe [...])` with `(get db ...)` would have returned `nil` — the computed values only exist in the subscription cache, not in app-db. Custom/homebrew content is especially tricky: `mi/all-weapons-map` (static) does NOT include user-imported weapons. + +#### Utility namespace: `orcpub.dnd.e5.event-utils` + +A circular dependency prevented `events.cljs` from importing subscription computation code: + +``` +equipment_subs.cljs → events.cljs (url-for-route, show-generic-error) +subs.cljs → events.cljs (url-for-route, show-generic-error, mod-cfg, default-mod-set) +spell_subs.cljs → events.cljs (dead import) +``` + +**Fix**: Shared utility functions extracted from `events.cljs` into `event_utils.cljs`: + +| Function | Purpose | +|----------|---------| +| `backend-url` | Rewrites path to localhost:8890 in dev | +| `url-for-route` | Builds backend URL from bidi route | +| `auth-headers` | Returns `Authorization` header map (was duplicated 3x) | +| `show-generic-error` | Returns generic error dispatch vector | +| `mod-cfg` | Builds modifier config map | +| `mod-key` | Multimethod: extracts comparison key from modifier | +| `compare-mod-keys` | Comparator for modifier configs | +| `default-mod-set` | Ensures modifier set is sorted | + +Subscription files now import `event-utils` instead of `events`, breaking the circle. `events.cljs` aliases the moved functions for backward compatibility. + +#### Template caching via `reagent.core/track!` + +**THIS IS A NEW PATTERN IN THIS PROJECT.** `track!` has never been used elsewhere in the codebase. It is stable Reagent API (since ~0.6) but warrants extra testing attention. + +**Problem**: The autosave handler (`::char5e/save-character`) needs `built-character`, which requires the full template. The template is computed by a 12-input subscription chain. The handler is dispatched from a timer (no component context), so it can't subscribe. + +**Solution**: `autosave_fx.cljs` creates a `track!` watcher that observes the `::char5e/template` subscription in a proper reactive context and caches the result in app-db: + +```clojure +;; autosave_fx.cljs +(defonce _init-template-cache + (js/setTimeout + (fn [] + (r/track! + (fn [] + (when-let [template @(subscribe [::char5e/template])] + (dispatch [::cache-template template]))))) + 0)) +``` + +The save handler then reads the cached template and computes `entity/build(character, template)` directly: + +```clojure +;; events.cljs — ::char5e/save-character +(let [cached-template (get db ::autosave-fx/cached-template)] + (if-not cached-template + {} ;; template not cached yet — skip this cycle + (let [built-character (entity/build character cached-template)] + ...))) +``` + +**Why `js/setTimeout 0`**: The `track!` must run after all subscription registrations. Since `autosave_fx.cljs` loads before `equipment_subs.cljs` (where `::char5e/template` is registered), the timeout defers creation to end-of-event-loop when all modules are loaded. + +**Why `built-template` is a no-op**: The `built-template` function in `subs.cljs:254-270` has its plugin-merging logic entirely commented out (`#_`). It returns the input template unchanged. This means the cached `::char5e/template` IS the effective `built-template`. + +**Safety**: If the template hasn't cached yet (e.g., app just loaded), the handler returns `{}` (no-op). The next autosave cycle (7.5s later) retries. + +**Risk factors**: +- `track!` creates a long-lived reactive watcher — ensure it doesn't leak if the app is torn down +- If `::char5e/template` subscription is ever renamed/removed, the `track!` will silently fail +- The `js/setTimeout 0` relies on module load order — all `reg-sub` calls must complete before the timeout fires +- If `built-template` plugin merging is ever re-enabled, the cached template would need to include plugin-option processing + +#### Files changed (subscribe refactor) + +| File | Changes | +|------|---------| +| `src/cljs/orcpub/dnd/e5/event_utils.cljs` | **NEW**: shared utilities extracted from events.cljs | +| `src/cljs/orcpub/dnd/e5/events.cljs` | All handler fixes, compute helpers, verify-user-session, subscribe removed from requires | +| `src/cljs/orcpub/dnd/e5/autosave_fx.cljs` | Template cache via `track!` | +| `src/cljs/orcpub/dnd/e5/subs.cljs` | Import event-utils, remove local auth-headers | +| `src/cljs/orcpub/dnd/e5/equipment_subs.cljs` | Import event-utils, remove local auth-headers | +| `src/cljs/orcpub/dnd/e5/spell_subs.cljs` | Remove dead events import | +| `src/cljs/orcpub/character_builder.cljs` | Pass built-char via dispatch (save, random name) | +| `src/cljc/orcpub/dnd/e5/options.cljc` | Multi-arity custom-option-builder (inject-template? flag) | +| `web/cljs/orcpub/core.cljs` | Replace `@(subscribe [:user false])` with `(dispatch-sync [:verify-user-session])` | + +#### Test coverage + +Pure function tests now cover many refactored modules. CLJS-only tests exist for re-frame handler integration. The JVM test suite includes tests for `compute-all-weapons-map`, feat-prereqs, pdf_spec pure functions, folder routes (CRUD + validation), and event handler round-trips. + +**Current**: 206 JVM tests, 945 assertions, 0 failures. + +**CLJS-only tests** (browser via `lein fig:test`): +- `test/cljs/orcpub/dnd/e5/events_test.cljs` — re-frame handler tests + +**Manual testing checklist**: +- [ ] Save character (manual) — verify character saves with correct summary +- [ ] Autosave — edit character, wait 7.5s, verify save fires +- [ ] Random name — verify race/subrace/sex-appropriate name generated +- [ ] Filter spells — type 3+ characters, verify spell list filters +- [ ] Filter items — type 3+ characters, verify item list filters +- [ ] Level up — verify character navigates to builder +- [ ] Export plugins — verify .orcbrew file downloads +- [ ] Custom subclass/feat — verify name input works in homebrew builder +- [ ] Login — verify auth check on app startup +- [ ] Homebrew import — import a 2-5MB .orcbrew file, verify content loads + +## Figwheel + +Migrated from **lein-figwheel** (deprecated) to **figwheel-main 0.2.20**. + +### What Changed + +| Aspect | Before | After | +|--------|--------|-------| +| Plugin | `lein-figwheel` | `com.bhauman/figwheel-main 0.2.20` | +| Config | `:figwheel {}` in project.clj | `dev.cljs.edn` + `:figwheel {}` in project.clj | +| REPL | `lein figwheel` | `lein fig:dev` | +| Port | 3449 | 3449 (unchanged) | + +### Port + +Figwheel runs on **port 3449**. This has not changed. The devcontainer forwards this port. + +### Figwheel Modes + +Three Leiningen aliases expose figwheel-main's different modes: + +| Alias | Command | Use when | +|-------|---------|----------| +| `lein fig:dev` | `--build dev --repl` | Interactive development (needs a terminal) | +| `lein fig:watch` | `--build dev` | Background/scripted startup (headless, works with nohup) | +| `lein fig:build` | `--build-once dev` | CI or quick compilation check | + +**Important**: `fig:dev` uses `--repl` which requires an interactive terminal. Running it under `nohup` causes the REPL to read EOF and the watcher dies. `start.sh figwheel` uses `fig:watch` (headless) for this reason. + +### user.clj Integration + +`dev/user.clj` lazy-loads figwheel-main to avoid pulling in CLJS tooling for server-only REPL sessions: + +```clojure +(def ^:private fig-api + (delay + (require 'figwheel.main.api) + (find-ns 'figwheel.main.api))) +``` + +REPL functions: `(fig-start)`, `(fig-stop)`, `(cljs-repl)` + +## Other Frontend Dependencies + +| Library | Before | After | Notes | +|---------|--------|-------|-------| +| `binaryage/devtools` | 0.x | 1.0.7 | Chrome devtools for CLJS | +| `cider/piggieback` | 0.3.x | 0.5.3 | nREPL middleware for CLJS REPL | +| `day8.re-frame/re-frame-10x` | old | 1.11.0 | re-frame debugging panel | +| `hiccup` | 1.x | 2.0.0 | HTML templating | +| `com.cognitect/transit-cljs` | 0.8.x | 0.8.280 | Transit serialization | diff --git a/docs/migration/library-upgrades.md b/docs/migration/library-upgrades.md new file mode 100644 index 000000000..8dc3551ca --- /dev/null +++ b/docs/migration/library-upgrades.md @@ -0,0 +1,156 @@ +# Library Upgrades + +All dependency changes in `project.clj`, grouped by category. + +## Clojure / ClojureScript Core + +| Library | Before | After | +|---------|--------|-------| +| `org.clojure/clojure` | 1.9.0 | 1.12.4 | +| `org.clojure/clojurescript` | 1.9.x | 1.12.134 | +| `org.clojure/core.async` | 0.3.x | 1.8.741 | +| `org.clojure/core.match` | old | 1.1.1 | +| `org.clojure/test.check` | old | 1.1.1 | +| `org.clojure/data.json` | old | 2.5.0 | + +## Date/Time: clj-time → java-time + +**clj-time** wraps Joda-Time, which is end-of-life. Replaced with **java-time** which wraps `java.time` (built into Java 8+). + +```clojure +;; Before +[clj-time "0.x"] + +;; After +[clojure.java-time "1.4.2"] +``` + +The require alias is preserved to minimize churn: + +```clojure +;; Same alias, different library +[clj-time.core :as t] → [java-time.api :as t] +``` + +**Files affected**: `src/clj/orcpub/pedestal.clj` (`parse-date` rewritten) + +**Frontend**: `com.andrewmcveigh/cljs-time 0.5.2` is unchanged (it wraps goog.date, not Joda). + +## PDF Generation + +```clojure +;; Before +[org.apache.pdfbox/pdfbox "2.x"] + +;; After +[org.apache.pdfbox/pdfbox "3.0.6"] +``` + +PDFBox 3 has API changes in font handling and document loading. The vendored JAR is in `lib/org/apache/pdfbox/` and resolves via the `file:lib` repository. + +## Authentication + +```clojure +;; Before +[buddy/buddy-auth "1.x"] +[buddy/buddy-hashers "1.x"] + +;; After +[buddy/buddy-auth "3.0.323"] +[buddy/buddy-hashers "2.0.167"] +``` + +Buddy 3 is compatible with Java 21. The JWS token handling in `orcpub.routes` is unchanged. + +## Web Framework + +See [pedestal-0.7.md](pedestal-0.7.md) for details. + +```clojure +;; Before +[io.pedestal/pedestal.service "0.5.1"] +[io.pedestal/pedestal.route "0.5.1"] +[io.pedestal/pedestal.jetty "0.5.1"] + +;; After (pinned — see pedestal-0.7.md for why) +[io.pedestal/pedestal.service "0.7.0"] +[io.pedestal/pedestal.route "0.7.0"] +[io.pedestal/pedestal.jetty "0.7.0"] +[io.pedestal/pedestal.error "0.7.0"] ;; new +[commons-io/commons-io "2.15.1"] ;; required by Pedestal 0.7 ring-middlewares +``` + +## Component / Infrastructure + +| Library | Before | After | +|---------|--------|-------| +| `com.stuartsierra/component` | old | 1.2.0 | +| `com.google.guava/guava` | old | 32.1.2-jre | +| `com.fasterxml.jackson.core/*` | old | 2.15.2 | +| `environ` | 1.1.0 | 1.2.0 | + +Jackson and Guava are pinned to resolve transitive dependency conflicts and CVEs. + +## Other + +| Library | Before | After | Notes | +|---------|--------|-------|-------| +| `funcool/cuerdas` | old | 2026.415 | String utilities (Clojars versioning) | +| `clj-http` | old | 3.13.1 | HTTP client (JVM) | +| `cljs-http` | 0.1.45 | 0.1.49 | HTTP client (CLJS) — fixes `no.en.core` shadowing (see below) | +| `hiccup` | 1.x | 2.0.0 | HTML rendering | +| `garden` → `com.lambdaisland/garden` | old | 1.9.606 | CSS-in-Clojure (see below) | +| `bidi` | old | 2.1.6 | Routing | +| `com.draines/postal` | old | 2.0.5 | Email | +| `javax.servlet/javax.servlet-api` | — | 4.0.1 | Required on Java 9+ | + +## Garden: noprompt → lambdaisland Fork + +```clojure +;; Before +[garden "1.3.10"] + +;; After +[com.lambdaisland/garden "1.9.606"] +``` + +The original `noprompt/garden` is unmaintained. On Clojure 1.12, it causes a `clojure.core/abs` shadowing warning because it only excludes `complement` from core, not `abs` (added in Clojure 1.11). + +The [lambdaisland/garden](https://github.com/lambdaisland/garden) fork fixes this by adding `abs` to `:refer-clojure :exclude`. It's a **drop-in replacement** — same namespaces, same API, same `lein-garden` plugin compatibility. + +## Test Dependencies + +| Library | Before | After | Notes | +|---------|--------|-------|-------| +| `org.clojars.favila/datomock` | — | 0.2.2-favila1 | Replaces `vvvvalvalval/datomock` (Datomic Pro compatible) | + +## Dev Dependencies + +| Library | Before | After | +|---------|--------|-------| +| `com.bhauman/figwheel-main` | — | 0.2.20 | +| `com.bhauman/rebel-readline-cljs` | — | 0.1.4 | +| `binaryage/devtools` | old | 1.0.7 | +| `cider/piggieback` | old | 0.5.3 | +| `day8.re-frame/re-frame-10x` | old | 1.11.0 | + +## Clojure 1.11+ Core Shadowing Pattern + +Clojure 1.11 added `abs`, `parse-long`, `parse-double`, and `parse-integer` to `clojure.core`. Libraries written before 1.11 that define their own versions of these functions produce shadowing warnings on modern Clojure/ClojureScript. + +This affected two dependencies in this project: + +| Library | Shadowed functions | Fix | +|---------|-------------------|-----| +| `garden` (noprompt) | `abs` | Switched to `com.lambdaisland/garden` fork | +| `noencore` (via `cljs-http`) | `parse-long`, `parse-double` | Upgraded `cljs-http` 0.1.45 → 0.1.49 | + +**If you see similar warnings from other libraries in the future**, the fix is usually: check for a newer version that adds the functions to `:refer-clojure :exclude`, or switch to a maintained fork. + +## Build Plugins + +| Plugin | Before | After | +|--------|--------|-------| +| `lein-cljsbuild` | 1.1.7 | 1.1.7 (unchanged) | +| `lein-garden` | 0.3.0 | 0.3.0 (unchanged) | +| `clj-kondo` (via lint profile) | — | 2024.05.22 | diff --git a/docs/migration/pedestal-0.7.md b/docs/migration/pedestal-0.7.md new file mode 100644 index 000000000..22c183413 --- /dev/null +++ b/docs/migration/pedestal-0.7.md @@ -0,0 +1,73 @@ +# Pedestal 0.5.1 → 0.7.0 + +## Version Pin + +Pedestal is pinned to **0.7.0** in `project.clj`. This is intentional: + +- **0.7.0** uses Jetty 11, which is compatible with figwheel-main's Ring adapter +- **0.7.1+** and **0.8.x** use Jetty 12, which removes `ScopedHandler` — causing `NoClassDefFoundError` at startup when figwheel-main is running + +This pin stays until figwheel-main supports Jetty 12. + +## Breaking Changes + +### 1. Interceptor Maps Must Be Wrapped + +Pedestal 0.7 requires explicit interceptor coercion. Plain maps cause `AssertionError`. + +**Before** (0.5.1): +```clojure +(def db-interceptor + {:name :db-interceptor + :enter (fn [context] ...)}) +``` + +**After** (0.7.0): +```clojure +(defn db-interceptor [conn] + (interceptor/interceptor + {:name :db-interceptor + :enter (fn [context] ...)})) +``` + +Changed in: `src/clj/orcpub/pedestal.clj` — `db-interceptor` and `etag-interceptor` + +### 2. Content Security Policy (CSP) + +Pedestal 0.5.1 had no CSP support. Pedestal 0.7.0 adds a default CSP via `::http/secure-headers` with `strict-dynamic`. + +In CSP Level 3 browsers (all modern browsers), `strict-dynamic` causes the browser to **ignore** `unsafe-inline` and `unsafe-eval`. Without nonces, this breaks: +- Figwheel's `document.write()` for hot-reload script injection +- Any inline `