diff --git a/.env.example b/.env.example index c071a74..a4ca7e3 100644 --- a/.env.example +++ b/.env.example @@ -1,41 +1,49 @@ -# Java Chat Application Environment Variables +# Java Chat environment variables +# +# Copy to .env: +# cp .env.example .env # Environment Configuration # Set to 'prod' for production (hides API key details in logs) # Set to 'dev' for development (shows last 4 chars of API keys in logs) SPRING_PROFILE=dev -# API Keys (Choose one of the following) -# Option 1: GitHub Models (Free tier available) -GITHUB_TOKEN=your_github_personal_access_token - -# Option 2: OpenAI API -OPENAI_API_KEY=your_openai_api_key - -# Optional: Override default model -OPENAI_MODEL=gpt-4o-mini - -# Optional: Override base URL for API -# For GitHub Models: https://models.github.ai/inference -# For OpenAI: https://api.openai.com -OPENAI_BASE_URL=https://models.github.ai/inference - # Server Configuration PORT=8085 +# LLM providers (set one or both) +# +# GitHub Models (free tier available) +GITHUB_TOKEN= +# Optional overrides (used by Makefile + streaming SDK) +GITHUB_MODELS_BASE_URL=https://models.github.ai/inference +GITHUB_MODELS_CHAT_MODEL=gpt-5 +# +# OpenAI (optional: can be used as primary or fallback) +OPENAI_API_KEY= +# For OpenAI: https://api.openai.com +OPENAI_BASE_URL=https://api.openai.com +OPENAI_MODEL=gpt-5.2 +# OPENAI_REASONING_EFFORT=high + # Local Embedding Server (if you have one running) APP_LOCAL_EMBEDDING_ENABLED=false LOCAL_EMBEDDING_SERVER_URL=http://127.0.0.1:8088 APP_LOCAL_EMBEDDING_MODEL=text-embedding-qwen3-embedding-8b +APP_LOCAL_EMBEDDING_DIMENSIONS=4096 +APP_LOCAL_EMBEDDING_USE_HASH_WHEN_DISABLED=true -# Qdrant Vector Database (if using) +# Qdrant Vector Database (local docker-compose defaults) QDRANT_HOST=localhost -QDRANT_PORT=8086 +QDRANT_SSL=false +QDRANT_PORT=8086 # gRPC (app) +QDRANT_REST_PORT=8087 # REST (scripts/monitors) QDRANT_COLLECTION=java-chat +QDRANT_API_KEY= # RAG Configuration RAG_CHUNK_MAX_TOKENS=900 RAG_CHUNK_OVERLAP_TOKENS=150 RAG_TOP_K=12 RAG_RETURN_K=6 -RAG_CITATIONS_K=3 \ No newline at end of file +RAG_CITATIONS_K=3 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..3bcc23f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,70 @@ +name: Build & Test + +on: + push: + branches: + - main + - dev + pull_request: + branches: + - main + - dev + +jobs: + build: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Set up Temurin JDK + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 25 + cache: gradle + + - name: Log system & Java info (drift detection) + run: | + echo "=== Environment Info ===" + uname -a + echo "=== Java Version ===" + java -version + echo "=== Gradle Version ===" + ./gradlew --version + + - name: Run tests + run: ./gradlew test --no-daemon + + - name: Build application + run: ./gradlew build -x test --no-daemon + + - name: Run static analysis + run: ./gradlew spotbugsMain pmdMain --no-daemon + + - name: Upload test reports on failure + if: failure() + uses: actions/upload-artifact@v6 + with: + name: test-results + path: build/test-results/ + retention-days: 7 + + - name: Upload SpotBugs report on failure + if: failure() + uses: actions/upload-artifact@v6 + with: + name: spotbugs-report + path: build/reports/spotbugs/ + retention-days: 7 + + - name: Upload PMD report on failure + if: failure() + uses: actions/upload-artifact@v6 + with: + name: pmd-report + path: build/reports/pmd/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index 442f79a..b8be761 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,9 @@ build/ frontend/node_modules/ frontend/.svelte-kit/ +# Local scratchpad (non-source artifacts) +tmp/ + # Built frontend assets (generated by npm run build) # Favicons in static/ are source files and should be committed src/main/resources/static/index.html diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..b9e9639 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: + - repo: local + hooks: + - id: spotless-java + name: Spotless Java (Gradle, auto-format) + entry: ./gradlew spotlessApply --no-daemon + language: system + pass_filenames: false + - id: frontend-check + name: Frontend check (Vite/Svelte) + entry: bash -lc "cd frontend && npm run check" + language: system + pass_filenames: false diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..35cfcf0 --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +java = temurin 25 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ed03508 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,41 @@ +# Contributing to Java Chat + +Feedback and contributions are welcome. If you find a bug or want a feature, please open an issue in this repository. + +## Getting started + +1) Fork the repository +2) Clone your fork +3) Create a feature branch +4) Make your changes +5) Run checks locally: + +```bash +make test +make build +make lint +``` + +6) Commit with a descriptive message +7) Push your branch and open a Pull Request + +## Guidelines + +- Use the deterministic toolchain: Gradle Wrapper + Gradle Toolchains + Temurin. +- Local Java is managed by mise/asdf via `.tool-versions` (see below). The CI uses the same vendor. +- We pin major Java version in Gradle toolchain and CI; patch is logged in CI and bumped intentionally. + +- Keep PRs focused (one change per PR when possible). +- Add tests for new behavior. +- Update docs when you change workflows or endpoints. +- Don’t commit secrets (use `.env`, and keep `.env.example` up to date). + +## Reporting issues + +When reporting an issue, please include: + +- Clear description of the problem +- Steps to reproduce +- Expected vs actual behavior +- OS, Java version, and how you’re running the app (`make dev`, `make run`, etc.) + diff --git a/Dockerfile b/Dockerfile index 48db660..4ba899c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN npm run build # ================================ # BACKEND BUILD STAGE # ================================ -FROM public.ecr.aws/docker/library/eclipse-temurin:21-jdk AS builder +FROM public.ecr.aws/docker/library/eclipse-temurin:25-jdk AS builder WORKDIR /app # Copy Gradle wrapper, build files, and static analysis configs @@ -51,7 +51,7 @@ RUN ./gradlew clean build -x test --no-daemon && \ # ================================ # RUNTIME STAGE # ================================ -FROM public.ecr.aws/docker/library/eclipse-temurin:21-jre AS runtime +FROM public.ecr.aws/docker/library/eclipse-temurin:25-jre AS runtime # Install curl for health check RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4cb3ebc --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +Copyright © 2026, William Callahan. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +No licensee or downstream recipient may use the Software (including any modified or derivative versions) to directly compete with the original Licensor by offering it to third parties as a hosted, managed, or Software-as-a-Service (SaaS) product or cloud service where the primary value of the service is the functionality of the Software itself. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/Makefile b/Makefile index 0b644ed..45083c9 100644 --- a/Makefile +++ b/Makefile @@ -21,19 +21,9 @@ export RED GREEN YELLOW CYAN NC export PROJECT_ROOT := $(shell pwd) export JAR_PATH = $(call get_jar) +.PHONY: all help clean build test lint format hooks run dev dev-backend compose-up compose-down compose-logs compose-ps health ingest citations fetch-all process-all full-pipeline frontend-install frontend-build -# Runtime arguments mapped from GitHub Models env vars -# - Requires GITHUB_TOKEN (PAT with models:read) -# - Base URL and model names have sensible defaults -# - CRITICAL: GitHub Models endpoint is https://models.github.ai/inference (NOT azure.com) -# - Model names differ by provider: GitHub Models uses gpt-5, OpenAI uses gpt-5.2 -RUN_ARGS := \ - --spring.ai.openai.api-key="$$GITHUB_TOKEN" \ - --spring.ai.openai.base-url="$${GITHUB_MODELS_BASE_URL:-https://models.github.ai/inference}" \ - --spring.ai.openai.chat.options.model="$${GITHUB_MODELS_CHAT_MODEL:-gpt-5}" \ - --spring.ai.openai.embedding.options.model="$${GITHUB_MODELS_EMBED_MODEL:-text-embedding-3-small}" - -.PHONY: help clean build test lint run dev dev-backend compose-up compose-down compose-logs compose-ps health ingest citations fetch-all process-all full-pipeline frontend-install frontend-build +all: help ## Default target (alias) help: ## Show available targets @grep -E '^[a-zA-Z0-9_.-]+:.*?## ' Makefile | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' @@ -52,18 +42,41 @@ lint: ## Run static analysis (Java: SpotBugs + PMD, Frontend: svelte-check) $(GRADLEW) spotbugsMain pmdMain cd frontend && npm run check +format: ## Apply Java formatting (Palantir via Spotless) + $(GRADLEW) spotlessApply + +hooks: ## Install git hooks via prek + @command -v prek >/dev/null 2>&1 || { echo "Error: 'prek' not found. Install it first: https://prek.j178.dev/" >&2; exit 1; } + prek install --install-hooks + run: build ## Run the packaged jar (loads .env if present) @if [ -f .env ]; then set -a; source .env; set +a; fi; \ - [ -n "$$GITHUB_TOKEN" ] || (echo "ERROR: GITHUB_TOKEN is not set. See README for setup." >&2; exit 1); \ + if [ -z "$$GITHUB_TOKEN" ] && [ -z "$$OPENAI_API_KEY" ]; then \ + echo "ERROR: Set GITHUB_TOKEN or OPENAI_API_KEY. See README and docs/configuration.md." >&2; \ + exit 1; \ + fi; \ SERVER_PORT=$${PORT:-$${port:-8085}}; \ if [ $$SERVER_PORT -lt 8085 ] || [ $$SERVER_PORT -gt 8090 ]; then echo "Requested port $$SERVER_PORT is outside allowed range 8085-8090; using 8085" >&2; SERVER_PORT=8085; fi; \ echo "Ensuring port $$SERVER_PORT is free..." >&2; \ PIDS=$$(lsof -ti tcp:$$SERVER_PORT 2>/dev/null || true); echo "Found PIDs on port $$SERVER_PORT: '$$PIDS'" >&2; if [ -n "$$PIDS" ]; then echo "Killing process(es) on port $$SERVER_PORT: $$PIDS" >&2; kill -9 $$PIDS 2>/dev/null || true; sleep 2; fi; \ echo "Binding app to port $$SERVER_PORT" >&2; \ + APP_ARGS=(--server.port=$$SERVER_PORT); \ + if [ -n "$$GITHUB_TOKEN" ]; then \ + APP_ARGS+=( \ + --spring.ai.openai.api-key="$$GITHUB_TOKEN" \ + --spring.ai.openai.base-url="$${GITHUB_MODELS_BASE_URL:-https://models.github.ai/inference}" \ + --spring.ai.openai.chat.options.model="$${GITHUB_MODELS_CHAT_MODEL:-gpt-5}" \ + --spring.ai.openai.embedding.options.model="$${GITHUB_MODELS_EMBED_MODEL:-text-embedding-3-small}" \ + ); \ + elif [ -n "$$OPENAI_API_KEY" ]; then \ + APP_ARGS+=( \ + --spring.ai.openai.api-key="$$OPENAI_API_KEY" \ + ); \ + fi; \ # Add conservative JVM memory limits to prevent OS-level SIGKILL (exit 137) under memory pressure # Tuned for local dev: override via JAVA_OPTS env if needed JAVA_OPTS="$${JAVA_OPTS:- -XX:+IgnoreUnrecognizedVMOptions -Xms512m -Xmx1g -XX:+UseG1GC -XX:MaxRAMPercentage=70 -XX:MaxDirectMemorySize=256m -Dio.netty.handler.ssl.noOpenSsl=true -Dio.grpc.netty.shaded.io.netty.handler.ssl.noOpenSsl=true}"; \ - java $$JAVA_OPTS -Djava.net.preferIPv4Stack=true -jar $(call get_jar) --server.port=$$SERVER_PORT $(RUN_ARGS) & disown + java $$JAVA_OPTS -Djava.net.preferIPv4Stack=true -jar $(call get_jar) "$${APP_ARGS[@]}" & disown dev: frontend-build ## Start both Spring Boot and Vite dev servers (Ctrl+C stops both) @echo "$(YELLOW)Starting full-stack development environment...$(NC)" @@ -71,19 +84,38 @@ dev: frontend-build ## Start both Spring Boot and Vite dev servers (Ctrl+C stops @echo "$(YELLOW)Backend API: http://localhost:8085/api/$(NC)" @echo "" @if [ -f .env ]; then set -a; source .env; set +a; fi; \ - [ -n "$$GITHUB_TOKEN" ] || (echo "ERROR: GITHUB_TOKEN is not set. See README for setup." >&2; exit 1); \ + if [ -z "$$GITHUB_TOKEN" ] && [ -z "$$OPENAI_API_KEY" ]; then \ + echo "ERROR: Set GITHUB_TOKEN or OPENAI_API_KEY. See README and docs/configuration.md." >&2; \ + exit 1; \ + fi; \ trap 'kill 0' INT TERM; \ (cd frontend && npm run dev 2>&1 | awk '{print "\033[36m[vite]\033[0m " $$0; fflush()}') & \ (if [ -f .env ]; then set -a; source .env; set +a; fi; \ + APP_ARGS=(--server.port=8085); \ + if [ -n "$$GITHUB_TOKEN" ]; then \ + APP_ARGS+=( \ + --spring.ai.openai.api-key="$$GITHUB_TOKEN" \ + --spring.ai.openai.base-url="$${GITHUB_MODELS_BASE_URL:-https://models.github.ai/inference}" \ + --spring.ai.openai.chat.options.model="$${GITHUB_MODELS_CHAT_MODEL:-gpt-5}" \ + --spring.ai.openai.embedding.options.model="$${GITHUB_MODELS_EMBED_MODEL:-text-embedding-3-small}" \ + ); \ + elif [ -n "$$OPENAI_API_KEY" ]; then \ + APP_ARGS+=( \ + --spring.ai.openai.api-key="$$OPENAI_API_KEY" \ + ); \ + fi; \ SPRING_PROFILES_ACTIVE=dev $(GRADLEW) bootRun \ - --args="--server.port=8085 $(RUN_ARGS)" \ + --args="$${APP_ARGS[*]}" \ -Dorg.gradle.jvmargs="-Xmx2g -Dspring.devtools.restart.enabled=true -Djava.net.preferIPv4Stack=true -Dio.netty.handler.ssl.noOpenSsl=true -Dio.grpc.netty.shaded.io.netty.handler.ssl.noOpenSsl=true" 2>&1 \ | awk '{print "\033[33m[java]\033[0m " $$0; fflush()}') & \ wait dev-backend: ## Run only Spring Boot backend (dev profile) @if [ -f .env ]; then set -a; source .env; set +a; fi; \ - [ -n "$$GITHUB_TOKEN" ] || (echo "ERROR: GITHUB_TOKEN is not set. See README for setup." >&2; exit 1); \ + if [ -z "$$GITHUB_TOKEN" ] && [ -z "$$OPENAI_API_KEY" ]; then \ + echo "ERROR: Set GITHUB_TOKEN or OPENAI_API_KEY. See README and docs/configuration.md." >&2; \ + exit 1; \ + fi; \ SERVER_PORT=$${PORT:-$${port:-8085}}; \ LIVERELOAD_PORT=$${LIVERELOAD_PORT:-35730}; \ if [ $$SERVER_PORT -lt 8085 ] || [ $$SERVER_PORT -gt 8090 ]; then echo "Requested port $$SERVER_PORT is outside allowed range 8085-8090; using 8085" >&2; SERVER_PORT=8085; fi; \ @@ -93,8 +125,21 @@ dev-backend: ## Run only Spring Boot backend (dev profile) if [ -n "$$PIDS" ]; then echo "Killing process(es) on port $$port: $$PIDS" >&2; kill -9 $$PIDS 2>/dev/null || true; sleep 1; fi; \ done; \ echo "Binding app (dev) to port $$SERVER_PORT, LiveReload on $$LIVERELOAD_PORT" >&2; \ + APP_ARGS=(--server.port=$$SERVER_PORT --spring.devtools.livereload.port=$$LIVERELOAD_PORT); \ + if [ -n "$$GITHUB_TOKEN" ]; then \ + APP_ARGS+=( \ + --spring.ai.openai.api-key="$$GITHUB_TOKEN" \ + --spring.ai.openai.base-url="$${GITHUB_MODELS_BASE_URL:-https://models.github.ai/inference}" \ + --spring.ai.openai.chat.options.model="$${GITHUB_MODELS_CHAT_MODEL:-gpt-5}" \ + --spring.ai.openai.embedding.options.model="$${GITHUB_MODELS_EMBED_MODEL:-text-embedding-3-small}" \ + ); \ + elif [ -n "$$OPENAI_API_KEY" ]; then \ + APP_ARGS+=( \ + --spring.ai.openai.api-key="$$OPENAI_API_KEY" \ + ); \ + fi; \ SPRING_PROFILES_ACTIVE=dev $(GRADLEW) bootRun \ - --args="--server.port=$$SERVER_PORT --spring.devtools.livereload.port=$$LIVERELOAD_PORT $(RUN_ARGS)" \ + --args="$${APP_ARGS[*]}" \ -Dorg.gradle.jvmargs="-Xmx2g -Dspring.devtools.restart.enabled=true -Djava.net.preferIPv4Stack=true -Dio.netty.handler.ssl.noOpenSsl=true -Dio.grpc.netty.shaded.io.netty.handler.ssl.noOpenSsl=true" frontend-install: ## Install frontend dependencies diff --git a/README.md b/README.md index 3058435..772f722 100644 --- a/README.md +++ b/README.md @@ -1,474 +1,67 @@ -# Java Chat (Spring Boot, Java 21) +# Java Chat -A modern, streaming RAG chat for Java learners, grounded in Java 24/25 documentation with precise citations. Backend-only (Spring WebFlux + Spring AI + Qdrant). Uses OpenAI API with local embeddings (LM Studio) and Qdrant Cloud for vector storage. +AI-powered Java learning with **streaming answers**, **citations**, and **guided lessons** grounded in ingested documentation (RAG). -## 🚀 Latest Updates +Built with Spring Boot + WebFlux, Svelte, and Qdrant. -- **Complete Documentation Coverage**: Successfully ingested 22,756+ documents from Java 24/25 and Spring ecosystem -- **Local Embeddings**: Integrated with LM Studio using text-embedding-qwen3-embedding-8b model (4096 dimensions) -- **Qdrant Cloud Integration**: Connected to cloud-hosted vector database with 22,756+ indexed vectors -- **Consolidated Pipeline**: Single-command fetch and process pipeline with SHA-256 hash-based deduplication -- **Smart Deduplication**: Prevents redundant processing and re-uploading of documents -- **Comprehensive Documentation**: Java 24 (10,743 files), Java 25 (10,510 files), Spring AI (218 files) - - **Dual-Mode UI (Chat + Guided Learning)**: Tabbed shell (`/`) loads isolated Chat (`/chat.html`) and new Guided Learning (`/guided.html`) - - **Guided Learning (Think Java)**: Curated lessons powered by the “Think Java — 2nd Edition” PDF with lesson-scoped chat, citations, and enrichment +## Features -## Quick start - -### 🚀 One-Command Setup (Recommended) -```bash -# Complete setup: fetch all docs and process to Qdrant -make full-pipeline -``` - -This single command will: -1. Fetch all Java 24/25/EA and Spring documentation (skips existing) -2. Process documents with embeddings -3. Upload to Qdrant with deduplication -4. Start the application on port 8085 - -### Manual Setup -```bash -# 1) Set env vars (example - use your real values) -# Create a .env file in repo root (see .env.example for all options): -# -# Authentication - You can use one or both: -# -# GitHub Models (free tier available): -# GITHUB_TOKEN=your_github_personal_access_token -# -# OpenAI API (separate, independent): -# OPENAI_API_KEY=sk-xxx -# -# How the app uses these: -# 1. Spring AI tries GITHUB_TOKEN first, then OPENAI_API_KEY -# 2. On auth failure, fallback tries direct OpenAI or GitHub Models -# -# Optional: Local embeddings (if using LM Studio) -# APP_LOCAL_EMBEDDING_ENABLED=true -# LOCAL_EMBEDDING_SERVER_URL=http://127.0.0.1:8088 -# APP_LOCAL_EMBEDDING_MODEL=text-embedding-qwen3-embedding-8b -# APP_LOCAL_EMBEDDING_DIMENSIONS=4096 # Note: 4096 for qwen3-embedding-8b -# -# Optional: Qdrant Cloud (for vector storage) -# QDRANT_HOST=xxx.us-west-1-0.aws.cloud.qdrant.io -# QDRANT_PORT=8086 -# QDRANT_SSL=true -# QDRANT_API_KEY=your-qdrant-api-key -# QDRANT_COLLECTION=java-chat - -# 2) Fetch documentation (checks for existing) -make fetch-all - -# 3) Process and run (auto-processes new docs) -make run -``` - -Health check: GET http://localhost:8085/actuator/health -Embeddings health: GET http://localhost:8085/api/chat/health/embeddings - -## Dual-Mode UI: Chat + Guided Learning - -The app now provides two complementary modes with a shared learning UX and formatting pipeline: - -- Chat (free-form): - - Location: `/chat.html` (also accessible via the “Chat” tab at `/`) - - Features: SSE streaming, server-side markdown, inline enrichments ({{hint}}, {{reminder}}, {{background}}, {{warning}}, {{example}}), citation pills, Prism highlighting, copy/export. - - APIs used: `/api/chat/stream`, `/api/chat/citations`, `/api/markdown/render`, `/api/chat/enrich` (alias: `/api/enrich`). - -- Guided Learning (curated): - - Location: `/guided.html` (also accessible via the “Guided Learning” tab at `/`) - - Content scope: “Think Java — 2nd Edition” PDF (mapped to `/pdfs/Think Java - 2nd Edition Book.pdf`) - - Features: lesson selector (TOC), lesson summary, book-scoped citations, enrichment cards, and an embedded chat scoped to the selected lesson. - - APIs used: `/api/guided/toc`, `/api/guided/lesson`, `/api/guided/citations`, `/api/guided/enrich`, `/api/guided/stream`. - -Frontend structure: -- `static/index.html`: tab shell only (a11y tabs + iframe loader for pages). -- `static/chat.html`: isolated Chat UI (migrated from original `index.html`). -- `static/guided.html`: Guided Learning UI. - -Guided Learning backend: -- `GET /api/guided/toc` → curated lessons (from `src/main/resources/guided/toc.json`). -- `GET /api/guided/lesson?slug=...` → lesson metadata. -- `GET /api/guided/citations?slug=...` → citations restricted to Think Java. -- `GET /api/guided/enrich?slug=...` → hints/background/reminders based on Think Java snippets. -- `POST /api/guided/stream` (SSE) → lesson-scoped chat. Body: `{ "sessionId":"guided:", "slug":"", "latest":"question" }`. - -All rendering quality is consistent across both modes: server-side markdown via `MarkdownService` preserves enrichment markers which the client rehydrates into styled blocks; spacing for paragraphs, lists, and code follows the same rules; Prism handles code highlighting. - -## Makefile (recommended) - -Common workflows are scripted via `Makefile`: - -```bash -# Discover commands -make help - -# Build / Test -make build -make test - -# Run packaged jar -make run - -# Live dev (Spring DevTools hot reload) -make dev - -# Local Qdrant via Docker Compose (optional) -make compose-up # start -make compose-logs # tail logs -make compose-ps # list services -make compose-down # stop - -# Convenience API helpers -make health -make ingest # ingest first 1000 docs -make citations # sample citations query -``` - -## Configuration - -All config is env-driven. See `src/main/resources/application.properties` for defaults. Key vars: - -### API Configuration -- `GITHUB_TOKEN`: GitHub personal access token for GitHub Models -- `OPENAI_API_KEY`: OpenAI API key (separate, independent service) -- `OPENAI_MODEL`: Model name, default `gpt-5.2` (used by all endpoints) -- `OPENAI_TEMPERATURE`: default `0.7` -- `OPENAI_BASE_URL`: Spring AI base URL (default: `https://models.github.ai/inference`) - - **CRITICAL**: Must be `https://models.github.ai/inference` for GitHub Models - - **DO NOT USE**: `models.inference.ai.azure.com` (this is a hallucinated URL) - - **DO NOT USE**: Any `azure.com` domain (we don't have Azure instances) +- Streaming chat over SSE (`/api/chat/stream`) with a final `citation` event +- Guided learning mode (`/learn`) with lesson-scoped chat (`/api/guided/*`) +- Documentation ingestion pipeline (fetch → chunk → embed → dedupe → index) +- Chunking uses JTokkit's CL100K_BASE tokenizer (GPT-3.5/4 style) for token counting +- Embedding fallbacks: local embedding server → remote/OpenAI → hash fallback -**How APIs are used:** -1. **Spring AI** (primary): Uses `OPENAI_BASE_URL` with `GITHUB_TOKEN` (preferred) or `OPENAI_API_KEY` -2. **Direct fallbacks** (on 401 auth errors): - - If `OPENAI_API_KEY` exists: Direct OpenAI API at `https://api.openai.com` - - If only `GITHUB_TOKEN` exists: GitHub Models at `https://models.github.ai/inference` (CORRECT endpoint) - -### Local Embeddings (LM Studio) -- `APP_LOCAL_EMBEDDING_ENABLED`: `true` to use local embeddings server -- `LOCAL_EMBEDDING_SERVER_URL`: URL of your local embeddings server (default: `http://127.0.0.1:8088`) -- `APP_LOCAL_EMBEDDING_DIMENSIONS`: `4096` (actual dimensions for qwen3-embedding-8b model) -- Recommended model: `text-embedding-qwen3-embedding-8b` (4096 dimensions) -- Note: LM Studio may show tokenizer warnings which are harmless - -### Qdrant Vector Database -- `QDRANT_HOST`: Cloud host (e.g., `xxx.us-west-1-0.aws.cloud.qdrant.io`) or `localhost` for Docker -- `QDRANT_PORT`: `8086` for gRPC (mapped from Docker's 6334) -- `QDRANT_API_KEY`: Your Qdrant Cloud API key (empty for local) -- `QDRANT_SSL`: `true` for cloud, `false` for local -- `QDRANT_COLLECTION`: default `java-chat` - -### Documentation Sources -- `DOCS_ROOT_URL`: default `https://docs.oracle.com/en/java/javase/24/` -- `DOCS_SNAPSHOT_DIR`: default `data/snapshots` (raw HTML) -- `DOCS_PARSED_DIR`: default `data/parsed` (chunk text) -- `DOCS_INDEX_DIR`: default `data/index` (ingest hash markers) -- **Containers**: point `DOCS_*` to a writable path (e.g., `/app/data/...`) and ensure the directories exist. +## Quick start -## Documentation Ingestion +### Prerequisites -### 🎯 Consolidated Pipeline (Recommended) +This project uses **Gradle Toolchains** with **Temurin JDK 25** and **mise** (or **asdf**) for reproducible builds. -We provide a unified pipeline that handles all documentation fetching and processing with intelligent deduplication: +#### Option 1: Using mise (recommended) ```bash -# Complete pipeline: fetch all docs and process to Qdrant -make full-pipeline - -# Or run steps separately: -make fetch-all # Fetch all documentation (checks for existing) -make process-all # Process and upload to Qdrant (deduplicates) +# Install mise if you don't have it: https://mise.jdnow.dev/ +mise install ``` -### Available Documentation -The pipeline automatically fetches and processes: -- **Java 24 API**: Complete Javadocs from docs.oracle.com (10,743 files ✅) -- **Java 25 API**: Complete Javadocs from docs.oracle.com (10,510 files ✅) -- **Spring Boot**: Full reference and API documentation (10,379 files) -- **Spring Framework**: Core Spring docs (13,342 files) -- **Spring AI**: AI/ML integration docs (218 files ✅) +#### Option 2: Using asdf -**Current Status**: Successfully indexed 22,756+ documents in Qdrant Cloud with automatic SHA-256 deduplication - -### Fetching Documentation - -#### Consolidated Fetch (Recommended) ```bash -# Fetch ALL documentation with deduplication checking -./scripts/fetch_all_docs.sh - -# Features: -# - Checks for existing documentation before fetching -# - Downloads only missing documentation -# - Creates metadata file with statistics -# - Logs all operations for debugging +# Install asdf if you don't have it: https://asdf-vm.com/ +asdf plugin add java https://github.com/halcyon/asdf-java.git +asdf install ``` -#### Legacy Scripts (for specific needs) -```bash -# Individual fetchers if you need specific docs -./scripts/fetch_java_complete.sh # Java 24/25 Javadocs -./scripts/fetch_spring_complete.sh # Spring ecosystem only -``` +**What happens**: Gradle Toolchains will auto-download Temurin JDK 25 on first build if not present locally. The `mise`/`asdf` setup ensures your shell and IDE (IntelliJ) use the correct Java version. -### Processing and Uploading to Qdrant +### Running -#### Consolidated Processing (Recommended) ```bash -# Process all documentation with deduplication -./scripts/process_all_to_qdrant.sh - -# Features: -# - SHA-256 hash-based deduplication -# - Tracks processed files in hash database -# - Prevents redundant embedding generation -# - Prevents duplicate uploads to Qdrant -# - Shows real-time progress -# - Generates processing statistics +cp .env.example .env +# edit .env and set GITHUB_TOKEN or OPENAI_API_KEY +make compose-up # optional local Qdrant +make dev ``` -#### Important Usage Notes - -**Resumable Processing**: The script is designed to handle interruptions gracefully: -- If the connection is lost or the process is killed, simply re-run the script -- It will automatically skip all previously indexed documents (via hash markers in `data/index/`) -- Progress is preserved in Qdrant - vectors are never lost -- Each successful chunk creates a persistent marker file +Open `http://localhost:8085/`. -**How Resume Works**: -1. **Hash Markers**: Each successfully indexed chunk creates a file in `data/index/` named with its SHA-256 hash -2. **On Restart**: The system checks for existing hash files before processing any chunk -3. **Skip Logic**: If `data/index/{hash}` exists, the chunk is skipped (already in Qdrant) -4. **Atomic Operations**: Markers are only created AFTER successful Qdrant insertion +## Index documentation (RAG) -**Monitoring Progress**: ```bash -# Check current vector count in Qdrant -source .env && curl -s -H "api-key: $QDRANT_API_KEY" \ - "https://$QDRANT_HOST/collections/$QDRANT_COLLECTION" | \ - grep -o '"points_count":[0-9]*' | cut -d: -f2 - -# Count processed chunks (hash markers) -ls data/index/ | wc -l - -# Monitor real-time progress (create monitor_progress.sh) -#!/bin/bash -source .env -while true; do - count=$(curl -s -H "api-key: $QDRANT_API_KEY" \ - "https://$QDRANT_HOST/collections/$QDRANT_COLLECTION" | \ - grep -o '"points_count":[0-9]*' | cut -d: -f2) - echo -ne "\r[$(date +%H:%M:%S)] Vectors in Qdrant: $count" - sleep 5 -done -``` - -**Performance Notes**: -- Local embeddings (LM Studio) process ~35-40 vectors/minute -- Full indexing of 60,000 documents takes ~24-30 hours -- The script has NO timeout - it will run until completion -- Safe to run multiple times - deduplication prevents any redundant work - -#### Manual Ingestion (if needed) -```bash -# The application automatically processes docs on startup -make run # Starts app and processes any new documents - -# Or trigger manual ingestion via API -curl -X POST "http://localhost:8085/api/ingest/local?path=data/docs&maxFiles=10000" +make full-pipeline ``` -### Deduplication & Quality - -#### How Deduplication Works -1. **Content Hashing**: Each document chunk gets a SHA-256 hash based on `url + chunkIndex + content` -2. **Hash Database**: Processed files are tracked in `data/.processed_hashes.db` -3. **Vector Store Check**: Before uploading, checks if hash already exists in Qdrant -4. **Skip Redundant Work**: Prevents: - - Re-downloading existing documentation - - Re-processing already embedded documents - - Duplicate vectors in Qdrant - -#### Quality Features -- **Smart chunking**: ~900 tokens with 150 token overlap for context preservation -- **Metadata enrichment**: URL, title, package name, chunk index for precise citations -- **Idempotent operations**: Safe to run multiple times without side effects -- **Automatic retries**: Handles network failures gracefully - -## Chat API (streaming) - -- POST `/api/chat/stream` (SSE) - - Body: `{ "sessionId": "s1", "latest": "How do I use records?" }` - - Streams text tokens; on completion, stores the assistant response in session memory. - -- GET `/api/chat/citations?q=your+query` - - Returns top citations (URL, title, snippet) for the query. - -- GET `/api/chat/export/last?sessionId=s1` - - Returns the last assistant message (markdown). - -- GET `/api/chat/export/session?sessionId=s1` - - Returns the full session conversation as markdown. - -## Retrieval & quality - -- Chunking: ~900 tokens with 150 overlap (CL100K_BASE tokenizer via JTokkit). -- Vector search: Qdrant similarity. Next steps: enable hybrid (BM25 + vector) and MMR diversity. -- Re-ranker: planned BGE reranker (DJL) or LLM rerank for top-k. Citations pinned to top-3 by score. - -## Citations & learning UX - -Responses are grounded with citations and “background tooltips”: -- Citation metadata: `package/module`, `JDK version`, `resource/framework + version`, `URL`, `title`. -- Background: tooltips with bigger-picture context, hints, and reminders to aid understanding. +See `docs/ingestion.md`. -Data structures (server): -- Citation: `{ url, title, anchor, snippet }` (see `com.williamcallahan.javachat.model.Citation`). -- TODO: `Enrichment` payload with fields: `packageName`, `jdkVersion`, `resource`, `resourceVersion`, `hints[]`, `reminders[]`, `background[]`. - - Guided: `GuidedLesson` `{ slug, title, summary, keywords[] }` + TOC from `src/main/resources/guided/toc.json`. +## Documentation -UI (server-rendered static placeholder): -- Return JSON with `citations` and `enrichment`. The client should render: - - Compact “source pills” with domain icon, title, and external-link affordance (open in new tab). - - Hover tooltips for background context (multi-paragraph allowed, markdown-safe). - - Clear, modern layout (Shadcn-inspired). Future: SPA frontend if needed. +Start with `docs/README.md`. -Modes & objectives: -- Chat: fast, accurate answers with layered insights and citations. -- Guided: structured progression through core topics with the same learning affordances, plus lesson-focused chat to deepen understanding. +## Contributing -## Models & Architecture +See `CONTRIBUTING.md`. -### Chat Model -- **OpenAI Java SDK (standardized)**: All streaming and non-streaming chat uses `OpenAIStreamingService` - - ✅ Official SDK streaming, no manual SSE parsing - - ✅ Prompt truncation for GPT‑5 context window (~400K tokens, 128K max output) handled centrally - - ✅ Clean, reliable streaming and consolidated error handling +## License -### Legacy Deletions -- Removed `ResilientApiClient` and all manual SSE parsing -- Controllers (`ChatController`, `GuidedLearningController`) stream via SDK only - -### Service Responsibilities -- `OpenAIStreamingService`: streaming + complete() helper -- `ChatService`: builds prompts (RAG-aware); may stream via SDK for internal flows -- `EnrichmentService` / `RerankerService`: use SDK `complete()` for JSON/ordering -- Session memory management for context preservation - -### Embeddings -- **Local LM Studio**: `text-embedding-qwen3-embedding-8b` (4096 dimensions) - - Running on Apple Silicon for fast, private embeddings - - No external API calls for document processing - - Server running at http://127.0.0.1:8088 (configurable) -- **Fallback**: OpenAI `text-embedding-3-small` if local server unavailable -- **Status**: ✅ Healthy and operational - -### Vector Search & RAG -- **Qdrant Cloud**: High-performance HNSW vector search - - Collection: `java-chat` with 22,756+ vectors - - Dimensions: 4096 (matching local embedding model) - - Connected via gRPC on port 8086 (mapped from container's 6334) with SSL -- **Smart Retrieval**: - - Top-K similarity search with configurable K (default: 12) - - MMR (Maximum Marginal Relevance) for result diversity - - TF-IDF reranking for relevance optimization -- **Citation System**: Top 3 sources with snippets and metadata - -## Maintenance - -- Re-ingesting docs: rerun `/api/ingest?maxPages=...` after a docs update. -- Qdrant housekeeping: snapshot/backup via Qdrant Cloud; set collection to HNSW + MMR/hybrid as needed. -- Env changes: restart app to pick up new model names or hosts. -- Logs/metrics: Spring Boot Actuator endpoints enabled for health/info/metrics. - - Observability TODO: add tracing and custom metrics (query time, tokens, hit rates). - -### Troubleshooting - -#### Qdrant Cloud -- Error `Invalid host or port` or `Expected closing bracket for IPv6 address`: - - Ensure `QDRANT_HOST` has no `https://` prefix; it must be the hostname only. - - Ensure `QDRANT_PORT=6334` and `QDRANT_SSL=true`. - - Makefile forces IPv4 (`-Djava.net.preferIPv4Stack=true`) to avoid macOS IPv6 resolver quirks. -- Dimension mismatch errors: - - Ensure `APP_LOCAL_EMBEDDING_DIMENSIONS=4096` matches your embedding model - - Delete and recreate Qdrant collection if dimensions change -- LM Studio tokenizer warnings: - - "[WARNING] At least one last token in strings embedded is not SEP" is harmless - -#### Rate Limiting -- **GitHub Models API**: ~15 requests/minute free tier. Set both `GITHUB_TOKEN` and `OPENAI_API_KEY` for automatic fallback. -- **Built-in retry**: 5 attempts with exponential backoff (2s → 30s max). Configurable via `AI_RETRY_*` env vars. -- **Fallback behavior**: On 429 errors, automatically switches to OpenAI API if `OPENAI_API_KEY` is available. - -## Roadmap - -- [ ] Hybrid retrieval (BM25 + vector), MMR, and re-ranker integration. -- [ ] Enrichment payload + endpoint for tooltips/hints/reminders with package/JDK metadata. -- [ ] Content hashing + upsert-by-hash for dedup and change detection. -- [ ] Minimal SPA with modern source pills, tooltips, and copy actions. -- [ ] Persist user chats + embeddings (future, configurable). - - [ ] Slash-commands (/search, /explain, /example) with semantic routing. - - [ ] Per-session rate limiting. - - [ ] DigitalOcean Spaces S3 offload for snapshots & parsed text. - - [ ] Docker Compose app service + optional local embedding model. -## 📱 Mobile Responsive Design - -The Java Chat application is fully optimized for mobile devices with comprehensive responsive design and mobile-specific safety measures. - -### Mobile Features -- **Full-width chat containers** on mobile with comfortable margins -- **16px minimum font size** on all inputs to prevent iOS Safari zoom -- **Enhanced touch targets** (44px minimum) for all interactive elements -- **Touch-optimized scrolling** with momentum scrolling support -- **Safe area insets** for devices with notches (iPhone X+) -- **Zoom prevention** on double-tap for chat areas -- **Horizontal scroll prevention** with proper text wrapping -- **Improved focus visibility** for keyboard navigation -- **Reduced motion support** for accessibility preferences - -### Mobile Breakpoints -- **Mobile**: ≤768px - Full mobile optimization -- **Tablet**: 769px-1024px - Intermediate responsive layout -- **Desktop**: >1024px - Full desktop experience - -### Mobile Safety Measures -- **Viewport Configuration**: Prevents unwanted zooming and ensures proper scaling -- **Text Size Adjustment**: Prevents browser text inflation on mobile -- **Touch Action Optimization**: Improves touch responsiveness and prevents conflicts -- **Performance Optimizations**: CSS containment and will-change for smooth animations -- **Accessibility**: Respects `prefers-reduced-motion` for users with motion sensitivity - -### Mobile Testing Checklist -- ✅ iOS Safari (iPhone/iPad) -- ✅ Chrome Mobile (Android) -- ✅ Samsung Internet -- ✅ Firefox Mobile -- ✅ Edge Mobile - -### Things to Avoid (Mobile Anti-Patterns) -1. **Font sizes < 16px on inputs** - Causes iOS Safari to zoom -2. **Touch targets < 44px** - Poor accessibility and usability -3. **Fixed positioning without safe-area-insets** - Content hidden by notches -4. **Horizontal overflow** - Breaks mobile UX -5. **user-scalable=yes without maximum-scale** - Allows accidental zoom -6. **Missing touch-action: manipulation** - Slower tap response (300ms delay) -7. **Viewport units (vh/vw) without fallbacks** - Inconsistent on mobile browsers -8. **Hover-only interactions** - Inaccessible on touch devices -9. **Small click areas** - Difficult to tap accurately -10. **Ignoring prefers-reduced-motion** - Accessibility violation - -## Stack details - -- Spring Boot 3.5.5 (WebFlux, Actuator) -- Spring AI 1.0.1 (OpenAI client, VectorStore Qdrant) -- Qdrant (HNSW vector DB); `docker-compose.yml` includes a local dev service -- JSoup (HTML parsing), JTokkit (tokenization), Fastutil (utils) -- **Mobile-First CSS**: Responsive design with mobile-specific optimizations - -Docker Compose (Qdrant only, optional fallback when you outgrow the free Qdrant Cloud plan or for offline dev): -```bash -docker compose up -d -# Then set QDRANT_HOST=localhost QDRANT_PORT=8086 -``` +See `LICENSE`. diff --git a/build.gradle.kts b/build.gradle.kts index 46fb69c..e000cc7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -4,9 +4,10 @@ plugins { id("io.spring.dependency-management") version "1.1.7" id("com.github.spotbugs") version "6.4.8" id("pmd") + id("com.diffplug.spotless") version "8.1.0" } -val javaVersion = 21 +val javaVersion = 25 val springAiVersion = "1.1.2" val openaiVersion = "4.16.0" val springDotenvVersion = "5.1.0" @@ -31,6 +32,7 @@ version = "0.0.1-SNAPSHOT" java { toolchain { languageVersion = JavaLanguageVersion.of(javaVersion) + vendor = JvmVendorSpec.ADOPTIUM } } @@ -163,6 +165,14 @@ tasks.withType().configureEach { } } +spotless { + java { + target("src/main/java/**/*.java", "src/test/java/**/*.java") + palantirJavaFormat("2.85.0") + removeUnusedImports() + } +} + // Test configuration - base settings for all Test tasks tasks.withType { useJUnitPlatform() diff --git a/docker-compose-qdrant.yml b/docker-compose-qdrant.yml index 11e8c6f..93c6ef2 100644 --- a/docker-compose-qdrant.yml +++ b/docker-compose-qdrant.yml @@ -4,7 +4,7 @@ services: # Pin to v1.13.0 to match io.qdrant:client:1.13.0 from Spring AI 1.1.2 BOM # Upgrading requires Spring AI BOM update to avoid gRPC protocol mismatches # Use GitHub Container Registry to avoid Docker Hub rate limits (per DK1) - image: ghcr.io/qdrant/qdrant:v1.13.0 + image: ghcr.io/qdrant/qdrant/qdrant:v1.13.0 container_name: qdrant ports: - "8087:6333" # REST (mapped into allowed range) @@ -17,4 +17,3 @@ volumes: qdrant_storage: - diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..b7ef5d0 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,16 @@ +# Documentation + +This folder contains developer documentation for **Java Chat**. + +## Start here + +- [Getting started](getting-started.md) +- [Configuration](configuration.md) +- [Documentation ingestion (RAG indexing)](ingestion.md) +- [HTTP API (endpoints + SSE format)](api.md) +- [Architecture](architecture.md) + +## Deep dives + +- `docs/domains/` contains design notes and domain-focused documentation (e.g., markdown processing, local-store details). + diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..5db3269 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,73 @@ +# HTTP API + +Base URL (local dev): `http://localhost:8085` + +## Streaming chat (SSE) + +### POST `/api/chat/stream` + +Request body: + +```json +{ "sessionId": "s1", "latest": "How do Java records work?" } +``` + +SSE event types (see `SseConstants`): + +- `status` → `{"message":"...","details":"..."}` +- `text` → `{"text":"..."}` +- `citation` → JSON array of citations +- `error` → `{"message":"...","details":"..."}` + +Example: + +```bash +curl -N -H "Content-Type: application/json" \ + -d '{"sessionId":"s1","latest":"Explain Java records"}' \ + http://localhost:8085/api/chat/stream +``` + +### Other chat endpoints + +- GET `/api/chat/citations?q=...` +- GET `/api/chat/health/embeddings` +- POST `/api/chat/clear?sessionId=...` +- GET `/api/chat/session/validate?sessionId=...` +- GET `/api/chat/export/last?sessionId=...` +- GET `/api/chat/export/session?sessionId=...` +- GET `/api/chat/diagnostics/retrieval?q=...` + +## Guided learning + +- GET `/api/guided/toc` +- GET `/api/guided/lesson?slug=...` +- GET `/api/guided/citations?slug=...` +- GET `/api/guided/enrich?slug=...` +- GET `/api/guided/content/stream?slug=...` (SSE, raw markdown) +- GET `/api/guided/content?slug=...` (JSON) +- GET `/api/guided/content/html?slug=...` (HTML) +- POST `/api/guided/stream` (SSE; request includes `sessionId`, `slug`, `latest`) + +## Markdown rendering + +- POST `/api/markdown/render` +- POST `/api/markdown/preview` +- POST `/api/markdown/render/structured` +- GET `/api/markdown/cache/stats` +- POST `/api/markdown/cache/clear` + +## Enrichment + +- GET `/api/enrich?q=...` +- GET `/api/chat/enrich?q=...` (alias) + +## Ingestion + embeddings cache + +- POST `/api/ingest?maxPages=...` +- POST `/api/ingest/local?dir=...&maxFiles=...` +- GET `/api/embeddings-cache/stats` +- POST `/api/embeddings-cache/upload?batchSize=...` +- POST `/api/embeddings-cache/snapshot` +- POST `/api/embeddings-cache/export?filename=...` +- POST `/api/embeddings-cache/import?filename=...` + diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..4955f15 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,36 @@ +# Architecture + +Java Chat is a Java-learning assistant focused on fast streaming answers and verifiable citations from ingested documentation. + +## High-level components + +- **Frontend**: Svelte 5 + Vite (built into `src/main/resources/static/`) +- **Backend**: Spring Boot (Web + WebFlux + Actuator) +- **Streaming**: Server-Sent Events (SSE) with typed event payloads +- **Retrieval**: Spring AI `VectorStore` (Qdrant) with local fallback search +- **LLM streaming**: OpenAI Java SDK (`OpenAIStreamingService`) supporting GitHub Models and OpenAI + +## Request flow (chat) + +1) UI calls `POST /api/chat/stream` +2) Backend retrieves candidate documents from Qdrant (`RetrievalService`) +3) Documents are reranked (`RerankerService`) and converted into citations +4) Prompt is built with retrieval context (`ChatService`) +5) Response streams via SSE (`SseSupport`) and emits a final `citation` event + +## Document ingestion (RAG indexing) + +The ingestion pipeline uses: + +- `scripts/fetch_all_docs.sh` to mirror docs into `data/docs/` +- `com.williamcallahan.javachat.cli.DocumentProcessor` (Spring `cli` profile) to ingest local docs +- `LocalStoreService` to store parsed chunks (`data/parsed/`) and hash markers (`data/index/`) + +See [ingestion.md](ingestion.md) and [local store directories](domains/local-store-directories.md). + +## Related design docs + +See also: + +- [All parsing and markdown logic](domains/all-parsing-and-markdown-logic.md) +- [Adding LLM source attribution](domains/adding-llm-source-attribution.md) diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..4182ec4 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,93 @@ +# Configuration + +Java Chat is configured primarily via environment variables (loaded from `.env` by `Makefile` targets and most scripts). + +For defaults, see `src/main/resources/application.properties`. + +## Ports + +- `PORT` (default `8085`) is restricted to `8085–8090` (see `server.port` and the app’s port initializer). + +## LLM providers (streaming chat) + +Streaming uses the OpenAI Java SDK (`OpenAIStreamingService`) and supports: + +- **GitHub Models** via `GITHUB_TOKEN` +- **OpenAI** via `OPENAI_API_KEY` + +If both keys are present, the service prefers OpenAI for streaming. There is no automatic cross-provider fallback; if the preferred provider fails or is rate-limited, the error is surfaced to the client rather than silently switching providers. + +Common variables: + +- `GITHUB_TOKEN` (GitHub Models auth) +- `GITHUB_MODELS_BASE_URL` (default `https://models.github.ai/inference/v1`) +- `GITHUB_MODELS_CHAT_MODEL` (default `gpt-5`) +- `OPENAI_API_KEY` (OpenAI auth) +- `OPENAI_BASE_URL` (default `https://api.openai.com/v1`) +- `OPENAI_MODEL` (default `gpt-5.2`) +- `OPENAI_REASONING_EFFORT` (optional, GPT‑5 family) +- `OPENAI_STREAMING_REQUEST_TIMEOUT_SECONDS` (default `600`) +- `OPENAI_STREAMING_READ_TIMEOUT_SECONDS` (default `75`) + +### Provider notes + +- GitHub Models uses `https://models.github.ai/inference` (the OpenAI SDK requires `/v1`, so the default is `.../inference/v1`). +- OpenAI uses `https://api.openai.com` (the OpenAI SDK requires `/v1`; the app normalizes URLs when needed). +- Avoid `azure.com`-style endpoints unless you are explicitly running an Azure OpenAI-compatible gateway; this project does not configure Azure by default. + +### Rate limiting + +- If you hit `429` errors on GitHub Models, either wait and retry or set `OPENAI_API_KEY` as an additional provider. + +## Embeddings + +Embeddings are configured with a fallback chain (see `EmbeddingFallbackConfig`): + +1) Local embedding server (when enabled) +2) Remote OpenAI-compatible embedding provider (optional) +3) OpenAI embeddings (optional; requires `OPENAI_API_KEY`) +4) Hash-based fallback (deterministic, not semantic) + +Common variables: + +- `APP_LOCAL_EMBEDDING_ENABLED` (`true|false`) +- `LOCAL_EMBEDDING_SERVER_URL` (default `http://127.0.0.1:8088`) +- `APP_LOCAL_EMBEDDING_MODEL` (default `text-embedding-qwen3-embedding-8b`) +- `APP_LOCAL_EMBEDDING_DIMENSIONS` (default `4096`) +- `APP_LOCAL_EMBEDDING_USE_HASH_WHEN_DISABLED` (default `true`) +- `REMOTE_EMBEDDING_SERVER_URL`, `REMOTE_EMBEDDING_API_KEY`, `REMOTE_EMBEDDING_MODEL_NAME`, `REMOTE_EMBEDDING_DIMENSIONS` (optional) + +## Qdrant + +The app uses Qdrant via Spring AI’s Qdrant vector store starter. + +Common variables: + +- `QDRANT_HOST` (default `localhost`) +- `QDRANT_PORT` (gRPC; local compose maps to `8086`) +- `QDRANT_REST_PORT` (REST; local compose maps to `8087`, mainly for scripts) +- `QDRANT_API_KEY` (required for cloud; empty for local) +- `QDRANT_SSL` (`true` for cloud, `false` for local) +- `QDRANT_COLLECTION` (default `java-chat`) + +Local Qdrant: + +```bash +make compose-up +``` + +### Qdrant troubleshooting + +- `QDRANT_HOST` must be a hostname only (no `http://` or `https://` prefix). +- Local compose maps Qdrant to allowed ports: gRPC `8086`, REST `8087` (`docker-compose-qdrant.yml`). +- Some scripts use REST for health checks; set `QDRANT_REST_PORT=8087` when using local compose. + +## RAG tuning + +Common variables (see `app.rag.*` defaults in `application.properties`): + +- `RAG_CHUNK_MAX_TOKENS` +- `RAG_CHUNK_OVERLAP_TOKENS` +- `RAG_TOP_K` +- `RAG_RETURN_K` +- `RAG_CITATIONS_K` diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..9838fcb --- /dev/null +++ b/docs/development.md @@ -0,0 +1,238 @@ +# Development Guide + +## Deterministic Build Setup + +This project uses a reproducible build system across macOS/Linux local dev and CI/CD. + +### Local Development + +#### Prerequisites + +The project is configured to use **Gradle Toolchains** with **Temurin JDK 25** for build-time determinism. + +##### Option 1: Using mise (recommended) + +[mise](https://mise.jdx.dev/) is a modern version manager that reads `.tool-versions` for Java versioning. + +```bash +# Install mise (one-time) +curl https://mise.jdx.dev/install.sh | sh + +# Then, in the repo: +mise install +``` + +This sets `JAVA_HOME` correctly for your terminal, Gradle, and IntelliJ. + +##### Option 2: Using asdf + +[asdf](https://asdf-vm.com/) is a general version manager. + +```bash +# Install asdf (one-time) +git clone https://github.com/asdf-vm/asdf.git ~/.asdf +cd ~/.asdf && git checkout "$(git describe --abbrev=0 --tags)" + +# Add the Java plugin +asdf plugin add java https://github.com/halcyon/asdf-java.git + +# In the repo: +asdf install +``` + +#### How It Works + +1. **`.tool-versions` file** pins `java = temurin 25` (or your desired version). +2. **Gradle Wrapper** (`./gradlew`) pins Gradle 9.2.1. +3. **Gradle Toolchains** auto-downloads Temurin JDK 25 if missing (enabled by Foojay resolver in `settings.gradle.kts`). +4. **Result**: Consistent Java version across: + - Shell commands (`./gradlew build`, `java -version`) + - IntelliJ "Gradle JVM" setting + - IntelliJ "Project SDK" setting + +#### Verifying Setup + +```bash +# Check Java version (should be Temurin 25.x.x) +java -version + +# Verify Gradle uses correct toolchain +./gradlew --version + +# Build (auto-downloads JDK if needed) +make build +``` + +#### Git Hooks (prek) + +This repo uses a `pre-commit` configuration compatible with [prek](https://prek.j178.dev/) to catch +formatting and frontend issues before commits. The Java hook auto-formats with Spotless, so commits +may update files. Hooks are local, so each developer installs once: + +```bash +prek install --install-hooks +``` + +Or use the Makefile helper: + +```bash +make hooks +``` + +### CI/CD (GitHub Actions) + +The `.github/workflows/build.yml` workflow: + +- Runs on **ubuntu-24.04** (pinned, not `-latest`) +- Uses `actions/setup-java@v5` with `distribution: temurin` + `java-version: 25` +- Logs Java, Gradle, and OS versions for drift detection +- Uploads test/lint reports on failure + +**Key insight**: CI uses the same JDK vendor as local dev, but may differ in patch version. See [JDK Patch Versioning](#jdk-patch-versioning) below. + +### Gradle Configuration + +#### `settings.gradle.kts` (Foojay Resolver) + +```kotlin +plugins { + id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0" +} +``` + +This enables **auto-download** of Temurin JDK if missing. Without it, Gradle fails to find the toolchain. + +#### `build.gradle.kts` (Toolchain Vendor) + +```kotlin +java { + toolchain { + languageVersion = JavaLanguageVersion.of(25) + vendor = JvmVendorSpec.ADOPTIUM // Explicitly specify Temurin + } +} +``` + +The explicit vendor removes ambiguity: Gradle will prefer Temurin JDK 25 over other vendors. + +#### `gradle.properties` (Daemon & Caching) + +```properties +org.gradle.parallel=true +org.gradle.caching=true +org.gradle.daemon=true +org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 +``` + +Enables: +- Parallel task execution +- Incremental builds (faster rebuilds) +- Gradle daemon (faster commands) +- 2GB max heap for Gradle itself + +### Docker + +The `Dockerfile` uses `eclipse-temurin:25-jdk` (build) and `eclipse-temurin:25-jre` (runtime), sourced from `public.ecr.aws` (not Docker Hub). + +**Future optimization**: Pin images by SHA256 digest for byte-for-byte reproducibility (e.g., `eclipse-temurin:25-jdk@sha256:abc...`). Trade-off: maintenance burden vs. max determinism. + +--- + +## JDK Patch Versioning + +### The Limitation + +**Gradle Toolchains cannot pin patch-level JDK versions** (e.g., 25.0.3). It can only pin major version (25) + vendor (Temurin). + +Example: +- ✅ **Supported**: Java 25 + Temurin +- ❌ **NOT supported**: Java 25.0.3 + Temurin + +### Strategy + +1. **Local dev**: Patch version determined by `mise`/`asdf` + Foojay resolver. +2. **CI/CD**: GitHub Actions logs exact Java patch version: + ```text + java -version + # Output: openjdk version "25.0.3" 2024-XX-XX + ``` +3. **When to bump patch**: Patch updates to JDK are **intentional commits**. Never rely on automatic patch upgrades. + +### Intentional Patch Bumping + +When you want to upgrade from 25.0.3 → 25.0.4: + +```bash +# Update local version manager +mise use java@temurin 25.0.4 # or asdf local java temurin-25.0.4 + +# Update .tool-versions +cat .tool-versions +# java = temurin 25.0.4 + +# Verify CI picks it up +git add .tool-versions +git commit -m "Bump JDK patch: 25.0.3 → 25.0.4" +``` + +CI will log the new patch version, and you'll have an audit trail. + +--- + +## Common Commands + +### Local Development + +```bash +# Install Java (one-time) +mise install # or: asdf install + +# Build +make build + +# Test +make test + +# Static analysis (SpotBugs + PMD) +make lint + +# Run dev server (Spring Boot + Vite) +make dev + +# Run backend only +make dev-backend + +# Docker stack (Qdrant) +make compose-up +make compose-down +``` + +### Troubleshooting + +#### "gradle: command not found" +- Run `mise install` or `asdf install` to set `JAVA_HOME` +- Verify: `echo $JAVA_HOME` should point to a Temurin 25 JDK + +#### Gradle downloads JDK every time +- Ensure `settings.gradle.kts` has Foojay resolver (added in `0.8.0`) +- Check `~/.gradle/jdks/` for cached toolchains + +#### IntelliJ doesn't pick up Java version +- Open IntelliJ settings → Build, Execution, Deployment → Gradle +- Set "Gradle JVM" to "Use JAVA_HOME" +- Set "Project SDK" to Temurin 25 (or refresh if it's auto-detected) + +#### CI build fails but local works +- Check CI logs for Java version (e.g., `java -version`) +- Ensure your local patch matches: `java -version 2>&1 | grep -o '"[^"]*"'` +- If CI is on 25.0.4 and you're on 25.0.3, bump locally and re-test + +--- + +## References + +- [Gradle Toolchains](https://docs.gradle.org/current/userguide/toolchains.html) +- [Foojay Resolver Convention](https://plugins.gradle.org/plugin/org.gradle.toolchains.foojay-resolver-convention) +- [Eclipse Temurin JDK](https://adoptium.net/) +- [mise version manager](https://mise.jdx.dev/) +- [asdf version manager](https://asdf-vm.com/) diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..278aece --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,73 @@ +# Getting started + +## Prerequisites + +- Java 25 (project toolchain) +- Node.js (frontend build/dev) +- Docker (optional, for local Qdrant) +- `wget` (optional, for `make fetch-all`) + +## Quick start (dev) + +1) Create your env file: + +```bash +cp .env.example .env +``` + +2) Edit `.env` and set `GITHUB_TOKEN` (GitHub Models) or `OPENAI_API_KEY` (OpenAI). + +3) Start local Qdrant (optional but recommended for full RAG): + +```bash +make compose-up +``` + +- gRPC: `localhost:8086` (set `QDRANT_PORT=8086`) +- REST: `localhost:8087` (used by some scripts; set `QDRANT_REST_PORT=8087` if needed) + +4) Run the app in dev mode (Svelte + Spring Boot): + +```bash +make dev +``` + +Open: + +- App: `http://localhost:8085/` +- Chat: `http://localhost:8085/chat` +- Guided learning: `http://localhost:8085/learn` + +## Run packaged JAR + +Build + run the packaged Spring Boot JAR (also builds the frontend): + +```bash +make run +``` + +Health: + +```bash +make health +``` + +## Documentation ingestion (optional) + +To mirror upstream docs into `data/docs/` and index them into Qdrant: + +```bash +make full-pipeline +``` + +See [ingestion.md](ingestion.md) for details and troubleshooting. + +## Common commands + +```bash +make help +make build +make test +make lint +make dev-backend +``` diff --git a/docs/ingestion.md b/docs/ingestion.md new file mode 100644 index 0000000..232f7be --- /dev/null +++ b/docs/ingestion.md @@ -0,0 +1,76 @@ +# Documentation ingestion (RAG indexing) + +Java Chat includes scripts and a CLI profile to mirror upstream documentation into `data/docs/` and ingest it into the vector store with content-hash deduplication. + +## Pipeline overview + +1) **Fetch** documentation into `data/docs/` (HTML mirrors). +2) **Process** docs into chunks + embeddings. +3) **Deduplicate** chunks by SHA‑256 hash. +4) **Upload** embeddings to Qdrant (or cache locally). + +## Fetch docs + +Fetch all configured sources: + +```bash +make fetch-all +``` + +This runs `scripts/fetch_all_docs.sh` (requires `wget`). Source URLs live in: + +- `src/main/resources/docs-sources.properties` + +## Process + upload to Qdrant + +Run the processor: + +```bash +make process-all +``` + +This runs `scripts/process_all_to_qdrant.sh`, which: + +- Loads `.env` +- Builds the app JAR (`./gradlew buildForScripts`) +- Runs the `cli` Spring profile (`com.williamcallahan.javachat.cli.DocumentProcessor`) + +### Modes + +- Default: `--upload` (uploads to Qdrant) +- Optional: `--local-only` (caches embeddings under `data/embeddings-cache/`) + +```bash +./scripts/process_all_to_qdrant.sh --local-only +./scripts/process_all_to_qdrant.sh --upload +``` + +## Deduplication markers + +Deduplication is based on per-chunk SHA‑256 markers stored locally: + +- `data/index/` contains one file per ingested chunk hash +- `data/parsed/` contains chunk text snapshots used for local fallback search and debugging + +See [local store directories](domains/local-store-directories.md) for details. + +## Ingest via HTTP API + +Ingest a local docs directory (must be under `data/docs/`): + +```bash +curl -sS -X POST "http://localhost:8085/api/ingest/local?dir=data/docs&maxFiles=50000" +``` + +Run a small remote crawl (dev/debug): + +```bash +curl -sS -X POST "http://localhost:8085/api/ingest?maxPages=100" +``` + +## Monitoring + +There are helper scripts in `scripts/`: + +- `scripts/monitor_progress.sh` (simple log-based view) +- `scripts/monitor_indexing.sh` (dashboard view; requires `jq` and `bc`) diff --git a/frontend/src/lib/components/ChatView.svelte b/frontend/src/lib/components/ChatView.svelte index 4dd5984..d85f2c6 100644 --- a/frontend/src/lib/components/ChatView.svelte +++ b/frontend/src/lib/components/ChatView.svelte @@ -7,6 +7,7 @@ import { streamChat, type ChatMessage, type Citation } from '../services/chat' import { isNearBottom, scrollToBottom } from '../utils/scroll' import { generateSessionId } from '../utils/session' + import { createChatMessageId } from '../utils/chatMessageId' import { createStreamingState } from '../composables/createStreamingState.svelte' /** Extended message type that includes inline citations from the stream. */ @@ -19,7 +20,7 @@ let messages = $state([]) let messagesContainer: HTMLElement | null = $state(null) let shouldAutoScroll = $state(true) - let pendingCitations = $state([]) + let activeStreamingMessageId = $state(null) // Streaming state from composable (with 800ms status persistence) const streaming = createStreamingState({ statusClearDelayMs: 800 }) @@ -32,6 +33,43 @@ // Session ID for chat continuity const sessionId = generateSessionId('chat') + let hasStreamingContent = $derived.by(() => { + if (!streaming.isStreaming || !activeStreamingMessageId) return false + const activeMessage = messages.find((existingMessage) => existingMessage.messageId === activeStreamingMessageId) + return !!activeMessage?.content + }) + + function findMessageIndex(messageId: string): number { + return messages.findIndex((existingMessage) => existingMessage.messageId === messageId) + } + + function ensureAssistantMessage(messageId: string): void { + if (findMessageIndex(messageId) >= 0) return + messages = [ + ...messages, + { + messageId, + role: 'assistant', + content: '', + timestamp: Date.now() + } + ] + } + + function updateAssistantMessage(messageId: string, updater: (message: MessageWithCitations) => MessageWithCitations): void { + const targetIndex = findMessageIndex(messageId) + if (targetIndex < 0) return + + const existingMessage = messages[targetIndex] + const updatedMessage = updater(existingMessage) + + messages = [ + ...messages.slice(0, targetIndex), + updatedMessage, + ...messages.slice(targetIndex + 1) + ] + } + function checkAutoScroll() { shouldAutoScroll = isNearBottom(messagesContainer) } @@ -40,39 +78,39 @@ await scrollToBottom(messagesContainer, shouldAutoScroll) } - async function executeChatStream(userQuery: string): Promise { + async function executeChatStream(userQuery: string, assistantMessageId: string): Promise { try { await streamChat( sessionId, userQuery, (chunk) => { - streaming.appendContent(chunk) + ensureAssistantMessage(assistantMessageId) + updateAssistantMessage(assistantMessageId, (existingMessage) => ({ + ...existingMessage, + content: existingMessage.content + chunk + })) doScrollToBottom() }, { onStatus: streaming.updateStatus, onError: streaming.updateStatus, onCitations: (citations) => { - pendingCitations = citations + ensureAssistantMessage(assistantMessageId) + updateAssistantMessage(assistantMessageId, (existingMessage) => ({ + ...existingMessage, + citations + })) } } ) - - // Add completed assistant message with inline citations - messages = [...messages, { - role: 'assistant', - content: streaming.streamingContent, - timestamp: Date.now(), - citations: pendingCitations - }] } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Sorry, I encountered an error. Please try again.' - messages = [...messages, { - role: 'assistant', + ensureAssistantMessage(assistantMessageId) + updateAssistantMessage(assistantMessageId, (existingMessage) => ({ + ...existingMessage, content: errorMessage, - timestamp: Date.now(), isError: true - }] + })) } } @@ -82,24 +120,30 @@ const userQuery = message.trim() // Add user message - messages = [...messages, { - role: 'user', - content: userQuery, - timestamp: Date.now() - }] + messages = [ + ...messages, + { + messageId: createChatMessageId('chat', sessionId), + role: 'user', + content: userQuery, + timestamp: Date.now() + } + ] shouldAutoScroll = true await doScrollToBottom() // Start streaming streaming.startStream() - pendingCitations = [] + activeStreamingMessageId = createChatMessageId('chat', sessionId) try { - await executeChatStream(userQuery) + const assistantMessageId = activeStreamingMessageId + if (!assistantMessageId) return + await executeChatStream(userQuery, assistantMessageId) } finally { streaming.finishStream() - pendingCitations = [] + activeStreamingMessageId = null await doScrollToBottom() } } @@ -122,15 +166,15 @@ - {#snippet messageRenderer({ message, index })} + {#snippet messageRenderer({ message, index, isStreaming })} {@const typedMessage = message as MessageWithCitations}
- + {#if typedMessage.role === 'assistant' && typedMessage.citations && typedMessage.citations.length > 0 && !typedMessage.isError} {/if} diff --git a/frontend/src/lib/components/ChatView.test.ts b/frontend/src/lib/components/ChatView.test.ts new file mode 100644 index 0000000..958907b --- /dev/null +++ b/frontend/src/lib/components/ChatView.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { render, fireEvent } from '@testing-library/svelte' +import { tick } from 'svelte' + +const streamChatMock = vi.fn() + +vi.mock('../services/chat', async () => { + const actualChatService = await vi.importActual('../services/chat') + return { + ...actualChatService, + streamChat: streamChatMock + } +}) + +async function renderChatView() { + const ChatViewComponent = (await import('./ChatView.svelte')).default + return render(ChatViewComponent) +} + +describe('ChatView streaming stability', () => { + beforeEach(() => { + streamChatMock.mockReset() + }) + + it('keeps the assistant message DOM node stable when the stream completes', async () => { + let completeStream: () => void = () => { + throw new Error('Expected stream completion callback to be set') + } + + streamChatMock.mockImplementation(async (_sessionId, _message, onChunk, options) => { + options?.onStatus?.({ message: 'Searching', details: 'Loading sources' }) + + await Promise.resolve() + onChunk('Hello') + + await Promise.resolve() + options?.onCitations?.([{ url: 'https://example.com', title: 'Example' }]) + + return new Promise((resolve) => { + completeStream = resolve + }) + }) + + const { getByLabelText, getByRole, container, findByText } = await renderChatView() + + const inputElement = getByLabelText('Message input') as HTMLTextAreaElement + await fireEvent.input(inputElement, { target: { value: 'Hi' } }) + + const sendButton = getByRole('button', { name: 'Send message' }) + await fireEvent.click(sendButton) + + const assistantTextElement = await findByText('Hello') + await tick() + + const assistantMessageElement = assistantTextElement.closest('.message.assistant') + expect(assistantMessageElement).not.toBeNull() + + expect(container.querySelector('.message.assistant .cursor.visible')).not.toBeNull() + + completeStream() + await tick() + + const assistantTextElementAfter = await findByText('Hello') + const assistantMessageElementAfter = assistantTextElementAfter.closest('.message.assistant') + + expect(assistantMessageElementAfter).toBe(assistantMessageElement) + expect(container.querySelector('.message.assistant .cursor.visible')).toBeNull() + }) +}) diff --git a/frontend/src/lib/components/GuidedLessonChatPanel.svelte b/frontend/src/lib/components/GuidedLessonChatPanel.svelte new file mode 100644 index 0000000..2d248fb --- /dev/null +++ b/frontend/src/lib/components/GuidedLessonChatPanel.svelte @@ -0,0 +1,237 @@ + + +
+
+ + Ask about this lesson + {#if messages.length > 0} + + {/if} +
+ +
+ {#if messages.length === 0 && !isStreaming} +
+

Have questions about {lessonTitle}?

+

Ask anything about the concepts in this lesson.

+
+ {:else} + + {#snippet messageRenderer({ message, index, isStreaming })} + {@const typedMessage = message as MessageWithCitations} +
+ + {#if typedMessage.role === 'assistant' && typedMessage.citations && typedMessage.citations.length > 0 && !typedMessage.isError} + + {/if} +
+ {/snippet} +
+ {/if} +
+ + +
+ + diff --git a/frontend/src/lib/components/LearnView.svelte b/frontend/src/lib/components/LearnView.svelte index 110c9aa..c0b3a1f 100644 --- a/frontend/src/lib/components/LearnView.svelte +++ b/frontend/src/lib/components/LearnView.svelte @@ -1,16 +1,18 @@ + +{#if loaded && error} +
+ Unable to load lesson sources +
+{:else if loaded && citations.length > 0} +
+ +
+{/if} + + + diff --git a/frontend/src/lib/components/MessageBubble.svelte b/frontend/src/lib/components/MessageBubble.svelte index 6f95cf7..bfebeca 100644 --- a/frontend/src/lib/components/MessageBubble.svelte +++ b/frontend/src/lib/components/MessageBubble.svelte @@ -30,6 +30,9 @@ // Debounced to avoid flicker during streaming $effect(() => { if (renderedContent && contentEl) { + if (isStreaming) { + return cleanupHighlighter + } // Apply Java language detection before highlighting (client-side DOM operation) applyJavaLanguageDetection(contentEl) scheduleHighlight(contentEl, isStreaming) @@ -90,32 +93,34 @@
{/if} -
- -
+ {#if message.role === 'assistant'} +
+ +
+ {/if} @@ -156,6 +161,7 @@ /* Bubble */ .bubble { position: relative; + overflow: visible; max-width: 85%; padding: var(--space-4); border-radius: var(--radius-xl); @@ -313,23 +319,27 @@ 50% { opacity: 0; } } - /* Actions - capability-based visibility */ + /* Actions - desktop hover only (never on touch / narrow viewports) */ .bubble-actions { + display: none; position: absolute; - bottom: var(--space-2); - right: var(--space-2); - opacity: 1; /* Default visible for touch devices */ + top: var(--space-2); + left: calc(100% + var(--space-2)); + opacity: 0; + pointer-events: none; transition: opacity var(--duration-fast) var(--ease-out); } - /* Only use hover behavior when device supports it */ - @media (hover: hover) and (pointer: fine) { + /* Only show hover actions on wide viewports with a fine pointer (mouse/trackpad). */ + @media (hover: hover) and (pointer: fine) and (min-width: 641px) { .bubble-actions { - opacity: 0; + display: block; } - .bubble:hover .bubble-actions { + .bubble:hover .bubble-actions, + .bubble:focus-within .bubble-actions { opacity: 1; + pointer-events: auto; } } @@ -379,17 +389,6 @@ height: 14px; } - .message.user .action-btn { - background: rgba(255, 255, 255, 0.15); - border-color: rgba(255, 255, 255, 0.2); - color: rgba(255, 255, 255, 0.7); - } - - .message.user .action-btn:hover { - background: rgba(255, 255, 255, 0.25); - color: white; - } - /* Tablet */ @media (max-width: 768px) { .bubble { diff --git a/frontend/src/lib/components/MessageBubble.test.ts b/frontend/src/lib/components/MessageBubble.test.ts new file mode 100644 index 0000000..cb87fec --- /dev/null +++ b/frontend/src/lib/components/MessageBubble.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect } from 'vitest' +import { render } from '@testing-library/svelte' +import MessageBubble from './MessageBubble.svelte' + +describe('MessageBubble', () => { + it('does not render copy action for user messages', () => { + const { container } = render(MessageBubble, { + props: { + message: { messageId: 'msg-test-user', role: 'user', content: 'Hello', timestamp: 1 }, + index: 0 + } + }) + + expect(container.querySelector('.bubble-actions')).toBeNull() + }) + + it('renders copy action for assistant messages', () => { + const { container, getByRole } = render(MessageBubble, { + props: { + message: { messageId: 'msg-test-assistant', role: 'assistant', content: 'Hello', timestamp: 1 }, + index: 0 + } + }) + + expect(container.querySelector('.bubble-actions')).not.toBeNull() + expect(getByRole('button', { name: /copy message/i, hidden: true })).toBeInTheDocument() + }) +}) diff --git a/frontend/src/lib/components/MobileChatDrawer.svelte b/frontend/src/lib/components/MobileChatDrawer.svelte index d5cea0c..5aec7c3 100644 --- a/frontend/src/lib/components/MobileChatDrawer.svelte +++ b/frontend/src/lib/components/MobileChatDrawer.svelte @@ -1,4 +1,5 @@
- {#each messages as message, messageIndex (message.timestamp)} + {#each messages as message, messageIndex (message.messageId)} + {@const messageIsStreaming = isStreaming && !!streamingMessageId && message.messageId === streamingMessageId} {#if messageRenderer} - {@render messageRenderer({ message, index: messageIndex })} + {@render messageRenderer({ message, index: messageIndex, isStreaming: messageIsStreaming })} {:else} - + {/if} {/each} - {#if isStreaming && streamingContent} - - {:else if isStreaming} + {#if isStreaming && !hasContent} {/if}
diff --git a/frontend/src/lib/composables/createStreamingState.svelte.ts b/frontend/src/lib/composables/createStreamingState.svelte.ts index cc307f5..6563997 100644 --- a/frontend/src/lib/composables/createStreamingState.svelte.ts +++ b/frontend/src/lib/composables/createStreamingState.svelte.ts @@ -24,8 +24,6 @@ export interface StreamingStateOptions { export interface StreamingState { /** Whether a stream is currently active. */ readonly isStreaming: boolean - /** Accumulated content from stream chunks. */ - readonly streamingContent: string /** Current status message (e.g., "Searching...", "Done"). */ readonly statusMessage: string /** Additional status details. */ @@ -33,8 +31,6 @@ export interface StreamingState { /** Marks stream as active and resets content/status. */ startStream: () => void - /** Appends a text chunk to streaming content. */ - appendContent: (chunk: string) => void /** Updates status message and optional details. */ updateStatus: (status: StreamStatus) => void /** Marks stream as complete and schedules status clearing. */ @@ -62,7 +58,7 @@ export interface StreamingState { * streaming.startStream() * try { * await streamChat(sessionId, message, (chunk) => { - * streaming.appendContent(chunk) + * // Update the active assistant message content in your messages list. * }, { * onStatus: streaming.updateStatus * }) @@ -73,7 +69,7 @@ export interface StreamingState { * * * {#if streaming.isStreaming} - * + * * {/if} * ``` */ @@ -82,7 +78,6 @@ export function createStreamingState(options: StreamingStateOptions = {}): Strea // Internal reactive state let isStreaming = $state(false) - let streamingContent = $state('') let statusMessage = $state('') let statusDetails = $state('') @@ -121,9 +116,6 @@ export function createStreamingState(options: StreamingStateOptions = {}): Strea get isStreaming() { return isStreaming }, - get streamingContent() { - return streamingContent - }, get statusMessage() { return statusMessage }, @@ -135,15 +127,10 @@ export function createStreamingState(options: StreamingStateOptions = {}): Strea startStream() { cancelStatusTimer() isStreaming = true - streamingContent = '' statusMessage = '' statusDetails = '' }, - appendContent(chunk: string) { - streamingContent += chunk - }, - updateStatus(status: StreamStatus) { statusMessage = status.message statusDetails = status.details ?? '' @@ -151,14 +138,12 @@ export function createStreamingState(options: StreamingStateOptions = {}): Strea finishStream() { isStreaming = false - streamingContent = '' clearStatusDelayed() }, reset() { cancelStatusTimer() isStreaming = false - streamingContent = '' statusMessage = '' statusDetails = '' }, diff --git a/frontend/src/lib/services/chat.ts b/frontend/src/lib/services/chat.ts index 826a8b1..6e5fd90 100644 --- a/frontend/src/lib/services/chat.ts +++ b/frontend/src/lib/services/chat.ts @@ -11,6 +11,8 @@ import type { StreamStatus, StreamError } from './stream-types' export type { StreamStatus, StreamError } export interface ChatMessage { + /** Stable client-side identifier for rendering and list keying. */ + messageId: string role: 'user' | 'assistant' content: string timestamp: number @@ -28,6 +30,7 @@ export interface StreamChatOptions { onStatus?: (status: StreamStatus) => void onError?: (error: StreamError) => void onCitations?: (citations: Citation[]) => void + signal?: AbortSignal } /** Result type for citation fetches - distinguishes empty results from errors. */ @@ -58,10 +61,33 @@ export async function streamChat( onError: options.onError, onCitations: options.onCitations }, - 'chat.ts' + 'chat.ts', + { signal: options.signal } ) } +/** + * Clears the server-side chat memory for a session. + * + * @param sessionId - Session identifier to clear on the backend. + */ +export async function clearChatSession(sessionId: string): Promise { + const normalizedSessionId = sessionId.trim() + if (!normalizedSessionId) { + throw new Error('Session ID is required') + } + + const response = await fetch(`/api/chat/clear?sessionId=${encodeURIComponent(normalizedSessionId)}`, { + method: 'POST' + }) + + if (!response.ok) { + const errorBody = await response.text().catch(() => '') + const suffix = errorBody ? `: ${errorBody}` : '' + throw new Error(`Failed to clear chat session (HTTP ${response.status})${suffix}`) + } +} + /** * Fetch citations for a query. * Used by LearnView to fetch lesson-level citations separately from the chat stream. diff --git a/frontend/src/lib/services/guided.ts b/frontend/src/lib/services/guided.ts index 1c4125a..591a267 100644 --- a/frontend/src/lib/services/guided.ts +++ b/frontend/src/lib/services/guided.ts @@ -7,6 +7,7 @@ import { streamSse } from './sse' import type { StreamStatus } from './stream-types' +import type { Citation, CitationFetchResult } from './chat' export type { StreamStatus } @@ -29,6 +30,8 @@ export interface GuidedStreamCallbacks { onChunk: (chunk: string) => void onStatus?: (status: StreamStatus) => void onError?: (error: Error) => void + onCitations?: (citations: Citation[]) => void + signal?: AbortSignal } /** @@ -64,6 +67,24 @@ export async function fetchLessonContent(slug: string): Promise { + try { + const response = await fetch(`/api/guided/citations?slug=${encodeURIComponent(slug)}`) + if (!response.ok) { + return { success: false, error: `HTTP ${response.status}: ${response.statusText}` } + } + const citations: Citation[] = await response.json() + return { success: true, citations } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Network error fetching lesson sources' + return { success: false, error: errorMessage } + } +} + /** * Stream a chat response within the guided lesson context. * Uses the same JSON-wrapped SSE format as the main chat for consistent whitespace handling. @@ -75,7 +96,7 @@ export async function streamGuidedChat( message: string, callbacks: GuidedStreamCallbacks ): Promise { - const { onChunk, onStatus, onError } = callbacks + const { onChunk, onStatus, onError, onCitations, signal } = callbacks let errorNotified = false try { @@ -85,12 +106,14 @@ export async function streamGuidedChat( { onText: onChunk, onStatus, + onCitations, onError: (streamError) => { errorNotified = true onError?.(new Error(streamError.message)) } }, - 'guided.ts' + 'guided.ts', + { signal } ) } catch (error) { // Re-throw after invoking callback to maintain dual error propagation diff --git a/frontend/src/lib/services/markdown.test.ts b/frontend/src/lib/services/markdown.test.ts index 113b655..87fcb5a 100644 --- a/frontend/src/lib/services/markdown.test.ts +++ b/frontend/src/lib/services/markdown.test.ts @@ -9,39 +9,39 @@ describe('parseMarkdown', () => { }) it('parses basic markdown to HTML', () => { - const result = parseMarkdown('**bold** and *italic*') - expect(result).toContain('bold') - expect(result).toContain('italic') + const renderedHtml = parseMarkdown('**bold** and *italic*') + expect(renderedHtml).toContain('bold') + expect(renderedHtml).toContain('italic') }) it('parses code blocks', () => { const markdown = '```java\npublic class Test {}\n```' - const result = parseMarkdown(markdown) - expect(result).toContain('
')
-    expect(result).toContain('')
+    expect(renderedHtml).toContain(' {
     const markdown = ''
-    const result = parseMarkdown(markdown)
-    expect(result).not.toContain(''
-    const result = escapeHtml(input)
-    expect(result).not.toContain('<')
-    expect(result).not.toContain('>')
-    expect(result).toContain('<')
-    expect(result).toContain('>')
+    const escapedHtml = escapeHtml(input)
+    expect(escapedHtml).not.toContain('<')
+    expect(escapedHtml).not.toContain('>')
+    expect(escapedHtml).toContain('<')
+    expect(escapedHtml).toContain('>')
   })
 
   it('is SSR-safe - uses pure string operations', () => {
     // This works without document APIs
-    const result = escapeHtml('
') - expect(result).toBe('<div class="test">') + const escapedHtml = escapeHtml('
') + expect(escapedHtml).toBe('<div class="test">') }) }) - diff --git a/frontend/src/lib/services/sse.test.ts b/frontend/src/lib/services/sse.test.ts new file mode 100644 index 0000000..da64a44 --- /dev/null +++ b/frontend/src/lib/services/sse.test.ts @@ -0,0 +1,59 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { streamSse } from './sse' + +describe('streamSse abort handling', () => { + afterEach(() => { + vi.unstubAllGlobals() + vi.restoreAllMocks() + }) + + it('returns without invoking callbacks when fetch is aborted', async () => { + const abortController = new AbortController() + abortController.abort() + + const fetchMock = vi.fn().mockRejectedValue(Object.assign(new Error('Aborted'), { name: 'AbortError' })) + vi.stubGlobal('fetch', fetchMock) + + const onText = vi.fn() + const onError = vi.fn() + + await streamSse( + '/api/test/stream', + { hello: 'world' }, + { onText, onError }, + 'sse.test.ts', + { signal: abortController.signal } + ) + + expect(onText).not.toHaveBeenCalled() + expect(onError).not.toHaveBeenCalled() + }) + + it('treats AbortError during read as a cancellation (no onError)', async () => { + const encoder = new TextEncoder() + const abortError = Object.assign(new Error('Aborted'), { name: 'AbortError' }) + let didEnqueue = false + + const responseBody = new ReadableStream({ + pull(controller) { + if (!didEnqueue) { + didEnqueue = true + controller.enqueue(encoder.encode('data: {"text":"Hello"}\n\n')) + return + } + controller.error(abortError) + } + }) + + const fetchMock = vi.fn().mockResolvedValue({ ok: true, body: responseBody, status: 200, statusText: 'OK' }) + vi.stubGlobal('fetch', fetchMock) + + const onText = vi.fn() + const onError = vi.fn() + + await streamSse('/api/test/stream', { hello: 'world' }, { onText, onError }, 'sse.test.ts') + + expect(onText).toHaveBeenCalledWith('Hello') + expect(onError).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/lib/services/sse.ts b/frontend/src/lib/services/sse.ts index 34fd31c..50a4024 100644 --- a/frontend/src/lib/services/sse.ts +++ b/frontend/src/lib/services/sse.ts @@ -14,6 +14,11 @@ const SSE_EVENT_STATUS = 'status' const SSE_EVENT_ERROR = 'error' const SSE_EVENT_CITATION = 'citation' +/** Optional request options for streaming fetch calls. */ +export interface StreamSseRequestOptions { + signal?: AbortSignal +} + /** Callbacks for SSE stream processing. */ export interface SseCallbacks { onText: (content: string) => void @@ -22,6 +27,10 @@ export interface SseCallbacks { onCitations?: (citations: Citation[]) => void } +function isAbortError(error: unknown): boolean { + return error instanceof Error && error.name === 'AbortError' +} + /** * Attempts JSON parsing only when content looks like JSON. * Returns parsed object or null for plain text content. @@ -121,15 +130,27 @@ export async function streamSse( url: string, body: object, callbacks: SseCallbacks, - source: string + source: string, + options: StreamSseRequestOptions = {} ): Promise { - const response = await fetch(url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify(body) - }) + const abortSignal = options.signal + let response: Response + + try { + response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(body), + signal: abortSignal + }) + } catch (fetchError) { + if (abortSignal?.aborted || isAbortError(fetchError)) { + return + } + throw fetchError + } if (!response.ok) { const httpError = new Error(`HTTP ${response.status}: ${response.statusText}`) @@ -231,6 +252,11 @@ export async function streamSse( } } } + } catch (streamError) { + if (abortSignal?.aborted || isAbortError(streamError)) { + return + } + throw streamError } finally { // Cancel reader on abnormal exit to prevent dangling connections if (!streamCompletedNormally) { diff --git a/frontend/src/lib/services/stream-types.ts b/frontend/src/lib/services/stream-types.ts index 34bad13..096ff4b 100644 --- a/frontend/src/lib/services/stream-types.ts +++ b/frontend/src/lib/services/stream-types.ts @@ -26,7 +26,6 @@ export interface StreamError { */ export interface StreamingChatFields { isStreaming: boolean - streamingContent: string statusMessage: string statusDetails: string } diff --git a/frontend/src/lib/utils/chatMessageId.ts b/frontend/src/lib/utils/chatMessageId.ts new file mode 100644 index 0000000..c3b4941 --- /dev/null +++ b/frontend/src/lib/utils/chatMessageId.ts @@ -0,0 +1,37 @@ +/** + * Chat message identifier utilities. + * + * Provides stable, collision-resistant IDs for message list keying across + * chat and guided chat rendering paths. + */ + +type MessageContext = 'chat' | 'guided' + +let sequenceNumber = 0 + +function nextSequenceNumber(): number { + sequenceNumber = (sequenceNumber + 1) % 1_000_000 + return sequenceNumber +} + +function createRandomSuffix(): string { + if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') { + return crypto.randomUUID() + } + return Math.random().toString(36).slice(2, 12) +} + +/** + * Creates a stable identifier for a chat message. + * + * @param context - Message origin (main chat vs guided chat) + * @param sessionId - Stable per-view session identifier + * @returns A stable unique message identifier + */ +export function createChatMessageId(context: MessageContext, sessionId: string): string { + const timestampMs = Date.now() + const sequence = nextSequenceNumber() + const randomSuffix = createRandomSuffix() + return `msg-${context}-${sessionId}-${timestampMs}-${sequence}-${randomSuffix}` +} + diff --git a/frontend/src/lib/utils/url.test.ts b/frontend/src/lib/utils/url.test.ts index 7ab3d24..8d8dede 100644 --- a/frontend/src/lib/utils/url.test.ts +++ b/frontend/src/lib/utils/url.test.ts @@ -60,8 +60,8 @@ describe('deduplicateCitations', () => { }) it('returns empty array for null/undefined input', () => { - expect(deduplicateCitations(null as unknown as [])).toEqual([]) - expect(deduplicateCitations(undefined as unknown as [])).toEqual([]) + expect(deduplicateCitations(null)).toEqual([]) + expect(deduplicateCitations(undefined)).toEqual([]) }) it('removes duplicate URLs', () => { @@ -70,9 +70,9 @@ describe('deduplicateCitations', () => { { url: 'https://b.com', title: 'B' }, { url: 'https://a.com', title: 'A duplicate' } ] - const result = deduplicateCitations(citations) - expect(result).toHaveLength(2) - expect(result.map(c => c.url)).toEqual(['https://a.com', 'https://b.com']) + const deduplicatedCitations = deduplicateCitations(citations) + expect(deduplicatedCitations).toHaveLength(2) + expect(deduplicatedCitations.map(c => c.url)).toEqual(['https://a.com', 'https://b.com']) }) it('keeps first occurrence when deduplicating', () => { @@ -80,8 +80,8 @@ describe('deduplicateCitations', () => { { url: 'https://a.com', title: 'First' }, { url: 'https://a.com', title: 'Second' } ] - const result = deduplicateCitations(citations) - expect(result[0].title).toBe('First') + const deduplicatedCitations = deduplicateCitations(citations) + expect(deduplicatedCitations[0].title).toBe('First') }) }) diff --git a/frontend/src/lib/utils/url.ts b/frontend/src/lib/utils/url.ts index 986e3d8..e635feb 100644 --- a/frontend/src/lib/utils/url.ts +++ b/frontend/src/lib/utils/url.ts @@ -177,16 +177,11 @@ interface HasUrl { * Deduplicates an array of objects by URL (case-insensitive). * Filters out objects with missing or invalid URL properties. * - * @param citations - Array of objects with url property + * @param citations - Array of objects with url property (null/undefined treated as empty) * @returns Deduplicated array preserving original order - * @throws Warning logged if input is not an array (indicates caller bug) */ -export function deduplicateCitations(citations: T[]): T[] { - if (!Array.isArray(citations)) { - console.warn('[url.ts] deduplicateCitations received non-array input:', { - receivedType: typeof citations, - value: citations - }) +export function deduplicateCitations(citations: readonly T[] | null | undefined): T[] { + if (!citations || citations.length === 0) { return [] } const seen = new Set() diff --git a/frontend/src/test/setup.ts b/frontend/src/test/setup.ts index 6422862..4a14fc8 100644 --- a/frontend/src/test/setup.ts +++ b/frontend/src/test/setup.ts @@ -14,3 +14,14 @@ Object.defineProperty(window, 'matchMedia', { dispatchEvent: () => false }) }) + +// jsdom doesn't implement scrollTo on elements; components use it for chat auto-scroll. +Object.defineProperty(HTMLElement.prototype, 'scrollTo', { + writable: true, + value: () => {} +}) + +// requestAnimationFrame is used for post-update DOM adjustments; provide a safe fallback. +if (typeof window.requestAnimationFrame !== 'function') { + window.requestAnimationFrame = (callback: FrameRequestCallback) => window.setTimeout(() => callback(performance.now()), 0) +} diff --git a/frontend/vitest.config.ts b/frontend/vitest.config.ts index 0c35597..b1337b1 100644 --- a/frontend/vitest.config.ts +++ b/frontend/vitest.config.ts @@ -3,6 +3,12 @@ import { svelte } from '@sveltejs/vite-plugin-svelte' export default defineConfig({ plugins: [svelte({ hot: !process.env.VITEST })], + // Vitest runs modules through Vite's SSR pipeline by default, which can cause + // conditional exports to resolve Svelte's server entry (where `mount()` is unavailable). + // Force browser conditions so component tests can mount under jsdom. + resolve: { + conditions: ['module', 'browser', 'development'] + }, test: { environment: 'jsdom', globals: true, diff --git a/scripts/docs_sources.sh b/scripts/docs_sources.sh deleted file mode 100644 index 23c26c5..0000000 --- a/scripts/docs_sources.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# Centralized documentation source URLs for fetching and cross-referencing - -# Oracle Java SE 24 API -export JAVA24_API_BASE="https://docs.oracle.com/en/java/javase/24/docs/api/" - -# Oracle Java SE 25 API -export JAVA25_API_BASE="https://docs.oracle.com/en/java/javase/25/docs/api/" - -# Spring ecosystem -export SPRING_BOOT_API_BASE="https://docs.spring.io/spring-boot/docs/current/api/" -export SPRING_FRAMEWORK_API_BASE="https://docs.spring.io/spring-framework/docs/current/javadoc-api/" -export SPRING_AI_API_BASE="https://docs.spring.io/spring-ai/reference/1.0/api/" - diff --git a/scripts/fetch_all_docs.sh b/scripts/fetch_all_docs.sh index 3c55f44..406f564 100755 --- a/scripts/fetch_all_docs.sh +++ b/scripts/fetch_all_docs.sh @@ -7,12 +7,7 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Optional: centralized source URLs -if [ -f "$SCRIPT_DIR/docs_sources.sh" ]; then - # shellcheck source=/dev/null - . "$SCRIPT_DIR/docs_sources.sh" -fi -# Also allow sourcing from the Java resources properties to keep a single source of truth +# Centralized source URLs (single source of truth) RES_PROPS="$SCRIPT_DIR/../src/main/resources/docs-sources.properties" if [ -f "$RES_PROPS" ]; then # Export variables defined as KEY=VALUE in the properties file diff --git a/scripts/monitor_indexing.sh b/scripts/monitor_indexing.sh index 8f12d69..ef93476 100755 --- a/scripts/monitor_indexing.sh +++ b/scripts/monitor_indexing.sh @@ -23,15 +23,29 @@ if [ -f "$PROJECT_ROOT/.env" ]; then set +a fi -# Configuration -QDRANT_URL="https://$QDRANT_HOST/collections/$QDRANT_COLLECTION" +# Configuration (use REST API) +QDRANT_PROTOCOL="https" +QDRANT_BASE_URL="" +if [ "${QDRANT_SSL:-false}" = "true" ] || [ "${QDRANT_SSL:-false}" = "1" ]; then + QDRANT_REST_PORT="${QDRANT_REST_PORT:-8087}" + QDRANT_BASE_URL="${QDRANT_PROTOCOL}://${QDRANT_HOST}:${QDRANT_REST_PORT}" +else + QDRANT_PROTOCOL="http" + QDRANT_REST_PORT="${QDRANT_REST_PORT:-8087}" + QDRANT_BASE_URL="${QDRANT_PROTOCOL}://${QDRANT_HOST}:${QDRANT_REST_PORT}" +fi +QDRANT_URL="${QDRANT_BASE_URL}/collections/${QDRANT_COLLECTION}" LOG_FILE="$PROJECT_ROOT/process_qdrant.log" REFRESH_INTERVAL=${1:-5} # Default 5 seconds, or pass as argument # Function to get Qdrant stats get_qdrant_stats() { - local response=$(curl -s -H "api-key: $QDRANT_API_KEY" "$QDRANT_URL" 2>/dev/null) - if [ $? -eq 0 ]; then + local auth=() + if [ -n "${QDRANT_API_KEY:-}" ]; then + auth=( -H "api-key: $QDRANT_API_KEY" ) + fi + local response + if response=$(curl -s "${auth[@]}" "$QDRANT_URL" 2>/dev/null); then echo "$response" | jq -r '.result | "\(.points_count)|\(.vectors_count)|\(.indexed_vectors_count)"' 2>/dev/null || echo "0|0|0" else echo "0|0|0" @@ -186,4 +200,4 @@ while true; do LAST_UPDATE_TIME=$CURRENT_TIME sleep $REFRESH_INTERVAL -done \ No newline at end of file +done diff --git a/scripts/monitor_progress.sh b/scripts/monitor_progress.sh index 86f3bcf..f38fbfa 100755 --- a/scripts/monitor_progress.sh +++ b/scripts/monitor_progress.sh @@ -3,10 +3,26 @@ source .env while true; do - count=$(curl -s -H "api-key: $QDRANT_API_KEY" "https://$QDRANT_HOST/collections/$QDRANT_COLLECTION" | grep -o '"points_count":[0-9]*' | cut -d: -f2) + protocol="https" + base_url="" + if [ "${QDRANT_SSL:-false}" = "true" ] || [ "${QDRANT_SSL:-false}" = "1" ]; then + rest_port="${QDRANT_REST_PORT:-8087}" + base_url="${protocol}://${QDRANT_HOST}:${rest_port}" + else + protocol="http" + rest_port="${QDRANT_REST_PORT:-8087}" + base_url="${protocol}://${QDRANT_HOST}:${rest_port}" + fi + + auth=() + if [ -n "${QDRANT_API_KEY:-}" ]; then + auth=(-H "api-key: $QDRANT_API_KEY") + fi + + count=$(curl -s "${auth[@]}" "${base_url}/collections/${QDRANT_COLLECTION}" | grep -o '"points_count":[0-9]*' | cut -d: -f2) embedding_calls=$(grep -c "EMBEDDING.*Calling API" process_qdrant.log) last_file=$(grep "Processing file:" process_qdrant.log | tail -1 | cut -d: -f2) echo -ne "\r[$(date +%H:%M:%S)] Vectors: $count | Embedding calls: $embedding_calls | Last: $last_file " sleep 5 -done \ No newline at end of file +done diff --git a/scripts/process_all_to_qdrant.sh b/scripts/process_all_to_qdrant.sh index b780d25..adb8d2e 100755 --- a/scripts/process_all_to_qdrant.sh +++ b/scripts/process_all_to_qdrant.sh @@ -73,7 +73,7 @@ fi # Verify required environment variables based on mode if [ "$UPLOAD_MODE" = "upload" ]; then - required_vars=("QDRANT_HOST" "QDRANT_PORT" "QDRANT_API_KEY" "QDRANT_COLLECTION" "APP_LOCAL_EMBEDDING_ENABLED") + required_vars=("QDRANT_HOST" "QDRANT_PORT" "QDRANT_COLLECTION" "APP_LOCAL_EMBEDDING_ENABLED") else # Local-only mode doesn't need Qdrant vars required_vars=("APP_LOCAL_EMBEDDING_ENABLED") @@ -137,16 +137,17 @@ check_qdrant_connection() { log "${YELLOW}Checking Qdrant connection...${NC}" - # Respect QDRANT_SSL setting for protocol selection - local protocol="https" - if [ "$QDRANT_SSL" = "false" ] || [ "$QDRANT_SSL" = "0" ] || [ -z "$QDRANT_SSL" ]; then - protocol="http" - # For local non-TLS, include the port (REST port, not gRPC) - local rest_port="${QDRANT_REST_PORT:-6333}" - local url="${protocol}://${QDRANT_HOST}:${rest_port}/collections/$QDRANT_COLLECTION" + # Use REST API for connectivity checks (not gRPC). For local docker-compose, REST is mapped to 8087. + local url + if [ "$QDRANT_SSL" = "true" ] || [ "$QDRANT_SSL" = "1" ]; then + if [ -n "${QDRANT_REST_PORT:-}" ]; then + url="https://${QDRANT_HOST}:${QDRANT_REST_PORT}/collections/$QDRANT_COLLECTION" + else + url="https://${QDRANT_HOST}/collections/$QDRANT_COLLECTION" + fi else - # For TLS (cloud), port 443 is implicit - local url="${protocol}://${QDRANT_HOST}/collections/$QDRANT_COLLECTION" + local rest_port="${QDRANT_REST_PORT:-8087}" + url="http://${QDRANT_HOST}:${rest_port}/collections/$QDRANT_COLLECTION" fi local curl_opts=(-s -o /dev/null -w "%{http_code}") @@ -174,6 +175,12 @@ check_qdrant_connection() { log "${BLUE}ℹ Current vectors: ${points:-0}${NC}" log "${BLUE}ℹ Dimensions: ${dimensions:-unknown}${NC}" return 0 + elif [ "$response" = "404" ]; then + log "${YELLOW}⚠ Qdrant reachable, but collection not found yet (HTTP 404)${NC} ($(percent_complete))" + log "${YELLOW} Collection: $QDRANT_COLLECTION${NC}" + log "${YELLOW} URL: $url${NC}" + log "${YELLOW} If schema init is enabled, the app will create the collection on first use.${NC}" + return 0 else log "${RED}✗ Failed to connect to Qdrant (HTTP $response)${NC}" log "${YELLOW} URL: $url${NC}" @@ -370,8 +377,23 @@ show_statistics() { if [ "$UPLOAD_MODE" = "upload" ]; then # Get Qdrant statistics - local url="https://$QDRANT_HOST/collections/$QDRANT_COLLECTION" - local info=$(curl -s -H "api-key: $QDRANT_API_KEY" "$url") + local base_url + if [ "${QDRANT_SSL:-false}" = "true" ] || [ "${QDRANT_SSL:-false}" = "1" ]; then + if [ -n "${QDRANT_REST_PORT:-}" ]; then + base_url="https://${QDRANT_HOST}:${QDRANT_REST_PORT}" + else + base_url="https://${QDRANT_HOST}" + fi + else + base_url="http://${QDRANT_HOST}:${QDRANT_REST_PORT:-8087}" + fi + local url="${base_url}/collections/$QDRANT_COLLECTION" + local info_opts=(-s) + if [ -n "${QDRANT_API_KEY:-}" ]; then + info_opts+=( -H "api-key: $QDRANT_API_KEY" ) + fi + local info + info=$(curl "${info_opts[@]}" "$url" || echo "{}") local points=$(echo "$info" | grep -o '"points_count":[0-9]*' | cut -d: -f2) log "${BLUE}📊 Qdrant Statistics:${NC}" @@ -458,4 +480,4 @@ main() { } # Run main function -main \ No newline at end of file +main diff --git a/scripts/test_single_index.sh b/scripts/test_single_index.sh index 0fb92a6..260c378 100755 --- a/scripts/test_single_index.sh +++ b/scripts/test_single_index.sh @@ -54,8 +54,25 @@ else fi # Check Qdrant -QDRANT_URL="https://$QDRANT_HOST/collections/$QDRANT_COLLECTION" -QDRANT_INFO=$(curl -s -H "api-key: $QDRANT_API_KEY" "$QDRANT_URL" 2>/dev/null) +QDRANT_PROTOCOL="https" +QDRANT_BASE_URL="" +if [ "${QDRANT_SSL:-false}" = "true" ] || [ "${QDRANT_SSL:-false}" = "1" ]; then + if [ -n "${QDRANT_REST_PORT:-}" ]; then + QDRANT_BASE_URL="${QDRANT_PROTOCOL}://${QDRANT_HOST}:${QDRANT_REST_PORT}" + else + QDRANT_BASE_URL="${QDRANT_PROTOCOL}://${QDRANT_HOST}" + fi +else + QDRANT_PROTOCOL="http" + QDRANT_REST_PORT="${QDRANT_REST_PORT:-8087}" + QDRANT_BASE_URL="${QDRANT_PROTOCOL}://${QDRANT_HOST}:${QDRANT_REST_PORT}" +fi +QDRANT_URL="${QDRANT_BASE_URL}/collections/${QDRANT_COLLECTION}" +QDRANT_AUTH=() +if [ -n "${QDRANT_API_KEY:-}" ]; then + QDRANT_AUTH=( -H "api-key: $QDRANT_API_KEY" ) +fi +QDRANT_INFO=$(curl -s "${QDRANT_AUTH[@]}" "$QDRANT_URL" 2>/dev/null) if [ $? -eq 0 ]; then VECTOR_COUNT=$(echo "$QDRANT_INFO" | jq -r '.result.points_count' 2>/dev/null || echo "0") echo -e "✅ Qdrant: ${GREEN}Connected${NC}" @@ -206,7 +223,7 @@ echo -e "\n${BOLD}${YELLOW}5️⃣ Results${NC}" echo -e "───────────────────────────" # Final vector count -FINAL_COUNT=$(curl -s -H "api-key: $QDRANT_API_KEY" "$QDRANT_URL" 2>/dev/null | \ +FINAL_COUNT=$(curl -s "${QDRANT_AUTH[@]}" "$QDRANT_URL" 2>/dev/null | \ jq -r '.result.points_count' 2>/dev/null || echo "$VECTOR_COUNT") if [ "$FINAL_COUNT" -gt "$VECTOR_COUNT" ]; then @@ -233,8 +250,8 @@ if [ "$FINAL_COUNT" -gt "$VECTOR_COUNT" ]; then jq '.data[0].embedding' 2>/dev/null) if [ -n "$SEARCH_VECTOR" ]; then - SEARCH_RESULT=$(curl -s -X POST "https://$QDRANT_HOST:6333/collections/$QDRANT_COLLECTION/points/search" \ - -H "api-key: $QDRANT_API_KEY" \ + SEARCH_RESULT=$(curl -s -X POST "${QDRANT_BASE_URL}/collections/${QDRANT_COLLECTION}/points/search" \ + "${QDRANT_AUTH[@]}" \ -H "Content-Type: application/json" \ -d "{\"vector\": $SEARCH_VECTOR, \"limit\": 3, \"with_payload\": true}" 2>/dev/null) @@ -250,4 +267,4 @@ fi echo -e "\n${BOLD}${CYAN}═══════════════════════════════════════════════════════════════${NC}" echo -e "Full log available at: ${CYAN}$TEST_LOG${NC}" -echo -e "${BOLD}${CYAN}═══════════════════════════════════════════════════════════════${NC}" \ No newline at end of file +echo -e "${BOLD}${CYAN}═══════════════════════════════════════════════════════════════${NC}" diff --git a/settings.gradle.kts b/settings.gradle.kts index 4e9ee14..103607a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1 +1,5 @@ +plugins { + id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0" +} + rootProject.name = "java-chat" diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml index 3c9d418..7d1f430 100644 --- a/spotbugs-exclude.xml +++ b/spotbugs-exclude.xml @@ -58,25 +58,30 @@ - - + + - - + + + + - - - - - - - - - - diff --git a/src/main/java/com/williamcallahan/javachat/JavaChatApplication.java b/src/main/java/com/williamcallahan/javachat/JavaChatApplication.java index caac23e..ea0fffa 100644 --- a/src/main/java/com/williamcallahan/javachat/JavaChatApplication.java +++ b/src/main/java/com/williamcallahan/javachat/JavaChatApplication.java @@ -26,7 +26,5 @@ public static void main(final String[] args) { SpringApplication.run(JavaChatApplication.class, args); } - private JavaChatApplication() { - } - + private JavaChatApplication() {} } diff --git a/src/main/java/com/williamcallahan/javachat/application/prompt/PromptTruncator.java b/src/main/java/com/williamcallahan/javachat/application/prompt/PromptTruncator.java index 1bda1fe..186fe6c 100644 --- a/src/main/java/com/williamcallahan/javachat/application/prompt/PromptTruncator.java +++ b/src/main/java/com/williamcallahan/javachat/application/prompt/PromptTruncator.java @@ -3,13 +3,12 @@ import com.williamcallahan.javachat.domain.prompt.ContextDocumentSegment; import com.williamcallahan.javachat.domain.prompt.ConversationTurnSegment; import com.williamcallahan.javachat.domain.prompt.StructuredPrompt; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; /** * Truncates structured prompts to fit within model token limits while preserving semantic boundaries. @@ -30,12 +29,10 @@ public class PromptTruncator { private static final Logger log = LoggerFactory.getLogger(PromptTruncator.class); /** Truncation notice for GPT-5 family models with 8K input limit. */ - private static final String TRUNCATION_NOTICE_GPT5 = - "[Context truncated due to GPT-5 8K input limit]\n\n"; + private static final String TRUNCATION_NOTICE_GPT5 = "[Context truncated due to GPT-5 8K input limit]\n\n"; /** Truncation notice for other models with larger limits. */ - private static final String TRUNCATION_NOTICE_GENERIC = - "[Context truncated due to model input limit]\n\n"; + private static final String TRUNCATION_NOTICE_GENERIC = "[Context truncated due to model input limit]\n\n"; /** * Truncates a structured prompt to fit within the specified token limit. @@ -50,20 +47,18 @@ public class PromptTruncator { * @return truncation result with the fitted prompt and truncation metadata */ public TruncatedPrompt truncate(StructuredPrompt prompt, int maxTokens, boolean isGpt5Family) { - int reservedTokens = prompt.system().estimatedTokens() + prompt.currentQuery().estimatedTokens(); + int reservedTokens = + prompt.system().estimatedTokens() + prompt.currentQuery().estimatedTokens(); if (reservedTokens >= maxTokens) { - log.warn("System prompt ({} tokens) + query ({} tokens) exceed limit ({} tokens)", + log.warn( + "System prompt ({} tokens) + query ({} tokens) exceed limit ({} tokens)", prompt.system().estimatedTokens(), prompt.currentQuery().estimatedTokens(), maxTokens); // Return prompt with only system and query - no room for context or history - StructuredPrompt minimalPrompt = new StructuredPrompt( - prompt.system(), - List.of(), - List.of(), - prompt.currentQuery() - ); + StructuredPrompt minimalPrompt = + new StructuredPrompt(prompt.system(), List.of(), List.of(), prompt.currentQuery()); return new TruncatedPrompt(minimalPrompt, true, isGpt5Family); } @@ -73,38 +68,33 @@ public TruncatedPrompt truncate(StructuredPrompt prompt, int maxTokens, boolean int originalTurnCount = prompt.conversationHistory().size(); // Fit conversation history (newest first - reverse to prioritize recent) - List fittingTurns = fitSegmentsNewestFirst( - prompt.conversationHistory(), available); + List fittingTurns = fitSegmentsNewestFirst(prompt.conversationHistory(), available); int turnsTokens = sumTokens(fittingTurns); available -= turnsTokens; if (fittingTurns.size() < prompt.conversationHistory().size()) { wasTruncated = true; - log.debug("Truncated conversation history from {} to {} turns", - originalTurnCount, fittingTurns.size()); + log.debug("Truncated conversation history from {} to {} turns", originalTurnCount, fittingTurns.size()); } // Fit context documents with remaining budget (most relevant first) - List fittingDocs = fitDocumentsByRelevance( - prompt.contextDocuments(), available); + List fittingDocs = fitDocumentsByRelevance(prompt.contextDocuments(), available); if (fittingDocs.size() < prompt.contextDocuments().size()) { wasTruncated = true; - log.debug("Truncated context documents from {} to {}", - originalDocCount, fittingDocs.size()); + log.debug("Truncated context documents from {} to {}", originalDocCount, fittingDocs.size()); } - StructuredPrompt truncated = new StructuredPrompt( - prompt.system(), - fittingDocs, - fittingTurns, - prompt.currentQuery() - ); + StructuredPrompt truncated = + new StructuredPrompt(prompt.system(), fittingDocs, fittingTurns, prompt.currentQuery()); if (wasTruncated) { - log.info("Prompt truncated: {} docs → {}, {} turns → {} (limit: {} tokens)", - originalDocCount, fittingDocs.size(), - originalTurnCount, fittingTurns.size(), + log.info( + "Prompt truncated: {} docs → {}, {} turns → {} (limit: {} tokens)", + originalDocCount, + fittingDocs.size(), + originalTurnCount, + fittingTurns.size(), maxTokens); } @@ -174,11 +164,7 @@ private List fitDocumentsByRelevance( for (int newIndex = 0; newIndex < fitting.size(); newIndex++) { ContextDocumentSegment original = fitting.get(newIndex); reindexed.add(new ContextDocumentSegment( - newIndex + 1, - original.sourceUrl(), - original.documentContent(), - original.estimatedTokens() - )); + newIndex + 1, original.sourceUrl(), original.documentContent(), original.estimatedTokens())); } return List.copyOf(reindexed); @@ -199,11 +185,7 @@ private int sumTokens(List DOCUMENTATION_SETS = List.of( - new DocumentationSet(DOCSET_PDF_BOOKS_NAME, DOCSET_PDF_BOOKS_PATH), - new DocumentationSet(DOCSET_JAVA_24_COMPLETE_NAME, DOCSET_JAVA_24_COMPLETE_PATH), - new DocumentationSet(DOCSET_JAVA_25_COMPLETE_NAME, DOCSET_JAVA_25_COMPLETE_PATH), - new DocumentationSet(DOCSET_JAVA_25_ALT_NAME, DOCSET_JAVA_25_ALT_PATH), - new DocumentationSet(DOCSET_SPRING_BOOT_COMPLETE_NAME, DOCSET_SPRING_BOOT_COMPLETE_PATH), - new DocumentationSet(DOCSET_SPRING_FRAMEWORK_COMPLETE_NAME, DOCSET_SPRING_FRAMEWORK_COMPLETE_PATH), - new DocumentationSet(DOCSET_SPRING_AI_COMPLETE_NAME, DOCSET_SPRING_AI_COMPLETE_PATH), - new DocumentationSet(DOCSET_JAVA_24_QUICK_NAME, DOCSET_JAVA_24_QUICK_PATH), - new DocumentationSet(DOCSET_JAVA_25_QUICK_NAME, DOCSET_JAVA_25_QUICK_PATH), - new DocumentationSet(DOCSET_SPRING_BOOT_QUICK_NAME, DOCSET_SPRING_BOOT_QUICK_PATH), - new DocumentationSet(DOCSET_SPRING_FRAMEWORK_QUICK_NAME, DOCSET_SPRING_FRAMEWORK_QUICK_PATH), - new DocumentationSet(DOCSET_SPRING_AI_QUICK_NAME, DOCSET_SPRING_AI_QUICK_PATH) - ); + new DocumentationSet(DOCSET_PDF_BOOKS_NAME, DOCSET_PDF_BOOKS_PATH), + new DocumentationSet(DOCSET_JAVA_24_COMPLETE_NAME, DOCSET_JAVA_24_COMPLETE_PATH), + new DocumentationSet(DOCSET_JAVA_25_COMPLETE_NAME, DOCSET_JAVA_25_COMPLETE_PATH), + new DocumentationSet(DOCSET_JAVA_25_ALT_NAME, DOCSET_JAVA_25_ALT_PATH), + new DocumentationSet(DOCSET_SPRING_BOOT_COMPLETE_NAME, DOCSET_SPRING_BOOT_COMPLETE_PATH), + new DocumentationSet(DOCSET_SPRING_FRAMEWORK_COMPLETE_NAME, DOCSET_SPRING_FRAMEWORK_COMPLETE_PATH), + new DocumentationSet(DOCSET_SPRING_AI_COMPLETE_NAME, DOCSET_SPRING_AI_COMPLETE_PATH), + new DocumentationSet(DOCSET_JAVA_24_QUICK_NAME, DOCSET_JAVA_24_QUICK_PATH), + new DocumentationSet(DOCSET_JAVA_25_QUICK_NAME, DOCSET_JAVA_25_QUICK_PATH), + new DocumentationSet(DOCSET_SPRING_BOOT_QUICK_NAME, DOCSET_SPRING_BOOT_QUICK_PATH), + new DocumentationSet(DOCSET_SPRING_FRAMEWORK_QUICK_NAME, DOCSET_SPRING_FRAMEWORK_QUICK_PATH), + new DocumentationSet(DOCSET_SPRING_AI_QUICK_NAME, DOCSET_SPRING_AI_QUICK_PATH)); private final DocsIngestionService ingestionService; private final ProgressTracker progressTracker; @@ -126,8 +127,7 @@ public class DocumentProcessor { * @param ingestionService service for ingesting documentation into the vector store * @param progressTracker tracker for monitoring ingestion progress */ - public DocumentProcessor(final DocsIngestionService ingestionService, - final ProgressTracker progressTracker) { + public DocumentProcessor(final DocsIngestionService ingestionService, final ProgressTracker progressTracker) { this.ingestionService = ingestionService; this.progressTracker = progressTracker; } @@ -157,15 +157,14 @@ private void runDocumentProcessing(final String... ignoredArgs) { final Path basePath = Path.of(config.docsDirectory()).toAbsolutePath().normalize(); final IngestionTotals totals = DOCUMENTATION_SETS.stream() - .map(docSet -> processDocumentationSet(basePath, docSet)) - .reduce(IngestionTotals.ZERO, this::accumulateOutcome, IngestionTotals::combine); + .map(docSet -> processDocumentationSet(basePath, docSet)) + .reduce(IngestionTotals.ZERO, this::accumulateOutcome, IngestionTotals::combine); logSummary(config, totals); if (totals.failedSets() > 0) { throw new DocumentProcessingException( - String.format(Locale.ROOT, PROCESSING_FAILED_TEMPLATE, totals.failedSets()) - ); + String.format(Locale.ROOT, PROCESSING_FAILED_TEMPLATE, totals.failedSets())); } } @@ -211,7 +210,7 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc final long startMillis = System.currentTimeMillis(); try { final DocsIngestionService.LocalIngestionOutcome outcome = - ingestionService.ingestLocalDirectory(docsPath.toString(), Integer.MAX_VALUE); + ingestionService.ingestLocalDirectory(docsPath.toString(), Integer.MAX_VALUE); final int processed = outcome.processedCount(); final long elapsedMillis = System.currentTimeMillis() - startMillis; logProcessingStats(processed, elapsedMillis); @@ -230,9 +229,10 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc } catch (IOException ioException) { if (LOGGER.isErrorEnabled()) { - LOGGER.error(LOG_PROCESSING_FAILED, - Integer.toHexString(Objects.hashCode(docSet.displayName())), - ioException.getClass().getSimpleName()); + LOGGER.error( + LOG_PROCESSING_FAILED, + Integer.toHexString(Objects.hashCode(docSet.displayName())), + ioException.getClass().getSimpleName()); } if (LOGGER.isDebugEnabled()) { LOGGER.debug(LOG_STACK_TRACE, ioException); @@ -243,15 +243,12 @@ private ProcessingOutcome processDocumentationSet(final Path basePath, final Doc private long countEligibleFiles(final Path docsPath) { try (Stream paths = Files.walk(docsPath)) { - return paths - .filter(path -> !Files.isDirectory(path)) - .filter(DocumentProcessor::isEligibleFile) - .count(); + return paths.filter(path -> !Files.isDirectory(path)) + .filter(DocumentProcessor::isEligibleFile) + .count(); } catch (IOException ioException) { throw new UncheckedIOException( - String.format(Locale.ROOT, FILE_ENUMERATION_FAILURE_TEMPLATE, docsPath), - ioException - ); + String.format(Locale.ROOT, FILE_ENUMERATION_FAILURE_TEMPLATE, docsPath), ioException); } } @@ -281,11 +278,12 @@ private void logProcessingStats(final int processed, final long elapsedMillis) { } final double elapsedSeconds = Math.max(elapsedMillis, 1) / MILLIS_PER_SECOND; final double rate = processed / elapsedSeconds; - LOGGER.info(LOG_PROCESSED_STATS, - processed, - String.format(Locale.ROOT, FORMAT_SECONDS, elapsedSeconds), - String.format(Locale.ROOT, FORMAT_RATE, rate), - progressTracker.formatPercent()); + LOGGER.info( + LOG_PROCESSED_STATS, + processed, + String.format(Locale.ROOT, FORMAT_SECONDS, elapsedSeconds), + String.format(Locale.ROOT, FORMAT_RATE, rate), + progressTracker.formatPercent()); } private void logSummary(final EnvironmentConfig config, final IngestionTotals totals) { @@ -318,12 +316,7 @@ private void logSummary(final EnvironmentConfig config, final IngestionTotals to * Environment configuration consolidated from system environment variables. */ private record EnvironmentConfig( - String docsDirectory, - String qdrantCollection, - String qdrantHost, - String qdrantPort, - String appPort - ) { + String docsDirectory, String qdrantCollection, String qdrantHost, String qdrantPort, String appPort) { private static final String DOCS_DIR_DEFAULT = "data/docs"; private static final String QDRANT_COLLECTION_DEFAULT = "(not set)"; private static final String QDRANT_HOST_DEFAULT = "localhost"; @@ -337,12 +330,11 @@ private record EnvironmentConfig( static EnvironmentConfig fromEnvironment() { return new EnvironmentConfig( - envOrDefault(ENV_DOCS_DIR, DOCS_DIR_DEFAULT), - envOrDefault(ENV_QDRANT_COLLECTION, QDRANT_COLLECTION_DEFAULT), - envOrDefault(ENV_QDRANT_HOST, QDRANT_HOST_DEFAULT), - envOrDefault(ENV_QDRANT_PORT, QDRANT_PORT_DEFAULT), - envOrDefault(ENV_APP_PORT, APP_PORT_DEFAULT) - ); + envOrDefault(ENV_DOCS_DIR, DOCS_DIR_DEFAULT), + envOrDefault(ENV_QDRANT_COLLECTION, QDRANT_COLLECTION_DEFAULT), + envOrDefault(ENV_QDRANT_HOST, QDRANT_HOST_DEFAULT), + envOrDefault(ENV_QDRANT_PORT, QDRANT_PORT_DEFAULT), + envOrDefault(ENV_APP_PORT, APP_PORT_DEFAULT)); } private static String envOrDefault(final String key, final String fallbackText) { @@ -371,11 +363,10 @@ IngestionTotals addFailed() { static IngestionTotals combine(final IngestionTotals left, final IngestionTotals right) { return new IngestionTotals( - left.processed + right.processed, - left.duplicates + right.duplicates, - left.skippedSets + right.skippedSets, - left.failedSets + right.failedSets - ); + left.processed + right.processed, + left.duplicates + right.duplicates, + left.skippedSets + right.skippedSets, + left.failedSets + right.failedSets); } } @@ -384,15 +375,16 @@ static IngestionTotals combine(final IngestionTotals left, final IngestionTotals */ private sealed interface ProcessingOutcome { record Success(long processed, long duplicates) implements ProcessingOutcome {} + record Skipped(String setName, String reason) implements ProcessingOutcome {} + record Failed(String setName) implements ProcessingOutcome {} } /** * A documentation set to process, defined by display name and relative path. */ - private record DocumentationSet(String displayName, String relativePath) { - } + private record DocumentationSet(String displayName, String relativePath) {} /** * Thrown when document processing completes but one or more documentation sets failed. diff --git a/src/main/java/com/williamcallahan/javachat/config/AiConfig.java b/src/main/java/com/williamcallahan/javachat/config/AiConfig.java index d3c6266..fa73b1f 100644 --- a/src/main/java/com/williamcallahan/javachat/config/AiConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/AiConfig.java @@ -1,17 +1,18 @@ package com.williamcallahan.javachat.config; +import io.netty.channel.ChannelOption; +import java.time.Duration; import org.springframework.ai.chat.client.ChatClient; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.ConfigurationPropertiesScan; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.client.RestClient; +import org.springframework.web.reactive.function.client.WebClient; import reactor.netty.http.client.HttpClient; -import io.netty.channel.ChannelOption; -import java.time.Duration; + /** * Spring configuration for AI clients and HTTP settings. */ @@ -29,8 +30,7 @@ public class AiConfig { /** * Creates the AI configuration. */ - public AiConfig() { - } + public AiConfig() {} /** * Builds the chat client for Spring AI. @@ -54,11 +54,10 @@ public WebClient.Builder webClientBuilder() { // Configure HTTP client with increased timeouts for GitHub Models API // GitHub Models can be slower than OpenAI, especially for complex requests final HttpClient httpClient = HttpClient.create() - .responseTimeout(Duration.ofMinutes(RESPONSE_TIMEOUT_MINUTES)) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT_MILLIS); + .responseTimeout(Duration.ofMinutes(RESPONSE_TIMEOUT_MINUTES)) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT_MILLIS); - return WebClient.builder() - .clientConnector(new ReactorClientHttpConnector(httpClient)); + return WebClient.builder().clientConnector(new ReactorClientHttpConnector(httpClient)); } /** @@ -70,7 +69,8 @@ public WebClient.Builder webClientBuilder() { public RestClient.Builder restClientBuilder() { SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); requestFactory.setConnectTimeout(CONNECT_TIMEOUT_MILLIS); - requestFactory.setReadTimeout((int) Duration.ofMinutes(RESPONSE_TIMEOUT_MINUTES).toMillis()); + requestFactory.setReadTimeout( + (int) Duration.ofMinutes(RESPONSE_TIMEOUT_MINUTES).toMillis()); return RestClient.builder().requestFactory(requestFactory); } } diff --git a/src/main/java/com/williamcallahan/javachat/config/ApiKeyLoggingConfig.java b/src/main/java/com/williamcallahan/javachat/config/ApiKeyLoggingConfig.java index 8490eb2..04d8b60 100644 --- a/src/main/java/com/williamcallahan/javachat/config/ApiKeyLoggingConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/ApiKeyLoggingConfig.java @@ -31,7 +31,7 @@ public class ApiKeyLoggingConfig { private static final String LOG_CHAT_GITHUB_FALLBACK = "Chat API: Using GitHub Models (fallback)"; private static final String LOG_CHAT_OPENAI_FALLBACK = "Chat API: Using OpenAI API (fallback)"; private static final String LOG_CHAT_MISSING = - "Chat API: No API key configured - chat functionality will not work!"; + "Chat API: No API key configured - chat functionality will not work!"; private static final String GITHUB_TOKEN_PROPERTY = "${GITHUB_TOKEN:}"; private static final String OPENAI_API_KEY_PROPERTY = "${OPENAI_API_KEY:}"; private static final String QDRANT_API_KEY_PROPERTY = "${QDRANT_API_KEY:}"; diff --git a/src/main/java/com/williamcallahan/javachat/config/AppProperties.java b/src/main/java/com/williamcallahan/javachat/config/AppProperties.java index a4c4731..b2437c0 100644 --- a/src/main/java/com/williamcallahan/javachat/config/AppProperties.java +++ b/src/main/java/com/williamcallahan/javachat/config/AppProperties.java @@ -1,9 +1,9 @@ package com.williamcallahan.javachat.config; +import com.williamcallahan.javachat.support.AsciiTextNormalizer; import jakarta.annotation.PostConstruct; import java.net.URI; import java.net.URISyntaxException; -import com.williamcallahan.javachat.support.AsciiTextNormalizer; import java.util.Locale; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,8 +55,7 @@ public class AppProperties { /** * Creates configuration sections with default values. */ - public AppProperties() { - } + public AppProperties() {} @PostConstruct void validateConfiguration() { @@ -92,11 +91,7 @@ public void setPublicBaseUrl(final String publicBaseUrl) { private static String validatePublicBaseUrl(final String configuredPublicBaseUrl) { if (configuredPublicBaseUrl == null || configuredPublicBaseUrl.isBlank()) { - log.warn( - "{} is not configured; defaulting to {}", - PUBLIC_BASE_URL_KEY, - DEFAULT_PUBLIC_BASE_URL - ); + log.warn("{} is not configured; defaulting to {}", PUBLIC_BASE_URL_KEY, DEFAULT_PUBLIC_BASE_URL); return DEFAULT_PUBLIC_BASE_URL; } @@ -107,64 +102,42 @@ private static String validatePublicBaseUrl(final String configuredPublicBaseUrl } catch (URISyntaxException syntaxError) { String sanitizedMessage = sanitizeLogMessage(syntaxError.getMessage()); log.warn( - "{} is invalid ({}); defaulting to {}", - PUBLIC_BASE_URL_KEY, - sanitizedMessage, - DEFAULT_PUBLIC_BASE_URL - ); + "{} is invalid ({}); defaulting to {}", + PUBLIC_BASE_URL_KEY, + sanitizedMessage, + DEFAULT_PUBLIC_BASE_URL); return DEFAULT_PUBLIC_BASE_URL; } final String scheme = parsed.getScheme(); if (scheme == null || scheme.isBlank()) { - log.warn( - "{} is missing a scheme; defaulting to {}", - PUBLIC_BASE_URL_KEY, - DEFAULT_PUBLIC_BASE_URL - ); + log.warn("{} is missing a scheme; defaulting to {}", PUBLIC_BASE_URL_KEY, DEFAULT_PUBLIC_BASE_URL); return DEFAULT_PUBLIC_BASE_URL; } final String normalizedScheme = AsciiTextNormalizer.toLowerAscii(scheme); if (!"http".equals(normalizedScheme) && !"https".equals(normalizedScheme)) { - log.warn( - "{} must use http/https scheme; defaulting to {}", - PUBLIC_BASE_URL_KEY, - DEFAULT_PUBLIC_BASE_URL - ); + log.warn("{} must use http/https scheme; defaulting to {}", PUBLIC_BASE_URL_KEY, DEFAULT_PUBLIC_BASE_URL); return DEFAULT_PUBLIC_BASE_URL; } final String host = parsed.getHost(); if (host == null || host.isBlank()) { - log.warn( - "{} is missing a host; defaulting to {}", - PUBLIC_BASE_URL_KEY, - DEFAULT_PUBLIC_BASE_URL - ); + log.warn("{} is missing a host; defaulting to {}", PUBLIC_BASE_URL_KEY, DEFAULT_PUBLIC_BASE_URL); return DEFAULT_PUBLIC_BASE_URL; } final int port = parsed.getPort(); try { // Strip any path/query/fragment; keep scheme/host/port only. - return new URI( - normalizedScheme, - null, - host, - port, - null, - null, - null - ).toString(); + return new URI(normalizedScheme, null, host, port, null, null, null).toString(); } catch (URISyntaxException syntaxError) { String sanitizedMessage = sanitizeLogMessage(syntaxError.getMessage()); log.warn( - "{} could not be normalized ({}); defaulting to {}", - PUBLIC_BASE_URL_KEY, - sanitizedMessage, - DEFAULT_PUBLIC_BASE_URL - ); + "{} could not be normalized ({}); defaulting to {}", + PUBLIC_BASE_URL_KEY, + sanitizedMessage, + DEFAULT_PUBLIC_BASE_URL); return DEFAULT_PUBLIC_BASE_URL; } } @@ -176,32 +149,77 @@ private static String sanitizeLogMessage(final String message) { return message.replace("\r", "").replace("\n", ""); } - public RetrievalAugmentationConfig getRag() { return rag; } - public void setRag(RetrievalAugmentationConfig rag) { this.rag = requireConfiguredSection(rag, RAG_KEY); } + public RetrievalAugmentationConfig getRag() { + return rag; + } + + public void setRag(RetrievalAugmentationConfig rag) { + this.rag = requireConfiguredSection(rag, RAG_KEY); + } + + public LocalEmbedding getLocalEmbedding() { + return localEmbedding; + } + + public void setLocalEmbedding(LocalEmbedding localEmbedding) { + this.localEmbedding = requireConfiguredSection(localEmbedding, LOCAL_EMBED_KEY); + } + + public RemoteEmbedding getRemoteEmbedding() { + return remoteEmbedding; + } + + public void setRemoteEmbedding(RemoteEmbedding remoteEmbedding) { + this.remoteEmbedding = requireConfiguredSection(remoteEmbedding, REMOTE_EMB_KEY); + } + + public DocumentationConfig getDocs() { + return docs; + } + + public void setDocs(DocumentationConfig docs) { + this.docs = requireConfiguredSection(docs, DOCS_KEY); + } + + public Diagnostics getDiagnostics() { + return diagnostics; + } + + public void setDiagnostics(Diagnostics diagnostics) { + this.diagnostics = requireConfiguredSection(diagnostics, DIAG_KEY); + } - public LocalEmbedding getLocalEmbedding() { return localEmbedding; } - public void setLocalEmbedding(LocalEmbedding localEmbedding) { this.localEmbedding = requireConfiguredSection(localEmbedding, LOCAL_EMBED_KEY); } + public Qdrant getQdrant() { + return qdrant; + } - public RemoteEmbedding getRemoteEmbedding() { return remoteEmbedding; } - public void setRemoteEmbedding(RemoteEmbedding remoteEmbedding) { this.remoteEmbedding = requireConfiguredSection(remoteEmbedding, REMOTE_EMB_KEY); } + public void setQdrant(Qdrant qdrant) { + this.qdrant = requireConfiguredSection(qdrant, QDRANT_KEY); + } - public DocumentationConfig getDocs() { return docs; } - public void setDocs(DocumentationConfig docs) { this.docs = requireConfiguredSection(docs, DOCS_KEY); } + public CorsConfig getCors() { + return cors; + } - public Diagnostics getDiagnostics() { return diagnostics; } - public void setDiagnostics(Diagnostics diagnostics) { this.diagnostics = requireConfiguredSection(diagnostics, DIAG_KEY); } + public void setCors(CorsConfig cors) { + this.cors = requireConfiguredSection(cors, CORS_KEY); + } - public Qdrant getQdrant() { return qdrant; } - public void setQdrant(Qdrant qdrant) { this.qdrant = requireConfiguredSection(qdrant, QDRANT_KEY); } + public Embeddings getEmbeddings() { + return embeddings; + } - public CorsConfig getCors() { return cors; } - public void setCors(CorsConfig cors) { this.cors = requireConfiguredSection(cors, CORS_KEY); } + public void setEmbeddings(Embeddings embeddings) { + this.embeddings = requireConfiguredSection(embeddings, EMBEDDINGS_KEY); + } - public Embeddings getEmbeddings() { return embeddings; } - public void setEmbeddings(Embeddings embeddings) { this.embeddings = requireConfiguredSection(embeddings, EMBEDDINGS_KEY); } + public Llm getLlm() { + return llm; + } - public Llm getLlm() { return llm; } - public void setLlm(Llm llm) { this.llm = requireConfiguredSection(llm, LLM_KEY); } + public void setLlm(Llm llm) { + this.llm = requireConfiguredSection(llm, LLM_KEY); + } private static T requireConfiguredSection(T section, String sectionKey) { if (section == null) { @@ -213,18 +231,36 @@ private static T requireConfiguredSection(T section, String sectionKey) { /** Qdrant vector store settings. */ public static class Qdrant { private boolean ensurePayloadIndexes = true; - public boolean isEnsurePayloadIndexes() { return ensurePayloadIndexes; } - public void setEnsurePayloadIndexes(boolean ensurePayloadIndexes) { this.ensurePayloadIndexes = ensurePayloadIndexes; } + + public boolean isEnsurePayloadIndexes() { + return ensurePayloadIndexes; + } + + public void setEnsurePayloadIndexes(boolean ensurePayloadIndexes) { + this.ensurePayloadIndexes = ensurePayloadIndexes; + } } /** Embedding vector configuration. */ public static class Embeddings { private int dimensions = 1536; private String cacheDir = "./data/embeddings-cache"; - public int getDimensions() { return dimensions; } - public void setDimensions(int dimensions) { this.dimensions = dimensions; } - public String getCacheDir() { return cacheDir; } - public void setCacheDir(String cacheDir) { this.cacheDir = cacheDir; } + + public int getDimensions() { + return dimensions; + } + + public void setDimensions(int dimensions) { + this.dimensions = dimensions; + } + + public String getCacheDir() { + return cacheDir; + } + + public void setCacheDir(String cacheDir) { + this.cacheDir = cacheDir; + } Embeddings validateConfiguration() { if (dimensions <= 0) { @@ -240,15 +276,23 @@ public static class Llm { private static final double MAX_TEMPERATURE = 2.0; private double temperature = 0.7; - public double getTemperature() { return temperature; } - public void setTemperature(double temperature) { this.temperature = temperature; } + + public double getTemperature() { + return temperature; + } + + public void setTemperature(double temperature) { + this.temperature = temperature; + } Llm validateConfiguration() { if (temperature < MIN_TEMPERATURE || temperature > MAX_TEMPERATURE) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, + throw new IllegalArgumentException(String.format( + Locale.ROOT, "app.llm.temperature must be in range [%.1f, %.1f], got: %.2f", - MIN_TEMPERATURE, MAX_TEMPERATURE, temperature)); + MIN_TEMPERATURE, + MAX_TEMPERATURE, + temperature)); } return this; } diff --git a/src/main/java/com/williamcallahan/javachat/config/CacheConfig.java b/src/main/java/com/williamcallahan/javachat/config/CacheConfig.java index 529dd0c..0922eaa 100644 --- a/src/main/java/com/williamcallahan/javachat/config/CacheConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/CacheConfig.java @@ -24,8 +24,7 @@ public class CacheConfig { /** * Creates cache configuration. */ - public CacheConfig() { - } + public CacheConfig() {} /** * Registers the cache manager with configured cache names. @@ -34,11 +33,7 @@ public CacheConfig() { */ @Bean public CacheManager cacheManager() { - return new ConcurrentMapCacheManager( - RERANKER_CACHE, - ENRICHMENT_CACHE, - CHAT_CACHE - ); + return new ConcurrentMapCacheManager(RERANKER_CACHE, ENRICHMENT_CACHE, CHAT_CACHE); } /** @@ -47,12 +42,11 @@ public CacheManager cacheManager() { @Scheduled(fixedRate = EVICT_INTERVAL) public void evictAllCachesAtIntervals() { final CacheManager manager = cacheManager(); - manager.getCacheNames() - .forEach(cacheName -> { - final Cache cache = manager.getCache(cacheName); - if (cache != null) { - cache.clear(); - } - }); + manager.getCacheNames().forEach(cacheName -> { + final Cache cache = manager.getCache(cacheName); + if (cache != null) { + cache.clear(); + } + }); } } diff --git a/src/main/java/com/williamcallahan/javachat/config/CorsConfig.java b/src/main/java/com/williamcallahan/javachat/config/CorsConfig.java index 9f9c43a..98b64b0 100644 --- a/src/main/java/com/williamcallahan/javachat/config/CorsConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/CorsConfig.java @@ -17,13 +17,8 @@ public class CorsConfig { private static final String METHOD_OPTIONS = "OPTIONS"; private static final String HEADER_WILD = "*"; private static final List ORIGINS_DEF = List.of(LOCAL_ORIGIN, LOOPBACK_ORIGIN); - private static final List METHODS_DEF = List.of( - METHOD_GET, - METHOD_POST, - METHOD_PUT, - METHOD_DELETE, - METHOD_OPTIONS - ); + private static final List METHODS_DEF = + List.of(METHOD_GET, METHOD_POST, METHOD_PUT, METHOD_DELETE, METHOD_OPTIONS); private static final List HEADERS_DEF = List.of(HEADER_WILD); private static final boolean CREDENTIALS_DEF = true; private static final long MAX_AGE_DEF = 3_600L; @@ -44,8 +39,7 @@ public class CorsConfig { /** * Creates CORS configuration. */ - public CorsConfig() { - } + public CorsConfig() {} /** * Validates CORS settings. diff --git a/src/main/java/com/williamcallahan/javachat/config/Diagnostics.java b/src/main/java/com/williamcallahan/javachat/config/Diagnostics.java index 2bc46b3..7f6ec2f 100644 --- a/src/main/java/com/williamcallahan/javachat/config/Diagnostics.java +++ b/src/main/java/com/williamcallahan/javachat/config/Diagnostics.java @@ -17,8 +17,7 @@ public class Diagnostics { /** * Creates diagnostics configuration. */ - public Diagnostics() { - } + public Diagnostics() {} /** * Validates diagnostics settings. diff --git a/src/main/java/com/williamcallahan/javachat/config/DocsLocalPathMapper.java b/src/main/java/com/williamcallahan/javachat/config/DocsLocalPathMapper.java index 32f9af7..27f1c37 100644 --- a/src/main/java/com/williamcallahan/javachat/config/DocsLocalPathMapper.java +++ b/src/main/java/com/williamcallahan/javachat/config/DocsLocalPathMapper.java @@ -17,10 +17,8 @@ final class DocsLocalPathMapper { private static final String SPRING_FRAMEWORK_MARKER = "spring-framework"; private static final String SPRING_BOOT_MARKER = "spring-boot"; - private static final String SPRING_FRAMEWORK_DUPLICATE_JAVADOC_PREFIX = - "docs/current/api/current/javadoc-api/"; - private static final String SPRING_FRAMEWORK_DUPLICATE_JAVADOC_BASE = - "docs/current/api/current/"; + private static final String SPRING_FRAMEWORK_DUPLICATE_JAVADOC_PREFIX = "docs/current/api/current/javadoc-api/"; + private static final String SPRING_FRAMEWORK_DUPLICATE_JAVADOC_BASE = "docs/current/api/current/"; private static final String SPRING_FRAMEWORK_API_CURRENT_PREFIX = "api/current/javadoc-api/"; private static final String SPRING_FRAMEWORK_DOCS_JAVADOC_PREFIX = "docs/current/javadoc-api/"; private static final String SPRING_FRAMEWORK_JAVADOC_JAVA_PREFIX = "javadoc-api/java/"; @@ -35,22 +33,18 @@ final class DocsLocalPathMapper { private static final String API_SUFFIX = "/api"; private static final String API_PREFIX = "api/"; - private DocsLocalPathMapper() { - } + private DocsLocalPathMapper() {} static Optional mapLocalPrefixToRemote( - final String localPath, - final Map localPrefixLookup - ) { + final String localPath, final Map localPrefixLookup) { Optional mappedUrl = Optional.empty(); if (localPath != null) { final String normalizedPath = localPath.replace(WINDOWS_PATH_SEPARATOR, UNIX_PATH_SEPARATOR); for (final Map.Entry prefixEntry : localPrefixLookup.entrySet()) { final String localPrefix = prefixEntry.getKey(); if (normalizedPath.contains(localPrefix)) { - final String relativePath = normalizedPath.substring( - normalizedPath.indexOf(localPrefix) + localPrefix.length() - ); + final String relativePath = + normalizedPath.substring(normalizedPath.indexOf(localPrefix) + localPrefix.length()); final String adjustedPath = normalizeRelativePath(localPrefix, relativePath); mappedUrl = joinBaseAndRel(prefixEntry.getValue(), adjustedPath); break; @@ -82,7 +76,7 @@ private static String normalizeSpringFrameworkRelativePath(final String relative } if (adjustedPath.startsWith(SPRING_FRAMEWORK_JAVADOC_JAVA_PREFIX)) { adjustedPath = SPRING_FRAMEWORK_JAVADOC_PREFIX - + adjustedPath.substring(SPRING_FRAMEWORK_JAVADOC_JAVA_PREFIX.length()); + + adjustedPath.substring(SPRING_FRAMEWORK_JAVADOC_JAVA_PREFIX.length()); } return adjustedPath; } @@ -93,8 +87,7 @@ private static String normalizeSpringBootRelativePath(final String relativePath) adjustedPath = adjustedPath.substring(SPRING_BOOT_DOCS_API_PREFIX.length()); } if (adjustedPath.startsWith(SPRING_BOOT_API_JAVA_PREFIX)) { - adjustedPath = SPRING_BOOT_API_PREFIX - + adjustedPath.substring(SPRING_BOOT_API_JAVA_PREFIX.length()); + adjustedPath = SPRING_BOOT_API_PREFIX + adjustedPath.substring(SPRING_BOOT_API_JAVA_PREFIX.length()); } return adjustedPath; } @@ -104,8 +97,8 @@ private static Optional joinBaseAndRel(final String baseUrl, final Strin if (baseUrl != null) { final String normalizedBase = trimTrailingSlashes(baseUrl); String normalizedRel = relativePath == null - ? EMPTY_TEXT - : relativePath.replace(WINDOWS_PATH_SEPARATOR, UNIX_PATH_SEPARATOR); + ? EMPTY_TEXT + : relativePath.replace(WINDOWS_PATH_SEPARATOR, UNIX_PATH_SEPARATOR); normalizedRel = trimLeadingSlashes(normalizedRel); // Avoid duplicate 'docs/api' or 'api' in path diff --git a/src/main/java/com/williamcallahan/javachat/config/DocsSourceRegistry.java b/src/main/java/com/williamcallahan/javachat/config/DocsSourceRegistry.java index 3598d17..fbbbdbb 100644 --- a/src/main/java/com/williamcallahan/javachat/config/DocsSourceRegistry.java +++ b/src/main/java/com/williamcallahan/javachat/config/DocsSourceRegistry.java @@ -1,9 +1,6 @@ package com.williamcallahan.javachat.config; import com.williamcallahan.javachat.support.AsciiTextNormalizer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.InputStream; import java.util.LinkedHashMap; @@ -11,6 +8,8 @@ import java.util.Objects; import java.util.Optional; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Central registry for documentation source URL mappings. @@ -26,12 +25,11 @@ public final class DocsSourceRegistry { private static final Logger LOGGER = LoggerFactory.getLogger(DocsSourceRegistry.class); private static final String DOCS_SOURCES_RESOURCE = "/docs-sources.properties"; - private static final String LOG_DOCS_SOURCES_LOADED = - "Loaded docs-sources.properties with {} entries"; + private static final String LOG_DOCS_SOURCES_LOADED = "Loaded docs-sources.properties with {} entries"; private static final String LOG_DOCS_SOURCES_MISSING = - "docs-sources.properties not found on classpath; using default URL mappings"; + "docs-sources.properties not found on classpath; using default URL mappings"; private static final String LOG_DOCS_SOURCES_LOAD_FAILED = - "Failed to load docs-sources.properties (exceptionType={}) - using default URL mappings"; + "Failed to load docs-sources.properties (exceptionType={}) - using default URL mappings"; private static final String JAVA24_API_BASE_KEY = "JAVA24_API_BASE"; private static final String JAVA25_API_BASE_KEY = "JAVA25_API_BASE"; @@ -43,12 +41,10 @@ public final class DocsSourceRegistry { private static final String DEFAULT_JAVA24 = "https://docs.oracle.com/en/java/javase/24/docs/api/"; private static final String DEFAULT_JAVA25 = "https://docs.oracle.com/en/java/javase/25/docs/api/"; - private static final String DEFAULT_SPRING_BOOT_API_BASE = - "https://docs.spring.io/spring-boot/docs/current/api/"; + private static final String DEFAULT_SPRING_BOOT_API_BASE = "https://docs.spring.io/spring-boot/docs/current/api/"; private static final String DEFAULT_SPRING_FRAMEWORK_API_BASE = - "https://docs.spring.io/spring-framework/docs/current/javadoc-api/"; - private static final String DEFAULT_SPRING_AI_API_BASE = - "https://docs.spring.io/spring-ai/reference/1.0/api/"; + "https://docs.spring.io/spring-framework/docs/current/javadoc-api/"; + private static final String DEFAULT_SPRING_AI_API_BASE = "https://docs.spring.io/spring-ai/reference/1.0/api/"; private static final String LOCAL_DOCS_ROOT = "/data/docs/"; private static final String LOCAL_DOCS_JAVA24 = LOCAL_DOCS_ROOT + "java24/"; @@ -79,20 +75,16 @@ public final class DocsSourceRegistry { public static final String JAVA24_API_BASE = resolveSetting(JAVA24_API_BASE_KEY, DEFAULT_JAVA24); public static final String JAVA25_API_BASE = resolveSetting(JAVA25_API_BASE_KEY, DEFAULT_JAVA25); public static final String SPRING_BOOT_API_BASE = - resolveSetting(SPRING_BOOT_API_BASE_KEY, DEFAULT_SPRING_BOOT_API_BASE); + resolveSetting(SPRING_BOOT_API_BASE_KEY, DEFAULT_SPRING_BOOT_API_BASE); public static final String SPRING_FRAMEWORK_API_BASE = - resolveSetting(SPRING_FRAMEWORK_API_BASE_KEY, DEFAULT_SPRING_FRAMEWORK_API_BASE); + resolveSetting(SPRING_FRAMEWORK_API_BASE_KEY, DEFAULT_SPRING_FRAMEWORK_API_BASE); public static final String SPRING_AI_API_BASE = resolveSetting(SPRING_AI_API_BASE_KEY, DEFAULT_SPRING_AI_API_BASE); - private static final String[] EMBEDDED_HOST_MARKERS = { - DOCS_ORACLE_HOST_MARKER, - SPRING_DOCS_HOST_MARKER - }; + private static final String[] EMBEDDED_HOST_MARKERS = {DOCS_ORACLE_HOST_MARKER, SPRING_DOCS_HOST_MARKER}; private static final Map LOCAL_PREFIX_TO_REMOTE_BASE = buildLocalPrefixLookup(); - private DocsSourceRegistry() { - } + private DocsSourceRegistry() {} private static Properties loadDocsSourceProperties() { final Properties docsSourceProperties = new Properties(); @@ -107,7 +99,8 @@ private static Properties loadDocsSourceProperties() { } } catch (IOException configLoadError) { if (LOGGER.isWarnEnabled()) { - LOGGER.warn(LOG_DOCS_SOURCES_LOAD_FAILED, configLoadError.getClass().getName()); + LOGGER.warn( + LOG_DOCS_SOURCES_LOAD_FAILED, configLoadError.getClass().getName()); } } return docsSourceProperties; @@ -163,8 +156,8 @@ public static Optional reconstructFromEmbeddedHost(final String localPat final String candidateUrl = HTTPS_PREFIX + normalizedPath.substring(markerIndex); // Fix Spring URLs using proper string parsing final String normalizedUrl = candidateUrl.startsWith(SPRING_DOCS_HTTPS_PREFIX) - ? SpringDocsUrlNormalizer.normalize(candidateUrl) - : candidateUrl; + ? SpringDocsUrlNormalizer.normalize(candidateUrl) + : candidateUrl; reconstructedUrl = Optional.of(normalizedUrl); break; } @@ -203,9 +196,7 @@ public static Optional mapBookLocalToPublic(final String localPath) { final String fileName = normalizedPath.substring(markerIndex + LOCAL_DOCS_BOOKS.length()); // Only map the basename to avoid subfolder leakage final int lastSlash = fileName.lastIndexOf('/'); - final String baseName = lastSlash >= 0 - ? fileName.substring(lastSlash + 1) - : fileName; + final String baseName = lastSlash >= 0 ? fileName.substring(lastSlash + 1) : fileName; publicPdfUrl = Optional.of(PUBLIC_PDFS_BASE + baseName); } } @@ -230,14 +221,9 @@ public static String canonicalizeHttpDocUrl(String url) { result = result.replace("/api/api/", "/api/"); // Fix malformed Spring docs paths that accidentally include '/java/' segment if (result.contains(SPRING_DOCS_HTTPS_PREFIX)) { + result = result.replace("/spring-boot/docs/current/api/java/", "/spring-boot/docs/current/api/"); result = result.replace( - "/spring-boot/docs/current/api/java/", - "/spring-boot/docs/current/api/" - ); - result = result.replace( - "/spring-framework/docs/current/javadoc-api/java/", - "/spring-framework/docs/current/javadoc-api/" - ); + "/spring-framework/docs/current/javadoc-api/java/", "/spring-framework/docs/current/javadoc-api/"); } // Remove accidental double slashes (but keep protocol) int protoIdx = result.indexOf("://"); @@ -260,8 +246,8 @@ public static Optional resolveLocalPath(String absolutePath) { return Optional.empty(); } return mapBookLocalToPublic(absolutePath) - .or(() -> reconstructFromEmbeddedHost(absolutePath)) - .or(() -> mapLocalPrefixToRemote(absolutePath)); + .or(() -> reconstructFromEmbeddedHost(absolutePath)) + .or(() -> mapLocalPrefixToRemote(absolutePath)); } /** @@ -283,9 +269,7 @@ public static String normalizeDocUrl(String rawUrl) { } // Map book PDFs to public server path - String resolvedPath = trimmedUrl.startsWith("file://") - ? trimmedUrl.substring("file://".length()) - : trimmedUrl; + String resolvedPath = trimmedUrl.startsWith("file://") ? trimmedUrl.substring("file://".length()) : trimmedUrl; Optional publicPdf = mapBookLocalToPublic(resolvedPath); if (publicPdf.isPresent()) { return publicPdf.get(); diff --git a/src/main/java/com/williamcallahan/javachat/config/DocumentationConfig.java b/src/main/java/com/williamcallahan/javachat/config/DocumentationConfig.java index f3ac93b..102f55a 100644 --- a/src/main/java/com/williamcallahan/javachat/config/DocumentationConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/DocumentationConfig.java @@ -32,8 +32,7 @@ public class DocumentationConfig { /** * Creates documentation configuration. */ - public DocumentationConfig() { - } + public DocumentationConfig() {} /** * Validates documentation settings. diff --git a/src/main/java/com/williamcallahan/javachat/config/EmbeddingFallbackConfig.java b/src/main/java/com/williamcallahan/javachat/config/EmbeddingFallbackConfig.java index f23be73..a2f5d9a 100644 --- a/src/main/java/com/williamcallahan/javachat/config/EmbeddingFallbackConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/EmbeddingFallbackConfig.java @@ -5,6 +5,8 @@ import com.williamcallahan.javachat.service.LocalHashingEmbeddingModel; import com.williamcallahan.javachat.service.OpenAiCompatibleEmbeddingModel; import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -12,12 +14,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Comprehensive embedding configuration with multiple fallback strategies: - * + * * 1. Local embedding server (if enabled and available) * 2. OpenAI embedding API (if API key provided) * 3. Hash-based fallback (deterministic but not semantic) @@ -26,7 +26,7 @@ @Configuration public class EmbeddingFallbackConfig { private static final Logger log = LoggerFactory.getLogger(EmbeddingFallbackConfig.class); - + /** * Creates a local embedding model with remote and hash-based fallbacks. * @@ -62,33 +62,35 @@ public EmbeddingModel localEmbeddingWithFallback( @Value("${spring.ai.openai.embedding.base-url:https://api.openai.com}") String openaiBaseUrl, @Value("${spring.ai.openai.embedding.options.model:text-embedding-3-small}") String openaiModel, RestTemplateBuilder restTemplateBuilder) { - + log.info("[EMBEDDING] Configuring local embedding with fallback strategies"); - + // Primary: Local embedding server - LocalEmbeddingModel primaryModel = new LocalEmbeddingModel(localUrl, localModel, dimensions, restTemplateBuilder); - + LocalEmbeddingModel primaryModel = + new LocalEmbeddingModel(localUrl, localModel, dimensions, restTemplateBuilder); + // Secondary: Prefer remote OpenAI-compatible provider; else OpenAI direct if key present EmbeddingModel secondaryModel = null; if (remoteUrl != null && !remoteUrl.isBlank() && remoteApiKey != null && !remoteApiKey.isBlank()) { - log.info("[EMBEDDING] Configured remote OpenAI-compatible embedding fallback (urlId={})", - Integer.toHexString(Objects.hashCode(remoteUrl))); - secondaryModel = OpenAiCompatibleEmbeddingModel.create(remoteUrl, remoteApiKey, remoteModel, - remoteDims > 0 ? remoteDims : dimensions); + log.info( + "[EMBEDDING] Configured remote OpenAI-compatible embedding fallback (urlId={})", + Integer.toHexString(Objects.hashCode(remoteUrl))); + secondaryModel = OpenAiCompatibleEmbeddingModel.create( + remoteUrl, remoteApiKey, remoteModel, remoteDims > 0 ? remoteDims : dimensions); } else if (openaiApiKey != null && !openaiApiKey.trim().isEmpty()) { log.info("[EMBEDDING] Configured OpenAI embedding fallback"); - secondaryModel = OpenAiCompatibleEmbeddingModel.create(openaiBaseUrl, openaiApiKey, openaiModel, - dimensions); + secondaryModel = + OpenAiCompatibleEmbeddingModel.create(openaiBaseUrl, openaiApiKey, openaiModel, dimensions); } else { log.info("[EMBEDDING] No remote/OpenAI embedding fallback configured"); } - + // Tertiary: Hash-based fallback LocalHashingEmbeddingModel hashingModel = new LocalHashingEmbeddingModel(dimensions); - + return new GracefulEmbeddingModel(primaryModel, secondaryModel, hashingModel, useHashFallback); } - + /** * Creates a remote embedding model with hash-based fallback when local embeddings are disabled. * @@ -119,20 +121,21 @@ public EmbeddingModel openaiEmbeddingWithFallback( @Value("${spring.ai.openai.embedding.options.model:text-embedding-3-small}") String openaiModel, @Value("${app.local-embedding.use-hash-when-disabled:false}") boolean useHashFallback) { int embeddingDimensions = appProperties.getEmbeddings().getDimensions(); - + log.info("[EMBEDDING] Configuring remote/OpenAI embeddings with fallback strategies"); // Primary: Prefer remote provider; else OpenAI direct EmbeddingModel primary = null; if (remoteUrl != null && !remoteUrl.isBlank() && remoteApiKey != null && !remoteApiKey.isBlank()) { - log.info("[EMBEDDING] Using remote OpenAI-compatible embedding provider (urlId={})", - Integer.toHexString(Objects.hashCode(remoteUrl))); - primary = OpenAiCompatibleEmbeddingModel.create(remoteUrl, remoteApiKey, remoteModel, - remoteDims > 0 ? remoteDims : embeddingDimensions); + log.info( + "[EMBEDDING] Using remote OpenAI-compatible embedding provider (urlId={})", + Integer.toHexString(Objects.hashCode(remoteUrl))); + primary = OpenAiCompatibleEmbeddingModel.create( + remoteUrl, remoteApiKey, remoteModel, remoteDims > 0 ? remoteDims : embeddingDimensions); } else if (openaiApiKey != null && !openaiApiKey.trim().isEmpty()) { log.info("[EMBEDDING] Using OpenAI embeddings as primary provider"); - primary = OpenAiCompatibleEmbeddingModel.create(openaiBaseUrl, openaiApiKey, openaiModel, - embeddingDimensions); + primary = OpenAiCompatibleEmbeddingModel.create( + openaiBaseUrl, openaiApiKey, openaiModel, embeddingDimensions); } LocalHashingEmbeddingModel hashingModel = new LocalHashingEmbeddingModel(embeddingDimensions); @@ -144,7 +147,7 @@ public EmbeddingModel openaiEmbeddingWithFallback( log.warn("[EMBEDDING] No remote/OpenAI embedding configured. Falling back to hash-only mode."); return useHashFallback ? hashingModel : new NoOpEmbeddingModel(embeddingDimensions); } - + /** * No-op embedding model that always throws exceptions to trigger keyword search fallback */ @@ -156,19 +159,19 @@ private NoOpEmbeddingModel(int embeddingDimensions) { } @Override - public org.springframework.ai.embedding.EmbeddingResponse call(org.springframework.ai.embedding.EmbeddingRequest request) { + public org.springframework.ai.embedding.EmbeddingResponse call( + org.springframework.ai.embedding.EmbeddingRequest request) { throw new GracefulEmbeddingModel.EmbeddingServiceUnavailableException("No embedding service configured"); } - + @Override public int dimensions() { return embeddingDimensions; } - + @Override public float[] embed(org.springframework.ai.document.Document document) { throw new GracefulEmbeddingModel.EmbeddingServiceUnavailableException("No embedding service configured"); } } - } diff --git a/src/main/java/com/williamcallahan/javachat/config/InferenceConfig.java b/src/main/java/com/williamcallahan/javachat/config/InferenceConfig.java index e69de29..8b13789 100644 --- a/src/main/java/com/williamcallahan/javachat/config/InferenceConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/InferenceConfig.java @@ -0,0 +1 @@ + diff --git a/src/main/java/com/williamcallahan/javachat/config/LocalEmbedding.java b/src/main/java/com/williamcallahan/javachat/config/LocalEmbedding.java index 98bd20a..bc79349 100644 --- a/src/main/java/com/williamcallahan/javachat/config/LocalEmbedding.java +++ b/src/main/java/com/williamcallahan/javachat/config/LocalEmbedding.java @@ -27,8 +27,7 @@ public class LocalEmbedding { /** * Creates local embedding configuration. */ - public LocalEmbedding() { - } + public LocalEmbedding() {} /** * Validates local embedding settings. diff --git a/src/main/java/com/williamcallahan/javachat/config/LocalEmbeddingConfig.java b/src/main/java/com/williamcallahan/javachat/config/LocalEmbeddingConfig.java index 5392656..4dbace2 100644 --- a/src/main/java/com/williamcallahan/javachat/config/LocalEmbeddingConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/LocalEmbeddingConfig.java @@ -24,7 +24,7 @@ public class LocalEmbeddingConfig { public RestTemplateBuilder restTemplateBuilder() { return new RestTemplateBuilder(); } - + /** * Creates the local embedding model backed by the configured local server. * @@ -44,4 +44,3 @@ public EmbeddingModel localEmbeddingModel( return new LocalEmbeddingModel(baseUrl, modelName, dimensions, restTemplateBuilder); } } - diff --git a/src/main/java/com/williamcallahan/javachat/config/PortInitializer.java b/src/main/java/com/williamcallahan/javachat/config/PortInitializer.java index 79ad123..95ee7fa 100644 --- a/src/main/java/com/williamcallahan/javachat/config/PortInitializer.java +++ b/src/main/java/com/williamcallahan/javachat/config/PortInitializer.java @@ -7,7 +7,6 @@ import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.core.env.PropertySource; - /** * Ensures a usable server port is selected before Spring Boot starts. */ @@ -31,7 +30,8 @@ public class PortInitializer implements EnvironmentPostProcessor, Ordered { public void postProcessEnvironment(ConfigurableEnvironment environment, SpringApplication application) { // Disable port manipulation entirely when running under the 'test' profile for (String activeProfileName : environment.getActiveProfiles()) { - String normalizedProfile = AsciiTextNormalizer.toLowerAscii(activeProfileName == null ? "" : activeProfileName.trim()); + String normalizedProfile = + AsciiTextNormalizer.toLowerAscii(activeProfileName == null ? "" : activeProfileName.trim()); if (PROFILE_TEST.equals(normalizedProfile)) { System.err.println("[startup] PortInitializer disabled under 'test' profile"); return; @@ -68,9 +68,9 @@ public void postProcessEnvironment(ConfigurableEnvironment environment, SpringAp preferred = min; } - environment.getPropertySources().addFirst( - new ServerPortPropertySource(PORT_SOURCE_NAME, Integer.toString(preferred)) - ); + environment + .getPropertySources() + .addFirst(new ServerPortPropertySource(PORT_SOURCE_NAME, Integer.toString(preferred))); System.err.println("[startup] Using server.port=" + preferred + " (allowed range " + min + "-" + max + ")"); } diff --git a/src/main/java/com/williamcallahan/javachat/config/QdrantClientConfig.java b/src/main/java/com/williamcallahan/javachat/config/QdrantClientConfig.java index 9f0e7f1..6301e45 100644 --- a/src/main/java/com/williamcallahan/javachat/config/QdrantClientConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/QdrantClientConfig.java @@ -1,7 +1,7 @@ package com.williamcallahan.javachat.config; -import io.grpc.ManagedChannelBuilder; import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; import io.qdrant.client.QdrantClient; import io.qdrant.client.QdrantGrpcClient; import java.util.concurrent.TimeUnit; @@ -53,8 +53,7 @@ public class QdrantClientConfig { @Bean @Primary public QdrantClient qdrantClient() { - log.info("Creating QdrantClient with gRPC keepalive (host={}, port={}, tls={})", - host, port, useTls); + log.info("Creating QdrantClient with gRPC keepalive"); ManagedChannelBuilder channelBuilder = ManagedChannelBuilder.forAddress(host, port); if (useTls) { @@ -64,12 +63,15 @@ public QdrantClient qdrantClient() { } channelBuilder - .keepAliveTime(KEEPALIVE_TIME_SECONDS, TimeUnit.SECONDS) - .keepAliveTimeout(KEEPALIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS) - .keepAliveWithoutCalls(true) - .idleTimeout(IDLE_TIMEOUT_MINUTES, TimeUnit.MINUTES); - log.debug("gRPC keepalive configured: time={}s, timeout={}s, idleTimeout={}m", - KEEPALIVE_TIME_SECONDS, KEEPALIVE_TIMEOUT_SECONDS, IDLE_TIMEOUT_MINUTES); + .keepAliveTime(KEEPALIVE_TIME_SECONDS, TimeUnit.SECONDS) + .keepAliveTimeout(KEEPALIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .idleTimeout(IDLE_TIMEOUT_MINUTES, TimeUnit.MINUTES); + log.debug( + "gRPC keepalive configured: time={}s, timeout={}s, idleTimeout={}m", + KEEPALIVE_TIME_SECONDS, + KEEPALIVE_TIMEOUT_SECONDS, + IDLE_TIMEOUT_MINUTES); ManagedChannel channel = channelBuilder.build(); QdrantGrpcClient.Builder grpcClientBuilder = QdrantGrpcClient.newBuilder(channel, true); diff --git a/src/main/java/com/williamcallahan/javachat/config/QdrantHealthIndicator.java b/src/main/java/com/williamcallahan/javachat/config/QdrantHealthIndicator.java index 31b45eb..77c5307 100644 --- a/src/main/java/com/williamcallahan/javachat/config/QdrantHealthIndicator.java +++ b/src/main/java/com/williamcallahan/javachat/config/QdrantHealthIndicator.java @@ -27,20 +27,16 @@ public QdrantHealthIndicator(ExternalServiceHealth externalServiceHealth) { @Override public Health health() { - ExternalServiceHealth.ServiceInfo info = externalServiceHealth.getServiceInfo( - ExternalServiceHealth.SERVICE_QDRANT - ); + ExternalServiceHealth.ServiceInfo info = + externalServiceHealth.getServiceInfo(ExternalServiceHealth.SERVICE_QDRANT); if (info.isHealthy()) { - return Health.up() - .withDetail("status", info.message()) - .build(); + return Health.up().withDetail("status", info.message()).build(); } // Include time until next check for debugging - Health.Builder builder = Health.down() - .withDetail("status", info.message()); - + Health.Builder builder = Health.down().withDetail("status", info.message()); + if (info.timeUntilNextCheck() != null) { builder.withDetail("nextCheckIn", info.timeUntilNextCheck().toString()); } diff --git a/src/main/java/com/williamcallahan/javachat/config/QdrantIndexInitializer.java b/src/main/java/com/williamcallahan/javachat/config/QdrantIndexInitializer.java index dd9dffe..b787efa 100644 --- a/src/main/java/com/williamcallahan/javachat/config/QdrantIndexInitializer.java +++ b/src/main/java/com/williamcallahan/javachat/config/QdrantIndexInitializer.java @@ -1,8 +1,8 @@ package com.williamcallahan.javachat.config; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -93,8 +93,8 @@ private List restBaseUrls() { * @param restTemplateBuilder shared RestTemplate builder for HTTP configuration * @param embeddingModel embedding model for dimension validation */ - public QdrantIndexInitializer(AppProperties appProperties, RestTemplateBuilder restTemplateBuilder, - EmbeddingModel embeddingModel) { + public QdrantIndexInitializer( + AppProperties appProperties, RestTemplateBuilder restTemplateBuilder, EmbeddingModel embeddingModel) { this.ensurePayloadIndexes = appProperties.getQdrant().isEnsurePayloadIndexes(); this.embeddingModel = embeddingModel; // Build RestTemplate once and reuse for all index creation requests @@ -115,7 +115,7 @@ public QdrantIndexInitializer(AppProperties appProperties, RestTemplateBuilder r public void ensurePayloadIndexes() { // Validate embedding dimensions match collection configuration validateCollectionDimensions(); - + if (!ensurePayloadIndexes) { log.info("[QDRANT] Skipping payload index ensure (app.qdrant.ensure-payload-indexes=false)"); return; @@ -125,8 +125,9 @@ public void ensurePayloadIndexes() { createPayloadIndex("hash", SCHEMA_TYPE_KEYWORD); createPayloadIndex("chunkIndex", SCHEMA_TYPE_INTEGER); } catch (RuntimeException indexCreationException) { - log.warn("Unable to ensure Qdrant payload indexes (exception type: {})", - indexCreationException.getClass().getSimpleName()); + log.warn( + "Unable to ensure Qdrant payload indexes (exception type: {})", + indexCreationException.getClass().getSimpleName()); throw new IllegalStateException("Unable to ensure Qdrant payload indexes", indexCreationException); } } @@ -148,37 +149,38 @@ private void validateCollectionDimensions() { } String collectionPath = "/collections/" + collection; - + for (String base : restBaseUrls()) { String url = base + collectionPath; try { ResponseEntity response = restTemplate.exchange( - url, HttpMethod.GET, new HttpEntity<>(headers), QdrantCollectionInfoResponse.class - ); + url, HttpMethod.GET, new HttpEntity<>(headers), QdrantCollectionInfoResponse.class); Integer collectionDimensions = extractVectorDimensions(response.getBody()); if (collectionDimensions != null) { if (collectionDimensions != expectedDimensions) { - log.error("[QDRANT] DIMENSION MISMATCH: Collection '{}' has {} dimensions, " - + "but embedding model produces {} dimensions. " - + "This will cause all similarity searches to fail. " - + "Recreate the collection or switch back to the original embedding model.", - collection, collectionDimensions, expectedDimensions); + log.error( + "[QDRANT] DIMENSION MISMATCH: Collection has {} dimensions, " + + "but embedding model produces {} dimensions. " + + "This will cause all similarity searches to fail. " + + "Recreate the collection or switch back to the original embedding model.", + collectionDimensions, + expectedDimensions); // Log warning but don't fail startup - let the application try to run // The actual failure will happen on first search/insert } else { - log.info("[QDRANT] Collection '{}' dimensions ({}) match embedding model", - collection, collectionDimensions); + log.info("[QDRANT] Collection dimensions ({}) match embedding model", collectionDimensions); } } return; // Success - we got a response } catch (RuntimeException exception) { - log.debug("[QDRANT] Could not validate dimensions via {} ({})", - url, exception.getClass().getSimpleName()); + log.debug( + "[QDRANT] Could not validate dimensions (exceptionType={})", + exception.getClass().getSimpleName()); // Continue to next URL candidate } } - + log.info("[QDRANT] Could not validate collection dimensions (Qdrant may be unavailable)"); } @@ -189,7 +191,8 @@ private Integer extractVectorDimensions(QdrantCollectionInfoResponse responseBod if (responseBody == null || responseBody.collectionInfoResult() == null) { return null; } - QdrantCollectionConfig collectionConfig = responseBody.collectionInfoResult().collectionConfig(); + QdrantCollectionConfig collectionConfig = + responseBody.collectionInfoResult().collectionConfig(); if (collectionConfig == null || collectionConfig.collectionParams() == null) { return null; } @@ -226,23 +229,19 @@ private Integer extractVectorSize(JsonNode vectorsNode) { @JsonIgnoreProperties(ignoreUnknown = true) private record QdrantCollectionInfoResponse( - @JsonProperty("result") QdrantCollectionInfoResult collectionInfoResult - ) {} + @JsonProperty("result") QdrantCollectionInfoResult collectionInfoResult) {} @JsonIgnoreProperties(ignoreUnknown = true) private record QdrantCollectionInfoResult( - @JsonProperty("config") QdrantCollectionConfig collectionConfig - ) {} + @JsonProperty("config") QdrantCollectionConfig collectionConfig) {} @JsonIgnoreProperties(ignoreUnknown = true) private record QdrantCollectionConfig( - @JsonProperty("params") QdrantCollectionParams collectionParams - ) {} + @JsonProperty("params") QdrantCollectionParams collectionParams) {} @JsonIgnoreProperties(ignoreUnknown = true) private record QdrantCollectionParams( - @JsonProperty("vectors") JsonNode vectorsNode - ) {} + @JsonProperty("vectors") JsonNode vectorsNode) {} private void createPayloadIndex(String field, String schemaType) { HttpHeaders headers = new HttpHeaders(); @@ -251,7 +250,7 @@ private void createPayloadIndex(String field, String schemaType) { if (apiKey != null && !apiKey.isBlank()) { headers.set("api-key", apiKey); } - + // Qdrant API requires field_schema as an object: {"type": "keyword"} not just "keyword" PayloadIndexRequest body = new PayloadIndexRequest(field, Map.of("type", schemaType)); @@ -263,31 +262,33 @@ private void createPayloadIndex(String field, String schemaType) { String url = base + indexPath; try { // Use PUT for Qdrant payload index creation (official API) - ResponseEntity resp = restTemplate.exchange(url, HttpMethod.PUT, - new HttpEntity<>(body, headers), String.class); - log.info("[QDRANT] Ensured payload index field={} (status={})", field, resp.getStatusCode().value()); + ResponseEntity resp = + restTemplate.exchange(url, HttpMethod.PUT, new HttpEntity<>(body, headers), String.class); + log.info( + "[QDRANT] Ensured payload index (status={})", + resp.getStatusCode().value()); return; } catch (RuntimeException putEx) { lastError = putEx; - log.debug("[QDRANT] PUT failed for index field={} url={} (exceptionType={})", - field, url, putEx.getClass().getSimpleName()); + log.debug( + "[QDRANT] PUT failed for payload index (exceptionType={})", + putEx.getClass().getSimpleName()); // Continue to next URL candidate if available } } // If we reach here, all attempts failed; log once at INFO to avoid noisy warnings. - log.info("[QDRANT] Could not ensure payload index field={}. Last error type: {}", - field, lastError != null ? lastError.getClass().getSimpleName() : "unknown"); + log.info( + "[QDRANT] Could not ensure payload index. Last error type: {}", + lastError != null ? lastError.getClass().getSimpleName() : "unknown"); } /** * Request body for Qdrant payload index creation. - * + * * @param fieldName the field to index * @param fieldSchema schema object containing type, e.g., {"type": "keyword"} */ private record PayloadIndexRequest( - @JsonProperty("field_name") String fieldName, - @JsonProperty("field_schema") Map fieldSchema - ) { - } + @JsonProperty("field_name") String fieldName, + @JsonProperty("field_schema") Map fieldSchema) {} } diff --git a/src/main/java/com/williamcallahan/javachat/config/ReactorHooksConfig.java b/src/main/java/com/williamcallahan/javachat/config/ReactorHooksConfig.java index 77f6ca5..d55b1de 100644 --- a/src/main/java/com/williamcallahan/javachat/config/ReactorHooksConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/ReactorHooksConfig.java @@ -1,5 +1,7 @@ package com.williamcallahan.javachat.config; +import java.io.InterruptedIOException; +import java.util.Locale; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.annotation.Configuration; @@ -7,9 +9,6 @@ import org.springframework.context.event.EventListener; import reactor.core.publisher.Hooks; -import java.io.InterruptedIOException; -import java.util.Locale; - /** * Configures Reactor hooks to handle dropped errors gracefully. * @@ -31,7 +30,8 @@ public class ReactorHooksConfig { public void configureDroppedErrorHandler() { Hooks.onErrorDropped(error -> { if (isExpectedCancellationError(error)) { - log.debug("Dropped expected cancellation error (exceptionType={})", + log.debug( + "Dropped expected cancellation error (exceptionType={})", error.getClass().getSimpleName()); } else { log.warn("Dropped unexpected error", error); diff --git a/src/main/java/com/williamcallahan/javachat/config/RemoteEmbedding.java b/src/main/java/com/williamcallahan/javachat/config/RemoteEmbedding.java index 3f7b09c..1915891 100644 --- a/src/main/java/com/williamcallahan/javachat/config/RemoteEmbedding.java +++ b/src/main/java/com/williamcallahan/javachat/config/RemoteEmbedding.java @@ -28,8 +28,7 @@ public class RemoteEmbedding { /** * Creates remote embedding configuration. */ - public RemoteEmbedding() { - } + public RemoteEmbedding() {} /** * Validates remote embedding settings. diff --git a/src/main/java/com/williamcallahan/javachat/config/RetrievalAugmentationConfig.java b/src/main/java/com/williamcallahan/javachat/config/RetrievalAugmentationConfig.java index 92201ef..1065a1a 100644 --- a/src/main/java/com/williamcallahan/javachat/config/RetrievalAugmentationConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/RetrievalAugmentationConfig.java @@ -37,8 +37,7 @@ public class RetrievalAugmentationConfig { /** * Creates retrieval augmentation configuration. */ - public RetrievalAugmentationConfig() { - } + public RetrievalAugmentationConfig() {} private static final String RETURN_K_BOUND_MSG = "%s must be less than or equal to %s (got %d > %d)."; private static final String OVERLAP_BOUND_MSG = "%s must be less than %s (got %d >= %d)."; @@ -54,12 +53,12 @@ public void validateConfiguration() { requireNonNegativeCount(CITE_KEY, searchCitations); requireLambdaRange(); if (searchReturnK > searchTopK) { - throw new IllegalArgumentException(String.format( - Locale.ROOT, RETURN_K_BOUND_MSG, RETURN_K_KEY, TOP_K_KEY, searchReturnK, searchTopK)); + throw new IllegalArgumentException( + String.format(Locale.ROOT, RETURN_K_BOUND_MSG, RETURN_K_KEY, TOP_K_KEY, searchReturnK, searchTopK)); } if (overlapTokens >= chunkMaxTokens) { throw new IllegalArgumentException(String.format( - Locale.ROOT, OVERLAP_BOUND_MSG, OVERLAP_KEY, CHUNK_MAX_KEY, overlapTokens, chunkMaxTokens)); + Locale.ROOT, OVERLAP_BOUND_MSG, OVERLAP_KEY, CHUNK_MAX_KEY, overlapTokens, chunkMaxTokens)); } } @@ -185,13 +184,8 @@ private void requireNonNegativeCount(final String propertyKey, final int count) private void requireLambdaRange() { if (searchMmrLambda < MMR_MIN || searchMmrLambda > MMR_MAX) { - throw new IllegalArgumentException(String.format( - Locale.ROOT, - RANGE_FMT, - MMR_KEY, - Double.toString(MMR_MIN), - Double.toString(MMR_MAX) - )); + throw new IllegalArgumentException( + String.format(Locale.ROOT, RANGE_FMT, MMR_KEY, Double.toString(MMR_MIN), Double.toString(MMR_MAX))); } } } diff --git a/src/main/java/com/williamcallahan/javachat/config/SecurityConfig.java b/src/main/java/com/williamcallahan/javachat/config/SecurityConfig.java index e950c56..24d31e1 100644 --- a/src/main/java/com/williamcallahan/javachat/config/SecurityConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/SecurityConfig.java @@ -1,5 +1,6 @@ package com.williamcallahan.javachat.config; +import java.util.List; import org.springframework.boot.actuate.autoconfigure.security.servlet.EndpointRequest; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -9,7 +10,6 @@ import org.springframework.web.cors.CorsConfiguration; import org.springframework.web.cors.CorsConfigurationSource; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; -import java.util.List; /** * Security configuration for API endpoints and static resources. @@ -50,19 +50,16 @@ public CorsConfigurationSource corsConfigurationSource(AppProperties appProperti */ @Bean @Order(0) - public SecurityFilterChain managementSecurityFilterChain(HttpSecurity http, - CorsConfigurationSource corsConfigurationSource) throws Exception { - http - .securityMatcher(EndpointRequest.toAnyEndpoint()) - .cors(c -> c.configurationSource(corsConfigurationSource)) - .authorizeHttpRequests(auth -> auth - .anyRequest().permitAll() - ) - // Allow same-origin iframes (used by tab shell loading chat.html/guided.html) - .headers(h -> h.frameOptions(fo -> fo.sameOrigin())) - .csrf(csrf -> csrf.ignoringRequestMatchers(EndpointRequest.toAnyEndpoint())) - .httpBasic(b -> b.disable()) - .formLogin(f -> f.disable()); + public SecurityFilterChain managementSecurityFilterChain( + HttpSecurity http, CorsConfigurationSource corsConfigurationSource) throws Exception { + http.securityMatcher(EndpointRequest.toAnyEndpoint()) + .cors(c -> c.configurationSource(corsConfigurationSource)) + .authorizeHttpRequests(auth -> auth.anyRequest().permitAll()) + // Allow same-origin iframes (used by tab shell loading chat.html/guided.html) + .headers(h -> h.frameOptions(fo -> fo.sameOrigin())) + .csrf(csrf -> csrf.ignoringRequestMatchers(EndpointRequest.toAnyEndpoint())) + .httpBasic(b -> b.disable()) + .formLogin(f -> f.disable()); return http.build(); } @@ -71,29 +68,28 @@ public SecurityFilterChain managementSecurityFilterChain(HttpSecurity http, */ @Bean @Order(1) - public SecurityFilterChain appSecurityFilterChain(HttpSecurity http, - CorsConfigurationSource corsConfigurationSource) throws Exception { - http - .cors(c -> c.configurationSource(corsConfigurationSource)) - .csrf(csrf -> csrf.ignoringRequestMatchers("/api/**")) - .authorizeHttpRequests(auth -> auth - .requestMatchers( - "/", - "/index.html", - "/chat.html", - "/guided.html", - "/favicon.ico", - "/app/**", - "/assets/**", - "/static/**" - ).permitAll() - .requestMatchers("/api/**").permitAll() - .anyRequest().permitAll() - ) - // Allow same-origin iframes (used by tab shell loading chat.html/guided.html) - .headers(h -> h.frameOptions(fo -> fo.sameOrigin())) - .httpBasic(b -> b.disable()) - .formLogin(f -> f.disable()); + public SecurityFilterChain appSecurityFilterChain( + HttpSecurity http, CorsConfigurationSource corsConfigurationSource) throws Exception { + http.cors(c -> c.configurationSource(corsConfigurationSource)) + .csrf(csrf -> csrf.ignoringRequestMatchers("/api/**")) + .authorizeHttpRequests(auth -> auth.requestMatchers( + "/", + "/index.html", + "/chat.html", + "/guided.html", + "/favicon.ico", + "/app/**", + "/assets/**", + "/static/**") + .permitAll() + .requestMatchers("/api/**") + .permitAll() + .anyRequest() + .permitAll()) + // Allow same-origin iframes (used by tab shell loading chat.html/guided.html) + .headers(h -> h.frameOptions(fo -> fo.sameOrigin())) + .httpBasic(b -> b.disable()) + .formLogin(f -> f.disable()); return http.build(); } } diff --git a/src/main/java/com/williamcallahan/javachat/config/SpringDocsUrlNormalizer.java b/src/main/java/com/williamcallahan/javachat/config/SpringDocsUrlNormalizer.java index 8e2eff3..8918613 100644 --- a/src/main/java/com/williamcallahan/javachat/config/SpringDocsUrlNormalizer.java +++ b/src/main/java/com/williamcallahan/javachat/config/SpringDocsUrlNormalizer.java @@ -24,59 +24,26 @@ final class SpringDocsUrlNormalizer { private static final int FIRST_PATH_SEGMENT_INDEX = 1; private static final int URL_BUFFER_PADDING = 64; - private static final String[] DOCS_CURRENT_REFERENCE_SEQUENCE = { - DOCS_SEGMENT, - CURRENT_SEGMENT, - REFERENCE_SEGMENT - }; + private static final String[] DOCS_CURRENT_REFERENCE_SEQUENCE = {DOCS_SEGMENT, CURRENT_SEGMENT, REFERENCE_SEGMENT}; private static final String[] REFERENCE_ROOT_SEQUENCE = {REFERENCE_SEGMENT}; private static final String[] FRAMEWORK_API_CURRENT_SEQUENCE = { - DOCS_SEGMENT, - CURRENT_SEGMENT, - API_SEGMENT, - CURRENT_SEGMENT, - JAVADOC_API_SEGMENT + DOCS_SEGMENT, CURRENT_SEGMENT, API_SEGMENT, CURRENT_SEGMENT, JAVADOC_API_SEGMENT }; private static final String[] FRAMEWORK_API_JAVA_SEQUENCE = { - DOCS_SEGMENT, - CURRENT_SEGMENT, - JAVADOC_API_SEGMENT, - JAVA_SEGMENT - }; - private static final String[] BOOT_API_JAVA_SEQUENCE = { - DOCS_SEGMENT, - CURRENT_SEGMENT, - API_SEGMENT, - JAVA_SEGMENT + DOCS_SEGMENT, CURRENT_SEGMENT, JAVADOC_API_SEGMENT, JAVA_SEGMENT }; + private static final String[] BOOT_API_JAVA_SEQUENCE = {DOCS_SEGMENT, CURRENT_SEGMENT, API_SEGMENT, JAVA_SEGMENT}; - private static final SpringDocsUrlPrefix SPRING_FRAMEWORK_REFERENCE_PREFIX = new SpringDocsUrlPrefix( - SPRING_FRAMEWORK_SEGMENT, - REFERENCE_SEGMENT, - CURRENT_SEGMENT, - null - ); - private static final SpringDocsUrlPrefix SPRING_FRAMEWORK_JAVADOC_PREFIX = new SpringDocsUrlPrefix( - SPRING_FRAMEWORK_SEGMENT, - DOCS_SEGMENT, - CURRENT_SEGMENT, - JAVADOC_API_SEGMENT - ); - private static final SpringDocsUrlPrefix SPRING_BOOT_REFERENCE_PREFIX = new SpringDocsUrlPrefix( - SPRING_BOOT_SEGMENT, - REFERENCE_SEGMENT, - CURRENT_SEGMENT, - null - ); - private static final SpringDocsUrlPrefix SPRING_BOOT_API_PREFIX = new SpringDocsUrlPrefix( - SPRING_BOOT_SEGMENT, - DOCS_SEGMENT, - CURRENT_SEGMENT, - API_SEGMENT - ); + private static final SpringDocsUrlPrefix SPRING_FRAMEWORK_REFERENCE_PREFIX = + new SpringDocsUrlPrefix(SPRING_FRAMEWORK_SEGMENT, REFERENCE_SEGMENT, CURRENT_SEGMENT, null); + private static final SpringDocsUrlPrefix SPRING_FRAMEWORK_JAVADOC_PREFIX = + new SpringDocsUrlPrefix(SPRING_FRAMEWORK_SEGMENT, DOCS_SEGMENT, CURRENT_SEGMENT, JAVADOC_API_SEGMENT); + private static final SpringDocsUrlPrefix SPRING_BOOT_REFERENCE_PREFIX = + new SpringDocsUrlPrefix(SPRING_BOOT_SEGMENT, REFERENCE_SEGMENT, CURRENT_SEGMENT, null); + private static final SpringDocsUrlPrefix SPRING_BOOT_API_PREFIX = + new SpringDocsUrlPrefix(SPRING_BOOT_SEGMENT, DOCS_SEGMENT, CURRENT_SEGMENT, API_SEGMENT); - private SpringDocsUrlNormalizer() { - } + private SpringDocsUrlNormalizer() {} static String normalize(final String url) { String normalizedUrl = url; @@ -197,18 +164,15 @@ private static String normalizeSpringBootApiJava(final String[] segments) { return normalizedUrl; } - private static String buildUrl( - final SpringDocsUrlPrefix prefix, - final String[] segments, - final int startIndex - ) { + private static String buildUrl(final SpringDocsUrlPrefix prefix, final String[] segments, final int startIndex) { final StringBuilder urlBuilder = new StringBuilder(SPRING_DOCS_PREFIX.length() + URL_BUFFER_PADDING); - urlBuilder.append(SPRING_DOCS_PREFIX) - .append(prefix.projectSegment()) - .append(PATH_SEPARATOR) - .append(prefix.primarySegment()) - .append(PATH_SEPARATOR) - .append(prefix.secondarySegment()); + urlBuilder + .append(SPRING_DOCS_PREFIX) + .append(prefix.projectSegment()) + .append(PATH_SEPARATOR) + .append(prefix.primarySegment()) + .append(PATH_SEPARATOR) + .append(prefix.secondarySegment()); if (prefix.tertiarySegment() != null) { urlBuilder.append(PATH_SEPARATOR).append(prefix.tertiarySegment()); } @@ -219,10 +183,7 @@ private static String buildUrl( } private static boolean matchesSequence( - final String[] segments, - final int startIndex, - final String[] expectedSegments - ) { + final String[] segments, final int startIndex, final String[] expectedSegments) { final int expectedLength = expectedSegments.length; boolean matches = false; if (segments.length >= startIndex + expectedLength) { @@ -254,17 +215,12 @@ private static boolean isVersionString(final String text) { boolean isVersion = false; if (text != null && !text.isEmpty()) { final char firstChar = text.charAt(0); - isVersion = text.contains(VERSION_SEPARATOR) - && (Character.isDigit(firstChar) || firstChar == VERSION_PREFIX); + isVersion = + text.contains(VERSION_SEPARATOR) && (Character.isDigit(firstChar) || firstChar == VERSION_PREFIX); } return isVersion; } private record SpringDocsUrlPrefix( - String projectSegment, - String primarySegment, - String secondarySegment, - String tertiarySegment - ) { - } + String projectSegment, String primarySegment, String secondarySegment, String tertiarySegment) {} } diff --git a/src/main/java/com/williamcallahan/javachat/config/SystemPromptConfig.java b/src/main/java/com/williamcallahan/javachat/config/SystemPromptConfig.java index 364d6a5..011cc53 100644 --- a/src/main/java/com/williamcallahan/javachat/config/SystemPromptConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/SystemPromptConfig.java @@ -1,7 +1,7 @@ package com.williamcallahan.javachat.config; -import org.springframework.context.annotation.Configuration; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; /** * Centralized system prompt configuration for DRY principle. @@ -18,7 +18,7 @@ public class SystemPromptConfig { When answering questions, follow this priority: 1. Use provided context from our RAG retrievals (Qdrant vector embeddings) containing: - Official Java JDK documentation - - Spring Framework documentation + - Spring Framework documentation - Think Java 2nd edition textbook - Related Java ecosystem documentation 2. If RAG data is unavailable or insufficient for the query, provide the most accurate answer based on your training knowledge @@ -51,27 +51,27 @@ Simply reference sources naturally in prose when relevant (e.g., "the JDK docume - When RAG data is unavailable, clearly state you're using knowledge from your training cutoff - Be explicit about version-specific features when relevant """; - + @Value("${DOCS_JDK_VERSION:24}") private String jdkVersion; - + /** * Core system prompt shared by all models (OpenAI, GitHub Models, etc.) */ public String getCoreSystemPrompt() { return CORE_PROMPT_TEMPLATE.replace(JDK_VERSION_PLACEHOLDER, jdkVersion); } - + /** * Get prompt for when search quality is poor */ public String getLowQualitySearchPrompt() { return """ - Note: Search results may be less relevant than usual. + Note: Search results may be less relevant than usual. Feel free to supplement with general Java knowledge while maintaining accuracy. """; } - + /** * Get prompt for guided/structured learning mode */ @@ -81,7 +81,7 @@ public String getGuidedLearningPrompt() { Break down complex concepts into digestible parts and build understanding progressively. """; } - + /** * Get prompt for code review/analysis mode */ @@ -95,7 +95,7 @@ public String getCodeReviewPrompt() { Use the learning markers to highlight key insights. """; } - + /** * Combine base prompt with context-specific additions */ diff --git a/src/main/java/com/williamcallahan/javachat/config/WebMvcConfig.java b/src/main/java/com/williamcallahan/javachat/config/WebMvcConfig.java index e59cc8c..6323916 100644 --- a/src/main/java/com/williamcallahan/javachat/config/WebMvcConfig.java +++ b/src/main/java/com/williamcallahan/javachat/config/WebMvcConfig.java @@ -1,10 +1,10 @@ package com.williamcallahan.javachat.config; +import java.util.List; import org.springframework.context.annotation.Configuration; import org.springframework.web.servlet.config.annotation.CorsRegistry; import org.springframework.web.servlet.config.annotation.ViewControllerRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; -import java.util.List; /** * MVC configuration for CORS and SPA routing. @@ -52,11 +52,10 @@ public void addCorsMappings(CorsRegistry registry) { } else { mapping.allowedOrigins(origins.toArray(String[]::new)); } - mapping - .allowedMethods(allowedMethods.toArray(String[]::new)) - .allowedHeaders(allowedHeaders.toArray(String[]::new)) - .allowCredentials(allowCredentials) - .maxAge(maxAgeSeconds); + mapping.allowedMethods(allowedMethods.toArray(String[]::new)) + .allowedHeaders(allowedHeaders.toArray(String[]::new)) + .allowCredentials(allowCredentials) + .maxAge(maxAgeSeconds); } /** diff --git a/src/main/java/com/williamcallahan/javachat/domain/RetrievedContent.java b/src/main/java/com/williamcallahan/javachat/domain/RetrievedContent.java new file mode 100644 index 0000000..7c04343 --- /dev/null +++ b/src/main/java/com/williamcallahan/javachat/domain/RetrievedContent.java @@ -0,0 +1,28 @@ +package com.williamcallahan.javachat.domain; + +import java.util.Optional; + +/** + * Domain abstraction for retrieved document content used in search quality evaluation. + * + *

Decouples domain logic from Spring AI's Document class, allowing the domain layer + * to remain framework-free per clean architecture principles (AR4).

+ */ +public interface RetrievedContent { + + /** + * Returns the text content of this retrieved item. + * + * @return the content text, empty if no content is available + */ + Optional getText(); + + /** + * Returns the source URL of this retrieved item, if available. + * + *

Used to identify retrieval source characteristics (e.g., keyword search markers).

+ * + * @return the source URL, empty if unknown + */ + Optional getSourceUrl(); +} diff --git a/src/main/java/com/williamcallahan/javachat/domain/SearchQualityLevel.java b/src/main/java/com/williamcallahan/javachat/domain/SearchQualityLevel.java new file mode 100644 index 0000000..bb4211f --- /dev/null +++ b/src/main/java/com/williamcallahan/javachat/domain/SearchQualityLevel.java @@ -0,0 +1,123 @@ +package com.williamcallahan.javachat.domain; + +import java.util.List; + +/** + * Categorizes the quality of search results for contextualizing LLM responses. + * + *

Replaces sequential boolean checks with self-describing behavior. Each level + * knows how to describe itself to the LLM so responses can be appropriately calibrated.

+ */ +public enum SearchQualityLevel { + /** + * No documents were retrieved; LLM must rely on general knowledge. + */ + NONE, + + /** + * Documents came from keyword/fallback search rather than semantic embeddings. + */ + KEYWORD_SEARCH, + + /** + * All retrieved documents are high-quality semantic matches. + */ + HIGH_QUALITY, + + /** + * Mix of high and lower quality results from semantic search. + */ + MIXED_QUALITY; + + private static final String MESSAGE_NONE = "No relevant documents found. Using general knowledge only."; + private static final String MESSAGE_KEYWORD_SEARCH = + "Found %d documents via keyword search (embedding service unavailable). Results may be less semantically relevant."; + private static final String MESSAGE_HIGH_QUALITY = "Found %d high-quality relevant documents via semantic search."; + private static final String MESSAGE_MIXED_QUALITY = + "Found %d documents (%d high-quality) via search. Some results may be less relevant."; + private static final String KEYWORD_MARKER_LOCAL_SEARCH = "local-search"; + private static final String KEYWORD_MARKER_KEYWORD = "keyword"; + + /** + * Minimum content length to consider a document as having substantial content. + * Documents shorter than this threshold are classified as lower quality. + */ + private static final int SUBSTANTIAL_CONTENT_THRESHOLD = 100; + + /** + * Formats the quality message with document counts. + * + * @param totalCount total number of documents + * @param highQualityCount number of high-quality documents + * @return formatted message for the LLM + */ + public String formatMessage(int totalCount, int highQualityCount) { + return switch (this) { + case NONE -> MESSAGE_NONE; + case KEYWORD_SEARCH -> MESSAGE_KEYWORD_SEARCH.formatted(totalCount); + case HIGH_QUALITY -> MESSAGE_HIGH_QUALITY.formatted(totalCount); + case MIXED_QUALITY -> MESSAGE_MIXED_QUALITY.formatted(totalCount, highQualityCount); + }; + } + + /** + * Counts documents with substantial content (length exceeds threshold). + * + * @param contents the retrieved contents to evaluate + * @return count of high-quality documents with substantial content + */ + private static long countHighQuality(List contents) { + if (contents == null) { + return 0; + } + return contents.stream() + .filter(content -> content.getText() + .filter(text -> text.length() > SUBSTANTIAL_CONTENT_THRESHOLD) + .isPresent()) + .count(); + } + + /** + * Determines the search quality level for a set of retrieved contents. + * + * @param contents the retrieved contents + * @return the appropriate quality level + */ + public static SearchQualityLevel determine(List contents) { + if (contents == null || contents.isEmpty()) { + return NONE; + } + + // Check if documents came from keyword/fallback search + boolean likelyKeywordSearch = contents.stream().anyMatch(content -> content.getSourceUrl() + .filter(url -> url.contains(KEYWORD_MARKER_LOCAL_SEARCH) || url.contains(KEYWORD_MARKER_KEYWORD)) + .isPresent()); + + if (likelyKeywordSearch) { + return KEYWORD_SEARCH; + } + + // Count high-quality documents (has substantial content) + long highQualityCount = countHighQuality(contents); + + if (highQualityCount == contents.size()) { + return HIGH_QUALITY; + } + + return MIXED_QUALITY; + } + + /** + * Generates the complete search quality note for the contents. + * + * @param contents the retrieved contents + * @return formatted quality message + */ + public static String describeQuality(List contents) { + SearchQualityLevel level = determine(contents); + int totalCount = contents != null ? contents.size() : 0; + long highQualityCount = countHighQuality(contents); + + return level.formatMessage(totalCount, (int) highQualityCount); + } +} diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/Background.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/Background.java index 3cb8f69..c6cde75 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/Background.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/Background.java @@ -5,7 +5,7 @@ * Provides contextual information and explanations. */ public record Background(String content, EnrichmentPriority priority, int position) implements MarkdownEnrichment { - + public Background { if (content == null || content.trim().isEmpty()) { throw new IllegalArgumentException("Background content cannot be null or empty"); @@ -17,7 +17,7 @@ public record Background(String content, EnrichmentPriority priority, int positi throw new IllegalArgumentException("Background position must be non-negative"); } } - + /** * Creates a background element with low priority. * @param content the background content @@ -27,7 +27,7 @@ public record Background(String content, EnrichmentPriority priority, int positi public static Background create(String content, int position) { return new Background(content, EnrichmentPriority.LOW, position); } - + @Override public String type() { return "background"; diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/CitationType.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/CitationType.java index db9343a..049f3d8 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/CitationType.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/CitationType.java @@ -11,38 +11,38 @@ public enum CitationType { * External HTTP/HTTPS link. */ EXTERNAL_LINK("external"), - + /** * PDF document reference. */ PDF_DOCUMENT("pdf"), - + /** * Local application link. */ LOCAL_LINK("local"), - + /** * API documentation reference. */ API_DOCUMENTATION("api-doc"), - + /** * Code repository reference. */ CODE_REPOSITORY("repo"), - + /** * Unknown or unclassified link type. */ UNKNOWN("unknown"); - + private final String identifier; - + CitationType(String identifier) { this.identifier = identifier; } - + /** * Gets the string identifier for this citation type. * @return string identifier @@ -50,7 +50,7 @@ public enum CitationType { public String getIdentifier() { return identifier; } - + /** * Determines citation type from URL. * @param url The URL to analyze @@ -60,29 +60,32 @@ public static CitationType fromUrl(String url) { if (url == null || url.isEmpty()) { return UNKNOWN; } - + String lowerUrl = url.toLowerCase(Locale.ROOT); - + if (lowerUrl.endsWith(".pdf")) { return PDF_DOCUMENT; } - + if (lowerUrl.startsWith("http://") || lowerUrl.startsWith("https://")) { - if (lowerUrl.contains("docs.oracle.com") || lowerUrl.contains("javadoc") || - lowerUrl.contains("/api/") || lowerUrl.contains("/docs/api/")) { + if (lowerUrl.contains("docs.oracle.com") + || lowerUrl.contains("javadoc") + || lowerUrl.contains("/api/") + || lowerUrl.contains("/docs/api/")) { return API_DOCUMENTATION; } - if (lowerUrl.contains("github.com") || lowerUrl.contains("gitlab.com") || - lowerUrl.contains("bitbucket.org")) { + if (lowerUrl.contains("github.com") + || lowerUrl.contains("gitlab.com") + || lowerUrl.contains("bitbucket.org")) { return CODE_REPOSITORY; } return EXTERNAL_LINK; } - + if (lowerUrl.startsWith("/")) { return LOCAL_LINK; } - + return UNKNOWN; } } diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/EnrichmentPriority.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/EnrichmentPriority.java index 4d27ed4..5198005 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/EnrichmentPriority.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/EnrichmentPriority.java @@ -9,33 +9,33 @@ public enum EnrichmentPriority { * Critical warnings that must be shown prominently. */ CRITICAL(100), - + /** * High priority items like warnings and important reminders. */ HIGH(75), - + /** * Medium priority items like hints and examples. */ MEDIUM(50), - + /** * Low priority items like background information. */ LOW(25), - + /** * Informational items with minimal visual impact. */ INFO(10); - + private final int value; - + EnrichmentPriority(int value) { this.value = value; } - + /** * Gets the numeric priority value. * @return priority value (higher = more important) @@ -43,7 +43,7 @@ public enum EnrichmentPriority { public int getValue() { return value; } - + /** * Compares this priority with another. * @param other the other priority diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/Example.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/Example.java index 80658aa..6e61f92 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/Example.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/Example.java @@ -5,7 +5,7 @@ * Provides code examples and demonstrations. */ public record Example(String content, EnrichmentPriority priority, int position) implements MarkdownEnrichment { - + public Example { if (content == null || content.trim().isEmpty()) { throw new IllegalArgumentException("Example content cannot be null or empty"); @@ -17,7 +17,7 @@ public record Example(String content, EnrichmentPriority priority, int position) throw new IllegalArgumentException("Example position must be non-negative"); } } - + /** * Creates an example with medium priority. * @param content the example content @@ -27,7 +27,7 @@ public record Example(String content, EnrichmentPriority priority, int position) public static Example create(String content, int position) { return new Example(content, EnrichmentPriority.MEDIUM, position); } - + @Override public String type() { return "example"; diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/Hint.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/Hint.java index 5ca5ffe..9d44505 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/Hint.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/Hint.java @@ -5,7 +5,7 @@ * Provides helpful tips and suggestions to users. */ public record Hint(String content, EnrichmentPriority priority, int position) implements MarkdownEnrichment { - + public Hint { if (content == null || content.trim().isEmpty()) { throw new IllegalArgumentException("Hint content cannot be null or empty"); @@ -17,7 +17,7 @@ public record Hint(String content, EnrichmentPriority priority, int position) im throw new IllegalArgumentException("Hint position must be non-negative"); } } - + /** * Creates a hint with medium priority. * @param content the hint content @@ -27,7 +27,7 @@ public record Hint(String content, EnrichmentPriority priority, int position) im public static Hint create(String content, int position) { return new Hint(content, EnrichmentPriority.MEDIUM, position); } - + @Override public String type() { return "hint"; diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheClearResponse.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheClearResponse.java index 4d93341..3d93be5 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheClearResponse.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheClearResponse.java @@ -3,6 +3,4 @@ /** * Represents the response variants for markdown cache clear requests. */ -public sealed interface MarkdownCacheClearResponse - permits MarkdownCacheClearOutcome, MarkdownErrorResponse { -} +public sealed interface MarkdownCacheClearResponse permits MarkdownCacheClearOutcome, MarkdownErrorResponse {} diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheStatsResponse.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheStatsResponse.java index 330b319..320ca84 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheStatsResponse.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheStatsResponse.java @@ -3,6 +3,4 @@ /** * Represents the response variants for markdown cache statistics requests. */ -public sealed interface MarkdownCacheStatsResponse - permits MarkdownCacheStatsSnapshot, MarkdownErrorResponse { -} +public sealed interface MarkdownCacheStatsResponse permits MarkdownCacheStatsSnapshot, MarkdownErrorResponse {} diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheStatsSnapshot.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheStatsSnapshot.java index 205ac07..91fdeb4 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheStatsSnapshot.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCacheStatsSnapshot.java @@ -5,13 +5,8 @@ /** * Captures cache statistics for markdown rendering. */ -public record MarkdownCacheStatsSnapshot( - long hitCount, - long missCount, - long evictionCount, - long size, - String hitRate -) implements MarkdownCacheStatsResponse { +public record MarkdownCacheStatsSnapshot(long hitCount, long missCount, long evictionCount, long size, String hitRate) + implements MarkdownCacheStatsResponse { public MarkdownCacheStatsSnapshot { Objects.requireNonNull(hitRate, "Hit rate string cannot be null"); if (hitCount < 0 || missCount < 0 || evictionCount < 0 || size < 0) { diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCitation.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCitation.java index d40377a..788a382 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCitation.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownCitation.java @@ -6,18 +6,12 @@ /** * Represents a structured citation extracted from markdown content. * This replaces string-based citation processing with typed objects. - * + * * Note: Named MarkdownCitation to avoid conflict with existing model.Citation class. */ public record MarkdownCitation( - String url, - String title, - String snippet, - CitationType type, - int position, - LocalDateTime extractedAt -) { - + String url, String title, String snippet, CitationType type, int position, LocalDateTime extractedAt) { + public MarkdownCitation { Objects.requireNonNull(url, "Citation URL cannot be null"); Objects.requireNonNull(title, "Citation title cannot be null"); @@ -26,7 +20,7 @@ public record MarkdownCitation( throw new IllegalArgumentException("Citation position must be non-negative"); } } - + /** * Creates a citation with current timestamp. * @param url The citation URL @@ -39,7 +33,7 @@ public record MarkdownCitation( public static MarkdownCitation create(String url, String title, String snippet, CitationType type, int position) { return new MarkdownCitation(url, title, snippet != null ? snippet : "", type, position, LocalDateTime.now()); } - + /** * Checks if this citation has a snippet. * @return true if snippet is not empty @@ -47,7 +41,7 @@ public static MarkdownCitation create(String url, String title, String snippet, public boolean hasSnippet() { return snippet != null && !snippet.trim().isEmpty(); } - + /** * Gets the domain from the URL for display purposes. * @return domain string or "unknown" if URL is invalid diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownEnrichment.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownEnrichment.java index 3b5fc70..5724434 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownEnrichment.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownEnrichment.java @@ -3,36 +3,35 @@ /** * Base interface for structured enrichment elements. * This replaces regex-based enrichment processing with type-safe objects. - * + * * Note: Named MarkdownEnrichment to avoid conflict with existing model.Enrichment class. */ -public sealed interface MarkdownEnrichment - permits Hint, Warning, Background, Example, Reminder { - +public sealed interface MarkdownEnrichment permits Hint, Warning, Background, Example, Reminder { + /** * Gets the enrichment type identifier. * @return type string */ String type(); - + /** * Gets the enrichment content. * @return content string */ String content(); - + /** * Gets the enrichment priority for rendering order. * @return priority level */ EnrichmentPriority priority(); - + /** * Gets the position in the document where this enrichment was found. * @return document position */ int position(); - + /** * Checks if this enrichment has non-empty content. * @return true if content is not empty diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownErrorResponse.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownErrorResponse.java index ed0d294..e8fb5b5 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownErrorResponse.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownErrorResponse.java @@ -6,7 +6,7 @@ * Describes a markdown rendering failure. */ public record MarkdownErrorResponse(String error, String details) - implements MarkdownRenderResponse, MarkdownCacheStatsResponse, MarkdownCacheClearResponse { + implements MarkdownRenderResponse, MarkdownCacheStatsResponse, MarkdownCacheClearResponse { public MarkdownErrorResponse { Objects.requireNonNull(error, "Error message cannot be null"); details = details == null ? "" : details; diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderOutcome.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderOutcome.java index c2a189d..ddb2408 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderOutcome.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderOutcome.java @@ -5,13 +5,8 @@ /** * Describes the outcome of rendering markdown to HTML for standard endpoints. */ -public record MarkdownRenderOutcome( - String html, - String source, - boolean cached, - int citations, - int enrichments -) implements MarkdownRenderResponse { +public record MarkdownRenderOutcome(String html, String source, boolean cached, int citations, int enrichments) + implements MarkdownRenderResponse { public MarkdownRenderOutcome { Objects.requireNonNull(html, "Rendered HTML cannot be null"); Objects.requireNonNull(source, "Render source cannot be null"); diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderRequest.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderRequest.java index b1887c7..e8c8f79 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderRequest.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderRequest.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; /** diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderResponse.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderResponse.java index 34f75ea..6fe4682 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderResponse.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownRenderResponse.java @@ -3,6 +3,4 @@ /** * Represents the response variants for standard markdown rendering endpoints. */ -public sealed interface MarkdownRenderResponse - permits MarkdownRenderOutcome, MarkdownErrorResponse { -} +public sealed interface MarkdownRenderResponse permits MarkdownRenderOutcome, MarkdownErrorResponse {} diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredErrorResponse.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredErrorResponse.java index 809dbb9..cd40997 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredErrorResponse.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredErrorResponse.java @@ -5,7 +5,8 @@ /** * Describes a structured markdown rendering failure with a source marker. */ -public record MarkdownStructuredErrorResponse(String error, String source, String details) implements MarkdownStructuredResponse { +public record MarkdownStructuredErrorResponse(String error, String source, String details) + implements MarkdownStructuredResponse { public MarkdownStructuredErrorResponse { Objects.requireNonNull(error, "Error message cannot be null"); Objects.requireNonNull(source, "Error source cannot be null"); diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredOutcome.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredOutcome.java index 948bce8..07c5e72 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredOutcome.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredOutcome.java @@ -7,21 +7,24 @@ * Represents structured markdown rendering with typed citations and enrichments. */ public record MarkdownStructuredOutcome( - String html, - List citations, - List enrichments, - List warnings, - long processingTimeMs, - String source, - int structuredElementCount, - boolean isClean -) implements MarkdownStructuredResponse { + String html, + List citations, + List enrichments, + List warnings, + long processingTimeMs, + String source, + int structuredElementCount, + boolean isClean) + implements MarkdownStructuredResponse { public MarkdownStructuredOutcome { Objects.requireNonNull(html, "Rendered HTML cannot be null"); Objects.requireNonNull(citations, "Citations cannot be null"); Objects.requireNonNull(enrichments, "Enrichments cannot be null"); Objects.requireNonNull(warnings, "Warnings cannot be null"); Objects.requireNonNull(source, "Render source cannot be null"); + citations = List.copyOf(citations); + enrichments = List.copyOf(enrichments); + warnings = List.copyOf(warnings); if (processingTimeMs < 0) { throw new IllegalArgumentException("Processing time must be non-negative"); } @@ -29,4 +32,28 @@ public record MarkdownStructuredOutcome( throw new IllegalArgumentException("Structured element count must be non-negative"); } } + + /** + * Returns citations as an unmodifiable snapshot to prevent mutation through the domain API. + */ + @Override + public List citations() { + return List.copyOf(citations); + } + + /** + * Returns enrichments as an unmodifiable snapshot to prevent mutation through the domain API. + */ + @Override + public List enrichments() { + return List.copyOf(enrichments); + } + + /** + * Returns warnings as an unmodifiable snapshot to prevent mutation through the domain API. + */ + @Override + public List warnings() { + return List.copyOf(warnings); + } } diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredResponse.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredResponse.java index 8339d49..ac1046c 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredResponse.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/MarkdownStructuredResponse.java @@ -3,6 +3,4 @@ /** * Represents the response variants for structured markdown rendering endpoints. */ -public sealed interface MarkdownStructuredResponse - permits MarkdownStructuredOutcome, MarkdownStructuredErrorResponse { -} +public sealed interface MarkdownStructuredResponse permits MarkdownStructuredOutcome, MarkdownStructuredErrorResponse {} diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/ProcessedMarkdown.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/ProcessedMarkdown.java index 60b1546..23378f9 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/ProcessedMarkdown.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/ProcessedMarkdown.java @@ -6,7 +6,7 @@ /** * Represents the result of markdown processing with structured data. * This replaces string-based processing with typed objects for better maintainability. - * + * * @param html The rendered HTML content * @param citations List of extracted citations with metadata * @param enrichments List of structured enrichment objects @@ -14,13 +14,12 @@ * @param processingTimeMs Time taken to process the markdown */ public record ProcessedMarkdown( - String html, - List citations, - List enrichments, - List warnings, - long processingTimeMs -) { - + String html, + List citations, + List enrichments, + List warnings, + long processingTimeMs) { + public ProcessedMarkdown { Objects.requireNonNull(html, "HTML content cannot be null"); Objects.requireNonNull(citations, "Citations list cannot be null"); @@ -45,7 +44,7 @@ public List enrichments() { public List warnings() { return List.copyOf(warnings); } - + /** * Checks if processing completed without warnings. * @return true if no warnings were generated during processing @@ -53,7 +52,7 @@ public List warnings() { public boolean isClean() { return warnings.isEmpty(); } - + /** * Gets the total number of structured elements (citations + enrichments). * @return count of structured elements diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/ProcessingWarning.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/ProcessingWarning.java index 1be4b19..9e0c625 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/ProcessingWarning.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/ProcessingWarning.java @@ -4,13 +4,8 @@ * Represents a non-fatal warning encountered during markdown processing. * Used for structured error reporting instead of silent failures. */ -public record ProcessingWarning( - String message, - WarningType type, - int position, - String context -) { - +public record ProcessingWarning(String message, WarningType type, int position, String context) { + public ProcessingWarning { if (message == null || message.trim().isEmpty()) { throw new IllegalArgumentException("Warning message cannot be null or empty"); @@ -22,7 +17,7 @@ public record ProcessingWarning( throw new IllegalArgumentException("Warning position must be non-negative"); } } - + /** * Creates a processing warning with minimal context. * @param message the warning message @@ -33,7 +28,7 @@ public record ProcessingWarning( public static ProcessingWarning create(String message, WarningType type, int position) { return new ProcessingWarning(message, type, position, ""); } - + /** * Warning types for categorization. */ @@ -42,27 +37,27 @@ public enum WarningType { * Malformed enrichment marker. */ MALFORMED_ENRICHMENT, - + /** * Invalid citation format. */ INVALID_CITATION, - + /** * Unclosed code block. */ UNCLOSED_CODE_BLOCK, - + /** * Nested structure issue. */ NESTED_STRUCTURE, - + /** * Unknown enrichment type. */ UNKNOWN_ENRICHMENT_TYPE, - + /** * General parsing issue. */ diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/Reminder.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/Reminder.java index 8fba5c2..d08652e 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/Reminder.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/Reminder.java @@ -5,7 +5,7 @@ * Highlights important points to remember. */ public record Reminder(String content, EnrichmentPriority priority, int position) implements MarkdownEnrichment { - + public Reminder { if (content == null || content.trim().isEmpty()) { throw new IllegalArgumentException("Reminder content cannot be null or empty"); @@ -17,7 +17,7 @@ public record Reminder(String content, EnrichmentPriority priority, int position throw new IllegalArgumentException("Reminder position must be non-negative"); } } - + /** * Creates a reminder with high priority. * @param content the reminder content @@ -27,7 +27,7 @@ public record Reminder(String content, EnrichmentPriority priority, int position public static Reminder create(String content, int position) { return new Reminder(content, EnrichmentPriority.HIGH, position); } - + @Override public String type() { return "reminder"; diff --git a/src/main/java/com/williamcallahan/javachat/domain/markdown/Warning.java b/src/main/java/com/williamcallahan/javachat/domain/markdown/Warning.java index f368e64..402eb73 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/markdown/Warning.java +++ b/src/main/java/com/williamcallahan/javachat/domain/markdown/Warning.java @@ -5,7 +5,7 @@ * Highlights important cautions and potential issues. */ public record Warning(String content, EnrichmentPriority priority, int position) implements MarkdownEnrichment { - + public Warning { if (content == null || content.trim().isEmpty()) { throw new IllegalArgumentException("Warning content cannot be null or empty"); @@ -17,7 +17,7 @@ public record Warning(String content, EnrichmentPriority priority, int position) throw new IllegalArgumentException("Warning position must be non-negative"); } } - + /** * Creates a warning with high priority. * @param content the warning content @@ -27,7 +27,7 @@ public record Warning(String content, EnrichmentPriority priority, int position) public static Warning create(String content, int position) { return new Warning(content, EnrichmentPriority.HIGH, position); } - + /** * Creates a critical warning with highest priority. * @param content the warning content @@ -37,7 +37,7 @@ public static Warning create(String content, int position) { public static Warning createCritical(String content, int position) { return new Warning(content, EnrichmentPriority.CRITICAL, position); } - + @Override public String type() { return "warning"; diff --git a/src/main/java/com/williamcallahan/javachat/domain/prompt/ContextDocumentSegment.java b/src/main/java/com/williamcallahan/javachat/domain/prompt/ContextDocumentSegment.java index 35815b2..0af484c 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/prompt/ContextDocumentSegment.java +++ b/src/main/java/com/williamcallahan/javachat/domain/prompt/ContextDocumentSegment.java @@ -12,12 +12,8 @@ * @param documentContent extracted text content from the source * @param estimatedTokens approximate token count for budget calculations */ -public record ContextDocumentSegment( - int index, - String sourceUrl, - String documentContent, - int estimatedTokens -) implements PromptSegment { +public record ContextDocumentSegment(int index, String sourceUrl, String documentContent, int estimatedTokens) + implements PromptSegment { /** Marker prefix for context document references. */ public static final String CONTEXT_MARKER = "[CTX "; diff --git a/src/main/java/com/williamcallahan/javachat/domain/prompt/ConversationTurnSegment.java b/src/main/java/com/williamcallahan/javachat/domain/prompt/ConversationTurnSegment.java index 00d0dcc..c2ad4ef 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/prompt/ConversationTurnSegment.java +++ b/src/main/java/com/williamcallahan/javachat/domain/prompt/ConversationTurnSegment.java @@ -11,11 +11,7 @@ * @param messageText the message content * @param estimatedTokens approximate token count for budget calculations */ -public record ConversationTurnSegment( - String role, - String messageText, - int estimatedTokens -) implements PromptSegment { +public record ConversationTurnSegment(String role, String messageText, int estimatedTokens) implements PromptSegment { /** Prefix for assistant messages in the rendered prompt. */ public static final String ASSISTANT_PREFIX = "Assistant: "; diff --git a/src/main/java/com/williamcallahan/javachat/domain/prompt/CurrentQuerySegment.java b/src/main/java/com/williamcallahan/javachat/domain/prompt/CurrentQuerySegment.java index 86a38e8..d7fcd68 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/prompt/CurrentQuerySegment.java +++ b/src/main/java/com/williamcallahan/javachat/domain/prompt/CurrentQuerySegment.java @@ -10,10 +10,7 @@ * @param queryText the user's current question * @param estimatedTokens approximate token count for budget calculations */ -public record CurrentQuerySegment( - String queryText, - int estimatedTokens -) implements PromptSegment { +public record CurrentQuerySegment(String queryText, int estimatedTokens) implements PromptSegment { /** * Creates a current query segment with validation. diff --git a/src/main/java/com/williamcallahan/javachat/domain/prompt/StructuredPrompt.java b/src/main/java/com/williamcallahan/javachat/domain/prompt/StructuredPrompt.java index 282fb8c..fa48dc8 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/prompt/StructuredPrompt.java +++ b/src/main/java/com/williamcallahan/javachat/domain/prompt/StructuredPrompt.java @@ -18,8 +18,7 @@ public record StructuredPrompt( SystemSegment system, List contextDocuments, List conversationHistory, - CurrentQuerySegment currentQuery -) { + CurrentQuerySegment currentQuery) { /** Separator between prompt segments. */ public static final String SEGMENT_SEPARATOR = "\n\n"; @@ -112,10 +111,6 @@ public StructuredPrompt withConversationHistory(List ne */ public static StructuredPrompt fromRawPrompt(String rawPrompt, int estimatedTokens) { return new StructuredPrompt( - new SystemSegment(rawPrompt, estimatedTokens), - List.of(), - List.of(), - new CurrentQuerySegment("", 0) - ); + new SystemSegment(rawPrompt, estimatedTokens), List.of(), List.of(), new CurrentQuerySegment("", 0)); } } diff --git a/src/main/java/com/williamcallahan/javachat/domain/prompt/SystemSegment.java b/src/main/java/com/williamcallahan/javachat/domain/prompt/SystemSegment.java index 03ea4b7..777253f 100644 --- a/src/main/java/com/williamcallahan/javachat/domain/prompt/SystemSegment.java +++ b/src/main/java/com/williamcallahan/javachat/domain/prompt/SystemSegment.java @@ -9,10 +9,7 @@ * @param systemPrompt the complete system instruction text * @param estimatedTokens approximate token count for budget calculations */ -public record SystemSegment( - String systemPrompt, - int estimatedTokens -) implements PromptSegment { +public record SystemSegment(String systemPrompt, int estimatedTokens) implements PromptSegment { /** * Creates a system segment with validation. diff --git a/src/main/java/com/williamcallahan/javachat/logging/ProcessingLogger.java b/src/main/java/com/williamcallahan/javachat/logging/ProcessingLogger.java index c2ac52a..690a66e 100644 --- a/src/main/java/com/williamcallahan/javachat/logging/ProcessingLogger.java +++ b/src/main/java/com/williamcallahan/javachat/logging/ProcessingLogger.java @@ -1,19 +1,18 @@ package com.williamcallahan.javachat.logging; +import java.util.Objects; import org.aspectj.lang.JoinPoint; import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Before; import org.aspectj.lang.annotation.After; import org.aspectj.lang.annotation.AfterReturning; import org.aspectj.lang.annotation.AfterThrowing; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import java.util.Objects; - /** * Comprehensive logging aspect for all processing steps in the RAG pipeline. * Logs each step with detailed information for debugging and monitoring. @@ -23,10 +22,9 @@ public class ProcessingLogger { private static final Logger PIPELINE_LOG = LoggerFactory.getLogger("PIPELINE"); // Thread-local storage for request tracking - private static final ThreadLocal REQUEST_ID = ThreadLocal.withInitial(() -> - "REQ-" + System.currentTimeMillis() + "-" + Thread.currentThread().threadId() - ); - + private static final ThreadLocal REQUEST_ID = ThreadLocal.withInitial(() -> + "REQ-" + System.currentTimeMillis() + "-" + Thread.currentThread().threadId()); + /** * Log embedding generation */ @@ -40,8 +38,7 @@ public Object logEmbeddingGeneration(ProceedingJoinPoint joinPoint) throws Throw Object embeddingOutcome = joinPoint.proceed(); long duration = System.currentTimeMillis() - startTime; - PIPELINE_LOG.info("[{}] STEP 1: EMBEDDING GENERATION - Completed in {}ms", - requestToken, duration); + PIPELINE_LOG.info("[{}] STEP 1: EMBEDDING GENERATION - Completed in {}ms", requestToken, duration); return embeddingOutcome; } @@ -50,15 +47,16 @@ public Object logEmbeddingGeneration(ProceedingJoinPoint joinPoint) throws Throw * Logs embedding generation failures without masking the underlying exception. */ @AfterThrowing( - pointcut = "execution(* com.williamcallahan.javachat.service.*EmbeddingModel.embed(..))", - throwing = "exception" - ) + pointcut = "execution(* com.williamcallahan.javachat.service.*EmbeddingModel.embed(..))", + throwing = "exception") public void logEmbeddingGenerationFailure(Throwable exception) { int requestToken = Objects.hashCode(REQUEST_ID.get()); - PIPELINE_LOG.error("[{}] STEP 1: EMBEDDING GENERATION - Failed (exception type: {})", - requestToken, exception.getClass().getSimpleName()); + PIPELINE_LOG.error( + "[{}] STEP 1: EMBEDDING GENERATION - Failed (exception type: {})", + requestToken, + exception.getClass().getSimpleName()); } - + /** * Log document parsing - DISABLED to avoid excessive logging during startup * The ChunkProcessingService processes thousands of documents on startup @@ -69,18 +67,17 @@ public Object logDocumentParsing(ProceedingJoinPoint joinPoint) throws Throwable // Uncomment the @Around annotation to re-enable if needed for debugging return joinPoint.proceed(); } - + /** * Log query intent interpretation */ - @Before("execution(* com.williamcallahan.javachat.service.ChatService.*(..)) && " + - "args(query,..)") + @Before("execution(* com.williamcallahan.javachat.service.ChatService.*(..)) && " + "args(query,..)") public void logQueryIntentInterpretation(JoinPoint joinPoint, Object query) { int requestToken = Objects.hashCode(REQUEST_ID.get()); PIPELINE_LOG.info("[{}] STEP 3: QUERY INTENT INTERPRETATION - Starting", requestToken); } - + /** * Log RAG retrieval */ @@ -88,14 +85,13 @@ public void logQueryIntentInterpretation(JoinPoint joinPoint, Object query) { public Object logRAGRetrieval(ProceedingJoinPoint joinPoint) throws Throwable { int requestToken = Objects.hashCode(REQUEST_ID.get()); long startTime = System.currentTimeMillis(); - + PIPELINE_LOG.info("[{}] STEP 4: RAG RETRIEVAL - Starting vector search", requestToken); - + Object retrievalOutcome = joinPoint.proceed(); long duration = System.currentTimeMillis() - startTime; - PIPELINE_LOG.info("[{}] STEP 4: RAG RETRIEVAL - Retrieved documents in {}ms", - requestToken, duration); + PIPELINE_LOG.info("[{}] STEP 4: RAG RETRIEVAL - Retrieved documents in {}ms", requestToken, duration); // Log retrieved document count if available if (retrievalOutcome instanceof java.util.List) { @@ -109,15 +105,16 @@ public Object logRAGRetrieval(ProceedingJoinPoint joinPoint) throws Throwable { * Logs RAG retrieval failures without masking the underlying exception. */ @AfterThrowing( - pointcut = "execution(* com.williamcallahan.javachat.service.RetrievalService.retrieve*(..))", - throwing = "exception" - ) + pointcut = "execution(* com.williamcallahan.javachat.service.RetrievalService.retrieve*(..))", + throwing = "exception") public void logRagRetrievalFailure(Throwable exception) { int requestToken = Objects.hashCode(REQUEST_ID.get()); - PIPELINE_LOG.error("[{}] STEP 4: RAG RETRIEVAL - Failed (exception type: {})", - requestToken, exception.getClass().getSimpleName()); + PIPELINE_LOG.error( + "[{}] STEP 4: RAG RETRIEVAL - Failed (exception type: {})", + requestToken, + exception.getClass().getSimpleName()); } - + /** * Log LLM interaction */ @@ -131,8 +128,7 @@ public Object logLLMInteraction(ProceedingJoinPoint joinPoint) throws Throwable Object llmOutcome = joinPoint.proceed(); long duration = System.currentTimeMillis() - startTime; - PIPELINE_LOG.info("[{}] STEP 5: LLM INTERACTION - Response streaming started in {}ms", - requestToken, duration); + PIPELINE_LOG.info("[{}] STEP 5: LLM INTERACTION - Response streaming started in {}ms", requestToken, duration); return llmOutcome; } @@ -141,22 +137,22 @@ public Object logLLMInteraction(ProceedingJoinPoint joinPoint) throws Throwable * Logs LLM interaction failures without masking the underlying exception. */ @AfterThrowing( - pointcut = "execution(* com.williamcallahan.javachat.service.ChatService.streamAnswer(..))", - throwing = "exception" - ) + pointcut = "execution(* com.williamcallahan.javachat.service.ChatService.streamAnswer(..))", + throwing = "exception") public void logLlmInteractionFailure(Throwable exception) { int requestToken = Objects.hashCode(REQUEST_ID.get()); - PIPELINE_LOG.error("[{}] STEP 5: LLM INTERACTION - Failed (exception type: {})", - requestToken, exception.getClass().getSimpleName()); + PIPELINE_LOG.error( + "[{}] STEP 5: LLM INTERACTION - Failed (exception type: {})", + requestToken, + exception.getClass().getSimpleName()); } - + /** * Log response categorization */ @AfterReturning( - pointcut = "execution(* com.williamcallahan.javachat.service.EnrichmentService.enrich*(..))", - returning = "responsePayload" - ) + pointcut = "execution(* com.williamcallahan.javachat.service.EnrichmentService.enrich*(..))", + returning = "responsePayload") public void logResponseCategorization(JoinPoint joinPoint, Object responsePayload) { int requestToken = Objects.hashCode(REQUEST_ID.get()); @@ -168,7 +164,7 @@ public void logResponseCategorization(JoinPoint joinPoint, Object responsePayloa PIPELINE_LOG.info("[{}] Response categories computed", requestToken); } } - + /** * Log citation generation */ @@ -193,26 +189,25 @@ public Object logCitationGeneration(ProceedingJoinPoint joinPoint) throws Throwa * Logs citation generation failures without masking the underlying exception. */ @AfterThrowing( - pointcut = "execution(* com.williamcallahan.javachat.service.ChatService.citationsFor(..))", - throwing = "exception" - ) + pointcut = "execution(* com.williamcallahan.javachat.service.ChatService.citationsFor(..))", + throwing = "exception") public void logCitationGenerationFailure(Throwable exception) { int requestToken = Objects.hashCode(REQUEST_ID.get()); - PIPELINE_LOG.error("[{}] STEP 7: CITATION GENERATION - Failed (exception type: {})", - requestToken, exception.getClass().getSimpleName()); + PIPELINE_LOG.error( + "[{}] STEP 7: CITATION GENERATION - Failed (exception type: {})", + requestToken, + exception.getClass().getSimpleName()); } - - + /** * Log pipeline summary at the end */ @AfterReturning( - pointcut = "execution(* com.williamcallahan.javachat.web.ChatController.stream(..))", - returning = "streamOutcome" - ) + pointcut = "execution(* com.williamcallahan.javachat.web.ChatController.stream(..))", + returning = "streamOutcome") public void logPipelineSummary(JoinPoint joinPoint, Object streamOutcome) { int requestToken = Objects.hashCode(REQUEST_ID.get()); - + PIPELINE_LOG.info("[{}] ============================================", requestToken); PIPELINE_LOG.info("[{}] PIPELINE COMPLETE - All steps processed", requestToken); PIPELINE_LOG.info("[{}] ============================================", requestToken); diff --git a/src/main/java/com/williamcallahan/javachat/model/AuditReport.java b/src/main/java/com/williamcallahan/javachat/model/AuditReport.java index 555fef2..f36f289 100644 --- a/src/main/java/com/williamcallahan/javachat/model/AuditReport.java +++ b/src/main/java/com/williamcallahan/javachat/model/AuditReport.java @@ -6,16 +6,15 @@ * Summary of audit results comparing parsed chunks to stored vector entries. */ public record AuditReport( - String url, - int parsedCount, - int qdrantCount, - int missingCount, - int extraCount, - List duplicateHashes, - boolean ok, - List missingHashes, - List extraHashes -) { + String url, + int parsedCount, + int qdrantCount, + int missingCount, + int extraCount, + List duplicateHashes, + boolean ok, + List missingHashes, + List extraHashes) { public AuditReport { duplicateHashes = duplicateHashes == null ? List.of() : List.copyOf(duplicateHashes); missingHashes = missingHashes == null ? List.of() : List.copyOf(missingHashes); diff --git a/src/main/java/com/williamcallahan/javachat/model/ChatTurn.java b/src/main/java/com/williamcallahan/javachat/model/ChatTurn.java index 8e3cb65..6ed8cc7 100644 --- a/src/main/java/com/williamcallahan/javachat/model/ChatTurn.java +++ b/src/main/java/com/williamcallahan/javachat/model/ChatTurn.java @@ -67,7 +67,3 @@ public void setText(String text) { this.text = Objects.requireNonNull(text, "text must not be null"); } } - - - - diff --git a/src/main/java/com/williamcallahan/javachat/model/Citation.java b/src/main/java/com/williamcallahan/javachat/model/Citation.java index 4b11958..216a715 100644 --- a/src/main/java/com/williamcallahan/javachat/model/Citation.java +++ b/src/main/java/com/williamcallahan/javachat/model/Citation.java @@ -110,7 +110,3 @@ public void setSnippet(String snippet) { this.snippet = Objects.requireNonNullElse(snippet, DEFAULT_SNIPPET); } } - - - - diff --git a/src/main/java/com/williamcallahan/javachat/model/Enrichment.java b/src/main/java/com/williamcallahan/javachat/model/Enrichment.java index 255f731..28ff22e 100644 --- a/src/main/java/com/williamcallahan/javachat/model/Enrichment.java +++ b/src/main/java/com/williamcallahan/javachat/model/Enrichment.java @@ -184,6 +184,3 @@ private static List trimFilter(List entries) { .toList(); } } - - - diff --git a/src/main/java/com/williamcallahan/javachat/model/GuidedLesson.java b/src/main/java/com/williamcallahan/javachat/model/GuidedLesson.java index bb780fe..290dc6c 100644 --- a/src/main/java/com/williamcallahan/javachat/model/GuidedLesson.java +++ b/src/main/java/com/williamcallahan/javachat/model/GuidedLesson.java @@ -14,8 +14,7 @@ public class GuidedLesson { /** * Creates an empty guided lesson container. */ - public GuidedLesson() { - } + public GuidedLesson() {} /** * Creates a guided lesson with the supplied metadata. diff --git a/src/main/java/com/williamcallahan/javachat/model/RetrievalResult.java b/src/main/java/com/williamcallahan/javachat/model/RetrievalResult.java index 04cdab3..2763537 100644 --- a/src/main/java/com/williamcallahan/javachat/model/RetrievalResult.java +++ b/src/main/java/com/williamcallahan/javachat/model/RetrievalResult.java @@ -1,19 +1,14 @@ package com.williamcallahan.javachat.model; -import org.springframework.ai.document.Document; - import java.util.List; +import org.springframework.ai.document.Document; /** * Encapsulates retrieval results with quality metadata. * Enables callers to distinguish between successful retrieval, degraded retrieval, * and failed retrieval, allowing appropriate handling and LLM context adjustment. */ -public record RetrievalResult( - List documents, - RetrievalQuality quality, - String qualityReason -) { +public record RetrievalResult(List documents, RetrievalQuality quality, String qualityReason) { public RetrievalResult { documents = documents == null ? List.of() : List.copyOf(documents); @@ -55,48 +50,49 @@ public enum RetrievalQuality { * Create a successful vector search result. */ public static RetrievalResult vectorSearch(List documents) { - return new RetrievalResult(documents, RetrievalQuality.VECTOR_SEARCH, - "Semantic vector search completed successfully"); + return new RetrievalResult( + documents, RetrievalQuality.VECTOR_SEARCH, "Semantic vector search completed successfully"); } /** * Create a version-filtered vector search result. */ public static RetrievalResult versionFiltered(List documents, String version) { - return new RetrievalResult(documents, RetrievalQuality.VECTOR_SEARCH_VERSION_FILTERED, - "Semantic vector search with Java " + version + " version filtering"); + return new RetrievalResult( + documents, + RetrievalQuality.VECTOR_SEARCH_VERSION_FILTERED, + "Semantic vector search with Java " + version + " version filtering"); } /** * Create a fallback keyword search result. */ public static RetrievalResult keywordFallback(List documents, String failureReason) { - return new RetrievalResult(documents, RetrievalQuality.LOCAL_KEYWORD_FALLBACK, - "Fell back to local keyword search: " + failureReason); + return new RetrievalResult( + documents, + RetrievalQuality.LOCAL_KEYWORD_FALLBACK, + "Fell back to local keyword search: " + failureReason); } /** * Create a failed search result. */ public static RetrievalResult failed(String reason) { - return new RetrievalResult(List.of(), RetrievalQuality.SEARCH_FAILED, - "Search failed: " + reason); + return new RetrievalResult(List.of(), RetrievalQuality.SEARCH_FAILED, "Search failed: " + reason); } /** * Returns true if retrieval used semantic vector search (not keyword fallback). */ public boolean isSemanticSearch() { - return quality == RetrievalQuality.VECTOR_SEARCH - || quality == RetrievalQuality.VECTOR_SEARCH_VERSION_FILTERED; + return quality == RetrievalQuality.VECTOR_SEARCH || quality == RetrievalQuality.VECTOR_SEARCH_VERSION_FILTERED; } /** * Returns true if retrieval degraded to a fallback mode. */ public boolean isDegraded() { - return quality == RetrievalQuality.LOCAL_KEYWORD_FALLBACK - || quality == RetrievalQuality.SEARCH_FAILED; + return quality == RetrievalQuality.LOCAL_KEYWORD_FALLBACK || quality == RetrievalQuality.SEARCH_FAILED; } /** @@ -112,14 +108,12 @@ public boolean isVersionFiltered() { public String getLlmContextNote() { return switch (quality) { case VECTOR_SEARCH -> ""; - case VECTOR_SEARCH_VERSION_FILTERED -> - "Note: Results filtered to match requested Java version."; + case VECTOR_SEARCH_VERSION_FILTERED -> "Note: Results filtered to match requested Java version."; case LOCAL_KEYWORD_FALLBACK -> - "IMPORTANT: Using keyword-based fallback search with limited semantic understanding. " + - "Results may be less relevant. " + qualityReason; + "IMPORTANT: Using keyword-based fallback search with limited semantic understanding. " + + "Results may be less relevant. " + qualityReason; case SEARCH_FAILED -> - "WARNING: Document retrieval failed. Responding based on training knowledge only. " + - qualityReason; + "WARNING: Document retrieval failed. Responding based on training knowledge only. " + qualityReason; }; } } diff --git a/src/main/java/com/williamcallahan/javachat/service/AuditService.java b/src/main/java/com/williamcallahan/javachat/service/AuditService.java index f3f7ee1..a9f34b6 100644 --- a/src/main/java/com/williamcallahan/javachat/service/AuditService.java +++ b/src/main/java/com/williamcallahan/javachat/service/AuditService.java @@ -5,15 +5,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.williamcallahan.javachat.model.AuditReport; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -26,6 +17,15 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; /** * Service for auditing ingested documents against the vector store. @@ -41,21 +41,26 @@ public class AuditService { * but scroll/REST operations require the REST port (6333 default, 8087 in docker). */ private static final Map GRPC_TO_REST_PORT = Map.of( - 6334, 6333, // Qdrant default: gRPC -> REST - 8086, 8087 // Docker compose mapping: gRPC -> REST - ); + 6334, 6333, // Qdrant default: gRPC -> REST + 8086, 8087 // Docker compose mapping: gRPC -> REST + ); private final LocalStoreService localStore; private final ContentHasher hasher; + private final RestTemplateBuilder restTemplateBuilder; @Value("${spring.ai.vectorstore.qdrant.host}") private String host; + @Value("${spring.ai.vectorstore.qdrant.port:6334}") private int port; + @Value("${spring.ai.vectorstore.qdrant.use-tls:false}") private boolean useTls; + @Value("${spring.ai.vectorstore.qdrant.api-key:}") private String apiKey; + @Value("${spring.ai.vectorstore.qdrant.collection-name}") private String collection; @@ -65,9 +70,10 @@ public class AuditService { * @param localStore local snapshot and chunk storage * @param hasher content hashing helper */ - public AuditService(LocalStoreService localStore, ContentHasher hasher) { + public AuditService(LocalStoreService localStore, ContentHasher hasher, RestTemplateBuilder restTemplateBuilder) { this.localStore = localStore; this.hasher = hasher; + this.restTemplateBuilder = restTemplateBuilder; } /** @@ -80,7 +86,7 @@ public AuditService(LocalStoreService localStore, ContentHasher hasher) { public AuditReport auditByUrl(String url) throws IOException { Set expectedHashes = getExpectedHashes(url); List qdrantHashes = fetchQdrantHashes(url); - + return compareAndReport(url, expectedHashes, qdrantHashes); } @@ -97,18 +103,17 @@ private Set getExpectedHashes(String url) throws IOException { } // pattern: _.txt - Pattern chunkPattern = Pattern.compile( - Pattern.quote(safeName) + "_" + "(\\d+)" + "_" + "([0-9a-f]{12})" + "\\.txt" - ); + Pattern chunkPattern = + Pattern.compile(Pattern.quote(safeName) + "_" + "(\\d+)" + "_" + "([0-9a-f]{12})" + "\\.txt"); List parsedFiles = new ArrayList<>(); try (var stream = Files.walk(parsedRoot)) { stream.filter(Files::isRegularFile) - .filter(filePath -> { - Path fileName = filePath.getFileName(); - return fileName != null && fileName.toString().startsWith(safeBase); - }) - .forEach(parsedFiles::add); + .filter(filePath -> { + Path fileName = filePath.getFileName(); + return fileName != null && fileName.toString().startsWith(safeBase); + }) + .forEach(parsedFiles::add); } Set expectedHashes = new LinkedHashSet<>(); @@ -145,37 +150,34 @@ private AuditReport compareAndReport(String url, Set expectedHashes, Lis duplicateCounts.merge(hashValue, 1, Integer::sum); } List duplicateHashes = duplicateCounts.entrySet().stream() - .filter(countEntry -> countEntry.getValue() != null && countEntry.getValue() > 1) - .map(Map.Entry::getKey) - .toList(); + .filter(countEntry -> countEntry.getValue() != null && countEntry.getValue() > 1) + .map(Map.Entry::getKey) + .toList(); - boolean auditOk = missingHashes.isEmpty() - && extraHashes.isEmpty() - && duplicateHashes.isEmpty(); + boolean auditOk = missingHashes.isEmpty() && extraHashes.isEmpty() && duplicateHashes.isEmpty(); List missingHashesSample = missingHashes.isEmpty() - ? List.of() - : missingHashes.stream().limit(20).toList(); + ? List.of() + : missingHashes.stream().limit(20).toList(); List extraHashesSample = extraHashes.isEmpty() - ? List.of() - : extraHashes.stream().limit(20).toList(); + ? List.of() + : extraHashes.stream().limit(20).toList(); return new AuditReport( - url, - expectedHashes.size(), - qdrantHashes.size(), - missingHashes.size(), - extraHashes.size(), - duplicateHashes, - auditOk, - missingHashesSample, - extraHashesSample - ); + url, + expectedHashes.size(), + qdrantHashes.size(), + missingHashes.size(), + extraHashes.size(), + duplicateHashes, + auditOk, + missingHashesSample, + extraHashesSample); } private List fetchQdrantHashes(String url) { List hashes = new ArrayList<>(); - RestTemplate restTemplate = new RestTemplate(); - + RestTemplate restTemplate = restTemplateBuilder.build(); + // Build REST base URL with correct port mapping // Note: spring.ai.vectorstore.qdrant.port is typically gRPC (6334); REST runs on 6333 String base = buildQdrantRestBaseUrl(); @@ -187,32 +189,25 @@ private List fetchQdrantHashes(String url) { headers.set("api-key", apiKey); } - QdrantScrollFilter scrollFilter = new QdrantScrollFilter( - List.of(new QdrantScrollMustCondition("url", new QdrantScrollMatch(url))) - ); + QdrantScrollFilter scrollFilter = + new QdrantScrollFilter(List.of(new QdrantScrollMustCondition("url", new QdrantScrollMatch(url)))); // Paginate through all results using next_page_offset JsonNode nextOffset = null; int pageCount = 0; int maxPages = 100; // Safety limit to prevent infinite loops int pageSize = 1000; // Reduced from 2048 for more reliable pagination - + do { - QdrantScrollRequest requestBody = new QdrantScrollRequest( - scrollFilter, - true, - pageSize, - nextOffset - ); + QdrantScrollRequest requestBody = new QdrantScrollRequest(scrollFilter, true, pageSize, nextOffset); try { var response = restTemplate.exchange( - endpoint, - org.springframework.http.HttpMethod.POST, - new HttpEntity<>(requestBody, headers), - QdrantScrollResponse.class - ); - + endpoint, + org.springframework.http.HttpMethod.POST, + new HttpEntity<>(requestBody, headers), + QdrantScrollResponse.class); + QdrantScrollResponse body = response.getBody(); if (body != null && body.scrollResult() != null) { hashes.addAll(body.scrollResult().hashes()); @@ -224,54 +219,48 @@ private List fetchQdrantHashes(String url) { nextOffset = null; } pageCount++; - + if (pageCount > 1) { log.debug("Scroll page {} fetched, total hashes so far: {}", pageCount, hashes.size()); } - + } catch (Exception requestFailure) { // Propagate failure so caller knows audit could not complete throw new IllegalStateException( - "Qdrant scroll failed for URL audit (endpoint: " + endpoint + ")", requestFailure); + "Qdrant scroll failed for URL audit (endpoint: " + endpoint + ")", requestFailure); } } while (nextOffset != null && pageCount < maxPages); - + if (pageCount >= maxPages) { log.warn("Scroll pagination reached safety limit of {} pages; results may be incomplete", maxPages); } - + return hashes; } @JsonInclude(JsonInclude.Include.NON_NULL) private record QdrantScrollRequest( - @JsonProperty("filter") QdrantScrollFilter filter, - @JsonProperty("with_payload") boolean withPayload, - @JsonProperty("limit") int limit, - @JsonProperty("offset") JsonNode offset - ) {} + @JsonProperty("filter") QdrantScrollFilter filter, + @JsonProperty("with_payload") boolean withPayload, + @JsonProperty("limit") int limit, + @JsonProperty("offset") JsonNode offset) {} - private record QdrantScrollFilter( - @JsonProperty("must") List must - ) {} + private record QdrantScrollFilter(@JsonProperty("must") List must) {} private record QdrantScrollMustCondition( - @JsonProperty("key") String key, - @JsonProperty("match") QdrantScrollMatch match - ) {} + @JsonProperty("key") String key, + @JsonProperty("match") QdrantScrollMatch match) {} private record QdrantScrollMatch(@JsonProperty("value") String value) {} @JsonIgnoreProperties(ignoreUnknown = true) private record QdrantScrollResponse( - @JsonProperty("result") QdrantScrollResult scrollResult - ) {} + @JsonProperty("result") QdrantScrollResult scrollResult) {} @JsonIgnoreProperties(ignoreUnknown = true) private record QdrantScrollResult( - @JsonProperty("points") List points, - @JsonProperty("next_page_offset") JsonNode nextPageOffset - ) { + @JsonProperty("points") List points, + @JsonProperty("next_page_offset") JsonNode nextPageOffset) { List hashes() { if (points == null || points.isEmpty()) { return List.of(); @@ -291,7 +280,8 @@ List hashes() { } @JsonIgnoreProperties(ignoreUnknown = true) - private record QdrantScrollPoint(@JsonProperty("payload") QdrantScrollPayload payload) {} + private record QdrantScrollPoint( + @JsonProperty("payload") QdrantScrollPayload payload) {} @JsonIgnoreProperties(ignoreUnknown = true) private record QdrantScrollPayload(@JsonProperty("hash") String hash) {} diff --git a/src/main/java/com/williamcallahan/javachat/service/ChatMemoryService.java b/src/main/java/com/williamcallahan/javachat/service/ChatMemoryService.java index a1899ea..c9aafca 100644 --- a/src/main/java/com/williamcallahan/javachat/service/ChatMemoryService.java +++ b/src/main/java/com/williamcallahan/javachat/service/ChatMemoryService.java @@ -22,19 +22,16 @@ public class ChatMemoryService { // Use synchronizedList wrapper to ensure thread-safe list operations. // ConcurrentHashMap only protects map operations, not the contained lists. - private final ConcurrentMap> sessionToMessages = - new ConcurrentHashMap<>(); - private final ConcurrentMap> sessionToTurns = - new ConcurrentHashMap<>(); + private final ConcurrentMap> sessionToMessages = new ConcurrentHashMap<>(); + private final ConcurrentMap> sessionToTurns = new ConcurrentHashMap<>(); /** * Returns a thread-safe snapshot of the history for the given session. * Callers receive an independent copy that can be safely iterated without synchronization. */ public List getHistory(String sessionId) { - List history = sessionToMessages.computeIfAbsent(sessionId, sessionKey -> - Collections.synchronizedList(new ArrayList<>()) - ); + List history = sessionToMessages.computeIfAbsent( + sessionId, sessionKey -> Collections.synchronizedList(new ArrayList<>())); // Return a snapshot to avoid ConcurrentModificationException during iteration synchronized (history) { return new ArrayList<>(history); @@ -46,9 +43,8 @@ public List getHistory(String sessionId) { * Use with care - prefer addUser/addAssistant for adding messages. */ List getHistoryInternal(String sessionId) { - return sessionToMessages.computeIfAbsent(sessionId, sessionKey -> - Collections.synchronizedList(new ArrayList<>()) - ); + return sessionToMessages.computeIfAbsent( + sessionId, sessionKey -> Collections.synchronizedList(new ArrayList<>())); } /** @@ -87,9 +83,8 @@ public void clear(String sessionId) { * Returns a thread-safe snapshot of the turns for the given session. */ public List getTurns(String sessionId) { - List turns = sessionToTurns.computeIfAbsent(sessionId, sessionKey -> - Collections.synchronizedList(new ArrayList<>()) - ); + List turns = sessionToTurns.computeIfAbsent( + sessionId, sessionKey -> Collections.synchronizedList(new ArrayList<>())); synchronized (turns) { return new ArrayList<>(turns); } @@ -99,9 +94,7 @@ public List getTurns(String sessionId) { * Returns the internal synchronized list for direct modification. */ List getTurnsInternal(String sessionId) { - return sessionToTurns.computeIfAbsent(sessionId, sessionKey -> - Collections.synchronizedList(new ArrayList<>()) - ); + return sessionToTurns.computeIfAbsent(sessionId, sessionKey -> Collections.synchronizedList(new ArrayList<>())); } // TODO: Persist chat history embeddings to Qdrant for long-term memory (future feature) diff --git a/src/main/java/com/williamcallahan/javachat/service/ChatService.java b/src/main/java/com/williamcallahan/javachat/service/ChatService.java index 21b808f..0499f84 100644 --- a/src/main/java/com/williamcallahan/javachat/service/ChatService.java +++ b/src/main/java/com/williamcallahan/javachat/service/ChatService.java @@ -1,15 +1,19 @@ package com.williamcallahan.javachat.service; import com.williamcallahan.javachat.config.AppProperties; -import com.williamcallahan.javachat.config.ModelConfiguration; -import com.williamcallahan.javachat.model.Citation; import com.williamcallahan.javachat.config.DocsSourceRegistry; +import com.williamcallahan.javachat.config.ModelConfiguration; import com.williamcallahan.javachat.config.SystemPromptConfig; +import com.williamcallahan.javachat.domain.SearchQualityLevel; import com.williamcallahan.javachat.domain.prompt.ContextDocumentSegment; import com.williamcallahan.javachat.domain.prompt.ConversationTurnSegment; import com.williamcallahan.javachat.domain.prompt.CurrentQuerySegment; import com.williamcallahan.javachat.domain.prompt.StructuredPrompt; import com.williamcallahan.javachat.domain.prompt.SystemSegment; +import com.williamcallahan.javachat.model.Citation; +import com.williamcallahan.javachat.support.DocumentContentAdapter; +import java.util.ArrayList; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.ai.chat.messages.AssistantMessage; @@ -19,9 +23,6 @@ import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; -import java.util.ArrayList; -import java.util.List; - /** * Builds chat prompts, enriches them with retrieval context, and delegates streaming to the LLM provider. */ @@ -42,10 +43,11 @@ public class ChatService { * @param systemPromptConfig system prompt configuration * @param appProperties application configuration for LLM settings */ - public ChatService(OpenAIStreamingService openAIStreamingService, - RetrievalService retrievalService, - SystemPromptConfig systemPromptConfig, - AppProperties appProperties) { + public ChatService( + OpenAIStreamingService openAIStreamingService, + RetrievalService retrievalService, + SystemPromptConfig systemPromptConfig, + AppProperties appProperties) { this.openAIStreamingService = openAIStreamingService; this.retrievalService = retrievalService; this.systemPromptConfig = systemPromptConfig; @@ -65,15 +67,15 @@ public ChatService(OpenAIStreamingService openAIStreamingService, public Flux streamAnswer(List history, String latestUserMessage) { logger.debug("ChatService.streamAnswer called"); - StructuredPromptOutcome outcome = buildStructuredPromptWithContextOutcome( - history, latestUserMessage, null); + StructuredPromptOutcome outcome = buildStructuredPromptWithContextOutcome(history, latestUserMessage, null); if (!openAIStreamingService.isAvailable()) { logger.error("OpenAI streaming service is not available - check API credentials"); return Flux.error(new IllegalStateException("Chat service unavailable - no API credentials configured")); } - return openAIStreamingService.streamResponse(outcome.structuredPrompt(), temperature) + return openAIStreamingService + .streamResponse(outcome.structuredPrompt(), temperature) .onErrorResume(streamingException -> { logger.error("Streaming failed", streamingException); return Flux.error(streamingException); @@ -92,16 +94,15 @@ public Flux streamAnswer(List history, String latestUserMessage * @param guidance custom system guidance to append * @return streaming response chunks */ - public Flux streamAnswerWithContext(List history, - String latestUserMessage, - List contextDocs, - String guidance) { + public Flux streamAnswerWithContext( + List history, String latestUserMessage, List contextDocs, String guidance) { if (contextDocs == null) contextDocs = List.of(); - StructuredPrompt structuredPrompt = buildStructuredPromptWithContextAndGuidance( - history, latestUserMessage, contextDocs, guidance); + StructuredPrompt structuredPrompt = + buildStructuredPromptWithContextAndGuidance(history, latestUserMessage, contextDocs, guidance); - return openAIStreamingService.streamResponse(structuredPrompt, temperature) + return openAIStreamingService + .streamResponse(structuredPrompt, temperature) .onErrorResume(exception -> { logger.error("Streaming failed", exception); return Flux.error(exception); @@ -129,10 +130,7 @@ public List citationsFor(String userQuery) { * @return structured prompt for intelligent truncation */ public StructuredPrompt buildStructuredPromptWithContextAndGuidance( - List history, - String latestUserMessage, - List contextDocs, - String guidance) { + List history, String latestUserMessage, List contextDocs, String guidance) { // Build system prompt with guidance String basePrompt = systemPromptConfig.getCoreSystemPrompt(); @@ -140,28 +138,17 @@ public StructuredPrompt buildStructuredPromptWithContextAndGuidance( ? systemPromptConfig.buildFullPrompt(basePrompt, guidance) : basePrompt; - SystemSegment systemSegment = new SystemSegment( - completePrompt, - estimateTokens(completePrompt) - ); + SystemSegment systemSegment = new SystemSegment(completePrompt, estimateTokens(completePrompt)); - List contextSegments = buildContextSegments( - contextDocs != null ? contextDocs : List.of() - ); + List contextSegments = + buildContextSegments(contextDocs != null ? contextDocs : List.of()); List conversationSegments = buildConversationSegments(history); - CurrentQuerySegment querySegment = new CurrentQuerySegment( - latestUserMessage, - estimateTokens(latestUserMessage) - ); - - return new StructuredPrompt( - systemSegment, - contextSegments, - conversationSegments, - querySegment - ); + CurrentQuerySegment querySegment = + new CurrentQuerySegment(latestUserMessage, estimateTokens(latestUserMessage)); + + return new StructuredPrompt(systemSegment, contextSegments, conversationSegments, querySegment); } /** @@ -176,22 +163,19 @@ public StructuredPrompt buildStructuredPromptWithContextAndGuidance( * @return structured prompt outcome with segments and retrieval metadata */ public StructuredPromptOutcome buildStructuredPromptWithContextOutcome( - List history, - String latestUserMessage, - String modelHint) { + List history, String latestUserMessage, String modelHint) { // Use reduced RAG for token-constrained models (GPT-5.x family) RetrievalService.RetrievalOutcome retrievalOutcome; if (ModelConfiguration.isTokenConstrained(modelHint)) { retrievalOutcome = retrievalService.retrieveWithLimitOutcome( - latestUserMessage, - ModelConfiguration.RAG_LIMIT_CONSTRAINED, - ModelConfiguration.RAG_TOKEN_LIMIT_CONSTRAINED - ); - logger.debug("Using reduced RAG for {}: {} documents with max {} tokens each", - modelHint, - retrievalOutcome.documents().size(), - ModelConfiguration.RAG_TOKEN_LIMIT_CONSTRAINED); + latestUserMessage, + ModelConfiguration.RAG_LIMIT_CONSTRAINED, + ModelConfiguration.RAG_TOKEN_LIMIT_CONSTRAINED); + logger.debug( + "Using reduced RAG: {} documents with max {} tokens each", + retrievalOutcome.documents().size(), + ModelConfiguration.RAG_TOKEN_LIMIT_CONSTRAINED); } else { retrievalOutcome = retrievalService.retrieveOutcome(latestUserMessage); } @@ -210,31 +194,18 @@ public StructuredPromptOutcome buildStructuredPromptWithContextOutcome( String systemPromptText = systemPromptBuilder.toString(); // Build structured segments - SystemSegment systemSegment = new SystemSegment( - systemPromptText, - estimateTokens(systemPromptText) - ); + SystemSegment systemSegment = new SystemSegment(systemPromptText, estimateTokens(systemPromptText)); List contextSegments = buildContextSegments(contextDocs); List conversationSegments = buildConversationSegments(history); - CurrentQuerySegment querySegment = new CurrentQuerySegment( - latestUserMessage, - estimateTokens(latestUserMessage) - ); - - StructuredPrompt structuredPrompt = new StructuredPrompt( - systemSegment, - contextSegments, - conversationSegments, - querySegment - ); - - return new StructuredPromptOutcome( - structuredPrompt, - retrievalOutcome.notices(), - retrievalOutcome.documents() - ); + CurrentQuerySegment querySegment = + new CurrentQuerySegment(latestUserMessage, estimateTokens(latestUserMessage)); + + StructuredPrompt structuredPrompt = + new StructuredPrompt(systemSegment, contextSegments, conversationSegments, querySegment); + + return new StructuredPromptOutcome(structuredPrompt, retrievalOutcome.notices(), retrievalOutcome.documents()); } /** @@ -249,12 +220,7 @@ private List buildContextSegments(List context String normalizedUrl = DocsSourceRegistry.normalizeDocUrl(rawUrl); String content = doc.getText(); - segments.add(new ContextDocumentSegment( - docIndex + 1, - normalizedUrl, - content, - estimateTokens(content) - )); + segments.add(new ContextDocumentSegment(docIndex + 1, normalizedUrl, content, estimateTokens(content))); } return segments; } @@ -299,38 +265,12 @@ private int estimateTokens(String text) { } /** - * Determine the quality of search results and provide context to the AI. + * Determines the quality of search results and provides context to the AI. + * + *

Delegates to {@link SearchQualityLevel} enum for self-describing quality categorization.

*/ private String determineSearchQuality(List docs) { - if (docs.isEmpty()) { - return "No relevant documents found. Using general knowledge only."; - } - - // Check if documents seem to be from keyword search (less semantic relevance) - boolean likelyKeywordSearch = docs.stream() - .anyMatch(doc -> { - String url = String.valueOf(doc.getMetadata().getOrDefault("url", "")); - return url.contains("local-search") || url.contains("keyword"); - }); - - if (likelyKeywordSearch) { - return String.format("Found %d documents via keyword search (embedding service unavailable). Results may be less semantically relevant.", docs.size()); - } - - // Check document relevance quality - long highQualityDocs = docs.stream() - .filter(doc -> { - String content = doc.getText(); // Use getText() instead of getContent() - return content != null && content.length() > 100; // Basic quality check - }) - .count(); - - if (highQualityDocs == docs.size()) { - return String.format("Found %d high-quality relevant documents via semantic search.", docs.size()); - } else { - return String.format("Found %d documents (%d high-quality) via search. Some results may be less relevant.", - docs.size(), highQualityDocs); - } + return SearchQualityLevel.describeQuality(DocumentContentAdapter.fromDocuments(docs)); } /** diff --git a/src/main/java/com/williamcallahan/javachat/service/ChunkProcessingService.java b/src/main/java/com/williamcallahan/javachat/service/ChunkProcessingService.java index 3d6d191..6659607 100644 --- a/src/main/java/com/williamcallahan/javachat/service/ChunkProcessingService.java +++ b/src/main/java/com/williamcallahan/javachat/service/ChunkProcessingService.java @@ -1,11 +1,10 @@ package com.williamcallahan.javachat.service; -import org.springframework.ai.document.Document; -import org.springframework.stereotype.Service; - import java.io.IOException; import java.util.ArrayList; import java.util.List; +import org.springframework.ai.document.Document; +import org.springframework.stereotype.Service; /** * Service for processing text into chunks with consistent metadata and storage. @@ -51,11 +50,8 @@ public ChunkProcessingService( * @return List of created documents * @throws IOException If file operations fail */ - public List processAndStoreChunks( - String text, - String url, - String title, - String packageName) throws IOException { + public List processAndStoreChunks(String text, String url, String title, String packageName) + throws IOException { // Chunk the text with standard parameters List chunks = chunker.chunkByTokens(text, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_OVERLAP); @@ -73,8 +69,7 @@ public List processAndStoreChunks( } // Create document with standardized metadata - Document document = documentFactory.createDocument( - chunkText, url, title, chunkIndex, packageName, hash); + Document document = documentFactory.createDocument(chunkText, url, title, chunkIndex, packageName, hash); documents.add(document); @@ -95,11 +90,7 @@ public List processAndStoreChunks( * @param packageName The Java package name * @return List of documents ready for vector storage */ - public List processChunksOnly( - String text, - String url, - String title, - String packageName) { + public List processChunksOnly(String text, String url, String title, String packageName) { List chunks = chunker.chunkByTokens(text, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_OVERLAP); List documents = new ArrayList<>(); @@ -108,8 +99,7 @@ public List processChunksOnly( String chunkText = chunks.get(chunkIndex); String hash = hasher.generateChunkHash(url, chunkIndex, chunkText); - Document document = documentFactory.createDocument( - chunkText, url, title, chunkIndex, packageName, hash); + Document document = documentFactory.createDocument(chunkText, url, title, chunkIndex, packageName, hash); documents.add(document); } @@ -122,10 +112,7 @@ public List processChunksOnly( * for long pages. Adds pageStart/pageEnd metadata to each resulting document. */ public List processPdfAndStoreWithPages( - java.nio.file.Path pdfPath, - String url, - String title, - String packageName) throws java.io.IOException { + java.nio.file.Path pdfPath, String url, String title, String packageName) throws java.io.IOException { List pages = pdfExtractor.extractPageTexts(pdfPath); List pageDocuments = new ArrayList<>(); @@ -141,14 +128,7 @@ public List processPdfAndStoreWithPages( String hash = hasher.generateChunkHash(url, globalIndex, chunkText); if (!localStore.isHashIngested(hash)) { Document doc = documentFactory.createDocumentWithPages( - chunkText, - url, - title, - globalIndex, - packageName, - hash, - pageIndex + 1, - pageIndex + 1); + chunkText, url, title, globalIndex, packageName, hash, pageIndex + 1, pageIndex + 1); pageDocuments.add(doc); localStore.saveChunkText(url, globalIndex, chunkText, hash); } diff --git a/src/main/java/com/williamcallahan/javachat/service/Chunker.java b/src/main/java/com/williamcallahan/javachat/service/Chunker.java index 6687acf..d0cf410 100644 --- a/src/main/java/com/williamcallahan/javachat/service/Chunker.java +++ b/src/main/java/com/williamcallahan/javachat/service/Chunker.java @@ -1,14 +1,13 @@ package com.williamcallahan.javachat.service; +import com.knuddels.jtokkit.Encodings; import com.knuddels.jtokkit.api.Encoding; -import com.knuddels.jtokkit.api.EncodingType; import com.knuddels.jtokkit.api.EncodingRegistry; -import com.knuddels.jtokkit.Encodings; +import com.knuddels.jtokkit.api.EncodingType; import com.knuddels.jtokkit.api.IntArrayList; -import org.springframework.stereotype.Component; - import java.util.ArrayList; import java.util.List; +import org.springframework.stereotype.Component; /** * Splits text into token-aware chunks using the shared encoding registry. @@ -69,4 +68,3 @@ public String keepLastTokens(String text, int maxTokens) { return encoding.decode(lastTokens); } } - diff --git a/src/main/java/com/williamcallahan/javachat/service/ContentHasher.java b/src/main/java/com/williamcallahan/javachat/service/ContentHasher.java index 0c1626a..d1704c2 100644 --- a/src/main/java/com/williamcallahan/javachat/service/ContentHasher.java +++ b/src/main/java/com/williamcallahan/javachat/service/ContentHasher.java @@ -1,10 +1,9 @@ package com.williamcallahan.javachat.service; -import org.springframework.stereotype.Component; - import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import org.springframework.stereotype.Component; /** * Computes stable hashes for content and chunk metadata. @@ -46,4 +45,3 @@ public String generateChunkHash(String url, int chunkIndex, String text) { return sha256(hashInput); } } - diff --git a/src/main/java/com/williamcallahan/javachat/service/DocsIngestionService.java b/src/main/java/com/williamcallahan/javachat/service/DocsIngestionService.java index d4f4378..faf0b1e 100644 --- a/src/main/java/com/williamcallahan/javachat/service/DocsIngestionService.java +++ b/src/main/java/com/williamcallahan/javachat/service/DocsIngestionService.java @@ -4,23 +4,22 @@ import com.williamcallahan.javachat.support.AsciiTextNormalizer; import com.williamcallahan.javachat.support.RetrievalErrorClassifier; import com.williamcallahan.javachat.support.RetrySupport; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.springframework.ai.vectorstore.VectorStore; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.stream.Stream; import java.time.Duration; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; /** * Ingests documentation content into the vector store with chunking, caching, and local snapshots. @@ -30,7 +29,7 @@ public class DocsIngestionService { private static final String DEFAULT_DOCS_ROOT = "data/docs"; private static final String SPRING_BOOT_REFERENCE_PATH = "/data/docs/spring-boot/reference.html"; private static final String SPRING_BOOT_REFERENCE_URL = - "https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/"; + "https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/"; private static final String FILE_URL_PREFIX = "file://"; private static final String API_PATH_SEGMENT = "/api/"; private static final Duration HTTP_CONNECT_TIMEOUT = Duration.ofSeconds(30); @@ -38,7 +37,7 @@ public class DocsIngestionService { private final ProgressTracker progressTracker; private static final Logger log = LoggerFactory.getLogger(DocsIngestionService.class); private static final Logger INDEXING_LOG = LoggerFactory.getLogger("INDEXING"); - + private final String rootUrl; private final VectorStore vectorStore; private final ChunkProcessingService chunkProcessingService; @@ -63,16 +62,17 @@ public class DocsIngestionService { * @param embeddingCache embedding cache for local-only mode * @param uploadMode ingestion upload mode */ - public DocsIngestionService(@Value("${app.docs.root-url}") String rootUrl, - VectorStore vectorStore, - ChunkProcessingService chunkProcessingService, - LocalStoreService localStore, - FileOperationsService fileOperationsService, - HtmlContentExtractor htmlExtractor, - PdfContentExtractor pdfExtractor, - ProgressTracker progressTracker, - EmbeddingCacheService embeddingCache, - @Value("${EMBEDDINGS_UPLOAD_MODE:upload}") String uploadMode) { + public DocsIngestionService( + @Value("${app.docs.root-url}") String rootUrl, + VectorStore vectorStore, + ChunkProcessingService chunkProcessingService, + LocalStoreService localStore, + FileOperationsService fileOperationsService, + HtmlContentExtractor htmlExtractor, + PdfContentExtractor pdfExtractor, + ProgressTracker progressTracker, + EmbeddingCacheService embeddingCache, + @Value("${EMBEDDINGS_UPLOAD_MODE:upload}") String uploadMode) { this.rootUrl = rootUrl; this.vectorStore = vectorStore; this.chunkProcessingService = chunkProcessingService; @@ -83,7 +83,7 @@ public DocsIngestionService(@Value("${app.docs.root-url}") String rootUrl, this.progressTracker = progressTracker; this.embeddingCache = embeddingCache; this.localOnlyMode = "local-only".equals(uploadMode); - + if (localOnlyMode) { INDEXING_LOG.info("[INDEXING] Running in LOCAL-ONLY mode - embeddings will be cached locally"); } else { @@ -103,17 +103,17 @@ public void crawlAndIngest(int maxPages) throws IOException { if (!visited.add(url)) continue; if (!url.startsWith(rootUrl)) continue; org.jsoup.Connection.Response response = Jsoup.connect(url) - .timeout((int) HTTP_CONNECT_TIMEOUT.toMillis()) - .maxBodySize(0) - .execute(); + .timeout((int) HTTP_CONNECT_TIMEOUT.toMillis()) + .maxBodySize(0) + .execute(); String rawHtml = Optional.ofNullable(response.body()).orElse(""); CrawlPageSnapshot pageSnapshot = prepareCrawlPageSnapshot(url, rawHtml); Document doc = pageSnapshot.document(); String title = Optional.ofNullable(doc.title()).orElse(""); Document extractionDoc = doc.clone(); - String bodyText = url.contains(API_PATH_SEGMENT) ? - htmlExtractor.extractJavaApiContent(extractionDoc) : - htmlExtractor.extractCleanContent(extractionDoc); + String bodyText = url.contains(API_PATH_SEGMENT) + ? htmlExtractor.extractJavaApiContent(extractionDoc) + : htmlExtractor.extractCleanContent(extractionDoc); String packageName = extractPackage(url, bodyText); localStore.saveHtml(url, pageSnapshot.rawHtml()); @@ -124,7 +124,7 @@ public void crawlAndIngest(int maxPages) throws IOException { } List documents = - chunkProcessingService.processAndStoreChunks(bodyText, url, title, packageName); + chunkProcessingService.processAndStoreChunks(bodyText, url, title, packageName); if (!documents.isEmpty()) { INDEXING_LOG.info("[INDEXING] Processing {} documents", documents.size()); @@ -135,7 +135,7 @@ public void crawlAndIngest(int maxPages) throws IOException { } } catch (RuntimeException storageException) { String destination = localOnlyMode ? "cache" : "Qdrant"; - INDEXING_LOG.error("[INDEXING] ✗ Failed to store documents to {}", destination, storageException); + INDEXING_LOG.error("[INDEXING] Failed to store documents"); throw new IOException("Failed to store documents to " + destination, storageException); } } else { @@ -156,9 +156,8 @@ public LocalIngestionOutcome ingestLocalDirectory(String rootDir, int maxFiles) Path root = Path.of(rootDir).toAbsolutePath().normalize(); Path baseDir = Path.of(DEFAULT_DOCS_ROOT).toAbsolutePath().normalize(); Path rootRoot = root.getRoot(); - Path absoluteBaseDir = rootRoot == null - ? baseDir - : rootRoot.resolve(DEFAULT_DOCS_ROOT).normalize(); + Path absoluteBaseDir = + rootRoot == null ? baseDir : rootRoot.resolve(DEFAULT_DOCS_ROOT).normalize(); if (!root.startsWith(baseDir) && !root.startsWith(absoluteBaseDir)) { throw new IllegalArgumentException("Local docs directory must be under " + absoluteBaseDir); } @@ -168,8 +167,7 @@ public LocalIngestionOutcome ingestLocalDirectory(String rootDir, int maxFiles) AtomicInteger processedCount = new AtomicInteger(0); List failures = new ArrayList<>(); try (Stream paths = Files.walk(root)) { - Iterator pathIterator = paths - .filter(pathCandidate -> !Files.isDirectory(pathCandidate)) + Iterator pathIterator = paths.filter(pathCandidate -> !Files.isDirectory(pathCandidate)) .filter(this::isIngestableFile) .iterator(); while (pathIterator.hasNext() && processedCount.get() < maxFiles) { @@ -200,15 +198,15 @@ private LocalFileProcessingOutcome processLocalFile(Path file) { long fileStartMillis = System.currentTimeMillis(); Path fileNamePath = file.getFileName(); if (fileNamePath == null) { - return new LocalFileProcessingOutcome(false, - new LocalIngestionFailure(file.toString(), "filename", "Missing filename")); + return new LocalFileProcessingOutcome( + false, new LocalIngestionFailure(file.toString(), "filename", "Missing filename")); } String fileName = fileNamePath.toString().toLowerCase(Locale.ROOT); String title; String bodyText = null; String url = mapLocalPathToUrl(file); String packageName; - + if (fileName.endsWith(".pdf")) { try { String metadata = pdfExtractor.getPdfMetadata(file); @@ -218,24 +216,25 @@ private LocalFileProcessingOutcome processLocalFile(Path file) { final Optional publicPdfUrl = DocsSourceRegistry.mapBookLocalToPublic(file.toString()); url = publicPdfUrl.orElse(url); } catch (IOException pdfExtractionException) { - log.error("Failed to extract PDF content (exception type: {})", - pdfExtractionException.getClass().getSimpleName()); - return new LocalFileProcessingOutcome(false, - failure(file, "pdf-extraction", pdfExtractionException)); + log.error( + "Failed to extract PDF content (exception type: {})", + pdfExtractionException.getClass().getSimpleName()); + return new LocalFileProcessingOutcome(false, failure(file, "pdf-extraction", pdfExtractionException)); } } else { try { String html = fileOperationsService.readTextFile(file); org.jsoup.nodes.Document doc = Jsoup.parse(html); title = Optional.ofNullable(doc.title()).orElse(""); - bodyText = url.contains(API_PATH_SEGMENT) ? - htmlExtractor.extractJavaApiContent(doc) : - htmlExtractor.extractCleanContent(doc); + bodyText = url.contains(API_PATH_SEGMENT) + ? htmlExtractor.extractJavaApiContent(doc) + : htmlExtractor.extractCleanContent(doc); packageName = extractPackage(url, bodyText); } catch (IOException htmlReadException) { - log.error("Failed to read HTML file (exception type: {})", htmlReadException.getClass().getSimpleName()); - return new LocalFileProcessingOutcome(false, - failure(file, "html-read", htmlReadException)); + log.error( + "Failed to read HTML file (exception type: {})", + htmlReadException.getClass().getSimpleName()); + return new LocalFileProcessingOutcome(false, failure(file, "html-read", htmlReadException)); } } @@ -247,23 +246,23 @@ private LocalFileProcessingOutcome processLocalFile(Path file) { documents = chunkProcessingService.processAndStoreChunks(bodyText, url, title, packageName); } } catch (IOException chunkingException) { - log.error("Chunking failed (exception type: {})", chunkingException.getClass().getSimpleName()); - return new LocalFileProcessingOutcome(false, - failure(file, "chunking", chunkingException)); + log.error( + "Chunking failed (exception type: {})", + chunkingException.getClass().getSimpleName()); + return new LocalFileProcessingOutcome(false, failure(file, "chunking", chunkingException)); } if (!documents.isEmpty()) { return processDocuments(file, documents, fileStartMillis); } else { INDEXING_LOG.debug("[INDEXING] Skipping empty document"); - return new LocalFileProcessingOutcome(false, - new LocalIngestionFailure(file.toString(), "empty-document", "No chunks generated")); + return new LocalFileProcessingOutcome( + false, new LocalIngestionFailure(file.toString(), "empty-document", "No chunks generated")); } } - private LocalFileProcessingOutcome processDocuments(Path file, - List documents, - long fileStartMillis) { + private LocalFileProcessingOutcome processDocuments( + Path file, List documents, long fileStartMillis) { INDEXING_LOG.info("[INDEXING] Processing file with {} chunks", documents.size()); try { @@ -271,10 +270,15 @@ private LocalFileProcessingOutcome processDocuments(Path file, // Per-file completion summary (end-to-end, including extraction + embedding + indexing) long totalDuration = System.currentTimeMillis() - fileStartMillis; - String destination = localOnlyMode ? "cache" : - (storageResult.usedPrimaryDestination() ? "Qdrant" : "cache (fallback)"); - INDEXING_LOG.info("[INDEXING] ✔ Completed processing {}/{} chunks to {} in {}ms (end-to-end) ({})", - documents.size(), documents.size(), destination, totalDuration, progressTracker.formatPercent()); + String destination = + localOnlyMode ? "cache" : (storageResult.usedPrimaryDestination() ? "Qdrant" : "cache (fallback)"); + INDEXING_LOG.info( + "[INDEXING] ✔ Completed processing {}/{} chunks to {} in {}ms (end-to-end) ({})", + documents.size(), + documents.size(), + destination, + totalDuration, + progressTracker.formatPercent()); // Mark hashes as processed only after confirmed primary destination write // Don't mark when we fell back to cache in upload mode - allows future re-upload @@ -289,10 +293,11 @@ private LocalFileProcessingOutcome processDocuments(Path file, throw indexingException; } String operation = localOnlyMode ? "cache" : "index"; - INDEXING_LOG.error("[INDEXING] ✗ Failed to {} file (exception type: {})", - operation, indexingException.getClass().getSimpleName()); - return new LocalFileProcessingOutcome(false, - failure(file, operation, indexingException)); + INDEXING_LOG.error( + "[INDEXING] ✗ Failed to {} file (exception type: {})", + operation, + indexingException.getClass().getSimpleName()); + return new LocalFileProcessingOutcome(false, failure(file, operation, indexingException)); } } @@ -309,8 +314,9 @@ private void markDocumentsIngested(List { vectorStore.add(documents); return null; }, - "Qdrant upload" - ); + () -> { + vectorStore.add(documents); + return null; + }, + "Qdrant upload"); long duration = System.currentTimeMillis() - startTime; - INDEXING_LOG.info("[INDEXING] ✓ Added {} vectors to Qdrant in {}ms ({})", - documents.size(), duration, progressTracker.formatPercent()); + INDEXING_LOG.info( + "[INDEXING] ✓ Added {} vectors to Qdrant in {}ms ({})", + documents.size(), + duration, + progressTracker.formatPercent()); return new DocumentStorageResult(true, true); } catch (RuntimeException qdrantError) { if (RetrievalErrorClassifier.isTransientVectorStoreError(qdrantError)) { - INDEXING_LOG.warn("[INDEXING] Qdrant upload failed after retries ({}), falling back to local cache", - qdrantError.getClass().getSimpleName()); + INDEXING_LOG.warn( + "[INDEXING] Qdrant upload failed after retries ({}), falling back to local cache", + qdrantError.getClass().getSimpleName()); embeddingCache.getOrComputeEmbeddings(documents); long duration = System.currentTimeMillis() - startTime; - INDEXING_LOG.info("[INDEXING] ✓ Cached {} vectors locally (fallback) in {}ms ({})", - documents.size(), duration, progressTracker.formatPercent()); + INDEXING_LOG.info( + "[INDEXING] ✓ Cached {} vectors locally (fallback) in {}ms ({})", + documents.size(), + duration, + progressTracker.formatPercent()); return new DocumentStorageResult(true, false); } - INDEXING_LOG.error("[INDEXING] Qdrant upload failed with non-transient error ({}), not falling back", - qdrantError.getClass().getSimpleName()); + INDEXING_LOG.error( + "[INDEXING] Qdrant upload failed with non-transient error ({}), not falling back", + qdrantError.getClass().getSimpleName()); throw qdrantError; } } @@ -386,8 +405,8 @@ private LocalIngestionFailure failure(Path file, String phase, Exception excepti details.append(" [").append(diagnosticHint).append("]"); } else if (exception.getCause() != null) { details.append(" [caused by: ") - .append(exception.getCause().getClass().getSimpleName()) - .append("]"); + .append(exception.getCause().getClass().getSimpleName()) + .append("]"); } return new LocalIngestionFailure(file.toString(), phase, details.toString()); @@ -422,8 +441,8 @@ static String getDiagnosticHint(Exception exception) { private String mapLocalPathToUrl(final Path file) { final String absolutePath = file.toAbsolutePath().toString().replace('\\', '/'); return DocsSourceRegistry.resolveLocalPath(absolutePath) - .or(() -> mapKnownMirrorUrl(absolutePath)) - .orElse(FILE_URL_PREFIX + absolutePath); + .or(() -> mapKnownMirrorUrl(absolutePath)) + .orElse(FILE_URL_PREFIX + absolutePath); } private Optional mapKnownMirrorUrl(final String absolutePath) { diff --git a/src/main/java/com/williamcallahan/javachat/service/DocumentFactory.java b/src/main/java/com/williamcallahan/javachat/service/DocumentFactory.java index 38050cf..ca6c76c 100644 --- a/src/main/java/com/williamcallahan/javachat/service/DocumentFactory.java +++ b/src/main/java/com/williamcallahan/javachat/service/DocumentFactory.java @@ -1,10 +1,9 @@ package com.williamcallahan.javachat.service; -import org.springframework.stereotype.Service; - import java.util.HashMap; import java.util.Map; import java.util.Objects; +import org.springframework.stereotype.Service; /** * Factory service for creating standardized Spring AI Document objects @@ -28,20 +27,14 @@ public class DocumentFactory { * @return A properly configured Spring AI Document */ public org.springframework.ai.document.Document createDocument( - String text, - String url, - String title, - int chunkIndex, - String packageName, - String hash) { + String text, String url, String title, int chunkIndex, String packageName, String hash) { Map metadata = Map.of( - "url", url, - "title", title, - "chunkIndex", chunkIndex, - "package", packageName, - "hash", hash - ); + "url", url, + "title", title, + "chunkIndex", chunkIndex, + "package", packageName, + "hash", hash); // Create and configure the document var document = createDocumentWithOptionalId(text, hash); @@ -59,10 +52,7 @@ public org.springframework.ai.document.Document createDocument( * @return A properly configured Spring AI Document */ public org.springframework.ai.document.Document createLocalDocument(String text, String url) { - Map metadata = Map.of( - "url", url, - "title", "Local Doc" - ); + Map metadata = Map.of("url", url, "title", "Local Doc"); var document = new org.springframework.ai.document.Document(text); document.getMetadata().putAll(metadata); @@ -81,9 +71,7 @@ public org.springframework.ai.document.Document createLocalDocument(String text, * @throws NullPointerException if any parameter is null */ public org.springframework.ai.document.Document createWithPreservedMetadata( - String newText, - Map existingMetadata, - Map additionalMetadata) { + String newText, Map existingMetadata, Map additionalMetadata) { Objects.requireNonNull(newText, "newText must not be null"); Objects.requireNonNull(existingMetadata, "existingMetadata must not be null; use Map.of() for empty"); diff --git a/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheEntry.java b/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheEntry.java index 4929d02..dd8d345 100644 --- a/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheEntry.java +++ b/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheEntry.java @@ -6,23 +6,21 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.TextNode; -import org.springframework.ai.document.Document; - import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Set; +import org.springframework.ai.document.Document; /** * Captures a cached embedding payload with metadata for disk persistence. */ @JsonAutoDetect( - fieldVisibility = JsonAutoDetect.Visibility.ANY, - getterVisibility = JsonAutoDetect.Visibility.NONE, - isGetterVisibility = JsonAutoDetect.Visibility.NONE, - setterVisibility = JsonAutoDetect.Visibility.NONE -) + fieldVisibility = JsonAutoDetect.Visibility.ANY, + getterVisibility = JsonAutoDetect.Visibility.NONE, + isGetterVisibility = JsonAutoDetect.Visibility.NONE, + setterVisibility = JsonAutoDetect.Visibility.NONE) @JsonIgnoreProperties(ignoreUnknown = true) final class EmbeddingCacheEntry { private String id; @@ -100,32 +98,29 @@ Document toDocument() { @JsonIgnoreProperties(ignoreUnknown = true) record EmbeddingCacheMetadata( - @JsonProperty("url") String url, - @JsonProperty("title") String title, - @JsonProperty("chunkIndex") Integer chunkIndex, - @JsonProperty("package") String packageName, - @JsonProperty("hash") String hash, - @JsonProperty("pageStart") Integer pageStart, - @JsonProperty("pageEnd") Integer pageEnd, - @JsonProperty("retrievalSource") String retrievalSource, - @JsonProperty("fallbackReason") String fallbackReason, - @JsonProperty("additionalMetadata") Map additionalMetadata -) { + @JsonProperty("url") String url, + @JsonProperty("title") String title, + @JsonProperty("chunkIndex") Integer chunkIndex, + @JsonProperty("package") String packageName, + @JsonProperty("hash") String hash, + @JsonProperty("pageStart") Integer pageStart, + @JsonProperty("pageEnd") Integer pageEnd, + @JsonProperty("retrievalSource") String retrievalSource, + @JsonProperty("fallbackReason") String fallbackReason, + @JsonProperty("additionalMetadata") Map additionalMetadata) { private static final Set KNOWN_KEYS = Set.of( - "url", - "title", - "chunkIndex", - "package", - "hash", - "pageStart", - "pageEnd", - "retrievalSource", - "fallbackReason" - ); - - private static final EmbeddingCacheMetadata EMPTY = new EmbeddingCacheMetadata( - null, null, null, null, null, null, null, null, null, Map.of() - ); + "url", + "title", + "chunkIndex", + "package", + "hash", + "pageStart", + "pageEnd", + "retrievalSource", + "fallbackReason"); + + private static final EmbeddingCacheMetadata EMPTY = + new EmbeddingCacheMetadata(null, null, null, null, null, null, null, null, null, Map.of()); static EmbeddingCacheMetadata empty() { return EMPTY; @@ -151,17 +146,16 @@ static EmbeddingCacheMetadata fromDocument(Document sourceDocument) { Map additionalMetadata = additionalMetadataFrom(springMetadata); return new EmbeddingCacheMetadata( - blankToNull(url), - blankToNull(title), - chunkIndex, - blankToNull(packageName), - blankToNull(hash), - pageStart, - pageEnd, - blankToNull(retrievalSource), - blankToNull(fallbackReason), - additionalMetadata - ); + blankToNull(url), + blankToNull(title), + chunkIndex, + blankToNull(packageName), + blankToNull(hash), + pageStart, + pageEnd, + blankToNull(retrievalSource), + blankToNull(fallbackReason), + additionalMetadata); } static EmbeddingCacheMetadata fromLegacyMetadataMap(Map legacyMetadata) { @@ -182,17 +176,16 @@ static EmbeddingCacheMetadata fromLegacyMetadataMap(Map legacyMetadata) { Map additionalMetadata = additionalMetadataFromLegacy(legacyMetadata); return new EmbeddingCacheMetadata( - blankToNull(url), - blankToNull(title), - chunkIndex, - blankToNull(packageName), - blankToNull(hash), - pageStart, - pageEnd, - blankToNull(retrievalSource), - blankToNull(fallbackReason), - additionalMetadata - ); + blankToNull(url), + blankToNull(title), + chunkIndex, + blankToNull(packageName), + blankToNull(hash), + pageStart, + pageEnd, + blankToNull(retrievalSource), + blankToNull(fallbackReason), + additionalMetadata); } void applyTo(Document document) { @@ -278,7 +271,9 @@ private static Map additionalMetadataFrom(Map sourc private static Map additionalMetadataFromLegacy(Map sourceMetadata) { LinkedHashMap additionalMetadata = new LinkedHashMap<>(); sourceMetadata.forEach((legacyKey, legacyValue) -> { - if (!(legacyKey instanceof String metadataKey) || metadataKey.isBlank() || KNOWN_KEYS.contains(metadataKey)) { + if (!(legacyKey instanceof String metadataKey) + || metadataKey.isBlank() + || KNOWN_KEYS.contains(metadataKey)) { return; } JsonNode node = jsonNodeFrom(legacyValue); diff --git a/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheFileImporter.java b/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheFileImporter.java index e8b9a08..c10442b 100644 --- a/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheFileImporter.java +++ b/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheFileImporter.java @@ -1,8 +1,9 @@ package com.williamcallahan.javachat.service; +import static java.io.ObjectInputFilter.Status; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; - import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -13,8 +14,6 @@ import java.util.Objects; import java.util.UUID; -import static java.io.ObjectInputFilter.Status; - /** * Reads embedding cache files in JSON or legacy serialized formats. */ @@ -112,18 +111,18 @@ private List readJsonCache(InputStream jsonStream) throws I String className = serializedClass.getName(); if (Objects.equals(className, EmbeddingCacheService.CachedEmbedding.class.getName()) - || Objects.equals(className, "java.util.ArrayList") - || Objects.equals(className, "java.util.HashMap") - || Objects.equals(className, "java.util.LinkedHashMap") - || Objects.equals(className, "java.util.Collections$UnmodifiableMap") - || Objects.equals(className, "java.lang.String") - || Objects.equals(className, "java.lang.Integer") - || Objects.equals(className, "java.lang.Long") - || Objects.equals(className, "java.lang.Double") - || Objects.equals(className, "java.lang.Float") - || Objects.equals(className, "java.lang.Boolean") - || Objects.equals(className, "java.time.LocalDateTime") - || Objects.equals(className, "java.time.Ser")) { + || Objects.equals(className, "java.util.ArrayList") + || Objects.equals(className, "java.util.HashMap") + || Objects.equals(className, "java.util.LinkedHashMap") + || Objects.equals(className, "java.util.Collections$UnmodifiableMap") + || Objects.equals(className, "java.lang.String") + || Objects.equals(className, "java.lang.Integer") + || Objects.equals(className, "java.lang.Long") + || Objects.equals(className, "java.lang.Double") + || Objects.equals(className, "java.lang.Float") + || Objects.equals(className, "java.lang.Boolean") + || Objects.equals(className, "java.time.LocalDateTime") + || Objects.equals(className, "java.time.Ser")) { return Status.ALLOWED; } @@ -140,23 +139,24 @@ private List readLegacyJavaSerializedCache(InputStream lega } if (!(deserialized instanceof List legacyList)) { throw new IOException("Unexpected legacy cache format; expected a List but got: " - + deserialized.getClass().getName()); + + deserialized.getClass().getName()); } List converted = new ArrayList<>(legacyList.size()); for (Object legacyEntry : legacyList) { if (!(legacyEntry instanceof EmbeddingCacheService.CachedEmbedding cachedEmbedding)) { throw new IOException("Unexpected legacy cache entry type: " - + (legacyEntry == null ? "null" : legacyEntry.getClass().getName())); + + (legacyEntry == null + ? "null" + : legacyEntry.getClass().getName())); } EmbeddingCacheEntry convertedEntry = new EmbeddingCacheEntry( - cachedEmbedding.id == null || cachedEmbedding.id.isBlank() - ? UUID.randomUUID().toString() - : cachedEmbedding.id, - cachedEmbedding.content, - cachedEmbedding.embedding, - EmbeddingCacheMetadata.fromLegacyMetadataMap(cachedEmbedding.metadata) - ); + cachedEmbedding.id == null || cachedEmbedding.id.isBlank() + ? UUID.randomUUID().toString() + : cachedEmbedding.id, + cachedEmbedding.content, + cachedEmbedding.embedding, + EmbeddingCacheMetadata.fromLegacyMetadataMap(cachedEmbedding.metadata)); convertedEntry.setCreatedAt(cachedEmbedding.createdAt); convertedEntry.setUploaded(cachedEmbedding.uploaded); converted.add(convertedEntry); diff --git a/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheService.java b/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheService.java index b616535..d5a8a7e 100644 --- a/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheService.java +++ b/src/main/java/com/williamcallahan/javachat/service/EmbeddingCacheService.java @@ -5,15 +5,6 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import jakarta.annotation.PostConstruct; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.ai.document.Document; -import org.springframework.ai.embedding.EmbeddingModel; -import org.springframework.ai.embedding.EmbeddingResponse; -import org.springframework.ai.vectorstore.VectorStore; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - import java.io.*; import java.nio.file.Files; import java.nio.file.Path; @@ -22,13 +13,21 @@ import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.ai.document.Document; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.embedding.EmbeddingResponse; +import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; /** * Service for caching embeddings locally to reduce API calls and enable batch processing. @@ -37,29 +36,30 @@ @Service public class EmbeddingCacheService { private static final Logger CACHE_LOG = LoggerFactory.getLogger("EMBEDDING_CACHE"); - + private final ObjectMapper cacheMapper; private final Path cacheDir; private final EmbeddingModel embeddingModel; private final VectorStore vectorStore; /** In-memory cache for fast lookup of computed embeddings */ private final Map memoryCache = new ConcurrentHashMap<>(); + private final EmbeddingCacheFileImporter cacheFileImporter; private final AtomicInteger cacheHits = new AtomicInteger(0); private final AtomicInteger cacheMisses = new AtomicInteger(0); - private final AtomicInteger embeddingsSinceLastSave = new AtomicInteger(0); - private static final int AUTO_SAVE_THRESHOLD = 50; // Save every 50 embeddings - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + private final AtomicInteger embeddingsSinceLastSave = new AtomicInteger(0); + private static final int AUTO_SAVE_THRESHOLD = 50; // Save every 50 embeddings + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); private final Object cacheFileLock = new Object(); - /** - * Wraps cache persistence failures as a runtime exception suitable for Spring initialization paths. - */ - private static final class EmbeddingCacheOperationException extends IllegalStateException { - private EmbeddingCacheOperationException(String message, Exception cause) { - super(message, cause); - } - } + /** + * Wraps cache persistence failures as a runtime exception suitable for Spring initialization paths. + */ + private static final class EmbeddingCacheOperationException extends IllegalStateException { + private EmbeddingCacheOperationException(String message, Exception cause) { + super(message, cause); + } + } /** * Validates filename and resolves to a safe path within cacheDir. @@ -69,16 +69,16 @@ private EmbeddingCacheOperationException(String message, Exception cause) { * @return the validated, normalized path * @throws InvalidParameterException if filename would escape cacheDir */ - private Path validateAndResolvePath(String filename) { + private Path validateAndResolvePath(String filename) { if (filename == null || filename.isBlank()) { throw new InvalidParameterException("Filename cannot be null or blank"); } // Reject obvious path traversal attempts and path separators if (filename.contains("..") - || filename.startsWith("/") - || filename.contains(":") - || filename.contains("/") - || filename.contains("\\")) { + || filename.startsWith("/") + || filename.contains(":") + || filename.contains("/") + || filename.contains("\\")) { throw new InvalidParameterException("Invalid filename: path traversal not allowed"); } Path resolved = cacheDir.resolve(filename).normalize(); @@ -86,8 +86,8 @@ private Path validateAndResolvePath(String filename) { if (!resolved.startsWith(cacheDir.normalize())) { throw new InvalidParameterException("Invalid filename: resolved path escapes cache directory"); } - return resolved; - } + return resolved; + } /** * Creates an embedding cache service rooted at the configured cache directory. @@ -100,18 +100,19 @@ public EmbeddingCacheService( this.cacheDir = Path.of(cacheDir); this.embeddingModel = embeddingModel; this.vectorStore = vectorStore; - this.cacheMapper = objectMapper.copy() - .registerModule(new JavaTimeModule()) - .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) - .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + this.cacheMapper = objectMapper + .copy() + .registerModule(new JavaTimeModule()) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); this.cacheFileImporter = new EmbeddingCacheFileImporter(this.cacheMapper); } - /** - * Initializes the cache directory and schedules periodic persistence for incremental updates. - */ - @PostConstruct - public void initializeCache() { + /** + * Initializes the cache directory and schedules periodic persistence for incremental updates. + */ + @PostConstruct + public void initializeCache() { try { Files.createDirectories(cacheDir); } catch (IOException exception) { @@ -128,26 +129,33 @@ public void initializeCache() { saveIncrementalCache(); CACHE_LOG.info("Cache saved successfully. Total embeddings cached: {}", memoryCache.size()); } catch (RuntimeException exception) { - CACHE_LOG.error("Failed to save cache on shutdown (exception type: {})", - exception.getClass().getSimpleName()); + CACHE_LOG.error( + "Failed to save cache on shutdown (exception type: {})", + exception.getClass().getSimpleName()); } })); // Start periodic save timer (every 2 minutes) - scheduler.scheduleAtFixedRate(() -> { - try { - if (embeddingsSinceLastSave.get() > 0) { - CACHE_LOG.info("Periodic save: {} new embeddings since last save", embeddingsSinceLastSave.get()); - saveIncrementalCache(); - embeddingsSinceLastSave.set(0); - } - } catch (RuntimeException exception) { - CACHE_LOG.error("Periodic save failed (exception type: {})", - exception.getClass().getSimpleName()); - } - }, 2, 2, TimeUnit.MINUTES); + scheduler.scheduleAtFixedRate( + () -> { + try { + if (embeddingsSinceLastSave.get() > 0) { + CACHE_LOG.info( + "Periodic save: {} new embeddings since last save", embeddingsSinceLastSave.get()); + saveIncrementalCache(); + embeddingsSinceLastSave.set(0); + } + } catch (RuntimeException exception) { + CACHE_LOG.error( + "Periodic save failed (exception type: {})", + exception.getClass().getSimpleName()); + } + }, + 2, + 2, + TimeUnit.MINUTES); } - + /** * Gets embeddings from cache or computes them if not cached * @param documents List of documents to get embeddings for @@ -157,12 +165,12 @@ public List getOrComputeEmbeddings(List documents) { List embeddings = new ArrayList<>(); List toCompute = new ArrayList<>(); Map indexMapping = new HashMap<>(); - + for (int documentIndex = 0; documentIndex < documents.size(); documentIndex++) { Document sourceDocument = documents.get(documentIndex); String cacheKey = generateCacheKey(sourceDocument); EmbeddingCacheEntry cachedEmbedding = memoryCache.get(cacheKey); - + if (cachedEmbedding != null) { embeddings.add(cachedEmbedding.getEmbedding()); cacheHits.incrementAndGet(); @@ -174,36 +182,34 @@ public List getOrComputeEmbeddings(List documents) { cacheMisses.incrementAndGet(); } } - + if (!toCompute.isEmpty()) { - CACHE_LOG.info("Computing {} new embeddings (cache hit rate: {:.1f}%)", - toCompute.size(), getCacheHitRate() * 100); - - List texts = toCompute.stream() - .map(Document::getText) - .collect(Collectors.toList()); - + CACHE_LOG.info( + "Computing {} new embeddings (cache hit rate: {}%)", + toCompute.size(), String.format("%.1f", getCacheHitRate() * 100)); + + List texts = toCompute.stream().map(Document::getText).collect(Collectors.toList()); + EmbeddingResponse response = embeddingModel.embedForResponse(texts); List computedEmbeddings = response.getResults().stream() - .map(embeddingResult -> embeddingResult.getOutput()) - .collect(Collectors.toList()); - + .map(embeddingResult -> embeddingResult.getOutput()) + .collect(Collectors.toList()); + for (int computedIndex = 0; computedIndex < computedEmbeddings.size(); computedIndex++) { Document sourceDocument = toCompute.get(computedIndex); float[] embeddingVector = computedEmbeddings.get(computedIndex); int originalIndex = indexMapping.get(computedIndex); embeddings.set(originalIndex, embeddingVector); - + EmbeddingCacheEntry cachedEmbedding = new EmbeddingCacheEntry( - UUID.randomUUID().toString(), - sourceDocument.getText(), - embeddingVector, - EmbeddingCacheMetadata.fromDocument(sourceDocument) - ); - + UUID.randomUUID().toString(), + sourceDocument.getText(), + embeddingVector, + EmbeddingCacheMetadata.fromDocument(sourceDocument)); + String cacheKey = generateCacheKey(sourceDocument); memoryCache.put(cacheKey, cachedEmbedding); - + // Auto-save every N embeddings if (embeddingsSinceLastSave.incrementAndGet() >= AUTO_SAVE_THRESHOLD) { CACHE_LOG.info("Auto-saving cache after {} new embeddings...", AUTO_SAVE_THRESHOLD); @@ -212,17 +218,17 @@ public List getOrComputeEmbeddings(List documents) { CACHE_LOG.info("Auto-save completed. Total cached: {}", memoryCache.size()); } } - + // Final save after batch completion if (embeddingsSinceLastSave.get() > 0) { saveIncrementalCache(); embeddingsSinceLastSave.set(0); } } - + return embeddings; } - + /** * Exports cache to compressed file. * Validates filename to prevent path traversal attacks per OWASP guidelines. @@ -237,7 +243,7 @@ public void exportCache(String filename) throws IOException { synchronized (cacheFileLock) { try (OutputStream fos = Files.newOutputStream(exportPath); - GZIPOutputStream gzos = new GZIPOutputStream(fos)) { + GZIPOutputStream gzos = new GZIPOutputStream(fos)) { cacheMapper.writeValue(gzos, new ArrayList<>(memoryCache.values())); CACHE_LOG.info("Successfully exported cache"); } @@ -257,7 +263,7 @@ public final void importCache(String filename) throws IOException { Path importPath = validateAndResolvePath(filename); importCacheFromPath(importPath); } - + /** * Uploads pending embeddings to Qdrant in batches * @param batchSize Number of embeddings per batch @@ -265,54 +271,55 @@ public final void importCache(String filename) throws IOException { */ public int uploadPendingToVectorStore(int batchSize) { List pendingEmbeddings = memoryCache.values().stream() - .filter(cachedEmbedding -> !cachedEmbedding.isUploaded()) - .collect(Collectors.toList()); - + .filter(cachedEmbedding -> !cachedEmbedding.isUploaded()) + .collect(Collectors.toList()); + if (pendingEmbeddings.isEmpty()) { CACHE_LOG.info("No pending embeddings to upload"); return 0; } - - CACHE_LOG.info("Uploading {} pending embeddings to vector store in batches of {}", - pendingEmbeddings.size(), batchSize); - + + CACHE_LOG.info( + "Uploading {} pending embeddings to vector store in batches of {}", + pendingEmbeddings.size(), + batchSize); + int uploadedCount = 0; for (int batchStartIndex = 0; batchStartIndex < pendingEmbeddings.size(); batchStartIndex += batchSize) { int batchEndIndex = Math.min(batchStartIndex + batchSize, pendingEmbeddings.size()); List batch = pendingEmbeddings.subList(batchStartIndex, batchEndIndex); - - List documents = batch.stream() - .map(EmbeddingCacheEntry::toDocument) - .collect(Collectors.toList()); - + + List documents = + batch.stream().map(EmbeddingCacheEntry::toDocument).collect(Collectors.toList()); + try { vectorStore.add(documents); - + for (EmbeddingCacheEntry cachedEmbedding : batch) { cachedEmbedding.setUploaded(true); } - + uploadedCount += batch.size(); - CACHE_LOG.info("Uploaded batch {}/{} ({} embeddings)", - (batchStartIndex / batchSize) + 1, - (pendingEmbeddings.size() + batchSize - 1) / batchSize, - batch.size()); - + CACHE_LOG.info( + "Uploaded batch {}/{} ({} embeddings)", + (batchStartIndex / batchSize) + 1, + (pendingEmbeddings.size() + batchSize - 1) / batchSize, + batch.size()); + } catch (Exception exception) { - CACHE_LOG.error("Failed to upload batch (exception type: {})", - exception.getClass().getSimpleName()); + CACHE_LOG.error( + "Failed to upload batch (exception type: {})", + exception.getClass().getSimpleName()); throw new EmbeddingCacheOperationException( - "Failed to upload embedding batch starting at index " + batchStartIndex, - exception - ); + "Failed to upload embedding batch starting at index " + batchStartIndex, exception); } } - + saveIncrementalCache(); CACHE_LOG.info("Successfully uploaded {} embeddings to vector store", uploadedCount); return uploadedCount; } - + /** * Saves timestamped snapshot of current cache */ @@ -321,7 +328,7 @@ public void saveSnapshot() throws IOException { String filename = String.format("embeddings_snapshot_%s.gz", timestamp); exportCache(filename); } - + /** * Returns cache statistics including hit rate and pending uploads. * @@ -329,14 +336,17 @@ public void saveSnapshot() throws IOException { */ public CacheStats getCacheStats() { return new CacheStats( - memoryCache.size(), - memoryCache.values().stream().filter(EmbeddingCacheEntry::isUploaded).count(), - memoryCache.values().stream().filter(cachedEmbedding -> !cachedEmbedding.isUploaded()).count(), - cacheHits.get(), - cacheMisses.get(), - getCacheHitRate(), - cacheDir.toString() - ); + memoryCache.size(), + memoryCache.values().stream() + .filter(EmbeddingCacheEntry::isUploaded) + .count(), + memoryCache.values().stream() + .filter(cachedEmbedding -> !cachedEmbedding.isUploaded()) + .count(), + cacheHits.get(), + cacheMisses.get(), + getCacheHitRate(), + cacheDir.toString()); } /** @@ -351,15 +361,14 @@ public CacheStats getCacheStats() { * @param cacheDirectory path to cache directory */ public record CacheStats( - long totalCached, - long uploaded, - long pending, - long cacheHits, - long cacheMisses, - double hitRate, - String cacheDirectory - ) {} - + long totalCached, + long uploaded, + long pending, + long cacheHits, + long cacheMisses, + double hitRate, + String cacheDirectory) {} + private void loadExistingCache() { Path latestCache = cacheDir.resolve("embeddings_cache.gz"); if (Files.exists(latestCache)) { @@ -367,30 +376,32 @@ private void loadExistingCache() { importCacheFromPath(latestCache); CACHE_LOG.info("Loaded existing cache with {} embeddings", memoryCache.size()); } catch (Exception exception) { - CACHE_LOG.error("Could not load existing cache (exception type: {})", - exception.getClass().getSimpleName()); + CACHE_LOG.error( + "Could not load existing cache (exception type: {})", + exception.getClass().getSimpleName()); throw new EmbeddingCacheOperationException("Failed to load existing cache", exception); } } } - + private void saveIncrementalCache() { try { exportCache("embeddings_cache.gz"); } catch (IOException exception) { - CACHE_LOG.error("Failed to save incremental cache (exception type: {})", - exception.getClass().getSimpleName()); + CACHE_LOG.error( + "Failed to save incremental cache (exception type: {})", + exception.getClass().getSimpleName()); throw new EmbeddingCacheOperationException("Failed to save incremental cache", exception); } } - + private String generateCacheKey(Document doc) { if (doc == null) { throw new IllegalArgumentException("Document is required"); } return generateCacheKey(doc.getText(), EmbeddingCacheMetadata.fromDocument(doc)); } - + private String generateCacheKey(String content, EmbeddingCacheMetadata metadata) { StringBuilder key = new StringBuilder(); String safeContent = content == null ? "" : content; @@ -406,8 +417,8 @@ private void importCacheFromPath(Path importPath) throws IOException { synchronized (cacheFileLock) { try (InputStream fileInputStream = Files.newInputStream(importPath); - GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); - BufferedInputStream bufferedInputStream = new BufferedInputStream(gzipInputStream)) { + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); + BufferedInputStream bufferedInputStream = new BufferedInputStream(gzipInputStream)) { List importedEmbeddings = cacheFileImporter.read(bufferedInputStream); @@ -441,7 +452,7 @@ static final class CachedEmbedding implements Serializable { CachedEmbedding() {} } - + private double getCacheHitRate() { int total = cacheHits.get() + cacheMisses.get(); return total == 0 ? 0.0 : (double) cacheHits.get() / total; diff --git a/src/main/java/com/williamcallahan/javachat/service/EnrichmentService.java b/src/main/java/com/williamcallahan/javachat/service/EnrichmentService.java index f7b0730..0bdefdb 100644 --- a/src/main/java/com/williamcallahan/javachat/service/EnrichmentService.java +++ b/src/main/java/com/williamcallahan/javachat/service/EnrichmentService.java @@ -1,16 +1,15 @@ package com.williamcallahan.javachat.service; -import com.williamcallahan.javachat.model.Enrichment; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.williamcallahan.javachat.model.Enrichment; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; -import java.util.List; - /** * Generates enrichment metadata by prompting the LLM with contextual snippets. */ @@ -24,8 +23,7 @@ public class EnrichmentService { /** * Creates the enrichment service with JSON handling and LLM access. */ - public EnrichmentService(ObjectMapper objectMapper, - OpenAIStreamingService openAIStreamingService) { + public EnrichmentService(ObjectMapper objectMapper, OpenAIStreamingService openAIStreamingService) { this.objectMapper = objectMapper.copy(); this.openAIStreamingService = openAIStreamingService; } @@ -69,7 +67,9 @@ public Enrichment enrich(String userQuery, String jdkVersion, List conte json = "{}"; } } catch (RuntimeException exception) { - logger.warn("Enrichment service failed (exception type: {})", exception.getClass().getSimpleName()); + logger.warn( + "Enrichment service failed (exception type: {})", + exception.getClass().getSimpleName()); json = "{}"; // Return empty JSON for graceful degradation } @@ -96,9 +96,6 @@ public Enrichment enrich(String userQuery, String jdkVersion, List conte } } - - - private String cleanJson(String raw) { if (raw == null) return "{}"; String trimmedJson = raw.trim(); diff --git a/src/main/java/com/williamcallahan/javachat/service/ExternalServiceHealth.java b/src/main/java/com/williamcallahan/javachat/service/ExternalServiceHealth.java index dde0bad..f5e1204 100644 --- a/src/main/java/com/williamcallahan/javachat/service/ExternalServiceHealth.java +++ b/src/main/java/com/williamcallahan/javachat/service/ExternalServiceHealth.java @@ -1,13 +1,5 @@ package com.williamcallahan.javachat.service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.springframework.web.reactive.function.client.WebClient; -import reactor.core.publisher.Mono; - import jakarta.annotation.PostConstruct; import java.time.Duration; import java.time.Instant; @@ -15,6 +7,13 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.core.publisher.Mono; /** * Monitors external service health with exponential backoff for failed services. @@ -127,18 +126,18 @@ public ServiceInfo getServiceInfo(String serviceName) { Duration timeUntilNextCheck = null; if (status.isHealthy.get()) { - message = String.format("Healthy (checked %s ago)", - formatDuration(Duration.between(status.lastCheck, Instant.now()))); + message = String.format( + "Healthy (checked %s ago)", formatDuration(Duration.between(status.lastCheck, Instant.now()))); } else { - timeUntilNextCheck = Duration.between(Instant.now(), - status.lastCheck.plus(status.currentBackoff)); + timeUntilNextCheck = Duration.between(Instant.now(), status.lastCheck.plus(status.currentBackoff)); if (timeUntilNextCheck.isNegative()) { message = "Unhealthy (checking now...)"; timeUntilNextCheck = Duration.ZERO; } else { - message = String.format("Unhealthy (failed %d times, next check in %s)", - status.consecutiveFailures.get(), formatDuration(timeUntilNextCheck)); + message = String.format( + "Unhealthy (failed %d times, next check in %s)", + status.consecutiveFailures.get(), formatDuration(timeUntilNextCheck)); } } @@ -184,20 +183,22 @@ private void checkQdrantHealth() { } requestSpec - .retrieve() - .toBodilessEntity() - .timeout(Duration.ofSeconds(5)) - .doOnSuccess(response -> { - status.markHealthy(); - log.debug("Qdrant health check succeeded"); - }) - .doOnError(error -> { - status.markUnhealthy(); - log.warn("Qdrant health check failed (exception type: {}) - Will retry in {}", - error.getClass().getSimpleName(), formatDuration(status.currentBackoff)); - }) - .onErrorResume(error -> Mono.empty()) - .subscribe(); + .retrieve() + .toBodilessEntity() + .timeout(Duration.ofSeconds(5)) + .doOnSuccess(response -> { + status.markHealthy(); + log.debug("Qdrant health check succeeded"); + }) + .doOnError(error -> { + status.markUnhealthy(); + log.warn( + "Qdrant health check failed (exception type: {}) - Will retry in {}", + error.getClass().getSimpleName(), + formatDuration(status.currentBackoff)); + }) + .onErrorResume(error -> Mono.empty()) + .subscribe(); } /** diff --git a/src/main/java/com/williamcallahan/javachat/service/FileOperationsService.java b/src/main/java/com/williamcallahan/javachat/service/FileOperationsService.java index bae24f3..90024c4 100644 --- a/src/main/java/com/williamcallahan/javachat/service/FileOperationsService.java +++ b/src/main/java/com/williamcallahan/javachat/service/FileOperationsService.java @@ -1,7 +1,5 @@ package com.williamcallahan.javachat.service; -import org.springframework.stereotype.Service; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.CharsetDecoder; @@ -10,6 +8,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import org.springframework.stereotype.Service; /** * Centralized service for file operations to eliminate duplication @@ -43,7 +42,8 @@ public String readTextFile(Path filePath) throws IOException { } catch (MalformedInputException mie) { // Fallback: decode with replacement to handle non-UTF8 bytes gracefully byte[] bytes = Files.readAllBytes(filePath); - CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder() + CharsetDecoder decoder = StandardCharsets.UTF_8 + .newDecoder() .onMalformedInput(CodingErrorAction.REPLACE) .onUnmappableCharacter(CodingErrorAction.REPLACE); return decoder.decode(ByteBuffer.wrap(bytes)).toString(); diff --git a/src/main/java/com/williamcallahan/javachat/service/GracefulEmbeddingModel.java b/src/main/java/com/williamcallahan/javachat/service/GracefulEmbeddingModel.java index f9b6ad3..a0e4dcb 100644 --- a/src/main/java/com/williamcallahan/javachat/service/GracefulEmbeddingModel.java +++ b/src/main/java/com/williamcallahan/javachat/service/GracefulEmbeddingModel.java @@ -17,9 +17,7 @@ */ public class GracefulEmbeddingModel implements EmbeddingModel { - private static final Logger log = LoggerFactory.getLogger( - GracefulEmbeddingModel.class - ); + private static final Logger log = LoggerFactory.getLogger(GracefulEmbeddingModel.class); private final EmbeddingModel primaryModel; private final EmbeddingModel secondaryModel; @@ -42,11 +40,10 @@ public class GracefulEmbeddingModel implements EmbeddingModel { * @param enableHashFallback enable hash-based fallback when remote providers fail */ public GracefulEmbeddingModel( - EmbeddingModel primaryModel, - EmbeddingModel secondaryModel, - EmbeddingModel hashingModel, - boolean enableHashFallback - ) { + EmbeddingModel primaryModel, + EmbeddingModel secondaryModel, + EmbeddingModel hashingModel, + boolean enableHashFallback) { this.primaryModel = primaryModel; this.secondaryModel = secondaryModel; this.hashingModel = hashingModel; @@ -61,10 +58,7 @@ public GracefulEmbeddingModel( * @param enableHashFallback enable hash-based fallback when remote providers fail */ public GracefulEmbeddingModel( - EmbeddingModel primaryModel, - EmbeddingModel hashingModel, - boolean enableHashFallback - ) { + EmbeddingModel primaryModel, EmbeddingModel hashingModel, boolean enableHashFallback) { this(primaryModel, null, hashingModel, enableHashFallback); } @@ -79,9 +73,7 @@ public EmbeddingResponse call(EmbeddingRequest request) { EmbeddingResponse response = primaryModel.call(request); if (!response.getResults().isEmpty()) { if (!primaryAvailable) { - log.info( - "[EMBEDDING] Primary embedding service recovered" - ); + log.info("[EMBEDDING] Primary embedding service recovered"); primaryAvailable = true; } return response; @@ -94,18 +86,13 @@ public EmbeddingResponse call(EmbeddingRequest request) { } // Try secondary model if available - if ( - secondaryModel != null && - (secondaryAvailable || shouldRetrySecondary()) - ) { + if (secondaryModel != null && (secondaryAvailable || shouldRetrySecondary())) { try { log.info("[EMBEDDING] Attempting secondary embedding service"); EmbeddingResponse response = secondaryModel.call(request); if (!response.getResults().isEmpty()) { if (!secondaryAvailable) { - log.info( - "[EMBEDDING] Secondary embedding service recovered" - ); + log.info("[EMBEDDING] Secondary embedding service recovered"); secondaryAvailable = true; } return response; @@ -120,9 +107,7 @@ public EmbeddingResponse call(EmbeddingRequest request) { // Try hash-based fallback if enabled if (enableHashFallback && hashingModel != null) { try { - log.info( - "[EMBEDDING] Using hash-based fallback embeddings (limited semantic meaning)" - ); + log.info("[EMBEDDING] Using hash-based fallback embeddings (limited semantic meaning)"); return hashingModel.call(request); } catch (Exception hashFallbackException) { log.error("[EMBEDDING] Hash-based fallback failed", hashFallbackException); @@ -130,26 +115,16 @@ public EmbeddingResponse call(EmbeddingRequest request) { } // Complete degradation - return empty response - log.error( - "[EMBEDDING] All embedding services failed. Vector search will be unavailable." - ); - throw new EmbeddingServiceUnavailableException( - "All embedding services are unavailable" - ); + log.error("[EMBEDDING] All embedding services failed. Vector search will be unavailable."); + throw new EmbeddingServiceUnavailableException("All embedding services are unavailable"); } private boolean shouldRetryPrimary() { - return ( - System.currentTimeMillis() - lastPrimaryCheck > - CIRCUIT_BREAKER_TIMEOUT - ); + return (System.currentTimeMillis() - lastPrimaryCheck > CIRCUIT_BREAKER_TIMEOUT); } private boolean shouldRetrySecondary() { - return ( - System.currentTimeMillis() - lastSecondaryCheck > - CIRCUIT_BREAKER_TIMEOUT - ); + return (System.currentTimeMillis() - lastSecondaryCheck > CIRCUIT_BREAKER_TIMEOUT); } /** @@ -194,26 +169,19 @@ public float[] embed(Document document) { // Delegate to call() and let exceptions propagate for consistent error handling. // Previously this method caught exceptions and returned zero vectors, which was // inconsistent with call() and caused silent data corruption in vector stores. - EmbeddingRequest request = new EmbeddingRequest( - List.of(document.getText()), - null - ); + EmbeddingRequest request = new EmbeddingRequest(List.of(document.getText()), null); EmbeddingResponse response = call(request); if (!response.getResults().isEmpty()) { return response.getResults().get(0).getOutput(); } // This shouldn't happen since call() either returns results or throws - throw new EmbeddingServiceUnavailableException( - "Embedding returned empty results" - ); + throw new EmbeddingServiceUnavailableException("Embedding returned empty results"); } /** * Custom exception for when all embedding services are unavailable */ - public static class EmbeddingServiceUnavailableException - extends RuntimeException - { + public static class EmbeddingServiceUnavailableException extends RuntimeException { /** * Creates an exception that signals all embedding backends are unavailable. diff --git a/src/main/java/com/williamcallahan/javachat/service/GuidedLearningService.java b/src/main/java/com/williamcallahan/javachat/service/GuidedLearningService.java index a88b9d2..b6dcd75 100644 --- a/src/main/java/com/williamcallahan/javachat/service/GuidedLearningService.java +++ b/src/main/java/com/williamcallahan/javachat/service/GuidedLearningService.java @@ -1,10 +1,19 @@ package com.williamcallahan.javachat.service; +import com.williamcallahan.javachat.config.SystemPromptConfig; import com.williamcallahan.javachat.domain.prompt.StructuredPrompt; import com.williamcallahan.javachat.model.Citation; import com.williamcallahan.javachat.model.Enrichment; import com.williamcallahan.javachat.model.GuidedLesson; - +import com.williamcallahan.javachat.support.PdfCitationEnhancer; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.ai.chat.messages.Message; @@ -13,23 +22,6 @@ import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Locale; -import java.util.stream.Collectors; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.time.Duration; -import java.time.Instant; - -import org.springframework.core.io.ClassPathResource; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.*; -import org.apache.pdfbox.Loader; -import org.apache.pdfbox.pdmodel.PDDocument; - /** * Orchestrates guided learning flows over curated lesson metadata using retrieval, enrichment, and streaming chat. */ @@ -41,46 +33,69 @@ public class GuidedLearningService { private final RetrievalService retrievalService; private final EnrichmentService enrichmentService; private final ChatService chatService; - private final LocalStoreService localStore; + private final SystemPromptConfig systemPromptConfig; + private final PdfCitationEnhancer pdfCitationEnhancer; // Public server path of the Think Java book (as mapped by DocsSourceRegistry) private static final String THINK_JAVA_PDF_PATH = "/pdfs/Think Java - 2nd Edition Book.pdf"; - /** System guidance for Think Java-grounded responses with learning aid markers. */ - private static final String THINK_JAVA_GUIDANCE = - "You are a Java learning assistant guiding the user through 'Think Java — 2nd Edition'. " + - "Use ONLY content grounded in this book for factual claims. " + - "Cite sources with [n] markers. Embed learning aids using {{hint:...}}, {{reminder:...}}, {{background:...}}, {{example:...}}, {{warning:...}}. " + - "Prefer short, correct explanations with clear code examples when appropriate. If unsure, state the limitation."; + /** + * Base guidance for Think Java-grounded responses with learning aid markers. + * + *

This template includes a placeholder for the current lesson context, which is + * filled in at runtime to keep responses focused on the active topic.

+ */ + private static final String THINK_JAVA_GUIDANCE_TEMPLATE = + "You are a Java learning assistant guiding the user through 'Think Java — 2nd Edition'. " + + "Use ONLY content grounded in this book for factual claims. " + + "Do NOT include footnote references like [1] or a citations section; the UI shows sources separately. " + + "Embed learning aids using {{hint:...}}, {{reminder:...}}, {{background:...}}, {{example:...}}, {{warning:...}}. " + + "Prefer short, correct explanations with clear code examples when appropriate. If unsure, state the limitation.%n%n" + + "## Current Lesson Context%n" + + "%s%n%n" + + "## Topic Handling Rules%n" + + "1. Keep all responses focused on the current lesson topic.%n" + + "2. If the user sends a greeting (hi, hello, hey, etc.) or off-topic message, " + + "acknowledge it briefly and redirect to the lesson topic with a helpful prompt.%n" + + "3. For off-topic Java questions, acknowledge the question and gently steer back to the current lesson, " + + "explaining how the lesson topic relates or suggesting they complete this lesson first.%n" + + "4. Never ignore the lesson context - every response should reinforce learning the current topic."; private final String jdkVersion; /** * Creates the guided learning orchestrator using retrieval and enrichment services plus the configured JDK version hint. */ - public GuidedLearningService(GuidedTOCProvider tocProvider, - RetrievalService retrievalService, - EnrichmentService enrichmentService, - ChatService chatService, - LocalStoreService localStore, - @Value("${app.docs.jdk-version}") String jdkVersion) { + public GuidedLearningService( + GuidedTOCProvider tocProvider, + RetrievalService retrievalService, + EnrichmentService enrichmentService, + ChatService chatService, + SystemPromptConfig systemPromptConfig, + PdfCitationEnhancer pdfCitationEnhancer, + @Value("${app.docs.jdk-version}") String jdkVersion) { this.tocProvider = tocProvider; this.retrievalService = retrievalService; this.enrichmentService = enrichmentService; this.chatService = chatService; - this.localStore = localStore; + this.systemPromptConfig = systemPromptConfig; + this.pdfCitationEnhancer = pdfCitationEnhancer; this.jdkVersion = jdkVersion; } /** * Returns the guided lesson table of contents. */ - public List getTOC() { return tocProvider.getTOC(); } + public List getTOC() { + return tocProvider.getTOC(); + } /** * Returns guided lesson metadata for a slug when present. */ - public Optional getLesson(String slug) { return tocProvider.findBySlug(slug); } + public Optional getLesson(String slug) { + return tocProvider.findBySlug(slug); + } /** * Retrieves citations for a lesson using book-focused retrieval and best-effort PDF page anchoring. @@ -93,7 +108,7 @@ public List citationsForLesson(String slug) { List filtered = filterToBook(docs); if (filtered.isEmpty()) return List.of(); List base = retrievalService.toCitations(filtered); - return enhancePdfCitationsWithPage(filtered, base); + return pdfCitationEnhancer.enhanceWithPageAnchors(filtered, base); } /** @@ -106,10 +121,14 @@ public Enrichment enrichmentForLesson(String slug) { String query = buildLessonQuery(lesson); List docs = retrievalService.retrieve(query); List filtered = filterToBook(docs); - List snippets = filtered.stream().map(Document::getText).limit(6).collect(Collectors.toList()); + List snippets = + filtered.stream().map(Document::getText).limit(6).collect(Collectors.toList()); Enrichment enrichment = enrichmentService.enrich(query, jdkVersion, snippets); - logger.debug("GuidedLearningService returning enrichment with hints: {}, reminders: {}, background: {}", - enrichment.getHints().size(), enrichment.getReminders().size(), enrichment.getBackground().size()); + logger.debug( + "GuidedLearningService returning enrichment with hints: {}, reminders: {}, background: {}", + enrichment.getHints().size(), + enrichment.getReminders().size(), + enrichment.getBackground().size()); return enrichment; } @@ -122,9 +141,10 @@ public Flux streamGuidedAnswer(List history, String slug, Strin List docs = retrievalService.retrieve(query); List filtered = filterToBook(docs); - return chatService.streamAnswerWithContext(history, userMessage, filtered, THINK_JAVA_GUIDANCE); + String guidance = buildLessonGuidance(lesson); + return chatService.streamAnswerWithContext(history, userMessage, filtered, guidance); } - + /** * Builds a structured prompt for guided learning with intelligent truncation support. * @@ -134,19 +154,72 @@ public Flux streamGuidedAnswer(List history, String slug, Strin * @param history conversation history * @param slug lesson slug for context retrieval * @param userMessage user's question - * @return structured prompt for the OpenAI streaming service + * @return guided prompt outcome including structured prompt and context documents */ - public StructuredPrompt buildStructuredGuidedPromptWithContext( - List history, - String slug, - String userMessage) { + public GuidedChatPromptOutcome buildStructuredGuidedPromptWithContext( + List history, String slug, String userMessage) { var lesson = tocProvider.findBySlug(slug).orElse(null); String query = lesson != null ? buildLessonQuery(lesson) + "\n" + userMessage : userMessage; List docs = retrievalService.retrieve(query); List filtered = filterToBook(docs); - return chatService.buildStructuredPromptWithContextAndGuidance( - history, userMessage, filtered, THINK_JAVA_GUIDANCE); + String guidance = buildLessonGuidance(lesson); + StructuredPrompt structuredPrompt = + chatService.buildStructuredPromptWithContextAndGuidance(history, userMessage, filtered, guidance); + return new GuidedChatPromptOutcome(structuredPrompt, filtered); + } + + /** + * Builds UI-ready citations from Think Java context documents, with best-effort PDF page anchors. + * + *

This method is designed for guided chat streaming: citation generation must never break the + * response stream. If citation conversion or page anchor enrichment fails, it returns a best-effort + * result and logs diagnostics.

+ * + * @param bookContextDocuments retrieved Think Java documents used to ground the response + * @return list of citations for display in the UI citation panel + */ + public List citationsForBookDocuments(List bookContextDocuments) { + if (bookContextDocuments == null || bookContextDocuments.isEmpty()) { + return List.of(); + } + List baseCitations; + try { + baseCitations = retrievalService.toCitations(bookContextDocuments); + } catch (RuntimeException conversionFailure) { + logger.warn( + "Unable to convert guided context documents into citations (exceptionType={})", + conversionFailure.getClass().getSimpleName()); + return List.of(); + } + + try { + return pdfCitationEnhancer.enhanceWithPageAnchors(bookContextDocuments, baseCitations); + } catch (RuntimeException anchorFailure) { + logger.warn( + "Unable to enhance guided citations with PDF page anchors (exceptionType={})", + anchorFailure.getClass().getSimpleName()); + return baseCitations; + } + } + + /** + * Represents the guided prompt and the Think Java context documents used for grounding. + * + *

Normalizes {@code bookContextDocuments} to an unmodifiable list: {@link List#of()} when null, + * otherwise {@link List#copyOf(List)}.

+ * + * @param structuredPrompt structured prompt for LLM streaming + * @param bookContextDocuments Think Java-only context documents used for grounding and citations + * @throws IllegalArgumentException when structuredPrompt is null + */ + public record GuidedChatPromptOutcome(StructuredPrompt structuredPrompt, List bookContextDocuments) { + public GuidedChatPromptOutcome { + if (structuredPrompt == null) { + throw new IllegalArgumentException("Structured prompt cannot be null"); + } + bookContextDocuments = bookContextDocuments == null ? List.of() : List.copyOf(bookContextDocuments); + } } /** @@ -195,40 +268,39 @@ public static void main(String[] args) { List filtered = filterToBook(docs); // Guidance: produce a clean, layered markdown lesson body - String guidance = String.join(" ", - "Create a concise, beautifully formatted Java lesson using markdown only.", - "Do NOT include any heading at the top; the UI provides the title.", - "Then 1-2 short paragraphs that define and motivate the topic.", - "Add a bullet list of 3-5 key points or rules.", - "Include one short Java example in a fenced ```java code block with comments.", - "Add a small numbered list (1-3 steps) when it helps understanding.", - "Use inline [n] citations that correspond to the provided context order.", - "Do NOT include enrichment markers like {{hint:...}}; they are handled separately.", - "Do NOT include a conclusion section; keep it compact and practical.", - "If context is insufficient, state what is missing briefly." - ); + String guidance = String.join( + " ", + "Create a concise, beautifully formatted Java lesson using markdown only.", + "Do NOT include any heading at the top; the UI provides the title.", + "Then 1-2 short paragraphs that define and motivate the topic.", + "Add a bullet list of 3-5 key points or rules.", + "Include one short Java example in a fenced ```java code block with comments.", + "Add a small numbered list (1-3 steps) when it helps understanding.", + "Do NOT include footnote references like [1] or a citations section; the UI shows sources separately.", + "Do NOT include enrichment markers like {{hint:...}}; they are handled separately.", + "Do NOT include a conclusion section; keep it compact and practical.", + "If context is insufficient, state what is missing briefly."); // We pass a synthetic latestUserMessage that instructs the model to write the lesson String latestUserMessage = "Write the lesson for: " + title + "\nFocus on: " + query; List emptyHistory = List.of(); StringBuilder lessonMarkdownBuilder = new StringBuilder(); - return chatService.streamAnswerWithContext(emptyHistory, latestUserMessage, filtered, guidance) + return chatService + .streamAnswerWithContext(emptyHistory, latestUserMessage, filtered, guidance) .doOnNext(lessonMarkdownBuilder::append) .doOnComplete(() -> putLessonCache(slug, lessonMarkdownBuilder.toString())); } // ===== In-memory cache for lesson markdown ===== private static final long LESSON_MARKDOWN_CACHE_TTL_MINUTES = 30; - private static final Duration LESSON_MARKDOWN_CACHE_TTL = - Duration.ofMinutes(LESSON_MARKDOWN_CACHE_TTL_MINUTES); + private static final Duration LESSON_MARKDOWN_CACHE_TTL = Duration.ofMinutes(LESSON_MARKDOWN_CACHE_TTL_MINUTES); /** * Stores lesson markdown alongside the time it was cached to enforce an in-memory TTL. */ private record LessonMarkdownCacheEntry(String markdown, Instant cachedAt) {} - private final ConcurrentMap lessonMarkdownCache = - new ConcurrentHashMap<>(); + private final ConcurrentMap lessonMarkdownCache = new ConcurrentHashMap<>(); /** * Returns cached lesson markdown when present and not expired. @@ -259,80 +331,6 @@ public void putLessonCache(String slug, String markdown) { lessonMarkdownCache.put(slug, new LessonMarkdownCacheEntry(markdown, Instant.now())); } - // ===== PDF Pagination heuristics for /pdfs/Think Java - 2nd Edition Book.pdf ===== - private volatile Integer cachedPdfPages = null; - - private int getThinkJavaPdfPages() { - if (cachedPdfPages != null) return cachedPdfPages; - synchronized (this) { - if (cachedPdfPages != null) return cachedPdfPages; - try { - ClassPathResource pdfResource = new ClassPathResource("public/pdfs/Think Java - 2nd Edition Book.pdf"); - try (InputStream pdfStream = pdfResource.getInputStream(); - PDDocument document = Loader.loadPDF(pdfStream.readAllBytes())) { - cachedPdfPages = document.getNumberOfPages(); - } - } catch (IOException ioException) { - logger.error("Failed to load Think Java PDF for pagination", ioException); - throw new IllegalStateException("Unable to read Think Java PDF", ioException); - } - return cachedPdfPages; - } - } - - private int totalChunksForUrl(String url) { - try { - String safe = localStore.toSafeName(url); - Path dir = localStore.getParsedDir(); - if (dir == null) { - return 0; - } - try (var stream = Files.list(dir)) { - return (int) stream - .filter(path -> { - Path fileNamePath = path.getFileName(); - if (fileNamePath == null) { - return false; - } - String fileName = fileNamePath.toString(); - return fileName.startsWith(safe + "_") && fileName.endsWith(".txt"); - }) - .count(); - } - } catch (IOException ioException) { - throw new IllegalStateException("Unable to count local chunks for URL", ioException); - } - } - - private List enhancePdfCitationsWithPage(List docs, List citations) { - if (docs.size() != citations.size()) return citations; - int pages = getThinkJavaPdfPages(); - for (int docIndex = 0; docIndex < docs.size(); docIndex++) { - Document document = docs.get(docIndex); - Citation citation = citations.get(docIndex); - String url = citation.getUrl(); - if (url == null || !url.toLowerCase(Locale.ROOT).endsWith(".pdf")) continue; - Object chunkIndexMetadata = document.getMetadata().get("chunkIndex"); - int chunkIndex = -1; - try { - if (chunkIndexMetadata != null) { - chunkIndex = Integer.parseInt(String.valueOf(chunkIndexMetadata)); - } - } catch (NumberFormatException chunkIndexParseException) { - logger.debug("Failed to parse chunkIndex from metadata: {}", - sanitizeForLogText(String.valueOf(chunkIndexMetadata))); - } - int totalChunks = totalChunksForUrl(url); - if (pages > 0 && chunkIndex >= 0 && totalChunks > 0) { - int page = Math.max(1, Math.min(pages, (int) Math.round(((chunkIndex + 1.0) / totalChunks) * pages))); - String withAnchor = url.contains("#page=") ? url : url + "#page=" + page; - citation.setUrl(withAnchor); - citation.setAnchor("page=" + page); - } - } - return citations; - } - private List filterToBook(List docs) { List filtered = new ArrayList<>(); for (Document document : docs) { @@ -347,13 +345,61 @@ private List filterToBook(List docs) { private String buildLessonQuery(GuidedLesson lesson) { StringBuilder queryBuilder = new StringBuilder(); if (lesson.getTitle() != null) queryBuilder.append(lesson.getTitle()).append(". "); - if (lesson.getSummary() != null) queryBuilder.append(lesson.getSummary()).append(" "); + if (lesson.getSummary() != null) + queryBuilder.append(lesson.getSummary()).append(" "); if (lesson.getKeywords() != null && !lesson.getKeywords().isEmpty()) { queryBuilder.append(String.join(", ", lesson.getKeywords())); } return queryBuilder.toString().trim(); } + /** + * Builds complete guidance for a guided learning chat, combining lesson context with system prompts. + * + *

When a lesson is provided, the guidance includes the lesson title, summary, and keywords + * to keep responses focused on the current topic. It also integrates the guided learning + * mode instructions from SystemPromptConfig.

+ * + * @param lesson current lesson or null if no lesson context + * @return complete guidance string for the LLM + */ + private String buildLessonGuidance(GuidedLesson lesson) { + String lessonContext = buildLessonContextDescription(lesson); + String thinkJavaGuidance = String.format(THINK_JAVA_GUIDANCE_TEMPLATE, lessonContext); + + // Combine with guided learning mode instructions from SystemPromptConfig + String guidedLearningPrompt = systemPromptConfig.getGuidedLearningPrompt(); + return systemPromptConfig.buildFullPrompt(thinkJavaGuidance, guidedLearningPrompt); + } + + /** + * Builds a human-readable description of the current lesson context for the LLM. + * + * @param lesson current lesson or null + * @return description of the lesson context + */ + private String buildLessonContextDescription(GuidedLesson lesson) { + if (lesson == null) { + return "No specific lesson selected. Provide general Java learning assistance."; + } + + StringBuilder contextBuilder = new StringBuilder(); + contextBuilder + .append("The user is currently studying the lesson: **") + .append(lesson.getTitle()) + .append("**"); + + if (lesson.getSummary() != null && !lesson.getSummary().isBlank()) { + contextBuilder.append("\n\nLesson Summary: ").append(lesson.getSummary()); + } + + if (lesson.getKeywords() != null && !lesson.getKeywords().isEmpty()) { + contextBuilder.append("\n\nKey concepts to cover: ").append(String.join(", ", lesson.getKeywords())); + } + + return contextBuilder.toString(); + } + private Enrichment emptyEnrichment() { Enrichment fallbackEnrichment = new Enrichment(); fallbackEnrichment.setJdkVersion(jdkVersion); @@ -362,11 +408,4 @@ private Enrichment emptyEnrichment() { fallbackEnrichment.setBackground(List.of()); return fallbackEnrichment; } - - private static String sanitizeForLogText(String rawText) { - if (rawText == null) { - return ""; - } - return rawText.replace("\r", "\\r").replace("\n", "\\n"); - } } diff --git a/src/main/java/com/williamcallahan/javachat/service/GuidedTOCProvider.java b/src/main/java/com/williamcallahan/javachat/service/GuidedTOCProvider.java index 992457d..8c7b53e 100644 --- a/src/main/java/com/williamcallahan/javachat/service/GuidedTOCProvider.java +++ b/src/main/java/com/williamcallahan/javachat/service/GuidedTOCProvider.java @@ -1,20 +1,19 @@ package com.williamcallahan.javachat.service; -import com.williamcallahan.javachat.support.AsciiTextNormalizer; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.williamcallahan.javachat.model.GuidedLesson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.io.ClassPathResource; -import org.springframework.stereotype.Service; - +import com.williamcallahan.javachat.support.AsciiTextNormalizer; import java.io.IOException; import java.io.InputStream; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.Objects; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Service; /** * Loads and caches guided lesson metadata from the classpath to support guided learning flows. @@ -43,18 +42,15 @@ public synchronized List getTOC() { try { ClassPathResource tocResource = new ClassPathResource("guided/toc.json"); try (InputStream tocStream = tocResource.getInputStream()) { - List loadedLessons = mapper - .readerFor(new TypeReference>() {}) - .without(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - .readValue(tocStream); + List loadedLessons = mapper.readerFor(new TypeReference>() {}) + .without(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + .readValue(tocStream); cache = List.copyOf(loadedLessons); } } catch (IOException exception) { - log.warn("Failed to load guided TOC (exceptionType={})", exception.getClass().getName()); - cache = Collections.emptyList(); - } finally { - tocLoaded = true; + throw new IllegalStateException("Failed to load guided TOC from classpath", exception); } + tocLoaded = true; return cache; } @@ -65,10 +61,10 @@ public Optional findBySlug(String slug) { if (slug == null || slug.isBlank()) return Optional.empty(); String normalizedSlug = AsciiTextNormalizer.toLowerAscii(slug); return getTOC().stream() - .filter(lesson -> { - String lessonSlug = lesson.getSlug(); - return lessonSlug != null && normalizedSlug.equals(AsciiTextNormalizer.toLowerAscii(lessonSlug)); - }) - .findFirst(); + .filter(lesson -> { + String lessonSlug = lesson.getSlug(); + return lessonSlug != null && normalizedSlug.equals(AsciiTextNormalizer.toLowerAscii(lessonSlug)); + }) + .findFirst(); } } diff --git a/src/main/java/com/williamcallahan/javachat/service/HtmlContentExtractor.java b/src/main/java/com/williamcallahan/javachat/service/HtmlContentExtractor.java index 8aee0c1..4a84d2f 100644 --- a/src/main/java/com/williamcallahan/javachat/service/HtmlContentExtractor.java +++ b/src/main/java/com/williamcallahan/javachat/service/HtmlContentExtractor.java @@ -1,63 +1,72 @@ package com.williamcallahan.javachat.service; import com.williamcallahan.javachat.support.AsciiTextNormalizer; +import java.util.Set; +import java.util.stream.Collectors; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.springframework.stereotype.Service; - -import java.util.Set; -import java.util.stream.Collectors; - /** * Enhanced HTML content extraction that filters out navigation noise * and JavaScript warnings to produce cleaner documentation content. */ @Service public class HtmlContentExtractor { - + // Common noise patterns to filter out private static final Set NOISE_PATTERNS = Set.of( - "JavaScript is disabled", - "Skip navigation links", - "Hide sidebar", - "Show sidebar", - "Report a bug", - "suggest an enhancement", - "Other versions", - "Use is subject to license terms", - "Scripting on this page tracks", - "but does not change the content" - ); - + "JavaScript is disabled", + "Skip navigation links", + "Hide sidebar", + "Show sidebar", + "Report a bug", + "suggest an enhancement", + "Other versions", + "Use is subject to license terms", + "Scripting on this page tracks", + "but does not change the content"); + // CSS selectors for navigation and other non-content elements private static final String[] REMOVE_SELECTORS = { - "nav", "header", "footer", // Navigation elements - ".navigation", ".nav", ".navbar", // Navigation classes - ".sidebar", ".toc", ".breadcrumb", // Navigation components - ".skip-nav", ".skip-link", // Skip links - "script", "style", "noscript", // Scripts and styles - ".footer", ".header", // Headers/footers - ".copyright", ".legal", // Legal notices - "#navigation", "#nav" // Navigation IDs + "nav", + "header", + "footer", // Navigation elements + ".navigation", + ".nav", + ".navbar", // Navigation classes + ".sidebar", + ".toc", + ".breadcrumb", // Navigation components + ".skip-nav", + ".skip-link", // Skip links + "script", + "style", + "noscript", // Scripts and styles + ".footer", + ".header", // Headers/footers + ".copyright", + ".legal", // Legal notices + "#navigation", + "#nav" // Navigation IDs }; - + // CSS selectors for main content areas (in priority order) private static final String[] CONTENT_SELECTORS = { - "main", // HTML5 main element - "article", // HTML5 article element - ".content-container", // Common content class - ".main-content", // Common content class - ".documentation", // Documentation class - ".doc-content", // Documentation content - "#content", // Content ID - ".block", // Java doc block - ".description", // Description sections - ".detail", // Detail sections - ".summary" // Summary sections + "main", // HTML5 main element + "article", // HTML5 article element + ".content-container", // Common content class + ".main-content", // Common content class + ".documentation", // Documentation class + ".doc-content", // Documentation content + "#content", // Content ID + ".block", // Java doc block + ".description", // Description sections + ".detail", // Detail sections + ".summary" // Summary sections }; - + /** * Extract clean content from HTML document, filtering out navigation * and JavaScript warnings. @@ -67,20 +76,20 @@ public String extractCleanContent(Document doc) { for (String selector : REMOVE_SELECTORS) { doc.select(selector).remove(); } - + // Try to find main content area Element contentElement = findMainContent(doc); if (contentElement == null) { contentElement = doc.body(); } - + // Extract and clean the text String text = extractTextFromElement(contentElement); - + // Filter out common noise patterns return filterNoise(text); } - + /** * Find the main content element in the document. */ @@ -90,48 +99,43 @@ private Element findMainContent(Document doc) { if (!elements.isEmpty()) { // Return the largest content area if multiple found return elements.stream() - .max((e1, e2) -> Integer.compare( - e1.text().length(), - e2.text().length())) - .orElse(elements.first()); + .max((e1, e2) -> + Integer.compare(e1.text().length(), e2.text().length())) + .orElse(elements.first()); } } - + // Fallback: look for divs with significant text content Elements divs = doc.select("div"); return divs.stream() - .filter(div -> { - String text = div.text(); - // Must have substantial content and not be navigation - return text.length() > 500 && - !isNavigationElement(div) && - !containsExcessiveNoise(text); - }) - .max((e1, e2) -> Integer.compare( - e1.text().length(), - e2.text().length())) - .orElse(null); + .filter(div -> { + String text = div.text(); + // Must have substantial content and not be navigation + return text.length() > 500 && !isNavigationElement(div) && !containsExcessiveNoise(text); + }) + .max((e1, e2) -> Integer.compare(e1.text().length(), e2.text().length())) + .orElse(null); } - + /** * Extract text from element with better formatting. */ private String extractTextFromElement(Element element) { // Use wholeText() for better preservation of structure StringBuilder sb = new StringBuilder(); - + // Process specific elements for better formatting Elements headers = element.select("h1, h2, h3, h4, h5, h6"); Elements paragraphs = element.select("p"); - + // If we have structured content, process it if (!headers.isEmpty() || !paragraphs.isEmpty()) { element.children().forEach(child -> { String tagName = child.tagName(); String text = child.text().trim(); - + if (text.isEmpty()) return; - + switch (tagName) { case "h1", "h2", "h3", "h4", "h5", "h6" -> { sb.append("\n\n").append(text).append("\n"); @@ -146,8 +150,7 @@ private String extractTextFromElement(Element element) { sb.append("\n```\n").append(rawCode).append("\n```\n"); } case "ul", "ol" -> { - child.select("li").forEach(li -> - sb.append("\n• ").append(li.text())); + child.select("li").forEach(li -> sb.append("\n• ").append(li.text())); sb.append("\n"); } case "table" -> { @@ -155,9 +158,7 @@ private String extractTextFromElement(Element element) { Elements rows = child.select("tr"); rows.forEach(row -> { Elements cells = row.select("td, th"); - String rowText = cells.stream() - .map(Element::text) - .collect(Collectors.joining(" | ")); + String rowText = cells.stream().map(Element::text).collect(Collectors.joining(" | ")); sb.append("\n").append(rowText); }); sb.append("\n"); @@ -173,10 +174,10 @@ private String extractTextFromElement(Element element) { // Fallback to simple text extraction sb.append(element.text()); } - + return sb.toString().trim(); } - + /** * Check if element is likely navigation. */ @@ -184,37 +185,36 @@ private boolean isNavigationElement(Element element) { String className = AsciiTextNormalizer.toLowerAscii(element.className()); String id = AsciiTextNormalizer.toLowerAscii(element.id()); String text = AsciiTextNormalizer.toLowerAscii(element.text()); - - return className.contains("nav") || - className.contains("menu") || - className.contains("sidebar") || - className.contains("header") || - className.contains("footer") || - id.contains("nav") || - id.contains("menu") || - text.startsWith("skip") || - text.startsWith("hide") || - text.startsWith("show"); + + return className.contains("nav") + || className.contains("menu") + || className.contains("sidebar") + || className.contains("header") + || className.contains("footer") + || id.contains("nav") + || id.contains("menu") + || text.startsWith("skip") + || text.startsWith("hide") + || text.startsWith("show"); } - /** * Check if text contains excessive noise. */ private boolean containsExcessiveNoise(String text) { int noiseCount = 0; String lowerText = AsciiTextNormalizer.toLowerAscii(text); - + for (String noise : NOISE_PATTERNS) { if (lowerText.contains(AsciiTextNormalizer.toLowerAscii(noise))) { noiseCount++; } } - + // If more than 3 noise patterns, likely navigation/footer return noiseCount > 3; } - + /** * Filter out remaining noise patterns from text. */ @@ -222,7 +222,7 @@ private String filterNoise(String text) { String[] lines = text.split("\n"); StringBuilder cleaned = new StringBuilder(); boolean inCodeFence = false; - + for (String line : lines) { String trimmed = line.trim(); @@ -236,18 +236,18 @@ private String filterNoise(String text) { cleaned.append(line).append("\n"); continue; } - + // Skip empty lines if (trimmed.isEmpty()) { cleaned.append("\n"); continue; } - + // Skip lines that are pure noise String trimmedLower = AsciiTextNormalizer.toLowerAscii(trimmed); boolean isNoise = NOISE_PATTERNS.stream() - .anyMatch(noise -> trimmedLower.equals(AsciiTextNormalizer.toLowerAscii(noise))); - + .anyMatch(noise -> trimmedLower.equals(AsciiTextNormalizer.toLowerAscii(noise))); + if (!isNoise) { // Also skip very short lines that are likely navigation if (trimmed.length() > 3 || startsWithUppercaseLetter(trimmed)) { @@ -255,7 +255,7 @@ private String filterNoise(String text) { } } } - + // Clean up excessive whitespace without disturbing code fences return normalizeWhitespaceOutsideCodeFences(cleaned.toString()).trim(); } @@ -275,9 +275,9 @@ private String normalizeWhitespaceOutsideCodeFences(String text) { int index = 0; while (index < text.length()) { if (index + 2 < text.length() - && text.charAt(index) == '`' - && text.charAt(index + 1) == '`' - && text.charAt(index + 2) == '`') { + && text.charAt(index) == '`' + && text.charAt(index + 1) == '`' + && text.charAt(index + 2) == '`') { normalized.append("```"); index += 3; inFence = !inFence; @@ -316,22 +316,23 @@ private String normalizeWhitespaceOutsideCodeFences(String text) { } return normalized.toString(); } - + /** * Extract Java API-specific content with focus on class/method documentation. */ public String extractJavaApiContent(Document doc) { // For Java API docs, focus on specific content areas StringBuilder content = new StringBuilder(); - + // Class/Interface name and description Elements classHeaders = doc.select(".header h1, .header h2, .title"); classHeaders.forEach(header -> content.append(header.text()).append("\n\n")); - + // Package info Elements packageInfo = doc.select(".subTitle, .package"); - packageInfo.forEach(pkgElement -> content.append("Package: ").append(pkgElement.text()).append("\n")); - + packageInfo.forEach(pkgElement -> + content.append("Package: ").append(pkgElement.text()).append("\n")); + // Main description Elements descriptions = doc.select(".description, .block"); descriptions.forEach(descElement -> { @@ -340,14 +341,15 @@ public String extractJavaApiContent(Document doc) { content.append("\n").append(text).append("\n"); } }); - + // Method summaries Elements methodSummaries = doc.select(".summary .memberSummary"); if (!methodSummaries.isEmpty()) { content.append("\n\nMethod Summary:\n"); - methodSummaries.forEach(methodSummary -> content.append("• ").append(methodSummary.text()).append("\n")); + methodSummaries.forEach(methodSummary -> + content.append("• ").append(methodSummary.text()).append("\n")); } - + // Method details Elements methodDetails = doc.select(".details .memberDetails"); if (!methodDetails.isEmpty()) { @@ -359,22 +361,22 @@ public String extractJavaApiContent(Document doc) { } }); } - + // Code examples Elements codeExamples = doc.select("pre.code, pre.prettyprint"); if (!codeExamples.isEmpty()) { content.append("\n\nCode Examples:\n"); - codeExamples.forEach(code -> - content.append("```java\n").append(code.text()).append("\n```\n\n")); + codeExamples.forEach( + code -> content.append("```java\n").append(code.text()).append("\n```\n\n")); } - + String result = content.toString().trim(); - + // If we didn't get much content, fall back to general extraction if (result.length() < 100) { return extractCleanContent(doc); } - + return filterNoise(result); } } diff --git a/src/main/java/com/williamcallahan/javachat/service/LocalEmbeddingModel.java b/src/main/java/com/williamcallahan/javachat/service/LocalEmbeddingModel.java index 6bb49a8..d6ce168 100644 --- a/src/main/java/com/williamcallahan/javachat/service/LocalEmbeddingModel.java +++ b/src/main/java/com/williamcallahan/javachat/service/LocalEmbeddingModel.java @@ -15,17 +15,15 @@ import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; -import org.springframework.web.client.RestTemplate; import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; /** * Embedding model wrapper that calls a local service with safe fallbacks. */ public class LocalEmbeddingModel implements EmbeddingModel { - private static final Logger log = LoggerFactory.getLogger( - LocalEmbeddingModel.class - ); + private static final Logger log = LoggerFactory.getLogger(LocalEmbeddingModel.class); private final String baseUrl; private final String modelName; @@ -52,18 +50,14 @@ public class LocalEmbeddingModel implements EmbeddingModel { * @param restTemplateBuilder RestTemplate builder */ public LocalEmbeddingModel( - String baseUrl, - String modelName, - int dimensions, - RestTemplateBuilder restTemplateBuilder - ) { + String baseUrl, String modelName, int dimensions, RestTemplateBuilder restTemplateBuilder) { this.baseUrl = baseUrl; this.modelName = modelName; this.dimensions = dimensions; this.restTemplate = restTemplateBuilder - .connectTimeout(java.time.Duration.ofSeconds(CONNECT_TIMEOUT_SECONDS)) - .readTimeout(java.time.Duration.ofSeconds(READ_TIMEOUT_SECONDS)) - .build(); + .connectTimeout(java.time.Duration.ofSeconds(CONNECT_TIMEOUT_SECONDS)) + .readTimeout(java.time.Duration.ofSeconds(READ_TIMEOUT_SECONDS)) + .build(); // Check server availability on startup checkServerAvailability(); } @@ -79,16 +73,12 @@ private void checkServerAvailability() { String healthUrl = baseUrl + "/v1/models"; restTemplate.getForObject(healthUrl, String.class); if (!serverAvailable) { - log.info( - "[EMBEDDING] Local embedding server is now available" - ); + log.info("[EMBEDDING] Local embedding server is now available"); } serverAvailable = true; } catch (RestClientException healthCheckException) { if (serverAvailable) { - log.warn( - "[EMBEDDING] Local embedding server not reachable. Using fallback embeddings." - ); + log.warn("[EMBEDDING] Local embedding server not reachable. Using fallback embeddings."); } serverAvailable = false; } @@ -121,9 +111,8 @@ public EmbeddingResponse call(EmbeddingRequest request) { private EmbeddingResponse createFallbackResponse(EmbeddingRequest request) { if (log.isTraceEnabled()) { log.trace( - "[EMBEDDING] Server unavailable, returning fallback embeddings for {} texts", - request.getInstructions().size() - ); + "[EMBEDDING] Server unavailable, returning fallback embeddings for {} texts", + request.getInstructions().size()); } List embeddings = new ArrayList<>(); @@ -138,9 +127,7 @@ private EmbeddingResponse createFallbackResponse(EmbeddingRequest request) { * Call the embedding API for all texts in the request. */ private EmbeddingResponse callEmbeddingApi(EmbeddingRequest request) { - log.debug( - "[EMBEDDING] Generating embeddings for request payload" - ); + log.debug("[EMBEDDING] Generating embeddings for request payload"); List embeddings = new ArrayList<>(); @@ -149,12 +136,7 @@ private EmbeddingResponse callEmbeddingApi(EmbeddingRequest request) { if (vector.length > 0) { embeddings.add(new Embedding(vector, embeddings.size())); } else { - embeddings.add( - new Embedding( - createFallbackEmbedding(text), - embeddings.size() - ) - ); + embeddings.add(new Embedding(createFallbackEmbedding(text), embeddings.size())); } } @@ -175,15 +157,9 @@ private float[] fetchEmbeddingFromApi(String text) { headers.setContentType(MediaType.APPLICATION_JSON); HttpEntity entity = new HttpEntity<>(requestBody, headers); - log.debug( - "[EMBEDDING] Calling embedding API" - ); + log.debug("[EMBEDDING] Calling embedding API"); - EmbeddingResponsePayload response = restTemplate.postForObject( - url, - entity, - EmbeddingResponsePayload.class - ); + EmbeddingResponsePayload response = restTemplate.postForObject(url, entity, EmbeddingResponsePayload.class); return parseEmbeddingResponse(response); } @@ -222,14 +198,10 @@ private float[] parseEmbeddingResponse(EmbeddingResponsePayload response) { /** * Handle API failure by marking server unavailable and returning fallback embeddings. */ - private EmbeddingResponse handleApiFailure( - RuntimeException exception, - EmbeddingRequest request - ) { + private EmbeddingResponse handleApiFailure(RuntimeException exception, EmbeddingRequest request) { log.warn( - "[EMBEDDING] Failed to get embeddings from server, using fallback (exceptionType={})", - exception.getClass().getName() - ); + "[EMBEDDING] Failed to get embeddings from server, using fallback (exceptionType={})", + exception.getClass().getName()); serverAvailable = false; lastCheckTime = System.currentTimeMillis(); @@ -281,10 +253,7 @@ public int dimensions() { */ @Override public float[] embed(org.springframework.ai.document.Document document) { - EmbeddingRequest request = new EmbeddingRequest( - List.of(document.getText()), - null - ); + EmbeddingRequest request = new EmbeddingRequest(List.of(document.getText()), null); EmbeddingResponse response = call(request); if (!response.getResults().isEmpty()) { return response.getResults().get(0).getOutput(); @@ -293,12 +262,9 @@ public float[] embed(org.springframework.ai.document.Document document) { return createFallbackEmbedding(document.getText()); } - private record EmbeddingRequestPayload(String model, String input) { - } + private record EmbeddingRequestPayload(String model, String input) {} - private record EmbeddingResponsePayload(List data) { - } + private record EmbeddingResponsePayload(List data) {} - private record EmbeddingData(List embedding) { - } + private record EmbeddingData(List embedding) {} } diff --git a/src/main/java/com/williamcallahan/javachat/service/LocalHashingEmbeddingModel.java b/src/main/java/com/williamcallahan/javachat/service/LocalHashingEmbeddingModel.java index f747000..6774419 100644 --- a/src/main/java/com/williamcallahan/javachat/service/LocalHashingEmbeddingModel.java +++ b/src/main/java/com/williamcallahan/javachat/service/LocalHashingEmbeddingModel.java @@ -1,16 +1,15 @@ package com.williamcallahan.javachat.service; -import org.springframework.ai.document.Document; -import org.springframework.ai.embedding.Embedding; -import org.springframework.ai.embedding.EmbeddingModel; -import org.springframework.ai.embedding.EmbeddingRequest; -import org.springframework.ai.embedding.EmbeddingResponse; - import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; +import org.springframework.ai.document.Document; +import org.springframework.ai.embedding.Embedding; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.embedding.EmbeddingRequest; +import org.springframework.ai.embedding.EmbeddingResponse; /** * Minimal CPU-only deterministic hashing embedding to unblock retrieval without remote calls. diff --git a/src/main/java/com/williamcallahan/javachat/service/LocalSearchService.java b/src/main/java/com/williamcallahan/javachat/service/LocalSearchService.java index 0b45d93..52cdb9a 100644 --- a/src/main/java/com/williamcallahan/javachat/service/LocalSearchService.java +++ b/src/main/java/com/williamcallahan/javachat/service/LocalSearchService.java @@ -1,15 +1,14 @@ package com.williamcallahan.javachat.service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.*; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; /** * Searches locally parsed documents using simple keyword scoring as a fallback when vector retrieval is unavailable. @@ -51,9 +50,10 @@ public SearchOutcome search(String query, int topK) { Map scores = new HashMap<>(); try (var files = Files.walk(parsedDir)) { - List textFiles = files.filter(pathCandidate -> pathCandidate.toString().endsWith(".txt")) - .limit(MAX_FILES_TO_SCAN) - .collect(Collectors.toList()); + List textFiles = files.filter( + pathCandidate -> pathCandidate.toString().endsWith(".txt")) + .limit(MAX_FILES_TO_SCAN) + .collect(Collectors.toList()); log.debug("Local search scanning {} files", textFiles.size()); @@ -68,29 +68,29 @@ public SearchOutcome search(String query, int topK) { } if (score > 0) { scores.put( - textFile, - score / Math.max(MIN_CONTENT_LENGTH_FOR_SCORING, normalizedContent.length()) - ); + textFile, score / Math.max(MIN_CONTENT_LENGTH_FOR_SCORING, normalizedContent.length())); } } catch (IOException fileReadError) { - log.debug("Skipping unreadable file (exception type: {})", - fileReadError.getClass().getSimpleName()); + log.debug( + "Skipping unreadable file (exception type: {})", + fileReadError.getClass().getSimpleName()); } } List searchHits = scores.entrySet().stream() - .sorted(Map.Entry.comparingByValue().reversed()) - .limit(topK) - .map(scoreEntry -> toSearchHit(scoreEntry.getKey(), scoreEntry.getValue())) - .flatMap(Optional::stream) - .collect(Collectors.toList()); + .sorted(Map.Entry.comparingByValue().reversed()) + .limit(topK) + .map(scoreEntry -> toSearchHit(scoreEntry.getKey(), scoreEntry.getValue())) + .flatMap(Optional::stream) + .collect(Collectors.toList()); log.info("Local search found {} hits", searchHits.size()); return SearchOutcome.success(searchHits); } catch (IOException walkError) { - log.error("Local search failed to walk directory (exception type: {})", - walkError.getClass().getSimpleName()); + log.error( + "Local search failed to walk directory (exception type: {})", + walkError.getClass().getSimpleName()); return SearchOutcome.ioError(walkError.getMessage()); } } @@ -108,14 +108,13 @@ private Optional toSearchHit(Path textPath, double score) { // Filename pattern: safeUrl_index_hash.txt // Defensive: handle files without underscore delimiter int underscoreIdx = fileName.indexOf("_"); - String safeName = underscoreIdx > 0 - ? fileName.substring(0, underscoreIdx) - : fileName.replace(".txt", ""); + String safeName = underscoreIdx > 0 ? fileName.substring(0, underscoreIdx) : fileName.replace(".txt", ""); String url = fromSafeName(safeName); return Optional.of(new SearchHit(url, text, score)); } catch (IOException readError) { - log.warn("Failed to read result file (exception type: {})", - readError.getClass().getSimpleName()); + log.warn( + "Failed to read result file (exception type: {})", + readError.getClass().getSimpleName()); return Optional.empty(); } } @@ -211,8 +210,8 @@ public static SearchOutcome success(List hits) { * Builds a failure outcome for a missing parsed document directory. */ public static SearchOutcome directoryNotFound(String path) { - return new SearchOutcome(List.of(), Status.DIRECTORY_NOT_FOUND, - Optional.of("Search directory not found: " + path)); + return new SearchOutcome( + List.of(), Status.DIRECTORY_NOT_FOUND, Optional.of("Search directory not found: " + path)); } /** diff --git a/src/main/java/com/williamcallahan/javachat/service/LocalStoreService.java b/src/main/java/com/williamcallahan/javachat/service/LocalStoreService.java index a81c220..7415c34 100644 --- a/src/main/java/com/williamcallahan/javachat/service/LocalStoreService.java +++ b/src/main/java/com/williamcallahan/javachat/service/LocalStoreService.java @@ -1,13 +1,6 @@ package com.williamcallahan.javachat.service; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - import jakarta.annotation.PostConstruct; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -16,6 +9,10 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HexFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; /** * Persists document snapshots, parsed chunks, and ingestion markers on the local filesystem. @@ -41,10 +38,11 @@ public class LocalStoreService { /** * Creates the local store using configured directory roots for snapshots, parsed content, and ingest markers. */ - public LocalStoreService(@Value("${app.docs.snapshot-dir}") String snapshotDir, - @Value("${app.docs.parsed-dir}") String parsedDir, - @Value("${app.docs.index-dir}") String indexDir, - ProgressTracker progressTracker) { + public LocalStoreService( + @Value("${app.docs.snapshot-dir}") String snapshotDir, + @Value("${app.docs.parsed-dir}") String parsedDir, + @Value("${app.docs.index-dir}") String indexDir, + ProgressTracker progressTracker) { this.snapshotDirConfig = snapshotDir; this.parsedDirConfig = parsedDir; this.indexDirConfig = indexDir; @@ -63,8 +61,7 @@ void createStoreDirectories() { Files.createDirectories(this.snapshotDir); Files.createDirectories(this.parsedDir); Files.createDirectories(this.indexDir); - log.info("Local store directories ready (snapshots={}, parsed={}, index={})", - snapshotDir, parsedDir, indexDir); + log.info("Local store directories ready"); } catch (InvalidPathException | IOException exception) { throw new IllegalStateException("Failed to create local store directories", exception); } @@ -83,9 +80,7 @@ public void saveHtml(String url, String html) throws IOException { * Stores a parsed chunk payload for later local search and attribution. */ public void saveChunkText(String url, int index, String text, String hash) throws IOException { - String shortHash = hash.length() >= HASH_PREFIX_LENGTH - ? hash.substring(0, HASH_PREFIX_LENGTH) - : hash; + String shortHash = hash.length() >= HASH_PREFIX_LENGTH ? hash.substring(0, HASH_PREFIX_LENGTH) : hash; Path chunkFilePath = parsedDir.resolve(safeName(url) + "_" + index + "_" + shortHash + ".txt"); ensureParentDirectoryExists(chunkFilePath); Files.writeString(chunkFilePath, text, StandardCharsets.UTF_8); @@ -174,5 +169,4 @@ private String shortSha256(String input) { throw new IllegalStateException("SHA-256 MessageDigest is not available", exception); } } - } diff --git a/src/main/java/com/williamcallahan/javachat/service/MarkdownService.java b/src/main/java/com/williamcallahan/javachat/service/MarkdownService.java index b0604a1..d29f899 100644 --- a/src/main/java/com/williamcallahan/javachat/service/MarkdownService.java +++ b/src/main/java/com/williamcallahan/javachat/service/MarkdownService.java @@ -36,5 +36,4 @@ public MarkdownService(UnifiedMarkdownService unifiedService) { public ProcessedMarkdown processStructured(String markdownText) { return unifiedService.process(markdownText); } - } diff --git a/src/main/java/com/williamcallahan/javachat/service/OpenAIStreamingService.java b/src/main/java/com/williamcallahan/javachat/service/OpenAIStreamingService.java index 671a818..acffac0 100644 --- a/src/main/java/com/williamcallahan/javachat/service/OpenAIStreamingService.java +++ b/src/main/java/com/williamcallahan/javachat/service/OpenAIStreamingService.java @@ -23,6 +23,8 @@ import com.williamcallahan.javachat.support.AsciiTextNormalizer; import com.williamcallahan.javachat.support.OpenAiSdkUrlNormalizer; import jakarta.annotation.PreDestroy; +import java.util.Objects; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -31,13 +33,10 @@ import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - /** * OpenAI Java SDK-based streaming service that provides clean, reliable streaming * without manual SSE parsing, token buffering artifacts, or spacing issues. - * + * * This service replaces the complex manual SSE handling in ResilientApiClient * with the OpenAI Java SDK's native streaming support. */ @@ -58,14 +57,12 @@ public class OpenAIStreamingService { private static final int MAX_TOKENS_DEFAULT_INPUT = 100_000; /** Truncation notice for GPT-5 family models with 8K input limit. */ - private static final String TRUNCATION_NOTICE_GPT5 = - "[Context truncated due to GPT-5 8K input limit]\n\n"; + private static final String TRUNCATION_NOTICE_GPT5 = "[Context truncated due to GPT-5 8K input limit]\n\n"; /** Truncation notice for other models with larger limits. */ - private static final String TRUNCATION_NOTICE_GENERIC = - "[Context truncated due to model input limit]\n\n"; + private static final String TRUNCATION_NOTICE_GENERIC = "[Context truncated due to model input limit]\n\n"; - private OpenAIClient clientPrimary; // Prefer GitHub Models when available + private OpenAIClient clientPrimary; // Prefer GitHub Models when available private OpenAIClient clientSecondary; // Fallback to OpenAI when available private volatile boolean isAvailable = false; private final RateLimitManager rateLimitManager; @@ -82,9 +79,7 @@ public class OpenAIStreamingService { * @param chunker token-aware text chunker for legacy string truncation * @param promptTruncator structure-aware prompt truncator */ - public OpenAIStreamingService(RateLimitManager rateLimitManager, - Chunker chunker, - PromptTruncator promptTruncator) { + public OpenAIStreamingService(RateLimitManager rateLimitManager, Chunker chunker, PromptTruncator promptTruncator) { this.rateLimitManager = rateLimitManager; this.chunker = chunker; this.promptTruncator = promptTruncator; @@ -92,10 +87,10 @@ public OpenAIStreamingService(RateLimitManager rateLimitManager, @Value("${GITHUB_TOKEN:}") private String githubToken; - + @Value("${OPENAI_API_KEY:}") private String openaiApiKey; - + @Value("${OPENAI_BASE_URL:https://api.openai.com/v1}") private String openaiBaseUrl; @@ -107,7 +102,7 @@ public OpenAIStreamingService(RateLimitManager rateLimitManager, @Value("${GITHUB_MODELS_BASE_URL:https://models.github.ai/inference/v1}") private String githubModelsBaseUrl; - + @Value("${OPENAI_REASONING_EFFORT:}") private String reasoningEffortSetting; @@ -119,7 +114,6 @@ public OpenAIStreamingService(RateLimitManager rateLimitManager, @Value("${LLM_PRIMARY_BACKOFF_SECONDS:600}") private long primaryBackoffSeconds; - /** * Initializes OpenAI-compatible clients for configured providers after Spring injects credentials. @@ -140,21 +134,20 @@ public void initializeClient() { } this.isAvailable = (clientPrimary != null) || (clientSecondary != null); if (!this.isAvailable) { - log.warn("No API credentials found (GITHUB_TOKEN or OPENAI_API_KEY) - OpenAI streaming will not be available"); + log.warn( + "No API credentials found (GITHUB_TOKEN or OPENAI_API_KEY) - OpenAI streaming will not be available"); } else { log.info( - "OpenAI streaming available (githubModel={}, openaiModel={}, primary={}, secondary={})", - githubModelsChatModel, - openaiModel, - clientPrimary != null, - clientSecondary != null - ); + "OpenAI streaming available (primaryConfigured={}, secondaryConfigured={})", + clientPrimary != null, + clientSecondary != null); } } catch (RuntimeException initializationException) { log.error("Failed to initialize OpenAI client", initializationException); this.isAvailable = (clientPrimary != null) || (clientSecondary != null); if (!this.isAvailable) { - log.warn("No API credentials found (GITHUB_TOKEN or OPENAI_API_KEY) - OpenAI streaming will not be available"); + log.warn( + "No API credentials found (GITHUB_TOKEN or OPENAI_API_KEY) - OpenAI streaming will not be available"); } } } @@ -174,9 +167,9 @@ private void closeClientSafely(OpenAIClient client, String clientName) { } try { client.close(); - log.debug("Closed OpenAI client ({})", clientName); + log.debug("Closed OpenAI client"); } catch (RuntimeException closeException) { - log.warn("Failed to close OpenAI client ({})", clientName, closeException); + log.warn("Failed to close OpenAI client"); } } @@ -189,45 +182,6 @@ private OpenAIClient createClient(String apiKey, String baseUrl) { .maxRetries(0) .build(); } - - /** - * Stream a response from the OpenAI API using clean, native streaming support. - * - * @param prompt The complete prompt to send to the model - * @param temperature The temperature setting for response generation - * @return A Flux of content strings as they arrive from the model - * @deprecated Use {@link #streamResponse(StructuredPrompt, double)} for structure-aware truncation. - */ - @Deprecated - public Flux streamResponse(String prompt, double temperature) { - log.debug("Starting OpenAI stream"); - - return Flux.defer(() -> { - // Select client first to determine which provider's model name to use - OpenAIClient streamingClient = selectClientForStreaming(); - - if (streamingClient == null) { - String error = "All LLM providers unavailable - check rate limits and API credentials"; - log.error("[LLM] {}", error); - return Flux.error(new IllegalStateException(error)); - } - - boolean useGitHubModels = isPrimaryClient(streamingClient); - RateLimitManager.ApiProvider activeProvider = useGitHubModels - ? RateLimitManager.ApiProvider.GITHUB_MODELS - : RateLimitManager.ApiProvider.OPENAI; - - // Build params with provider-specific model after client selection - String truncatedPrompt = truncatePromptForModel(prompt); - ResponseCreateParams params = buildResponseParams(truncatedPrompt, temperature, useGitHubModels); - log.info("[LLM] [{}] Streaming started", activeProvider.getName()); - - return executeStreamingRequest(streamingClient, params, activeProvider); - }) - // Move blocking SDK stream consumption off the servlet thread. - // Prevents thread starvation and aligns with Reactor best practices. - .subscribeOn(Schedulers.boundedElastic()); - } /** * Stream a response using a structured prompt with intelligent truncation. @@ -244,71 +198,69 @@ public Flux streamResponse(StructuredPrompt structuredPrompt, double tem log.debug("Starting OpenAI stream with structured prompt"); return Flux.defer(() -> { - OpenAIClient streamingClient = selectClientForStreaming(); - - if (streamingClient == null) { - String error = "All LLM providers unavailable - check rate limits and API credentials"; - log.error("[LLM] {}", error); - return Flux.error(new IllegalStateException(error)); - } - - boolean useGitHubModels = isPrimaryClient(streamingClient); - RateLimitManager.ApiProvider activeProvider = useGitHubModels - ? RateLimitManager.ApiProvider.GITHUB_MODELS - : RateLimitManager.ApiProvider.OPENAI; - - // Determine model and token limits - String modelId = normalizedModelId(useGitHubModels); - boolean isGpt5 = isGpt5Family(modelId); - int tokenLimit = isGpt5 ? MAX_TOKENS_GPT5_INPUT : MAX_TOKENS_DEFAULT_INPUT; - - // Truncate using structure-aware truncator - PromptTruncator.TruncatedPrompt truncated = - promptTruncator.truncate(structuredPrompt, tokenLimit, isGpt5); - String finalPrompt = truncated.render(); - - if (truncated.wasTruncated()) { - log.info("[LLM] Prompt truncated: {} context docs, {} conversation turns", - truncated.contextDocumentCount(), truncated.conversationTurnCount()); - } - - ResponseCreateParams params = buildResponseParams(finalPrompt, temperature, useGitHubModels); - log.info("[LLM] [{}] Streaming started (structured)", activeProvider.getName()); - - return executeStreamingRequest(streamingClient, params, activeProvider); - }) - .subscribeOn(Schedulers.boundedElastic()); + OpenAIClient streamingClient = selectClientForStreaming(); + + if (streamingClient == null) { + String error = "All LLM providers unavailable - check rate limits and API credentials"; + log.error("[LLM] {}", error); + return Flux.error(new IllegalStateException(error)); + } + + boolean useGitHubModels = isPrimaryClient(streamingClient); + RateLimitManager.ApiProvider activeProvider = useGitHubModels + ? RateLimitManager.ApiProvider.GITHUB_MODELS + : RateLimitManager.ApiProvider.OPENAI; + + // Determine model and token limits + String modelId = normalizedModelId(useGitHubModels); + boolean isGpt5 = isGpt5Family(modelId); + int tokenLimit = isGpt5 ? MAX_TOKENS_GPT5_INPUT : MAX_TOKENS_DEFAULT_INPUT; + + // Truncate using structure-aware truncator + PromptTruncator.TruncatedPrompt truncated = + promptTruncator.truncate(structuredPrompt, tokenLimit, isGpt5); + String finalPrompt = truncated.render(); + + if (truncated.wasTruncated()) { + log.info( + "[LLM] Prompt truncated: {} context docs, {} conversation turns", + truncated.contextDocumentCount(), + truncated.conversationTurnCount()); + } + + ResponseCreateParams params = buildResponseParams(finalPrompt, temperature, useGitHubModels); + log.info("[LLM] Streaming started (structured, providerId={})", activeProvider.ordinal()); + + return executeStreamingRequest(streamingClient, params, activeProvider); + }) + .subscribeOn(Schedulers.boundedElastic()); } /** * Executes the streaming request and handles completion/error callbacks. */ - private Flux executeStreamingRequest(OpenAIClient client, - ResponseCreateParams params, - RateLimitManager.ApiProvider activeProvider) { - RequestOptions requestOptions = RequestOptions.builder() - .timeout(streamingTimeout()) - .build(); + private Flux executeStreamingRequest( + OpenAIClient client, ResponseCreateParams params, RateLimitManager.ApiProvider activeProvider) { + RequestOptions requestOptions = + RequestOptions.builder().timeout(streamingTimeout()).build(); return Flux.>using( - () -> client.responses().createStreaming(params, requestOptions), - (StreamResponse responseStream) -> Flux.fromStream(responseStream.stream()) - .concatMap(event -> Mono.justOrEmpty(extractTextDelta(event))) - ) - .doOnComplete(() -> { - log.debug("[LLM] [{}] Stream completed successfully", activeProvider.getName()); - if (rateLimitManager != null) { - rateLimitManager.recordSuccess(activeProvider); - } - }) - .doOnError(exception -> { - log.error("[LLM] [{}] Streaming failed: {}", - activeProvider.getName(), exception.getMessage(), exception); - if (rateLimitManager != null) { - recordProviderFailure(activeProvider, exception); - } - maybeBackoffPrimary(client, exception); - }); + () -> client.responses().createStreaming(params, requestOptions), + (StreamResponse responseStream) -> Flux.fromStream(responseStream.stream()) + .concatMap(event -> Mono.justOrEmpty(extractTextDelta(event)))) + .doOnComplete(() -> { + log.debug("[LLM] Stream completed successfully (providerId={})", activeProvider.ordinal()); + if (rateLimitManager != null) { + rateLimitManager.recordSuccess(activeProvider); + } + }) + .doOnError(exception -> { + log.error("[LLM] Streaming failed (providerId={})", activeProvider.ordinal()); + if (rateLimitManager != null) { + recordProviderFailure(activeProvider, exception); + } + maybeBackoffPrimary(client, exception); + }); } /** @@ -317,52 +269,52 @@ private Flux executeStreamingRequest(OpenAIClient client, public Mono complete(String prompt, double temperature) { final String truncatedPrompt = truncatePromptForModel(prompt); return Mono.defer(() -> { - OpenAIClient blockingClient = selectClientForBlocking(); - if (blockingClient == null) { - String error = "All LLM providers unavailable - check rate limits and API credentials"; - log.error("[LLM] {}", error); - return Mono.error(new IllegalStateException(error)); - } - boolean useGitHubModels = isPrimaryClient(blockingClient); - RateLimitManager.ApiProvider activeProvider = useGitHubModels - ? RateLimitManager.ApiProvider.GITHUB_MODELS - : RateLimitManager.ApiProvider.OPENAI; - ResponseCreateParams params = buildResponseParams(truncatedPrompt, temperature, useGitHubModels); - try { - log.info("[LLM] [{}] Complete started", activeProvider.getName()); - RequestOptions requestOptions = RequestOptions.builder() - .timeout(completeTimeout()) - .build(); - Response completion = blockingClient.responses().create(params, requestOptions); - if (rateLimitManager != null) { - rateLimitManager.recordSuccess(activeProvider); - } - log.debug("[LLM] [{}] Complete succeeded", activeProvider.getName()); - String response = extractTextFromResponse(completion); - return Mono.just(response); - } catch (RuntimeException completionException) { - log.error("[LLM] [{}] Complete failed: {}", - activeProvider.getName(), completionException.getMessage(), completionException); - if (rateLimitManager != null) { - recordProviderFailure(activeProvider, completionException); - } - maybeBackoffPrimary(blockingClient, completionException); - return Mono.error(completionException); - } - }).subscribeOn(Schedulers.boundedElastic()); + OpenAIClient blockingClient = selectClientForBlocking(); + if (blockingClient == null) { + String error = "All LLM providers unavailable - check rate limits and API credentials"; + log.error("[LLM] {}", error); + return Mono.error(new IllegalStateException(error)); + } + boolean useGitHubModels = isPrimaryClient(blockingClient); + RateLimitManager.ApiProvider activeProvider = useGitHubModels + ? RateLimitManager.ApiProvider.GITHUB_MODELS + : RateLimitManager.ApiProvider.OPENAI; + ResponseCreateParams params = buildResponseParams(truncatedPrompt, temperature, useGitHubModels); + try { + log.info("[LLM] Complete started (providerId={})", activeProvider.ordinal()); + RequestOptions requestOptions = RequestOptions.builder() + .timeout(completeTimeout()) + .build(); + Response completion = blockingClient.responses().create(params, requestOptions); + if (rateLimitManager != null) { + rateLimitManager.recordSuccess(activeProvider); + } + log.debug("[LLM] Complete succeeded (providerId={})", activeProvider.ordinal()); + String response = extractTextFromResponse(completion); + return Mono.just(response); + } catch (RuntimeException completionException) { + log.error("[LLM] Complete failed (providerId={})", activeProvider.ordinal()); + if (rateLimitManager != null) { + recordProviderFailure(activeProvider, completionException); + } + maybeBackoffPrimary(blockingClient, completionException); + return Mono.error(completionException); + } + }) + .subscribeOn(Schedulers.boundedElastic()); } private Timeout streamingTimeout() { return Timeout.builder() - .request(java.time.Duration.ofSeconds(Math.max(1, streamingRequestTimeoutSeconds))) - .read(java.time.Duration.ofSeconds(Math.max(1, streamingReadTimeoutSeconds))) - .build(); + .request(java.time.Duration.ofSeconds(Math.max(1, streamingRequestTimeoutSeconds))) + .read(java.time.Duration.ofSeconds(Math.max(1, streamingReadTimeoutSeconds))) + .build(); } private Timeout completeTimeout() { return Timeout.builder() - .request(java.time.Duration.ofSeconds(COMPLETE_REQUEST_TIMEOUT_SECONDS)) - .build(); + .request(java.time.Duration.ofSeconds(COMPLETE_REQUEST_TIMEOUT_SECONDS)) + .build(); } private void recordProviderFailure(RateLimitManager.ApiProvider provider, Throwable throwable) { @@ -392,9 +344,8 @@ private ResponseCreateParams buildResponseParams(String prompt, double temperatu boolean gpt5Family = isGpt5Family(normalizedModelId); boolean reasoningModel = gpt5Family || normalizedModelId.startsWith("o"); - ResponseCreateParams.Builder builder = ResponseCreateParams.builder() - .input(prompt) - .model(ResponsesModel.ofString(normalizedModelId)); + ResponseCreateParams.Builder builder = + ResponseCreateParams.builder().input(prompt).model(ResponsesModel.ofString(normalizedModelId)); if (gpt5Family) { // GPT-5 family: omit temperature and set conservative max output tokens @@ -413,9 +364,7 @@ private ResponseCreateParams buildResponseParams(String prompt, double temperatu } private String extractTextDelta(ResponseStreamEvent event) { - return event.outputTextDelta() - .map(ResponseTextDeltaEvent::delta) - .orElse(null); + return event.outputTextDelta().map(ResponseTextDeltaEvent::delta).orElse(null); } private String extractTextFromResponse(Response response) { @@ -437,9 +386,9 @@ private String extractTextFromResponse(Response response) { } return outputBuilder.toString(); } - + // Model mapping removed to prevent unintended regression; use configured model id - + /** * Truncate prompt conservatively based on model limits to avoid 413 errors. * @@ -464,7 +413,7 @@ private String truncatePromptForModel(String prompt) { String notice = isGpt5 ? TRUNCATION_NOTICE_GPT5 : TRUNCATION_NOTICE_GENERIC; return notice + truncated; } - + return prompt; } @@ -501,7 +450,7 @@ private ReasoningEffort resolveReasoningEffort(String normalizedModelId) { private String normalizeBaseUrl(String baseUrl) { return OpenAiSdkUrlNormalizer.normalize(baseUrl); } - + /** * Check if the OpenAI streaming service is properly configured and available. */ @@ -512,9 +461,10 @@ public boolean isAvailable() { private OpenAIClient selectClientForStreaming() { // Prefer OpenAI when available (more reliable, shorter rate limit windows) if (clientSecondary != null) { - if (rateLimitManager == null - || rateLimitManager.isProviderAvailable(RateLimitManager.ApiProvider.OPENAI)) { - log.debug("[{}] Selected for streaming", RateLimitManager.ApiProvider.OPENAI.getName()); + if (rateLimitManager == null || rateLimitManager.isProviderAvailable(RateLimitManager.ApiProvider.OPENAI)) { + log.debug( + "Selected provider for streaming (providerId={})", + RateLimitManager.ApiProvider.OPENAI.ordinal()); return clientSecondary; } } @@ -523,7 +473,9 @@ private OpenAIClient selectClientForStreaming() { if (clientPrimary != null && !isPrimaryInBackoff()) { if (rateLimitManager == null || rateLimitManager.isProviderAvailable(RateLimitManager.ApiProvider.GITHUB_MODELS)) { - log.debug("[{}] Selected for streaming", RateLimitManager.ApiProvider.GITHUB_MODELS.getName()); + log.debug( + "Selected provider for streaming (providerId={})", + RateLimitManager.ApiProvider.GITHUB_MODELS.ordinal()); return clientPrimary; } } @@ -531,15 +483,14 @@ private OpenAIClient selectClientForStreaming() { // All providers marked as rate limited - try OpenAI anyway (short rate limit windows). // Let the API decide if we're actually still rate limited. if (clientSecondary != null) { - log.warn("[{}] All providers marked as rate limited; attempting OpenAI anyway " - + "(typical rate limits are 1-60 seconds)", RateLimitManager.ApiProvider.OPENAI.getName()); + log.warn("All providers marked as rate limited; attempting OpenAI anyway " + + "(typical rate limits are 1-60 seconds)"); return clientSecondary; } // OpenAI not configured - try GitHub Models as last resort if (clientPrimary != null) { - log.warn("[{}] All providers marked as rate limited; attempting GitHub Models as last resort", - RateLimitManager.ApiProvider.GITHUB_MODELS.getName()); + log.warn("All providers marked as rate limited; attempting GitHub Models as last resort"); return clientPrimary; } @@ -557,10 +508,10 @@ private boolean isPrimaryClient(OpenAIClient client) { private boolean isRateLimit(Throwable throwable) { return throwable instanceof RateLimitException - || (throwable instanceof OpenAIServiceException serviceException - && serviceException.statusCode() == 429); + || (throwable instanceof OpenAIServiceException serviceException + && serviceException.statusCode() == 429); } - + private boolean isRetryablePrimaryFailure(Throwable throwable) { if (isRateLimit(throwable)) { return true; @@ -579,17 +530,15 @@ private boolean isRetryablePrimaryFailure(Throwable throwable) { String message = throwable.getMessage(); return message != null && AsciiTextNormalizer.toLowerAscii(message).contains("sleep interrupted"); } - + private boolean isPrimaryInBackoff() { return System.currentTimeMillis() < primaryBackoffUntilEpochMs; } - + private void markPrimaryBackoff() { long until = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(Math.max(1, primaryBackoffSeconds)); this.primaryBackoffUntilEpochMs = until; long seconds = Math.max(1, (until - System.currentTimeMillis()) / 1000); - log.warn("[{}] Temporarily disabled for {}s due to failure", - RateLimitManager.ApiProvider.GITHUB_MODELS.getName(), seconds); + log.warn("Primary provider temporarily disabled for {}s due to failure", seconds); } - } diff --git a/src/main/java/com/williamcallahan/javachat/service/OpenAiCompatibleEmbeddingModel.java b/src/main/java/com/williamcallahan/javachat/service/OpenAiCompatibleEmbeddingModel.java index 162c9d9..623d177 100644 --- a/src/main/java/com/williamcallahan/javachat/service/OpenAiCompatibleEmbeddingModel.java +++ b/src/main/java/com/williamcallahan/javachat/service/OpenAiCompatibleEmbeddingModel.java @@ -7,17 +7,16 @@ import com.openai.models.embeddings.CreateEmbeddingResponse; import com.openai.models.embeddings.EmbeddingCreateParams; import com.williamcallahan.javachat.support.OpenAiSdkUrlNormalizer; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.ai.embedding.Embedding; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.embedding.EmbeddingRequest; import org.springframework.ai.embedding.EmbeddingResponse; -import org.springframework.ai.embedding.Embedding; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.time.Duration; /** * Simple OpenAI-compatible EmbeddingModel. @@ -25,7 +24,7 @@ */ public final class OpenAiCompatibleEmbeddingModel implements EmbeddingModel, AutoCloseable { private static final Logger log = LoggerFactory.getLogger(OpenAiCompatibleEmbeddingModel.class); - + private static final int CONNECT_TIMEOUT_SECONDS = 10; private static final int READ_TIMEOUT_SECONDS = 60; @@ -33,18 +32,18 @@ public final class OpenAiCompatibleEmbeddingModel implements EmbeddingModel, Aut private final String modelName; private final int dimensionsHint; - /** - * Wraps remote embedding API failures as a runtime exception with concise context. - */ - private static final class EmbeddingApiResponseException extends IllegalStateException { - private EmbeddingApiResponseException(String message, Exception cause) { - super(message, cause); - } + /** + * Wraps remote embedding API failures as a runtime exception with concise context. + */ + private static final class EmbeddingApiResponseException extends IllegalStateException { + private EmbeddingApiResponseException(String message, Exception cause) { + super(message, cause); + } private EmbeddingApiResponseException(String message) { - super(message); - } - } + super(message); + } + } /** * Creates an OpenAI-compatible embedding model backed by a remote REST API endpoint. @@ -55,63 +54,54 @@ private EmbeddingApiResponseException(String message) { * @param dimensionsHint expected embedding dimensions (used as a hint) * @return embedding model configured for the remote endpoint */ - public static OpenAiCompatibleEmbeddingModel create(String baseUrl, - String apiKey, - String modelName, - int dimensionsHint) { + public static OpenAiCompatibleEmbeddingModel create( + String baseUrl, String apiKey, String modelName, int dimensionsHint) { validateDimensions(dimensionsHint); OpenAIClient client = OpenAIOkHttpClient.builder() - .apiKey(requireConfiguredApiKey(apiKey)) - .baseUrl(normalizeSdkBaseUrl(baseUrl)) - .build(); + .apiKey(requireConfiguredApiKey(apiKey)) + .baseUrl(normalizeSdkBaseUrl(baseUrl)) + .build(); return new OpenAiCompatibleEmbeddingModel(client, requireConfiguredModel(modelName), dimensionsHint); } - static OpenAiCompatibleEmbeddingModel create(OpenAIClient client, - String modelName, - int dimensionsHint) { + static OpenAiCompatibleEmbeddingModel create(OpenAIClient client, String modelName, int dimensionsHint) { validateDimensions(dimensionsHint); return new OpenAiCompatibleEmbeddingModel( - Objects.requireNonNull(client, "client"), - requireConfiguredModel(modelName), - dimensionsHint - ); + Objects.requireNonNull(client, "client"), requireConfiguredModel(modelName), dimensionsHint); } - private OpenAiCompatibleEmbeddingModel(OpenAIClient client, - String modelName, - int dimensionsHint) { + private OpenAiCompatibleEmbeddingModel(OpenAIClient client, String modelName, int dimensionsHint) { this.client = client; this.modelName = modelName; this.dimensionsHint = dimensionsHint; } - /** - * Calls the OpenAI-compatible embeddings endpoint for all inputs in the request. - */ - @Override + /** + * Calls the OpenAI-compatible embeddings endpoint for all inputs in the request. + */ + @Override public EmbeddingResponse call(EmbeddingRequest request) { List instructions = request.getInstructions(); if (instructions.isEmpty()) { return new EmbeddingResponse(List.of()); } - try { - EmbeddingCreateParams params = EmbeddingCreateParams.builder() + try { + EmbeddingCreateParams params = EmbeddingCreateParams.builder() .model(modelName) .inputOfArrayOfStrings(instructions) .build(); - RequestOptions requestOptions = RequestOptions.builder() - .timeout(embeddingTimeout()) - .build(); - CreateEmbeddingResponse response = client.embeddings().create(params, requestOptions); - List embeddings = parseResponse(response, instructions.size()); - return new EmbeddingResponse(embeddings); - - } catch (EmbeddingApiResponseException exception) { - throw exception; - } catch (RuntimeException exception) { - log.warn("[EMBEDDING] Remote embedding call failed (exception type: {})", - exception.getClass().getSimpleName()); + RequestOptions requestOptions = + RequestOptions.builder().timeout(embeddingTimeout()).build(); + CreateEmbeddingResponse response = client.embeddings().create(params, requestOptions); + List embeddings = parseResponse(response, instructions.size()); + return new EmbeddingResponse(embeddings); + + } catch (EmbeddingApiResponseException exception) { + throw exception; + } catch (RuntimeException exception) { + log.warn( + "[EMBEDDING] Remote embedding call failed (exception type: {})", + exception.getClass().getSimpleName()); throw new EmbeddingApiResponseException("Remote embedding call failed", exception); } } @@ -120,57 +110,55 @@ private Timeout embeddingTimeout() { Duration connectTimeout = Duration.ofSeconds(CONNECT_TIMEOUT_SECONDS); Duration requestTimeout = Duration.ofSeconds(READ_TIMEOUT_SECONDS); return Timeout.builder() - .connect(connectTimeout) - .request(requestTimeout) - .read(requestTimeout) - .build(); + .connect(connectTimeout) + .request(requestTimeout) + .read(requestTimeout) + .build(); } private List parseResponse(CreateEmbeddingResponse response, int expectedCount) { - if (response == null) { - throw new EmbeddingApiResponseException("Remote embedding response was null"); - } - List data = response.data(); - if (data.isEmpty()) { - throw new EmbeddingApiResponseException("Remote embedding response missing embedding entries"); - } + if (response == null) { + throw new EmbeddingApiResponseException("Remote embedding response was null"); + } + List data = response.data(); + if (data.isEmpty()) { + throw new EmbeddingApiResponseException("Remote embedding response missing embedding entries"); + } - List embeddingsByIndex = new ArrayList<>(expectedCount); - for (int index = 0; index < expectedCount; index++) { - embeddingsByIndex.add(null); - } + List embeddingsByIndex = new ArrayList<>(expectedCount); + for (int index = 0; index < expectedCount; index++) { + embeddingsByIndex.add(null); + } - for (int itemIndex = 0; itemIndex < data.size(); itemIndex++) { - com.openai.models.embeddings.Embedding item = data.get(itemIndex); - if (item == null) { - throw new EmbeddingApiResponseException( + for (int itemIndex = 0; itemIndex < data.size(); itemIndex++) { + com.openai.models.embeddings.Embedding item = data.get(itemIndex); + if (item == null) { + throw new EmbeddingApiResponseException( "Remote embedding response contained null entry at index " + itemIndex); - } - int targetIndex = safeEmbeddingIndex(itemIndex, item, expectedCount); - if (targetIndex < 0 || targetIndex >= expectedCount) { - continue; - } - float[] vector = toFloatVector(item.embedding()); - embeddingsByIndex.set(targetIndex, new Embedding(vector, targetIndex)); } + int targetIndex = safeEmbeddingIndex(itemIndex, item, expectedCount); + if (targetIndex < 0 || targetIndex >= expectedCount) { + continue; + } + float[] vector = toFloatVector(item.embedding()); + embeddingsByIndex.set(targetIndex, new Embedding(vector, targetIndex)); + } - List orderedEmbeddings = new ArrayList<>(expectedCount); - for (int index = 0; index < expectedCount; index++) { - Embedding embedding = embeddingsByIndex.get(index); - if (embedding == null) { - throw new EmbeddingApiResponseException("Remote embedding response missing embedding for index " + index); - } - orderedEmbeddings.add(embedding); + List orderedEmbeddings = new ArrayList<>(expectedCount); + for (int index = 0; index < expectedCount; index++) { + Embedding embedding = embeddingsByIndex.get(index); + if (embedding == null) { + throw new EmbeddingApiResponseException( + "Remote embedding response missing embedding for index " + index); } + orderedEmbeddings.add(embedding); + } - return orderedEmbeddings; - } + return orderedEmbeddings; + } private int safeEmbeddingIndex(int fallbackIndex, com.openai.models.embeddings.Embedding item, int expectedCount) { - long responseIndex = item._index() - .asNumber() - .map(Number::longValue) - .orElse((long) fallbackIndex); + long responseIndex = item._index().asNumber().map(Number::longValue).orElse((long) fallbackIndex); if (responseIndex < 0L || responseIndex > (long) Integer.MAX_VALUE) { log.debug("[EMBEDDING] Ignoring out-of-range embedding index={}", responseIndex); return -1; @@ -201,23 +189,23 @@ public float[] embed(org.springframework.ai.document.Document document) { if (embeddingResponse.getResults().isEmpty()) { throw new EmbeddingApiResponseException("Embedding response was empty"); } - return embeddingResponse.getResults().get(0).getOutput(); - } + return embeddingResponse.getResults().get(0).getOutput(); + } - private float[] toFloatVector(List embeddingEntries) { - if (embeddingEntries == null || embeddingEntries.isEmpty()) { - throw new EmbeddingApiResponseException("Remote embedding response missing embedding values"); - } - float[] vector = new float[embeddingEntries.size()]; - for (int vectorIndex = 0; vectorIndex < embeddingEntries.size(); vectorIndex++) { - Float entry = embeddingEntries.get(vectorIndex); - if (entry == null) { - throw new EmbeddingApiResponseException("Null embedding value at index " + vectorIndex); - } - vector[vectorIndex] = entry; + private float[] toFloatVector(List embeddingEntries) { + if (embeddingEntries == null || embeddingEntries.isEmpty()) { + throw new EmbeddingApiResponseException("Remote embedding response missing embedding values"); + } + float[] vector = new float[embeddingEntries.size()]; + for (int vectorIndex = 0; vectorIndex < embeddingEntries.size(); vectorIndex++) { + Float entry = embeddingEntries.get(vectorIndex); + if (entry == null) { + throw new EmbeddingApiResponseException("Null embedding value at index " + vectorIndex); } - return vector; + vector[vectorIndex] = entry; } + return vector; + } /** * Closes the underlying OpenAI client and releases its resources. diff --git a/src/main/java/com/williamcallahan/javachat/service/PdfContentExtractor.java b/src/main/java/com/williamcallahan/javachat/service/PdfContentExtractor.java index 363563c..3ea3146 100644 --- a/src/main/java/com/williamcallahan/javachat/service/PdfContentExtractor.java +++ b/src/main/java/com/williamcallahan/javachat/service/PdfContentExtractor.java @@ -1,14 +1,13 @@ package com.williamcallahan.javachat.service; +import java.io.IOException; +import java.nio.file.Path; import org.apache.pdfbox.Loader; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.text.PDFTextStripper; -import org.springframework.stereotype.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Path; +import org.springframework.stereotype.Service; /** * Extracts text and basic metadata from PDF documents using Apache PDFBox. @@ -19,40 +18,39 @@ @Service public class PdfContentExtractor { private static final Logger log = LoggerFactory.getLogger(PdfContentExtractor.class); - + /** * Extract text content from a PDF file. - * + * * @param pdfPath Path to the PDF file * @return Extracted text content * @throws IOException if the PDF cannot be read */ public String extractTextFromPdf(Path pdfPath) throws IOException { log.info("Extracting text from PDF"); - + try (PDDocument document = Loader.loadPDF(pdfPath.toFile())) { PDFTextStripper stripper = new PDFTextStripper(); - + // Configure the text stripper for better extraction stripper.setSortByPosition(true); stripper.setStartPage(1); stripper.setEndPage(document.getNumberOfPages()); - + String text = stripper.getText(document); - - log.info("Successfully extracted {} characters from {} pages", - text.length(), document.getNumberOfPages()); - + + log.info("Successfully extracted {} characters from {} pages", text.length(), document.getNumberOfPages()); + return text; } catch (IOException exception) { // Let caller decide how to log/handle this; avoid duplicate stack traces throw exception; } } - + /** * Extract text from a specific page range in a PDF. - * + * * @param pdfPath Path to the PDF file * @param startPage Starting page (1-indexed) * @param endPage Ending page (inclusive) @@ -61,30 +59,29 @@ public String extractTextFromPdf(Path pdfPath) throws IOException { */ public String extractTextFromPdfRange(Path pdfPath, int startPage, int endPage) throws IOException { log.info("Extracting text from PDF pages {}-{}", startPage, endPage); - + try (PDDocument document = Loader.loadPDF(pdfPath.toFile())) { PDFTextStripper stripper = new PDFTextStripper(); - + // Configure the text stripper stripper.setSortByPosition(true); stripper.setStartPage(startPage); stripper.setEndPage(Math.min(endPage, document.getNumberOfPages())); - + String text = stripper.getText(document); - - log.info("Successfully extracted {} characters from pages {}-{}", - text.length(), startPage, endPage); - + + log.info("Successfully extracted {} characters from pages {}-{}", text.length(), startPage, endPage); + return text; } catch (IOException exception) { // Let caller decide how to log/handle this; avoid duplicate stack traces throw exception; } } - + /** * Get metadata about a PDF file. - * + * * @param pdfPath Path to the PDF file * @return PDF metadata as a formatted string * @throws IOException if the PDF cannot be read @@ -92,10 +89,10 @@ public String extractTextFromPdfRange(Path pdfPath, int startPage, int endPage) public String getPdfMetadata(Path pdfPath) throws IOException { try (PDDocument document = Loader.loadPDF(pdfPath.toFile())) { StringBuilder metadata = new StringBuilder(); - + if (document.getDocumentInformation() != null) { var info = document.getDocumentInformation(); - + if (info.getTitle() != null) { metadata.append("Title: ").append(info.getTitle()).append("\n"); } @@ -106,9 +103,9 @@ public String getPdfMetadata(Path pdfPath) throws IOException { metadata.append("Subject: ").append(info.getSubject()).append("\n"); } } - + metadata.append("Pages: ").append(document.getNumberOfPages()).append("\n"); - + return metadata.toString(); } } diff --git a/src/main/java/com/williamcallahan/javachat/service/ProgressTracker.java b/src/main/java/com/williamcallahan/javachat/service/ProgressTracker.java index ddd9b5d..4e0bc8d 100644 --- a/src/main/java/com/williamcallahan/javachat/service/ProgressTracker.java +++ b/src/main/java/com/williamcallahan/javachat/service/ProgressTracker.java @@ -1,15 +1,14 @@ package com.williamcallahan.javachat.service; import jakarta.annotation.PostConstruct; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; /** * Tracks ingestion progress by counting parsed and indexed chunk artifacts on disk and in memory. @@ -27,8 +26,7 @@ public class ProgressTracker { * Creates a tracker rooted at the parsed and index directories. */ public ProgressTracker( - @Value("${app.docs.parsed-dir}") String parsedDir, - @Value("${app.docs.index-dir}") String indexDir) { + @Value("${app.docs.parsed-dir}") String parsedDir, @Value("${app.docs.index-dir}") String indexDir) { this.parsedDir = Path.of(parsedDir); this.indexDir = Path.of(indexDir); } @@ -42,18 +40,20 @@ public void init() { long parsed = 0; if (parsedDir != null && Files.isDirectory(parsedDir)) { try (var pathStream = Files.walk(parsedDir)) { - parsed = pathStream.filter(pathCandidate -> !Files.isDirectory(pathCandidate)) - .filter(pathCandidate -> { - Path fileName = pathCandidate.getFileName(); - return fileName != null && fileName.toString().endsWith(".txt"); - }) - .count(); + parsed = pathStream + .filter(pathCandidate -> !Files.isDirectory(pathCandidate)) + .filter(pathCandidate -> { + Path fileName = pathCandidate.getFileName(); + return fileName != null && fileName.toString().endsWith(".txt"); + }) + .count(); } } parsedCount.set(parsed); } catch (IOException exception) { - log.debug("Progress init: unable to count parsed chunks (exception type: {})", - exception.getClass().getSimpleName()); + log.debug( + "Progress init: unable to count parsed chunks (exception type: {})", + exception.getClass().getSimpleName()); } try { long indexed = 0; @@ -64,8 +64,9 @@ public void init() { } indexedCount.set(indexed); } catch (IOException exception) { - log.debug("Progress init: unable to count indexed chunks (exception type: {})", - exception.getClass().getSimpleName()); + log.debug( + "Progress init: unable to count indexed chunks (exception type: {})", + exception.getClass().getSimpleName()); } log.info("[INDEXING] Progress initialized: parsed={}, indexed={}", parsedCount.get(), indexedCount.get()); } diff --git a/src/main/java/com/williamcallahan/javachat/service/RateLimitHeaderParser.java b/src/main/java/com/williamcallahan/javachat/service/RateLimitHeaderParser.java index f70f5c1..d531351 100644 --- a/src/main/java/com/williamcallahan/javachat/service/RateLimitHeaderParser.java +++ b/src/main/java/com/williamcallahan/javachat/service/RateLimitHeaderParser.java @@ -38,7 +38,9 @@ boolean matches(String normalized) { } String extractNumber(String normalized) { - return normalized.substring(0, normalized.length() - suffix.length()).trim(); + return normalized + .substring(0, normalized.length() - suffix.length()) + .trim(); } Duration convert(long amount) { @@ -114,7 +116,7 @@ long parseRetryAfterSeconds(Headers headers) { } try { ZonedDateTime httpDate = - ZonedDateTime.parse(trimmed, java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME); + ZonedDateTime.parse(trimmed, java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME); long seconds = Duration.between(Instant.now(), httpDate.toInstant()).getSeconds(); return Math.max(0, seconds); } catch (RuntimeException parseFailure) { @@ -136,12 +138,11 @@ Instant parseResetInstant(Headers headers) { } long candidateSeconds = minPositive( - parseDurationSeconds(firstHeaderValue(headers, "x-ratelimit-reset-requests")), - parseDurationSeconds(firstHeaderValue(headers, "x-ratelimit-reset-tokens")), - parseDurationSeconds(firstHeaderValue(headers, "x-ratelimit-reset")), - parseDurationSeconds(firstHeaderValue(headers, "X-RateLimit-Reset-Requests")), - parseDurationSeconds(firstHeaderValue(headers, "X-RateLimit-Reset-Tokens")) - ); + parseDurationSeconds(firstHeaderValue(headers, "x-ratelimit-reset-requests")), + parseDurationSeconds(firstHeaderValue(headers, "x-ratelimit-reset-tokens")), + parseDurationSeconds(firstHeaderValue(headers, "x-ratelimit-reset")), + parseDurationSeconds(firstHeaderValue(headers, "X-RateLimit-Reset-Requests")), + parseDurationSeconds(firstHeaderValue(headers, "X-RateLimit-Reset-Tokens"))); if (candidateSeconds > 0) { return Instant.now().plusSeconds(candidateSeconds); } diff --git a/src/main/java/com/williamcallahan/javachat/service/RateLimitManager.java b/src/main/java/com/williamcallahan/javachat/service/RateLimitManager.java index 0a63c29..fcbfcdf 100644 --- a/src/main/java/com/williamcallahan/javachat/service/RateLimitManager.java +++ b/src/main/java/com/williamcallahan/javachat/service/RateLimitManager.java @@ -1,13 +1,13 @@ package com.williamcallahan.javachat.service; +import com.openai.core.http.Headers; +import com.openai.errors.OpenAIServiceException; import java.time.Duration; import java.time.Instant; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; -import com.openai.core.http.Headers; -import com.openai.errors.OpenAIServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.env.Environment; @@ -21,26 +21,20 @@ @Component public class RateLimitManager { - private static final Logger log = LoggerFactory.getLogger( - RateLimitManager.class - ); + private static final Logger log = LoggerFactory.getLogger(RateLimitManager.class); /** * Encapsulates rate limit information parsed from HTTP headers or error messages. * Eliminates data clump where resetTime and retrySeconds travel together. */ - private record ParsedRateLimitInfo( - Instant resetTime, - long retryAfterSeconds - ) { + private record ParsedRateLimitInfo(Instant resetTime, long retryAfterSeconds) { boolean hasResetTime() { return resetTime != null; } } private final RateLimitState rateLimitState; - private final Map endpointStates = - new ConcurrentHashMap<>(); + private final Map endpointStates = new ConcurrentHashMap<>(); private final Environment env; private final RateLimitHeaderParser headerParser; @@ -58,11 +52,7 @@ public enum ApiProvider { private final int dailyLimit; private final String typicalRateLimitWindow; - ApiProvider( - String name, - int dailyLimit, - String typicalRateLimitWindow - ) { + ApiProvider(String name, int dailyLimit, String typicalRateLimitWindow) { this.name = name; this.dailyLimit = dailyLimit; this.typicalRateLimitWindow = typicalRateLimitWindow; @@ -99,9 +89,7 @@ public static class ApiEndpointState { private volatile int backoffMultiplier = 1; private final AtomicInteger requestsToday = new AtomicInteger(0); - private volatile Instant dayReset = Instant.now().plus( - Duration.ofDays(1) - ); + private volatile Instant dayReset = Instant.now().plus(Duration.ofDays(1)); /** * Reports whether the provider is eligible for requests based on the current circuit state. @@ -177,10 +165,7 @@ private boolean isProviderConfigured(ApiProvider provider) { } private ApiEndpointState getOrCreateEndpointState(ApiProvider provider) { - return endpointStates.computeIfAbsent( - provider.getName(), - providerKey -> new ApiEndpointState() - ); + return endpointStates.computeIfAbsent(provider.getName(), providerKey -> new ApiEndpointState()); } private boolean hasText(String valueText) { @@ -200,9 +185,7 @@ public boolean isProviderAvailable(ApiProvider provider) { // Then check persistent rate limit state if (!rateLimitState.isAvailable(provider.getName())) { - Duration remaining = rateLimitState.getRemainingWaitTime( - provider.getName() - ); + Duration remaining = rateLimitState.getRemainingWaitTime(provider.getName()); if (!remaining.isZero()) { log.debug("[{}] Rate limited (persistent state)", providerName); return false; @@ -262,11 +245,7 @@ public void recordRateLimit(ApiProvider provider, String errorMessage) { state.recordRateLimit(retryAfterSeconds); // Update persistent state with proper window - rateLimitState.recordRateLimit( - provider.getName(), - resetTime, - provider.getTypicalRateLimitWindow() - ); + rateLimitState.recordRateLimit(provider.getName(), resetTime, provider.getTypicalRateLimitWindow()); log.warn("[{}] Rate limited (errorMessage={})", providerName, safeErrorMessage); } @@ -286,7 +265,10 @@ public void recordRateLimitFromOpenAiServiceException(ApiProvider provider, Open } ParsedRateLimitInfo rateLimitInfo = parseRateLimitFromHeaders(exception.headers()); if (rateLimitInfo.retryAfterSeconds > 0) { - applyRateLimit(provider, Instant.now().plusSeconds(rateLimitInfo.retryAfterSeconds), rateLimitInfo.retryAfterSeconds); + applyRateLimit( + provider, + Instant.now().plusSeconds(rateLimitInfo.retryAfterSeconds), + rateLimitInfo.retryAfterSeconds); return; } @@ -302,10 +284,7 @@ public void recordRateLimitFromOpenAiServiceException(ApiProvider provider, Open /** * Parse rate limit reset time from WebClientResponseException */ - public void recordRateLimitFromException( - ApiProvider provider, - Throwable error - ) { + public void recordRateLimitFromException(ApiProvider provider, Throwable error) { if (error instanceof WebClientResponseException webError) { ParsedRateLimitInfo info = parseRateLimitHeaders(webError); @@ -322,15 +301,10 @@ public void recordRateLimitFromException( /** * Parse rate limit information from HTTP response headers. */ - private ParsedRateLimitInfo parseRateLimitHeaders( - WebClientResponseException webError - ) { - Instant resetTime = headerParser.parseResetHeader( - webError.getHeaders().getFirst("X-RateLimit-Reset") - ); - long retrySeconds = headerParser.parseRetryAfterHeader( - webError.getHeaders().getFirst("Retry-After") - ); + private ParsedRateLimitInfo parseRateLimitHeaders(WebClientResponseException webError) { + Instant resetTime = headerParser.parseResetHeader(webError.getHeaders().getFirst("X-RateLimit-Reset")); + long retrySeconds = + headerParser.parseRetryAfterHeader(webError.getHeaders().getFirst("Retry-After")); // If we have retry seconds but no reset time, compute reset time from retry if (resetTime == null && retrySeconds > 0) { @@ -340,7 +314,6 @@ private ParsedRateLimitInfo parseRateLimitHeaders( return new ParsedRateLimitInfo(resetTime, retrySeconds); } - /** * Applies a rate limit using reset time and optional explicit retry-after seconds. * Updates both persistent state and in-memory circuit breaker, then logs the event. @@ -357,20 +330,16 @@ private void applyRateLimit(ApiProvider provider, Instant resetTime, long retryA if (retryAfterSecondsOverride > 0) { retryAfterSeconds = retryAfterSecondsOverride; } else if (resetTime != null) { - retryAfterSeconds = Math.max(0, Duration.between(Instant.now(), resetTime).getSeconds()); + retryAfterSeconds = + Math.max(0, Duration.between(Instant.now(), resetTime).getSeconds()); } else { retryAfterSeconds = 0; } state.recordRateLimit(retryAfterSeconds); - rateLimitState.recordRateLimit( - provider.getName(), - resetTime, - provider.getTypicalRateLimitWindow() - ); - - log.warn("[{}] Rate limited (resetTime={}, retryAfterSeconds={})", - providerName, resetTime, retryAfterSeconds); + rateLimitState.recordRateLimit(provider.getName(), resetTime, provider.getTypicalRateLimitWindow()); + + log.warn("[{}] Rate limited (resetTime={}, retryAfterSeconds={})", providerName, resetTime, retryAfterSeconds); } private long extractRetryAfter(String errorMessage) { @@ -419,9 +388,7 @@ private ParsedRateLimitInfo parseRateLimitFromHeaders(Headers headers) { public Optional selectBestProvider() { // Priority order: OpenAI > GitHub Models > Local for (ApiProvider provider : new ApiProvider[] { - ApiProvider.OPENAI, - ApiProvider.GITHUB_MODELS, - ApiProvider.LOCAL, + ApiProvider.OPENAI, ApiProvider.GITHUB_MODELS, ApiProvider.LOCAL, }) { String providerName = sanitizeLogValue(provider.getName()); if (!isProviderConfigured(provider)) { @@ -441,9 +408,7 @@ public Optional selectBestProvider() { log.warn("[{}] Unavailable - missing configuration (API key/token)", providerName); continue; } - Duration remaining = rateLimitState.getRemainingWaitTime( - provider.getName() - ); + Duration remaining = rateLimitState.getRemainingWaitTime(provider.getName()); if (!remaining.isZero()) { log.warn("[{}] Unavailable - rate limited for {} more", providerName, formatDuration(remaining)); } diff --git a/src/main/java/com/williamcallahan/javachat/service/RateLimitState.java b/src/main/java/com/williamcallahan/javachat/service/RateLimitState.java index f02e59b..f27cf84 100644 --- a/src/main/java/com/williamcallahan/javachat/service/RateLimitState.java +++ b/src/main/java/com/williamcallahan/javachat/service/RateLimitState.java @@ -2,16 +2,12 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import java.io.File; import java.io.IOException; -import java.time.Instant; import java.time.Duration; +import java.time.Instant; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -19,6 +15,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; /** * Persistent rate limit state manager that survives application restarts. @@ -67,11 +66,11 @@ public void shutdown() { } catch (Exception shutdownException) { // Use stderr during teardown - logging framework may be partially unloaded System.err.println("[RateLimitState] Failed to save state on shutdown: " - + shutdownException.getClass().getName() + ": " + shutdownException.getMessage()); + + shutdownException.getClass().getName() + ": " + shutdownException.getMessage()); } catch (NoClassDefFoundError classLoadError) { // Explicitly handle classloading issues during shutdown (expected in some JVM teardown scenarios) - System.err.println("[RateLimitState] Classloader issue during shutdown (expected): " - + classLoadError.getMessage()); + System.err.println( + "[RateLimitState] Classloader issue during shutdown (expected): " + classLoadError.getMessage()); } scheduler.shutdown(); } @@ -104,8 +103,11 @@ public void recordRateLimit(String provider, Instant resetTime, String rateLimit } state.rateLimitedUntil = state.rateLimitedUntil.plus(additionalBackoff); - log.warn("[{}] Consecutive failures (count={}). Extended backoff until {}", - sanitizeLogValue(provider), failures, state.rateLimitedUntil); + log.warn( + "[{}] Consecutive failures (count={}). Extended backoff until {}", + sanitizeLogValue(provider), + failures, + state.rateLimitedUntil); } safeSaveState(); @@ -188,9 +190,13 @@ private boolean loadState() { return true; } catch (IOException exception) { String safeMessage = sanitizeLogValue(exception.getMessage()); - log.error("Failed to load rate limit state from {}: {} - {}", - STATE_FILE, exception.getClass().getSimpleName(), safeMessage); - log.warn("Continuing with empty rate limit state; previously rate-limited providers may be retried prematurely"); + log.error( + "Failed to load rate limit state from {}: {} - {}", + STATE_FILE, + exception.getClass().getSimpleName(), + safeMessage); + log.warn( + "Continuing with empty rate limit state; previously rate-limited providers may be retried prematurely"); return false; } } @@ -199,8 +205,7 @@ private void logCurrentRateLimitStatus() { for (Map.Entry entry : providerStates.entrySet()) { if (!isAvailable(entry.getKey())) { Duration remaining = getRemainingWaitTime(entry.getKey()); - log.warn("[{}] Rate limited for {} more", - sanitizeLogValue(entry.getKey()), formatDuration(remaining)); + log.warn("[{}] Rate limited for {} more", sanitizeLogValue(entry.getKey()), formatDuration(remaining)); } } } @@ -217,13 +222,18 @@ private boolean trySaveState() { return true; } catch (IOException ioException) { String safeMessage = sanitizeLogValue(ioException.getMessage()); - log.error("Failed to persist rate limit state to {}: {} - {}", - STATE_FILE, ioException.getClass().getSimpleName(), safeMessage); + log.error( + "Failed to persist rate limit state to {}: {} - {}", + STATE_FILE, + ioException.getClass().getSimpleName(), + safeMessage); return false; } catch (RuntimeException runtimeException) { String safeMessage = sanitizeLogValue(runtimeException.getMessage()); - log.error("Unexpected error persisting rate limit state: {} - {}", - runtimeException.getClass().getSimpleName(), safeMessage); + log.error( + "Unexpected error persisting rate limit state: {} - {}", + runtimeException.getClass().getSimpleName(), + safeMessage); return false; } } diff --git a/src/main/java/com/williamcallahan/javachat/service/RerankerService.java b/src/main/java/com/williamcallahan/javachat/service/RerankerService.java index 77a7bf2..52328c8 100644 --- a/src/main/java/com/williamcallahan/javachat/service/RerankerService.java +++ b/src/main/java/com/williamcallahan/javachat/service/RerankerService.java @@ -22,9 +22,7 @@ @Service public class RerankerService { - private static final Logger log = LoggerFactory.getLogger( - RerankerService.class - ); + private static final Logger log = LoggerFactory.getLogger(RerankerService.class); private final OpenAIStreamingService openAIStreamingService; private final ObjectMapper mapper; @@ -44,14 +42,10 @@ public RerankerService(OpenAIStreamingService openAIStreamingService, ObjectMapp * Cache key includes document URLs to prevent returning results for wrong document sets. */ @Cacheable( - value = "reranker-cache", - key = "#query + ':' + T(com.williamcallahan.javachat.service.RerankerService).computeDocsHash(#docs) + ':' + #returnK" - ) - public List rerank( - String query, - List docs, - int returnK - ) { + value = "reranker-cache", + key = + "#query + ':' + T(com.williamcallahan.javachat.service.RerankerService).computeDocsHash(#docs) + ':' + #returnK") + public List rerank(String query, List docs, int returnK) { if (docs.size() <= 1) { return docs; } @@ -83,10 +77,7 @@ public List rerank( * @return reranking response, or empty if service unavailable or times out */ private Optional callLlmForReranking(String query, List docs) { - if ( - openAIStreamingService == null || - !openAIStreamingService.isAvailable() - ) { + if (openAIStreamingService == null || !openAIStreamingService.isAvailable()) { log.warn("OpenAIStreamingService unavailable; skipping LLM rerank"); throw new RerankingFailureException("Reranking service unavailable"); } @@ -95,11 +86,10 @@ private Optional callLlmForReranking(String query, List docs) // Cap reranker latency aggressively; fall back on original order fast return openAIStreamingService - .complete(prompt, 0.0) - .timeout(java.time.Duration.ofSeconds(4)) - .doOnError(timeoutOrApiError -> - log.debug("Reranker LLM call timed out or failed", timeoutOrApiError)) - .blockOptional(); + .complete(prompt, 0.0) + .timeout(java.time.Duration.ofSeconds(4)) + .doOnError(timeoutOrApiError -> log.debug("Reranker LLM call timed out or failed", timeoutOrApiError)) + .blockOptional(); } /** @@ -107,19 +97,11 @@ private Optional callLlmForReranking(String query, List docs) */ private String buildRerankPrompt(String query, List docs) { StringBuilder prompt = new StringBuilder(); - prompt.append( - "You are a document re-ranker for the Java learning assistant system.\n" - ); - prompt.append( - "Reorder the following documents by relevance to the query.\n" - ); - prompt.append( - "Consider Java-specific context, version relevance, and learning value.\n" - ); - prompt.append( - "Return JSON: {\"order\":[indices...]} with 0-based indices.\n\n" - ); - prompt.append("Query: ").append(query).append("\n\n"); + prompt.append("You are a document re-ranker for the Java learning assistant system.\n"); + prompt.append("Reorder the following documents by relevance to the query.\n"); + prompt.append("Consider Java-specific context, version relevance, and learning value.\n"); + prompt.append("Return JSON: {\"order\":[indices...]} with 0-based indices.\n\n"); + prompt.append("Query: ").append(query).append("\n\n"); for (int docIndex = 0; docIndex < docs.size(); docIndex++) { Document document = docs.get(docIndex); @@ -127,16 +109,15 @@ private String buildRerankPrompt(String query, List docs) { String title = extractMetadataString(metadata, "title"); String url = extractMetadataString(metadata, "url"); String text = document.getText(); - prompt - .append("[") - .append(docIndex) - .append("] ") - .append(title) - .append(" | ") - .append(url) - .append("\n") - .append(trim(text == null ? "" : text, 500)) - .append("\n\n"); + prompt.append("[") + .append(docIndex) + .append("] ") + .append(title) + .append(" | ") + .append(url) + .append("\n") + .append(trim(text == null ? "" : text, 500)) + .append("\n\n"); } return prompt.toString(); @@ -145,10 +126,7 @@ private String buildRerankPrompt(String query, List docs) { /** * Parse the LLM response to extract document ordering. */ - private List parseRerankResponse( - String response, - List docs - ) throws JsonProcessingException { + private List parseRerankResponse(String response, List docs) throws JsonProcessingException { List reordered = new ArrayList<>(); RerankOrderResponse orderResponse = parseRerankOrderResponse(response); if (orderResponse == null || orderResponse.order() == null) { @@ -248,7 +226,8 @@ private String findFirstJsonObject(String text) { } @JsonIgnoreProperties(ignoreUnknown = true) - private record RerankOrderResponse(@JsonProperty("order") List order) {} + private record RerankOrderResponse( + @JsonProperty("order") List order) {} /** * Limit document list to returnK elements. diff --git a/src/main/java/com/williamcallahan/javachat/service/RetrievalService.java b/src/main/java/com/williamcallahan/javachat/service/RetrievalService.java index 3d39e9d..412ccd3 100644 --- a/src/main/java/com/williamcallahan/javachat/service/RetrievalService.java +++ b/src/main/java/com/williamcallahan/javachat/service/RetrievalService.java @@ -24,9 +24,7 @@ @Service public class RetrievalService { - private static final Logger log = LoggerFactory.getLogger( - RetrievalService.class - ); + private static final Logger log = LoggerFactory.getLogger(RetrievalService.class); /** Max chars for first document preview in debug logs */ private static final int DEBUG_FIRST_DOC_PREVIEW_LENGTH = 200; @@ -60,12 +58,11 @@ public class RetrievalService { * @param documentFactory document factory for metadata preservation */ public RetrievalService( - VectorStore vectorStore, - AppProperties props, - RerankerService rerankerService, - LocalSearchService localSearch, - DocumentFactory documentFactory - ) { + VectorStore vectorStore, + AppProperties props, + RerankerService rerankerService, + LocalSearchService localSearch, + DocumentFactory documentFactory) { this.vectorStore = vectorStore; this.props = props; this.rerankerService = rerankerService; @@ -129,11 +126,15 @@ public RetrievalOutcome retrieveOutcome(String query) { docs = executeVersionAwareSearch(query, boostedQuery, versionFilter, baseTopK); } catch (RuntimeException exception) { return handleVectorSearchFailureOutcome( - exception, query, notices, props.getRag().getSearchTopK(), props.getRag().getSearchReturnK() - ); + exception, + query, + notices, + props.getRag().getSearchTopK(), + props.getRag().getSearchReturnK()); } - List reranked = rerankWithDiagnostics(query, docs, props.getRag().getSearchReturnK(), notices); + List reranked = + rerankWithDiagnostics(query, docs, props.getRag().getSearchReturnK(), notices); return new RetrievalOutcome(reranked, notices); } @@ -141,22 +142,14 @@ public RetrievalOutcome retrieveOutcome(String query) { * Retrieve documents with custom limits for token-constrained models. * Used for GPT-5.2 which has an 8K input token limit. */ - public List retrieveWithLimit( - String query, - int maxDocs, - int maxTokensPerDoc - ) { + public List retrieveWithLimit(String query, int maxDocs, int maxTokensPerDoc) { return retrieveWithLimitOutcome(query, maxDocs, maxTokensPerDoc).documents(); } /** * Retrieves documents with token limits and returns diagnostic notices. */ - public RetrievalOutcome retrieveWithLimitOutcome( - String query, - int maxDocs, - int maxTokensPerDoc - ) { + public RetrievalOutcome retrieveWithLimitOutcome(String query, int maxDocs, int maxTokensPerDoc) { List notices = new ArrayList<>(); if (query == null || query.isBlank()) { return new RetrievalOutcome(List.of(), notices); @@ -171,10 +164,7 @@ public RetrievalOutcome retrieveWithLimitOutcome( // Initial vector search with custom topK List docs; try { - int baseTopK = Math.max( - 1, - Math.max(maxDocs, props.getRag().getSearchTopK()) - ); + int baseTopK = Math.max(1, Math.max(maxDocs, props.getRag().getSearchTopK())); docs = executeVersionAwareSearch(query, boostedQuery, versionFilter, baseTopK); } catch (RuntimeException exception) { return executeFallbackSearchOutcome(query, maxDocs, exception, notices); @@ -186,22 +176,17 @@ public RetrievalOutcome retrieveWithLimitOutcome( } // Truncate documents to token limits and return limited count - List truncatedDocs = docs - .stream() - .limit(maxDocs) - .map(doc -> truncateDocumentToTokenLimit(doc, maxTokensPerDoc)) - .collect(Collectors.toList()); + List truncatedDocs = docs.stream() + .limit(maxDocs) + .map(doc -> truncateDocumentToTokenLimit(doc, maxTokensPerDoc)) + .collect(Collectors.toList()); List reranked = rerankWithDiagnostics(query, truncatedDocs, maxDocs, notices); return new RetrievalOutcome(reranked, notices); } private List executeVersionAwareSearch( - String query, - String boostedQuery, - Optional versionFilter, - int baseTopK - ) { + String query, String boostedQuery, Optional versionFilter, int baseTopK) { // Fetch more candidates when version filtering is active int topK = versionFilter.isPresent() ? baseTopK * 2 : baseTopK; @@ -210,10 +195,8 @@ private List executeVersionAwareSearch( } log.info("TopK requested: {}", topK); - SearchRequest searchRequest = SearchRequest.builder() - .query(boostedQuery) - .topK(topK) - .build(); + SearchRequest searchRequest = + SearchRequest.builder().query(boostedQuery).topK(topK).build(); List docs = vectorStore.similaritySearch(searchRequest); log.info("VectorStore returned {} documents", docs.size()); @@ -222,24 +205,26 @@ private List executeVersionAwareSearch( if (versionFilter.isPresent()) { VersionFilterPatterns filter = versionFilter.get(); List versionMatchedDocs = docs.stream() - .filter(doc -> filter.matchesUrl(String.valueOf(doc.getMetadata().get(METADATA_URL)))) - .collect(Collectors.toList()); + .filter(doc -> + filter.matchesUrl(String.valueOf(doc.getMetadata().get(METADATA_URL)))) + .collect(Collectors.toList()); - log.info("Version filter matched {} of {} documents", - versionMatchedDocs.size(), docs.size()); + log.info("Version filter matched {} of {} documents", versionMatchedDocs.size(), docs.size()); // Use version-matched docs if we have enough, otherwise fall back to all docs if (versionMatchedDocs.size() >= 2) { docs = versionMatchedDocs; } else { - log.info("Insufficient version-specific docs ({}), using all {} candidates", - versionMatchedDocs.size(), docs.size()); + log.info( + "Insufficient version-specific docs ({}), using all {} candidates", + versionMatchedDocs.size(), + docs.size()); } } if (!docs.isEmpty()) { Map metadata = docs.get(0).getMetadata(); - int metadataSize = metadata == null ? 0 : metadata.size(); + int metadataSize = metadata.size(); String docText = Optional.ofNullable(docs.get(0).getText()).orElse(""); int previewLength = Math.min(DEBUG_FIRST_DOC_PREVIEW_LENGTH, docText.length()); log.info("First doc metadata size: {}", metadataSize); @@ -281,16 +266,9 @@ private Document truncateDocumentToTokenLimit(Document doc, int maxTokens) { // Create new document with truncated content, preserving all original metadata // and adding truncation-specific metadata - Map truncationMetadata = Map.of( - "truncated", true, - "originalLength", content.length() - ); - - return documentFactory.createWithPreservedMetadata( - truncated, - doc.getMetadata(), - truncationMetadata - ); + Map truncationMetadata = Map.of("truncated", true, "originalLength", content.length()); + + return documentFactory.createWithPreservedMetadata(truncated, doc.getMetadata(), truncationMetadata); } /** @@ -310,19 +288,11 @@ public List toCitations(List docs) { String title = stringMetadataValue(metadata, METADATA_TITLE); String url = DocsSourceRegistry.normalizeDocUrl(rawUrl); // Refine Javadoc URLs to nested type pages where the chunk references them - url = - com.williamcallahan.javachat.util.JavadocLinkResolver.refineNestedTypeUrl( - url, - sourceDoc.getText() - ); + url = com.williamcallahan.javachat.util.JavadocLinkResolver.refineNestedTypeUrl(url, sourceDoc.getText()); // Append member anchors (methods/constructors) when confidently derivable String pkg = stringMetadataValue(metadata, METADATA_PACKAGE); - url = - com.williamcallahan.javachat.util.JavadocLinkResolver.refineMemberAnchorUrl( - url, - sourceDoc.getText(), - pkg - ); + url = com.williamcallahan.javachat.util.JavadocLinkResolver.refineMemberAnchorUrl( + url, sourceDoc.getText(), pkg); // Final canonicalization in case of any accidental duplications if (url.startsWith("http://") || url.startsWith("https://")) { url = DocsSourceRegistry.canonicalizeHttpDocUrl(url); @@ -331,16 +301,13 @@ public List toCitations(List docs) { // For book sources, we now link to public /pdfs path (handled by normalizeCitationUrl) - citations.add( - new Citation( + citations.add(new Citation( url, title, "", snippet.length() > CITATION_SNIPPET_MAX_LENGTH - ? snippet.substring(0, CITATION_SNIPPET_MAX_LENGTH) + "…" - : snippet - ) - ); + ? snippet.substring(0, CITATION_SNIPPET_MAX_LENGTH) + "…" + : snippet)); } return citations; } @@ -354,20 +321,12 @@ private static String stringMetadataValue(Map metadata, String key) { } private RetrievalOutcome handleVectorSearchFailureOutcome( - RuntimeException exception, - String query, - List notices, - int maxDocs, - int returnLimit - ) { + RuntimeException exception, String query, List notices, int maxDocs, int returnLimit) { String errorType = RetrievalErrorClassifier.determineErrorType(exception); log.warn("Vector search unavailable; falling back to local keyword search"); RetrievalErrorClassifier.logUserFriendlyErrorContext(log, errorType, exception); - notices.add(new RetrievalNotice( - "Vector search failed; using keyword fallback", - describeFailure(exception) - )); + notices.add(new RetrievalNotice("Vector search failed; using keyword fallback", describeFailure(exception))); // Use searchTopK (not searchReturnK) to match the primary path's candidate pool size LocalSearchService.SearchOutcome outcome = localSearch.search(query, maxDocs); @@ -375,66 +334,45 @@ private RetrievalOutcome handleVectorSearchFailureOutcome( if (outcome.isFailed()) { String outcomeMessage = outcome.errorMessage().orElse("Local keyword search failed"); log.error("Local keyword search also failed - returning empty hits"); - notices.add(new RetrievalNotice( - "Keyword fallback failed; returning empty results", - outcomeMessage - )); + notices.add(new RetrievalNotice("Keyword fallback failed; returning empty results", outcomeMessage)); return new RetrievalOutcome(List.of(), notices); } log.info("Local keyword search returned {} hits", outcome.hits().size()); - List fallbackDocs = outcome.hits() - .stream() - .map(hit -> { - Document doc = documentFactory.createLocalDocument(hit.text(), hit.url()); - // Mark document as coming from fallback search - doc.getMetadata().put(METADATA_RETRIEVAL_SOURCE, SOURCE_KEYWORD_FALLBACK); - doc.getMetadata().put(METADATA_FALLBACK_REASON, errorType); - return doc; - }) - .limit(returnLimit) - .collect(Collectors.toList()); + List fallbackDocs = outcome.hits().stream() + .map(hit -> { + Document doc = documentFactory.createLocalDocument(hit.text(), hit.url()); + // Mark document as coming from fallback search + doc.getMetadata().put(METADATA_RETRIEVAL_SOURCE, SOURCE_KEYWORD_FALLBACK); + doc.getMetadata().put(METADATA_FALLBACK_REASON, errorType); + return doc; + }) + .limit(returnLimit) + .collect(Collectors.toList()); return new RetrievalOutcome(fallbackDocs, notices); } private RetrievalOutcome executeFallbackSearchOutcome( - String query, - int maxDocs, - RuntimeException exception, - List notices - ) { + String query, int maxDocs, RuntimeException exception, List notices) { return handleVectorSearchFailureOutcome(exception, query, notices, maxDocs, maxDocs); } private List rerankWithDiagnostics( - String query, - List docs, - int returnLimit, - List notices - ) { - List uniqueByUrl = docs - .stream() - .collect( - Collectors.toMap( - doc -> String.valueOf(doc.getMetadata().get(METADATA_URL)), - doc -> doc, - (first, dup) -> first - ) - ) - .values() - .stream() - .collect(Collectors.toList()); + String query, List docs, int returnLimit, List notices) { + List uniqueByUrl = docs.stream() + .collect(Collectors.toMap( + doc -> String.valueOf(doc.getMetadata().get(METADATA_URL)), doc -> doc, (first, dup) -> first)) + .values() + .stream() + .collect(Collectors.toList()); List reranked; try { reranked = rerankerService.rerank(query, uniqueByUrl, returnLimit); } catch (RerankingFailureException exception) { log.error("Reranking failed; returning original order", exception); - notices.add(new RetrievalNotice( - "Reranking failed; using original order", - describeFailure(exception) - )); + notices.add(new RetrievalNotice("Reranking failed; using original order", describeFailure(exception))); reranked = uniqueByUrl; } @@ -456,5 +394,4 @@ private String describeFailure(Throwable exception) { } return exception.getClass().getSimpleName() + ": " + message; } - } diff --git a/src/main/java/com/williamcallahan/javachat/service/TooltipRegistry.java b/src/main/java/com/williamcallahan/javachat/service/TooltipRegistry.java index 8ef5506..e6b46c5 100644 --- a/src/main/java/com/williamcallahan/javachat/service/TooltipRegistry.java +++ b/src/main/java/com/williamcallahan/javachat/service/TooltipRegistry.java @@ -1,7 +1,7 @@ package com.williamcallahan.javachat.service; -import org.springframework.stereotype.Component; import java.util.*; +import org.springframework.stereotype.Component; /** * Provides a curated glossary of Java terms for UI tooltips. @@ -61,14 +61,38 @@ public String getLink() { */ public TooltipRegistry() { // Seed with top Java terms (expand as needed) - add("primitive", "A basic value type like int, double, boolean stored directly (not an object).", "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/datatypes.html"); - add("reference", "A value that points to an object on the heap rather than containing the value directly.", "https://docs.oracle.com/javase/specs/"); - add("variable", "A named storage location with a declared type that determines the kind of data it can hold.", "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html"); - add("assignment", "The operation of storing a value into a variable using the = operator.", "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/op1.html"); - add("autoboxing", "Automatic conversion between primitives and their wrapper types (e.g., int ↔ Integer).", "https://docs.oracle.com/javase/tutorial/java/data/autoboxing.html"); - add("immutable", "An object whose state cannot be changed after creation (e.g., String).", "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/strings.html"); - add("record", "A special class that provides a compact syntax for immutable data carriers.", "https://docs.oracle.com/en/java/javase/24/language/records.html"); - add("optional", "A container object which may or may not contain a non-null value, used to avoid null checks.", "https://docs.oracle.com/javase/8/docs/api/java/util/Optional.html"); + add( + "primitive", + "A basic value type like int, double, boolean stored directly (not an object).", + "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/datatypes.html"); + add( + "reference", + "A value that points to an object on the heap rather than containing the value directly.", + "https://docs.oracle.com/javase/specs/"); + add( + "variable", + "A named storage location with a declared type that determines the kind of data it can hold.", + "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html"); + add( + "assignment", + "The operation of storing a value into a variable using the = operator.", + "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/op1.html"); + add( + "autoboxing", + "Automatic conversion between primitives and their wrapper types (e.g., int ↔ Integer).", + "https://docs.oracle.com/javase/tutorial/java/data/autoboxing.html"); + add( + "immutable", + "An object whose state cannot be changed after creation (e.g., String).", + "https://docs.oracle.com/javase/tutorial/java/nutsandbolts/strings.html"); + add( + "record", + "A special class that provides a compact syntax for immutable data carriers.", + "https://docs.oracle.com/en/java/javase/24/language/records.html"); + add( + "optional", + "A container object which may or may not contain a non-null value, used to avoid null checks.", + "https://docs.oracle.com/javase/8/docs/api/java/util/Optional.html"); } private void add(String term, String definition, String link) { diff --git a/src/main/java/com/williamcallahan/javachat/service/markdown/CitationProcessor.java b/src/main/java/com/williamcallahan/javachat/service/markdown/CitationProcessor.java index d1561c8..06ded1d 100644 --- a/src/main/java/com/williamcallahan/javachat/service/markdown/CitationProcessor.java +++ b/src/main/java/com/williamcallahan/javachat/service/markdown/CitationProcessor.java @@ -7,25 +7,24 @@ import com.vladsch.flexmark.util.ast.VisitHandler; import com.williamcallahan.javachat.domain.markdown.CitationType; import com.williamcallahan.javachat.domain.markdown.MarkdownCitation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; import java.util.Locale; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * AST-based citation processor that replaces regex-based citation extraction. * Uses Flexmark's visitor pattern for reliable parsing. */ public class CitationProcessor { - + private static final Logger logger = LoggerFactory.getLogger(CitationProcessor.class); - + /** * Extracts citations from a Flexmark AST document. * This replaces regex-based citation processing with structured AST traversal. - * + * * @param document the parsed markdown document * @return list of extracted citations */ @@ -33,16 +32,16 @@ public List extractCitations(Node document) { if (document == null) { return List.of(); } - + CitationVisitor visitor = new CitationVisitor(); visitor.visit(document); - + List citations = visitor.citations(); logger.debug("Extracted {} citations using AST processing", citations.size()); - + return citations; } - + /** * Visitor implementation for extracting citations from AST nodes. * This is the AGENTS.md compliant approach using proper AST traversal. @@ -50,20 +49,18 @@ public List extractCitations(Node document) { private static class CitationVisitor { private final List citations = new ArrayList<>(); private int position = 0; - + private final NodeVisitor visitor = new NodeVisitor( - new VisitHandler<>(Link.class, this::visitLink), - new VisitHandler<>(Text.class, this::visitText) - ); - + new VisitHandler<>(Link.class, this::visitLink), new VisitHandler<>(Text.class, this::visitText)); + void visit(Node node) { visitor.visit(node); } - + List citations() { return List.copyOf(citations); } - + /** * Processes Link nodes to extract citation information. * @param link the link node to process @@ -71,18 +68,18 @@ List citations() { private void visitLink(Link link) { String url = link.getUrl().toString(); CitationType type = CitationType.fromUrl(url); - + String title = extractLinkTitle(link); if (isValidCitation(url)) { MarkdownCitation citation = MarkdownCitation.create(url, title, "", type, position++); citations.add(citation); logger.debug("Found citation at position {}", citation.position()); } - + // Continue visiting child nodes visitor.visitChildren(link); } - + /** * Advances position counter through text content to maintain citation ordering. * Citations extracted from links receive positions reflecting their document order. @@ -90,7 +87,7 @@ private void visitLink(Link link) { private void visitText(Text text) { position += text.getChars().length(); } - + /** * Extracts title from a link node, preferring explicit title over link text. * @param link the link node @@ -101,7 +98,7 @@ private String extractLinkTitle(Link link) { if (link.getTitle().isNotNull() && !link.getTitle().isEmpty()) { return link.getTitle().toString(); } - + // Fall back to link text content StringBuilder titleBuilder = new StringBuilder(); Node child = link.getFirstChild(); @@ -111,11 +108,11 @@ private String extractLinkTitle(Link link) { } child = child.getNext(); } - + String title = titleBuilder.toString().trim(); return title.isEmpty() ? "Source" : title; } - + /** * Validates if a URL and title constitute a valid citation. * @param url the URL to validate @@ -126,12 +123,12 @@ private boolean isValidCitation(String url) { if (url == null || url.trim().isEmpty()) { return false; } - + // Skip common non-citation links String lowerUrl = url.toLowerCase(Locale.ROOT); return !lowerUrl.startsWith("mailto:") - && !lowerUrl.startsWith("tel:") - && !lowerUrl.startsWith("javascript:"); + && !lowerUrl.startsWith("tel:") + && !lowerUrl.startsWith("javascript:"); } } } diff --git a/src/main/java/com/williamcallahan/javachat/service/markdown/CodeFenceStateTracker.java b/src/main/java/com/williamcallahan/javachat/service/markdown/CodeFenceStateTracker.java index a2259c9..8c301b5 100644 --- a/src/main/java/com/williamcallahan/javachat/service/markdown/CodeFenceStateTracker.java +++ b/src/main/java/com/williamcallahan/javachat/service/markdown/CodeFenceStateTracker.java @@ -103,7 +103,7 @@ static boolean hasClosingBacktickRun(String text, int startIndex, int runLength) } int matchLength = 0; while (nextBacktickIndex + matchLength < text.length() - && text.charAt(nextBacktickIndex + matchLength) == BACKTICK) { + && text.charAt(nextBacktickIndex + matchLength) == BACKTICK) { matchLength++; } if (matchLength == runLength) { diff --git a/src/main/java/com/williamcallahan/javachat/service/markdown/EnrichmentPlaceholderizer.java b/src/main/java/com/williamcallahan/javachat/service/markdown/EnrichmentPlaceholderizer.java index 5bb5041..b352f38 100644 --- a/src/main/java/com/williamcallahan/javachat/service/markdown/EnrichmentPlaceholderizer.java +++ b/src/main/java/com/williamcallahan/javachat/service/markdown/EnrichmentPlaceholderizer.java @@ -10,14 +10,13 @@ import com.williamcallahan.javachat.domain.markdown.Reminder; import com.williamcallahan.javachat.domain.markdown.Warning; import com.williamcallahan.javachat.support.AsciiTextNormalizer; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; /** * Extracts inline enrichment markers and replaces them with HTML placeholders. @@ -35,11 +34,26 @@ class EnrichmentPlaceholderizer { * Defines supported enrichment marker kinds and their rendering metadata. */ private enum EnrichmentKind { - HINT("hint", "Helpful Hints", ""), - BACKGROUND("background", "Background Context", ""), - REMINDER("reminder", "Important Reminders", ""), - WARNING("warning", "Warning", ""), - EXAMPLE("example", "Example", ""); + HINT( + "hint", + "Helpful Hints", + ""), + BACKGROUND( + "background", + "Background Context", + ""), + REMINDER( + "reminder", + "Important Reminders", + ""), + WARNING( + "warning", + "Warning", + ""), + EXAMPLE( + "example", + "Example", + ""); private final String token; private final String title; @@ -75,7 +89,6 @@ static Optional fromToken(String rawToken) { } return Optional.empty(); } - } EnrichmentPlaceholderizer(Parser parser, HtmlRenderer renderer) { @@ -84,17 +97,13 @@ static Optional fromToken(String rawToken) { } private record EnrichmentContext( - String markdown, - List enrichments, - Map placeholders, - StringBuilder outputBuilder - ) {} + String markdown, + List enrichments, + Map placeholders, + StringBuilder outputBuilder) {} String extractAndPlaceholderizeEnrichments( - String markdown, - List enrichments, - Map placeholders - ) { + String markdown, List enrichments, Map placeholders) { if (markdown == null || markdown.isEmpty()) { return markdown; } @@ -147,12 +156,7 @@ String extractAndPlaceholderizeEnrichments( EnrichmentParseResult parseResult = parseEnrichmentMarker(markdown, cursor); if (parseResult.isValid()) { EnrichmentProcessingResult processingResult = processEnrichment( - context, - cursor, - parseResult.contentStartIndex(), - parseResult.kind(), - absolutePosition - ); + context, cursor, parseResult.contentStartIndex(), parseResult.kind(), absolutePosition); if (processingResult != null) { cursor = processingResult.nextIndex(); @@ -254,7 +258,8 @@ private int findEnrichmentEndIndex(String markdown, int startIndex) { } if (!fenceTracker.isInsideFence()) { - CodeFenceStateTracker.BacktickRun backtickRun = CodeFenceStateTracker.scanBacktickRun(markdown, scanIndex); + CodeFenceStateTracker.BacktickRun backtickRun = + CodeFenceStateTracker.scanBacktickRun(markdown, scanIndex); if (backtickRun != null) { fenceTracker.processCharacter(markdown, scanIndex, isStartOfLine); scanIndex += backtickRun.length(); @@ -281,18 +286,18 @@ private MarkdownEnrichment createEnrichment(EnrichmentKind kind, String content, } private EnrichmentProcessingResult processEnrichment( - EnrichmentContext context, - int openingIndex, - int contentStartIndex, - EnrichmentKind kind, - int absolutePosition - ) { + EnrichmentContext context, + int openingIndex, + int contentStartIndex, + EnrichmentKind kind, + int absolutePosition) { int closingIndex = findEnrichmentEndIndex(context.markdown, contentStartIndex); if (closingIndex == -1) { return null; } - String content = context.markdown.substring(contentStartIndex, closingIndex).trim(); + String content = + context.markdown.substring(contentStartIndex, closingIndex).trim(); int consumedLength = (closingIndex + 2) - openingIndex; if (content.isEmpty()) { @@ -301,7 +306,7 @@ private EnrichmentProcessingResult processEnrichment( MarkdownEnrichment enrichment = createEnrichment(kind, content, absolutePosition); context.enrichments.add(enrichment); - + String placeholderId = PLACEHOLDER_PREFIX + UUID.randomUUID().toString().replace("-", ""); context.placeholders.put(placeholderId, buildEnrichmentHtmlUnified(kind, content)); context.outputBuilder.append(placeholderId); @@ -348,7 +353,7 @@ private EnrichmentParseResult parseEnrichmentMarker(String markdown, int markerS String rawToken = markdown.substring(markerStart + MARKER_START.length(), colonIndex); return EnrichmentKind.fromToken(rawToken) - .map(kind -> EnrichmentParseResult.of(kind, colonIndex + 1)) - .orElse(EnrichmentParseResult.invalid()); + .map(kind -> EnrichmentParseResult.of(kind, colonIndex + 1)) + .orElse(EnrichmentParseResult.invalid()); } } diff --git a/src/main/java/com/williamcallahan/javachat/service/markdown/InlineListParser.java b/src/main/java/com/williamcallahan/javachat/service/markdown/InlineListParser.java index d191a65..96d4f1a 100644 --- a/src/main/java/com/williamcallahan/javachat/service/markdown/InlineListParser.java +++ b/src/main/java/com/williamcallahan/javachat/service/markdown/InlineListParser.java @@ -1,8 +1,8 @@ package com.williamcallahan.javachat.service.markdown; -import org.jsoup.nodes.Element; import java.util.ArrayList; import java.util.List; +import org.jsoup.nodes.Element; /** * Parses inline list markers from flat text into structured list elements. @@ -82,11 +82,10 @@ private static List renderNestedListsRecursively(Parse parse, int depth * @param trailingText text after the last list item */ record Conversion( - String leadingText, - Element primaryListElement, - List additionalListElements, - String trailingText - ) {} + String leadingText, + Element primaryListElement, + List additionalListElements, + String trailingText) {} /** * Represents a parsed inline list with its items and structure. @@ -104,12 +103,7 @@ record Block(String tagName, List entryLabels) {} * @param nestedSegments segments that may contain nested lists * @param trailingText text after the last item */ - record Parse( - String leadingText, - Block primaryBlock, - List nestedSegments, - String trailingText - ) { + record Parse(String leadingText, Block primaryBlock, List nestedSegments, String trailingText) { static Parse tryParse(String input) { if (input == null) return null; String text = input.strip(); @@ -142,16 +136,20 @@ private static Parse parseInlineListOrderedKind(String text, InlineListOrderedKi String trailingText = ""; for (int markerIndex = 0; markerIndex < markers.size(); markerIndex++) { int contentStart = markers.get(markerIndex).contentStartIndex(); - int nextMarkerStart = markerIndex + 1 < markers.size() ? markers.get(markerIndex + 1).markerStartIndex() : text.length(); - String rawEntryText = text.substring(contentStart, nextMarkerStart).trim(); + int nextMarkerStart = markerIndex + 1 < markers.size() + ? markers.get(markerIndex + 1).markerStartIndex() + : text.length(); + String rawEntryText = + text.substring(contentStart, nextMarkerStart).trim(); if (rawEntryText.isEmpty()) continue; boolean isLastMarker = markerIndex == markers.size() - 1; - EntryTextSplit entryTextSplit = isLastMarker ? extractTrailingText(rawEntryText) - : new EntryTextSplit(rawEntryText, ""); + EntryTextSplit entryTextSplit = + isLastMarker ? extractTrailingText(rawEntryText) : new EntryTextSplit(rawEntryText, ""); ParsedEntry parsedEntry = splitNestedList(entryTextSplit.entryText()); entryLabels.add(parsedEntry.label()); - if (parsedEntry.nestedSegment() != null && !parsedEntry.nestedSegment().isBlank()) { + if (parsedEntry.nestedSegment() != null + && !parsedEntry.nestedSegment().isBlank()) { nestedSegments.add(parsedEntry.nestedSegment()); } if (!entryTextSplit.trailingText().isBlank()) { @@ -180,15 +178,19 @@ private static Parse tryParseBulleted(String text) { String trailingText = ""; for (int markerIndex = 0; markerIndex < markers.size(); markerIndex++) { int contentStart = markers.get(markerIndex).contentStartIndex(); - int nextMarkerStart = markerIndex + 1 < markers.size() ? markers.get(markerIndex + 1).markerStartIndex() : text.length(); - String rawEntryText = text.substring(contentStart, nextMarkerStart).trim(); + int nextMarkerStart = markerIndex + 1 < markers.size() + ? markers.get(markerIndex + 1).markerStartIndex() + : text.length(); + String rawEntryText = + text.substring(contentStart, nextMarkerStart).trim(); if (rawEntryText.isEmpty()) continue; boolean isLastMarker = markerIndex == markers.size() - 1; - EntryTextSplit entryTextSplit = isLastMarker ? extractTrailingText(rawEntryText) - : new EntryTextSplit(rawEntryText, ""); + EntryTextSplit entryTextSplit = + isLastMarker ? extractTrailingText(rawEntryText) : new EntryTextSplit(rawEntryText, ""); ParsedEntry parsedEntry = splitNestedList(entryTextSplit.entryText()); entryLabels.add(parsedEntry.label()); - if (parsedEntry.nestedSegment() != null && !parsedEntry.nestedSegment().isBlank()) { + if (parsedEntry.nestedSegment() != null + && !parsedEntry.nestedSegment().isBlank()) { nestedSegments.add(parsedEntry.nestedSegment()); } if (!entryTextSplit.trailingText().isBlank()) { @@ -237,15 +239,16 @@ private static boolean isBulletListIntro(String text, int markerIndex) { if (markerIndex == 0) return true; char previousChar = text.charAt(markerIndex - 1); return previousChar == ':' - || previousChar == '\n' - || (previousChar == ' ' && markerIndex >= COLON_BACKTRACK_OFFSET - && text.charAt(markerIndex - COLON_BACKTRACK_OFFSET) == ':'); + || previousChar == '\n' + || (previousChar == ' ' + && markerIndex >= COLON_BACKTRACK_OFFSET + && text.charAt(markerIndex - COLON_BACKTRACK_OFFSET) == ':'); } private static boolean isBulletMarker(String text, int markerIndex, InlineListBulletKind kind) { return text.charAt(markerIndex) == kind.markerChar() - && markerIndex + 1 < text.length() - && text.charAt(markerIndex + 1) == ' '; + && markerIndex + 1 < text.length() + && text.charAt(markerIndex + 1) == ' '; } private static List findBulletMarkers(String text, InlineListBulletKind kind) { @@ -286,7 +289,8 @@ private static Marker tryReadOrderedMarkerAt(String text, int index, InlineListO if (index < 0 || index >= text.length()) return null; if (!isMarkerBoundary(text, index)) return null; - OrderedMarkerScanner.MarkerMatch match = OrderedMarkerScanner.scanAt(text, index, kind).orElse(null); + OrderedMarkerScanner.MarkerMatch match = + OrderedMarkerScanner.scanAt(text, index, kind).orElse(null); if (match == null) return null; if (!isContentStartValid(text, match.afterIndex())) return null; @@ -355,7 +359,7 @@ private static int findTrailingTextStart(String rawEntryText) { int candidateStart = index + 1; boolean sawWhitespace = false; while (candidateStart < rawEntryText.length() - && Character.isWhitespace(rawEntryText.charAt(candidateStart))) { + && Character.isWhitespace(rawEntryText.charAt(candidateStart))) { sawWhitespace = true; candidateStart++; } diff --git a/src/main/java/com/williamcallahan/javachat/service/markdown/MarkdownNormalizer.java b/src/main/java/com/williamcallahan/javachat/service/markdown/MarkdownNormalizer.java index 600873c..aaddb5a 100644 --- a/src/main/java/com/williamcallahan/javachat/service/markdown/MarkdownNormalizer.java +++ b/src/main/java/com/williamcallahan/javachat/service/markdown/MarkdownNormalizer.java @@ -16,7 +16,7 @@ static String preNormalizeForListsAndFences(String markdownText) { StringBuilder normalizedBuilder = new StringBuilder(markdownText.length() + 64); CodeFenceStateTracker fenceTracker = new CodeFenceStateTracker(); - for (int cursor = 0; cursor < markdownText.length();) { + for (int cursor = 0; cursor < markdownText.length(); ) { boolean isStartOfLine = cursor == 0 || markdownText.charAt(cursor - 1) == '\n'; CodeFenceStateTracker.FenceMarker fenceMarker = CodeFenceStateTracker.scanFenceMarker(markdownText, cursor); boolean isAttachedFenceStart = isAttachedFenceStart(markdownText, cursor); @@ -59,7 +59,8 @@ static String preNormalizeForListsAndFences(String markdownText) { } } if (!fenceTracker.isInsideFence()) { - CodeFenceStateTracker.BacktickRun backtickRun = CodeFenceStateTracker.scanBacktickRun(markdownText, cursor); + CodeFenceStateTracker.BacktickRun backtickRun = + CodeFenceStateTracker.scanBacktickRun(markdownText, cursor); if (backtickRun != null) { fenceTracker.processCharacter(markdownText, cursor, isStartOfLine); appendBacktickRun(normalizedBuilder, markdownText, cursor, backtickRun.length()); @@ -72,8 +73,8 @@ static String preNormalizeForListsAndFences(String markdownText) { } if (fenceTracker.isInsideFence()) { char closingChar = fenceTracker.getFenceChar() == 0 - ? CodeFenceStateTracker.DEFAULT_FENCE_CHAR - : fenceTracker.getFenceChar(); + ? CodeFenceStateTracker.DEFAULT_FENCE_CHAR + : fenceTracker.getFenceChar(); int closingLength = Math.max(CodeFenceStateTracker.FENCE_MIN_LENGTH, fenceTracker.getFenceLength()); normalizedBuilder.append('\n').append(String.valueOf(closingChar).repeat(closingLength)); } @@ -171,8 +172,8 @@ private static boolean isNumericHeader(String trimmedLine) { return digitIndex + 1 < trimmedLine.length() && trimmedLine.charAt(digitIndex + 1) == ' '; } - private static void appendFenceMarker(StringBuilder builder, CodeFenceStateTracker.FenceMarker marker, - String text, int index) { + private static void appendFenceMarker( + StringBuilder builder, CodeFenceStateTracker.FenceMarker marker, String text, int index) { for (int offset = 0; offset < marker.length(); offset++) { builder.append(text.charAt(index + offset)); } diff --git a/src/main/java/com/williamcallahan/javachat/service/markdown/OrderedMarkerScanner.java b/src/main/java/com/williamcallahan/javachat/service/markdown/OrderedMarkerScanner.java index 3385f28..90636d9 100644 --- a/src/main/java/com/williamcallahan/javachat/service/markdown/OrderedMarkerScanner.java +++ b/src/main/java/com/williamcallahan/javachat/service/markdown/OrderedMarkerScanner.java @@ -71,7 +71,7 @@ static boolean startsWithOrderedMarker(String trimmedLine) { int posAfterDelimiter = cursor + 1; // Valid if: at end of line, or next char is whitespace return posAfterDelimiter >= trimmedLine.length() - || Character.isWhitespace(trimmedLine.charAt(posAfterDelimiter)); + || Character.isWhitespace(trimmedLine.charAt(posAfterDelimiter)); } } return false; @@ -130,7 +130,8 @@ private static MarkerStrategy strategyFor(InlineListOrderedKind kind) { }; } - private static MarkerMatch finalizeMarker(String text, int startIndex, int sequenceEnd, InlineListOrderedKind kind) { + private static MarkerMatch finalizeMarker( + String text, int startIndex, int sequenceEnd, InlineListOrderedKind kind) { if (sequenceEnd >= text.length()) return null; char markerChar = text.charAt(sequenceEnd); @@ -138,8 +139,8 @@ private static MarkerMatch finalizeMarker(String text, int startIndex, int seque // Reject version numbers like 1.8 for numeric markers if (kind == InlineListOrderedKind.NUMERIC - && sequenceEnd + 1 < text.length() - && Character.isDigit(text.charAt(sequenceEnd + 1))) { + && sequenceEnd + 1 < text.length() + && Character.isDigit(text.charAt(sequenceEnd + 1))) { return null; } @@ -154,8 +155,7 @@ private static MarkerMatch finalizeMarker(String text, int startIndex, int seque private static int readNumericSequence(String text, int index) { int cursor = index; int digitCount = 0; - while (cursor < text.length() && Character.isDigit(text.charAt(cursor)) - && digitCount < MAX_NUMERIC_DIGITS) { + while (cursor < text.length() && Character.isDigit(text.charAt(cursor)) && digitCount < MAX_NUMERIC_DIGITS) { digitCount++; cursor++; } diff --git a/src/main/java/com/williamcallahan/javachat/service/markdown/UnifiedMarkdownService.java b/src/main/java/com/williamcallahan/javachat/service/markdown/UnifiedMarkdownService.java index 78fdb31..a62db94 100644 --- a/src/main/java/com/williamcallahan/javachat/service/markdown/UnifiedMarkdownService.java +++ b/src/main/java/com/williamcallahan/javachat/service/markdown/UnifiedMarkdownService.java @@ -2,36 +2,32 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; -import com.williamcallahan.javachat.domain.markdown.MarkdownCitation; -import com.williamcallahan.javachat.domain.markdown.MarkdownEnrichment; -import com.williamcallahan.javachat.domain.markdown.ProcessedMarkdown; +import com.vladsch.flexmark.ext.autolink.AutolinkExtension; +import com.vladsch.flexmark.ext.gfm.strikethrough.StrikethroughExtension; +import com.vladsch.flexmark.ext.gfm.tasklist.TaskListExtension; +import com.vladsch.flexmark.ext.tables.TablesExtension; import com.vladsch.flexmark.html.HtmlRenderer; import com.vladsch.flexmark.parser.Parser; import com.vladsch.flexmark.util.ast.Node; import com.vladsch.flexmark.util.data.MutableDataSet; -import com.vladsch.flexmark.ext.tables.TablesExtension; -import com.vladsch.flexmark.ext.gfm.strikethrough.StrikethroughExtension; -import com.vladsch.flexmark.ext.gfm.tasklist.TaskListExtension; -import com.vladsch.flexmark.ext.autolink.AutolinkExtension; - +import com.williamcallahan.javachat.domain.markdown.MarkdownCitation; +import com.williamcallahan.javachat.domain.markdown.MarkdownEnrichment; +import com.williamcallahan.javachat.domain.markdown.ProcessedMarkdown; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.time.Duration; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; - import org.springframework.stereotype.Service; /** * Unified markdown service that uses AST-based processing instead of regex. * This is the AGENTS.md compliant replacement for regex-based markdown processing. - * + * * Key improvements: * - Uses Flexmark AST visitors instead of regex for structured data extraction * - Provides type-safe citation and enrichment objects @@ -40,12 +36,12 @@ */ @Service public class UnifiedMarkdownService { - + private static final Logger logger = LoggerFactory.getLogger(UnifiedMarkdownService.class); private static final int MAX_INPUT_LENGTH = 100000; // 100KB max private static final int CACHE_SIZE = 500; private static final Duration CACHE_DURATION = Duration.ofMinutes(30); - + private final Parser parser; private final HtmlRenderer renderer; private final CitationProcessor citationProcessor; @@ -53,58 +49,59 @@ public class UnifiedMarkdownService { private final Cache processCache; // Enrichment marker parsing is handled by a streaming scanner (not regex) - + /** * Creates the unified markdown processor with AST parsing and caching. */ public UnifiedMarkdownService() { // Configure Flexmark with optimal settings MutableDataSet options = new MutableDataSet() - .set(Parser.EXTENSIONS, Arrays.asList( - TablesExtension.create(), - StrikethroughExtension.create(), - TaskListExtension.create(), - AutolinkExtension.create() - )) - .set(Parser.BLANK_LINES_IN_AST, false) - .set(Parser.HTML_BLOCK_DEEP_PARSER, false) - .set(Parser.INDENTED_CODE_NO_TRAILING_BLANK_LINES, true) - .set(HtmlRenderer.ESCAPE_HTML, true) - .set(HtmlRenderer.SUPPRESS_HTML, false) - // Convert soft-breaks (single newlines) to
tags to preserve LLM line structure. - // Matches client-side marked.js with breaks: true for consistent streaming/final render. - .set(HtmlRenderer.SOFT_BREAK, "
\n") - .set(HtmlRenderer.HARD_BREAK, "
\n") - .set(HtmlRenderer.FENCED_CODE_LANGUAGE_CLASS_PREFIX, "language-") - .set(HtmlRenderer.SUPPRESSED_LINKS, "(?i)^(javascript|data|vbscript):.*") - .set(HtmlRenderer.INDENT_SIZE, 2) - // Strict blank line control: never allow consecutive blank lines in output - .set(HtmlRenderer.MAX_BLANK_LINES, 0) - .set(HtmlRenderer.MAX_TRAILING_BLANK_LINES, 0) - .set(TablesExtension.COLUMN_SPANS, false) - .set(TablesExtension.APPEND_MISSING_COLUMNS, true) - .set(TablesExtension.DISCARD_EXTRA_COLUMNS, true) - .set(TablesExtension.HEADER_SEPARATOR_COLUMN_MATCH, true); - + .set( + Parser.EXTENSIONS, + Arrays.asList( + TablesExtension.create(), + StrikethroughExtension.create(), + TaskListExtension.create(), + AutolinkExtension.create())) + .set(Parser.BLANK_LINES_IN_AST, false) + .set(Parser.HTML_BLOCK_DEEP_PARSER, false) + .set(Parser.INDENTED_CODE_NO_TRAILING_BLANK_LINES, true) + .set(HtmlRenderer.ESCAPE_HTML, true) + .set(HtmlRenderer.SUPPRESS_HTML, false) + // Convert soft-breaks (single newlines) to
tags to preserve LLM line structure. + // Matches client-side marked.js with breaks: true for consistent streaming/final render. + .set(HtmlRenderer.SOFT_BREAK, "
\n") + .set(HtmlRenderer.HARD_BREAK, "
\n") + .set(HtmlRenderer.FENCED_CODE_LANGUAGE_CLASS_PREFIX, "language-") + .set(HtmlRenderer.SUPPRESSED_LINKS, "(?i)^(javascript|data|vbscript):.*") + .set(HtmlRenderer.INDENT_SIZE, 2) + // Strict blank line control: never allow consecutive blank lines in output + .set(HtmlRenderer.MAX_BLANK_LINES, 0) + .set(HtmlRenderer.MAX_TRAILING_BLANK_LINES, 0) + .set(TablesExtension.COLUMN_SPANS, false) + .set(TablesExtension.APPEND_MISSING_COLUMNS, true) + .set(TablesExtension.DISCARD_EXTRA_COLUMNS, true) + .set(TablesExtension.HEADER_SEPARATOR_COLUMN_MATCH, true); + this.parser = Parser.builder(options).build(); this.renderer = HtmlRenderer.builder(options).build(); this.citationProcessor = new CitationProcessor(); this.enrichmentPlaceholderizer = new EnrichmentPlaceholderizer(parser, renderer); // init - + // Initialize cache this.processCache = Caffeine.newBuilder() - .maximumSize(CACHE_SIZE) - .expireAfterWrite(CACHE_DURATION) - .recordStats() - .build(); - + .maximumSize(CACHE_SIZE) + .expireAfterWrite(CACHE_DURATION) + .recordStats() + .build(); + logger.info("UnifiedMarkdownService initialized with AST-based processing"); } - + /** * Processes markdown using AST-based approach instead of regex. * This is the main entry point for AGENTS.md compliant markdown processing. - * + * * @param markdown the markdown text to process * @return structured ProcessedMarkdown result */ @@ -126,8 +123,7 @@ public ProcessedMarkdown process(String markdown) { long startTime = System.currentTimeMillis(); if (markdown.length() > MAX_INPUT_LENGTH) { - logger.warn("Markdown input exceeds maximum length: {} > {}", - markdown.length(), MAX_INPUT_LENGTH); + logger.warn("Markdown input exceeds maximum length: {} > {}", markdown.length(), MAX_INPUT_LENGTH); markdown = markdown.substring(0, MAX_INPUT_LENGTH); } @@ -137,46 +133,49 @@ public ProcessedMarkdown process(String markdown) { // Replace enrichment markers with placeholders to prevent cross-node splits (e.g., example code fences) java.util.Map placeholders = new java.util.HashMap<>(); java.util.List placeholderEnrichments = new java.util.ArrayList<>(); - String placeholderMarkdown = enrichmentPlaceholderizer.extractAndPlaceholderizeEnrichments(markdown, placeholderEnrichments, placeholders); - + String placeholderMarkdown = enrichmentPlaceholderizer.extractAndPlaceholderizeEnrichments( + markdown, placeholderEnrichments, placeholders); + try { // Parse markdown to AST - this is the foundation of AGENTS.md compliance Node document = parser.parse(placeholderMarkdown); - + // AST-level cleanups prior to HTML rendering transformAst(document); - + // Extract structured data using AST visitors (not regex) List citations = citationProcessor.extractCitations(document); List enrichments = new java.util.ArrayList<>(placeholderEnrichments); - + // Render HTML from AST String html = renderer.render(document); // Reinsert enrichment cards from placeholders (handles example blocks) html = enrichmentPlaceholderizer.renderEnrichmentBlocksFromPlaceholders(html, placeholders); - + // Post-process HTML using DOM-safe methods html = postProcessHtml(html); - + long processingTime = System.currentTimeMillis() - startTime; - + ProcessedMarkdown processedMarkdown = new ProcessedMarkdown( - html, - citations, - enrichments, - List.of(), // No warnings for now - will be added in future iterations - processingTime - ); - + html, + citations, + enrichments, + List.of(), // No warnings for now - will be added in future iterations + processingTime); + // Cache the result using the original input key for consistency processCache.put(cacheKey, processedMarkdown); - - logger.debug("Processed markdown in {}ms: {} citations, {} enrichments", - processingTime, citations.size(), enrichments.size()); - + + logger.debug( + "Processed markdown in {}ms: {} citations, {} enrichments", + processingTime, + citations.size(), + enrichments.size()); + return processedMarkdown; - + } catch (Exception processingFailure) { logger.error("Error processing markdown with AST approach", processingFailure); throw new MarkdownProcessingException("Markdown processing failed", processingFailure); @@ -196,7 +195,7 @@ private void transformAst(Node document) { /** * Post-processes HTML using safe string operations. * This replaces regex-based post-processing with safer alternatives. - * + * * @param html the HTML to post-process * @return cleaned HTML */ @@ -214,7 +213,10 @@ private String postProcessHtml(String html) { } // Remove orphan brace-only paragraphs left by fragmented generations for (Element paragraphElement : new java.util.ArrayList<>(document.select("p"))) { - if (!paragraphElement.parents().select("pre, code, .inline-enrichment").isEmpty()) continue; + if (!paragraphElement + .parents() + .select("pre, code, .inline-enrichment") + .isEmpty()) continue; String paragraphText = paragraphElement.text(); if (paragraphText != null) { String trimmedParagraphText = paragraphText.trim(); @@ -288,7 +290,7 @@ private String collapseConsecutiveNewlines(String html) { // Track
 and  tags to preserve whitespace inside them
             if (currentChar == '<' && cursor + 4 < html.length()) {
                 String ahead = html.substring(cursor, Math.min(cursor + 6, html.length()))
-                    .toLowerCase(Locale.ROOT);
+                        .toLowerCase(Locale.ROOT);
                 if (ahead.startsWith("
") || ahead.startsWith("
")) {
@@ -325,7 +327,10 @@ private void renderInlineLists(Document document) {
         }
         java.util.List paragraphElements = new java.util.ArrayList<>(document.select("p"));
         for (Element paragraphElement : paragraphElements) {
-            if (!paragraphElement.parents().select("pre, code, .inline-enrichment").isEmpty()) continue;
+            if (!paragraphElement
+                    .parents()
+                    .select("pre, code, .inline-enrichment")
+                    .isEmpty()) continue;
             // Only transform plain-text paragraphs to avoid breaking links/code spans.
             if (!paragraphElement.children().isEmpty()) continue;
 
@@ -350,23 +355,18 @@ private void renderInlineLists(Document document) {
     }
 
     // === Pre-normalization and paragraph utilities (no regex) ===
-    
+
     // fixSentenceSpacing and splitLongParagraphs removed to ensure clean rendering
-    
+
     /**
      * Gets cache statistics for monitoring.
      * @return cache statistics
      */
     public CacheStats getCacheStats() {
         var stats = processCache.stats();
-        return new CacheStats(
-            stats.hitCount(),
-            stats.missCount(),
-            stats.evictionCount(),
-            processCache.estimatedSize()
-        );
+        return new CacheStats(stats.hitCount(), stats.missCount(), stats.evictionCount(), processCache.estimatedSize());
     }
-    
+
     /**
      * Clears the processing cache.
      */
@@ -374,23 +374,17 @@ public void clearCache() {
         processCache.invalidateAll();
         logger.info("Unified markdown processing cache cleared");
     }
-    
+
     /**
      * Cache statistics record.
      */
-	    public record CacheStats(
-	        long hitCount,
-	        long missCount,
-	        long evictionCount,
-	        long size
-	    ) {
-	        /**
-	         * Computes the cache hit rate as a fraction between 0.0 and 1.0.
-	         */
-	        public double hitRate() {
-	            long total = hitCount + missCount;
-	            return total == 0 ? 0.0 : (double) hitCount / total;
-	        }
-	    }
-
+    public record CacheStats(long hitCount, long missCount, long evictionCount, long size) {
+        /**
+         * Computes the cache hit rate as a fraction between 0.0 and 1.0.
+         */
+        public double hitRate() {
+            long total = hitCount + missCount;
+            return total == 0 ? 0.0 : (double) hitCount / total;
+        }
+    }
 }
diff --git a/src/main/java/com/williamcallahan/javachat/support/DocumentContentAdapter.java b/src/main/java/com/williamcallahan/javachat/support/DocumentContentAdapter.java
new file mode 100644
index 0000000..40b0a1a
--- /dev/null
+++ b/src/main/java/com/williamcallahan/javachat/support/DocumentContentAdapter.java
@@ -0,0 +1,62 @@
+package com.williamcallahan.javachat.support;
+
+import com.williamcallahan.javachat.domain.RetrievedContent;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.springframework.ai.document.Document;
+
+/**
+ * Adapts Spring AI Document to the domain RetrievedContent interface.
+ *
+ * 

This adapter allows domain logic to remain framework-free while the + * infrastructure layer handles the Spring AI integration.

+ */ +public final class DocumentContentAdapter implements RetrievedContent { + + private static final String METADATA_URL = "url"; + + private final Document document; + + private DocumentContentAdapter(Document document) { + this.document = document; + } + + /** + * Wraps a Spring AI Document as RetrievedContent. + * + * @param document the Spring AI document to wrap + * @return the domain-facing content representation + * @throws IllegalArgumentException if document is null + */ + public static RetrievedContent fromDocument(Document document) { + if (document == null) { + throw new IllegalArgumentException("Document cannot be null"); + } + return new DocumentContentAdapter(document); + } + + /** + * Converts a list of Spring AI Documents to RetrievedContent list. + * + * @param documents the documents to convert, may be null + * @return list of domain content representations, empty if input is null + */ + public static List fromDocuments(List documents) { + if (documents == null) { + return List.of(); + } + return documents.stream().map(DocumentContentAdapter::fromDocument).toList(); + } + + @Override + public Optional getText() { + return Optional.ofNullable(document.getText()); + } + + @Override + public Optional getSourceUrl() { + Map metadata = document.getMetadata(); + return Optional.ofNullable(metadata.get(METADATA_URL)).map(String::valueOf); + } +} diff --git a/src/main/java/com/williamcallahan/javachat/support/PdfCitationEnhancer.java b/src/main/java/com/williamcallahan/javachat/support/PdfCitationEnhancer.java new file mode 100644 index 0000000..dfbdc22 --- /dev/null +++ b/src/main/java/com/williamcallahan/javachat/support/PdfCitationEnhancer.java @@ -0,0 +1,222 @@ +package com.williamcallahan.javachat.support; + +import com.williamcallahan.javachat.model.Citation; +import com.williamcallahan.javachat.service.LocalStoreService; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Locale; +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.ai.document.Document; +import org.springframework.core.io.ClassPathResource; +import org.springframework.stereotype.Component; + +/** + * Enhances PDF citations with page number anchors based on chunk position heuristics. + * + *

For PDFs where chunk ordering correlates with page ordering (like Think Java), + * this service estimates the page number from the chunk index and total chunk count, + * then adds a #page=N anchor to the citation URL.

+ */ +@Component +public class PdfCitationEnhancer { + private static final Logger logger = LoggerFactory.getLogger(PdfCitationEnhancer.class); + + private final LocalStoreService localStore; + + /** Cached page count for the Think Java PDF to avoid repeated I/O. */ + private volatile Integer cachedThinkJavaPdfPages = null; + + /** Classpath location of the Think Java PDF. */ + private static final String THINK_JAVA_PDF_CLASSPATH = "public/pdfs/Think Java - 2nd Edition Book.pdf"; + + /** Filename pattern used to identify Think Java PDF URLs. */ + private static final String THINK_JAVA_PDF_FILENAME = "think java"; + + /** URL-encoded variant of Think Java filename. */ + private static final String THINK_JAVA_PDF_FILENAME_ENCODED = "think%20java"; + + /** + * Creates an enhancer that estimates PDF page anchors using locally stored chunk metadata. + * + * @param localStore service that provides access to parsed chunk storage + */ + public PdfCitationEnhancer(LocalStoreService localStore) { + this.localStore = localStore; + } + + /** + * Enhances PDF citations with estimated page anchors. + * + *

For each citation pointing to a PDF, this method attempts to calculate + * the page number based on the document's chunk index and the total number + * of chunks for that PDF. The citation URL is updated with a #page=N anchor.

+ * + * @param docs the retrieved documents with chunk metadata + * @param citations the citations to enhance (must be same size as docs) + * @return the enhanced citations list (same list, mutated) + * @throws UncheckedIOException when the PDF or chunk listing cannot be read + */ + public List enhanceWithPageAnchors(List docs, List citations) { + if (docs.size() != citations.size()) { + logger.warn( + "Skipping PDF anchor enhancement because docs/citations sizes differ (docs={}, citations={})", + docs.size(), + citations.size()); + return citations; + } + + int thinkJavaPages = getThinkJavaPdfPages(); + + for (int docIndex = 0; docIndex < docs.size(); docIndex++) { + Document document = docs.get(docIndex); + Citation citation = citations.get(docIndex); + String url = citation.getUrl(); + + if (url == null || !url.toLowerCase(Locale.ROOT).endsWith(".pdf")) { + continue; + } + + // Only apply page estimation to the Think Java PDF where we know the page count + if (!isThinkJavaPdf(url)) { + continue; + } + + int chunkIndex = parseChunkIndex(document); + if (chunkIndex < 0) { + continue; + } + + int totalChunks = countChunksForUrl(url); + if (thinkJavaPages > 0 && totalChunks > 0) { + int page = estimatePage(chunkIndex, totalChunks, thinkJavaPages); + String withAnchor = url.contains("#page=") ? url : url + "#page=" + page; + citation.setUrl(withAnchor); + citation.setAnchor("page=" + page); + } + } + return citations; + } + + /** + * Provides the total page count for the Think Java PDF. + * + *

The result is cached after the first load to avoid repeated I/O.

+ * + * @return page count + * @throws UncheckedIOException if the PDF cannot be loaded + */ + public int getThinkJavaPdfPages() { + if (cachedThinkJavaPdfPages != null) { + return cachedThinkJavaPdfPages; + } + + synchronized (this) { + if (cachedThinkJavaPdfPages != null) { + return cachedThinkJavaPdfPages; + } + + try { + ClassPathResource pdfResource = new ClassPathResource(THINK_JAVA_PDF_CLASSPATH); + try (InputStream pdfStream = pdfResource.getInputStream(); + PDDocument document = Loader.loadPDF(pdfStream.readAllBytes())) { + cachedThinkJavaPdfPages = document.getNumberOfPages(); + } + } catch (IOException ioException) { + throw new UncheckedIOException("Failed to load Think Java PDF for pagination", ioException); + } + return cachedThinkJavaPdfPages; + } + } + + /** + * Counts the total number of chunks stored for a given URL. + * + * @param url the source URL + * @return count of chunk files, or 0 if the directory is unavailable + * @throws UncheckedIOException if listing files fails + */ + int countChunksForUrl(String url) { + try { + String safe = localStore.toSafeName(url); + Path dir = localStore.getParsedDir(); + if (dir == null) { + return 0; + } + + try (var stream = Files.list(dir)) { + return (int) stream.filter(path -> { + Path fileNamePath = path.getFileName(); + if (fileNamePath == null) { + return false; + } + String fileName = fileNamePath.toString(); + return fileName.startsWith(safe + "_") && fileName.endsWith(".txt"); + }) + .count(); + } + } catch (IOException ioException) { + throw new UncheckedIOException("Unable to count local chunks for URL: " + url, ioException); + } + } + + /** + * Parses the chunk index from document metadata. + * + * @param document the document with metadata + * @return chunk index or -1 if not available + */ + private int parseChunkIndex(Document document) { + Object chunkIndexMetadata = document.getMetadata().get("chunkIndex"); + if (chunkIndexMetadata == null) { + return -1; + } + + try { + return Integer.parseInt(String.valueOf(chunkIndexMetadata)); + } catch (NumberFormatException parseException) { + logger.debug( + "Failed to parse chunkIndex from metadata: {}", + sanitizeForLogText(String.valueOf(chunkIndexMetadata))); + return -1; + } + } + + /** + * Estimates the page number based on chunk position within the document. + * + * @param chunkIndex zero-based chunk index + * @param totalChunks total number of chunks + * @param totalPages total number of pages in the PDF + * @return estimated page number (1-based, clamped to valid range) + */ + private int estimatePage(int chunkIndex, int totalChunks, int totalPages) { + double position = (chunkIndex + 1.0) / totalChunks; + int page = (int) Math.round(position * totalPages); + return Math.max(1, Math.min(totalPages, page)); + } + + private static String sanitizeForLogText(String rawText) { + if (rawText == null) { + return ""; + } + return rawText.replace("\r", "\\r").replace("\n", "\\n"); + } + + /** + * Checks if the URL refers to the Think Java PDF. + * + * @param url the citation URL + * @return true if the URL appears to be the Think Java PDF + */ + private static boolean isThinkJavaPdf(String url) { + String normalized = url.toLowerCase(Locale.ROOT); + return normalized.contains(THINK_JAVA_PDF_FILENAME) || normalized.contains(THINK_JAVA_PDF_FILENAME_ENCODED); + } +} diff --git a/src/main/java/com/williamcallahan/javachat/support/RetrievalErrorClassifier.java b/src/main/java/com/williamcallahan/javachat/support/RetrievalErrorClassifier.java index 548a017..4b99436 100644 --- a/src/main/java/com/williamcallahan/javachat/support/RetrievalErrorClassifier.java +++ b/src/main/java/com/williamcallahan/javachat/support/RetrievalErrorClassifier.java @@ -59,29 +59,29 @@ public static String determineErrorType(Throwable error) { */ public static boolean isTransientVectorStoreError(Throwable error) { String errorType = determineErrorType(error); - + // Connection errors and rate limits are transient if ("Connection Error".equals(errorType) || "429 Rate Limited".equals(errorType)) { return true; } - + // Check for specific Qdrant/gRPC error patterns in exception chain Throwable current = error; while (current != null) { String exceptionName = current.getClass().getName().toLowerCase(Locale.ROOT); String message = current.getMessage(); String lowerMessage = message != null ? message.toLowerCase(Locale.ROOT) : ""; - + // gRPC errors from Qdrant client if (exceptionName.contains("grpc") || exceptionName.contains("qdrant")) { // Service unavailable, deadline exceeded are transient - if (lowerMessage.contains("unavailable") - || lowerMessage.contains("deadline exceeded") - || lowerMessage.contains("resource exhausted")) { + if (lowerMessage.contains("unavailable") + || lowerMessage.contains("deadline exceeded") + || lowerMessage.contains("resource exhausted")) { return true; } } - + // ExecutionException wrapping gRPC failures if (current instanceof java.util.concurrent.ExecutionException) { // Check the cause for gRPC issues @@ -93,16 +93,15 @@ public static boolean isTransientVectorStoreError(Throwable error) { } } } - + // IllegalArgumentException for UUID parsing is NOT transient (programming error) - if (current instanceof IllegalArgumentException - && lowerMessage.contains("uuid")) { + if (current instanceof IllegalArgumentException && lowerMessage.contains("uuid")) { return false; } - + current = current.getCause(); } - + // Default: unknown errors are not assumed to be transient return false; } @@ -115,22 +114,17 @@ public static boolean isTransientVectorStoreError(Throwable error) { * @param error original exception */ public static void logUserFriendlyErrorContext(Logger log, String errorType, Throwable error) { - if (error.getCause() instanceof com.williamcallahan.javachat.service.GracefulEmbeddingModel.EmbeddingServiceUnavailableException) { + if (error.getCause() + instanceof + com.williamcallahan.javachat.service.GracefulEmbeddingModel.EmbeddingServiceUnavailableException) { log.info( - "Embedding services are unavailable. Using keyword-based search with limited semantic understanding." - ); + "Embedding services are unavailable. Using keyword-based search with limited semantic understanding."); } else if (errorType.contains("404")) { - log.info( - "Embedding API endpoint not found. Check configuration for spring.ai.openai.embedding.base-url" - ); + log.info("Embedding API endpoint not found. Check configuration for spring.ai.openai.embedding.base-url"); } else if (errorType.contains("401") || errorType.contains("403")) { - log.info( - "Embedding API authentication failed. Check OPENAI_API_KEY or GITHUB_TOKEN configuration" - ); + log.info("Embedding API authentication failed. Check OPENAI_API_KEY or GITHUB_TOKEN configuration"); } else if (errorType.contains("429")) { - log.info( - "Embedding API rate limit exceeded. Consider using local embeddings or upgrading API tier" - ); + log.info("Embedding API rate limit exceeded. Consider using local embeddings or upgrading API tier"); } } } diff --git a/src/main/java/com/williamcallahan/javachat/support/RetrySupport.java b/src/main/java/com/williamcallahan/javachat/support/RetrySupport.java index 935e5df..7bd9921 100644 --- a/src/main/java/com/williamcallahan/javachat/support/RetrySupport.java +++ b/src/main/java/com/williamcallahan/javachat/support/RetrySupport.java @@ -51,43 +51,47 @@ public static T executeWithRetry(Supplier operation, String operationName * @throws RuntimeException if all retries are exhausted or a non-transient error occurs */ public static T executeWithRetry( - Supplier operation, - String operationName, - int maxAttempts, - Duration initialBackoff) { + Supplier operation, String operationName, int maxAttempts, Duration initialBackoff) { String safeOperationName = sanitizeLogValue(operationName); - + if (maxAttempts < 1) { throw new IllegalArgumentException("maxAttempts must be at least 1"); } RuntimeException lastException = null; Duration currentBackoff = initialBackoff; - + for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { return operation.get(); } catch (RuntimeException exception) { lastException = exception; - + // Check if this is a transient error worth retrying if (!RetrievalErrorClassifier.isTransientVectorStoreError(exception)) { - log.warn("{} failed with non-transient error on attempt {}/{}, not retrying", - safeOperationName, attempt, maxAttempts); + log.warn( + "{} failed with non-transient error on attempt {}/{}, not retrying", + safeOperationName, + attempt, + maxAttempts); throw exception; } - + if (attempt < maxAttempts) { - log.warn("{} failed with transient error on attempt {}/{}, retrying in {}ms", - safeOperationName, attempt, maxAttempts, currentBackoff.toMillis()); - + log.warn( + "{} failed with transient error on attempt {}/{}, retrying in {}ms", + safeOperationName, + attempt, + maxAttempts, + currentBackoff.toMillis()); + try { Thread.sleep(currentBackoff.toMillis()); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); throw new IllegalStateException("Retry interrupted", interrupted); } - + // Exponential backoff with cap long nextBackoffMillis = (long) (currentBackoff.toMillis() * DEFAULT_MULTIPLIER); currentBackoff = Duration.ofMillis(Math.min(nextBackoffMillis, MAX_BACKOFF.toMillis())); @@ -96,7 +100,7 @@ public static T executeWithRetry( } } } - + // Guard for static analysis: loop always sets lastException before reaching here if (lastException == null) { throw new IllegalStateException("Retry loop completed without exception - should not happen"); @@ -111,10 +115,12 @@ public static T executeWithRetry( * @param operationName name for logging purposes */ public static void executeWithRetry(Runnable operation, String operationName) { - executeWithRetry(() -> { - operation.run(); - return null; - }, operationName); + executeWithRetry( + () -> { + operation.run(); + return null; + }, + operationName); } private static String sanitizeLogValue(String rawValue) { diff --git a/src/main/java/com/williamcallahan/javachat/util/JavadocMemberAnchorResolver.java b/src/main/java/com/williamcallahan/javachat/util/JavadocMemberAnchorResolver.java index 9de1662..5c6368a 100644 --- a/src/main/java/com/williamcallahan/javachat/util/JavadocMemberAnchorResolver.java +++ b/src/main/java/com/williamcallahan/javachat/util/JavadocMemberAnchorResolver.java @@ -32,9 +32,7 @@ static String refineMemberAnchorUrl(String url, String text, String packageName) String classFileName = url.substring(url.lastIndexOf('/') + 1); String className = classFileName.substring(0, classFileName.length() - ".html".length()); - String outerSimpleName = className.contains(".") - ? className.substring(0, className.indexOf('.')) - : className; + String outerSimpleName = className.contains(".") ? className.substring(0, className.indexOf('.')) : className; String constructorAnchor = findConstructorAnchor(outerSimpleName, text, packageName, className); if (constructorAnchor != null) { @@ -49,7 +47,8 @@ static String refineMemberAnchorUrl(String url, String text, String packageName) return url; } - private static String findConstructorAnchor(String classSimpleName, String text, String packageName, String fullClassName) { + private static String findConstructorAnchor( + String classSimpleName, String text, String packageName, String fullClassName) { Pattern constructorPattern = Pattern.compile("\\b" + Pattern.quote(classSimpleName) + "\\s*\\(([^)]*)\\)"); Matcher constructorMatcher = constructorPattern.matcher(text); if (constructorMatcher.find()) { @@ -128,5 +127,4 @@ private static String[] splitParams(String rawParams) { } return tokens.toArray(new String[0]); } - } diff --git a/src/main/java/com/williamcallahan/javachat/util/JavadocNestedTypeResolver.java b/src/main/java/com/williamcallahan/javachat/util/JavadocNestedTypeResolver.java index 18d5399..b5c0b5a 100644 --- a/src/main/java/com/williamcallahan/javachat/util/JavadocNestedTypeResolver.java +++ b/src/main/java/com/williamcallahan/javachat/util/JavadocNestedTypeResolver.java @@ -26,9 +26,8 @@ static String refineNestedTypeUrl(String url, String text) { String fileName = url.substring(url.lastIndexOf('/') + 1); String basePath = url.substring(0, url.length() - fileName.length()); - String outerTypeName = fileName.endsWith(".html") - ? fileName.substring(0, fileName.length() - ".html".length()) - : fileName; + String outerTypeName = + fileName.endsWith(".html") ? fileName.substring(0, fileName.length() - ".html".length()) : fileName; // Already a nested type page if (outerTypeName.contains(".")) { @@ -38,8 +37,7 @@ static String refineNestedTypeUrl(String url, String text) { // Look for Outer.Inner or deeper in the text // Pattern: Outer.(Inner(.Deep)*) where Outer is the current page type name Pattern nestedTypePattern = Pattern.compile( - "\\b" + Pattern.quote(outerTypeName) + "\\.([A-Z][A-Za-z0-9_]*(?:\\.[A-Z][A-Za-z0-9_]*)*)\\b" - ); + "\\b" + Pattern.quote(outerTypeName) + "\\.([A-Z][A-Za-z0-9_]*(?:\\.[A-Z][A-Za-z0-9_]*)*)\\b"); Matcher nestedTypeMatcher = nestedTypePattern.matcher(text); if (nestedTypeMatcher.find()) { String nestedSuffix = nestedTypeMatcher.group(1); // e.g., "Operator" or "Inner.Deep" diff --git a/src/main/java/com/williamcallahan/javachat/util/JavadocTypeCanonicalizer.java b/src/main/java/com/williamcallahan/javachat/util/JavadocTypeCanonicalizer.java index cf3afde..7d208f8 100644 --- a/src/main/java/com/williamcallahan/javachat/util/JavadocTypeCanonicalizer.java +++ b/src/main/java/com/williamcallahan/javachat/util/JavadocTypeCanonicalizer.java @@ -70,11 +70,7 @@ static String canonicalizeType(String rawType, String packageName, String fullCl } private static String resolveRelativeQualifiedType( - String trimmedType, - String packageName, - int arrayDimensions, - boolean isVarargs - ) { + String trimmedType, String packageName, int arrayDimensions, boolean isVarargs) { if (!Character.isUpperCase(trimmedType.charAt(0))) { return null; } @@ -151,14 +147,14 @@ private static String stripParamName(String typeText) { private static boolean isPrimitiveOrVoid(String typeText) { return typeText.equals("byte") - || typeText.equals("short") - || typeText.equals("int") - || typeText.equals("long") - || typeText.equals("char") - || typeText.equals("float") - || typeText.equals("double") - || typeText.equals("boolean") - || typeText.equals("void"); + || typeText.equals("short") + || typeText.equals("int") + || typeText.equals("long") + || typeText.equals("char") + || typeText.equals("float") + || typeText.equals("double") + || typeText.equals("boolean") + || typeText.equals("void"); } private static String mapJavaLang(String typeText) { diff --git a/src/main/java/com/williamcallahan/javachat/util/QueryVersionExtractor.java b/src/main/java/com/williamcallahan/javachat/util/QueryVersionExtractor.java index 8782960..4cfa0a3 100644 --- a/src/main/java/com/williamcallahan/javachat/util/QueryVersionExtractor.java +++ b/src/main/java/com/williamcallahan/javachat/util/QueryVersionExtractor.java @@ -16,10 +16,8 @@ private QueryVersionExtractor() {} * Pattern to match Java version references in queries. * Matches: "Java 25", "JDK 24", "java25", "jdk-25", "Java SE 24", "JavaSE 25" */ - private static final Pattern VERSION_PATTERN = Pattern.compile( - "\\b(?:java\\s*se|javase|java|jdk)[\\s-]*(\\d{1,2})\\b", - Pattern.CASE_INSENSITIVE - ); + private static final Pattern VERSION_PATTERN = + Pattern.compile("\\b(?:java\\s*se|javase|java|jdk)[\\s-]*(\\d{1,2})\\b", Pattern.CASE_INSENSITIVE); /** * Extract the Java version number from a query string. @@ -46,8 +44,7 @@ public static Optional extractVersionNumber(String query) { * @return Optional containing the source identifier if detected, empty otherwise */ public static Optional extractSourceIdentifier(String query) { - return extractVersionNumber(query) - .map(version -> "java" + version); + return extractVersionNumber(query).map(version -> "java" + version); } /** @@ -58,8 +55,7 @@ public static Optional extractSourceIdentifier(String query) { * @return Optional containing filter patterns if version detected, empty otherwise */ public static Optional extractFilterPatterns(String query) { - return extractVersionNumber(query) - .map(VersionFilterPatterns::new); + return extractVersionNumber(query).map(VersionFilterPatterns::new); } /** @@ -109,9 +105,7 @@ public boolean matchesUrl(String url) { return false; } String lowerUrl = url.toLowerCase(Locale.ROOT); - return lowerUrl.contains(javaPattern) - || lowerUrl.contains(jdkPattern) - || lowerUrl.contains(eaPattern); + return lowerUrl.contains(javaPattern) || lowerUrl.contains(jdkPattern) || lowerUrl.contains(eaPattern); } } } diff --git a/src/main/java/com/williamcallahan/javachat/web/ApiResponse.java b/src/main/java/com/williamcallahan/javachat/web/ApiResponse.java index 83fed41..d100be4 100644 --- a/src/main/java/com/williamcallahan/javachat/web/ApiResponse.java +++ b/src/main/java/com/williamcallahan/javachat/web/ApiResponse.java @@ -15,4 +15,3 @@ public sealed interface ApiResponse permits ApiErrorResponse, ApiSuccessResponse */ String status(); } - diff --git a/src/main/java/com/williamcallahan/javachat/web/AuditController.java b/src/main/java/com/williamcallahan/javachat/web/AuditController.java index bd43067..461b53e 100644 --- a/src/main/java/com/williamcallahan/javachat/web/AuditController.java +++ b/src/main/java/com/williamcallahan/javachat/web/AuditController.java @@ -1,8 +1,9 @@ package com.williamcallahan.javachat.web; import com.williamcallahan.javachat.model.AuditReport; -import jakarta.annotation.security.PermitAll; import com.williamcallahan.javachat.service.AuditService; +import jakarta.annotation.security.PermitAll; +import java.io.IOException; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; @@ -10,8 +11,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.io.IOException; - /** * REST controller for auditing ingested content against the vector store. */ diff --git a/src/main/java/com/williamcallahan/javachat/web/BaseController.java b/src/main/java/com/williamcallahan/javachat/web/BaseController.java index a9f922c..b6a9ac6 100644 --- a/src/main/java/com/williamcallahan/javachat/web/BaseController.java +++ b/src/main/java/com/williamcallahan/javachat/web/BaseController.java @@ -1,4 +1,5 @@ package com.williamcallahan.javachat.web; + import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -24,13 +25,9 @@ protected BaseController(ExceptionResponseBuilder exceptionBuilder) { * @param operation Description of the operation that failed * @return Standardized error response */ - protected ResponseEntity handleServiceException( - Exception e, String operation) { + protected ResponseEntity handleServiceException(Exception e, String operation) { return exceptionBuilder.buildErrorResponse( - HttpStatus.INTERNAL_SERVER_ERROR, - "Failed to " + operation + ": " + e.getMessage(), - e - ); + HttpStatus.INTERNAL_SERVER_ERROR, "Failed to " + operation + ": " + e.getMessage(), e); } /** @@ -39,12 +36,8 @@ protected ResponseEntity handleServiceException( * @param validationException The validation exception * @return Bad request error response */ - protected ResponseEntity handleValidationException( - IllegalArgumentException validationException) { - return exceptionBuilder.buildErrorResponse( - HttpStatus.BAD_REQUEST, - validationException.getMessage() - ); + protected ResponseEntity handleValidationException(IllegalArgumentException validationException) { + return exceptionBuilder.buildErrorResponse(HttpStatus.BAD_REQUEST, validationException.getMessage()); } /** diff --git a/src/main/java/com/williamcallahan/javachat/web/ChatController.java b/src/main/java/com/williamcallahan/javachat/web/ChatController.java index 1c614c6..2af9479 100644 --- a/src/main/java/com/williamcallahan/javachat/web/ChatController.java +++ b/src/main/java/com/williamcallahan/javachat/web/ChatController.java @@ -1,5 +1,7 @@ package com.williamcallahan.javachat.web; +import static com.williamcallahan.javachat.web.SseConstants.*; + import com.openai.errors.OpenAIIoException; import com.openai.errors.RateLimitException; import com.williamcallahan.javachat.config.AppProperties; @@ -7,32 +9,29 @@ import com.williamcallahan.javachat.model.ChatTurn; import com.williamcallahan.javachat.model.Citation; import com.williamcallahan.javachat.service.ChatMemoryService; -import com.williamcallahan.javachat.support.AsciiTextNormalizer; import com.williamcallahan.javachat.service.ChatService; -import com.williamcallahan.javachat.service.RetrievalService; import com.williamcallahan.javachat.service.OpenAIStreamingService; +import com.williamcallahan.javachat.service.RetrievalService; +import com.williamcallahan.javachat.support.AsciiTextNormalizer; +import jakarta.annotation.security.PermitAll; +import jakarta.servlet.http.HttpServletResponse; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.ai.chat.messages.Message; +import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.http.codec.ServerSentEvent; import org.springframework.security.access.prepost.PreAuthorize; -import jakarta.annotation.security.PermitAll; -import jakarta.servlet.http.HttpServletResponse; -import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.web.bind.annotation.*; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; import reactor.core.publisher.Flux; -import org.springframework.http.codec.ServerSentEvent; - -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import static com.williamcallahan.javachat.web.SseConstants.*; /** * Exposes chat endpoints for streaming responses, session history management, and diagnostics. @@ -66,13 +65,15 @@ public class ChatController extends BaseController { * @param exceptionBuilder shared exception response builder * @param appProperties centralized application configuration */ - public ChatController(ChatService chatService, ChatMemoryService chatMemory, - OpenAIStreamingService openAIStreamingService, - RetrievalService retrievalService, - SseSupport sseSupport, - RestTemplateBuilder restTemplateBuilder, - ExceptionResponseBuilder exceptionBuilder, - AppProperties appProperties) { + public ChatController( + ChatService chatService, + ChatMemoryService chatMemory, + OpenAIStreamingService openAIStreamingService, + RetrievalService retrievalService, + SseSupport sseSupport, + RestTemplateBuilder restTemplateBuilder, + ExceptionResponseBuilder exceptionBuilder, + AppProperties appProperties) { super(exceptionBuilder); this.chatService = chatService; this.chatMemory = chatMemory; @@ -88,11 +89,10 @@ public ChatController(ChatService chatService, ChatMemoryService chatMemory, */ private String normalizeRole(ChatTurn turn) { return turn.getRole() == null - ? "" - : AsciiTextNormalizer.toLowerAscii(turn.getRole().trim()); + ? "" + : AsciiTextNormalizer.toLowerAscii(turn.getRole().trim()); } - /** * Streams a response to a user's chat message using Server-Sent Events (SSE). * Uses the OpenAI Java SDK for clean, reliable streaming without manual SSE parsing. @@ -117,18 +117,18 @@ public Flux> stream(@RequestBody ChatStreamRequest reque PIPELINE_LOG.info("[{}] ============================================", requestToken); PIPELINE_LOG.info("[{}] NEW CHAT REQUEST", requestToken); PIPELINE_LOG.info("[{}] ============================================", requestToken); - + List history = new ArrayList<>(chatMemory.getHistory(sessionId)); PIPELINE_LOG.info("[{}] Chat history loaded", requestToken); - + chatMemory.addUser(sessionId, userQuery); StringBuilder fullResponse = new StringBuilder(); AtomicInteger chunkCount = new AtomicInteger(0); // Build structured prompt for intelligent truncation // Pass model hint to optimize RAG for token-constrained models - ChatService.StructuredPromptOutcome promptOutcome = - chatService.buildStructuredPromptWithContextOutcome(history, userQuery, ModelConfiguration.DEFAULT_MODEL); + ChatService.StructuredPromptOutcome promptOutcome = chatService.buildStructuredPromptWithContextOutcome( + history, userQuery, ModelConfiguration.DEFAULT_MODEL); // Use OpenAI streaming only (legacy fallback removed) if (openAIStreamingService.isAvailable()) { @@ -157,8 +157,7 @@ public Flux> stream(@RequestBody ChatStreamRequest reque try { citations = retrievalService.toCitations(promptOutcome.documents()); } catch (Exception citationError) { - PIPELINE_LOG.warn("[{}] Citation conversion failed, continuing without citations: {}", - requestToken, citationError.getMessage()); + PIPELINE_LOG.warn("[{}] Citation conversion failed, continuing without citations", requestToken); citations = List.of(); } Flux> citationEvent = Flux.just(sseSupport.citationEvent(citations)); @@ -171,10 +170,9 @@ public Flux> stream(@RequestBody ChatStreamRequest reque .onErrorResume(error -> { // Log and send error event to client - errors must be communicated, not silently dropped String errorDetail = buildUserFacingErrorMessage(error); - String diagnostics = error instanceof Exception exception - ? describeException(exception) - : error.toString(); - PIPELINE_LOG.error("[{}] STREAMING ERROR: {}", requestToken, errorDetail, error); + String diagnostics = + error instanceof Exception exception ? describeException(exception) : error.toString(); + PIPELINE_LOG.error("[{}] STREAMING ERROR", requestToken); return sseSupport.sseError(errorDetail, diagnostics); }); @@ -193,19 +191,16 @@ public Flux> stream(@RequestBody ChatStreamRequest reque public RetrievalDiagnosticsResponse retrievalDiagnostics(@RequestParam("q") String query) { // Mirror token-constrained model constraints used in buildPromptWithContext RetrievalService.RetrievalOutcome outcome = retrievalService.retrieveWithLimitOutcome( - query, - ModelConfiguration.RAG_LIMIT_CONSTRAINED, - ModelConfiguration.RAG_TOKEN_LIMIT_CONSTRAINED - ); + query, ModelConfiguration.RAG_LIMIT_CONSTRAINED, ModelConfiguration.RAG_TOKEN_LIMIT_CONSTRAINED); // Normalize URLs the same way as citations so we never emit file:// links List citations = retrievalService.toCitations(outcome.documents()); if (outcome.notices().isEmpty()) { return RetrievalDiagnosticsResponse.success(citations); } String noticeDetails = outcome.notices().stream() - .map(notice -> notice.summary() + ": " + notice.details()) - .reduce((first, second) -> first + "; " + second) - .orElse("Retrieval warnings present"); + .map(notice -> notice.summary() + ": " + notice.details()) + .reduce((first, second) -> first + "; " + second) + .orElse("Retrieval warnings present"); return new RetrievalDiagnosticsResponse(citations, noticeDetails); } @@ -259,12 +254,16 @@ public ResponseEntity exportSession(@RequestParam(name = "sessionId") St StringBuilder formatted = new StringBuilder(); for (var turn : turns) { String role = "user".equals(normalizeRole(turn)) ? "User" : "Assistant"; - formatted.append("### ").append(role).append("\n\n").append(turn.getText()).append("\n\n"); + formatted + .append("### ") + .append(role) + .append("\n\n") + .append(turn.getText()) + .append("\n\n"); } return ResponseEntity.ok(formatted.toString()); } - /** * Clears the chat history for a given session. * @@ -292,16 +291,16 @@ public ResponseEntity clearSession(@RequestParam(name = "sessionId", req public ResponseEntity validateSession( @RequestParam(name = "sessionId") String sessionId) { if (sessionId == null || sessionId.isEmpty()) { - return ResponseEntity.badRequest().body( - new SessionValidationResponse("", 0, false, "Session ID is required")); + return ResponseEntity.badRequest() + .body(new SessionValidationResponse("", 0, false, "Session ID is required")); } var turns = chatMemory.getTurns(sessionId); int turnCount = turns.size(); boolean exists = turnCount > 0; return ResponseEntity.ok(new SessionValidationResponse( - sessionId, turnCount, exists, exists ? "Session found" : "Session not found on server")); + sessionId, turnCount, exists, exists ? "Session found" : "Session not found on server")); } - + /** * Reports whether the configured local embedding server is reachable when the feature is enabled. */ @@ -320,8 +319,7 @@ public ResponseEntity checkEmbeddingsHealth() { } catch (RestClientException httpError) { log.debug("Embedding server health check failed", httpError); String details = describeException(httpError); - return ResponseEntity.ok(EmbeddingsHealthResponse.unhealthy( - serverUrl, "UNREACHABLE: " + details)); + return ResponseEntity.ok(EmbeddingsHealthResponse.unhealthy(serverUrl, "UNREACHABLE: " + details)); } } @@ -341,14 +339,16 @@ private String buildUserFacingErrorMessage(Throwable error) { if (error instanceof OpenAIIoException ioError) { Throwable cause = ioError.getCause(); - if (cause != null && cause.getMessage() != null + if (cause != null + && cause.getMessage() != null && cause.getMessage().toLowerCase(Locale.ROOT).contains("interrupt")) { return "Request cancelled - LLM provider did not respond in time"; } return "LLM provider connection failed - " + error.getClass().getSimpleName(); } - if (error instanceof IllegalStateException && error.getMessage() != null + if (error instanceof IllegalStateException + && error.getMessage() != null && error.getMessage().contains("providers unavailable")) { return error.getMessage(); } diff --git a/src/main/java/com/williamcallahan/javachat/web/ChatStreamRequest.java b/src/main/java/com/williamcallahan/javachat/web/ChatStreamRequest.java index ddaa2b9..646b66b 100644 --- a/src/main/java/com/williamcallahan/javachat/web/ChatStreamRequest.java +++ b/src/main/java/com/williamcallahan/javachat/web/ChatStreamRequest.java @@ -8,11 +8,7 @@ * @param message The user's chat message (preferred field name for API clients) * @param latest The user's chat message (alternative field name used by web UI) */ -public record ChatStreamRequest( - String sessionId, - String message, - String latest -) { +public record ChatStreamRequest(String sessionId, String message, String latest) { private static final String GENERATED_SESSION_PREFIX = "chat-"; /** diff --git a/src/main/java/com/williamcallahan/javachat/web/CustomErrorController.java b/src/main/java/com/williamcallahan/javachat/web/CustomErrorController.java index 57b0122..20934d0 100644 --- a/src/main/java/com/williamcallahan/javachat/web/CustomErrorController.java +++ b/src/main/java/com/williamcallahan/javachat/web/CustomErrorController.java @@ -39,7 +39,7 @@ public class CustomErrorController implements ErrorController { private static final String ERROR_VIEW_UNSUPPORTED_MEDIA = "forward:/errors/unsupported-media-type"; private final ExceptionResponseBuilder exceptionBuilder; - + /** * Creates the error controller backed by the shared exception response builder. * @@ -48,40 +48,47 @@ public class CustomErrorController implements ErrorController { public CustomErrorController(ExceptionResponseBuilder exceptionBuilder) { this.exceptionBuilder = exceptionBuilder; } - + /** * Handles error requests and returns appropriate error pages or JSON responses. - * + * * @param request The HTTP request * @param model Spring MVC model for template rendering * @return ModelAndView for HTML requests or ResponseEntity for API requests */ // Explicitly specify all HTTP methods - error handlers must respond to any request type // that might generate an error. This is intentional, not a CSRF risk. - @RequestMapping(value = ERROR_PATH, method = { - RequestMethod.GET, RequestMethod.POST, RequestMethod.PUT, - RequestMethod.DELETE, RequestMethod.PATCH, RequestMethod.HEAD, RequestMethod.OPTIONS - }) + @RequestMapping( + value = ERROR_PATH, + method = { + RequestMethod.GET, + RequestMethod.POST, + RequestMethod.PUT, + RequestMethod.DELETE, + RequestMethod.PATCH, + RequestMethod.HEAD, + RequestMethod.OPTIONS + }) public Object handleError(HttpServletRequest request, Model model) { // Get error details from request attributes Object status = request.getAttribute(RequestDispatcher.ERROR_STATUS_CODE); Object message = request.getAttribute(RequestDispatcher.ERROR_MESSAGE); Object exception = request.getAttribute(RequestDispatcher.ERROR_EXCEPTION); Object requestUri = request.getAttribute(RequestDispatcher.ERROR_REQUEST_URI); - + int statusCode = status != null ? (Integer) status : 500; String errorMessage = message != null ? message.toString() : "An unexpected error occurred"; String uri = requestUri != null ? requestUri.toString() : request.getRequestURI(); - + // Log the error for monitoring without echoing request-derived strings log.error("Error {} occurred while handling request", statusCode); if (exception instanceof Exception exceptionInstance) { log.error("Exception type: {}", exceptionInstance.getClass().getSimpleName()); } - + // Determine if this is an API request or a page request boolean isApiRequest = uri.startsWith("/api/"); - + if (isApiRequest) { // Return JSON error response for API requests return handleApiError(statusCode, errorMessage, (Exception) exception); @@ -90,7 +97,7 @@ public Object handleError(HttpServletRequest request, Model model) { return handlePageError(statusCode, resolveUserFacingMessage(statusCode), uri, model); } } - + /** * Handles API error responses with JSON format. */ @@ -99,33 +106,33 @@ private ResponseEntity handleApiError(int statusCode, String m if (httpStatus == null) { httpStatus = HttpStatus.INTERNAL_SERVER_ERROR; } - + if (exception != null) { return exceptionBuilder.buildErrorResponse(httpStatus, message, exception); } else { return exceptionBuilder.buildErrorResponse(httpStatus, message); } } - + /** * Handles page error responses with HTML error pages. */ private ModelAndView handlePageError(int statusCode, String message, String uri, Model model) { ModelAndView modelAndView = new ModelAndView(); - + // Add error details to model for potential template use model.addAttribute("status", statusCode); model.addAttribute("error", HttpStatus.resolve(statusCode)); model.addAttribute("message", message); model.addAttribute("path", uri); model.addAttribute("timestamp", System.currentTimeMillis()); - + HttpStatus resolvedStatus = HttpStatus.resolve(statusCode); if (resolvedStatus == null) { resolvedStatus = HttpStatus.INTERNAL_SERVER_ERROR; } modelAndView.setViewName(resolveErrorViewName(resolvedStatus)); - + return modelAndView; } @@ -152,7 +159,7 @@ private String resolveErrorViewName(HttpStatus status) { default -> ERROR_VIEW_INTERNAL; }; } - + /** * Returns the error path for Spring Boot's ErrorController interface. */ diff --git a/src/main/java/com/williamcallahan/javachat/web/EmbeddingCacheController.java b/src/main/java/com/williamcallahan/javachat/web/EmbeddingCacheController.java index 9d898b8..cb2b109 100644 --- a/src/main/java/com/williamcallahan/javachat/web/EmbeddingCacheController.java +++ b/src/main/java/com/williamcallahan/javachat/web/EmbeddingCacheController.java @@ -2,6 +2,7 @@ import com.williamcallahan.javachat.service.EmbeddingCacheService; import jakarta.annotation.security.PermitAll; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.ResponseEntity; @@ -12,8 +13,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.io.IOException; - /** * REST API for managing embedding cache operations. */ @@ -52,21 +51,14 @@ public ResponseEntity getCacheStats() { public ResponseEntity saveSnapshot() { try { embeddingCacheService.saveSnapshot(); - return ResponseEntity.ok(new SnapshotResponse( - true, - "Snapshot saved successfully", - embeddingCacheService.getCacheStats() - )); + return ResponseEntity.ok( + new SnapshotResponse(true, "Snapshot saved successfully", embeddingCacheService.getCacheStats())); } catch (IOException snapshotFailure) { log.error("Failed to save snapshot", snapshotFailure); String errorMessage = snapshotFailure.getMessage() != null - ? snapshotFailure.getMessage() - : snapshotFailure.getClass().getSimpleName(); - return ResponseEntity.internalServerError().body(new SnapshotResponse( - false, - errorMessage, - null - )); + ? snapshotFailure.getMessage() + : snapshotFailure.getClass().getSimpleName(); + return ResponseEntity.internalServerError().body(new SnapshotResponse(false, errorMessage, null)); } } @@ -77,15 +69,10 @@ public ResponseEntity saveSnapshot() { * @return response with upload count and updated stats */ @PostMapping("/upload") - public ResponseEntity uploadToVectorStore( - @RequestParam(defaultValue = "100") int batchSize) { + public ResponseEntity uploadToVectorStore(@RequestParam(defaultValue = "100") int batchSize) { int uploaded = embeddingCacheService.uploadPendingToVectorStore(batchSize); - return ResponseEntity.ok(new UploadResponse( - true, - uploaded, - embeddingCacheService.getCacheStats() - )); + return ResponseEntity.ok(new UploadResponse(true, uploaded, embeddingCacheService.getCacheStats())); } /** @@ -95,8 +82,7 @@ public ResponseEntity uploadToVectorStore( * @return response with operation status and updated stats */ @PostMapping("/export") - public ResponseEntity exportCache( - @RequestParam(required = false) String filename) { + public ResponseEntity exportCache(@RequestParam(required = false) String filename) { try { if (filename == null || filename.isEmpty()) { @@ -105,22 +91,13 @@ public ResponseEntity exportCache( embeddingCacheService.exportCache(filename); } return ResponseEntity.ok(new ExportResponse( - true, - "Cache exported successfully", - filename, - embeddingCacheService.getCacheStats() - )); + true, "Cache exported successfully", filename, embeddingCacheService.getCacheStats())); } catch (IOException exportFailure) { log.error("Failed to export cache", exportFailure); String errorMessage = exportFailure.getMessage() != null - ? exportFailure.getMessage() - : exportFailure.getClass().getSimpleName(); - return ResponseEntity.internalServerError().body(new ExportResponse( - false, - errorMessage, - filename, - null - )); + ? exportFailure.getMessage() + : exportFailure.getClass().getSimpleName(); + return ResponseEntity.internalServerError().body(new ExportResponse(false, errorMessage, filename, null)); } } @@ -131,66 +108,40 @@ public ResponseEntity exportCache( * @return response with operation status and updated stats */ @PostMapping("/import") - public ResponseEntity importCache( - @RequestParam String filename) { + public ResponseEntity importCache(@RequestParam String filename) { try { embeddingCacheService.importCache(filename); return ResponseEntity.ok(new ImportResponse( - true, - "Cache imported successfully", - filename, - embeddingCacheService.getCacheStats() - )); + true, "Cache imported successfully", filename, embeddingCacheService.getCacheStats())); } catch (IOException importFailure) { log.error("Failed to import cache", importFailure); String errorMessage = importFailure.getMessage() != null - ? importFailure.getMessage() - : importFailure.getClass().getSimpleName(); - return ResponseEntity.internalServerError().body(new ImportResponse( - false, - errorMessage, - filename, - null - )); + ? importFailure.getMessage() + : importFailure.getClass().getSimpleName(); + return ResponseEntity.internalServerError().body(new ImportResponse(false, errorMessage, filename, null)); } } /** * Response for snapshot operations. */ - public record SnapshotResponse( - boolean success, - String message, - EmbeddingCacheService.CacheStats stats - ) {} + public record SnapshotResponse(boolean success, String message, EmbeddingCacheService.CacheStats stats) {} /** * Response for upload operations. */ - public record UploadResponse( - boolean success, - int uploaded, - EmbeddingCacheService.CacheStats stats - ) {} + public record UploadResponse(boolean success, int uploaded, EmbeddingCacheService.CacheStats stats) {} /** * Response for export operations. */ public record ExportResponse( - boolean success, - String message, - String filename, - EmbeddingCacheService.CacheStats stats - ) {} + boolean success, String message, String filename, EmbeddingCacheService.CacheStats stats) {} /** * Response for import operations. */ public record ImportResponse( - boolean success, - String message, - String filename, - EmbeddingCacheService.CacheStats stats - ) {} + boolean success, String message, String filename, EmbeddingCacheService.CacheStats stats) {} } diff --git a/src/main/java/com/williamcallahan/javachat/web/EmbeddingsHealthResponse.java b/src/main/java/com/williamcallahan/javachat/web/EmbeddingsHealthResponse.java index 0f11d8e..a5f9350 100644 --- a/src/main/java/com/williamcallahan/javachat/web/EmbeddingsHealthResponse.java +++ b/src/main/java/com/williamcallahan/javachat/web/EmbeddingsHealthResponse.java @@ -10,12 +10,7 @@ * @param error Error message if unhealthy */ public record EmbeddingsHealthResponse( - boolean localEmbeddingEnabled, - String serverUrl, - String status, - Boolean serverReachable, - String error -) { + boolean localEmbeddingEnabled, String serverUrl, String status, Boolean serverReachable, String error) { /** * Creates a healthy response. */ diff --git a/src/main/java/com/williamcallahan/javachat/web/EnrichmentController.java b/src/main/java/com/williamcallahan/javachat/web/EnrichmentController.java index 4ae1d4a..88d2e96 100644 --- a/src/main/java/com/williamcallahan/javachat/web/EnrichmentController.java +++ b/src/main/java/com/williamcallahan/javachat/web/EnrichmentController.java @@ -1,18 +1,17 @@ package com.williamcallahan.javachat.web; import com.williamcallahan.javachat.model.Enrichment; -import jakarta.annotation.security.PermitAll; import com.williamcallahan.javachat.service.EnrichmentService; import com.williamcallahan.javachat.service.RetrievalService; +import jakarta.annotation.security.PermitAll; +import java.util.List; +import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Value; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.util.List; -import java.util.stream.Collectors; - /** * Exposes enrichment marker generation backed by retrieval and enrichment services. */ @@ -27,9 +26,10 @@ public class EnrichmentController { /** * Creates the enrichment controller using retrieval for context and a service to synthesize enrichment markers. */ - public EnrichmentController(RetrievalService retrievalService, - EnrichmentService enrichmentService, - @Value("${app.docs.jdk-version}") String jdkVersion) { + public EnrichmentController( + RetrievalService retrievalService, + EnrichmentService enrichmentService, + @Value("${app.docs.jdk-version}") String jdkVersion) { this.retrievalService = retrievalService; this.enrichmentService = enrichmentService; this.jdkVersion = jdkVersion; diff --git a/src/main/java/com/williamcallahan/javachat/web/ErrorDocumentationController.java b/src/main/java/com/williamcallahan/javachat/web/ErrorDocumentationController.java index b39b156..e8006dd 100644 --- a/src/main/java/com/williamcallahan/javachat/web/ErrorDocumentationController.java +++ b/src/main/java/com/williamcallahan/javachat/web/ErrorDocumentationController.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; - import org.springframework.core.io.ClassPathResource; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -36,9 +35,8 @@ public class ErrorDocumentationController { * @return HTML for the error catalog */ @GetMapping( - value = {ROOT_PATH, ROOT_SLASH}, - produces = MediaType.TEXT_HTML_VALUE - ) + value = {ROOT_PATH, ROOT_SLASH}, + produces = MediaType.TEXT_HTML_VALUE) public ResponseEntity index() { return serveHtmlFile(INDEX_FILE); } diff --git a/src/main/java/com/williamcallahan/javachat/web/ErrorTestController.java b/src/main/java/com/williamcallahan/javachat/web/ErrorTestController.java index 4720337..ea352a2 100644 --- a/src/main/java/com/williamcallahan/javachat/web/ErrorTestController.java +++ b/src/main/java/com/williamcallahan/javachat/web/ErrorTestController.java @@ -1,6 +1,7 @@ package com.williamcallahan.javachat.web; import jakarta.annotation.security.PermitAll; +import org.springframework.context.annotation.Profile; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; @@ -12,15 +13,16 @@ /** * Test controller for demonstrating error pages. * This controller provides endpoints to test different error scenarios. - * + * * IMPORTANT: This controller should be removed or disabled in production. */ @Controller @RequestMapping("/test-errors") +@Profile("!prod") @PermitAll @PreAuthorize("permitAll()") public class ErrorTestController { - + /** * Test 404 error by accessing a non-existent page. * URL: /test-errors/404 @@ -29,7 +31,7 @@ public class ErrorTestController { public String test404() { throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Test 404 error page"); } - + /** * Test 500 internal server error. * URL: /test-errors/500 @@ -38,7 +40,7 @@ public String test404() { public String test500() { throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Test internal server error"); } - + /** * Test 400 bad request error. * URL: /test-errors/400 @@ -47,7 +49,7 @@ public String test500() { public String test400() { throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Test bad request error"); } - + /** * Test 403 forbidden error. * URL: /test-errors/403 @@ -56,7 +58,7 @@ public String test400() { public String test403() { throw new ResponseStatusException(HttpStatus.FORBIDDEN, "Test forbidden access error"); } - + /** * Test 503 service unavailable error. * URL: /test-errors/503 @@ -65,7 +67,7 @@ public String test403() { public String test503() { throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Test service unavailable error"); } - + /** * Test generic error with custom status code. * URL: /test-errors/{statusCode} @@ -78,16 +80,17 @@ public String testCustomError(@PathVariable int statusCode) { } throw new ResponseStatusException(status, "Test error with status code " + statusCode); } - + /** * Test runtime exception (will result in 500 error). * URL: /test-errors/runtime-exception */ @GetMapping("/runtime-exception") public String testRuntimeException() { - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Test runtime exception for error handling"); + throw new ResponseStatusException( + HttpStatus.INTERNAL_SERVER_ERROR, "Test runtime exception for error handling"); } - + /** * Test null pointer exception (will result in 500 error). * URL: /test-errors/null-pointer diff --git a/src/main/java/com/williamcallahan/javachat/web/ExceptionResponseBuilder.java b/src/main/java/com/williamcallahan/javachat/web/ExceptionResponseBuilder.java index 07f8d81..be75c24 100644 --- a/src/main/java/com/williamcallahan/javachat/web/ExceptionResponseBuilder.java +++ b/src/main/java/com/williamcallahan/javachat/web/ExceptionResponseBuilder.java @@ -1,6 +1,7 @@ package com.williamcallahan.javachat.web; import com.openai.errors.OpenAIServiceException; +import java.util.Optional; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -8,7 +9,6 @@ import org.springframework.web.client.RestClientResponseException; import org.springframework.web.reactive.function.client.WebClientResponseException; import org.springframework.web.server.ResponseStatusException; -import java.util.Optional; /** * Centralized utility for building consistent error responses across controllers. @@ -25,8 +25,7 @@ public class ExceptionResponseBuilder { * @return ResponseEntity with error details */ public ResponseEntity buildErrorResponse(HttpStatus status, String message) { - return ResponseEntity.status(status) - .body(ApiErrorResponse.error(message)); + return ResponseEntity.status(status).body(ApiErrorResponse.error(message)); } /** @@ -38,8 +37,7 @@ public ResponseEntity buildErrorResponse(HttpStatus status, St * @return ResponseEntity with error details */ public ResponseEntity buildErrorResponse(HttpStatus status, String message, Exception exception) { - return ResponseEntity.status(status) - .body(ApiErrorResponse.error(message, describeException(exception))); + return ResponseEntity.status(status).body(ApiErrorResponse.error(message, describeException(exception))); } /** @@ -87,15 +85,15 @@ public String describeException(Exception exception) { private void appendRestClientDetails(StringBuilder details, RestClientResponseException exception) { details.append(" [httpStatus=").append(exception.getStatusCode().value()); String statusText = exception.getStatusText(); - if (statusText != null && !statusText.isBlank()) { + if (!statusText.isBlank()) { details.append(" ").append(statusText); } String responseBody = exception.getResponseBodyAsString(); if (!responseBody.isBlank()) { details.append(", body=").append(responseBody); } - HttpHeaders headers = Optional.ofNullable(exception.getResponseHeaders()) - .orElseGet(HttpHeaders::new); + HttpHeaders headers = + Optional.ofNullable(exception.getResponseHeaders()).orElseGet(HttpHeaders::new); if (!headers.isEmpty()) { details.append(", headers=").append(headers); } @@ -105,11 +103,11 @@ private void appendRestClientDetails(StringBuilder details, RestClientResponseEx private void appendWebClientDetails(StringBuilder details, WebClientResponseException exception) { details.append(" [httpStatus=").append(exception.getStatusCode().value()); String statusText = exception.getStatusText(); - if (statusText != null && !statusText.isBlank()) { + if (!statusText.isBlank()) { details.append(" ").append(statusText); } String responseBody = exception.getResponseBodyAsString(); - if (responseBody != null && !responseBody.isBlank()) { + if (!responseBody.isBlank()) { details.append(", body=").append(responseBody); } HttpHeaders headers = exception.getHeaders(); @@ -122,15 +120,13 @@ private void appendWebClientDetails(StringBuilder details, WebClientResponseExce private void appendOpenAiDetails(StringBuilder details, OpenAIServiceException exception) { details.append(" [httpStatus=").append(exception.statusCode()); var headers = exception.headers(); - if (headers != null && !headers.isEmpty()) { + if (!headers.isEmpty()) { details.append(", headers=").append(headers); } var bodyJson = exception.body(); - if (bodyJson != null) { - String body = bodyJson.toString(); - if (!body.isBlank()) { - details.append(", body=").append(body); - } + String body = bodyJson.toString(); + if (!body.isBlank()) { + details.append(", body=").append(body); } exception.code().ifPresent(code -> details.append(", code=").append(code)); exception.param().ifPresent(param -> details.append(", param=").append(param)); @@ -139,6 +135,8 @@ private void appendOpenAiDetails(StringBuilder details, OpenAIServiceException e } private void appendStatusExceptionDetails(StringBuilder details, ResponseStatusException exception) { - details.append(" [httpStatus=").append(exception.getStatusCode().value()).append("]"); + details.append(" [httpStatus=") + .append(exception.getStatusCode().value()) + .append("]"); } } diff --git a/src/main/java/com/williamcallahan/javachat/web/GuidedLearningController.java b/src/main/java/com/williamcallahan/javachat/web/GuidedLearningController.java index 438ccd5..3e75bed 100644 --- a/src/main/java/com/williamcallahan/javachat/web/GuidedLearningController.java +++ b/src/main/java/com/williamcallahan/javachat/web/GuidedLearningController.java @@ -1,28 +1,26 @@ package com.williamcallahan.javachat.web; -import com.williamcallahan.javachat.domain.prompt.StructuredPrompt; +import static com.williamcallahan.javachat.web.SseConstants.*; + import com.williamcallahan.javachat.model.Citation; import com.williamcallahan.javachat.model.Enrichment; import com.williamcallahan.javachat.model.GuidedLesson; import com.williamcallahan.javachat.service.ChatMemoryService; import com.williamcallahan.javachat.service.GuidedLearningService; +import com.williamcallahan.javachat.service.MarkdownService; import com.williamcallahan.javachat.service.OpenAIStreamingService; import jakarta.annotation.security.PermitAll; - +import jakarta.servlet.http.HttpServletResponse; +import java.time.Duration; +import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.http.codec.ServerSentEvent; import org.springframework.security.access.prepost.PreAuthorize; -import jakarta.servlet.http.HttpServletResponse; import org.springframework.web.bind.annotation.*; import reactor.core.publisher.Flux; -import com.williamcallahan.javachat.service.MarkdownService; - -import java.util.*; -import java.time.Duration; - -import static com.williamcallahan.javachat.web.SseConstants.*; /** * Exposes guided learning endpoints for lesson metadata, citations, enrichment, and streaming lesson content. @@ -46,12 +44,13 @@ public class GuidedLearningController extends BaseController { /** * Creates the guided learning controller wired to the guided learning orchestration services. */ - public GuidedLearningController(GuidedLearningService guidedService, - ChatMemoryService chatMemory, - OpenAIStreamingService openAIStreamingService, - ExceptionResponseBuilder exceptionBuilder, - MarkdownService markdownService, - SseSupport sseSupport) { + public GuidedLearningController( + GuidedLearningService guidedService, + ChatMemoryService chatMemory, + OpenAIStreamingService openAIStreamingService, + ExceptionResponseBuilder exceptionBuilder, + MarkdownService markdownService, + SseSupport sseSupport) { super(exceptionBuilder); this.guidedService = guidedService; this.chatMemory = chatMemory; @@ -79,7 +78,9 @@ public List toc() { */ @GetMapping("/lesson") public GuidedLesson lesson(@RequestParam("slug") String slug) { - return guidedService.getLesson(slug).orElseThrow(() -> new NoSuchElementException("Unknown lesson slug: " + slug)); + return guidedService + .getLesson(slug) + .orElseThrow(() -> new NoSuchElementException("Unknown lesson slug: " + slug)); } /** @@ -120,7 +121,7 @@ public Flux streamLesson(@RequestParam("slug") String slug) { // Return raw content and let Spring handle SSE formatting automatically return Flux.just(payload); } - + // Stream raw content and let Spring handle SSE formatting automatically return guidedService.streamLessonContent(slug); } @@ -138,14 +139,7 @@ public LessonContentResponse content(@RequestParam("slug") String slug) { if (cached.isPresent()) { return new LessonContentResponse(cached.get(), true); } - // Generate synchronously (best-effort) and cache - List chunks = guidedService.streamLessonContent(slug).collectList().block(LESSON_CONTENT_TIMEOUT); - if (chunks == null || chunks.isEmpty()) { - log.error("Content generation timed out or returned empty for lesson"); - throw new IllegalStateException("Content generation failed for lesson"); - } - String md = String.join("", chunks); - guidedService.putLessonCache(slug, md); + String md = generateAndCacheLessonContent(slug); return new LessonContentResponse(md, false); } @@ -159,19 +153,29 @@ public LessonContentResponse content(@RequestParam("slug") String slug) { @GetMapping(value = "/content/html", produces = MediaType.TEXT_HTML_VALUE) public String contentHtml(@RequestParam("slug") String slug) { var cached = guidedService.getCachedLessonMarkdown(slug); - String md = cached.orElseGet(() -> { - List chunks = guidedService.streamLessonContent(slug).collectList().block(LESSON_CONTENT_TIMEOUT); - if (chunks == null || chunks.isEmpty()) { - log.error("Content generation timed out or returned empty for lesson HTML"); - throw new IllegalStateException("Content generation failed for lesson"); - } - String text = String.join("", chunks); - guidedService.putLessonCache(slug, text); - return text; - }); + String md = cached.orElseGet(() -> generateAndCacheLessonContent(slug)); return markdownService.processStructured(md).html(); } + /** + * Generates lesson content synchronously and caches the result. + * + * @param slug lesson identifier + * @return generated markdown content + * @throws IllegalStateException if generation times out or returns empty + */ + private String generateAndCacheLessonContent(String slug) { + List chunks = + guidedService.streamLessonContent(slug).collectList().block(LESSON_CONTENT_TIMEOUT); + if (chunks == null || chunks.isEmpty()) { + log.error("Content generation timed out or returned empty for lesson"); + throw new IllegalStateException("Content generation failed for lesson"); + } + String content = String.join("", chunks); + guidedService.putLessonCache(slug, content); + return content; + } + /** * Streams a response to a user's chat message within the context of a guided lesson. * Uses the same JSON-wrapped SSE format as ChatController for consistent whitespace handling. @@ -180,53 +184,71 @@ public String contentHtml(@RequestParam("slug") String slug) { * @return A {@link Flux} of ServerSentEvents with JSON-wrapped text chunks. */ @PostMapping(value = "/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE) - public Flux> stream(@RequestBody GuidedStreamRequest request, HttpServletResponse response) { + public Flux> stream( + @RequestBody GuidedStreamRequest request, HttpServletResponse response) { sseSupport.configureStreamingHeaders(response); String sessionId = request.resolvedSessionId(); - String userQuery = request.userQuery(); - String lessonSlug = request.lessonSlug(); + String userQuery = + request.userQuery().orElseThrow(() -> new IllegalArgumentException("User query is required")); + String lessonSlug = + request.lessonSlug().orElseThrow(() -> new IllegalArgumentException("Lesson slug is required")); - // Load history BEFORE adding user message to avoid duplication in prompt - // (buildGuidedPromptWithContext adds latestUserMessage separately) + if (!openAIStreamingService.isAvailable()) { + log.warn("OpenAI streaming service unavailable for guided session"); + return sseSupport.sseError("Service temporarily unavailable", "The streaming service is not ready"); + } + + return streamGuidedResponse(sessionId, userQuery, lessonSlug); + } + + /** + * Builds and streams the guided lesson response via OpenAI. + * + * @param sessionId chat session identifier + * @param userQuery user's question + * @param lessonSlug lesson context identifier + * @return SSE stream of response chunks with heartbeats + */ + private Flux> streamGuidedResponse(String sessionId, String userQuery, String lessonSlug) { List history = new ArrayList<>(chatMemory.getHistory(sessionId)); chatMemory.addUser(sessionId, userQuery); StringBuilder fullResponse = new StringBuilder(); - // Use OpenAI streaming only (legacy fallback removed) - if (openAIStreamingService.isAvailable()) { - // Build structured prompt for intelligent truncation - StructuredPrompt structuredPrompt = - guidedService.buildStructuredGuidedPromptWithContext(history, lessonSlug, userQuery); - - // Stream with structure-aware truncation - preserves semantic boundaries - Flux dataStream = sseSupport.prepareDataStream( - openAIStreamingService.streamResponse(structuredPrompt, DEFAULT_TEMPERATURE), - chunk -> fullResponse.append(chunk)); - - // Heartbeats terminate when data stream completes - Flux> heartbeats = sseSupport.heartbeats(dataStream); - - // Wrap chunks in JSON to preserve whitespace - Flux> dataEvents = dataStream.map(sseSupport::textEvent); - - return Flux.merge(dataEvents, heartbeats) - .doOnComplete(() -> chatMemory.addAssistant(sessionId, fullResponse.toString())) - .onErrorResume(error -> { - // SSE streams require error events rather than thrown exceptions. - // Log full details server-side, send sanitized message to client. - String errorType = error.getClass().getSimpleName(); - log.error("Guided streaming error (exception type: {})", errorType, error); - return sseSupport.sseError( + GuidedLearningService.GuidedChatPromptOutcome promptOutcome = + guidedService.buildStructuredGuidedPromptWithContext(history, lessonSlug, userQuery); + + Flux dataStream = sseSupport.prepareDataStream( + openAIStreamingService.streamResponse(promptOutcome.structuredPrompt(), DEFAULT_TEMPERATURE), + fullResponse::append); + + Flux> heartbeats = sseSupport.heartbeats(dataStream); + Flux> dataEvents = dataStream.map(sseSupport::textEvent); + + Flux> citationEvent = Flux.defer(() -> { + List citations = guidedService.citationsForBookDocuments(promptOutcome.bookContextDocuments()); + return Flux.just(sseSupport.citationEvent(citations)); + }); + + return Flux.concat(Flux.merge(dataEvents, heartbeats), citationEvent) + .doOnComplete(() -> chatMemory.addAssistant(sessionId, fullResponse.toString())) + .onErrorResume(error -> { + String errorType = error.getClass().getSimpleName(); + log.error("Guided streaming error"); + return sseSupport.sseError( "Streaming error: " + errorType, - "The response stream encountered an error. Please try again." - ); - }); + "The response stream encountered an error. Please try again."); + }); + } - } else { - // Service unavailable - send structured error event - log.warn("OpenAI streaming service unavailable for guided session"); - return sseSupport.sseError("Service temporarily unavailable", "The streaming service is not ready"); - } + /** + * Maps validation exceptions from missing request fields to HTTP 400 responses. + * + * @param validationException the validation exception with the error details + * @return standardized bad request error response + */ + @ExceptionHandler(IllegalArgumentException.class) + public ResponseEntity handleValidationException(IllegalArgumentException validationException) { + return super.handleValidationException(validationException); } } diff --git a/src/main/java/com/williamcallahan/javachat/web/GuidedStreamRequest.java b/src/main/java/com/williamcallahan/javachat/web/GuidedStreamRequest.java index 53aed52..2ea9045 100644 --- a/src/main/java/com/williamcallahan/javachat/web/GuidedStreamRequest.java +++ b/src/main/java/com/williamcallahan/javachat/web/GuidedStreamRequest.java @@ -1,5 +1,7 @@ package com.williamcallahan.javachat.web; +import java.util.Optional; + /** * Request body for guided learning streaming endpoint. * @@ -7,11 +9,7 @@ * @param slug The lesson slug identifier * @param latest The user's chat message */ -public record GuidedStreamRequest( - String sessionId, - String slug, - String latest -) { +public record GuidedStreamRequest(String sessionId, String slug, String latest) { /** * Returns a valid session ID, using default if not provided. */ @@ -23,16 +21,23 @@ public String resolvedSessionId() { } /** - * Returns the user query, defaulting to empty string if null. + * Returns the user query when present and non-blank. + * + *

Callers should use {@link Optional#orElseThrow} or {@link Optional#orElse} + * to handle the missing case explicitly, avoiding silent empty-string defaults.

+ * + * @return the user's query if present and non-blank */ - public String userQuery() { - return latest != null ? latest : ""; + public Optional userQuery() { + return Optional.ofNullable(latest).filter(s -> !s.isBlank()); } /** - * Returns the lesson slug, defaulting to empty string if null. + * Returns the lesson slug when present and non-blank. + * + * @return the lesson slug if present and non-blank */ - public String lessonSlug() { - return slug != null ? slug : ""; + public Optional lessonSlug() { + return Optional.ofNullable(slug).filter(s -> !s.isBlank()); } } diff --git a/src/main/java/com/williamcallahan/javachat/web/IngestionController.java b/src/main/java/com/williamcallahan/javachat/web/IngestionController.java index e442d1c..7c84f78 100644 --- a/src/main/java/com/williamcallahan/javachat/web/IngestionController.java +++ b/src/main/java/com/williamcallahan/javachat/web/IngestionController.java @@ -2,6 +2,9 @@ import com.williamcallahan.javachat.service.DocsIngestionService; import jakarta.annotation.security.PermitAll; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.ResponseEntity; @@ -12,10 +15,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import jakarta.validation.constraints.Max; -import jakarta.validation.constraints.Min; -import java.io.IOException; - /** * Exposes ingestion endpoints for crawling remote docs and ingesting local directories. */ @@ -42,11 +41,11 @@ public IngestionController(DocsIngestionService docsIngestionService, ExceptionR */ @PostMapping public ResponseEntity ingest( - @RequestParam(name = "maxPages", defaultValue = "100") - @Min(value = 1, message = "maxPages must be at least 1") - @Max(value = MAX_ALLOWED_PAGES, message = "maxPages cannot exceed " + MAX_ALLOWED_PAGES) - int maxPages) { - + @RequestParam(name = "maxPages", defaultValue = "100") + @Min(value = 1, message = "maxPages must be at least 1") + @Max(value = MAX_ALLOWED_PAGES, message = "maxPages cannot exceed " + MAX_ALLOWED_PAGES) + int maxPages) { + try { log.info("Starting ingestion for up to {} pages", maxPages); docsIngestionService.crawlAndIngest(maxPages); @@ -54,43 +53,45 @@ public ResponseEntity ingest( return createSuccessResponse(String.format("Ingestion completed for up to %d pages", maxPages)); } catch (IOException ioException) { - log.error("IO error during ingestion (exception type: {})", ioException.getClass().getSimpleName()); + log.error( + "IO error during ingestion (exception type: {})", + ioException.getClass().getSimpleName()); return handleServiceException(ioException, "ingest documents"); } catch (RuntimeException runtimeException) { - log.error("Unexpected error during ingestion (exception type: {})", - runtimeException.getClass().getSimpleName()); + log.error( + "Unexpected error during ingestion (exception type: {})", + runtimeException.getClass().getSimpleName()); return handleServiceException(runtimeException, "perform ingestion"); } } - + /** * Ingests documents from a local directory, primarily for offline or development workflows. */ @PostMapping("/local") public ResponseEntity ingestLocal( @RequestParam(name = "dir", defaultValue = "data/docs") String directory, - @RequestParam(name = "maxFiles", defaultValue = "50000") - @Min(1) @Max(1000000) int maxFiles) { + @RequestParam(name = "maxFiles", defaultValue = "50000") @Min(1) @Max(1000000) int maxFiles) { try { DocsIngestionService.LocalIngestionOutcome outcome = - docsIngestionService.ingestLocalDirectory(directory, maxFiles); - return ResponseEntity.ok(IngestionLocalResponse.success( - outcome.processedCount(), - directory, - outcome.failures() - )); + docsIngestionService.ingestLocalDirectory(directory, maxFiles); + return ResponseEntity.ok( + IngestionLocalResponse.success(outcome.processedCount(), directory, outcome.failures())); } catch (IllegalArgumentException illegalArgumentException) { return handleValidationException(illegalArgumentException); } catch (IOException ioException) { - log.error("Local ingestion IO error (exception type: {})", ioException.getClass().getSimpleName()); + log.error( + "Local ingestion IO error (exception type: {})", + ioException.getClass().getSimpleName()); return handleServiceException(ioException, "perform local ingestion"); } catch (RuntimeException runtimeException) { - log.error("Local ingestion error (exception type: {})", - runtimeException.getClass().getSimpleName()); + log.error( + "Local ingestion error (exception type: {})", + runtimeException.getClass().getSimpleName()); return handleServiceException(runtimeException, "perform local ingestion"); } } - + /** * Returns a standardized validation error response for invalid ingestion request parameters. */ diff --git a/src/main/java/com/williamcallahan/javachat/web/IngestionLocalResponse.java b/src/main/java/com/williamcallahan/javachat/web/IngestionLocalResponse.java index 0b11ad4..02f1348 100644 --- a/src/main/java/com/williamcallahan/javachat/web/IngestionLocalResponse.java +++ b/src/main/java/com/williamcallahan/javachat/web/IngestionLocalResponse.java @@ -11,12 +11,13 @@ * @param dir ingested directory path * @param failures per-file failures encountered during ingestion */ -public record IngestionLocalResponse(String status, - int processed, - String dir, - List failures) +public record IngestionLocalResponse(String status, int processed, String dir, List failures) implements ApiResponse { + public IngestionLocalResponse { + failures = failures == null ? List.of() : List.copyOf(failures); + } + /** * Creates a local ingestion success response. * @@ -25,11 +26,16 @@ public record IngestionLocalResponse(String status, * @param failures per-file failures encountered during ingestion * @return standardized local ingestion payload */ - public static IngestionLocalResponse success(int processed, - String dir, - List failures) { + public static IngestionLocalResponse success(int processed, String dir, List failures) { String status = failures == null || failures.isEmpty() ? "success" : "partial-success"; - return new IngestionLocalResponse(status, processed, dir, - failures == null ? List.of() : List.copyOf(failures)); + return new IngestionLocalResponse(status, processed, dir, failures); + } + + /** + * Returns per-file ingestion failures as an unmodifiable snapshot. + */ + @Override + public List failures() { + return List.copyOf(failures); } } diff --git a/src/main/java/com/williamcallahan/javachat/web/LessonContentResponse.java b/src/main/java/com/williamcallahan/javachat/web/LessonContentResponse.java index eaefa23..307a95e 100644 --- a/src/main/java/com/williamcallahan/javachat/web/LessonContentResponse.java +++ b/src/main/java/com/williamcallahan/javachat/web/LessonContentResponse.java @@ -6,7 +6,4 @@ * @param markdown The lesson markdown content * @param cached Whether the content was served from cache */ -public record LessonContentResponse( - String markdown, - boolean cached -) {} +public record LessonContentResponse(String markdown, boolean cached) {} diff --git a/src/main/java/com/williamcallahan/javachat/web/MarkdownController.java b/src/main/java/com/williamcallahan/javachat/web/MarkdownController.java index 04a69cb..991371b 100644 --- a/src/main/java/com/williamcallahan/javachat/web/MarkdownController.java +++ b/src/main/java/com/williamcallahan/javachat/web/MarkdownController.java @@ -2,12 +2,12 @@ import com.williamcallahan.javachat.domain.markdown.MarkdownCacheClearOutcome; import com.williamcallahan.javachat.domain.markdown.MarkdownCacheClearResponse; -import com.williamcallahan.javachat.domain.markdown.MarkdownCacheStatsSnapshot; import com.williamcallahan.javachat.domain.markdown.MarkdownCacheStatsResponse; +import com.williamcallahan.javachat.domain.markdown.MarkdownCacheStatsSnapshot; import com.williamcallahan.javachat.domain.markdown.MarkdownErrorResponse; import com.williamcallahan.javachat.domain.markdown.MarkdownRenderOutcome; -import com.williamcallahan.javachat.domain.markdown.MarkdownRenderResponse; import com.williamcallahan.javachat.domain.markdown.MarkdownRenderRequest; +import com.williamcallahan.javachat.domain.markdown.MarkdownRenderResponse; import com.williamcallahan.javachat.domain.markdown.MarkdownStructuredErrorResponse; import com.williamcallahan.javachat.domain.markdown.MarkdownStructuredOutcome; import com.williamcallahan.javachat.domain.markdown.MarkdownStructuredResponse; @@ -30,13 +30,13 @@ @PermitAll @PreAuthorize("permitAll()") public class MarkdownController { - + private static final Logger logger = LoggerFactory.getLogger(MarkdownController.class); - + private final MarkdownService markdownService; private final UnifiedMarkdownService unifiedMarkdownService; private final ExceptionResponseBuilder exceptionBuilder; - + /** * Creates a markdown controller with required services. * @@ -44,14 +44,15 @@ public class MarkdownController { * @param unifiedMarkdownService AST-based unified markdown processor * @param exceptionBuilder shared exception response builder */ - public MarkdownController(MarkdownService markdownService, - UnifiedMarkdownService unifiedMarkdownService, - ExceptionResponseBuilder exceptionBuilder) { + public MarkdownController( + MarkdownService markdownService, + UnifiedMarkdownService unifiedMarkdownService, + ExceptionResponseBuilder exceptionBuilder) { this.markdownService = markdownService; this.unifiedMarkdownService = unifiedMarkdownService; this.exceptionBuilder = exceptionBuilder; } - + /** * Renders markdown text to HTML. This endpoint uses server-side caching to improve performance * for frequently rendered content. @@ -64,40 +65,35 @@ public MarkdownController(MarkdownService markdownService, * }
* @return A {@link ResponseEntity} containing the rendered markdown outcome. */ - @PostMapping(value = "/render", - consumes = MediaType.APPLICATION_JSON_VALUE, - produces = MediaType.APPLICATION_JSON_VALUE) + @PostMapping( + value = "/render", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity renderMarkdown(@RequestBody MarkdownRenderRequest renderRequest) { try { if (renderRequest.isBlank()) { - return ResponseEntity.ok(new MarkdownRenderOutcome( - "", - "server", - false, - 0, - 0 - )); + return ResponseEntity.ok(new MarkdownRenderOutcome("", "server", false, 0, 0)); } - - logger.debug("Processing markdown of length: {}", renderRequest.content().length()); - + + logger.debug( + "Processing markdown of length: {}", renderRequest.content().length()); + var processed = markdownService.processStructured(renderRequest.content()); - + return ResponseEntity.ok(new MarkdownRenderOutcome( - processed.html(), - "server", - false, - processed.citations().size(), - processed.enrichments().size() - )); - + processed.html(), + "server", + false, + processed.citations().size(), + processed.enrichments().size())); + } catch (RuntimeException renderException) { - logger.error("Error rendering markdown (exception type: {})", - renderException.getClass().getSimpleName()); - return ResponseEntity.status(500).body(new MarkdownErrorResponse( - "Failed to render markdown", - exceptionBuilder.describeException(renderException) - )); + logger.error( + "Error rendering markdown (exception type: {})", + renderException.getClass().getSimpleName()); + return ResponseEntity.status(500) + .body(new MarkdownErrorResponse( + "Failed to render markdown", exceptionBuilder.describeException(renderException))); } } @@ -113,41 +109,35 @@ public ResponseEntity renderMarkdown(@RequestBody Markdo * }
* @return A {@link ResponseEntity} containing the rendered markdown outcome. */ - @PostMapping(value = "/preview", - consumes = MediaType.APPLICATION_JSON_VALUE, - produces = MediaType.APPLICATION_JSON_VALUE) + @PostMapping( + value = "/preview", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity previewMarkdown(@RequestBody MarkdownRenderRequest renderRequest) { try { if (renderRequest.isBlank()) { - return ResponseEntity.ok(new MarkdownRenderOutcome( - "", - "preview", - false, - 0, - 0 - )); + return ResponseEntity.ok(new MarkdownRenderOutcome("", "preview", false, 0, 0)); } - + var processed = markdownService.processStructured(renderRequest.content()); - + return ResponseEntity.ok(new MarkdownRenderOutcome( - processed.html(), - "preview", - false, - processed.citations().size(), - processed.enrichments().size() - )); - + processed.html(), + "preview", + false, + processed.citations().size(), + processed.enrichments().size())); + } catch (RuntimeException previewException) { - logger.error("Error rendering preview markdown (exception type: {})", - previewException.getClass().getSimpleName()); - return ResponseEntity.status(500).body(new MarkdownErrorResponse( - "Failed to render preview", - exceptionBuilder.describeException(previewException) - )); + logger.error( + "Error rendering preview markdown (exception type: {})", + previewException.getClass().getSimpleName()); + return ResponseEntity.status(500) + .body(new MarkdownErrorResponse( + "Failed to render preview", exceptionBuilder.describeException(previewException))); } } - + /** * Retrieves statistics about the server-side markdown render cache. * Provides metrics like hit count, miss count, size, and hit rate. @@ -158,25 +148,24 @@ public ResponseEntity previewMarkdown(@RequestBody Markd public ResponseEntity getCacheStats() { try { var stats = unifiedMarkdownService.getCacheStats(); - + return ResponseEntity.ok(new MarkdownCacheStatsSnapshot( - stats.hitCount(), - stats.missCount(), - stats.evictionCount(), - stats.size(), - String.format("%.2f%%", stats.hitRate() * 100) - )); - + stats.hitCount(), + stats.missCount(), + stats.evictionCount(), + stats.size(), + String.format("%.2f%%", stats.hitRate() * 100))); + } catch (RuntimeException statsException) { - logger.error("Error getting cache stats (exception type: {})", - statsException.getClass().getSimpleName()); - return ResponseEntity.status(500).body(new MarkdownErrorResponse( - "Failed to get cache stats", - exceptionBuilder.describeException(statsException) - )); + logger.error( + "Error getting cache stats (exception type: {})", + statsException.getClass().getSimpleName()); + return ResponseEntity.status(500) + .body(new MarkdownErrorResponse( + "Failed to get cache stats", exceptionBuilder.describeException(statsException))); } } - + /** * Clears the server-side markdown render cache. * @@ -187,70 +176,70 @@ public ResponseEntity clearCache() { try { unifiedMarkdownService.clearCache(); logger.info("Markdown cache cleared via API"); - - return ResponseEntity.ok(new MarkdownCacheClearOutcome( - "success", - "Cache cleared successfully" - )); - + + return ResponseEntity.ok(new MarkdownCacheClearOutcome("success", "Cache cleared successfully")); + } catch (RuntimeException clearException) { - logger.error("Error clearing cache (exception type: {})", - clearException.getClass().getSimpleName()); - return ResponseEntity.status(500).body(new MarkdownCacheClearOutcome( - "error", - "Failed to clear cache: " + exceptionBuilder.describeException(clearException) - )); + logger.error( + "Error clearing cache (exception type: {})", + clearException.getClass().getSimpleName()); + return ResponseEntity.status(500) + .body(new MarkdownCacheClearOutcome( + "error", "Failed to clear cache: " + exceptionBuilder.describeException(clearException))); } } - + /** * Renders markdown using the new AST-based UnifiedMarkdownService directly. * This endpoint provides structured output with type-safe citations and enrichments. - * + * * @param request A JSON object containing the markdown to render * @return A {@link ResponseEntity} with structured markdown processing results */ - @PostMapping(value = "/render/structured", - consumes = MediaType.APPLICATION_JSON_VALUE, - produces = MediaType.APPLICATION_JSON_VALUE) - public ResponseEntity renderStructured(@RequestBody MarkdownRenderRequest renderRequest) { + @PostMapping( + value = "/render/structured", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity renderStructured( + @RequestBody MarkdownRenderRequest renderRequest) { try { if (renderRequest.isBlank()) { return ResponseEntity.ok(new MarkdownStructuredOutcome( - "", - java.util.List.of(), - java.util.List.of(), - java.util.List.of(), - 0L, - "unified-service", - 0, - true - )); + "", + java.util.List.of(), + java.util.List.of(), + java.util.List.of(), + 0L, + "unified-service", + 0, + true)); } - - logger.debug("Processing markdown with UnifiedMarkdownService, length: {}", renderRequest.content().length()); - + + logger.debug( + "Processing markdown with UnifiedMarkdownService, length: {}", + renderRequest.content().length()); + var processed = unifiedMarkdownService.process(renderRequest.content()); - + return ResponseEntity.ok(new MarkdownStructuredOutcome( - processed.html(), - processed.citations(), - processed.enrichments(), - processed.warnings(), - processed.processingTimeMs(), - "unified-service", - processed.getStructuredElementCount(), - processed.isClean() - )); - + processed.html(), + processed.citations(), + processed.enrichments(), + processed.warnings(), + processed.processingTimeMs(), + "unified-service", + processed.getStructuredElementCount(), + processed.isClean())); + } catch (RuntimeException structuredException) { - logger.error("Error rendering structured markdown (exception type: {})", - structuredException.getClass().getSimpleName()); - return ResponseEntity.status(500).body(new MarkdownStructuredErrorResponse( - "Failed to render structured markdown", - "unified-service", - exceptionBuilder.describeException(structuredException) - )); + logger.error( + "Error rendering structured markdown (exception type: {})", + structuredException.getClass().getSimpleName()); + return ResponseEntity.status(500) + .body(new MarkdownStructuredErrorResponse( + "Failed to render structured markdown", + "unified-service", + exceptionBuilder.describeException(structuredException))); } } } diff --git a/src/main/java/com/williamcallahan/javachat/web/RetrievalDiagnosticsResponse.java b/src/main/java/com/williamcallahan/javachat/web/RetrievalDiagnosticsResponse.java index 00a7262..af71e92 100644 --- a/src/main/java/com/williamcallahan/javachat/web/RetrievalDiagnosticsResponse.java +++ b/src/main/java/com/williamcallahan/javachat/web/RetrievalDiagnosticsResponse.java @@ -9,10 +9,7 @@ * @param docs The list of retrieved citations * @param error Optional error message if retrieval failed */ -public record RetrievalDiagnosticsResponse( - List docs, - String error -) { +public record RetrievalDiagnosticsResponse(List docs, String error) { public RetrievalDiagnosticsResponse { docs = docs == null ? List.of() : List.copyOf(docs); } diff --git a/src/main/java/com/williamcallahan/javachat/web/RobotsController.java b/src/main/java/com/williamcallahan/javachat/web/RobotsController.java index efc9696..6186d37 100644 --- a/src/main/java/com/williamcallahan/javachat/web/RobotsController.java +++ b/src/main/java/com/williamcallahan/javachat/web/RobotsController.java @@ -2,7 +2,6 @@ import jakarta.annotation.security.PermitAll; import jakarta.servlet.http.HttpServletRequest; - import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; diff --git a/src/main/java/com/williamcallahan/javachat/web/SeoController.java b/src/main/java/com/williamcallahan/javachat/web/SeoController.java index f8f31f4..2d59f2d 100644 --- a/src/main/java/com/williamcallahan/javachat/web/SeoController.java +++ b/src/main/java/com/williamcallahan/javachat/web/SeoController.java @@ -32,19 +32,17 @@ public class SeoController { private final SiteUrlResolver siteUrlResolver; private final Map metadataMap = new ConcurrentHashMap<>(); - // Cache the parsed document to avoid re-reading files, but clone it per request to modify - private Document cachedIndexDocument; - - /** - * Creates the SEO controller using the built SPA index.html template and a base URL resolver. - */ - public SeoController( - @Value("classpath:/static/index.html") Resource indexHtml, - SiteUrlResolver siteUrlResolver) { - this.indexHtml = indexHtml; - this.siteUrlResolver = siteUrlResolver; - initMetadata(); - } + // Cache the parsed document to avoid re-reading files, but clone it per request to modify + private Document cachedIndexDocument; + + /** + * Creates the SEO controller using the built SPA index.html template and a base URL resolver. + */ + public SeoController(@Value("classpath:/static/index.html") Resource indexHtml, SiteUrlResolver siteUrlResolver) { + this.indexHtml = indexHtml; + this.siteUrlResolver = siteUrlResolver; + initMetadata(); + } private void initMetadata() { String defaultImage = "/mstile-310x310.png"; @@ -52,33 +50,34 @@ private void initMetadata() { PageMetadata base = new PageMetadata( "Java Chat - AI-Powered Java Learning With Citations", "Learn Java faster with an AI tutor: streaming answers, code examples, and citations to official docs.", - defaultImage - ); + defaultImage); metadataMap.put("/", base); - metadataMap.put("/chat", new PageMetadata( - "Java Chat - Streaming Java Tutor With Citations", - "Ask Java questions and get streaming answers with citations to official docs and practical examples.", - defaultImage - )); - + metadataMap.put( + "/chat", + new PageMetadata( + "Java Chat - Streaming Java Tutor With Citations", + "Ask Java questions and get streaming answers with citations to official docs and practical examples.", + defaultImage)); + PageMetadata guided = new PageMetadata( "Guided Java Learning - Java Chat", "Structured, step-by-step Java learning paths with examples and explanations.", - defaultImage - ); - metadataMap.put("/guided", guided); - metadataMap.put("/learn", guided); - } - - /** - * Serves the SPA index.html with path-specific SEO metadata for crawlers and social previews. - */ - @GetMapping(value = {"/", "/chat", "/guided", "/learn"}, produces = MediaType.TEXT_HTML_VALUE) - public ResponseEntity serveIndexWithSeo(HttpServletRequest request) { - try { - Document doc = getIndexDocument(); - + defaultImage); + metadataMap.put("/guided", guided); + metadataMap.put("/learn", guided); + } + + /** + * Serves the SPA index.html with path-specific SEO metadata for crawlers and social previews. + */ + @GetMapping( + value = {"/", "/chat", "/guided", "/learn"}, + produces = MediaType.TEXT_HTML_VALUE) + public ResponseEntity serveIndexWithSeo(HttpServletRequest request) { + try { + Document doc = getIndexDocument(); + String path = resolvePath(request); PageMetadata metadata = metadataMap.getOrDefault(path, metadataMap.get("/")); String baseUrl = siteUrlResolver.resolvePublicBaseUrl(request); @@ -106,7 +105,7 @@ private void updateDocumentMetadata(Document doc, PageMetadata metadata, String // Basic Metadata doc.title(metadata.title); setMeta(doc, "name", "description", metadata.description); - + // Open Graph setMeta(doc, "property", "og:title", metadata.title); setMeta(doc, "property", "og:description", metadata.description); @@ -120,7 +119,7 @@ private void updateDocumentMetadata(Document doc, PageMetadata metadata, String // Canonical Link updateCanonicalLink(doc, fullUrl); - + // Structured Data (JSON-LD) updateJsonLd(doc, fullUrl, metadata.description); } @@ -146,9 +145,8 @@ private void updateJsonLd(Document doc, String fullUrl, String description) { "applicationCategory": "EducationalApplication", "operatingSystem": "Web", "description": "__DESCRIPTION__" - }""" - .replace("__FULL_URL__", escapeJson(fullUrl)) - .replace("__DESCRIPTION__", escapeJson(description)); + }""".replace("__FULL_URL__", escapeJson(fullUrl)) + .replace("__DESCRIPTION__", escapeJson(description)); jsonLd.text(json); } } @@ -170,7 +168,7 @@ private void setMeta(Document doc, String attrKey, String attrValue, String cont doc.head().appendElement("meta").attr(attrKey, attrValue).attr("content", content); } } - + private String escapeJson(String input) { if (input == null) return ""; return input.replace("\"", "\\\""); diff --git a/src/main/java/com/williamcallahan/javachat/web/SessionValidationResponse.java b/src/main/java/com/williamcallahan/javachat/web/SessionValidationResponse.java index 345fc4f..b774fe8 100644 --- a/src/main/java/com/williamcallahan/javachat/web/SessionValidationResponse.java +++ b/src/main/java/com/williamcallahan/javachat/web/SessionValidationResponse.java @@ -9,9 +9,4 @@ * @param exists true if the session has any history on the server * @param message human-readable status message */ -public record SessionValidationResponse( - String sessionId, - int turnCount, - boolean exists, - String message) { -} +public record SessionValidationResponse(String sessionId, int turnCount, boolean exists, String message) {} diff --git a/src/main/java/com/williamcallahan/javachat/web/SitemapController.java b/src/main/java/com/williamcallahan/javachat/web/SitemapController.java index 3535371..b355a94 100644 --- a/src/main/java/com/williamcallahan/javachat/web/SitemapController.java +++ b/src/main/java/com/williamcallahan/javachat/web/SitemapController.java @@ -6,7 +6,6 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.util.List; - import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; @@ -28,12 +27,7 @@ public class SitemapController { private static final int INITIAL_BUFFER_CAPACITY = 512; private static final String SITEMAP_NAMESPACE = "http://www.sitemaps.org/schemas/sitemap/0.9"; - private static final List PUBLIC_ROUTES = List.of( - "/", - "/chat", - "/learn", - "/guided" - ); + private static final List PUBLIC_ROUTES = List.of("/", "/chat", "/learn", "/guided"); private final SiteUrlResolver siteUrlResolver; diff --git a/src/main/java/com/williamcallahan/javachat/web/SseSupport.java b/src/main/java/com/williamcallahan/javachat/web/SseSupport.java index 46a6b8a..063c76e 100644 --- a/src/main/java/com/williamcallahan/javachat/web/SseSupport.java +++ b/src/main/java/com/williamcallahan/javachat/web/SseSupport.java @@ -1,8 +1,13 @@ package com.williamcallahan.javachat.web; +import static com.williamcallahan.javachat.web.SseConstants.*; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import jakarta.servlet.http.HttpServletResponse; +import java.time.Duration; +import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpHeaders; @@ -10,11 +15,6 @@ import org.springframework.stereotype.Component; import reactor.core.publisher.Flux; -import java.time.Duration; -import java.util.function.Consumer; - -import static com.williamcallahan.javachat.web.SseConstants.*; - /** * Shared SSE support utilities for streaming controllers. * @@ -29,9 +29,9 @@ public class SseSupport { /** Fallback JSON payload when SSE error serialization fails. */ private static final String ERROR_FALLBACK_JSON = - "{\"message\":\"Error serialization failed\",\"details\":\"See server logs\"}"; + "{\"message\":\"Error serialization failed\",\"details\":\"See server logs\"}"; - private final ObjectMapper objectMapper; + private final ObjectWriter jsonWriter; /** * Creates SSE support wired to the application's ObjectMapper. @@ -39,7 +39,7 @@ public class SseSupport { * @param objectMapper JSON mapper for safe SSE serialization */ public SseSupport(ObjectMapper objectMapper) { - this.objectMapper = objectMapper; + this.jsonWriter = objectMapper.writer(); } /** @@ -62,11 +62,10 @@ public void configureStreamingHeaders(HttpServletResponse response) { * @return a shared flux configured for SSE streaming */ public Flux prepareDataStream(Flux source, Consumer chunkConsumer) { - return source - .filter(chunk -> chunk != null && !chunk.isEmpty()) - .doOnNext(chunkConsumer) - .onBackpressureBuffer(BACKPRESSURE_BUFFER_SIZE) - .share(); + return source.filter(chunk -> chunk != null && !chunk.isEmpty()) + .doOnNext(chunkConsumer) + .onBackpressureBuffer(BACKPRESSURE_BUFFER_SIZE) + .share(); } /** @@ -78,7 +77,7 @@ public Flux prepareDataStream(Flux source, Consumer chun */ public String jsonSerialize(Object payload) { try { - return objectMapper.writeValueAsString(payload); + return jsonWriter.writeValueAsString(payload); } catch (JsonProcessingException e) { throw new IllegalStateException("Failed to serialize SSE data", e); } @@ -94,15 +93,13 @@ public String jsonSerialize(Object payload) { public Flux> sseError(String message, String details) { String json; try { - json = objectMapper.writeValueAsString(new ErrorPayload(message, details)); + json = jsonWriter.writeValueAsString(new ErrorPayload(message, details)); } catch (JsonProcessingException e) { - log.error("Failed to serialize SSE error: {}", message, e); + log.error("Failed to serialize SSE error payload", e); json = ERROR_FALLBACK_JSON; } - return Flux.just(ServerSentEvent.builder() - .event(EVENT_ERROR) - .data(json) - .build()); + return Flux.just( + ServerSentEvent.builder().event(EVENT_ERROR).data(json).build()); } /** @@ -116,15 +113,13 @@ public Flux> sseError(String message, String details) { public Flux> sseStatus(String message, String details) { String json; try { - json = objectMapper.writeValueAsString(new StatusPayload(message, details)); + json = jsonWriter.writeValueAsString(new StatusPayload(message, details)); } catch (JsonProcessingException e) { - log.error("Failed to serialize SSE status: {}", message, e); + log.error("Failed to serialize SSE status payload", e); json = ERROR_FALLBACK_JSON; } - return Flux.just(ServerSentEvent.builder() - .event(EVENT_STATUS) - .data(json) - .build()); + return Flux.just( + ServerSentEvent.builder().event(EVENT_STATUS).data(json).build()); } /** @@ -136,8 +131,10 @@ public Flux> sseStatus(String message, String details) { */ public Flux> heartbeats(Flux terminateOn) { return Flux.interval(Duration.ofSeconds(HEARTBEAT_INTERVAL_SECONDS)) - .takeUntilOther(terminateOn.ignoreElements()) - .map(tick -> ServerSentEvent.builder().comment(COMMENT_KEEPALIVE).build()); + .takeUntilOther(terminateOn.ignoreElements()) + .map(tick -> ServerSentEvent.builder() + .comment(COMMENT_KEEPALIVE) + .build()); } /** @@ -149,9 +146,9 @@ public Flux> heartbeats(Flux terminateOn) { */ public ServerSentEvent textEvent(String chunk) { return ServerSentEvent.builder() - .event(EVENT_TEXT) - .data(jsonSerialize(new ChunkPayload(chunk))) - .build(); + .event(EVENT_TEXT) + .data(jsonSerialize(new ChunkPayload(chunk))) + .build(); } /** @@ -163,9 +160,9 @@ public ServerSentEvent textEvent(String chunk) { */ public ServerSentEvent statusEvent(String summary, String details) { return ServerSentEvent.builder() - .event(EVENT_STATUS) - .data(jsonSerialize(new StatusPayload(summary, details))) - .build(); + .event(EVENT_STATUS) + .data(jsonSerialize(new StatusPayload(summary, details))) + .build(); } /** @@ -176,9 +173,9 @@ public ServerSentEvent statusEvent(String summary, String details) { */ public ServerSentEvent citationEvent(Object citations) { return ServerSentEvent.builder() - .event(EVENT_CITATION) - .data(jsonSerialize(citations)) - .build(); + .event(EVENT_CITATION) + .data(jsonSerialize(citations)) + .build(); } /** Payload record for text chunks - preserves whitespace in JSON. */ diff --git a/src/main/java/com/williamcallahan/javachat/web/TooltipController.java b/src/main/java/com/williamcallahan/javachat/web/TooltipController.java index a1f7664..071ae5f 100644 --- a/src/main/java/com/williamcallahan/javachat/web/TooltipController.java +++ b/src/main/java/com/williamcallahan/javachat/web/TooltipController.java @@ -2,13 +2,12 @@ import com.williamcallahan.javachat.service.TooltipRegistry; import jakarta.annotation.security.PermitAll; +import java.util.List; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.util.List; - /** * Exposes tooltip glossary entries for the frontend. */ diff --git a/src/test/java/TestCompleteStreaming.java b/src/test/java/TestCompleteStreaming.java index f723ae0..6a3de6b 100644 --- a/src/test/java/TestCompleteStreaming.java +++ b/src/test/java/TestCompleteStreaming.java @@ -7,7 +7,6 @@ import com.openai.models.ReasoningEffort; import com.openai.models.chat.completions.ChatCompletionChunk; import com.openai.models.chat.completions.ChatCompletionCreateParams; - import java.time.Duration; /** @@ -15,7 +14,7 @@ */ public class TestCompleteStreaming { private static final String OPENAI_API_KEY = System.getenv("OPENAI_API_KEY"); - + /** * Runs the streaming extraction test against the OpenAI API. * @@ -26,45 +25,45 @@ public static void main(String[] args) { System.err.println("Please set OPENAI_API_KEY environment variable"); System.exit(1); } - + System.out.println("=== Testing Complete GPT-5.2 Streaming Pipeline ===\n"); - + StringBuilder fullResponse = new StringBuilder(); System.out.println("Sending request to GPT-5.2...\n"); System.out.println("=== STREAMING RESPONSE ==="); OpenAIClient client = OpenAIOkHttpClient.builder() - .apiKey(OPENAI_API_KEY) - .maxRetries(0) - .build(); + .apiKey(OPENAI_API_KEY) + .maxRetries(0) + .build(); Timeout timeout = Timeout.builder() - .request(Duration.ofSeconds(60)) - .read(Duration.ofSeconds(60)) - .build(); + .request(Duration.ofSeconds(60)) + .read(Duration.ofSeconds(60)) + .build(); - RequestOptions requestOptions = RequestOptions.builder() - .timeout(timeout) - .build(); + RequestOptions requestOptions = + RequestOptions.builder().timeout(timeout).build(); ChatCompletionCreateParams params = ChatCompletionCreateParams.builder() - .model(ChatModel.of("gpt-5.2")) - .maxCompletionTokens(200) - .reasoningEffort(ReasoningEffort.of("minimal")) - .addUserMessage("What is Spring Boot? Give a very brief answer.") - .build(); + .model(ChatModel.of("gpt-5.2")) + .maxCompletionTokens(200) + .reasoningEffort(ReasoningEffort.of("minimal")) + .addUserMessage("What is Spring Boot? Give a very brief answer.") + .build(); try (StreamResponse responseStream = - client.chat().completions().createStreaming(params, requestOptions)) { + client.chat().completions().createStreaming(params, requestOptions)) { responseStream.stream() - .flatMap(chunk -> chunk.choices().stream()) - .flatMap(choice -> choice.delta().content().stream()) - .forEach(contentChunk -> { - System.out.print(contentChunk); - fullResponse.append(contentChunk); - }); + .flatMap(chunk -> chunk.choices().stream()) + .flatMap(choice -> choice.delta().content().stream()) + .forEach(contentChunk -> { + System.out.print(contentChunk); + fullResponse.append(contentChunk); + }); } catch (RuntimeException streamingFailure) { - System.err.println("\nError during streaming: " + streamingFailure.getClass().getSimpleName()); + System.err.println( + "\nError during streaming: " + streamingFailure.getClass().getSimpleName()); } finally { client.close(); } @@ -76,7 +75,7 @@ public static void main(String[] args) { } else { System.out.println("SUCCESS: Content was properly extracted and displayed!"); } - + System.out.println("\nTest complete!"); } } diff --git a/src/test/java/TestGPT5Streaming.java b/src/test/java/TestGPT5Streaming.java index 385320c..41b1cbb 100644 --- a/src/test/java/TestGPT5Streaming.java +++ b/src/test/java/TestGPT5Streaming.java @@ -7,7 +7,6 @@ import com.openai.models.ReasoningEffort; import com.openai.models.chat.completions.ChatCompletionChunk; import com.openai.models.chat.completions.ChatCompletionCreateParams; - import java.time.Duration; /** @@ -15,7 +14,7 @@ */ public class TestGPT5Streaming { private static final String OPENAI_API_KEY = System.getenv("OPENAI_API_KEY"); - + /** * Runs the SSE probe against the OpenAI API. * @@ -26,46 +25,45 @@ public static void main(String[] args) { System.err.println("Please set OPENAI_API_KEY environment variable"); System.exit(1); } - + System.out.println("=== Testing GPT-5.2 Streaming ==="); System.out.println("API Key present: " + (OPENAI_API_KEY.length() > 0)); System.out.println("\nSending request to OpenAI...\n"); OpenAIClient client = OpenAIOkHttpClient.builder() - .apiKey(OPENAI_API_KEY) - .maxRetries(0) - .build(); + .apiKey(OPENAI_API_KEY) + .maxRetries(0) + .build(); Timeout timeout = Timeout.builder() - .request(Duration.ofSeconds(60)) - .read(Duration.ofSeconds(60)) - .build(); + .request(Duration.ofSeconds(60)) + .read(Duration.ofSeconds(60)) + .build(); - RequestOptions requestOptions = RequestOptions.builder() - .timeout(timeout) - .build(); + RequestOptions requestOptions = + RequestOptions.builder().timeout(timeout).build(); ChatCompletionCreateParams params = ChatCompletionCreateParams.builder() - .model(ChatModel.of("gpt-5.2")) - .maxCompletionTokens(100) - .reasoningEffort(ReasoningEffort.of("minimal")) - .addUserMessage("Say 'Hello World' and nothing else") - .build(); + .model(ChatModel.of("gpt-5.2")) + .maxCompletionTokens(100) + .reasoningEffort(ReasoningEffort.of("minimal")) + .addUserMessage("Say 'Hello World' and nothing else") + .build(); StringBuilder fullResponse = new StringBuilder(); java.util.concurrent.atomic.AtomicInteger chunkCount = new java.util.concurrent.atomic.AtomicInteger(0); try (StreamResponse responseStream = - client.chat().completions().createStreaming(params, requestOptions)) { + client.chat().completions().createStreaming(params, requestOptions)) { responseStream.stream().forEach(chunk -> { chunkCount.incrementAndGet(); chunk.choices().stream() - .flatMap(choice -> choice.delta().content().stream()) - .forEach(contentChunk -> { - System.out.print(contentChunk); - fullResponse.append(contentChunk); - }); + .flatMap(choice -> choice.delta().content().stream()) + .forEach(contentChunk -> { + System.out.print(contentChunk); + fullResponse.append(contentChunk); + }); }); } catch (RuntimeException streamingFailure) { System.err.println("Error: " + streamingFailure.getClass().getSimpleName()); diff --git a/src/test/java/TestWebFluxSSE.java b/src/test/java/TestWebFluxSSE.java index 4b2daff..73ed793 100644 --- a/src/test/java/TestWebFluxSSE.java +++ b/src/test/java/TestWebFluxSSE.java @@ -7,7 +7,6 @@ import com.openai.models.ReasoningEffort; import com.openai.models.chat.completions.ChatCompletionChunk; import com.openai.models.chat.completions.ChatCompletionCreateParams; - import java.time.Duration; /** @@ -15,7 +14,7 @@ */ public class TestWebFluxSSE { private static final String OPENAI_API_KEY = System.getenv("OPENAI_API_KEY"); - + /** * Runs the raw SSE chunk probe. * @@ -26,41 +25,40 @@ public static void main(String[] args) { System.err.println("Please set OPENAI_API_KEY environment variable"); System.exit(1); } - + System.out.println("=== Testing OpenAI Java SDK Streaming ==="); System.out.println("Sending request..."); OpenAIClient client = OpenAIOkHttpClient.builder() - .apiKey(OPENAI_API_KEY) - .maxRetries(0) - .build(); + .apiKey(OPENAI_API_KEY) + .maxRetries(0) + .build(); Timeout timeout = Timeout.builder() - .request(Duration.ofSeconds(30)) - .read(Duration.ofSeconds(30)) - .build(); + .request(Duration.ofSeconds(30)) + .read(Duration.ofSeconds(30)) + .build(); - RequestOptions requestOptions = RequestOptions.builder() - .timeout(timeout) - .build(); + RequestOptions requestOptions = + RequestOptions.builder().timeout(timeout).build(); ChatCompletionCreateParams params = ChatCompletionCreateParams.builder() - .model(ChatModel.of("gpt-5.2")) - .maxCompletionTokens(100) - .reasoningEffort(ReasoningEffort.of("minimal")) - .addUserMessage("Say 'Hello World' and nothing else") - .build(); - + .model(ChatModel.of("gpt-5.2")) + .maxCompletionTokens(100) + .reasoningEffort(ReasoningEffort.of("minimal")) + .addUserMessage("Say 'Hello World' and nothing else") + .build(); + System.out.println("\n=== RAW CHUNKS FROM OPENAI-JAVA ==="); try (StreamResponse responseStream = - client.chat().completions().createStreaming(params, requestOptions)) { + client.chat().completions().createStreaming(params, requestOptions)) { responseStream.stream().forEach(chunk -> { System.out.println("\n--- CHUNK START ---"); System.out.println("Chunk: " + chunk); chunk.choices().stream() - .flatMap(choice -> choice.delta().content().stream()) - .forEach(contentChunk -> System.out.println("Delta: " + contentChunk)); + .flatMap(choice -> choice.delta().content().stream()) + .forEach(contentChunk -> System.out.println("Delta: " + contentChunk)); System.out.println("--- CHUNK END ---"); }); System.out.println("\n=== STREAM COMPLETE ==="); @@ -69,7 +67,7 @@ public static void main(String[] args) { } finally { client.close(); } - + System.out.println("\nTest complete!"); } } diff --git a/src/test/java/TestWhatGetsStreamed.java b/src/test/java/TestWhatGetsStreamed.java index e0e2c3f..ed59aea 100644 --- a/src/test/java/TestWhatGetsStreamed.java +++ b/src/test/java/TestWhatGetsStreamed.java @@ -8,7 +8,6 @@ import com.openai.models.ReasoningEffort; import com.openai.models.chat.completions.ChatCompletionChunk; import com.openai.models.chat.completions.ChatCompletionCreateParams; - import java.time.Duration; /** @@ -17,7 +16,7 @@ public class TestWhatGetsStreamed { private static final String OPENAI_API_KEY = System.getenv("OPENAI_API_KEY"); private static final ObjectMapper objectMapper = new ObjectMapper(); - + /** * Runs the streaming probe against the OpenAI API. * @@ -28,46 +27,46 @@ public static void main(String[] args) { System.err.println("Please set OPENAI_API_KEY environment variable"); System.exit(1); } - + System.out.println("=== Testing What Gets Sent to Browser ===\n"); System.out.println("=== SIMULATING ChatController BEHAVIOR ===\n"); OpenAIClient client = OpenAIOkHttpClient.builder() - .apiKey(OPENAI_API_KEY) - .maxRetries(0) - .build(); + .apiKey(OPENAI_API_KEY) + .maxRetries(0) + .build(); Timeout timeout = Timeout.builder() - .request(Duration.ofSeconds(30)) - .read(Duration.ofSeconds(30)) - .build(); + .request(Duration.ofSeconds(30)) + .read(Duration.ofSeconds(30)) + .build(); - RequestOptions requestOptions = RequestOptions.builder() - .timeout(timeout) - .build(); + RequestOptions requestOptions = + RequestOptions.builder().timeout(timeout).build(); ChatCompletionCreateParams params = ChatCompletionCreateParams.builder() - .model(ChatModel.of("gpt-5.2")) - .maxCompletionTokens(50) - .reasoningEffort(ReasoningEffort.of("minimal")) - .addUserMessage("Say hello") - .build(); + .model(ChatModel.of("gpt-5.2")) + .maxCompletionTokens(50) + .reasoningEffort(ReasoningEffort.of("minimal")) + .addUserMessage("Say hello") + .build(); try (StreamResponse responseStream = - client.chat().completions().createStreaming(params, requestOptions)) { + client.chat().completions().createStreaming(params, requestOptions)) { responseStream.stream() - .flatMap(chunk -> chunk.choices().stream()) - .flatMap(choice -> choice.delta().content().stream()) - .filter(contentChunk -> !contentChunk.isEmpty()) - .forEach(contentChunk -> { - System.out.println("Extracted: '" + contentChunk + "'"); - String jsonPayload = jsonTextPayload(contentChunk); - String sseFrame = "data: " + jsonPayload + "\n\n"; - System.out.println("Sending to browser: '" + sseFrame.replace("\n", "\\n") + "'"); - }); + .flatMap(chunk -> chunk.choices().stream()) + .flatMap(choice -> choice.delta().content().stream()) + .filter(contentChunk -> !contentChunk.isEmpty()) + .forEach(contentChunk -> { + System.out.println("Extracted: '" + contentChunk + "'"); + String jsonPayload = jsonTextPayload(contentChunk); + String sseFrame = "data: " + jsonPayload + "\n\n"; + System.out.println("Sending to browser: '" + sseFrame.replace("\n", "\\n") + "'"); + }); } catch (RuntimeException streamingFailure) { - System.err.println("\nError during streaming: " + streamingFailure.getClass().getSimpleName()); + System.err.println( + "\nError during streaming: " + streamingFailure.getClass().getSimpleName()); } finally { client.close(); } @@ -77,7 +76,8 @@ private static String jsonTextPayload(String chunk) { try { return objectMapper.writeValueAsString(new TextEvent(chunk)); } catch (Exception jsonFailure) { - System.err.println("JSON serialization error: " + jsonFailure.getClass().getSimpleName()); + System.err.println( + "JSON serialization error: " + jsonFailure.getClass().getSimpleName()); return "{\"text\":\"\"}"; } } diff --git a/src/test/java/com/williamcallahan/javachat/ExtractorQualityTest.java b/src/test/java/com/williamcallahan/javachat/ExtractorQualityTest.java index 4328fa2..fe5adf5 100644 --- a/src/test/java/com/williamcallahan/javachat/ExtractorQualityTest.java +++ b/src/test/java/com/williamcallahan/javachat/ExtractorQualityTest.java @@ -1,24 +1,22 @@ package com.williamcallahan.javachat; import com.williamcallahan.javachat.service.HtmlContentExtractor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; -import org.springframework.context.annotation.Bean; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; // NOTE: This is a manual utility runner, not part of test boot configuration. // IMPORTANT: Do NOT annotate with @SpringBootApplication here to avoid @@ -29,7 +27,7 @@ @Configuration @Profile("manual") public class ExtractorQualityTest { - + /** * Launches the extraction quality runner in non-web mode. * @@ -41,7 +39,7 @@ public static void main(String[] args) { ConfigurableApplicationContext context = application.run(args); SpringApplication.exit(context); } - + /** * Executes the extraction comparison against a sample of documentation files. * @@ -52,7 +50,7 @@ public static void main(String[] args) { CommandLineRunner testExtraction(HtmlContentExtractor extractor) { return args -> { String docsRoot = "data/docs"; - + // Test sources String[][] sources = { {"java/java24-complete", "Java 24"}, @@ -61,62 +59,62 @@ CommandLineRunner testExtraction(HtmlContentExtractor extractor) { {"spring-boot-complete", "Spring Boot"}, {"spring-framework-complete", "Spring Framework"} }; - + System.out.println("\n========================================"); System.out.println("HTML EXTRACTION QUALITY CONTROL TEST"); System.out.println("========================================\n"); - + for (String[] source : sources) { String dir = source[0]; String name = source[1]; Path sourcePath = Paths.get(docsRoot, dir); - + if (!Files.exists(sourcePath)) { System.out.println("⚠ Skipping " + name + " (not found)"); continue; } - + System.out.println("\n=== Testing " + name + " ==="); - + // Get sample files List htmlFiles; try (Stream paths = Files.walk(sourcePath)) { - htmlFiles = paths - .filter(Files::isRegularFile) - .filter(p -> p.toString().endsWith(".html")) - .collect(Collectors.toList()); + htmlFiles = paths.filter(Files::isRegularFile) + .filter(p -> p.toString().endsWith(".html")) + .collect(Collectors.toList()); } - + if (htmlFiles.isEmpty()) { System.out.println("No HTML files found"); continue; } - + // Sample up to 5 random files Collections.shuffle(htmlFiles); int sampleSize = Math.min(5, htmlFiles.size()); - + System.out.println("Total files: " + htmlFiles.size()); System.out.println("Sampling: " + sampleSize + " files\n"); - + int totalOldNoise = 0; int totalNewNoise = 0; int totalOldLength = 0; int totalNewLength = 0; - + for (int i = 0; i < sampleSize; i++) { Path file = htmlFiles.get(i); String html = Files.readString(file); Document doc = Jsoup.parse(html); - + // Old extraction String oldText = doc.body() != null ? doc.body().text() : ""; - + // New extraction - String newText = file.toString().contains("/api/") ? - extractor.extractJavaApiContent(doc) : - extractor.extractCleanContent(doc); - + String path = file.toString(); + boolean isApiDoc = path.contains("/api/") || path.contains("\\api\\"); + String newText = + isApiDoc ? extractor.extractJavaApiContent(doc) : extractor.extractCleanContent(doc); + // Count noise patterns String[] noisePatterns = { "JavaScript is disabled", @@ -128,48 +126,57 @@ CommandLineRunner testExtraction(HtmlContentExtractor extractor) { "Use is subject to license", "Scripting on this page" }; - + int oldNoise = 0; int newNoise = 0; - + for (String noise : noisePatterns) { if (oldText.contains(noise)) oldNoise++; if (newText.contains(noise)) newNoise++; } - + totalOldNoise += oldNoise; totalNewNoise += newNoise; totalOldLength += oldText.length(); totalNewLength += newText.length(); - - System.out.println("File " + (i+1) + ": " + file.getFileName()); + + System.out.println("File " + (i + 1) + ": " + file.getFileName()); System.out.println(" Old: " + oldText.length() + " chars, " + oldNoise + " noise patterns"); System.out.println(" New: " + newText.length() + " chars, " + newNoise + " noise patterns"); - System.out.println(" Reduction: " + - String.format("%.1f%%", (1.0 - (double)newText.length()/oldText.length()) * 100) + - " size, " + (oldNoise - newNoise) + " noise patterns removed"); - + double reduction = + oldText.length() > 0 ? (1.0 - (double) newText.length() / oldText.length()) * 100 : 0; + System.out.println(" Reduction: " + + String.format("%.1f%%", reduction) + + " size, " + + (oldNoise - newNoise) + " noise patterns removed"); + // Show preview of improvement if (i == 0) { System.out.println("\n Preview (first 300 chars):"); - System.out.println(" OLD: " + oldText.substring(0, Math.min(300, oldText.length())).replace("\n", " ")); - System.out.println(" NEW: " + newText.substring(0, Math.min(300, newText.length())).replace("\n", " ")); + System.out.println(" OLD: " + + oldText.substring(0, Math.min(300, oldText.length())) + .replace("\n", " ")); + System.out.println(" NEW: " + + newText.substring(0, Math.min(300, newText.length())) + .replace("\n", " ")); } System.out.println(); } - + // Summary for this source System.out.println("Summary for " + name + ":"); - System.out.println(" Total noise patterns: " + totalOldNoise + " → " + totalNewNoise + - " (" + (totalOldNoise - totalNewNoise) + " removed)"); - System.out.println(" Average size reduction: " + - String.format("%.1f%%", (1.0 - (double)totalNewLength/totalOldLength) * 100)); - System.out.println(" Quality improvement: " + - (totalNewNoise == 0 ? "✅ EXCELLENT - No noise patterns" : - totalNewNoise < totalOldNoise/4 ? "✅ GOOD - Significant reduction" : - "⚠ NEEDS REVIEW")); + System.out.println(" Total noise patterns: " + totalOldNoise + " → " + totalNewNoise + " (" + + (totalOldNoise - totalNewNoise) + " removed)"); + double avgReduction = totalOldLength > 0 ? (1.0 - (double) totalNewLength / totalOldLength) * 100 : 0; + System.out.println(" Average size reduction: " + String.format("%.1f%%", avgReduction)); + System.out.println(" Quality improvement: " + + (totalNewNoise == 0 + ? "✅ EXCELLENT - No noise patterns" + : totalNewNoise < totalOldNoise / 4 + ? "✅ GOOD - Significant reduction" + : "⚠ NEEDS REVIEW")); } - + System.out.println("\n========================================"); System.out.println("TEST COMPLETE"); System.out.println("========================================\n"); diff --git a/src/test/java/com/williamcallahan/javachat/JavaChatApplicationTests.java b/src/test/java/com/williamcallahan/javachat/JavaChatApplicationTests.java index a293b6d..1adba44 100644 --- a/src/test/java/com/williamcallahan/javachat/JavaChatApplicationTests.java +++ b/src/test/java/com/williamcallahan/javachat/JavaChatApplicationTests.java @@ -8,21 +8,20 @@ /** * Verifies the Spring Boot test context loads with mocked AI dependencies. */ -@SpringBootTest(properties = { - "spring.ai.openai.api-key=test", - "spring.ai.openai.chat.api-key=test", - "spring.ai.vectorstore.qdrant.host=localhost", - "spring.ai.vectorstore.qdrant.use-tls=false", - "spring.ai.vectorstore.qdrant.port=8086", - "spring.ai.vectorstore.qdrant.initialize-schema=false" -}) +@SpringBootTest( + properties = { + "spring.ai.openai.api-key=test", + "spring.ai.openai.chat.api-key=test", + "spring.ai.vectorstore.qdrant.host=localhost", + "spring.ai.vectorstore.qdrant.use-tls=false", + "spring.ai.vectorstore.qdrant.port=8086", + "spring.ai.vectorstore.qdrant.initialize-schema=false" + }) class JavaChatApplicationTests { @MockitoBean ChatModel chatModel; @Test - void contextLoads() { - } - + void contextLoads() {} } diff --git a/src/test/java/com/williamcallahan/javachat/StandaloneExtractionTest.java b/src/test/java/com/williamcallahan/javachat/StandaloneExtractionTest.java index a00ffb0..381a7c2 100644 --- a/src/test/java/com/williamcallahan/javachat/StandaloneExtractionTest.java +++ b/src/test/java/com/williamcallahan/javachat/StandaloneExtractionTest.java @@ -2,9 +2,6 @@ import com.williamcallahan.javachat.service.HtmlContentExtractor; import com.williamcallahan.javachat.support.AsciiTextNormalizer; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -14,12 +11,14 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; /** * Manual smoke test for HTML extraction that compares noise removal between old and new paths. */ public class StandaloneExtractionTest { - + /** * Runs the extraction comparison against sampled documentation files. * @@ -29,7 +28,7 @@ public class StandaloneExtractionTest { public static void main(String[] args) throws IOException { HtmlContentExtractor extractor = new HtmlContentExtractor(); String docsRoot = "data/docs"; - + // Test sources String[][] sources = { {"java/java24-complete", "Java 24"}, @@ -38,66 +37,65 @@ public static void main(String[] args) throws IOException { {"spring-boot-complete", "Spring Boot"}, {"spring-framework-complete", "Spring Framework"} }; - + System.out.println("\n========================================"); System.out.println("HTML EXTRACTION QUALITY CONTROL TEST"); System.out.println("========================================\n"); - + int grandTotalOldNoise = 0; int grandTotalNewNoise = 0; List problematicFiles = new ArrayList<>(); - + for (String[] source : sources) { String dir = source[0]; String name = source[1]; Path sourcePath = Paths.get(docsRoot, dir); - + if (!Files.exists(sourcePath)) { System.out.println("⚠ Skipping " + name + " (not found)"); continue; } - + System.out.println("\n=== Testing " + name + " ==="); - + // Get sample files List htmlFiles; try (Stream paths = Files.walk(sourcePath)) { - htmlFiles = paths - .filter(Files::isRegularFile) - .filter(p -> p.toString().endsWith(".html")) - .collect(Collectors.toList()); + htmlFiles = paths.filter(Files::isRegularFile) + .filter(p -> p.toString().endsWith(".html")) + .collect(Collectors.toList()); } - + if (htmlFiles.isEmpty()) { System.out.println("No HTML files found"); continue; } - + // Sample up to 5 random files Collections.shuffle(htmlFiles); int sampleSize = Math.min(5, htmlFiles.size()); - + System.out.println("Total files: " + htmlFiles.size()); System.out.println("Sampling: " + sampleSize + " files\n"); - + int totalOldNoise = 0; int totalNewNoise = 0; int totalOldLength = 0; int totalNewLength = 0; - + for (int i = 0; i < sampleSize; i++) { Path file = htmlFiles.get(i); String html = Files.readString(file); Document doc = Jsoup.parse(html); - + // Old extraction String oldText = doc.body() != null ? doc.body().text() : ""; - + // New extraction - String newText = file.toString().contains("/api/") ? - extractor.extractJavaApiContent(doc) : - extractor.extractCleanContent(doc); - + String path = file.toString(); + boolean isApiDoc = path.contains("/api/") || path.contains("\\api\\"); + String newText = isApiDoc ? extractor.extractJavaApiContent(doc) : extractor.extractCleanContent(doc); + // Count noise patterns String[] noisePatterns = { "JavaScript is disabled", @@ -110,7 +108,7 @@ public static void main(String[] args) throws IOException { "Scripting on this page", "Other versions" }; - + int oldNoise = 0; int newNoise = 0; String oldTextLower = AsciiTextNormalizer.toLowerAscii(oldText); @@ -121,78 +119,79 @@ public static void main(String[] args) throws IOException { if (oldTextLower.contains(normalizedNoise)) oldNoise++; if (newTextLower.contains(normalizedNoise)) newNoise++; } - + totalOldNoise += oldNoise; totalNewNoise += newNoise; grandTotalOldNoise += oldNoise; grandTotalNewNoise += newNoise; totalOldLength += oldText.length(); totalNewLength += newText.length(); - - System.out.println("File " + (i+1) + ": " + file.getFileName()); + + System.out.println("File " + (i + 1) + ": " + file.getFileName()); System.out.println(" Old: " + oldText.length() + " chars, " + oldNoise + " noise patterns"); System.out.println(" New: " + newText.length() + " chars, " + newNoise + " noise patterns"); - - double reduction = oldText.length() > 0 ? - (1.0 - (double)newText.length()/oldText.length()) * 100 : 0; - System.out.println(" Reduction: " + - String.format("%.1f%%", reduction) + - " size, " + (oldNoise - newNoise) + " noise patterns removed"); - + + double reduction = + oldText.length() > 0 ? (1.0 - (double) newText.length() / oldText.length()) * 100 : 0; + System.out.println(" Reduction: " + String.format("%.1f%%", reduction) + + " size, " + + (oldNoise - newNoise) + " noise patterns removed"); + if (newNoise > 0) { problematicFiles.add(file.toString()); System.out.println(" ⚠ Still contains noise patterns!"); } - + // Show preview for first file if (i == 0) { System.out.println("\n Preview (first 400 chars):"); - System.out.println(" OLD: " + - oldText.substring(0, Math.min(400, oldText.length())) - .replace("\n", " ").replaceAll("\\s+", " ")); - System.out.println("\n NEW: " + - newText.substring(0, Math.min(400, newText.length())) - .replace("\n", " ").replaceAll("\\s+", " ")); + System.out.println(" OLD: " + + oldText.substring(0, Math.min(400, oldText.length())) + .replace("\n", " ") + .replaceAll("\\s+", " ")); + System.out.println("\n NEW: " + + newText.substring(0, Math.min(400, newText.length())) + .replace("\n", " ") + .replaceAll("\\s+", " ")); } System.out.println(); } - + // Summary for this source System.out.println("Summary for " + name + ":"); - System.out.println(" Total noise patterns: " + totalOldNoise + " → " + totalNewNoise + - " (" + (totalOldNoise - totalNewNoise) + " removed)"); - - double avgReduction = totalOldLength > 0 ? - (1.0 - (double)totalNewLength/totalOldLength) * 100 : 0; - System.out.println(" Average size reduction: " + - String.format("%.1f%%", avgReduction)); - - System.out.println(" Quality improvement: " + - (totalNewNoise == 0 ? "✅ EXCELLENT - No noise patterns" : - totalNewNoise < totalOldNoise/4 ? "✅ GOOD - Significant reduction" : - "⚠ NEEDS REVIEW")); + System.out.println(" Total noise patterns: " + totalOldNoise + " → " + totalNewNoise + " (" + + (totalOldNoise - totalNewNoise) + " removed)"); + + double avgReduction = totalOldLength > 0 ? (1.0 - (double) totalNewLength / totalOldLength) * 100 : 0; + System.out.println(" Average size reduction: " + String.format("%.1f%%", avgReduction)); + + System.out.println(" Quality improvement: " + + (totalNewNoise == 0 + ? "✅ EXCELLENT - No noise patterns" + : totalNewNoise < totalOldNoise / 4 ? "✅ GOOD - Significant reduction" : "⚠ NEEDS REVIEW")); } - + System.out.println("\n========================================"); System.out.println("OVERALL RESULTS"); System.out.println("========================================"); System.out.println("Total noise patterns across all samples:"); System.out.println(" OLD METHOD: " + grandTotalOldNoise + " patterns"); System.out.println(" NEW METHOD: " + grandTotalNewNoise + " patterns"); - System.out.println(" IMPROVEMENT: " + (grandTotalOldNoise - grandTotalNewNoise) + - " patterns removed (" + - String.format("%.1f%%", - grandTotalOldNoise > 0 ? - ((double)(grandTotalOldNoise - grandTotalNewNoise) / grandTotalOldNoise * 100) : 0) + - " reduction)"); - + System.out.println(" IMPROVEMENT: " + (grandTotalOldNoise - grandTotalNewNoise) + " patterns removed (" + + String.format( + "%.1f%%", + grandTotalOldNoise > 0 + ? ((double) (grandTotalOldNoise - grandTotalNewNoise) / grandTotalOldNoise * 100) + : 0) + + " reduction)"); + if (problematicFiles.isEmpty()) { System.out.println("\n✅ ALL SAMPLES CLEAN - No noise patterns remaining!"); } else { System.out.println("\n⚠ Files still containing noise (" + problematicFiles.size() + "):"); problematicFiles.forEach(f -> System.out.println(" - " + f)); } - + System.out.println("\n========================================"); System.out.println("TEST COMPLETE"); System.out.println("========================================\n"); diff --git a/src/test/java/com/williamcallahan/javachat/TestConfiguration.java b/src/test/java/com/williamcallahan/javachat/TestConfiguration.java index 78c1df8..f05f048 100644 --- a/src/test/java/com/williamcallahan/javachat/TestConfiguration.java +++ b/src/test/java/com/williamcallahan/javachat/TestConfiguration.java @@ -1,16 +1,15 @@ package com.williamcallahan.javachat; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.extension.ConditionEvaluationResult; import org.junit.jupiter.api.extension.ExecutionCondition; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtensionContext; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - /** * Test configuration utilities for conditional test execution */ @@ -45,27 +44,27 @@ public static class RequiresExternalServicesCondition implements ExecutionCondit @Override public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { // Check for API keys - boolean hasApiKeys = System.getenv("OPENAI_API_KEY") != null || - System.getenv("GITHUB_TOKEN") != null; - + boolean hasApiKeys = System.getenv("OPENAI_API_KEY") != null || System.getenv("GITHUB_TOKEN") != null; + // Check for integration test flag boolean integrationEnabled = "true".equals(System.getProperty("test.integration.enabled")); - + if (!hasApiKeys) { return ConditionEvaluationResult.disabled("Skipping test - no API keys configured"); } - + if (!integrationEnabled && !isRunningInCI()) { - return ConditionEvaluationResult.disabled("Skipping integration test - set -Dtest.integration.enabled=true to run"); + return ConditionEvaluationResult.disabled( + "Skipping integration test - set -Dtest.integration.enabled=true to run"); } - + return ConditionEvaluationResult.enabled("External services available"); } - + private boolean isRunningInCI() { - return System.getenv("CI") != null || - System.getenv("GITHUB_ACTIONS") != null || - System.getenv("JENKINS_HOME") != null; + return System.getenv("CI") != null + || System.getenv("GITHUB_ACTIONS") != null + || System.getenv("JENKINS_HOME") != null; } } } diff --git a/src/test/java/com/williamcallahan/javachat/application/prompt/PromptTruncatorTest.java b/src/test/java/com/williamcallahan/javachat/application/prompt/PromptTruncatorTest.java index cd14b9f..ec2088c 100644 --- a/src/test/java/com/williamcallahan/javachat/application/prompt/PromptTruncatorTest.java +++ b/src/test/java/com/williamcallahan/javachat/application/prompt/PromptTruncatorTest.java @@ -1,19 +1,18 @@ package com.williamcallahan.javachat.application.prompt; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.williamcallahan.javachat.domain.prompt.ContextDocumentSegment; import com.williamcallahan.javachat.domain.prompt.ConversationTurnSegment; import com.williamcallahan.javachat.domain.prompt.CurrentQuerySegment; import com.williamcallahan.javachat.domain.prompt.StructuredPrompt; import com.williamcallahan.javachat.domain.prompt.SystemSegment; +import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Verifies structure-aware prompt truncation preserves semantic boundaries. */ @@ -32,8 +31,7 @@ void noTruncationWhenWithinLimit() { new SystemSegment("System instructions", 50), List.of(new ContextDocumentSegment(1, "url1", "content1", 100)), List.of(new ConversationTurnSegment("user", "Hello", 10)), - new CurrentQuerySegment("What is Java?", 20) - ); + new CurrentQuerySegment("What is Java?", 20)); PromptTruncator.TruncatedPrompt result = truncator.truncate(prompt, 500, false); @@ -49,11 +47,9 @@ void truncatesContextDocumentsFirst() { List.of( new ContextDocumentSegment(1, "url1", "doc1", 200), new ContextDocumentSegment(2, "url2", "doc2", 200), - new ContextDocumentSegment(3, "url3", "doc3", 200) - ), + new ContextDocumentSegment(3, "url3", "doc3", 200)), List.of(new ConversationTurnSegment("user", "history", 50)), - new CurrentQuerySegment("query", 50) - ); + new CurrentQuerySegment("query", 50)); // Total: 100 + 600 + 50 + 50 = 800 tokens // Limit 350: 100 (system) + 50 (query) = 150 reserved, 200 available @@ -75,10 +71,8 @@ void truncatesOldConversationHistoryFirst() { List.of( new ConversationTurnSegment("user", "old1", 100), new ConversationTurnSegment("assistant", "old2", 100), - new ConversationTurnSegment("user", "recent", 100) - ), - new CurrentQuerySegment("query", 50) - ); + new ConversationTurnSegment("user", "recent", 100)), + new CurrentQuerySegment("query", 50)); // Total: 100 + 0 + 300 + 50 = 450 tokens // Limit 300: 100 + 50 = 150 reserved, 150 available @@ -100,8 +94,7 @@ void neverTruncatesSystemPromptOrCurrentQuery() { new SystemSegment("Critical system instructions", 200), List.of(new ContextDocumentSegment(1, "url", "doc", 100)), List.of(new ConversationTurnSegment("user", "history", 100)), - new CurrentQuerySegment("Important question", 200) - ); + new CurrentQuerySegment("Important question", 200)); // Limit smaller than system + query alone PromptTruncator.TruncatedPrompt result = truncator.truncate(prompt, 350, true); @@ -122,11 +115,9 @@ void prependsTruncationNoticeForGpt5() { new SystemSegment("System", 100), List.of( new ContextDocumentSegment(1, "url1", "doc1", 500), - new ContextDocumentSegment(2, "url2", "doc2", 500) - ), + new ContextDocumentSegment(2, "url2", "doc2", 500)), List.of(), - new CurrentQuerySegment("query", 50) - ); + new CurrentQuerySegment("query", 50)); PromptTruncator.TruncatedPrompt result = truncator.truncate(prompt, 400, true); @@ -140,8 +131,7 @@ void prependsGenericTruncationNoticeForOtherModels() { new SystemSegment("System", 100), List.of(new ContextDocumentSegment(1, "url1", "doc1", 500)), List.of(), - new CurrentQuerySegment("query", 50) - ); + new CurrentQuerySegment("query", 50)); PromptTruncator.TruncatedPrompt result = truncator.truncate(prompt, 200, false); @@ -157,11 +147,9 @@ void reindexesContextDocumentsAfterTruncation() { List.of( new ContextDocumentSegment(1, "url1", "doc1", 100), new ContextDocumentSegment(2, "url2", "doc2", 100), - new ContextDocumentSegment(3, "url3", "doc3", 100) - ), + new ContextDocumentSegment(3, "url3", "doc3", 100)), List.of(), - new CurrentQuerySegment("query", 50) - ); + new CurrentQuerySegment("query", 50)); // Should keep first 2 docs (most relevant), re-indexed as 1 and 2 PromptTruncator.TruncatedPrompt result = truncator.truncate(prompt, 350, false); @@ -182,11 +170,7 @@ void reindexesContextDocumentsAfterTruncation() { @Test void handlesEmptyContextAndHistory() { StructuredPrompt prompt = new StructuredPrompt( - new SystemSegment("System", 100), - List.of(), - List.of(), - new CurrentQuerySegment("query", 50) - ); + new SystemSegment("System", 100), List.of(), List.of(), new CurrentQuerySegment("query", 50)); PromptTruncator.TruncatedPrompt result = truncator.truncate(prompt, 200, false); @@ -206,10 +190,8 @@ void preservesAssistantPrefixInRenderedOutput() { List.of(), List.of( new ConversationTurnSegment("user", "question", 20), - new ConversationTurnSegment("assistant", "answer", 20) - ), - new CurrentQuerySegment("follow-up", 20) - ); + new ConversationTurnSegment("assistant", "answer", 20)), + new CurrentQuerySegment("follow-up", 20)); PromptTruncator.TruncatedPrompt result = truncator.truncate(prompt, 500, false); diff --git a/src/test/java/com/williamcallahan/javachat/domain/SearchQualityLevelTest.java b/src/test/java/com/williamcallahan/javachat/domain/SearchQualityLevelTest.java new file mode 100644 index 0000000..e245b5f --- /dev/null +++ b/src/test/java/com/williamcallahan/javachat/domain/SearchQualityLevelTest.java @@ -0,0 +1,78 @@ +package com.williamcallahan.javachat.domain; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.williamcallahan.javachat.support.DocumentContentAdapter; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.springframework.ai.document.Document; + +/** + * Tests for {@link SearchQualityLevel} enum behavior. + */ +class SearchQualityLevelTest { + + @Test + void determineReturnsNoneForEmptyList() { + SearchQualityLevel level = SearchQualityLevel.determine(List.of()); + assertThat(level).isEqualTo(SearchQualityLevel.NONE); + } + + @Test + void determineReturnsNoneForNullList() { + SearchQualityLevel level = SearchQualityLevel.determine(null); + assertThat(level).isEqualTo(SearchQualityLevel.NONE); + } + + @Test + void determineReturnsKeywordSearchWhenUrlContainsKeyword() { + Document keywordDoc = new Document("some content", Map.of("url", "local-search://query")); + SearchQualityLevel level = + SearchQualityLevel.determine(DocumentContentAdapter.fromDocuments(List.of(keywordDoc))); + assertThat(level).isEqualTo(SearchQualityLevel.KEYWORD_SEARCH); + } + + @Test + void determineReturnsHighQualityWhenAllDocsHaveSubstantialContent() { + String longContent = "a".repeat(150); + Document doc1 = new Document(longContent, Map.of("url", "https://example.com/doc1")); + Document doc2 = new Document(longContent, Map.of("url", "https://example.com/doc2")); + SearchQualityLevel level = + SearchQualityLevel.determine(DocumentContentAdapter.fromDocuments(List.of(doc1, doc2))); + assertThat(level).isEqualTo(SearchQualityLevel.HIGH_QUALITY); + } + + @Test + void determineReturnsMixedQualityWhenSomeDocsHaveShortContent() { + String longContent = "a".repeat(150); + String shortContent = "short"; + Document highQualityDoc = new Document(longContent, Map.of("url", "https://example.com/doc1")); + Document lowQualityDoc = new Document(shortContent, Map.of("url", "https://example.com/doc2")); + SearchQualityLevel level = SearchQualityLevel.determine( + DocumentContentAdapter.fromDocuments(List.of(highQualityDoc, lowQualityDoc))); + assertThat(level).isEqualTo(SearchQualityLevel.MIXED_QUALITY); + } + + @Test + void formatMessageReturnsCorrectStringForEachLevel() { + assertThat(SearchQualityLevel.NONE.formatMessage(0, 0)).contains("No relevant documents"); + + assertThat(SearchQualityLevel.KEYWORD_SEARCH.formatMessage(5, 0)) + .contains("5 documents") + .contains("keyword search"); + + assertThat(SearchQualityLevel.HIGH_QUALITY.formatMessage(3, 3)).contains("3 high-quality"); + + assertThat(SearchQualityLevel.MIXED_QUALITY.formatMessage(5, 2)) + .contains("5 documents") + .contains("2 high-quality"); + } + + @Test + void describeQualityReturnsFormattedMessage() { + String description = SearchQualityLevel.describeQuality(List.of()); + assertThat(description).isNotBlank(); + assertThat(description).contains("No relevant documents"); + } +} diff --git a/src/test/java/com/williamcallahan/javachat/service/ChatMemoryServiceTest.java b/src/test/java/com/williamcallahan/javachat/service/ChatMemoryServiceTest.java index c9cc3c2..9711bbd 100644 --- a/src/test/java/com/williamcallahan/javachat/service/ChatMemoryServiceTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/ChatMemoryServiceTest.java @@ -1,5 +1,8 @@ package com.williamcallahan.javachat.service; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -7,10 +10,6 @@ import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.messages.UserMessage; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - /** * Validates chat memory and conversation context handling. * Ensures that what the user sees in the UI matches what the AI receives in context. diff --git a/src/test/java/com/williamcallahan/javachat/service/ComprehensiveListFormattingTest.java b/src/test/java/com/williamcallahan/javachat/service/ComprehensiveListFormattingTest.java index 7a8ebfd..6b9fcdf 100644 --- a/src/test/java/com/williamcallahan/javachat/service/ComprehensiveListFormattingTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/ComprehensiveListFormattingTest.java @@ -1,22 +1,23 @@ package com.williamcallahan.javachat.service; +import static org.junit.jupiter.api.Assertions.*; + import com.williamcallahan.javachat.service.markdown.UnifiedMarkdownService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; /** * Covers list formatting scenarios for markdown processing. */ class ComprehensiveListFormattingTest { - + private MarkdownService markdownService; - + @BeforeEach void setUp() { markdownService = new MarkdownService(new UnifiedMarkdownService()); } - + @Test void testNumberedListWithPeriod() { String input = "The types are:1. boolean 2. byte 3. int 4. long"; @@ -32,7 +33,7 @@ void testNumberedListWithPeriod() { assertTrue(html.contains("
  • int
  • "), "Should contain third item"); assertTrue(html.contains("
  • long
  • "), "Should contain fourth item"); } - + @Test void testNumberedListWithParenthesis() { String input = "The steps include:1) Setup 2) Configure 3) Deploy"; @@ -47,7 +48,7 @@ void testNumberedListWithParenthesis() { assertTrue(html.contains("
  • Configure
  • "), "Should contain second item"); assertTrue(html.contains("
  • Deploy
  • "), "Should contain third item"); } - + @Test void testRomanNumeralsLowercase() { String input = "The stages are:i. Planning ii. Development iii. Testing"; @@ -92,7 +93,7 @@ void testDashBulletList() { assertTrue(html.contains("
  • High accuracy
  • "), "Should contain second item"); assertTrue(html.contains("
  • Low latency
  • "), "Should contain third item"); } - + @Test void testAsteriskBulletList() { String input = "Benefits are:* Cost effective* Time saving* Easy to use"; @@ -136,7 +137,7 @@ void testMixedListMarkersAfterColon() { assertTrue(html.contains("
  • primitives
  • "), "Should contain first item"); assertTrue(html.contains("
  • references
  • "), "Should contain second item"); } - + @Test void testListIntroducedByKeywords() { String input = "The benefits include 1. performance 2. reliability 3. scalability"; @@ -151,7 +152,7 @@ void testListIntroducedByKeywords() { assertTrue(html.contains("
  • reliability
  • "), "Should contain second item"); assertTrue(html.contains("
  • scalability
  • "), "Should contain third item"); } - + @Test void testDirectAttachmentToPunctuation() { String input = "See below:1.First item.2.Second item!3.Third item"; @@ -166,7 +167,7 @@ void testDirectAttachmentToPunctuation() { assertTrue(html.contains("
  • Second item
  • "), "Should contain second item"); assertTrue(html.contains("
  • Third item
  • "), "Should contain third item"); } - + @Test void testSpecialBulletCharacters() { String input = "Options:• First option• Second option• Third option"; @@ -181,7 +182,7 @@ void testSpecialBulletCharacters() { assertTrue(html.contains("
  • Second option
  • "), "Should contain second item"); assertTrue(html.contains("
  • Third option
  • "), "Should contain third item"); } - + @Test void testNoFalsePositivesInSentences() { // These should NOT be converted to lists @@ -205,10 +206,11 @@ void testNoFalsePositivesInSentences() { System.out.println("Math expression preserved: " + !html2.contains("
      ") + !html2.contains("
        ")); System.out.println("Normal sentences preserved: " + !html3.contains("
          ") + !html3.contains("
            ")); } - + @Test void testComplexRealWorldExample() { - String input = "Java provides:1. Primitive types:a. boolean: true/false b. byte: 8-bit 2. Reference types:- Arrays- Classes- Interfaces"; + String input = + "Java provides:1. Primitive types:a. boolean: true/false b. byte: 8-bit 2. Reference types:- Arrays- Classes- Interfaces"; String html = markdownService.processStructured(input).html(); System.out.println("\nTest: Complex real-world example"); diff --git a/src/test/java/com/williamcallahan/javachat/service/DocsIngestionServiceTest.java b/src/test/java/com/williamcallahan/javachat/service/DocsIngestionServiceTest.java index a46cffb..66cd9a6 100644 --- a/src/test/java/com/williamcallahan/javachat/service/DocsIngestionServiceTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/DocsIngestionServiceTest.java @@ -1,12 +1,11 @@ package com.williamcallahan.javachat.service; -import org.junit.jupiter.api.Test; - -import java.util.List; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.List; +import org.junit.jupiter.api.Test; + /** * Verifies crawl snapshot handling preserves raw HTML and link discovery. */ @@ -22,14 +21,11 @@ void capturesRawHtmlAndDiscoversLinksBeforeMutation() { """; - DocsIngestionService.CrawlPageSnapshot snapshot = - DocsIngestionService.prepareCrawlPageSnapshot(baseUrl, html); + DocsIngestionService.CrawlPageSnapshot snapshot = DocsIngestionService.prepareCrawlPageSnapshot(baseUrl, html); List discoveredLinks = snapshot.discoveredLinks(); assertEquals(html, snapshot.rawHtml(), "Should preserve the raw HTML snapshot"); - assertTrue(discoveredLinks.contains("https://docs.example.com/root/hidden"), - "Should include navigation links"); - assertTrue(discoveredLinks.contains("https://docs.example.com/root/visible"), - "Should include content links"); + assertTrue(discoveredLinks.contains("https://docs.example.com/root/hidden"), "Should include navigation links"); + assertTrue(discoveredLinks.contains("https://docs.example.com/root/visible"), "Should include content links"); } } diff --git a/src/test/java/com/williamcallahan/javachat/service/EmbeddingCacheServiceLegacyImportTest.java b/src/test/java/com/williamcallahan/javachat/service/EmbeddingCacheServiceLegacyImportTest.java index 5b64863..04670fd 100644 --- a/src/test/java/com/williamcallahan/javachat/service/EmbeddingCacheServiceLegacyImportTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/EmbeddingCacheServiceLegacyImportTest.java @@ -1,13 +1,10 @@ package com.williamcallahan.javachat.service; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mockito; -import org.springframework.ai.embedding.EmbeddingModel; -import org.springframework.ai.vectorstore.VectorStore; - import java.io.IOException; import java.io.ObjectOutputStream; import java.io.OutputStream; @@ -18,9 +15,11 @@ import java.util.HashMap; import java.util.List; import java.util.zip.GZIPOutputStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; +import org.springframework.ai.embedding.EmbeddingModel; +import org.springframework.ai.vectorstore.VectorStore; /** * Verifies legacy embedding cache imports remain compatible with new metadata handling. @@ -32,12 +31,8 @@ void importsLegacyJavaSerializedCacheAndPreservesAdditionalMetadata(@TempDir Pat VectorStore vectorStore = Mockito.mock(VectorStore.class); ObjectMapper objectMapper = new ObjectMapper(); - EmbeddingCacheService cacheService = new EmbeddingCacheService( - tempDir.toString(), - embeddingModel, - vectorStore, - objectMapper - ); + EmbeddingCacheService cacheService = + new EmbeddingCacheService(tempDir.toString(), embeddingModel, vectorStore, objectMapper); Path legacyCachePath = tempDir.resolve("legacy_cache.gz"); writeLegacyCache(legacyCachePath); @@ -68,7 +63,7 @@ private static void writeLegacyCache(Path legacyCachePath) throws IOException { EmbeddingCacheService.CachedEmbedding legacyEntry = new EmbeddingCacheService.CachedEmbedding(); legacyEntry.id = "legacy-id"; legacyEntry.content = "Legacy content"; - legacyEntry.embedding = new float[] { 0.1f, 0.2f, 0.3f }; + legacyEntry.embedding = new float[] {0.1f, 0.2f, 0.3f}; legacyEntry.createdAt = LocalDateTime.of(2024, 1, 1, 0, 0); legacyEntry.uploaded = true; @@ -81,15 +76,15 @@ private static void writeLegacyCache(Path legacyCachePath) throws IOException { entries.add(legacyEntry); try (OutputStream fileOutputStream = Files.newOutputStream(legacyCachePath); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); - ObjectOutputStream objectOutputStream = new ObjectOutputStream(gzipOutputStream)) { + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream); + ObjectOutputStream objectOutputStream = new ObjectOutputStream(gzipOutputStream)) { objectOutputStream.writeObject(entries); } } private static JsonNode readGzipJson(Path gzipPath, ObjectMapper objectMapper) throws IOException { try (var fileInputStream = Files.newInputStream(gzipPath); - var gzipInputStream = new java.util.zip.GZIPInputStream(fileInputStream)) { + var gzipInputStream = new java.util.zip.GZIPInputStream(fileInputStream)) { return objectMapper.readTree(gzipInputStream); } } diff --git a/src/test/java/com/williamcallahan/javachat/service/HtmlContentExtractorTest.java b/src/test/java/com/williamcallahan/javachat/service/HtmlContentExtractorTest.java index bb69337..85ccf5b 100644 --- a/src/test/java/com/williamcallahan/javachat/service/HtmlContentExtractorTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/HtmlContentExtractorTest.java @@ -1,11 +1,11 @@ package com.williamcallahan.javachat.service; +import static org.junit.jupiter.api.Assertions.assertTrue; + import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertTrue; - /** * Verifies HTML extraction preserves code formatting while normalizing prose whitespace. */ diff --git a/src/test/java/com/williamcallahan/javachat/service/MarkdownPreprocessingTest.java b/src/test/java/com/williamcallahan/javachat/service/MarkdownPreprocessingTest.java index e769374..af44262 100644 --- a/src/test/java/com/williamcallahan/javachat/service/MarkdownPreprocessingTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/MarkdownPreprocessingTest.java @@ -1,22 +1,23 @@ package com.williamcallahan.javachat.service; +import static org.junit.jupiter.api.Assertions.*; + import com.williamcallahan.javachat.service.markdown.UnifiedMarkdownService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; /** * Verifies preprocessing rules applied before markdown rendering. */ class MarkdownPreprocessingTest { - + private MarkdownService markdownService; - + @BeforeEach void setUp() { markdownService = new MarkdownService(new UnifiedMarkdownService()); } - + @Test void testColonDashListPattern() { String input = "The remainder operator has several uses, such as:- Checking divisibility- Extracting digits"; @@ -30,7 +31,7 @@ void testColonDashListPattern() { assertTrue(html.contains("
          • Checking divisibility
          • "), "Should contain first list item"); assertTrue(html.contains("
          • Extracting digits
          • "), "Should contain second list item"); } - + @Test void testInlineNumberedList() { String input = "The primitive types are:1. boolean: true or false. 2. byte: 8-bit signed."; @@ -45,7 +46,7 @@ void testInlineNumberedList() { assertTrue(html.contains("
          • boolean"), "Should contain first list item"); assertTrue(html.contains("
          • byte"), "Should contain second list item"); } - + @Test void testInlineTripleBackticksRemainText() { String input = "Use ``` to denote a code fence in markdown."; @@ -64,7 +65,7 @@ void testSentenceSpacingAvoidsUrlsAndPackages() { assertFalse(html.contains("java. lang"), "Package names must not be split"); assertTrue(html.contains("href=\"https://example.com/Test\""), "URL should remain intact"); } - + @Test void testCodeBlockSpacing() { String input = "Here's an example:```java\nint x = 10 % 3;\n```The result is 1."; @@ -79,7 +80,7 @@ void testCodeBlockSpacing() { assertTrue(html.contains(""), "Should contain code with language class"); assertTrue(html.contains("int x = 10 % 3"), "Should contain code content"); } - + @Test void testClosingFenceSeparatesProse() { String input = "Here's an example:```java\nint x = 10 % 3;\n```The result is 1."; @@ -105,7 +106,7 @@ void testTildeFencePreserved() { assertTrue(html.contains("{{warning:still code}}"), "Markers inside fences should not render as cards"); assertTrue(html.contains("After fence."), "Prose after fence should render"); } - + @Test void testColonDirectlyBeforeCodeFence() { // This is the exact issue from the screenshot @@ -121,7 +122,7 @@ void testColonDirectlyBeforeCodeFence() { assertTrue(html.contains(" tag"); assertTrue(html.contains("import java.util.Scanner"), "Should contain code content"); } - + @Test void testPeriodDirectlyBeforeCodeFence() { String input = "Here is the code.```python\nprint('hello')"; @@ -138,7 +139,8 @@ void testPeriodDirectlyBeforeCodeFence() { @Test void testJavaCodeBlockWithComplexLanguageTag() { - String input = "Here's a Java example:```java\npublic class Hello {\n public static void main(String[] args) {\n System.out.println(\"Hello, World!\");\n }\n}\n```"; + String input = + "Here's a Java example:```java\npublic class Hello {\n public static void main(String[] args) {\n System.out.println(\"Hello, World!\");\n }\n}\n```"; String html = markdownService.processStructured(input).html(); assertTrue(html.contains("
            "), "HTML should contain 
             tag");
            @@ -148,7 +150,8 @@ void testJavaCodeBlockWithComplexLanguageTag() {
             
                 @Test
                 void testMultipleJavaCodeBlocks() {
            -        String input = "First example:```java\nSystem.out.println(\"First\");\n```\n\nSecond example:```java\nSystem.out.println(\"Second\");\n```";
            +        String input =
            +                "First example:```java\nSystem.out.println(\"First\");\n```\n\nSecond example:```java\nSystem.out.println(\"Second\");\n```";
                     String html = markdownService.processStructured(input).html();
             
                     System.out.println("\nTest: Multiple Java code blocks");
            @@ -164,7 +167,8 @@ void testMultipleJavaCodeBlocks() {
             
                 @Test
                 void testJavaCodeBlockAfterColon() {
            -        String input = "The solution is:```java\npublic static void main(String[] args) {\n    // Java code here\n}\n```";
            +        String input =
            +                "The solution is:```java\npublic static void main(String[] args) {\n    // Java code here\n}\n```";
                     String html = markdownService.processStructured(input).html();
             
                     System.out.println("\nTest: Java code block after colon");
            @@ -178,7 +182,8 @@ void testJavaCodeBlockAfterColon() {
             
                 @Test
                 void testJavaCodeBlockWithSpecialCharacters() {
            -        String input = "Advanced Java features:```java\n// Using generics and lambdas\nList names = Arrays.asList(\"Alice\", \"Bob\");\nnames.stream().filter(name -> name.length() > 3).forEach(System.out::println);\n```";
            +        String input =
            +                "Advanced Java features:```java\n// Using generics and lambdas\nList names = Arrays.asList(\"Alice\", \"Bob\");\nnames.stream().filter(name -> name.length() > 3).forEach(System.out::println);\n```";
                     String html = markdownService.processStructured(input).html();
             
                     System.out.println("\nTest: Java code block with special characters");
            @@ -205,7 +210,8 @@ void testEmptyJavaCodeBlock() {
             
                 @Test
                 void testJavaCodeBlockWithAnnotations() {
            -        String input = "Spring Boot example:```java\n@RestController\npublic class UserController {\n    @GetMapping(\"/users\")\n    public List getUsers() {\n        return userService.findAll();\n    }\n}\n```";
            +        String input =
            +                "Spring Boot example:```java\n@RestController\npublic class UserController {\n    @GetMapping(\"/users\")\n    public List getUsers() {\n        return userService.findAll();\n    }\n}\n```";
                     String html = markdownService.processStructured(input).html();
             
                     System.out.println("\nTest: Java code block with annotations");
            diff --git a/src/test/java/com/williamcallahan/javachat/service/MarkdownServiceTest.java b/src/test/java/com/williamcallahan/javachat/service/MarkdownServiceTest.java
            index 0690d80..2d9b70c 100644
            --- a/src/test/java/com/williamcallahan/javachat/service/MarkdownServiceTest.java
            +++ b/src/test/java/com/williamcallahan/javachat/service/MarkdownServiceTest.java
            @@ -1,21 +1,21 @@
             package com.williamcallahan.javachat.service;
             
            +import static org.junit.jupiter.api.Assertions.*;
            +
             import com.williamcallahan.javachat.service.markdown.UnifiedMarkdownService;
             import com.williamcallahan.javachat.support.AsciiTextNormalizer;
            +import java.util.Locale;
             import org.junit.jupiter.api.BeforeEach;
             import org.junit.jupiter.api.DisplayName;
             import org.junit.jupiter.api.Test;
            -import java.util.Locale;
            -
            -import static org.junit.jupiter.api.Assertions.*;
             
             /**
              * Exercises markdown normalization and rendering paths for the unified service.
              */
             class MarkdownServiceTest {
            -    
            +
                 private MarkdownService markdownService;
            -    
            +
                 @BeforeEach
                 void setUp() {
                     markdownService = new MarkdownService(new UnifiedMarkdownService());
            @@ -34,22 +34,24 @@ void testParagraphBreaksQuestionExclamation() {
                 void testHeaders() {
                     String markdown = "# Header 1\n## Header 2\n### Header 3";
                     String html = markdownService.processStructured(markdown).html();
            -        
            +
                     assertTrue(html.contains("

            Header 1

            "), "Should contain H1"); assertTrue(html.contains("

            Header 2

            "), "Should contain H2"); assertTrue(html.contains("

            Header 3

            "), "Should contain H3"); } - + @Test @DisplayName("Should render bold and italic text") void testBoldAndItalic() { String markdown = "**bold text** and *italic text* and ***bold italic***"; String html = markdownService.processStructured(markdown).html(); - + assertTrue(html.contains("bold text"), "Should contain bold"); assertTrue(html.contains("italic text"), "Should contain italic"); - assertTrue(html.contains("bold italic") || - html.contains("bold italic"), "Should contain bold italic"); + assertTrue( + html.contains("bold italic") + || html.contains("bold italic"), + "Should contain bold italic"); } @Test @@ -66,58 +68,57 @@ void testBoldWithSpacesInsideMarkers() { void testEnrichmentNotBrokenByPreprocessing() { String markdown = "A sentence. {{hint:This should remain intact even after paragraph logic.}} Next."; String html = markdownService.processStructured(markdown).html(); - assertTrue(html.contains("inline-enrichment hint"), - "Enrichment card should render as a single unit"); + assertTrue(html.contains("inline-enrichment hint"), "Enrichment card should render as a single unit"); } - + @Test @DisplayName("Should render unordered lists") void testUnorderedLists() { String markdown = "- Item 1\n- Item 2\n- Item 3"; String html = markdownService.processStructured(markdown).html(); - + assertTrue(html.contains("
              "), "Should contain UL tag"); assertTrue(html.contains("
            • Item 1
            • "), "Should contain list item 1"); assertTrue(html.contains("
            • Item 2
            • "), "Should contain list item 2"); assertTrue(html.contains("
            • Item 3
            • "), "Should contain list item 3"); assertTrue(html.contains("
            "), "Should close UL tag"); } - + @Test @DisplayName("Should render ordered lists") void testOrderedLists() { String markdown = "1. First item\n2. Second item\n3. Third item"; String html = markdownService.processStructured(markdown).html(); System.out.println("[DEBUG testOrderedLists] HTML=\n" + html); - + assertTrue(html.contains("
              "), "Should contain OL tag"); assertTrue(html.contains("
            1. First item
            2. "), "Should contain list item 1"); assertTrue(html.contains("
            3. Second item
            4. "), "Should contain list item 2"); assertTrue(html.contains("
            5. Third item
            6. "), "Should contain list item 3"); assertTrue(html.contains("
            "), "Should close OL tag"); } - + @Test @DisplayName("Should render code blocks with language class") void testCodeBlocks() { String markdown = "```java\npublic class Test {}\n```"; String html = markdownService.processStructured(markdown).html(); - + assertTrue(html.contains("
            "), "Should contain PRE tag");
                     assertTrue(html.contains(""), "Should contain code with language class");
                     assertTrue(html.contains("public class Test {}"), "Should contain code content");
                 }
            -    
            +
                 @Test
                 @DisplayName("Should render inline code")
                 void testInlineCode() {
                     String markdown = "Use `System.out.println()` to print";
                     String html = markdownService.processStructured(markdown).html();
                     System.out.println("[DEBUG testInlineCode] HTML=\n" + html);
            -        
            +
                     assertTrue(html.contains("System.out.println()"), "Should contain inline code");
                 }
            -    
            +
                 @Test
                 @DisplayName("Should preserve enrichment markers")
                 void testEnrichmentMarkers() {
            @@ -127,19 +128,20 @@ void testEnrichmentMarkers() {
                     assertTrue(html.contains("inline-enrichment hint"), "Hint card should render");
                     assertTrue(html.contains("inline-enrichment warning"), "Warning card should render");
                 }
            -    
            +
                 @Test
                 @DisplayName("Should handle mixed markdown with enrichments")
                 void testMixedContent() {
            -        String markdown = "# Java 24\n\n**Key features:**\n\n1. Source Version24\n2. Type System\n\n{{hint:Always check the docs}}";
            +        String markdown =
            +                "# Java 24\n\n**Key features:**\n\n1. Source Version24\n2. Type System\n\n{{hint:Always check the docs}}";
                     String html = markdownService.processStructured(markdown).html();
            -        
            +
                     assertTrue(html.contains("

            Java 24

            "), "Should have header"); assertTrue(html.contains("Key features:"), "Should have bold text"); assertTrue(html.contains("
              "), "Should have ordered list"); assertTrue(html.contains("inline-enrichment hint"), "Should render hint card"); } - + @Test @DisplayName("Should handle line breaks properly") void testLineBreaks() { @@ -149,47 +151,47 @@ void testLineBreaks() { // With SOFT_BREAK=\n we do not force
              ; ensure second paragraph exists assertTrue(html.contains("

              New paragraph

              "), "Should have new paragraph"); } - + @Test @DisplayName("Should escape raw HTML for security") void testHTMLEscaping() { String markdown = "\n\n**Safe bold**"; String html = markdownService.processStructured(markdown).html(); - + assertFalse(html.contains("