From 36622452e234c882ca40bcdf8198ce4b1ae5097a Mon Sep 17 00:00:00 2001 From: Marco Ferreira Date: Wed, 19 Nov 2025 20:19:59 -0500 Subject: [PATCH 1/3] feat(http-server) Added http server, tests, updated readme, added workflow --- .env | 10 ++ .env.example | 10 ++ .github/workflows/frontend-test.yml | 35 ++++ Makefile | 51 +++++- README.md | 16 +- src/Backend/opti-sql-go/go.mod | 2 +- src/FrontEnd/.dockerignore | 38 +++++ src/FrontEnd/.env | 10 ++ src/FrontEnd/Dockerfile | 17 ++ src/FrontEnd/README.md | 186 +++++++++++++++++++++ src/FrontEnd/app/__init__.py | 0 src/FrontEnd/app/api/__init__.py | 0 src/FrontEnd/app/api/v1/__init__.py | 0 src/FrontEnd/app/api/v1/routes/__init__.py | 0 src/FrontEnd/app/api/v1/routes/health.py | 17 ++ src/FrontEnd/app/api/v1/routes/query.py | 79 +++++++++ src/FrontEnd/app/core/__init__.py | 0 src/FrontEnd/app/core/config.py | 27 +++ src/FrontEnd/app/core/logging.py | 44 +++++ src/FrontEnd/app/main.py | 51 ++++++ src/FrontEnd/app/models/__init__.py | 0 src/FrontEnd/app/models/schemas.py | 32 ++++ src/FrontEnd/docker-compose.yml | 24 +++ src/FrontEnd/generate_swagger.py | 12 ++ src/FrontEnd/main.cpp | 6 - src/FrontEnd/pytest.ini | 9 + src/FrontEnd/requirements.txt | 10 ++ src/FrontEnd/swagger.yml | 155 +++++++++++++++++ src/FrontEnd/tests/README.md | 119 +++++++++++++ src/FrontEnd/tests/__init__.py | 0 src/FrontEnd/tests/conftest.py | 26 +++ src/FrontEnd/tests/test_health.py | 28 ++++ src/FrontEnd/tests/test_integration.py | 116 +++++++++++++ src/FrontEnd/tests/test_query.py | 124 ++++++++++++++ 34 files changed, 1243 insertions(+), 11 deletions(-) create mode 100644 .env create mode 100644 .env.example create mode 100644 .github/workflows/frontend-test.yml create mode 100644 src/FrontEnd/.dockerignore create mode 100644 src/FrontEnd/.env create mode 100644 src/FrontEnd/Dockerfile create mode 100644 src/FrontEnd/README.md create mode 100644 src/FrontEnd/app/__init__.py create mode 100644 src/FrontEnd/app/api/__init__.py create mode 100644 src/FrontEnd/app/api/v1/__init__.py create mode 100644 src/FrontEnd/app/api/v1/routes/__init__.py create mode 100644 src/FrontEnd/app/api/v1/routes/health.py create mode 100644 src/FrontEnd/app/api/v1/routes/query.py create mode 100644 src/FrontEnd/app/core/__init__.py create mode 100644 src/FrontEnd/app/core/config.py create mode 100644 src/FrontEnd/app/core/logging.py create mode 100644 src/FrontEnd/app/main.py create mode 100644 src/FrontEnd/app/models/__init__.py create mode 100644 src/FrontEnd/app/models/schemas.py create mode 100644 src/FrontEnd/docker-compose.yml create mode 100644 src/FrontEnd/generate_swagger.py delete mode 100644 src/FrontEnd/main.cpp create mode 100644 src/FrontEnd/pytest.ini create mode 100644 src/FrontEnd/requirements.txt create mode 100644 src/FrontEnd/swagger.yml create mode 100644 src/FrontEnd/tests/README.md create mode 100644 src/FrontEnd/tests/__init__.py create mode 100644 src/FrontEnd/tests/conftest.py create mode 100644 src/FrontEnd/tests/test_health.py create mode 100644 src/FrontEnd/tests/test_integration.py create mode 100644 src/FrontEnd/tests/test_query.py diff --git a/.env b/.env new file mode 100644 index 0000000..0070cc1 --- /dev/null +++ b/.env @@ -0,0 +1,10 @@ +# Server Configuration +PORT=8000 +HOST=0.0.0.0 + +# Logging Configuration +# Options: prod, info, debug +LOGGING_MODE=info + +# Testing Configuration +TEST_SERVER_URL=http://localhost:8000 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0070cc1 --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +# Server Configuration +PORT=8000 +HOST=0.0.0.0 + +# Logging Configuration +# Options: prod, info, debug +LOGGING_MODE=info + +# Testing Configuration +TEST_SERVER_URL=http://localhost:8000 diff --git a/.github/workflows/frontend-test.yml b/.github/workflows/frontend-test.yml new file mode 100644 index 0000000..9208ef6 --- /dev/null +++ b/.github/workflows/frontend-test.yml @@ -0,0 +1,35 @@ +name: Frontend Tests + +on: + push: + branches: [ main, pre-release ] + paths: + - 'src/FrontEnd/**' + - '.github/workflows/frontend-test.yml' + pull_request: + branches: [ main, pre-release ] + paths: + - 'src/FrontEnd/**' + - '.github/workflows/frontend-test.yml' + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r src/FrontEnd/requirements.txt + + - name: Run tests + run: | + cd src/FrontEnd + pytest -v -m "not integration" diff --git a/Makefile b/Makefile index 0afcce1..0c24562 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help go-test rust-test go-run rust-run go-lint rust-lint go-fmt rust-fmt test-all lint-all fmt-all pre-push +.PHONY: help go-test rust-test go-run rust-run go-lint rust-lint go-fmt rust-fmt frontend-test frontend-run frontend-docker-build frontend-docker-run frontend-docker-down frontend-setup test-all lint-all fmt-all pre-push # Default target help: @@ -11,7 +11,13 @@ help: @echo " make rust-lint - Run Rust linter and formatter check" @echo " make go-fmt - Format Go code" @echo " make rust-fmt - Format Rust code" - @echo " make test-all - Run all tests (Go + Rust)" + @echo " make frontend-test - Run Python/Frontend tests" + @echo " make frontend-run - Run Frontend server (without Docker)" + @echo " make frontend-setup - Setup Python virtual environment and install dependencies" + @echo " make frontend-docker-build - Build Frontend Docker image" + @echo " make frontend-docker-run - Run Frontend using Docker Compose" + @echo " make frontend-docker-down - Stop Frontend Docker containers" + @echo " make test-all - Run all tests (Go + Rust + Frontend)" @echo " make lint-all - Run all linters (Go + Rust)" @echo " make fmt-all - Format all code (Go + Rust)" @echo " make pre-push - Run fmt, lint, and test (use before pushing)" @@ -71,8 +77,47 @@ rust-fmt-check: @echo "Checking Rust formatting..." cd src/Backend/opti-sql-rs && cargo fmt --check +# Frontend targets +frontend-setup: + @echo "Setting up Python virtual environment..." + rm -rf src/FrontEnd/venv + cd src/FrontEnd && python3.12 -m venv --without-pip venv + @echo "Installing pip..." + cd src/FrontEnd && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python + @echo "Installing dependencies..." + cd src/FrontEnd && . venv/bin/activate && pip install --upgrade pip && pip install -r requirements.txt + @echo "Frontend setup completed! Activate with: cd src/FrontEnd && source venv/bin/activate" + +frontend-test: frontend-setup + @echo "Running Frontend/Python tests..." + cd src/FrontEnd && . venv/bin/activate && pytest -m "not integration" + +frontend-run: + @echo "Running Frontend server..." + cd src/FrontEnd && . venv/bin/activate && python -m uvicorn app.main:app --reload --host 0.0.0.0 --port 8005 + +frontend-docker-build: + @echo "Building Frontend Docker image..." + @if [ ! -f src/FrontEnd/.env ]; then \ + echo "Creating .env file from root .env..."; \ + cp .env src/FrontEnd/.env; \ + fi + cd src/FrontEnd && docker compose build + +frontend-docker-run: + @echo "Running Frontend with Docker Compose..." + @if [ ! -f src/FrontEnd/.env ]; then \ + echo "Creating .env file from root .env..."; \ + cp .env src/FrontEnd/.env; \ + fi + cd src/FrontEnd && docker compose up -d + +frontend-docker-down: + @echo "Stopping Frontend Docker containers..." + cd src/FrontEnd && docker compose down + # Combined targets -test-all: go-test rust-test +test-all: go-test rust-test frontend-test @echo "All tests completed!" lint-all: go-lint rust-lint diff --git a/README.md b/README.md index 061a643..52dedf9 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ A high-performance, in-memory query execution engine. ![Go Tests](https://github.com/Rich-T-kid/OptiSQL/actions/workflows/go-test.yml/badge.svg) ![Rust Tests](https://github.com/Rich-T-kid/OptiSQL/actions/workflows/rust-test.yml/badge.svg) +![Frontend Tests](https://github.com/Rich-T-kid/OptiSQL/actions/workflows/frontend-test.yml/badge.svg) + ## Overview @@ -20,6 +22,8 @@ OptiSQL is a custom in-memory query execution engine. The backend (physical exec - Go 1.24+ - Rust 1.70+ - C++23 +- Python 3.11+ +- Docker 29+ - Make - git @@ -36,6 +40,13 @@ make go-run # Build and run Rust backend make rust-run +# Frontend setup and run +make frontend-setup # Create venv and install dependencies +make frontend-run # Run locally without Docker +# OR with Docker +make frontend-docker-build +make frontend-docker-run + # Run all tests make test-all @@ -58,7 +69,10 @@ OptiSQL/ │ │ └── opti-sql-rs/ # Rust implementation (Go clone for learning) │ │ ├── src/project/ # Core project logic │ │ └── src/ # Query processing modules -│ └── FrontEnd/ # C++ frontend (in development) +│ └── FrontEnd/ # Python/FastAPI HTTP server (C++ query processing in progress) +│ ├── app/ # API endpoints and logic +│ ├── tests/ # Frontend tests +│ └── Dockerfile # Docker configuration ├── .github/workflows/ # CI/CD pipelines ├── Makefile # Development commands └── CONTRIBUTING.md # Contribution guidelines diff --git a/src/Backend/opti-sql-go/go.mod b/src/Backend/opti-sql-go/go.mod index 49182e3..665dc8c 100644 --- a/src/Backend/opti-sql-go/go.mod +++ b/src/Backend/opti-sql-go/go.mod @@ -1,6 +1,6 @@ module opti-sql-go -go 1.24.0 +go 1.23 require ( github.com/apache/arrow/go/v17 v17.0.0 diff --git a/src/FrontEnd/.dockerignore b/src/FrontEnd/.dockerignore new file mode 100644 index 0000000..99162ba --- /dev/null +++ b/src/FrontEnd/.dockerignore @@ -0,0 +1,38 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +venv/ +ENV/ +*.egg-info/ +dist/ +build/ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Git +.git/ +.gitignore + +# Documentation +*.md +tests/ + +# Other +.DS_Store +*.log +swagger.yml +generate_swagger.py +.env.example diff --git a/src/FrontEnd/.env b/src/FrontEnd/.env new file mode 100644 index 0000000..0070cc1 --- /dev/null +++ b/src/FrontEnd/.env @@ -0,0 +1,10 @@ +# Server Configuration +PORT=8000 +HOST=0.0.0.0 + +# Logging Configuration +# Options: prod, info, debug +LOGGING_MODE=info + +# Testing Configuration +TEST_SERVER_URL=http://localhost:8000 diff --git a/src/FrontEnd/Dockerfile b/src/FrontEnd/Dockerfile new file mode 100644 index 0000000..282a56c --- /dev/null +++ b/src/FrontEnd/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY . . +COPY ../../.env + +# Expose port (will be read from config.yml) +EXPOSE 8000 + +# Run the application +CMD ["python", "-m", "app.main"] diff --git a/src/FrontEnd/README.md b/src/FrontEnd/README.md new file mode 100644 index 0000000..2131f8e --- /dev/null +++ b/src/FrontEnd/README.md @@ -0,0 +1,186 @@ +# OptiSQL Frontend API + +FastAPI server for SQL query processing and optimization. + +## Features + +- **Health Check Endpoint**: `/api/v1/health` +- **SQL Query Processing**: `/api/v1/query` + - Supports file uploads (CSV, JSON, Parquet, Excel) + - Supports file URIs (HTTP/HTTPS) + - Configurable logging levels + +## Project Structure + +``` +FrontEnd/ +├── app/ +│ ├── main.py # FastAPI application +│ ├── api/v1/routes/ # API endpoints +│ ├── core/ # Config and logging +│ └── models/ # Pydantic schemas +├── tests/ # Test suite +├── config.yml # Configuration +├── requirements.txt # Python dependencies +├── Dockerfile # Docker image +├── docker-compose.yml # Docker Compose config +└── pytest.ini # Pytest configuration +``` + +## Configuration + +Create a `.env` file (or copy from `.env.example`): + +```bash +cp .env.example .env +``` + +Edit `.env` to configure: + +```env +# Server Configuration +PORT=8000 +HOST=0.0.0.0 + +# Logging Configuration +# Options: prod, info, debug +LOGGING_MODE=info +``` + +## Running Locally + +### Install Dependencies + +```bash +pip install -r requirements.txt +``` + +### Run the Server + +```bash +python -m app.main +``` + +The server will start on the port specified in `config.yml` (default: 8000). + +### Access API Documentation + +- Swagger UI: http://localhost:8000/docs +- ReDoc: http://localhost:8000/redoc + +## Running with Docker + +### Build and Run + +```bash +docker compose up --build +``` + +### Run in Background + +```bash +docker compose up -d +``` + +### View Logs + +```bash +docker compose logs -f +``` + +### Stop the Service + +```bash +docker compose down +``` + +Note: Use `docker compose` (without hyphen) for Docker Compose V2. + +## Testing + + +### Run Unit Tests Only (No Server Required) + +```bash +pytest -m "not integration" +``` + +### Run Integration Tests (Requires Running Server) + +```bash +# Start server first +docker compose up -d + +# Run integration tests +pytest -m integration +``` + +### Run All Tests + +```bash +pytest +``` + +### Run with Verbose Output + +```bash +pytest -v +``` + +### Run Specific Tests + +```bash +pytest tests/test_health.py +pytest tests/test_query.py +pytest tests/test_integration.py +``` + +### Test Against Different Server + +```bash +TEST_SERVER_URL=http://localhost:9000 pytest -m integration +``` + +## API Endpoints + +### Health Check + +```bash +curl http://localhost:8000/api/v1/health +``` + +### SQL Query Processing (File Upload) + +```bash +curl -X POST http://localhost:8000/api/v1/query \ + -F "sql_query=SELECT * FROM data" \ + -F "file=@data.csv" +``` + +### SQL Query Processing (File URI) + +```bash +curl -X POST http://localhost:8000/api/v1/query \ + -F "sql_query=SELECT * FROM data" \ + -F "file_uri=https://example.com/data.csv" +``` + +## Logging Modes + +- **prod**: WARNING level, minimal output +- **info**: INFO level, standard logging +- **debug**: DEBUG level, detailed logs with file/line numbers + +## Development + +### Hot Reload + +When running with Docker Compose, the app directory is mounted as a volume, enabling hot-reload during development. + +### Generate Swagger YAML + +```bash +python generate_swagger.py +``` + +This generates `swagger.yml` with the OpenAPI specification. diff --git a/src/FrontEnd/app/__init__.py b/src/FrontEnd/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/FrontEnd/app/api/__init__.py b/src/FrontEnd/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/FrontEnd/app/api/v1/__init__.py b/src/FrontEnd/app/api/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/FrontEnd/app/api/v1/routes/__init__.py b/src/FrontEnd/app/api/v1/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/FrontEnd/app/api/v1/routes/health.py b/src/FrontEnd/app/api/v1/routes/health.py new file mode 100644 index 0000000..c46ba37 --- /dev/null +++ b/src/FrontEnd/app/api/v1/routes/health.py @@ -0,0 +1,17 @@ +from fastapi import APIRouter +from app.models.schemas import HealthResponse +import logging + +router = APIRouter() +logger = logging.getLogger(__name__) + +@router.get("/health", response_model=HealthResponse, tags=["Health"]) +async def health_check(): + """ + Health check endpoint to verify the service is running. + + Returns: + HealthResponse: Current health status of the service + """ + logger.info("Health check requested") + return HealthResponse(status="healthy", version="0.1.0") diff --git a/src/FrontEnd/app/api/v1/routes/query.py b/src/FrontEnd/app/api/v1/routes/query.py new file mode 100644 index 0000000..9914824 --- /dev/null +++ b/src/FrontEnd/app/api/v1/routes/query.py @@ -0,0 +1,79 @@ +from fastapi import APIRouter, UploadFile, File, Form, HTTPException +from app.models.schemas import SQLQueryResponse +import logging +import time +from typing import Optional + +router = APIRouter() +logger = logging.getLogger(__name__) + +@router.post("/query", response_model=SQLQueryResponse, tags=["Query Processing"]) +async def process_sql_query( + sql_query: str = Form(..., description="SQL query to process"), + file_uri: Optional[str] = Form(None, description="URI to a remote file (optional if file is provided)"), + file: Optional[UploadFile] = File(default=None, description="Data file to process (optional if file_uri is provided)") +): + """ + Process a SQL query against an uploaded file or a file from a URI. + + Args: + sql_query: SQL query string to execute + file: Uploaded file (CSV, JSON, Parquet, etc.) - optional + file_uri: URI to a remote file (e.g., s3://bucket/file.csv, https://example.com/data.csv) - optional + + Returns: + SQLQueryResponse: Query processing results + + Note: + Either 'file' or 'file_uri' must be provided, but not both. + """ + start_time = time.time() + + logger.info(f"Processing SQL query: {sql_query[:100]}...") + + # Normalize inputs: treat empty strings as None + if file_uri is not None and file_uri.strip() == "": + file_uri = None + + # Check if file is actually empty (no filename means no file uploaded) + if file is not None and (not file.filename or file.filename == ""): + file = None + + # Validate input: must have either file or file_uri + if file is None and file_uri is None: + raise HTTPException( + status_code=400, + detail="Either 'file' (uploaded file) or 'file_uri' (file URI) must be provided" + ) + + if file is not None and file_uri is not None: + raise HTTPException( + status_code=400, + detail="Cannot provide both 'file' and 'file_uri'. Please provide only one." + ) + + try: + + # TODO: Implement actual SQL query processing logic + + result = { + "query_length": len(sql_query), + "message": "Query processing not yet implemented" + } + + execution_time = (time.time() - start_time) * 1000 + + logger.info(f"Query processed successfully in {execution_time:.2f}ms") + + return SQLQueryResponse( + status="success", + query=sql_query, + result=result, + execution_time_ms=execution_time + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error processing query: {str(e)}") + raise HTTPException(status_code=500, detail=f"Query processing failed: {str(e)}") \ No newline at end of file diff --git a/src/FrontEnd/app/core/__init__.py b/src/FrontEnd/app/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/FrontEnd/app/core/config.py b/src/FrontEnd/app/core/config.py new file mode 100644 index 0000000..e59f6b4 --- /dev/null +++ b/src/FrontEnd/app/core/config.py @@ -0,0 +1,27 @@ +import os +from typing import Literal +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv() + +class Config: + def __init__(self): + pass + + @property + def port(self) -> int: + return int(os.getenv('PORT', '8000')) + + @property + def host(self) -> str: + return os.getenv('HOST', '0.0.0.0') + + @property + def logging_mode(self) -> Literal['prod', 'info', 'debug']: + mode = os.getenv('LOGGING_MODE', 'info').lower() + if mode not in ['prod', 'info', 'debug']: + return 'info' + return mode + +config = Config() diff --git a/src/FrontEnd/app/core/logging.py b/src/FrontEnd/app/core/logging.py new file mode 100644 index 0000000..1a0671f --- /dev/null +++ b/src/FrontEnd/app/core/logging.py @@ -0,0 +1,44 @@ +import logging +import sys +from typing import Literal + +def setup_logging(mode: Literal['prod', 'info', 'debug']) -> None: + """ + Configure logging based on the mode specified in config.yml + + Args: + mode: Logging mode - 'prod', 'info', or 'debug' + """ + # Define logging levels + level_map = { + 'prod': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG + } + + log_level = level_map.get(mode, logging.INFO) + + # Configure format based on mode + if mode == 'debug': + log_format = '%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - %(message)s' + elif mode == 'info': + log_format = '%(asctime)s - %(levelname)s - %(message)s' + else: # prod + log_format = '%(asctime)s - %(levelname)s - %(message)s' + + # Configure root logger + logging.basicConfig( + level=log_level, + format=log_format, + handlers=[ + logging.StreamHandler(sys.stdout) + ] + ) + + # Set uvicorn logger levels + logging.getLogger("uvicorn").setLevel(log_level) + logging.getLogger("uvicorn.access").setLevel(log_level) + logging.getLogger("uvicorn.error").setLevel(log_level) + + logger = logging.getLogger(__name__) + logger.info(f"Logging configured with mode: {mode} (level: {logging.getLevelName(log_level)})") diff --git a/src/FrontEnd/app/main.py b/src/FrontEnd/app/main.py new file mode 100644 index 0000000..f8fa204 --- /dev/null +++ b/src/FrontEnd/app/main.py @@ -0,0 +1,51 @@ +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from app.core.config import config +from app.core.logging import setup_logging +from app.api.v1.routes import health, query +import logging + +# Setup logging +setup_logging(config.logging_mode) +logger = logging.getLogger(__name__) + +# Create FastAPI app +app = FastAPI( + title="OptiSQL Frontend API", + description="FastAPI server for SQL query processing and optimization", + version="0.1.0", + docs_url="/docs", + redoc_url="/redoc" +) + +# Add CORS middleware +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Include routers +app.include_router(health.router, prefix="/api/v1") +app.include_router(query.router, prefix="/api/v1") + +@app.on_event("startup") +async def startup_event(): + logger.info("Starting OptiSQL Frontend API") + logger.info(f"Server configuration - Host: {config.host}, Port: {config.port}") + logger.info(f"Logging mode: {config.logging_mode}") + +@app.on_event("shutdown") +async def shutdown_event(): + logger.info("Shutting down OptiSQL Frontend API") + +if __name__ == "__main__": + import uvicorn + uvicorn.run( + "app.main:app", + host=config.host, + port=config.port, + reload=True if config.logging_mode == "debug" else False + ) diff --git a/src/FrontEnd/app/models/__init__.py b/src/FrontEnd/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/FrontEnd/app/models/schemas.py b/src/FrontEnd/app/models/schemas.py new file mode 100644 index 0000000..ec626cb --- /dev/null +++ b/src/FrontEnd/app/models/schemas.py @@ -0,0 +1,32 @@ +from pydantic import BaseModel, Field +from typing import Optional, Any + +class HealthResponse(BaseModel): + status: str = Field(..., description="Health status of the service") + version: str = Field(default="0.1.0", description="API version") + +class SQLQueryRequest(BaseModel): + sql_query: str = Field(..., description="SQL query to process") + + class Config: + json_schema_extra = { + "example": { + "sql_query": "SELECT * FROM users WHERE age > 25" + } + } + +class SQLQueryResponse(BaseModel): + status: str = Field(..., description="Processing status") + query: str = Field(..., description="Original SQL query") + result: Optional[Any] = Field(None, description="Query processing result") + execution_time_ms: Optional[float] = Field(None, description="Execution time in milliseconds") + + class Config: + json_schema_extra = { + "example": { + "status": "success", + "query": "SELECT * FROM users WHERE age > 25", + "result": {"rows": 42}, + "execution_time_ms": 123.45 + } + } diff --git a/src/FrontEnd/docker-compose.yml b/src/FrontEnd/docker-compose.yml new file mode 100644 index 0000000..2afaa98 --- /dev/null +++ b/src/FrontEnd/docker-compose.yml @@ -0,0 +1,24 @@ +services: + fastapi: + build: + context: . + dockerfile: Dockerfile + container_name: optisql-frontend + ports: + - "${PORT:-8000}:${PORT:-8000}" + volumes: + # Mount the app directory + # TODO: need to add cpp library + - ./app:/app/app + - ./.env:/app/.env + env_file: + - .env + environment: + - PYTHONUNBUFFERED=1 + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:${PORT:-8000}/api/v1/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s diff --git a/src/FrontEnd/generate_swagger.py b/src/FrontEnd/generate_swagger.py new file mode 100644 index 0000000..7e0725c --- /dev/null +++ b/src/FrontEnd/generate_swagger.py @@ -0,0 +1,12 @@ +import yaml +import json +from app.main import app + +# Generate OpenAPI schema +openapi_schema = app.openapi() + +# Write to YAML file +with open('swagger.yml', 'w') as f: + yaml.dump(openapi_schema, f, default_flow_style=False, sort_keys=False) + +print("Swagger YAML file generated: swagger.yml") diff --git a/src/FrontEnd/main.cpp b/src/FrontEnd/main.cpp deleted file mode 100644 index fa5c04d..0000000 --- a/src/FrontEnd/main.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main() { - std::cout << "Hello World" << std::endl; - return 0; -} diff --git a/src/FrontEnd/pytest.ini b/src/FrontEnd/pytest.ini new file mode 100644 index 0000000..7193e46 --- /dev/null +++ b/src/FrontEnd/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_functions = test_* +addopts = -v --tb=short + +markers = + integration: marks tests as integration tests (requires running server) + unit: marks tests as unit tests (no server required) diff --git a/src/FrontEnd/requirements.txt b/src/FrontEnd/requirements.txt new file mode 100644 index 0000000..c3d97d2 --- /dev/null +++ b/src/FrontEnd/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.109.0 +uvicorn[standard]==0.27.0 +pydantic==2.5.3 +python-multipart==0.0.6 +python-dotenv==1.0.0 + +# Testing +pytest==7.4.4 +pytest-asyncio==0.23.3 +httpx==0.26.0 diff --git a/src/FrontEnd/swagger.yml b/src/FrontEnd/swagger.yml new file mode 100644 index 0000000..5a5163a --- /dev/null +++ b/src/FrontEnd/swagger.yml @@ -0,0 +1,155 @@ +openapi: 3.1.0 +info: + title: OptiSQL Frontend API + description: FastAPI server for SQL query processing and optimization + version: 0.1.0 +paths: + /api/v1/health: + get: + tags: + - Health + summary: Health Check + description: "Health check endpoint to verify the service is running.\n\nReturns:\n\ + \ HealthResponse: Current health status of the service" + operationId: health_check_api_v1_health_get + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/HealthResponse' + /api/v1/query: + post: + tags: + - Query Processing + summary: Process Sql Query + description: "Process a SQL query against an uploaded file or a file from a\ + \ URI.\n\nArgs:\n sql_query: SQL query string to execute\n file: Uploaded\ + \ file (CSV, JSON, Parquet, etc.) - optional\n file_uri: URI to a remote\ + \ file (e.g., s3://bucket/file.csv, https://example.com/data.csv) - optional\n\ + \nReturns:\n SQLQueryResponse: Query processing results\n\nNote:\n Either\ + \ 'file' or 'file_uri' must be provided, but not both." + operationId: process_sql_query_api_v1_query_post + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/Body_process_sql_query_api_v1_query_post' + required: true + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/SQLQueryResponse' + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' +components: + schemas: + Body_process_sql_query_api_v1_query_post: + properties: + sql_query: + type: string + title: Sql Query + description: SQL query to process + file_uri: + anyOf: + - type: string + - type: 'null' + title: File Uri + description: URI to a remote file (optional if file is provided) + file: + anyOf: + - type: string + format: binary + - type: 'null' + title: File + description: Data file to process (optional if file_uri is provided) + type: object + required: + - sql_query + title: Body_process_sql_query_api_v1_query_post + HTTPValidationError: + properties: + detail: + items: + $ref: '#/components/schemas/ValidationError' + type: array + title: Detail + type: object + title: HTTPValidationError + HealthResponse: + properties: + status: + type: string + title: Status + description: Health status of the service + version: + type: string + title: Version + description: API version + default: 0.1.0 + type: object + required: + - status + title: HealthResponse + SQLQueryResponse: + properties: + status: + type: string + title: Status + description: Processing status + query: + type: string + title: Query + description: Original SQL query + result: + anyOf: + - {} + - type: 'null' + title: Result + description: Query processing result + execution_time_ms: + anyOf: + - type: number + - type: 'null' + title: Execution Time Ms + description: Execution time in milliseconds + type: object + required: + - status + - query + title: SQLQueryResponse + example: + execution_time_ms: 123.45 + query: SELECT * FROM users WHERE age > 25 + result: + rows: 42 + status: success + ValidationError: + properties: + loc: + items: + anyOf: + - type: string + - type: integer + type: array + title: Location + msg: + type: string + title: Message + type: + type: string + title: Error Type + type: object + required: + - loc + - msg + - type + title: ValidationError diff --git a/src/FrontEnd/tests/README.md b/src/FrontEnd/tests/README.md new file mode 100644 index 0000000..f0d94b0 --- /dev/null +++ b/src/FrontEnd/tests/README.md @@ -0,0 +1,119 @@ +# Tests + +This directory contains tests for the OptiSQL FastAPI server. + +## Test Types + +### Unit Tests +Unit tests use FastAPI's `TestClient` and don't require a running server. They are fast and can be run anywhere. + +- `test_health.py` - Health endpoint tests +- `test_query.py` - Query endpoint tests + +### Integration Tests +Integration tests connect to a real running server. They test the full stack including networking, Docker, etc. + +- `test_integration.py` - Tests against a running server + +## Running Tests + +### Run all tests (unit only, no server required) +```bash +pytest -m "not integration" +``` + +### Run only unit tests +```bash +pytest tests/test_health.py tests/test_query.py +``` + +### Run integration tests (requires running server) + +First, start the server: +```bash +# Using Docker +docker compose up -d + +# Or locally +python -m app.main +``` + +Then run integration tests: +```bash +pytest -m integration +``` + +### Run all tests (unit + integration) +```bash +# Make sure server is running first! +docker compose up -d + +# Run all tests +pytest +``` + +### Run specific test file +```bash +pytest tests/test_health.py +pytest tests/test_query.py +pytest tests/test_integration.py +``` + +### Run with verbose output +```bash +pytest -v +``` + +### Run with coverage +```bash +pytest --cov=app --cov-report=html +``` + +## Environment Variables + +### `TEST_SERVER_URL` +Set this to test against a different server URL (default: `http://localhost:8000`) + +```bash +TEST_SERVER_URL=http://localhost:9000 pytest -m integration +``` + +## Complete Test Workflow + +```bash +# 1. Run unit tests (no server needed) +pytest -m "not integration" + +# 2. Start the server +docker compose up -d + +# 3. Wait for server to be ready +sleep 3 + +# 4. Run integration tests +pytest -m integration + +# 5. Run all tests together +pytest + +# 6. Stop the server +docker compose down +``` + +## Test Structure + +- `conftest.py` - Shared fixtures for all tests +- `test_health.py` - Unit tests for the health endpoint +- `test_query.py` - Unit tests for the SQL query processing endpoint +- `test_integration.py` - Integration tests against running server + +## Test Fixtures + +### `client` +A TestClient instance for making requests to the API without running the server. + +### `sample_csv_file` +A sample CSV file fixture for testing file uploads. + +### `sample_json_file` +A sample JSON file fixture for testing file uploads. diff --git a/src/FrontEnd/tests/__init__.py b/src/FrontEnd/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/FrontEnd/tests/conftest.py b/src/FrontEnd/tests/conftest.py new file mode 100644 index 0000000..b52365c --- /dev/null +++ b/src/FrontEnd/tests/conftest.py @@ -0,0 +1,26 @@ +import pytest +from fastapi.testclient import TestClient +from app.main import app + + +@pytest.fixture +def client(): + """ + Create a test client for the FastAPI application. + This client can be used to make requests to the API without running the server. + """ + return TestClient(app) + + +@pytest.fixture +def sample_csv_file(): + """Create a sample CSV file content for testing.""" + content = b"name,age,city\nJohn,30,NYC\nJane,25,LA\nBob,35,Chicago" + return ("test_data.csv", content, "text/csv") + + +@pytest.fixture +def sample_json_file(): + """Create a sample JSON file content for testing.""" + content = b'[{"name":"John","age":30,"city":"NYC"},{"name":"Jane","age":25,"city":"LA"}]' + return ("test_data.json", content, "application/json") diff --git a/src/FrontEnd/tests/test_health.py b/src/FrontEnd/tests/test_health.py new file mode 100644 index 0000000..bda593d --- /dev/null +++ b/src/FrontEnd/tests/test_health.py @@ -0,0 +1,28 @@ +import pytest +from fastapi.testclient import TestClient + + +def test_health_endpoint(client: TestClient): + """Test that the health endpoint returns a successful response.""" + response = client.get("/api/v1/health") + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + assert "version" in data + + +def test_health_endpoint_structure(client: TestClient): + """Test that the health endpoint returns the correct structure.""" + response = client.get("/api/v1/health") + + assert response.status_code == 200 + data = response.json() + + # Check required fields + assert "status" in data + assert "version" in data + + # Check data types + assert isinstance(data["status"], str) + assert isinstance(data["version"], str) diff --git a/src/FrontEnd/tests/test_integration.py b/src/FrontEnd/tests/test_integration.py new file mode 100644 index 0000000..9c8ddaa --- /dev/null +++ b/src/FrontEnd/tests/test_integration.py @@ -0,0 +1,116 @@ +import pytest +import httpx +import os +import io + + +# Get server URL from environment or use default +SERVER_URL = os.getenv("TEST_SERVER_URL", "http://localhost:8000") + + +@pytest.mark.integration +def test_server_is_running(): + """Test that the server is accessible.""" + try: + response = httpx.get(f"{SERVER_URL}/api/v1/health", timeout=5.0) + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + except httpx.ConnectError: + pytest.fail(f"Could not connect to server at {SERVER_URL}. Is it running?") + + +@pytest.mark.integration +def test_health_endpoint_live(): + """Test the health endpoint on a running server.""" + response = httpx.get(f"{SERVER_URL}/api/v1/health", timeout=5.0) + + assert response.status_code == 200 + data = response.json() + + assert data["status"] == "healthy" + assert "version" in data + assert isinstance(data["version"], str) + + +@pytest.mark.integration +def test_query_endpoint_with_file_live(): + """Test the query endpoint with file upload on a running server.""" + # Create a test CSV file + csv_content = b"name,age,city\nJohn,30,NYC\nJane,25,LA" + + files = {"file": ("test.csv", io.BytesIO(csv_content), "text/csv")} + data = {"sql_query": "SELECT * FROM data"} + + response = httpx.post( + f"{SERVER_URL}/api/v1/query", + files=files, + data=data, + timeout=10.0 + ) + + assert response.status_code == 200 + result = response.json() + + assert result["status"] == "success" + assert result["query"] == "SELECT * FROM data" + assert "execution_time_ms" in result + + +@pytest.mark.integration +def test_query_endpoint_with_uri_live(): + """Test the query endpoint with file URI on a running server.""" + data = { + "sql_query": "SELECT * FROM data", + "file_uri": "https://example.com/data.csv" + } + + response = httpx.post( + f"{SERVER_URL}/api/v1/query", + data=data, + timeout=10.0 + ) + + assert response.status_code == 200 + result = response.json() + + assert result["status"] == "success" + assert result["query"] == "SELECT * FROM data" + + +@pytest.mark.integration +def test_query_endpoint_validation_live(): + """Test that validation works on a running server.""" + # Missing both file and file_uri + response = httpx.post( + f"{SERVER_URL}/api/v1/query", + data={"sql_query": "SELECT * FROM data"}, + timeout=10.0 + ) + + assert response.status_code == 400 + error = response.json() + assert "detail" in error + + +@pytest.mark.integration +def test_openapi_docs_accessible(): + """Test that the OpenAPI/Swagger documentation is accessible.""" + response = httpx.get(f"{SERVER_URL}/docs", timeout=5.0) + assert response.status_code == 200 + + response = httpx.get(f"{SERVER_URL}/redoc", timeout=5.0) + assert response.status_code == 200 + + +@pytest.mark.integration +def test_openapi_schema_available(): + """Test that the OpenAPI schema is available.""" + response = httpx.get(f"{SERVER_URL}/openapi.json", timeout=5.0) + assert response.status_code == 200 + + schema = response.json() + assert "openapi" in schema + assert "paths" in schema + assert "/api/v1/health" in schema["paths"] + assert "/api/v1/query" in schema["paths"] diff --git a/src/FrontEnd/tests/test_query.py b/src/FrontEnd/tests/test_query.py new file mode 100644 index 0000000..cf99cda --- /dev/null +++ b/src/FrontEnd/tests/test_query.py @@ -0,0 +1,124 @@ +import pytest +from fastapi.testclient import TestClient +import io + + +def test_query_endpoint_with_file(client: TestClient, sample_csv_file): + """Test the query endpoint with a file upload.""" + filename, content, content_type = sample_csv_file + + response = client.post( + "/api/v1/query", + data={"sql_query": "SELECT * FROM data"}, + files={"file": (filename, io.BytesIO(content), content_type)} + ) + + assert response.status_code == 200 + data = response.json() + + assert data["status"] == "success" + assert data["query"] == "SELECT * FROM data" + assert "result" in data + assert "execution_time_ms" in data + + +def test_query_endpoint_with_file_uri(client: TestClient): + """Test the query endpoint with a file URI.""" + response = client.post( + "/api/v1/query", + data={ + "sql_query": "SELECT * FROM data", + "file_uri": "https://example.com/data.csv" + } + ) + + assert response.status_code == 200 + data = response.json() + + assert data["status"] == "success" + assert data["query"] == "SELECT * FROM data" + assert "result" in data + assert "execution_time_ms" in data + + +def test_query_endpoint_missing_both_file_and_uri(client: TestClient): + """Test that the endpoint rejects requests without file or file_uri.""" + response = client.post( + "/api/v1/query", + data={"sql_query": "SELECT * FROM data"} + ) + + assert response.status_code == 400 + data = response.json() + assert "detail" in data + assert "file" in data["detail"].lower() or "uri" in data["detail"].lower() + + +def test_query_endpoint_with_both_file_and_uri(client: TestClient, sample_csv_file): + """Test that the endpoint rejects requests with both file and file_uri.""" + filename, content, content_type = sample_csv_file + + response = client.post( + "/api/v1/query", + data={ + "sql_query": "SELECT * FROM data", + "file_uri": "https://example.com/data.csv" + }, + files={"file": (filename, io.BytesIO(content), content_type)} + ) + + assert response.status_code == 400 + data = response.json() + assert "detail" in data + + +def test_query_endpoint_missing_sql_query(client: TestClient, sample_csv_file): + """Test that the endpoint requires sql_query parameter.""" + filename, content, content_type = sample_csv_file + + response = client.post( + "/api/v1/query", + files={"file": (filename, io.BytesIO(content), content_type)} + ) + + assert response.status_code == 422 # Validation error + + +def test_query_endpoint_response_structure(client: TestClient, sample_csv_file): + """Test that the query endpoint returns the correct response structure.""" + filename, content, content_type = sample_csv_file + + response = client.post( + "/api/v1/query", + data={"sql_query": "SELECT * FROM data WHERE age > 25"}, + files={"file": (filename, io.BytesIO(content), content_type)} + ) + + assert response.status_code == 200 + data = response.json() + + # Check required fields + assert "status" in data + assert "query" in data + assert "result" in data + assert "execution_time_ms" in data + + # Check data types + assert isinstance(data["status"], str) + assert isinstance(data["query"], str) + assert isinstance(data["execution_time_ms"], (int, float)) + + +def test_query_endpoint_with_json_file(client: TestClient, sample_json_file): + """Test the query endpoint with a JSON file upload.""" + filename, content, content_type = sample_json_file + + response = client.post( + "/api/v1/query", + data={"sql_query": "SELECT * FROM data"}, + files={"file": (filename, io.BytesIO(content), content_type)} + ) + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" From 2ff516ea6bb4703e4cded3a57f5cbcc833247d4a Mon Sep 17 00:00:00 2001 From: Marco Ferreira Date: Wed, 19 Nov 2025 20:23:02 -0500 Subject: [PATCH 2/3] docs(Contributing) added frontend test information to Contributing.md --- CONTRIBUTING.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7e5fd6c..61a39a1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,13 @@ We use a Makefile to simplify common development tasks. All commands should be r make rust-test ``` -### Run All Tests (Go + Rust) +### Frontend Tests +- Run all tests + ```bash + make frontend-test + ``` + +### Run All Tests (Go + Rust + Frontend) - Run tests for both backends ```bash make test-all From a32b36645521aed4c0282e6e0ef8c18951dfaa93 Mon Sep 17 00:00:00 2001 From: Marco Ferreira Date: Wed, 19 Nov 2025 20:32:21 -0500 Subject: [PATCH 3/3] docs(Contributing) added example information to Contributing.md --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61a39a1..0e212de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,6 +116,11 @@ make go-run make rust-run ``` +### Build and Run Frontend +```bash +make frontend-run +``` + ### Run All Tests ```bash make test-all @@ -125,6 +130,7 @@ Or run individually: ```bash make go-test make rust-test +make frontend-test ``` ### Run Linters